1 /* Blue Sky: File Systems in the Cloud
3 * Copyright (C) 2009 The Regents of the University of California
4 * Written by Michael Vrable <mvrable@cs.ucsd.edu>
14 #include "bluesky-private.h"
16 /* Serialization of in-memory filesystem data structures to bytestrings which
17 * can be written to persistent storage. All data is stored in little-endian
20 /* Magic signature and structure of serialized superblocks. */
22 #define SUPERBLOCK_MAGIC 0x65ca91e91b124234ULL
24 struct serialized_superblock {
25 uint64_t signature; /* SUPERBLOCK_MAGIC */
27 } __attribute__((packed));
29 /* Magic signature for serialized inodes. */
31 #define INODE_MAGIC 0xa6832100943d71e6ULL
33 struct serialized_inode {
34 uint64_t signature; /* INODE_MAGIC */
40 uint64_t change_count;
45 } __attribute__((packed));
47 void bluesky_serialize_superblock(GString *out, BlueSkyFS *fs)
49 struct serialized_superblock buf;
51 buf.signature = GUINT64_TO_LE(SUPERBLOCK_MAGIC);
52 buf.next_inum = GUINT64_TO_LE(fs->next_inum);
54 g_string_append_len(out, (gchar *)&buf, sizeof(buf));
57 BlueSkyFS *bluesky_deserialize_superblock(const gchar *buf)
59 struct serialized_superblock *raw = (struct serialized_superblock *)buf;
61 if (GUINT64_FROM_LE(raw->signature) != SUPERBLOCK_MAGIC)
64 BlueSkyFS *fs = bluesky_new_fs("deserialized");
65 fs->next_inum = GUINT64_FROM_LE(raw->next_inum);
70 BlueSkyCloudLog *bluesky_serialize_inode(BlueSkyInode *inode)
72 BlueSkyFS *fs = inode->fs;
73 GString *out = g_string_new("");
74 struct serialized_inode buf;
76 BlueSkyCloudLog *cloudlog = bluesky_cloudlog_new(fs, NULL);
77 cloudlog->type = LOGTYPE_INODE;
78 cloudlog->inum = inode->inum;
80 buf.signature = GUINT64_TO_LE(INODE_MAGIC);
81 buf.type = GUINT32_TO_LE(inode->type);
82 buf.mode = GUINT32_TO_LE(inode->mode);
83 buf.uid = GUINT32_TO_LE(inode->uid);
84 buf.gid = GUINT32_TO_LE(inode->gid);
85 buf.nlink = GUINT32_TO_LE(inode->nlink);
86 buf.inum = GUINT64_TO_LE(inode->inum);
87 buf.change_count = GUINT64_TO_LE(inode->change_count);
88 buf.atime = GINT64_TO_LE(inode->atime);
89 buf.ctime = GINT64_TO_LE(inode->ctime);
90 buf.mtime = GINT64_TO_LE(inode->mtime);
91 buf.ntime = GINT64_TO_LE(inode->ntime);
93 g_string_append_len(out, (gchar *)&buf, sizeof(buf));
95 switch (inode->type) {
98 uint64_t size = GUINT64_TO_LE(inode->size);
99 g_string_append_len(out, (gchar *)&size, sizeof(uint64_t));
100 for (int i = 0; i < inode->blocks->len; i++) {
101 BlueSkyBlock *b = &g_array_index(inode->blocks, BlueSkyBlock, i);
102 BlueSkyCloudLog *ref = (b->type == BLUESKY_BLOCK_REF ? b->ref : NULL);
103 bluesky_cloudlog_ref(ref);
104 g_array_append_val(cloudlog->links, ref);
109 case BLUESKY_DIRECTORY:
113 GSequenceIter *i = g_sequence_get_begin_iter(inode->dirents);
115 while (!g_sequence_iter_is_end(i)) {
116 BlueSkyDirent *d = g_sequence_get(i);
118 seq = GUINT32_TO_LE(d->cookie);
119 inum = GUINT64_TO_LE(d->inum);
120 g_string_append_len(out, (gchar *)&seq, sizeof(uint32_t));
121 g_string_append_len(out, (gchar *)&inum, sizeof(uint64_t));
122 g_string_append(out, d->name);
123 g_string_append_c(out, '\0');
125 i = g_sequence_iter_next(i);
128 seq = GUINT32_TO_LE(0);
129 g_string_append_len(out, (gchar *)&seq, sizeof(uint32_t));
134 case BLUESKY_SYMLINK:
136 g_string_append(out, inode->symlink_contents);
137 g_string_append_c(out, '\0');
142 g_warning("Serialization for inode type %d not implemented!\n",
146 cloudlog->data = bluesky_string_new_from_gstring(out);
147 bluesky_cloudlog_insert(cloudlog);
148 bluesky_cloudlog_stats_update(cloudlog, 1);
153 /* Deserialize an inode into an in-memory representation. Returns a boolean
154 * indicating whether the deserialization was successful. */
155 gboolean bluesky_deserialize_inode(BlueSkyInode *inode, BlueSkyCloudLog *item)
157 g_assert(item->data != NULL);
158 const char *buf = item->data->data;
160 if (bluesky_verbose) {
161 g_log("bluesky/serialize", G_LOG_LEVEL_DEBUG,
162 "Deserializing inode %lld...", (long long)inode->inum);
165 struct serialized_inode *raw = (struct serialized_inode *)buf;
167 if (GUINT64_FROM_LE(raw->signature) != INODE_MAGIC)
170 if (inode->inum != GUINT64_FROM_LE(raw->inum))
173 bluesky_init_inode(inode, GUINT32_FROM_LE(raw->type));
175 inode->mode = GUINT32_FROM_LE(raw->mode);
176 inode->uid = GUINT32_FROM_LE(raw->uid);
177 inode->gid = GUINT32_FROM_LE(raw->gid);
178 inode->nlink = GUINT32_FROM_LE(raw->nlink);
179 inode->change_count = GUINT64_FROM_LE(raw->change_count);
180 inode->change_commit = inode->change_count;
181 inode->atime = GINT64_FROM_LE(raw->atime);
182 inode->ctime = GINT64_FROM_LE(raw->ctime);
183 inode->mtime = GINT64_FROM_LE(raw->mtime);
184 inode->ntime = GINT64_FROM_LE(raw->ntime);
186 buf += sizeof(struct serialized_inode);
188 /* TODO: Bounds checking */
189 switch (inode->type) {
190 case BLUESKY_REGULAR:
191 inode->size = GINT64_FROM_LE(*(uint64_t *)buf);
192 buf += sizeof(uint64_t);
193 g_array_set_size(inode->blocks,
194 (inode->size + BLUESKY_BLOCK_SIZE - 1)
195 / BLUESKY_BLOCK_SIZE);
196 g_assert(inode->blocks->len == item->links->len);
197 for (int i = 0; i < inode->blocks->len; i++) {
198 BlueSkyBlock *b = &g_array_index(inode->blocks, BlueSkyBlock, i);
199 b->type = BLUESKY_BLOCK_REF;
200 b->ref = g_array_index(item->links, BlueSkyCloudLog *, i);
201 bluesky_cloudlog_ref(b->ref);
206 case BLUESKY_DIRECTORY:
208 struct serialized_dirent {
212 } __attribute__((packed));
214 struct serialized_dirent *d = (struct serialized_dirent *)buf;
215 while (GUINT32_FROM_LE(d->seq) != 0) {
216 BlueSkyDirent *dirent = g_new(BlueSkyDirent, 1);
217 dirent->cookie = GUINT64_FROM_LE(d->seq);
218 dirent->inum = GUINT64_FROM_LE(d->inum);
219 dirent->name = g_strdup(d->name);
220 dirent->name_folded = bluesky_lowercase(d->name);
222 g_sequence_insert_sorted(inode->dirents, dirent,
223 bluesky_dirent_compare, NULL);
224 g_hash_table_insert(inode->dirhash, dirent->name, dirent);
225 g_hash_table_insert(inode->dirhash_folded, dirent->name_folded,
228 buf = strchr(d->name, '\0') + 1;
229 d = (struct serialized_dirent *)buf;
234 case BLUESKY_SYMLINK:
236 inode->symlink_contents = g_strdup(buf);
241 g_warning("Deserialization for inode type %d not implemented!\n",
248 /* Convert an in-memory cloud log item to a more serialized form, suitable
249 * either for writing to the local journal or the the cloud. */
250 void bluesky_serialize_cloudlog(BlueSkyCloudLog *log,
251 GString *encrypted, // Raw data payload
252 GString *authenticated, // Block links
253 GString *writable) // Writable block links
255 g_string_append_len(encrypted, log->data->data, log->data->len);
256 for (int i = 0; i < log->links->len; i++) {
257 BlueSkyCloudLog *ref = g_array_index(log->links, BlueSkyCloudLog *, i);
259 g_string_append_len(authenticated,
260 (const char *)&ref->id,
261 sizeof(BlueSkyCloudID));
262 // TODO: Fix endianness of output
263 g_string_append_len(writable,
264 (const char *)&ref->location,
265 sizeof(ref->location));
268 memset(&id, 0, sizeof(id));
269 g_string_append_len(authenticated, (const char *)&id, sizeof(id));
274 /* Deserialize data from the journal or a cloud segment back into the in-memory
275 * cloud log item format. */
276 void bluesky_deserialize_cloudlog(BlueSkyCloudLog *item,
280 const char *data1, *data2, *data3;
281 size_t len1, len2, len3;
286 /* Auto-detect the format: either the journal or cloud log, based on the
287 * magic number at the start */
288 if (memcmp(data, JOURNAL_MAGIC, 4) == 0) {
289 g_assert(len >= sizeof(struct log_header));
290 struct log_header *header = (struct log_header *)data;
291 type = header->type - '0';
292 len1 = GUINT32_FROM_LE(header->size1);
293 len2 = GUINT32_FROM_LE(header->size2);
294 len3 = GUINT32_FROM_LE(header->size3);
296 data1 = data + sizeof(struct log_header);
297 data2 = data1 + len1;
298 data3 = data2 + len2;
299 g_assert(data3 + len3 - data <= len);
300 item->type = header->type - '0';
301 item->inum = GUINT64_FROM_LE(header->inum);
302 } else if (memcmp(data, CLOUDLOG_MAGIC, 4) == 0) {
303 g_assert(len >= sizeof(struct cloudlog_header));
304 struct cloudlog_header *header = (struct cloudlog_header *)data;
305 type = header->type - '0';
306 len1 = GUINT32_FROM_LE(header->size1);
307 len2 = GUINT32_FROM_LE(header->size2);
308 len3 = GUINT32_FROM_LE(header->size3);
310 data1 = data + sizeof(struct cloudlog_header);
311 data2 = data1 + len1;
312 data3 = data2 + len2;
313 g_assert(data3 + len3 - data <= len);
314 item->type = header->type - '0';
315 item->inum = GUINT64_FROM_LE(header->inum);
317 g_warning("Deserializing garbage cloud log item!");
321 if (memcmp(&id, &item->id, sizeof(BlueSkyCloudID)) != 0) {
322 g_warning("ID does not match expected value!\n");
325 BlueSkyFS *fs = item->fs;
327 bluesky_string_unref(item->data);
329 item->data_size = len1;
331 int link_count = len2 / sizeof(BlueSkyCloudID);
332 GArray *new_links = g_array_new(FALSE, TRUE, sizeof(BlueSkyCloudLog *));
333 for (int i = 0; i < link_count; i++) {
335 g_assert(len2 >= sizeof(id));
336 memcpy(&id, data2, sizeof(id));
337 data2 += sizeof(id); len2 -= sizeof(id);
339 BlueSkyCloudLog *ref = bluesky_cloudlog_get(fs, id);
341 g_mutex_lock(ref->lock);
342 g_assert(len3 >= sizeof(ref->location));
343 memcpy(&ref->location, data3, sizeof(ref->location));
344 ref->location_flags |= CLOUDLOG_CLOUD;
345 data3 += sizeof(ref->location); len3 -= sizeof(ref->location);
346 g_mutex_unlock(ref->lock);
349 g_array_append_val(new_links, ref);
352 for (int i = 0; i < item->links->len; i++) {
353 BlueSkyCloudLog *c = g_array_index(item->links,
354 BlueSkyCloudLog *, i);
355 bluesky_cloudlog_unref(c);
357 g_array_unref(item->links);
358 item->links = new_links;