1 /* Blue Sky: File Systems in the Cloud
3 * Copyright (C) 2009 The Regents of the University of California
4 * Written by Michael Vrable <mvrable@cs.ucsd.edu>
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions
9 * 1. Redistributions of source code must retain the above copyright
10 * notice, this list of conditions and the following disclaimer.
11 * 2. Redistributions in binary form must reproduce the above copyright
12 * notice, this list of conditions and the following disclaimer in the
13 * documentation and/or other materials provided with the distribution.
14 * 3. Neither the name of the University nor the names of its contributors
15 * may be used to endorse or promote products derived from this software
16 * without specific prior written permission.
18 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
19 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
20 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
21 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
22 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
23 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
24 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
25 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
26 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
27 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
36 #include "bluesky-private.h"
38 /* Serialization of in-memory filesystem data structures to bytestrings which
39 * can be written to persistent storage. All data is stored in little-endian
42 /* Magic signature and structure of serialized superblocks. */
44 #define SUPERBLOCK_MAGIC 0x65ca91e91b124234ULL
46 struct serialized_superblock {
47 uint64_t signature; /* SUPERBLOCK_MAGIC */
49 } __attribute__((packed));
51 /* Magic signature for serialized inodes. */
53 #define INODE_MAGIC 0xa6832100943d71e6ULL
55 struct serialized_inode {
56 uint64_t signature; /* INODE_MAGIC */
62 uint64_t change_count;
67 } __attribute__((packed));
69 void bluesky_serialize_superblock(GString *out, BlueSkyFS *fs)
71 struct serialized_superblock buf;
73 buf.signature = GUINT64_TO_LE(SUPERBLOCK_MAGIC);
74 buf.next_inum = GUINT64_TO_LE(fs->next_inum);
76 g_string_append_len(out, (gchar *)&buf, sizeof(buf));
79 BlueSkyFS *bluesky_deserialize_superblock(const gchar *buf)
81 struct serialized_superblock *raw = (struct serialized_superblock *)buf;
83 if (GUINT64_FROM_LE(raw->signature) != SUPERBLOCK_MAGIC)
86 BlueSkyFS *fs = bluesky_new_fs("deserialized");
87 fs->next_inum = GUINT64_FROM_LE(raw->next_inum);
92 BlueSkyCloudLog *bluesky_serialize_inode(BlueSkyInode *inode)
94 BlueSkyFS *fs = inode->fs;
95 GString *out = g_string_new("");
96 struct serialized_inode buf;
98 BlueSkyCloudLog *cloudlog = bluesky_cloudlog_new(fs, NULL);
99 cloudlog->type = LOGTYPE_INODE;
100 cloudlog->inum = inode->inum;
102 buf.signature = GUINT64_TO_LE(INODE_MAGIC);
103 buf.type = GUINT32_TO_LE(inode->type);
104 buf.mode = GUINT32_TO_LE(inode->mode);
105 buf.uid = GUINT32_TO_LE(inode->uid);
106 buf.gid = GUINT32_TO_LE(inode->gid);
107 buf.nlink = GUINT32_TO_LE(inode->nlink);
108 buf.inum = GUINT64_TO_LE(inode->inum);
109 buf.change_count = GUINT64_TO_LE(inode->change_count);
110 buf.atime = GINT64_TO_LE(inode->atime);
111 buf.ctime = GINT64_TO_LE(inode->ctime);
112 buf.mtime = GINT64_TO_LE(inode->mtime);
113 buf.ntime = GINT64_TO_LE(inode->ntime);
115 g_string_append_len(out, (gchar *)&buf, sizeof(buf));
117 switch (inode->type) {
118 case BLUESKY_REGULAR:
120 uint64_t size = GUINT64_TO_LE(inode->size);
121 g_string_append_len(out, (gchar *)&size, sizeof(uint64_t));
122 for (int i = 0; i < inode->blocks->len; i++) {
123 BlueSkyBlock *b = &g_array_index(inode->blocks, BlueSkyBlock, i);
124 BlueSkyCloudLog *ref = (b->type == BLUESKY_BLOCK_REF ? b->ref : NULL);
125 bluesky_cloudlog_ref(ref);
126 g_array_append_val(cloudlog->links, ref);
131 case BLUESKY_DIRECTORY:
135 GSequenceIter *i = g_sequence_get_begin_iter(inode->dirents);
137 while (!g_sequence_iter_is_end(i)) {
138 BlueSkyDirent *d = g_sequence_get(i);
140 seq = GUINT32_TO_LE(d->cookie);
141 inum = GUINT64_TO_LE(d->inum);
142 g_string_append_len(out, (gchar *)&seq, sizeof(uint32_t));
143 g_string_append_len(out, (gchar *)&inum, sizeof(uint64_t));
144 g_string_append(out, d->name);
145 g_string_append_c(out, '\0');
147 i = g_sequence_iter_next(i);
150 seq = GUINT32_TO_LE(0);
151 g_string_append_len(out, (gchar *)&seq, sizeof(uint32_t));
156 case BLUESKY_SYMLINK:
158 g_string_append(out, inode->symlink_contents);
159 g_string_append_c(out, '\0');
164 g_warning("Serialization for inode type %d not implemented!\n",
168 cloudlog->data = bluesky_string_new_from_gstring(out);
169 bluesky_cloudlog_insert(cloudlog);
170 bluesky_cloudlog_stats_update(cloudlog, 1);
175 /* Deserialize an inode into an in-memory representation. Returns a boolean
176 * indicating whether the deserialization was successful. */
177 gboolean bluesky_deserialize_inode(BlueSkyInode *inode, BlueSkyCloudLog *item)
179 g_assert(item->data != NULL);
180 const char *buf = item->data->data;
182 if (bluesky_verbose) {
183 g_log("bluesky/serialize", G_LOG_LEVEL_DEBUG,
184 "Deserializing inode %lld...", (long long)inode->inum);
187 struct serialized_inode *raw = (struct serialized_inode *)buf;
189 if (GUINT64_FROM_LE(raw->signature) != INODE_MAGIC)
192 if (inode->inum != GUINT64_FROM_LE(raw->inum))
195 bluesky_init_inode(inode, GUINT32_FROM_LE(raw->type));
197 inode->mode = GUINT32_FROM_LE(raw->mode);
198 inode->uid = GUINT32_FROM_LE(raw->uid);
199 inode->gid = GUINT32_FROM_LE(raw->gid);
200 inode->nlink = GUINT32_FROM_LE(raw->nlink);
201 inode->change_count = GUINT64_FROM_LE(raw->change_count);
202 inode->change_commit = inode->change_count;
203 inode->atime = GINT64_FROM_LE(raw->atime);
204 inode->ctime = GINT64_FROM_LE(raw->ctime);
205 inode->mtime = GINT64_FROM_LE(raw->mtime);
206 inode->ntime = GINT64_FROM_LE(raw->ntime);
208 buf += sizeof(struct serialized_inode);
210 /* TODO: Bounds checking */
211 switch (inode->type) {
212 case BLUESKY_REGULAR:
213 inode->size = GINT64_FROM_LE(*(uint64_t *)buf);
214 buf += sizeof(uint64_t);
215 g_array_set_size(inode->blocks,
216 (inode->size + BLUESKY_BLOCK_SIZE - 1)
217 / BLUESKY_BLOCK_SIZE);
218 g_assert(inode->blocks->len == item->links->len);
219 for (int i = 0; i < inode->blocks->len; i++) {
220 BlueSkyBlock *b = &g_array_index(inode->blocks, BlueSkyBlock, i);
221 b->type = BLUESKY_BLOCK_REF;
222 b->ref = g_array_index(item->links, BlueSkyCloudLog *, i);
223 bluesky_cloudlog_ref(b->ref);
228 case BLUESKY_DIRECTORY:
230 struct serialized_dirent {
234 } __attribute__((packed));
236 struct serialized_dirent *d = (struct serialized_dirent *)buf;
237 while (GUINT32_FROM_LE(d->seq) != 0) {
238 BlueSkyDirent *dirent = g_new(BlueSkyDirent, 1);
239 dirent->cookie = GUINT64_FROM_LE(d->seq);
240 dirent->inum = GUINT64_FROM_LE(d->inum);
241 dirent->name = g_strdup(d->name);
242 dirent->name_folded = bluesky_lowercase(d->name);
244 g_sequence_insert_sorted(inode->dirents, dirent,
245 bluesky_dirent_compare, NULL);
246 g_hash_table_insert(inode->dirhash, dirent->name, dirent);
247 g_hash_table_insert(inode->dirhash_folded, dirent->name_folded,
250 buf = strchr(d->name, '\0') + 1;
251 d = (struct serialized_dirent *)buf;
256 case BLUESKY_SYMLINK:
258 inode->symlink_contents = g_strdup(buf);
263 g_warning("Deserialization for inode type %d not implemented!\n",
270 /* Convert an in-memory cloud log item to a more serialized form, suitable
271 * either for writing to the local journal or the the cloud. */
272 void bluesky_serialize_cloudlog(BlueSkyCloudLog *log,
273 GString *encrypted, // Raw data payload
274 GString *authenticated, // Block links
275 GString *writable) // Writable block links
277 g_string_append_len(encrypted, log->data->data, log->data->len);
278 for (int i = 0; i < log->links->len; i++) {
279 BlueSkyCloudLog *ref = g_array_index(log->links, BlueSkyCloudLog *, i);
281 g_string_append_len(authenticated,
282 (const char *)&ref->id,
283 sizeof(BlueSkyCloudID));
284 // TODO: Fix endianness of output
285 g_string_append_len(writable,
286 (const char *)&ref->location,
287 sizeof(ref->location));
290 memset(&id, 0, sizeof(id));
291 g_string_append_len(authenticated, (const char *)&id, sizeof(id));
296 /* Deserialize data from the journal or a cloud segment back into the in-memory
297 * cloud log item format. */
298 void bluesky_deserialize_cloudlog(BlueSkyCloudLog *item,
302 const char *data1, *data2, *data3;
303 size_t len1, len2, len3;
308 /* Auto-detect the format: either the journal or cloud log, based on the
309 * magic number at the start */
310 if (memcmp(data, JOURNAL_MAGIC, 4) == 0) {
311 g_assert(len >= sizeof(struct log_header));
312 struct log_header *header = (struct log_header *)data;
313 type = header->type - '0';
314 len1 = GUINT32_FROM_LE(header->size1);
315 len2 = GUINT32_FROM_LE(header->size2);
316 len3 = GUINT32_FROM_LE(header->size3);
318 data1 = data + sizeof(struct log_header);
319 data2 = data1 + len1;
320 data3 = data2 + len2;
321 g_assert(data3 + len3 - data <= len);
322 item->type = header->type - '0';
323 item->inum = GUINT64_FROM_LE(header->inum);
324 } else if (memcmp(data, CLOUDLOG_MAGIC, 4) == 0) {
325 g_assert(len >= sizeof(struct cloudlog_header));
326 struct cloudlog_header *header = (struct cloudlog_header *)data;
327 type = header->type - '0';
328 len1 = GUINT32_FROM_LE(header->size1);
329 len2 = GUINT32_FROM_LE(header->size2);
330 len3 = GUINT32_FROM_LE(header->size3);
332 data1 = data + sizeof(struct cloudlog_header);
333 data2 = data1 + len1;
334 data3 = data2 + len2;
335 g_assert(data3 + len3 - data <= len);
336 item->type = header->type - '0';
337 item->inum = GUINT64_FROM_LE(header->inum);
339 g_warning("Deserializing garbage cloud log item!");
343 if (memcmp(&id, &item->id, sizeof(BlueSkyCloudID)) != 0) {
344 g_warning("ID does not match expected value!\n");
347 BlueSkyFS *fs = item->fs;
349 bluesky_string_unref(item->data);
351 item->data_size = len1;
353 int link_count = len2 / sizeof(BlueSkyCloudID);
354 GArray *new_links = g_array_new(FALSE, TRUE, sizeof(BlueSkyCloudLog *));
355 for (int i = 0; i < link_count; i++) {
357 g_assert(len2 >= sizeof(id));
358 memcpy(&id, data2, sizeof(id));
359 data2 += sizeof(id); len2 -= sizeof(id);
361 BlueSkyCloudLog *ref = bluesky_cloudlog_get(fs, id);
363 g_mutex_lock(ref->lock);
364 g_assert(len3 >= sizeof(ref->location));
365 memcpy(&ref->location, data3, sizeof(ref->location));
366 ref->location_flags |= CLOUDLOG_CLOUD;
367 data3 += sizeof(ref->location); len3 -= sizeof(ref->location);
368 g_mutex_unlock(ref->lock);
371 g_array_append_val(new_links, ref);
374 for (int i = 0; i < item->links->len; i++) {
375 BlueSkyCloudLog *c = g_array_index(item->links,
376 BlueSkyCloudLog *, i);
377 bluesky_cloudlog_unref(c);
379 g_array_unref(item->links);
380 item->links = new_links;