From 2b3dd7b15178288761500a6896376b4d44d3bfae Mon Sep 17 00:00:00 2001 From: Michael Vrable Date: Thu, 2 Sep 2010 15:53:07 -0700 Subject: [PATCH] Start at writing out inode maps to cloud storage. --- bluesky/bluesky-private.h | 5 ++-- bluesky/cache.c | 5 ++++ bluesky/cloudlog.c | 43 ++++++++++++++------------- bluesky/imap.c | 61 +++++++++++++++++++++++++++++++++++++-- 4 files changed, 87 insertions(+), 27 deletions(-) diff --git a/bluesky/bluesky-private.h b/bluesky/bluesky-private.h index 3cfec78..9f973a3 100644 --- a/bluesky/bluesky-private.h +++ b/bluesky/bluesky-private.h @@ -350,8 +350,8 @@ typedef struct { /* A sorted list (by inode number) of InodeMapEntry objects. */ GSequence *map_entries; - /* The location where this inode map section is stored in the cloud. */ - BlueSkyCloudPointer location; + /* The serialized version of the inode map data. */ + BlueSkyCloudLog *serialized; /* Have there been changes that require writing this section out again? */ gboolean dirty; @@ -359,6 +359,7 @@ typedef struct { InodeMapEntry *bluesky_inode_map_lookup(GSequence *inode_map, uint64_t inum, int action); +BlueSkyCloudLog *bluesky_inode_map_serialize(BlueSkyFS *fs); #ifdef __cplusplus } diff --git a/bluesky/cache.c b/bluesky/cache.c index d94c18e..454e06f 100644 --- a/bluesky/cache.c +++ b/bluesky/cache.c @@ -150,6 +150,11 @@ static void flushd_cloud(BlueSkyFS *fs) g_mutex_lock(fs->lock); } + /* Write out any updated inode map entries, so that all inodes just written + * can be located, and then a final commit record. */ + BlueSkyCloudLog *commit_record = bluesky_inode_map_serialize(fs); + bluesky_cloudlog_serialize(commit_record, fs); + g_mutex_unlock(fs->lock); bluesky_cloudlog_flush(fs); diff --git a/bluesky/cloudlog.c b/bluesky/cloudlog.c index bf7d22e..beaf1b4 100644 --- a/bluesky/cloudlog.c +++ b/bluesky/cloudlog.c @@ -180,22 +180,14 @@ void bluesky_cloudlog_insert(BlueSkyCloudLog *log) g_mutex_unlock(log->fs->lock); } -struct log_header { +struct cloudlog_header { char magic[4]; - uint32_t size; + uint8_t type; BlueSkyCloudID id; - uint32_t pointer_count; + uint32_t size1, size2, size3; } __attribute__((packed)); -struct logref { - BlueSkyCloudID id; - BlueSkyCloudPointer location; -} __attribute__((packed)); - -struct log_footer { - char refmagic[4]; - struct logref refs[0]; -}; +#define CLOUDLOG_MAGIC "AgI-" /* Ensure that a cloud log item is loaded in memory, and if not read it in. * TODO: Make asynchronous, and make this also fetch from the cloud. Right now @@ -251,22 +243,29 @@ BlueSkyCloudPointer bluesky_cloudlog_serialize(BlueSkyCloudLog *log, bluesky_cloudlog_stats_update(log, -1); + GString *data1 = g_string_new(""); + GString *data2 = g_string_new(""); + GString *data3 = g_string_new(""); + bluesky_serialize_cloudlog(log, data1, data2, data3); + /* TODO: Right now offset/size are set to the raw data, but we should add * header parsing to the code which loads objects back in. */ log->location = state->location; - log->location.offset = state->data->len + sizeof(struct log_header); - log->location.size = log->data->len; - /* = sizeof(struct log_header) + sizeof(BlueSkyCloudID) * 0 - + log->data->len; */ - - struct log_header header; - memcpy(header.magic, "AgI ", 4); - header.size = GUINT32_TO_LE(log->location.size); + log->location.offset = state->data->len + sizeof(struct cloudlog_header); + log->location.size = data1->len; + + struct cloudlog_header header; + memcpy(header.magic, CLOUDLOG_MAGIC, 4); + header.type = log->type + '0'; + header.size1 = GUINT32_TO_LE(data1->len); + header.size2 = GUINT32_TO_LE(data2->len); + header.size3 = GUINT32_TO_LE(data3->len); header.id = log->id; - header.pointer_count = GUINT32_TO_LE(0); g_string_append_len(state->data, (const char *)&header, sizeof(header)); - g_string_append_len(state->data, log->data->data, log->data->len); + g_string_append_len(state->data, data1->str, data1->len); + g_string_append_len(state->data, data2->str, data2->len); + g_string_append_len(state->data, data3->str, data3->len); /* If the object we flushed was an inode, update the inode map. */ if (log->type == LOGTYPE_INODE) { diff --git a/bluesky/imap.c b/bluesky/imap.c index e9dd1d9..484f24f 100644 --- a/bluesky/imap.c +++ b/bluesky/imap.c @@ -70,7 +70,7 @@ InodeMapEntry *bluesky_inode_map_lookup(GSequence *inode_map, uint64_t inum, range->start = 0; range->end = G_MAXUINT64; range->map_entries = g_sequence_new(NULL); - range->dirty = TRUE; + range->serialized = NULL; g_print("Creating inode map range, 1: start=%"PRIu64" end=%"PRIu64"\n", range->start, range->end); @@ -112,8 +112,10 @@ InodeMapEntry *bluesky_inode_map_lookup(GSequence *inode_map, uint64_t inum, g_print("Created inode map entry for %"PRIu64"\n", inum); } - if (action != 0) - range->dirty = TRUE; + if (action != 0) { + bluesky_cloudlog_unref(range->serialized); + range->serialized = NULL; + } /* Were we requested to delete the item? */ if (action < 0) { @@ -123,3 +125,56 @@ InodeMapEntry *bluesky_inode_map_lookup(GSequence *inode_map, uint64_t inum, return entry; } + +/* Convert a section of the inode map to serialized form, in preparation for + * writing it out to the cloud. */ +static void bluesky_inode_map_serialize_section(BlueSkyFS *fs, + InodeMapRange *range) +{ + if (range->serialized != NULL) + return; + + GString *buf = g_string_new(""); + BlueSkyCloudLog *log = bluesky_cloudlog_new(fs, NULL); + log->type = LOGTYPE_INODE_MAP; + log->inum = 0; + + GSequenceIter *i = g_sequence_get_begin_iter(range->map_entries); + while (!g_sequence_iter_is_end(i)) { + InodeMapEntry *entry = (InodeMapEntry *)g_sequence_get(i); + uint64_t inum = GUINT64_TO_LE(entry->inum); + g_string_append_len(buf, (const char *)&inum, sizeof(inum)); + g_array_append_val(log->links, entry->item); + i = g_sequence_iter_next(i); + } + + log->data = bluesky_string_new_from_gstring(buf); + bluesky_cloudlog_unref(range->serialized); + range->serialized = log; +} + +BlueSkyCloudLog *bluesky_inode_map_serialize(BlueSkyFS *fs) +{ + GString *buf = g_string_new(""); + BlueSkyCloudLog *log = bluesky_cloudlog_new(fs, NULL); + log->type = LOGTYPE_CHECKPOINT; + log->inum = 0; + + GSequenceIter *i = g_sequence_get_begin_iter(fs->inode_map); + while (!g_sequence_iter_is_end(i)) { + InodeMapRange *range = (InodeMapRange *)g_sequence_get(i); + uint64_t inum = GUINT64_TO_LE(range->start); + g_string_append_len(buf, (const char *)&inum, sizeof(inum)); + inum = GUINT64_TO_LE(range->end); + g_string_append_len(buf, (const char *)&inum, sizeof(inum)); + + if (range->serialized == NULL) + bluesky_inode_map_serialize_section(fs, range); + bluesky_cloudlog_ref(range->serialized); + g_array_append_val(log->links, range->serialized); + i = g_sequence_iter_next(i); + } + + log->data = bluesky_string_new_from_gstring(buf); + return log; +} -- 2.20.1