Start work on log replay for filesystem recovery.

Right now this implements scanning of one journal segment with consistency
checking to find items that were written out.

Also fix the checksum calculation on log entries so that they will validate
properly (we want to compute the checksum so that on validation, computing
the checksum of the entire object results in a value of zero).

diff --git a/bluesky/bluesky-private.h b/bluesky/bluesky-private.h
index e5b5b58..8bb9e0c 100644 (file)
--- a/bluesky/bluesky-private.h
+++ b/bluesky/bluesky-private.h
@@ -305,6 +305,8 @@ BlueSkyCacheFile *bluesky_cachefile_lookup(BlueSkyFS *fs,
                                            int clouddir, int log_seq);
 void bluesky_cachefile_gc(BlueSkyFS *fs);
 
+void bluesky_replay(BlueSkyFS *fs);
+
 /* Used to track log segments that are being written to the cloud. */
 typedef struct {
     BlueSkyRCStr *data;

diff --git a/bluesky/crc32c.c b/bluesky/crc32c.c
index b29f18d..8e11bbf 100644 (file)
--- a/bluesky/crc32c.c
+++ b/bluesky/crc32c.c
@@ -101,5 +101,5 @@ uint32_t crc32c(uint32_t crc, const char *buf, unsigned int length)
 
 uint32_t crc32c_finalize(uint32_t crc)
 {
-    return ~GUINT32_TO_LE(crc);
+    return GUINT32_TO_LE(crc);
 }

diff --git a/bluesky/inode.c b/bluesky/inode.c
index a79bcc1..b8ab4e8 100644 (file)
--- a/bluesky/inode.c
+++ b/bluesky/inode.c
@@ -130,6 +130,8 @@ BlueSkyFS *bluesky_init_fs(gchar *name, BlueSkyStore *store)
     bluesky_inode_do_sync(root);
     bluesky_superblock_flush(fs);
 
+    bluesky_replay(fs);
+
     return fs;
 }
 

diff --git a/bluesky/log.c b/bluesky/log.c
index 1fe6176..1459ee4 100644 (file)
--- a/bluesky/log.c
+++ b/bluesky/log.c
@@ -43,8 +43,10 @@
 
 struct log_header {
     uint32_t magic;             // HEADER_MAGIC
-    uint64_t offset;            // Starting byte offset of the log header
+    uint8_t type;               // Object type + '0'
+    uint32_t offset;            // Starting byte offset of the log header
     uint32_t size;              // Size of the data item (bytes)
+    uint64_t inum;              // Inode which owns this data, if any
     BlueSkyCloudID id;          // Object identifier
 } __attribute__((packed));
 
@@ -189,9 +191,11 @@ static gpointer log_thread(gpointer d)
         }
 
         header.magic = GUINT32_TO_LE(HEADER_MAGIC);
-        header.offset = GUINT64_TO_LE(offset);
+        header.offset = GUINT32_TO_LE(offset);
         header.size = GUINT32_TO_LE(item->data->len);
+        header.type = item->type + '0';
         header.id = item->id;
+        header.inum = GUINT64_TO_LE(item->inum);
         footer.magic = GUINT32_TO_LE(FOOTER_MAGIC);
 
         uint32_t crc = BLUESKY_CRC32C_SEED;
@@ -557,3 +561,82 @@ void bluesky_cachefile_gc(BlueSkyFS *fs)
 
     g_mutex_unlock(fs->log->mmap_lock);
 }
+
+/******************************* JOURNAL REPLAY *******************************
+ * The journal replay code is used to recover filesystem state after a
+ * filesystem restart.  We first look for the most recent commit record in the
+ * journal, which indicates the point before which all data in the journal has
+ * also been committed to the cloud.  Then, we read in all data in the log past
+ * that point.
+ */
+
+static gboolean validate_journal_item(const char *buf, size_t len, off_t offset)
+{
+    const struct log_header *header;
+    const struct log_footer *footer;
+
+    if (offset + sizeof(struct log_header) + sizeof(struct log_footer) > len)
+        return FALSE;
+
+    header = (const struct log_header *)(buf + offset);
+    if (GUINT32_FROM_LE(header->magic) != HEADER_MAGIC)
+        return FALSE;
+    if (GUINT32_FROM_LE(header->offset) != offset)
+        return FALSE;
+    size_t size = GUINT32_FROM_LE(header->size);
+
+    off_t footer_offset = offset + sizeof(struct log_header) + size;
+    if (footer_offset + sizeof(struct log_footer) > len)
+        return FALSE;
+    footer = (const struct log_footer *)(buf + footer_offset);
+
+    if (GUINT32_FROM_LE(footer->magic) != FOOTER_MAGIC)
+        return FALSE;
+
+    uint32_t crc = crc32c(BLUESKY_CRC32C_SEED, buf + offset,
+                          sizeof(struct log_header) + sizeof(struct log_footer)
+                          + size);
+    if (crc != 0)
+        return FALSE;
+
+    return TRUE;
+}
+
+/* Scan through a journal segment to extract correctly-written items (those
+ * that pass sanity checks and have a valid checksum). */
+static void bluesky_replay_scan_journal(const char *buf, size_t len)
+{
+    const struct log_header *header;
+    off_t offset = 0;
+
+    while (validate_journal_item(buf, len, offset)) {
+        header = (const struct log_header *)(buf + offset);
+        g_print("In replay found valid item at offset %zd\n", offset);
+
+        size_t size = GUINT32_FROM_LE(header->size);
+
+        offset += sizeof(struct log_header) + size + sizeof(struct log_footer);
+    }
+}
+
+void bluesky_replay(BlueSkyFS *fs)
+{
+    BlueSkyLog *log = fs->log;
+    char logname[64];
+    int seq_num = 0;
+    int fd;
+
+    g_snprintf(logname, sizeof(logname), "journal-%08d", seq_num);
+    fd = openat(log->dirfd, logname, O_RDONLY);
+    if (fd < 0)
+        return;
+
+    off_t length = lseek(fd, 0, SEEK_END);
+    void *addr = mmap(NULL, length, PROT_READ, MAP_SHARED, fd, 0);
+    if (addr == NULL)
+        return;
+
+    bluesky_replay_scan_journal((const char *)addr, length);
+
+    munmap(addr, length);
+}