Initial implementation of metadata log sharing.

Allow metadata written to segments to be reused between snapshots.  Keep
track of what metadata was written out on the client, and when identicial
metadata would be written on a subsequent backup, instead emit a reference
to the old metadata.

This needs more testing and verification.  There also needs to be a
mechanism for performing the equivalent of segment cleaning for metadata,
so that the metadata log does not become excessively fragmented over time.

diff --git a/metadata.cc b/metadata.cc
index 68a5664..7aa9822 100644 (file)
--- a/metadata.cc
+++ b/metadata.cc
@@ -11,6 +11,7 @@
 #include <iostream>
 
 #include "metadata.h"
+#include "ref.h"
 #include "store.h"
 #include "statcache.h"
 #include "util.h"
@@ -25,6 +26,44 @@ static const size_t LBS_METADATA_BLOCK_SIZE = 65536;
 /* TODO: Move to header file */
 void add_segment(const string& segment);
 
+/* Like strcmp, but sorts in the order that files will be visited in the
+ * filesystem.  That is, we break paths apart at slashes, and compare path
+ * components separately. */
+static int pathcmp(const char *path1, const char *path2)
+{
+    /* Find the first component in each path. */
+    const char *slash1 = strchr(path1, '/');
+    const char *slash2 = strchr(path2, '/');
+
+    {
+        string comp1, comp2;
+        if (slash1 == NULL)
+            comp1 = path1;
+        else
+            comp1 = string(path1, slash1 - path1);
+
+        if (slash2 == NULL)
+            comp2 = path2;
+        else
+            comp2 = string(path2, slash2 - path2);
+
+        /* Directly compare the two components first. */
+        if (comp1 < comp2)
+            return -1;
+        if (comp1 > comp2)
+            return 1;
+    }
+
+    if (slash1 == NULL && slash2 == NULL)
+        return 0;
+    if (slash1 == NULL)
+        return -1;
+    if (slash2 == NULL)
+        return 1;
+
+    return pathcmp(slash1 + 1, slash2 + 1);
+}
+
 MetadataWriter::MetadataWriter(TarSegmentStore *store,
                                const char *path,
                                const char *snapshot_name,
@@ -36,6 +75,8 @@ MetadataWriter::MetadataWriter(TarSegmentStore *store,
         statcache_path = statcache_path + "-" + snapshot_scheme;
     statcache_tmp_path = statcache_path + "." + snapshot_name;
 
+    statcache_in = fopen(statcache_path.c_str(), "r");
+
     statcache_out = fopen(statcache_tmp_path.c_str(), "w");
     if (statcache_out == NULL) {
         fprintf(stderr, "Error opening statcache %s: %m\n",
@@ -43,22 +84,137 @@ MetadataWriter::MetadataWriter(TarSegmentStore *store,
         throw IOException("Error opening statcache");
     }
 
+    old_metadata_eof = false;
+
     this->store = store;
     chunk_size = 0;
 }
 
+/* Read the next entry from the old statcache file, loading it into
+ * old_metadata. */
+void MetadataWriter::read_statcache()
+{
+    if (statcache_in == NULL) {
+        old_metadata_eof = true;
+        return;
+    }
+
+    old_metadata.clear();
+
+    char *buf = NULL;
+    size_t n = 0;
+    string field = "";          // Last field to be read in
+
+    /* Look for a first line starting with "@@", which tells where the metadata
+     * can be found in the metadata log of an old snapshot. */
+    if (getline(&buf, &n, statcache_in) < 0
+        || buf == NULL || buf[0] != '@' || buf[1] != '@') {
+        old_metadata_eof = true;
+        return;
+    }
+
+    if (strchr(buf, '\n') != NULL)
+        *strchr(buf, '\n') = '\0';
+    old_metadata_loc = buf + 2;
+
+    /* After the initial line follows the metadata, as key-value pairs. */
+    while (!feof(statcache_in)) {
+        if (getline(&buf, &n, statcache_in) < 0)
+            break;
+
+        char *eol = strchr(buf, '\n');
+        if (eol != NULL)
+            *eol = '\0';
+
+        /* Is the line blank?  If so, we have reached the end of this entry. */
+        if (buf[0] == '\0')
+            break;
+
+        /* Is this a continuation line?  (Does it start with whitespace?) */
+        if (isspace(buf[0]) && field != "") {
+            old_metadata[field] += string("\n") + buf;
+            continue;
+        }
+
+        /* For lines of the form "Key: Value" look for ':' and split the line
+         * apart. */
+        char *value = strchr(buf, ':');
+        if (value == NULL)
+            continue;
+        *value = '\0';
+        field = buf;
+
+        value++;
+        while (isspace(*value))
+            value++;
+
+        old_metadata[field] = value;
+    }
+
+    if (feof(statcache_in) && old_metadata.size() == 0) {
+        old_metadata_eof = true;
+    }
+
+    free(buf);
+}
+
+bool MetadataWriter::find(const string& path)
+{
+    const char *path_str = path.c_str();
+    while (!old_metadata_eof) {
+        string old_path = uri_decode(old_metadata["path"]);
+        int cmp = pathcmp(old_path.c_str(), path_str);
+        if (cmp == 0)
+            return true;
+        else if (cmp > 0)
+            return false;
+        else
+            read_statcache();
+    }
+
+    return false;
+}
+
 /* Ensure contents of metadata are flushed to an object. */
 void MetadataWriter::metadata_flush()
 {
     int offset = 0;
 
     ostringstream metadata;
+    ObjectReference indirect;
     for (list<MetadataItem>::iterator i = items.begin();
          i != items.end(); ++i) {
-        metadata << i->text;
-        i->offset = offset;
-        offset += i->text.size();
+        // Write out an indirect reference to any previous objects which could
+        // be reused
+        if (!i->reused || !indirect.merge(i->ref)) {
+            if (!indirect.is_null()) {
+                string refstr = indirect.to_string();
+                metadata << "@" << refstr << "\n";
+                offset += refstr.size() + 2;
+                if (!i->reused) {
+                    metadata << "\n";
+                    offset += 1;
+                }
+            }
+            if (i->reused)
+                indirect = i->ref;
+            else
+                indirect = ObjectReference();
+        }
+
+        if (!i->reused) {
+            metadata << i->text;
+            i->offset = offset;
+            offset += i->text.size();
+        }
     }
+    if (!indirect.is_null()) {
+        string refstr = indirect.to_string();
+        metadata << "@" << refstr << "\n";
+        offset += refstr.size() + 2;
+        indirect = ObjectReference();
+    }
+
     string m = metadata.str();
     if (m.size() == 0)
         return;
@@ -84,6 +240,9 @@ void MetadataWriter::metadata_flush()
         ObjectReference r = ref;
         r.set_range(i->offset, i->text.size());
 
+        if (i->reused)
+            r = i->ref;
+
         string refstr = r.to_string();
         fprintf(statcache_out, "@@%s\n%s", refstr.c_str(), i->text.c_str());
     }
@@ -92,13 +251,22 @@ void MetadataWriter::metadata_flush()
     items.clear();
 }
 
-void MetadataWriter::add(const string& path, dictionary info)
+void MetadataWriter::add(dictionary info)
 {
     MetadataItem item;
     item.offset = 0;
-    item.text = "path: " + uri_encode(path) + "\n";
+    item.reused = false;
     item.text += encode_dict(info) + "\n";
 
+    if (info == old_metadata) {
+        ObjectReference *ref = ObjectReference::parse(old_metadata_loc);
+        if (ref != NULL) {
+            item.reused = true;
+            item.ref = *ref;
+            delete ref;
+        }
+    }
+
     items.push_back(item);
     chunk_size += item.text.size();
 

diff --git a/metadata.h b/metadata.h
index f86244c..9325338 100644 (file)
--- a/metadata.h
+++ b/metadata.h
@@ -23,29 +23,45 @@
 struct MetadataItem {
     int offset;
     std::string text;
+
+    bool reused;
+    ObjectReference ref;
 };
 
 class MetadataWriter {
 public:
     MetadataWriter(TarSegmentStore *store, const char *path,
                    const char *snapshot_name, const char *snapshot_scheme);
-    void add(const std::string& path, dictionary info);
+    void add(dictionary info);
     ObjectReference close();
 
+    bool find(const std::string& path);
+    ObjectReference *old_ref() const {
+        return ObjectReference::parse(old_metadata_loc);
+    }
+
+    dictionary get_old_metadata() const { return old_metadata; }
+
 private:
     void metadata_flush();
+    void read_statcache();
 
     // Where are objects eventually written to?
     TarSegmentStore *store;
 
     // File descriptors for reading/writing local statcache data
     std::string statcache_path, statcache_tmp_path;
-    FILE *statcache_out;
+    FILE *statcache_in, *statcache_out;
 
     // Metadata not yet written out to the segment store
     size_t chunk_size;
     std::list<MetadataItem> items;
     std::ostringstream metadata_root;
+
+    // Statcache information read back in from a previous run
+    bool old_metadata_eof;
+    dictionary old_metadata;
+    std::string old_metadata_loc;   // Reference to where the metadata is found
 };
 
 #endif // _LBS_METADATA_H

diff --git a/ref.cc b/ref.cc
index b5c02d6..158fec3 100644 (file)
--- a/ref.cc
+++ b/ref.cc
@@ -142,3 +142,36 @@ ObjectReference *ObjectReference::parse(const std::string& str)
 
     return ref;
 }
+
+/* Attempt to merge a new object reference into the current one.  Returns a
+ * boolean indicating success; if successful this reference is modified so that
+ * it refers to the range of bytes originally covered by this reference plus
+ * the reference passed in.  Merging only succeeds if both references refer to
+ * the same object and the byte ranges are contiguous. */
+bool ObjectReference::merge(ObjectReference ref)
+{
+    // Exception: We can always merge into a null object
+    if (is_null()) {
+        *this = ref;
+        return true;
+    }
+
+    if (segment != ref.segment)
+        return false;
+    if (object != ref.object)
+        return false;
+
+    // TODO: Allow the case where only one checksum was filled in
+    if (checksum_valid != ref.checksum_valid || checksum != ref.checksum)
+        return false;
+
+    if (!range_valid || !ref.range_valid)
+        return false;
+
+    if (range_start + range_length == ref.range_start) {
+        range_length += ref.range_length;
+        return true;
+    } else {
+        return false;
+    }
+}

diff --git a/ref.h b/ref.h
index cc21a5a..7ed7ff0 100644 (file)
--- a/ref.h
+++ b/ref.h
@@ -68,6 +68,7 @@ public:
     ObjectReference(const std::string& segment, int sequence);
     ObjectReference(const std::string& segment, const std::string& sequence);
 
+    bool is_null() { return segment.size() == 0; }
     std::string to_string() const;
     static ObjectReference *parse(const std::string& s);
 
@@ -88,6 +89,8 @@ public:
     void set_range(size_t start, size_t length)
         { range_start = start; range_length = length; range_valid = true; }
 
+    bool merge(ObjectReference ref);
+
 private:
     std::string segment, object, checksum;
     size_t range_start, range_length;

diff --git a/scandir.cc b/scandir.cc
index 4f84955..91970f5 100644 (file)
--- a/scandir.cc
+++ b/scandir.cc
@@ -256,6 +256,16 @@ void dump_inode(const string& path,         // Path within snapshot
 
     printf("%s\n", path.c_str());
 
+    if (metawriter->find(path)) {
+        ObjectReference *r = metawriter->old_ref();
+        if (r != NULL) {
+            string s = r->to_string();
+            printf("    cached at %s\n", s.c_str());
+            delete r;
+        }
+    }
+
+    file_info["path"] = uri_encode(path);
     file_info["mode"] = encode_int(stat_buf.st_mode & 07777, 8);
     file_info["ctime"] = encode_int(stat_buf.st_ctime);
     file_info["mtime"] = encode_int(stat_buf.st_mtime);
@@ -340,7 +350,7 @@ void dump_inode(const string& path,         // Path within snapshot
 
     file_info["type"] = string(1, inode_type);
 
-    metawriter->add(path, file_info);
+    metawriter->add(file_info);
 }
 
 void scanfile(const string& path, bool include)