Initial framework for direct transfer of backups to remote storage.

[cumulus.git] / metadata.cc

diff --git a/metadata.cc b/metadata.cc
index 7aa9822..2b5f12a 100644 (file)
--- a/metadata.cc
+++ b/metadata.cc
@@ -9,21 +9,28 @@
 
 #include <string>
 #include <iostream>
+#include <map>
 
 #include "metadata.h"
+#include "localdb.h"
 #include "ref.h"
 #include "store.h"
-#include "statcache.h"
 #include "util.h"
 
 using std::list;
+using std::map;
 using std::string;
 using std::ostream;
 using std::ostringstream;
 
 static const size_t LBS_METADATA_BLOCK_SIZE = 65536;
 
+// If true, forces a full write of metadata: will not include pointers to
+// metadata in old snapshots.
+bool flag_full_metadata = false;
+
 /* TODO: Move to header file */
+extern LocalDb *db;
 void add_segment(const string& segment);
 
 /* Like strcmp, but sorts in the order that files will be visited in the
@@ -64,6 +71,27 @@ static int pathcmp(const char *path1, const char *path2)
     return pathcmp(slash1 + 1, slash2 + 1);
 }
 
+/* Encode a dictionary of string key/value pairs into a sequence of lines of
+ * the form "key: value".  If it exists, the key "name" is treated specially
+ * and will be listed first. */
+static string encode_dict(const map<string, string>& dict)
+{
+    string result;
+
+    if (dict.find("name") != dict.end()) {
+        result += "name: " + dict.at("name") + "\n";
+    }
+
+    for (map<string, string>::const_iterator i = dict.begin();
+         i != dict.end(); ++i) {
+        if (i->first == "name")
+            continue;
+        result += i->first + ": " + i->second + "\n";
+    }
+
+    return result;
+}
+
 MetadataWriter::MetadataWriter(TarSegmentStore *store,
                                const char *path,
                                const char *snapshot_name,
@@ -162,7 +190,7 @@ bool MetadataWriter::find(const string& path)
 {
     const char *path_str = path.c_str();
     while (!old_metadata_eof) {
-        string old_path = uri_decode(old_metadata["path"]);
+        string old_path = uri_decode(old_metadata["name"]);
         int cmp = pathcmp(old_path.c_str(), path_str);
         if (cmp == 0)
             return true;
@@ -175,6 +203,69 @@ bool MetadataWriter::find(const string& path)
     return false;
 }
 
+/* Does a file appear to be unchanged from the previous time it was backed up,
+ * based on stat information? */
+bool MetadataWriter::is_unchanged(const struct stat *stat_buf)
+{
+    if (old_metadata.find("volatile") != old_metadata.end()
+        && parse_int(old_metadata["volatile"]) != 0)
+        return false;
+
+    if (old_metadata.find("ctime") == old_metadata.end())
+        return false;
+    if (stat_buf->st_ctime != parse_int(old_metadata["ctime"]))
+        return false;
+
+    if (old_metadata.find("mtime") == old_metadata.end())
+        return false;
+    if (stat_buf->st_mtime != parse_int(old_metadata["mtime"]))
+        return false;
+
+    if (old_metadata.find("size") == old_metadata.end())
+        return false;
+    if (stat_buf->st_size != parse_int(old_metadata["size"]))
+        return false;
+
+    if (old_metadata.find("inode") == old_metadata.end())
+        return false;
+    string inode = encode_int(major(stat_buf->st_dev))
+        + "/" + encode_int(minor(stat_buf->st_dev))
+        + "/" + encode_int(stat_buf->st_ino);
+    if (inode != old_metadata["inode"])
+        return false;
+
+    return true;
+}
+
+list<ObjectReference> MetadataWriter::get_blocks()
+{
+    list<ObjectReference> blocks;
+
+    /* Parse the list of blocks. */
+    const char *s = old_metadata["data"].c_str();
+    while (*s != '\0') {
+        if (isspace(*s)) {
+            s++;
+            continue;
+        }
+
+        string ref = "";
+        while (*s != '\0' && !isspace(*s)) {
+            char buf[2];
+            buf[0] = *s;
+            buf[1] = '\0';
+            ref += buf;
+            s++;
+        }
+
+        ObjectReference r = ObjectReference::parse(ref);
+        if (!r.is_null())
+            blocks.push_back(r);
+    }
+
+    return blocks;
+}
+
 /* Ensure contents of metadata are flushed to an object. */
 void MetadataWriter::metadata_flush()
 {
@@ -184,6 +275,13 @@ void MetadataWriter::metadata_flush()
     ObjectReference indirect;
     for (list<MetadataItem>::iterator i = items.begin();
          i != items.end(); ++i) {
+        // If indirectly referencing any other metadata logs, be sure those
+        // segments are properly referenced.
+        if (i->reused) {
+            add_segment(i->ref.get_segment());
+            db->UseSegment(i->ref.get_segment(), 1.0);
+        }
+
         // Write out an indirect reference to any previous objects which could
         // be reused
         if (!i->reused || !indirect.merge(i->ref)) {
@@ -230,6 +328,7 @@ void MetadataWriter::metadata_flush()
     ObjectReference ref = meta->get_ref();
     metadata_root << "@" << ref.to_string() << "\n";
     add_segment(ref.get_segment());
+    db->UseSegment(ref.get_segment(), 1.0);
 
     delete meta;
 
@@ -258,12 +357,11 @@ void MetadataWriter::add(dictionary info)
     item.reused = false;
     item.text += encode_dict(info) + "\n";
 
-    if (info == old_metadata) {
-        ObjectReference *ref = ObjectReference::parse(old_metadata_loc);
-        if (ref != NULL) {
+    if (info == old_metadata && !flag_full_metadata) {
+        ObjectReference ref = ObjectReference::parse(old_metadata_loc);
+        if (!ref.is_null()) {
             item.reused = true;
-            item.ref = *ref;
-            delete ref;
+            item.ref = ref;
         }
     }
 
@@ -285,6 +383,7 @@ ObjectReference MetadataWriter::close()
     root->write(store);
     root->checksum();
     add_segment(root->get_ref().get_segment());
+    db->UseSegment(root->get_ref().get_segment(), 1.0);
 
     ObjectReference ref = root->get_ref();
     delete root;