From: Michael Vrable <vrable@cs.hmc.edu>
Date: Thu, 2 May 2013 04:17:58 +0000 (-0700)
Subject: Write out a generic segment metadata file with backups.
X-Git-Url: https://git.vrable.net/?a=commitdiff_plain;h=ca9732d7fe1cc7023f5642fd50774982bb430d64;p=cumulus.git

Write out a generic segment metadata file with backups.

This can be used to rebuild local database state without redownloading all
segments.
---

diff --git a/localdb.cc b/localdb.cc
index 15b58e6..fd86605 100644
--- a/localdb.cc
+++ b/localdb.cc
@@ -32,12 +32,14 @@
 #include <sqlite3.h>
 
 #include <algorithm>
+#include <map>
 #include <string>
 
 #include "localdb.h"
 #include "store.h"
 #include "util.h"
 
+using std::map;
 using std::max;
 using std::min;
 using std::set;
@@ -516,36 +518,33 @@ void LocalDb::SetSegmentMetadata(const std::string &segment,
     sqlite3_finalize(stmt);
 }
 
-bool LocalDb::GetSegmentMetadata(const string &segment,
-                                 string *seg_path,
-                                 string *seg_checksum)
+map<string, string> LocalDb::GetSegmentMetadata(const string &segment)
 {
     int rc;
     sqlite3_stmt *stmt;
-    ObjectReference ref;
-    int found = false;
+    map<string, string> info;
+
+    // Names in the returned map, in the order returned from the select
+    // statement below.
+    static const char *fields[] = {
+        "datetime", "path", "checksum", "data_size", "disk_size", "type", NULL
+    };
 
-    stmt = Prepare("select path, checksum from segments where segment = ?");
+    stmt = Prepare("select datetime(timestamp), path, checksum, "
+                   "    data_size, disk_size, type "
+                   "from segments where segment = ?");
     sqlite3_bind_text(stmt, 1, segment.c_str(), segment.size(),
                       SQLITE_TRANSIENT);
 
     rc = sqlite3_step(stmt);
     if (rc == SQLITE_DONE) {
     } else if (rc == SQLITE_ROW) {
-        found = true;
-        const char *val;
-
-        val = (const char *)sqlite3_column_text(stmt, 0);
-        if (val == NULL)
-            found = false;
-        else
-            *seg_path = val;
-
-        val = (const char *)sqlite3_column_text(stmt, 1);
-        if (val == NULL)
-            found = false;
-        else
-            *seg_checksum = val;
+        info["segment"] = segment;
+        for (int i = 0; fields[i] != NULL; i++) {
+            const char *val = (const char *)sqlite3_column_text(stmt, i);
+            if (val != NULL)
+                info[fields[i]] = val;
+        }
     } else {
         fprintf(stderr, "Could not execute SELECT statement!\n");
         ReportError(rc);
@@ -553,7 +552,7 @@ bool LocalDb::GetSegmentMetadata(const string &segment,
 
     sqlite3_finalize(stmt);
 
-    return found;
+    return info;
 }
 
 /* Look up and return the packed representation of the subblock chunk
diff --git a/localdb.h b/localdb.h
index 2ffb196..4e1292f 100644
--- a/localdb.h
+++ b/localdb.h
@@ -53,8 +53,8 @@ public:
                             const std::string &checksum,
                             const std::string &type, int data_size,
                             int disk_size);
-    bool GetSegmentMetadata(const std::string &segment,
-                            std::string *seg_path, std::string *seg_checksum);
+    std::map<std::string, std::string> GetSegmentMetadata(
+        const std::string &segment);
 
     bool LoadChunkSignatures(ObjectReference ref,
                              void **buf, size_t *len,
diff --git a/main.cc b/main.cc
index ef8f2d7..ede33c6 100644
--- a/main.cc
+++ b/main.cc
@@ -860,8 +860,12 @@ int main(int argc, char *argv[])
     std::set<string> segment_list = db->GetUsedSegments();
     for (std::set<string>::iterator i = segment_list.begin();
          i != segment_list.end(); ++i) {
-        string seg_path, seg_csum;
-        if (db->GetSegmentMetadata(*i, &seg_path, &seg_csum)) {
+        map<string, string> segment_metadata = db->GetSegmentMetadata(*i);
+        if (segment_metadata.count("path")
+            && segment_metadata.count("checksum"))
+        {
+            string seg_path = segment_metadata["path"];
+            string seg_csum = segment_metadata["checksum"];
             const char *raw_checksum = NULL;
             if (strncmp(seg_csum.c_str(), csum_type,
                         strlen(csum_type)) == 0) {
@@ -888,6 +892,33 @@ int main(int argc, char *argv[])
 
     checksum_file->send();
 
+    /* Write out a summary file with metadata for all the segments in this
+     * snapshot (can be used to reconstruct database contents if needed). */
+    string dbmeta_filename = "snapshot-";
+    if (backup_scheme.size() > 0)
+        dbmeta_filename += backup_scheme + "-";
+    dbmeta_filename += timestamp + ".meta";
+    RemoteFile *dbmeta_file = remote->alloc_file(dbmeta_filename,
+                                                   "meta");
+    FILE *dbmeta = fdopen(dbmeta_file->get_fd(), "w");
+
+    for (std::set<string>::iterator i = segment_list.begin();
+         i != segment_list.end(); ++i) {
+        map<string, string> segment_metadata = db->GetSegmentMetadata(*i);
+        if (segment_metadata.size() > 0) {
+            map<string, string>::const_iterator j;
+            for (j = segment_metadata.begin();
+                 j != segment_metadata.end(); ++j)
+            {
+                fprintf(dbmeta, "%s: %s\n",
+                        j->first.c_str(), j->second.c_str());
+            }
+            fprintf(dbmeta, "\n");
+        }
+    }
+    fclose(dbmeta);
+    dbmeta_file->send();
+
     db->Close();
 
     /* All other files should be flushed to remote storage before writing the
diff --git a/remote.h b/remote.h
index ac7d9dc..10a0a96 100644
--- a/remote.h
+++ b/remote.h
@@ -71,6 +71,7 @@ public:
     int get_fd() const { return fd; }
 
     const std::string &get_local_path() const { return local_path; }
+    const std::string &get_remote_path() const { return remote_path; }
 
     /* Called when the file is finished--request that it be sent to the remote
      * server.  This will delete the RemoteFile object. */
diff --git a/store.cc b/store.cc
index 115529f..ccd055d 100644
--- a/store.cc
+++ b/store.cc
@@ -315,8 +315,8 @@ void TarSegmentStore::close_segment(const string &group)
             checksum = segment_checksum.checksum_str();
         }
 
-        db->SetSegmentMetadata(segment->name, segment->basename, checksum,
-                               group, segment->data_size, disk_size);
+        db->SetSegmentMetadata(segment->name, segment->rf->get_remote_path(),
+                               checksum, group, segment->data_size, disk_size);
     }
 
     segment->rf->send();