From ca9732d7fe1cc7023f5642fd50774982bb430d64 Mon Sep 17 00:00:00 2001 From: Michael Vrable Date: Wed, 1 May 2013 21:17:58 -0700 Subject: [PATCH] Write out a generic segment metadata file with backups. This can be used to rebuild local database state without redownloading all segments. --- localdb.cc | 41 ++++++++++++++++++++--------------------- localdb.h | 4 ++-- main.cc | 35 +++++++++++++++++++++++++++++++++-- remote.h | 1 + store.cc | 4 ++-- 5 files changed, 58 insertions(+), 27 deletions(-) diff --git a/localdb.cc b/localdb.cc index 15b58e6..fd86605 100644 --- a/localdb.cc +++ b/localdb.cc @@ -32,12 +32,14 @@ #include #include +#include #include #include "localdb.h" #include "store.h" #include "util.h" +using std::map; using std::max; using std::min; using std::set; @@ -516,36 +518,33 @@ void LocalDb::SetSegmentMetadata(const std::string &segment, sqlite3_finalize(stmt); } -bool LocalDb::GetSegmentMetadata(const string &segment, - string *seg_path, - string *seg_checksum) +map LocalDb::GetSegmentMetadata(const string &segment) { int rc; sqlite3_stmt *stmt; - ObjectReference ref; - int found = false; + map info; + + // Names in the returned map, in the order returned from the select + // statement below. + static const char *fields[] = { + "datetime", "path", "checksum", "data_size", "disk_size", "type", NULL + }; - stmt = Prepare("select path, checksum from segments where segment = ?"); + stmt = Prepare("select datetime(timestamp), path, checksum, " + " data_size, disk_size, type " + "from segments where segment = ?"); sqlite3_bind_text(stmt, 1, segment.c_str(), segment.size(), SQLITE_TRANSIENT); rc = sqlite3_step(stmt); if (rc == SQLITE_DONE) { } else if (rc == SQLITE_ROW) { - found = true; - const char *val; - - val = (const char *)sqlite3_column_text(stmt, 0); - if (val == NULL) - found = false; - else - *seg_path = val; - - val = (const char *)sqlite3_column_text(stmt, 1); - if (val == NULL) - found = false; - else - *seg_checksum = val; + info["segment"] = segment; + for (int i = 0; fields[i] != NULL; i++) { + const char *val = (const char *)sqlite3_column_text(stmt, i); + if (val != NULL) + info[fields[i]] = val; + } } else { fprintf(stderr, "Could not execute SELECT statement!\n"); ReportError(rc); @@ -553,7 +552,7 @@ bool LocalDb::GetSegmentMetadata(const string &segment, sqlite3_finalize(stmt); - return found; + return info; } /* Look up and return the packed representation of the subblock chunk diff --git a/localdb.h b/localdb.h index 2ffb196..4e1292f 100644 --- a/localdb.h +++ b/localdb.h @@ -53,8 +53,8 @@ public: const std::string &checksum, const std::string &type, int data_size, int disk_size); - bool GetSegmentMetadata(const std::string &segment, - std::string *seg_path, std::string *seg_checksum); + std::map GetSegmentMetadata( + const std::string &segment); bool LoadChunkSignatures(ObjectReference ref, void **buf, size_t *len, diff --git a/main.cc b/main.cc index ef8f2d7..ede33c6 100644 --- a/main.cc +++ b/main.cc @@ -860,8 +860,12 @@ int main(int argc, char *argv[]) std::set segment_list = db->GetUsedSegments(); for (std::set::iterator i = segment_list.begin(); i != segment_list.end(); ++i) { - string seg_path, seg_csum; - if (db->GetSegmentMetadata(*i, &seg_path, &seg_csum)) { + map segment_metadata = db->GetSegmentMetadata(*i); + if (segment_metadata.count("path") + && segment_metadata.count("checksum")) + { + string seg_path = segment_metadata["path"]; + string seg_csum = segment_metadata["checksum"]; const char *raw_checksum = NULL; if (strncmp(seg_csum.c_str(), csum_type, strlen(csum_type)) == 0) { @@ -888,6 +892,33 @@ int main(int argc, char *argv[]) checksum_file->send(); + /* Write out a summary file with metadata for all the segments in this + * snapshot (can be used to reconstruct database contents if needed). */ + string dbmeta_filename = "snapshot-"; + if (backup_scheme.size() > 0) + dbmeta_filename += backup_scheme + "-"; + dbmeta_filename += timestamp + ".meta"; + RemoteFile *dbmeta_file = remote->alloc_file(dbmeta_filename, + "meta"); + FILE *dbmeta = fdopen(dbmeta_file->get_fd(), "w"); + + for (std::set::iterator i = segment_list.begin(); + i != segment_list.end(); ++i) { + map segment_metadata = db->GetSegmentMetadata(*i); + if (segment_metadata.size() > 0) { + map::const_iterator j; + for (j = segment_metadata.begin(); + j != segment_metadata.end(); ++j) + { + fprintf(dbmeta, "%s: %s\n", + j->first.c_str(), j->second.c_str()); + } + fprintf(dbmeta, "\n"); + } + } + fclose(dbmeta); + dbmeta_file->send(); + db->Close(); /* All other files should be flushed to remote storage before writing the diff --git a/remote.h b/remote.h index ac7d9dc..10a0a96 100644 --- a/remote.h +++ b/remote.h @@ -71,6 +71,7 @@ public: int get_fd() const { return fd; } const std::string &get_local_path() const { return local_path; } + const std::string &get_remote_path() const { return remote_path; } /* Called when the file is finished--request that it be sent to the remote * server. This will delete the RemoteFile object. */ diff --git a/store.cc b/store.cc index 115529f..ccd055d 100644 --- a/store.cc +++ b/store.cc @@ -315,8 +315,8 @@ void TarSegmentStore::close_segment(const string &group) checksum = segment_checksum.checksum_str(); } - db->SetSegmentMetadata(segment->name, segment->basename, checksum, - group, segment->data_size, disk_size); + db->SetSegmentMetadata(segment->name, segment->rf->get_remote_path(), + checksum, group, segment->data_size, disk_size); } segment->rf->send(); -- 2.20.1