Write out new-style statcache data.

Write out statcache-style data from the metadata logging module of lbs.
This will eventually replace the old statcache implementation, but is not
complete.  This new statcache data is not yet read in or used elsewhere.

In the new format, the data in the statcache file has the same format as
the data in the metadata log itself.  Each stanza with file information is
prefixed with a @@reference line that gives a reference to the location of
the metadata.  If the metadata has not changed, this will allow metadata
log data to be re-used between snapshots.

diff --git a/metadata.cc b/metadata.cc
index 5d30baf..6661f39 100644 (file)
--- a/metadata.cc
+++ b/metadata.cc
@@ -15,22 +15,50 @@
 #include "statcache.h"
 #include "util.h"
 
+using std::list;
 using std::string;
 using std::ostream;
+using std::ostringstream;
 
 static const size_t LBS_METADATA_BLOCK_SIZE = 65536;
 
 /* TODO: Move to header file */
 void add_segment(const string& segment);
 
-MetadataWriter::MetadataWriter(TarSegmentStore *store)
+MetadataWriter::MetadataWriter(TarSegmentStore *store,
+                               const char *path,
+                               const char *snapshot_name,
+                               const char *snapshot_scheme)
 {
+    statcache_path = path;
+    statcache_path += "/statcache2";
+    if (snapshot_scheme != NULL)
+        statcache_path = statcache_path + "-" + snapshot_scheme;
+    statcache_tmp_path = statcache_path + "." + snapshot_name;
+
+    statcache_out = fopen(statcache_tmp_path.c_str(), "w");
+    if (statcache_out == NULL) {
+        fprintf(stderr, "Error opening statcache %s: %m\n",
+                statcache_tmp_path.c_str());
+        throw IOException("Error opening statcache");
+    }
+
     this->store = store;
+    chunk_size = 0;
 }
 
 /* Ensure contents of metadata are flushed to an object. */
 void MetadataWriter::metadata_flush()
 {
+    int offset = 0;
+
+    ostringstream metadata;
+    for (list<MetadataItem>::iterator i = items.begin();
+         i != items.end(); ++i) {
+        metadata << i->text;
+        i->offset = offset;
+        offset += i->text.size();
+    }
     string m = metadata.str();
     if (m.size() == 0)
         return;
@@ -49,16 +77,31 @@ void MetadataWriter::metadata_flush()
 
     delete meta;
 
-    metadata.str("");
+    /* Write these files out to the statcache, and include a reference to where
+     * the metadata lives (so we can re-use it if it has not changed). */
+    for (list<MetadataItem>::const_iterator i = items.begin();
+         i != items.end(); ++i) {
+        ObjectReference r = ref;
+        r.set_range(i->offset, i->text.size());
+
+        string refstr = r.to_string();
+        fprintf(statcache_out, "@@%s\n%s", refstr.c_str(), i->text.c_str());
+    }
+
+    items.clear();
 }
 
 void MetadataWriter::add(const string& path, dictionary info)
 {
-    metadata << "path: " << uri_encode(path) << "\n";
-    metadata << encode_dict(info);
-    metadata << "\n";
+    MetadataItem item;
+    item.offset = 0;
+    item.text = "path: " + uri_encode(path) + "\n";
+    item.text += encode_dict(info) + "\n";
 
-    if (metadata.str().size() > LBS_METADATA_BLOCK_SIZE)
+    items.push_back(item);
+    chunk_size += item.text.size();
+
+    if (chunk_size > LBS_METADATA_BLOCK_SIZE)
         metadata_flush();
 }
 
@@ -76,5 +119,12 @@ ObjectReference MetadataWriter::close()
 
     ObjectReference ref = root->get_ref();
     delete root;
+
+    fclose(statcache_out);
+    if (rename(statcache_tmp_path.c_str(), statcache_path.c_str()) < 0) {
+        fprintf(stderr, "Error renaming statcache from %s to %s: %m\n",
+                statcache_tmp_path.c_str(), statcache_path.c_str());
+    }
+
     return ref;
 }

diff --git a/metadata.h b/metadata.h
index 7a09c74..f86244c 100644 (file)
--- a/metadata.h
+++ b/metadata.h
@@ -10,6 +10,8 @@
 #ifndef _LBS_METADATA_H
 #define _LBS_METADATA_H
 
+#include <stdio.h>
+#include <list>
 #include <string>
 #include <sstream>
 
@@ -17,17 +19,33 @@
 #include "ref.h"
 #include "util.h"
 
+/* Metadata for a single inode, ready to be written out. */
+struct MetadataItem {
+    int offset;
+    std::string text;
+};
+
 class MetadataWriter {
 public:
-    MetadataWriter(TarSegmentStore *store);
+    MetadataWriter(TarSegmentStore *store, const char *path,
+                   const char *snapshot_name, const char *snapshot_scheme);
     void add(const std::string& path, dictionary info);
     ObjectReference close();
 
 private:
     void metadata_flush();
 
+    // Where are objects eventually written to?
     TarSegmentStore *store;
-    std::ostringstream metadata, metadata_root;
+
+    // File descriptors for reading/writing local statcache data
+    std::string statcache_path, statcache_tmp_path;
+    FILE *statcache_out;
+
+    // Metadata not yet written out to the segment store
+    size_t chunk_size;
+    std::list<MetadataItem> items;
+    std::ostringstream metadata_root;
 };
 
 #endif // _LBS_METADATA_H

diff --git a/scandir.cc b/scandir.cc
index d5a8c86..4f84955 100644 (file)
--- a/scandir.cc
+++ b/scandir.cc
@@ -257,6 +257,7 @@ void dump_inode(const string& path,         // Path within snapshot
     printf("%s\n", path.c_str());
 
     file_info["mode"] = encode_int(stat_buf.st_mode & 07777, 8);
+    file_info["ctime"] = encode_int(stat_buf.st_ctime);
     file_info["mtime"] = encode_int(stat_buf.st_mtime);
     file_info["user"] = encode_int(stat_buf.st_uid);
     file_info["group"] = encode_int(stat_buf.st_gid);
@@ -273,11 +274,12 @@ void dump_inode(const string& path,         // Path within snapshot
 
     if (stat_buf.st_nlink > 1 && (stat_buf.st_mode & S_IFMT) != S_IFDIR) {
         file_info["links"] = encode_int(stat_buf.st_nlink);
-        file_info["inode"] = encode_int(major(stat_buf.st_dev))
-            + "/" + encode_int(minor(stat_buf.st_dev))
-            + "/" + encode_int(stat_buf.st_ino);
     }
 
+    file_info["inode"] = encode_int(major(stat_buf.st_dev))
+        + "/" + encode_int(minor(stat_buf.st_dev))
+        + "/" + encode_int(stat_buf.st_ino);
+
     char inode_type;
 
     switch (stat_buf.st_mode & S_IFMT) {
@@ -664,13 +666,16 @@ int main(int argc, char *argv[])
 
     tss = new TarSegmentStore(backup_dest, db);
 
-    metawriter = new MetadataWriter(tss);
-
     /* Initialize the stat cache, for skipping over unchanged files. */
     statcache = new StatCache;
     statcache->Open(localdb_dir.c_str(), desc_buf,
                     backup_scheme.size() ? backup_scheme.c_str() : NULL);
 
+    metawriter = new MetadataWriter(tss, localdb_dir.c_str(), desc_buf,
+                                    backup_scheme.size()
+                                        ? backup_scheme.c_str()
+                                        : NULL);
+
     scanfile(".", false);
 
     ObjectReference root_ref = metawriter->close();