Preserve the "timestamp" database field when expiring segments.

When segments are repacked, we would like to keep the original timestamp
for the old objects which are written into new segments.  These changes
should now propagate that timestamp value.

diff --git a/localdb.cc b/localdb.cc
index 951f35c..1388881 100644 (file)
--- a/localdb.cc
+++ b/localdb.cc
@@ -144,14 +144,21 @@ string LocalDb::IdToSegment(int64_t segmentid)
 }
 
 void LocalDb::StoreObject(const ObjectReference& ref,
-                          const string &checksum, int64_t size)
+                          const string &checksum, int64_t size,
+                          double age)
 {
     int rc;
     sqlite3_stmt *stmt;
 
-    stmt = Prepare("insert into "
-                   "block_index(segmentid, object, checksum, size, timestamp) "
-                   "values (?, ?, ?, ?, julianday('now'))");
+    if (age == 0.0) {
+        stmt = Prepare("insert into block_index("
+                       "segmentid, object, checksum, size, timestamp) "
+                       "values (?, ?, ?, ?, julianday('now'))");
+    } else {
+        stmt = Prepare("insert into block_index("
+                       "segmentid, object, checksum, size, timestamp) "
+                       "values (?, ?, ?, ?, ?)");
+    }
 
     sqlite3_bind_int64(stmt, 1, SegmentToId(ref.get_segment()));
     string obj = ref.get_sequence();
@@ -159,6 +166,8 @@ void LocalDb::StoreObject(const ObjectReference& ref,
     sqlite3_bind_text(stmt, 3, checksum.c_str(), checksum.size(),
                       SQLITE_TRANSIENT);
     sqlite3_bind_int64(stmt, 4, size);
+    if (age != 0.0)
+        sqlite3_bind_double(stmt, 5, age);
 
     rc = sqlite3_step(stmt);
     if (rc != SQLITE_DONE) {
@@ -194,13 +203,13 @@ ObjectReference LocalDb::FindObject(const string &checksum, int64_t size)
     return ref;
 }
 
-bool LocalDb::IsOldObject(const string &checksum, int64_t size)
+bool LocalDb::IsOldObject(const string &checksum, int64_t size, double *age)
 {
     int rc;
     sqlite3_stmt *stmt;
     bool found = false;
 
-    stmt = Prepare("select segmentid, object from block_index "
+    stmt = Prepare("select segmentid, object, timestamp from block_index "
                    "where checksum = ? and size = ?");
     sqlite3_bind_text(stmt, 1, checksum.c_str(), checksum.size(),
                       SQLITE_TRANSIENT);
@@ -211,6 +220,7 @@ bool LocalDb::IsOldObject(const string &checksum, int64_t size)
         found = false;
     } else if (rc == SQLITE_ROW) {
         found = true;
+        *age = sqlite3_column_double(stmt, 2);
     } else {
         fprintf(stderr, "Could not execute SELECT statement!\n");
     }

diff --git a/localdb.h b/localdb.h
index bdf3b7c..3f5e5ba 100644 (file)
--- a/localdb.h
+++ b/localdb.h
@@ -24,9 +24,9 @@ public:
     void Open(const char *path, const char *snapshot_name);
     void Close();
     void StoreObject(const ObjectReference& ref,
-                     const std::string &checksum, int64_t size);
+                     const std::string &checksum, int64_t size, double age);
     ObjectReference FindObject(const std::string &checksum, int64_t size);
-    bool IsOldObject(const std::string &checksum, int64_t size);
+    bool IsOldObject(const std::string &checksum, int64_t size, double *age);
     void UseObject(const ObjectReference& ref);
 private:
     sqlite3 *db;

diff --git a/scandir.cc b/scandir.cc
index 99498c4..71fba9d 100644 (file)
--- a/scandir.cc
+++ b/scandir.cc
@@ -142,6 +142,7 @@ int64_t dumpfile(int fd, dictionary &file_info, const string &path)
 
         // Either find a copy of this block in an already-existing segment, or
         // index it so it can be re-used in the future
+        double block_age = 0.0;
         SHA1Checksum block_hash;
         block_hash.process(block_buf, bytes);
         string block_csum = block_hash.checksum_str();
@@ -158,8 +159,10 @@ int64_t dumpfile(int fd, dictionary &file_info, const string &path)
              * put it in a separate group, so that old objects get grouped
              * together.  The hope is that these old objects will continue to
              * be used in the future, and we obtain segments which will
-             * continue to be well-utilized. */
-            if (db->IsOldObject(block_csum, bytes))
+             * continue to be well-utilized.  Additionally, keep track of the
+             * age of the data by looking up the age of the block which was
+             * expired and using that instead of the current time. */
+            if (db->IsOldObject(block_csum, bytes, &block_age))
                 o->set_group("compacted");
             else
                 o->set_group("data");
@@ -167,7 +170,7 @@ int64_t dumpfile(int fd, dictionary &file_info, const string &path)
             o->set_data(block_buf, bytes);
             o->write(tss);
             ref = o->get_ref();
-            db->StoreObject(ref, block_csum, bytes);
+            db->StoreObject(ref, block_csum, bytes, block_age);
             delete o;
         }