Extend tracking of used segments to cover metadata segments.

In the segments_used table in the local database, include segments that
contain metadata in addition to data segments.  Additionally, slightly
extend the segment tracking code so that the modification time of segments
is written out.

The exact utilization of the metadata segments is not yet computed; for now
the utilization is listed as 1.0 even if it is actually less.

diff --git a/contrib/upgrade0.6-localdb.sql b/contrib/upgrade0.6-localdb.sql
index d1449ac..7256f9f 100644 (file)
--- a/contrib/upgrade0.6-localdb.sql
+++ b/contrib/upgrade0.6-localdb.sql
@@ -23,6 +23,8 @@ create table segments_used (
     segmentid integer not null,
     utilization real
 );
+create unique index segments_used_index
+    on segments_used(snapshotid, segmentid);
 
 alter table snapshots add column intent real;
 

diff --git a/localdb.cc b/localdb.cc
index 50c15e9..1c44fa4 100644 (file)
--- a/localdb.cc
+++ b/localdb.cc
@@ -125,14 +125,21 @@ void LocalDb::Close()
     int rc;
 
     /* Summarize the snapshot_refs table into segments_used. */
-    sqlite3_stmt *stmt = Prepare("insert into segments_used "
-                                 "select ? as snapshotid, segmentid, "
-                                 "cast(used as real) / size as utilization "
-                                 "from "
-                                 "(select segmentid, sum(size) as used "
-                                 "from snapshot_refs group by segmentid) "
-                                 "join segments using (segmentid)");
+    sqlite3_stmt *stmt = Prepare(
+        "insert or replace into segments_used "
+        "select ? as snapshotid, segmentid, max(utilization) from ("
+        "    select segmentid, cast(used as real) / size as utilization "
+        "    from "
+        "    (select segmentid, sum(size) as used from snapshot_refs "
+        "       group by segmentid) "
+        "    join segments using (segmentid) "
+        "  union "
+        "    select segmentid, utilization from segments_used "
+        "    where snapshotid = ? "
+        ") group by segmentid"
+    );
     sqlite3_bind_int64(stmt, 1, snapshotid);
+    sqlite3_bind_int64(stmt, 2, snapshotid);
     rc = sqlite3_step(stmt);
     if (rc != SQLITE_OK && rc != SQLITE_DONE) {
         ReportError(rc);
@@ -239,6 +246,15 @@ void LocalDb::StoreObject(const ObjectReference& ref,
     }
 
     sqlite3_finalize(stmt);
+
+    if (age != 0.0) {
+        stmt = Prepare("update segments set mtime = max(mtime, ?) "
+                       "where segmentid = ?");
+        sqlite3_bind_double(stmt, 1, age);
+        sqlite3_bind_int64(stmt, 2, SegmentToId(ref.get_segment()));
+        rc = sqlite3_step(stmt);
+        sqlite3_finalize(stmt);
+    }
 }
 
 ObjectReference LocalDb::FindObject(const string &checksum, int64_t size)
@@ -357,6 +373,27 @@ void LocalDb::UseObject(const ObjectReference& ref)
     sqlite3_finalize(stmt);
 }
 
+void LocalDb::UseSegment(const std::string &segment, double utilization)
+{
+    int rc;
+    sqlite3_stmt *stmt;
+
+    stmt = Prepare("insert or replace "
+                   "into segments_used(snapshotid, segmentid, utilization) "
+                   "values (?, ?, ?)");
+    sqlite3_bind_int64(stmt, 1, snapshotid);
+    sqlite3_bind_int64(stmt, 2, SegmentToId(segment));
+    sqlite3_bind_double(stmt, 3, utilization);
+
+    rc = sqlite3_step(stmt);
+    if (rc != SQLITE_DONE) {
+        fprintf(stderr, "Could not insert segment use record!\n");
+        ReportError(rc);
+    }
+
+    sqlite3_finalize(stmt);
+}
+
 void LocalDb::SetSegmentChecksum(const std::string &segment,
                                  const std::string &path,
                                  const std::string &checksum,
@@ -365,7 +402,8 @@ void LocalDb::SetSegmentChecksum(const std::string &segment,
     int rc;
     sqlite3_stmt *stmt;
 
-    stmt = Prepare("update segments set path = ?, checksum = ?, size = ? "
+    stmt = Prepare("update segments set path = ?, checksum = ?, size = ?, "
+                   "mtime = coalesce(mtime, julianday('now')) "
                    "where segmentid = ?");
     sqlite3_bind_text(stmt, 1, path.c_str(), path.size(),
                       SQLITE_TRANSIENT);

diff --git a/localdb.h b/localdb.h
index a89923f..1bfaf3b 100644 (file)
--- a/localdb.h
+++ b/localdb.h
@@ -31,6 +31,7 @@ public:
                      int *group);
     bool IsAvailable(const ObjectReference &ref);
     void UseObject(const ObjectReference& ref);
+    void UseSegment(const std::string &segment, double utilization);
 
     void SetSegmentChecksum(const std::string &segment, const std::string &path,
                             const std::string &checksum, int size);

diff --git a/metadata.cc b/metadata.cc
index cb3dabb..2b5f12a 100644 (file)
--- a/metadata.cc
+++ b/metadata.cc
@@ -12,6 +12,7 @@
 #include <map>
 
 #include "metadata.h"
+#include "localdb.h"
 #include "ref.h"
 #include "store.h"
 #include "util.h"
@@ -29,6 +30,7 @@ static const size_t LBS_METADATA_BLOCK_SIZE = 65536;
 bool flag_full_metadata = false;
 
 /* TODO: Move to header file */
+extern LocalDb *db;
 void add_segment(const string& segment);
 
 /* Like strcmp, but sorts in the order that files will be visited in the
@@ -275,8 +277,10 @@ void MetadataWriter::metadata_flush()
          i != items.end(); ++i) {
         // If indirectly referencing any other metadata logs, be sure those
         // segments are properly referenced.
-        if (i->reused)
+        if (i->reused) {
             add_segment(i->ref.get_segment());
+            db->UseSegment(i->ref.get_segment(), 1.0);
+        }
 
         // Write out an indirect reference to any previous objects which could
         // be reused
@@ -324,6 +328,7 @@ void MetadataWriter::metadata_flush()
     ObjectReference ref = meta->get_ref();
     metadata_root << "@" << ref.to_string() << "\n";
     add_segment(ref.get_segment());
+    db->UseSegment(ref.get_segment(), 1.0);
 
     delete meta;
 
@@ -378,6 +383,7 @@ ObjectReference MetadataWriter::close()
     root->write(store);
     root->checksum();
     add_segment(root->get_ref().get_segment());
+    db->UseSegment(root->get_ref().get_segment(), 1.0);
 
     ObjectReference ref = root->get_ref();
     delete root;

diff --git a/schema.sql b/schema.sql
index 3cf175b..8529e32 100644 (file)
--- a/schema.sql
+++ b/schema.sql
@@ -41,6 +41,8 @@ create table segments_used (
     segmentid integer not null,
     utilization real
 );
+create unique index segments_used_index
+    on segments_used(snapshotid, segmentid);
 
 -- Overall estimate of segment utilization, for all snapshots combined.
 create view segment_info as