From: Michael Vrable <mvrable@cs.ucsd.edu>
Date: Fri, 30 May 2008 22:13:56 +0000 (-0700)
Subject: Track quantity of data referenced in old segments more precisely.
X-Git-Url: http://git.vrable.net/?p=cumulus.git;a=commitdiff_plain;h=c10290a5e5b066bedfec4ae23a799e5d5a1ff009

Track quantity of data referenced in old segments more precisely.

When calculating used space in segments, treat a reference to just a subset
of data in an object (for example, with subfile incrementals) as using just
that data, instead of the entire object.  This should provide better
information to the segment cleaner.
---

diff --git a/localdb.cc b/localdb.cc
index c74e27d..ecf83aa 100644
--- a/localdb.cc
+++ b/localdb.cc
@@ -32,11 +32,13 @@
 #include <string.h>
 #include <sqlite3.h>
 
+#include <algorithm>
 #include <string>
 
 #include "localdb.h"
 #include "store.h"
 
+using std::min;
 using std::string;
 
 /* Helper function to prepare a statement for execution in the current
@@ -376,20 +378,61 @@ void LocalDb::UseObject(const ObjectReference& ref)
     if (!ref.is_normal())
         return;
 
-    stmt = Prepare("insert or ignore into snapshot_refs "
-                   "select segmentid, object, size from block_index "
+    int64_t old_size = 0;
+    stmt = Prepare("select size from snapshot_refs "
                    "where segmentid = ? and object = ?");
     sqlite3_bind_int64(stmt, 1, SegmentToId(ref.get_segment()));
     string obj = ref.get_sequence();
     sqlite3_bind_text(stmt, 2, obj.c_str(), obj.size(), SQLITE_TRANSIENT);
-
     rc = sqlite3_step(stmt);
-    if (rc != SQLITE_DONE) {
-        fprintf(stderr, "Could not execute INSERT statement!\n");
-        ReportError(rc);
+    if (rc == SQLITE_ROW) {
+        old_size = sqlite3_column_int64(stmt, 0);
     }
+    sqlite3_finalize(stmt);
 
+    int64_t block_size = 0;
+    stmt = Prepare("select size from block_index "
+                   "where segmentid = ? and object = ?");
+    sqlite3_bind_int64(stmt, 1, SegmentToId(ref.get_segment()));
+    obj = ref.get_sequence();
+    sqlite3_bind_text(stmt, 2, obj.c_str(), obj.size(), SQLITE_TRANSIENT);
+    rc = sqlite3_step(stmt);
+    if (rc == SQLITE_ROW) {
+        block_size = sqlite3_column_int64(stmt, 0);
+    } else {
+        string refstr = ref.to_string();
+        fprintf(stderr, "No block found in block_index for %s\n",
+                refstr.c_str());
+        sqlite3_finalize(stmt);
+        return;
+    }
     sqlite3_finalize(stmt);
+
+    int64_t new_size = old_size;
+    if (ref.has_range()) {
+        new_size += ref.get_range_length();
+        new_size = min(new_size, block_size);
+    } else {
+        new_size = block_size;
+    }
+
+    if (new_size != old_size) {
+        stmt = Prepare("insert or replace "
+                       "into snapshot_refs(segmentid, object, size) "
+                       "values (?, ?, ?)");
+        sqlite3_bind_int64(stmt, 1, SegmentToId(ref.get_segment()));
+        obj = ref.get_sequence();
+        sqlite3_bind_text(stmt, 2, obj.c_str(), obj.size(), SQLITE_TRANSIENT);
+        sqlite3_bind_int64(stmt, 3, new_size);
+
+        rc = sqlite3_step(stmt);
+        if (rc != SQLITE_DONE) {
+            fprintf(stderr, "Could not execute INSERT statement!\n");
+            ReportError(rc);
+        }
+
+        sqlite3_finalize(stmt);
+    }
 }
 
 void LocalDb::UseSegment(const std::string &segment, double utilization)