X-Git-Url: http://git.vrable.net/?a=blobdiff_plain;f=lbs.py;h=507b9e13bcef1fb5ec078637ed1f95d588ca9245;hb=d3f9ee42a4023631059f16cc2a8b96b9540750b2;hp=4d2641f22e5cdb072b2d5135a3de4a430d9c0fd2;hpb=4749293dc3e518f5de8a50c9da7384b1c7794ce2;p=cumulus.git

diff --git a/lbs.py b/lbs.py
index 4d2641f..507b9e1 100644
--- a/lbs.py
+++ b/lbs.py
@@ -263,7 +263,7 @@ def parse(lines, terminate=None):
             last_key = None
             continue
 
-        m = re.match(r"^(\w+):\s*(.*)$", l)
+        m = re.match(r"^([-\w]+):\s*(.*)$", l)
         if m:
             dict[m.group(1)] = m.group(2)
             last_key = m.group(1)
@@ -483,6 +483,13 @@ class LocalDatabase:
 
         cur = self.cursor()
 
+        # Find the id of the last snapshot to be created.  This is used for
+        # measuring time in a way: we record this value in each segment we
+        # expire on this run, and then on a future run can tell if there have
+        # been intervening backups made.
+        cur.execute("select max(snapshotid) from snapshots")
+        last_snapshotid = cur.fetchone()[0]
+
         # Get the list of old snapshots for this scheme.  Delete all the old
         # ones.  Rules for what to keep:
         #   - Always keep the most recent snapshot.
@@ -523,12 +530,21 @@ class LocalDatabase:
         cur.execute("""delete from segments where segmentid not in
                            (select segmentid from segments_used)""")
 
-        # Finally, delete objects contained in non-existent segments.  We can't
-        # simply delete unused objects, since we use the set of unused objects
-        # to determine the used/free ratio of segments.
+        # Delete unused objects in the block_index table.  By "unused", we mean
+        # any object which was stored in a segment which has been deleted, and
+        # any object in a segment which was marked for cleaning and has had
+        # cleaning performed already (the expired time is less than the current
+        # largest snapshot id).
         cur.execute("""delete from block_index
-                       where segmentid not in
-                           (select segmentid from segments)""")
+                       where segmentid not in (select segmentid from segments)
+                          or segmentid in (select segmentid from segments
+                                           where expire_time < ?)""",
+                    (last_snapshotid,))
+
+        # Remove sub-block signatures for deleted objects.
+        cur.execute("""delete from subblock_signatures
+                       where blockid not in
+                           (select blockid from block_index)""")
 
     # Segment cleaning.
     class SegmentInfo(Struct): pass
@@ -553,7 +569,8 @@ class LocalDatabase:
         cur = self.cursor()
         segments = []
         cur.execute("""select segmentid, used, size, mtime,
-                       julianday('now') - mtime as age from segment_info""")
+                       julianday('now') - mtime as age from segment_info
+                       where expire_time is null""")
         for row in cur:
             info = self.SegmentInfo()
             info.id = row[0]
@@ -604,7 +621,11 @@ class LocalDatabase:
             raise TypeError("Invalid segment: %s, must be of type int or SegmentInfo, not %s" % (segment, type(segment)))
 
         cur = self.cursor()
-        cur.execute("update block_index set expired = 1 where segmentid = ?",
+        cur.execute("select max(snapshotid) from snapshots")
+        last_snapshotid = cur.fetchone()[0]
+        cur.execute("update segments set expire_time = ? where segmentid = ?",
+                    (last_snapshotid, id))
+        cur.execute("update block_index set expired = 0 where segmentid = ?",
                     (id,))
 
     def balance_expired_objects(self):
@@ -733,5 +754,6 @@ class LocalDatabase:
         cutoffs.reverse()
         for i in range(len(cutoffs)):
             cur.execute("""update block_index set expired = ?
-                           where round(? - timestamp) > ?""",
+                           where round(? - timestamp) > ?
+                             and expired is not null""",
                         (i, now, cutoffs[i]))