X-Git-Url: http://git.vrable.net/?a=blobdiff_plain;f=cleaner%2Fcleaner;h=b018d0ddccf3f0c3bda24e20a5ce77a558e8b874;hb=f5a625989f95db5466d358af892301aee8acdc7f;hp=fb3ce0559ae99f751b4ecbf623510e126eea119c;hpb=c665095a0fb48ac1304f906eaca563ba8d01e6e8;p=bluesky.git

diff --git a/cleaner/cleaner b/cleaner/cleaner
index fb3ce05..b018d0d 100755
--- a/cleaner/cleaner
+++ b/cleaner/cleaner
@@ -49,6 +49,9 @@ class FileBackend:
         fp.write(data)
         fp.close()
 
+    def delete(self, filename):
+        os.unlink(os.path.join(self.path, filename))
+
     def loc_to_name(self, location):
         return "log-%08d-%08d" % (location)
 
@@ -268,8 +271,10 @@ class InodeMap:
         print
         print "Segment utilizations:"
         for (s, u) in sorted(util.segments.items()):
-            #if u[1] > 0:
             print "%s: %s %s" % (s, u, float(u[1]) / u[0])
+            if u[1] == 0:
+                print "Deleting..."
+                backend.delete(s)
 
         self.inodes = inodes
         self.util = util
@@ -325,14 +330,15 @@ class InodeMap:
         log.write(new_checkpoint, 2)
         self.checkpoint_record = new_checkpoint
 
-def rewrite_inode(backend, inode_map, inum, log):
+def rewrite_inode(backend, inode_map, inum, log, copy_data=True):
     inode = inode_map.inodes[inum]
-    blocks = []
-    for l in inode.links:
-        data = load_item(backend, l[1])
-        blocks.append(data)
-        log.write(data, 0)
-    inode.links = [(b.id, b.location) for b in blocks]
+    if copy_data:
+        blocks = []
+        for l in inode.links:
+            data = load_item(backend, l[1])
+            blocks.append(data)
+            log.write(data, 0)
+        inode.links = [(b.id, b.location) for b in blocks]
     log.write(inode, 1)
     inode_map.mark_updated(inum)
 
@@ -340,26 +346,38 @@ def run_cleaner(backend, inode_map, log):
     # Determine which segments are poorly utilized and should be cleaned.  We
     # need better heuristics here.
     for (s, u) in sorted(inode_map.util.segments.items()):
-        if float(u[1]) / u[0] < 0.99 and u[1] > 0:
+        if float(u[1]) / u[0] < 0.6 and u[1] > 0:
             print "Should clean segment", s
             loc = backend.name_to_loc(s)
             if s: inode_map.obsolete_segments.add(loc)
 
+    # TODO: We probably also want heuristics that will find inodes with
+    # badly-fragmented data and rewrite that to achieve better locality.
+
     # Given that list of segments to clean, scan through those segments to find
     # data which is still live and mark relevant inodes as needing to be
     # rewritten.
     dirty_inodes = set()
+    dirty_inode_data = set()
     for s in inode_map.obsolete_segments:
         filename = backend.loc_to_name(s)
         print "Scanning", filename, "for live data"
         for item in parse_log(backend.read(filename), filename):
             if item.type in (ITEM_TYPE.DATA, ITEM_TYPE.INODE):
                 if item.inum != 0:
-                    dirty_inodes.add(item.inum)
+                    inode = inode_map.inodes[item.inum]
+                    if s == inode.location[0:2]:
+                        dirty_inodes.add(item.inum)
+                    if item.inum not in dirty_inode_data:
+                        for b in inode.links:
+                            if s == b[1][0:2]:
+                                dirty_inode_data.add(item.inum)
+                                break
 
     print "Inodes to rewrite:", dirty_inodes
-    for i in sorted(dirty_inodes):
-        rewrite_inode(backend, inode_map, i, log)
+    print "Inodes with data to rewrite:", dirty_inode_data
+    for i in sorted(dirty_inodes.union(dirty_inode_data)):
+        rewrite_inode(backend, inode_map, i, log, i in dirty_inode_data)
 
 if __name__ == '__main__':
     backend = FileBackend(".")