From: Michael Vrable <vrable@cs.hmc.edu>
Date: Mon, 26 Sep 2011 04:36:12 +0000 (-0700)
Subject: Have cleaner wait on proxy when needed
X-Git-Url: https://git.vrable.net/?a=commitdiff_plain;h=cfb095f9f5cee9c5f0dc674b7775755f07e4266f;p=bluesky.git

Have cleaner wait on proxy when needed
---

diff --git a/cleaner/cleaner b/cleaner/cleaner
index 9cdba54..f2dcfcb 100755
--- a/cleaner/cleaner
+++ b/cleaner/cleaner
@@ -21,6 +21,15 @@ HEADER_SIZE = struct.calcsize(HEADER_FORMAT)
 
 CHECKPOINT_MAGIC = struct.pack('<Q', 0x7ad7dafb42a498b4)
 
+# Log file to write benchmark data to
+benchlog = None
+def benchlog_write(msg, *args):
+    m = msg % args
+    print "LOG:", m
+    if benchlog is not None:
+        benchlog.write(msg % args)
+        benchlog.write("\n")
+
 class ITEM_TYPE:
     DATA = '1'
     INODE = '2'
@@ -163,6 +172,8 @@ class S3Backend(Backend):
         print "S3 statistics:"
         print "GET: %d ops / %d bytes" % tuple(self.stats_get)
         print "PUT: %d ops / %d bytes" % tuple(self.stats_put)
+        benchlog_write("s3_get: %d", self.stats_get[1])
+        benchlog_write("s3_put: %d", self.stats_put[1])
 
 class SimpleBackend(Backend):
     """An interface to the simple BlueSky test network server."""
@@ -488,6 +499,9 @@ class InodeMap:
 
         print "%d bytes total / %d bytes used" % tuple(total_data)
         print "would delete %d segments (%d bytes)" % tuple(deletions)
+        benchlog_write("bytes_used: %d", total_data[1])
+        benchlog_write("bytes_wasted: %d", total_data[0] - total_data[1])
+        benchlog_write("bytes_freed: %d", deletions[1])
 
     def mark_updated(self, inum):
         self.updated_inodes.add(inum)
@@ -600,6 +614,8 @@ def run_cleaner(backend, inode_map, log, repack_inodes=False):
         rewrite_inode(backend, inode_map, i, log, i in dirty_inode_data)
 
 if __name__ == '__main__':
+    benchlog = open('cleaner.log', 'a')
+    benchlog_write("*** START CLEANER RUN ***")
     start_time = time.time()
     backend = S3Backend("mvrable-bluesky-west", cachedir="/tmp/bluesky-cache")
     #backend = FileBackend(".")
@@ -610,10 +626,18 @@ if __name__ == '__main__':
     imap.build(backend, chkpt)
     print chkpt
 
+    print "Version vector:", imap.version_vector
+    print "Last cleaner log file:", log_dir.seq_num - 1
+    if imap.version_vector.get(1, -1) != log_dir.seq_num - 1:
+        print "Proxy hasn't updated to latest cleaner segment yet!"
+        benchlog_write("waiting for proxy...")
+        sys.exit(0)
+
     run_cleaner(backend, imap, log_dir)
     print "Version vector:", imap.version_vector
     imap.write(backend, log_dir)
     log_dir.close_all()
     end_time = time.time()
-    print "Cleaner running time:", end_time - start_time
     backend.dump_stats()
+    benchlog_write("running_time: %s", end_time - start_time)
+    benchlog_write("")