projects
/
bluesky.git
/ commitdiff
commit
grep
author
committer
pickaxe
?
search:
re
summary
|
shortlog
|
log
|
commit
| commitdiff |
tree
raw
|
patch
|
inline
| side by side (parent:
63bc891
)
Have cleaner wait on proxy when needed
author
Michael Vrable
<vrable@cs.hmc.edu>
Mon, 26 Sep 2011 04:36:12 +0000
(21:36 -0700)
committer
Michael Vrable
<vrable@cs.hmc.edu>
Mon, 26 Sep 2011 04:36:12 +0000
(21:36 -0700)
cleaner/cleaner
patch
|
blob
|
history
diff --git
a/cleaner/cleaner
b/cleaner/cleaner
index
9cdba54
..
f2dcfcb
100755
(executable)
--- a/
cleaner/cleaner
+++ b/
cleaner/cleaner
@@
-21,6
+21,15
@@
HEADER_SIZE = struct.calcsize(HEADER_FORMAT)
CHECKPOINT_MAGIC = struct.pack('<Q', 0x7ad7dafb42a498b4)
CHECKPOINT_MAGIC = struct.pack('<Q', 0x7ad7dafb42a498b4)
+# Log file to write benchmark data to
+benchlog = None
+def benchlog_write(msg, *args):
+ m = msg % args
+ print "LOG:", m
+ if benchlog is not None:
+ benchlog.write(msg % args)
+ benchlog.write("\n")
+
class ITEM_TYPE:
DATA = '1'
INODE = '2'
class ITEM_TYPE:
DATA = '1'
INODE = '2'
@@
-163,6
+172,8
@@
class S3Backend(Backend):
print "S3 statistics:"
print "GET: %d ops / %d bytes" % tuple(self.stats_get)
print "PUT: %d ops / %d bytes" % tuple(self.stats_put)
print "S3 statistics:"
print "GET: %d ops / %d bytes" % tuple(self.stats_get)
print "PUT: %d ops / %d bytes" % tuple(self.stats_put)
+ benchlog_write("s3_get: %d", self.stats_get[1])
+ benchlog_write("s3_put: %d", self.stats_put[1])
class SimpleBackend(Backend):
"""An interface to the simple BlueSky test network server."""
class SimpleBackend(Backend):
"""An interface to the simple BlueSky test network server."""
@@
-488,6
+499,9
@@
class InodeMap:
print "%d bytes total / %d bytes used" % tuple(total_data)
print "would delete %d segments (%d bytes)" % tuple(deletions)
print "%d bytes total / %d bytes used" % tuple(total_data)
print "would delete %d segments (%d bytes)" % tuple(deletions)
+ benchlog_write("bytes_used: %d", total_data[1])
+ benchlog_write("bytes_wasted: %d", total_data[0] - total_data[1])
+ benchlog_write("bytes_freed: %d", deletions[1])
def mark_updated(self, inum):
self.updated_inodes.add(inum)
def mark_updated(self, inum):
self.updated_inodes.add(inum)
@@
-600,6
+614,8
@@
def run_cleaner(backend, inode_map, log, repack_inodes=False):
rewrite_inode(backend, inode_map, i, log, i in dirty_inode_data)
if __name__ == '__main__':
rewrite_inode(backend, inode_map, i, log, i in dirty_inode_data)
if __name__ == '__main__':
+ benchlog = open('cleaner.log', 'a')
+ benchlog_write("*** START CLEANER RUN ***")
start_time = time.time()
backend = S3Backend("mvrable-bluesky-west", cachedir="/tmp/bluesky-cache")
#backend = FileBackend(".")
start_time = time.time()
backend = S3Backend("mvrable-bluesky-west", cachedir="/tmp/bluesky-cache")
#backend = FileBackend(".")
@@
-610,10
+626,18
@@
if __name__ == '__main__':
imap.build(backend, chkpt)
print chkpt
imap.build(backend, chkpt)
print chkpt
+ print "Version vector:", imap.version_vector
+ print "Last cleaner log file:", log_dir.seq_num - 1
+ if imap.version_vector.get(1, -1) != log_dir.seq_num - 1:
+ print "Proxy hasn't updated to latest cleaner segment yet!"
+ benchlog_write("waiting for proxy...")
+ sys.exit(0)
+
run_cleaner(backend, imap, log_dir)
print "Version vector:", imap.version_vector
imap.write(backend, log_dir)
log_dir.close_all()
end_time = time.time()
run_cleaner(backend, imap, log_dir)
print "Version vector:", imap.version_vector
imap.write(backend, log_dir)
log_dir.close_all()
end_time = time.time()
- print "Cleaner running time:", end_time - start_time
backend.dump_stats()
backend.dump_stats()
+ benchlog_write("running_time: %s", end_time - start_time)
+ benchlog_write("")