projects
/
bluesky.git
/ commitdiff
commit
grep
author
committer
pickaxe
?
search:
re
summary
|
shortlog
|
log
|
commit
| commitdiff |
tree
raw
|
patch
|
inline
| side by side (parent:
13e7c2e
)
Dump some statistics when the cleaner runs
author
Michael Vrable
<mvrable@cs.ucsd.edu>
Fri, 18 Mar 2011 04:18:12 +0000
(21:18 -0700)
committer
Michael Vrable
<mvrable@cs.ucsd.edu>
Fri, 18 Mar 2011 04:18:12 +0000
(21:18 -0700)
cleaner/cleaner
patch
|
blob
|
history
diff --git
a/cleaner/cleaner
b/cleaner/cleaner
index
4cda7c7
..
93feef4
100755
(executable)
--- a/
cleaner/cleaner
+++ b/
cleaner/cleaner
@@
-37,6
+37,8
@@
class Backend:
m = re.match(r"^log-(\d+)-(\d+)$", name)
if m: return (int(m.group(1)), int(m.group(2)))
m = re.match(r"^log-(\d+)-(\d+)$", name)
if m: return (int(m.group(1)), int(m.group(2)))
+ def dump_stats(self):
+ pass
class FileBackend(Backend):
"""An interface to BlueSky where the log segments are on local disk.
class FileBackend(Backend):
"""An interface to BlueSky where the log segments are on local disk.
@@
-95,6
+97,8
@@
class S3Backend(Backend):
self.cachedir = cachedir
self.cache = {}
self.connect()
self.cachedir = cachedir
self.cache = {}
self.connect()
+ self.stats_get = [0, 0]
+ self.stats_put = [0, 0]
def connect(self):
self.conn = boto.connect_s3(is_secure=False)
def connect(self):
self.conn = boto.connect_s3(is_secure=False)
@@
-125,6
+129,8
@@
class S3Backend(Backend):
fp.write(data)
fp.close()
self.cache[filename] = True
fp.write(data)
fp.close()
self.cache[filename] = True
+ self.stats_get[0] += 1
+ self.stats_get[1] += len(data)
if offset > 0:
data = data[offset:]
if length is not None:
if offset > 0:
data = data[offset:]
if length is not None:
@@
-136,6
+142,8
@@
class S3Backend(Backend):
k = Key(self.bucket)
k.key = self.path + filename
k.set_contents_from_string(data)
k = Key(self.bucket)
k.key = self.path + filename
k.set_contents_from_string(data)
+ self.stats_put[0] += 1
+ self.stats_put[1] += len(data)
if filename in self.cache:
del self.cache[filename]
if filename in self.cache:
del self.cache[filename]
@@
-147,6
+155,11
@@
class S3Backend(Backend):
if filename in self.cache:
del self.cache[filename]
if filename in self.cache:
del self.cache[filename]
+ def dump_stats(self):
+ print "S3 statistics:"
+ print "GET: %d ops / %d bytes" % tuple(self.stats_get)
+ print "PUT: %d ops / %d bytes" % tuple(self.stats_put)
+
class SimpleBackend(Backend):
"""An interface to the simple BlueSky test network server."""
class SimpleBackend(Backend):
"""An interface to the simple BlueSky test network server."""
@@
-446,17
+459,24
@@
class InodeMap:
print
print "Segment utilizations:"
print
print "Segment utilizations:"
+ total_data = [0, 0]
+ deletions = [0, 0]
for (s, u) in sorted(util.segments.items()):
for (s, u) in sorted(util.segments.items()):
+ for i in range(2): total_data[i] += u[i]
print "%s: %s %s" % (s, u, float(u[1]) / u[0])
if u[1] == 0:
print "%s: %s %s" % (s, u, float(u[1]) / u[0])
if u[1] == 0:
- # print "Deleting..."
- # backend.delete(s)
- pass
+ print "Would delete..."
+ #backend.delete(s)
+ deletions[0] += 1
+ deletions[1] += u[0]
self.inodes = inodes
self.util = util
self.updated_inodes = set()
self.inodes = inodes
self.util = util
self.updated_inodes = set()
+ print "%d bytes total / %d bytes used" % tuple(total_data)
+ print "would delete %d segments (%d bytes)" % tuple(deletions)
+
def mark_updated(self, inum):
self.updated_inodes.add(inum)
def mark_updated(self, inum):
self.updated_inodes.add(inum)
@@
-565,7
+585,8
@@
def run_cleaner(backend, inode_map, log, repack_inodes=False):
rewrite_inode(backend, inode_map, i, log, i in dirty_inode_data)
if __name__ == '__main__':
rewrite_inode(backend, inode_map, i, log, i in dirty_inode_data)
if __name__ == '__main__':
- backend = S3Backend("mvrable-bluesky", cachedir=".")
+ start_time = time.time()
+ backend = S3Backend("mvrable-bluesky-west", cachedir="/export/cache")
#backend = FileBackend(".")
chkpt = load_checkpoint_record(backend)
print backend.list()
#backend = FileBackend(".")
chkpt = load_checkpoint_record(backend)
print backend.list()
@@
-578,3
+599,6
@@
if __name__ == '__main__':
print "Version vector:", imap.version_vector
imap.write(backend, log_dir)
log_dir.close_all()
print "Version vector:", imap.version_vector
imap.write(backend, log_dir)
log_dir.close_all()
+ end_time = time.time()
+ print "Cleaner running time:", end_time - start_time
+ backend.dump_stats()