From: Michael Vrable Date: Thu, 9 Sep 2010 19:09:52 +0000 (-0700) Subject: Improve segment cleaning. X-Git-Url: http://git.vrable.net/?a=commitdiff_plain;h=885814beba8d6191c042d56e7734bda0e680ac6a;hp=c665095a0fb48ac1304f906eaca563ba8d01e6e8;p=bluesky.git Improve segment cleaning. --- diff --git a/cleaner/cleaner b/cleaner/cleaner index fb3ce05..445bdeb 100755 --- a/cleaner/cleaner +++ b/cleaner/cleaner @@ -325,14 +325,15 @@ class InodeMap: log.write(new_checkpoint, 2) self.checkpoint_record = new_checkpoint -def rewrite_inode(backend, inode_map, inum, log): +def rewrite_inode(backend, inode_map, inum, log, copy_data=True): inode = inode_map.inodes[inum] - blocks = [] - for l in inode.links: - data = load_item(backend, l[1]) - blocks.append(data) - log.write(data, 0) - inode.links = [(b.id, b.location) for b in blocks] + if copy_data: + blocks = [] + for l in inode.links: + data = load_item(backend, l[1]) + blocks.append(data) + log.write(data, 0) + inode.links = [(b.id, b.location) for b in blocks] log.write(inode, 1) inode_map.mark_updated(inum) @@ -340,26 +341,38 @@ def run_cleaner(backend, inode_map, log): # Determine which segments are poorly utilized and should be cleaned. We # need better heuristics here. for (s, u) in sorted(inode_map.util.segments.items()): - if float(u[1]) / u[0] < 0.99 and u[1] > 0: + if float(u[1]) / u[0] < 0.95 and u[1] > 0: print "Should clean segment", s loc = backend.name_to_loc(s) if s: inode_map.obsolete_segments.add(loc) + # TODO: We probably also want heuristics that will find inodes with + # badly-fragmented data and rewrite that to achieve better locality. + # Given that list of segments to clean, scan through those segments to find # data which is still live and mark relevant inodes as needing to be # rewritten. dirty_inodes = set() + dirty_inode_data = set() for s in inode_map.obsolete_segments: filename = backend.loc_to_name(s) print "Scanning", filename, "for live data" for item in parse_log(backend.read(filename), filename): if item.type in (ITEM_TYPE.DATA, ITEM_TYPE.INODE): if item.inum != 0: - dirty_inodes.add(item.inum) + inode = inode_map.inodes[item.inum] + if s == inode.location[0:2]: + dirty_inodes.add(item.inum) + if item.inum not in dirty_inode_data: + for b in inode.links: + if s == b[1][0:2]: + dirty_inode_data.add(item.inum) + break print "Inodes to rewrite:", dirty_inodes - for i in sorted(dirty_inodes): - rewrite_inode(backend, inode_map, i, log) + print "Inodes with data to rewrite:", dirty_inode_data + for i in sorted(dirty_inodes.union(dirty_inode_data)): + rewrite_inode(backend, inode_map, i, log, i in dirty_inode_data) if __name__ == '__main__': backend = FileBackend(".") @@ -368,7 +381,7 @@ if __name__ == '__main__': imap.build(backend, chkpt) print chkpt - log_dir = LogDirectory(backend, 0) + log_dir = LogDirectory(backend, 1) run_cleaner(backend, imap, log_dir) imap.write(backend, log_dir) log_dir.close_all()