From 8260042a674abb9f47a331963180cc3b3560885f Mon Sep 17 00:00:00 2001 From: Michael Vrable Date: Wed, 8 Sep 2010 22:07:17 -0700 Subject: [PATCH] Extend cleaner with a simple policy for choosing segments to clean. --- cleaner/cleaner | 38 +++++++++++++++++++++++++++++++------- 1 file changed, 31 insertions(+), 7 deletions(-) diff --git a/cleaner/cleaner b/cleaner/cleaner index e249ed2..42aef01 100755 --- a/cleaner/cleaner +++ b/cleaner/cleaner @@ -234,6 +234,7 @@ class InodeMap: util = UtilizationTracker(backend) util.add_item(checkpoint_record) inodes = {} + self.obsolete_segments = set() print "Inode map:" for i in range(len(checkpoint_record.data) // 16): @@ -282,11 +283,11 @@ class InodeMap: # Case 1: No inodes in this range of the old inode map have # changed. Simply emit a new pointer to the same inode map block. - # TODO: Add the ability to rewrite the inode map block if we choose - # to do so for cleaning, even if no inodes have changed. if len(updated_inodes) == 0 or updated_inodes[-1] > end: - new_checkpoint.links.append(self.checkpoint_record.links[i]) - continue + old_location = self.checkpoint_record.links[i][1][0:2] + if old_location not in self.obsolete_segments: + new_checkpoint.links.append(self.checkpoint_record.links[i]) + continue # Case 2: Some inodes have been updated. Create a new inode map # block, write it out, and point the new checkpoint at it. @@ -323,6 +324,31 @@ def rewrite_inode(backend, inode_map, inum, log): log.write(inode, 1) inode_map.mark_updated(inum) +def run_cleaner(backend, inode_map, log): + # Determine which segments are poorly utilized and should be cleaned. We + # need better heuristics here. + for (s, u) in sorted(inode_map.util.segments.items()): + if float(u[1]) / u[0] < 0.99 and u[1] > 0: + print "Should clean segment", s + m = re.match(r"^log-(\d+)-(\d+)$", s) + if m: inode_map.obsolete_segments.add((int(m.group(1)), int(m.group(2)))) + + # Given that list of segments to clean, scan through those segments to find + # data which is still live and mark relevant inodes as needing to be + # rewritten. + dirty_inodes = set() + for s in inode_map.obsolete_segments: + filename = "log-%08d-%08d" % s + print "Scanning", filename, "for live data" + for item in parse_log(backend.read(filename), filename): + if item.type in (ITEM_TYPE.DATA, ITEM_TYPE.INODE): + if item.inum != 0: + dirty_inodes.add(item.inum) + + print "Inodes to rewrite:", dirty_inodes + for i in sorted(dirty_inodes): + rewrite_inode(backend, inode_map, i, log) + if __name__ == '__main__': backend = FileBackend(".") chkpt = load_checkpoint_record(backend) @@ -330,9 +356,7 @@ if __name__ == '__main__': imap.build(backend, chkpt) print chkpt - print repr(chkpt.serialize()) - log_dir = LogDirectory(backend, 1, 0) - rewrite_inode(backend, imap, 147, log_dir) + run_cleaner(backend, imap, log_dir) imap.write(backend, log_dir) log_dir.close_all() -- 2.20.1