X-Git-Url: http://git.vrable.net/?a=blobdiff_plain;f=python%2Fcumulus%2Fcmd_util.py;h=9d97190b1c605ac8ac1c4475d72bcf9de868e5c4;hb=5949214bc01b2c762adfb724d1e63b7e130c91f4;hp=c3733f0e9cd44dd8a43299dd81f78f6c9e2cbfa9;hpb=e21af7d71c2c53e2f2cacea9eb9e277ed50b0eac;p=cumulus.git diff --git a/python/cumulus/cmd_util.py b/python/cumulus/cmd_util.py index c3733f0..9d97190 100644 --- a/python/cumulus/cmd_util.py +++ b/python/cumulus/cmd_util.py @@ -1,8 +1,6 @@ -# Cumulus: Smart Filesystem Backup to Dumb Servers -# -# Copyright (C) 2006-2009 The Regents of the University of California -# Copyright (C) 2012 Google Inc. -# Written by Michael Vrable +# Cumulus: Efficient Filesystem Backup to the Cloud +# Copyright (C) 2006-2009, 2012 The Cumulus Developers +# See the AUTHORS file for a list of contributors. # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -61,7 +59,7 @@ def cmd_clean(args, clean_threshold=7.0): # Delete old snapshots from the local database. intent = float(options.intent) for s in db.list_schemes(): - db.garbage_collect(s, intent) + db.prune_old_snapshots(s, intent) # Expire segments which are poorly-utilized. for s in db.get_segment_cleaning_list(): @@ -78,97 +76,75 @@ def cmd_list_snapshots(args): """ List snapshots stored. Syntax: $0 --data=DATADIR list-snapshots """ - store = cumulus.LowlevelDataStore(options.store) - for s in sorted(store.list_snapshots()): - print s + store = cumulus.CumulusStore(options.store) + for s in sorted(store.list_snapshots()): print s def cmd_list_snapshot_sizes(args): """ List size of data needed for each snapshot. Syntax: $0 --data=DATADIR list-snapshot-sizes """ - lowlevel = cumulus.LowlevelDataStore(options.store) - lowlevel.scan() - store = cumulus.ObjectStore(lowlevel) + store = cumulus.CumulusStore(options.store) + backend = store.backend previous = set() - exts = {} - for seg in lowlevel.store.list('segments'): - exts.update ([seg.split ('.', 1)]) - for s in sorted(lowlevel.list_snapshots()): + size = 0 + def get_size(segment): + return backend.stat_generic(segment + ".tar", "segments")["size"] + for s in sorted(store.list_snapshots()): d = cumulus.parse_full(store.load_snapshot(s)) check_version(d['Format']) - try: - intent = float(d['Backup-Intent']) - except: - intent = 1.0 - - segments = d['Segments'].split() - (size, added, removed, addcount, remcount) = (0, 0, 0, 0, 0) - lo_stat = lowlevel.lowlevel_stat - for seg in segments: - segsize = lo_stat('.'.join ((seg, exts[seg])))['size'] - size += segsize - if seg not in previous: - added += segsize - addcount += 1 - for seg in previous: - if seg not in segments: - removed += lo_stat('.'.join((seg, exts[seg])))['size'] - remcount += 1 - previous = set(segments) - print "%s [%s]: %.3f +%.3f -%.3f (+%d/-%d segments)" % (s, intent, size / 1024.0**2, added / 1024.0**2, removed / 1024.0**2, addcount, remcount) + segments = set(d['Segments'].split()) + (added, removed, addcount, remcount) = (0, 0, 0, 0) + for seg in segments.difference(previous): + added += get_size(seg) + addcount += 1 + for seg in previous.difference(segments): + removed += get_size(seg) + remcount += 1 + size += added - removed + previous = segments + print "%s: %.3f +%.3f -%.3f (+%d/-%d segments)" % (s, size / 1024.0**2, added / 1024.0**2, removed / 1024.0**2, addcount, remcount) def cmd_garbage_collect(args): """ Search for any files which are not needed by any current snapshots and offer to delete them. Syntax: $0 --store=DATADIR gc """ - lowlevel = cumulus.LowlevelDataStore(options.store) - lowlevel.scan() - store = cumulus.ObjectStore(lowlevel) - snapshots = set(lowlevel.list_snapshots()) - segments = set() - for s in snapshots: + store = cumulus.CumulusStore(options.store) + backend = store.backend + referenced = set() + for s in store.list_snapshots(): d = cumulus.parse_full(store.load_snapshot(s)) check_version(d['Format']) - segments.update(d['Segments'].split()) - - referenced = snapshots.union(segments) - reclaimed = 0 - for (t, r) in cumulus.store.type_patterns.items(): - for f in lowlevel.store.list(t): - m = r.match(f) - if m is None or m.group(1) not in referenced: - print "Garbage:", (t, f) - reclaimed += lowlevel.store.stat(t, f)['size'] - if not options.dry_run: - lowlevel.store.delete(t, f) - print "Reclaimed space:", reclaimed + referenced.add(s) + referenced.update(d['Segments'].split()) -cmd_gc = cmd_garbage_collect + print referenced -def cmd_object_checksums(segments): - """ Build checksum list for objects in the given segments, or all - segments if none are specified. - """ - get_passphrase() - lowlevel = cumulus.LowlevelDataStore(options.store) - store = cumulus.ObjectStore(lowlevel) - if len(segments) == 0: - segments = sorted(lowlevel.list_segments()) - for s in segments: - for (o, data) in store.load_segment(s): - csum = cumulus.ChecksumCreator().update(data).compute() - print "%s/%s:%d:%s" % (s, o, len(data), csum) - store.cleanup() -object_sums = cmd_object_checksums + to_delete = [] + to_preserve = [] + for filetype in cumulus.SEARCH_PATHS: + for (name, path) in store.backend.list_generic(filetype): + if name in referenced: + to_preserve.append(path) + else: + to_delete.append(path) + + print to_preserve + print to_delete + + raw_backend = backend.raw_backend + for f in to_delete: + print "Delete:", f + if not options.dry_run: + raw_backend.delete(f) +cmd_gc = cmd_garbage_collect def cmd_read_snapshots(snapshots): """ Read a snapshot file """ get_passphrase() - lowlevel = cumulus.LowlevelDataStore(options.store) - store = cumulus.ObjectStore(lowlevel) + store = cumulus.CumulusStore(options.store) for s in snapshots: d = cumulus.parse_full(store.load_snapshot(s)) check_version(d['Format']) @@ -181,8 +157,7 @@ def cmd_read_metadata(args): """ snapshot = args [0] get_passphrase() - lowlevel = cumulus.LowlevelDataStore(options.store) - store = cumulus.ObjectStore(lowlevel) + store = cumulus.CumulusStore(options.store) d = cumulus.parse_full(store.load_snapshot(snapshot)) check_version(d['Format']) metadata = cumulus.read_metadata(store, d['Root']) @@ -200,8 +175,7 @@ def cmd_verify_snapshots(snapshots): """ Verify snapshot integrity """ get_passphrase() - lowlevel = cumulus.LowlevelDataStore(options.store) - store = cumulus.ObjectStore(lowlevel) + store = cumulus.CumulusStore(options.store) for s in snapshots: cumulus.accessed_segments.clear() print "#### Snapshot", s @@ -239,8 +213,7 @@ def cmd_restore_snapshot(args): """ Restore a snapshot, or some subset of files from it """ get_passphrase() - lowlevel = cumulus.LowlevelDataStore(options.store) - store = cumulus.ObjectStore(lowlevel) + store = cumulus.CumulusStore(options.store) snapshot = cumulus.parse_full(store.load_snapshot(args[0])) check_version(snapshot['Format']) destdir = args[1] @@ -281,7 +254,7 @@ def cmd_restore_snapshot(args): metadata_paths[pathname] = m for block in m.data(): (segment, object, checksum, slice) \ - = cumulus.ObjectStore.parse_ref(block) + = cumulus.CumulusStore.parse_ref(block) if segment not in metadata_segments: metadata_segments[segment] = set() metadata_segments[segment].add(pathname)