X-Git-Url: http://git.vrable.net/?a=blobdiff_plain;f=python%2Fcumulus%2Fcmd_util.py;h=0827cb05025e90aaa49ccf8d90ecfc5cc401b112;hb=a5f66616b1ec0c38328ad5131bf1c889ccc43659;hp=e0b094dc908b59a8ebec90343f93089e7358f9f1;hpb=3d780590edec4583eb3ef0ca16120afd0f7451f9;p=cumulus.git diff --git a/python/cumulus/cmd_util.py b/python/cumulus/cmd_util.py index e0b094d..0827cb0 100644 --- a/python/cumulus/cmd_util.py +++ b/python/cumulus/cmd_util.py @@ -36,7 +36,7 @@ def check_version(format): # environment variable. def get_passphrase(): ENV_KEY = 'LBS_GPG_PASSPHRASE' - if not os.environ.has_key(ENV_KEY): + if ENV_KEY not in os.environ: os.environ[ENV_KEY] = getpass.getpass() def cmd_prune_db(args): @@ -64,8 +64,8 @@ def cmd_clean(args, clean_threshold=7.0): # Expire segments which are poorly-utilized. for s in db.get_segment_cleaning_list(): if s.cleaning_benefit > clean_threshold: - print "Cleaning segment %d (benefit %.2f)" % (s.id, - s.cleaning_benefit) + print("Cleaning segment %d (benefit %.2f)" % (s.id, + s.cleaning_benefit)) db.mark_segment_expired(s) else: break @@ -76,102 +76,81 @@ def cmd_list_snapshots(args): """ List snapshots stored. Syntax: $0 --data=DATADIR list-snapshots """ - store = cumulus.LowlevelDataStore(options.store) - for s in sorted(store.list_snapshots()): - print s + store = cumulus.CumulusStore(options.store) + for s in sorted(store.list_snapshots()): print(s) def cmd_list_snapshot_sizes(args): """ List size of data needed for each snapshot. Syntax: $0 --data=DATADIR list-snapshot-sizes """ - lowlevel = cumulus.LowlevelDataStore(options.store) - lowlevel.scan() - store = cumulus.ObjectStore(lowlevel) + store = cumulus.CumulusStore(options.store) + backend = store.backend + backend.prefetch_generic() previous = set() - exts = {} - for seg in lowlevel.store.list('segments'): - exts.update ([seg.split ('.', 1)]) - for s in sorted(lowlevel.list_snapshots()): + size = 0 + def get_size(segment): + return backend.stat_generic(segment + ".tar", "segments")["size"] + for s in sorted(store.list_snapshots()): d = cumulus.parse_full(store.load_snapshot(s)) check_version(d['Format']) - try: - intent = float(d['Backup-Intent']) - except: - intent = 1.0 - - segments = d['Segments'].split() - (size, added, removed, addcount, remcount) = (0, 0, 0, 0, 0) - lo_stat = lowlevel.lowlevel_stat - for seg in segments: - segsize = lo_stat('.'.join ((seg, exts[seg])))['size'] - size += segsize - if seg not in previous: - added += segsize - addcount += 1 - for seg in previous: - if seg not in segments: - removed += lo_stat('.'.join((seg, exts[seg])))['size'] - remcount += 1 - previous = set(segments) - print "%s [%s]: %.3f +%.3f -%.3f (+%d/-%d segments)" % (s, intent, size / 1024.0**2, added / 1024.0**2, removed / 1024.0**2, addcount, remcount) + segments = set(d['Segments'].split()) + (added, removed, addcount, remcount) = (0, 0, 0, 0) + for seg in segments.difference(previous): + added += get_size(seg) + addcount += 1 + for seg in previous.difference(segments): + removed += get_size(seg) + remcount += 1 + size += added - removed + previous = segments + print("%s: %.3f +%.3f -%.3f (+%d/-%d segments)" % (s, size / 1024.0**2, added / 1024.0**2, removed / 1024.0**2, addcount, remcount)) def cmd_garbage_collect(args): """ Search for any files which are not needed by any current snapshots and offer to delete them. Syntax: $0 --store=DATADIR gc """ - lowlevel = cumulus.LowlevelDataStore(options.store) - lowlevel.scan() - store = cumulus.ObjectStore(lowlevel) - snapshots = set(lowlevel.list_snapshots()) - segments = set() - for s in snapshots: + store = cumulus.CumulusStore(options.store) + backend = store.backend + referenced = set() + for s in store.list_snapshots(): d = cumulus.parse_full(store.load_snapshot(s)) check_version(d['Format']) - segments.update(d['Segments'].split()) - - referenced = snapshots.union(segments) - reclaimed = 0 - for (t, r) in cumulus.store.type_patterns.items(): - for f in lowlevel.store.list(t): - m = r.match(f) - if m is None or m.group(1) not in referenced: - print "Garbage:", (t, f) - reclaimed += lowlevel.store.stat(t, f)['size'] - if not options.dry_run: - lowlevel.store.delete(t, f) - print "Reclaimed space:", reclaimed + referenced.add(s) + referenced.update(d['Segments'].split()) -cmd_gc = cmd_garbage_collect + print(referenced) -def cmd_object_checksums(segments): - """ Build checksum list for objects in the given segments, or all - segments if none are specified. - """ - get_passphrase() - lowlevel = cumulus.LowlevelDataStore(options.store) - store = cumulus.ObjectStore(lowlevel) - if len(segments) == 0: - segments = sorted(lowlevel.list_segments()) - for s in segments: - for (o, data) in store.load_segment(s): - csum = cumulus.ChecksumCreator().update(data).compute() - print "%s/%s:%d:%s" % (s, o, len(data), csum) - store.cleanup() -object_sums = cmd_object_checksums + to_delete = [] + to_preserve = [] + for filetype in cumulus.SEARCH_PATHS: + for (name, path) in store.backend.list_generic(filetype): + if name in referenced: + to_preserve.append(path) + else: + to_delete.append(path) + + print(to_preserve) + print(to_delete) + + raw_backend = backend.raw_backend + for f in to_delete: + print("Delete:", f) + if not options.dry_run: + raw_backend.delete(f) +cmd_gc = cmd_garbage_collect def cmd_read_snapshots(snapshots): """ Read a snapshot file """ get_passphrase() - lowlevel = cumulus.LowlevelDataStore(options.store) - store = cumulus.ObjectStore(lowlevel) + store = cumulus.CumulusStore(options.store) for s in snapshots: d = cumulus.parse_full(store.load_snapshot(s)) check_version(d['Format']) - print d - print d['Segments'].split() + print(d) + print(d['Segments'].split()) store.cleanup() def cmd_read_metadata(args): @@ -179,8 +158,7 @@ def cmd_read_metadata(args): """ snapshot = args [0] get_passphrase() - lowlevel = cumulus.LowlevelDataStore(options.store) - store = cumulus.ObjectStore(lowlevel) + store = cumulus.CumulusStore(options.store) d = cumulus.parse_full(store.load_snapshot(snapshot)) check_version(d['Format']) metadata = cumulus.read_metadata(store, d['Root']) @@ -198,18 +176,17 @@ def cmd_verify_snapshots(snapshots): """ Verify snapshot integrity """ get_passphrase() - lowlevel = cumulus.LowlevelDataStore(options.store) - store = cumulus.ObjectStore(lowlevel) + store = cumulus.CumulusStore(options.store) for s in snapshots: cumulus.accessed_segments.clear() - print "#### Snapshot", s + print("#### Snapshot", s) d = cumulus.parse_full(store.load_snapshot(s)) check_version(d['Format']) - print "## Root:", d['Root'] + print("## Root:", d['Root']) metadata = cumulus.iterate_metadata(store, d['Root']) for m in metadata: if m.fields['type'] not in ('-', 'f'): continue - print "%s [%d bytes]" % (m.fields['name'], int(m.fields['size'])) + print("%s [%d bytes]" % (m.fields['name'], int(m.fields['size']))) verifier = cumulus.ChecksumVerifier(m.fields['checksum']) size = 0 for block in m.data(): @@ -226,19 +203,18 @@ def cmd_verify_snapshots(snapshots): # doesn't contain duplicates. listed_segments = set(d['Segments'].split()) if cumulus.accessed_segments - listed_segments: - print "Error: Some segments not listed in descriptor!" - print sorted(list(cumulus.accessed_segments - listed_segments)) + print("Error: Some segments not listed in descriptor!") + print(sorted(list(cumulus.accessed_segments - listed_segments))) if listed_segments - cumulus.accessed_segments : - print "Warning: Extra unused segments listed in descriptor!" - print sorted(list(listed_segments - cumulus.accessed_segments)) + print("Warning: Extra unused segments listed in descriptor!") + print(sorted(list(listed_segments - cumulus.accessed_segments))) store.cleanup() def cmd_restore_snapshot(args): """ Restore a snapshot, or some subset of files from it """ get_passphrase() - lowlevel = cumulus.LowlevelDataStore(options.store) - store = cumulus.ObjectStore(lowlevel) + store = cumulus.CumulusStore(options.store) snapshot = cumulus.parse_full(store.load_snapshot(args[0])) check_version(snapshot['Format']) destdir = args[1] @@ -256,7 +232,7 @@ def cmd_restore_snapshot(args): return False def warn(m, msg): - print "Warning: %s: %s" % (m.items.name, msg) + print("Warning: %s: %s" % (m.items.name, msg)) # Phase 1: Read the complete metadata log and create directory structure. metadata_items = [] @@ -279,23 +255,23 @@ def cmd_restore_snapshot(args): metadata_paths[pathname] = m for block in m.data(): (segment, object, checksum, slice) \ - = cumulus.ObjectStore.parse_ref(block) + = cumulus.CumulusStore.parse_ref(block) if segment not in metadata_segments: metadata_segments[segment] = set() metadata_segments[segment].add(pathname) try: if not os.path.isdir(path): - print "mkdir:", path + print("mkdir:", path) os.makedirs(path) - except Exception, e: + except Exception as e: warn(m, "Error creating directory structure: %s" % (e,)) continue # Phase 2: Restore files, ordered by how data is stored in segments. def restore_file(pathname, m): assert m.items.type in ('-', 'f') - print "extract:", pathname + print("extract:", pathname) destpath = os.path.join(destdir, pathname) file = open(destpath, 'wb') @@ -314,13 +290,13 @@ def cmd_restore_snapshot(args): while metadata_segments: (segment, items) = metadata_segments.popitem() - print "+ Segment", segment + print("+ Segment", segment) for pathname in sorted(items): if pathname in metadata_paths: restore_file(pathname, metadata_paths[pathname]) del metadata_paths[pathname] - print "+ Remaining files" + print("+ Remaining files") while metadata_paths: (pathname, m) = metadata_paths.popitem() restore_file(pathname, m) @@ -328,7 +304,7 @@ def cmd_restore_snapshot(args): # Phase 3: Restore special files (symlinks, devices). # Phase 4: Restore directory permissions and modification times. for (pathname, m) in reversed(metadata_items): - print "permissions:", pathname + print("permissions:", pathname) destpath = os.path.join(destdir, pathname) (path, filename) = os.path.split(destpath) @@ -350,9 +326,9 @@ def cmd_restore_snapshot(args): os.mkfifo(destpath) elif m.items.type in ('c', 'b'): if m.items.type == 'c': - mode = 0600 | stat.S_IFCHR + mode = 0o600 | stat.S_IFCHR else: - mode = 0600 | stat.S_IFBLK + mode = 0o600 | stat.S_IFBLK os.mknod(destpath, mode, os.makedev(*m.items.device)) elif m.items.type == 's': pass # TODO: Implement @@ -360,7 +336,7 @@ def cmd_restore_snapshot(args): warn(m, "Unknown type code: " + m.items.type) continue - except Exception, e: + except Exception as e: warn(m, "Error restoring: %s" % (e,)) continue @@ -368,7 +344,7 @@ def cmd_restore_snapshot(args): uid = m.items.user[0] gid = m.items.group[0] os.lchown(destpath, uid, gid) - except Exception, e: + except Exception as e: warn(m, "Error restoring file ownership: %s" % (e,)) if m.items.type == 'l': @@ -376,12 +352,12 @@ def cmd_restore_snapshot(args): try: os.chmod(destpath, m.items.mode) - except Exception, e: + except Exception as e: warn(m, "Error restoring file permissions: %s" % (e,)) try: os.utime(destpath, (time.time(), m.items.mtime)) - except Exception, e: + except Exception as e: warn(m, "Error restoring file timestamps: %s" % (e,)) store.cleanup() @@ -389,7 +365,7 @@ def cmd_restore_snapshot(args): def main(argv): usage = ["%prog [option]... command [arg]...", "", "Commands:"] cmd = method = None - for cmd, method in globals().iteritems(): + for cmd, method in globals().items(): if cmd.startswith ('cmd_'): usage.append(cmd[4:].replace('_', '-') + ':' + method.__doc__) parser = OptionParser(usage="\n".join(usage)) @@ -415,6 +391,6 @@ def main(argv): if method: method (args) else: - print "Unknown command:", cmd + print("Unknown command:", cmd) parser.print_usage() sys.exit(1)