X-Git-Url: http://git.vrable.net/?a=blobdiff_plain;f=lbs-util;h=5c124386c72b2ecf24c511aa34debb33e79c6a94;hb=18978c991772448ba77d6e11f921f1ab3d0b8fd2;hp=33c9c7599b7c555a815d56a84866a3dd1d71a81b;hpb=378d7decebc6eff227d8c22558faca3214d08660;p=cumulus.git diff --git a/lbs-util b/lbs-util index 33c9c75..5c12438 100755 --- a/lbs-util +++ b/lbs-util @@ -2,10 +2,19 @@ # # Utility for managing LBS archives. -import getpass, os, sys +import getpass, os, stat, sys, time from optparse import OptionParser import lbs +# We support up to "LBS Snapshot v0.2" formats, but are also limited by the lbs +# module. +FORMAT_VERSION = min(lbs.FORMAT_VERSION, (0, 2)) + +def check_version(format): + ver = lbs.parse_metadata_version(format) + if ver > FORMAT_VERSION: + raise RuntimeError("Unsupported LBS format: " + format) + parser = OptionParser(usage="%prog [option]... command [arg]...") parser.add_option("-v", action="store_true", dest="verbose", default=False, help="increase verbosity") @@ -22,6 +31,16 @@ def get_passphrase(): if not os.environ.has_key(ENV_KEY): os.environ[ENV_KEY] = getpass.getpass() +# Delete old snapshots from the local database, though do not actually schedule +# any segment cleaning. +# Syntax: $0 --localdb=LOCALDB prune-db +def cmd_prune_db(): + db = lbs.LocalDatabase(options.localdb) + + # Delete old snapshots from the local database. + db.garbage_collect() + db.commit() + # Run the segment cleaner. # Syntax: $0 --localdb=LOCALDB clean def cmd_clean(clean_threshold=7.0): @@ -56,6 +75,7 @@ def cmd_list_snapshot_sizes(): previous = set() for s in sorted(lowlevel.list_snapshots()): d = lbs.parse_full(store.load_snapshot(s)) + check_version(d['Format']) segments = d['Segments'].split() (size, added, removed) = (0, 0, 0) for seg in segments: @@ -68,10 +88,14 @@ def cmd_list_snapshot_sizes(): previous = set(segments) print "%s: %.3f +%.3f -%.3f" % (s, size / 1024.0**2, added / 1024.0**2, removed / 1024.0**2) -# Build checksum list for objects in the given segments +# Build checksum list for objects in the given segments, or all segments if +# none are specified. def cmd_object_checksums(segments): + get_passphrase() lowlevel = lbs.LowlevelDataStore(options.store) store = lbs.ObjectStore(lowlevel) + if len(segments) == 0: + segments = sorted(lowlevel.list_segments()) for s in segments: for (o, data) in store.load_segment(s): csum = lbs.ChecksumCreator().update(data).compute() @@ -85,10 +109,23 @@ def cmd_read_snapshots(snapshots): store = lbs.ObjectStore(lowlevel) for s in snapshots: d = lbs.parse_full(store.load_snapshot(s)) + check_version(d['Format']) print d print d['Segments'].split() store.cleanup() +# Produce a flattened metadata dump from a snapshot +def cmd_read_metadata(snapshot): + get_passphrase() + lowlevel = lbs.LowlevelDataStore(options.store) + store = lbs.ObjectStore(lowlevel) + d = lbs.parse_full(store.load_snapshot(snapshot)) + check_version(d['Format']) + metadata = lbs.read_metadata(store, d['Root']) + for l in metadata: + sys.stdout.write(l) + store.cleanup() + # Verify snapshot integrity def cmd_verify_snapshots(snapshots): get_passphrase() @@ -97,6 +134,7 @@ def cmd_verify_snapshots(snapshots): for s in snapshots: print "#### Snapshot", s d = lbs.parse_full(store.load_snapshot(s)) + check_version(d['Format']) print "## Root:", d['Root'] metadata = lbs.iterate_metadata(store, d['Root']) for m in metadata: @@ -114,6 +152,94 @@ def cmd_verify_snapshots(snapshots): raise ValueError("Bad checksum found") store.cleanup() +# Restore a snapshot, or some subset of files from it +def cmd_restore_snapshot(args): + get_passphrase() + lowlevel = lbs.LowlevelDataStore(options.store) + store = lbs.ObjectStore(lowlevel) + snapshot = lbs.parse_full(store.load_snapshot(args[0])) + check_version(snapshot['Format']) + destdir = args[1] + paths = args[2:] + + def warn(m, msg): + print "Warning: %s: %s" % (m.items.name, msg) + + for m in lbs.iterate_metadata(store, snapshot['Root']): + pathname = os.path.normpath(m.items.name) + while os.path.isabs(pathname): + pathname = pathname[1:] + print pathname + destpath = os.path.join(destdir, pathname) + (path, filename) = os.path.split(destpath) + + # TODO: Check for ../../../paths that might attempt to write outside + # the destination directory. Maybe also check attempts to follow + # symlinks pointing outside? + + try: + if not os.path.isdir(path): + os.makedirs(path) + + if m.items.type == '-': + file = open(destpath, 'wb') + verifier = lbs.ChecksumVerifier(m.items.checksum) + size = 0 + for block in m.data(): + data = store.get(block) + verifier.update(data) + size += len(data) + file.write(data) + file.close() + if int(m.fields['size']) != size: + raise ValueError("File size does not match!") + if not verifier.valid(): + raise ValueError("Bad checksum found") + elif m.items.type == 'd': + if filename != '.': + os.mkdir(destpath) + elif m.items.type == 'l': + os.symlink(m.items.contents, destpath) + elif m.items.type == 'p': + os.mkfifo(destpath) + elif m.items.type in ('c', 'b'): + if m.items.type == 'c': + mode = 0600 | stat.S_IFCHR + else: + mode = 0600 | stat.S_IFBLK + os.mknod(destpath, mode, os.makedev(*m.items.device)) + elif m.items.type == 's': + pass # TODO: Implement + else: + warn(m, "Unknown type code: " + m.items.type) + continue + + except Exception, e: + warn(m, "Error restoring: %s" % (e,)) + continue + + try: + uid = m.items.user[0] + gid = m.items.group[0] + os.lchown(destpath, uid, gid) + except Exception, e: + warn(m, "Error restoring file ownership: %s" % (e,)) + + if m.items.type == 'l': + continue + + try: + os.chmod(destpath, m.items.mode) + except Exception, e: + warn(m, "Error restoring file permissions: %s" % (e,)) + + try: + os.utime(destpath, (time.time(), m.items.mtime)) + except Exception, e: + warn(m, "Error restoring file timestamps: %s" % (e,)) + + store.cleanup() + if len(args) == 0: parser.print_usage() sys.exit(1) @@ -121,16 +247,22 @@ cmd = args[0] args = args[1:] if cmd == 'clean': cmd_clean() +elif cmd == 'prune-db': + cmd_prune_db() elif cmd == 'list-snapshots': cmd_list_snapshots() elif cmd == 'object-sums': cmd_object_checksums(args) elif cmd == 'read-snapshots': cmd_read_snapshots(args) +elif cmd == 'read-metadata': + cmd_read_metadata(args[0]) elif cmd == 'list-snapshot-sizes': cmd_list_snapshot_sizes() elif cmd == 'verify-snapshots': cmd_verify_snapshots(args) +elif cmd == 'restore-snapshot': + cmd_restore_snapshot(args) else: print "Unknown command:", cmd parser.print_usage()