#!/usr/bin/python # # Utility for managing LBS archives. import getpass, os, stat, sys, time from optparse import OptionParser import lbs parser = OptionParser(usage="%prog [option]... command [arg]...") parser.add_option("-v", action="store_true", dest="verbose", default=False, help="increase verbosity") parser.add_option("--store", dest="store", help="specify path to backup data store") parser.add_option("--localdb", dest="localdb", help="specify path to local database") (options, args) = parser.parse_args(sys.argv[1:]) # Read a passphrase from the user and store it in the LBS_GPG_PASSPHRASE # environment variable. def get_passphrase(): ENV_KEY = 'LBS_GPG_PASSPHRASE' if not os.environ.has_key(ENV_KEY): os.environ[ENV_KEY] = getpass.getpass() # Delete old snapshots from the local database, though do not actually schedule # any segment cleaning. # Syntax: $0 --localdb=LOCALDB prune-db def cmd_prune_db(): db = lbs.LocalDatabase(options.localdb) # Delete old snapshots from the local database. db.garbage_collect() db.commit() # Run the segment cleaner. # Syntax: $0 --localdb=LOCALDB clean def cmd_clean(clean_threshold=7.0): db = lbs.LocalDatabase(options.localdb) # Delete old snapshots from the local database. db.garbage_collect() # Expire segments which are poorly-utilized. for s in db.get_segment_cleaning_list(): if s.cleaning_benefit > clean_threshold: print "Cleaning segment %d (benefit %.2f)" % (s.id, s.cleaning_benefit) db.mark_segment_expired(s) else: break db.balance_expired_objects() db.commit() # List snapshots stored. # Syntax: $0 --data=DATADIR list-snapshots def cmd_list_snapshots(): store = lbs.LowlevelDataStore(options.store) for s in sorted(store.list_snapshots()): print s # List size of data needed for each snapshot. # Syntax: $0 --data=DATADIR list-snapshot-sizes def cmd_list_snapshot_sizes(): lowlevel = lbs.LowlevelDataStore(options.store) store = lbs.ObjectStore(lowlevel) previous = set() for s in sorted(lowlevel.list_snapshots()): d = lbs.parse_full(store.load_snapshot(s)) segments = d['Segments'].split() (size, added, removed) = (0, 0, 0) for seg in segments: segsize = lowlevel.lowlevel_stat(seg + ".tar.gpg")['size'] size += segsize if seg not in previous: added += segsize for seg in previous: if seg not in segments: removed += lowlevel.lowlevel_stat(seg + ".tar.gpg")['size'] previous = set(segments) print "%s: %.3f +%.3f -%.3f" % (s, size / 1024.0**2, added / 1024.0**2, removed / 1024.0**2) # Build checksum list for objects in the given segments, or all segments if # none are specified. def cmd_object_checksums(segments): get_passphrase() lowlevel = lbs.LowlevelDataStore(options.store) store = lbs.ObjectStore(lowlevel) if len(segments) == 0: segments = sorted(lowlevel.list_segments()) for s in segments: for (o, data) in store.load_segment(s): csum = lbs.ChecksumCreator().update(data).compute() print "%s/%s:%d:%s" % (s, o, len(data), csum) store.cleanup() # Read a snapshot file def cmd_read_snapshots(snapshots): get_passphrase() lowlevel = lbs.LowlevelDataStore(options.store) store = lbs.ObjectStore(lowlevel) for s in snapshots: d = lbs.parse_full(store.load_snapshot(s)) print d print d['Segments'].split() store.cleanup() # Verify snapshot integrity def cmd_verify_snapshots(snapshots): get_passphrase() lowlevel = lbs.LowlevelDataStore(options.store) store = lbs.ObjectStore(lowlevel) for s in snapshots: print "#### Snapshot", s d = lbs.parse_full(store.load_snapshot(s)) print "## Root:", d['Root'] metadata = lbs.iterate_metadata(store, d['Root']) for m in metadata: if m.fields['type'] != '-': continue print "%s [%d bytes]" % (m.fields['name'], int(m.fields['size'])) verifier = lbs.ChecksumVerifier(m.fields['checksum']) size = 0 for block in m.data(): data = store.get(block) verifier.update(data) size += len(data) if int(m.fields['size']) != size: raise ValueError("File size does not match!") if not verifier.valid(): raise ValueError("Bad checksum found") store.cleanup() # Restore a snapshot, or some subset of files from it def cmd_restore_snapshot(args): get_passphrase() lowlevel = lbs.LowlevelDataStore(options.store) store = lbs.ObjectStore(lowlevel) snapshot = lbs.parse_full(store.load_snapshot(args[0])) destdir = args[1] paths = args[2:] def warn(m, msg): print "Warning: %s: %s" % (m.items.name, msg) for m in lbs.iterate_metadata(store, snapshot['Root']): pathname = os.path.normpath(m.items.name) while os.path.isabs(pathname): pathname = pathname[1:] print pathname destpath = os.path.join(destdir, pathname) (path, filename) = os.path.split(destpath) # TODO: Check for ../../../paths that might attempt to write outside # the destination directory. Maybe also check attempts to follow # symlinks pointing outside? try: if not os.path.isdir(path): os.makedirs(path) if m.items.type == '-': file = open(destpath, 'wb') verifier = lbs.ChecksumVerifier(m.items.checksum) size = 0 for block in m.data(): data = store.get(block) verifier.update(data) size += len(data) file.write(data) file.close() if int(m.fields['size']) != size: raise ValueError("File size does not match!") if not verifier.valid(): raise ValueError("Bad checksum found") elif m.items.type == 'd': if filename != '.': os.mkdir(destpath) elif m.items.type == 'l': os.symlink(m.items.contents, destpath) elif m.items.type == 'p': os.mkfifo(destpath) elif m.items.type in ('c', 'b'): if m.items.type == 'c': mode = 0600 | stat.S_IFCHR else: mode = 0600 | stat.S_IFBLK os.mknod(destpath, mode, os.makedev(*m.items.device)) elif m.items.type == 's': pass # TODO: Implement else: warn(m, "Unknown type code: " + m.items.type) continue except Exception, e: warn(m, "Error restoring: %s" % (e,)) continue try: uid = m.items.user[0] gid = m.items.group[0] os.lchown(destpath, uid, gid) except Exception, e: warn(m, "Error restoring file ownership: %s" % (e,)) if m.items.type == 'l': continue try: os.chmod(destpath, m.items.mode) except Exception, e: warn(m, "Error restoring file permissions: %s" % (e,)) try: os.utime(destpath, (time.time(), m.items.mtime)) except Exception, e: warn(m, "Error restoring file timestamps: %s" % (e,)) store.cleanup() if len(args) == 0: parser.print_usage() sys.exit(1) cmd = args[0] args = args[1:] if cmd == 'clean': cmd_clean() elif cmd == 'prune-db': cmd_prune_db() elif cmd == 'list-snapshots': cmd_list_snapshots() elif cmd == 'object-sums': cmd_object_checksums(args) elif cmd == 'read-snapshots': cmd_read_snapshots(args) elif cmd == 'list-snapshot-sizes': cmd_list_snapshot_sizes() elif cmd == 'verify-snapshots': cmd_verify_snapshots(args) elif cmd == 'restore-snapshot': cmd_restore_snapshot(args) else: print "Unknown command:", cmd parser.print_usage() sys.exit(1)