From: Michael Vrable Date: Wed, 30 May 2012 04:47:50 +0000 (-0700) Subject: Update to support new snapshot version in Python tools. X-Git-Url: http://git.vrable.net/?p=cumulus.git;a=commitdiff_plain;h=78376bd08b8f19a73fde4a1d922c743bd0a9dfbf Update to support new snapshot version in Python tools. Also, remove some older references to "lbs". --- diff --git a/cumulus-util b/cumulus-util index c905308..1608b63 100755 --- a/cumulus-util +++ b/cumulus-util @@ -12,17 +12,14 @@ sys.path.append(os.path.join(script_directory, 'python')) import cumulus -# Compatibility -lbs = cumulus - -# We support up to "LBS Snapshot v0.8" formats, but are also limited by the lbs -# module. -FORMAT_VERSION = min(lbs.FORMAT_VERSION, (0, 8)) +# We support up to "Cumulus Snapshot v0.11" formats, but are also limited by +# the cumulus module. +FORMAT_VERSION = min(cumulus.FORMAT_VERSION, (0, 11)) def check_version(format): - ver = lbs.parse_metadata_version(format) + ver = cumulus.parse_metadata_version(format) if ver > FORMAT_VERSION: - raise RuntimeError("Unsupported LBS format: " + format) + raise RuntimeError("Unsupported Cumulus format: " + format) # Read a passphrase from the user and store it in the LBS_GPG_PASSPHRASE # environment variable. @@ -36,7 +33,7 @@ def cmd_prune_db(args): actually schedule any segment cleaning. Syntax: $0 --localdb=LOCALDB prune-db """ - db = lbs.LocalDatabase(options.localdb) + db = cumulus.LocalDatabase(options.localdb) # Delete old snapshots from the local database. #db.garbage_collect() @@ -46,7 +43,7 @@ def cmd_clean(args, clean_threshold=7.0): """ Run the segment cleaner. Syntax: $0 --localdb=LOCALDB clean """ - db = lbs.LocalDatabase(options.localdb) + db = cumulus.LocalDatabase(options.localdb) # Delete old snapshots from the local database. intent = float(options.intent) @@ -68,7 +65,7 @@ def cmd_list_snapshots(args): """ List snapshots stored. Syntax: $0 --data=DATADIR list-snapshots """ - store = lbs.LowlevelDataStore(options.store) + store = cumulus.LowlevelDataStore(options.store) for s in sorted(store.list_snapshots()): print s @@ -76,15 +73,15 @@ def cmd_list_snapshot_sizes(args): """ List size of data needed for each snapshot. Syntax: $0 --data=DATADIR list-snapshot-sizes """ - lowlevel = lbs.LowlevelDataStore(options.store) + lowlevel = cumulus.LowlevelDataStore(options.store) lowlevel.scan() - store = lbs.ObjectStore(lowlevel) + store = cumulus.ObjectStore(lowlevel) previous = set() exts = {} for seg in lowlevel.store.list('segments'): exts.update ([seg.split ('.', 1)]) for s in sorted(lowlevel.list_snapshots()): - d = lbs.parse_full(store.load_snapshot(s)) + d = cumulus.parse_full(store.load_snapshot(s)) check_version(d['Format']) try: @@ -113,13 +110,13 @@ def cmd_garbage_collect(args): snapshots and offer to delete them. Syntax: $0 --store=DATADIR gc """ - lowlevel = lbs.LowlevelDataStore(options.store) + lowlevel = cumulus.LowlevelDataStore(options.store) lowlevel.scan() - store = lbs.ObjectStore(lowlevel) + store = cumulus.ObjectStore(lowlevel) snapshots = set(lowlevel.list_snapshots()) segments = set() for s in snapshots: - d = lbs.parse_full(store.load_snapshot(s)) + d = cumulus.parse_full(store.load_snapshot(s)) check_version(d['Format']) segments.update(d['Segments'].split()) @@ -142,13 +139,13 @@ def cmd_object_checksums(segments): segments if none are specified. """ get_passphrase() - lowlevel = lbs.LowlevelDataStore(options.store) - store = lbs.ObjectStore(lowlevel) + lowlevel = cumulus.LowlevelDataStore(options.store) + store = cumulus.ObjectStore(lowlevel) if len(segments) == 0: segments = sorted(lowlevel.list_segments()) for s in segments: for (o, data) in store.load_segment(s): - csum = lbs.ChecksumCreator().update(data).compute() + csum = cumulus.ChecksumCreator().update(data).compute() print "%s/%s:%d:%s" % (s, o, len(data), csum) store.cleanup() object_sums = cmd_object_checksums @@ -157,10 +154,10 @@ def cmd_read_snapshots(snapshots): """ Read a snapshot file """ get_passphrase() - lowlevel = lbs.LowlevelDataStore(options.store) - store = lbs.ObjectStore(lowlevel) + lowlevel = cumulus.LowlevelDataStore(options.store) + store = cumulus.ObjectStore(lowlevel) for s in snapshots: - d = lbs.parse_full(store.load_snapshot(s)) + d = cumulus.parse_full(store.load_snapshot(s)) check_version(d['Format']) print d print d['Segments'].split() @@ -171,11 +168,11 @@ def cmd_read_metadata(args): """ snapshot = args [0] get_passphrase() - lowlevel = lbs.LowlevelDataStore(options.store) - store = lbs.ObjectStore(lowlevel) - d = lbs.parse_full(store.load_snapshot(snapshot)) + lowlevel = cumulus.LowlevelDataStore(options.store) + store = cumulus.ObjectStore(lowlevel) + d = cumulus.parse_full(store.load_snapshot(snapshot)) check_version(d['Format']) - metadata = lbs.read_metadata(store, d['Root']) + metadata = cumulus.read_metadata(store, d['Root']) blank = True for l in metadata: if l == '\n': @@ -190,19 +187,19 @@ def cmd_verify_snapshots(snapshots): """ Verify snapshot integrity """ get_passphrase() - lowlevel = lbs.LowlevelDataStore(options.store) - store = lbs.ObjectStore(lowlevel) + lowlevel = cumulus.LowlevelDataStore(options.store) + store = cumulus.ObjectStore(lowlevel) for s in snapshots: - lbs.accessed_segments.clear() + cumulus.accessed_segments.clear() print "#### Snapshot", s - d = lbs.parse_full(store.load_snapshot(s)) + d = cumulus.parse_full(store.load_snapshot(s)) check_version(d['Format']) print "## Root:", d['Root'] - metadata = lbs.iterate_metadata(store, d['Root']) + metadata = cumulus.iterate_metadata(store, d['Root']) for m in metadata: if m.fields['type'] not in ('-', 'f'): continue print "%s [%d bytes]" % (m.fields['name'], int(m.fields['size'])) - verifier = lbs.ChecksumVerifier(m.fields['checksum']) + verifier = cumulus.ChecksumVerifier(m.fields['checksum']) size = 0 for block in m.data(): data = store.get(block) @@ -217,21 +214,21 @@ def cmd_verify_snapshots(snapshots): # actually accurate: covered all segments that were really read, and # doesn't contain duplicates. listed_segments = set(d['Segments'].split()) - if lbs.accessed_segments - listed_segments: + if cumulus.accessed_segments - listed_segments: print "Error: Some segments not listed in descriptor!" - print sorted(list(lbs.accessed_segments - listed_segments)) - if listed_segments - lbs.accessed_segments : + print sorted(list(cumulus.accessed_segments - listed_segments)) + if listed_segments - cumulus.accessed_segments : print "Warning: Extra unused segments listed in descriptor!" - print sorted(list(listed_segments - lbs.accessed_segments)) + print sorted(list(listed_segments - cumulus.accessed_segments)) store.cleanup() def cmd_restore_snapshot(args): """ Restore a snapshot, or some subset of files from it """ get_passphrase() - lowlevel = lbs.LowlevelDataStore(options.store) - store = lbs.ObjectStore(lowlevel) - snapshot = lbs.parse_full(store.load_snapshot(args[0])) + lowlevel = cumulus.LowlevelDataStore(options.store) + store = cumulus.ObjectStore(lowlevel) + snapshot = cumulus.parse_full(store.load_snapshot(args[0])) check_version(snapshot['Format']) destdir = args[1] paths = args[2:] @@ -254,7 +251,7 @@ def cmd_restore_snapshot(args): metadata_items = [] metadata_paths = {} metadata_segments = {} - for m in lbs.iterate_metadata(store, snapshot['Root']): + for m in cumulus.iterate_metadata(store, snapshot['Root']): pathname = os.path.normpath(m.items.name) while os.path.isabs(pathname): pathname = pathname[1:] @@ -271,7 +268,7 @@ def cmd_restore_snapshot(args): metadata_paths[pathname] = m for block in m.data(): (segment, object, checksum, slice) \ - = lbs.ObjectStore.parse_ref(block) + = cumulus.ObjectStore.parse_ref(block) if segment not in metadata_segments: metadata_segments[segment] = set() metadata_segments[segment].add(pathname) @@ -291,7 +288,7 @@ def cmd_restore_snapshot(args): destpath = os.path.join(destdir, pathname) file = open(destpath, 'wb') - verifier = lbs.ChecksumVerifier(m.items.checksum) + verifier = cumulus.ChecksumVerifier(m.items.checksum) size = 0 for block in m.data(): data = store.get(block) diff --git a/python/cumulus/__init__.py b/python/cumulus/__init__.py index 5418d3c..ff811b0 100644 --- a/python/cumulus/__init__.py +++ b/python/cumulus/__init__.py @@ -1,7 +1,7 @@ -"""High-level interface for working with LBS archives. +"""High-level interface for working with Cumulus archives. This module provides an easy interface for reading from and manipulating -various parts of an LBS archive: +various parts of a Cumulus archive: - listing the snapshots and segments present - reading segment contents - parsing snapshot descriptors and snapshot metadata logs @@ -15,7 +15,7 @@ from pysqlite2 import dbapi2 as sqlite3 import cumulus.store, cumulus.store.file # The largest supported snapshot format that can be understood. -FORMAT_VERSION = (0, 8) # LBS Snapshot v0.8 +FORMAT_VERSION = (0, 11) # Cumulus Snapshot v0.11 # Maximum number of nested indirect references allowed in a snapshot. MAX_RECURSION_DEPTH = 3 @@ -60,7 +60,7 @@ CHECKSUM_ALGORITHMS = { } class ChecksumCreator: - """Compute an LBS checksum for provided data. + """Compute a Cumulus checksum for provided data. The algorithm used is selectable, but currently defaults to sha1. """ @@ -324,7 +324,7 @@ def parse_full(lines): def parse_metadata_version(s): """Convert a string with the snapshot version format to a tuple.""" - m = re.match(r"^LBS Snapshot v(\d+(\.\d+)*)$", s) + m = re.match(r"^(?:Cumulus|LBS) Snapshot v(\d+(\.\d+)*)$", s) if m is None: return () else: @@ -682,7 +682,7 @@ class LocalDatabase: """ # The expired column of the block_index table is used when generating a - # new LBS snapshot. A null value indicates that an object may be + # new Cumulus snapshot. A null value indicates that an object may be # re-used. Otherwise, an object must be written into a new segment if # needed. Objects with distinct expired values will be written into # distinct segments, to allow for some grouping by age. The value 0 is