"""Implementation of the Cumulus command-line utility program."""
+from __future__ import division, print_function, unicode_literals
+
import getpass, os, stat, sys, time
from optparse import OptionParser
# environment variable.
def get_passphrase():
ENV_KEY = 'LBS_GPG_PASSPHRASE'
- if not os.environ.has_key(ENV_KEY):
+ if ENV_KEY not in os.environ:
os.environ[ENV_KEY] = getpass.getpass()
def cmd_prune_db(args):
# Expire segments which are poorly-utilized.
for s in db.get_segment_cleaning_list():
if s.cleaning_benefit > clean_threshold:
- print "Cleaning segment %d (benefit %.2f)" % (s.id,
- s.cleaning_benefit)
+ print("Cleaning segment %d (benefit %.2f)" % (s.id,
+ s.cleaning_benefit))
db.mark_segment_expired(s)
else:
break
""" List snapshots stored.
Syntax: $0 --data=DATADIR list-snapshots
"""
- store = cumulus.LowlevelDataStore(options.store)
- for s in sorted(store.list_snapshots()):
- print s
+ store = cumulus.CumulusStore(options.store)
+ for s in sorted(store.list_snapshots()): print(s)
def cmd_list_snapshot_sizes(args):
""" List size of data needed for each snapshot.
Syntax: $0 --data=DATADIR list-snapshot-sizes
"""
- lowlevel = cumulus.LowlevelDataStore(options.store)
- lowlevel.scan()
- store = cumulus.ObjectStore(lowlevel)
+ store = cumulus.CumulusStore(options.store)
+ backend = store.backend
+ backend.prefetch_generic()
previous = set()
- exts = {}
- for seg in lowlevel.store.list('segments'):
- exts.update ([seg.split ('.', 1)])
- for s in sorted(lowlevel.list_snapshots()):
+ size = 0
+ def get_size(segment):
+ return backend.stat_generic(segment + ".tar", "segments")["size"]
+ for s in sorted(store.list_snapshots()):
d = cumulus.parse_full(store.load_snapshot(s))
check_version(d['Format'])
- try:
- intent = float(d['Backup-Intent'])
- except:
- intent = 1.0
-
- segments = d['Segments'].split()
- (size, added, removed, addcount, remcount) = (0, 0, 0, 0, 0)
- lo_stat = lowlevel.lowlevel_stat
- for seg in segments:
- segsize = lo_stat('.'.join ((seg, exts[seg])))['size']
- size += segsize
- if seg not in previous:
- added += segsize
- addcount += 1
- for seg in previous:
- if seg not in segments:
- removed += lo_stat('.'.join((seg, exts[seg])))['size']
- remcount += 1
- previous = set(segments)
- print "%s [%s]: %.3f +%.3f -%.3f (+%d/-%d segments)" % (s, intent, size / 1024.0**2, added / 1024.0**2, removed / 1024.0**2, addcount, remcount)
+ segments = set(d['Segments'].split())
+ (added, removed, addcount, remcount) = (0, 0, 0, 0)
+ for seg in segments.difference(previous):
+ added += get_size(seg)
+ addcount += 1
+ for seg in previous.difference(segments):
+ removed += get_size(seg)
+ remcount += 1
+ size += added - removed
+ previous = segments
+ print("%s: %.3f +%.3f -%.3f (+%d/-%d segments)" % (s, size / 1024.0**2, added / 1024.0**2, removed / 1024.0**2, addcount, remcount))
def cmd_garbage_collect(args):
""" Search for any files which are not needed by any current
snapshots and offer to delete them.
Syntax: $0 --store=DATADIR gc
"""
- lowlevel = cumulus.LowlevelDataStore(options.store)
- lowlevel.scan()
- store = cumulus.ObjectStore(lowlevel)
- snapshots = set(lowlevel.list_snapshots())
- segments = set()
- for s in snapshots:
+ store = cumulus.CumulusStore(options.store)
+ backend = store.backend
+ referenced = set()
+ for s in store.list_snapshots():
d = cumulus.parse_full(store.load_snapshot(s))
check_version(d['Format'])
- segments.update(d['Segments'].split())
-
- referenced = snapshots.union(segments)
- reclaimed = 0
- for (t, r) in cumulus.store.type_patterns.items():
- for f in lowlevel.store.list(t):
- m = r.match(f)
- if m is None or m.group(1) not in referenced:
- print "Garbage:", (t, f)
- reclaimed += lowlevel.store.stat(t, f)['size']
- if not options.dry_run:
- lowlevel.store.delete(t, f)
- print "Reclaimed space:", reclaimed
+ referenced.add(s)
+ referenced.update(d['Segments'].split())
-cmd_gc = cmd_garbage_collect
+ print(referenced)
-def cmd_object_checksums(segments):
- """ Build checksum list for objects in the given segments, or all
- segments if none are specified.
- """
- get_passphrase()
- lowlevel = cumulus.LowlevelDataStore(options.store)
- store = cumulus.ObjectStore(lowlevel)
- if len(segments) == 0:
- segments = sorted(lowlevel.list_segments())
- for s in segments:
- for (o, data) in store.load_segment(s):
- csum = cumulus.ChecksumCreator().update(data).compute()
- print "%s/%s:%d:%s" % (s, o, len(data), csum)
- store.cleanup()
-object_sums = cmd_object_checksums
+ to_delete = []
+ to_preserve = []
+ for filetype in cumulus.SEARCH_PATHS:
+ for (name, path) in store.backend.list_generic(filetype):
+ if name in referenced:
+ to_preserve.append(path)
+ else:
+ to_delete.append(path)
+
+ print(to_preserve)
+ print(to_delete)
+
+ raw_backend = backend.raw_backend
+ for f in to_delete:
+ print("Delete:", f)
+ if not options.dry_run:
+ raw_backend.delete(f)
+cmd_gc = cmd_garbage_collect
def cmd_read_snapshots(snapshots):
""" Read a snapshot file
"""
get_passphrase()
- lowlevel = cumulus.LowlevelDataStore(options.store)
- store = cumulus.ObjectStore(lowlevel)
+ store = cumulus.CumulusStore(options.store)
for s in snapshots:
d = cumulus.parse_full(store.load_snapshot(s))
check_version(d['Format'])
- print d
- print d['Segments'].split()
+ print(d)
+ print(d['Segments'].split())
store.cleanup()
def cmd_read_metadata(args):
"""
snapshot = args [0]
get_passphrase()
- lowlevel = cumulus.LowlevelDataStore(options.store)
- store = cumulus.ObjectStore(lowlevel)
+ store = cumulus.CumulusStore(options.store)
d = cumulus.parse_full(store.load_snapshot(snapshot))
check_version(d['Format'])
metadata = cumulus.read_metadata(store, d['Root'])
""" Verify snapshot integrity
"""
get_passphrase()
- lowlevel = cumulus.LowlevelDataStore(options.store)
- store = cumulus.ObjectStore(lowlevel)
+ store = cumulus.CumulusStore(options.store)
for s in snapshots:
cumulus.accessed_segments.clear()
- print "#### Snapshot", s
+ print("#### Snapshot", s)
d = cumulus.parse_full(store.load_snapshot(s))
check_version(d['Format'])
- print "## Root:", d['Root']
+ print("## Root:", d['Root'])
metadata = cumulus.iterate_metadata(store, d['Root'])
for m in metadata:
if m.fields['type'] not in ('-', 'f'): continue
- print "%s [%d bytes]" % (m.fields['name'], int(m.fields['size']))
+ print("%s [%d bytes]" % (m.fields['name'], int(m.fields['size'])))
verifier = cumulus.ChecksumVerifier(m.fields['checksum'])
size = 0
for block in m.data():
# doesn't contain duplicates.
listed_segments = set(d['Segments'].split())
if cumulus.accessed_segments - listed_segments:
- print "Error: Some segments not listed in descriptor!"
- print sorted(list(cumulus.accessed_segments - listed_segments))
+ print("Error: Some segments not listed in descriptor!")
+ print(sorted(list(cumulus.accessed_segments - listed_segments)))
if listed_segments - cumulus.accessed_segments :
- print "Warning: Extra unused segments listed in descriptor!"
- print sorted(list(listed_segments - cumulus.accessed_segments))
+ print("Warning: Extra unused segments listed in descriptor!")
+ print(sorted(list(listed_segments - cumulus.accessed_segments)))
store.cleanup()
def cmd_restore_snapshot(args):
""" Restore a snapshot, or some subset of files from it
"""
get_passphrase()
- lowlevel = cumulus.LowlevelDataStore(options.store)
- store = cumulus.ObjectStore(lowlevel)
+ store = cumulus.CumulusStore(options.store)
snapshot = cumulus.parse_full(store.load_snapshot(args[0]))
check_version(snapshot['Format'])
destdir = args[1]
return False
def warn(m, msg):
- print "Warning: %s: %s" % (m.items.name, msg)
+ print("Warning: %s: %s" % (m.items.name, msg))
# Phase 1: Read the complete metadata log and create directory structure.
metadata_items = []
metadata_paths[pathname] = m
for block in m.data():
(segment, object, checksum, slice) \
- = cumulus.ObjectStore.parse_ref(block)
+ = cumulus.CumulusStore.parse_ref(block)
if segment not in metadata_segments:
metadata_segments[segment] = set()
metadata_segments[segment].add(pathname)
try:
if not os.path.isdir(path):
- print "mkdir:", path
+ print("mkdir:", path)
os.makedirs(path)
- except Exception, e:
+ except Exception as e:
warn(m, "Error creating directory structure: %s" % (e,))
continue
# Phase 2: Restore files, ordered by how data is stored in segments.
def restore_file(pathname, m):
assert m.items.type in ('-', 'f')
- print "extract:", pathname
+ print("extract:", pathname)
destpath = os.path.join(destdir, pathname)
file = open(destpath, 'wb')
while metadata_segments:
(segment, items) = metadata_segments.popitem()
- print "+ Segment", segment
+ print("+ Segment", segment)
for pathname in sorted(items):
if pathname in metadata_paths:
restore_file(pathname, metadata_paths[pathname])
del metadata_paths[pathname]
- print "+ Remaining files"
+ print("+ Remaining files")
while metadata_paths:
(pathname, m) = metadata_paths.popitem()
restore_file(pathname, m)
# Phase 3: Restore special files (symlinks, devices).
# Phase 4: Restore directory permissions and modification times.
for (pathname, m) in reversed(metadata_items):
- print "permissions:", pathname
+ print("permissions:", pathname)
destpath = os.path.join(destdir, pathname)
(path, filename) = os.path.split(destpath)
os.mkfifo(destpath)
elif m.items.type in ('c', 'b'):
if m.items.type == 'c':
- mode = 0600 | stat.S_IFCHR
+ mode = 0o600 | stat.S_IFCHR
else:
- mode = 0600 | stat.S_IFBLK
+ mode = 0o600 | stat.S_IFBLK
os.mknod(destpath, mode, os.makedev(*m.items.device))
elif m.items.type == 's':
pass # TODO: Implement
warn(m, "Unknown type code: " + m.items.type)
continue
- except Exception, e:
+ except Exception as e:
warn(m, "Error restoring: %s" % (e,))
continue
uid = m.items.user[0]
gid = m.items.group[0]
os.lchown(destpath, uid, gid)
- except Exception, e:
+ except Exception as e:
warn(m, "Error restoring file ownership: %s" % (e,))
if m.items.type == 'l':
try:
os.chmod(destpath, m.items.mode)
- except Exception, e:
+ except Exception as e:
warn(m, "Error restoring file permissions: %s" % (e,))
try:
os.utime(destpath, (time.time(), m.items.mtime))
- except Exception, e:
+ except Exception as e:
warn(m, "Error restoring file timestamps: %s" % (e,))
store.cleanup()
def main(argv):
usage = ["%prog [option]... command [arg]...", "", "Commands:"]
cmd = method = None
- for cmd, method in globals().iteritems():
+ for cmd, method in globals().items():
if cmd.startswith ('cmd_'):
usage.append(cmd[4:].replace('_', '-') + ':' + method.__doc__)
parser = OptionParser(usage="\n".join(usage))
if method:
method (args)
else:
- print "Unknown command:", cmd
+ print("Unknown command:", cmd)
parser.print_usage()
sys.exit(1)
projects / cumulus.git / blobdiff