Add a few items to the TODO list.

[cumulus.git] / cumulus-util

diff --git a/cumulus-util b/cumulus-util
index 05043f1..1bac12b 100755 (executable)
--- a/cumulus-util
+++ b/cumulus-util
@@ -4,6 +4,12 @@
 
 import getpass, os, stat, sys, time
 from optparse import OptionParser
+
+# Automatically set Python path, based on script directory.  This should be
+# removed if the tools are properly installed somewhere.
+script_directory = os.path.dirname(sys.argv[0])
+sys.path.append(os.path.join(script_directory, 'python'))
+
 import cumulus
 
 # Compatibility
@@ -21,6 +27,8 @@ def check_version(format):
 parser = OptionParser(usage="%prog [option]... command [arg]...")
 parser.add_option("-v", action="store_true", dest="verbose", default=False,
                   help="increase verbosity")
+parser.add_option("-n", action="store_true", dest="dry_run", default=False,
+                  help="dry run")
 parser.add_option("--store", dest="store",
                   help="specify path to backup data store")
 parser.add_option("--localdb", dest="localdb",
@@ -78,6 +86,7 @@ def cmd_list_snapshots():
 # Syntax: $0 --data=DATADIR list-snapshot-sizes
 def cmd_list_snapshot_sizes():
     lowlevel = lbs.LowlevelDataStore(options.store)
+    lowlevel.scan()
     store = lbs.ObjectStore(lowlevel)
     previous = set()
     for s in sorted(lowlevel.list_snapshots()):
@@ -90,16 +99,45 @@ def cmd_list_snapshot_sizes():
             intent = 1.0
 
         segments = d['Segments'].split()
-        (size, added, removed) = (0, 0, 0)
+        (size, added, removed, addcount, remcount) = (0, 0, 0, 0, 0)
         for seg in segments:
             segsize = lowlevel.lowlevel_stat(seg + ".tar.gpg")['size']
             size += segsize
-            if seg not in previous: added += segsize
+            if seg not in previous:
+                added += segsize
+                addcount += 1
         for seg in previous:
             if seg not in segments:
                 removed += lowlevel.lowlevel_stat(seg + ".tar.gpg")['size']
+                remcount += 1
         previous = set(segments)
-        print "%s [%s]: %.3f +%.3f -%.3f" % (s, intent, size / 1024.0**2, added / 1024.0**2, removed / 1024.0**2)
+        print "%s [%s]: %.3f +%.3f -%.3f (+%d/-%d segments)" % (s, intent, size / 1024.0**2, added / 1024.0**2, removed / 1024.0**2, addcount, remcount)
+
+# Search for any files which are not needed by any current snapshots and offer
+# to delete them.
+# Syntax: $0 --store=DATADIR gc
+def cmd_garbage_collect():
+    lowlevel = lbs.LowlevelDataStore(options.store)
+    lowlevel.scan()
+    store = lbs.ObjectStore(lowlevel)
+    snapshots = set(lowlevel.list_snapshots())
+    segments = set()
+    for s in snapshots:
+        d = lbs.parse_full(store.load_snapshot(s))
+        check_version(d['Format'])
+        segments.update(d['Segments'].split())
+
+    referenced = snapshots.union(segments)
+    reclaimed = 0
+    for (t, r) in cumulus.store.type_patterns.items():
+        for f in lowlevel.store.list(t):
+            m = r.match(f)
+            if m is None or m.group(1) not in referenced:
+                print "Garbage:", (t, f)
+                reclaimed += lowlevel.store.stat(t, f)['size']
+                if not options.dry_run:
+                    lowlevel.store.delete(t, f)
+    print "Reclaimed space:", reclaimed
 
 # Build checksum list for objects in the given segments, or all segments if
 # none are specified.
@@ -354,6 +392,8 @@ elif cmd == 'read-metadata':
     cmd_read_metadata(args[0])
 elif cmd == 'list-snapshot-sizes':
     cmd_list_snapshot_sizes()
+elif cmd == 'gc':
+    cmd_garbage_collect()
 elif cmd == 'verify-snapshots':
     cmd_verify_snapshots(args)
 elif cmd == 'restore-snapshot':