cumulus-sync: Create a tool for copying snapshots between locations.

This will automatically find and copy all needed segments that are not
already present, and can handle both local filesystems and remote storage
with Amazon S3.

diff --git a/cumulus-sync b/cumulus-sync
new file mode 100755 (executable)
index 0000000..9fb835c
--- /dev/null
+++ b/cumulus-sync
@@ -0,0 +1,38 @@
+#!/usr/bin/python
+#
+# Tool for copying cumulus archives from one source to another.
+
+import os, sys
+
+# Automatically set Python path, based on script directory.  This should be
+# removed if the tools are properly installed somewhere.
+script_directory = os.path.dirname(sys.argv[0])
+sys.path.append(os.path.join(script_directory, 'python'))
+
+import cumulus
+import cumulus.store
+
+store1 = cumulus.store.open(sys.argv[1])
+store2 = cumulus.store.open(sys.argv[2])
+
+source = cumulus.ObjectStore(cumulus.LowlevelDataStore(store1))
+
+filter = set()
+for s in sys.argv[3:]:
+    filter.add(s)
+    d = cumulus.parse_full(source.load_snapshot(s))
+    filter.update(d['Segments'].split())
+
+for ty in ('segments', 'checksums', 'snapshots'):
+    for f in sorted(store1.list(ty)):
+        m = cumulus.store.type_patterns[ty].match(f)
+        if not m: continue
+        if filter and m.group(1) not in filter:
+            continue
+
+        print ty, f
+        try:
+            store2.stat(ty, f)
+        except cumulus.store.NotFoundError:
+            store2.put(ty, f, store1.get(ty, f))
+            print "    [sent]"

diff --git a/python/cumulus/__init__.py b/python/cumulus/__init__.py
index e1dac84..424e38c 100644 (file)
--- a/python/cumulus/__init__.py
+++ b/python/cumulus/__init__.py
@@ -97,7 +97,9 @@ class LowlevelDataStore:
     """
 
     def __init__(self, path):
-        if path.find(":") >= 0:
+        if isinstance(path, cumulus.store.Store):
+            self.store = path
+        elif path.find(":") >= 0:
             self.store = cumulus.store.open(path)
         else:
             self.store = cumulus.store.file.FileStore(path)