From: Michael Vrable <mvrable@cs.ucsd.edu>
Date: Mon, 15 Dec 2008 22:29:13 +0000 (-0800)
Subject: cumulus-sync: Create a tool for copying snapshots between locations.
X-Git-Url: http://git.vrable.net/?p=cumulus.git;a=commitdiff_plain;h=5d0038c47ba5926b1e3e8713da75078602a06f62

cumulus-sync: Create a tool for copying snapshots between locations.

This will automatically find and copy all needed segments that are not
already present, and can handle both local filesystems and remote storage
with Amazon S3.
---

diff --git a/cumulus-sync b/cumulus-sync
new file mode 100755
index 0000000..9fb835c
--- /dev/null
+++ b/cumulus-sync
@@ -0,0 +1,38 @@
+#!/usr/bin/python
+#
+# Tool for copying cumulus archives from one source to another.
+
+import os, sys
+
+# Automatically set Python path, based on script directory.  This should be
+# removed if the tools are properly installed somewhere.
+script_directory = os.path.dirname(sys.argv[0])
+sys.path.append(os.path.join(script_directory, 'python'))
+
+import cumulus
+import cumulus.store
+
+store1 = cumulus.store.open(sys.argv[1])
+store2 = cumulus.store.open(sys.argv[2])
+
+source = cumulus.ObjectStore(cumulus.LowlevelDataStore(store1))
+
+filter = set()
+for s in sys.argv[3:]:
+    filter.add(s)
+    d = cumulus.parse_full(source.load_snapshot(s))
+    filter.update(d['Segments'].split())
+
+for ty in ('segments', 'checksums', 'snapshots'):
+    for f in sorted(store1.list(ty)):
+        m = cumulus.store.type_patterns[ty].match(f)
+        if not m: continue
+        if filter and m.group(1) not in filter:
+            continue
+
+        print ty, f
+        try:
+            store2.stat(ty, f)
+        except cumulus.store.NotFoundError:
+            store2.put(ty, f, store1.get(ty, f))
+            print "    [sent]"
diff --git a/python/cumulus/__init__.py b/python/cumulus/__init__.py
index e1dac84..424e38c 100644
--- a/python/cumulus/__init__.py
+++ b/python/cumulus/__init__.py
@@ -97,7 +97,9 @@ class LowlevelDataStore:
     """
 
     def __init__(self, path):
-        if path.find(":") >= 0:
+        if isinstance(path, cumulus.store.Store):
+            self.store = path
+        elif path.find(":") >= 0:
             self.store = cumulus.store.open(path)
         else:
             self.store = cumulus.store.file.FileStore(path)