X-Git-Url: http://git.vrable.net/?p=cumulus.git;a=blobdiff_plain;f=cumulus-sync;h=9f31a29b8131a6b8e2641aab6d20c3f20dfd07c9;hp=b76ebb53188bf2ac56ab7db3599de09767a69781;hb=HEAD;hpb=64bff41cb3ccdd60e767a5bb9ed8525d2dda1966

diff --git a/cumulus-sync b/cumulus-sync
index b76ebb5..9f31a29 100755
--- a/cumulus-sync
+++ b/cumulus-sync
@@ -20,6 +20,8 @@
 
 # Tool for copying cumulus archives from one source to another.
 
+from __future__ import division, print_function, unicode_literals
+
 import os, sys
 
 # Automatically set Python path, based on script directory.  This should be
@@ -30,27 +32,37 @@ sys.path.append(os.path.join(script_directory, 'python'))
 import cumulus
 import cumulus.store
 
-store1 = cumulus.store.open(sys.argv[1])
-store2 = cumulus.store.open(sys.argv[2])
+store1 = cumulus.BackendWrapper(sys.argv[1])
+store2 = cumulus.BackendWrapper(sys.argv[2])
 
-source = cumulus.ObjectStore(cumulus.LowlevelDataStore(store1))
+source = cumulus.CumulusStore(store1)
 
-filter = set()
-for s in sys.argv[3:]:
-    filter.add(s)
+items_required = set()
+snapshots = sys.argv[3:]
+if not snapshots:
+    snapshots = list(source.list_snapshots())
+for s in snapshots:
+    items_required.add(s)
     d = cumulus.parse_full(source.load_snapshot(s))
-    filter.update(d['Segments'].split())
-
-for ty in ('segments', 'checksums', 'snapshots'):
-    for f in sorted(store1.list(ty)):
-        m = cumulus.store.type_patterns[ty].match(f)
-        if not m: continue
-        if filter and m.group(1) not in filter:
-            continue
-
-        print ty, f
-        try:
-            store2.stat(ty, f)
-        except cumulus.store.NotFoundError:
-            store2.put(ty, f, store1.get(ty, f))
-            print "    [sent]"
+    items_required.update(d['Segments'].split())
+print("Required:", len(items_required))
+
+files_present = set()
+for filetype in cumulus.SEARCH_PATHS:
+    for (name, path) in store2.list_generic(filetype):
+        items_required.discard(name)
+        files_present.add(path)
+print("Files already present:", len(sorted(files_present)))
+
+files_required = []
+items_found = set()
+for filetype in cumulus.SEARCH_PATHS:
+    for (name, path) in store1.list_generic(filetype):
+        if name in items_required:
+            files_required.append(path)
+            items_found.add(name)
+files_required.sort()
+
+for i, f in enumerate(files_required):
+    print("[%d/%d] %s" % (i + 1, len(files_required), f))
+    store2.raw_backend.put(f, store1.raw_backend.get(f))