Allow a URL to be used in cumulus-util to specify a store location.

Both file:/// and s3:// URLs are supported, reading data from the local
filesystem or Amazon S3, respectively.  Local paths (without a file:///
prefix) can still be specified.

diff --git a/python/cumulus/__init__.py b/python/cumulus/__init__.py
index 602e70e..51d3ee8 100644 (file)
--- a/python/cumulus/__init__.py
+++ b/python/cumulus/__init__.py
@@ -84,7 +84,10 @@ class LowlevelDataStore:
     """
 
     def __init__(self, path):
-        self.store = cumulus.store.file.FileStore(path)
+        if path.find(":") >= 0:
+            self.store = cumulus.store.open(path)
+        else:
+            self.store = cumulus.store.file.FileStore(path)
 
     def _classify(self, filename):
         for (t, r) in cumulus.store.type_patterns.items():

diff --git a/python/cumulus/store/__init__.py b/python/cumulus/store/__init__.py
index 5ebf0d7..08b4517 100644 (file)
--- a/python/cumulus/store/__init__.py
+++ b/python/cumulus/store/__init__.py
@@ -1,4 +1,4 @@
-import re
+import re, urlparse
 
 type_patterns = {
     'checksums': re.compile(r"^snapshot-(.*)\.(\w+)sums$"),
@@ -20,3 +20,18 @@ class Store:
 
     def delete(self, type, name):
         raise NotImplementedException
+
+def open(url):
+    (scheme, netloc, path, params, query, fragment) \
+        = urlparse.urlparse(url)
+
+    if scheme == "file":
+        import cumulus.store.file
+        return cumulus.store.file.FileStore(path)
+    elif scheme == "s3":
+        import cumulus.store.s3
+        while path.startswith("/"): path = path[1:]
+        (bucket, path) = path.split("/", 1)
+        return cumulus.store.s3.S3Store(bucket, path)
+    else:
+        raise NotImplementedException

diff --git a/python/cumulus/store/s3.py b/python/cumulus/store/s3.py
index 0453c21..6460fb6 100644 (file)
--- a/python/cumulus/store/s3.py
+++ b/python/cumulus/store/s3.py
@@ -36,3 +36,7 @@ class S3Store(cumulus.store.Store):
 
     def delete(self, type, name):
         self.bucket.delete_key("%s/%s/%s" % (self.prefix, type, name))
+
+    def stat(self, type, name):
+        k = self.bucket.get_key("%s/%s/%s" % (self.prefix, type, name))
+        return {'size': int(k.size)}