From 2ee97034047db53780a52d803b1c577b4c23c303 Mon Sep 17 00:00:00 2001 From: Michael Vrable Date: Mon, 11 Aug 2008 14:38:30 -0700 Subject: [PATCH] Allow a URL to be used in cumulus-util to specify a store location. Both file:/// and s3:// URLs are supported, reading data from the local filesystem or Amazon S3, respectively. Local paths (without a file:/// prefix) can still be specified. --- python/cumulus/__init__.py | 5 ++++- python/cumulus/store/__init__.py | 17 ++++++++++++++++- python/cumulus/store/s3.py | 4 ++++ 3 files changed, 24 insertions(+), 2 deletions(-) diff --git a/python/cumulus/__init__.py b/python/cumulus/__init__.py index 602e70e..51d3ee8 100644 --- a/python/cumulus/__init__.py +++ b/python/cumulus/__init__.py @@ -84,7 +84,10 @@ class LowlevelDataStore: """ def __init__(self, path): - self.store = cumulus.store.file.FileStore(path) + if path.find(":") >= 0: + self.store = cumulus.store.open(path) + else: + self.store = cumulus.store.file.FileStore(path) def _classify(self, filename): for (t, r) in cumulus.store.type_patterns.items(): diff --git a/python/cumulus/store/__init__.py b/python/cumulus/store/__init__.py index 5ebf0d7..08b4517 100644 --- a/python/cumulus/store/__init__.py +++ b/python/cumulus/store/__init__.py @@ -1,4 +1,4 @@ -import re +import re, urlparse type_patterns = { 'checksums': re.compile(r"^snapshot-(.*)\.(\w+)sums$"), @@ -20,3 +20,18 @@ class Store: def delete(self, type, name): raise NotImplementedException + +def open(url): + (scheme, netloc, path, params, query, fragment) \ + = urlparse.urlparse(url) + + if scheme == "file": + import cumulus.store.file + return cumulus.store.file.FileStore(path) + elif scheme == "s3": + import cumulus.store.s3 + while path.startswith("/"): path = path[1:] + (bucket, path) = path.split("/", 1) + return cumulus.store.s3.S3Store(bucket, path) + else: + raise NotImplementedException diff --git a/python/cumulus/store/s3.py b/python/cumulus/store/s3.py index 0453c21..6460fb6 100644 --- a/python/cumulus/store/s3.py +++ b/python/cumulus/store/s3.py @@ -36,3 +36,7 @@ class S3Store(cumulus.store.Store): def delete(self, type, name): self.bucket.delete_key("%s/%s/%s" % (self.prefix, type, name)) + + def stat(self, type, name): + k = self.bucket.get_key("%s/%s/%s" % (self.prefix, type, name)) + return {'size': int(k.size)} -- 2.20.1