Introduce a script to provide access to remote repositories.

cumulus-store is a Python script that uses the cumulus library code to
access various storage repositories (local file, S3, and extensible to
others).  It allows non-Python code to access these storage repositories
through a simple interface through stdin/stdout.

Additionally, make a few extensions and fixes to the cumulus Python
libraries.

diff --git a/cumulus-store b/cumulus-store
new file mode 100755 (executable)
index 0000000..54d5515
--- /dev/null
+++ b/cumulus-store
@@ -0,0 +1,46 @@
+#!/usr/bin/python
+#
+# Generic storage hook for writing LBS backups directly to Amazon's Simple
+# Storage Service (S3), or any other service supported by the Python storage
+# implementation.
+#
+# Storage protocol: After launching this script (with the remote location
+# specified on the command-line), send any number of commands as lines to
+# stdin.  Available commands are:
+#     PUT <type> <name> <local file>
+#     LIST <type>
+# Tokens are whitespace-separated, but may contain any characters by
+# URI-encoding them.  After completing each operation, a response line is
+# written to stdout, which is either "OK" (for success) or "ERR" (if an error
+# occurred).
+
+import os, sys, traceback
+
+# Automatically set Python path, based on script directory.  This should be
+# removed if the tools are properly installed somewhere.
+script_directory = os.path.dirname(sys.argv[0])
+sys.path.append(os.path.join(script_directory, 'python'))
+
+import cumulus
+from cumulus import store
+
+remote = store.open(sys.argv[1])
+while True:
+    cmd = sys.stdin.readline()
+    if cmd == "": break
+    cmd = [cumulus.uri_decode(s) for s in cmd.strip().split()]
+
+    try:
+        if cmd[0] == 'PUT':
+            remote.put(cmd[1], cmd[2], open(cmd[3], 'r'))
+            sys.stdout.write('OK\n')
+        elif cmd[0] == 'LIST':
+            files = remote.list(cmd[1])
+            for f in files:
+                sys.stdout.write("* " + cumulus.uri_encode(f) + "\n")
+            sys.stdout.write('OK\n')
+    except Exception:
+        traceback.print_exc()
+        sys.stdout.write('ERR\n')
+
+    sys.stdout.flush()

diff --git a/python/cumulus/__init__.py b/python/cumulus/__init__.py
index 51d3ee8..e1dac84 100644 (file)
--- a/python/cumulus/__init__.py
+++ b/python/cumulus/__init__.py
@@ -23,6 +23,19 @@ MAX_RECURSION_DEPTH = 3
 # All segments which have been accessed this session.
 accessed_segments = set()
 
+def uri_decode(s):
+    """Decode a URI-encoded (%xx escapes) string."""
+    def hex_decode(m): return chr(int(m.group(1), 16))
+    return re.sub(r"%([0-9a-f]{2})", hex_decode, s)
+def uri_encode(s):
+    """Encode a string to URI-encoded (%xx escapes) form."""
+    def hex_encode(c):
+        if c > '+' and c < '\x7f' and c != '@':
+            return c
+        else:
+            return "%%%02x" % (ord(c),)
+    return ''.join(hex_encode(c) for c in s)
+
 class Struct:
     """A class which merely acts as a data container.
 
@@ -347,8 +360,7 @@ class MetadataItem:
     @staticmethod
     def decode_str(s):
         """Decode a URI-encoded (%xx escapes) string."""
-        def hex_decode(m): return chr(int(m.group(1), 16))
-        return re.sub(r"%([0-9a-f]{2})", hex_decode, s)
+        return uri_decode(s)
 
     @staticmethod
     def raw_str(s):

diff --git a/python/cumulus/store/s3.py b/python/cumulus/store/s3.py
index 6460fb6..9fa006e 100644 (file)
--- a/python/cumulus/store/s3.py
+++ b/python/cumulus/store/s3.py
@@ -32,7 +32,7 @@ class S3Store(cumulus.store.Store):
 
     def put(self, type, name, fp):
         k = self._get_key(type, name)
-        k.send_file(fp)
+        k.set_contents_from_file(fp)
 
     def delete(self, type, name):
         self.bucket.delete_key("%s/%s/%s" % (self.prefix, type, name))