X-Git-Url: http://git.vrable.net/?a=blobdiff_plain;f=python%2Fcumulus%2F__init__.py;h=323a7c75b4679ca6293ff9a6ee67d7aa6806e217;hb=da1d95d3242ee9d596e60b8d5bfcf9e5bedcd80f;hp=8dc4c9875e0f77ef95232f866f37a28e7ae9949e;hpb=a5f66616b1ec0c38328ad5131bf1c889ccc43659;p=cumulus.git

diff --git a/python/cumulus/__init__.py b/python/cumulus/__init__.py
index 8dc4c98..323a7c7 100644
--- a/python/cumulus/__init__.py
+++ b/python/cumulus/__init__.py
@@ -26,19 +26,31 @@ various parts of a Cumulus archive:
   - reading and maintaining the local object database
 """
 
+from __future__ import division, print_function, unicode_literals
 
+import codecs
 import hashlib
 import itertools
 import os
 import re
 import sqlite3
+import subprocess
+import sys
 import tarfile
 import tempfile
-import _thread
+try:
+    import _thread
+except ImportError:
+    import thread as _thread
 
 import cumulus.store
 import cumulus.store.file
 
+if sys.version < "3":
+    StringTypes = (str, unicode)
+else:
+    StringTypes = (str,)
+
 # The largest supported snapshot format that can be understood.
 FORMAT_VERSION = (0, 11)        # Cumulus Snapshot v0.11
 
@@ -58,6 +70,12 @@ SEGMENT_FILTERS = [
     ("", None),
 ]
 
+def to_lines(data):
+    """Decode binary data from a file into a sequence of lines.
+
+    Newline markers are retained."""
+    return list(codecs.iterdecode(data.splitlines(True), "utf-8"))
+
 def uri_decode(s):
     """Decode a URI-encoded (%xx escapes) string."""
     def hex_decode(m): return chr(int(m.group(1), 16))
@@ -264,11 +282,8 @@ class BackendWrapper(object):
 
         store may either be a Store object or URL.
         """
-        if type(backend) in (str, str):
-            if backend.find(":") >= 0:
-                self._backend = cumulus.store.open(backend)
-            else:
-                self._backend = cumulus.store.file.FileStore(backend)
+        if type(backend) in StringTypes:
+            self._backend = cumulus.store.open(backend)
         else:
             self._backend = backend
 
@@ -295,7 +310,7 @@ class BackendWrapper(object):
     def prefetch_generic(self):
         """Calls scan on directories to prefetch file metadata."""
         directories = set()
-        for typeinfo in list(SEARCH_PATHS.values()):
+        for typeinfo in SEARCH_PATHS.values():
             directories.update(typeinfo.directories())
         for d in directories:
             print("Prefetch", d)
@@ -328,7 +343,7 @@ class CumulusStore:
         if m:
             return ("zero", None, None, (0, int(m.group(1)), False))
 
-        m = re.match(r"^([-0-9a-f]+)\/([0-9a-f]+)(\(\S+\))?(\[(((\d+)\+)?(\d+)|=(\d+))\])?$", refstr)
+        m = re.match(r"^([-0-9a-f]+)\/([0-9a-f]+)(\(\S+\))?(\[(=?(\d+)|(\d+)\+(\d+))\])?$", refstr)
         if not m: return
 
         segment = m.group(1)
@@ -340,12 +355,9 @@ class CumulusStore:
             checksum = checksum.lstrip("(").rstrip(")")
 
         if slice is not None:
-            if m.group(9) is not None:
+            if m.group(6) is not None:
                 # Size-assertion slice
-                slice = (0, int(m.group(9)), True)
-            elif m.group(6) is None:
-                # Abbreviated slice
-                slice = (0, int(m.group(8)), False)
+                slice = (0, int(m.group(6)), True)
             else:
                 slice = (int(m.group(7)), int(m.group(8)), False)
 
@@ -359,13 +371,15 @@ class CumulusStore:
 
     def load_snapshot(self, snapshot):
         snapshot_file = self.backend.open_snapshot(snapshot)[0]
-        return snapshot_file.read().splitlines(True)
+        return to_lines(snapshot_file.read())
 
     @staticmethod
     def filter_data(filehandle, filter_cmd):
         if filter_cmd is None:
             return filehandle
-        (input, output) = os.popen2(filter_cmd)
+        p = subprocess.Popen(filter_cmd, shell=True, stdin=subprocess.PIPE,
+                             stdout=subprocess.PIPE, close_fds=True)
+        input, output = p.stdin, p.stdout
         def copy_thread(src, dst):
             BLOCK_SIZE = 4096
             while True:
@@ -374,6 +388,7 @@ class CumulusStore:
                 dst.write(block)
             src.close()
             dst.close()
+            p.wait()
         _thread.start_new_thread(copy_thread, (filehandle, input))
         return output
 
@@ -433,6 +448,9 @@ class CumulusStore:
 
         if slice is not None:
             (start, length, exact) = slice
+            # Note: The following assertion check may need to be commented out
+            # to restore from pre-v0.8 snapshots, as the syntax for
+            # size-assertion slices has changed.
             if exact and len(data) != length: raise ValueError
             data = data[start:start+length]
             if len(data) != length: raise IndexError
@@ -506,7 +524,7 @@ def read_metadata(object_store, root):
 
     def follow_ref(refstr):
         if len(stack) >= MAX_RECURSION_DEPTH: raise OverflowError
-        lines = object_store.get(refstr).splitlines(True)
+        lines = to_lines(object_store.get(refstr))
         lines.reverse()
         stack.append(lines)
 
@@ -566,7 +584,7 @@ class MetadataItem:
     @staticmethod
     def decode_device(s):
         """Decode a device major/minor number."""
-        (major, minor) = list(map(MetadataItem.decode_int, s.split("/")))
+        (major, minor) = map(MetadataItem.decode_int, s.split("/"))
         return (major, minor)
 
     class Items: pass
@@ -578,7 +596,7 @@ class MetadataItem:
         self.object_store = object_store
         self.keys = []
         self.items = self.Items()
-        for (k, v) in list(fields.items()):
+        for (k, v) in fields.items():
             if k in self.field_types:
                 decoder = self.field_types[k]
                 setattr(self.items, k, decoder(v))