From: Michael Vrable <vrable@cs.hmc.edu>
Date: Fri, 31 Jan 2014 00:06:53 +0000 (-0800)
Subject: Byte/string handling fixes for Python 3.
X-Git-Url: https://git.vrable.net/?a=commitdiff_plain;h=ee98274cfd9e9383214a9792c01fdfe4f22ef677;p=cumulus.git

Byte/string handling fixes for Python 3.
---

diff --git a/python/cumulus/__init__.py b/python/cumulus/__init__.py
index 4b7bd6c..6e03bd2 100644
--- a/python/cumulus/__init__.py
+++ b/python/cumulus/__init__.py
@@ -28,6 +28,7 @@ various parts of a Cumulus archive:
 
 from __future__ import division, print_function, unicode_literals
 
+import codecs
 import hashlib
 import itertools
 import os
@@ -44,7 +45,7 @@ except ImportError:
 import cumulus.store
 import cumulus.store.file
 
-if sys.version < '3':
+if sys.version < "3":
     StringTypes = (str, unicode)
 else:
     StringTypes = (str,)
@@ -68,6 +69,12 @@ SEGMENT_FILTERS = [
     ("", None),
 ]
 
+def to_lines(data):
+    """Decode binary data from a file into a sequence of lines.
+
+    Newline markers are retained."""
+    return list(codecs.iterdecode(data.splitlines(True), "utf-8"))
+
 def uri_decode(s):
     """Decode a URI-encoded (%xx escapes) string."""
     def hex_decode(m): return chr(int(m.group(1), 16))
@@ -366,7 +373,7 @@ class CumulusStore:
 
     def load_snapshot(self, snapshot):
         snapshot_file = self.backend.open_snapshot(snapshot)[0]
-        return snapshot_file.read().splitlines(True)
+        return to_lines(snapshot_file.read())
 
     @staticmethod
     def filter_data(filehandle, filter_cmd):
@@ -513,7 +520,7 @@ def read_metadata(object_store, root):
 
     def follow_ref(refstr):
         if len(stack) >= MAX_RECURSION_DEPTH: raise OverflowError
-        lines = object_store.get(refstr).splitlines(True)
+        lines = to_lines(object_store.get(refstr))
         lines.reverse()
         stack.append(lines)
 
diff --git a/python/cumulus/store/file.py b/python/cumulus/store/file.py
index fd6805f..833a5f0 100644
--- a/python/cumulus/store/file.py
+++ b/python/cumulus/store/file.py
@@ -36,7 +36,7 @@ class Store(cumulus.store.Store):
 
     def get(self, path):
         try:
-            return open(os.path.join(self.prefix, path), 'rb')
+            return open(os.path.join(self.prefix, path), "rb")
         except IOError:
             raise cumulus.store.NotFoundError(path)