X-Git-Url: http://git.vrable.net/?p=cumulus.git;a=blobdiff_plain;f=python%2Fcumulus%2F__init__.py;h=6e03bd2f7275f23897c20d3c52d1c7e8e6a1a5ad;hp=4b7bd6caf3ed40bdf89f9dc94c9db4e680bc8394;hb=ee98274cfd9e9383214a9792c01fdfe4f22ef677;hpb=8637c84e5657ec8aa677091a1cde4c2fe666cbfb

diff --git a/python/cumulus/__init__.py b/python/cumulus/__init__.py
index 4b7bd6c..6e03bd2 100644
--- a/python/cumulus/__init__.py
+++ b/python/cumulus/__init__.py
@@ -28,6 +28,7 @@ various parts of a Cumulus archive:
 
 from __future__ import division, print_function, unicode_literals
 
+import codecs
 import hashlib
 import itertools
 import os
@@ -44,7 +45,7 @@ except ImportError:
 import cumulus.store
 import cumulus.store.file
 
-if sys.version < '3':
+if sys.version < "3":
     StringTypes = (str, unicode)
 else:
     StringTypes = (str,)
@@ -68,6 +69,12 @@ SEGMENT_FILTERS = [
     ("", None),
 ]
 
+def to_lines(data):
+    """Decode binary data from a file into a sequence of lines.
+
+    Newline markers are retained."""
+    return list(codecs.iterdecode(data.splitlines(True), "utf-8"))
+
 def uri_decode(s):
     """Decode a URI-encoded (%xx escapes) string."""
     def hex_decode(m): return chr(int(m.group(1), 16))
@@ -366,7 +373,7 @@ class CumulusStore:
 
     def load_snapshot(self, snapshot):
         snapshot_file = self.backend.open_snapshot(snapshot)[0]
-        return snapshot_file.read().splitlines(True)
+        return to_lines(snapshot_file.read())
 
     @staticmethod
     def filter_data(filehandle, filter_cmd):
@@ -513,7 +520,7 @@ def read_metadata(object_store, root):
 
     def follow_ref(refstr):
         if len(stack) >= MAX_RECURSION_DEPTH: raise OverflowError
-        lines = object_store.get(refstr).splitlines(True)
+        lines = to_lines(object_store.get(refstr))
         lines.reverse()
         stack.append(lines)