Flag "volatile" files when creating a snapshot.

[cumulus.git] / lbs.py

diff --git a/lbs.py b/lbs.py
index e3ca9ab..3f647c4 100644 (file)
--- a/lbs.py
+++ b/lbs.py
@@ -12,6 +12,9 @@ from __future__ import division
 import os, re, sha, tarfile, tempfile, thread
 from pysqlite2 import dbapi2 as sqlite3
 
+# The largest supported snapshot format that can be understood.
+FORMAT_VERSION = (0, 2)         # LBS Snapshot v0.2
+
 # Maximum number of nested indirect references allowed in a snapshot.
 MAX_RECURSION_DEPTH = 3
 
@@ -128,7 +131,7 @@ class ObjectStore:
     def cleanup(self):
         if self.cachedir is not None:
             # TODO: Avoid use of system, make this safer
-            os.system("rm -rv " + self.cachedir)
+            os.system("rm -rf " + self.cachedir)
         self.cachedir = None
 
     @staticmethod
@@ -187,12 +190,11 @@ class ObjectStore:
     def load_object(self, segment, object):
         path = os.path.join(self.get_cachedir(), segment, object)
         if not os.access(path, os.R_OK):
-            print "Extracting", segment
             self.extract_segment(segment)
         if segment in self.lru_list: self.lru_list.remove(segment)
         self.lru_list.append(segment)
         while len(self.lru_list) > self.CACHE_SIZE:
-            os.system("rm -rv " + os.path.join(self.cachedir, self.lru_list[0]))
+            os.system("rm -rf " + os.path.join(self.cachedir, self.lru_list[0]))
             self.lru_list = self.lru_list[1:]
         return open(path, 'rb').read()
 
@@ -262,6 +264,15 @@ def parse_full(lines):
     except StopIteration:
         return {}
 
+def parse_metadata_version(s):
+    """Convert a string with the snapshot version format to a tuple."""
+
+    m = re.match(r"^LBS Snapshot v(\d+(\.\d+)*)$", s)
+    if m is None:
+        return ()
+    else:
+        return tuple([int(d) for d in m.group(1).split(".")])
+
 def read_metadata(object_store, root):
     """Iterate through all lines in the metadata log, following references."""
 
@@ -298,11 +309,60 @@ def read_metadata(object_store, root):
 class MetadataItem:
     """Metadata for a single file (or directory or...) from a snapshot."""
 
+    # Functions for parsing various datatypes that can appear in a metadata log
+    # item.
+    @staticmethod
+    def decode_int(s):
+        """Decode an integer, expressed in decimal, octal, or hexadecimal."""
+        if s.startswith("0x"):
+            return int(s, 16)
+        elif s.startswith("0"):
+            return int(s, 8)
+        else:
+            return int(s, 10)
+
+    @staticmethod
+    def decode_str(s):
+        """Decode a URI-encoded (%xx escapes) string."""
+        def hex_decode(m): return chr(int(m.group(1), 16))
+        return re.sub(r"%([0-9a-f]{2})", hex_decode, s)
+
+    @staticmethod
+    def raw_str(s):
+        """An unecoded string."""
+        return s
+
+    @staticmethod
+    def decode_user(s):
+        """Decode a user/group to a tuple of uid/gid followed by name."""
+        items = s.split()
+        uid = MetadataItem.decode_int(items[0])
+        name = None
+        if len(items) > 1:
+            if items[1].startswith("(") and items[1].endswith(")"):
+                name = MetadataItem.decode_str(items[1][1:-1])
+        return (uid, name)
+
+    @staticmethod
+    def decode_device(s):
+        """Decode a device major/minor number."""
+        (major, minor) = map(MetadataItem.decode_int, s.split("/"))
+        return (major, minor)
+
+    class Items: pass
+
     def __init__(self, fields, object_store):
         """Initialize from a dictionary of key/value pairs from metadata log."""
 
         self.fields = fields
         self.object_store = object_store
+        self.keys = []
+        self.items = self.Items()
+        for (k, v) in fields.items():
+            if k in self.field_types:
+                decoder = self.field_types[k]
+                setattr(self.items, k, decoder(v))
+                self.keys.append(k)
 
     def data(self):
         """Return an iterator for the data blocks that make up a file."""
@@ -334,6 +394,22 @@ class MetadataItem:
             else:
                 yield ref
 
+# Description of fields that might appear, and how they should be parsed.
+MetadataItem.field_types = {
+    'name': MetadataItem.decode_str,
+    'type': MetadataItem.raw_str,
+    'mode': MetadataItem.decode_int,
+    'device': MetadataItem.decode_device,
+    'user': MetadataItem.decode_user,
+    'group': MetadataItem.decode_user,
+    'mtime': MetadataItem.decode_int,
+    'links': MetadataItem.decode_int,
+    'inode': MetadataItem.raw_str,
+    'checksum': MetadataItem.decode_str,
+    'size': MetadataItem.decode_int,
+    'contents': MetadataItem.decode_str,
+}
+
 def iterate_metadata(object_store, root):
     for d in parse(read_metadata(object_store, root), lambda l: len(l) == 0):
         yield MetadataItem(d, object_store)