import os, re, sha, tarfile, tempfile, thread
from pysqlite2 import dbapi2 as sqlite3
+# The largest supported snapshot format that can be understood.
+FORMAT_VERSION = (0, 2) # LBS Snapshot v0.2
+
# Maximum number of nested indirect references allowed in a snapshot.
MAX_RECURSION_DEPTH = 3
def cleanup(self):
if self.cachedir is not None:
# TODO: Avoid use of system, make this safer
- os.system("rm -rv " + self.cachedir)
+ os.system("rm -rf " + self.cachedir)
self.cachedir = None
@staticmethod
def load_object(self, segment, object):
path = os.path.join(self.get_cachedir(), segment, object)
if not os.access(path, os.R_OK):
- print "Extracting", segment
self.extract_segment(segment)
if segment in self.lru_list: self.lru_list.remove(segment)
self.lru_list.append(segment)
while len(self.lru_list) > self.CACHE_SIZE:
- os.system("rm -rv " + os.path.join(self.cachedir, self.lru_list[0]))
+ os.system("rm -rf " + os.path.join(self.cachedir, self.lru_list[0]))
self.lru_list = self.lru_list[1:]
return open(path, 'rb').read()
except StopIteration:
return {}
+def parse_metadata_version(s):
+ """Convert a string with the snapshot version format to a tuple."""
+
+ m = re.match(r"^LBS Snapshot v(\d+(\.\d+)*)$", s)
+ if m is None:
+ return ()
+ else:
+ return tuple([int(d) for d in m.group(1).split(".")])
+
def read_metadata(object_store, root):
"""Iterate through all lines in the metadata log, following references."""
class MetadataItem:
"""Metadata for a single file (or directory or...) from a snapshot."""
+ # Functions for parsing various datatypes that can appear in a metadata log
+ # item.
+ @staticmethod
+ def decode_int(s):
+ """Decode an integer, expressed in decimal, octal, or hexadecimal."""
+ if s.startswith("0x"):
+ return int(s, 16)
+ elif s.startswith("0"):
+ return int(s, 8)
+ else:
+ return int(s, 10)
+
+ @staticmethod
+ def decode_str(s):
+ """Decode a URI-encoded (%xx escapes) string."""
+ def hex_decode(m): return chr(int(m.group(1), 16))
+ return re.sub(r"%([0-9a-f]{2})", hex_decode, s)
+
+ @staticmethod
+ def raw_str(s):
+ """An unecoded string."""
+ return s
+
+ @staticmethod
+ def decode_user(s):
+ """Decode a user/group to a tuple of uid/gid followed by name."""
+ items = s.split()
+ uid = MetadataItem.decode_int(items[0])
+ name = None
+ if len(items) > 1:
+ if items[1].startswith("(") and items[1].endswith(")"):
+ name = MetadataItem.decode_str(items[1][1:-1])
+ return (uid, name)
+
+ @staticmethod
+ def decode_device(s):
+ """Decode a device major/minor number."""
+ (major, minor) = map(MetadataItem.decode_int, s.split("/"))
+ return (major, minor)
+
+ class Items: pass
+
def __init__(self, fields, object_store):
"""Initialize from a dictionary of key/value pairs from metadata log."""
self.fields = fields
self.object_store = object_store
+ self.keys = []
+ self.items = self.Items()
+ for (k, v) in fields.items():
+ if k in self.field_types:
+ decoder = self.field_types[k]
+ setattr(self.items, k, decoder(v))
+ self.keys.append(k)
def data(self):
"""Return an iterator for the data blocks that make up a file."""
else:
yield ref
+# Description of fields that might appear, and how they should be parsed.
+MetadataItem.field_types = {
+ 'name': MetadataItem.decode_str,
+ 'type': MetadataItem.raw_str,
+ 'mode': MetadataItem.decode_int,
+ 'device': MetadataItem.decode_device,
+ 'user': MetadataItem.decode_user,
+ 'group': MetadataItem.decode_user,
+ 'mtime': MetadataItem.decode_int,
+ 'links': MetadataItem.decode_int,
+ 'inode': MetadataItem.raw_str,
+ 'checksum': MetadataItem.decode_str,
+ 'size': MetadataItem.decode_int,
+ 'contents': MetadataItem.decode_str,
+}
+
def iterate_metadata(object_store, root):
for d in parse(read_metadata(object_store, root), lambda l: len(l) == 0):
yield MetadataItem(d, object_store)