projects
/
cumulus.git
/ blobdiff
commit
grep
author
committer
pickaxe
?
search:
re
summary
|
shortlog
|
log
|
commit
|
commitdiff
|
tree
raw
|
inline
| side by side
Fix typo in restore.pl.
[cumulus.git]
/
lbs.py
diff --git
a/lbs.py
b/lbs.py
index
4224cf9
..
b8de982
100644
(file)
--- a/
lbs.py
+++ b/
lbs.py
@@
-13,7
+13,7
@@
import os, re, sha, tarfile, tempfile, thread
from pysqlite2 import dbapi2 as sqlite3
# The largest supported snapshot format that can be understood.
from pysqlite2 import dbapi2 as sqlite3
# The largest supported snapshot format that can be understood.
-FORMAT_VERSION = (0,
6) # LBS Snapshot v0.6
+FORMAT_VERSION = (0,
8) # LBS Snapshot v0.8
# Maximum number of nested indirect references allowed in a snapshot.
MAX_RECURSION_DEPTH = 3
# Maximum number of nested indirect references allowed in a snapshot.
MAX_RECURSION_DEPTH = 3
@@
-143,7
+143,7
@@
class ObjectStore:
if m:
return ("zero", None, None, (0, int(m.group(1))))
if m:
return ("zero", None, None, (0, int(m.group(1))))
- m = re.match(r"^([-0-9a-f]+)\/([0-9a-f]+)(\(\S+\))?(\[((
\d+)\+)?(\d+
)\])?$", refstr)
+ m = re.match(r"^([-0-9a-f]+)\/([0-9a-f]+)(\(\S+\))?(\[((
(\d+)\+)?(\d+)|=(\d+)
)\])?$", refstr)
if not m: return
segment = m.group(1)
if not m: return
segment = m.group(1)
@@
-155,11
+155,14
@@
class ObjectStore:
checksum = checksum.lstrip("(").rstrip(")")
if slice is not None:
checksum = checksum.lstrip("(").rstrip(")")
if slice is not None:
- if m.group(5) is None:
+ if m.group(9) is not None:
+ # Size-assertion slice
+ slice = (0, int(m.group(9)), True)
+ elif m.group(6) is None:
# Abbreviated slice
# Abbreviated slice
- slice = (0, int(m.group(
7))
)
+ slice = (0, int(m.group(
8)), False
)
else:
else:
- slice = (int(m.group(
6)), int(m.group(7))
)
+ slice = (int(m.group(
7)), int(m.group(8)), False
)
return (segment, object, checksum, slice)
return (segment, object, checksum, slice)
@@
-231,7
+234,8
@@
class ObjectStore:
raise ValueError
if slice is not None:
raise ValueError
if slice is not None:
- (start, length) = slice
+ (start, length, exact) = slice
+ if exact and len(data) != length: raise ValueError
data = data[start:start+length]
if len(data) != length: raise IndexError
data = data[start:start+length]
if len(data) != length: raise IndexError
@@
-263,7
+267,7
@@
def parse(lines, terminate=None):
last_key = None
continue
last_key = None
continue
- m = re.match(r"^(
\w
+):\s*(.*)$", l)
+ m = re.match(r"^(
[-\w]
+):\s*(.*)$", l)
if m:
dict[m.group(1)] = m.group(2)
last_key = m.group(1)
if m:
dict[m.group(1)] = m.group(2)
last_key = m.group(1)
@@
-541,6
+545,11
@@
class LocalDatabase:
where expire_time < ?)""",
(last_snapshotid,))
where expire_time < ?)""",
(last_snapshotid,))
+ # Remove sub-block signatures for deleted objects.
+ cur.execute("""delete from subblock_signatures
+ where blockid not in
+ (select blockid from block_index)""")
+
# Segment cleaning.
class SegmentInfo(Struct): pass
# Segment cleaning.
class SegmentInfo(Struct): pass
@@
-564,7
+573,8
@@
class LocalDatabase:
cur = self.cursor()
segments = []
cur.execute("""select segmentid, used, size, mtime,
cur = self.cursor()
segments = []
cur.execute("""select segmentid, used, size, mtime,
- julianday('now') - mtime as age from segment_info""")
+ julianday('now') - mtime as age from segment_info
+ where expire_time is null""")
for row in cur:
info = self.SegmentInfo()
info.id = row[0]
for row in cur:
info = self.SegmentInfo()
info.id = row[0]
@@
-573,9
+583,11
@@
class LocalDatabase:
info.mtime = row[3]
info.age_days = row[4]
info.mtime = row[3]
info.age_days = row[4]
- # If
age
is not available for whatever reason, treat it as 0.0.
+ # If
data
is not available for whatever reason, treat it as 0.0.
if info.age_days is None:
info.age_days = 0.0
if info.age_days is None:
info.age_days = 0.0
+ if info.used_bytes is None:
+ info.used_bytes = 0.0
# Benefit calculation: u is the estimated fraction of each segment
# which is utilized (bytes belonging to objects still in use
# Benefit calculation: u is the estimated fraction of each segment
# which is utilized (bytes belonging to objects still in use