projects
/
bluesky.git
/ blobdiff
commit
grep
author
committer
pickaxe
?
search:
re
summary
|
shortlog
|
log
|
commit
|
commitdiff
|
tree
raw
|
inline
| side by side
Improve cleaner: make sure new logs are written after existing ones.
[bluesky.git]
/
cleaner
/
cleaner
diff --git
a/cleaner/cleaner
b/cleaner/cleaner
index
42aef01
..
fb3ce05
100755
(executable)
--- a/
cleaner/cleaner
+++ b/
cleaner/cleaner
@@
-49,6
+49,13
@@
class FileBackend:
fp.write(data)
fp.close()
fp.write(data)
fp.close()
+ def loc_to_name(self, location):
+ return "log-%08d-%08d" % (location)
+
+ def name_to_loc(self, name):
+ m = re.match(r"^log-(\d+)-(\d+)$", name)
+ if m: return (int(m.group(1)), int(m.group(2)))
+
class LogItem:
"""In-memory representation of a single item stored in a log file."""
class LogItem:
"""In-memory representation of a single item stored in a log file."""
@@
-91,18
+98,23
@@
class LogSegment:
def close(self):
data = ''.join(self.data)
def close(self):
data = ''.join(self.data)
- filename =
"log-%08d-%08d" %
(self.location)
+ filename =
self.backend.loc_to_name
(self.location)
print "Would write %d bytes of data to %s" % (len(data), filename)
self.backend.write(filename, data)
class LogDirectory:
TARGET_SIZE = 4 << 20
print "Would write %d bytes of data to %s" % (len(data), filename)
self.backend.write(filename, data)
class LogDirectory:
TARGET_SIZE = 4 << 20
- def __init__(self, backend, dir
, seq
):
+ def __init__(self, backend, dir):
self.backend = backend
self.dir_num = dir
self.backend = backend
self.dir_num = dir
- self.seq_num = seq
+ self.seq_num = 0
+ for logname in backend.list():
+ loc = backend.name_to_loc(logname[0])
+ if loc is not None and loc[0] == dir:
+ self.seq_num = max(self.seq_num, loc[1] + 1)
self.groups = {}
self.groups = {}
+ print "Starting sequence number is", self.seq_num
def open_segment(self):
seg = LogSegment(self.backend, (self.dir_num, self.seq_num))
def open_segment(self):
seg = LogSegment(self.backend, (self.dir_num, self.seq_num))
@@
-182,7
+194,7
@@
def load_item(backend, location):
The elements of the tuple are (directory, sequence, offset, size)."""
The elements of the tuple are (directory, sequence, offset, size)."""
- filename =
"log-%08d-%08d" % (location[0], location[1]
)
+ filename =
backend.loc_to_name((location[0], location[1])
)
data = backend.read(filename)[location[2] : location[2] + location[3]]
item = parse_item(data)
item.location = location
data = backend.read(filename)[location[2] : location[2] + location[3]]
item = parse_item(data)
item.location = location
@@
-330,15
+342,15
@@
def run_cleaner(backend, inode_map, log):
for (s, u) in sorted(inode_map.util.segments.items()):
if float(u[1]) / u[0] < 0.99 and u[1] > 0:
print "Should clean segment", s
for (s, u) in sorted(inode_map.util.segments.items()):
if float(u[1]) / u[0] < 0.99 and u[1] > 0:
print "Should clean segment", s
-
m = re.match(r"^log-(\d+)-(\d+)$",
s)
- if
m: inode_map.obsolete_segments.add((int(m.group(1)), int(m.group(2)))
)
+
loc = backend.name_to_loc(
s)
+ if
s: inode_map.obsolete_segments.add(loc
)
# Given that list of segments to clean, scan through those segments to find
# data which is still live and mark relevant inodes as needing to be
# rewritten.
dirty_inodes = set()
for s in inode_map.obsolete_segments:
# Given that list of segments to clean, scan through those segments to find
# data which is still live and mark relevant inodes as needing to be
# rewritten.
dirty_inodes = set()
for s in inode_map.obsolete_segments:
- filename =
"log-%08d-%08d" % s
+ filename =
backend.loc_to_name(s)
print "Scanning", filename, "for live data"
for item in parse_log(backend.read(filename), filename):
if item.type in (ITEM_TYPE.DATA, ITEM_TYPE.INODE):
print "Scanning", filename, "for live data"
for item in parse_log(backend.read(filename), filename):
if item.type in (ITEM_TYPE.DATA, ITEM_TYPE.INODE):
@@
-356,7
+368,7
@@
if __name__ == '__main__':
imap.build(backend, chkpt)
print chkpt
imap.build(backend, chkpt)
print chkpt
- log_dir = LogDirectory(backend,
1,
0)
+ log_dir = LogDirectory(backend, 0)
run_cleaner(backend, imap, log_dir)
imap.write(backend, log_dir)
log_dir.close_all()
run_cleaner(backend, imap, log_dir)
imap.write(backend, log_dir)
log_dir.close_all()