From 63bc8919d5c02cf1c1179ed3528a0dfeee8e7454 Mon Sep 17 00:00:00 2001
From: Michael Vrable <vrable@cs.hmc.edu>
Date: Sun, 25 Sep 2011 20:40:35 -0700
Subject: [PATCH] Cleaner updates

---
 cleaner/cleaner                               |  54 +++++++++++-------
 .../20110815/.synwrite-2048M-summary.data.swp | Bin 12288 -> 0 bytes
 .../20110815/.synwrite-256-summary.data.swp   | Bin 12288 -> 0 bytes
 3 files changed, 33 insertions(+), 21 deletions(-)
 delete mode 100644 results/20110815/.synwrite-2048M-summary.data.swp
 delete mode 100644 results/20110815/.synwrite-256-summary.data.swp

diff --git a/cleaner/cleaner b/cleaner/cleaner
index 60c91f5..9cdba54 100755
--- a/cleaner/cleaner
+++ b/cleaner/cleaner
@@ -83,6 +83,7 @@ def retry_wrap(method):
                 return method(self, *args, **kwargs)
             except:
                 print >>sys.stderr, "S3 operation failed, retrying..."
+                print >>sys.stderr, "  %s %s %s" % (method, args, kwargs)
                 self.connect()
                 time.sleep(1.0)
         return method(self, *args, **kwargs)
@@ -98,7 +99,7 @@ class S3Backend(Backend):
         self.cache = {}
         for f in os.listdir(cachedir):
             self.cache[f] = True
-        print "Initial cache contents:", list(self.cache.keys())
+        #print "Initial cache contents:", list(self.cache.keys())
         self.connect()
         self.stats_get = [0, 0]
         self.stats_put = [0, 0]
@@ -284,7 +285,7 @@ class LogDirectory:
         self.dir_num = dir
         self.seq_num = 0
         for logname in backend.list(dir):
-            print "Old log file:", logname
+            #print "Old log file:", logname
             loc = backend.name_to_loc(logname[0])
             if loc is not None and loc[0] == dir:
                 self.seq_num = max(self.seq_num, loc[1] + 1)
@@ -443,12 +444,12 @@ class InodeMap:
 
         data = data[self.vvsize:]
 
-        print "Inode map:"
+        #print "Inode map:"
         for i in range(len(data) // 16):
             (start, end) = struct.unpack_from("<QQ", data, 16*i)
             imap = load_item(backend, checkpoint_record.links[i][1])
             util.add_item(imap)
-            print "[%d, %d]: %s" % (start, end, imap)
+            #print "[%d, %d]: %s" % (start, end, imap)
             for j in range(len(imap.data) // 8):
                 (inum,) = struct.unpack_from("<Q", imap.data, 8*j)
                 inode = load_item(backend, imap.links[j][1])
@@ -457,10 +458,11 @@ class InodeMap:
                 util.add_item(inode)
                 for i in inode.links:
                     util.add_item(i[1])
-                    data_segments.add(i[1][0:2])
-                print "  %d: %s (%d data segments)" % (inum, inode, len(data_segments))
+                    if i[1] is not None:
+                        data_segments.add(i[1][0:2])
+                #print "  %d: %s (%d data segments)" % (inum, inode, len(data_segments))
 
-        print
+        #print
         print "Segment utilizations:"
         total_data = [0, 0]
         deletions = [0, 0]
@@ -469,9 +471,16 @@ class InodeMap:
             print "%s: %s %s" % (s, u, float(u[1]) / u[0])
             if u[1] == 0:
                 print "Would delete..."
-                backend.delete(s)
-                deletions[0] += 1
-                deletions[1] += u[0]
+                (d, n) = backend.name_to_loc(s)
+                try:
+                    if n < self.version_vector[d]:
+                        backend.delete(s)
+                        deletions[0] += 1
+                        deletions[1] += u[0]
+                    else:
+                        print "Not deleting log file newer than checkpoint!"
+                except:
+                    print "Error determining age of log segment, keeping"
 
         self.inodes = inodes
         self.util = util
@@ -538,12 +547,15 @@ class InodeMap:
 def rewrite_inode(backend, inode_map, inum, log, copy_data=True):
     inode = inode_map.inodes[inum]
     if copy_data:
-        blocks = []
+        newlinks = []
         for l in inode.links:
-            data = load_item(backend, l[1])
-            blocks.append(data)
-            log.write(data, 0)
-        inode.links = [(b.id, b.location) for b in blocks]
+            if l[1] is not None:
+                data = load_item(backend, l[1])
+                log.write(data, 0)
+                newlinks.append((data.id, data.location))
+            else:
+                newlinks.append(l)
+        inode.links = newlinks
     log.write(inode, 1)
     inode_map.mark_updated(inum)
 
@@ -569,7 +581,7 @@ def run_cleaner(backend, inode_map, log, repack_inodes=False):
     dirty_inode_data = set()
     for s in inode_map.obsolete_segments:
         filename = backend.loc_to_name(s)
-        print "Scanning", filename, "for live data"
+        #print "Scanning", filename, "for live data"
         for item in parse_log(backend.read(filename), filename):
             if item.type in (ITEM_TYPE.DATA, ITEM_TYPE.INODE):
                 if item.inum != 0:
@@ -578,12 +590,12 @@ def run_cleaner(backend, inode_map, log, repack_inodes=False):
                         dirty_inodes.add(item.inum)
                     if item.inum not in dirty_inode_data:
                         for b in inode.links:
-                            if s == b[1][0:2]:
+                            if b[1] is not None and s == b[1][0:2]:
                                 dirty_inode_data.add(item.inum)
                                 break
 
-    print "Inodes to rewrite:", dirty_inodes
-    print "Inodes with data to rewrite:", dirty_inode_data
+    #print "Inodes to rewrite:", dirty_inodes
+    #print "Inodes with data to rewrite:", dirty_inode_data
     for i in sorted(dirty_inodes.union(dirty_inode_data)):
         rewrite_inode(backend, inode_map, i, log, i in dirty_inode_data)
 
@@ -592,12 +604,12 @@ if __name__ == '__main__':
     backend = S3Backend("mvrable-bluesky-west", cachedir="/tmp/bluesky-cache")
     #backend = FileBackend(".")
     chkpt = load_checkpoint_record(backend)
-    print backend.list()
+    #print backend.list()
+    log_dir = LogDirectory(backend, 1)
     imap = InodeMap()
     imap.build(backend, chkpt)
     print chkpt
 
-    log_dir = LogDirectory(backend, 1)
     run_cleaner(backend, imap, log_dir)
     print "Version vector:", imap.version_vector
     imap.write(backend, log_dir)
diff --git a/results/20110815/.synwrite-2048M-summary.data.swp b/results/20110815/.synwrite-2048M-summary.data.swp
deleted file mode 100644
index c8fc0779f1c3d4e33d9b9124db400733c5436762..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 12288
zcmeI2zi!k(5QispNCXh{$bcv)&NsX3wY?__kf5MQD*i}R_&5U=zBsad3D*RkfEH@1
zP#{`(3#h26pymmf<MtqOlmrR{LBEwgS-YMY?|$bktlJ^qy1o)5p&@A}iC#^%R$t_n
zUT+eOH|lIy=CeEdwqjHl<-QT~n_IqMzpU<MWq(*s^5)Srd{DIgI&UUr+w?Un#i_FW
zX1can7j53t9HlG0W-=aU^)wu1ZT1UKLI@1Nz-|M}WVGVh`1x~ZgVU#O?he5S48Q;k
zzyJ)u01UtY48XvDYoJ|Uq9?MtBYuU;{<_q;`YtXo00S@p126ysFaQHE00S@p126ys
zhtNPaB6@e6=;#TVy!`*a`2PPQ{rAFK;f?TIcqTj+?h3bs%fdw=5N7$GrOroTTeu{A
zm15h%NVp+f5I!9v`XIa%o(k*272%A41_Lkv126ysFaQHE00S`aHw_r0Lk^^IXmx6w
zvdU`f$nbo)H_Q%>XdVuN#kt;IH@w^bCm(R|B>(ste{%}o&rU1$r>EuTYKw8+41(1v
zI7F51<aKqm*eIH!+GEx`BG#_ebyW|7`(>52-SBSTzh*nq_lVHx{7okvEp`GMV>n4u
z7pDnnzY{ogE{Rj69NWZ@TBND?XPQnDj+J2(#iV>W*<VIE8#xuFlIFJ?sUKz8Xm(ad
sY>ji|%Le4j%Epuc?P41#R&2?aosUP(CXQoHE&0Opac3ndR+=?^19Z0oj{pDw

diff --git a/results/20110815/.synwrite-256-summary.data.swp b/results/20110815/.synwrite-256-summary.data.swp
deleted file mode 100644
index 45ab76c538f9a06380c168418b9064e19715cdea..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 12288
zcmeI&y>1gh5C`xH)Byw?)j?|8JG0;E=pa(kL5t4#1Qut4eCI@NLPdqr@&vRzLEZw1
zhd`o7bj*>nEuko)b@V@JwDRoU%<j*X<sF+B&kl<?w4~2HqJzcR(dWw3*Hfa|$-Erb
zRX20(Go8$*^|dQ5C(rtVqx$$&S&zo`qH5kQhp(sYXkImox@|^cm`!3oYL@R#=hL<t
zh!2BiF`Je1<#1BA<$v_S5(q%xDuG?H!ls+qd;F+)aR2#Li;#l=1Rwwb2tWV=5P$##
zZm2-}euqBj*>3gc+wI4lf5!eJ1_(d^0uX=z1Rwwb2tWV=5P$##woss)5PiHu^j)L-
z|9|!S|HoaTZyFaGUo_4&PBdO>Jk?;0hZ?sve)jKw((#GLFD>$WRcH%Muw@8900Izz
z00bZa0SG_<0yjv&%Fq=0m|+OPN#ex9j`pm%%n!z8^Jc$TUB@2$y^GDJyN!J~&_(TZ
zRyF&@(Q&awl_aeXokB7uryOD;ahtTwIeWGumZYapqL>3`k(6yD-V});8xe6<i*Q&M
ziB_U;FnX1pcvCy1Y+Taox83SBzmGmT4#FH9@w%nYA+ynqIOnME*nL?xryx#W(RANY
j(@u9Cy?B>`r@mvi+GU%?vgSSY9W|}JtaUDCNs0ad>eHNL

-- 
2.20.1