From 1d2357b567318ccc6172f4b9edbbadbeba713086 Mon Sep 17 00:00:00 2001
From: Michael Vrable <mvrable@cs.ucsd.edu>
Date: Fri, 10 Sep 2010 17:11:05 -0700
Subject: [PATCH] Add very basic caching to the cleaner S3 backend.
---
cleaner/cleaner | 23 +++++++++++++++++++----
1 file changed, 19 insertions(+), 4 deletions(-)
diff --git a/cleaner/cleaner b/cleaner/cleaner
index 4267b6e..336537d 100755
--- a/cleaner/cleaner
+++ b/cleaner/cleaner
@@ -63,10 +63,12 @@ class FileBackend:
class S3Backend:
"""An interface to BlueSky where the log segments are on in Amazon S3."""
- def __init__(self, bucket, path='', cachedir=None):
+ def __init__(self, bucket, path='', cachedir="."):
self.conn = boto.connect_s3(is_secure=False)
self.bucket = self.conn.get_bucket(bucket)
self.path = path
+ self.cachedir = cachedir
+ self.cache = {}
def list(self):
files = []
@@ -75,19 +77,32 @@ class S3Backend:
return files
def read(self, filename):
- k = Key(self.bucket)
- k.key = self.path + filename
- return k.get_contents_as_string()
+ if filename in self.cache:
+ fp = open(os.path.join(self.cachedir, filename), 'rb')
+ return fp.read()
+ else:
+ k = Key(self.bucket)
+ k.key = self.path + filename
+ data = k.get_contents_as_string()
+ fp = open(os.path.join(self.cachedir, filename), 'wb')
+ fp.write(data)
+ fp.close()
+ self.cache[filename] = True
+ return data
def write(self, filename, data):
k = Key(self.bucket)
k.key = self.path + filename
k.set_contents_from_string(data)
+ if filename in self.cache:
+ del self.cache[filename]
def delete(self, filename):
k = Key(self.bucket)
k.key = self.path + filename
k.delete()
+ if filename in self.cache:
+ del self.cache[filename]
def loc_to_name(self, location):
return "log-%08d-%08d" % (location)
--
2.20.1