1 # Amazon S3 storage backend. Uses a URL of the form s3://BUCKET/PATH/.
2 import os, sys, tempfile
4 from boto.s3.bucket import Bucket
5 from boto.s3.key import Key
9 class S3Store(cumulus.store.Store):
10 def __init__(self, url, **kw):
11 # Old versions of the Python urlparse library will take a URL like
12 # s3://bucket/path/ and include the bucket with the path, while new
13 # versions (2.6 and later) treat it as the netloc (which seems more
16 # But, so that we can work with either behavior, for now just combine
17 # the netloc and path together before we do any further processing
18 # (which will then split the combined path apart into a bucket and path
19 # again). If we didn't want to support Python 2.5, this would be
20 # easier as we could just use the netloc as the bucket directly.
21 path = self.netloc + '/' + self.path
22 (bucket, prefix) = path.lstrip("/").split("/", 1)
23 self.conn = boto.connect_s3(is_secure=False)
24 self.bucket = self.conn.create_bucket(bucket)
25 self.prefix = prefix.strip("/")
28 def _get_key(self, type, name):
30 k.key = "%s/%s/%s" % (self.prefix, type, name)
34 prefix = "%s/" % (self.prefix,)
35 for i in self.bucket.list(prefix):
36 assert i.key.startswith(prefix)
37 self.scan_cache[i.key] = i
40 prefix = "%s/%s/" % (self.prefix, type)
41 for i in self.bucket.list(prefix):
42 assert i.key.startswith(prefix)
43 yield i.key[len(prefix):]
45 def get(self, type, name):
46 fp = tempfile.TemporaryFile()
47 k = self._get_key(type, name)
52 def put(self, type, name, fp):
53 k = self._get_key(type, name)
54 k.set_contents_from_file(fp)
56 def delete(self, type, name):
57 self.bucket.delete_key("%s/%s/%s" % (self.prefix, type, name))
59 def stat(self, type, name):
60 path = "%s/%s/%s" % (self.prefix, type, name)
61 if path in self.scan_cache:
62 k = self.scan_cache[path]
64 k = self.bucket.get_key(path)
66 raise cumulus.store.NotFoundError
68 return {'size': int(k.size)}