python/cumulus/store/s3.py

   1 import os, sys, tempfile
   2 import boto
   3 from boto.s3.bucket import Bucket
   4 from boto.s3.key import Key
   5
   6 import cumulus.store
   7
   8 class S3Store(cumulus.store.Store):
   9     def __init__(self, bucket, prefix):
  10         self.conn = boto.connect_s3(is_secure=False)
  11         self.bucket = self.conn.create_bucket(bucket)
  12         while prefix.endswith("/"): prefix = prefix[:-1]
  13         self.prefix = prefix
  14         self.scan_cache = {}
  15
  16     def _get_key(self, type, name):
  17         k = Key(self.bucket)
  18         k.key = "%s/%s/%s" % (self.prefix, type, name)
  19         return k
  20
  21     def scan(self):
  22         prefix = "%s/" % (self.prefix,)
  23         for i in self.bucket.list(prefix):
  24             assert i.key.startswith(prefix)
  25             self.scan_cache[i.key] = i
  26
  27     def list(self, type):
  28         prefix = "%s/%s/" % (self.prefix, type)
  29         for i in self.bucket.list(prefix):
  30             assert i.key.startswith(prefix)
  31             yield i.key[len(prefix):]
  32
  33     def get(self, type, name):
  34         fp = tempfile.TemporaryFile()
  35         k = self._get_key(type, name)
  36         k.get_file(fp)
  37         fp.seek(0)
  38         return fp
  39
  40     def put(self, type, name, fp):
  41         k = self._get_key(type, name)
  42         k.set_contents_from_file(fp)
  43
  44     def delete(self, type, name):
  45         self.bucket.delete_key("%s/%s/%s" % (self.prefix, type, name))
  46
  47     def stat(self, type, name):
  48         path = "%s/%s/%s" % (self.prefix, type, name)
  49         if path in self.scan_cache:
  50             k = self.scan_cache[path]
  51         else:
  52             k = self.bucket.get_key(path)
  53         if k is None:
  54             raise cumulus.store.NotFoundError
  55
  56         return {'size': int(k.size)}