1 # Blue Sky: File Systems in the Cloud
3 # Copyright (C) 2011 The Regents of the University of California
4 # Written by Michael Vrable <mvrable@cs.ucsd.edu>
6 # Redistribution and use in source and binary forms, with or without
7 # modification, are permitted provided that the following conditions
9 # 1. Redistributions of source code must retain the above copyright
10 # notice, this list of conditions and the following disclaimer.
11 # 2. Redistributions in binary form must reproduce the above copyright
12 # notice, this list of conditions and the following disclaimer in the
13 # documentation and/or other materials provided with the distribution.
14 # 3. Neither the name of the University nor the names of its contributors
15 # may be used to endorse or promote products derived from this software
16 # without specific prior written permission.
18 # THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
19 # ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
20 # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
21 # ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
22 # FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
23 # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
24 # OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
25 # HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
26 # LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
27 # OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
30 """A simple Python library for accessing the Windows Azure blob service."""
32 import base64, hashlib, hmac, httplib, os, time, urllib
33 import xml.dom.minidom
35 # The version of the Azure API we implement; sent in the x-ms-version header.
36 API_VERSION = '2009-09-19'
39 return urllib.unquote_plus(s)
42 return urllib.quote_plus(s)
44 def xmlGetText(nodelist):
48 if node.nodeType == node.TEXT_NODE:
49 text.append(node.data)
56 return xml.dom.minidom.parseString(s)
58 def buildQueryString(uri, params={}):
59 for (k, v) in params.items():
60 if v is None: continue
61 kv = '%s=%s' % (uri_encode(k), uri_encode(v))
68 def add_auth_headers(headers, method, path, account, key):
69 header_order = ['Content-Encoding', 'Content-Language', 'Content-Length',
70 'Content-MD5', 'Content-Type', 'Date', 'If-Modified-Since',
71 'If-Match', 'If-None-Match', 'If-Unmodified-Since',
74 if not headers.has_key('Date'):
75 headers['Date'] = time.strftime("%a, %d %b %Y %H:%M:%S GMT",
77 if not headers.has_key('x-ms-version'):
78 headers['x-ms-version'] = API_VERSION
80 StringToSign = method + "\n"
81 for h in header_order:
83 StringToSign += headers[h] + "\n"
87 # Add Canonicalized Headers
89 for (k, v) in headers.items():
91 if k.startswith('x-ms-'):
92 canonized.append((k, v))
94 for (k, v) in canonized:
95 StringToSign += "%s:%s\n" % (k, v)
97 resource = "/" + account
101 (path, params) = path.split('?', 1)
102 params = [p.split('=') for p in params.split("&")]
103 params = dict((k.lower(), uri_decode(v)) for (k, v) in params)
105 for k in sorted(params):
106 resource += "\n%s:%s" % (k, params[k])
107 StringToSign += resource
109 h = hmac.new(key, digestmod=hashlib.sha256)
110 h.update(StringToSign)
112 signature = base64.b64encode(h.digest())
113 headers['Authorization'] = "SharedKey %s:%s" % (account, signature)
115 class AzureError(RuntimeError):
116 def __init__(self, response):
117 self.response = response
118 self.details = response.read()
120 class AzureConnection:
121 def __init__(self, account=None, key=None):
123 account = os.environ['AZURE_ACCOUNT_NAME']
124 self.account = account
125 self.host = account + ".blob.core.windows.net"
126 #self.conn = httplib.HTTPConnection(self.host)
129 key = os.environ['AZURE_SECRET_KEY']
130 self.key = base64.b64decode(key)
132 def _make_request(self, path, method='GET', body="", headers={}):
133 headers = headers.copy()
134 headers['Content-Length'] = str(len(body))
136 headers['Content-MD5'] \
137 = base64.b64encode(hashlib.md5(body).digest())
138 add_auth_headers(headers, method, path, self.account, self.key)
140 conn = httplib.HTTPConnection(self.host)
141 conn.request(method, path, body, headers)
142 response = conn.getresponse()
143 if response.status // 100 != 2:
144 raise AzureError(response)
146 #print "Response:", response.status
147 #print "Headers:", response.getheaders()
148 #body = response.read()
150 def list(self, container, prefix=''):
153 path = '/' + container + '?restype=container&comp=list'
154 path = buildQueryString(path, {'prefix': prefix, 'marker': marker})
155 r = self._make_request(path)
156 xml = xmlParse(r.read())
158 blobs = xml.getElementsByTagName('Blob')
160 yield xmlGetText(b.getElementsByTagName('Name'))
162 marker = xmlGetText(xml.getElementsByTagName('NextMarker'))
166 def get(self, container, key):
167 path = "/%s/%s" % (container, key)
168 r = self._make_request(path)
171 def put(self, container, key, value):
172 path = "/%s/%s" % (container, key)
173 r = self._make_request(path, method='PUT', body=value,
174 headers={'x-ms-blob-type': 'BlockBlob'})
176 def delete(self, container, key):
177 path = "/%s/%s" % (container, key)
178 r = self._make_request(path, method='DELETE')
180 def parallel_delete(container, keys):
182 from threading import Lock, Thread
184 keys = list(iter(keys))
186 q = Queue.Queue(16384)
190 conn = AzureConnection()
196 conn.delete(container, k)
200 t = Thread(target=deletion_task)
208 if __name__ == '__main__':
209 container = 'bluesky'
210 conn = AzureConnection()
212 conn.put(container, "testkey", "A" * 40)
213 print "Fetch result:", conn.get(container, "testkey")
215 parallel_delete(container, conn.list(container))