X-Git-Url: http://git.vrable.net/?a=blobdiff_plain;f=cleaner%2Fazure.py;h=7a07694cd772ec338028ea98ea993a6927eaa162;hb=388030970805a70cb4fad34ade5e3de7a3607a57;hp=9f0ac519abd96dab5422c4ff352b20ae10e802ea;hpb=fcfa65ce8923a13cc3d6022b0c6e2bd5d4262a28;p=bluesky.git

diff --git a/cleaner/azure.py b/cleaner/azure.py
index 9f0ac51..7a07694 100644
--- a/cleaner/azure.py
+++ b/cleaner/azure.py
@@ -148,16 +148,39 @@ class AzureConnection:
         path = "/%s/%s" % (container, key)
         r = self._make_request(path, method='DELETE')
 
+def parallel_delete(container, keys):
+    import Queue
+    from threading import Lock, Thread
+
+    keys = list(iter(keys))
+
+    q = Queue.Queue(16384)
+    l = Lock()
+
+    def deletion_task():
+        conn = AzureConnection()
+        while True:
+            k = q.get()
+            l.acquire()
+            print k
+            l.release()
+            conn.delete(container, k)
+            q.task_done()
+
+    for i in range(128):
+        t = Thread(target=deletion_task)
+        t.setDaemon(True)
+        t.start()
+
+    for k in keys:
+        q.put(k)
+    q.join()
+
 if __name__ == '__main__':
     container = 'bluesky'
     conn = AzureConnection()
 
-    print "Container contents:"
-    for k in conn.list(container):
-        print k
-
     conn.put(container, "testkey", "A" * 40)
     print "Fetch result:", conn.get(container, "testkey")
 
-    for k in list(iter(conn.list(container))):
-        conn.delete(container, k)
+    parallel_delete(container, conn.list(container))