From fa2344c8e4f01f2a44dd44bed9448934d318fe84 Mon Sep 17 00:00:00 2001
From: Michael Vrable <mvrable@cs.ucsd.edu>
Date: Mon, 14 Mar 2011 20:56:08 -0700
Subject: [PATCH] Speed up the deletion of all files in an Azure container
---
cleaner/azure.py | 32 +++++++++++++++++++++++++++++---
1 file changed, 29 insertions(+), 3 deletions(-)
diff --git a/cleaner/azure.py b/cleaner/azure.py
index 89b4f23..7a07694 100644
--- a/cleaner/azure.py
+++ b/cleaner/azure.py
@@ -148,6 +148,34 @@ class AzureConnection:
path = "/%s/%s" % (container, key)
r = self._make_request(path, method='DELETE')
+def parallel_delete(container, keys):
+ import Queue
+ from threading import Lock, Thread
+
+ keys = list(iter(keys))
+
+ q = Queue.Queue(16384)
+ l = Lock()
+
+ def deletion_task():
+ conn = AzureConnection()
+ while True:
+ k = q.get()
+ l.acquire()
+ print k
+ l.release()
+ conn.delete(container, k)
+ q.task_done()
+
+ for i in range(128):
+ t = Thread(target=deletion_task)
+ t.setDaemon(True)
+ t.start()
+
+ for k in keys:
+ q.put(k)
+ q.join()
+
if __name__ == '__main__':
container = 'bluesky'
conn = AzureConnection()
@@ -155,6 +183,4 @@ if __name__ == '__main__':
conn.put(container, "testkey", "A" * 40)
print "Fetch result:", conn.get(container, "testkey")
- for k in list(iter(conn.list(container))):
- print "Deleting", k
- conn.delete(container, k)
+ parallel_delete(container, conn.list(container))
--
2.20.1