From fa2344c8e4f01f2a44dd44bed9448934d318fe84 Mon Sep 17 00:00:00 2001 From: Michael Vrable Date: Mon, 14 Mar 2011 20:56:08 -0700 Subject: [PATCH] Speed up the deletion of all files in an Azure container --- cleaner/azure.py | 32 +++++++++++++++++++++++++++++--- 1 file changed, 29 insertions(+), 3 deletions(-) diff --git a/cleaner/azure.py b/cleaner/azure.py index 89b4f23..7a07694 100644 --- a/cleaner/azure.py +++ b/cleaner/azure.py @@ -148,6 +148,34 @@ class AzureConnection: path = "/%s/%s" % (container, key) r = self._make_request(path, method='DELETE') +def parallel_delete(container, keys): + import Queue + from threading import Lock, Thread + + keys = list(iter(keys)) + + q = Queue.Queue(16384) + l = Lock() + + def deletion_task(): + conn = AzureConnection() + while True: + k = q.get() + l.acquire() + print k + l.release() + conn.delete(container, k) + q.task_done() + + for i in range(128): + t = Thread(target=deletion_task) + t.setDaemon(True) + t.start() + + for k in keys: + q.put(k) + q.join() + if __name__ == '__main__': container = 'bluesky' conn = AzureConnection() @@ -155,6 +183,4 @@ if __name__ == '__main__': conn.put(container, "testkey", "A" * 40) print "Fetch result:", conn.get(container, "testkey") - for k in list(iter(conn.list(container))): - print "Deleting", k - conn.delete(container, k) + parallel_delete(container, conn.list(container)) -- 2.20.1