microbench/setup/s3-cleanup.py

   1 #!/usr/bin/env python
   2 #
   3 # Delete all files on S3 used for a benchmarking run (we don't want to be
   4 # charged for storage of them).
   5
   6 import boto, sys, time, Queue
   7 from boto.s3.connection import SubdomainCallingFormat
   8 from threading import Lock, Thread
   9
  10 if len(sys.argv) > 1:
  11     bucket_name = sys.argv[1]
  12 else:
  13     bucket_name = 'mvrable-bluesky'
  14
  15 DELAY = 4.0
  16 print "Will delete all contents of %s starting in %s seconds..." \
  17         % (bucket_name, DELAY)
  18 time.sleep(DELAY)
  19
  20 THREADS = 32
  21 print "Using %d threads" % (THREADS,)
  22
  23 q = Queue.Queue(THREADS * 4096)
  24 l = Lock()
  25
  26 def deletion_task():
  27     conn = boto.connect_s3()
  28     bucket = conn.get_bucket(bucket_name)
  29     while True:
  30         k = q.get()
  31         l.acquire()
  32         print k
  33         l.release()
  34         bucket.delete_key(k)
  35         q.task_done()
  36
  37 for i in range(THREADS):
  38     t = Thread(target=deletion_task)
  39     t.setDaemon(True)
  40     t.start()
  41
  42 conn = boto.connect_s3(is_secure=False,
  43                        calling_format=SubdomainCallingFormat())
  44 bucket = conn.get_bucket(bucket_name)
  45 for k in bucket:
  46     q.put(k.key)
  47
  48 q.join()
  49 time.sleep(0.5)
  50
  51 print "Finished"