Add proper per-file copyright notices/licenses and top-level license.
[bluesky.git] / microbench / setup / s3-cleanup.py
1 #!/usr/bin/env python
2 #
3 # Delete all files on S3 used for a benchmarking run (we don't want to be
4 # charged for storage of them).
5
6 import boto, sys, time, Queue
7 from boto.s3.connection import SubdomainCallingFormat
8 from threading import Lock, Thread
9
10 if len(sys.argv) > 1:
11     bucket_name = sys.argv[1]
12 else:
13     bucket_name = 'mvrable-bluesky'
14
15 DELAY = 4.0
16 print "Will delete all contents of %s starting in %s seconds..." \
17         % (bucket_name, DELAY)
18 time.sleep(DELAY)
19
20 THREADS = 32
21 print "Using %d threads" % (THREADS,)
22
23 q = Queue.Queue(THREADS * 4096)
24 l = Lock()
25
26 def deletion_task():
27     conn = boto.connect_s3()
28     bucket = conn.get_bucket(bucket_name)
29     while True:
30         k = q.get()
31         l.acquire()
32         print k
33         l.release()
34         bucket.delete_key(k)
35         q.task_done()
36
37 for i in range(THREADS):
38     t = Thread(target=deletion_task)
39     t.setDaemon(True)
40     t.start()
41
42 conn = boto.connect_s3(is_secure=False,
43                        calling_format=SubdomainCallingFormat())
44 bucket = conn.get_bucket(bucket_name)
45 for k in bucket:
46     q.put(k.key)
47
48 q.join()
49 time.sleep(0.5)
50
51 print "Finished"