3 # Run a series of simple test requests against S3 for gathering some basic
7 from boto.s3.connection import SubdomainCallingFormat
8 from boto.s3.key import Key
9 import sys, threading, time, Queue
12 BUCKET_NAME = 'mvrable-benchmark'
13 SIZES = [64, 4096, 32 << 10, 256 << 10, 1 << 20, 4 << 20, 32 << 20]
15 class S3TestConnection:
17 self.conn = boto.connect_s3(is_secure=False,
18 calling_format=SubdomainCallingFormat())
19 self.bucket = self.conn.get_bucket(BUCKET_NAME)
21 def put_object(self, name, size):
23 k = Key(self.bucket, name)
24 start_time = time.time()
25 k.set_contents_from_string(buf)
26 #print "%s: %f" % (name, time.time() - start_time)
28 def get_object(self, name):
29 k = Key(self.bucket, name)
30 start_time = time.time()
31 buf = k.get_contents_as_string()
32 duration = time.time() - start_time
33 #print "%s: %f" % (name, duration)
36 def parallel_get(name, connections, delay1=0.0):
37 #print "Get: %s x %d" % (name, len(connections))
40 def launcher(c, name, result_queue):
41 result_queue.put(c.get_object(name))
42 for i in range(len(connections)):
44 threads.append(threading.Thread(target=launcher, args=(c, name, q)))
45 for i in range(len(threads)):
47 for t in threads: t.join()
53 def run_test(size, threads, num):
54 connections = [S3TestConnection() for _ in range(threads)]
56 res = parallel_get('file-%d-%d' % (size, i), connections)
60 run_test(32768, 4, 500)
63 if __name__ == '__main__':
64 # Pass 1: Identical downloads in parallel
65 connections = [S3TestConnection() for _ in range(8)]
66 SIZES = [4096, 32 << 10, 256 << 10, 1 << 20, 4 << 20]
67 PRIME = (1 << 20) + (1 << 10)
68 c = S3TestConnection()
71 parallel_get('file-%d-%d' % (size, i), connections)
73 # Pass 1: Downloads in parallel, but downloads staggered so one request
75 connections = [S3TestConnection() for _ in range(8)]
76 SIZES = [4096, 32 << 10, 256 << 10, 1 << 20, 4 << 20]
77 PRIME = (1 << 20) + (1 << 10)
78 c = S3TestConnection()
81 parallel_get('file-%d-%d' % (size, i), connections, delay1=1.0)