From: Michael Vrable <mvrable@cs.ucsd.edu>
Date: Fri, 28 May 2010 19:08:59 +0000 (-0700)
Subject: Testing of multiple requests in parallel.
X-Git-Url: http://git.vrable.net/?p=bluesky.git;a=commitdiff_plain;h=b49973e78300a264aa8e3b985aa125e0e0e89d8c

Testing of multiple requests in parallel.
---

diff --git a/cloudbench/paralleltest.py b/cloudbench/paralleltest.py
index f1b0be2..5084b62 100755
--- a/cloudbench/paralleltest.py
+++ b/cloudbench/paralleltest.py
@@ -6,10 +6,10 @@
 import boto, time
 from boto.s3.connection import SubdomainCallingFormat
 from boto.s3.key import Key
-import sys, threading, time
+import sys, threading, time, Queue
 import azure
 
-BUCKET_NAME = 'mvrable-benchmark-west'
+BUCKET_NAME = 'mvrable-benchmark'
 SIZES = [64, 4096, 32 << 10, 256 << 10, 1 << 20, 4 << 20, 32 << 20]
 
 class S3TestConnection:
@@ -23,38 +23,42 @@ class S3TestConnection:
         k = Key(self.bucket, name)
         start_time = time.time()
         k.set_contents_from_string(buf)
-        print "%s: %f" % (name, time.time() - start_time)
+        #print "%s: %f" % (name, time.time() - start_time)
 
     def get_object(self, name):
         k = Key(self.bucket, name)
         start_time = time.time()
         buf = k.get_contents_as_string()
-        print "%s: %f" % (name, time.time() - start_time)
+        duration = time.time() - start_time
+        #print "%s: %f" % (name, duration)
+        return duration
 
 def parallel_get(name, connections, delay1=0.0):
-    print "Get: %s x %d" % (name, len(connections))
+    #print "Get: %s x %d" % (name, len(connections))
     threads = []
+    q = Queue.Queue()
+    def launcher(c, name, result_queue):
+        result_queue.put(c.get_object(name))
     for i in range(len(connections)):
         c = connections[i]
-        threads.append(threading.Thread(target=c.get_object, args=(name,)))
+        threads.append(threading.Thread(target=launcher, args=(c, name, q)))
     for i in range(len(threads)):
         threads[i].start()
-        if i == 0: time.sleep(delay1)
     for t in threads: t.join()
-    time.sleep(1.0)
+    res = []
+    while not q.empty():
+        res.append(q.get())
+    return res
 
-def run_test():
-    print "==== S3 ===="
-    c = S3TestConnection()
-    for repeat in range(4):
-        for size in SIZES:
-            #c.put_object('file-%d-%d' % (size, repeat), size)
-            pass
+def run_test(size, threads, num):
+    connections = [S3TestConnection() for _ in range(threads)]
+    for i in range(num):
+        res = parallel_get('file-%d-%d' % (size, i), connections)
+        print res
+        time.sleep(1.0)
 
-    c = S3TestConnection()
-    for repeat in range(4):
-        for size in SIZES:
-            c.get_object('file-%d-%d' % (size, repeat))
+run_test(32768, 4, 500)
+sys.exit(0)
 
 if __name__ == '__main__':
     # Pass 1: Identical downloads in parallel