X-Git-Url: http://git.vrable.net/?a=blobdiff_plain;f=cloudbench%2Fparalleltest.py;h=b447938ecdbc77397e096e708b84c919e0082683;hb=6e95f90cc53d2a230b9cf22d3b5906a66b4e71de;hp=f1b0be2418ec2da3015009d6bcacea4a98a997bc;hpb=7902607f424c9b68f16adf56bab03fedb599c4ef;p=bluesky.git

diff --git a/cloudbench/paralleltest.py b/cloudbench/paralleltest.py
index f1b0be2..b447938 100755
--- a/cloudbench/paralleltest.py
+++ b/cloudbench/paralleltest.py
@@ -6,11 +6,11 @@
 import boto, time
 from boto.s3.connection import SubdomainCallingFormat
 from boto.s3.key import Key
-import sys, threading, time
+import sys, threading, time, Queue
 import azure
 
-BUCKET_NAME = 'mvrable-benchmark-west'
-SIZES = [64, 4096, 32 << 10, 256 << 10, 1 << 20, 4 << 20, 32 << 20]
+BUCKET_NAME = 'mvrable-benchmark'
+SIZES = [(1 << s) for s in range(12, 23)]
 
 class S3TestConnection:
     def __init__(self):
@@ -23,38 +23,55 @@ class S3TestConnection:
         k = Key(self.bucket, name)
         start_time = time.time()
         k.set_contents_from_string(buf)
-        print "%s: %f" % (name, time.time() - start_time)
+        #print "%s: %f" % (name, time.time() - start_time)
 
     def get_object(self, name):
         k = Key(self.bucket, name)
         start_time = time.time()
         buf = k.get_contents_as_string()
-        print "%s: %f" % (name, time.time() - start_time)
+        duration = time.time() - start_time
+        #print "%s: %f" % (name, duration)
+        return duration
 
 def parallel_get(name, connections, delay1=0.0):
-    print "Get: %s x %d" % (name, len(connections))
+    #print "Get: %s x %d" % (name, len(connections))
     threads = []
+    q = Queue.Queue()
+    def launcher(c, name, result_queue):
+        result_queue.put(c.get_object(name))
     for i in range(len(connections)):
         c = connections[i]
-        threads.append(threading.Thread(target=c.get_object, args=(name,)))
+        threads.append(threading.Thread(target=launcher, args=(c, name, q)))
     for i in range(len(threads)):
         threads[i].start()
-        if i == 0: time.sleep(delay1)
     for t in threads: t.join()
-    time.sleep(1.0)
+    res = []
+    while not q.empty():
+        res.append(q.get())
 
-def run_test():
-    print "==== S3 ===="
-    c = S3TestConnection()
-    for repeat in range(4):
-        for size in SIZES:
-            #c.put_object('file-%d-%d' % (size, repeat), size)
-            pass
+    if len(res) == len(connections):
+        return res
 
-    c = S3TestConnection()
-    for repeat in range(4):
-        for size in SIZES:
-            c.get_object('file-%d-%d' % (size, repeat))
+def run_test(size, threads, num, logfile=sys.stdout, delay=1.0):
+    connections = [S3TestConnection() for _ in range(threads)]
+    for i in range(num):
+        print "    ...test", i
+        res = parallel_get('file-%d-%d' % (size, i), connections)
+        if res is not None:
+            logfile.write(str(min(res)) + "\n")
+        if delay > 0:
+            time.sleep(delay)
+
+for s in SIZES:
+    print "Priming objects: %d-byte objects" % (s,)
+    logfile = open('/dev/null', 'w')
+    run_test(s, 1, 100, logfile, 0.0)
+
+    for t in [4, 2, 1]:
+        print "Running tests: %d-byte objects, %d parallel fetches" % (s, t)
+        logfile = open('parallel-%d-%d.data' % (s, t), 'w')
+        run_test(s, t, 100, logfile)
+sys.exit(0)
 
 if __name__ == '__main__':
     # Pass 1: Identical downloads in parallel