More updates to S3 pipelining test.

It seems there is a bug in S3 with many pipelined requests that we are
triggering in testing:
http://developer.amazonwebservices.com/connect/thread.jspa?messageID=39907

diff --git a/s3bench/s3pipeline.py b/s3bench/s3pipeline.py
index bad40df..47216b1 100755 (executable)
--- a/s3bench/s3pipeline.py
+++ b/s3bench/s3pipeline.py
@@ -24,7 +24,7 @@ class HttpResponseParser:
 
         if not self.header and self.content_length is not None:
             consumed = min(self.content_length, len(self.buf))
-            print "Got", consumed, "bytes of data"
+            #print "Got", consumed, "bytes of data"
             self.content_length -= consumed
             self.buf = self.buf[consumed:]
 
@@ -41,7 +41,7 @@ class HttpResponseParser:
 
         line = self.buf[0:crlf]
         self.buf = self.buf[crlf+2:]
-        print "Header line:", line
+        #print "Header line:", line
         if line.lower().startswith('content-length'):
             self.content_length = int(line[16:])
             print "Content length:", self.content_length
@@ -72,7 +72,7 @@ class PipelinedRequester:
         auth_path = conn.calling_format.build_auth_path(self.bucket, key)
 
         headers = {'User-Agent': boto.UserAgent + " (pipelined)",
-                   'Content-Length': str(0), 'Connection': 'keep-alive'}
+                   'Content-Length': str(0)}
         conn.add_aws_auth_header(headers, method, auth_path)
 
         req = "%s %s HTTP/1.1\r\nHost: %s\r\n" % (method, path, self.host)
@@ -80,9 +80,11 @@ class PipelinedRequester:
         self.sock.sendall(req)
 
 requester = PipelinedRequester(bucket_name)
-for i in range(16):
-    requester.send_request('file-8192-' + str(i))
-    if i == 0:
+for i in range(12, 18):
+    requester.send_request('file-%d-1' % (1 << i,))
+    if i == 12:
         time.sleep(2)
+for i in range(32):
+    requester.send_request('file-8192-%d' % (i,))
 
 time.sleep(5)