BLUESKY_RUN_NAME=$(date +%Y%m%d)-$$
BENCHER=vrable1.sysnet.ucsd.edu
PROXY=vrable2.sysnet.ucsd.edu
+BLUESKY_EXTRA_OPTS=""
SSH_ARGS="-i $HOME/.ssh/id_bluesky"
echo "$p=${!p}"
fi
done
+ for p in $BLUESKY_EXTRA_OPTS; do
+ echo $p
+ done
}
run_cmd() {
BLUESKY_RUN_NAME=$(date +%Y%m%d)-$1
SPEC_CONF=$2
- $HOME/bin/s3-cleanup.py mvrable-bluesky
- $HOME/bin/s3-cleanup.py mvrable-bluesky-west
+ #$HOME/bin/s3-cleanup.py mvrable-bluesky
+ #$HOME/bin/s3-cleanup.py mvrable-bluesky-west
run_cmd $PROXY cleanup-proxy
run_cmd $PROXY run-proxy >$BLUESKY_RUN_NAME.proxy 2>&1 &
proxy_pid=$!
wait $proxy_pid
}
-BLUESKY_TARGET=s3:mvrable-bluesky-west
-BLUESKY_OPT_FULL_SEGMENTS=1
-run_spec s3-west-fullseg sfs_bluesky
+# Test of using full segment fetches instead of range requests
+#BLUESKY_TARGET=s3:mvrable-bluesky-west
+#BLUESKY_OPT_FULL_SEGMENTS=1
+#run_spec s3-west-fullseg sfs_bluesky
+
+# Run a simulation of not using a log-structured system at all
+#BLUESKY_TARGET=s3:mvrable-bluesky-west
+#BLUESKY_EXTRA_OPTS="BLUESKY_OPT_FULL_SEGMENTS=1 BLUESKY_OPT_NO_AGGREGATION=1"
+#run_spec s3-west-noseg sfs_bluesky
+
+# Standard test with Azure instead of Amazon S3 for storage
+BLUESKY_TARGET=azure
+run_spec azure sfs_bluesky
--- /dev/null
+# target_ops actual_ops latency_avg read write create setattr lookup getattr
+50 50 0.6 - 2.47 2.15 0.88 0.18 0.17
+100 100 2.5 7.59 7.76 2.82 1.0 0.18 0.18
+150 150 2.3 6.26 8.05 3.45 1.23 0.18 0.18
+200 201 2.8 8.41 9.29 4.96 1.51 0.19 0.18
+250 251 4.3 14.58 11.65 6.93 2.62 0.19 0.18
+300 301 4.6 13.14 12.62 12.0 6.01 0.19 0.18
+350 352 4.1 12.05 12.22 9.1 3.35 0.19 0.18
+400 401 4.6 13.81 13.35 11.43 3.55 0.19 0.19
+450 454 5.0 15.96 12.74 8.88 3.93 0.19 0.19
+500 505 6.2 21.02 15.38 11.52 4.36 0.19 0.19
+550 555 6.6 22.01 16.71 11.55 4.47 0.19 0.19
+600 610 7.4 25.23 18.71 14.05 6.04 0.19 0.19
+650 612 8.2 27.13 21.6 15.05 6.97 0.19 0.18
+700 517 9.8 33.35 24.8 18.13 8.28 0.21 0.22
+750 509 9.9 34.67 22.94 17.24 7.04 0.25 0.24
+800 474 10.6 37.45 23.38 23.34 7.81 0.27 0.25
+850 459 10.8 37.22 24.45 19.81 10.55 0.19 0.18
+900 421 11.8 41.63 28.42 22.94 8.55 0.19 0.18
+950 382 12.9 45.03 31.24 25.52 8.59 0.19 0.18
+1000 354 14.0 49.88 31.59 23.47 13.46 0.19 0.18
+1050 352 14.2 51.77 33.18 30.27 9.55 0.19 0.18
+1100 292 16.6 59.15 36.41 29.74 16.28 0.19 0.18
+1150 301 16.2 57.78 35.66 35.41 10.34 0.19 0.18
+1200 287 17.0 60.72 38.04 40.44 12.04 0.19 0.18
+1250 263 18.5 67.99 42.68 28.96 13.67 0.2 0.18
--- /dev/null
+# target_ops actual_ops latency_avg read write create setattr lookup getattr
+50 50 0.7 - 2.53 2.22 1.21 0.18 0.18
+100 100 3.0 10.17 5.92 2.98 3.01 0.18 0.18
+150 150 2.9 9.95 7.06 3.63 1.44 0.18 0.18
+200 201 3.9 14.51 7.49 5.33 2.4 0.18 0.18
+250 251 4.8 18.36 8.96 5.02 3.09 0.19 0.18
+300 302 4.9 17.7 8.75 8.75 4.78 0.19 0.18
+350 353 4.9 17.37 9.07 8.45 5.48 0.19 0.18
+400 404 5.4 20.04 8.88 12.64 5.28 0.19 0.19
+450 455 8.3 29.98 12.38 12.04 13.35 0.3 0.24
+500 232 20.5 73.39 28.87 26.75 24.4 5.98 1.0
+550 20 136.4 377.36 92.16 98.16 127.19 51.6 63.53
--- /dev/null
+# target_ops actual_ops latency_avg read write create setattr lookup getattr
+50 50 0.9 0.6 0.87 5.06 3.41 0.15 0.14
+100 100 2.4 2.41 4.08 10.39 11.58 0.15 0.14
+150 150 2.8 2.91 5.18 20.69 13.38 0.15 0.14
+200 200 3.2 3.23 3.72 24.77 15.75 0.15 0.14
+250 248 4.0 4.4 4.15 29.96 24.75 0.15 0.14
+300 308 5.0 4.28 4.68 45.52 34.37 0.15 0.14
+350 355 6.1 4.89 4.5 63.39 45.96 0.15 0.14
+400 397 7.3 6.27 5.96 87.67 50.18 0.16 0.15
+450 421 9.7 8.11 6.07 135.19 69.25 0.16 0.15
+500 366 11.3 10.22 7.33 153.18 84.46 0.16 0.15
+550 341 12.1 12.07 8.47 142.13 87.97 0.16 0.15
+600 338 12.3 13.39 9.28 177.98 79.28 0.17 0.15
+650 279 14.9 18.83 11.8 173.63 106.86 0.16 0.15
+700 285 14.6 18.22 10.35 185.34 96.68 0.19 0.15
+750 276 15.1 18.31 12.1 183.76 101.98 0.17 0.15
+800 274 15.2 19.03 10.9 198.73 98.85 0.18 0.15
+850 265 15.8 20.26 12.0 193.63 105.41 0.17 0.15
+900 242 17.2 24.35 12.34 209.12 114.67 0.19 0.15
+950 235 17.7 24.05 13.96 214.2 116.21 0.2 0.15
+1000 238 17.5 22.29 13.16 215.77 119.35 0.28 0.15
+1050 226 18.4 27.05 14.04 202.13 122.22 0.26 0.15
+1100 245 17.1 24.07 13.93 184.61 117.15 0.21 0.15
+1150 228 18.3 24.67 13.71 223.73 118.08 0.23 0.15
+1200 224 18.7 26.07 13.51 213.98 124.15 0.36 0.15
+1250 220 19.0 28.53 14.49 225.1 125.71 0.31 0.15
--- /dev/null
+# target_ops actual_ops latency_avg read write create setattr lookup getattr
+50 50 0.6 - 2.23 2.14 0.86 0.18 0.17
+100 100 1.8 4.18 7.23 2.84 1.04 0.18 0.18
+150 150 3.3 7.96 14.93 3.96 1.52 0.18 0.18
+200 201 3.2 7.3 15.15 4.5 1.44 0.2 0.18
+250 252 9.9 28.95 37.98 10.33 3.92 0.29 0.22
+300 244 18.9 57.35 68.87 15.77 5.5 0.22 0.19
+350 142 32.1 110.09 107.32 23.23 9.32 0.26 0.18
+400 103 44.2 153.65 143.79 35.41 10.03 0.24 0.18
+450 70 64.2 228.74 203.17 38.95 12.93 0.38 0.19
+500 60 74.6 264.03 229.13 55.05 16.33 0.18 0.18
+550 51 87.4 326.31 238.64 45.76 11.96 0.2 0.23
+600 54 83.5 317.46 229.61 62.68 17.94 0.28 0.22
+650 42 105.3 401.14 281.91 65.28 18.56 0.19 0.18
+700 43 102.0 373.66 283.82 89.0 19.31 0.28 0.18
+750 35 127.1 476.85 353.94 75.13 19.64 0.21 0.18
+800 38 116.1 422.79 322.12 94.72 25.34 0.19 0.32
+850 35 125.9 482.94 328.19 82.82 19.9 0.2 0.18
+900 33 135.4 511.89 360.49 100.49 26.17 0.26 0.18
+950 33 135.7 522.47 359.12 90.21 17.35 0.19 0.18
+1000 31 140.1 527.36 377.98 133.21 29.83 0.19 0.18
+1050 29 149.9 550.69 392.93 89.19 25.05 0.19 0.18
+1100 30 149.2 593.24 371.8 99.28 21.5 0.44 0.18
+1150 31 140.0 543.0 372.45 120.19 24.16 0.24 0.18
+1200 31 142.4 552.34 354.58 100.58 21.8 0.22 0.22
+1250 27 161.9 604.93 434.33 115.2 18.86 0.2 0.18
--- /dev/null
+# target_ops actual_ops latency_avg read write create setattr lookup getattr
+50 50 0.7 - 2.18 2.14 1.92 0.18 0.17
+100 100 2.6 7.66 7.2 7.24 1.64 0.18 0.18
+150 150 2.7 8.22 7.48 4.06 2.1 0.18 0.18
+200 201 3.1 10.29 8.81 3.75 1.17 0.18 0.18
+250 251 4.2 15.4 10.73 4.2 1.34 0.18 0.18
+300 302 4.1 14.04 10.64 4.89 1.7 0.19 0.18
+350 353 4.0 13.55 10.72 5.32 1.61 0.19 0.18
+400 402 4.2 14.77 11.24 5.86 1.77 0.19 0.18
+450 453 4.8 17.52 11.26 6.24 1.74 0.19 0.19
+500 505 5.3 19.31 12.74 7.4 1.94 0.19 0.19
+550 554 5.6 20.48 13.83 8.01 2.23 0.2 0.19
+600 606 6.2 23.31 14.74 8.58 2.06 0.2 0.19
+650 657 7.2 27.08 17.81 10.16 2.05 0.2 0.19
+700 708 7.4 28.17 17.29 10.3 2.12 0.2 0.19
+750 678 7.7 29.27 18.15 11.5 2.38 0.2 0.19
+800 661 8.0 30.71 18.05 12.25 2.38 0.2 0.19
+850 585 9.0 34.85 20.14 13.62 2.58 0.2 0.2
+900 497 10.5 40.74 24.9 14.27 2.58 0.19 0.19
+950 478 10.9 42.55 25.08 14.79 2.38 0.19 0.18
+1000 464 11.2 44.42 24.47 17.33 2.4 0.2 0.19
+1050 446 11.6 45.94 27.36 17.09 2.31 0.2 0.19
+1100 441 11.8 46.46 26.66 18.6 2.48 0.2 0.19
+1150 414 12.5 50.1 27.36 19.32 2.68 0.2 0.19
+1200 382 13.5 54.12 29.24 20.6 2.57 0.2 0.19
+1250 376 13.7 54.69 29.71 21.63 2.7 0.2 0.19
--- /dev/null
+# target_ops actual_ops latency_avg read write create setattr lookup getattr
+50 50 0.7 0.97 2.67 2.12 0.93 0.18 0.18
+100 101 5.1 15.44 19.95 3.21 1.03 0.18 0.18
+150 151 5.3 15.31 16.38 5.18 3.99 0.3 0.29
+200 201 6.8 20.12 26.88 5.11 2.64 0.18 0.18
+250 251 11.3 37.35 40.05 7.68 3.38 0.19 0.18
+300 303 10.8 36.25 34.53 8.78 3.54 0.19 0.18
+350 303 15.6 51.51 44.69 23.58 11.56 0.24 0.23
+400 244 20.4 72.6 63.4 12.12 3.71 0.18 0.17
+450 185 17.2 63.7 49.71 13.63 3.81 0.19 0.18
+500 32 27.2 74.72 63.96 16.18 6.9 0.51 15.85
+550 133 35.3 131.6 105.62 7.08 6.36 0.18 0.17
+600 144 33.6 123.36 103.22 17.44 7.83 0.18 0.17
+650 118 40.2 148.44 116.56 13.85 9.05 0.18 0.17
+700 109 43.5 163.09 126.11 28.79 6.5 0.18 0.17
+750 90 49.7 194.78 124.69 22.33 12.88 0.18 0.17
m = regexp.match(l)
if m: return m
+OPERATIONS = ('read', 'write', 'create', 'setattr', 'lookup', 'getattr')
+
def parse_run(lines, timestamp, outfp=sys.stdout):
#print timestamp
requested_load = extract_re(lines, r"\s*Requested Load.*= (\d+)")
for l in lines:
m = regexp.match(l)
if m:
- table[m.group(1)] = [float(m.group(i)) for i in range(2, 11)]
+ try:
+ table[m.group(1)] = [float(m.group(i)) for i in range(2, 11)]
+ except:
+ #sys.stderr.write("Error parsing line: " + l.strip() + "\n")
+ pass
outfp.write("%d\t%s\t%s" % (load, results.group(1), results.group(2)))
- for o in ('read', 'write', 'getattr'):
- outfp.write("\t%s\t%s" % (table[o][5], table[o][6]))
+ for o in OPERATIONS:
+ val = '-'
+ try: val = table[o][5]
+ except: pass
+ outfp.write("\t%s" % (val,))
outfp.write("\n")
def parse_sfsres(fp):
+ sys.stdout.write("# target_ops actual_ops latency_avg")
+ for o in OPERATIONS:
+ sys.stdout.write(" " + o)
+ sys.stdout.write("\n")
timestamp = None
run_data = []
for line in fp:
projects / bluesky.git / commitdiff