Set up for benchmark runs with small reads/writes within larger files.

diff --git a/microbench/mixedbench.c b/microbench/mixedbench.c
index a93d5f9..4755424 100644 (file)
--- a/microbench/mixedbench.c
+++ b/microbench/mixedbench.c
@@ -11,6 +11,7 @@
  *   Target operations per second (aggregate across all threads)
  *   Interval count (how many times to report results during the run)
  *   Directory size (number of files per numbered subdirectory)
+ *   Read/write block size (0 for the entire file)
  */
 
 #include <errno.h>
@@ -26,7 +27,7 @@
 #include <unistd.h>
 #include <math.h>
 
-int opt_filesize, opt_filecount, opt_threads, opt_duration, opt_intervals, opt_dirsize;
+int opt_filesize, opt_filecount, opt_threads, opt_duration, opt_intervals, opt_dirsize, opt_blocksize;
 double opt_writeratio, opt_ops;
 
 int write_threads;
@@ -111,6 +112,13 @@ void benchmark_op(struct thread_state *ts)
     char filename[256];
     sprintf(filename, "%d/%d", n1, n2);
 
+    /* If a smaller blocksize was requested, choose a random offset within the
+     * file to use. */
+    int offset = 0;
+    if (opt_blocksize > 0) {
+        offset = get_random(opt_filesize / opt_blocksize) * opt_blocksize;
+    }
+
     if (ts->thread_num >= write_threads) {
         /* Read */
         FILE *f = fopen(filename, "rb");
@@ -119,8 +127,18 @@ void benchmark_op(struct thread_state *ts)
             return;
         }
 
+        if (offset != 0)
+            fseek(f, offset, SEEK_SET);
+
         char buf[65536];
-        while (fread(buf, 1, sizeof(buf), f) > 0) { }
+        int bytes_left = opt_blocksize > 0 ? opt_blocksize : opt_filesize;
+        while (bytes_left > 0) {
+            size_t read = fread(buf, 1, bytes_left < sizeof(buf)
+                                         ? bytes_left : sizeof(buf), f);
+            if (ferror(f))
+                return;
+            bytes_left -= read;
+        }
         fclose(f);
 
         end = now_hires();
@@ -131,14 +149,17 @@ void benchmark_op(struct thread_state *ts)
         pthread_mutex_unlock(&ts->lock);
     } else {
         /* Write */
-        FILE *f = fopen(filename, "wb");
+        FILE *f = fopen(filename, "r+b");
         if (f == NULL) {
             fprintf(stderr, "fopen(%s): %m\n", filename);
             return;
         }
 
+        if (offset != 0)
+            fseek(f, offset, SEEK_SET);
+
         char buf[65536];
-        int bytes_left = opt_filesize;
+        int bytes_left = opt_blocksize > 0 ? opt_blocksize : opt_filesize;
         while (bytes_left > 0) {
             size_t written = fwrite(buf, 1,
                                     bytes_left < sizeof(buf)
@@ -228,7 +249,7 @@ void reset_stats(int print, double duration)
 
 int main(int argc, char *argv[])
 {
-    if (argc != 9) {
+    if (argc != 10) {
         fprintf(stderr, "Usage: TODO\n");
         return 1;
     }
@@ -241,6 +262,7 @@ int main(int argc, char *argv[])
     opt_ops = atof(argv[6]);
     opt_intervals = atoi(argv[7]);
     opt_dirsize = atoi(argv[8]);
+    opt_blocksize = atoi(argv[9]);
 
     srandom(time(NULL));
 

diff --git a/microbench/run2.sh b/microbench/run2.sh
index dd801c0..12d946f 100755 (executable)
--- a/microbench/run2.sh
+++ b/microbench/run2.sh
@@ -1,6 +1,6 @@
 #!/bin/bash
 
-PARAMS="BLUESKY_CACHE_SIZE BLUESKY_TARGET BENCH_DURATION BENCH_FILESIZE BENCH_FILECOUNT BENCH_WRITERATIO BENCH_THREADS BENCH_OPS BENCH_INTERVALS BENCH_DIRSIZE"
+PARAMS="BLUESKY_CACHE_SIZE BLUESKY_TARGET BENCH_DURATION BENCH_FILESIZE BENCH_FILECOUNT BENCH_WRITERATIO BENCH_THREADS BENCH_OPS BENCH_INTERVALS BENCH_DIRSIZE BENCH_BLOCKSIZE"
 
 BENCHER=c09-44.sysnet.ucsd.edu
 PROXY=c09-45.sysnet.ucsd.edu
@@ -22,23 +22,25 @@ run_cmd() {
     ssh $SSH_ARGS -l root $host /scratch/bluesky.git/microbench/setup/$cmd "$@" $(get_params)
 }
 
-BLUESKY_TARGET=s3
 BLUESKY_CACHE_SIZE=$((512 * 1024))
 BENCH_FILESIZE=$((1 << 20))
+BENCH_BLOCKSIZE=0
 BENCH_FILECOUNT=$((62))
-BENCH_WRITERATIO=0.0
+BENCH_WRITERATIO=0.5
 BENCH_THREADS=4
 BENCH_DURATION=120
 BENCH_INTERVALS=10
 BENCH_DIRSIZE=128
 
+sizes="128 512 1024 2048"
+
 #for size in 128 256 512 768 1024 1536 2048; do
 #    for filesize in 4 32 256 2048; do
 run_experiments() {
-for size in 128 512 1024 2048; do
-    for filesize in 1024; do
-        BENCH_FILESIZE=$(($filesize * 1024))
-        BENCH_FILECOUNT=$(($size * 1024 / $filesize))
+for size in $sizes; do
+    for filesize in 4 32; do
+        BENCH_BLOCKSIZE=$(($filesize * 1024))
+        BENCH_FILECOUNT=$(($size * 1024 / ($BENCH_FILESIZE / 1024)))
         PREFIX=$BLUESKY_TARGET-$BENCH_WRITERATIO-${size}M-${filesize}k
 
         $HOME/bin/s3-cleanup.py
@@ -65,13 +67,9 @@ for size in 128 512 1024 2048; do
 done
 }
 
-BLUESKY_TARGET=native
-run_experiments
-
-BENCH_WRITERATIO=1.0
+BLUESKY_TARGET=s3
 run_experiments
 
-ssh -l root $PROXY pkill -TERM lockmem
-
-BLUESKY_TARGET=s3
+BLUESKY_TARGET=native
+sizes=128
 run_experiments

diff --git a/microbench/setup/run-benchmark b/microbench/setup/run-benchmark
index 80bd635..6eb40ff 100755 (executable)
--- a/microbench/setup/run-benchmark
+++ b/microbench/setup/run-benchmark
@@ -8,6 +8,7 @@ SERVER=c09-45.sysnet.ucsd.edu
 MNTDIR=/mnt/bluesky
 
 BENCH_FILESIZE=$((1 << 12))
+BENCH_BLOCKSIZE=0
 BENCH_FILECOUNT=$((8))
 BENCH_WRITERATIO=1.0
 BENCH_THREADS=8
@@ -52,7 +53,8 @@ pushd $MNTDIR >/dev/null
     $BENCH_DURATION \
     $BENCH_OPS \
     $BENCH_INTERVALS \
-    $BENCH_DIRSIZE
+    $BENCH_DIRSIZE \
+    $BENCH_BLOCKSIZE
 
 pkill -TERM lockmem