From: Michael Vrable Date: Mon, 21 Feb 2011 04:40:21 +0000 (-0800) Subject: Set up for benchmark runs with small reads/writes within larger files. X-Git-Url: https://git.vrable.net/?a=commitdiff_plain;h=4d584aa004dd09de2cb75091ac8d13432829b3fb;p=bluesky.git Set up for benchmark runs with small reads/writes within larger files. --- diff --git a/microbench/mixedbench.c b/microbench/mixedbench.c index a93d5f9..4755424 100644 --- a/microbench/mixedbench.c +++ b/microbench/mixedbench.c @@ -11,6 +11,7 @@ * Target operations per second (aggregate across all threads) * Interval count (how many times to report results during the run) * Directory size (number of files per numbered subdirectory) + * Read/write block size (0 for the entire file) */ #include @@ -26,7 +27,7 @@ #include #include -int opt_filesize, opt_filecount, opt_threads, opt_duration, opt_intervals, opt_dirsize; +int opt_filesize, opt_filecount, opt_threads, opt_duration, opt_intervals, opt_dirsize, opt_blocksize; double opt_writeratio, opt_ops; int write_threads; @@ -111,6 +112,13 @@ void benchmark_op(struct thread_state *ts) char filename[256]; sprintf(filename, "%d/%d", n1, n2); + /* If a smaller blocksize was requested, choose a random offset within the + * file to use. */ + int offset = 0; + if (opt_blocksize > 0) { + offset = get_random(opt_filesize / opt_blocksize) * opt_blocksize; + } + if (ts->thread_num >= write_threads) { /* Read */ FILE *f = fopen(filename, "rb"); @@ -119,8 +127,18 @@ void benchmark_op(struct thread_state *ts) return; } + if (offset != 0) + fseek(f, offset, SEEK_SET); + char buf[65536]; - while (fread(buf, 1, sizeof(buf), f) > 0) { } + int bytes_left = opt_blocksize > 0 ? opt_blocksize : opt_filesize; + while (bytes_left > 0) { + size_t read = fread(buf, 1, bytes_left < sizeof(buf) + ? bytes_left : sizeof(buf), f); + if (ferror(f)) + return; + bytes_left -= read; + } fclose(f); end = now_hires(); @@ -131,14 +149,17 @@ void benchmark_op(struct thread_state *ts) pthread_mutex_unlock(&ts->lock); } else { /* Write */ - FILE *f = fopen(filename, "wb"); + FILE *f = fopen(filename, "r+b"); if (f == NULL) { fprintf(stderr, "fopen(%s): %m\n", filename); return; } + if (offset != 0) + fseek(f, offset, SEEK_SET); + char buf[65536]; - int bytes_left = opt_filesize; + int bytes_left = opt_blocksize > 0 ? opt_blocksize : opt_filesize; while (bytes_left > 0) { size_t written = fwrite(buf, 1, bytes_left < sizeof(buf) @@ -228,7 +249,7 @@ void reset_stats(int print, double duration) int main(int argc, char *argv[]) { - if (argc != 9) { + if (argc != 10) { fprintf(stderr, "Usage: TODO\n"); return 1; } @@ -241,6 +262,7 @@ int main(int argc, char *argv[]) opt_ops = atof(argv[6]); opt_intervals = atoi(argv[7]); opt_dirsize = atoi(argv[8]); + opt_blocksize = atoi(argv[9]); srandom(time(NULL)); diff --git a/microbench/run2.sh b/microbench/run2.sh index dd801c0..12d946f 100755 --- a/microbench/run2.sh +++ b/microbench/run2.sh @@ -1,6 +1,6 @@ #!/bin/bash -PARAMS="BLUESKY_CACHE_SIZE BLUESKY_TARGET BENCH_DURATION BENCH_FILESIZE BENCH_FILECOUNT BENCH_WRITERATIO BENCH_THREADS BENCH_OPS BENCH_INTERVALS BENCH_DIRSIZE" +PARAMS="BLUESKY_CACHE_SIZE BLUESKY_TARGET BENCH_DURATION BENCH_FILESIZE BENCH_FILECOUNT BENCH_WRITERATIO BENCH_THREADS BENCH_OPS BENCH_INTERVALS BENCH_DIRSIZE BENCH_BLOCKSIZE" BENCHER=c09-44.sysnet.ucsd.edu PROXY=c09-45.sysnet.ucsd.edu @@ -22,23 +22,25 @@ run_cmd() { ssh $SSH_ARGS -l root $host /scratch/bluesky.git/microbench/setup/$cmd "$@" $(get_params) } -BLUESKY_TARGET=s3 BLUESKY_CACHE_SIZE=$((512 * 1024)) BENCH_FILESIZE=$((1 << 20)) +BENCH_BLOCKSIZE=0 BENCH_FILECOUNT=$((62)) -BENCH_WRITERATIO=0.0 +BENCH_WRITERATIO=0.5 BENCH_THREADS=4 BENCH_DURATION=120 BENCH_INTERVALS=10 BENCH_DIRSIZE=128 +sizes="128 512 1024 2048" + #for size in 128 256 512 768 1024 1536 2048; do # for filesize in 4 32 256 2048; do run_experiments() { -for size in 128 512 1024 2048; do - for filesize in 1024; do - BENCH_FILESIZE=$(($filesize * 1024)) - BENCH_FILECOUNT=$(($size * 1024 / $filesize)) +for size in $sizes; do + for filesize in 4 32; do + BENCH_BLOCKSIZE=$(($filesize * 1024)) + BENCH_FILECOUNT=$(($size * 1024 / ($BENCH_FILESIZE / 1024))) PREFIX=$BLUESKY_TARGET-$BENCH_WRITERATIO-${size}M-${filesize}k $HOME/bin/s3-cleanup.py @@ -65,13 +67,9 @@ for size in 128 512 1024 2048; do done } -BLUESKY_TARGET=native -run_experiments - -BENCH_WRITERATIO=1.0 +BLUESKY_TARGET=s3 run_experiments -ssh -l root $PROXY pkill -TERM lockmem - -BLUESKY_TARGET=s3 +BLUESKY_TARGET=native +sizes=128 run_experiments diff --git a/microbench/setup/run-benchmark b/microbench/setup/run-benchmark index 80bd635..6eb40ff 100755 --- a/microbench/setup/run-benchmark +++ b/microbench/setup/run-benchmark @@ -8,6 +8,7 @@ SERVER=c09-45.sysnet.ucsd.edu MNTDIR=/mnt/bluesky BENCH_FILESIZE=$((1 << 12)) +BENCH_BLOCKSIZE=0 BENCH_FILECOUNT=$((8)) BENCH_WRITERATIO=1.0 BENCH_THREADS=8 @@ -52,7 +53,8 @@ pushd $MNTDIR >/dev/null $BENCH_DURATION \ $BENCH_OPS \ $BENCH_INTERVALS \ - $BENCH_DIRSIZE + $BENCH_DIRSIZE \ + $BENCH_BLOCKSIZE pkill -TERM lockmem