X-Git-Url: http://git.vrable.net/?a=blobdiff_plain;f=logbench%2Flogbench.c;h=8ab082cc9d91ec52ed8ab56b740707adf2a5f89a;hb=HEAD;hp=722a2ded6b48aa2b65b122afaa5dec03c81045f4;hpb=c7cfbc512b8eb08ead281df1865cb3b241a8070a;p=bluesky.git diff --git a/logbench/logbench.c b/logbench/logbench.c index 722a2de..8ab082c 100644 --- a/logbench/logbench.c +++ b/logbench/logbench.c @@ -1,3 +1,33 @@ +/* Blue Sky: File Systems in the Cloud + * + * Copyright (C) 2010 The Regents of the University of California + * Written by Michael Vrable + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. Neither the name of the University nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + /* A simple tool for benchmarking various logging strategies. * * We want to log a series of key/value pairs. Approaches that we try include: @@ -15,6 +45,7 @@ #include #include #include +#include #include #include #include @@ -33,12 +64,22 @@ int queue_capacity = 1024; int item_size = 1024; int opt_threads = 1; int opt_batchsize = 1; +int opt_writes = (1 << 12); +int opt_bdb_async = FALSE; GAsyncQueue *queue; int outstanding = 0; GMutex *lock; GCond *cond_empty, *cond_full; +int64_t get_ns() +{ + struct timespec ts; + clock_gettime(CLOCK_MONOTONIC, &ts); + + return ts.tv_sec * 1000000000LL + ts.tv_nsec; +} + struct item *get_item() { return (struct item *)g_async_queue_pop(queue); @@ -73,16 +114,14 @@ void writebuf(int fd, const char *buf, size_t len) } /************************ Direct-to-filesystem logging ***********************/ -static int dirfd = -1; +static int dir_fd = -1; gpointer fslog_thread(gpointer d) { - g_print("Launching filesystem writer thread...\n"); - while (TRUE) { struct item *item = get_item(); - int fd = openat(dirfd, item->key, O_CREAT|O_WRONLY|O_TRUNC, 0666); + int fd = openat(dir_fd, item->key, O_CREAT|O_WRONLY|O_TRUNC, 0666); g_assert(fd >= 0); writebuf(fd, item->data, item->len); @@ -90,7 +129,7 @@ gpointer fslog_thread(gpointer d) finish_item(item); fsync(fd); - fsync(dirfd); + fsync(dir_fd); close(fd); } @@ -99,8 +138,8 @@ gpointer fslog_thread(gpointer d) void launch_fslog() { - dirfd = open(".", O_DIRECTORY); - g_assert(dirfd >= 0); + dir_fd = open(".", O_DIRECTORY); + g_assert(dir_fd >= 0); for (int i = 0; i < 1; i++) g_thread_create(fslog_thread, NULL, FALSE, NULL); @@ -109,8 +148,6 @@ void launch_fslog() /****************************** Single-File Log ******************************/ gpointer flatlog_thread(gpointer d) { - g_print("Launching flat log writer thread...\n"); - int fd = open("logfile", O_CREAT|O_WRONLY|O_TRUNC, 0666); g_assert(fd >= 0); @@ -141,8 +178,6 @@ void launch_flatlog() /************************* Transactional Berkeley DB *************************/ gpointer bdb_thread(gpointer d) { - g_print("Launching BDB log writer thread...\n"); - int res; DB_ENV *env; DB *db; @@ -157,6 +192,11 @@ gpointer bdb_thread(gpointer d) | DB_INIT_MPOOL | DB_INIT_TXN | DB_THREAD, 0644); g_assert(res == 0); + if (opt_bdb_async) { + res = env->set_flags(env, DB_TXN_WRITE_NOSYNC, 1); + g_assert(res == 0); + } + res = db_create(&db, env, 0); g_assert(res == 0); @@ -165,7 +205,7 @@ gpointer bdb_thread(gpointer d) g_assert(res == 0); while (TRUE) { - if (txn == NULL) { + if (txn == NULL && !opt_bdb_async) { res = env->txn_begin(env, NULL, &txn, 0); g_assert(res == 0); } @@ -182,13 +222,17 @@ gpointer bdb_thread(gpointer d) value.data = item->data; value.size = item->len; - res = db->put(db, NULL, &key, &value, 0); + res = db->put(db, opt_bdb_async ? NULL : txn, &key, &value, 0); g_assert(res == 0); count++; if (count % opt_batchsize == 0) { - txn->commit(txn, 0); - txn = NULL; + if (opt_bdb_async) { + env->txn_checkpoint(env, 0, 0, 0); + } else { + txn->commit(txn, 0); + txn = NULL; + } } finish_item(item); @@ -204,6 +248,8 @@ void launch_bdb() int main(int argc, char *argv[]) { + int64_t time_start, time_end; + g_thread_init(NULL); queue = g_async_queue_new(); lock = g_mutex_new(); @@ -212,8 +258,12 @@ int main(int argc, char *argv[]) int opt; int backend = 0; - while ((opt = getopt(argc, argv, "t:s:b:BFD")) != -1) { + while ((opt = getopt(argc, argv, "at:s:b:n:BFD")) != -1) { switch (opt) { + case 'a': + // Make BDB log writes more asynchronous + opt_bdb_async = TRUE; + break; case 't': // Set number of log worker threads opt_threads = atoi(optarg); @@ -226,6 +276,10 @@ int main(int argc, char *argv[]) // Set batch size opt_batchsize = atoi(optarg); break; + case 'n': + // Set object count + opt_writes = atoi(optarg); + break; case 'B': // Select BDB backend backend = 'b'; @@ -260,7 +314,8 @@ int main(int argc, char *argv[]) return EXIT_FAILURE; } - for (int i = 0; i < (1 << 12); i++) { + time_start = get_ns(); + for (int i = 0; i < opt_writes; i++) { struct item *item = g_new(struct item, 1); item->key = g_strdup_printf("item-%06d", i); item->data = g_malloc(item_size); @@ -278,6 +333,23 @@ int main(int argc, char *argv[]) while (outstanding > 0) g_cond_wait(cond_empty, lock); g_mutex_unlock(lock); + time_end = get_ns(); + + double elapsed = (time_end - time_start) / 1e9; + printf("Elapsed: %f s\nThroughput: %f txn/s, %f MiB/s\n", + elapsed, opt_writes / elapsed, + opt_writes / elapsed * item_size / (1 << 20)); + + if (backend == 'b' && opt_bdb_async) + backend = 'B'; + + FILE *f = fopen("../logbench.data", "a"); + g_assert(f != NULL); + fprintf(f, "%c\t%d\t%d\t%d\t%f\t%f\t%f\n", + backend, item_size, opt_writes, opt_batchsize, + elapsed, opt_writes / elapsed, + opt_writes / elapsed * item_size / (1 << 20)); + fclose(f); return 0; }