From 6e95f90cc53d2a230b9cf22d3b5906a66b4e71de Mon Sep 17 00:00:00 2001 From: Michael Vrable Date: Fri, 4 Jun 2010 10:47:25 -0700 Subject: [PATCH] Add a "multi" storage backend which doubles all GET requests. This can be used to test whether the performance with S3 improves when making parallel requests to reduce latency. --- bluesky/CMakeLists.txt | 2 +- bluesky/init.c | 2 + bluesky/store-multi.c | 124 +++++++++++++++++++++++++++++++++++++++++ 3 files changed, 127 insertions(+), 1 deletion(-) create mode 100644 bluesky/store-multi.c diff --git a/bluesky/CMakeLists.txt b/bluesky/CMakeLists.txt index 2dfe8ef..1a6353a 100644 --- a/bluesky/CMakeLists.txt +++ b/bluesky/CMakeLists.txt @@ -4,7 +4,7 @@ link_directories("${LIBS3_BUILD_DIR}/lib" ${KVSTORE_DIR}) add_library(bluesky SHARED cache.c crypto.c debug.c dir.c file.c init.c inode.c serialize.c - store.c store-kv.cc store-s3.c util.c) + store.c store-kv.cc store-multi.c store-s3.c util.c) add_executable(bluesky-test main.c) set(CMAKE_C_FLAGS "-Wall -std=gnu99 ${CMAKE_C_FLAGS}") diff --git a/bluesky/init.c b/bluesky/init.c index fee9aad..2171030 100644 --- a/bluesky/init.c +++ b/bluesky/init.c @@ -54,6 +54,7 @@ static struct { void bluesky_store_init_s3(void); void bluesky_store_init_kv(void); +void bluesky_store_init_multi(void); /* Initialize the BlueSky library and dependent libraries. */ void bluesky_init(void) @@ -73,4 +74,5 @@ void bluesky_init(void) bluesky_store_init(); bluesky_store_init_kv(); bluesky_store_init_s3(); + bluesky_store_init_multi(); } diff --git a/bluesky/store-multi.c b/bluesky/store-multi.c new file mode 100644 index 0000000..f0b1e5e --- /dev/null +++ b/bluesky/store-multi.c @@ -0,0 +1,124 @@ +/* Blue Sky: File Systems in the Cloud + * + * Copyright (C) 2009 The Regents of the University of California + * Written by Michael Vrable + * + * TODO: Licensing + */ + +/* A stacked storage implementation which tries to improve performance by + * duplicating GET requests. Create using a name like "multi:s3" and each GET + * request will be translated into two GET requests to the "s3" backend. The + * first to complete will have its results returned. */ + +#include +#include +#include +#include + +#include "bluesky-private.h" + +struct MultiRequest { +}; + +static gpointer multistore_new(const gchar *path) +{ + BlueSkyStore *base = bluesky_store_new(path); + if (base == NULL) { + g_warning("Unable to create base store %s for multirequest stacking.", + path); + } + + return base; +} + +static void multistore_destroy(gpointer store) +{ + bluesky_store_free(store); +} + +static void multistore_completion_handler(BlueSkyStoreAsync *async, + BlueSkyStoreAsync *top_async) +{ + g_mutex_lock(top_async->lock); + + g_print("Completed a subrequest for %s\n", async->key); + + /* This might be the second request to finish; in that case we don't do + * anything. */ + if (top_async->status == ASYNC_RUNNING) { + if (top_async->op == STORE_OP_GET) { + bluesky_string_unref(top_async->data); + top_async->data = async->data; + bluesky_string_ref(top_async->data); + } + top_async->result = async->result; + bluesky_store_async_mark_complete(top_async); + } + + g_mutex_unlock(top_async->lock); + bluesky_store_async_unref(top_async); +} + +static void multistore_submit(gpointer store, BlueSkyStoreAsync *async) +{ + BlueSkyStore *base = (BlueSkyStore *)store; + + g_return_if_fail(async->status == ASYNC_NEW); + g_return_if_fail(async->op != STORE_OP_NONE); + + switch (async->op) { + case STORE_OP_GET: + g_print("Received multirequest for %s\n", async->key); + async->status = ASYNC_RUNNING; + async->exec_time = bluesky_now_hires(); + for (int i = 0; i < 2; i++) { + BlueSkyStoreAsync *a = bluesky_store_async_new(base); + a->op = STORE_OP_GET; + a->key = g_strdup(async->key); + bluesky_store_async_ref(async); + bluesky_store_async_add_notifier(a, (GFunc)multistore_completion_handler, async); + bluesky_store_async_submit(a); + bluesky_store_async_unref(a); + } + break; + + case STORE_OP_PUT: + { + async->status = ASYNC_RUNNING; + async->exec_time = bluesky_now_hires(); + + bluesky_store_async_ref(async); + BlueSkyStoreAsync *a = bluesky_store_async_new(base); + a->op = STORE_OP_PUT; + a->key = g_strdup(async->key); + a->data = async->data; + bluesky_string_ref(a->data); + bluesky_store_async_add_notifier(a, (GFunc)multistore_completion_handler, async); + bluesky_store_async_submit(a); + bluesky_store_async_unref(a); + break; + } + + default: + g_warning("Uknown operation type for multistore: %d\n", async->op); + bluesky_store_async_mark_complete(async); + break; + } +} + +static void multistore_cleanup(gpointer store, BlueSkyStoreAsync *async) +{ +} + +static BlueSkyStoreImplementation store_impl = { + .create = multistore_new, + .destroy = multistore_destroy, + .submit = multistore_submit, + .cleanup = multistore_cleanup, +}; + +void bluesky_store_init_multi(void) +{ + bluesky_store_register(&store_impl, "multi"); +} -- 2.20.1