From: Michael Vrable Date: Tue, 16 Feb 2010 22:48:12 +0000 (-0800) Subject: Link kvstore into main build system. X-Git-Url: https://git.vrable.net/?a=commitdiff_plain;h=544dad3ef87acd87be934b09e861baa5c2406686;p=bluesky.git Link kvstore into main build system. --- diff --git a/CMakeLists.txt b/CMakeLists.txt index f7c573d..50bf118 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -8,6 +8,7 @@ pkg_check_modules(GLIB REQUIRED glib-2.0 gthread-2.0) set(LIBS3_BUILD_DIR "${CMAKE_SOURCE_DIR}/libs3-1.4/build") add_definitions(-D_FILE_OFFSET_BITS=64) +add_subdirectory(kvstore) add_subdirectory(bluesky) add_subdirectory(nfs3) add_subdirectory(microbench) diff --git a/bluesky/CMakeLists.txt b/bluesky/CMakeLists.txt index 93b7cb9..2dfe8ef 100644 --- a/bluesky/CMakeLists.txt +++ b/bluesky/CMakeLists.txt @@ -4,7 +4,7 @@ link_directories("${LIBS3_BUILD_DIR}/lib" ${KVSTORE_DIR}) add_library(bluesky SHARED cache.c crypto.c debug.c dir.c file.c init.c inode.c serialize.c - store.c store-kv.cc s3store.c util.c) + store.c store-kv.cc store-s3.c util.c) add_executable(bluesky-test main.c) set(CMAKE_C_FLAGS "-Wall -std=gnu99 ${CMAKE_C_FLAGS}") diff --git a/bluesky/s3store.c b/bluesky/s3store.c deleted file mode 100644 index ccf0878..0000000 --- a/bluesky/s3store.c +++ /dev/null @@ -1,209 +0,0 @@ -/* Blue Sky: File Systems in the Cloud - * - * Copyright (C) 2009 The Regents of the University of California - * Written by Michael Vrable - * - * TODO: Licensing - */ - -#include -#include -#include -#include - -#include "bluesky-private.h" -#include "libs3.h" - -/* Interface to Amazon S3 storage. */ - -typedef struct { - GThreadPool *thread_pool; - S3BucketContext bucket; - uint8_t encryption_key[CRYPTO_KEY_SIZE]; -} S3Store; - -typedef struct { - enum { S3_GET, S3_PUT } op; - gchar *key; - BlueSkyRCStr *data; -} S3Op; - -struct get_info { - int success; - GString *buf; -}; - -struct put_info { - BlueSkyRCStr *val; - gint offset; -}; - -static S3Status s3store_get_handler(int bufferSize, const char *buffer, - void *callbackData) -{ - struct get_info *info = (struct get_info *)callbackData; - g_string_append_len(info->buf, buffer, bufferSize); - return S3StatusOK; -} - -static int s3store_put_handler(int bufferSize, char *buffer, - void *callbackData) -{ - struct put_info *info = (struct put_info *)callbackData; - gint bytes = MIN(bufferSize, (int)(info->val->len - info->offset)); - memcpy(buffer, (char *)info->val->data + info->offset, bytes); - info->offset += bytes; - return bytes; -} - -static S3Status s3store_properties_callback(const S3ResponseProperties *properties, - void *callbackData) -{ - return S3StatusOK; -} - -static void s3store_response_callback(S3Status status, - const S3ErrorDetails *errorDetails, - void *callbackData) -{ - struct get_info *info = (struct get_info *)callbackData; - - if (status == 0) { - info->success = 1; - } - - if (errorDetails != NULL && errorDetails->message != NULL) { - g_print(" Error message: %s\n", errorDetails->message); - } -} - -static void s3store_task(gpointer a, gpointer s) -{ - BlueSkyStoreAsync *async = (BlueSkyStoreAsync *)a; - S3Store *store = (S3Store *)s; - - async->status = ASYNC_RUNNING; - - if (async->op == STORE_OP_GET) { - struct get_info info; - info.buf = g_string_new(""); - info.success = 0; - - struct S3GetObjectHandler handler; - handler.responseHandler.propertiesCallback = s3store_properties_callback; - handler.responseHandler.completeCallback = s3store_response_callback; - handler.getObjectDataCallback = s3store_get_handler; - - S3_get_object(&store->bucket, async->key, NULL, 0, 0, NULL, - &handler, &info); - - if (info.success) { - BlueSkyRCStr *raw, *decrypted; - raw = bluesky_string_new_from_gstring(info.buf); - decrypted = bluesky_crypt_decrypt(raw, store->encryption_key); - bluesky_string_unref(raw); - async->data = decrypted; - async->result = 0; - } else { - g_string_free(info.buf, TRUE); - } - - } else if (async->op == STORE_OP_PUT) { - BlueSkyRCStr *encrypted = bluesky_crypt_encrypt(async->data, - store->encryption_key); - - struct put_info info; - info.val = encrypted; - info.offset = 0; - - struct S3PutObjectHandler handler; - handler.responseHandler.propertiesCallback - = s3store_properties_callback; - handler.responseHandler.completeCallback = s3store_response_callback; - handler.putObjectDataCallback = s3store_put_handler; - - S3_put_object(&store->bucket, async->key, encrypted->len, NULL, NULL, - &handler, &info); - - bluesky_string_unref(encrypted); - - async->result = 0; - } - - bluesky_store_async_mark_complete(async); - bluesky_store_async_unref(async); -} - -static gpointer s3store_new() -{ - S3Store *store = g_new(S3Store, 1); - store->thread_pool = g_thread_pool_new(s3store_task, store, -1, FALSE, - NULL); - store->bucket.bucketName = "mvrable-bluesky"; - store->bucket.protocol = S3ProtocolHTTP; - store->bucket.uriStyle = S3UriStylePath; - store->bucket.accessKeyId = getenv("AWS_ACCESS_KEY_ID"); - store->bucket.secretAccessKey = getenv("AWS_SECRET_ACCESS_KEY"); - - const char *key = getenv("BLUESKY_KEY"); - if (key == NULL) { - g_error("Encryption key not defined; please set BLUESKY_KEY environment variable"); - exit(1); - } - - bluesky_crypt_hash_key(key, store->encryption_key); - - g_print("Initializing S3 with bucket %s, access key %s, encryption key %s\n", - store->bucket.bucketName, store->bucket.accessKeyId, key); - - return store; -} - -static void s3store_destroy(gpointer store) -{ - g_free(store); -} - -static void s3store_submit(gpointer s, BlueSkyStoreAsync *async) -{ - S3Store *store = (S3Store *)s; - g_return_if_fail(async->status == ASYNC_NEW); - g_return_if_fail(async->op != STORE_OP_NONE); - - switch (async->op) { - case STORE_OP_GET: - case STORE_OP_PUT: - async->status = ASYNC_PENDING; - bluesky_store_async_ref(async); - g_thread_pool_push(store->thread_pool, async, NULL); - break; - - default: - g_warning("Uknown operation type for S3Store: %d\n", async->op); - bluesky_store_async_mark_complete(async); - break; - } -} - -static void s3store_cleanup(gpointer store, BlueSkyStoreAsync *async) -{ - GString *buf = (GString *)async->store_private; - - if (buf != NULL) { - g_string_free(buf, TRUE); - async->store_private = NULL; - } -} - -static BlueSkyStoreImplementation store_impl = { - .create = s3store_new, - .destroy = s3store_destroy, - .submit = s3store_submit, - .cleanup = s3store_cleanup, -}; - -void bluesky_store_init_s3(void) -{ - S3_initialize(NULL, S3_INIT_ALL); - bluesky_store_register(&store_impl, "s3"); -} diff --git a/bluesky/store-s3.c b/bluesky/store-s3.c new file mode 100644 index 0000000..ccf0878 --- /dev/null +++ b/bluesky/store-s3.c @@ -0,0 +1,209 @@ +/* Blue Sky: File Systems in the Cloud + * + * Copyright (C) 2009 The Regents of the University of California + * Written by Michael Vrable + * + * TODO: Licensing + */ + +#include +#include +#include +#include + +#include "bluesky-private.h" +#include "libs3.h" + +/* Interface to Amazon S3 storage. */ + +typedef struct { + GThreadPool *thread_pool; + S3BucketContext bucket; + uint8_t encryption_key[CRYPTO_KEY_SIZE]; +} S3Store; + +typedef struct { + enum { S3_GET, S3_PUT } op; + gchar *key; + BlueSkyRCStr *data; +} S3Op; + +struct get_info { + int success; + GString *buf; +}; + +struct put_info { + BlueSkyRCStr *val; + gint offset; +}; + +static S3Status s3store_get_handler(int bufferSize, const char *buffer, + void *callbackData) +{ + struct get_info *info = (struct get_info *)callbackData; + g_string_append_len(info->buf, buffer, bufferSize); + return S3StatusOK; +} + +static int s3store_put_handler(int bufferSize, char *buffer, + void *callbackData) +{ + struct put_info *info = (struct put_info *)callbackData; + gint bytes = MIN(bufferSize, (int)(info->val->len - info->offset)); + memcpy(buffer, (char *)info->val->data + info->offset, bytes); + info->offset += bytes; + return bytes; +} + +static S3Status s3store_properties_callback(const S3ResponseProperties *properties, + void *callbackData) +{ + return S3StatusOK; +} + +static void s3store_response_callback(S3Status status, + const S3ErrorDetails *errorDetails, + void *callbackData) +{ + struct get_info *info = (struct get_info *)callbackData; + + if (status == 0) { + info->success = 1; + } + + if (errorDetails != NULL && errorDetails->message != NULL) { + g_print(" Error message: %s\n", errorDetails->message); + } +} + +static void s3store_task(gpointer a, gpointer s) +{ + BlueSkyStoreAsync *async = (BlueSkyStoreAsync *)a; + S3Store *store = (S3Store *)s; + + async->status = ASYNC_RUNNING; + + if (async->op == STORE_OP_GET) { + struct get_info info; + info.buf = g_string_new(""); + info.success = 0; + + struct S3GetObjectHandler handler; + handler.responseHandler.propertiesCallback = s3store_properties_callback; + handler.responseHandler.completeCallback = s3store_response_callback; + handler.getObjectDataCallback = s3store_get_handler; + + S3_get_object(&store->bucket, async->key, NULL, 0, 0, NULL, + &handler, &info); + + if (info.success) { + BlueSkyRCStr *raw, *decrypted; + raw = bluesky_string_new_from_gstring(info.buf); + decrypted = bluesky_crypt_decrypt(raw, store->encryption_key); + bluesky_string_unref(raw); + async->data = decrypted; + async->result = 0; + } else { + g_string_free(info.buf, TRUE); + } + + } else if (async->op == STORE_OP_PUT) { + BlueSkyRCStr *encrypted = bluesky_crypt_encrypt(async->data, + store->encryption_key); + + struct put_info info; + info.val = encrypted; + info.offset = 0; + + struct S3PutObjectHandler handler; + handler.responseHandler.propertiesCallback + = s3store_properties_callback; + handler.responseHandler.completeCallback = s3store_response_callback; + handler.putObjectDataCallback = s3store_put_handler; + + S3_put_object(&store->bucket, async->key, encrypted->len, NULL, NULL, + &handler, &info); + + bluesky_string_unref(encrypted); + + async->result = 0; + } + + bluesky_store_async_mark_complete(async); + bluesky_store_async_unref(async); +} + +static gpointer s3store_new() +{ + S3Store *store = g_new(S3Store, 1); + store->thread_pool = g_thread_pool_new(s3store_task, store, -1, FALSE, + NULL); + store->bucket.bucketName = "mvrable-bluesky"; + store->bucket.protocol = S3ProtocolHTTP; + store->bucket.uriStyle = S3UriStylePath; + store->bucket.accessKeyId = getenv("AWS_ACCESS_KEY_ID"); + store->bucket.secretAccessKey = getenv("AWS_SECRET_ACCESS_KEY"); + + const char *key = getenv("BLUESKY_KEY"); + if (key == NULL) { + g_error("Encryption key not defined; please set BLUESKY_KEY environment variable"); + exit(1); + } + + bluesky_crypt_hash_key(key, store->encryption_key); + + g_print("Initializing S3 with bucket %s, access key %s, encryption key %s\n", + store->bucket.bucketName, store->bucket.accessKeyId, key); + + return store; +} + +static void s3store_destroy(gpointer store) +{ + g_free(store); +} + +static void s3store_submit(gpointer s, BlueSkyStoreAsync *async) +{ + S3Store *store = (S3Store *)s; + g_return_if_fail(async->status == ASYNC_NEW); + g_return_if_fail(async->op != STORE_OP_NONE); + + switch (async->op) { + case STORE_OP_GET: + case STORE_OP_PUT: + async->status = ASYNC_PENDING; + bluesky_store_async_ref(async); + g_thread_pool_push(store->thread_pool, async, NULL); + break; + + default: + g_warning("Uknown operation type for S3Store: %d\n", async->op); + bluesky_store_async_mark_complete(async); + break; + } +} + +static void s3store_cleanup(gpointer store, BlueSkyStoreAsync *async) +{ + GString *buf = (GString *)async->store_private; + + if (buf != NULL) { + g_string_free(buf, TRUE); + async->store_private = NULL; + } +} + +static BlueSkyStoreImplementation store_impl = { + .create = s3store_new, + .destroy = s3store_destroy, + .submit = s3store_submit, + .cleanup = s3store_cleanup, +}; + +void bluesky_store_init_s3(void) +{ + S3_initialize(NULL, S3_INIT_ALL); + bluesky_store_register(&store_impl, "s3"); +}