Some test work with using Berkeley DB for a local disk cache.

Implement this at first by simply making it a new storage backend.  Later
we will have to make it a separate layer so we can stack a Berkeley DB
cache/write log with a remote storage option.

diff --git a/bluesky/CMakeLists.txt b/bluesky/CMakeLists.txt
index 1a6353a..70ebf30 100644 (file)
--- a/bluesky/CMakeLists.txt
+++ b/bluesky/CMakeLists.txt
@@ -4,7 +4,7 @@ link_directories("${LIBS3_BUILD_DIR}/lib" ${KVSTORE_DIR})
 
 add_library(bluesky SHARED
             cache.c crypto.c debug.c dir.c file.c init.c inode.c serialize.c
-            store.c store-kv.cc store-multi.c store-s3.c util.c)
+            store.c store-bdb.c store-kv.cc store-multi.c store-s3.c util.c)
 add_executable(bluesky-test main.c)
 
 set(CMAKE_C_FLAGS "-Wall -std=gnu99 ${CMAKE_C_FLAGS}")

diff --git a/bluesky/init.c b/bluesky/init.c
index 2171030..1db4283 100644 (file)
--- a/bluesky/init.c
+++ b/bluesky/init.c
@@ -55,6 +55,7 @@ static struct {
 void bluesky_store_init_s3(void);
 void bluesky_store_init_kv(void);
 void bluesky_store_init_multi(void);
+void bluesky_store_init_bdb(void);
 
 /* Initialize the BlueSky library and dependent libraries. */
 void bluesky_init(void)
@@ -75,4 +76,5 @@ void bluesky_init(void)
     bluesky_store_init_kv();
     bluesky_store_init_s3();
     bluesky_store_init_multi();
+    bluesky_store_init_bdb();
 }

diff --git a/bluesky/store-bdb.c b/bluesky/store-bdb.c
new file mode 100644 (file)
index 0000000..8565314
--- /dev/null
+++ b/bluesky/store-bdb.c
@@ -0,0 +1,178 @@
+/* Blue Sky: File Systems in the Cloud
+ *
+ * Copyright (C) 2009  The Regents of the University of California
+ * Written by Michael Vrable <mvrable@cs.ucsd.edu>
+ *
+ * TODO: Licensing
+ */
+
+#include <stdint.h>
+#include <stdlib.h>
+#include <glib.h>
+#include <string.h>
+#include <db.h>
+#include <errno.h>
+
+#include "bluesky-private.h"
+#include "libs3.h"
+
+/* A storage layer that writes to Berkeley DB locally. */
+
+typedef struct {
+    GThreadPool *thread_pool;
+    DB_ENV *env;
+    DB *db;
+} BDBStore;
+
+static void bdbstore_task(gpointer a, gpointer s)
+{
+    int res;
+    BlueSkyStoreAsync *async = (BlueSkyStoreAsync *)a;
+    BDBStore *store = (BDBStore *)s;
+
+    async->status = ASYNC_RUNNING;
+    async->exec_time = bluesky_now_hires();
+
+    DBT key;
+    memset(&key, 0, sizeof(key));
+
+    key.data = async->key;
+    key.size = strlen(async->key);
+
+    DBT value;
+    memset(&value, 0, sizeof(value));
+
+    if (async->op == STORE_OP_GET) {
+        value.flags = DB_DBT_MALLOC;
+
+        res = store->db->get(store->db, NULL, &key, &value, 0);
+
+        async->result = res;
+        async->data = NULL;
+
+        if (res != 0) {
+            fprintf(stderr, "BDB read failure: %s\n", db_strerror(res));
+        } else {
+            async->data = bluesky_string_new(value.data, value.size);
+            async->result = 0;
+        }
+
+    } else if (async->op == STORE_OP_PUT) {
+        value.data = async->data->data;
+        value.size = async->data->len;
+
+        res = store->db->put(store->db, NULL, &key, &value, 0);
+
+        if (res != 0) {
+            fprintf(stderr, "BDB write failure: %s\n", db_strerror(res));
+        }
+
+        async->result = 0;
+    }
+
+    bluesky_store_async_mark_complete(async);
+    bluesky_store_async_unref(async);
+}
+
+static gpointer bdbstore_new(const gchar *path)
+{
+    int res;
+    BDBStore *store = g_new0(BDBStore, 1);
+    store->thread_pool = g_thread_pool_new(bdbstore_task, store, 16, FALSE,
+                                           NULL);
+
+    res = db_env_create(&store->env, 0);
+
+    if (res != 0) {
+        fprintf(stderr, "db_env_create failure: %s\n", db_strerror(res));
+        return NULL;
+    }
+
+    res = store->env->open(store->env, path,
+                           DB_CREATE | DB_RECOVER | DB_INIT_LOCK | DB_INIT_LOG
+                            | DB_INIT_MPOOL | DB_INIT_TXN | DB_THREAD,
+                           0644);
+
+    if (res != 0) {
+        fprintf(stderr, "BDB open failure: %s\n",
+                db_strerror(res));
+        return NULL;
+    }
+
+    res = db_create(&store->db, store->env, 0);
+
+    if (res != 0) {
+        fprintf(stderr, "DB create failed: %s\n", db_strerror(res));
+        return NULL;
+    }
+
+    uint32_t flags = DB_CREATE | DB_THREAD | DB_AUTO_COMMIT;
+
+    res = store->db->open(store->db,
+                          NULL, /* TXN */
+                          "store.db",
+                          "store",
+                          DB_BTREE,
+                          flags,
+                          0644);
+
+    if (res != 0) {
+        fprintf(stderr, "DB open failed: %s\n",
+                db_strerror(res));
+    }
+
+    return store;
+}
+
+static void bdbstore_destroy(gpointer s)
+{
+    BDBStore *store = (BDBStore *)store;
+
+    if (store->db) {
+        store->db->close(store->db, 0);
+    }
+
+    if (store->env) {
+        store->env->close(store->env, 0);
+    }
+
+    g_free(store);
+}
+
+static void bdbstore_submit(gpointer s, BlueSkyStoreAsync *async)
+{
+    BDBStore *store = (BDBStore *)s;
+    g_return_if_fail(async->status == ASYNC_NEW);
+    g_return_if_fail(async->op != STORE_OP_NONE);
+
+    switch (async->op) {
+    case STORE_OP_GET:
+    case STORE_OP_PUT:
+        async->status = ASYNC_PENDING;
+        bluesky_store_async_ref(async);
+        g_thread_pool_push(store->thread_pool, async, NULL);
+        break;
+
+    default:
+        g_warning("Uknown operation type for BDBStore: %d\n", async->op);
+        bluesky_store_async_mark_complete(async);
+        break;
+    }
+}
+
+static void bdbstore_cleanup(gpointer store, BlueSkyStoreAsync *async)
+{
+}
+
+static BlueSkyStoreImplementation store_impl = {
+    .create = bdbstore_new,
+    .destroy = bdbstore_destroy,
+    .submit = bdbstore_submit,
+    .cleanup = bdbstore_cleanup,
+};
+
+void bluesky_store_init_bdb(void)
+{
+    bluesky_store_register(&store_impl, "bdb");
+}
+