Implement flushing of file blocks to Amazon S3.

Right now many details are hardcoded.  Data can't be read back yet.  But it
implements some of the needed functionality.

diff --git a/Makefile b/Makefile
index 8d986f4..fa9d6f6 100644 (file)
--- a/Makefile
+++ b/Makefile
@@ -3,11 +3,13 @@ PACKAGES=glib-2.0 gthread-2.0
 DEBUG=-g
 CFLAGS=-O -std=gnu99 -Wall -D_FILE_OFFSET_BITS=64 $(DEBUG) \
        $(shell pkg-config --cflags $(PACKAGES))
+CXXFLAGS=-O -Wall -D_FILE_OFFSET_BITS=64 $(DEBUG) \
+       $(shell pkg-config --cflags $(PACKAGES))
 LDFLAGS=$(DEBUG) $(shell pkg-config --libs $(PACKAGES))
 SUBDIRS=nfs3
 
 SRCS=dir.c inode.c store.c
-OBJS=$(SRCS:.c=.o)
+OBJS=$(SRCS:.c=.o) s3store.o
 
 all : bluesky
        for d in $(SUBDIRS); do $(MAKE) -C $$d; done
@@ -17,7 +19,7 @@ bluesky.a : $(OBJS)
        ranlib $@
 
 bluesky : main.o bluesky.a
-       $(CC) $(LDFLAGS) -o $@ $^ -Wl,-rpath=$(LIBS3_PATH)/lib -L$(LIBS3_PATH)/lib -ls3
+       $(CXX) $(LDFLAGS) -o $@ $^ -Wl,-rpath=$(LIBS3_PATH)/lib -L$(LIBS3_PATH)/lib -ls3
 
 clean :
        rm -f $(OBJS) bluesky

diff --git a/bluesky.h b/bluesky.h
index 5571db6..d7bf8e3 100644 (file)
--- a/bluesky.h
+++ b/bluesky.h
@@ -12,6 +12,12 @@
 #include <stdint.h>
 #include <glib.h>
 
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+struct S3Store;
+
 /* Reference-counted blocks of memory, used for passing data in and out of
  * storage backends and in other places. */
 typedef struct {
@@ -45,6 +51,8 @@ typedef struct {
     gchar *name;                /* Descriptive name for the filesystem */
     GHashTable *inodes;         /* Cached inodes */
     uint64_t next_inum;         /* Next available inode for allocation */
+
+    struct S3Store *store;
 } BlueSkyFS;
 
 /* Inode number of the root directory. */
@@ -60,6 +68,8 @@ typedef struct {
     gint refcnt;                /* May be accessed atomically without lock */
     GMutex *lock;
 
+    BlueSkyFS *fs;
+
     BlueSkyFileType type;
     uint32_t mode;
     uint32_t uid, gid;
@@ -120,7 +130,7 @@ BlueSkyFS *bluesky_new_fs(gchar *name);
 int64_t bluesky_get_current_time();
 void bluesky_inode_update_ctime(BlueSkyInode *inode, gboolean update_mtime);
 uint64_t bluesky_fs_alloc_inode(BlueSkyFS *fs);
-BlueSkyInode *bluesky_new_inode(uint64_t inum, BlueSkyFileType type);
+BlueSkyInode *bluesky_new_inode(uint64_t inum, BlueSkyFS *fs, BlueSkyFileType type);
 
 BlueSkyInode *bluesky_get_inode(BlueSkyFS *fs, uint64_t inum);
 void bluesky_insert_inode(BlueSkyFS *fs, BlueSkyInode *inode);
@@ -133,11 +143,19 @@ gboolean bluesky_directory_insert(BlueSkyInode *dir, gchar *name,
 void bluesky_directory_dump(BlueSkyInode *dir);
 
 void bluesky_block_touch(BlueSkyInode *inode, uint64_t i);
-void bluesky_block_flush(BlueSkyBlock *block);
+void bluesky_block_flush(BlueSkyFS *fs, BlueSkyBlock *block);
 void bluesky_file_truncate(BlueSkyInode *inode, uint64_t size);
 void bluesky_file_write(BlueSkyInode *inode, uint64_t offset,
                         const char *data, gint len);
 void bluesky_file_read(BlueSkyInode *inode, uint64_t offset,
                        char *buf, gint len);
 
+struct S3Store *s3store_new();
+BlueSkyRCStr *s3store_get(struct S3Store *store, const gchar *key);
+void s3store_put(struct S3Store *store, const gchar *key, BlueSkyRCStr *val);
+
+#ifdef __cplusplus
+}
+#endif
+
 #endif

diff --git a/inode.c b/inode.c
index 162e154..b1071a6 100644 (file)
--- a/inode.c
+++ b/inode.c
@@ -114,6 +114,7 @@ BlueSkyFS *bluesky_new_fs(gchar *name)
     fs->inodes = g_hash_table_new(bluesky_fs_key_hash_func,
                                   bluesky_fs_key_equal_func);
     fs->next_inum = BLUESKY_ROOT_INUM + 1;
+    fs->store = s3store_new();
 
     return fs;
 }
@@ -132,12 +133,14 @@ uint64_t bluesky_fs_alloc_inode(BlueSkyFS *fs)
     return inum;
 }
 
-BlueSkyInode *bluesky_new_inode(uint64_t inum, BlueSkyFileType type)
+BlueSkyInode *bluesky_new_inode(uint64_t inum, BlueSkyFS *fs,
+                                BlueSkyFileType type)
 {
     BlueSkyInode *i = g_new0(BlueSkyInode, 1);
 
     i->lock = g_mutex_new();
     i->type = type;
+    i->fs = fs;
     i->inum = inum;
 
     switch (type) {
@@ -268,7 +271,7 @@ void bluesky_file_write(BlueSkyInode *inode, uint64_t offset,
         BlueSkyBlock *b = &g_array_index(inode->blocks, BlueSkyBlock,
                                          block_num);
         memcpy(&b->data[block_offset], data, bytes);
-        bluesky_block_flush(b);
+        bluesky_block_flush(inode->fs, b);
 
         offset += bytes;
         data += bytes;
@@ -313,7 +316,7 @@ void bluesky_file_read(BlueSkyInode *inode, uint64_t offset,
 }
 
 /* Write the given block to cloud-backed storage and mark it clean. */
-void bluesky_block_flush(BlueSkyBlock *block)
+void bluesky_block_flush(BlueSkyFS *fs, BlueSkyBlock *block)
 {
     if (block->type != BLUESKY_BLOCK_DIRTY)
         return;
@@ -327,7 +330,7 @@ void bluesky_block_flush(BlueSkyBlock *block)
     const gchar *name = g_checksum_get_string(csum);
 
     g_print("Flushing block as %s\n", name);
-    //memstore_put(store, name, data);
+    s3store_put(fs->store, name, data);
     g_free(block->ref);
     block->ref = g_strdup(name);
     block->type = BLUESKY_BLOCK_CACHED;

diff --git a/main.c b/main.c
index dc34d50..c02bc22 100644 (file)
--- a/main.c
+++ b/main.c
@@ -26,25 +26,25 @@ int main(int argc, char *argv[])
     BlueSkyFS *fs = bluesky_new_fs("export");
 
     BlueSkyInode *root;
-    root = bluesky_new_inode(BLUESKY_ROOT_INUM, BLUESKY_DIRECTORY);
+    root = bluesky_new_inode(BLUESKY_ROOT_INUM, fs, BLUESKY_DIRECTORY);
     root->nlink = 1;
     root->mode = 0755;
     bluesky_insert_inode(fs, root);
 
     BlueSkyInode *file;
-    file = bluesky_new_inode(bluesky_fs_alloc_inode(fs), BLUESKY_REGULAR);
+    file = bluesky_new_inode(bluesky_fs_alloc_inode(fs), fs, BLUESKY_REGULAR);
     file->nlink = 1;
     file->mode = 0755;
     bluesky_insert_inode(fs, file);
     bluesky_directory_insert(root, "foo", file->inum);
 
-    file = bluesky_new_inode(bluesky_fs_alloc_inode(fs), BLUESKY_REGULAR);
+    file = bluesky_new_inode(bluesky_fs_alloc_inode(fs), fs, BLUESKY_REGULAR);
     file->nlink = 1;
     file->mode = 0755;
     bluesky_insert_inode(fs, file);
     bluesky_directory_insert(root, "bar", file->inum);
 
-    file = bluesky_new_inode(bluesky_fs_alloc_inode(fs), BLUESKY_REGULAR);
+    file = bluesky_new_inode(bluesky_fs_alloc_inode(fs), fs, BLUESKY_REGULAR);
     file->nlink = 1;
     file->mode = 0755;
     bluesky_insert_inode(fs, file);

diff --git a/nfs3/Makefile b/nfs3/Makefile
index 3b09716..9905682 100644 (file)
--- a/nfs3/Makefile
+++ b/nfs3/Makefile
@@ -9,7 +9,7 @@ SRCS=nfsd.c rpc.c mount.c nfs3.c mount_prot_xdr.c nfs3_prot_xdr.c
 OBJS=$(SRCS:.c=.o)
 
 nfsproxy : $(OBJS)
-       $(CC) $(LDFLAGS) -o $@ $^ ../bluesky.a -Wl,-rpath=$(LIBS3_PATH)/lib -L$(LIBS3_PATH)/lib -ls3
+       $(CXX) $(LDFLAGS) -o $@ $^ ../bluesky.a -Wl,-rpath=$(LIBS3_PATH)/lib -L$(LIBS3_PATH)/lib -ls3
 
 clean :
        rm -f $(OBJS) nfsproxy

diff --git a/nfs3/nfs3.c b/nfs3/nfs3.c
index 95185e5..051d728 100644 (file)
--- a/nfs3/nfs3.c
+++ b/nfs3/nfs3.c
@@ -293,7 +293,7 @@ nfsproc3_create_3_svc(create3args *argp, struct svc_req *rqstp)
     }
 
     BlueSkyInode *file;
-    file = bluesky_new_inode(bluesky_fs_alloc_inode(fs), BLUESKY_REGULAR);
+    file = bluesky_new_inode(bluesky_fs_alloc_inode(fs), fs, BLUESKY_REGULAR);
     file->nlink = 1;
     file->mode = 0755;
     int64_t time = bluesky_get_current_time();

diff --git a/nfs3/nfsd.c b/nfs3/nfsd.c
index f33a945..ab7b749 100644 (file)
--- a/nfs3/nfsd.c
+++ b/nfs3/nfsd.c
@@ -36,13 +36,13 @@ int main(int argc, char *argv[])
     fs = bluesky_new_fs("export");
 
     BlueSkyInode *root;
-    root = bluesky_new_inode(BLUESKY_ROOT_INUM, BLUESKY_DIRECTORY);
+    root = bluesky_new_inode(BLUESKY_ROOT_INUM, fs, BLUESKY_DIRECTORY);
     root->nlink = 1;
     root->mode = 0755;
     bluesky_insert_inode(fs, root);
 
     BlueSkyInode *file;
-    file = bluesky_new_inode(bluesky_fs_alloc_inode(fs), BLUESKY_REGULAR);
+    file = bluesky_new_inode(bluesky_fs_alloc_inode(fs), fs, BLUESKY_REGULAR);
     file->nlink = 1;
     file->mode = 0755;
     bluesky_insert_inode(fs, file);

diff --git a/s3store.cc b/s3store.cc
new file mode 100644 (file)
index 0000000..aff9080
--- /dev/null
+++ b/s3store.cc
@@ -0,0 +1,91 @@
+/* Blue Sky: File Systems in the Cloud
+ *
+ * Copyright (C) 2009  The Regents of the University of California
+ * Written by Michael Vrable <mvrable@cs.ucsd.edu>
+ *
+ * TODO: Licensing
+ */
+
+#include <stdint.h>
+#include <stdlib.h>
+#include <glib.h>
+#include <string.h>
+
+#include "bluesky.h"
+#include "libs3.h"
+
+/* Interface to Amazon S3 storage. */
+
+/* Simple in-memory data store for test purposes. */
+struct S3Store {
+    S3BucketContext bucket;
+};
+
+S3Store *s3store_new()
+{
+    S3Store *store = g_new(S3Store, 1);
+    store->bucket.bucketName = "mvrable-bluesky";
+    store->bucket.protocol = S3ProtocolHTTP;
+    store->bucket.uriStyle = S3UriStylePath;
+    store->bucket.accessKeyId = getenv("AWS_ACCESS_KEY_ID");
+    store->bucket.secretAccessKey = getenv("AWS_SECRET_ACCESS_KEY");
+
+    g_print("Initializing S3 with bucket %s, access key %s\n",
+            store->bucket.bucketName, store->bucket.accessKeyId);
+
+    return store;
+}
+
+BlueSkyRCStr *s3store_get(S3Store *store, const gchar *key)
+{
+    return NULL;
+}
+
+struct put_info {
+    BlueSkyRCStr *val;
+    gint offset;
+};
+
+static int s3store_put_handler(int bufferSize, char *buffer,
+                               void *callbackData)
+{
+    struct put_info *info = (struct put_info *)callbackData;
+    gint bytes = MIN(bufferSize, (int)(info->val->len - info->offset));
+    memcpy(buffer, (char *)info->val->data + info->offset, bytes);
+    info->offset += bytes;
+    return bytes;
+}
+
+static S3Status s3store_properties_callback(const S3ResponseProperties *properties,
+                                     void *callbackData)
+{
+    g_print("(Properties callback)\n");
+    return S3StatusOK;
+}
+
+void s3store_response_callback(S3Status status,
+                               const S3ErrorDetails *errorDetails,
+                               void *callbackData)
+{
+    g_print("S3 operation complete, status=%s\n",
+            S3_get_status_name(status));
+    if (errorDetails != NULL) {
+        g_print("  Error message: %s\n", errorDetails->message);
+    }
+}
+
+void s3store_put(S3Store *store, const gchar *key, BlueSkyRCStr *val)
+{
+    struct put_info info;
+    info.val = val;
+    info.offset = 0;
+
+    struct S3PutObjectHandler handler;
+    handler.responseHandler.propertiesCallback = s3store_properties_callback;
+    handler.responseHandler.completeCallback = s3store_response_callback;
+    handler.putObjectDataCallback = s3store_put_handler;
+
+    g_print("Starting store of %s to S3...\n", key);
+    S3_put_object(&store->bucket, key, val->len, NULL, NULL,
+                  &handler, &info);
+}

diff --git a/store.c b/store.c
new file mode 100644 (file)
index 0000000..63228b2
--- /dev/null
+++ b/store.c
@@ -0,0 +1,92 @@
+/* Blue Sky: File Systems in the Cloud
+ *
+ * Copyright (C) 2009  The Regents of the University of California
+ * Written by Michael Vrable <mvrable@cs.ucsd.edu>
+ *
+ * TODO: Licensing
+ */
+
+#include <stdint.h>
+#include <glib.h>
+#include <string.h>
+
+#include "bluesky.h"
+
+/* Interaction with cloud storage.  We expose very simple GET/PUT style
+ * interface, which different backends can implement.  Available backends
+ * (will) include Amazon S3 and a simple local store for testing purposes. */
+
+/* Create and return a new reference-counted string.  The reference count is
+ * initially one.  The newly-returned string takes ownership of the memory
+ * pointed at by data, and will call g_free on it when the reference count
+ * drops to zero. */
+BlueSkyRCStr *bluesky_string_new(gpointer data, gsize len)
+{
+    BlueSkyRCStr *string = g_new(BlueSkyRCStr, 1);
+    string->data = data;
+    string->len = len;
+    g_atomic_int_set(&string->refcount, 1);
+    return string;
+}
+
+void bluesky_string_ref(BlueSkyRCStr *string)
+{
+    g_atomic_int_inc(&string->refcount);
+}
+
+void bluesky_string_unref(BlueSkyRCStr *string)
+{
+    if (g_atomic_int_dec_and_test(&string->refcount)) {
+        g_free(string->data);
+        g_free(string);
+    }
+}
+
+/* Duplicate and return a new reference-counted string, containing a copy of
+ * the original data, with a reference count of 1.  As an optimization, if the
+ * passed-in string already has a reference count of 1, the original is
+ * returned.   Can be used to make a mutable copy of a shared string. */
+BlueSkyRCStr *bluesky_string_dup(BlueSkyRCStr *string)
+{
+    if (g_atomic_int_dec_and_test(&string->refcount)) {
+        /* There are no other shared copies, so return this one. */
+        g_atomic_int_inc(&string->refcount);
+        return string;
+    } else {
+        return bluesky_string_new(g_memdup(string->data, string->len),
+                                  string->len);
+    }
+}
+
+/* Simple in-memory data store for test purposes. */
+typedef struct {
+    GMutex *lock;
+
+    /* TODO: A hashtable isn't optimal for list queries... */
+    GHashTable *store;
+} MemStore;
+
+MemStore *memstore_new()
+{
+    MemStore *store = g_new(MemStore, 1);
+    store->lock = g_mutex_new();
+    store->store = g_hash_table_new_full(g_str_hash, g_str_equal,
+                                         g_free,
+                                         (GDestroyNotify)bluesky_string_unref);
+
+    return store;
+}
+
+BlueSkyRCStr *memstore_get(MemStore *store, const gchar *key)
+{
+    BlueSkyRCStr *s = g_hash_table_lookup(store->store, key);
+    if (s != NULL)
+        bluesky_string_ref(s);
+    return s;
+}
+
+void memstore_put(MemStore *store, const gchar *key, BlueSkyRCStr *val)
+{
+    bluesky_string_ref(val);
+    g_hash_table_insert(store->store, g_strdup(key), val);
+}