From 3ebe1efb5570c4a678774ebdf33b36be6dc49bbc Mon Sep 17 00:00:00 2001 From: Michael Vrable Date: Mon, 31 Aug 2009 12:09:14 -0700 Subject: [PATCH] Implement flushing of file blocks to Amazon S3. Right now many details are hardcoded. Data can't be read back yet. But it implements some of the needed functionality. --- Makefile | 6 ++-- bluesky.h | 22 ++++++++++-- inode.c | 11 +++--- main.c | 8 ++--- nfs3/Makefile | 2 +- nfs3/nfs3.c | 2 +- nfs3/nfsd.c | 4 +-- s3store.cc | 91 ++++++++++++++++++++++++++++++++++++++++++++++++++ store.c | 92 +++++++++++++++++++++++++++++++++++++++++++++++++++ 9 files changed, 222 insertions(+), 16 deletions(-) create mode 100644 s3store.cc create mode 100644 store.c diff --git a/Makefile b/Makefile index 8d986f4..fa9d6f6 100644 --- a/Makefile +++ b/Makefile @@ -3,11 +3,13 @@ PACKAGES=glib-2.0 gthread-2.0 DEBUG=-g CFLAGS=-O -std=gnu99 -Wall -D_FILE_OFFSET_BITS=64 $(DEBUG) \ $(shell pkg-config --cflags $(PACKAGES)) +CXXFLAGS=-O -Wall -D_FILE_OFFSET_BITS=64 $(DEBUG) \ + $(shell pkg-config --cflags $(PACKAGES)) LDFLAGS=$(DEBUG) $(shell pkg-config --libs $(PACKAGES)) SUBDIRS=nfs3 SRCS=dir.c inode.c store.c -OBJS=$(SRCS:.c=.o) +OBJS=$(SRCS:.c=.o) s3store.o all : bluesky for d in $(SUBDIRS); do $(MAKE) -C $$d; done @@ -17,7 +19,7 @@ bluesky.a : $(OBJS) ranlib $@ bluesky : main.o bluesky.a - $(CC) $(LDFLAGS) -o $@ $^ -Wl,-rpath=$(LIBS3_PATH)/lib -L$(LIBS3_PATH)/lib -ls3 + $(CXX) $(LDFLAGS) -o $@ $^ -Wl,-rpath=$(LIBS3_PATH)/lib -L$(LIBS3_PATH)/lib -ls3 clean : rm -f $(OBJS) bluesky diff --git a/bluesky.h b/bluesky.h index 5571db6..d7bf8e3 100644 --- a/bluesky.h +++ b/bluesky.h @@ -12,6 +12,12 @@ #include #include +#ifdef __cplusplus +extern "C" { +#endif + +struct S3Store; + /* Reference-counted blocks of memory, used for passing data in and out of * storage backends and in other places. */ typedef struct { @@ -45,6 +51,8 @@ typedef struct { gchar *name; /* Descriptive name for the filesystem */ GHashTable *inodes; /* Cached inodes */ uint64_t next_inum; /* Next available inode for allocation */ + + struct S3Store *store; } BlueSkyFS; /* Inode number of the root directory. */ @@ -60,6 +68,8 @@ typedef struct { gint refcnt; /* May be accessed atomically without lock */ GMutex *lock; + BlueSkyFS *fs; + BlueSkyFileType type; uint32_t mode; uint32_t uid, gid; @@ -120,7 +130,7 @@ BlueSkyFS *bluesky_new_fs(gchar *name); int64_t bluesky_get_current_time(); void bluesky_inode_update_ctime(BlueSkyInode *inode, gboolean update_mtime); uint64_t bluesky_fs_alloc_inode(BlueSkyFS *fs); -BlueSkyInode *bluesky_new_inode(uint64_t inum, BlueSkyFileType type); +BlueSkyInode *bluesky_new_inode(uint64_t inum, BlueSkyFS *fs, BlueSkyFileType type); BlueSkyInode *bluesky_get_inode(BlueSkyFS *fs, uint64_t inum); void bluesky_insert_inode(BlueSkyFS *fs, BlueSkyInode *inode); @@ -133,11 +143,19 @@ gboolean bluesky_directory_insert(BlueSkyInode *dir, gchar *name, void bluesky_directory_dump(BlueSkyInode *dir); void bluesky_block_touch(BlueSkyInode *inode, uint64_t i); -void bluesky_block_flush(BlueSkyBlock *block); +void bluesky_block_flush(BlueSkyFS *fs, BlueSkyBlock *block); void bluesky_file_truncate(BlueSkyInode *inode, uint64_t size); void bluesky_file_write(BlueSkyInode *inode, uint64_t offset, const char *data, gint len); void bluesky_file_read(BlueSkyInode *inode, uint64_t offset, char *buf, gint len); +struct S3Store *s3store_new(); +BlueSkyRCStr *s3store_get(struct S3Store *store, const gchar *key); +void s3store_put(struct S3Store *store, const gchar *key, BlueSkyRCStr *val); + +#ifdef __cplusplus +} +#endif + #endif diff --git a/inode.c b/inode.c index 162e154..b1071a6 100644 --- a/inode.c +++ b/inode.c @@ -114,6 +114,7 @@ BlueSkyFS *bluesky_new_fs(gchar *name) fs->inodes = g_hash_table_new(bluesky_fs_key_hash_func, bluesky_fs_key_equal_func); fs->next_inum = BLUESKY_ROOT_INUM + 1; + fs->store = s3store_new(); return fs; } @@ -132,12 +133,14 @@ uint64_t bluesky_fs_alloc_inode(BlueSkyFS *fs) return inum; } -BlueSkyInode *bluesky_new_inode(uint64_t inum, BlueSkyFileType type) +BlueSkyInode *bluesky_new_inode(uint64_t inum, BlueSkyFS *fs, + BlueSkyFileType type) { BlueSkyInode *i = g_new0(BlueSkyInode, 1); i->lock = g_mutex_new(); i->type = type; + i->fs = fs; i->inum = inum; switch (type) { @@ -268,7 +271,7 @@ void bluesky_file_write(BlueSkyInode *inode, uint64_t offset, BlueSkyBlock *b = &g_array_index(inode->blocks, BlueSkyBlock, block_num); memcpy(&b->data[block_offset], data, bytes); - bluesky_block_flush(b); + bluesky_block_flush(inode->fs, b); offset += bytes; data += bytes; @@ -313,7 +316,7 @@ void bluesky_file_read(BlueSkyInode *inode, uint64_t offset, } /* Write the given block to cloud-backed storage and mark it clean. */ -void bluesky_block_flush(BlueSkyBlock *block) +void bluesky_block_flush(BlueSkyFS *fs, BlueSkyBlock *block) { if (block->type != BLUESKY_BLOCK_DIRTY) return; @@ -327,7 +330,7 @@ void bluesky_block_flush(BlueSkyBlock *block) const gchar *name = g_checksum_get_string(csum); g_print("Flushing block as %s\n", name); - //memstore_put(store, name, data); + s3store_put(fs->store, name, data); g_free(block->ref); block->ref = g_strdup(name); block->type = BLUESKY_BLOCK_CACHED; diff --git a/main.c b/main.c index dc34d50..c02bc22 100644 --- a/main.c +++ b/main.c @@ -26,25 +26,25 @@ int main(int argc, char *argv[]) BlueSkyFS *fs = bluesky_new_fs("export"); BlueSkyInode *root; - root = bluesky_new_inode(BLUESKY_ROOT_INUM, BLUESKY_DIRECTORY); + root = bluesky_new_inode(BLUESKY_ROOT_INUM, fs, BLUESKY_DIRECTORY); root->nlink = 1; root->mode = 0755; bluesky_insert_inode(fs, root); BlueSkyInode *file; - file = bluesky_new_inode(bluesky_fs_alloc_inode(fs), BLUESKY_REGULAR); + file = bluesky_new_inode(bluesky_fs_alloc_inode(fs), fs, BLUESKY_REGULAR); file->nlink = 1; file->mode = 0755; bluesky_insert_inode(fs, file); bluesky_directory_insert(root, "foo", file->inum); - file = bluesky_new_inode(bluesky_fs_alloc_inode(fs), BLUESKY_REGULAR); + file = bluesky_new_inode(bluesky_fs_alloc_inode(fs), fs, BLUESKY_REGULAR); file->nlink = 1; file->mode = 0755; bluesky_insert_inode(fs, file); bluesky_directory_insert(root, "bar", file->inum); - file = bluesky_new_inode(bluesky_fs_alloc_inode(fs), BLUESKY_REGULAR); + file = bluesky_new_inode(bluesky_fs_alloc_inode(fs), fs, BLUESKY_REGULAR); file->nlink = 1; file->mode = 0755; bluesky_insert_inode(fs, file); diff --git a/nfs3/Makefile b/nfs3/Makefile index 3b09716..9905682 100644 --- a/nfs3/Makefile +++ b/nfs3/Makefile @@ -9,7 +9,7 @@ SRCS=nfsd.c rpc.c mount.c nfs3.c mount_prot_xdr.c nfs3_prot_xdr.c OBJS=$(SRCS:.c=.o) nfsproxy : $(OBJS) - $(CC) $(LDFLAGS) -o $@ $^ ../bluesky.a -Wl,-rpath=$(LIBS3_PATH)/lib -L$(LIBS3_PATH)/lib -ls3 + $(CXX) $(LDFLAGS) -o $@ $^ ../bluesky.a -Wl,-rpath=$(LIBS3_PATH)/lib -L$(LIBS3_PATH)/lib -ls3 clean : rm -f $(OBJS) nfsproxy diff --git a/nfs3/nfs3.c b/nfs3/nfs3.c index 95185e5..051d728 100644 --- a/nfs3/nfs3.c +++ b/nfs3/nfs3.c @@ -293,7 +293,7 @@ nfsproc3_create_3_svc(create3args *argp, struct svc_req *rqstp) } BlueSkyInode *file; - file = bluesky_new_inode(bluesky_fs_alloc_inode(fs), BLUESKY_REGULAR); + file = bluesky_new_inode(bluesky_fs_alloc_inode(fs), fs, BLUESKY_REGULAR); file->nlink = 1; file->mode = 0755; int64_t time = bluesky_get_current_time(); diff --git a/nfs3/nfsd.c b/nfs3/nfsd.c index f33a945..ab7b749 100644 --- a/nfs3/nfsd.c +++ b/nfs3/nfsd.c @@ -36,13 +36,13 @@ int main(int argc, char *argv[]) fs = bluesky_new_fs("export"); BlueSkyInode *root; - root = bluesky_new_inode(BLUESKY_ROOT_INUM, BLUESKY_DIRECTORY); + root = bluesky_new_inode(BLUESKY_ROOT_INUM, fs, BLUESKY_DIRECTORY); root->nlink = 1; root->mode = 0755; bluesky_insert_inode(fs, root); BlueSkyInode *file; - file = bluesky_new_inode(bluesky_fs_alloc_inode(fs), BLUESKY_REGULAR); + file = bluesky_new_inode(bluesky_fs_alloc_inode(fs), fs, BLUESKY_REGULAR); file->nlink = 1; file->mode = 0755; bluesky_insert_inode(fs, file); diff --git a/s3store.cc b/s3store.cc new file mode 100644 index 0000000..aff9080 --- /dev/null +++ b/s3store.cc @@ -0,0 +1,91 @@ +/* Blue Sky: File Systems in the Cloud + * + * Copyright (C) 2009 The Regents of the University of California + * Written by Michael Vrable + * + * TODO: Licensing + */ + +#include +#include +#include +#include + +#include "bluesky.h" +#include "libs3.h" + +/* Interface to Amazon S3 storage. */ + +/* Simple in-memory data store for test purposes. */ +struct S3Store { + S3BucketContext bucket; +}; + +S3Store *s3store_new() +{ + S3Store *store = g_new(S3Store, 1); + store->bucket.bucketName = "mvrable-bluesky"; + store->bucket.protocol = S3ProtocolHTTP; + store->bucket.uriStyle = S3UriStylePath; + store->bucket.accessKeyId = getenv("AWS_ACCESS_KEY_ID"); + store->bucket.secretAccessKey = getenv("AWS_SECRET_ACCESS_KEY"); + + g_print("Initializing S3 with bucket %s, access key %s\n", + store->bucket.bucketName, store->bucket.accessKeyId); + + return store; +} + +BlueSkyRCStr *s3store_get(S3Store *store, const gchar *key) +{ + return NULL; +} + +struct put_info { + BlueSkyRCStr *val; + gint offset; +}; + +static int s3store_put_handler(int bufferSize, char *buffer, + void *callbackData) +{ + struct put_info *info = (struct put_info *)callbackData; + gint bytes = MIN(bufferSize, (int)(info->val->len - info->offset)); + memcpy(buffer, (char *)info->val->data + info->offset, bytes); + info->offset += bytes; + return bytes; +} + +static S3Status s3store_properties_callback(const S3ResponseProperties *properties, + void *callbackData) +{ + g_print("(Properties callback)\n"); + return S3StatusOK; +} + +void s3store_response_callback(S3Status status, + const S3ErrorDetails *errorDetails, + void *callbackData) +{ + g_print("S3 operation complete, status=%s\n", + S3_get_status_name(status)); + if (errorDetails != NULL) { + g_print(" Error message: %s\n", errorDetails->message); + } +} + +void s3store_put(S3Store *store, const gchar *key, BlueSkyRCStr *val) +{ + struct put_info info; + info.val = val; + info.offset = 0; + + struct S3PutObjectHandler handler; + handler.responseHandler.propertiesCallback = s3store_properties_callback; + handler.responseHandler.completeCallback = s3store_response_callback; + handler.putObjectDataCallback = s3store_put_handler; + + g_print("Starting store of %s to S3...\n", key); + S3_put_object(&store->bucket, key, val->len, NULL, NULL, + &handler, &info); +} diff --git a/store.c b/store.c new file mode 100644 index 0000000..63228b2 --- /dev/null +++ b/store.c @@ -0,0 +1,92 @@ +/* Blue Sky: File Systems in the Cloud + * + * Copyright (C) 2009 The Regents of the University of California + * Written by Michael Vrable + * + * TODO: Licensing + */ + +#include +#include +#include + +#include "bluesky.h" + +/* Interaction with cloud storage. We expose very simple GET/PUT style + * interface, which different backends can implement. Available backends + * (will) include Amazon S3 and a simple local store for testing purposes. */ + +/* Create and return a new reference-counted string. The reference count is + * initially one. The newly-returned string takes ownership of the memory + * pointed at by data, and will call g_free on it when the reference count + * drops to zero. */ +BlueSkyRCStr *bluesky_string_new(gpointer data, gsize len) +{ + BlueSkyRCStr *string = g_new(BlueSkyRCStr, 1); + string->data = data; + string->len = len; + g_atomic_int_set(&string->refcount, 1); + return string; +} + +void bluesky_string_ref(BlueSkyRCStr *string) +{ + g_atomic_int_inc(&string->refcount); +} + +void bluesky_string_unref(BlueSkyRCStr *string) +{ + if (g_atomic_int_dec_and_test(&string->refcount)) { + g_free(string->data); + g_free(string); + } +} + +/* Duplicate and return a new reference-counted string, containing a copy of + * the original data, with a reference count of 1. As an optimization, if the + * passed-in string already has a reference count of 1, the original is + * returned. Can be used to make a mutable copy of a shared string. */ +BlueSkyRCStr *bluesky_string_dup(BlueSkyRCStr *string) +{ + if (g_atomic_int_dec_and_test(&string->refcount)) { + /* There are no other shared copies, so return this one. */ + g_atomic_int_inc(&string->refcount); + return string; + } else { + return bluesky_string_new(g_memdup(string->data, string->len), + string->len); + } +} + +/* Simple in-memory data store for test purposes. */ +typedef struct { + GMutex *lock; + + /* TODO: A hashtable isn't optimal for list queries... */ + GHashTable *store; +} MemStore; + +MemStore *memstore_new() +{ + MemStore *store = g_new(MemStore, 1); + store->lock = g_mutex_new(); + store->store = g_hash_table_new_full(g_str_hash, g_str_equal, + g_free, + (GDestroyNotify)bluesky_string_unref); + + return store; +} + +BlueSkyRCStr *memstore_get(MemStore *store, const gchar *key) +{ + BlueSkyRCStr *s = g_hash_table_lookup(store->store, key); + if (s != NULL) + bluesky_string_ref(s); + return s; +} + +void memstore_put(MemStore *store, const gchar *key, BlueSkyRCStr *val) +{ + bluesky_string_ref(val); + g_hash_table_insert(store->store, g_strdup(key), val); +} -- 2.20.1