1 /* Blue Sky: File Systems in the Cloud
3 * Copyright (C) 2009 The Regents of the University of California
4 * Written by Michael Vrable <mvrable@cs.ucsd.edu>
14 #include "bluesky-private.h"
17 /* Interface to Amazon S3 storage. */
20 GThreadPool *thread_pool;
21 S3BucketContext bucket;
22 uint8_t encryption_key[CRYPTO_KEY_SIZE];
26 enum { S3_GET, S3_PUT } op;
31 static void s3store_task(gpointer s, gpointer o);
33 static gpointer s3store_new()
35 S3Store *store = g_new(S3Store, 1);
36 store->thread_pool = g_thread_pool_new(s3store_task, store, -1, FALSE,
38 store->bucket.bucketName = "mvrable-bluesky";
39 store->bucket.protocol = S3ProtocolHTTP;
40 store->bucket.uriStyle = S3UriStylePath;
41 store->bucket.accessKeyId = getenv("AWS_ACCESS_KEY_ID");
42 store->bucket.secretAccessKey = getenv("AWS_SECRET_ACCESS_KEY");
44 const char *key = getenv("BLUESKY_KEY");
46 g_error("Encryption key not defined; please set BLUESKY_KEY environment variable");
50 bluesky_crypt_hash_key(key, store->encryption_key);
52 g_print("Initializing S3 with bucket %s, access key %s, encryption key %s\n",
53 store->bucket.bucketName, store->bucket.accessKeyId, key);
58 static void s3store_destroy(gpointer store)
68 static S3Status s3store_get_handler(int bufferSize, const char *buffer,
71 struct get_info *info = (struct get_info *)callbackData;
72 gint bytes = MIN(bufferSize, (int)(BLUESKY_BLOCK_SIZE - info->offset));
73 memcpy(info->buf + info->offset, buffer, bytes);
74 info->offset += bytes;
83 static int s3store_put_handler(int bufferSize, char *buffer,
86 struct put_info *info = (struct put_info *)callbackData;
87 gint bytes = MIN(bufferSize, (int)(info->val->len - info->offset));
88 memcpy(buffer, (char *)info->val->data + info->offset, bytes);
89 info->offset += bytes;
93 static S3Status s3store_properties_callback(const S3ResponseProperties *properties,
96 g_print("(Properties callback)\n");
100 static void s3store_response_callback(S3Status status,
101 const S3ErrorDetails *errorDetails,
104 g_print("S3 operation complete, status=%s, now=%ld\n",
105 S3_get_status_name(status), bluesky_now_hires());
106 if (errorDetails != NULL) {
107 g_print(" Error message: %s\n", errorDetails->message);
111 static BlueSkyRCStr *s3store_get(gpointer s, const gchar *key)
113 S3Store *store = (S3Store *)s;
115 struct get_info info;
116 info.buf = (char *)g_malloc0(BLUESKY_BLOCK_SIZE);
119 struct S3GetObjectHandler handler;
120 handler.responseHandler.propertiesCallback = s3store_properties_callback;
121 handler.responseHandler.completeCallback = s3store_response_callback;
122 handler.getObjectDataCallback = s3store_get_handler;
124 g_print("Starting fetch of %s from S3...\n", key);
125 S3_get_object(&store->bucket, key, NULL, 0, 0, NULL,
128 BlueSkyRCStr *raw, *decrypted;
129 raw = bluesky_string_new(info.buf, BLUESKY_BLOCK_SIZE);
130 decrypted = bluesky_crypt_decrypt(raw, store->encryption_key);
131 bluesky_string_unref(raw);
135 static void s3store_put(gpointer s, const gchar *key, BlueSkyRCStr *val)
137 S3Store *store = (S3Store *)s;
139 S3Op *op = g_new(S3Op, 1);
141 op->key = g_strdup(key);
142 bluesky_string_ref(val);
145 g_thread_pool_push(store->thread_pool, op, NULL);
148 static void s3store_task(gpointer o, gpointer s)
150 S3Store *store = (S3Store *)s;
151 S3Op *op = (S3Op *)o;
153 g_print("Start task [key=%s]...\n", op->key);
155 if (op->op == S3_PUT) {
156 BlueSkyRCStr *encrypted = bluesky_crypt_encrypt(op->data,
157 store->encryption_key);
159 struct put_info info;
160 info.val = encrypted;
163 struct S3PutObjectHandler handler;
164 handler.responseHandler.propertiesCallback
165 = s3store_properties_callback;
166 handler.responseHandler.completeCallback = s3store_response_callback;
167 handler.putObjectDataCallback = s3store_put_handler;
169 g_print("Starting store of %s to S3 at %ld...\n",
170 op->key, bluesky_now_hires());
171 S3_put_object(&store->bucket, op->key, encrypted->len, NULL, NULL,
174 bluesky_string_unref(encrypted);
177 bluesky_string_unref(op->data);
181 g_print("Finish task...\n");
184 static BlueSkyStoreImplementation store_impl = {
185 .create = s3store_new,
186 .destroy = s3store_destroy,
191 void bluesky_store_init_s3(void)
193 S3_initialize(NULL, S3_INIT_ALL);
194 bluesky_store_register(&store_impl, "s3");