link_directories("${LIBS3_BUILD_DIR}/lib" ${KVSTORE_DIR})
add_library(bluesky SHARED
- cache.c cloudlog.c crc32c.c crypto.c debug.c dir.c file.c imap.c
- init.c inode.c log.c serialize.c store.c store-bdb.c store-kv.cc
- store-multi.c store-s3.c util.c)
+ cache.c cleaner.c cloudlog.c crc32c.c crypto.c debug.c dir.c file.c
+ imap.c init.c inode.c log.c serialize.c store.c store-bdb.c
+ store-kv.cc store-multi.c store-s3.c util.c)
add_executable(bluesky-test main.c)
set(CMAKE_C_FLAGS "-Wall -std=gnu99 ${CMAKE_C_FLAGS}")
void bluesky_crypt_block_encrypt(gchar *cloud_block, size_t len,
BlueSkyCryptKeys *keys);
gboolean bluesky_crypt_block_decrypt(gchar *cloud_block, size_t len,
- BlueSkyCryptKeys *keys);
+ BlueSkyCryptKeys *keys,
+ gboolean allow_unauth);
void bluesky_cloudlog_encrypt(GString *segment, BlueSkyCryptKeys *keys);
void bluesky_cloudlog_decrypt(char *segment, size_t len,
BlueSkyCryptKeys *keys,
- BlueSkyRangeset *items);
+ BlueSkyRangeset *items,
+ gboolean allow_unauth);
/* Storage layer. Requests can be performed asynchronously, so these objects
* help keep track of operations in progress. */
int disk_used;
BlueSkyFS *fs;
BlueSkyLog *log;
- gboolean fetching, ready; // Cloud data: downloading or ready for use
+ gboolean fetching; // Cloud data: downloading or ready for use
+ gboolean complete; // Complete file has been fetched from cloud
int64_t atime; // Access time, for cache management
BlueSkyRangeset *items; // Locations of valid items
BlueSkyRangeset *prefetches;// Locations we have been requested to prefetch
gboolean bluesky_checkpoint_load(BlueSkyFS *fs);
+/* Merging of log state with the work of the cleaner. */
+void bluesky_cleaner_find_checkpoint(BlueSkyFS *fs);
+
#ifdef __cplusplus
}
#endif
--- /dev/null
+/* Blue Sky: File Systems in the Cloud
+ *
+ * Copyright (C) 2010 The Regents of the University of California
+ * Written by Michael Vrable <mvrable@cs.ucsd.edu>
+ *
+ * TODO: Licensing
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <stdint.h>
+#include <inttypes.h>
+#include <glib.h>
+#include <string.h>
+
+#include "bluesky-private.h"
+
+/* Proxy component of the file system cleaner. This consists effectively of
+ * code for merging multiple versions of the file system logs: that generated
+ * by us and that generated by the in-cloud cleaner. Other logic, such as for
+ * rewriting log segments where needed and deleting old segments, is handled by
+ * the in-cloud cleaner component. */
+
+/* Check the cleaner's logs to find the a more recent checkpoint record. This
+ * should be called occasionally to see if the cleaner has done any work since
+ * our last check. */
+void bluesky_cleaner_find_checkpoint(BlueSkyFS *fs)
+{
+ char *prefix = g_strdup_printf("log-%08d", BLUESKY_CLOUD_DIR_CLEANER);
+ char *last_segment = bluesky_store_lookup_last(fs->store, prefix);
+ g_free(prefix);
+ if (last_segment == NULL)
+ return;
+
+ g_print("Last cloud log segment: %s\n", last_segment);
+ int seq = atoi(last_segment + 13);
+ g_free(last_segment);
+
+ if (seq <= fs->log_state->latest_cleaner_seq_seen)
+ return;
+
+ g_print("New log segment appeared in cleaner directory: %d\n", seq);
+
+ BlueSkyCacheFile *cachefile;
+ cachefile = bluesky_cachefile_lookup(fs, BLUESKY_CLOUD_DIR_CLEANER, seq,
+ TRUE);
+ while (!cachefile->complete)
+ g_cond_wait(cachefile->cond, cachefile->lock);
+
+ g_print("Downloaded latest cleaner segment.\n");
+
+ int64_t offset = -1, length = 0;
+ while (TRUE) {
+ const BlueSkyRangesetItem *item;
+ item = bluesky_rangeset_lookup_next(cachefile->items, offset + 1);
+ if (item == NULL)
+ break;
+ offset = item->start;
+ length = item->length;
+ }
+
+ if (length > 0) {
+ g_print("Last object: %"PRIu64" + %"PRIu64"\n", offset, length);
+ }
+
+ bluesky_cachefile_unref(cachefile);
+ g_mutex_unlock(cachefile->lock);
+}
/* Make an decryption pass over a cloud log segment to decrypt items which were
* encrypted. Also computes a list of all offsets which at which valid
- * cloud log items are found and adds those offsets to items (if non-NULL). */
+ * cloud log items are found and adds those offsets to items (if non-NULL).
+ *
+ * If allow_unauth is set to true, then allow a limited set of unauthenticated
+ * items that may have been rewritten by a file system cleaner. These include
+ * the checkpoint and inode map records only; other items must still pass
+ * authentication. */
void bluesky_cloudlog_decrypt(char *segment, size_t len,
BlueSkyCryptKeys *keys,
- BlueSkyRangeset *items)
+ BlueSkyRangeset *items,
+ gboolean allow_unauth)
{
char *data = segment;
size_t remaining_size = len;
+ GUINT32_FROM_LE(header->size3);
if (item_size > remaining_size)
break;
- if (bluesky_crypt_block_decrypt(data, item_size, keys)) {
+ if (bluesky_crypt_block_decrypt(data, item_size, keys, allow_unauth)) {
if (items != NULL) {
if (bluesky_verbose)
g_print(" data item at %zx\n", offset);
}
gboolean bluesky_crypt_block_decrypt(gchar *cloud_block, size_t len,
- BlueSkyCryptKeys *keys)
+ BlueSkyCryptKeys *keys,
+ gboolean allow_unauth)
{
gcry_error_t status;
uint8_t hmac_check[CRYPTO_HASH_SIZE];
keys->authentication_key,
hmac_check);
if (memcmp(hmac_check, header->crypt_auth, CRYPTO_HASH_SIZE) != 0) {
- g_warning("Cloud block HMAC does not match!\n");
- return FALSE;
+ g_warning("Cloud block HMAC does not match!");
+ if (allow_unauth
+ && (header->type == LOGTYPE_INODE_MAP + '0'
+ || header->type == LOGTYPE_CHECKPOINT + '0'))
+ {
+ g_warning("Allowing unauthenticated data from cleaner");
+ } else {
+ return FALSE;
+ }
}
if (encrypted) {
}
last = bluesky_string_dup(last);
- bluesky_cloudlog_decrypt(last->data, last->len, fs->keys, NULL);
+ bluesky_cloudlog_decrypt(last->data, last->len, fs->keys, NULL, FALSE);
/* Scan through the contents of the last log segment to find a checkpoint
* record. We need to do a linear scan since at this point we don't have a
bluesky_inode_do_sync(root);
}
+ bluesky_cleaner_find_checkpoint(fs);
+
return fs;
}
g_mutex_lock(map->lock);
map->cond = g_cond_new();
map->filename = g_strdup(logname);
+ map->log_dir = clouddir;
map->log_seq = log_seq;
map->log = log;
g_atomic_int_set(&map->mapcount, 0);
static void cloudlog_partial_fetch_complete(BlueSkyStoreAsync *async,
BlueSkyCacheFile *cachefile)
{
- g_print("Partial fetch of %s from cloud complete, status = %d\n",
+ g_print("Fetch of %s from cloud complete, status = %d\n",
async->key, async->result);
g_mutex_lock(cachefile->lock);
if (async->result >= 0) {
+ if (async->len == 0) {
+ g_print("Complete object was fetched.\n");
+ cachefile->complete = TRUE;
+ }
+
/* Descrypt items fetched and write valid items out to the local log,
* but only if they do not overlap existing objects. This will protect
* against an attack by the cloud provider where one valid object is
BlueSkyRangeset *items = bluesky_rangeset_new();
int fd = openat(cachefile->log->dirfd, cachefile->filename, O_WRONLY);
if (fd >= 0) {
+ gboolean allow_unauth;
async->data = bluesky_string_dup(async->data);
+ allow_unauth = cachefile->log_dir == BLUESKY_CLOUD_DIR_CLEANER;
bluesky_cloudlog_decrypt(async->data->data, async->data->len,
- cachefile->fs->keys, items);
+ cachefile->fs->keys, items, allow_unauth);
uint64_t item_offset = 0;
while (TRUE) {
const BlueSkyRangesetItem *item;
projects / bluesky.git / commitdiff