X-Git-Url: http://git.vrable.net/?a=blobdiff_plain;f=bluesky%2Finode.c;h=0a6166b6ebe0a15b241351d2453cae5e829133d4;hb=8ff0fd08d6e1cc97cdb7e94b7cd97dc28c29e674;hp=26967136cf18d6f03bd0fac52e19d542367e4448;hpb=83fd6b61a6e092a22d4d5e59ed95f05f5e287f11;p=bluesky.git diff --git a/bluesky/inode.c b/bluesky/inode.c index 2696713..0a6166b 100644 --- a/bluesky/inode.c +++ b/bluesky/inode.c @@ -3,7 +3,29 @@ * Copyright (C) 2009 The Regents of the University of California * Written by Michael Vrable * - * TODO: Licensing + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. Neither the name of the University nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. */ #include @@ -14,6 +36,8 @@ #include "bluesky-private.h" +static void inode_fetch_task(gpointer a, gpointer b); + /* Core filesystem. Different proxies, such as the NFSv3 one, interface to * this, but the core actually tracks the data which is stored. So far we just * implement an in-memory filesystem, but eventually this will be state which @@ -50,14 +74,9 @@ void bluesky_inode_update_ctime(BlueSkyInode *inode, gboolean update_mtime) g_mutex_lock(inode->fs->lock); bluesky_list_unlink(&inode->fs->unlogged_list, inode->unlogged_list); inode->unlogged_list = bluesky_list_prepend(&inode->fs->unlogged_list, inode); - bluesky_list_unlink(&inode->fs->dirty_list, inode->dirty_list); - inode->dirty_list = bluesky_list_prepend(&inode->fs->dirty_list, inode); bluesky_list_unlink(&inode->fs->accessed_list, inode->accessed_list); inode->accessed_list = bluesky_list_prepend(&inode->fs->accessed_list, inode); g_mutex_unlock(inode->fs->lock); - - bluesky_cloudlog_unref(inode->committed_item); - inode->committed_item = NULL; } /* Unfortunately a glib hash table is only guaranteed to be able to store @@ -94,43 +113,50 @@ BlueSkyFS *bluesky_new_fs(gchar *name) fs->flushd_cond = g_cond_new(); fs->locations = g_hash_table_new(bluesky_cloudlog_hash, bluesky_cloudlog_equal); + fs->inode_map = g_sequence_new(NULL); fs->log_state = g_new0(BlueSkyCloudLogState, 1); fs->log_state->data = g_string_new(""); + fs->log_state->latest_cleaner_seq_seen = -1; + fs->log_state->uploads_pending_lock = g_mutex_new(); + fs->log_state->uploads_pending_cond = g_cond_new(); + + bluesky_cloudlog_threads_init(fs); + fs->inode_fetch_thread_pool = g_thread_pool_new(inode_fetch_task, NULL, + bluesky_max_threads, + FALSE, NULL); return fs; } -BlueSkyFS *bluesky_init_fs(gchar *name, BlueSkyStore *store) +BlueSkyFS *bluesky_init_fs(gchar *name, BlueSkyStore *store, + const gchar *master_key) { - BlueSkyRCStr *data = bluesky_store_get(store, "superblock"); - if (data != NULL) { - BlueSkyFS *fs = bluesky_deserialize_superblock(data->data); - if (fs != NULL) { - fs->store = store; - fs->log = bluesky_log_new("journal"); - g_print("Loaded filesystem superblock\n"); - g_free(fs->name); - fs->name = g_strdup(name); - return fs; - } - bluesky_string_unref(data); - } - - g_print("Initializing fresh filesystem\n"); BlueSkyFS *fs = bluesky_new_fs(name); + fs->master_key = g_strdup(master_key); + fs->keys = g_new(BlueSkyCryptKeys, 1); + bluesky_crypt_derive_keys(fs->keys, master_key); fs->store = store; fs->log = bluesky_log_new("journal"); + fs->log->fs = fs; + + if (bluesky_checkpoint_load(fs)) { + g_print("Filesystem checkpoint loaded, starting journal replay...\n"); + bluesky_replay(fs); + g_print("Journal replay complete, filesystem ready.\n"); + } else { + /* Initialize a fresh filesystem */ + g_print("Initializing new filesystem...\n"); + BlueSkyInode *root = bluesky_new_inode(BLUESKY_ROOT_INUM, fs, + BLUESKY_DIRECTORY); + root->nlink = 1; + root->mode = 0755; + bluesky_insert_inode(fs, root); + bluesky_inode_update_ctime(root, TRUE); + bluesky_inode_do_sync(root); + } - BlueSkyInode *root = bluesky_new_inode(BLUESKY_ROOT_INUM, fs, - BLUESKY_DIRECTORY); - root->nlink = 1; - root->mode = 0755; - bluesky_insert_inode(fs, root); - bluesky_inode_update_ctime(root, TRUE); - - bluesky_inode_do_sync(root); - bluesky_superblock_flush(fs); + bluesky_cleaner_thread_launch(fs); return fs; } @@ -141,6 +167,50 @@ void bluesky_inode_ref(BlueSkyInode *inode) g_atomic_int_inc(&inode->refcount); } +/* Free most of the resources used by an inode structure, but do not free the + * inode itself. Can be used if the inode data will be reloaded from + * serialized form to clear out old information first. */ +void bluesky_inode_free_resources(BlueSkyInode *inode) +{ + switch (inode->type) { + case BLUESKY_REGULAR: + if (inode->blocks != NULL) { + for (int i = 0; i < inode->blocks->len; i++) { + BlueSkyBlock *b = &g_array_index(inode->blocks, + BlueSkyBlock, i); + if (b->type == BLUESKY_BLOCK_DIRTY) { + g_error("Deleting an inode with dirty file data!"); + } + bluesky_cloudlog_unref(b->ref); + bluesky_string_unref(b->dirty); + } + g_array_unref(inode->blocks); + inode->blocks = NULL; + } + break; + + case BLUESKY_DIRECTORY: + if (inode->dirhash != NULL) + g_hash_table_destroy(inode->dirhash); + inode->dirhash = NULL; + if (inode->dirhash_folded != NULL) + g_hash_table_destroy(inode->dirhash_folded); + inode->dirhash_folded = NULL; + if (inode->dirents != NULL) + g_sequence_free(inode->dirents); + inode->dirents = NULL; + break; + + case BLUESKY_SYMLINK: + g_free(inode->symlink_contents); + inode->symlink_contents = NULL; + break; + + default: + break; + } +} + void bluesky_inode_unref(BlueSkyInode *inode) { if (g_atomic_int_dec_and_test(&inode->refcount)) { @@ -167,38 +237,9 @@ void bluesky_inode_unref(BlueSkyInode *inode) bluesky_list_unlink(&inode->fs->unlogged_list, inode->unlogged_list); g_mutex_unlock(inode->fs->lock); - /* Free file type specific data. It should be an error for there to be - * dirty data to commit when the reference count has reaches zero. */ - switch (inode->type) { - case BLUESKY_REGULAR: - for (int i = 0; i < inode->blocks->len; i++) { - BlueSkyBlock *b = &g_array_index(inode->blocks, - BlueSkyBlock, i); - if (b->type == BLUESKY_BLOCK_DIRTY) { - g_error("Deleting an inode with dirty file data!"); - } - bluesky_cloudlog_unref(b->ref); - bluesky_string_unref(b->dirty); - } - g_array_unref(inode->blocks); - break; - - case BLUESKY_DIRECTORY: - g_hash_table_destroy(inode->dirhash); - g_hash_table_destroy(inode->dirhash_folded); - g_sequence_free(inode->dirents); - break; - - case BLUESKY_SYMLINK: - g_free(inode->symlink_contents); - break; - - default: - break; - } + bluesky_inode_free_resources(inode); g_mutex_free(inode->lock); - g_free(inode); } } @@ -212,8 +253,6 @@ uint64_t bluesky_fs_alloc_inode(BlueSkyFS *fs) inum = fs->next_inum; fs->next_inum++; - bluesky_superblock_flush(fs); - return inum; } @@ -253,6 +292,30 @@ BlueSkyInode *bluesky_new_inode(uint64_t inum, BlueSkyFS *fs, return i; } +/* Issue a prefetch hint for an inode. This signals that the inode may be + * needed soon. Does not return any useful data. */ +void bluesky_inode_prefetch(BlueSkyFS *fs, uint64_t inum) +{ + BlueSkyInode *inode = NULL; + + g_mutex_lock(fs->lock); + inode = (BlueSkyInode *)g_hash_table_lookup(fs->inodes, &inum); + + if (inode != NULL) { + /* Inode is already available, no need for any prefetching... */ + g_mutex_unlock(fs->lock); + return; + } + + InodeMapEntry *entry = bluesky_inode_map_lookup(fs->inode_map, inum, 0); + if (entry != NULL) { + bluesky_cloudlog_prefetch(entry->item); + } + + g_mutex_unlock(fs->lock); + return; +} + /* Retrieve an inode from the filesystem. Eventually this will be a cache and * so we might need to go fetch the inode from elsewhere; for now all * filesystem state is stored here. inode is returned with a reference held @@ -318,6 +381,13 @@ void bluesky_inode_start_sync(BlueSkyInode *inode) g_mutex_lock(inode->fs->lock); bluesky_list_unlink(&inode->fs->unlogged_list, inode->unlogged_list); inode->unlogged_list = NULL; + + /* Since a new version of the inode has been written to the log, also + * schedule a future flush of the new data to cloud storage. */ + bluesky_list_unlink(&inode->fs->dirty_list, inode->dirty_list); + inode->dirty_list = bluesky_list_prepend(&inode->fs->dirty_list, inode); + inode->change_cloud = inode->change_count; + g_mutex_unlock(inode->fs->lock); } @@ -336,27 +406,21 @@ void bluesky_inode_do_sync(BlueSkyInode *inode) } } -static void complete_inode_fetch(BlueSkyStoreAsync *async, BlueSkyInode *inode) +static void inode_fetch_task(gpointer a, gpointer b) { - if (bluesky_verbose) { - g_log("bluesky/inode", G_LOG_LEVEL_DEBUG, - "Completing fetch of inode %"PRIu64"...", inode->inum); - } + BlueSkyInode *inode = (BlueSkyInode *)a; - if (async->result != 0 - || !bluesky_deserialize_inode(inode, async->data->data)) - { - if (bluesky_verbose) { - g_log("bluesky/inode", G_LOG_LEVEL_DEBUG, - " failed to load inode, cleaning up"); - } - g_mutex_lock(inode->fs->lock); - g_hash_table_remove(inode->fs->inodes, &inode->inum); - bluesky_list_unlink(&inode->fs->accessed_list, inode->accessed_list); - inode->accessed_list = NULL; - g_mutex_unlock(inode->fs->lock); - bluesky_inode_unref(inode); - } + bluesky_profile_set((BlueSkyProfile *)inode->private_data); + + BlueSkyCloudLog *item = inode->committed_item; + inode->committed_item = NULL; + g_print("Completing fetch of inode %"PRIu64"...\n", inode->inum); + + g_mutex_lock(item->lock); + bluesky_cloudlog_fetch(item); + if (!bluesky_deserialize_inode(inode, item)) + g_print("Error deserializing inode %"PRIu64"\n", inode->inum); + g_mutex_unlock(item->lock); inode->access_time = bluesky_get_current_time(); g_mutex_lock(inode->fs->lock); @@ -365,6 +429,7 @@ static void complete_inode_fetch(BlueSkyStoreAsync *async, BlueSkyInode *inode) g_mutex_unlock(inode->fs->lock); g_mutex_unlock(inode->lock); + bluesky_cloudlog_unref(item); bluesky_inode_unref(inode); } @@ -375,44 +440,23 @@ static void complete_inode_fetch(BlueSkyStoreAsync *async, BlueSkyInode *inode) * fs lock must be held. */ void bluesky_inode_fetch(BlueSkyFS *fs, uint64_t inum) { - char key[64]; - sprintf(key, "inode-%016"PRIx64, inum); + InodeMapEntry *entry = bluesky_inode_map_lookup(fs->inode_map, inum, 0); + if (entry == NULL) + return; + + /* Non-portable behavior: We take the inode lock here, and release it in + * the fetching thread. This works with the default Linux pthreads + * implementation but is not guaranteed. */ BlueSkyInode *inode = bluesky_new_inode(inum, fs, BLUESKY_PENDING); inode->change_count = 0; bluesky_inode_ref(inode); // Extra ref held by fetching process g_mutex_lock(inode->lock); - bluesky_insert_inode(fs, inode); - - BlueSkyStoreAsync *async = bluesky_store_async_new(fs->store); - async->op = STORE_OP_GET; - async->key = g_strdup(key); - - bluesky_store_async_add_notifier(async, (GFunc)complete_inode_fetch, inode); - bluesky_store_async_submit(async); - - if (bluesky_options.sync_inode_fetches) { - bluesky_store_async_wait(async); - } - bluesky_store_async_unref(async); -} + inode->committed_item = entry->item; + bluesky_cloudlog_ref(entry->item); + bluesky_insert_inode(fs, inode); -/* Synchronize filesystem superblock to stable storage. */ -void bluesky_superblock_flush(BlueSkyFS *fs) -{ -#if 0 - GString *buf = g_string_new(""); - bluesky_serialize_superblock(buf, fs); - BlueSkyRCStr *data = bluesky_string_new_from_gstring(buf); - - BlueSkyStoreAsync *async = bluesky_store_async_new(fs->store); - async->op = STORE_OP_PUT; - async->key = g_strdup("superblock"); - async->data = data; - bluesky_store_async_submit(async); - bluesky_store_async_unref(async); - - //bluesky_store_sync(fs->store); -#endif + inode->private_data = bluesky_profile_get(); + g_thread_pool_push(fs->inode_fetch_thread_pool, inode, NULL); }