From e1ca9ea030f0adc562642adca4b18557b3c9fe9c Mon Sep 17 00:00:00 2001 From: Michael Vrable Date: Wed, 18 Aug 2010 17:03:29 -0700 Subject: [PATCH] Add a target size for the cache, and prune the cache when it gets larger. --- bluesky/cloudlog.c | 9 +++++++-- bluesky/file.c | 3 --- bluesky/log.c | 32 ++++++++++++++++++++++++-------- 3 files changed, 31 insertions(+), 13 deletions(-) diff --git a/bluesky/cloudlog.c b/bluesky/cloudlog.c index 196eeb8..36895fe 100644 --- a/bluesky/cloudlog.c +++ b/bluesky/cloudlog.c @@ -209,14 +209,19 @@ void bluesky_cloudlog_fetch(BlueSkyCloudLog *log) log->data = bluesky_log_map_object(log->fs, -1, log->log_seq, log->log_offset, log->log_size); bluesky_cloudlog_stats_update(log, 1); - } else if (log->location_flags & CLOUDLOG_CLOUD) { + } + + if (log->data == NULL && (log->location_flags & CLOUDLOG_CLOUD)) { + log->location_flags &= ~CLOUDLOG_JOURNAL; bluesky_cloudlog_stats_update(log, -1); log->data = bluesky_log_map_object(log->fs, log->location.directory, log->location.sequence, log->location.offset, log->location.size); bluesky_cloudlog_stats_update(log, 1); - } else { + } + + if (log->data == NULL) { g_error("Unable to fetch cloudlog entry!"); } diff --git a/bluesky/file.c b/bluesky/file.c index 0c92e69..94677f6 100644 --- a/bluesky/file.c +++ b/bluesky/file.c @@ -287,9 +287,6 @@ void bluesky_file_drop_cached(BlueSkyInode *inode) b->ref->data = NULL; bluesky_cloudlog_stats_update(b->ref, 1); } - if (b->ref->location_flags & CLOUDLOG_CLOUD) { - b->ref->location_flags &= ~CLOUDLOG_JOURNAL; - } g_mutex_unlock(b->ref->lock); } } diff --git a/bluesky/log.c b/bluesky/log.c index 66b4edb..7146506 100644 --- a/bluesky/log.c +++ b/bluesky/log.c @@ -36,7 +36,11 @@ // Rough size limit for a log segment. This is not a firm limit and there are // no absolute guarantees on the size of a log segment. -#define LOG_SEGMENT_SIZE (1 << 24) +#define LOG_SEGMENT_SIZE (1 << 22) + +// Target amount of disk space to use for the journal and cache files, in +// kilobytes. +#define DISK_CACHE_SIZE_TARGET (64 * 1024) #define HEADER_MAGIC 0x676f4c0a #define FOOTER_MAGIC 0x2e435243 @@ -217,8 +221,10 @@ static gpointer log_thread(gpointer d) * increment the count of live dirty objects in that journal file. The * count will be decremented when objects are deleted or written to the * cloud. */ - g_atomic_int_add(&log->current_log->dirty_refs, 1); - item->dirty_journal = log->current_log; + if (!(item->location_flags & CLOUDLOG_CLOUD)) { + g_atomic_int_add(&log->current_log->dirty_refs, 1); + item->dirty_journal = log->current_log; + } /* Replace the log item's string data with a memory-mapped copy of the * data, now that it has been written to the log file. (Even if it @@ -330,6 +336,7 @@ BlueSkyCacheFile *bluesky_cachefile_lookup(BlueSkyFS *fs, BlueSkyLog *log = fs->log; + struct stat statbuf; char logname[64]; int type; @@ -346,10 +353,16 @@ BlueSkyCacheFile *bluesky_cachefile_lookup(BlueSkyFS *fs, g_mutex_lock(log->mmap_lock); map = g_hash_table_lookup(log->mmap_cache, logname); - if (map == NULL) { - /* TODO: stat() call */ + if (map == NULL + && type == CLOUDLOG_JOURNAL + && fstatat(log->dirfd, logname, &statbuf, 0) < 0) { + /* A stale reference to a journal file which doesn't exist any longer + * because it was reclaimed. Return NULL. */ + } else if (map == NULL) { + g_print("Adding cache file %s\n", logname); + map = g_new0(BlueSkyCacheFile, 1); - map->type = CLOUDLOG_JOURNAL; + map->type = type; map->lock = g_mutex_new(); map->type = type; g_mutex_lock(map->lock); @@ -381,7 +394,8 @@ BlueSkyCacheFile *bluesky_cachefile_lookup(BlueSkyFS *fs, } g_mutex_unlock(log->mmap_lock); - g_atomic_int_inc(&map->refcount); + if (map != NULL) + g_atomic_int_inc(&map->refcount); return map; } @@ -504,7 +518,8 @@ void bluesky_cachefile_gc(BlueSkyFS *fs) g_print(" (fetching)"); g_print("\n"); - if (g_atomic_int_get(&cachefile->refcount) == 0 + if (g_atomic_int_get(&fs->log->disk_used) > DISK_CACHE_SIZE_TARGET + && g_atomic_int_get(&cachefile->refcount) == 0 && g_atomic_int_get(&cachefile->mapcount) == 0 && g_atomic_int_get(&cachefile->dirty_refs) == 0) { @@ -514,6 +529,7 @@ void bluesky_cachefile_gc(BlueSkyFS *fs) cachefile->filename); } + g_atomic_int_add(&fs->log->disk_used, -(cachefile->len / 1024)); g_hash_table_remove(fs->log->mmap_cache, cachefile->filename); g_mutex_unlock(cachefile->lock); g_mutex_free(cachefile->lock); -- 2.20.1