Add a target size for the cache, and prune the cache when it gets larger.

[bluesky.git] / bluesky / log.c

diff --git a/bluesky/log.c b/bluesky/log.c
index 66b4edb..7146506 100644 (file)
--- a/bluesky/log.c
+++ b/bluesky/log.c
@@ -36,7 +36,11 @@
 
 // Rough size limit for a log segment.  This is not a firm limit and there are
 // no absolute guarantees on the size of a log segment.
-#define LOG_SEGMENT_SIZE (1 << 24)
+#define LOG_SEGMENT_SIZE (1 << 22)
+
+// Target amount of disk space to use for the journal and cache files, in
+// kilobytes.
+#define DISK_CACHE_SIZE_TARGET (64 * 1024)
 
 #define HEADER_MAGIC 0x676f4c0a
 #define FOOTER_MAGIC 0x2e435243
@@ -217,8 +221,10 @@ static gpointer log_thread(gpointer d)
          * increment the count of live dirty objects in that journal file.  The
          * count will be decremented when objects are deleted or written to the
          * cloud. */
-        g_atomic_int_add(&log->current_log->dirty_refs, 1);
-        item->dirty_journal = log->current_log;
+        if (!(item->location_flags & CLOUDLOG_CLOUD)) {
+            g_atomic_int_add(&log->current_log->dirty_refs, 1);
+            item->dirty_journal = log->current_log;
+        }
 
         /* Replace the log item's string data with a memory-mapped copy of the
          * data, now that it has been written to the log file.  (Even if it
@@ -330,6 +336,7 @@ BlueSkyCacheFile *bluesky_cachefile_lookup(BlueSkyFS *fs,
 
     BlueSkyLog *log = fs->log;
 
+    struct stat statbuf;
     char logname[64];
     int type;
 
@@ -346,10 +353,16 @@ BlueSkyCacheFile *bluesky_cachefile_lookup(BlueSkyFS *fs,
     g_mutex_lock(log->mmap_lock);
     map = g_hash_table_lookup(log->mmap_cache, logname);
 
-    if (map == NULL) {
-        /* TODO: stat() call */
+    if (map == NULL
+        && type == CLOUDLOG_JOURNAL
+        && fstatat(log->dirfd, logname, &statbuf, 0) < 0) {
+        /* A stale reference to a journal file which doesn't exist any longer
+         * because it was reclaimed.  Return NULL. */
+    } else if (map == NULL) {
+        g_print("Adding cache file %s\n", logname);
+
         map = g_new0(BlueSkyCacheFile, 1);
-        map->type = CLOUDLOG_JOURNAL;
+        map->type = type;
         map->lock = g_mutex_new();
         map->type = type;
         g_mutex_lock(map->lock);
@@ -381,7 +394,8 @@ BlueSkyCacheFile *bluesky_cachefile_lookup(BlueSkyFS *fs,
     }
 
     g_mutex_unlock(log->mmap_lock);
-    g_atomic_int_inc(&map->refcount);
+    if (map != NULL)
+        g_atomic_int_inc(&map->refcount);
     return map;
 }
 
@@ -504,7 +518,8 @@ void bluesky_cachefile_gc(BlueSkyFS *fs)
                 g_print(" (fetching)");
             g_print("\n");
 
-            if (g_atomic_int_get(&cachefile->refcount) == 0
+            if (g_atomic_int_get(&fs->log->disk_used) > DISK_CACHE_SIZE_TARGET
+                && g_atomic_int_get(&cachefile->refcount) == 0
                 && g_atomic_int_get(&cachefile->mapcount) == 0
                 && g_atomic_int_get(&cachefile->dirty_refs) == 0)
             {
@@ -514,6 +529,7 @@ void bluesky_cachefile_gc(BlueSkyFS *fs)
                             cachefile->filename);
                 }
 
+                g_atomic_int_add(&fs->log->disk_used, -(cachefile->len / 1024));
                 g_hash_table_remove(fs->log->mmap_cache, cachefile->filename);
                 g_mutex_unlock(cachefile->lock);
                 g_mutex_free(cachefile->lock);