Add write throttling based on the size of the uncommitted journal

[bluesky.git] / bluesky / log.c

diff --git a/bluesky/log.c b/bluesky/log.c
index 2e5c920..ccc9a7a 100644 (file)
--- a/bluesky/log.c
+++ b/bluesky/log.c
@@ -34,10 +34,6 @@
  * only incompletely written out before a crash, which should only happen for
  * log records that were not considered committed). */
 
-// Rough size limit for a log segment.  This is not a firm limit and there are
-// no absolute guarantees on the size of a log segment.
-#define LOG_SEGMENT_SIZE (1 << 22)
-
 #define HEADER_MAGIC 0x676f4c0a
 #define FOOTER_MAGIC 0x2e435243
 
@@ -443,14 +439,16 @@ BlueSkyCacheFile *bluesky_cachefile_lookup(BlueSkyFS *fs,
             ftruncate(fd, 5 << 20);     // FIXME
             close(fd);
         }
-
-        // If the log file is stored in the cloud, we may need to fetch it
-        if (clouddir >= 0 && start_fetch)
-            cloudlog_fetch_start(map);
     } else {
         g_mutex_lock(map->lock);
     }
 
+
+    /* If the log file is stored in the cloud and has not been fully fetched,
+     * we may need to initiate a fetch now. */
+    if (clouddir >= 0 && start_fetch && !map->complete && !map->fetching)
+        cloudlog_fetch_start(map);
+
     g_mutex_unlock(log->mmap_lock);
     if (map != NULL)
         g_atomic_int_inc(&map->refcount);
@@ -480,7 +478,9 @@ static void cloudlog_partial_fetch_start(BlueSkyCacheFile *cachefile,
                                          size_t offset, size_t length)
 {
     g_atomic_int_inc(&cachefile->refcount);
-    g_print("Starting fetch of %s from cloud\n", cachefile->filename);
+    if (bluesky_verbose)
+        g_print("Starting partial fetch of %s from cloud (%zd + %zd)\n",
+                cachefile->filename, offset, length);
     BlueSkyStoreAsync *async = bluesky_store_async_new(cachefile->fs->store);
     async->op = STORE_OP_GET;
     async->key = g_strdup(cachefile->filename);
@@ -497,13 +497,15 @@ static void cloudlog_partial_fetch_start(BlueSkyCacheFile *cachefile,
 static void cloudlog_partial_fetch_complete(BlueSkyStoreAsync *async,
                                             BlueSkyCacheFile *cachefile)
 {
-    g_print("Fetch of %s from cloud complete, status = %d\n",
-            async->key, async->result);
+    if (bluesky_verbose || async->result != 0)
+        g_print("Fetch of %s from cloud complete, status = %d\n",
+                async->key, async->result);
 
     g_mutex_lock(cachefile->lock);
     if (async->result >= 0) {
         if (async->len == 0) {
-            g_print("Complete object was fetched.\n");
+            if (bluesky_verbose)
+                g_print("Complete object was fetched.\n");
             cachefile->complete = TRUE;
         }
 
@@ -547,8 +549,10 @@ static void cloudlog_partial_fetch_complete(BlueSkyStoreAsync *async,
             g_warning("Unable to open and write to cache file %s: %m",
                       cachefile->filename);
         }
+
+        bluesky_rangeset_free(items);
     } else {
-        g_print("Error fetching from cloud, retrying...\n");
+        g_print("Error fetching %s from cloud, retrying...\n", async->key);
         cloudlog_partial_fetch_start(cachefile, async->start, async->len);
     }
 
@@ -571,7 +575,8 @@ static void cloudlog_fetch_start(BlueSkyCacheFile *cachefile)
 {
     g_atomic_int_inc(&cachefile->refcount);
     cachefile->fetching = TRUE;
-    g_print("Starting fetch of %s from cloud\n", cachefile->filename);
+    if (bluesky_verbose)
+        g_print("Starting fetch of %s from cloud\n", cachefile->filename);
     BlueSkyStoreAsync *async = bluesky_store_async_new(cachefile->fs->store);
     async->op = STORE_OP_GET;
     async->key = g_strdup(cachefile->filename);
@@ -646,7 +651,8 @@ BlueSkyRCStr *bluesky_log_map_object(BlueSkyCloudLog *item, gboolean map_data)
     BlueSkyRCStr *str = NULL;
     int location = 0;
     size_t file_offset = 0, file_size = 0;
-    gboolean range_request = TRUE;
+    gboolean range_request = bluesky_options.full_segment_fetches
+                              ? FALSE : TRUE;
 
     if (page_size == 0) {
         page_size = getpagesize();
@@ -698,7 +704,7 @@ BlueSkyRCStr *bluesky_log_map_object(BlueSkyCloudLog *item, gboolean map_data)
                             file_offset);
                 }
                 if (range_request) {
-                    uint64_t start = 0, length = 0, end;
+                    uint64_t start = file_offset, length = file_size, end;
                     if (map->prefetches != NULL)
                         bluesky_rangeset_get_extents(map->prefetches,
                                                      &start, &length);
@@ -715,7 +721,7 @@ BlueSkyRCStr *bluesky_log_map_object(BlueSkyCloudLog *item, gboolean map_data)
             } else if (rangeitem->start == file_offset
                        && rangeitem->length == file_size) {
                 if (bluesky_verbose)
-                    g_print("Item now available.\n");
+                    g_print("Item %zd now available.\n", file_offset);
                 break;
             }
         }
@@ -746,7 +752,7 @@ void bluesky_mmap_unref(BlueSkyCacheFile *mmap)
 
     if (g_atomic_int_dec_and_test(&mmap->mapcount)) {
         g_mutex_lock(mmap->lock);
-        if (g_atomic_int_get(&mmap->mapcount) == 0) {
+        if (mmap->addr != NULL && g_atomic_int_get(&mmap->mapcount) == 0) {
             if (bluesky_verbose)
                 g_print("Unmapped log segment %d...\n", mmap->log_seq);
             munmap((void *)mmap->addr, mmap->len);
@@ -755,6 +761,7 @@ void bluesky_mmap_unref(BlueSkyCacheFile *mmap)
         }
         g_mutex_unlock(mmap->lock);
     }
+    g_assert(g_atomic_int_get(&mmap->mapcount) >= 0);
 }
 
 /******************************* JOURNAL REPLAY *******************************