Add write throttling based on the size of the uncommitted journal

[bluesky.git] / bluesky / log.c

diff --git a/bluesky/log.c b/bluesky/log.c
index 6475b5d..ccc9a7a 100644 (file)
--- a/bluesky/log.c
+++ b/bluesky/log.c
@@ -34,10 +34,6 @@
  * only incompletely written out before a crash, which should only happen for
  * log records that were not considered committed). */
 
-// Rough size limit for a log segment.  This is not a firm limit and there are
-// no absolute guarantees on the size of a log segment.
-#define LOG_SEGMENT_SIZE (1 << 22)
-
 #define HEADER_MAGIC 0x676f4c0a
 #define FOOTER_MAGIC 0x2e435243
 
@@ -482,7 +478,9 @@ static void cloudlog_partial_fetch_start(BlueSkyCacheFile *cachefile,
                                          size_t offset, size_t length)
 {
     g_atomic_int_inc(&cachefile->refcount);
-    g_print("Starting fetch of %s from cloud\n", cachefile->filename);
+    if (bluesky_verbose)
+        g_print("Starting partial fetch of %s from cloud (%zd + %zd)\n",
+                cachefile->filename, offset, length);
     BlueSkyStoreAsync *async = bluesky_store_async_new(cachefile->fs->store);
     async->op = STORE_OP_GET;
     async->key = g_strdup(cachefile->filename);
@@ -499,13 +497,15 @@ static void cloudlog_partial_fetch_start(BlueSkyCacheFile *cachefile,
 static void cloudlog_partial_fetch_complete(BlueSkyStoreAsync *async,
                                             BlueSkyCacheFile *cachefile)
 {
-    g_print("Fetch of %s from cloud complete, status = %d\n",
-            async->key, async->result);
+    if (bluesky_verbose || async->result != 0)
+        g_print("Fetch of %s from cloud complete, status = %d\n",
+                async->key, async->result);
 
     g_mutex_lock(cachefile->lock);
     if (async->result >= 0) {
         if (async->len == 0) {
-            g_print("Complete object was fetched.\n");
+            if (bluesky_verbose)
+                g_print("Complete object was fetched.\n");
             cachefile->complete = TRUE;
         }
 
@@ -549,8 +549,10 @@ static void cloudlog_partial_fetch_complete(BlueSkyStoreAsync *async,
             g_warning("Unable to open and write to cache file %s: %m",
                       cachefile->filename);
         }
+
+        bluesky_rangeset_free(items);
     } else {
-        g_print("Error fetching from cloud, retrying...\n");
+        g_print("Error fetching %s from cloud, retrying...\n", async->key);
         cloudlog_partial_fetch_start(cachefile, async->start, async->len);
     }
 
@@ -573,7 +575,8 @@ static void cloudlog_fetch_start(BlueSkyCacheFile *cachefile)
 {
     g_atomic_int_inc(&cachefile->refcount);
     cachefile->fetching = TRUE;
-    g_print("Starting fetch of %s from cloud\n", cachefile->filename);
+    if (bluesky_verbose)
+        g_print("Starting fetch of %s from cloud\n", cachefile->filename);
     BlueSkyStoreAsync *async = bluesky_store_async_new(cachefile->fs->store);
     async->op = STORE_OP_GET;
     async->key = g_strdup(cachefile->filename);
@@ -648,7 +651,8 @@ BlueSkyRCStr *bluesky_log_map_object(BlueSkyCloudLog *item, gboolean map_data)
     BlueSkyRCStr *str = NULL;
     int location = 0;
     size_t file_offset = 0, file_size = 0;
-    gboolean range_request = TRUE;
+    gboolean range_request = bluesky_options.full_segment_fetches
+                              ? FALSE : TRUE;
 
     if (page_size == 0) {
         page_size = getpagesize();
@@ -700,7 +704,7 @@ BlueSkyRCStr *bluesky_log_map_object(BlueSkyCloudLog *item, gboolean map_data)
                             file_offset);
                 }
                 if (range_request) {
-                    uint64_t start = 0, length = 0, end;
+                    uint64_t start = file_offset, length = file_size, end;
                     if (map->prefetches != NULL)
                         bluesky_rangeset_get_extents(map->prefetches,
                                                      &start, &length);
@@ -717,7 +721,7 @@ BlueSkyRCStr *bluesky_log_map_object(BlueSkyCloudLog *item, gboolean map_data)
             } else if (rangeitem->start == file_offset
                        && rangeitem->length == file_size) {
                 if (bluesky_verbose)
-                    g_print("Item now available.\n");
+                    g_print("Item %zd now available.\n", file_offset);
                 break;
             }
         }
@@ -748,7 +752,7 @@ void bluesky_mmap_unref(BlueSkyCacheFile *mmap)
 
     if (g_atomic_int_dec_and_test(&mmap->mapcount)) {
         g_mutex_lock(mmap->lock);
-        if (g_atomic_int_get(&mmap->mapcount) == 0) {
+        if (mmap->addr != NULL && g_atomic_int_get(&mmap->mapcount) == 0) {
             if (bluesky_verbose)
                 g_print("Unmapped log segment %d...\n", mmap->log_seq);
             munmap((void *)mmap->addr, mmap->len);
@@ -757,6 +761,7 @@ void bluesky_mmap_unref(BlueSkyCacheFile *mmap)
         }
         g_mutex_unlock(mmap->lock);
     }
+    g_assert(g_atomic_int_get(&mmap->mapcount) >= 0);
 }
 
 /******************************* JOURNAL REPLAY *******************************