Add write throttling based on the size of the uncommitted journal

Previously, throttling was based only on data in memory not committed to
the journal.  With limited upload bandwidth to the cloud, we need both.

diff --git a/bluesky/bluesky-private.h b/bluesky/bluesky-private.h
index 5a55b79..2595367 100644 (file)
--- a/bluesky/bluesky-private.h
+++ b/bluesky/bluesky-private.h
@@ -268,6 +268,10 @@ struct cloudlog_header {
     uint32_t size1, size2, size3;
 } __attribute__((packed));
 
+// Rough size limit for a log segment.  This is not a firm limit and there are
+// no absolute guarantees on the size of a log segment.
+#define LOG_SEGMENT_SIZE (1 << 22)
+
 #define JOURNAL_MAGIC "\nLog"
 #define CLOUDLOG_MAGIC "AgI-"
 #define CLOUDLOG_MAGIC_ENCRYPTED "AgI="     // CLOUDLOG_MAGIC[3] ^= 0x10

diff --git a/bluesky/cache.c b/bluesky/cache.c
index 16e3d81..b20960c 100644 (file)
--- a/bluesky/cache.c
+++ b/bluesky/cache.c
@@ -421,9 +421,50 @@ void bluesky_flushd_invoke(BlueSkyFS *fs)
     g_thread_create((GThreadFunc)flushd_task, fs, FALSE, NULL);
 }
 
-void bluesky_flushd_invoke_conditional(BlueSkyFS *fs)
+/* How urgent is flushing out data?  Returns one of several values:
+ *   0 - memory state is fine
+ *   1 - should launch flushd if not already running
+ *   2 - should block writers until memory frees up
+ */
+static int compute_pressure(BlueSkyFS *fs)
 {
+    /* LEVEL 2 */
+    /* Too much dirty data in memory? */
+    if (g_atomic_int_get(&fs->cache_dirty)
+                + g_atomic_int_get(&fs->cache_log_dirty)
+           > bluesky_watermark_high_dirty)
+        return 2;
+
+    /* Too much uncommitted data in the journal on disk, not yet flushed to the
+     * cloud? */
+    printf("Dirty journals: %d to %d\n",
+           fs->log->journal_watermark, fs->log->seq_num);
+    int dirty_limit;
+    dirty_limit = bluesky_options.cache_size / (LOG_SEGMENT_SIZE / 1024) / 2;
+    int dirty_journals = fs->log->seq_num - fs->log->journal_watermark + 1;
+    if (dirty_journals > 1 && dirty_journals >= dirty_limit) {
+        printf("Too many dirty journals (%d >= %d)\n",
+               dirty_journals, dirty_limit);
+        return 2;
+    }
+
+    /* LEVEL 1 */
     if (g_atomic_int_get(&fs->cache_dirty) < bluesky_watermark_medium_dirty)
+        return 1;
+
+    if (dirty_journals > 1 && dirty_journals > dirty_limit / 2) {
+        printf("Many dirty journals (%d), should start writeback\n",
+               dirty_journals);
+        return 1;
+    }
+
+    return 0;
+}
+
+void bluesky_flushd_invoke_conditional(BlueSkyFS *fs)
+{
+    int pressure = compute_pressure(fs);
+    if (pressure == 0)
         return;
 
     if (bluesky_verbose) {
@@ -436,16 +477,17 @@ void bluesky_flushd_invoke_conditional(BlueSkyFS *fs)
 
     /* If the system is under heavy memory pressure, actually delay execution
      * so the flush daemon can catch up. */
-    while (g_atomic_int_get(&fs->cache_dirty)
-                + g_atomic_int_get(&fs->cache_log_dirty)
-           > bluesky_watermark_high_dirty) {
+    while (pressure > 1) {
         g_log("bluesky/flushd", G_LOG_LEVEL_DEBUG,
               "Waiting due to memory pressure, dirty=%d + %d",
               g_atomic_int_get(&fs->cache_dirty),
               g_atomic_int_get(&fs->cache_log_dirty));
         g_mutex_lock(fs->lock);
-        g_cond_wait(fs->flushd_cond, fs->lock);
+        pressure = compute_pressure(fs);
+        if (pressure > 1)
+            g_cond_wait(fs->flushd_cond, fs->lock);
         g_mutex_unlock(fs->lock);
+        pressure = compute_pressure(fs);
     }
 }
 

diff --git a/bluesky/log.c b/bluesky/log.c
index 7728720..ccc9a7a 100644 (file)
--- a/bluesky/log.c
+++ b/bluesky/log.c
@@ -34,10 +34,6 @@
  * only incompletely written out before a crash, which should only happen for
  * log records that were not considered committed). */
 
-// Rough size limit for a log segment.  This is not a firm limit and there are
-// no absolute guarantees on the size of a log segment.
-#define LOG_SEGMENT_SIZE (1 << 22)
-
 #define HEADER_MAGIC 0x676f4c0a
 #define FOOTER_MAGIC 0x2e435243