From: Michael Vrable <mvrable@cs.ucsd.edu>
Date: Thu, 21 Oct 2010 20:07:18 +0000 (-0700)
Subject: Fix Amazon S3 store_lookup_last implementation.
X-Git-Url: http://git.vrable.net/?p=bluesky.git;a=commitdiff_plain;h=21b992d479c38c68a173f7b9905babd0cc62fac0

Fix Amazon S3 store_lookup_last implementation.

Amazon will return directory listings in chunks.  Be sure to repeat list
calls until we find the actual end of the directory listing.
---

diff --git a/bluesky/imap.c b/bluesky/imap.c
index 9b5f8e3..1b9feb8 100644
--- a/bluesky/imap.c
+++ b/bluesky/imap.c
@@ -315,6 +315,10 @@ gboolean bluesky_checkpoint_load(BlueSkyFS *fs)
         len -= size;
     }
 
+    if (checkpoint_size == 0) {
+        g_error("Unable to locate checkpoint record!\n");
+    }
+
     g_print("Found checkpoint record at %zd (size %zd)\n",
             checkpoint - last->data, checkpoint_size);
 
diff --git a/bluesky/store-s3.c b/bluesky/store-s3.c
index 7f23434..4735de4 100644
--- a/bluesky/store-s3.c
+++ b/bluesky/store-s3.c
@@ -42,6 +42,7 @@ struct put_info {
 struct list_info {
     int success;
     char *last_entry;
+    gboolean truncated;
 };
 
 static S3Status s3store_get_handler(int bufferSize, const char *buffer,
@@ -145,13 +146,14 @@ static S3Status s3store_list_handler(int isTruncated,
         g_free(info->last_entry);
         info->last_entry = g_strdup(contents[contentsCount - 1].key);
     }
+    info->truncated = isTruncated;
     return S3StatusOK;
 }
 
 static char *s3store_lookup_last(gpointer s, const char *prefix)
 {
     S3Store *store = (S3Store *)s;
-    struct list_info info = {0, NULL};
+    struct list_info info = {0, NULL, FALSE};
 
     struct S3ListBucketHandler handler;
     handler.responseHandler.propertiesCallback
@@ -161,9 +163,15 @@ static char *s3store_lookup_last(gpointer s, const char *prefix)
 
     char *marker = NULL;
 
-    S3_list_bucket(&store->bucket, prefix, marker, NULL, 1024, NULL, &handler, &info);
+    do {
+        S3_list_bucket(&store->bucket, prefix, marker, NULL, 1024, NULL,
+                       &handler, &info);
+        g_free(marker);
+        marker = g_strdup(info.last_entry);
+        g_print("Last key: %s\n", info.last_entry);
+    } while (info.truncated);
 
-    g_print("Last key: %s\n", info.last_entry);
+    g_free(marker);
 
     return info.last_entry;
 }