From: Michael Vrable <mvrable@cs.ucsd.edu>
Date: Sat, 30 Dec 2006 03:24:48 +0000 (-0800)
Subject: Support for spreading objects across segments.
X-Git-Url: http://git.vrable.net/?p=cumulus.git;a=commitdiff_plain;h=358b7ac6f741088838fb84979c9127976eb34fdf

Support for spreading objects across segments.

New segments are created automatically whenever an old segment becomes too
large (current limit is set at something over 1 MB, but this could be
adjusted).
---

diff --git a/scandir.cc b/scandir.cc
index d0ed67d..043d35e 100644
--- a/scandir.cc
+++ b/scandir.cc
@@ -20,8 +20,11 @@
 using std::string;
 using std::vector;
 
+static SegmentStore *segment_store;
 static OutputStream *info_dump = NULL;
 
+static SegmentPartitioner *index_segment, *data_segment;
+
 void scandir(const string& path);
 
 /* Converts time to microseconds since the epoch. */
@@ -55,6 +58,8 @@ void dumpfile(int fd, dictionary &file_info)
             break;
         } else {
             hash.process(buf, res);
+            OutputStream *block = data_segment->new_object();
+            block->write(buf, res);
             size += res;
         }
     }
@@ -201,10 +206,13 @@ void scandir(const string& path)
 
 int main(int argc, char *argv[])
 {
-    SegmentStore ss(".");
-    SegmentWriter *sw = ss.new_segment();
+    segment_store = new SegmentStore(".");
+    SegmentWriter *sw = segment_store->new_segment();
     info_dump = sw->new_object();
 
+    index_segment = new SegmentPartitioner(segment_store);
+    data_segment = new SegmentPartitioner(segment_store);
+
     string uuid = SegmentWriter::format_uuid(sw->get_uuid());
     printf("Backup UUID: %s\n", uuid.c_str());
 
@@ -214,6 +222,8 @@ int main(int argc, char *argv[])
         fprintf(stderr, "IOException: %s\n", e.getError().c_str());
     }
 
+    delete index_segment;
+    delete data_segment;
     delete sw;
 
     return 0;
diff --git a/store.cc b/store.cc
index 52304c2..06e9453 100644
--- a/store.cc
+++ b/store.cc
@@ -288,3 +288,31 @@ SegmentWriter *SegmentStore::new_segment()
 
     return new SegmentWriter(new FileOutputStream(f), id);
 }
+
+SegmentPartitioner::SegmentPartitioner(SegmentStore *s)
+    : store(s),
+      segment(NULL),
+      object(NULL)
+{
+    // Default target size is around 1 MB
+    target_size = 1024 * 1024;
+}
+
+SegmentPartitioner::~SegmentPartitioner()
+{
+    if (segment)
+        delete segment;
+}
+
+OutputStream *SegmentPartitioner::new_object()
+{
+    if (segment != NULL && segment->get_size() > target_size) {
+        delete segment;
+        segment = NULL;
+    }
+
+    if (segment == NULL)
+        segment = store->new_segment();
+
+    return segment->new_object();
+}
diff --git a/store.h b/store.h
index 0bc2171..7629673 100644
--- a/store.h
+++ b/store.h
@@ -161,6 +161,9 @@ public:
     OutputStream *new_object();
     void finish_object();
 
+    // Determine size of segment data written out so far.
+    size_t get_size() const { return raw_out->get_pos(); }
+
     // Utility functions for generating and formatting UUIDs for display.
     static struct uuid generate_uuid();
     static std::string format_uuid(const struct uuid u);
@@ -193,4 +196,23 @@ private:
     std::string directory;
 };
 
+/* A SegmentPartitioner helps to divide objects up among a collection of
+ * segments to meet a rough size limit per segment.  Like a SegmentWriter, only
+ * one object should be written at a time; however, multiple
+ * SegmentPartitioners can be created using the same base SegmentStore. */
+class SegmentPartitioner {
+public:
+    explicit SegmentPartitioner(SegmentStore *s);
+    ~SegmentPartitioner();
+
+    OutputStream *new_object();
+
+private:
+    size_t target_size;
+
+    SegmentStore *store;
+    SegmentWriter *segment;
+    OutputStream *object;
+};
+
 #endif // _LBS_STORE_H