From 358b7ac6f741088838fb84979c9127976eb34fdf Mon Sep 17 00:00:00 2001 From: Michael Vrable Date: Fri, 29 Dec 2006 19:24:48 -0800 Subject: [PATCH] Support for spreading objects across segments. New segments are created automatically whenever an old segment becomes too large (current limit is set at something over 1 MB, but this could be adjusted). --- scandir.cc | 14 ++++++++++++-- store.cc | 28 ++++++++++++++++++++++++++++ store.h | 22 ++++++++++++++++++++++ 3 files changed, 62 insertions(+), 2 deletions(-) diff --git a/scandir.cc b/scandir.cc index d0ed67d..043d35e 100644 --- a/scandir.cc +++ b/scandir.cc @@ -20,8 +20,11 @@ using std::string; using std::vector; +static SegmentStore *segment_store; static OutputStream *info_dump = NULL; +static SegmentPartitioner *index_segment, *data_segment; + void scandir(const string& path); /* Converts time to microseconds since the epoch. */ @@ -55,6 +58,8 @@ void dumpfile(int fd, dictionary &file_info) break; } else { hash.process(buf, res); + OutputStream *block = data_segment->new_object(); + block->write(buf, res); size += res; } } @@ -201,10 +206,13 @@ void scandir(const string& path) int main(int argc, char *argv[]) { - SegmentStore ss("."); - SegmentWriter *sw = ss.new_segment(); + segment_store = new SegmentStore("."); + SegmentWriter *sw = segment_store->new_segment(); info_dump = sw->new_object(); + index_segment = new SegmentPartitioner(segment_store); + data_segment = new SegmentPartitioner(segment_store); + string uuid = SegmentWriter::format_uuid(sw->get_uuid()); printf("Backup UUID: %s\n", uuid.c_str()); @@ -214,6 +222,8 @@ int main(int argc, char *argv[]) fprintf(stderr, "IOException: %s\n", e.getError().c_str()); } + delete index_segment; + delete data_segment; delete sw; return 0; diff --git a/store.cc b/store.cc index 52304c2..06e9453 100644 --- a/store.cc +++ b/store.cc @@ -288,3 +288,31 @@ SegmentWriter *SegmentStore::new_segment() return new SegmentWriter(new FileOutputStream(f), id); } + +SegmentPartitioner::SegmentPartitioner(SegmentStore *s) + : store(s), + segment(NULL), + object(NULL) +{ + // Default target size is around 1 MB + target_size = 1024 * 1024; +} + +SegmentPartitioner::~SegmentPartitioner() +{ + if (segment) + delete segment; +} + +OutputStream *SegmentPartitioner::new_object() +{ + if (segment != NULL && segment->get_size() > target_size) { + delete segment; + segment = NULL; + } + + if (segment == NULL) + segment = store->new_segment(); + + return segment->new_object(); +} diff --git a/store.h b/store.h index 0bc2171..7629673 100644 --- a/store.h +++ b/store.h @@ -161,6 +161,9 @@ public: OutputStream *new_object(); void finish_object(); + // Determine size of segment data written out so far. + size_t get_size() const { return raw_out->get_pos(); } + // Utility functions for generating and formatting UUIDs for display. static struct uuid generate_uuid(); static std::string format_uuid(const struct uuid u); @@ -193,4 +196,23 @@ private: std::string directory; }; +/* A SegmentPartitioner helps to divide objects up among a collection of + * segments to meet a rough size limit per segment. Like a SegmentWriter, only + * one object should be written at a time; however, multiple + * SegmentPartitioners can be created using the same base SegmentStore. */ +class SegmentPartitioner { +public: + explicit SegmentPartitioner(SegmentStore *s); + ~SegmentPartitioner(); + + OutputStream *new_object(); + +private: + size_t target_size; + + SegmentStore *store; + SegmentWriter *segment; + OutputStream *object; +}; + #endif // _LBS_STORE_H -- 2.20.1