From: Michael Vrable Date: Sun, 24 Dec 2006 04:15:31 +0000 (-0800) Subject: Initial support for objects encapsulated into segments. X-Git-Url: http://git.vrable.net/?p=cumulus.git;a=commitdiff_plain;h=08721c894385b530ed501498a02d824b8eba7228 Initial support for objects encapsulated into segments. --- diff --git a/scandir.cc b/scandir.cc index 81ddd4b..98285ac 100644 --- a/scandir.cc +++ b/scandir.cc @@ -71,6 +71,9 @@ void scanfile(const string& path) char *buf; ssize_t len; + // Set to true if the item is a directory and we should recursively scan + bool recurse = false; + dictionary file_info; lstat(path.c_str(), &stat_buf); @@ -148,7 +151,7 @@ void scanfile(const string& path) break; case S_IFDIR: inode_type = 'd'; - scandir(path); + recurse = true; break; default: @@ -160,6 +163,11 @@ void scanfile(const string& path) info_dump->write_string(path); info_dump->write_dictionary(file_info); + + // If we hit a directory, now that we've written the directory itself, + // recursively scan the directory. + if (recurse) + scandir(path); } void scandir(const string& path) @@ -193,14 +201,19 @@ void scandir(const string& path) int main(int argc, char *argv[]) { - FILE *dump = fopen("fileinfo", "w"); + struct uuid id = SegmentWriter::generate_uuid(); + string filename = SegmentWriter::format_uuid(id); + + printf("Backup UUID: %s\n", filename.c_str()); + FILE *dump = fopen(filename.c_str(), "w"); if (dump == NULL) { - fprintf(stderr, "Cannot open fileinfo: %m\n"); + fprintf(stderr, "Cannot open file %s: %m\n", filename.c_str()); return 1; } FileOutputStream os(dump); - info_dump = &os; + SegmentWriter sw(os, id); + info_dump = sw.new_object(); try { scanfile("."); diff --git a/store.cc b/store.cc index 6770e1c..2dc6f84 100644 --- a/store.cc +++ b/store.cc @@ -6,6 +6,7 @@ * reading and writing objects and segments. */ #include +#include #include "store.h" @@ -133,6 +134,16 @@ void FileOutputStream::write_internal(const void *data, size_t len) } } +WrapperOutputStream::WrapperOutputStream(OutputStream &o) + : real(o) +{ +} + +void WrapperOutputStream::write_internal(const void *data, size_t len) +{ + real.write(data, len); +} + /* Utility functions, for encoding data types to strings. */ string encode_u16(uint16_t val) { @@ -154,3 +165,78 @@ string encode_u64(uint64_t val) s.write_u64(val); return s.contents(); } + +SegmentWriter::SegmentWriter(OutputStream &output, struct uuid u) + : out(output), + id(u), + object_stream(NULL) +{ + /* Write out the segment header first. */ + static const char signature[] = "LBSSEG0\n"; + out.write(signature, strlen(signature)); + out.write(id.bytes, sizeof(struct uuid)); +} + +SegmentWriter::~SegmentWriter() +{ + if (object_stream) + finish_object(); + + // Write out the object table which gives the sizes and locations of all + // objects, and then add the trailing signature, which indicates the end of + // the segment and gives the offset of the object table. + int64_t index_offset = out.get_pos(); + + for (object_table::const_iterator i = objects.begin(); + i != objects.end(); ++i) { + out.write_s64(i->first); + out.write_s64(i->second); + } + + static const char signature2[] = "LBSEND"; + out.write(signature2, strlen(signature2)); + out.write_s64(index_offset); + out.write_u32(objects.size()); +} + +OutputStream *SegmentWriter::new_object() +{ + if (object_stream) + finish_object(); + + object_start_offset = out.get_pos(); + object_stream = new WrapperOutputStream(out); + + return object_stream; +} + +void SegmentWriter::finish_object() +{ + assert(object_stream != NULL); + + // store (start, length) information for locating this object + objects.push_back(std::make_pair(object_start_offset, + object_stream->get_pos())); + + delete object_stream; + object_stream = NULL; +} + +struct uuid SegmentWriter::generate_uuid() +{ + struct uuid u; + + uuid_generate(u.bytes); + + return u; +} + +string SegmentWriter::format_uuid(const struct uuid u) +{ + // A UUID only takes 36 bytes, plus the trailing '\0', so this is safe. + char buf[40]; + + uuid_unparse_lower(u.bytes, buf); + + return string(buf); +} diff --git a/store.h b/store.h index 3725a9f..8344dda 100644 --- a/store.h +++ b/store.h @@ -14,6 +14,7 @@ #include #include #include +#include /* In memory datatype to represent key/value pairs of information, such as file * metadata. Currently implemented as map. */ @@ -97,10 +98,55 @@ private: FILE *f; }; +/* An OutputStream which is simply sends writes to another OutputStream, but + * does provide separate tracking of bytes written. */ +class WrapperOutputStream : public OutputStream { +public: + explicit WrapperOutputStream(OutputStream &o); + virtual ~WrapperOutputStream() { } + +protected: + virtual void write_internal(const void *data, size_t len); + +private: + OutputStream ℜ +}; + /* Simple wrappers that encode integers using a StringOutputStream and return * the encoded result. */ std::string encode_u16(uint16_t val); std::string encode_u32(uint32_t val); std::string encode_u64(uint64_t val); +struct uuid { + uint8_t bytes[16]; +}; + +class SegmentWriter { +public: + SegmentWriter(OutputStream &output, struct uuid u); + ~SegmentWriter(); + + struct uuid get_uuid() const { return id; } + + // Start writing out a new object to this segment. + OutputStream *new_object(); + void finish_object(); + + // Utility functions for generating and formatting UUIDs for display. + static struct uuid generate_uuid(); + static std::string format_uuid(const struct uuid u); + +private: + typedef std::vector > object_table; + + OutputStream &out; + struct uuid id; + + int64_t object_start_offset; + OutputStream *object_stream; + + object_table objects; +}; + #endif // _LBS_STORE_H