Add checksums to tarstore, and partially migrate to using it for storage.

[cumulus.git] / tarstore.cc

diff --git a/tarstore.cc b/tarstore.cc
index 1fad845..78b089b 100644 (file)
--- a/tarstore.cc
+++ b/tarstore.cc
@@ -15,6 +15,7 @@
 #include <uuid/uuid.h>
 
 #include <string>
+#include <iostream>
 
 #include "tarstore.h"
 
@@ -30,6 +31,9 @@ Tarfile::Tarfile(const string &path, const string &segment)
 
 Tarfile::~Tarfile()
 {
+    string checksum_list = checksums.str();
+    internal_write_object(segment_name + "/checksums",
+                          checksum_list.data(), checksum_list.size());
     tar_append_eof(t);
 
     if (tar_close(t) != 0)
@@ -38,13 +42,26 @@ Tarfile::~Tarfile()
 
 void Tarfile::write_object(int id, const char *data, size_t len)
 {
-    memset(&t->th_buf, 0, sizeof(struct tar_header));
-
     char buf[64];
     sprintf(buf, "%08x", id);
     string path = segment_name + "/" + buf;
     printf("path: %s\n", path.c_str());
 
+    internal_write_object(path, data, len);
+
+    // Compute a checksum for the data block, which will be stored at the end
+    // of the TAR file.
+    SHA1Checksum hash;
+    hash.process(data, len);
+    sprintf(buf, "%08x", id);
+    checksums << buf << " " << hash.checksum_str() << "\n";
+}
+
+void Tarfile::internal_write_object(const string &path,
+                                    const char *data, size_t len)
+{
+    memset(&t->th_buf, 0, sizeof(struct tar_header));
+
     th_set_type(t, S_IFREG | 0600);
     th_set_user(t, 0);
     th_set_group(t, 0);
@@ -76,3 +93,54 @@ void Tarfile::write_object(int id, const char *data, size_t len)
     if (tar_block_write(t, block) == -1)
         throw IOException("Error writing final tar block");
 }
+
+string TarSegmentStore::write_object(const char *data, size_t len, const
+                                     std::string &group)
+{
+    struct segment_info *segment;
+
+    // Find the segment into which the object should be written, looking up by
+    // group.  If no segment exists yet, create one.
+    if (segments.find(group) == segments.end()) {
+        segment = new segment_info;
+
+        uint8_t uuid[16];
+        char uuid_buf[40];
+        uuid_generate(uuid);
+        uuid_unparse_lower(uuid, uuid_buf);
+        segment->name = uuid_buf;
+
+        string filename = path + "/" + segment->name + ".tar";
+        segment->file = new Tarfile(filename, segment->name);
+
+        segment->count = 0;
+
+        segments[group] = segment;
+    } else {
+        segment = segments[group];
+    }
+
+    int id = segment->count;
+    char id_buf[64];
+    sprintf(id_buf, "%08x", id);
+
+    segment->file->write_object(id, data, len);
+    segment->count++;
+
+    return segment->name + "/" + id_buf;
+}
+
+void TarSegmentStore::sync()
+{
+    while (!segments.empty()) {
+        const string &name = segments.begin()->first;
+        struct segment_info *segment = segments[name];
+
+        fprintf(stderr, "Closing segment group %s (%s)\n",
+                name.c_str(), segment->name.c_str());
+
+        delete segment->file;
+        segments.erase(segments.begin());
+        delete segment;
+    }
+}