1 /* LBS: An LFS-inspired filesystem backup system
2 * Copyright (C) 2007 Michael Vrable
4 * Backup data is stored in a collection of objects, which are grouped together
5 * into segments for storage purposes. This implementation of the object store
6 * is built on top of libtar, and represents segments as TAR files and objects
7 * as files within them. */
10 #include <sys/types.h>
15 #include <uuid/uuid.h>
28 list<string> TarSegmentStore::norefs;
30 Tarfile::Tarfile(const string &path, const string &segment)
34 if (tar_open(&t, (char *)path.c_str(), NULL, O_WRONLY | O_CREAT, 0600,
35 TAR_VERBOSE | TAR_GNU) == -1)
36 throw IOException("Error opening Tarfile");
41 string checksum_list = checksums.str();
42 internal_write_object(segment_name + "/checksums",
43 checksum_list.data(), checksum_list.size());
46 if (tar_close(t) != 0)
47 throw IOException("Error closing Tarfile");
50 void Tarfile::write_object(int id, const char *data, size_t len)
53 sprintf(buf, "%08x", id);
54 string path = segment_name + "/" + buf;
56 internal_write_object(path, data, len);
58 // Compute a checksum for the data block, which will be stored at the end
61 hash.process(data, len);
62 sprintf(buf, "%08x", id);
63 checksums << buf << " " << hash.checksum_str() << "\n";
66 void Tarfile::internal_write_object(const string &path,
67 const char *data, size_t len)
69 memset(&t->th_buf, 0, sizeof(struct tar_header));
71 th_set_type(t, S_IFREG | 0600);
76 th_set_mtime(t, time(NULL));
77 th_set_path(t, const_cast<char *>(path.c_str()));
81 throw IOException("Error writing tar header");
88 size_t blocks = (len + T_BLOCKSIZE - 1) / T_BLOCKSIZE;
89 size_t padding = blocks * T_BLOCKSIZE - len;
91 for (size_t i = 0; i < blocks - 1; i++) {
92 if (tar_block_write(t, &data[i * T_BLOCKSIZE]) == -1)
93 throw IOException("Error writing tar block");
96 char block[T_BLOCKSIZE];
97 memset(block, 0, sizeof(block));
98 memcpy(block, &data[T_BLOCKSIZE * (blocks - 1)], T_BLOCKSIZE - padding);
99 if (tar_block_write(t, block) == -1)
100 throw IOException("Error writing final tar block");
102 size += blocks * T_BLOCKSIZE;
105 static const size_t SEGMENT_SIZE = 4 * 1024 * 1024;
107 string TarSegmentStore::write_object(const char *data, size_t len, const
109 const std::list<std::string> &refs)
111 struct segment_info *segment;
113 // Find the segment into which the object should be written, looking up by
114 // group. If no segment exists yet, create one.
115 if (segments.find(group) == segments.end()) {
116 segment = new segment_info;
121 uuid_unparse_lower(uuid, uuid_buf);
122 segment->name = uuid_buf;
124 string filename = path + "/" + segment->name + ".tar";
125 segment->file = new Tarfile(filename, segment->name);
129 segments[group] = segment;
131 segment = segments[group];
134 int id = segment->count;
136 sprintf(id_buf, "%08x", id);
138 segment->file->write_object(id, data, len);
141 string full_name = segment->name + "/" + id_buf;
143 // Store any dependencies this object has on other segments, so they can be
144 // written when the segment is closed.
145 for (list<string>::const_iterator i = refs.begin(); i != refs.end(); ++i) {
146 segment->refs.insert(*i);
149 // If this segment meets or exceeds the size target, close it so that
150 // future objects will go into a new segment.
151 if (segment->file->size_estimate() >= SEGMENT_SIZE)
152 close_segment(group);
157 void TarSegmentStore::sync()
159 while (!segments.empty())
160 close_segment(segments.begin()->first);
163 void TarSegmentStore::close_segment(const string &group)
165 struct segment_info *segment = segments[group];
166 fprintf(stderr, "Closing segment group %s (%s)\n",
167 group.c_str(), segment->name.c_str());
170 for (set<string>::iterator i = segment->refs.begin();
171 i != segment->refs.end(); ++i) {
172 reflist += *i + "\n";
174 segment->file->internal_write_object(segment->name + "/references",
175 reflist.data(), reflist.size());
177 delete segment->file;
178 segments.erase(segments.find(group));
182 string TarSegmentStore::object_reference_to_segment(const string &object)