1 /* LBS: An LFS-inspired filesystem backup system
2 * Copyright (C) 2007 Michael Vrable
4 * Handling of metadata written to backup snapshots. This manages the writing
5 * of file metadata into new backup snapshots, including breaking the metadata
6 * log apart across separate objects. Eventually this should include unified
7 * handling of the statcache, and re-use of metadata between snapshots.
23 using std::ostringstream;
25 static const size_t LBS_METADATA_BLOCK_SIZE = 65536;
27 /* TODO: Move to header file */
28 void add_segment(const string& segment);
30 /* Like strcmp, but sorts in the order that files will be visited in the
31 * filesystem. That is, we break paths apart at slashes, and compare path
32 * components separately. */
33 static int pathcmp(const char *path1, const char *path2)
35 /* Find the first component in each path. */
36 const char *slash1 = strchr(path1, '/');
37 const char *slash2 = strchr(path2, '/');
44 comp1 = string(path1, slash1 - path1);
49 comp2 = string(path2, slash2 - path2);
51 /* Directly compare the two components first. */
58 if (slash1 == NULL && slash2 == NULL)
65 return pathcmp(slash1 + 1, slash2 + 1);
68 /* Encode a dictionary of string key/value pairs into a sequence of lines of
69 * the form "key: value". If it exists, the key "name" is treated specially
70 * and will be listed first. */
71 static string encode_dict(const map<string, string>& dict)
75 if (dict.find("name") != dict.end()) {
76 result += "name: " + dict.at("name") + "\n";
79 for (map<string, string>::const_iterator i = dict.begin();
80 i != dict.end(); ++i) {
81 if (i->first == "name")
83 result += i->first + ": " + i->second + "\n";
89 MetadataWriter::MetadataWriter(TarSegmentStore *store,
91 const char *snapshot_name,
92 const char *snapshot_scheme)
94 statcache_path = path;
95 statcache_path += "/statcache2";
96 if (snapshot_scheme != NULL)
97 statcache_path = statcache_path + "-" + snapshot_scheme;
98 statcache_tmp_path = statcache_path + "." + snapshot_name;
100 statcache_in = fopen(statcache_path.c_str(), "r");
102 statcache_out = fopen(statcache_tmp_path.c_str(), "w");
103 if (statcache_out == NULL) {
104 fprintf(stderr, "Error opening statcache %s: %m\n",
105 statcache_tmp_path.c_str());
106 throw IOException("Error opening statcache");
109 old_metadata_eof = false;
115 /* Read the next entry from the old statcache file, loading it into
117 void MetadataWriter::read_statcache()
119 if (statcache_in == NULL) {
120 old_metadata_eof = true;
124 old_metadata.clear();
128 string field = ""; // Last field to be read in
130 /* Look for a first line starting with "@@", which tells where the metadata
131 * can be found in the metadata log of an old snapshot. */
132 if (getline(&buf, &n, statcache_in) < 0
133 || buf == NULL || buf[0] != '@' || buf[1] != '@') {
134 old_metadata_eof = true;
138 if (strchr(buf, '\n') != NULL)
139 *strchr(buf, '\n') = '\0';
140 old_metadata_loc = buf + 2;
142 /* After the initial line follows the metadata, as key-value pairs. */
143 while (!feof(statcache_in)) {
144 if (getline(&buf, &n, statcache_in) < 0)
147 char *eol = strchr(buf, '\n');
151 /* Is the line blank? If so, we have reached the end of this entry. */
155 /* Is this a continuation line? (Does it start with whitespace?) */
156 if (isspace(buf[0]) && field != "") {
157 old_metadata[field] += string("\n") + buf;
161 /* For lines of the form "Key: Value" look for ':' and split the line
163 char *value = strchr(buf, ':');
170 while (isspace(*value))
173 old_metadata[field] = value;
176 if (feof(statcache_in) && old_metadata.size() == 0) {
177 old_metadata_eof = true;
183 bool MetadataWriter::find(const string& path)
185 const char *path_str = path.c_str();
186 while (!old_metadata_eof) {
187 string old_path = uri_decode(old_metadata["name"]);
188 int cmp = pathcmp(old_path.c_str(), path_str);
200 /* Does a file appear to be unchanged from the previous time it was backed up,
201 * based on stat information?
203 * TODO: Notice files that were modified as they were being backed up the last
205 bool MetadataWriter::is_unchanged(const struct stat *stat_buf)
207 if (old_metadata.find("ctime") == old_metadata.end())
209 if (stat_buf->st_ctime != parse_int(old_metadata["ctime"]))
212 if (old_metadata.find("mtime") == old_metadata.end())
214 if (stat_buf->st_mtime != parse_int(old_metadata["mtime"]))
217 if (old_metadata.find("size") == old_metadata.end())
219 if (stat_buf->st_size != parse_int(old_metadata["size"]))
222 if (old_metadata.find("inode") == old_metadata.end())
224 string inode = encode_int(major(stat_buf->st_dev))
225 + "/" + encode_int(minor(stat_buf->st_dev))
226 + "/" + encode_int(stat_buf->st_ino);
227 if (inode != old_metadata["inode"])
233 list<ObjectReference> MetadataWriter::get_blocks()
235 list<ObjectReference> blocks;
237 /* Parse the list of blocks. */
238 const char *s = old_metadata["data"].c_str();
246 while (*s != '\0' && !isspace(*s)) {
254 ObjectReference r = ObjectReference::parse(ref);
262 /* Ensure contents of metadata are flushed to an object. */
263 void MetadataWriter::metadata_flush()
267 ostringstream metadata;
268 ObjectReference indirect;
269 for (list<MetadataItem>::iterator i = items.begin();
270 i != items.end(); ++i) {
271 // Write out an indirect reference to any previous objects which could
273 if (!i->reused || !indirect.merge(i->ref)) {
274 if (!indirect.is_null()) {
275 string refstr = indirect.to_string();
276 metadata << "@" << refstr << "\n";
277 offset += refstr.size() + 2;
286 indirect = ObjectReference();
292 offset += i->text.size();
295 if (!indirect.is_null()) {
296 string refstr = indirect.to_string();
297 metadata << "@" << refstr << "\n";
298 offset += refstr.size() + 2;
299 indirect = ObjectReference();
302 string m = metadata.str();
306 /* Write current metadata information to a new object. */
307 LbsObject *meta = new LbsObject;
308 meta->set_group("metadata");
309 meta->set_data(m.data(), m.size());
313 /* Write a reference to this block in the root. */
314 ObjectReference ref = meta->get_ref();
315 metadata_root << "@" << ref.to_string() << "\n";
316 add_segment(ref.get_segment());
320 /* Write these files out to the statcache, and include a reference to where
321 * the metadata lives (so we can re-use it if it has not changed). */
322 for (list<MetadataItem>::const_iterator i = items.begin();
323 i != items.end(); ++i) {
324 ObjectReference r = ref;
325 r.set_range(i->offset, i->text.size());
330 string refstr = r.to_string();
331 fprintf(statcache_out, "@@%s\n%s", refstr.c_str(), i->text.c_str());
338 void MetadataWriter::add(dictionary info)
343 item.text += encode_dict(info) + "\n";
345 if (info == old_metadata) {
346 ObjectReference ref = ObjectReference::parse(old_metadata_loc);
347 if (!ref.is_null()) {
353 items.push_back(item);
354 chunk_size += item.text.size();
356 if (chunk_size > LBS_METADATA_BLOCK_SIZE)
360 ObjectReference MetadataWriter::close()
363 const string root_data = metadata_root.str();
365 LbsObject *root = new LbsObject;
366 root->set_group("metadata");
367 root->set_data(root_data.data(), root_data.size());
370 add_segment(root->get_ref().get_segment());
372 ObjectReference ref = root->get_ref();
375 fclose(statcache_out);
376 if (rename(statcache_tmp_path.c_str(), statcache_path.c_str()) < 0) {
377 fprintf(stderr, "Error renaming statcache from %s to %s: %m\n",
378 statcache_tmp_path.c_str(), statcache_path.c_str());