X-Git-Url: http://git.vrable.net/?p=cumulus.git;a=blobdiff_plain;f=metadata.cc;h=15bd7a51647d7ac02665fdd7f606516b804a8412;hp=7aa9822cee71e6a7c9ea9b7fa7b9ea2d35866573;hb=3d780590edec4583eb3ef0ca16120afd0f7451f9;hpb=0ac19625c9ad293791b93c931331aa5a0c876f36 diff --git a/metadata.cc b/metadata.cc index 7aa9822..15bd7a5 100644 --- a/metadata.cc +++ b/metadata.cc @@ -1,30 +1,54 @@ -/* LBS: An LFS-inspired filesystem backup system - * Copyright (C) 2007 Michael Vrable +/* Cumulus: Efficient Filesystem Backup to the Cloud + * Copyright (C) 2007-2008 The Cumulus Developers + * See the AUTHORS file for a list of contributors. * - * Handling of metadata written to backup snapshots. This manages the writing + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +/* Handling of metadata written to backup snapshots. This manages the writing * of file metadata into new backup snapshots, including breaking the metadata * log apart across separate objects. Eventually this should include unified * handling of the statcache, and re-use of metadata between snapshots. */ +#include +#include #include #include +#include #include "metadata.h" +#include "localdb.h" #include "ref.h" #include "store.h" -#include "statcache.h" #include "util.h" using std::list; +using std::map; using std::string; using std::ostream; using std::ostringstream; static const size_t LBS_METADATA_BLOCK_SIZE = 65536; +// If true, forces a full write of metadata: will not include pointers to +// metadata in old snapshots. +bool flag_full_metadata = false; + /* TODO: Move to header file */ -void add_segment(const string& segment); +extern LocalDb *db; /* Like strcmp, but sorts in the order that files will be visited in the * filesystem. That is, we break paths apart at slashes, and compare path @@ -64,6 +88,27 @@ static int pathcmp(const char *path1, const char *path2) return pathcmp(slash1 + 1, slash2 + 1); } +/* Encode a dictionary of string key/value pairs into a sequence of lines of + * the form "key: value". If it exists, the key "name" is treated specially + * and will be listed first. */ +static string encode_dict(const map& dict) +{ + string result; + + if (dict.find("name") != dict.end()) { + result += "name: " + dict.find("name")->second + "\n"; + } + + for (map::const_iterator i = dict.begin(); + i != dict.end(); ++i) { + if (i->first == "name") + continue; + result += i->first + ": " + i->second + "\n"; + } + + return result; +} + MetadataWriter::MetadataWriter(TarSegmentStore *store, const char *path, const char *snapshot_name, @@ -71,7 +116,7 @@ MetadataWriter::MetadataWriter(TarSegmentStore *store, { statcache_path = path; statcache_path += "/statcache2"; - if (snapshot_scheme != NULL) + if (snapshot_scheme != NULL && strlen(snapshot_scheme) > 0) statcache_path = statcache_path + "-" + snapshot_scheme; statcache_tmp_path = statcache_path + "." + snapshot_name; @@ -81,7 +126,7 @@ MetadataWriter::MetadataWriter(TarSegmentStore *store, if (statcache_out == NULL) { fprintf(stderr, "Error opening statcache %s: %m\n", statcache_tmp_path.c_str()); - throw IOException("Error opening statcache"); + fatal("Error opening statcache"); } old_metadata_eof = false; @@ -162,7 +207,7 @@ bool MetadataWriter::find(const string& path) { const char *path_str = path.c_str(); while (!old_metadata_eof) { - string old_path = uri_decode(old_metadata["path"]); + string old_path = uri_decode(old_metadata["name"]); int cmp = pathcmp(old_path.c_str(), path_str); if (cmp == 0) return true; @@ -175,6 +220,69 @@ bool MetadataWriter::find(const string& path) return false; } +/* Does a file appear to be unchanged from the previous time it was backed up, + * based on stat information? */ +bool MetadataWriter::is_unchanged(const struct stat *stat_buf) +{ + if (old_metadata.find("volatile") != old_metadata.end() + && parse_int(old_metadata["volatile"]) != 0) + return false; + + if (old_metadata.find("ctime") == old_metadata.end()) + return false; + if (stat_buf->st_ctime != parse_int(old_metadata["ctime"])) + return false; + + if (old_metadata.find("mtime") == old_metadata.end()) + return false; + if (stat_buf->st_mtime != parse_int(old_metadata["mtime"])) + return false; + + if (old_metadata.find("size") == old_metadata.end()) + return false; + if (stat_buf->st_size != parse_int(old_metadata["size"])) + return false; + + if (old_metadata.find("inode") == old_metadata.end()) + return false; + string inode = encode_int(major(stat_buf->st_dev)) + + "/" + encode_int(minor(stat_buf->st_dev)) + + "/" + encode_int(stat_buf->st_ino); + if (inode != old_metadata["inode"]) + return false; + + return true; +} + +list MetadataWriter::get_blocks() +{ + list blocks; + + /* Parse the list of blocks. */ + const char *s = old_metadata["data"].c_str(); + while (*s != '\0') { + if (isspace(*s)) { + s++; + continue; + } + + string ref = ""; + while (*s != '\0' && !isspace(*s)) { + char buf[2]; + buf[0] = *s; + buf[1] = '\0'; + ref += buf; + s++; + } + + ObjectReference r = ObjectReference::parse(ref); + if (!r.is_null()) + blocks.push_back(r); + } + + return blocks; +} + /* Ensure contents of metadata are flushed to an object. */ void MetadataWriter::metadata_flush() { @@ -184,6 +292,12 @@ void MetadataWriter::metadata_flush() ObjectReference indirect; for (list::iterator i = items.begin(); i != items.end(); ++i) { + // If indirectly referencing any other metadata logs, be sure those + // segments are properly referenced. + if (i->reused) { + db->UseObject(i->ref); + } + // Write out an indirect reference to any previous objects which could // be reused if (!i->reused || !indirect.merge(i->ref)) { @@ -222,14 +336,13 @@ void MetadataWriter::metadata_flush() /* Write current metadata information to a new object. */ LbsObject *meta = new LbsObject; meta->set_group("metadata"); - meta->set_data(m.data(), m.size()); + meta->set_data(m.data(), m.size(), NULL); meta->write(store); - meta->checksum(); /* Write a reference to this block in the root. */ ObjectReference ref = meta->get_ref(); metadata_root << "@" << ref.to_string() << "\n"; - add_segment(ref.get_segment()); + db->UseObject(ref); delete meta; @@ -258,12 +371,11 @@ void MetadataWriter::add(dictionary info) item.reused = false; item.text += encode_dict(info) + "\n"; - if (info == old_metadata) { - ObjectReference *ref = ObjectReference::parse(old_metadata_loc); - if (ref != NULL) { + if (info == old_metadata && !flag_full_metadata) { + ObjectReference ref = ObjectReference::parse(old_metadata_loc); + if (!ref.is_null()) { item.reused = true; - item.ref = *ref; - delete ref; + item.ref = ref; } } @@ -281,10 +393,9 @@ ObjectReference MetadataWriter::close() LbsObject *root = new LbsObject; root->set_group("metadata"); - root->set_data(root_data.data(), root_data.size()); + root->set_data(root_data.data(), root_data.size(), NULL); root->write(store); - root->checksum(); - add_segment(root->get_ref().get_segment()); + db->UseObject(root->get_ref()); ObjectReference ref = root->get_ref(); delete root;