X-Git-Url: http://git.vrable.net/?p=cumulus.git;a=blobdiff_plain;f=metadata.cc;h=96d21ea27d31e512c9e910aa727bee3d2eb75117;hp=2517d7341fb1b9d5c4b04402a750d65123ed8836;hb=HEAD;hpb=bae7cf459803abce603c8b3d3009153936378b42 diff --git a/metadata.cc b/metadata.cc index 2517d73..96d21ea 100644 --- a/metadata.cc +++ b/metadata.cc @@ -1,29 +1,55 @@ -/* LBS: An LFS-inspired filesystem backup system - * Copyright (C) 2007 Michael Vrable +/* Cumulus: Efficient Filesystem Backup to the Cloud + * Copyright (C) 2007-2008 The Cumulus Developers + * See the AUTHORS file for a list of contributors. * - * Handling of metadata written to backup snapshots. This manages the writing + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +/* Handling of metadata written to backup snapshots. This manages the writing * of file metadata into new backup snapshots, including breaking the metadata * log apart across separate objects. Eventually this should include unified * handling of the statcache, and re-use of metadata between snapshots. */ +#include +#include +#include #include #include +#include #include "metadata.h" +#include "localdb.h" #include "ref.h" #include "store.h" #include "util.h" using std::list; +using std::map; using std::string; using std::ostream; using std::ostringstream; static const size_t LBS_METADATA_BLOCK_SIZE = 65536; +// If true, forces a full write of metadata: will not include pointers to +// metadata in old snapshots. +bool flag_full_metadata = false; + /* TODO: Move to header file */ -void add_segment(const string& segment); +extern LocalDb *db; /* Like strcmp, but sorts in the order that files will be visited in the * filesystem. That is, we break paths apart at slashes, and compare path @@ -63,6 +89,27 @@ static int pathcmp(const char *path1, const char *path2) return pathcmp(slash1 + 1, slash2 + 1); } +/* Encode a dictionary of string key/value pairs into a sequence of lines of + * the form "key: value". If it exists, the key "name" is treated specially + * and will be listed first. */ +static string encode_dict(const map& dict) +{ + string result; + + if (dict.find("name") != dict.end()) { + result += "name: " + dict.find("name")->second + "\n"; + } + + for (map::const_iterator i = dict.begin(); + i != dict.end(); ++i) { + if (i->first == "name") + continue; + result += i->first + ": " + i->second + "\n"; + } + + return result; +} + MetadataWriter::MetadataWriter(TarSegmentStore *store, const char *path, const char *snapshot_name, @@ -70,7 +117,7 @@ MetadataWriter::MetadataWriter(TarSegmentStore *store, { statcache_path = path; statcache_path += "/statcache2"; - if (snapshot_scheme != NULL) + if (snapshot_scheme != NULL && strlen(snapshot_scheme) > 0) statcache_path = statcache_path + "-" + snapshot_scheme; statcache_tmp_path = statcache_path + "." + snapshot_name; @@ -80,10 +127,9 @@ MetadataWriter::MetadataWriter(TarSegmentStore *store, if (statcache_out == NULL) { fprintf(stderr, "Error opening statcache %s: %m\n", statcache_tmp_path.c_str()); - throw IOException("Error opening statcache"); + fatal("Error opening statcache"); } - found_match = false; old_metadata_eof = false; this->store = store; @@ -162,30 +208,27 @@ bool MetadataWriter::find(const string& path) { const char *path_str = path.c_str(); while (!old_metadata_eof) { - string old_path = uri_decode(old_metadata["path"]); + string old_path = uri_decode(old_metadata["name"]); int cmp = pathcmp(old_path.c_str(), path_str); - if (cmp == 0) { - found_match = true; + if (cmp == 0) return true; - } else if (cmp > 0) { - found_match = false; + else if (cmp > 0) return false; - } else { + else read_statcache(); - } } - found_match = false; return false; } /* Does a file appear to be unchanged from the previous time it was backed up, - * based on stat information? - * - * TODO: Notice files that were modified as they were being backed up the last - * time. */ + * based on stat information? */ bool MetadataWriter::is_unchanged(const struct stat *stat_buf) { + if (old_metadata.find("volatile") != old_metadata.end() + && parse_int(old_metadata["volatile"]) != 0) + return false; + if (old_metadata.find("ctime") == old_metadata.end()) return false; if (stat_buf->st_ctime != parse_int(old_metadata["ctime"])) @@ -233,11 +276,9 @@ list MetadataWriter::get_blocks() s++; } - ObjectReference *r = ObjectReference::parse(ref); - if (r != NULL) { - blocks.push_back(*r); - delete r; - } + ObjectReference r = ObjectReference::parse(ref); + if (!r.is_null()) + blocks.push_back(r); } return blocks; @@ -252,6 +293,12 @@ void MetadataWriter::metadata_flush() ObjectReference indirect; for (list::iterator i = items.begin(); i != items.end(); ++i) { + // If indirectly referencing any other metadata logs, be sure those + // segments are properly referenced. + if (i->reused) { + db->UseObject(i->ref); + } + // Write out an indirect reference to any previous objects which could // be reused if (!i->reused || !indirect.merge(i->ref)) { @@ -290,14 +337,13 @@ void MetadataWriter::metadata_flush() /* Write current metadata information to a new object. */ LbsObject *meta = new LbsObject; meta->set_group("metadata"); - meta->set_data(m.data(), m.size()); + meta->set_data(m.data(), m.size(), NULL); meta->write(store); - meta->checksum(); /* Write a reference to this block in the root. */ ObjectReference ref = meta->get_ref(); metadata_root << "@" << ref.to_string() << "\n"; - add_segment(ref.get_segment()); + db->UseObject(ref); delete meta; @@ -326,12 +372,11 @@ void MetadataWriter::add(dictionary info) item.reused = false; item.text += encode_dict(info) + "\n"; - if (info == old_metadata) { - ObjectReference *ref = ObjectReference::parse(old_metadata_loc); - if (ref != NULL) { + if (info == old_metadata && !flag_full_metadata) { + ObjectReference ref = ObjectReference::parse(old_metadata_loc); + if (!ref.is_null() && db->IsAvailable(ref)) { item.reused = true; - item.ref = *ref; - delete ref; + item.ref = ref; } } @@ -349,10 +394,9 @@ ObjectReference MetadataWriter::close() LbsObject *root = new LbsObject; root->set_group("metadata"); - root->set_data(root_data.data(), root_data.size()); + root->set_data(root_data.data(), root_data.size(), NULL); root->write(store); - root->checksum(); - add_segment(root->get_ref().get_segment()); + db->UseObject(root->get_ref()); ObjectReference ref = root->get_ref(); delete root;