#include <iostream>
#include <list>
#include <map>
+#include <memory>
#include <set>
#include <sstream>
#include <string>
using std::string;
using std::vector;
using std::ostream;
+using std::unique_ptr;
/* Version information. This will be filled in by the Makefile. */
#ifndef CUMULUS_VERSION
/* If the file is new or changed, we must read in the contents a block at a
* time. */
if (!cached) {
- Hash *hash = Hash::New();
+ unique_ptr<Hash> file_hash(Hash::New());
Subfile subfile(db);
subfile.load_old_blocks(old_blocks);
break;
}
- hash->update(block_buf, bytes);
+ file_hash->update(block_buf, bytes);
// Sparse file processing: if we read a block of all zeroes, encode
// that explicitly.
double block_age = 0.0;
ObjectReference ref;
- Hash *hash = Hash::New();
- hash->update(block_buf, bytes);
- string block_csum = hash->digest_str();
- delete hash;
+ unique_ptr<Hash> block_hash(Hash::New());
+ block_hash->update(block_buf, bytes);
+ string block_csum = block_hash->digest_str();
if (all_zero) {
ref = ObjectReference(ObjectReference::REF_ZERO);
if (object_group == 0) {
o->set_group("data");
} else {
- char group[32];
- sprintf(group, "compacted-%d", object_group);
- o->set_group(group);
+ o->set_group(string_printf("compacted-%d",
+ object_group));
}
if (status == NULL)
status = "partial";
while (!refs.empty()) {
ref = refs.front(); refs.pop_front();
+
+ // The file-level checksum guarantees integrity of the data.
+ // To reduce the metadata log size, do not include checksums on
+ // individual objects.
+ ref.clear_checksum();
+
object_list.push_back(ref.to_string());
db->UseObject(ref);
}
status = "old";
}
- file_info["checksum"] = hash->digest_str();
- delete hash;
+ file_info["checksum"] = file_hash->digest_str();
}
- // Sanity check: if we are rebuilding the statcache, but the file looks
- // like it hasn't changed, then the newly-computed checksum should match
- // the checksum in the statcache. If not, we have possible disk corruption
- // and report a warning.
- if (flag_rebuild_statcache) {
- if (found
- && metawriter->is_unchanged(&stat_buf)
- && file_info["checksum"] != metawriter->get_checksum()) {
- fprintf(stderr,
- "Warning: Checksum for %s does not match expected value\n"
- " expected: %s\n"
- " actual: %s\n",
- path.c_str(),
- metawriter->get_checksum().c_str(),
- file_info["checksum"].c_str());
- }
+ // Sanity check: if the file looks like it hasn't changed, then the
+ // newly-computed checksum should match the checksum in the statcache. If
+ // not, we have possible disk corruption and report a warning.
+ if (found
+ && metawriter->is_unchanged(&stat_buf)
+ && file_info["checksum"] != metawriter->get_checksum()) {
+ fprintf(stderr,
+ "Warning: Checksum for %s does not match expected value\n"
+ " expected: %s\n"
+ " actual: %s\n",
+ path.c_str(),
+ metawriter->get_checksum().c_str(),
+ file_info["checksum"].c_str());
}
if (verbose && status != NULL)
tss->dump_stats();
delete tss;
- /* Write out a checksums file which lists the checksums for all the
- * segments included in this snapshot. The format is designed so that it
- * may be easily verified using the sha1sums command. */
- const char csum_type[] = "sha1";
- string checksum_filename = "snapshot-";
- if (backup_scheme.size() > 0)
- checksum_filename += backup_scheme + "-";
- checksum_filename
- = checksum_filename + timestamp + "." + csum_type + "sums";
- RemoteFile *checksum_file = remote->alloc_file(checksum_filename,
- "meta");
- FILE *checksums = fdopen(checksum_file->get_fd(), "w");
-
- std::set<string> segment_list = db->GetUsedSegments();
- for (std::set<string>::iterator i = segment_list.begin();
- i != segment_list.end(); ++i) {
- map<string, string> segment_metadata = db->GetSegmentMetadata(*i);
- if (segment_metadata.count("path")
- && segment_metadata.count("checksum"))
- {
- string seg_path = segment_metadata["path"];
- string seg_csum = segment_metadata["checksum"];
- const char *raw_checksum = NULL;
- if (strncmp(seg_csum.c_str(), csum_type,
- strlen(csum_type)) == 0) {
- raw_checksum = seg_csum.c_str() + strlen(csum_type);
- if (*raw_checksum == '=')
- raw_checksum++;
- else
- raw_checksum = NULL;
- }
-
- if (raw_checksum != NULL)
- fprintf(checksums, "%s *%s\n",
- raw_checksum, seg_path.c_str());
- }
- }
- fclose(checksums);
-
- SHA1Checksum checksum_csum;
- string csum;
- checksum_filename = checksum_file->get_local_path();
- if (checksum_csum.process_file(checksum_filename.c_str())) {
- csum = checksum_csum.checksum_str();
- }
-
- checksum_file->send();
-
/* Write out a summary file with metadata for all the segments in this
- * snapshot (can be used to reconstruct database contents if needed). */
+ * snapshot (can be used to reconstruct database contents if needed), and
+ * contains hash values for the segments for quick integrity checks. */
string dbmeta_filename = "snapshot-";
if (backup_scheme.size() > 0)
dbmeta_filename += backup_scheme + "-";
- dbmeta_filename += timestamp + ".meta";
- RemoteFile *dbmeta_file = remote->alloc_file(dbmeta_filename,
- "meta");
- FILE *dbmeta = fdopen(dbmeta_file->get_fd(), "w");
+ dbmeta_filename += timestamp + ".meta" + filter_extension;
+ RemoteFile *dbmeta_file = remote->alloc_file(dbmeta_filename, "meta");
+ unique_ptr<FileFilter> dbmeta_filter(FileFilter::New(dbmeta_file->get_fd(),
+ filter_program));
+ if (dbmeta_filter == NULL) {
+ fprintf(stderr, "Unable to open descriptor output file: %m\n");
+ return 1;
+ }
+ FILE *dbmeta = fdopen(dbmeta_filter->get_wrapped_fd(), "w");
+ std::set<string> segment_list = db->GetUsedSegments();
for (std::set<string>::iterator i = segment_list.begin();
i != segment_list.end(); ++i) {
map<string, string> segment_metadata = db->GetSegmentMetadata(*i);
}
}
fclose(dbmeta);
+ dbmeta_filter->wait();
string dbmeta_csum
= Hash::hash_file(dbmeta_file->get_local_path().c_str());
RemoteFile *descriptor_file = remote->alloc_file(desc_filename,
"snapshots");
- int descriptor_fd = descriptor_file->get_fd();
- if (descriptor_fd < 0) {
+ unique_ptr<FileFilter> descriptor_filter(
+ FileFilter::New(descriptor_file->get_fd(), signature_filter.c_str()));
+ if (descriptor_filter == NULL) {
fprintf(stderr, "Unable to open descriptor output file: %m\n");
return 1;
}
- pid_t signature_pid = 0;
- if (signature_filter.size() > 0) {
- int new_fd = spawn_filter(descriptor_fd, signature_filter.c_str(),
- &signature_pid);
- close(descriptor_fd);
- descriptor_fd = new_fd;
- }
- FILE *descriptor = fdopen(descriptor_fd, "w");
+ FILE *descriptor = fdopen(descriptor_filter->get_wrapped_fd(), "w");
fprintf(descriptor, "Format: Cumulus Snapshot v0.11\n");
fprintf(descriptor, "Producer: Cumulus %s\n", cumulus_version);
fprintf(descriptor, "Root: %s\n", backup_root.c_str());
if (dbmeta_csum.size() > 0) {
- fprintf(descriptor, "Database-state: %s\n", dbmeta_csum.c_str());
- }
-
- if (csum.size() > 0) {
- fprintf(descriptor, "Checksums: %s\n", csum.c_str());
+ fprintf(descriptor, "Segment-metadata: %s\n", dbmeta_csum.c_str());
}
fprintf(descriptor, "Segments:\n");
}
fclose(descriptor);
-
- if (signature_pid) {
- int status;
- waitpid(signature_pid, &status, 0);
-
- if (!WIFEXITED(status) || WEXITSTATUS(status) != 0) {
- fatal("Signature filter process error");
- }
+ if (descriptor_filter->wait() < 0) {
+ fatal("Signature filter process error");
}
descriptor_file->send();
projects / cumulus.git / blobdiff