-/* Recursively descend the filesystem and visit each file. */
+/* LBS: An LFS-inspired filesystem backup system
+ * Copyright (C) 2006-2008 Michael Vrable
+ *
+ * Main entry point for LBS. Contains logic for traversing the filesystem and
+ * constructing a backup.
+ */
#include <dirent.h>
#include <errno.h>
#include "localdb.h"
#include "metadata.h"
+#include "remote.h"
#include "store.h"
#include "sha1.h"
#include "util.h"
#define LBS_STRINGIFY2(s) #s
static const char lbs_version[] = LBS_STRINGIFY(LBS_VERSION);
+static RemoteStore *remote = NULL;
static TarSegmentStore *tss = NULL;
static MetadataWriter *metawriter = NULL;
/* Keep track of all segments which are needed to reconstruct the snapshot. */
std::set<string> segment_list;
+/* Snapshot intent: 1=daily, 7=weekly, etc. This is not used directly, but is
+ * stored in the local database and can help guide segment cleaning and
+ * snapshot expiration policies. */
+double snapshot_intent = 1.0;
+
/* Selection of files to include/exclude in the snapshot. */
std::list<string> includes; // Paths in which files should be saved
std::list<string> excludes; // Paths which will not be saved
* re-reading the entire contents. */
bool cached = false;
- if (metawriter->matched() && metawriter->is_unchanged(&stat_buf)) {
+ if (metawriter->find(path) && metawriter->is_unchanged(&stat_buf)) {
cached = true;
list<ObjectReference> blocks = metawriter->get_blocks();
i != blocks.end(); ++i) {
const ObjectReference &ref = *i;
object_list.push_back(ref.to_string());
- segment_list.insert(ref.get_segment());
+ if (ref.is_normal())
+ add_segment(ref.get_segment());
db->UseObject(ref);
}
size = stat_buf.st_size;
hash.process(block_buf, bytes);
+ // Sparse file processing: if we read a block of all zeroes, encode
+ // that explicitly.
+ bool all_zero = true;
+ for (int i = 0; i < bytes; i++) {
+ if (block_buf[i] != 0) {
+ all_zero = false;
+ break;
+ }
+ }
+
// Either find a copy of this block in an already-existing segment,
// or index it so it can be re-used in the future
double block_age = 0.0;
+ ObjectReference ref;
+
SHA1Checksum block_hash;
block_hash.process(block_buf, bytes);
string block_csum = block_hash.checksum_str();
- ObjectReference ref = db->FindObject(block_csum, bytes);
+
+ if (all_zero) {
+ ref = ObjectReference(ObjectReference::REF_ZERO);
+ ref.set_range(0, bytes);
+ } else {
+ ref = db->FindObject(block_csum, bytes);
+ }
// Store a copy of the object if one does not yet exist
- if (ref.get_segment().size() == 0) {
+ if (ref.is_null()) {
LbsObject *o = new LbsObject;
int object_group;
o->write(tss);
ref = o->get_ref();
db->StoreObject(ref, block_csum, bytes, block_age);
+ ref.set_range(0, bytes);
delete o;
}
object_list.push_back(ref.to_string());
- segment_list.insert(ref.get_segment());
+ if (ref.is_normal())
+ add_segment(ref.get_segment());
db->UseObject(ref);
size += bytes;
ssize_t len;
printf("%s\n", path.c_str());
-
metawriter->find(path);
- file_info["path"] = uri_encode(path);
+ file_info["name"] = uri_encode(path);
file_info["mode"] = encode_int(stat_buf.st_mode & 07777, 8);
file_info["ctime"] = encode_int(stat_buf.st_ctime);
file_info["mtime"] = encode_int(stat_buf.st_mtime);
file_info["user"] = encode_int(stat_buf.st_uid);
file_info["group"] = encode_int(stat_buf.st_gid);
+ time_t now = time(NULL);
+ if (now - stat_buf.st_ctime < 30 || now - stat_buf.st_mtime < 30)
+ if ((stat_buf.st_mode & S_IFMT) != S_IFDIR)
+ file_info["volatile"] = "1";
+
struct passwd *pwd = getpwuid(stat_buf.st_uid);
if (pwd != NULL) {
file_info["user"] += " (" + uri_encode(pwd->pw_name) + ")";
if (file_size != stat_buf.st_size) {
fprintf(stderr, "Warning: Size of %s changed during reading\n",
path.c_str());
+ file_info["volatile"] = "1";
}
break;
"Produce backup snapshot of files in SOURCE and store to DEST.\n"
"\n"
"Options:\n"
- " --dest=PATH path where backup is to be written [REQUIRED]\n"
+ " --dest=PATH path where backup is to be written\n"
+ " --upload-script=COMMAND\n"
+ " program to invoke for each backup file generated\n"
" --exclude=PATH exclude files in PATH from snapshot\n"
" --localdb=PATH local backup metadata is stored in PATH\n"
+ " --tmpdir=PATH path for temporarily storing backup files\n"
+ " (defaults to TMPDIR environment variable or /tmp)\n"
" --filter=COMMAND program through which to filter segment data\n"
" (defaults to \"bzip2 -c\")\n"
" --filter-extension=EXT\n"
" (defaults to \".bz2\")\n"
" --signature-filter=COMMAND\n"
" program though which to filter descriptor\n"
- " --scheme=NAME optional name for this snapshot\n",
+ " --scheme=NAME optional name for this snapshot\n"
+ " --intent=FLOAT intended backup type: 1=daily, 7=weekly, ...\n"
+ " (defaults to \"1\")\n"
+ " --full-metadata do not re-use metadata from previous backups\n"
+ "\n"
+ "Exactly one of --dest or --upload-script must be specified.\n",
lbs_version, program
);
}
int main(int argc, char *argv[])
{
- string backup_dest = "";
+ string backup_dest = "", backup_script = "";
string localdb_dir = "";
string backup_scheme = "";
string signature_filter = "";
+ string tmp_dir = "/tmp";
+ if (getenv("TMPDIR") != NULL)
+ tmp_dir = getenv("TMPDIR");
+
while (1) {
static struct option long_options[] = {
{"localdb", 1, 0, 0}, // 0
{"dest", 1, 0, 0}, // 4
{"scheme", 1, 0, 0}, // 5
{"signature-filter", 1, 0, 0}, // 6
+ {"intent", 1, 0, 0}, // 7
+ {"full-metadata", 0, 0, 0}, // 8
+ {"tmpdir", 1, 0, 0}, // 9
+ {"upload-script", 1, 0, 0}, // 10
{NULL, 0, 0, 0},
};
case 6: // --signature-filter
signature_filter = optarg;
break;
+ case 7: // --intent
+ snapshot_intent = atof(optarg);
+ if (snapshot_intent <= 0)
+ snapshot_intent = 1;
+ break;
+ case 8: // --full-metadata
+ flag_full_metadata = true;
+ break;
+ case 9: // --tmpdir
+ tmp_dir = optarg;
+ break;
+ case 10: // --upload-script
+ backup_script = optarg;
+ break;
default:
fprintf(stderr, "Unhandled long option!\n");
return 1;
for (int i = optind; i < argc; i++)
add_include(argv[i]);
- if (backup_dest == "") {
+ if (backup_dest == "" && backup_script == "") {
fprintf(stderr,
- "Error: Backup destination must be specified with --dest=\n");
+ "Error: Backup destination must be specified using --dest= or --upload-script=\n");
+ usage(argv[0]);
+ return 1;
+ }
+
+ if (backup_dest != "" && backup_script != "") {
+ fprintf(stderr,
+ "Error: Cannot specify both --dest= and --upload-script=\n");
usage(argv[0]);
return 1;
}
if (localdb_dir == "") {
localdb_dir = backup_dest;
}
+ if (localdb_dir == "") {
+ fprintf(stderr,
+ "Error: Must specify local database path with --localdb=\n");
+ usage(argv[0]);
+ return 1;
+ }
// Dump paths for debugging/informational purposes
{
printf("LBS Version: %s\n", lbs_version);
- printf("--dest=%s\n--localdb=%s\n\n",
- backup_dest.c_str(), localdb_dir.c_str());
+ printf("--dest=%s\n--localdb=%s\n--upload-script=%s\n",
+ backup_dest.c_str(), localdb_dir.c_str(), backup_script.c_str());
printf("Includes:\n");
for (i = includes.begin(); i != includes.end(); ++i)
block_buf = new char[LBS_BLOCK_SIZE];
+ /* Initialize the remote storage layer. If using an upload script, create
+ * a temporary directory for staging files. Otherwise, write backups
+ * directly to the destination directory. */
+ if (backup_script != "") {
+ tmp_dir = tmp_dir + "/lbs." + generate_uuid();
+ if (mkdir(tmp_dir.c_str(), 0700) < 0) {
+ fprintf(stderr, "Cannot create temporary directory %s: %m\n",
+ tmp_dir.c_str());
+ return 1;
+ }
+ remote = new RemoteStore(tmp_dir);
+ remote->set_script(backup_script);
+ } else {
+ remote = new RemoteStore(backup_dest);
+ }
+
/* Store the time when the backup started, so it can be included in the
* snapshot name. */
time_t now;
string database_path = localdb_dir + "/localdb.sqlite";
db = new LocalDb;
db->Open(database_path.c_str(), desc_buf,
- backup_scheme.size() ? backup_scheme.c_str() : NULL);
+ backup_scheme.size() ? backup_scheme.c_str() : NULL,
+ snapshot_intent);
- tss = new TarSegmentStore(backup_dest, db);
+ tss = new TarSegmentStore(remote, db);
/* Initialize the stat cache, for skipping over unchanged files. */
metawriter = new MetadataWriter(tss, localdb_dir.c_str(), desc_buf,
* segments included in this snapshot. The format is designed so that it
* may be easily verified using the sha1sums command. */
const char csum_type[] = "sha1";
- string checksum_filename = backup_dest + "/snapshot-";
+ string checksum_filename = "snapshot-";
if (backup_scheme.size() > 0)
checksum_filename += backup_scheme + "-";
checksum_filename = checksum_filename + desc_buf + "." + csum_type + "sums";
- FILE *checksums = fopen(checksum_filename.c_str(), "w");
- if (checksums != NULL) {
- for (std::set<string>::iterator i = segment_list.begin();
- i != segment_list.end(); ++i) {
- string seg_path, seg_csum;
- if (db->GetSegmentChecksum(*i, &seg_path, &seg_csum)) {
- const char *raw_checksum = NULL;
- if (strncmp(seg_csum.c_str(), csum_type,
- strlen(csum_type)) == 0) {
- raw_checksum = seg_csum.c_str() + strlen(csum_type);
- if (*raw_checksum == '=')
- raw_checksum++;
- else
- raw_checksum = NULL;
- }
+ RemoteFile *checksum_file = remote->alloc_file(checksum_filename,
+ "checksums");
+ FILE *checksums = fdopen(checksum_file->get_fd(), "w");
- if (raw_checksum != NULL)
- fprintf(checksums, "%s *%s\n",
- raw_checksum, seg_path.c_str());
+ for (std::set<string>::iterator i = segment_list.begin();
+ i != segment_list.end(); ++i) {
+ string seg_path, seg_csum;
+ if (db->GetSegmentChecksum(*i, &seg_path, &seg_csum)) {
+ const char *raw_checksum = NULL;
+ if (strncmp(seg_csum.c_str(), csum_type,
+ strlen(csum_type)) == 0) {
+ raw_checksum = seg_csum.c_str() + strlen(csum_type);
+ if (*raw_checksum == '=')
+ raw_checksum++;
+ else
+ raw_checksum = NULL;
}
+
+ if (raw_checksum != NULL)
+ fprintf(checksums, "%s *%s\n",
+ raw_checksum, seg_path.c_str());
}
- fclose(checksums);
- } else {
- fprintf(stderr, "ERROR: Unable to write checksums file: %m\n");
}
+ fclose(checksums);
+ checksum_file->send();
db->Close();
+ /* All other files should be flushed to remote storage before writing the
+ * backup descriptor below, so that it is not possible to have a backup
+ * descriptor written out depending on non-existent (not yet written)
+ * files. */
+ remote->sync();
+
/* Write a backup descriptor file, which says which segments are needed and
* where to start to restore this snapshot. The filename is based on the
* current time. If a signature filter program was specified, filter the
* data through that to give a chance to sign the descriptor contents. */
- string desc_filename = backup_dest + "/snapshot-";
+ string desc_filename = "snapshot-";
if (backup_scheme.size() > 0)
desc_filename += backup_scheme + "-";
desc_filename = desc_filename + desc_buf + ".lbs";
- int descriptor_fd = open(desc_filename.c_str(), O_WRONLY | O_CREAT, 0666);
+ RemoteFile *descriptor_file = remote->alloc_file(desc_filename,
+ "snapshots");
+ int descriptor_fd = descriptor_file->get_fd();
if (descriptor_fd < 0) {
fprintf(stderr, "Unable to open descriptor output file: %m\n");
return 1;
fprintf(descriptor, "Date: %s\n", desc_buf);
if (backup_scheme.size() > 0)
fprintf(descriptor, "Scheme: %s\n", backup_scheme.c_str());
+ fprintf(descriptor, "Backup-Intent: %g\n", snapshot_intent);
fprintf(descriptor, "Root: %s\n", backup_root.c_str());
SHA1Checksum checksum_csum;
}
}
+ descriptor_file->send();
+
+ remote->sync();
+ delete remote;
+
+ if (backup_script != "") {
+ if (rmdir(tmp_dir.c_str()) < 0) {
+ fprintf(stderr,
+ "Warning: Cannot delete temporary directory %s: %m\n",
+ tmp_dir.c_str());
+ }
+ }
+
return 0;
}
projects / cumulus.git / blobdiff