X-Git-Url: http://git.vrable.net/?p=cumulus.git;a=blobdiff_plain;f=store.cc;h=bfeb18a77ceadcaaeca8f555283430847a4fa24e;hp=930794d50d12beccaa4caa5d223d82bea2b3b0e8;hb=dd553bb0b2ac2fe7a1904d2f2670f37e3bb25ae6;hpb=a4cf5f4d8df46fa00992a210d587cd824cedcb08 diff --git a/store.cc b/store.cc index 930794d..bfeb18a 100644 --- a/store.cc +++ b/store.cc @@ -1,13 +1,31 @@ -/* LBS: An LFS-inspired filesystem backup system - * Copyright (C) 2007 Michael Vrable +/* Cumulus: Smart Filesystem Backup to Dumb Servers * - * Backup data is stored in a collection of objects, which are grouped together + * Copyright (C) 2008 The Regents of the University of California + * Written by Michael Vrable + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +/* Backup data is stored in a collection of objects, which are grouped together * into segments for storage purposes. This implementation of the object store * represents segments as TAR files and objects as files within them. */ #include #include #include +#include #include #include #include @@ -25,10 +43,12 @@ #include "store.h" #include "ref.h" +#include "util.h" using std::max; using std::list; using std::map; +using std::pair; using std::set; using std::string; @@ -36,27 +56,15 @@ using std::string; const char *filter_program = "bzip2 -c"; const char *filter_extension = ".bz2"; -static void cloexec(int fd) -{ - long flags = fcntl(fd, F_GETFD); - - if (flags < 0) - return; - - fcntl(fd, F_SETFD, flags | FD_CLOEXEC); -} - -Tarfile::Tarfile(const string &path, const string &segment) +Tarfile::Tarfile(RemoteFile *file, const string &segment) : size(0), segment_name(segment) { assert(sizeof(struct tar_header) == TAR_BLOCK_SIZE); - real_fd = open(path.c_str(), O_WRONLY | O_CREAT, 0666); - if (real_fd < 0) - throw IOException("Error opening output file"); - - filter_fd = spawn_filter(real_fd); + this->file = file; + real_fd = file->get_fd(); + filter_fd = spawn_filter(real_fd, filter_program, &filter_pid); } Tarfile::~Tarfile() @@ -86,9 +94,10 @@ Tarfile::~Tarfile() * on the TAR output. The file descriptor to which output should be written * must be specified; the return value is the file descriptor which will be * attached to the standard input of the filter program. */ -int Tarfile::spawn_filter(int fd_out) +int spawn_filter(int fd_out, const char *program, pid_t *filter_pid) { int fds[2]; + pid_t pid; /* Create a pipe for communicating with the filter process. */ if (pipe(fds) < 0) { @@ -96,14 +105,16 @@ int Tarfile::spawn_filter(int fd_out) } /* Create a child process which can exec() the filter program. */ - filter_pid = fork(); - if (filter_pid < 0) + pid = fork(); + if (pid < 0) throw IOException("Unable to fork filter process"); - if (filter_pid > 0) { + if (pid > 0) { /* Parent process */ close(fds[0]); cloexec(fds[1]); + if (filter_pid != NULL) + *filter_pid = pid; } else { /* Child process. Rearrange file descriptors. stdin is fds[0], stdout * is fd_out, stderr is unchanged. */ @@ -118,7 +129,7 @@ int Tarfile::spawn_filter(int fd_out) close(fd_out); /* Exec the filter program. */ - execlp("/bin/sh", "/bin/sh", "-c", filter_program, NULL); + execlp("/bin/sh", "/bin/sh", "-c", program, NULL); /* Should not reach here except for error cases. */ fprintf(stderr, "Could not exec filter: %m\n"); @@ -149,19 +160,13 @@ void Tarfile::tar_write(const char *data, size_t len) void Tarfile::write_object(int id, const char *data, size_t len) { + struct tar_header header; + memset(&header, 0, sizeof(header)); + char buf[64]; sprintf(buf, "%08x", id); string path = segment_name + "/" + buf; - internal_write_object(path, data, len); -} - -void Tarfile::internal_write_object(const string &path, - const char *data, size_t len) -{ - struct tar_header header; - memset(&header, 0, sizeof(header)); - assert(path.size() < 100); memcpy(header.name, path.data(), path.size()); sprintf(header.mode, "%07o", 0600); @@ -217,7 +222,8 @@ size_t Tarfile::size_estimate() static const size_t SEGMENT_SIZE = 4 * 1024 * 1024; -static map group_sizes; +/* Backup size summary: segment type -> (uncompressed size, compressed size) */ +static map > group_sizes; ObjectReference TarSegmentStore::write_object(const char *data, size_t len, const std::string &group) @@ -230,11 +236,13 @@ ObjectReference TarSegmentStore::write_object(const char *data, size_t len, segment = new segment_info; segment->name = generate_uuid(); + segment->group = group; segment->basename = segment->name + ".tar"; segment->basename += filter_extension; - segment->fullname = path + "/" + segment->basename; - segment->file = new Tarfile(segment->fullname, segment->name); segment->count = 0; + segment->size = 0; + segment->rf = remote->alloc_file(segment->basename, "segments"); + segment->file = new Tarfile(segment->rf, segment->name); segments[group] = segment; } else { @@ -247,8 +255,9 @@ ObjectReference TarSegmentStore::write_object(const char *data, size_t len, segment->file->write_object(id, data, len); segment->count++; + segment->size += len; - group_sizes[group] += len; + group_sizes[group].first += len; ObjectReference ref(segment->name, id_buf); @@ -269,9 +278,10 @@ void TarSegmentStore::sync() void TarSegmentStore::dump_stats() { printf("Data written:\n"); - for (map::iterator i = group_sizes.begin(); + for (map >::iterator i = group_sizes.begin(); i != group_sizes.end(); ++i) { - printf(" %s: %lld\n", i->first.c_str(), i->second); + printf(" %s: %lld (%lld compressed)\n", i->first.c_str(), + i->second.first, i->second.second); } } @@ -283,12 +293,20 @@ void TarSegmentStore::close_segment(const string &group) if (db != NULL) { SHA1Checksum segment_checksum; - if (segment_checksum.process_file(segment->fullname.c_str())) { + if (segment_checksum.process_file(segment->rf->get_local_path().c_str())) { string checksum = segment_checksum.checksum_str(); - db->SetSegmentChecksum(segment->name, segment->basename, checksum); + db->SetSegmentChecksum(segment->name, segment->basename, checksum, + segment->size); + } + + struct stat stat_buf; + if (stat(segment->rf->get_local_path().c_str(), &stat_buf) == 0) { + group_sizes[segment->group].second += stat_buf.st_size; } } + segment->rf->send(); + segments.erase(segments.find(group)); delete segment; }