X-Git-Url: http://git.vrable.net/?p=cumulus.git;a=blobdiff_plain;f=remote.cc;h=e8e046eb5f5bb72da0569ef7b06cc23f71deaabf;hp=92b1f890ea0b0ae83ae2d7c41a5147f5af3c4df5;hb=HEAD;hpb=013636b17f3e44f5ac4837750a3888999d35d2f8 diff --git a/remote.cc b/remote.cc index 92b1f89..e8e046e 100644 --- a/remote.cc +++ b/remote.cc @@ -1,7 +1,23 @@ -/* LBS: An LFS-inspired filesystem backup system - * Copyright (C) 2008 Michael Vrable +/* Cumulus: Efficient Filesystem Backup to the Cloud + * Copyright (C) 2008-2009 The Cumulus Developers + * See the AUTHORS file for a list of contributors. * - * Backup data (segments and backup descriptors) may be stored on a remote + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +/* Backup data (segments and backup descriptors) may be stored on a remote * fileserver instead of locally. The only local storage needed is for the * local database and some temporary space for staging files before they are * transferred to the remote server. @@ -10,7 +26,11 @@ * scripts that are called when a file is to be transferred. */ #include +#include #include +#include +#include +#include #include #include #include @@ -21,12 +41,35 @@ #include "remote.h" #include "store.h" +#include "util.h" using std::string; -RemoteStore::RemoteStore(const string &stagedir) +static const char *backup_directories[] = { + "meta", + "segments0", + "segments1", + "snapshots", + NULL +}; + +RemoteStore::RemoteStore(const string &stagedir, const string &script) { staging_dir = stagedir; + backup_script = script; + + /* Ensure all necessary directories exist for each type of backup file. */ + for (size_t i = 0; backup_directories[i]; i++) { + string path = stagedir + "/" + backup_directories[i]; + if (mkdir(path.c_str(), 0777) < 0) { + /* Ignore errors for already-existing directories. */ + if (errno != EEXIST) { + fprintf(stderr, + "Warning: Cannot create backup directory %s: %m!", + path.c_str()); + } + } + } /* A background thread is created for each RemoteStore to manage the actual * transfers to a remote server. The main program thread can enqueue @@ -40,7 +83,7 @@ RemoteStore::RemoteStore(const string &stagedir) if (pthread_create(&thread, NULL, RemoteStore::start_transfer_thread, (void *)this) != 0) { fprintf(stderr, "Cannot create remote storage thread: %m\n"); - throw IOException("pthread_create"); + fatal("pthread_create"); } } @@ -69,11 +112,11 @@ RemoteStore::~RemoteStore() * which will upload it to the remote server. */ RemoteFile *RemoteStore::alloc_file(const string &name, const string &type) { - fprintf(stderr, "Allocate file: %s\n", name.c_str()); pthread_mutex_lock(&lock); files_outstanding++; pthread_mutex_unlock(&lock); - return new RemoteFile(this, name, type, staging_dir + "/" + name); + return new RemoteFile(this, name, type, + staging_dir + "/" + type + "/" + name); } /* Request that a file be transferred to the remote server. The actual @@ -83,8 +126,6 @@ RemoteFile *RemoteStore::alloc_file(const string &name, const string &type) * responsible for its destruction. */ void RemoteStore::enqueue(RemoteFile *file) { - fprintf(stderr, "Enqueue: %s\n", file->remote_path.c_str()); - pthread_mutex_lock(&lock); while (transfer_queue.size() >= MAX_QUEUE_SIZE) @@ -101,14 +142,12 @@ void RemoteStore::enqueue(RemoteFile *file) /* Wait for all transfers to finish. */ void RemoteStore::sync() { - fprintf(stderr, "RemoteStore::sync() start\n"); pthread_mutex_lock(&lock); while (busy) pthread_cond_wait(&cond, &lock); pthread_mutex_unlock(&lock); - fprintf(stderr, "RemoteStore::sync() end\n"); } void *RemoteStore::start_transfer_thread(void *arg) @@ -121,6 +160,48 @@ void *RemoteStore::start_transfer_thread(void *arg) /* Background thread for transferring backups to a remote server. */ void RemoteStore::transfer_thread() { + /* If a transfer script was specified, launch it and connect to both stdin + * and stdout. fd_in is stdin of the child, and fd_out is stdout for the + * child. */ + pid_t pid = 0; + FILE *fd_in = NULL, *fd_out = NULL; + + if (backup_script != "") { + int fds[4]; + + if (pipe(&fds[0]) < 0) { + fatal("Unable to create pipe for upload script"); + } + if (pipe(&fds[2]) < 0) { + fatal("Unable to create pipe for upload script"); + } + + pid = fork(); + if (pid < 0) { + fprintf(stderr, "Unable to fork for upload script: %m\n"); + fatal("fork: upload script"); + } + + if (pid > 0) { + /* Parent */ + close(fds[0]); + close(fds[3]); + cloexec(fds[1]); fd_in = fdopen(fds[1], "w"); + cloexec(fds[2]); fd_out = fdopen(fds[2], "r"); + } else if (pid == 0) { + /* Child */ + if (dup2(fds[0], 0) < 0) + exit(1); + if (dup2(fds[3], 1) < 0) + exit(1); + for (int i = 0; i < 3; i++) + close(fds[i]); + + execlp("/bin/sh", "/bin/sh", "-c", backup_script.c_str(), NULL); + fatal("exec failed"); + } + } + while (true) { RemoteFile *file = NULL; @@ -144,37 +225,45 @@ void RemoteStore::transfer_thread() pthread_mutex_unlock(&lock); // Transfer the file - fprintf(stderr, "Start transfer: %s\n", file->remote_path.c_str()); if (backup_script != "") { - pid_t pid = fork(); - if (pid < 0) { - fprintf(stderr, "Unable to fork for upload script: %m\n"); - throw IOException("fork: upload script"); - } - if (pid == 0) { - string cmd = backup_script; - cmd += " " + file->local_path + " " + file->type + " " - + file->remote_path; - execlp("/bin/sh", "/bin/sh", "-c", cmd.c_str(), NULL); - throw IOException("exec failed"); - } + string cmd = "PUT "; + cmd += uri_encode(file->type) + " "; + cmd += uri_encode(file->remote_path) + " "; + cmd += uri_encode(file->local_path) + "\n"; - int status = 0; - waitpid(pid, &status, 0); - if (!WIFEXITED(status) || WEXITSTATUS(status) != 0) { - fprintf(stderr, "Warning: error code from upload script: %d\n", - status); - } + fputs(cmd.c_str(), fd_in); + fflush(fd_in); + + char *resp = NULL; + size_t n; + if (getline(&resp, &n, fd_out) < 0 || resp == NULL) + fatal("error reading response from upload script"); + if (strchr(resp, '\n')) + *strchr(resp, '\n') = '\0'; + if (strcmp(resp, "OK") != 0) + fatal("error response from upload script"); if (unlink(file->local_path.c_str()) < 0) { fprintf(stderr, "Warning: Deleting temporary file %s: %m\n", file->local_path.c_str()); } } - fprintf(stderr, "Finish transfer: %s\n", file->remote_path.c_str()); delete file; } + + if (fd_in) fclose(fd_in); + + if (pid) { + int status = 0; + waitpid(pid, &status, 0); + if (!WIFEXITED(status) || WEXITSTATUS(status) != 0) { + fprintf(stderr, "Warning: error code from upload script: %d\n", + status); + } + } + + if (fd_out) fclose(fd_out); } RemoteFile::RemoteFile(RemoteStore *remote, @@ -184,9 +273,9 @@ RemoteFile::RemoteFile(RemoteStore *remote, remote_store = remote; this->type = type; this->local_path = local_path; - this->remote_path = name; + this->remote_path = type + "/" + name; fd = open(local_path.c_str(), O_WRONLY | O_CREAT, 0666); if (fd < 0) - throw IOException("Error opening output file"); + fatal("Error opening output file"); }