From dd553bb0b2ac2fe7a1904d2f2670f37e3bb25ae6 Mon Sep 17 00:00:00 2001 From: Michael Vrable Date: Thu, 20 Nov 2008 12:10:06 -0800 Subject: [PATCH] Re-do cumulus side of upload script interface. Update the cumulus executable so that the interface for the remote upload script is compatible with the new cumulus-store script, allowing cumulus to easily target different storage backends. --- remote.cc | 96 +++++++++++++++++++++++++++++++++++++++++++------------ store.cc | 11 +------ util.cc | 20 ++++++++++++ util.h | 3 ++ 4 files changed, 100 insertions(+), 30 deletions(-) diff --git a/remote.cc b/remote.cc index 1384087..6dd4900 100644 --- a/remote.cc +++ b/remote.cc @@ -28,6 +28,9 @@ #include #include +#include +#include +#include #include #include #include @@ -38,6 +41,7 @@ #include "remote.h" #include "store.h" +#include "util.h" using std::string; @@ -57,7 +61,7 @@ RemoteStore::RemoteStore(const string &stagedir) if (pthread_create(&thread, NULL, RemoteStore::start_transfer_thread, (void *)this) != 0) { fprintf(stderr, "Cannot create remote storage thread: %m\n"); - throw IOException("pthread_create"); + fatal("pthread_create"); } } @@ -133,6 +137,48 @@ void *RemoteStore::start_transfer_thread(void *arg) /* Background thread for transferring backups to a remote server. */ void RemoteStore::transfer_thread() { + /* If a transfer script was specified, launch it and connect to both stdin + * and stdout. fd_in is stdin of the child, and fd_out is stdout for the + * child. */ + pid_t pid = 0; + FILE *fd_in = NULL, *fd_out = NULL; + + if (backup_script != "") { + int fds[4]; + + if (pipe(&fds[0]) < 0) { + fatal("Unable to create pipe for upload script"); + } + if (pipe(&fds[2]) < 0) { + fatal("Unable to create pipe for upload script"); + } + + pid = fork(); + if (pid < 0) { + fprintf(stderr, "Unable to fork for upload script: %m\n"); + fatal("fork: upload script"); + } + + if (pid > 0) { + /* Parent */ + close(fds[0]); + close(fds[3]); + cloexec(fds[1]); fd_in = fdopen(fds[1], "w"); + cloexec(fds[2]); fd_out = fdopen(fds[2], "r"); + } else if (pid == 0) { + /* Child */ + if (dup2(fds[0], 0) < 0) + exit(1); + if (dup2(fds[3], 1) < 0) + exit(1); + for (int i = 0; i < 3; i++) + close(fds[i]); + + execlp("/bin/sh", "/bin/sh", "-c", backup_script.c_str(), NULL); + fatal("exec failed"); + } + } + while (true) { RemoteFile *file = NULL; @@ -157,25 +203,22 @@ void RemoteStore::transfer_thread() // Transfer the file if (backup_script != "") { - pid_t pid = fork(); - if (pid < 0) { - fprintf(stderr, "Unable to fork for upload script: %m\n"); - throw IOException("fork: upload script"); - } - if (pid == 0) { - string cmd = backup_script; - cmd += " " + file->local_path + " " + file->type + " " - + file->remote_path; - execlp("/bin/sh", "/bin/sh", "-c", cmd.c_str(), NULL); - throw IOException("exec failed"); - } + string cmd = "PUT "; + cmd += uri_encode(file->type) + " "; + cmd += uri_encode(file->remote_path) + " "; + cmd += uri_encode(file->local_path) + "\n"; - int status = 0; - waitpid(pid, &status, 0); - if (!WIFEXITED(status) || WEXITSTATUS(status) != 0) { - fprintf(stderr, "Warning: error code from upload script: %d\n", - status); - } + fputs(cmd.c_str(), fd_in); + fflush(fd_in); + + char *resp = NULL; + size_t n; + if (getline(&resp, &n, fd_out) < 0 || resp == NULL) + fatal("error reading response from upload script"); + if (strchr(resp, '\n')) + *strchr(resp, '\n') = '\0'; + if (strcmp(resp, "OK") != 0) + fatal("error response from upload script"); if (unlink(file->local_path.c_str()) < 0) { fprintf(stderr, "Warning: Deleting temporary file %s: %m\n", @@ -185,6 +228,19 @@ void RemoteStore::transfer_thread() delete file; } + + if (fd_in) fclose(fd_in); + + if (pid) { + int status = 0; + waitpid(pid, &status, 0); + if (!WIFEXITED(status) || WEXITSTATUS(status) != 0) { + fprintf(stderr, "Warning: error code from upload script: %d\n", + status); + } + } + + if (fd_out) fclose(fd_out); } RemoteFile::RemoteFile(RemoteStore *remote, @@ -198,5 +254,5 @@ RemoteFile::RemoteFile(RemoteStore *remote, fd = open(local_path.c_str(), O_WRONLY | O_CREAT, 0666); if (fd < 0) - throw IOException("Error opening output file"); + fatal("Error opening output file"); } diff --git a/store.cc b/store.cc index 9e7fc99..bfeb18a 100644 --- a/store.cc +++ b/store.cc @@ -43,6 +43,7 @@ #include "store.h" #include "ref.h" +#include "util.h" using std::max; using std::list; @@ -55,16 +56,6 @@ using std::string; const char *filter_program = "bzip2 -c"; const char *filter_extension = ".bz2"; -static void cloexec(int fd) -{ - long flags = fcntl(fd, F_GETFD); - - if (flags < 0) - return; - - fcntl(fd, F_SETFD, flags | FD_CLOEXEC); -} - Tarfile::Tarfile(RemoteFile *file, const string &segment) : size(0), segment_name(segment) diff --git a/util.cc b/util.cc index b595eea..0aa5efb 100644 --- a/util.cc +++ b/util.cc @@ -23,6 +23,8 @@ #include #include +#include +#include #include #include @@ -115,3 +117,21 @@ long long parse_int(const string &s) { return strtoll(s.c_str(), NULL, 0); } + +/* Mark a file descriptor as close-on-exec. */ +void cloexec(int fd) +{ + long flags = fcntl(fd, F_GETFD); + + if (flags < 0) + return; + + fcntl(fd, F_SETFD, flags | FD_CLOEXEC); +} + +/* Report a fatal error and exit. */ +void fatal(string msg) +{ + fprintf(stderr, "FATAL: %s\n", msg.c_str()); + exit(1); +} diff --git a/util.h b/util.h index c6ab6c4..5922a42 100644 --- a/util.h +++ b/util.h @@ -33,5 +33,8 @@ std::string uri_decode(const std::string &in); std::string encode_int(long long n, int base=10); long long parse_int(const std::string &s); +void cloexec(int fd); + +void fatal(std::string msg); #endif // _LBS_TARSTORE_H -- 2.20.1