From: Michael Vrable Date: Thu, 3 Apr 2008 20:07:11 +0000 (-0700) Subject: Preliminary support for external file upload scripts. X-Git-Url: https://git.vrable.net/?a=commitdiff_plain;h=09533a1615813c343e1244275531f933d9b48ddf;p=cumulus.git Preliminary support for external file upload scripts. This adds initial support for calling out to an external script to transfer files to a backup server. Storage requirements on the client using this are minimal: space for the local database and for spooling several files for upload. Local temporary files are deleted as they are uploaded, and the backup rate is throttled to the upload rate. --- diff --git a/remote.cc b/remote.cc index 1e77333..a279751 100644 --- a/remote.cc +++ b/remote.cc @@ -14,6 +14,7 @@ #include #include #include +#include #include #include @@ -144,7 +145,31 @@ void RemoteStore::transfer_thread() // Transfer the file fprintf(stderr, "Start transfer: %s\n", file->remote_path.c_str()); - // TODO + if (backup_script != "") { + pid_t pid = fork(); + if (pid < 0) { + fprintf(stderr, "Unable to fork for upload script: %m\n"); + throw IOException("fork: upload script"); + } + if (pid == 0) { + string cmd = backup_script; + cmd += " " + file->local_path + " " + file->remote_path; + execlp("/bin/sh", "/bin/sh", "-c", cmd.c_str(), NULL); + throw IOException("exec failed"); + } + + int status = 0; + waitpid(pid, &status, 0); + if (!WIFEXITED(status) || WEXITSTATUS(status) != 0) { + fprintf(stderr, "Warning: error code from upload script: %d\n", + status); + } + + if (unlink(file->local_path.c_str()) < 0) { + fprintf(stderr, "Warning: Deleting temporary file %s: %m\n", + file->local_path.c_str()); + } + } fprintf(stderr, "Finish transfer: %s\n", file->remote_path.c_str()); delete file; diff --git a/remote.h b/remote.h index 705fe80..85de06c 100644 --- a/remote.h +++ b/remote.h @@ -24,6 +24,8 @@ public: RemoteStore(const std::string &stagedir); ~RemoteStore(); + void set_script(const std::string &script) + { backup_script = script; } RemoteFile *alloc_file(const std::string &name); void enqueue(RemoteFile *file); void sync(); @@ -33,7 +35,7 @@ private: pthread_mutex_t lock; pthread_cond_t cond; - std::string staging_dir; + std::string staging_dir, backup_script; bool terminate; // Set when thread should shut down bool busy; // True while there are pending transfers std::list transfer_queue; diff --git a/scandir.cc b/scandir.cc index 92c9a4d..1daf352 100644 --- a/scandir.cc +++ b/scandir.cc @@ -555,9 +555,13 @@ void usage(const char *program) "Produce backup snapshot of files in SOURCE and store to DEST.\n" "\n" "Options:\n" - " --dest=PATH path where backup is to be written [REQUIRED]\n" + " --dest=PATH path where backup is to be written\n" + " --upload-script=COMMAND\n" + " program to invoke for each backup file generated\n" " --exclude=PATH exclude files in PATH from snapshot\n" " --localdb=PATH local backup metadata is stored in PATH\n" + " --tmpdir=PATH path for temporarily storing backup files\n" + " (defaults to TMPDIR environment variable or /tmp)\n" " --filter=COMMAND program through which to filter segment data\n" " (defaults to \"bzip2 -c\")\n" " --filter-extension=EXT\n" @@ -568,18 +572,24 @@ void usage(const char *program) " --scheme=NAME optional name for this snapshot\n" " --intent=FLOAT intended backup type: 1=daily, 7=weekly, ...\n" " (defaults to \"1\")\n" - " --full-metadata do not re-use metadata from previous backups\n", + " --full-metadata do not re-use metadata from previous backups\n" + "\n" + "Exactly one of --dest or --upload-script must be specified.\n", lbs_version, program ); } int main(int argc, char *argv[]) { - string backup_dest = ""; + string backup_dest = "", backup_script = ""; string localdb_dir = ""; string backup_scheme = ""; string signature_filter = ""; + string tmp_dir = "/tmp"; + if (getenv("TMPDIR") != NULL) + tmp_dir = getenv("TMPDIR"); + while (1) { static struct option long_options[] = { {"localdb", 1, 0, 0}, // 0 @@ -591,6 +601,8 @@ int main(int argc, char *argv[]) {"signature-filter", 1, 0, 0}, // 6 {"intent", 1, 0, 0}, // 7 {"full-metadata", 0, 0, 0}, // 8 + {"tmpdir", 1, 0, 0}, // 9 + {"upload-script", 1, 0, 0}, // 10 {NULL, 0, 0, 0}, }; @@ -634,6 +646,12 @@ int main(int argc, char *argv[]) case 8: // --full-metadata flag_full_metadata = true; break; + case 9: // --tmpdir + tmp_dir = optarg; + break; + case 10: // --upload-script + backup_script = optarg; + break; default: fprintf(stderr, "Unhandled long option!\n"); return 1; @@ -653,9 +671,16 @@ int main(int argc, char *argv[]) for (int i = optind; i < argc; i++) add_include(argv[i]); - if (backup_dest == "") { + if (backup_dest == "" && backup_script == "") { fprintf(stderr, - "Error: Backup destination must be specified with --dest=\n"); + "Error: Backup destination must be specified using --dest= or --upload-script=\n"); + usage(argv[0]); + return 1; + } + + if (backup_dest != "" && backup_script != "") { + fprintf(stderr, + "Error: Cannot specify both --dest= and --upload-script=\n"); usage(argv[0]); return 1; } @@ -664,6 +689,12 @@ int main(int argc, char *argv[]) if (localdb_dir == "") { localdb_dir = backup_dest; } + if (localdb_dir == "") { + fprintf(stderr, + "Error: Must specify local database path with --localdb=\n"); + usage(argv[0]); + return 1; + } // Dump paths for debugging/informational purposes { @@ -671,8 +702,8 @@ int main(int argc, char *argv[]) printf("LBS Version: %s\n", lbs_version); - printf("--dest=%s\n--localdb=%s\n\n", - backup_dest.c_str(), localdb_dir.c_str()); + printf("--dest=%s\n--localdb=%s\n--upload-script=%s\n", + backup_dest.c_str(), localdb_dir.c_str(), backup_script.c_str()); printf("Includes:\n"); for (i = includes.begin(); i != includes.end(); ++i) @@ -689,8 +720,21 @@ int main(int argc, char *argv[]) block_buf = new char[LBS_BLOCK_SIZE]; - /* Initialize the remote storage layer. */ - remote = new RemoteStore(backup_dest); + /* Initialize the remote storage layer. If using an upload script, create + * a temporary directory for staging files. Otherwise, write backups + * directly to the destination directory. */ + if (backup_script != "") { + tmp_dir = tmp_dir + "/lbs." + generate_uuid(); + if (mkdir(tmp_dir.c_str(), 0700) < 0) { + fprintf(stderr, "Cannot create temporary directory %s: %m\n", + tmp_dir.c_str()); + return 1; + } + remote = new RemoteStore(tmp_dir); + remote->set_script(backup_script); + } else { + remote = new RemoteStore(backup_dest); + } /* Store the time when the backup started, so it can be included in the * snapshot name. */ @@ -832,5 +876,13 @@ int main(int argc, char *argv[]) remote->sync(); delete remote; + if (backup_script != "") { + if (rmdir(tmp_dir.c_str()) < 0) { + fprintf(stderr, + "Warning: Cannot delete temporary directory %s: %m\n", + tmp_dir.c_str()); + } + } + return 0; }