-/* Cumulus: Smart Filesystem Backup to Dumb Servers
- *
- * Copyright (C) 2008 The Regents of the University of California
- * Written by Michael Vrable <mvrable@cs.ucsd.edu>
+/* Cumulus: Efficient Filesystem Backup to the Cloud
+ * Copyright (C) 2008-2009 The Cumulus Developers
+ * See the AUTHORS file for a list of contributors.
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* scripts that are called when a file is to be transferred. */
#include <assert.h>
+#include <errno.h>
#include <fcntl.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
#include <unistd.h>
#include <sys/types.h>
#include <sys/stat.h>
#include "remote.h"
#include "store.h"
+#include "util.h"
using std::string;
-RemoteStore::RemoteStore(const string &stagedir)
+static const char *backup_directories[] = {
+ "meta",
+ "segments0",
+ "segments1",
+ "snapshots",
+ NULL
+};
+
+RemoteStore::RemoteStore(const string &stagedir, const string &script)
{
staging_dir = stagedir;
+ backup_script = script;
+
+ /* Ensure all necessary directories exist for each type of backup file. */
+ for (size_t i = 0; backup_directories[i]; i++) {
+ string path = stagedir + "/" + backup_directories[i];
+ if (mkdir(path.c_str(), 0777) < 0) {
+ /* Ignore errors for already-existing directories. */
+ if (errno != EEXIST) {
+ fprintf(stderr,
+ "Warning: Cannot create backup directory %s: %m!",
+ path.c_str());
+ }
+ }
+ }
/* A background thread is created for each RemoteStore to manage the actual
* transfers to a remote server. The main program thread can enqueue
if (pthread_create(&thread, NULL, RemoteStore::start_transfer_thread,
(void *)this) != 0) {
fprintf(stderr, "Cannot create remote storage thread: %m\n");
- throw IOException("pthread_create");
+ fatal("pthread_create");
}
}
* which will upload it to the remote server. */
RemoteFile *RemoteStore::alloc_file(const string &name, const string &type)
{
- fprintf(stderr, "Allocate file: %s\n", name.c_str());
pthread_mutex_lock(&lock);
files_outstanding++;
pthread_mutex_unlock(&lock);
- return new RemoteFile(this, name, type, staging_dir + "/" + name);
+ return new RemoteFile(this, name, type,
+ staging_dir + "/" + type + "/" + name);
}
/* Request that a file be transferred to the remote server. The actual
* responsible for its destruction. */
void RemoteStore::enqueue(RemoteFile *file)
{
- fprintf(stderr, "Enqueue: %s\n", file->remote_path.c_str());
-
pthread_mutex_lock(&lock);
while (transfer_queue.size() >= MAX_QUEUE_SIZE)
/* Wait for all transfers to finish. */
void RemoteStore::sync()
{
- fprintf(stderr, "RemoteStore::sync() start\n");
pthread_mutex_lock(&lock);
while (busy)
pthread_cond_wait(&cond, &lock);
pthread_mutex_unlock(&lock);
- fprintf(stderr, "RemoteStore::sync() end\n");
}
void *RemoteStore::start_transfer_thread(void *arg)
/* Background thread for transferring backups to a remote server. */
void RemoteStore::transfer_thread()
{
+ /* If a transfer script was specified, launch it and connect to both stdin
+ * and stdout. fd_in is stdin of the child, and fd_out is stdout for the
+ * child. */
+ pid_t pid = 0;
+ FILE *fd_in = NULL, *fd_out = NULL;
+
+ if (backup_script != "") {
+ int fds[4];
+
+ if (pipe(&fds[0]) < 0) {
+ fatal("Unable to create pipe for upload script");
+ }
+ if (pipe(&fds[2]) < 0) {
+ fatal("Unable to create pipe for upload script");
+ }
+
+ pid = fork();
+ if (pid < 0) {
+ fprintf(stderr, "Unable to fork for upload script: %m\n");
+ fatal("fork: upload script");
+ }
+
+ if (pid > 0) {
+ /* Parent */
+ close(fds[0]);
+ close(fds[3]);
+ cloexec(fds[1]); fd_in = fdopen(fds[1], "w");
+ cloexec(fds[2]); fd_out = fdopen(fds[2], "r");
+ } else if (pid == 0) {
+ /* Child */
+ if (dup2(fds[0], 0) < 0)
+ exit(1);
+ if (dup2(fds[3], 1) < 0)
+ exit(1);
+ for (int i = 0; i < 3; i++)
+ close(fds[i]);
+
+ execlp("/bin/sh", "/bin/sh", "-c", backup_script.c_str(), NULL);
+ fatal("exec failed");
+ }
+ }
+
while (true) {
RemoteFile *file = NULL;
pthread_mutex_unlock(&lock);
// Transfer the file
- fprintf(stderr, "Start transfer: %s\n", file->remote_path.c_str());
if (backup_script != "") {
- pid_t pid = fork();
- if (pid < 0) {
- fprintf(stderr, "Unable to fork for upload script: %m\n");
- throw IOException("fork: upload script");
- }
- if (pid == 0) {
- string cmd = backup_script;
- cmd += " " + file->local_path + " " + file->type + " "
- + file->remote_path;
- execlp("/bin/sh", "/bin/sh", "-c", cmd.c_str(), NULL);
- throw IOException("exec failed");
- }
+ string cmd = "PUT ";
+ cmd += uri_encode(file->type) + " ";
+ cmd += uri_encode(file->remote_path) + " ";
+ cmd += uri_encode(file->local_path) + "\n";
- int status = 0;
- waitpid(pid, &status, 0);
- if (!WIFEXITED(status) || WEXITSTATUS(status) != 0) {
- fprintf(stderr, "Warning: error code from upload script: %d\n",
- status);
- }
+ fputs(cmd.c_str(), fd_in);
+ fflush(fd_in);
+
+ char *resp = NULL;
+ size_t n;
+ if (getline(&resp, &n, fd_out) < 0 || resp == NULL)
+ fatal("error reading response from upload script");
+ if (strchr(resp, '\n'))
+ *strchr(resp, '\n') = '\0';
+ if (strcmp(resp, "OK") != 0)
+ fatal("error response from upload script");
if (unlink(file->local_path.c_str()) < 0) {
fprintf(stderr, "Warning: Deleting temporary file %s: %m\n",
file->local_path.c_str());
}
}
- fprintf(stderr, "Finish transfer: %s\n", file->remote_path.c_str());
delete file;
}
+
+ if (fd_in) fclose(fd_in);
+
+ if (pid) {
+ int status = 0;
+ waitpid(pid, &status, 0);
+ if (!WIFEXITED(status) || WEXITSTATUS(status) != 0) {
+ fprintf(stderr, "Warning: error code from upload script: %d\n",
+ status);
+ }
+ }
+
+ if (fd_out) fclose(fd_out);
}
RemoteFile::RemoteFile(RemoteStore *remote,
remote_store = remote;
this->type = type;
this->local_path = local_path;
- this->remote_path = name;
+ this->remote_path = type + "/" + name;
fd = open(local_path.c_str(), O_WRONLY | O_CREAT, 0666);
if (fd < 0)
- throw IOException("Error opening output file");
+ fatal("Error opening output file");
}
projects / cumulus.git / blobdiff