1 /* Cumulus: Smart Filesystem Backup to Dumb Servers
3 * Copyright (C) 2008 The Regents of the University of California
4 * Written by Michael Vrable <mvrable@cs.ucsd.edu>
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License as published by
8 * the Free Software Foundation; either version 2 of the License, or
9 * (at your option) any later version.
11 * This program is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 * GNU General Public License for more details.
16 * You should have received a copy of the GNU General Public License along
17 * with this program; if not, write to the Free Software Foundation, Inc.,
18 * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
21 /* Backup data (segments and backup descriptors) may be stored on a remote
22 * fileserver instead of locally. The only local storage needed is for the
23 * local database and some temporary space for staging files before they are
24 * transferred to the remote server.
26 * Like encryption, remote storage is handled through the use of external
27 * scripts that are called when a file is to be transferred. */
35 #include <sys/types.h>
48 RemoteStore::RemoteStore(const string &stagedir)
50 staging_dir = stagedir;
52 /* A background thread is created for each RemoteStore to manage the actual
53 * transfers to a remote server. The main program thread can enqueue
54 * RemoteFile objects to be transferred asynchronously. */
55 pthread_mutex_init(&lock, NULL);
56 pthread_cond_init(&cond, NULL);
59 files_outstanding = 0;
61 if (pthread_create(&thread, NULL, RemoteStore::start_transfer_thread,
63 fprintf(stderr, "Cannot create remote storage thread: %m\n");
64 fatal("pthread_create");
68 /* The RemoteStore destructor will terminate the background transfer thread.
69 * It will wait for all work to finish. */
70 RemoteStore::~RemoteStore()
72 pthread_mutex_lock(&lock);
74 pthread_cond_broadcast(&cond);
75 pthread_mutex_unlock(&lock);
77 if (pthread_join(thread, NULL) != 0) {
78 fprintf(stderr, "Warning: Unable to join storage thread: %m\n");
81 assert(files_outstanding == 0);
83 pthread_cond_destroy(&cond);
84 pthread_mutex_destroy(&lock);
87 /* Prepare to write out a new file. Returns a RemoteFile object. The file
88 * will initially be created in a temporary directory. When the file is
89 * written out, the RemoteFile object should be passed to RemoteStore::enqueue,
90 * which will upload it to the remote server. */
91 RemoteFile *RemoteStore::alloc_file(const string &name, const string &type)
93 pthread_mutex_lock(&lock);
95 pthread_mutex_unlock(&lock);
96 return new RemoteFile(this, name, type, staging_dir + "/" + name);
99 /* Request that a file be transferred to the remote server. The actual
100 * transfer will happen asynchronously in another thread. The call to enqueue
101 * may block, however, if there is a backlog of data to be transferred.
102 * Ownership of the RemoteFile object is transferred; the RemoteStore will be
103 * responsible for its destruction. */
104 void RemoteStore::enqueue(RemoteFile *file)
106 pthread_mutex_lock(&lock);
108 while (transfer_queue.size() >= MAX_QUEUE_SIZE)
109 pthread_cond_wait(&cond, &lock);
111 transfer_queue.push_back(file);
115 pthread_cond_broadcast(&cond);
116 pthread_mutex_unlock(&lock);
119 /* Wait for all transfers to finish. */
120 void RemoteStore::sync()
122 pthread_mutex_lock(&lock);
125 pthread_cond_wait(&cond, &lock);
127 pthread_mutex_unlock(&lock);
130 void *RemoteStore::start_transfer_thread(void *arg)
132 RemoteStore *store = static_cast<RemoteStore *>(arg);
133 store->transfer_thread();
137 /* Background thread for transferring backups to a remote server. */
138 void RemoteStore::transfer_thread()
140 /* If a transfer script was specified, launch it and connect to both stdin
141 * and stdout. fd_in is stdin of the child, and fd_out is stdout for the
144 FILE *fd_in = NULL, *fd_out = NULL;
146 if (backup_script != "") {
149 if (pipe(&fds[0]) < 0) {
150 fatal("Unable to create pipe for upload script");
152 if (pipe(&fds[2]) < 0) {
153 fatal("Unable to create pipe for upload script");
158 fprintf(stderr, "Unable to fork for upload script: %m\n");
159 fatal("fork: upload script");
166 cloexec(fds[1]); fd_in = fdopen(fds[1], "w");
167 cloexec(fds[2]); fd_out = fdopen(fds[2], "r");
168 } else if (pid == 0) {
170 if (dup2(fds[0], 0) < 0)
172 if (dup2(fds[3], 1) < 0)
174 for (int i = 0; i < 3; i++)
177 execlp("/bin/sh", "/bin/sh", "-c", backup_script.c_str(), NULL);
178 fatal("exec failed");
183 RemoteFile *file = NULL;
185 // Wait for a file to transfer
186 pthread_mutex_lock(&lock);
187 while (transfer_queue.empty() && !terminate) {
189 pthread_cond_broadcast(&cond);
190 pthread_cond_wait(&cond, &lock);
192 if (terminate && transfer_queue.empty()) {
194 pthread_cond_broadcast(&cond);
195 pthread_mutex_unlock(&lock);
199 file = transfer_queue.front();
200 transfer_queue.pop_front();
201 pthread_cond_broadcast(&cond);
202 pthread_mutex_unlock(&lock);
205 if (backup_script != "") {
207 cmd += uri_encode(file->type) + " ";
208 cmd += uri_encode(file->remote_path) + " ";
209 cmd += uri_encode(file->local_path) + "\n";
211 fputs(cmd.c_str(), fd_in);
216 if (getline(&resp, &n, fd_out) < 0 || resp == NULL)
217 fatal("error reading response from upload script");
218 if (strchr(resp, '\n'))
219 *strchr(resp, '\n') = '\0';
220 if (strcmp(resp, "OK") != 0)
221 fatal("error response from upload script");
223 if (unlink(file->local_path.c_str()) < 0) {
224 fprintf(stderr, "Warning: Deleting temporary file %s: %m\n",
225 file->local_path.c_str());
232 if (fd_in) fclose(fd_in);
236 waitpid(pid, &status, 0);
237 if (!WIFEXITED(status) || WEXITSTATUS(status) != 0) {
238 fprintf(stderr, "Warning: error code from upload script: %d\n",
243 if (fd_out) fclose(fd_out);
246 RemoteFile::RemoteFile(RemoteStore *remote,
247 const string &name, const string &type,
248 const string &local_path)
250 remote_store = remote;
252 this->local_path = local_path;
253 this->remote_path = name;
255 fd = open(local_path.c_str(), O_WRONLY | O_CREAT, 0666);
257 fatal("Error opening output file");