-/* LBS: An LFS-inspired filesystem backup system
- * Copyright (C) 2007 Michael Vrable
+/* Cumulus: Smart Filesystem Backup to Dumb Servers
*
- * When creating backup snapshots, maintain a local database of data blocks and
+ * Copyright (C) 2007-2008 The Regents of the University of California
+ * Written by Michael Vrable <mvrable@cs.ucsd.edu>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+ */
+
+/* When creating backup snapshots, maintain a local database of data blocks and
* checksums, in addition to the data contents (which may be stored remotely).
* This database is consulted when attempting to build incremental snapshots,
* as it says which objects can be reused.
#include <string.h>
#include <sqlite3.h>
+#include <algorithm>
#include <string>
#include "localdb.h"
#include "store.h"
+#include "util.h"
+using std::min;
using std::string;
/* Helper function to prepare a statement for execution in the current
rc = sqlite3_prepare_v2(db, sql, strlen(sql), &stmt, &tail);
if (rc != SQLITE_OK) {
- throw IOException(string("Error preparing statement: ") + sql);
+ ReportError(rc);
+ fatal(string("Error preparing statement: ") + sql);
}
return stmt;
}
void LocalDb::Open(const char *path, const char *snapshot_name,
- const char *snapshot_scheme)
+ const char *snapshot_scheme, double intent)
{
int rc;
if (rc) {
fprintf(stderr, "Can't open database: %s\n", sqlite3_errmsg(db));
sqlite3_close(db);
- throw IOException("Error opening local database");
+ fatal("Error opening local database");
}
rc = sqlite3_exec(db, "begin", NULL, NULL, NULL);
if (rc) {
fprintf(stderr, "Can't open database: %s\n", sqlite3_errmsg(db));
sqlite3_close(db);
- throw IOException("Error starting transaction");
+ fatal("Error starting transaction");
}
sqlite3_extended_result_codes(db, 1);
+ if (snapshot_scheme == NULL)
+ snapshot_scheme = "";
+
/* Insert this snapshot into the database, and determine the integer key
* which will be used to identify it. */
sqlite3_stmt *stmt = Prepare("insert into "
- "snapshots(name, scheme, timestamp) "
- "values (?, ?, julianday('now'))");
+ "snapshots(name, scheme, timestamp, intent) "
+ "values (?, ?, julianday('now'), ?)");
sqlite3_bind_text(stmt, 1, snapshot_name, strlen(snapshot_name),
SQLITE_TRANSIENT);
- if (snapshot_scheme == NULL)
- sqlite3_bind_null(stmt, 2);
- else
- sqlite3_bind_text(stmt, 2, snapshot_scheme, strlen(snapshot_scheme),
- SQLITE_TRANSIENT);
+ sqlite3_bind_text(stmt, 2, snapshot_scheme, strlen(snapshot_scheme),
+ SQLITE_TRANSIENT);
+ sqlite3_bind_double(stmt, 3, intent);
rc = sqlite3_step(stmt);
if (rc != SQLITE_DONE) {
ReportError(rc);
sqlite3_close(db);
- throw IOException("Database execution error!");
+ fatal("Database execution error!");
}
snapshotid = sqlite3_last_insert_rowid(db);
if (snapshotid == 0) {
ReportError(rc);
sqlite3_close(db);
- throw IOException("Find snapshot id");
+ fatal("Find snapshot id");
+ }
+
+ /* Create a temporary table which will be used to keep track of the objects
+ * used by this snapshot. When the database is closed, we will summarize
+ * the results of this table into segments_used. */
+ rc = sqlite3_exec(db,
+ "create temporary table snapshot_refs ("
+ " segmentid integer not null,"
+ " object text not null,"
+ " size integer not null"
+ ")", NULL, NULL, NULL);
+ if (rc != SQLITE_OK) {
+ ReportError(rc);
+ sqlite3_close(db);
+ fatal("Database initialization");
+ }
+ rc = sqlite3_exec(db,
+ "create unique index snapshot_refs_index "
+ "on snapshot_refs(segmentid, object)",
+ NULL, NULL, NULL);
+ if (rc != SQLITE_OK) {
+ ReportError(rc);
+ sqlite3_close(db);
+ fatal("Database initialization");
}
}
void LocalDb::Close()
{
int rc;
+
+ /* Summarize the snapshot_refs table into segments_used. */
+ sqlite3_stmt *stmt = Prepare(
+ "insert or replace into segments_used "
+ "select ? as snapshotid, segmentid, max(utilization) from ("
+ " select segmentid, cast(used as real) / size as utilization "
+ " from "
+ " (select segmentid, sum(size) as used from snapshot_refs "
+ " group by segmentid) "
+ " join segments using (segmentid) "
+ " union "
+ " select segmentid, utilization from segments_used "
+ " where snapshotid = ? "
+ ") group by segmentid"
+ );
+ sqlite3_bind_int64(stmt, 1, snapshotid);
+ sqlite3_bind_int64(stmt, 2, snapshotid);
+ rc = sqlite3_step(stmt);
+ if (rc != SQLITE_OK && rc != SQLITE_DONE) {
+ ReportError(rc);
+ sqlite3_close(db);
+ fprintf(stderr, "DATABASE ERROR: Unable to create segment summary!\n");
+ }
+ sqlite3_finalize(stmt);
+
+ /* Commit changes to the database and close. */
rc = sqlite3_exec(db, "commit", NULL, NULL, NULL);
if (rc != SQLITE_OK) {
fprintf(stderr, "DATABASE ERROR: Can't commit database!\n");
SQLITE_TRANSIENT);
rc = sqlite3_step(stmt);
if (rc != SQLITE_DONE) {
- throw IOException("Could not execute INSERT statement!");
+ fatal("Could not execute INSERT statement!");
}
sqlite3_finalize(stmt);
rc = sqlite3_step(stmt);
if (rc == SQLITE_DONE) {
- throw IOException("No segment found by id");
+ fatal("No segment found by id");
} else if (rc == SQLITE_ROW) {
result = sqlite3_column_int64(stmt, 0);
} else {
- throw IOException("Error executing find segment by id query");
+ fatal("Error executing find segment by id query");
}
sqlite3_finalize(stmt);
rc = sqlite3_step(stmt);
if (rc == SQLITE_DONE) {
- throw IOException("No segment found by id");
+ fatal("No segment found by id");
} else if (rc == SQLITE_ROW) {
result = (const char *)sqlite3_column_text(stmt, 0);
} else {
- throw IOException("Error executing find segment by id query");
+ fatal("Error executing find segment by id query");
}
sqlite3_finalize(stmt);
}
sqlite3_finalize(stmt);
+
+ if (age != 0.0) {
+ stmt = Prepare("update segments "
+ "set mtime = coalesce(max(mtime, ?), ?) "
+ "where segmentid = ?");
+ sqlite3_bind_double(stmt, 1, age);
+ sqlite3_bind_double(stmt, 2, age);
+ sqlite3_bind_int64(stmt, 3, SegmentToId(ref.get_segment()));
+ rc = sqlite3_step(stmt);
+ sqlite3_finalize(stmt);
+ }
}
ObjectReference LocalDb::FindObject(const string &checksum, int64_t size)
} else if (rc == SQLITE_ROW) {
ref = ObjectReference(IdToSegment(sqlite3_column_int64(stmt, 0)),
(const char *)sqlite3_column_text(stmt, 1));
+ ref.set_range(0, size, true);
} else {
fprintf(stderr, "Could not execute SELECT statement!\n");
ReportError(rc);
sqlite3_stmt *stmt;
bool found = false;
+ // Special objects (such as the zero object) aren't stored in segments, and
+ // so are always available.
+ if (!ref.is_normal())
+ return true;
+
stmt = Prepare("select count(*) from block_index "
"where segmentid = ? and object = ? and expired is null");
sqlite3_bind_int64(stmt, 1, SegmentToId(ref.get_segment()));
int rc;
sqlite3_stmt *stmt;
- stmt = Prepare("insert or ignore into snapshot_contents "
- "select blockid, ? as snapshotid from block_index "
+ if (!ref.is_normal())
+ return;
+
+ int64_t old_size = 0;
+ stmt = Prepare("select size from snapshot_refs "
"where segmentid = ? and object = ?");
- sqlite3_bind_int64(stmt, 1, snapshotid);
- sqlite3_bind_int64(stmt, 2, SegmentToId(ref.get_segment()));
+ sqlite3_bind_int64(stmt, 1, SegmentToId(ref.get_segment()));
string obj = ref.get_sequence();
- sqlite3_bind_text(stmt, 3, obj.c_str(), obj.size(), SQLITE_TRANSIENT);
+ sqlite3_bind_text(stmt, 2, obj.c_str(), obj.size(), SQLITE_TRANSIENT);
+ rc = sqlite3_step(stmt);
+ if (rc == SQLITE_ROW) {
+ old_size = sqlite3_column_int64(stmt, 0);
+ }
+ sqlite3_finalize(stmt);
+
+ int64_t block_size = 0;
+ stmt = Prepare("select size from block_index "
+ "where segmentid = ? and object = ?");
+ sqlite3_bind_int64(stmt, 1, SegmentToId(ref.get_segment()));
+ obj = ref.get_sequence();
+ sqlite3_bind_text(stmt, 2, obj.c_str(), obj.size(), SQLITE_TRANSIENT);
+ rc = sqlite3_step(stmt);
+ if (rc == SQLITE_ROW) {
+ block_size = sqlite3_column_int64(stmt, 0);
+ } else {
+ string refstr = ref.to_string();
+ fprintf(stderr, "No block found in block_index for %s\n",
+ refstr.c_str());
+ sqlite3_finalize(stmt);
+ return;
+ }
+ sqlite3_finalize(stmt);
+
+ int64_t new_size = old_size;
+ if (ref.has_range()) {
+ new_size += ref.get_range_length();
+ new_size = min(new_size, block_size);
+ } else {
+ new_size = block_size;
+ }
+
+ if (new_size != old_size) {
+ stmt = Prepare("insert or replace "
+ "into snapshot_refs(segmentid, object, size) "
+ "values (?, ?, ?)");
+ sqlite3_bind_int64(stmt, 1, SegmentToId(ref.get_segment()));
+ obj = ref.get_sequence();
+ sqlite3_bind_text(stmt, 2, obj.c_str(), obj.size(), SQLITE_TRANSIENT);
+ sqlite3_bind_int64(stmt, 3, new_size);
+
+ rc = sqlite3_step(stmt);
+ if (rc != SQLITE_DONE) {
+ fprintf(stderr, "Could not execute INSERT statement!\n");
+ ReportError(rc);
+ }
+
+ sqlite3_finalize(stmt);
+ }
+}
+
+void LocalDb::UseSegment(const std::string &segment, double utilization)
+{
+ int rc;
+ sqlite3_stmt *stmt;
+
+ stmt = Prepare("insert or replace "
+ "into segments_used(snapshotid, segmentid, utilization) "
+ "values (?, ?, ?)");
+ sqlite3_bind_int64(stmt, 1, snapshotid);
+ sqlite3_bind_int64(stmt, 2, SegmentToId(segment));
+ sqlite3_bind_double(stmt, 3, utilization);
rc = sqlite3_step(stmt);
if (rc != SQLITE_DONE) {
- fprintf(stderr, "Could not execute INSERT statement!\n");
+ fprintf(stderr, "Could not insert segment use record!\n");
ReportError(rc);
}
void LocalDb::SetSegmentChecksum(const std::string &segment,
const std::string &path,
- const std::string &checksum)
+ const std::string &checksum,
+ int size)
{
int rc;
sqlite3_stmt *stmt;
- stmt = Prepare("update segments set path = ?, checksum = ? "
+ stmt = Prepare("update segments set path = ?, checksum = ?, size = ?, "
+ "mtime = coalesce(mtime, julianday('now')) "
"where segmentid = ?");
sqlite3_bind_text(stmt, 1, path.c_str(), path.size(),
SQLITE_TRANSIENT);
sqlite3_bind_text(stmt, 2, checksum.c_str(), checksum.size(),
SQLITE_TRANSIENT);
- sqlite3_bind_int64(stmt, 3, SegmentToId(segment));
+ sqlite3_bind_int64(stmt, 3, size);
+ sqlite3_bind_int64(stmt, 4, SegmentToId(segment));
rc = sqlite3_step(stmt);
if (rc != SQLITE_DONE) {
return found;
}
+
+/* Look up and return the packed representation of the subblock chunk
+ * signatures. Returns true if signatures were found for the specified object,
+ * and if so sets *buf to point at a buffer of memory (allocated with malloc;
+ * the caller should free it), and *len to the length of the buffer. */
+bool LocalDb::LoadChunkSignatures(ObjectReference ref,
+ void **buf, size_t *len,
+ string *algorithm)
+{
+ int rc;
+ sqlite3_stmt *stmt;
+ int found = false;
+
+ stmt = Prepare("select signatures, algorithm from subblock_signatures "
+ "where blockid = (select blockid from block_index "
+ " where segmentid = ? and object = ?)");
+ sqlite3_bind_int64(stmt, 1, SegmentToId(ref.get_segment()));
+ string obj = ref.get_sequence();
+ sqlite3_bind_text(stmt, 2, obj.c_str(), obj.size(), SQLITE_TRANSIENT);
+
+ rc = sqlite3_step(stmt);
+ if (rc == SQLITE_DONE) {
+ } else if (rc == SQLITE_ROW) {
+ const void *data = sqlite3_column_blob(stmt, 0);
+ *len = sqlite3_column_bytes(stmt, 0);
+
+ if (*len > 0) {
+ *buf = malloc(*len);
+ if (*buf != NULL) {
+ memcpy(*buf, data, *len);
+ *algorithm = (const char *)sqlite3_column_text(stmt, 1);
+ found = true;
+ }
+ }
+ } else {
+ fprintf(stderr, "Could not execute SELECT statement!\n");
+ ReportError(rc);
+ }
+
+ sqlite3_finalize(stmt);
+
+ return found;
+}
+
+/* Store the subblock chunk signatures for a specified object. The object
+ * itself must have already been indexed in the database. */
+void LocalDb::StoreChunkSignatures(ObjectReference ref,
+ const void *buf, size_t len,
+ const string& algorithm)
+{
+ int rc;
+ sqlite3_stmt *stmt;
+
+ stmt = Prepare("select blockid from block_index "
+ "where segmentid = ? and object = ?");
+ sqlite3_bind_int64(stmt, 1, SegmentToId(ref.get_segment()));
+ string obj = ref.get_sequence();
+ sqlite3_bind_text(stmt, 2, obj.c_str(), obj.size(), SQLITE_TRANSIENT);
+
+ rc = sqlite3_step(stmt);
+ if (rc != SQLITE_ROW) {
+ fprintf(stderr,
+ "Could not determine blockid in StoreChunkSignatures!\n");
+ ReportError(rc);
+ fatal("Error getting blockid");
+ }
+ int64_t blockid = sqlite3_column_int64(stmt, 0);
+ sqlite3_finalize(stmt);
+
+ stmt = Prepare("insert or replace "
+ "into subblock_signatures(blockid, algorithm, signatures) "
+ "values (?, ?, ?)");
+ sqlite3_bind_int64(stmt, 1, blockid);
+ sqlite3_bind_text(stmt, 2, algorithm.c_str(), algorithm.size(),
+ SQLITE_TRANSIENT);
+ sqlite3_bind_blob(stmt, 3, buf, len, SQLITE_TRANSIENT);
+
+ rc = sqlite3_step(stmt);
+ if (rc != SQLITE_DONE) {
+ fprintf(stderr, "Could not insert sub-block checksums!\n");
+ ReportError(rc);
+ }
+
+ sqlite3_finalize(stmt);
+}
projects / cumulus.git / blobdiff