-/* Cumulus: Smart Filesystem Backup to Dumb Servers
- *
- * Copyright (C) 2007-2008 The Regents of the University of California
- * Written by Michael Vrable <mvrable@cs.ucsd.edu>
+/* Cumulus: Efficient Filesystem Backup to the Cloud
+ * Copyright (C) 2007-2008 The Cumulus Developers
+ * See the AUTHORS file for a list of contributors.
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
#include <string.h>
#include <sqlite3.h>
+#include <algorithm>
+#include <map>
#include <string>
#include "localdb.h"
#include "store.h"
+#include "util.h"
+using std::map;
+using std::max;
+using std::min;
+using std::set;
using std::string;
+static const int SCHEMA_MAJOR = 0;
+static const int SCHEMA_MINOR = 11;
+
/* Helper function to prepare a statement for execution in the current
* database. */
sqlite3_stmt *LocalDb::Prepare(const char *sql)
rc = sqlite3_prepare_v2(db, sql, strlen(sql), &stmt, &tail);
if (rc != SQLITE_OK) {
ReportError(rc);
- throw IOException(string("Error preparing statement: ") + sql);
+ fatal(string("Error preparing statement: ") + sql);
}
return stmt;
}
void LocalDb::Open(const char *path, const char *snapshot_name,
- const char *snapshot_scheme, double intent)
+ const char *snapshot_scheme)
{
int rc;
if (rc) {
fprintf(stderr, "Can't open database: %s\n", sqlite3_errmsg(db));
sqlite3_close(db);
- throw IOException("Error opening local database");
+ fatal("Error opening local database");
}
rc = sqlite3_exec(db, "begin", NULL, NULL, NULL);
if (rc) {
fprintf(stderr, "Can't open database: %s\n", sqlite3_errmsg(db));
sqlite3_close(db);
- throw IOException("Error starting transaction");
+ fatal("Error starting transaction");
}
sqlite3_extended_result_codes(db, 1);
+ /* Check that the local database format is the correct version; if not,
+ * report an error. */
+ sqlite3_stmt *stmt = Prepare("select major, minor from schema_version");
+
+ rc = sqlite3_step(stmt);
+ if (rc != SQLITE_ROW) {
+ fatal("Unable to read local database version from database");
+ } else if (rc == SQLITE_ROW) {
+ int major = sqlite3_column_int(stmt, 0);
+ int minor = sqlite3_column_int(stmt, 1);
+ if (major != SCHEMA_MAJOR || minor != SCHEMA_MINOR) {
+ fprintf(stderr,
+ "Local database does not have required schema version!\n"
+ " expected: %d.%d, found: %d.%d\n",
+ SCHEMA_MAJOR, SCHEMA_MINOR, major, minor);
+ fatal("Unable to continue.");
+ }
+ }
+ sqlite3_finalize(stmt);
+
+ if (snapshot_scheme == NULL)
+ snapshot_scheme = "";
+
/* Insert this snapshot into the database, and determine the integer key
* which will be used to identify it. */
- sqlite3_stmt *stmt = Prepare("insert into "
- "snapshots(name, scheme, timestamp, intent) "
- "values (?, ?, julianday('now'), ?)");
+ stmt = Prepare("insert into snapshots(name, scheme, timestamp) "
+ "values (?, ?, julianday('now'))");
sqlite3_bind_text(stmt, 1, snapshot_name, strlen(snapshot_name),
SQLITE_TRANSIENT);
- if (snapshot_scheme == NULL)
- sqlite3_bind_null(stmt, 2);
- else
- sqlite3_bind_text(stmt, 2, snapshot_scheme, strlen(snapshot_scheme),
- SQLITE_TRANSIENT);
- sqlite3_bind_double(stmt, 3, intent);
+ sqlite3_bind_text(stmt, 2, snapshot_scheme, strlen(snapshot_scheme),
+ SQLITE_TRANSIENT);
rc = sqlite3_step(stmt);
if (rc != SQLITE_DONE) {
ReportError(rc);
sqlite3_close(db);
- throw IOException("Database execution error!");
+ fatal("Database execution error!");
}
snapshotid = sqlite3_last_insert_rowid(db);
if (snapshotid == 0) {
ReportError(rc);
sqlite3_close(db);
- throw IOException("Find snapshot id");
+ fatal("Find snapshot id");
}
/* Create a temporary table which will be used to keep track of the objects
if (rc != SQLITE_OK) {
ReportError(rc);
sqlite3_close(db);
- throw IOException("Database initialization");
+ fatal("Database initialization");
}
rc = sqlite3_exec(db,
"create unique index snapshot_refs_index "
if (rc != SQLITE_OK) {
ReportError(rc);
sqlite3_close(db);
- throw IOException("Database initialization");
+ fatal("Database initialization");
}
}
{
int rc;
- /* Summarize the snapshot_refs table into segments_used. */
+ /* Summarize the snapshot_refs table into segment_utilization. */
sqlite3_stmt *stmt = Prepare(
- "insert or replace into segments_used "
- "select ? as snapshotid, segmentid, max(utilization) from ("
- " select segmentid, cast(used as real) / size as utilization "
- " from "
- " (select segmentid, sum(size) as used from snapshot_refs "
- " group by segmentid) "
- " join segments using (segmentid) "
- " union "
- " select segmentid, utilization from segments_used "
- " where snapshotid = ? "
- ") group by segmentid"
+ "insert or replace into segment_utilization "
+ "select ? as snapshotid, segmentid, sum(size) "
+ "from snapshot_refs group by segmentid"
);
sqlite3_bind_int64(stmt, 1, snapshotid);
- sqlite3_bind_int64(stmt, 2, snapshotid);
rc = sqlite3_step(stmt);
if (rc != SQLITE_OK && rc != SQLITE_DONE) {
ReportError(rc);
SQLITE_TRANSIENT);
rc = sqlite3_step(stmt);
if (rc != SQLITE_DONE) {
- throw IOException("Could not execute INSERT statement!");
+ fatal("Could not execute INSERT statement!");
}
sqlite3_finalize(stmt);
rc = sqlite3_step(stmt);
if (rc == SQLITE_DONE) {
- throw IOException("No segment found by id");
+ fatal("No segment found by id");
} else if (rc == SQLITE_ROW) {
result = sqlite3_column_int64(stmt, 0);
} else {
- throw IOException("Error executing find segment by id query");
+ fatal("Error executing find segment by id query");
}
sqlite3_finalize(stmt);
rc = sqlite3_step(stmt);
if (rc == SQLITE_DONE) {
- throw IOException("No segment found by id");
+ fatal("No segment found by id");
} else if (rc == SQLITE_ROW) {
result = (const char *)sqlite3_column_text(stmt, 0);
} else {
- throw IOException("Error executing find segment by id query");
+ fatal("Error executing find segment by id query");
}
sqlite3_finalize(stmt);
return result;
}
-void LocalDb::StoreObject(const ObjectReference& ref,
- const string &checksum, int64_t size,
- double age)
+void LocalDb::StoreObject(const ObjectReference& ref, double age)
{
int rc;
sqlite3_stmt *stmt;
+ assert(ref.has_checksum());
+ string checksum = ref.get_checksum();
+ assert(ref.range_is_exact());
+ int64_t size = ref.get_range_length();
+
if (age == 0.0) {
stmt = Prepare("insert into block_index("
"segmentid, object, checksum, size, timestamp) "
}
sqlite3_finalize(stmt);
-
- if (age != 0.0) {
- stmt = Prepare("update segments "
- "set mtime = coalesce(max(mtime, ?), ?) "
- "where segmentid = ?");
- sqlite3_bind_double(stmt, 1, age);
- sqlite3_bind_double(stmt, 2, age);
- sqlite3_bind_int64(stmt, 3, SegmentToId(ref.get_segment()));
- rc = sqlite3_step(stmt);
- sqlite3_finalize(stmt);
- }
}
ObjectReference LocalDb::FindObject(const string &checksum, int64_t size)
} else if (rc == SQLITE_ROW) {
ref = ObjectReference(IdToSegment(sqlite3_column_int64(stmt, 0)),
(const char *)sqlite3_column_text(stmt, 1));
- ref.set_range(0, size);
+ ref.set_range(0, size, true);
} else {
fprintf(stderr, "Could not execute SELECT statement!\n");
ReportError(rc);
sqlite3_stmt *stmt;
bool found = false;
- stmt = Prepare("select segmentid, object, timestamp, expired "
+ stmt = Prepare("select segmentid, object, julianday(timestamp), expired "
"from block_index where checksum = ? and size = ?");
sqlite3_bind_text(stmt, 1, checksum.c_str(), checksum.size(),
SQLITE_TRANSIENT);
return found;
}
+set<string> LocalDb::GetUsedSegments()
+{
+ int rc;
+ sqlite3_stmt *stmt;
+ set<string> result;
+
+ stmt = Prepare("select segment from segments "
+ "where segmentid in (select segmentid from snapshot_refs)");
+
+ while (true) {
+ rc = sqlite3_step(stmt);
+ if (rc == SQLITE_ROW) {
+ const char *segment
+ = reinterpret_cast<const char *>(sqlite3_column_text(stmt, 0));
+ result.insert(segment);
+ } else if (rc == SQLITE_DONE) {
+ break;
+ } else {
+ ReportError(rc);
+ }
+ }
+
+ sqlite3_finalize(stmt);
+
+ return result;
+}
+
void LocalDb::UseObject(const ObjectReference& ref)
{
int rc;
if (!ref.is_normal())
return;
- stmt = Prepare("insert or ignore into snapshot_refs "
- "select segmentid, object, size from block_index "
+ int64_t old_size = 0;
+ stmt = Prepare("select size from snapshot_refs "
"where segmentid = ? and object = ?");
sqlite3_bind_int64(stmt, 1, SegmentToId(ref.get_segment()));
string obj = ref.get_sequence();
sqlite3_bind_text(stmt, 2, obj.c_str(), obj.size(), SQLITE_TRANSIENT);
-
rc = sqlite3_step(stmt);
- if (rc != SQLITE_DONE) {
- fprintf(stderr, "Could not execute INSERT statement!\n");
- ReportError(rc);
+ if (rc == SQLITE_ROW) {
+ old_size = sqlite3_column_int64(stmt, 0);
}
-
sqlite3_finalize(stmt);
-}
-
-void LocalDb::UseSegment(const std::string &segment, double utilization)
-{
- int rc;
- sqlite3_stmt *stmt;
- stmt = Prepare("insert or replace "
- "into segments_used(snapshotid, segmentid, utilization) "
- "values (?, ?, ?)");
- sqlite3_bind_int64(stmt, 1, snapshotid);
- sqlite3_bind_int64(stmt, 2, SegmentToId(segment));
- sqlite3_bind_double(stmt, 3, utilization);
+ // Attempt to determine the underlying size of the object. This may
+ // require a database lookup if the length is not encoded into the object
+ // reference already.
+ int64_t object_size = 0;
+ if (ref.range_is_exact()) {
+ object_size = ref.get_range_length();
+ } else {
+ stmt = Prepare("select size from block_index "
+ "where segmentid = ? and object = ?");
+ sqlite3_bind_int64(stmt, 1, SegmentToId(ref.get_segment()));
+ obj = ref.get_sequence();
+ sqlite3_bind_text(stmt, 2, obj.c_str(), obj.size(), SQLITE_TRANSIENT);
+ rc = sqlite3_step(stmt);
+ if (rc == SQLITE_ROW) {
+ object_size = sqlite3_column_int64(stmt, 0);
+ } else {
+ fprintf(stderr, "Warning: No block found in block_index for %s\n",
+ ref.to_string().c_str());
+ }
+ sqlite3_finalize(stmt);
+ }
- rc = sqlite3_step(stmt);
- if (rc != SQLITE_DONE) {
- fprintf(stderr, "Could not insert segment use record!\n");
- ReportError(rc);
+ // Possibly mark additional bytes as being referenced. The number of bytes
+ // referenced can only be increased (up to the object size). The bytes
+ // referenced will be set to the object size only if the entire object is
+ // referenced at once: a series of partial ranges that add up to the total
+ // size will have a reference size capped at just less than the full object
+ // size (we can't tell if some bytes were referenced multiple times, and
+ // thus we conservatively assume some bytes might still be unreferenced).
+ int64_t new_refs;
+ if (ref.has_range()) {
+ new_refs = ref.get_range_length();
+ } else {
+ new_refs = object_size;
}
+ int64_t new_size = old_size + new_refs;
+ if (old_size < object_size && new_refs < object_size)
+ new_size = min(new_size, object_size - 1);
+ new_size = min(object_size, new_size);
+ new_size = max(new_size, (int64_t)0);
+
+ if (new_size != old_size) {
+ stmt = Prepare("insert or replace "
+ "into snapshot_refs(segmentid, object, size) "
+ "values (?, ?, ?)");
+ sqlite3_bind_int64(stmt, 1, SegmentToId(ref.get_segment()));
+ obj = ref.get_sequence();
+ sqlite3_bind_text(stmt, 2, obj.c_str(), obj.size(), SQLITE_TRANSIENT);
+ sqlite3_bind_int64(stmt, 3, new_size);
- sqlite3_finalize(stmt);
+ rc = sqlite3_step(stmt);
+ if (rc != SQLITE_DONE) {
+ fprintf(stderr, "Could not execute INSERT statement!\n");
+ ReportError(rc);
+ }
+
+ sqlite3_finalize(stmt);
+ }
}
-void LocalDb::SetSegmentChecksum(const std::string &segment,
+void LocalDb::SetSegmentMetadata(const std::string &segment,
const std::string &path,
const std::string &checksum,
- int size)
+ const std::string &type,
+ int data_size, int disk_size)
{
int rc;
sqlite3_stmt *stmt;
- stmt = Prepare("update segments set path = ?, checksum = ?, size = ?, "
- "mtime = coalesce(mtime, julianday('now')) "
+ stmt = Prepare("update segments set path = ?, checksum = ?, "
+ "type = ?, data_size = ?, disk_size = ?, "
+ "timestamp = coalesce(julianday(timestamp), "
+ " julianday('now')) "
"where segmentid = ?");
sqlite3_bind_text(stmt, 1, path.c_str(), path.size(),
SQLITE_TRANSIENT);
sqlite3_bind_text(stmt, 2, checksum.c_str(), checksum.size(),
SQLITE_TRANSIENT);
- sqlite3_bind_int64(stmt, 3, size);
- sqlite3_bind_int64(stmt, 4, SegmentToId(segment));
+ sqlite3_bind_text(stmt, 3, type.c_str(), type.size(),
+ SQLITE_TRANSIENT);
+ sqlite3_bind_int64(stmt, 4, data_size);
+ sqlite3_bind_int64(stmt, 5, disk_size);
+ sqlite3_bind_int64(stmt, 6, SegmentToId(segment));
rc = sqlite3_step(stmt);
if (rc != SQLITE_DONE) {
sqlite3_finalize(stmt);
}
-bool LocalDb::GetSegmentChecksum(const string &segment,
- string *seg_path,
- string *seg_checksum)
+map<string, string> LocalDb::GetSegmentMetadata(const string &segment)
{
int rc;
sqlite3_stmt *stmt;
- ObjectReference ref;
- int found = false;
+ map<string, string> info;
- stmt = Prepare("select path, checksum from segments where segment = ?");
+ // Names in the returned map, in the order returned from the select
+ // statement below.
+ static const char *fields[] = {
+ "datetime", "path", "checksum", "data_size", "disk_size", "type", NULL
+ };
+
+ stmt = Prepare("select datetime(timestamp), path, checksum, "
+ " data_size, disk_size, type "
+ "from segments where segment = ?");
sqlite3_bind_text(stmt, 1, segment.c_str(), segment.size(),
SQLITE_TRANSIENT);
rc = sqlite3_step(stmt);
if (rc == SQLITE_DONE) {
} else if (rc == SQLITE_ROW) {
- found = true;
- const char *val;
-
- val = (const char *)sqlite3_column_text(stmt, 0);
- if (val == NULL)
- found = false;
- else
- *seg_path = val;
-
- val = (const char *)sqlite3_column_text(stmt, 1);
- if (val == NULL)
- found = false;
- else
- *seg_checksum = val;
+ info["segment"] = segment;
+ for (int i = 0; fields[i] != NULL; i++) {
+ const char *val = (const char *)sqlite3_column_text(stmt, i);
+ if (val != NULL)
+ info[fields[i]] = val;
+ }
} else {
fprintf(stderr, "Could not execute SELECT statement!\n");
ReportError(rc);
sqlite3_finalize(stmt);
- return found;
+ return info;
}
/* Look up and return the packed representation of the subblock chunk
sqlite3_stmt *stmt;
int found = false;
- stmt = Prepare("select algorithm, signatures from subblock_signatures "
+ stmt = Prepare("select signatures, algorithm from subblock_signatures "
"where blockid = (select blockid from block_index "
" where segmentid = ? and object = ?)");
sqlite3_bind_int64(stmt, 1, SegmentToId(ref.get_segment()));
fprintf(stderr,
"Could not determine blockid in StoreChunkSignatures!\n");
ReportError(rc);
- throw IOException("Error getting blockid");
+ fatal("Error getting blockid");
}
int64_t blockid = sqlite3_column_int64(stmt, 0);
sqlite3_finalize(stmt);
projects / cumulus.git / blobdiff