X-Git-Url: http://git.vrable.net/?p=cumulus.git;a=blobdiff_plain;f=localdb.cc;h=7304d49ba4c70aae53b69d5f9a092c1a969588a9;hp=c0ad7d68cfbab8f0fd21593cad6e73167462ecc1;hb=a9d83d1e32291aaa449442eaf19ade56bfd42778;hpb=4a312a9c567e5acae9f58587287795e0f7b0cd6c diff --git a/localdb.cc b/localdb.cc index c0ad7d6..7304d49 100644 --- a/localdb.cc +++ b/localdb.cc @@ -1,7 +1,24 @@ -/* LBS: An LFS-inspired filesystem backup system - * Copyright (C) 2007 Michael Vrable +/* Cumulus: Smart Filesystem Backup to Dumb Servers * - * When creating backup snapshots, maintain a local database of data blocks and + * Copyright (C) 2007-2008 The Regents of the University of California + * Written by Michael Vrable + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +/* When creating backup snapshots, maintain a local database of data blocks and * checksums, in addition to the data contents (which may be stored remotely). * This database is consulted when attempting to build incremental snapshots, * as it says which objects can be reused. @@ -15,11 +32,14 @@ #include #include +#include #include #include "localdb.h" #include "store.h" +#include "util.h" +using std::min; using std::string; /* Helper function to prepare a statement for execution in the current @@ -33,7 +53,7 @@ sqlite3_stmt *LocalDb::Prepare(const char *sql) rc = sqlite3_prepare_v2(db, sql, strlen(sql), &stmt, &tail); if (rc != SQLITE_OK) { ReportError(rc); - throw IOException(string("Error preparing statement: ") + sql); + fatal(string("Error preparing statement: ") + sql); } return stmt; @@ -46,7 +66,7 @@ void LocalDb::ReportError(int rc) } void LocalDb::Open(const char *path, const char *snapshot_name, - const char *snapshot_scheme) + const char *snapshot_scheme, double intent) { int rc; @@ -54,36 +74,37 @@ void LocalDb::Open(const char *path, const char *snapshot_name, if (rc) { fprintf(stderr, "Can't open database: %s\n", sqlite3_errmsg(db)); sqlite3_close(db); - throw IOException("Error opening local database"); + fatal("Error opening local database"); } rc = sqlite3_exec(db, "begin", NULL, NULL, NULL); if (rc) { fprintf(stderr, "Can't open database: %s\n", sqlite3_errmsg(db)); sqlite3_close(db); - throw IOException("Error starting transaction"); + fatal("Error starting transaction"); } sqlite3_extended_result_codes(db, 1); + if (snapshot_scheme == NULL) + snapshot_scheme = ""; + /* Insert this snapshot into the database, and determine the integer key * which will be used to identify it. */ sqlite3_stmt *stmt = Prepare("insert into " - "snapshots(name, scheme, timestamp) " - "values (?, ?, julianday('now'))"); + "snapshots(name, scheme, timestamp, intent) " + "values (?, ?, julianday('now'), ?)"); sqlite3_bind_text(stmt, 1, snapshot_name, strlen(snapshot_name), SQLITE_TRANSIENT); - if (snapshot_scheme == NULL) - sqlite3_bind_null(stmt, 2); - else - sqlite3_bind_text(stmt, 2, snapshot_scheme, strlen(snapshot_scheme), - SQLITE_TRANSIENT); + sqlite3_bind_text(stmt, 2, snapshot_scheme, strlen(snapshot_scheme), + SQLITE_TRANSIENT); + sqlite3_bind_double(stmt, 3, intent); rc = sqlite3_step(stmt); if (rc != SQLITE_DONE) { ReportError(rc); sqlite3_close(db); - throw IOException("Database execution error!"); + fatal("Database execution error!"); } snapshotid = sqlite3_last_insert_rowid(db); @@ -91,7 +112,7 @@ void LocalDb::Open(const char *path, const char *snapshot_name, if (snapshotid == 0) { ReportError(rc); sqlite3_close(db); - throw IOException("Find snapshot id"); + fatal("Find snapshot id"); } /* Create a temporary table which will be used to keep track of the objects @@ -106,7 +127,7 @@ void LocalDb::Open(const char *path, const char *snapshot_name, if (rc != SQLITE_OK) { ReportError(rc); sqlite3_close(db); - throw IOException("Database initialization"); + fatal("Database initialization"); } rc = sqlite3_exec(db, "create unique index snapshot_refs_index " @@ -115,7 +136,7 @@ void LocalDb::Open(const char *path, const char *snapshot_name, if (rc != SQLITE_OK) { ReportError(rc); sqlite3_close(db); - throw IOException("Database initialization"); + fatal("Database initialization"); } } @@ -124,14 +145,21 @@ void LocalDb::Close() int rc; /* Summarize the snapshot_refs table into segments_used. */ - sqlite3_stmt *stmt = Prepare("insert into segments_used " - "select ? as snapshotid, segmentid, " - "cast(used as real) / size as utilization " - "from " - "(select segmentid, sum(size) as used " - "from snapshot_refs group by segmentid) " - "join segments using (segmentid)"); + sqlite3_stmt *stmt = Prepare( + "insert or replace into segments_used " + "select ? as snapshotid, segmentid, max(utilization) from (" + " select segmentid, cast(used as real) / size as utilization " + " from " + " (select segmentid, sum(size) as used from snapshot_refs " + " group by segmentid) " + " join segments using (segmentid) " + " union " + " select segmentid, utilization from segments_used " + " where snapshotid = ? " + ") group by segmentid" + ); sqlite3_bind_int64(stmt, 1, snapshotid); + sqlite3_bind_int64(stmt, 2, snapshotid); rc = sqlite3_step(stmt); if (rc != SQLITE_OK && rc != SQLITE_DONE) { ReportError(rc); @@ -160,7 +188,7 @@ int64_t LocalDb::SegmentToId(const string &segment) SQLITE_TRANSIENT); rc = sqlite3_step(stmt); if (rc != SQLITE_DONE) { - throw IOException("Could not execute INSERT statement!"); + fatal("Could not execute INSERT statement!"); } sqlite3_finalize(stmt); @@ -170,11 +198,11 @@ int64_t LocalDb::SegmentToId(const string &segment) rc = sqlite3_step(stmt); if (rc == SQLITE_DONE) { - throw IOException("No segment found by id"); + fatal("No segment found by id"); } else if (rc == SQLITE_ROW) { result = sqlite3_column_int64(stmt, 0); } else { - throw IOException("Error executing find segment by id query"); + fatal("Error executing find segment by id query"); } sqlite3_finalize(stmt); @@ -193,11 +221,11 @@ string LocalDb::IdToSegment(int64_t segmentid) rc = sqlite3_step(stmt); if (rc == SQLITE_DONE) { - throw IOException("No segment found by id"); + fatal("No segment found by id"); } else if (rc == SQLITE_ROW) { result = (const char *)sqlite3_column_text(stmt, 0); } else { - throw IOException("Error executing find segment by id query"); + fatal("Error executing find segment by id query"); } sqlite3_finalize(stmt); @@ -238,6 +266,17 @@ void LocalDb::StoreObject(const ObjectReference& ref, } sqlite3_finalize(stmt); + + if (age != 0.0) { + stmt = Prepare("update segments " + "set mtime = coalesce(max(mtime, ?), ?) " + "where segmentid = ?"); + sqlite3_bind_double(stmt, 1, age); + sqlite3_bind_double(stmt, 2, age); + sqlite3_bind_int64(stmt, 3, SegmentToId(ref.get_segment())); + rc = sqlite3_step(stmt); + sqlite3_finalize(stmt); + } } ObjectReference LocalDb::FindObject(const string &checksum, int64_t size) @@ -257,6 +296,7 @@ ObjectReference LocalDb::FindObject(const string &checksum, int64_t size) } else if (rc == SQLITE_ROW) { ref = ObjectReference(IdToSegment(sqlite3_column_int64(stmt, 0)), (const char *)sqlite3_column_text(stmt, 1)); + ref.set_range(0, size, true); } else { fprintf(stderr, "Could not execute SELECT statement!\n"); ReportError(rc); @@ -304,6 +344,11 @@ bool LocalDb::IsAvailable(const ObjectReference &ref) sqlite3_stmt *stmt; bool found = false; + // Special objects (such as the zero object) aren't stored in segments, and + // so are always available. + if (!ref.is_normal()) + return true; + stmt = Prepare("select count(*) from block_index " "where segmentid = ? and object = ? and expired is null"); sqlite3_bind_int64(stmt, 1, SegmentToId(ref.get_segment())); @@ -331,16 +376,81 @@ void LocalDb::UseObject(const ObjectReference& ref) int rc; sqlite3_stmt *stmt; - stmt = Prepare("insert or ignore into snapshot_refs " - "select segmentid, object, size from block_index " + if (!ref.is_normal()) + return; + + int64_t old_size = 0; + stmt = Prepare("select size from snapshot_refs " "where segmentid = ? and object = ?"); sqlite3_bind_int64(stmt, 1, SegmentToId(ref.get_segment())); string obj = ref.get_sequence(); sqlite3_bind_text(stmt, 2, obj.c_str(), obj.size(), SQLITE_TRANSIENT); + rc = sqlite3_step(stmt); + if (rc == SQLITE_ROW) { + old_size = sqlite3_column_int64(stmt, 0); + } + sqlite3_finalize(stmt); + + int64_t block_size = 0; + stmt = Prepare("select size from block_index " + "where segmentid = ? and object = ?"); + sqlite3_bind_int64(stmt, 1, SegmentToId(ref.get_segment())); + obj = ref.get_sequence(); + sqlite3_bind_text(stmt, 2, obj.c_str(), obj.size(), SQLITE_TRANSIENT); + rc = sqlite3_step(stmt); + if (rc == SQLITE_ROW) { + block_size = sqlite3_column_int64(stmt, 0); + } else { + string refstr = ref.to_string(); + fprintf(stderr, "No block found in block_index for %s\n", + refstr.c_str()); + sqlite3_finalize(stmt); + return; + } + sqlite3_finalize(stmt); + + int64_t new_size = old_size; + if (ref.has_range()) { + new_size += ref.get_range_length(); + new_size = min(new_size, block_size); + } else { + new_size = block_size; + } + + if (new_size != old_size) { + stmt = Prepare("insert or replace " + "into snapshot_refs(segmentid, object, size) " + "values (?, ?, ?)"); + sqlite3_bind_int64(stmt, 1, SegmentToId(ref.get_segment())); + obj = ref.get_sequence(); + sqlite3_bind_text(stmt, 2, obj.c_str(), obj.size(), SQLITE_TRANSIENT); + sqlite3_bind_int64(stmt, 3, new_size); + + rc = sqlite3_step(stmt); + if (rc != SQLITE_DONE) { + fprintf(stderr, "Could not execute INSERT statement!\n"); + ReportError(rc); + } + + sqlite3_finalize(stmt); + } +} + +void LocalDb::UseSegment(const std::string &segment, double utilization) +{ + int rc; + sqlite3_stmt *stmt; + + stmt = Prepare("insert or replace " + "into segments_used(snapshotid, segmentid, utilization) " + "values (?, ?, ?)"); + sqlite3_bind_int64(stmt, 1, snapshotid); + sqlite3_bind_int64(stmt, 2, SegmentToId(segment)); + sqlite3_bind_double(stmt, 3, utilization); rc = sqlite3_step(stmt); if (rc != SQLITE_DONE) { - fprintf(stderr, "Could not execute INSERT statement!\n"); + fprintf(stderr, "Could not insert segment use record!\n"); ReportError(rc); } @@ -349,20 +459,20 @@ void LocalDb::UseObject(const ObjectReference& ref) void LocalDb::SetSegmentChecksum(const std::string &segment, const std::string &path, - const std::string &checksum) + const std::string &checksum, + int size) { int rc; sqlite3_stmt *stmt; - stmt = Prepare("update segments set path = ?, checksum = ?, " - "size = (select sum(size) from block_index " - " where segmentid = ?) " + stmt = Prepare("update segments set path = ?, checksum = ?, size = ?, " + "mtime = coalesce(mtime, julianday('now')) " "where segmentid = ?"); sqlite3_bind_text(stmt, 1, path.c_str(), path.size(), SQLITE_TRANSIENT); sqlite3_bind_text(stmt, 2, checksum.c_str(), checksum.size(), SQLITE_TRANSIENT); - sqlite3_bind_int64(stmt, 3, SegmentToId(segment)); + sqlite3_bind_int64(stmt, 3, size); sqlite3_bind_int64(stmt, 4, SegmentToId(segment)); rc = sqlite3_step(stmt); @@ -413,3 +523,88 @@ bool LocalDb::GetSegmentChecksum(const string &segment, return found; } + +/* Look up and return the packed representation of the subblock chunk + * signatures. Returns true if signatures were found for the specified object, + * and if so sets *buf to point at a buffer of memory (allocated with malloc; + * the caller should free it), and *len to the length of the buffer. */ +bool LocalDb::LoadChunkSignatures(ObjectReference ref, + void **buf, size_t *len, + string *algorithm) +{ + int rc; + sqlite3_stmt *stmt; + int found = false; + + stmt = Prepare("select signatures, algorithm from subblock_signatures " + "where blockid = (select blockid from block_index " + " where segmentid = ? and object = ?)"); + sqlite3_bind_int64(stmt, 1, SegmentToId(ref.get_segment())); + string obj = ref.get_sequence(); + sqlite3_bind_text(stmt, 2, obj.c_str(), obj.size(), SQLITE_TRANSIENT); + + rc = sqlite3_step(stmt); + if (rc == SQLITE_DONE) { + } else if (rc == SQLITE_ROW) { + const void *data = sqlite3_column_blob(stmt, 0); + *len = sqlite3_column_bytes(stmt, 0); + + if (*len > 0) { + *buf = malloc(*len); + if (*buf != NULL) { + memcpy(*buf, data, *len); + *algorithm = (const char *)sqlite3_column_text(stmt, 1); + found = true; + } + } + } else { + fprintf(stderr, "Could not execute SELECT statement!\n"); + ReportError(rc); + } + + sqlite3_finalize(stmt); + + return found; +} + +/* Store the subblock chunk signatures for a specified object. The object + * itself must have already been indexed in the database. */ +void LocalDb::StoreChunkSignatures(ObjectReference ref, + const void *buf, size_t len, + const string& algorithm) +{ + int rc; + sqlite3_stmt *stmt; + + stmt = Prepare("select blockid from block_index " + "where segmentid = ? and object = ?"); + sqlite3_bind_int64(stmt, 1, SegmentToId(ref.get_segment())); + string obj = ref.get_sequence(); + sqlite3_bind_text(stmt, 2, obj.c_str(), obj.size(), SQLITE_TRANSIENT); + + rc = sqlite3_step(stmt); + if (rc != SQLITE_ROW) { + fprintf(stderr, + "Could not determine blockid in StoreChunkSignatures!\n"); + ReportError(rc); + fatal("Error getting blockid"); + } + int64_t blockid = sqlite3_column_int64(stmt, 0); + sqlite3_finalize(stmt); + + stmt = Prepare("insert or replace " + "into subblock_signatures(blockid, algorithm, signatures) " + "values (?, ?, ?)"); + sqlite3_bind_int64(stmt, 1, blockid); + sqlite3_bind_text(stmt, 2, algorithm.c_str(), algorithm.size(), + SQLITE_TRANSIENT); + sqlite3_bind_blob(stmt, 3, buf, len, SQLITE_TRANSIENT); + + rc = sqlite3_step(stmt); + if (rc != SQLITE_DONE) { + fprintf(stderr, "Could not insert sub-block checksums!\n"); + ReportError(rc); + } + + sqlite3_finalize(stmt); +}