1 /* Cumulus: Smart Filesystem Backup to Dumb Servers
3 * Copyright (C) 2007-2008 The Regents of the University of California
4 * Written by Michael Vrable <mvrable@cs.ucsd.edu>
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License as published by
8 * the Free Software Foundation; either version 2 of the License, or
9 * (at your option) any later version.
11 * This program is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 * GNU General Public License for more details.
16 * You should have received a copy of the GNU General Public License along
17 * with this program; if not, write to the Free Software Foundation, Inc.,
18 * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
21 /* When creating backup snapshots, maintain a local database of data blocks and
22 * checksums, in addition to the data contents (which may be stored remotely).
23 * This database is consulted when attempting to build incremental snapshots,
24 * as it says which objects can be reused.
26 * The database is implemented as an SQLite3 database, but this implementation
27 * detail is kept internal to this file, so that the storage format may be
44 /* Helper function to prepare a statement for execution in the current
46 sqlite3_stmt *LocalDb::Prepare(const char *sql)
52 rc = sqlite3_prepare_v2(db, sql, strlen(sql), &stmt, &tail);
53 if (rc != SQLITE_OK) {
55 throw IOException(string("Error preparing statement: ") + sql);
61 void LocalDb::ReportError(int rc)
63 fprintf(stderr, "Result code: %d\n", rc);
64 fprintf(stderr, "Error message: %s\n", sqlite3_errmsg(db));
67 void LocalDb::Open(const char *path, const char *snapshot_name,
68 const char *snapshot_scheme, double intent)
72 rc = sqlite3_open(path, &db);
74 fprintf(stderr, "Can't open database: %s\n", sqlite3_errmsg(db));
76 throw IOException("Error opening local database");
79 rc = sqlite3_exec(db, "begin", NULL, NULL, NULL);
81 fprintf(stderr, "Can't open database: %s\n", sqlite3_errmsg(db));
83 throw IOException("Error starting transaction");
86 sqlite3_extended_result_codes(db, 1);
88 if (snapshot_scheme == NULL)
91 /* Insert this snapshot into the database, and determine the integer key
92 * which will be used to identify it. */
93 sqlite3_stmt *stmt = Prepare("insert into "
94 "snapshots(name, scheme, timestamp, intent) "
95 "values (?, ?, julianday('now'), ?)");
96 sqlite3_bind_text(stmt, 1, snapshot_name, strlen(snapshot_name),
98 sqlite3_bind_text(stmt, 2, snapshot_scheme, strlen(snapshot_scheme),
100 sqlite3_bind_double(stmt, 3, intent);
102 rc = sqlite3_step(stmt);
103 if (rc != SQLITE_DONE) {
106 throw IOException("Database execution error!");
109 snapshotid = sqlite3_last_insert_rowid(db);
110 sqlite3_finalize(stmt);
111 if (snapshotid == 0) {
114 throw IOException("Find snapshot id");
117 /* Create a temporary table which will be used to keep track of the objects
118 * used by this snapshot. When the database is closed, we will summarize
119 * the results of this table into segments_used. */
120 rc = sqlite3_exec(db,
121 "create temporary table snapshot_refs ("
122 " segmentid integer not null,"
123 " object text not null,"
124 " size integer not null"
125 ")", NULL, NULL, NULL);
126 if (rc != SQLITE_OK) {
129 throw IOException("Database initialization");
131 rc = sqlite3_exec(db,
132 "create unique index snapshot_refs_index "
133 "on snapshot_refs(segmentid, object)",
135 if (rc != SQLITE_OK) {
138 throw IOException("Database initialization");
142 void LocalDb::Close()
146 /* Summarize the snapshot_refs table into segments_used. */
147 sqlite3_stmt *stmt = Prepare(
148 "insert or replace into segments_used "
149 "select ? as snapshotid, segmentid, max(utilization) from ("
150 " select segmentid, cast(used as real) / size as utilization "
152 " (select segmentid, sum(size) as used from snapshot_refs "
153 " group by segmentid) "
154 " join segments using (segmentid) "
156 " select segmentid, utilization from segments_used "
157 " where snapshotid = ? "
158 ") group by segmentid"
160 sqlite3_bind_int64(stmt, 1, snapshotid);
161 sqlite3_bind_int64(stmt, 2, snapshotid);
162 rc = sqlite3_step(stmt);
163 if (rc != SQLITE_OK && rc != SQLITE_DONE) {
166 fprintf(stderr, "DATABASE ERROR: Unable to create segment summary!\n");
168 sqlite3_finalize(stmt);
170 /* Commit changes to the database and close. */
171 rc = sqlite3_exec(db, "commit", NULL, NULL, NULL);
172 if (rc != SQLITE_OK) {
173 fprintf(stderr, "DATABASE ERROR: Can't commit database!\n");
179 int64_t LocalDb::SegmentToId(const string &segment)
185 stmt = Prepare("insert or ignore into segments(segment) values (?)");
186 sqlite3_bind_text(stmt, 1, segment.c_str(), segment.size(),
188 rc = sqlite3_step(stmt);
189 if (rc != SQLITE_DONE) {
190 throw IOException("Could not execute INSERT statement!");
192 sqlite3_finalize(stmt);
194 stmt = Prepare("select segmentid from segments where segment = ?");
195 sqlite3_bind_text(stmt, 1, segment.c_str(), segment.size(),
198 rc = sqlite3_step(stmt);
199 if (rc == SQLITE_DONE) {
200 throw IOException("No segment found by id");
201 } else if (rc == SQLITE_ROW) {
202 result = sqlite3_column_int64(stmt, 0);
204 throw IOException("Error executing find segment by id query");
207 sqlite3_finalize(stmt);
212 string LocalDb::IdToSegment(int64_t segmentid)
218 stmt = Prepare("select segment from segments where segmentid = ?");
219 sqlite3_bind_int64(stmt, 1, segmentid);
221 rc = sqlite3_step(stmt);
222 if (rc == SQLITE_DONE) {
223 throw IOException("No segment found by id");
224 } else if (rc == SQLITE_ROW) {
225 result = (const char *)sqlite3_column_text(stmt, 0);
227 throw IOException("Error executing find segment by id query");
230 sqlite3_finalize(stmt);
235 void LocalDb::StoreObject(const ObjectReference& ref,
236 const string &checksum, int64_t size,
243 stmt = Prepare("insert into block_index("
244 "segmentid, object, checksum, size, timestamp) "
245 "values (?, ?, ?, ?, julianday('now'))");
247 stmt = Prepare("insert into block_index("
248 "segmentid, object, checksum, size, timestamp) "
249 "values (?, ?, ?, ?, ?)");
252 sqlite3_bind_int64(stmt, 1, SegmentToId(ref.get_segment()));
253 string obj = ref.get_sequence();
254 sqlite3_bind_text(stmt, 2, obj.c_str(), obj.size(), SQLITE_TRANSIENT);
255 sqlite3_bind_text(stmt, 3, checksum.c_str(), checksum.size(),
257 sqlite3_bind_int64(stmt, 4, size);
259 sqlite3_bind_double(stmt, 5, age);
261 rc = sqlite3_step(stmt);
262 if (rc != SQLITE_DONE) {
263 fprintf(stderr, "Could not execute INSERT statement!\n");
267 sqlite3_finalize(stmt);
270 stmt = Prepare("update segments "
271 "set mtime = coalesce(max(mtime, ?), ?) "
272 "where segmentid = ?");
273 sqlite3_bind_double(stmt, 1, age);
274 sqlite3_bind_double(stmt, 2, age);
275 sqlite3_bind_int64(stmt, 3, SegmentToId(ref.get_segment()));
276 rc = sqlite3_step(stmt);
277 sqlite3_finalize(stmt);
281 ObjectReference LocalDb::FindObject(const string &checksum, int64_t size)
287 stmt = Prepare("select segmentid, object from block_index "
288 "where checksum = ? and size = ? and expired is null");
289 sqlite3_bind_text(stmt, 1, checksum.c_str(), checksum.size(),
291 sqlite3_bind_int64(stmt, 2, size);
293 rc = sqlite3_step(stmt);
294 if (rc == SQLITE_DONE) {
295 } else if (rc == SQLITE_ROW) {
296 ref = ObjectReference(IdToSegment(sqlite3_column_int64(stmt, 0)),
297 (const char *)sqlite3_column_text(stmt, 1));
298 ref.set_range(0, size);
300 fprintf(stderr, "Could not execute SELECT statement!\n");
304 sqlite3_finalize(stmt);
309 bool LocalDb::IsOldObject(const string &checksum, int64_t size, double *age,
316 stmt = Prepare("select segmentid, object, timestamp, expired "
317 "from block_index where checksum = ? and size = ?");
318 sqlite3_bind_text(stmt, 1, checksum.c_str(), checksum.size(),
320 sqlite3_bind_int64(stmt, 2, size);
322 rc = sqlite3_step(stmt);
323 if (rc == SQLITE_DONE) {
325 } else if (rc == SQLITE_ROW) {
327 *age = sqlite3_column_double(stmt, 2);
328 *group = sqlite3_column_int(stmt, 3);
330 fprintf(stderr, "Could not execute SELECT statement!\n");
334 sqlite3_finalize(stmt);
339 /* Does this object still exist in the database (and not expired)? */
340 bool LocalDb::IsAvailable(const ObjectReference &ref)
346 // Special objects (such as the zero object) aren't stored in segments, and
347 // so are always available.
348 if (!ref.is_normal())
351 stmt = Prepare("select count(*) from block_index "
352 "where segmentid = ? and object = ? and expired is null");
353 sqlite3_bind_int64(stmt, 1, SegmentToId(ref.get_segment()));
354 sqlite3_bind_text(stmt, 2, ref.get_sequence().c_str(),
355 ref.get_sequence().size(), SQLITE_TRANSIENT);
357 rc = sqlite3_step(stmt);
358 if (rc == SQLITE_DONE) {
360 } else if (rc == SQLITE_ROW) {
361 if (sqlite3_column_int(stmt, 0) > 0)
364 fprintf(stderr, "Could not execute SELECT statement!\n");
368 sqlite3_finalize(stmt);
373 void LocalDb::UseObject(const ObjectReference& ref)
378 if (!ref.is_normal())
381 int64_t old_size = 0;
382 stmt = Prepare("select size from snapshot_refs "
383 "where segmentid = ? and object = ?");
384 sqlite3_bind_int64(stmt, 1, SegmentToId(ref.get_segment()));
385 string obj = ref.get_sequence();
386 sqlite3_bind_text(stmt, 2, obj.c_str(), obj.size(), SQLITE_TRANSIENT);
387 rc = sqlite3_step(stmt);
388 if (rc == SQLITE_ROW) {
389 old_size = sqlite3_column_int64(stmt, 0);
391 sqlite3_finalize(stmt);
393 int64_t block_size = 0;
394 stmt = Prepare("select size from block_index "
395 "where segmentid = ? and object = ?");
396 sqlite3_bind_int64(stmt, 1, SegmentToId(ref.get_segment()));
397 obj = ref.get_sequence();
398 sqlite3_bind_text(stmt, 2, obj.c_str(), obj.size(), SQLITE_TRANSIENT);
399 rc = sqlite3_step(stmt);
400 if (rc == SQLITE_ROW) {
401 block_size = sqlite3_column_int64(stmt, 0);
403 string refstr = ref.to_string();
404 fprintf(stderr, "No block found in block_index for %s\n",
406 sqlite3_finalize(stmt);
409 sqlite3_finalize(stmt);
411 int64_t new_size = old_size;
412 if (ref.has_range()) {
413 new_size += ref.get_range_length();
414 new_size = min(new_size, block_size);
416 new_size = block_size;
419 if (new_size != old_size) {
420 stmt = Prepare("insert or replace "
421 "into snapshot_refs(segmentid, object, size) "
423 sqlite3_bind_int64(stmt, 1, SegmentToId(ref.get_segment()));
424 obj = ref.get_sequence();
425 sqlite3_bind_text(stmt, 2, obj.c_str(), obj.size(), SQLITE_TRANSIENT);
426 sqlite3_bind_int64(stmt, 3, new_size);
428 rc = sqlite3_step(stmt);
429 if (rc != SQLITE_DONE) {
430 fprintf(stderr, "Could not execute INSERT statement!\n");
434 sqlite3_finalize(stmt);
438 void LocalDb::UseSegment(const std::string &segment, double utilization)
443 stmt = Prepare("insert or replace "
444 "into segments_used(snapshotid, segmentid, utilization) "
446 sqlite3_bind_int64(stmt, 1, snapshotid);
447 sqlite3_bind_int64(stmt, 2, SegmentToId(segment));
448 sqlite3_bind_double(stmt, 3, utilization);
450 rc = sqlite3_step(stmt);
451 if (rc != SQLITE_DONE) {
452 fprintf(stderr, "Could not insert segment use record!\n");
456 sqlite3_finalize(stmt);
459 void LocalDb::SetSegmentChecksum(const std::string &segment,
460 const std::string &path,
461 const std::string &checksum,
467 stmt = Prepare("update segments set path = ?, checksum = ?, size = ?, "
468 "mtime = coalesce(mtime, julianday('now')) "
469 "where segmentid = ?");
470 sqlite3_bind_text(stmt, 1, path.c_str(), path.size(),
472 sqlite3_bind_text(stmt, 2, checksum.c_str(), checksum.size(),
474 sqlite3_bind_int64(stmt, 3, size);
475 sqlite3_bind_int64(stmt, 4, SegmentToId(segment));
477 rc = sqlite3_step(stmt);
478 if (rc != SQLITE_DONE) {
479 fprintf(stderr, "Could not update segment checksum in database!\n");
483 sqlite3_finalize(stmt);
486 bool LocalDb::GetSegmentChecksum(const string &segment,
488 string *seg_checksum)
495 stmt = Prepare("select path, checksum from segments where segment = ?");
496 sqlite3_bind_text(stmt, 1, segment.c_str(), segment.size(),
499 rc = sqlite3_step(stmt);
500 if (rc == SQLITE_DONE) {
501 } else if (rc == SQLITE_ROW) {
505 val = (const char *)sqlite3_column_text(stmt, 0);
511 val = (const char *)sqlite3_column_text(stmt, 1);
517 fprintf(stderr, "Could not execute SELECT statement!\n");
521 sqlite3_finalize(stmt);
526 /* Look up and return the packed representation of the subblock chunk
527 * signatures. Returns true if signatures were found for the specified object,
528 * and if so sets *buf to point at a buffer of memory (allocated with malloc;
529 * the caller should free it), and *len to the length of the buffer. */
530 bool LocalDb::LoadChunkSignatures(ObjectReference ref,
531 void **buf, size_t *len,
538 stmt = Prepare("select signatures, algorithm from subblock_signatures "
539 "where blockid = (select blockid from block_index "
540 " where segmentid = ? and object = ?)");
541 sqlite3_bind_int64(stmt, 1, SegmentToId(ref.get_segment()));
542 string obj = ref.get_sequence();
543 sqlite3_bind_text(stmt, 2, obj.c_str(), obj.size(), SQLITE_TRANSIENT);
545 rc = sqlite3_step(stmt);
546 if (rc == SQLITE_DONE) {
547 } else if (rc == SQLITE_ROW) {
548 const void *data = sqlite3_column_blob(stmt, 0);
549 *len = sqlite3_column_bytes(stmt, 0);
554 memcpy(*buf, data, *len);
555 *algorithm = (const char *)sqlite3_column_text(stmt, 1);
560 fprintf(stderr, "Could not execute SELECT statement!\n");
564 sqlite3_finalize(stmt);
569 /* Store the subblock chunk signatures for a specified object. The object
570 * itself must have already been indexed in the database. */
571 void LocalDb::StoreChunkSignatures(ObjectReference ref,
572 const void *buf, size_t len,
573 const string& algorithm)
578 stmt = Prepare("select blockid from block_index "
579 "where segmentid = ? and object = ?");
580 sqlite3_bind_int64(stmt, 1, SegmentToId(ref.get_segment()));
581 string obj = ref.get_sequence();
582 sqlite3_bind_text(stmt, 2, obj.c_str(), obj.size(), SQLITE_TRANSIENT);
584 rc = sqlite3_step(stmt);
585 if (rc != SQLITE_ROW) {
587 "Could not determine blockid in StoreChunkSignatures!\n");
589 throw IOException("Error getting blockid");
591 int64_t blockid = sqlite3_column_int64(stmt, 0);
592 sqlite3_finalize(stmt);
594 stmt = Prepare("insert or replace "
595 "into subblock_signatures(blockid, algorithm, signatures) "
597 sqlite3_bind_int64(stmt, 1, blockid);
598 sqlite3_bind_text(stmt, 2, algorithm.c_str(), algorithm.size(),
600 sqlite3_bind_blob(stmt, 3, buf, len, SQLITE_TRANSIENT);
602 rc = sqlite3_step(stmt);
603 if (rc != SQLITE_DONE) {
604 fprintf(stderr, "Could not insert sub-block checksums!\n");
608 sqlite3_finalize(stmt);