1 /* Cumulus: Smart Filesystem Backup to Dumb Servers
3 * Copyright (C) 2007-2008 The Regents of the University of California
4 * Written by Michael Vrable <mvrable@cs.ucsd.edu>
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License as published by
8 * the Free Software Foundation; either version 2 of the License, or
9 * (at your option) any later version.
11 * This program is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 * GNU General Public License for more details.
16 * You should have received a copy of the GNU General Public License along
17 * with this program; if not, write to the Free Software Foundation, Inc.,
18 * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
21 /* When creating backup snapshots, maintain a local database of data blocks and
22 * checksums, in addition to the data contents (which may be stored remotely).
23 * This database is consulted when attempting to build incremental snapshots,
24 * as it says which objects can be reused.
26 * The database is implemented as an SQLite3 database, but this implementation
27 * detail is kept internal to this file, so that the storage format may be
42 /* Helper function to prepare a statement for execution in the current
44 sqlite3_stmt *LocalDb::Prepare(const char *sql)
50 rc = sqlite3_prepare_v2(db, sql, strlen(sql), &stmt, &tail);
51 if (rc != SQLITE_OK) {
53 throw IOException(string("Error preparing statement: ") + sql);
59 void LocalDb::ReportError(int rc)
61 fprintf(stderr, "Result code: %d\n", rc);
62 fprintf(stderr, "Error message: %s\n", sqlite3_errmsg(db));
65 void LocalDb::Open(const char *path, const char *snapshot_name,
66 const char *snapshot_scheme, double intent)
70 rc = sqlite3_open(path, &db);
72 fprintf(stderr, "Can't open database: %s\n", sqlite3_errmsg(db));
74 throw IOException("Error opening local database");
77 rc = sqlite3_exec(db, "begin", NULL, NULL, NULL);
79 fprintf(stderr, "Can't open database: %s\n", sqlite3_errmsg(db));
81 throw IOException("Error starting transaction");
84 sqlite3_extended_result_codes(db, 1);
86 /* Insert this snapshot into the database, and determine the integer key
87 * which will be used to identify it. */
88 sqlite3_stmt *stmt = Prepare("insert into "
89 "snapshots(name, scheme, timestamp, intent) "
90 "values (?, ?, julianday('now'), ?)");
91 sqlite3_bind_text(stmt, 1, snapshot_name, strlen(snapshot_name),
93 if (snapshot_scheme == NULL)
94 sqlite3_bind_null(stmt, 2);
96 sqlite3_bind_text(stmt, 2, snapshot_scheme, strlen(snapshot_scheme),
98 sqlite3_bind_double(stmt, 3, intent);
100 rc = sqlite3_step(stmt);
101 if (rc != SQLITE_DONE) {
104 throw IOException("Database execution error!");
107 snapshotid = sqlite3_last_insert_rowid(db);
108 sqlite3_finalize(stmt);
109 if (snapshotid == 0) {
112 throw IOException("Find snapshot id");
115 /* Create a temporary table which will be used to keep track of the objects
116 * used by this snapshot. When the database is closed, we will summarize
117 * the results of this table into segments_used. */
118 rc = sqlite3_exec(db,
119 "create temporary table snapshot_refs ("
120 " segmentid integer not null,"
121 " object text not null,"
122 " size integer not null"
123 ")", NULL, NULL, NULL);
124 if (rc != SQLITE_OK) {
127 throw IOException("Database initialization");
129 rc = sqlite3_exec(db,
130 "create unique index snapshot_refs_index "
131 "on snapshot_refs(segmentid, object)",
133 if (rc != SQLITE_OK) {
136 throw IOException("Database initialization");
140 void LocalDb::Close()
144 /* Summarize the snapshot_refs table into segments_used. */
145 sqlite3_stmt *stmt = Prepare(
146 "insert or replace into segments_used "
147 "select ? as snapshotid, segmentid, max(utilization) from ("
148 " select segmentid, cast(used as real) / size as utilization "
150 " (select segmentid, sum(size) as used from snapshot_refs "
151 " group by segmentid) "
152 " join segments using (segmentid) "
154 " select segmentid, utilization from segments_used "
155 " where snapshotid = ? "
156 ") group by segmentid"
158 sqlite3_bind_int64(stmt, 1, snapshotid);
159 sqlite3_bind_int64(stmt, 2, snapshotid);
160 rc = sqlite3_step(stmt);
161 if (rc != SQLITE_OK && rc != SQLITE_DONE) {
164 fprintf(stderr, "DATABASE ERROR: Unable to create segment summary!\n");
166 sqlite3_finalize(stmt);
168 /* Commit changes to the database and close. */
169 rc = sqlite3_exec(db, "commit", NULL, NULL, NULL);
170 if (rc != SQLITE_OK) {
171 fprintf(stderr, "DATABASE ERROR: Can't commit database!\n");
177 int64_t LocalDb::SegmentToId(const string &segment)
183 stmt = Prepare("insert or ignore into segments(segment) values (?)");
184 sqlite3_bind_text(stmt, 1, segment.c_str(), segment.size(),
186 rc = sqlite3_step(stmt);
187 if (rc != SQLITE_DONE) {
188 throw IOException("Could not execute INSERT statement!");
190 sqlite3_finalize(stmt);
192 stmt = Prepare("select segmentid from segments where segment = ?");
193 sqlite3_bind_text(stmt, 1, segment.c_str(), segment.size(),
196 rc = sqlite3_step(stmt);
197 if (rc == SQLITE_DONE) {
198 throw IOException("No segment found by id");
199 } else if (rc == SQLITE_ROW) {
200 result = sqlite3_column_int64(stmt, 0);
202 throw IOException("Error executing find segment by id query");
205 sqlite3_finalize(stmt);
210 string LocalDb::IdToSegment(int64_t segmentid)
216 stmt = Prepare("select segment from segments where segmentid = ?");
217 sqlite3_bind_int64(stmt, 1, segmentid);
219 rc = sqlite3_step(stmt);
220 if (rc == SQLITE_DONE) {
221 throw IOException("No segment found by id");
222 } else if (rc == SQLITE_ROW) {
223 result = (const char *)sqlite3_column_text(stmt, 0);
225 throw IOException("Error executing find segment by id query");
228 sqlite3_finalize(stmt);
233 void LocalDb::StoreObject(const ObjectReference& ref,
234 const string &checksum, int64_t size,
241 stmt = Prepare("insert into block_index("
242 "segmentid, object, checksum, size, timestamp) "
243 "values (?, ?, ?, ?, julianday('now'))");
245 stmt = Prepare("insert into block_index("
246 "segmentid, object, checksum, size, timestamp) "
247 "values (?, ?, ?, ?, ?)");
250 sqlite3_bind_int64(stmt, 1, SegmentToId(ref.get_segment()));
251 string obj = ref.get_sequence();
252 sqlite3_bind_text(stmt, 2, obj.c_str(), obj.size(), SQLITE_TRANSIENT);
253 sqlite3_bind_text(stmt, 3, checksum.c_str(), checksum.size(),
255 sqlite3_bind_int64(stmt, 4, size);
257 sqlite3_bind_double(stmt, 5, age);
259 rc = sqlite3_step(stmt);
260 if (rc != SQLITE_DONE) {
261 fprintf(stderr, "Could not execute INSERT statement!\n");
265 sqlite3_finalize(stmt);
268 stmt = Prepare("update segments "
269 "set mtime = coalesce(max(mtime, ?), ?) "
270 "where segmentid = ?");
271 sqlite3_bind_double(stmt, 1, age);
272 sqlite3_bind_double(stmt, 2, age);
273 sqlite3_bind_int64(stmt, 3, SegmentToId(ref.get_segment()));
274 rc = sqlite3_step(stmt);
275 sqlite3_finalize(stmt);
279 ObjectReference LocalDb::FindObject(const string &checksum, int64_t size)
285 stmt = Prepare("select segmentid, object from block_index "
286 "where checksum = ? and size = ? and expired is null");
287 sqlite3_bind_text(stmt, 1, checksum.c_str(), checksum.size(),
289 sqlite3_bind_int64(stmt, 2, size);
291 rc = sqlite3_step(stmt);
292 if (rc == SQLITE_DONE) {
293 } else if (rc == SQLITE_ROW) {
294 ref = ObjectReference(IdToSegment(sqlite3_column_int64(stmt, 0)),
295 (const char *)sqlite3_column_text(stmt, 1));
296 ref.set_range(0, size);
298 fprintf(stderr, "Could not execute SELECT statement!\n");
302 sqlite3_finalize(stmt);
307 bool LocalDb::IsOldObject(const string &checksum, int64_t size, double *age,
314 stmt = Prepare("select segmentid, object, timestamp, expired "
315 "from block_index where checksum = ? and size = ?");
316 sqlite3_bind_text(stmt, 1, checksum.c_str(), checksum.size(),
318 sqlite3_bind_int64(stmt, 2, size);
320 rc = sqlite3_step(stmt);
321 if (rc == SQLITE_DONE) {
323 } else if (rc == SQLITE_ROW) {
325 *age = sqlite3_column_double(stmt, 2);
326 *group = sqlite3_column_int(stmt, 3);
328 fprintf(stderr, "Could not execute SELECT statement!\n");
332 sqlite3_finalize(stmt);
337 /* Does this object still exist in the database (and not expired)? */
338 bool LocalDb::IsAvailable(const ObjectReference &ref)
344 // Special objects (such as the zero object) aren't stored in segments, and
345 // so are always available.
346 if (!ref.is_normal())
349 stmt = Prepare("select count(*) from block_index "
350 "where segmentid = ? and object = ? and expired is null");
351 sqlite3_bind_int64(stmt, 1, SegmentToId(ref.get_segment()));
352 sqlite3_bind_text(stmt, 2, ref.get_sequence().c_str(),
353 ref.get_sequence().size(), SQLITE_TRANSIENT);
355 rc = sqlite3_step(stmt);
356 if (rc == SQLITE_DONE) {
358 } else if (rc == SQLITE_ROW) {
359 if (sqlite3_column_int(stmt, 0) > 0)
362 fprintf(stderr, "Could not execute SELECT statement!\n");
366 sqlite3_finalize(stmt);
371 void LocalDb::UseObject(const ObjectReference& ref)
376 if (!ref.is_normal())
379 stmt = Prepare("insert or ignore into snapshot_refs "
380 "select segmentid, object, size from block_index "
381 "where segmentid = ? and object = ?");
382 sqlite3_bind_int64(stmt, 1, SegmentToId(ref.get_segment()));
383 string obj = ref.get_sequence();
384 sqlite3_bind_text(stmt, 2, obj.c_str(), obj.size(), SQLITE_TRANSIENT);
386 rc = sqlite3_step(stmt);
387 if (rc != SQLITE_DONE) {
388 fprintf(stderr, "Could not execute INSERT statement!\n");
392 sqlite3_finalize(stmt);
395 void LocalDb::UseSegment(const std::string &segment, double utilization)
400 stmt = Prepare("insert or replace "
401 "into segments_used(snapshotid, segmentid, utilization) "
403 sqlite3_bind_int64(stmt, 1, snapshotid);
404 sqlite3_bind_int64(stmt, 2, SegmentToId(segment));
405 sqlite3_bind_double(stmt, 3, utilization);
407 rc = sqlite3_step(stmt);
408 if (rc != SQLITE_DONE) {
409 fprintf(stderr, "Could not insert segment use record!\n");
413 sqlite3_finalize(stmt);
416 void LocalDb::SetSegmentChecksum(const std::string &segment,
417 const std::string &path,
418 const std::string &checksum,
424 stmt = Prepare("update segments set path = ?, checksum = ?, size = ?, "
425 "mtime = coalesce(mtime, julianday('now')) "
426 "where segmentid = ?");
427 sqlite3_bind_text(stmt, 1, path.c_str(), path.size(),
429 sqlite3_bind_text(stmt, 2, checksum.c_str(), checksum.size(),
431 sqlite3_bind_int64(stmt, 3, size);
432 sqlite3_bind_int64(stmt, 4, SegmentToId(segment));
434 rc = sqlite3_step(stmt);
435 if (rc != SQLITE_DONE) {
436 fprintf(stderr, "Could not update segment checksum in database!\n");
440 sqlite3_finalize(stmt);
443 bool LocalDb::GetSegmentChecksum(const string &segment,
445 string *seg_checksum)
452 stmt = Prepare("select path, checksum from segments where segment = ?");
453 sqlite3_bind_text(stmt, 1, segment.c_str(), segment.size(),
456 rc = sqlite3_step(stmt);
457 if (rc == SQLITE_DONE) {
458 } else if (rc == SQLITE_ROW) {
462 val = (const char *)sqlite3_column_text(stmt, 0);
468 val = (const char *)sqlite3_column_text(stmt, 1);
474 fprintf(stderr, "Could not execute SELECT statement!\n");
478 sqlite3_finalize(stmt);