1 /* LBS: An LFS-inspired filesystem backup system
2 * Copyright (C) 2007 Michael Vrable
4 * When creating backup snapshots, maintain a local database of data blocks and
5 * checksums, in addition to the data contents (which may be stored remotely).
6 * This database is consulted when attempting to build incremental snapshots,
7 * as it says which objects can be reused.
9 * The database is implemented as an SQLite3 database, but this implementation
10 * detail is kept internal to this file, so that the storage format may be
25 /* Helper function to prepare a statement for execution in the current
27 sqlite3_stmt *LocalDb::Prepare(const char *sql)
33 rc = sqlite3_prepare_v2(db, sql, strlen(sql), &stmt, &tail);
34 if (rc != SQLITE_OK) {
36 throw IOException(string("Error preparing statement: ") + sql);
42 void LocalDb::ReportError(int rc)
44 fprintf(stderr, "Result code: %d\n", rc);
45 fprintf(stderr, "Error message: %s\n", sqlite3_errmsg(db));
48 void LocalDb::Open(const char *path, const char *snapshot_name,
49 const char *snapshot_scheme, double intent)
53 rc = sqlite3_open(path, &db);
55 fprintf(stderr, "Can't open database: %s\n", sqlite3_errmsg(db));
57 throw IOException("Error opening local database");
60 rc = sqlite3_exec(db, "begin", NULL, NULL, NULL);
62 fprintf(stderr, "Can't open database: %s\n", sqlite3_errmsg(db));
64 throw IOException("Error starting transaction");
67 sqlite3_extended_result_codes(db, 1);
69 /* Insert this snapshot into the database, and determine the integer key
70 * which will be used to identify it. */
71 sqlite3_stmt *stmt = Prepare("insert into "
72 "snapshots(name, scheme, timestamp, intent) "
73 "values (?, ?, julianday('now'), ?)");
74 sqlite3_bind_text(stmt, 1, snapshot_name, strlen(snapshot_name),
76 if (snapshot_scheme == NULL)
77 sqlite3_bind_null(stmt, 2);
79 sqlite3_bind_text(stmt, 2, snapshot_scheme, strlen(snapshot_scheme),
81 sqlite3_bind_double(stmt, 3, intent);
83 rc = sqlite3_step(stmt);
84 if (rc != SQLITE_DONE) {
87 throw IOException("Database execution error!");
90 snapshotid = sqlite3_last_insert_rowid(db);
91 sqlite3_finalize(stmt);
92 if (snapshotid == 0) {
95 throw IOException("Find snapshot id");
98 /* Create a temporary table which will be used to keep track of the objects
99 * used by this snapshot. When the database is closed, we will summarize
100 * the results of this table into segments_used. */
101 rc = sqlite3_exec(db,
102 "create temporary table snapshot_refs ("
103 " segmentid integer not null,"
104 " object text not null,"
105 " size integer not null"
106 ")", NULL, NULL, NULL);
107 if (rc != SQLITE_OK) {
110 throw IOException("Database initialization");
112 rc = sqlite3_exec(db,
113 "create unique index snapshot_refs_index "
114 "on snapshot_refs(segmentid, object)",
116 if (rc != SQLITE_OK) {
119 throw IOException("Database initialization");
123 void LocalDb::Close()
127 /* Summarize the snapshot_refs table into segments_used. */
128 sqlite3_stmt *stmt = Prepare(
129 "insert or replace into segments_used "
130 "select ? as snapshotid, segmentid, max(utilization) from ("
131 " select segmentid, cast(used as real) / size as utilization "
133 " (select segmentid, sum(size) as used from snapshot_refs "
134 " group by segmentid) "
135 " join segments using (segmentid) "
137 " select segmentid, utilization from segments_used "
138 " where snapshotid = ? "
139 ") group by segmentid"
141 sqlite3_bind_int64(stmt, 1, snapshotid);
142 sqlite3_bind_int64(stmt, 2, snapshotid);
143 rc = sqlite3_step(stmt);
144 if (rc != SQLITE_OK && rc != SQLITE_DONE) {
147 fprintf(stderr, "DATABASE ERROR: Unable to create segment summary!\n");
149 sqlite3_finalize(stmt);
151 /* Commit changes to the database and close. */
152 rc = sqlite3_exec(db, "commit", NULL, NULL, NULL);
153 if (rc != SQLITE_OK) {
154 fprintf(stderr, "DATABASE ERROR: Can't commit database!\n");
160 int64_t LocalDb::SegmentToId(const string &segment)
166 stmt = Prepare("insert or ignore into segments(segment) values (?)");
167 sqlite3_bind_text(stmt, 1, segment.c_str(), segment.size(),
169 rc = sqlite3_step(stmt);
170 if (rc != SQLITE_DONE) {
171 throw IOException("Could not execute INSERT statement!");
173 sqlite3_finalize(stmt);
175 stmt = Prepare("select segmentid from segments where segment = ?");
176 sqlite3_bind_text(stmt, 1, segment.c_str(), segment.size(),
179 rc = sqlite3_step(stmt);
180 if (rc == SQLITE_DONE) {
181 throw IOException("No segment found by id");
182 } else if (rc == SQLITE_ROW) {
183 result = sqlite3_column_int64(stmt, 0);
185 throw IOException("Error executing find segment by id query");
188 sqlite3_finalize(stmt);
193 string LocalDb::IdToSegment(int64_t segmentid)
199 stmt = Prepare("select segment from segments where segmentid = ?");
200 sqlite3_bind_int64(stmt, 1, segmentid);
202 rc = sqlite3_step(stmt);
203 if (rc == SQLITE_DONE) {
204 throw IOException("No segment found by id");
205 } else if (rc == SQLITE_ROW) {
206 result = (const char *)sqlite3_column_text(stmt, 0);
208 throw IOException("Error executing find segment by id query");
211 sqlite3_finalize(stmt);
216 void LocalDb::StoreObject(const ObjectReference& ref,
217 const string &checksum, int64_t size,
224 stmt = Prepare("insert into block_index("
225 "segmentid, object, checksum, size, timestamp) "
226 "values (?, ?, ?, ?, julianday('now'))");
228 stmt = Prepare("insert into block_index("
229 "segmentid, object, checksum, size, timestamp) "
230 "values (?, ?, ?, ?, ?)");
233 sqlite3_bind_int64(stmt, 1, SegmentToId(ref.get_segment()));
234 string obj = ref.get_sequence();
235 sqlite3_bind_text(stmt, 2, obj.c_str(), obj.size(), SQLITE_TRANSIENT);
236 sqlite3_bind_text(stmt, 3, checksum.c_str(), checksum.size(),
238 sqlite3_bind_int64(stmt, 4, size);
240 sqlite3_bind_double(stmt, 5, age);
242 rc = sqlite3_step(stmt);
243 if (rc != SQLITE_DONE) {
244 fprintf(stderr, "Could not execute INSERT statement!\n");
248 sqlite3_finalize(stmt);
251 stmt = Prepare("update segments "
252 "set mtime = coalesce(max(mtime, ?), ?) "
253 "where segmentid = ?");
254 sqlite3_bind_double(stmt, 1, age);
255 sqlite3_bind_double(stmt, 2, age);
256 sqlite3_bind_int64(stmt, 3, SegmentToId(ref.get_segment()));
257 rc = sqlite3_step(stmt);
258 sqlite3_finalize(stmt);
262 ObjectReference LocalDb::FindObject(const string &checksum, int64_t size)
268 stmt = Prepare("select segmentid, object from block_index "
269 "where checksum = ? and size = ? and expired is null");
270 sqlite3_bind_text(stmt, 1, checksum.c_str(), checksum.size(),
272 sqlite3_bind_int64(stmt, 2, size);
274 rc = sqlite3_step(stmt);
275 if (rc == SQLITE_DONE) {
276 } else if (rc == SQLITE_ROW) {
277 ref = ObjectReference(IdToSegment(sqlite3_column_int64(stmt, 0)),
278 (const char *)sqlite3_column_text(stmt, 1));
279 ref.set_range(0, size);
281 fprintf(stderr, "Could not execute SELECT statement!\n");
285 sqlite3_finalize(stmt);
290 bool LocalDb::IsOldObject(const string &checksum, int64_t size, double *age,
297 stmt = Prepare("select segmentid, object, timestamp, expired "
298 "from block_index where checksum = ? and size = ?");
299 sqlite3_bind_text(stmt, 1, checksum.c_str(), checksum.size(),
301 sqlite3_bind_int64(stmt, 2, size);
303 rc = sqlite3_step(stmt);
304 if (rc == SQLITE_DONE) {
306 } else if (rc == SQLITE_ROW) {
308 *age = sqlite3_column_double(stmt, 2);
309 *group = sqlite3_column_int(stmt, 3);
311 fprintf(stderr, "Could not execute SELECT statement!\n");
315 sqlite3_finalize(stmt);
320 /* Does this object still exist in the database (and not expired)? */
321 bool LocalDb::IsAvailable(const ObjectReference &ref)
327 // Special objects (such as the zero object) aren't stored in segments, and
328 // so are always available.
329 if (!ref.is_normal())
332 stmt = Prepare("select count(*) from block_index "
333 "where segmentid = ? and object = ? and expired is null");
334 sqlite3_bind_int64(stmt, 1, SegmentToId(ref.get_segment()));
335 sqlite3_bind_text(stmt, 2, ref.get_sequence().c_str(),
336 ref.get_sequence().size(), SQLITE_TRANSIENT);
338 rc = sqlite3_step(stmt);
339 if (rc == SQLITE_DONE) {
341 } else if (rc == SQLITE_ROW) {
342 if (sqlite3_column_int(stmt, 0) > 0)
345 fprintf(stderr, "Could not execute SELECT statement!\n");
349 sqlite3_finalize(stmt);
354 void LocalDb::UseObject(const ObjectReference& ref)
359 if (!ref.is_normal())
362 stmt = Prepare("insert or ignore into snapshot_refs "
363 "select segmentid, object, size from block_index "
364 "where segmentid = ? and object = ?");
365 sqlite3_bind_int64(stmt, 1, SegmentToId(ref.get_segment()));
366 string obj = ref.get_sequence();
367 sqlite3_bind_text(stmt, 2, obj.c_str(), obj.size(), SQLITE_TRANSIENT);
369 rc = sqlite3_step(stmt);
370 if (rc != SQLITE_DONE) {
371 fprintf(stderr, "Could not execute INSERT statement!\n");
375 sqlite3_finalize(stmt);
378 void LocalDb::UseSegment(const std::string &segment, double utilization)
383 stmt = Prepare("insert or replace "
384 "into segments_used(snapshotid, segmentid, utilization) "
386 sqlite3_bind_int64(stmt, 1, snapshotid);
387 sqlite3_bind_int64(stmt, 2, SegmentToId(segment));
388 sqlite3_bind_double(stmt, 3, utilization);
390 rc = sqlite3_step(stmt);
391 if (rc != SQLITE_DONE) {
392 fprintf(stderr, "Could not insert segment use record!\n");
396 sqlite3_finalize(stmt);
399 void LocalDb::SetSegmentChecksum(const std::string &segment,
400 const std::string &path,
401 const std::string &checksum,
407 stmt = Prepare("update segments set path = ?, checksum = ?, size = ?, "
408 "mtime = coalesce(mtime, julianday('now')) "
409 "where segmentid = ?");
410 sqlite3_bind_text(stmt, 1, path.c_str(), path.size(),
412 sqlite3_bind_text(stmt, 2, checksum.c_str(), checksum.size(),
414 sqlite3_bind_int64(stmt, 3, size);
415 sqlite3_bind_int64(stmt, 4, SegmentToId(segment));
417 rc = sqlite3_step(stmt);
418 if (rc != SQLITE_DONE) {
419 fprintf(stderr, "Could not update segment checksum in database!\n");
423 sqlite3_finalize(stmt);
426 bool LocalDb::GetSegmentChecksum(const string &segment,
428 string *seg_checksum)
435 stmt = Prepare("select path, checksum from segments where segment = ?");
436 sqlite3_bind_text(stmt, 1, segment.c_str(), segment.size(),
439 rc = sqlite3_step(stmt);
440 if (rc == SQLITE_DONE) {
441 } else if (rc == SQLITE_ROW) {
445 val = (const char *)sqlite3_column_text(stmt, 0);
451 val = (const char *)sqlite3_column_text(stmt, 1);
457 fprintf(stderr, "Could not execute SELECT statement!\n");
461 sqlite3_finalize(stmt);