1 -- We maintain a local index of data blocks that have been previously stored
2 -- for constructing incremental snapshots.
4 -- The index is stored in an SQLite3 database. This is its schema.
6 -- A note on date formats: values which represent timestamps are generally
7 -- stored in SQLite's julianday format (Julian day stored as a 64-bit float).
8 -- However, application code should allow any value accepted by SQLite's
9 -- date/time functions: Julian day or ISO8601 text string. Code reading from
10 -- the database should select julianday(timestamp) or datetime(timestamp),
11 -- depending on the desired format for parsing.
13 -- Because the julianday format uses floating point, code should not rely on
14 -- exact equality tests between timestamps, or precise round-tripping of
17 -- Timestamps are always in UTC.
19 -- Versioning information, describing the revision for which the table schema
21 create table schema_version(
22 version text, -- Program version, dotted decimal string
23 major integer, -- Major version number
24 minor integer -- Minor version number
26 insert into schema_version values ('0.11', 0, 11);
28 -- List of snapshots which have been created and which we are still tracking.
29 -- There may be more snapshots than this actually stored at the remote server,
30 -- but the reverse should not ever be true: Cumulus may depend on data stored
31 -- in these snapshots when writing a new snapshot.
32 create table snapshots (
33 snapshotid integer primary key,
36 timestamp datetime -- should match the timestamp of the snapshot
39 -- List of segments which have been created.
40 create table segments (
41 segmentid integer primary key,
42 segment text unique not null,
43 timestamp datetime, -- when was the segment written?
46 data_size integer, -- sum of bytes in all objects in the segment
47 disk_size integer, -- size of segment on disk, after compression
50 create unique index segment_name_index on segments(segment);
52 -- Index of all data blocks in stored segments. This is indexed by content
53 -- hash to allow for coarse block-level data deduplication.
54 create table block_index (
55 blockid integer primary key,
56 segmentid integer not null,
60 timestamp datetime, -- when a block with this data was first stored
63 create index block_content_index on block_index(checksum);
64 create unique index block_name_index on block_index(segmentid, object);
66 -- Checksums for the decomposition of blocks into even smaller chunks
67 -- (variable-sized, but generally ~4 kB, and maximum 64 kB). Chunk boundaries
68 -- are determined based on the contents using Rabin fingerprints. These
69 -- checksums can be used for computing sub-file incrementals.
71 -- Each block stored in block_index may have an entry in the
72 -- subblock_signatures table. The signatures field is a binary blob consisting
73 -- of a packed sequence of (chunk length [16-bit unsigned, big-endian],
74 -- checksum [28 bytes if SHA-224]) tuples that should cover the entire block.
76 -- algorithm specifies the method used for computing break points as well as
77 -- the hash function used, so that signatures can be discarded if the algorithm
78 -- changes. The current algorithm used is 'lbfs-4096/sha224', which specifies
79 -- a target 4 kB block size with parameters set to match LBFS, and SHA-224 as
80 -- the hash algorithm.
81 create table subblock_signatures (
82 blockid integer primary key,
83 algorithm text not null,
84 signatures blob not null
87 -- Summary of segment utilization for each snapshot.
88 create table segment_utilization (
89 snapshotid integer not null,
90 segmentid integer not null,
92 -- Estimate for the number of live bytes in data objects: this is capped at
93 -- segments.data_size if all data in the segment is referenced.
94 bytes_referenced integer not null
96 create unique index segment_utilization_index
97 on segment_utilization(snapshotid, segmentid);