From e053c8481ac962711348e466496e8a0c5a53f100 Mon Sep 17 00:00:00 2001 From: Michael Vrable Date: Mon, 18 Jun 2007 13:15:05 -0700 Subject: [PATCH] Partial commit of statcache support. This will cache the results of stat calls from previous backups, so that future backups do not need to read the files in their entirety if no data has changed. --- Makefile | 2 +- scandir.cc | 20 +++++++++++-- statcache.cc | 79 ++++++++++++++++++++++++++++++++++++++++++++++++++++ statcache.h | 48 +++++++++++++++++++++++++++++++ 4 files changed, 145 insertions(+), 4 deletions(-) create mode 100644 statcache.cc create mode 100644 statcache.h diff --git a/Makefile b/Makefile index 33aa0c3..6eb0d67 100644 --- a/Makefile +++ b/Makefile @@ -4,7 +4,7 @@ CXXFLAGS=-O -Wall -D_FILE_OFFSET_BITS=64 $(DEBUG) \ `pkg-config --cflags $(PACKAGES)` LDFLAGS=$(DEBUG) -ltar `pkg-config --libs $(PACKAGES)` -SRCS=format.cc localdb.cc ref.cc scandir.cc sha1.cc store.cc +SRCS=format.cc localdb.cc ref.cc scandir.cc sha1.cc statcache.cc store.cc OBJS=$(SRCS:.cc=.o) scandir : $(OBJS) diff --git a/scandir.cc b/scandir.cc index 27f2de4..99498c4 100644 --- a/scandir.cc +++ b/scandir.cc @@ -26,6 +26,7 @@ #include "localdb.h" #include "store.h" #include "sha1.h" +#include "statcache.h" using std::list; using std::string; @@ -44,6 +45,10 @@ static const size_t LBS_METADATA_BLOCK_SIZE = 65536; * invocations to help in creating incremental snapshots. */ LocalDb *db; +/* Stat cache, which stored data locally to speed the backup process by quickly + * skipping files which have not changed. */ +StatCache *statcache; + /* Contents of the root object. This will contain a set of indirect links to * the metadata objects. */ std::ostringstream metadata_root; @@ -111,7 +116,7 @@ size_t file_read(int fd, char *buf, size_t maxlen) /* Read the contents of a file (specified by an open file descriptor) and copy * the data to the store. Returns the size of the file (number of bytes * dumped), or -1 on error. */ -int64_t dumpfile(int fd, dictionary &file_info) +int64_t dumpfile(int fd, dictionary &file_info, const string &path) { struct stat stat_buf; fstat(fd, &stat_buf); @@ -174,6 +179,8 @@ int64_t dumpfile(int fd, dictionary &file_info) file_info["checksum"] = hash.checksum_str(); + statcache->Save(path, &stat_buf, file_info["checksum"], object_list); + /* For files that only need to be broken apart into a few objects, store * the list of objects directly. For larger files, store the data * out-of-line and provide a pointer to the indrect object. */ @@ -312,7 +319,7 @@ void scanfile(const string& path) flags = fcntl(fd, F_GETFL); fcntl(fd, F_SETFL, flags & ~O_NONBLOCK); - file_size = dumpfile(fd, file_info); + file_size = dumpfile(fd, file_info, path); file_info["size"] = encode_int(file_size); close(fd); @@ -465,6 +472,7 @@ int main(int argc, char *argv[]) char desc_buf[256]; time(&now); localtime_r(&now, &time_buf); + strftime(desc_buf, sizeof(desc_buf), "%Y%m%dT%H%M%S", &time_buf); /* Open the local database which tracks all objects that are stored * remotely, for efficient incrementals. Provide it with the name of this @@ -473,6 +481,10 @@ int main(int argc, char *argv[]) db = new LocalDb; db->Open(database_path.c_str(), desc_buf); + /* Initialize the stat cache, for skipping over unchanged files. */ + statcache = new StatCache; + statcache->Open(localdb_dir.c_str(), desc_buf); + try { scanfile("."); } catch (IOException e) { @@ -492,7 +504,6 @@ int main(int argc, char *argv[]) /* Write a backup descriptor file, which says which segments are needed and * where to start to restore this snapshot. The filename is based on the * current time. */ - strftime(desc_buf, sizeof(desc_buf), "%Y%m%dT%H%M%S", &time_buf); string desc_filename = backup_dest + "/snapshot-" + desc_buf + ".lbs"; std::ofstream descriptor(desc_filename.c_str()); @@ -511,6 +522,9 @@ int main(int argc, char *argv[]) db->Close(); + statcache->Close(); + delete statcache; + tss->sync(); delete tss; diff --git a/statcache.cc b/statcache.cc new file mode 100644 index 0000000..71f6b36 --- /dev/null +++ b/statcache.cc @@ -0,0 +1,79 @@ +/* LBS: An LFS-inspired filesystem backup system Copyright (C) 2007 Michael + * Vrable + * + * To speed backups, we maintain a "stat cache" containing selected information + * about all regular files, including modification times and the list of blocks + * that comprised the file in the last backup. If the file has not changed + * according to a stat() call, we may re-use the information contained in the + * stat cache instead of re-reading the entire file. It is always safe to + * discard information from the stat cache; this will only cause a file to be + * re-read to determine that it contains the same data as before. + * + * The stat cache is stored in a file called "statcache" in the local backup + * directory. During a backup, a new statcache file is written out with a + * suffix based on the current time; at the end of a successful backup this + * file is renamed over the original statcache file. + * + * The information in the statcache file is stored in sorted order as we + * traverse the filesystem, so that we can read and write it in a purely + * streaming manner. (This is why we don't include the information in the + * SQLite local database; doing so is likely less efficient.) + */ + +#include +#include +#include + +#include +#include +#include + +#include "format.h" +#include "statcache.h" + +using std::list; +using std::string; +using std::ifstream; +using std::ofstream; + +void StatCache::Open(const char *path, const char *snapshot_name) +{ + oldpath = path; + oldpath += "/statcache"; + newpath = oldpath + "." + snapshot_name; + + oldcache = NULL; + newcache = new ofstream(newpath.c_str()); +} + +void StatCache::Close() +{ + if (oldcache != NULL) + delete oldcache; + + delete newcache; + + if (rename(newpath.c_str(), oldpath.c_str()) < 0) { + fprintf(stderr, "Error renaming statcache from %s to %s: %m\n", + newpath.c_str(), oldpath.c_str()); + } +} + +/* Save stat information about a regular file for future invocations. */ +void StatCache::Save(const string &path, struct stat *stat_buf, + const string &checksum, const list &blocks) +{ + *newcache << uri_encode(path) << "\n"; + *newcache << "mtime: " << encode_int(stat_buf->st_mtime) << "\n" + << "ctime: " << encode_int(stat_buf->st_ctime) << "\n" + << "inode: " << encode_int(stat_buf->st_ino) << "\n" + << "checksum: " << checksum << "\n"; + + *newcache << "blocks:"; + for (list::const_iterator i = blocks.begin(); + i != blocks.end(); ++i) { + *newcache << " " << *i << "\n"; + } + + *newcache << "\n"; +} diff --git a/statcache.h b/statcache.h new file mode 100644 index 0000000..10c0397 --- /dev/null +++ b/statcache.h @@ -0,0 +1,48 @@ +/* LBS: An LFS-inspired filesystem backup system Copyright (C) 2007 Michael + * Vrable + * + * To speed backups, we maintain a "stat cache" containing selected information + * about all regular files, including modification times and the list of blocks + * that comprised the file in the last backup. If the file has not changed + * according to a stat() call, we may re-use the information contained in the + * stat cache instead of re-reading the entire file. It is always safe to + * discard information from the stat cache; this will only cause a file to be + * re-read to determine that it contains the same data as before. + * + * The stat cache is stored in a file called "statcache" in the local backup + * directory. During a backup, a new statcache file is written out with a + * suffix based on the current time; at the end of a successful backup this + * file is renamed over the original statcache file. + * + * The information in the statcache file is stored in sorted order as we + * traverse the filesystem, so that we can read and write it in a purely + * streaming manner. (This is why we don't include the information in the + * SQLite local database; doing so is likely less efficient.) + */ + +#ifndef _LBS_STATCACHE_H +#define _LBS_STATCACHE_H + +#include +#include +#include + +#include +#include +#include + +class StatCache { +public: + void Open(const char *path, const char *snapshot_name); + void Close(); + void Save(const std::string &path, struct stat *stat_buf, + const std::string &checksum, + const std::list &blocks); + +private: + std::string oldpath, newpath; + std::ifstream *oldcache; + std::ofstream *newcache; +}; + +#endif // _LBS_STATCACHE_H -- 2.20.1