From 38c66f088ed65d2f42264c92add6e0b33eac2bfc Mon Sep 17 00:00:00 2001 From: Michael Vrable Date: Fri, 22 Dec 2006 22:53:06 -0800 Subject: [PATCH] Extend basic support for serializing simple data types. Add support for serializing integers, strings, and dictionaries. Then, extend the directory scanner to output a summary of stat information for each file/directory. --- Makefile | 5 +- scandir.cc | 67 +++++++++++++++++++++++-- store.cc | 145 +++++++++++++++++++++++++++++++++++++++++++++++++++++ store.h | 69 +++++++++++++++++++++++++ 4 files changed, 281 insertions(+), 5 deletions(-) create mode 100644 store.cc create mode 100644 store.h diff --git a/Makefile b/Makefile index 8fc6f05..45fb135 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,7 @@ -CXXFLAGS=-O -Wall -D_FILE_OFFSET_BITS=64 +CXXFLAGS=-O -Wall -D_FILE_OFFSET_BITS=64 -g +LDFLAGS=-g -OBJS=scandir.o +OBJS=scandir.o store.o scandir : $(OBJS) $(CXX) $(LDFLAGS) -o $@ $^ diff --git a/scandir.cc b/scandir.cc index c8e43a8..d0f76d1 100644 --- a/scandir.cc +++ b/scandir.cc @@ -14,11 +14,21 @@ #include #include +#include "store.h" + using std::string; using std::vector; +static OutputStream *info_dump = NULL; + void scandir(const string& path); +/* Converts time to microseconds since the epoch. */ +int64_t encode_time(time_t time) +{ + return (int64_t)time * 1000000; +} + void dumpfile(int fd) { struct stat stat_buf; @@ -57,6 +67,8 @@ void scanfile(const string& path) char *buf; ssize_t len; + dictionary file_info; + lstat(path.c_str(), &stat_buf); printf("%s:\n", path.c_str()); @@ -65,15 +77,35 @@ void scanfile(const string& path) stat_buf.st_uid, stat_buf.st_gid, stat_buf.st_nlink, (int)stat_buf.st_blksize, (int64_t)stat_buf.st_size); + file_info["mode"] = encode_u16(stat_buf.st_mode & 07777); + file_info["atime"] = encode_u64(encode_time(stat_buf.st_atime)); + file_info["ctime"] = encode_u64(encode_time(stat_buf.st_ctime)); + file_info["mtime"] = encode_u64(encode_time(stat_buf.st_mtime)); + file_info["user"] = encode_u32(stat_buf.st_uid); + file_info["group"] = encode_u32(stat_buf.st_gid); + + char inode_type; + switch (stat_buf.st_mode & S_IFMT) { case S_IFIFO: + printf(" FIFO\n"); + inode_type = 'p'; + break; case S_IFSOCK: + printf(" socket\n"); + inode_type = 's'; + break; case S_IFCHR: + printf(" character device\n"); + inode_type = 'c'; + break; case S_IFBLK: - printf(" special file\n"); + printf(" block device\n"); + inode_type = 'b'; break; case S_IFLNK: printf(" symlink\n"); + inode_type = 'l'; /* Use the reported file size to allocate a buffer large enough to read * the symlink. Allocate slightly more space, so that we ask for more @@ -88,10 +120,15 @@ void scanfile(const string& path) } else if (len > stat_buf.st_size) { printf("error reading symlink: name truncated\n"); } + + file_info["contents"] = buf; + delete[] buf; break; case S_IFREG: printf(" regular file\n"); + inode_type = '-'; + /* Be paranoid when opening the file. We have no guarantee that the * file was not replaced between the stat() call above and the open() * call below, so we might not even be opening a regular file. That @@ -110,15 +147,26 @@ void scanfile(const string& path) flags = fcntl(fd, F_GETFL); fcntl(fd, F_SETFL, flags & ~O_NONBLOCK); - dumpfile(fd); + //dumpfile(fd); + file_info["size"] = encode_u64(stat_buf.st_size); close(fd); break; case S_IFDIR: printf(" directory\n"); + inode_type = 'd'; scandir(path); break; + + default: + fprintf(stderr, "Unknown inode type: mode=%x\n", stat_buf.st_mode); + return; } + + file_info["type"] = string(1, inode_type); + + info_dump->write_string(path); + info_dump->write_dictionary(file_info); } void scandir(const string& path) @@ -152,7 +200,20 @@ void scandir(const string& path) int main(int argc, char *argv[]) { - scandir("."); + FILE *dump = fopen("fileinfo", "w"); + if (dump == NULL) { + fprintf(stderr, "Cannot open fileinfo: %m\n"); + return 1; + } + + FileOutputStream os(dump); + info_dump = &os; + + try { + scandir("."); + } catch (IOException e) { + fprintf(stderr, "IOException: %s\n", e.getError().c_str()); + } return 0; } diff --git a/store.cc b/store.cc new file mode 100644 index 0000000..5df4b77 --- /dev/null +++ b/store.cc @@ -0,0 +1,145 @@ +/* LBS: An LFS-inspired filesystem backup system + * Copyright (C) 2006 Michael Vrable + * + * Backup data is stored in a collection of objects, which are grouped together + * into segments for storage purposes. This file provides interfaces for + * reading and writing objects and segments. */ + +#include + +#include "store.h" + +using std::string; + +void OutputStream::write_u8(uint8_t val) +{ + write(&val, 1); +} + +void OutputStream::write_u16(uint16_t val) +{ + unsigned char buf[2]; + + buf[0] = val & 0xff; + buf[1] = (val >> 8) & 0xff; + write(buf, 2); +} + +void OutputStream::write_u32(uint32_t val) +{ + unsigned char buf[4]; + + buf[0] = val & 0xff; + buf[1] = (val >> 8) & 0xff; + buf[2] = (val >> 16) & 0xff; + buf[3] = (val >> 24) & 0xff; + write(buf, 4); +} + +void OutputStream::write_u64(uint64_t val) +{ + unsigned char buf[8]; + + buf[0] = val & 0xff; + buf[1] = (val >> 8) & 0xff; + buf[2] = (val >> 16) & 0xff; + buf[3] = (val >> 24) & 0xff; + buf[4] = (val >> 32) & 0xff; + buf[5] = (val >> 40) & 0xff; + buf[6] = (val >> 48) & 0xff; + buf[7] = (val >> 56) & 0xff; + write(buf, 8); +} + +/* Writes an integer to an output stream using a variable-sized representation: + * seven bits are written at a time (little-endian), and the eigth bit of each + * byte is set if more data follows. */ +void OutputStream::write_varint(uint64_t val) +{ + do { + uint8_t remainder = (val & 0x7f); + val >>= 7; + if (val) + remainder |= 0x80; + write_u8(remainder); + } while (val); +} + +/* Write an arbitrary string by first writing out the length, followed by the + * data itself. */ +void OutputStream::write_string(const string &s) +{ + size_t len = s.length(); + write_varint(len); + write(s.data(), len); +} + +void OutputStream::write_dictionary(const dictionary &d) +{ + size_t size = d.size(); + size_t written = 0; + + write_varint(size); + + for (dictionary::const_iterator i = d.begin(); i != d.end(); ++i) { + write_string(i->first); + write_string(i->second); + written++; + } + + assert(written == size); +} + +StringOutputStream::StringOutputStream() + : buf(std::ios_base::out) +{ +} + +void StringOutputStream::write(const void *data, size_t len) +{ + buf.write((const char *)data, len); + if (!buf.good()) + throw IOException("error writing to StringOutputStream"); +} + +FileOutputStream::FileOutputStream(FILE *file) +{ + f = file; +} + +FileOutputStream::~FileOutputStream() +{ + fclose(f); +} + +void FileOutputStream::write(const void *data, size_t len) +{ + size_t res; + + res = fwrite(data, 1, len, f); + if (res != len) { + throw IOException("write error"); + } +} + +/* Utility functions, for encoding data types to strings. */ +string encode_u16(uint16_t val) +{ + StringOutputStream s; + s.write_u16(val); + return s.contents(); +} + +string encode_u32(uint32_t val) +{ + StringOutputStream s; + s.write_u32(val); + return s.contents(); +} + +string encode_u64(uint64_t val) +{ + StringOutputStream s; + s.write_u64(val); + return s.contents(); +} diff --git a/store.h b/store.h new file mode 100644 index 0000000..d484780 --- /dev/null +++ b/store.h @@ -0,0 +1,69 @@ +/* LBS: An LFS-inspired filesystem backup system + * Copyright (C) 2006 Michael Vrable + * + * Backup data is stored in a collection of objects, which are grouped together + * into segments for storage purposes. This file provides interfaces for + * reading and writing objects and segments. */ + +#include + +#include +#include +#include +#include + +typedef std::map dictionary; + +class IOException : public std::exception { +private: + std::string error; +public: + explicit IOException(const std::string &err) { error = err; } + virtual ~IOException() throw () { } + std::string getError() const { return error; } +}; + +class OutputStream { +public: + virtual ~OutputStream() { } + virtual void write(const void *data, size_t len) = 0; + + /* Convenience functions for writing other data types. Values are always + * written out in little-endian order. */ + void write_u8(uint8_t val); + void write_u16(uint16_t val); + void write_u32(uint32_t val); + void write_u64(uint64_t val); + + void write_s32(int32_t val) { write_u32((uint32_t)val); } + void write_s64(int64_t val) { write_u64((uint64_t)val); } + + void write_varint(uint64_t val); + + void write_string(const std::string &s); + void write_dictionary(const dictionary &d); +}; + +class StringOutputStream : public OutputStream { +private: + std::stringstream buf; +public: + StringOutputStream(); + + virtual void write(const void *data, size_t len); + std::string contents() const { return buf.str(); } +}; + +class FileOutputStream : public OutputStream { +private: + FILE *f; +public: + explicit FileOutputStream(FILE *file); + virtual ~FileOutputStream(); + + virtual void write(const void *data, size_t len); +}; + +std::string encode_u16(uint16_t val); +std::string encode_u32(uint32_t val); +std::string encode_u64(uint64_t val); -- 2.20.1