X-Git-Url: http://git.vrable.net/?a=blobdiff_plain;f=scandir.cc;h=a82bc5eb749a5f97b01a2416c92d2362eeed597a;hb=3702f2531d1bedb56ec4e0ecd80ea007e447c128;hp=a227bb3b92dc4d1185bdb43c22896dc8e7e5fead;hpb=76eb457e0c0bd682cc841c409199f15286343bb1;p=cumulus.git

diff --git a/scandir.cc b/scandir.cc
index a227bb3..a82bc5e 100644
--- a/scandir.cc
+++ b/scandir.cc
@@ -1,27 +1,29 @@
 /* Recursively descend the filesystem and visit each file. */
 
-#include <stdio.h>
-#include <stdlib.h>
-#include <stdint.h>
 #include <dirent.h>
 #include <errno.h>
 #include <fcntl.h>
-#include <pwd.h>
+#include <getopt.h>
 #include <grp.h>
-#include <sys/types.h>
+#include <pwd.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
 #include <sys/stat.h>
+#include <sys/types.h>
 #include <unistd.h>
 
 #include <algorithm>
-#include <string>
-#include <list>
-#include <vector>
-#include <iostream>
 #include <fstream>
-#include <sstream>
+#include <iostream>
+#include <list>
 #include <set>
+#include <sstream>
+#include <string>
+#include <vector>
 
 #include "format.h"
+#include "localdb.h"
 #include "store.h"
 #include "sha1.h"
 
@@ -38,6 +40,10 @@ static char *block_buf;
 
 static const size_t LBS_METADATA_BLOCK_SIZE = 65536;
 
+/* Local database, which tracks objects written in this and previous
+ * invocations to help in creating incremental snapshots. */
+LocalDb *db;
+
 /* Contents of the root object.  This will contain a set of indirect links to
  * the metadata objects. */
 std::ostringstream metadata_root;
@@ -50,6 +56,9 @@ std::set<string> segment_list;
 
 void scandir(const string& path);
 
+/* Selection of files to include/exclude in the snapshot. */
+std::list<string> excludes;
+
 /* Ensure contents of metadata are flushed to an object. */
 void metadata_flush()
 {
@@ -126,15 +135,27 @@ int64_t dumpfile(int fd, dictionary &file_info)
 
         hash.process(block_buf, bytes);
 
-        // tarstore processing
-        LbsObject *o = new LbsObject;
-        o->set_group("data");
-        o->set_data(block_buf, bytes);
-        o->write(tss);
-        object_list.push_back(o->get_name());
-        segment_list.insert(o->get_ref().get_segment());
-        delete o;
+        // Either find a copy of this block in an already-existing segment, or
+        // index it so it can be re-used in the future
+        SHA1Checksum block_hash;
+        block_hash.process(block_buf, bytes);
+        string block_csum = block_hash.checksum_str();
+        ObjectReference ref = db->FindObject(block_csum, bytes);
+
+        // Store a copy of the object if one does not yet exist
+        if (ref.get_segment().size() == 0) {
+            LbsObject *o = new LbsObject;
+            o->set_group("data");
+            o->set_data(block_buf, bytes);
+            o->write(tss);
+            ref = o->get_ref();
+            db->StoreObject(ref, block_csum, bytes);
+            delete o;
+        }
 
+        object_list.push_back(ref.to_string());
+        segment_list.insert(ref.get_segment());
+        db->UseObject(ref);
         size += bytes;
     }
 
@@ -184,6 +205,16 @@ void scanfile(const string& path)
     // Set to true if the item is a directory and we should recursively scan
     bool recurse = false;
 
+    // Check this file against the include/exclude list to see if it should be
+    // considered
+    for (list<string>::iterator i = excludes.begin();
+         i != excludes.end(); ++i) {
+        if (path == *i) {
+            printf("Excluding %s\n", path.c_str());
+            return;
+        }
+    }
+
     dictionary file_info;
 
     lstat(path.c_str(), &stat_buf);
@@ -323,22 +354,81 @@ void scandir(const string& path)
     for (vector<string>::iterator i = contents.begin();
          i != contents.end(); ++i) {
         const string& filename = *i;
-        scanfile(path + "/" + filename);
+        if (path == ".")
+            scanfile(filename);
+        else
+            scanfile(path + "/" + filename);
     }
 
     closedir(dir);
 }
 
-int main(int argc, char *argv[])
+void usage(const char *program)
 {
-    block_buf = new char[LBS_BLOCK_SIZE];
+    fprintf(stderr,
+            "Usage: %s [OPTION]... SOURCE DEST\n"
+            "Produce backup snapshot of files in SOURCE and store to DEST.\n"
+            "\n"
+            "Options:\n"
+            "  --exclude=PATH       exclude files in PATH from snapshot\n"
+            "  --localdb=PATH       local backup metadata is stored in PATH\n",
+            program);
+}
 
+int main(int argc, char *argv[])
+{
+    string backup_source = ".";
     string backup_dest = ".";
+    string localdb_dir = "";
+
+    while (1) {
+        static struct option long_options[] = {
+            {"localdb", 1, 0, 0},   // 0
+            {"exclude", 1, 0, 0},   // 1
+            {NULL, 0, 0, 0},
+        };
+
+        int long_index;
+        int c = getopt_long(argc, argv, "", long_options, &long_index);
 
-    if (argc > 1)
-        backup_dest = argv[1];
+        if (c == -1)
+            break;
+
+        if (c == 0) {
+            switch (long_index) {
+            case 0:     // --localdb
+                localdb_dir = optarg;
+                break;
+            case 1:     // --exclude
+                excludes.push_back(optarg);
+                break;
+            default:
+                fprintf(stderr, "Unhandled long option!\n");
+                return 1;
+            }
+        } else {
+            usage(argv[0]);
+            return 1;
+        }
+    }
+
+    if (argc < optind + 2) {
+        usage(argv[0]);
+        return 1;
+    }
+
+    backup_source = argv[optind];
+    backup_dest = argv[argc - 1];
+
+    if (localdb_dir == "") {
+        localdb_dir = backup_dest;
+    }
+
+    printf("Source: %s, Dest: %s\n",
+           backup_source.c_str(), backup_dest.c_str());
 
     tss = new TarSegmentStore(backup_dest);
+    block_buf = new char[LBS_BLOCK_SIZE];
 
     /* Write a backup descriptor file, which says which segments are needed and
      * where to start to restore this snapshot.  The filename is based on the
@@ -352,6 +442,13 @@ int main(int argc, char *argv[])
     string desc_filename = backup_dest + "/" + desc_buf + ".lbs";
     std::ofstream descriptor(desc_filename.c_str());
 
+    /* Open the local database which tracks all objects that are stored
+     * remotely, for efficient incrementals.  Provide it with the name of this
+     * snapshot. */
+    string database_path = backup_dest + "/localdb.sqlite";
+    db = new LocalDb;
+    db->Open(database_path.c_str(), desc_buf);
+
     try {
         scanfile(".");
     } catch (IOException e) {
@@ -380,6 +477,8 @@ int main(int argc, char *argv[])
         descriptor << "    " << *i << "\n";
     }
 
+    db->Close();
+
     tss->sync();
     delete tss;