1 /* A simple tool for benchmarking various logging strategies.
3 * We want to log a series of key/value pairs. Approaches that we try include:
4 * - Data written directly into the filesystem.
5 * - Data is written to a Berkeley DB.
6 * - Data is appended to a log file.
7 * In all cases we want to ensure that data is persistent on disk so it could
8 * be used for crash recovery. We measure how many log records we can write
9 * per second to gauge performance. */
12 #define _ATFILE_SOURCE
18 #include <sys/types.h>
32 int queue_capacity = 1024;
35 int opt_batchsize = 1;
40 GCond *cond_empty, *cond_full;
42 struct item *get_item()
44 return (struct item *)g_async_queue_pop(queue);
47 void finish_item(struct item *item)
56 g_cond_signal(cond_empty);
57 if (outstanding < queue_capacity)
58 g_cond_signal(cond_full);
62 void writebuf(int fd, const char *buf, size_t len)
66 written = write(fd, buf, len);
67 if (written < 0 && errno == EINTR)
69 g_assert(written >= 0);
75 /************************ Direct-to-filesystem logging ***********************/
76 static int dirfd = -1;
78 gpointer fslog_thread(gpointer d)
80 g_print("Launching filesystem writer thread...\n");
83 struct item *item = get_item();
85 int fd = openat(dirfd, item->key, O_CREAT|O_WRONLY|O_TRUNC, 0666);
88 writebuf(fd, item->data, item->len);
102 dirfd = open(".", O_DIRECTORY);
103 g_assert(dirfd >= 0);
105 for (int i = 0; i < 1; i++)
106 g_thread_create(fslog_thread, NULL, FALSE, NULL);
109 /****************************** Single-File Log ******************************/
110 gpointer flatlog_thread(gpointer d)
112 g_print("Launching flat log writer thread...\n");
114 int fd = open("logfile", O_CREAT|O_WRONLY|O_TRUNC, 0666);
120 struct item *item = get_item();
122 writebuf(fd, item->key, strlen(item->key) + 1);
123 writebuf(fd, (char *)&item->len, sizeof(item->len));
124 writebuf(fd, item->data, item->len);
127 if (count % opt_batchsize == 0)
136 void launch_flatlog()
138 g_thread_create(flatlog_thread, NULL, FALSE, NULL);
141 /************************* Transactional Berkeley DB *************************/
142 gpointer bdb_thread(gpointer d)
144 g_print("Launching BDB log writer thread...\n");
152 res = db_env_create(&env, 0);
155 res = env->open(env, ".",
156 DB_CREATE | DB_RECOVER | DB_INIT_LOCK | DB_INIT_LOG
157 | DB_INIT_MPOOL | DB_INIT_TXN | DB_THREAD, 0644);
160 res = db_create(&db, env, 0);
163 res = db->open(db, NULL, "log.db", "log", DB_BTREE,
164 DB_CREATE | DB_THREAD | DB_AUTO_COMMIT, 0644);
169 res = env->txn_begin(env, NULL, &txn, 0);
173 struct item *item = get_item();
176 memset(&key, 0, sizeof(key));
177 memset(&value, 0, sizeof(value));
179 key.data = item->key;
180 key.size = strlen(item->key);
182 value.data = item->data;
183 value.size = item->len;
185 res = db->put(db, NULL, &key, &value, 0);
189 if (count % opt_batchsize == 0) {
202 g_thread_create(bdb_thread, NULL, FALSE, NULL);
205 int main(int argc, char *argv[])
208 queue = g_async_queue_new();
209 lock = g_mutex_new();
210 cond_empty = g_cond_new();
211 cond_full = g_cond_new();
215 while ((opt = getopt(argc, argv, "t:s:b:BFD")) != -1) {
218 // Set number of log worker threads
219 opt_threads = atoi(optarg);
222 // Set item size (in bytes)
223 item_size = atoi(optarg);
227 opt_batchsize = atoi(optarg);
230 // Select BDB backend
234 // Select flat file backend
238 // Select file system directory backend
242 fprintf(stderr, "Usage: %s [-t threads] {-B|-F|-D}\n",
259 fprintf(stderr, "Backend not selected!\n");
263 for (int i = 0; i < (1 << 12); i++) {
264 struct item *item = g_new(struct item, 1);
265 item->key = g_strdup_printf("item-%06d", i);
266 item->data = g_malloc(item_size);
267 item->len = item_size;
270 g_async_queue_push(queue, item);
272 if (outstanding == opt_batchsize)
273 g_cond_wait(cond_empty, lock);
274 g_mutex_unlock(lock);
278 while (outstanding > 0)
279 g_cond_wait(cond_empty, lock);
280 g_mutex_unlock(lock);