1 /* Blue Sky: File Systems in the Cloud
3 * Copyright (C) 2009 The Regents of the University of California
4 * Written by Michael Vrable <mvrable@cs.ucsd.edu>
15 /* Core filesystem. Different proxies, such as the NFSv3 one, interface to
16 * this, but the core actually tracks the data which is stored. So far we just
17 * implement an in-memory filesystem, but eventually this will be state which
18 * is persisted to the cloud. */
20 /* Return the current time, in microseconds since the epoch. */
21 int64_t bluesky_get_current_time()
24 g_get_current_time(&t);
25 return (int64_t)t.tv_sec * 1000000 + t.tv_usec;
28 /* Update an inode to indicate that a modification was made. This increases
29 * the change counter, updates the ctime to the current time, and optionally
30 * updates the mtime. */
31 void bluesky_inode_update_ctime(BlueSkyInode *inode, gboolean update_mtime)
33 int64_t now = bluesky_get_current_time();
34 inode->change_count++;
40 /* Compute the HMAC keyed-hash function using the given hash algorithm, data,
42 void compute_hmac(GChecksumType algo,
43 const guchar *data, gsize data_len,
44 const guchar *key, gsize key_len,
45 guint8 *buffer, gsize *digest_len)
52 case G_CHECKSUM_SHA256:
56 g_error("Unknown hash algorithm for HMAC: %d\n", algo);
59 gsize digest_size = g_checksum_type_get_length(algo);
61 guchar keybuf[block_size];
62 memset(keybuf, 0, block_size);
63 memcpy(keybuf, key, MIN(block_size, key_len));
64 for (int i = 0; i < block_size; i++)
67 GChecksum *csum1 = g_checksum_new(algo);
68 g_checksum_update(csum1, keybuf, block_size);
69 g_checksum_update(csum1, data, data_len);
70 guint8 digest[digest_size];
71 g_checksum_get_digest(csum1, digest, &digest_size);
73 memset(keybuf, 0, block_size);
74 memcpy(keybuf, key, MIN(block_size, key_len));
75 for (int i = 0; i < block_size; i++)
78 GChecksum *csum2 = g_checksum_new(algo);
79 g_checksum_update(csum2, keybuf, block_size);
80 g_checksum_update(csum2, digest, digest_size);
82 g_checksum_get_digest(csum2, buffer, digest_len);
84 g_checksum_free(csum1);
85 g_checksum_free(csum2);
88 /* Unfortunately a glib hash table is only guaranteed to be able to store
89 * 32-bit keys if we use the key directly. If we want 64-bit inode numbers,
90 * we'll have to allocate memory to store the 64-bit inumber, and use a pointer
91 * to it. Rather than allocate the memory for the key, we'll just include a
92 * pointer to the 64-bit inum stored in the inode itself, so that we don't need
93 * to do any more memory management. */
94 static guint bluesky_fs_key_hash_func(gconstpointer key)
96 uint64_t inum = *(const uint64_t *)key;
100 static gboolean bluesky_fs_key_equal_func(gconstpointer a, gconstpointer b)
102 uint64_t i1 = *(const uint64_t *)a;
103 uint64_t i2 = *(const uint64_t *)b;
107 /* Filesystem-level operations. A filesystem is like a directory tree that we
108 * are willing to export. */
109 BlueSkyFS *bluesky_new_fs(gchar *name)
111 BlueSkyFS *fs = g_new0(BlueSkyFS, 1);
112 fs->lock = g_mutex_new();
113 fs->name = g_strdup(name);
114 fs->inodes = g_hash_table_new(bluesky_fs_key_hash_func,
115 bluesky_fs_key_equal_func);
116 fs->next_inum = BLUESKY_ROOT_INUM + 1;
117 fs->store = s3store_new();
122 /* Allocate a fresh inode number which has not been used before within a
124 uint64_t bluesky_fs_alloc_inode(BlueSkyFS *fs)
128 g_mutex_lock(fs->lock);
129 inum = fs->next_inum;
131 g_mutex_unlock(fs->lock);
136 BlueSkyInode *bluesky_new_inode(uint64_t inum, BlueSkyFS *fs,
137 BlueSkyFileType type)
139 BlueSkyInode *i = g_new0(BlueSkyInode, 1);
141 i->lock = g_mutex_new();
147 case BLUESKY_REGULAR:
148 i->blocks = g_array_new(FALSE, TRUE, sizeof(BlueSkyBlock));
150 case BLUESKY_DIRECTORY:
151 i->dirents = g_sequence_new(bluesky_dirent_destroy);
152 i->dirhash = g_hash_table_new(g_str_hash, g_str_equal);
155 case BLUESKY_CHARACTER:
156 case BLUESKY_SYMLINK:
165 /* Retrieve an inode from the filesystem. Eventually this will be a cache and
166 * so we might need to go fetch the inode from elsewhere; for now all
167 * filesystem state is stored here. */
168 BlueSkyInode *bluesky_get_inode(BlueSkyFS *fs, uint64_t inum)
170 BlueSkyInode *inode = NULL;
172 g_mutex_lock(fs->lock);
173 inode = (BlueSkyInode *)g_hash_table_lookup(fs->inodes, &inum);
174 g_mutex_unlock(fs->lock);
179 /* Insert an inode into the filesystem inode cache. */
180 void bluesky_insert_inode(BlueSkyFS *fs, BlueSkyInode *inode)
182 g_mutex_lock(fs->lock);
183 g_hash_table_insert(fs->inodes, &inode->inum, inode);
184 g_mutex_unlock(fs->lock);
187 /* Mark a given block dirty and make sure that data is faulted in so that it
188 * can be written to. */
189 void bluesky_block_touch(BlueSkyInode *inode, uint64_t i)
191 g_return_if_fail(i < inode->blocks->len);
192 BlueSkyBlock *block = &g_array_index(inode->blocks, BlueSkyBlock, i);
194 switch (block->type) {
195 case BLUESKY_BLOCK_ZERO:
196 block->data = bluesky_string_new(g_malloc0(BLUESKY_BLOCK_SIZE),
199 case BLUESKY_BLOCK_REF:
200 bluesky_block_fetch(inode->fs, block);
201 g_assert(block->type == BLUESKY_BLOCK_CACHED);
203 case BLUESKY_BLOCK_CACHED:
204 case BLUESKY_BLOCK_DIRTY:
205 block->data = bluesky_string_dup(block->data);
209 block->type = BLUESKY_BLOCK_DIRTY;
212 /* Set the size of a file. This will truncate or extend the file as needed.
213 * Newly-allocated bytes are zeroed. */
214 void bluesky_file_truncate(BlueSkyInode *inode, uint64_t size)
216 g_return_if_fail(size <= BLUESKY_MAX_FILE_SIZE);
218 if (size == inode->size)
221 uint64_t blocks = (size + BLUESKY_BLOCK_SIZE - 1) / BLUESKY_BLOCK_SIZE;
223 if (blocks > inode->blocks->len) {
224 /* Need to add new blocks to the end of a file. New block structures
225 * are automatically zeroed, which initializes them to be pointers to
226 * zero blocks so we don't need to do any more work. */
227 g_array_set_size(inode->blocks, blocks);
228 } else if (blocks < inode->blocks->len) {
229 /* Delete blocks from a file. Must reclaim memory. */
230 for (guint i = inode->blocks->len; i < blocks; i++) {
231 BlueSkyBlock *b = &g_array_index(inode->blocks, BlueSkyBlock, i);
233 bluesky_string_unref(b->data);
235 g_array_set_size(inode->blocks, blocks);
238 /* If the file size is being decreased, ensure that any trailing data in
239 * the last block is zeroed. */
240 if (size < inode->size) {
241 BlueSkyBlock *b = &g_array_index(inode->blocks, BlueSkyBlock,
243 if (b->type != BLUESKY_BLOCK_ZERO) {
244 bluesky_block_touch(inode, blocks - 1);
245 int end_offset = size % BLUESKY_BLOCK_SIZE;
246 if (end_offset > 0) {
247 memset(&b->data->data[end_offset], 0,
248 BLUESKY_BLOCK_SIZE - end_offset);
254 bluesky_inode_update_ctime(inode, 1);
257 void bluesky_file_write(BlueSkyInode *inode, uint64_t offset,
258 const char *data, gint len)
260 g_return_if_fail(inode->type == BLUESKY_REGULAR);
261 g_return_if_fail(offset < inode->size);
262 g_return_if_fail(len <= inode->size - offset);
268 uint64_t block_num = offset / BLUESKY_BLOCK_SIZE;
269 gint block_offset = offset % BLUESKY_BLOCK_SIZE;
270 gint bytes = MIN(BLUESKY_BLOCK_SIZE - block_offset, len);
272 bluesky_block_touch(inode, block_num);
273 BlueSkyBlock *b = &g_array_index(inode->blocks, BlueSkyBlock,
275 memcpy(&b->data->data[block_offset], data, bytes);
276 bluesky_block_flush(inode->fs, b);
283 bluesky_inode_update_ctime(inode, 1);
286 void bluesky_file_read(BlueSkyInode *inode, uint64_t offset,
289 g_return_if_fail(inode->type == BLUESKY_REGULAR);
290 g_return_if_fail(offset < inode->size);
291 g_return_if_fail(len <= inode->size - offset);
294 uint64_t block_num = offset / BLUESKY_BLOCK_SIZE;
295 gint block_offset = offset % BLUESKY_BLOCK_SIZE;
296 gint bytes = MIN(BLUESKY_BLOCK_SIZE - block_offset, len);
298 BlueSkyBlock *b = &g_array_index(inode->blocks, BlueSkyBlock,
301 case BLUESKY_BLOCK_ZERO:
302 memset(buf, 0, bytes);
304 case BLUESKY_BLOCK_REF:
305 bluesky_block_fetch(inode->fs, b);
307 case BLUESKY_BLOCK_CACHED:
308 case BLUESKY_BLOCK_DIRTY:
309 memcpy(buf, &b->data->data[block_offset], bytes);
319 /* Read the given block from cloud-backed storage if the data is not already
321 void bluesky_block_fetch(BlueSkyFS *fs, BlueSkyBlock *block)
323 if (block->type != BLUESKY_BLOCK_REF)
326 g_print("Fetching block from %s\n", block->ref);
327 BlueSkyRCStr *string = s3store_get(fs->store, block->ref);
329 bluesky_string_unref(block->data);
330 block->data = string;
331 block->type = BLUESKY_BLOCK_CACHED;
334 /* Write the given block to cloud-backed storage and mark it clean. */
335 void bluesky_block_flush(BlueSkyFS *fs, BlueSkyBlock *block)
337 if (block->type != BLUESKY_BLOCK_DIRTY)
340 GChecksum *csum = g_checksum_new(G_CHECKSUM_SHA256);
341 g_checksum_update(csum, block->data->data, block->data->len);
342 gchar *name = g_strdup(g_checksum_get_string(csum));
344 g_print("Flushing block as %s\n", name);
345 s3store_put(fs->store, name, block->data);
349 /* block->type = BLUESKY_BLOCK_CACHED; */
350 bluesky_string_unref(block->data);
352 block->type = BLUESKY_BLOCK_REF;
354 g_checksum_free(csum);