1 /* Blue Sky: File Systems in the Cloud
3 * Copyright (C) 2009 The Regents of the University of California
4 * Written by Michael Vrable <mvrable@cs.ucsd.edu>
15 /* Core filesystem. Different proxies, such as the NFSv3 one, interface to
16 * this, but the core actually tracks the data which is stored. So far we just
17 * implement an in-memory filesystem, but eventually this will be state which
18 * is persisted to the cloud. */
20 /* Return the current time, in microseconds since the epoch. */
21 int64_t bluesky_get_current_time()
24 g_get_current_time(&t);
25 return (int64_t)t.tv_sec * 1000000 + t.tv_usec;
28 /* Update an inode to indicate that a modification was made. This increases
29 * the change counter, updates the ctime to the current time, and optionally
30 * updates the mtime. */
31 void bluesky_inode_update_ctime(BlueSkyInode *inode, gboolean update_mtime)
33 int64_t now = bluesky_get_current_time();
34 inode->change_count++;
40 /* Compute the HMAC keyed-hash function using the given hash algorithm, data,
42 void compute_hmac(GChecksumType algo,
43 const guchar *data, gsize data_len,
44 const guchar *key, gsize key_len,
45 guint8 *buffer, gsize *digest_len)
52 case G_CHECKSUM_SHA256:
56 g_error("Unknown hash algorithm for HMAC: %d\n", algo);
59 gsize digest_size = g_checksum_type_get_length(algo);
61 guchar keybuf[block_size];
62 memset(keybuf, 0, block_size);
63 memcpy(keybuf, key, MIN(block_size, key_len));
64 for (int i = 0; i < block_size; i++)
67 GChecksum *csum1 = g_checksum_new(algo);
68 g_checksum_update(csum1, keybuf, block_size);
69 g_checksum_update(csum1, data, data_len);
70 guint8 digest[digest_size];
71 g_checksum_get_digest(csum1, digest, &digest_size);
73 memset(keybuf, 0, block_size);
74 memcpy(keybuf, key, MIN(block_size, key_len));
75 for (int i = 0; i < block_size; i++)
78 GChecksum *csum2 = g_checksum_new(algo);
79 g_checksum_update(csum2, keybuf, block_size);
80 g_checksum_update(csum2, digest, digest_size);
82 g_checksum_get_digest(csum2, buffer, digest_len);
84 g_checksum_free(csum1);
85 g_checksum_free(csum2);
88 /* Unfortunately a glib hash table is only guaranteed to be able to store
89 * 32-bit keys if we use the key directly. If we want 64-bit inode numbers,
90 * we'll have to allocate memory to store the 64-bit inumber, and use a pointer
91 * to it. Rather than allocate the memory for the key, we'll just include a
92 * pointer to the 64-bit inum stored in the inode itself, so that we don't need
93 * to do any more memory management. */
94 static guint bluesky_fs_key_hash_func(gconstpointer key)
96 uint64_t inum = *(const uint64_t *)key;
100 static gboolean bluesky_fs_key_equal_func(gconstpointer a, gconstpointer b)
102 uint64_t i1 = *(const uint64_t *)a;
103 uint64_t i2 = *(const uint64_t *)b;
107 /* Filesystem-level operations. A filesystem is like a directory tree that we
108 * are willing to export. */
109 BlueSkyFS *bluesky_new_fs(gchar *name)
111 BlueSkyFS *fs = g_new0(BlueSkyFS, 1);
112 fs->lock = g_mutex_new();
113 fs->name = g_strdup(name);
114 fs->inodes = g_hash_table_new(bluesky_fs_key_hash_func,
115 bluesky_fs_key_equal_func);
116 fs->next_inum = BLUESKY_ROOT_INUM + 1;
121 /* Allocate a fresh inode number which has not been used before within a
123 uint64_t bluesky_fs_alloc_inode(BlueSkyFS *fs)
127 g_mutex_lock(fs->lock);
128 inum = fs->next_inum;
130 g_mutex_unlock(fs->lock);
135 BlueSkyInode *bluesky_new_inode(uint64_t inum, BlueSkyFileType type)
137 BlueSkyInode *i = g_new0(BlueSkyInode, 1);
139 i->lock = g_mutex_new();
144 case BLUESKY_REGULAR:
145 i->blocks = g_array_new(FALSE, TRUE, sizeof(BlueSkyBlock));
147 case BLUESKY_DIRECTORY:
148 i->dirents = g_sequence_new(bluesky_dirent_destroy);
149 i->dirhash = g_hash_table_new(g_str_hash, g_str_equal);
152 case BLUESKY_CHARACTER:
153 case BLUESKY_SYMLINK:
162 /* Retrieve an inode from the filesystem. Eventually this will be a cache and
163 * so we might need to go fetch the inode from elsewhere; for now all
164 * filesystem state is stored here. */
165 BlueSkyInode *bluesky_get_inode(BlueSkyFS *fs, uint64_t inum)
167 BlueSkyInode *inode = NULL;
169 g_mutex_lock(fs->lock);
170 inode = (BlueSkyInode *)g_hash_table_lookup(fs->inodes, &inum);
171 g_mutex_unlock(fs->lock);
176 /* Insert an inode into the filesystem inode cache. */
177 void bluesky_insert_inode(BlueSkyFS *fs, BlueSkyInode *inode)
179 g_mutex_lock(fs->lock);
180 g_hash_table_insert(fs->inodes, &inode->inum, inode);
181 g_mutex_unlock(fs->lock);
184 /* Mark a given block dirty and make sure that data is faulted in so that it
185 * can be written to. */
186 void bluesky_block_touch(BlueSkyInode *inode, uint64_t i)
188 g_return_if_fail(i < inode->blocks->len);
189 BlueSkyBlock *block = &g_array_index(inode->blocks, BlueSkyBlock, i);
191 switch (block->type) {
192 case BLUESKY_BLOCK_ZERO:
193 block->data = g_malloc0(BLUESKY_BLOCK_SIZE);
195 case BLUESKY_BLOCK_REF:
196 /* TODO: Pull in data first */
197 block->data = g_malloc0(BLUESKY_BLOCK_SIZE);
199 case BLUESKY_BLOCK_CACHED:
200 case BLUESKY_BLOCK_DIRTY:
204 block->type = BLUESKY_BLOCK_DIRTY;
207 /* Set the size of a file. This will truncate or extend the file as needed.
208 * Newly-allocated bytes are zeroed. */
209 void bluesky_file_truncate(BlueSkyInode *inode, uint64_t size)
211 g_return_if_fail(size <= BLUESKY_MAX_FILE_SIZE);
213 if (size == inode->size)
216 uint64_t blocks = (size + BLUESKY_BLOCK_SIZE - 1) / BLUESKY_BLOCK_SIZE;
218 if (blocks > inode->blocks->len) {
219 /* Need to add new blocks to the end of a file. New block structures
220 * are automatically zeroed, which initializes them to be pointers to
221 * zero blocks so we don't need to do any more work. */
222 g_array_set_size(inode->blocks, blocks);
223 } else if (blocks < inode->blocks->len) {
224 /* Delete blocks from a file. Must reclaim memory. */
225 for (guint i = inode->blocks->len; i < blocks; i++) {
226 BlueSkyBlock *b = &g_array_index(inode->blocks, BlueSkyBlock, i);
230 g_array_set_size(inode->blocks, blocks);
233 /* If the file size is being decreased, ensure that any trailing data in
234 * the last block is zeroed. */
235 if (size < inode->size) {
236 BlueSkyBlock *b = &g_array_index(inode->blocks, BlueSkyBlock,
238 if (b->type != BLUESKY_BLOCK_ZERO) {
239 bluesky_block_touch(inode, blocks - 1);
240 int end_offset = size % BLUESKY_BLOCK_SIZE;
241 if (end_offset > 0) {
242 memset(&b->data[end_offset], 0,
243 BLUESKY_BLOCK_SIZE - end_offset);
249 bluesky_inode_update_ctime(inode, 1);
252 void bluesky_file_write(BlueSkyInode *inode, uint64_t offset,
253 const char *data, gint len)
255 g_return_if_fail(inode->type == BLUESKY_REGULAR);
256 g_return_if_fail(offset < inode->size);
257 g_return_if_fail(len <= inode->size - offset);
263 uint64_t block_num = offset / BLUESKY_BLOCK_SIZE;
264 gint block_offset = offset % BLUESKY_BLOCK_SIZE;
265 gint bytes = MIN(BLUESKY_BLOCK_SIZE - block_offset, len);
267 bluesky_block_touch(inode, block_num);
268 BlueSkyBlock *b = &g_array_index(inode->blocks, BlueSkyBlock,
270 memcpy(&b->data[block_offset], data, bytes);
271 bluesky_block_flush(b);
278 bluesky_inode_update_ctime(inode, 1);
281 void bluesky_file_read(BlueSkyInode *inode, uint64_t offset,
284 g_return_if_fail(inode->type == BLUESKY_REGULAR);
285 g_return_if_fail(offset < inode->size);
286 g_return_if_fail(len <= inode->size - offset);
289 uint64_t block_num = offset / BLUESKY_BLOCK_SIZE;
290 gint block_offset = offset % BLUESKY_BLOCK_SIZE;
291 gint bytes = MIN(BLUESKY_BLOCK_SIZE - block_offset, len);
293 BlueSkyBlock *b = &g_array_index(inode->blocks, BlueSkyBlock,
296 case BLUESKY_BLOCK_ZERO:
297 memset(buf, 0, bytes);
299 case BLUESKY_BLOCK_REF:
300 /* TODO: Pull in data first */
301 memset(buf, 0, bytes);
303 case BLUESKY_BLOCK_CACHED:
304 case BLUESKY_BLOCK_DIRTY:
305 memcpy(buf, &b->data[block_offset], bytes);
315 /* Write the given block to cloud-backed storage and mark it clean. */
316 void bluesky_block_flush(BlueSkyBlock *block)
318 if (block->type != BLUESKY_BLOCK_DIRTY)
321 BlueSkyRCStr *data = bluesky_string_new(g_memdup(block->data,
325 GChecksum *csum = g_checksum_new(G_CHECKSUM_SHA256);
326 g_checksum_update(csum, data->data, data->len);
327 const gchar *name = g_checksum_get_string(csum);
329 g_print("Flushing block as %s\n", name);
330 //memstore_put(store, name, data);
332 block->ref = g_strdup(name);
333 block->type = BLUESKY_BLOCK_CACHED;
335 g_checksum_free(csum);
336 bluesky_string_unref(data);