1 /* Blue Sky: File Systems in the Cloud
3 * Copyright (C) 2009 The Regents of the University of California
4 * Written by Michael Vrable <mvrable@cs.ucsd.edu>
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions
9 * 1. Redistributions of source code must retain the above copyright
10 * notice, this list of conditions and the following disclaimer.
11 * 2. Redistributions in binary form must reproduce the above copyright
12 * notice, this list of conditions and the following disclaimer in the
13 * documentation and/or other materials provided with the distribution.
14 * 3. Neither the name of the University nor the names of its contributors
15 * may be used to endorse or promote products derived from this software
16 * without specific prior written permission.
18 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
19 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
20 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
21 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
22 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
23 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
24 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
25 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
26 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
27 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
31 /* RPC handling: registration, marshalling and unmarshalling of messages. For
32 * now this uses the standard Sun RPC mechanisms in the standard C library.
33 * Later, it might be changed to use something better. Much of this code was
34 * generated with rpcgen from the XDR specifications, but has been hand-edited
37 #include "mount_prot.h"
38 #include "nfs3_prot.h"
41 #include <rpc/pmap_clnt.h>
45 #include <sys/socket.h>
46 #include <netinet/in.h>
47 #include <netinet/ip.h>
52 static int outstanding_rpcs = 0;
53 static struct bluesky_stats *rpc_recv_stats, *rpc_send_stats;
55 /* TCP port number to use for NFS protocol. (Should be 2049.) */
56 #define NFS_SERVICE_PORT 2051
58 /* Maximum size of a single RPC message that we will accept (8 MB). */
59 #define MAX_RPC_MSGSIZE (8 << 20)
62 mount_program_3(struct svc_req *rqstp, register SVCXPRT *transp)
65 dirpath mountproc3_mnt_3_arg;
66 dirpath mountproc3_umnt_3_arg;
69 xdrproc_t _xdr_argument, _xdr_result;
70 char *(*local)(char *, struct svc_req *);
72 switch (rqstp->rq_proc) {
74 _xdr_argument = (xdrproc_t) xdr_void;
75 _xdr_result = (xdrproc_t) xdr_void;
76 local = (char *(*)(char *, struct svc_req *)) mountproc3_null_3_svc;
80 _xdr_argument = (xdrproc_t) xdr_dirpath;
81 _xdr_result = (xdrproc_t) xdr_mountres3;
82 local = (char *(*)(char *, struct svc_req *)) mountproc3_mnt_3_svc;
86 _xdr_argument = (xdrproc_t) xdr_void;
87 _xdr_result = (xdrproc_t) xdr_mountlist;
88 local = (char *(*)(char *, struct svc_req *)) mountproc3_dump_3_svc;
92 _xdr_argument = (xdrproc_t) xdr_dirpath;
93 _xdr_result = (xdrproc_t) xdr_void;
94 local = (char *(*)(char *, struct svc_req *)) mountproc3_umnt_3_svc;
97 case MOUNTPROC3_UMNTALL:
98 _xdr_argument = (xdrproc_t) xdr_void;
99 _xdr_result = (xdrproc_t) xdr_void;
100 local = (char *(*)(char *, struct svc_req *)) mountproc3_umntall_3_svc;
103 case MOUNTPROC3_EXPORT:
104 _xdr_argument = (xdrproc_t) xdr_void;
105 _xdr_result = (xdrproc_t) xdr_exports;
106 local = (char *(*)(char *, struct svc_req *)) mountproc3_export_3_svc;
110 svcerr_noproc (transp);
113 memset ((char *)&argument, 0, sizeof (argument));
114 if (!svc_getargs (transp, (xdrproc_t) _xdr_argument, (caddr_t) &argument)) {
115 svcerr_decode (transp);
118 result = (*local)((char *)&argument, rqstp);
119 if (result != NULL && !svc_sendreply(transp, (xdrproc_t) _xdr_result, result)) {
120 svcerr_systemerr (transp);
122 if (!svc_freeargs (transp, (xdrproc_t) _xdr_argument, (caddr_t) &argument)) {
123 fprintf (stderr, "%s", "unable to free arguments");
133 uint32_t verf_flavor;
135 uint32_t accept_stat;
138 static void async_rpc_write(RPCConnection *rpc,
139 const char *buf, gsize len);
140 static void async_rpc_flush(RPCConnection *rpc);
142 struct rpc_fail_reply {
146 uint32_t verf_flavor;
148 uint32_t accept_stat;
152 async_rpc_send_failure(RPCRequest *req, enum accept_stat stat)
154 struct rpc_fail_reply header;
156 g_atomic_int_add(&outstanding_rpcs, -1);
158 header.xid = htonl(req->xid);
159 header.type = htonl(1); /* REPLY */
160 header.stat = htonl(MSG_ACCEPTED);
161 header.verf_flavor = 0;
163 header.accept_stat = htonl(stat);
165 g_mutex_lock(req->connection->send_lock);
166 uint32_t fragment = htonl(sizeof(header) | 0x80000000);
167 if (!req->connection->udp_transport)
168 async_rpc_write(req->connection, (const char *)&fragment,
170 async_rpc_write(req->connection, (const char *)&header, sizeof(header));
171 async_rpc_flush(req->connection);
172 g_mutex_unlock(req->connection->send_lock);
174 bluesky_profile_free(req->profile);
176 if (req->args != NULL) {
179 xdrmem_create(&xdr, buf, sizeof(buf), XDR_FREE);
180 if (!req->xdr_args_free(&xdr, req->args)) {
181 fprintf(stderr, "unable to free arguments");
186 if (req->raw_args != NULL)
187 g_string_free(req->raw_args, TRUE);
189 while (req->cleanup != NULL) {
190 struct cleanup_list *c = req->cleanup;
191 req->cleanup = c->next;
196 if (req->connection->udp_transport) {
197 /* For UDP, a connection only exists for the duration of a single
199 g_mutex_free(req->connection->send_lock);
200 g_string_free(req->connection->msgbuf, TRUE);
201 g_string_free(req->connection->sendbuf, TRUE);
202 g_free(req->connection);
209 async_rpc_send_reply(RPCRequest *req, void *result)
211 bluesky_time_hires time_end;
213 bluesky_profile_add_event(req->profile,
214 g_strdup("Start encoding NFS response"));
216 GString *str = g_string_new("");
218 xdr_string_create(&xdr_out, str, XDR_ENCODE);
219 if (!req->xdr_result(&xdr_out, result)) {
220 async_rpc_send_failure(req, SYSTEM_ERR);
221 g_string_free(str, TRUE);
225 g_atomic_int_add(&outstanding_rpcs, -1);
226 bluesky_stats_add(rpc_send_stats, str->len);
228 struct rpc_reply header;
229 header.xid = htonl(req->xid);
230 header.type = htonl(1); /* REPLY */
231 header.stat = htonl(MSG_ACCEPTED);
232 header.verf_flavor = 0;
234 header.accept_stat = 0;
236 g_mutex_lock(req->connection->send_lock);
237 gsize msg_size = str->len;
238 uint32_t fragment = htonl((msg_size + sizeof(header)) | 0x80000000);
239 if (!req->connection->udp_transport)
240 async_rpc_write(req->connection, (const char *)&fragment,
242 async_rpc_write(req->connection, (const char *)&header, sizeof(header));
243 async_rpc_write(req->connection, str->str, str->len);
244 async_rpc_flush(req->connection);
245 g_mutex_unlock(req->connection->send_lock);
247 time_end = bluesky_now_hires();
250 printf("RPC[%"PRIx32"]: time = %"PRId64" ns\n",
251 req->xid, time_end - req->time_start);
254 bluesky_profile_add_event(req->profile,
255 g_strdup("NFS reply sent"));
256 bluesky_profile_print(req->profile);
259 bluesky_profile_free(req->profile);
260 g_string_free(str, TRUE);
262 if (req->args != NULL) {
265 xdrmem_create(&xdr, buf, sizeof(buf), XDR_FREE);
266 if (!req->xdr_args_free(&xdr, req->args)) {
267 fprintf(stderr, "unable to free arguments");
272 if (req->raw_args != NULL)
273 g_string_free(req->raw_args, TRUE);
275 while (req->cleanup != NULL) {
276 struct cleanup_list *c = req->cleanup;
277 req->cleanup = c->next;
282 if (req->connection->udp_transport) {
283 /* For UDP, a connection only exists for the duration of a single
285 g_mutex_free(req->connection->send_lock);
286 g_string_free(req->connection->msgbuf, TRUE);
287 g_string_free(req->connection->sendbuf, TRUE);
288 g_free(req->connection);
294 static const char *nfs_proc_names[] = {
295 [NFSPROC3_NULL] = "NULL",
296 [NFSPROC3_GETATTR] = "GETATTR",
297 [NFSPROC3_SETATTR] = "SETATTR",
298 [NFSPROC3_LOOKUP] = "LOOKUP",
299 [NFSPROC3_ACCESS] = "ACCESS",
300 [NFSPROC3_READLINK] = "READLINK",
301 [NFSPROC3_READ] = "READ",
302 [NFSPROC3_WRITE] = "WRITE",
303 [NFSPROC3_CREATE] = "CREATE",
304 [NFSPROC3_MKDIR] = "MKDIR",
305 [NFSPROC3_SYMLINK] = "SYMLINK",
306 [NFSPROC3_MKNOD] = "MKNOD",
307 [NFSPROC3_REMOVE] = "REMOVE",
308 [NFSPROC3_RMDIR] = "RMDIR",
309 [NFSPROC3_RENAME] = "RENAME",
310 [NFSPROC3_LINK] = "LINK",
311 [NFSPROC3_READDIR] = "READDIR",
312 [NFSPROC3_READDIRPLUS] = "READDIRPLUS",
313 [NFSPROC3_FSSTAT] = "FSSTAT",
314 [NFSPROC3_FSINFO] = "FSINFO",
315 [NFSPROC3_PATHCONF] = "PATHCONF",
316 [NFSPROC3_COMMIT] = "COMMIT",
320 nfs_program_3(RPCRequest *req)
322 RPCConnection *connection = req->connection;
323 uint32_t xid = req->xid;
324 const char *msg_buf = req->raw_args->str + req->raw_args_header_bytes;
325 size_t msg_len = req->raw_args->len - req->raw_args_header_bytes;
328 nfs_fh3 nfsproc3_getattr_3_arg;
329 setattr3args nfsproc3_setattr_3_arg;
330 diropargs3 nfsproc3_lookup_3_arg;
331 access3args nfsproc3_access_3_arg;
332 nfs_fh3 nfsproc3_readlink_3_arg;
333 read3args nfsproc3_read_3_arg;
334 write3args nfsproc3_write_3_arg;
335 create3args nfsproc3_create_3_arg;
336 mkdir3args nfsproc3_mkdir_3_arg;
337 symlink3args nfsproc3_symlink_3_arg;
338 mknod3args nfsproc3_mknod_3_arg;
339 diropargs3 nfsproc3_remove_3_arg;
340 diropargs3 nfsproc3_rmdir_3_arg;
341 rename3args nfsproc3_rename_3_arg;
342 link3args nfsproc3_link_3_arg;
343 readdir3args nfsproc3_readdir_3_arg;
344 readdirplus3args nfsproc3_readdirplus_3_arg;
345 nfs_fh3 nfsproc3_fsstat_3_arg;
346 nfs_fh3 nfsproc3_fsinfo_3_arg;
347 nfs_fh3 nfsproc3_pathconf_3_arg;
348 commit3args nfsproc3_commit_3_arg;
351 xdrproc_t _xdr_argument, _xdr_result;
352 char *(*local)(char *, RPCRequest *);
354 bluesky_profile_set(req->profile);
356 if (req->req_proc < sizeof(nfs_proc_names) / sizeof(const char *)) {
357 bluesky_profile_add_event(
359 g_strdup_printf("Dispatching NFS %s request",
360 nfs_proc_names[req->req_proc])
364 switch (req->req_proc) {
366 _xdr_argument = (xdrproc_t) xdr_void;
367 _xdr_result = (xdrproc_t) xdr_void;
368 local = (char *(*)(char *, RPCRequest *)) nfsproc3_null_3_svc;
371 case NFSPROC3_GETATTR:
372 _xdr_argument = (xdrproc_t) xdr_nfs_fh3;
373 _xdr_result = (xdrproc_t) xdr_getattr3res;
374 local = (char *(*)(char *, RPCRequest *)) nfsproc3_getattr_3_svc;
377 case NFSPROC3_SETATTR:
378 _xdr_argument = (xdrproc_t) xdr_setattr3args;
379 _xdr_result = (xdrproc_t) xdr_wccstat3;
380 local = (char *(*)(char *, RPCRequest *)) nfsproc3_setattr_3_svc;
383 case NFSPROC3_LOOKUP:
384 _xdr_argument = (xdrproc_t) xdr_diropargs3;
385 _xdr_result = (xdrproc_t) xdr_lookup3res;
386 local = (char *(*)(char *, RPCRequest *)) nfsproc3_lookup_3_svc;
389 case NFSPROC3_ACCESS:
390 _xdr_argument = (xdrproc_t) xdr_access3args;
391 _xdr_result = (xdrproc_t) xdr_access3res;
392 local = (char *(*)(char *, RPCRequest *)) nfsproc3_access_3_svc;
395 case NFSPROC3_READLINK:
396 _xdr_argument = (xdrproc_t) xdr_nfs_fh3;
397 _xdr_result = (xdrproc_t) xdr_readlink3res;
398 local = (char *(*)(char *, RPCRequest *)) nfsproc3_readlink_3_svc;
402 _xdr_argument = (xdrproc_t) xdr_read3args;
403 _xdr_result = (xdrproc_t) xdr_read3res;
404 local = (char *(*)(char *, RPCRequest *)) nfsproc3_read_3_svc;
408 _xdr_argument = (xdrproc_t) xdr_write3args;
409 _xdr_result = (xdrproc_t) xdr_write3res;
410 local = (char *(*)(char *, RPCRequest *)) nfsproc3_write_3_svc;
413 case NFSPROC3_CREATE:
414 _xdr_argument = (xdrproc_t) xdr_create3args;
415 _xdr_result = (xdrproc_t) xdr_diropres3;
416 local = (char *(*)(char *, RPCRequest *)) nfsproc3_create_3_svc;
420 _xdr_argument = (xdrproc_t) xdr_mkdir3args;
421 _xdr_result = (xdrproc_t) xdr_diropres3;
422 local = (char *(*)(char *, RPCRequest *)) nfsproc3_mkdir_3_svc;
425 case NFSPROC3_SYMLINK:
426 _xdr_argument = (xdrproc_t) xdr_symlink3args;
427 _xdr_result = (xdrproc_t) xdr_diropres3;
428 local = (char *(*)(char *, RPCRequest *)) nfsproc3_symlink_3_svc;
432 _xdr_argument = (xdrproc_t) xdr_mknod3args;
433 _xdr_result = (xdrproc_t) xdr_diropres3;
434 local = (char *(*)(char *, RPCRequest *)) nfsproc3_mknod_3_svc;
437 case NFSPROC3_REMOVE:
438 _xdr_argument = (xdrproc_t) xdr_diropargs3;
439 _xdr_result = (xdrproc_t) xdr_wccstat3;
440 local = (char *(*)(char *, RPCRequest *)) nfsproc3_remove_3_svc;
444 _xdr_argument = (xdrproc_t) xdr_diropargs3;
445 _xdr_result = (xdrproc_t) xdr_wccstat3;
446 local = (char *(*)(char *, RPCRequest *)) nfsproc3_rmdir_3_svc;
449 case NFSPROC3_RENAME:
450 _xdr_argument = (xdrproc_t) xdr_rename3args;
451 _xdr_result = (xdrproc_t) xdr_rename3res;
452 local = (char *(*)(char *, RPCRequest *)) nfsproc3_rename_3_svc;
456 _xdr_argument = (xdrproc_t) xdr_link3args;
457 _xdr_result = (xdrproc_t) xdr_link3res;
458 local = (char *(*)(char *, RPCRequest *)) nfsproc3_link_3_svc;
461 case NFSPROC3_READDIR:
462 _xdr_argument = (xdrproc_t) xdr_readdir3args;
463 _xdr_result = (xdrproc_t) xdr_readdir3res;
464 local = (char *(*)(char *, RPCRequest *)) nfsproc3_readdir_3_svc;
467 case NFSPROC3_READDIRPLUS:
468 _xdr_argument = (xdrproc_t) xdr_readdirplus3args;
469 _xdr_result = (xdrproc_t) xdr_readdirplus3res;
470 local = (char *(*)(char *, RPCRequest *)) nfsproc3_readdirplus_3_svc;
473 case NFSPROC3_FSSTAT:
474 _xdr_argument = (xdrproc_t) xdr_nfs_fh3;
475 _xdr_result = (xdrproc_t) xdr_fsstat3res;
476 local = (char *(*)(char *, RPCRequest *)) nfsproc3_fsstat_3_svc;
479 case NFSPROC3_FSINFO:
480 _xdr_argument = (xdrproc_t) xdr_nfs_fh3;
481 _xdr_result = (xdrproc_t) xdr_fsinfo3res;
482 local = (char *(*)(char *, RPCRequest *)) nfsproc3_fsinfo_3_svc;
485 case NFSPROC3_PATHCONF:
486 _xdr_argument = (xdrproc_t) xdr_nfs_fh3;
487 _xdr_result = (xdrproc_t) xdr_pathconf3res;
488 local = (char *(*)(char *, RPCRequest *)) nfsproc3_pathconf_3_svc;
491 case NFSPROC3_COMMIT:
492 _xdr_argument = (xdrproc_t) xdr_commit3args;
493 _xdr_result = (xdrproc_t) xdr_commit3res;
494 local = (char *(*)(char *, RPCRequest *)) nfsproc3_commit_3_svc;
498 async_rpc_send_failure(req, PROC_UNAVAIL);
502 /* Decode incoming message */
503 req->xdr_args_free = _xdr_argument;
504 req->args = g_new0(union argtype, 1);
506 xdrmem_create(&xdr_in, (char *)msg_buf, msg_len, XDR_DECODE);
507 if (!_xdr_argument(&xdr_in, req->args)) {
508 async_rpc_send_failure(req, GARBAGE_ARGS);
509 fprintf(stderr, "RPC decode error!\n");
513 /* Perform the call. */
514 req->xdr_result = _xdr_result;
515 result = (*local)((char *)req->args, req);
520 /* Enhanced, asynchronous-friendly RPC layer. This is a replacement for the
521 * built-in sunrpc parsing and dispatch that will allow for processing multiple
522 * requests at the same time. */
523 static GMainContext *main_context;
524 static GMainLoop *main_loop;
526 static GThreadPool *rpc_thread_pool;
528 static volatile int fs_dump_requested = 0;
530 static void sig_handler(int sig)
532 if (sig == SIGUSR1) {
533 fs_dump_requested = 1;
537 static gboolean async_flushd(gpointer data)
540 int rpc_count = g_atomic_int_get(&outstanding_rpcs);
541 if (rpc_count != 0) {
542 g_print("Currently outstanding RPC requests: %d\n", rpc_count);
546 if (fs_dump_requested) {
547 bluesky_debug_dump(fs);
548 bluesky_stats_dump_all();
549 fs_dump_requested = 0;
552 bluesky_flushd_invoke(fs);
556 static void async_rpc_task(gpointer data, gpointer user_data)
558 nfs_program_3((RPCRequest *)data);
561 static async_rpc_init()
563 main_context = g_main_context_new();
564 main_loop = g_main_loop_new(main_context, FALSE);
566 rpc_thread_pool = g_thread_pool_new(async_rpc_task, NULL,
567 bluesky_max_threads, FALSE, NULL);
569 /* Arrange to have the cache writeback code run every five seconds. */
570 GSource *source = g_timeout_source_new_seconds(5);
571 g_source_set_callback(source, async_flushd, NULL, NULL);
572 g_source_attach(source, main_context);
573 g_source_unref(source);
575 /* Signal USR1 is used to request a debugging dump of filesyste info */
577 sa.sa_handler = sig_handler;
578 sigemptyset(&sa.sa_mask);
579 sa.sa_flags = SA_RESTART;
580 if (sigaction(SIGUSR1, &sa, NULL) < 0) {
585 struct rpc_call_header {
599 /* Decode an RPC message and process it. Returns a boolean indicating whether
600 * the message could be processed; if false, an unrecoverable error occurred
601 * and the transport should be closed. */
602 static gboolean async_rpc_dispatch(RPCConnection *rpc)
604 bluesky_time_hires time_start = bluesky_now_hires();
606 GString *msg = rpc->msgbuf;
607 const char *buf = msg->str;
609 bluesky_stats_add(rpc_recv_stats, msg->len);
611 if (msg->len < sizeof(struct rpc_call_header)) {
612 fprintf(stderr, "Short RPC message: only %zd bytes!\n", msg->len);
616 struct rpc_call_header *header = (struct rpc_call_header *)(msg->str);
617 uint32_t xid = ntohl(header->xid);
619 if (ntohl(header->mtype) != 0) {
620 /* Not an RPC call */
624 if (ntohl(header->rpcvers) != 2) {
628 g_atomic_int_add(&outstanding_rpcs, 1);
630 RPCRequest *req = g_new0(RPCRequest, 1);
631 req->connection = rpc;
632 req->profile = bluesky_profile_new();
633 bluesky_profile_add_event(req->profile, g_strdup("Receive NFS request"));
636 if (ntohl(header->prog) != NFS_PROGRAM) {
637 async_rpc_send_failure(req, PROG_UNAVAIL);
639 } else if (ntohl(header->vers) != NFS_V3) {
640 /* FIXME: Should be PROG_MISMATCH */
641 async_rpc_send_failure(req, PROG_UNAVAIL);
645 uint32_t proc = ntohl(header->proc);
647 /* Next, skip over authentication headers. */
648 buf += sizeof(struct rpc_call_header);
649 for (i = 0; i < 2; i++) {
650 struct rpc_auth *auth = (struct rpc_auth *)buf;
651 if (buf - msg->str + sizeof(struct rpc_auth) > msg->len) {
652 g_atomic_int_add(&outstanding_rpcs, -1);
656 gsize authsize = ntohl(auth->len) + sizeof(struct rpc_auth);
657 if (authsize > MAX_RPC_MSGSIZE) {
658 g_atomic_int_add(&outstanding_rpcs, -1);
665 if (buf - msg->str > msg->len) {
666 g_atomic_int_add(&outstanding_rpcs, -1);
671 req->raw_args_header_bytes = buf - msg->str;
672 req->req_proc = ntohl(header->proc);
673 rpc->msgbuf = g_string_new("");
675 if (bluesky_options.sync_frontends) {
678 g_thread_pool_push(rpc_thread_pool, req, NULL);
684 /* Write the given data to the RPC socket. */
685 static void async_rpc_write(RPCConnection *rpc,
686 const char *buf, gsize len)
688 if (rpc->udp_transport) {
689 g_string_append_len(rpc->sendbuf, buf, len);
693 /* Normal TCP path */
696 switch (g_io_channel_write_chars(rpc->channel, buf, len,
698 case G_IO_STATUS_ERROR:
699 case G_IO_STATUS_EOF:
700 case G_IO_STATUS_AGAIN:
701 fprintf(stderr, "Error writing to socket!\n");
703 case G_IO_STATUS_NORMAL:
710 // g_io_channel_flush(rpc->channel, NULL);
713 /* Flush a completed message out to the RPC socket */
714 static void async_rpc_flush(RPCConnection *rpc)
716 if (rpc->udp_transport) {
717 sendto(g_io_channel_unix_get_fd(rpc->channel),
718 rpc->sendbuf->str, rpc->sendbuf->len, 0,
719 (struct sockaddr *)&rpc->peer, sizeof(struct sockaddr_in));
722 g_io_channel_flush(rpc->channel, NULL);
726 static gboolean async_rpc_do_read(GIOChannel *channel,
727 GIOCondition condition,
730 RPCConnection *rpc = (RPCConnection *)data;
732 gsize bytes_to_read = 0; /* Number of bytes to attempt to read. */
734 /* If we have not yet read in the fragment header, do that first. This is
735 * 4 bytes that indicates the number of bytes in the message to follow
736 * (with the high bit set if this is the last fragment making up the
738 if (rpc->frag_len == 0) {
739 bytes_to_read = 4 - rpc->frag_hdr_bytes;
741 bytes_to_read = rpc->frag_len & 0x7fffffff;
744 if (bytes_to_read > MAX_RPC_MSGSIZE
745 || rpc->msgbuf->len + bytes_to_read > MAX_RPC_MSGSIZE)
747 fprintf(stderr, "Excessive fragment size for RPC: %zd bytes\n",
749 g_io_channel_shutdown(rpc->channel, TRUE, NULL);
753 gsize bytes_read = 0;
754 g_string_set_size(rpc->msgbuf, rpc->msgbuf->len + bytes_to_read);
755 char *buf = &rpc->msgbuf->str[rpc->msgbuf->len - bytes_to_read];
756 switch (g_io_channel_read_chars(rpc->channel, buf,
757 bytes_to_read, &bytes_read, NULL)) {
758 case G_IO_STATUS_NORMAL:
760 case G_IO_STATUS_AGAIN:
762 case G_IO_STATUS_EOF:
763 if (bytes_read == bytes_to_read)
765 /* else fall through */
766 case G_IO_STATUS_ERROR:
767 fprintf(stderr, "Unexpected error or end of file on RPC stream %d!\n",
768 g_io_channel_unix_get_fd(rpc->channel));
769 g_io_channel_shutdown(rpc->channel, TRUE, NULL);
770 /* TODO: Clean up connection object. */
774 g_assert(bytes_read >= 0 && bytes_read <= bytes_to_read);
776 g_string_set_size(rpc->msgbuf,
777 rpc->msgbuf->len - (bytes_to_read - bytes_read));
779 if (rpc->frag_len == 0) {
780 /* Handle reading in the fragment header. If we've read the complete
781 * header, store the fragment size. */
782 rpc->frag_hdr_bytes += bytes_read;
783 if (rpc->frag_hdr_bytes == 4) {
784 memcpy((char *)&rpc->frag_len,
785 &rpc->msgbuf->str[rpc->msgbuf->len - 4], 4);
786 rpc->frag_len = ntohl(rpc->frag_len);
787 g_string_set_size(rpc->msgbuf, rpc->msgbuf->len - 4);
788 rpc->frag_hdr_bytes = 0;
791 /* We were reading in the fragment body. */
792 rpc->frag_len -= bytes_read;
794 if (rpc->frag_len == 0x80000000) {
795 /* We have a complete message since this was the last fragment and
796 * there are no more bytes in it. Dispatch the message. */
797 if (!async_rpc_dispatch(rpc)) {
798 fprintf(stderr, "Invalid RPC message, closing channel\n");
799 g_io_channel_shutdown(rpc->channel, TRUE, NULL);
803 g_string_set_size(rpc->msgbuf, 0);
810 static gboolean async_rpc_do_accept(GIOChannel *channel,
811 GIOCondition condition,
814 int fd = g_io_channel_unix_get_fd(channel);
815 struct sockaddr_in addr;
816 socklen_t addrlen = sizeof(addr);
818 g_print("Received new connection on fd %d!\n", fd);
819 int nfd = accept(fd, (struct sockaddr *)&addr, &addrlen);
821 fprintf(stderr, "Error accepting connection: %m\n");
825 RPCConnection *rpc = g_new0(RPCConnection, 1);
826 rpc->channel = g_io_channel_unix_new(nfd);
827 rpc->msgbuf = g_string_new("");
828 g_io_channel_set_encoding(rpc->channel, NULL, NULL);
829 rpc->send_lock = g_mutex_new();
830 GSource *source = g_io_create_watch(rpc->channel, G_IO_IN);
831 g_source_set_callback(source, (GSourceFunc)async_rpc_do_read,
833 g_source_attach(source, main_context);
834 g_source_unref(source);
839 static async_rpc_register_listening(int fd)
841 GIOChannel *channel = g_io_channel_unix_new(fd);
842 g_io_channel_set_encoding(channel, NULL, NULL);
843 GSource *source = g_io_create_watch(channel, G_IO_IN);
844 g_source_set_callback(source, (GSourceFunc)async_rpc_do_accept,
846 g_source_attach(source, main_context);
847 g_source_unref(source);
850 static gboolean async_rpc_do_udp(GIOChannel *channel,
851 GIOCondition condition,
856 struct sockaddr_in src;
857 socklen_t addrlen = sizeof(struct sockaddr_in);
858 ssize_t len = recvfrom(g_io_channel_unix_get_fd(channel),
860 (struct sockaddr *)&src, &addrlen);
862 fprintf(stderr, "UDP read error: %m, shutting down UDP\n");
866 g_assert(len < sizeof(buf));
868 RPCConnection *rpc = g_new0(RPCConnection, 1);
869 rpc->channel = channel;
870 rpc->msgbuf = g_string_new_len(buf, len);
871 rpc->send_lock = g_mutex_new();
872 rpc->udp_transport = TRUE;
873 memcpy(&rpc->peer, &src, sizeof(struct sockaddr_in));
874 rpc->sendbuf = g_string_new("");
876 /* We have a complete message since this was the last fragment and
877 * there are no more bytes in it. Dispatch the message. */
878 async_rpc_dispatch(rpc);
883 static async_rpc_register_listening_udp(int fd)
885 GIOChannel *channel = g_io_channel_unix_new(fd);
886 g_io_channel_set_encoding(channel, NULL, NULL);
887 GSource *source = g_io_create_watch(channel, G_IO_IN);
888 g_source_set_callback(source, (GSourceFunc)async_rpc_do_udp,
890 g_source_attach(source, main_context);
891 g_source_unref(source);
894 static gpointer async_rpc_run(gpointer data)
896 g_print("Starting NFS main loop...\n");
897 g_main_loop_run(main_loop);
904 rpc_recv_stats = bluesky_stats_new("NFS RPC Messages In");
905 rpc_send_stats = bluesky_stats_new("NFS RPC Messages Out");
910 pmap_unset (MOUNT_PROGRAM, MOUNT_V3);
912 transp = svcudp_create(RPC_ANYSOCK);
913 if (transp == NULL) {
914 fprintf(stderr, "%s", "cannot create udp service.");
917 if (!svc_register(transp, MOUNT_PROGRAM, MOUNT_V3, mount_program_3, IPPROTO_UDP)) {
918 fprintf(stderr, "%s", "unable to register (MOUNT_PROGRAM, MOUNT_V3, udp).");
922 transp = svctcp_create(RPC_ANYSOCK, 0, 0);
923 if (transp == NULL) {
924 fprintf(stderr, "%s", "cannot create tcp service.");
927 if (!svc_register(transp, MOUNT_PROGRAM, MOUNT_V3, mount_program_3, IPPROTO_TCP)) {
928 fprintf(stderr, "%s", "unable to register (MOUNT_PROGRAM, MOUNT_V3, tcp).");
932 /* NFS protocol (version 3) */
933 pmap_unset (NFS_PROGRAM, NFS_V3);
935 int fd = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP);
937 fprintf(stderr, "Unable to create NFS TCP socket: %m\n");
942 setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, (char *)&n, sizeof(n));
944 struct sockaddr_in addr;
945 addr.sin_family = AF_INET;
946 addr.sin_port = htons(NFS_SERVICE_PORT);
947 addr.sin_addr.s_addr = INADDR_ANY;
948 if (bind(fd, (struct sockaddr *)&addr, sizeof(addr)) < 0) {
949 fprintf(stderr, "Unable to bind to NFS TCP address: %m\n");
953 if (listen(fd, SOMAXCONN) < 0) {
954 fprintf(stderr, "Unable to listen on NFS TCP socket: %m\n");
958 if (!pmap_set(NFS_PROGRAM, NFS_V3, IPPROTO_TCP, NFS_SERVICE_PORT)) {
959 fprintf(stderr, "Could not register NFS RPC service!\n");
963 async_rpc_register_listening(fd);
965 /* Minimal UDP NFSv3 support */
966 fd = socket(AF_INET, SOCK_DGRAM, IPPROTO_UDP);
968 fprintf(stderr, "Unable to create NFS UDP socket: %m\n");
972 addr.sin_family = AF_INET;
973 addr.sin_port = htons(NFS_SERVICE_PORT);
974 addr.sin_addr.s_addr = INADDR_ANY;
975 if (bind(fd, (struct sockaddr *)&addr, sizeof(addr)) < 0) {
976 fprintf(stderr, "Unable to bind to NFS UDP address: %m\n");
980 if (!pmap_set(NFS_PROGRAM, NFS_V3, IPPROTO_UDP, NFS_SERVICE_PORT)) {
981 fprintf(stderr, "Could not register NFS UDP RPC service!\n");
985 async_rpc_register_listening_udp(fd);
987 g_thread_create(async_rpc_run, NULL, TRUE, NULL);