1 /* Blue Sky: File Systems in the Cloud
3 * Copyright (C) 2009 The Regents of the University of California
4 * Written by Michael Vrable <mvrable@cs.ucsd.edu>
9 /* RPC handling: registration, marshalling and unmarshalling of messages. For
10 * now this uses the standard Sun RPC mechanisms in the standard C library.
11 * Later, it might be changed to use something better. Much of this code was
12 * generated with rpcgen from the XDR specifications, but has been hand-edited
15 #include "mount_prot.h"
16 #include "nfs3_prot.h"
19 #include <rpc/pmap_clnt.h>
22 #include <sys/socket.h>
23 #include <netinet/in.h>
24 #include <netinet/ip.h>
29 /* TCP port number to use for NFS protocol. (Should be 2049.) */
30 #define NFS_SERVICE_PORT 2051
32 /* Maximum size of a single RPC message that we will accept (8 MB). */
33 #define MAX_RPC_MSGSIZE (8 << 20)
35 /* For now, used for NFS only. */
39 /* The reassembled message, thus far. */
42 /* Remaining number of bytes in this message fragment; 0 if we next expect
43 * another fragment header. */
46 /* If frag_len is zero: the number of bytes of the fragment header that
47 * have been read so far. */
52 mount_program_3(struct svc_req *rqstp, register SVCXPRT *transp)
55 dirpath mountproc3_mnt_3_arg;
56 dirpath mountproc3_umnt_3_arg;
59 xdrproc_t _xdr_argument, _xdr_result;
60 char *(*local)(char *, struct svc_req *);
62 switch (rqstp->rq_proc) {
64 _xdr_argument = (xdrproc_t) xdr_void;
65 _xdr_result = (xdrproc_t) xdr_void;
66 local = (char *(*)(char *, struct svc_req *)) mountproc3_null_3_svc;
70 _xdr_argument = (xdrproc_t) xdr_dirpath;
71 _xdr_result = (xdrproc_t) xdr_mountres3;
72 local = (char *(*)(char *, struct svc_req *)) mountproc3_mnt_3_svc;
76 _xdr_argument = (xdrproc_t) xdr_void;
77 _xdr_result = (xdrproc_t) xdr_mountlist;
78 local = (char *(*)(char *, struct svc_req *)) mountproc3_dump_3_svc;
82 _xdr_argument = (xdrproc_t) xdr_dirpath;
83 _xdr_result = (xdrproc_t) xdr_void;
84 local = (char *(*)(char *, struct svc_req *)) mountproc3_umnt_3_svc;
87 case MOUNTPROC3_UMNTALL:
88 _xdr_argument = (xdrproc_t) xdr_void;
89 _xdr_result = (xdrproc_t) xdr_void;
90 local = (char *(*)(char *, struct svc_req *)) mountproc3_umntall_3_svc;
93 case MOUNTPROC3_EXPORT:
94 _xdr_argument = (xdrproc_t) xdr_void;
95 _xdr_result = (xdrproc_t) xdr_exports;
96 local = (char *(*)(char *, struct svc_req *)) mountproc3_export_3_svc;
100 svcerr_noproc (transp);
103 memset ((char *)&argument, 0, sizeof (argument));
104 if (!svc_getargs (transp, (xdrproc_t) _xdr_argument, (caddr_t) &argument)) {
105 svcerr_decode (transp);
108 result = (*local)((char *)&argument, rqstp);
109 if (result != NULL && !svc_sendreply(transp, (xdrproc_t) _xdr_result, result)) {
110 svcerr_systemerr (transp);
112 if (!svc_freeargs (transp, (xdrproc_t) _xdr_argument, (caddr_t) &argument)) {
113 fprintf (stderr, "%s", "unable to free arguments");
123 uint32_t verf_flavor;
125 uint32_t accept_stat;
128 static void async_rpc_write(RPCConnection *rpc,
129 const char *buf, gsize len);
131 struct rpc_fail_reply {
135 uint32_t verf_flavor;
137 uint32_t accept_stat;
141 async_rpc_send_failure(RPCConnection *rpc, uint32_t xid, enum accept_stat stat)
143 struct rpc_fail_reply header;
145 fprintf(stderr, "Sending RPC failure status %d\n", stat);
147 header.xid = htonl(xid);
148 header.type = htonl(1); /* REPLY */
149 header.stat = htonl(MSG_ACCEPTED);
150 header.verf_flavor = 0;
152 header.accept_stat = htonl(stat);
154 uint32_t fragment = htonl(sizeof(header) | 0x80000000);
155 async_rpc_write(rpc, (const char *)&fragment, sizeof(fragment));
156 async_rpc_write(rpc, (const char *)&header, sizeof(header));
157 g_io_channel_flush(rpc->channel, NULL);
161 nfs_program_3(struct svc_req *rqstp, RPCConnection *connection, uint32_t xid,
162 const char *msg_buf, size_t msg_len)
165 nfs_fh3 nfsproc3_getattr_3_arg;
166 setattr3args nfsproc3_setattr_3_arg;
167 diropargs3 nfsproc3_lookup_3_arg;
168 access3args nfsproc3_access_3_arg;
169 nfs_fh3 nfsproc3_readlink_3_arg;
170 read3args nfsproc3_read_3_arg;
171 write3args nfsproc3_write_3_arg;
172 create3args nfsproc3_create_3_arg;
173 mkdir3args nfsproc3_mkdir_3_arg;
174 symlink3args nfsproc3_symlink_3_arg;
175 mknod3args nfsproc3_mknod_3_arg;
176 diropargs3 nfsproc3_remove_3_arg;
177 diropargs3 nfsproc3_rmdir_3_arg;
178 rename3args nfsproc3_rename_3_arg;
179 link3args nfsproc3_link_3_arg;
180 readdir3args nfsproc3_readdir_3_arg;
181 readdirplus3args nfsproc3_readdirplus_3_arg;
182 nfs_fh3 nfsproc3_fsstat_3_arg;
183 nfs_fh3 nfsproc3_fsinfo_3_arg;
184 nfs_fh3 nfsproc3_pathconf_3_arg;
185 commit3args nfsproc3_commit_3_arg;
188 xdrproc_t _xdr_argument, _xdr_result;
189 char *(*local)(char *, struct svc_req *);
191 switch (rqstp->rq_proc) {
193 _xdr_argument = (xdrproc_t) xdr_void;
194 _xdr_result = (xdrproc_t) xdr_void;
195 local = (char *(*)(char *, struct svc_req *)) nfsproc3_null_3_svc;
198 case NFSPROC3_GETATTR:
199 _xdr_argument = (xdrproc_t) xdr_nfs_fh3;
200 _xdr_result = (xdrproc_t) xdr_getattr3res;
201 local = (char *(*)(char *, struct svc_req *)) nfsproc3_getattr_3_svc;
204 case NFSPROC3_SETATTR:
205 _xdr_argument = (xdrproc_t) xdr_setattr3args;
206 _xdr_result = (xdrproc_t) xdr_wccstat3;
207 local = (char *(*)(char *, struct svc_req *)) nfsproc3_setattr_3_svc;
210 case NFSPROC3_LOOKUP:
211 _xdr_argument = (xdrproc_t) xdr_diropargs3;
212 _xdr_result = (xdrproc_t) xdr_lookup3res;
213 local = (char *(*)(char *, struct svc_req *)) nfsproc3_lookup_3_svc;
216 case NFSPROC3_ACCESS:
217 _xdr_argument = (xdrproc_t) xdr_access3args;
218 _xdr_result = (xdrproc_t) xdr_access3res;
219 local = (char *(*)(char *, struct svc_req *)) nfsproc3_access_3_svc;
222 case NFSPROC3_READLINK:
223 _xdr_argument = (xdrproc_t) xdr_nfs_fh3;
224 _xdr_result = (xdrproc_t) xdr_readlink3res;
225 local = (char *(*)(char *, struct svc_req *)) nfsproc3_readlink_3_svc;
229 _xdr_argument = (xdrproc_t) xdr_read3args;
230 _xdr_result = (xdrproc_t) xdr_read3res;
231 local = (char *(*)(char *, struct svc_req *)) nfsproc3_read_3_svc;
235 _xdr_argument = (xdrproc_t) xdr_write3args;
236 _xdr_result = (xdrproc_t) xdr_write3res;
237 local = (char *(*)(char *, struct svc_req *)) nfsproc3_write_3_svc;
240 case NFSPROC3_CREATE:
241 _xdr_argument = (xdrproc_t) xdr_create3args;
242 _xdr_result = (xdrproc_t) xdr_diropres3;
243 local = (char *(*)(char *, struct svc_req *)) nfsproc3_create_3_svc;
247 _xdr_argument = (xdrproc_t) xdr_mkdir3args;
248 _xdr_result = (xdrproc_t) xdr_diropres3;
249 local = (char *(*)(char *, struct svc_req *)) nfsproc3_mkdir_3_svc;
252 case NFSPROC3_SYMLINK:
253 _xdr_argument = (xdrproc_t) xdr_symlink3args;
254 _xdr_result = (xdrproc_t) xdr_diropres3;
255 local = (char *(*)(char *, struct svc_req *)) nfsproc3_symlink_3_svc;
259 _xdr_argument = (xdrproc_t) xdr_mknod3args;
260 _xdr_result = (xdrproc_t) xdr_diropres3;
261 local = (char *(*)(char *, struct svc_req *)) nfsproc3_mknod_3_svc;
264 case NFSPROC3_REMOVE:
265 _xdr_argument = (xdrproc_t) xdr_diropargs3;
266 _xdr_result = (xdrproc_t) xdr_wccstat3;
267 local = (char *(*)(char *, struct svc_req *)) nfsproc3_remove_3_svc;
271 _xdr_argument = (xdrproc_t) xdr_diropargs3;
272 _xdr_result = (xdrproc_t) xdr_wccstat3;
273 local = (char *(*)(char *, struct svc_req *)) nfsproc3_rmdir_3_svc;
276 case NFSPROC3_RENAME:
277 _xdr_argument = (xdrproc_t) xdr_rename3args;
278 _xdr_result = (xdrproc_t) xdr_rename3res;
279 local = (char *(*)(char *, struct svc_req *)) nfsproc3_rename_3_svc;
283 _xdr_argument = (xdrproc_t) xdr_link3args;
284 _xdr_result = (xdrproc_t) xdr_link3res;
285 local = (char *(*)(char *, struct svc_req *)) nfsproc3_link_3_svc;
288 case NFSPROC3_READDIR:
289 _xdr_argument = (xdrproc_t) xdr_readdir3args;
290 _xdr_result = (xdrproc_t) xdr_readdir3res;
291 local = (char *(*)(char *, struct svc_req *)) nfsproc3_readdir_3_svc;
294 case NFSPROC3_READDIRPLUS:
295 _xdr_argument = (xdrproc_t) xdr_readdirplus3args;
296 _xdr_result = (xdrproc_t) xdr_readdirplus3res;
297 local = (char *(*)(char *, struct svc_req *)) nfsproc3_readdirplus_3_svc;
300 case NFSPROC3_FSSTAT:
301 _xdr_argument = (xdrproc_t) xdr_nfs_fh3;
302 _xdr_result = (xdrproc_t) xdr_fsstat3res;
303 local = (char *(*)(char *, struct svc_req *)) nfsproc3_fsstat_3_svc;
306 case NFSPROC3_FSINFO:
307 _xdr_argument = (xdrproc_t) xdr_nfs_fh3;
308 _xdr_result = (xdrproc_t) xdr_fsinfo3res;
309 local = (char *(*)(char *, struct svc_req *)) nfsproc3_fsinfo_3_svc;
312 case NFSPROC3_PATHCONF:
313 _xdr_argument = (xdrproc_t) xdr_nfs_fh3;
314 _xdr_result = (xdrproc_t) xdr_pathconf3res;
315 local = (char *(*)(char *, struct svc_req *)) nfsproc3_pathconf_3_svc;
318 case NFSPROC3_COMMIT:
319 _xdr_argument = (xdrproc_t) xdr_commit3args;
320 _xdr_result = (xdrproc_t) xdr_commit3res;
321 local = (char *(*)(char *, struct svc_req *)) nfsproc3_commit_3_svc;
325 async_rpc_send_failure(connection, xid, PROC_UNAVAIL);
329 /* Decode incoming message */
330 memset ((char *)&argument, 0, sizeof (argument));
332 xdrmem_create(&xdr_in, (char *)msg_buf, msg_len, XDR_DECODE);
334 printf("Call XDR: ");
335 for (i = 0; i < msg_len; i++) {
336 printf("%02x ", (uint8_t)msg_buf[i]);
339 if (!_xdr_argument(&xdr_in, (caddr_t)&argument)) {
340 async_rpc_send_failure(connection, xid, GARBAGE_ARGS);
341 fprintf(stderr, "RPC decode error!\n");
345 /* Perform the call. */
346 result = (*local)((char *)&argument, rqstp);
348 /* Encode result and send reply. */
349 static char reply_buf[MAX_RPC_MSGSIZE];
351 xdrmem_create(&xdr_out, reply_buf, MAX_RPC_MSGSIZE, XDR_ENCODE);
352 if (result != NULL && !_xdr_result(&xdr_out, result)) {
353 async_rpc_send_failure(connection, xid, SYSTEM_ERR);
356 struct rpc_reply header;
357 header.xid = htonl(xid);
358 header.type = htonl(1); /* REPLY */
359 header.stat = htonl(MSG_ACCEPTED);
360 header.verf_flavor = 0;
362 header.accept_stat = 0;
364 gsize msg_size = xdr_out.x_ops->x_getpostn(&xdr_out);
365 printf("Have an RPC reply of size %zd bytes\n", msg_size);
366 uint32_t fragment = htonl((msg_size + sizeof(header)) | 0x80000000);
367 async_rpc_write(connection, (const char *)&fragment, sizeof(fragment));
368 async_rpc_write(connection, (const char *)&header, sizeof(header));
369 async_rpc_write(connection, reply_buf, msg_size);
370 g_io_channel_flush(connection->channel, NULL);
373 xdr_in.x_op = XDR_FREE;
374 if (!_xdr_argument(&xdr_in, (caddr_t)&argument)) {
375 fprintf (stderr, "%s", "unable to free arguments");
379 bluesky_flushd_invoke(fs);
384 /* Enhanced, asynchronous-friendly RPC layer. This is a replacement for the
385 * built-in sunrpc parsing and dispatch that will allow for processing multiple
386 * requests at the same time. */
387 static GMainContext *main_context;
388 static GMainLoop *main_loop;
390 static async_rpc_init()
392 main_context = g_main_context_new();
393 main_loop = g_main_loop_new(main_context, FALSE);
396 struct rpc_call_header {
410 /* Decode an RPC message and process it. Returns a boolean indicating whether
411 * the message could be processed; if false, an unrecoverable error occurred
412 * and the transport should be closed. */
413 static gboolean async_rpc_dispatch(RPCConnection *rpc)
416 GString *msg = rpc->msgbuf;
417 const char *buf = msg->str;
419 if (msg->len < sizeof(struct rpc_call_header)) {
420 fprintf(stderr, "Short RPC message: only %zd bytes!\n", msg->len);
424 struct rpc_call_header *header = (struct rpc_call_header *)(msg->str);
425 uint32_t xid = ntohl(header->xid);
427 if (ntohl(header->mtype) != 0) {
428 /* Not an RPC call */
432 if (ntohl(header->rpcvers) != 2) {
434 } else if (ntohl(header->prog) != NFS_PROGRAM) {
435 async_rpc_send_failure(rpc, xid, PROG_UNAVAIL);
437 } else if (ntohl(header->vers) != NFS_V3) {
438 /* FIXME: Should be PROG_MISMATCH */
439 async_rpc_send_failure(rpc, xid, PROG_UNAVAIL);
443 uint32_t proc = ntohl(header->proc);
445 /* Next, skip over authentication headers. */
446 buf += sizeof(struct rpc_call_header);
447 for (i = 0; i < 2; i++) {
448 struct rpc_auth *auth = (struct rpc_auth *)buf;
449 if (buf - msg->str + sizeof(struct rpc_auth) > msg->len)
452 gsize authsize = ntohl(auth->len) + sizeof(struct rpc_auth);
453 if (authsize > MAX_RPC_MSGSIZE)
459 if (buf - msg->str > msg->len)
462 printf("Dispatching RPC procedure %d...\n", proc);
465 req.rq_prog = ntohl(header->prog);
466 req.rq_vers = ntohl(header->vers);
467 req.rq_proc = ntohl(header->proc);
468 req.rq_cred.oa_flavor = 0;
469 req.rq_cred.oa_base = NULL;
470 req.rq_cred.oa_length = 0;
471 req.rq_clntcred = NULL;
474 nfs_program_3(&req, rpc, ntohl(header->xid), buf,
475 (msg->str + msg->len) - buf);
480 /* Write the given data to the RPC socket. */
481 static void async_rpc_write(RPCConnection *rpc,
482 const char *buf, gsize len)
486 switch (g_io_channel_write_chars(rpc->channel, buf, len,
488 case G_IO_STATUS_ERROR:
489 case G_IO_STATUS_EOF:
490 case G_IO_STATUS_AGAIN:
491 fprintf(stderr, "Error writing to socket!\n");
493 case G_IO_STATUS_NORMAL:
500 // g_io_channel_flush(rpc->channel, NULL);
503 static gboolean async_rpc_do_read(GIOChannel *channel,
504 GIOCondition condition,
507 RPCConnection *rpc = (RPCConnection *)data;
509 gsize bytes_to_read = 0; /* Number of bytes to attempt to read. */
511 /* If we have not yet read in the fragment header, do that first. This is
512 * 4 bytes that indicates the number of bytes in the message to follow
513 * (with the high bit set if this is the last fragment making up the
515 if (rpc->frag_len == 0) {
516 bytes_to_read = 4 - rpc->frag_hdr_bytes;
518 bytes_to_read = rpc->frag_len & 0x7fffffff;
521 if (bytes_to_read > MAX_RPC_MSGSIZE
522 || rpc->msgbuf->len + bytes_to_read > MAX_RPC_MSGSIZE)
524 fprintf(stderr, "Excessive fragment size for RPC: %zd bytes\n",
526 g_io_channel_shutdown(rpc->channel, TRUE, NULL);
530 gsize bytes_read = 0;
531 g_string_set_size(rpc->msgbuf, rpc->msgbuf->len + bytes_to_read);
532 char *buf = &rpc->msgbuf->str[rpc->msgbuf->len - bytes_to_read];
533 switch (g_io_channel_read_chars(rpc->channel, buf,
534 bytes_to_read, &bytes_read, NULL)) {
535 case G_IO_STATUS_NORMAL:
537 case G_IO_STATUS_AGAIN:
539 case G_IO_STATUS_EOF:
540 if (bytes_read == bytes_to_read)
542 /* else fall through */
543 case G_IO_STATUS_ERROR:
544 fprintf(stderr, "Unexpected error or end of file on RPC stream %d!\n",
545 g_io_channel_unix_get_fd(rpc->channel));
546 g_io_channel_shutdown(rpc->channel, TRUE, NULL);
550 g_assert(bytes_read >= 0 && bytes_read <= bytes_to_read);
552 g_string_set_size(rpc->msgbuf,
553 rpc->msgbuf->len - (bytes_to_read - bytes_read));
555 if (rpc->frag_len == 0) {
556 /* Handle reading in the fragment header. If we've read the complete
557 * header, store the fragment size. */
558 rpc->frag_hdr_bytes += bytes_read;
559 if (rpc->frag_hdr_bytes == 4) {
560 memcpy((char *)&rpc->frag_len,
561 &rpc->msgbuf->str[rpc->msgbuf->len - 4], 4);
562 rpc->frag_len = ntohl(rpc->frag_len);
563 g_string_set_size(rpc->msgbuf, rpc->msgbuf->len - 4);
564 rpc->frag_hdr_bytes = 0;
565 g_print("RPC fragment header: %08x\n", rpc->frag_len);
568 /* We were reading in the fragment body. */
569 rpc->frag_len -= bytes_read;
571 if (rpc->frag_len = 0x80000000) {
572 /* We have a complete message since this was the last fragment and
573 * there are no more bytes in it. Dispatch the message. */
574 g_print("Complete RPC message: %zd bytes\n", rpc->msgbuf->len);
575 if (!async_rpc_dispatch(rpc)) {
576 fprintf(stderr, "Invalid RPC message, closing channel\n");
577 g_io_channel_shutdown(rpc->channel, TRUE, NULL);
581 g_string_set_size(rpc->msgbuf, 0);
588 static gboolean async_rpc_do_accept(GIOChannel *channel,
589 GIOCondition condition,
592 int fd = g_io_channel_unix_get_fd(channel);
593 struct sockaddr_in addr;
594 socklen_t addrlen = sizeof(addr);
596 g_print("Received new connection on fd %d!\n", fd);
597 int nfd = accept(fd, (struct sockaddr *)&addr, &addrlen);
599 fprintf(stderr, "Error accepting connection: %m\n");
603 RPCConnection *rpc = g_new0(RPCConnection, 1);
604 rpc->channel = g_io_channel_unix_new(nfd);
605 rpc->msgbuf = g_string_new("");
606 g_io_channel_set_encoding(rpc->channel, NULL, NULL);
607 GSource *source = g_io_create_watch(rpc->channel, G_IO_IN);
608 g_source_set_callback(source, (GSourceFunc)async_rpc_do_read,
610 g_source_attach(source, main_context);
611 g_source_unref(source);
616 static async_rpc_register_listening(int fd)
618 GIOChannel *channel = g_io_channel_unix_new(fd);
619 g_io_channel_set_encoding(channel, NULL, NULL);
620 GSource *source = g_io_create_watch(channel, G_IO_IN);
621 g_source_set_callback(source, (GSourceFunc)async_rpc_do_accept,
623 g_source_attach(source, main_context);
624 g_source_unref(source);
627 static gpointer async_rpc_run(gpointer data)
629 g_print("Starting NFS main loop...\n");
630 g_main_loop_run(main_loop);
640 pmap_unset (MOUNT_PROGRAM, MOUNT_V3);
642 transp = svcudp_create(RPC_ANYSOCK);
643 if (transp == NULL) {
644 fprintf(stderr, "%s", "cannot create udp service.");
647 if (!svc_register(transp, MOUNT_PROGRAM, MOUNT_V3, mount_program_3, IPPROTO_UDP)) {
648 fprintf(stderr, "%s", "unable to register (MOUNT_PROGRAM, MOUNT_V3, udp).");
652 transp = svctcp_create(RPC_ANYSOCK, 0, 0);
653 if (transp == NULL) {
654 fprintf(stderr, "%s", "cannot create tcp service.");
657 if (!svc_register(transp, MOUNT_PROGRAM, MOUNT_V3, mount_program_3, IPPROTO_TCP)) {
658 fprintf(stderr, "%s", "unable to register (MOUNT_PROGRAM, MOUNT_V3, tcp).");
662 /* NFS protocol (version 3) */
663 pmap_unset (NFS_PROGRAM, NFS_V3);
665 int fd = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP);
667 fprintf(stderr, "Unable to create NFS TCP socket: %m\n");
672 setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, (char *)&n, sizeof(n));
674 struct sockaddr_in addr;
675 addr.sin_family = AF_INET;
676 addr.sin_port = htons(NFS_SERVICE_PORT);
677 addr.sin_addr.s_addr = INADDR_ANY;
678 if (bind(fd, (struct sockaddr *)&addr, sizeof(addr)) < 0) {
679 fprintf(stderr, "Unable to bind to NFS TCP address: %m\n");
683 if (listen(fd, SOMAXCONN) < 0) {
684 fprintf(stderr, "Unable to listen on NFS TCP socket: %m\n");
688 if (!pmap_set(NFS_PROGRAM, NFS_V3, IPPROTO_TCP, NFS_SERVICE_PORT)) {
689 fprintf(stderr, "Could not register NFS RPC service!\n");
693 async_rpc_register_listening(fd);
695 g_thread_create(async_rpc_run, NULL, TRUE, NULL);