From a832b64006b6e8b8f3127dd4541d9f2f88f7a88b Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Thu, 7 Aug 2014 19:26:06 +1000 Subject: [PATCH] Fix getfd function to really send a full struct msghdr returning a new fd to copy sockets across processes --- src/ckpool.c | 56 ++++++++++++++++++---- src/ckpool.h | 2 + src/connector.c | 25 ---------- src/libckpool.c | 120 ++++++++++++++++++++++++++++++++++++++---------- src/libckpool.h | 13 +++++- 5 files changed, 157 insertions(+), 59 deletions(-) diff --git a/src/ckpool.c b/src/ckpool.c index 4dcc607b..f9d3511e 100644 --- a/src/ckpool.c +++ b/src/ckpool.c @@ -173,6 +173,39 @@ static void broadcast_proc(ckpool_t *ckp, const char *buf) } } +static int send_procmsg(proc_instance_t *pi, const char *buf) +{ + char *path = pi->us.path; + int ret = -1; + int sockd; + + if (unlikely(!path || !strlen(path))) { + LOGERR("Attempted to send message %s to null path in send_proc", buf ? buf : ""); + goto out; + } + if (unlikely(!buf || !strlen(buf))) { + LOGERR("Attempted to send null message to socket %s in send_proc", path); + goto out; + } + if (unlikely(kill(pi->pid, 0))) { + LOGALERT("Attempting to send message %s to dead process %s", buf, pi->processname); + goto out; + } + sockd = open_unix_client(path); + if (unlikely(sockd < 0)) { + LOGWARNING("Failed to open socket %s in send_recv_proc", path); + goto out; + } + if (unlikely(!send_unix_msg(sockd, buf))) + LOGWARNING("Failed to send %s to socket %s", buf, path); + else + ret = sockd; +out: + if (unlikely(ret == -1)) + LOGERR("Failure in send_procmsg"); + return ret; +} + /* Listen for incoming global requests. Always returns a response if possible */ static void *listener(void *arg) { @@ -217,15 +250,20 @@ retry: send_unix_msg(sockd, "success"); } } else if (cmdmatch(buf, "getfd")) { - char *msg; - - msg = send_recv_proc(ckp->connector, "getfd"); - if (!msg) - LOGWARNING("Failed to receive fd data from connector"); - else { - send_unix_data(sockd, msg, sizeof(struct msghdr)); - free(msg); - } + int connfd = send_procmsg(ckp->connector, "getfd"); + + if (connfd > 0) { + int newfd = get_fd(connfd); + + if (newfd > 0) { + LOGDEBUG("Sending new fd %d", newfd); + send_fd(newfd, sockd); + close(newfd); + } else + LOGWARNING("Failed to get_fd"); + close(connfd); + } else + LOGWARNING("Failed to send_procmsg to connector"); } else { LOGINFO("Listener received unhandled message: %s", buf); send_unix_msg(sockd, "unknown"); diff --git a/src/ckpool.h b/src/ckpool.h index 94d64e13..2d28aa0b 100644 --- a/src/ckpool.h +++ b/src/ckpool.h @@ -13,6 +13,8 @@ #include "config.h" #include +#include +#include #include "libckpool.h" #include "uthash.h" diff --git a/src/connector.c b/src/connector.c index 676f23dc..0ad53b8b 100644 --- a/src/connector.c +++ b/src/connector.c @@ -458,31 +458,6 @@ static client_instance_t *client_by_id(conn_instance_t *ci, int id) return client; } -static void send_fd(int fd, int sockd) -{ - struct cmsghdr cmptr; - struct iovec iov[1]; - struct msghdr msg; - char buf[2]; - int *cd; - - memset(&cmptr, 0, sizeof(struct cmsghdr)); - iov[0].iov_base = buf; - iov[0].iov_len = 2; - msg.msg_iov = iov; - msg.msg_iovlen = 1; - msg.msg_name = NULL; - msg.msg_namelen = 0; - cmptr.cmsg_level = SOL_SOCKET; - cmptr.cmsg_type = SCM_RIGHTS; - cmptr.cmsg_len = CMSG_LEN(sizeof(int)); - cd = (int *)CMSG_DATA(&cmptr); - *cd = fd; - buf[1] = 0; - buf[0] = 0; - send_unix_data(sockd, &msg, sizeof(struct msghdr)); -} - static int connector_loop(proc_instance_t *pi, conn_instance_t *ci) { int sockd = -1, client_id, ret = 0, selret; diff --git a/src/libckpool.c b/src/libckpool.c index d4fbaa37..b9d1e4d3 100644 --- a/src/libckpool.c +++ b/src/libckpool.c @@ -714,16 +714,21 @@ int write_length(int sockd, const void *buf, int len) return ofs; } -bool _send_unix_data(int sockd, void *buf, uint32_t len, const char *file, const char *func, const int line) +bool _send_unix_msg(int sockd, const char *buf, const char *file, const char *func, const int line) { + uint32_t msglen, len; bool retval = false; - uint32_t msglen; int ret; if (unlikely(!buf)) { LOGWARNING("Null message sent to send_unix_msg"); goto out; } + len = strlen(buf); + if (unlikely(!len)) { + LOGWARNING("Zero length message sent to send_unix_msg"); + goto out; + } msglen = htole32(len); ret = wait_write_select(sockd, 5); if (unlikely(ret < 1)) { @@ -747,56 +752,125 @@ bool _send_unix_data(int sockd, void *buf, uint32_t len, const char *file, const } retval = true; out: + shutdown(sockd, SHUT_WR); if (unlikely(!retval)) LOGERR("Failure in send_unix_msg from %s %s:%d", file, func, line); return retval; } - -bool _send_unix_msg(int sockd, const char *buf, const char *file, const char *func, const int line) +bool _send_unix_data(int sockd, const struct msghdr *msg, const char *file, const char *func, const int line) { - uint32_t msglen, len; bool retval = false; int ret; - if (unlikely(!buf)) { - LOGWARNING("Null message sent to send_unix_msg"); + if (unlikely(!msg)) { + LOGWARNING("Null message sent to send_unix_data"); goto out; } - len = strlen(buf); - if (unlikely(!len)) { - LOGWARNING("Zero length message sent to send_unix_msg"); - goto out; - } - msglen = htole32(len); ret = wait_write_select(sockd, 5); if (unlikely(ret < 1)) { - LOGERR("Select1 failed in send_unix_msg"); + LOGERR("Select1 failed in send_unix_data"); goto out; } - ret = write_length(sockd, &msglen, 4); - if (unlikely(ret < 4)) { - LOGERR("Failed to write 4 byte length in send_unix_msg"); + ret = sendmsg(sockd, msg, 0); + if (unlikely(ret < 1)) { + LOGERR("Failed to send in send_unix_data"); goto out; } - ret = wait_write_select(sockd, 5); + retval = true; +out: + shutdown(sockd, SHUT_WR); + if (unlikely(!retval)) + LOGERR("Failure in send_unix_data from %s %s:%d", file, func, line); + return retval; +} + +bool _recv_unix_data(int sockd, struct msghdr *msg, const char *file, const char *func, const int line) +{ + bool retval = false; + int ret; + + ret = wait_read_select(sockd, 5); if (unlikely(ret < 1)) { - LOGERR("Select2 failed in send_unix_msg"); + LOGERR("Select1 failed in recv_unix_data"); goto out; } - ret = write_length(sockd, buf, len); + ret = recvmsg(sockd, msg, MSG_WAITALL); if (unlikely(ret < 0)) { - LOGERR("Failed to write %d bytes in send_unix_msg", len); + LOGERR("Failed to recv in recv_unix_data"); goto out; } retval = true; out: - shutdown(sockd, SHUT_WR); + shutdown(sockd, SHUT_RD); if (unlikely(!retval)) - LOGERR("Failure in send_unix_msg from %s %s:%d", file, func, line); + LOGERR("Failure in recv_unix_data from %s %s:%d", file, func, line); return retval; } +#define CONTROLLLEN CMSG_LEN(sizeof(int)) +#define MAXLINE 4096 + +/* Send a msghdr containing fd via the unix socket sockd */ +bool _send_fd(int fd, int sockd, const char *file, const char *func, const int line) +{ + struct cmsghdr *cmptr = ckzalloc(CONTROLLLEN); + struct iovec iov[1]; + struct msghdr msg; + char buf[2]; + bool ret; + + memset(&msg, 0, sizeof(struct msghdr)); + iov[0].iov_base = buf; + iov[0].iov_len = 2; + msg.msg_iov = iov; + msg.msg_iovlen = 1; + msg.msg_name = NULL; + msg.msg_namelen = 0; + msg.msg_controllen = CONTROLLLEN; + msg.msg_control = cmptr; + cmptr->cmsg_level = SOL_SOCKET; + cmptr->cmsg_type = SCM_RIGHTS; + cmptr->cmsg_len = CONTROLLLEN; + *(int *)CMSG_DATA(cmptr) = fd; + buf[1] = 0; + buf[0] = 0; + ret = send_unix_data(sockd, &msg); + free(cmptr); + if (!ret) + LOGERR("Failed to send_unix_data in send_fd from %s %s:%d", file, func, line); + return ret; +} + +/* Receive an fd by reading a msghdr from the unix socket sockd */ +int _get_fd(int sockd, const char *file, const char *func, const int line) +{ + int newfd = -1; + char buf[MAXLINE]; + struct iovec iov[1]; + struct msghdr msg; + struct cmsghdr *cmptr = ckzalloc(CONTROLLLEN); + + memset(&msg, 0, sizeof(struct msghdr)); + iov[0].iov_base = buf; + iov[0].iov_len = sizeof(buf); + msg.msg_iov = iov; + msg.msg_name = NULL; + msg.msg_namelen = 0; + msg.msg_control = cmptr; + msg.msg_controllen = CONTROLLLEN; + if (!recv_unix_data(sockd, &msg)) { + LOGERR("Failed to recv_unix_data in get_fd from %s %s:%d", file, func, line); + goto out; + } +out: + close(sockd); + newfd = *(int *)CMSG_DATA(cmptr); + free(cmptr); + return newfd; +} + + /* Extracts a string value from a json array with error checking. To be used * when the value of the string returned is only examined and not to be stored. * See json_array_string below */ diff --git a/src/libckpool.h b/src/libckpool.h index 62407f4f..c59b24f7 100644 --- a/src/libckpool.h +++ b/src/libckpool.h @@ -32,6 +32,9 @@ # include #endif +#include +#include + #include "utlist.h" #ifndef bswap_16 @@ -389,8 +392,14 @@ int wait_write_select(int sockd, int timeout); int write_length(int sockd, const void *buf, int len); bool _send_unix_msg(int sockd, const char *buf, const char *file, const char *func, const int line); #define send_unix_msg(sockd, buf) _send_unix_msg(sockd, buf, __FILE__, __func__, __LINE__) -bool _send_unix_data(int sockd, void *buf, uint32_t len, const char *file, const char *func, const int line); -#define send_unix_data(sockd, buf, len) _send_unix_data(sockd, buf, len, __FILE__, __func__, __LINE__) +bool _send_unix_data(int sockd, const struct msghdr *msg, const char *file, const char *func, const int line); +#define send_unix_data(sockd, msg) _send_unix_data(sockd, msg, __FILE__, __func__, __LINE__) +bool _recv_unix_data(int sockd, struct msghdr *msg, const char *file, const char *func, const int line); +#define recv_unix_data(sockd, msg) _recv_unix_data(sockd, msg, __FILE__, __func__, __LINE__) +bool _send_fd(int fd, int sockd, const char *file, const char *func, const int line); +#define send_fd(fd, sockd) _send_fd(fd, sockd, __FILE__, __func__, __LINE__) +int _get_fd(int sockd, const char *file, const char *func, const int line); +#define get_fd(sockd) _get_fd(sockd, __FILE__, __func__, __LINE__) const char *__json_array_string(json_t *val, unsigned int entry); char *json_array_string(json_t *val, unsigned int entry);