Fix getfd function to really send a full struct msghdr returning a new fd to copy sockets across processes

src/ckpool.c

@@ -173,6 +173,39 @@ static void broadcast_proc(ckpool_t *ckp, const char *buf)
 	}
 }
 
+static int send_procmsg(proc_instance_t *pi, const char *buf)
+{
+	char *path = pi->us.path;
+	int ret = -1;
+	int sockd;
+
+	if (unlikely(!path || !strlen(path))) {
+		LOGERR("Attempted to send message %s to null path in send_proc", buf ? buf : "");
+		goto out;
+	}
+	if (unlikely(!buf || !strlen(buf))) {
+		LOGERR("Attempted to send null message to socket %s in send_proc", path);
+		goto out;
+	}
+	if (unlikely(kill(pi->pid, 0))) {
+		LOGALERT("Attempting to send message %s to dead process %s", buf, pi->processname);
+		goto out;
+	}
+	sockd = open_unix_client(path);
+	if (unlikely(sockd < 0)) {
+		LOGWARNING("Failed to open socket %s in send_recv_proc", path);
+		goto out;
+	}
+	if (unlikely(!send_unix_msg(sockd, buf)))
+		LOGWARNING("Failed to send %s to socket %s", buf, path);
+	else
+		ret = sockd;
+out:
+	if (unlikely(ret == -1))
+		LOGERR("Failure in send_procmsg");
+	return ret;
+}
+
 /* Listen for incoming global requests. Always returns a response if possible */
 static void *listener(void *arg)
 {
@@ -217,15 +250,20 @@ retry:
 			send_unix_msg(sockd, "success");
 		}
 	} else if (cmdmatch(buf, "getfd")) {
-		char *msg;
-
-		msg = send_recv_proc(ckp->connector, "getfd");
-		if (!msg)
-			LOGWARNING("Failed to receive fd data from connector");
-		else {
-			send_unix_data(sockd, msg, sizeof(struct msghdr));
-			free(msg);
-		}
+		int connfd = send_procmsg(ckp->connector, "getfd");
+
+		if (connfd > 0) {
+			int newfd = get_fd(connfd);
+
+			if (newfd > 0) {
+				LOGDEBUG("Sending new fd %d", newfd);
+				send_fd(newfd, sockd);
+				close(newfd);
+			} else
+				LOGWARNING("Failed to get_fd");
+			close(connfd);
+		} else
+			LOGWARNING("Failed to send_procmsg to connector");
 	} else {
 		LOGINFO("Listener received unhandled message: %s", buf);
 		send_unix_msg(sockd, "unknown");

src/ckpool.h

@@ -13,6 +13,8 @@
 #include "config.h"
 
 #include <sys/file.h>
+#include <sys/socket.h>
+#include <sys/types.h>
 
 #include "libckpool.h"
 #include "uthash.h"

src/connector.c

@@ -458,31 +458,6 @@ static client_instance_t *client_by_id(conn_instance_t *ci, int id)
 	return client;
 }
 
-static void send_fd(int fd, int sockd)
-{
-	struct cmsghdr cmptr;
-	struct iovec iov[1];
-	struct msghdr msg;
-	char buf[2];
-	int *cd;
-
-	memset(&cmptr, 0, sizeof(struct cmsghdr));
-	iov[0].iov_base = buf;
-	iov[0].iov_len = 2;
-	msg.msg_iov = iov;
-	msg.msg_iovlen = 1;
-	msg.msg_name = NULL;
-	msg.msg_namelen = 0;
-	cmptr.cmsg_level = SOL_SOCKET;
-	cmptr.cmsg_type = SCM_RIGHTS;
-	cmptr.cmsg_len = CMSG_LEN(sizeof(int));
-	cd = (int *)CMSG_DATA(&cmptr);
-	*cd = fd;
-	buf[1] = 0;
-	buf[0] = 0;
-	send_unix_data(sockd, &msg, sizeof(struct msghdr));
-}
-
 static int connector_loop(proc_instance_t *pi, conn_instance_t *ci)
 {
 	int sockd = -1, client_id, ret = 0, selret;

src/libckpool.c

@@ -714,16 +714,21 @@ int write_length(int sockd, const void *buf, int len)
 	return ofs;
 }
 
-bool _send_unix_data(int sockd, void *buf, uint32_t len, const char *file, const char *func, const int line)
+bool _send_unix_msg(int sockd, const char *buf, const char *file, const char *func, const int line)
 {
+	uint32_t msglen, len;
 	bool retval = false;
-	uint32_t msglen;
 	int ret;
 
 	if (unlikely(!buf)) {
 		LOGWARNING("Null message sent to send_unix_msg");
 		goto out;
 	}
+	len = strlen(buf);
+	if (unlikely(!len)) {
+		LOGWARNING("Zero length message sent to send_unix_msg");
+		goto out;
+	}
 	msglen = htole32(len);
 	ret = wait_write_select(sockd, 5);
 	if (unlikely(ret < 1)) {
@@ -747,56 +752,125 @@ bool _send_unix_data(int sockd, void *buf, uint32_t len, const char *file, const
 	}
 	retval = true;
 out:
+	shutdown(sockd, SHUT_WR);
 	if (unlikely(!retval))
 		LOGERR("Failure in send_unix_msg from %s %s:%d", file, func, line);
 	return retval;
 }
 
-
-bool _send_unix_msg(int sockd, const char *buf, const char *file, const char *func, const int line)
+bool _send_unix_data(int sockd, const struct msghdr *msg, const char *file, const char *func, const int line)
 {
-	uint32_t msglen, len;
 	bool retval = false;
 	int ret;
 
-	if (unlikely(!buf)) {
-		LOGWARNING("Null message sent to send_unix_msg");
+	if (unlikely(!msg)) {
+		LOGWARNING("Null message sent to send_unix_data");
 		goto out;
 	}
-	len = strlen(buf);
-	if (unlikely(!len)) {
-		LOGWARNING("Zero length message sent to send_unix_msg");
-		goto out;
-	}
-	msglen = htole32(len);
 	ret = wait_write_select(sockd, 5);
 	if (unlikely(ret < 1)) {
-		LOGERR("Select1 failed in send_unix_msg");
+		LOGERR("Select1 failed in send_unix_data");
 		goto out;
 	}
-	ret = write_length(sockd, &msglen, 4);
-	if (unlikely(ret < 4)) {
-		LOGERR("Failed to write 4 byte length in send_unix_msg");
+	ret = sendmsg(sockd, msg, 0);
+	if (unlikely(ret < 1)) {
+		LOGERR("Failed to send in send_unix_data");
 		goto out;
 	}
-	ret = wait_write_select(sockd, 5);
+	retval = true;
+out:
+	shutdown(sockd, SHUT_WR);
+	if (unlikely(!retval))
+		LOGERR("Failure in send_unix_data from %s %s:%d", file, func, line);
+	return retval;
+}
+
+bool _recv_unix_data(int sockd, struct msghdr *msg, const char *file, const char *func, const int line)
+{
+	bool retval = false;
+	int ret;
+
+	ret = wait_read_select(sockd, 5);
 	if (unlikely(ret < 1)) {
-		LOGERR("Select2 failed in send_unix_msg");
+		LOGERR("Select1 failed in recv_unix_data");
 		goto out;
 	}
-	ret = write_length(sockd, buf, len);
+	ret = recvmsg(sockd, msg, MSG_WAITALL);
 	if (unlikely(ret < 0)) {
-		LOGERR("Failed to write %d bytes in send_unix_msg", len);
+		LOGERR("Failed to recv in recv_unix_data");
 		goto out;
 	}
 	retval = true;
 out:
-	shutdown(sockd, SHUT_WR);
+	shutdown(sockd, SHUT_RD);
 	if (unlikely(!retval))
-		LOGERR("Failure in send_unix_msg from %s %s:%d", file, func, line);
+		LOGERR("Failure in recv_unix_data from %s %s:%d", file, func, line);
 	return retval;
 }
 
+#define CONTROLLLEN CMSG_LEN(sizeof(int))
+#define MAXLINE 4096
+
+/* Send a msghdr containing fd via the unix socket sockd */
+bool _send_fd(int fd, int sockd, const char *file, const char *func, const int line)
+{
+	struct cmsghdr *cmptr = ckzalloc(CONTROLLLEN);
+	struct iovec iov[1];
+	struct msghdr msg;
+	char buf[2];
+	bool ret;
+
+	memset(&msg, 0, sizeof(struct msghdr));
+	iov[0].iov_base = buf;
+	iov[0].iov_len = 2;
+	msg.msg_iov = iov;
+	msg.msg_iovlen = 1;
+	msg.msg_name = NULL;
+	msg.msg_namelen = 0;
+	msg.msg_controllen = CONTROLLLEN;
+	msg.msg_control = cmptr;
+	cmptr->cmsg_level = SOL_SOCKET;
+	cmptr->cmsg_type = SCM_RIGHTS;
+	cmptr->cmsg_len = CONTROLLLEN;
+	*(int *)CMSG_DATA(cmptr) = fd;
+	buf[1] = 0;
+	buf[0] = 0;
+	ret = send_unix_data(sockd, &msg);
+	free(cmptr);
+	if (!ret)
+		LOGERR("Failed to send_unix_data in send_fd from %s %s:%d", file, func, line);
+	return ret;
+}
+
+/* Receive an fd by reading a msghdr from the unix socket sockd */
+int _get_fd(int sockd, const char *file, const char *func, const int line)
+{
+	int newfd = -1;
+	char buf[MAXLINE];
+	struct iovec iov[1];
+	struct msghdr msg;
+	struct cmsghdr *cmptr = ckzalloc(CONTROLLLEN);
+
+	memset(&msg, 0, sizeof(struct msghdr));
+	iov[0].iov_base = buf;
+	iov[0].iov_len = sizeof(buf);
+	msg.msg_iov = iov;
+	msg.msg_name = NULL;
+	msg.msg_namelen = 0;
+	msg.msg_control = cmptr;
+	msg.msg_controllen = CONTROLLLEN;
+	if (!recv_unix_data(sockd, &msg)) {
+		LOGERR("Failed to recv_unix_data in get_fd from %s %s:%d", file, func, line);
+		goto out;
+	}
+out:
+	close(sockd);
+	newfd = *(int *)CMSG_DATA(cmptr);
+	free(cmptr);
+	return newfd;
+}
+
+
 /* Extracts a string value from a json array with error checking. To be used
  * when the value of the string returned is only examined and not to be stored.
  * See json_array_string below */

src/libckpool.h

@@ -32,6 +32,9 @@
 # include <sys/endian.h>
 #endif
 
+#include <sys/socket.h>
+#include <sys/types.h>
+
 #include "utlist.h"
 
 #ifndef bswap_16
@@ -389,8 +392,14 @@ int wait_write_select(int sockd, int timeout);
 int write_length(int sockd, const void *buf, int len);
 bool _send_unix_msg(int sockd, const char *buf, const char *file, const char *func, const int line);
 #define send_unix_msg(sockd, buf) _send_unix_msg(sockd, buf, __FILE__, __func__, __LINE__)
-bool _send_unix_data(int sockd, void *buf, uint32_t len, const char *file, const char *func, const int line);
-#define send_unix_data(sockd, buf, len) _send_unix_data(sockd, buf, len, __FILE__, __func__, __LINE__)
+bool _send_unix_data(int sockd, const struct msghdr *msg, const char *file, const char *func, const int line);
+#define send_unix_data(sockd, msg) _send_unix_data(sockd, msg, __FILE__, __func__, __LINE__)
+bool _recv_unix_data(int sockd, struct msghdr *msg, const char *file, const char *func, const int line);
+#define recv_unix_data(sockd, msg) _recv_unix_data(sockd, msg, __FILE__, __func__, __LINE__)
+bool _send_fd(int fd, int sockd, const char *file, const char *func, const int line);
+#define send_fd(fd, sockd) _send_fd(fd, sockd, __FILE__, __func__, __LINE__)
+int _get_fd(int sockd, const char *file, const char *func, const int line);
+#define get_fd(sockd) _get_fd(sockd, __FILE__, __func__, __LINE__)
 
 const char *__json_array_string(json_t *val, unsigned int entry);
 char *json_array_string(json_t *val, unsigned int entry);