From da20ac7f3e9d2063dc9387c2955885974ed3b684 Mon Sep 17 00:00:00 2001 From: ckolivas Date: Wed, 22 Apr 2015 12:10:45 +1000 Subject: [PATCH 1/5] Disable user of nolinger on client sockets --- src/connector.c | 1 - 1 file changed, 1 deletion(-) diff --git a/src/connector.c b/src/connector.c index 43f268e5..5fc9fcec 100644 --- a/src/connector.c +++ b/src/connector.c @@ -219,7 +219,6 @@ static int accept_client(cdata_t *cdata, const int epfd, const uint64_t server) } keep_sockalive(fd); - nolinger_socket(fd); LOGINFO("Connected new client %d on socket %d to %d active clients from %s:%d", cdata->nfds, fd, no_clients, client->address_name, port); From 49ce58794930d4a4b58b9fed686c007a62076b7c Mon Sep 17 00:00:00 2001 From: ckolivas Date: Wed, 22 Apr 2015 14:17:52 +1000 Subject: [PATCH 2/5] nolinger client sockets when we're about to close them --- src/connector.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/connector.c b/src/connector.c index 5fc9fcec..ea1a0a45 100644 --- a/src/connector.c +++ b/src/connector.c @@ -257,7 +257,8 @@ static int drop_client(cdata_t *cdata, client_instance_t *client) if (fd != -1) { client_id = client->id; - epoll_ctl(cdata->epfd, EPOLL_CTL_DEL, client->fd, NULL); + epoll_ctl(cdata->epfd, EPOLL_CTL_DEL, fd, NULL); + nolinger_socket(fd); Close(client->fd); HASH_DEL(cdata->clients, client); DL_APPEND(cdata->dead_clients, client); From 8313628629ad452c4eb62961026aad1614e94645 Mon Sep 17 00:00:00 2001 From: ckolivas Date: Wed, 22 Apr 2015 14:54:00 +1000 Subject: [PATCH 3/5] Differentiate ready for writes from hangups in wait_write_select --- src/libckpool.c | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/libckpool.c b/src/libckpool.c index f20a8803..fee42439 100644 --- a/src/libckpool.c +++ b/src/libckpool.c @@ -983,14 +983,19 @@ out: int wait_write_select(int sockd, int timeout) { struct pollfd sfd; + int ret = -1; if (unlikely(sockd < 0)) - return -1; + goto out; sfd.fd = sockd; sfd.events = POLLOUT; sfd.revents = 0; timeout *= 1000; - return poll(&sfd, 1, timeout); + ret = poll(&sfd, 1, timeout); + if (ret && !(sfd.revents & POLLOUT)) + ret = -1; +out: + return ret; } int write_length(int sockd, const void *buf, int len) From 57e3aa83c6fff2fc5ddc89239ac48091c925fd71 Mon Sep 17 00:00:00 2001 From: ckolivas Date: Wed, 22 Apr 2015 14:55:49 +1000 Subject: [PATCH 4/5] Differentiate pollin from pollhup in wait_read_select --- src/libckpool.c | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/libckpool.c b/src/libckpool.c index fee42439..84e8ce8c 100644 --- a/src/libckpool.c +++ b/src/libckpool.c @@ -906,14 +906,19 @@ int wait_close(int sockd, int timeout) int wait_read_select(int sockd, int timeout) { struct pollfd sfd; + int ret = -1; if (unlikely(sockd < 0)) - return -1; + goto out; sfd.fd = sockd; sfd.events = POLLIN; sfd.revents = 0; timeout *= 1000; - return poll(&sfd, 1, timeout); + ret = poll(&sfd, 1, timeout); + if (ret && !(sfd.revents & POLLIN)) + ret = -1; +out: + return ret; } int read_length(int sockd, void *buf, int len) From 91d2aca51f3934cc8bf8ee133d652be2b99dec96 Mon Sep 17 00:00:00 2001 From: ckolivas Date: Wed, 22 Apr 2015 15:07:04 +1000 Subject: [PATCH 5/5] Detect pollhup in wait_read_select and not through recv fail conditions --- src/connector.c | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/src/connector.c b/src/connector.c index ea1a0a45..baad4c30 100644 --- a/src/connector.c +++ b/src/connector.c @@ -318,31 +318,26 @@ static void send_client(cdata_t *cdata, int64_t id, char *buf); /* Client is holding a reference count from being on the epoll list */ static void parse_client_msg(cdata_t *cdata, client_instance_t *client) { - int buflen, ret, selfail = 0; ckpool_t *ckp = cdata->ckp; char msg[PAGESIZE], *eol; + int buflen, ret; json_t *val; retry: - /* Select should always return positive after poll unless we have - * been disconnected. On retries, decdatade whether we should do further - * reads based on select readiness and only fail if we get an error. */ ret = wait_read_select(client->fd, 0); if (ret < 1) { - if (ret > selfail) + if (!ret) return; LOGINFO("Client fd %d disconnected - select fail with bufofs %d ret %d errno %d %s", client->fd, client->bufofs, ret, errno, ret && errno ? strerror(errno) : ""); invalidate_client(ckp, cdata, client); return; } - selfail = -1; buflen = PAGESIZE - client->bufofs; ret = recv(client->fd, client->buf + client->bufofs, buflen, 0); if (ret < 1) { - /* We should have something to read if called since poll set - * this fd's revents status so if there's nothing it means the - * client has disconnected. */ + if (!ret) + return; LOGINFO("Client fd %d disconnected - recv fail with bufofs %d ret %d errno %d %s", client->fd, client->bufofs, ret, errno, ret && errno ? strerror(errno) : ""); invalidate_client(ckp, cdata, client);