Recycle client instances in the connector

src/connector.c

@@ -23,6 +23,8 @@
 
 #define MAX_MSGSIZE 1024
 
+typedef struct client_instance client_instance_t;
+
 struct client_instance {
 	/* For clients hashtable */
 	UT_hash_handle hh;
@@ -34,8 +36,8 @@ struct client_instance {
 	int ref;
 
 	/* For dead_clients list */
-	struct client_instance *next;
-	struct client_instance *prev;
+	client_instance_t *next;
+	client_instance_t *prev;
 
 	struct sockaddr address;
 	char address_name[INET6_ADDRSTRLEN];
@@ -49,8 +51,6 @@ struct client_instance {
 	bool passthrough;
 };
 
-typedef struct client_instance client_instance_t;
-
 struct sender_send {
 	struct sender_send *next;
 	struct sender_send *prev;
@@ -81,6 +81,8 @@ struct connector_data {
 	client_instance_t *clients;
 	/* Linked list of dead clients no longer in use but may still have references */
 	client_instance_t *dead_clients;
+	/* Linked list of client structures we can reuse */
+	client_instance_t *recycled_clients;
 
 	int clients_generated;
 	int dead_generated;
@@ -120,6 +122,41 @@ static void dec_instance_ref(cdata_t *cdata, client_instance_t *client)
 	ck_wunlock(&cdata->lock);
 }
 
+/* Recruit a client structure from a recycled one if available, creating a
+ * new structure only if we have none to reuse. */
+static client_instance_t *recruit_client(cdata_t *cdata)
+{
+	client_instance_t *client = NULL;
+
+	ck_wlock(&cdata->lock);
+	if (cdata->recycled_clients) {
+		client = cdata->recycled_clients;
+		DL_DELETE(cdata->recycled_clients, client);
+	} else
+		cdata->clients_generated++;
+	ck_wunlock(&cdata->lock);
+
+	if (!client) {
+		LOGDEBUG("Connector created new client instance");
+		client = ckzalloc(sizeof(client_instance_t));
+	} else
+		LOGDEBUG("Connector recycled client instance");
+	return client;
+}
+
+static void __recycle_client(cdata_t *cdata, client_instance_t *client)
+{
+	memset(client, 0, sizeof(client_instance_t));
+	DL_APPEND(cdata->recycled_clients, client);
+}
+
+static void recycle_client(cdata_t *cdata, client_instance_t *client)
+{
+	ck_wlock(&cdata->lock);
+	__recycle_client(cdata, client);
+	ck_wunlock(&cdata->lock);
+}
+
 /* Accepts incoming connections on the server socket and generates client
  * instances */
 static int accept_client(cdata_t *cdata, const int epfd, const uint64_t server)
@@ -140,7 +177,7 @@ static int accept_client(cdata_t *cdata, const int epfd, const uint64_t server)
 	}
 
 	sockd = cdata->serverfd[server];
-	client = ckzalloc(sizeof(client_instance_t));
+	client = recruit_client(cdata);
 	client->server = server;
 	address_len = sizeof(client->address);
 	fd = accept(sockd, &client->address, &address_len);
@@ -152,7 +189,7 @@ static int accept_client(cdata_t *cdata, const int epfd, const uint64_t server)
 			return 0;
 		}
 		LOGERR("Failed to accept on socket %d in acceptor", sockd);
-		dealloc(client);
+		recycle_client(cdata, client);
 		return -1;
 	}
 
@@ -174,7 +211,7 @@ static int accept_client(cdata_t *cdata, const int epfd, const uint64_t server)
 			LOGWARNING("Unknown INET type for client %d on socket %d",
 				   cdata->nfds, fd);
 			Close(fd);
-			free(client);
+			recycle_client(cdata, client);
 			return 0;
 	}
 
@@ -189,7 +226,7 @@ static int accept_client(cdata_t *cdata, const int epfd, const uint64_t server)
 	event.events = EPOLLIN;
 	if (unlikely(epoll_ctl(epfd, EPOLL_CTL_ADD, fd, &event) < 0)) {
 		LOGERR("Failed to epoll_ctl add in accept_client");
-		free(client);
+		recycle_client(cdata, client);
 		return 0;
 	}
 
@@ -199,7 +236,6 @@ static int accept_client(cdata_t *cdata, const int epfd, const uint64_t server)
 	__inc_instance_ref(client);
 
 	ck_wlock(&cdata->lock);
-	cdata->clients_generated++;
 	client->id = cdata->client_id++;
 	HASH_ADD_I64(cdata->clients, id, client);
 	cdata->nfds++;
@@ -263,8 +299,8 @@ static int invalidate_client(ckpool_t *ckp, cdata_t *cdata, client_instance_t *c
 	DL_FOREACH_SAFE(cdata->dead_clients, client, tmp) {
 		if (!client->ref) {
 			DL_DELETE(cdata->dead_clients, client);
-			LOGINFO("Connector discarding client %"PRId64, client->id);
-			dealloc(client);
+			LOGINFO("Connector recycling client %"PRId64, client->id);
+			__recycle_client(cdata, client);
 		}
 	}
 	ck_wunlock(&cdata->lock);