From f85089eeec55311c2e0ff3bd52e8d89f059c469e Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Fri, 7 Nov 2014 11:48:27 +1100 Subject: [PATCH 01/47] Remove EOS marker in userstats in preparation for moving from client to worker stats --- src/stratifier.c | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/src/stratifier.c b/src/stratifier.c index d905d081..9c4c1715 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -3173,9 +3173,9 @@ out: static void update_userstats(ckpool_t *ckp) { stratum_instance_t *client, *tmp; - json_t *val = NULL; char cdfield[64]; time_t now_t; + json_t *val; ts_t ts_now; if (++stats.userstats_cycle > 0x1f) @@ -3204,11 +3204,6 @@ static void update_userstats(ckpool_t *ckp) if (cycle_mask != stats.userstats_cycle) continue; - if (val) { - json_set_bool(val,"eos", false); - ckdbq_add(ckp, ID_USERSTATS, val); - val = NULL; - } elapsed = now_t - client->start_time; ghs1 = client->dsps1 * nonces; ghs5 = client->dsps5 * nonces; @@ -3230,12 +3225,8 @@ static void update_userstats(ckpool_t *ckp) "createcode", __func__, "createinet", ckp->serverurl); client->notified_idle = client->idle; - } - /* Mark the last userstats sent on this pass of stats with an end of - * stats marker. */ - if (val) { - json_set_bool(val,"eos", true); ckdbq_add(ckp, ID_USERSTATS, val); + val = NULL; } ck_runlock(&instance_lock); } From 93e0e55cf089c02181c40fa06236b6c72549db41 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Fri, 7 Nov 2014 12:23:24 +1100 Subject: [PATCH 02/47] Store start time and idle status per worker --- src/stratifier.c | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/src/stratifier.c b/src/stratifier.c index 9c4c1715..2fb02f92 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -247,8 +247,11 @@ struct worker_instance { double dsps60; double dsps1440; tv_t last_share; + time_t start_time; int mindiff; /* User chosen mindiff */ + + bool idle; }; /* Per client stratum instance == workers */ @@ -1738,11 +1741,14 @@ static user_instance_t *generate_user(ckpool_t *ckp, stratum_instance_t *client, /* Create one worker instance for combined data from workers of the * same name */ if (!client->worker_instance) { - client->worker_instance = ckzalloc(sizeof(worker_instance_t)); - client->worker_instance->workername = strdup(workername); - client->worker_instance->instance = instance; - DL_APPEND(instance->worker_instances, client->worker_instance); - read_workerstats(ckp, client->worker_instance); + worker_instance_t *worker = ckzalloc(sizeof(worker_instance_t)); + + worker->workername = strdup(workername); + worker->instance = instance; + DL_APPEND(instance->worker_instances, worker); + read_workerstats(ckp, worker); + worker->start_time = time(NULL); + client->worker_instance = worker; } DL_APPEND(instance->instances, client); ck_wunlock(&instance_lock); @@ -2030,6 +2036,7 @@ static void add_submit(ckpool_t *ckp, stratum_instance_t *client, int diff, bool decay_time(&worker->dsps60, diff, tdiff, 3600); decay_time(&worker->dsps1440, diff, tdiff, 86400); copy_tv(&worker->last_share, &now_t); + worker->idle = false; tdiff = sane_tdiff(&now_t, &instance->last_share); decay_time(&instance->dsps1, diff, tdiff, 60); @@ -3367,6 +3374,7 @@ static void *statsupdate(void *arg) decay_time(&worker->dsps5, 0, per_tdiff, 300); decay_time(&worker->dsps60, 0, per_tdiff, 3600); decay_time(&worker->dsps1440, 0, per_tdiff, 86400); + worker->idle = true; } ghs = worker->dsps1 * nonces; suffix_string(ghs, suffix1, 16, 0); From 5ccd3b53408566ece858a5803b63c002be80fa12 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Fri, 7 Nov 2014 12:43:11 +1100 Subject: [PATCH 03/47] Send workerstats to ckdb instead of client stats --- src/stratifier.c | 85 ++++++++++++++++++++++++------------------------ 1 file changed, 42 insertions(+), 43 deletions(-) diff --git a/src/stratifier.c b/src/stratifier.c index 2fb02f92..d6279b58 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -252,6 +252,7 @@ struct worker_instance { int mindiff; /* User chosen mindiff */ bool idle; + bool notified_idle; }; /* Per client stratum instance == workers */ @@ -286,7 +287,6 @@ struct stratum_instance { bool subscribed; bool authorised; bool idle; - bool notified_idle; int reject; /* Indicator that this client is having a run of rejects * or other problem and should be dropped lazily if * this is set to 2 */ @@ -344,7 +344,7 @@ static int gen_priority; #define ID_SHARES 3 #define ID_SHAREERR 4 #define ID_POOLSTATS 5 -#define ID_USERSTATS 6 +#define ID_WORKERSTATS 6 #define ID_BLOCK 7 #define ID_ADDRAUTH 8 #define ID_HEARTBEAT 9 @@ -356,7 +356,7 @@ static const char *ckdb_ids[] = { "shares", "shareerror", "poolstats", - "userstats", + "workerstats", "block", "addrauth", "heartbeat" @@ -3177,12 +3177,11 @@ out: /* Called every 20 seconds, we send the updated stats to ckdb of those users * who have gone 10 minutes between updates. This ends up staggering stats to * avoid floods of stat data coming at once. */ -static void update_userstats(ckpool_t *ckp) +static void update_workerstats(ckpool_t *ckp) { - stratum_instance_t *client, *tmp; + user_instance_t *user, *tmp; char cdfield[64]; time_t now_t; - json_t *val; ts_t ts_now; if (++stats.userstats_cycle > 0x1f) @@ -3193,47 +3192,47 @@ static void update_userstats(ckpool_t *ckp) now_t = ts_now.tv_sec; ck_rlock(&instance_lock); - HASH_ITER(hh, stratum_instances, client, tmp) { - double ghs1, ghs5, ghs60, ghs1440; + HASH_ITER(hh, user_instances, user, tmp) { + worker_instance_t *worker; uint8_t cycle_mask; - int elapsed; - if (!client->authorised) - continue; - - /* Send one lot of stats once the client is idle if they have submitted - * no shares in the last 10 minutes with the idle bool set. */ - if (client->idle && client->notified_idle) - continue; - /* Select clients using a mask to return each user's stats once + /* Select users using a mask to return each user's stats once * every ~10 minutes */ - cycle_mask = client->user_id & 0x1f; + cycle_mask = user->id & 0x1f; if (cycle_mask != stats.userstats_cycle) continue; - - elapsed = now_t - client->start_time; - ghs1 = client->dsps1 * nonces; - ghs5 = client->dsps5 * nonces; - ghs60 = client->dsps60 * nonces; - ghs1440 = client->dsps1440 * nonces; - JSON_CPACK(val, "{ss,sI,si,ss,ss,sf,sf,sf,sf,sb,ss,ss,ss,ss}", - "poolinstance", ckp->name, - "instanceid", client->id, - "elapsed", elapsed, - "username", client->user_instance->username, - "workername", client->workername, - "hashrate", ghs1, - "hashrate5m", ghs5, - "hashrate1hr", ghs60, - "hashrate24hr", ghs1440, - "idle", client->idle, - "createdate", cdfield, - "createby", "code", - "createcode", __func__, - "createinet", ckp->serverurl); - client->notified_idle = client->idle; - ckdbq_add(ckp, ID_USERSTATS, val); - val = NULL; + DL_FOREACH(user->worker_instances, worker) { + double ghs1, ghs5, ghs60, ghs1440; + int elapsed; + json_t *val; + + /* Send one lot of stats once the worker is idle if + * they have submitted no shares in the last 10 minutes + * with the idle bool set. */ + if (worker->idle && worker->notified_idle) + continue; + elapsed = now_t - worker->start_time; + ghs1 = worker->dsps1 * nonces; + ghs5 = worker->dsps5 * nonces; + ghs60 = worker->dsps60 * nonces; + ghs1440 = worker->dsps1440 * nonces; + JSON_CPACK(val, "{ss,si,ss,ss,sf,sf,sf,sf,sb,ss,ss,ss,ss}", + "poolinstance", ckp->name, + "elapsed", elapsed, + "username", user->username, + "workername", worker->workername, + "hashrate", ghs1, + "hashrate5m", ghs5, + "hashrate1hr", ghs60, + "hashrate24hr", ghs1440, + "idle", worker->idle, + "createdate", cdfield, + "createby", "code", + "createcode", __func__, + "createinet", ckp->serverurl); + worker->notified_idle = worker->idle; + ckdbq_add(ckp, ID_WORKERSTATS, val); + } } ck_runlock(&instance_lock); } @@ -3478,7 +3477,7 @@ static void *statsupdate(void *arg) for (i = 0; i < 3; i++) { cksleep_ms_r(&stats.last_update, 20000); cksleep_prepare_r(&stats.last_update); - update_userstats(ckp); + update_workerstats(ckp); mutex_lock(&stats_lock); stats.accounted_shares += stats.unaccounted_shares; From fe15f9384dd5cb7b7bd0ef904afd01671b2c4978 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Fri, 7 Nov 2014 12:58:55 +1100 Subject: [PATCH 04/47] Privatise all global variables within the generator to allow multiple ckpools in the future --- src/ckpool.h | 3 +++ src/generator.c | 23 ++++++++++++++++------- 2 files changed, 19 insertions(+), 7 deletions(-) diff --git a/src/ckpool.h b/src/ckpool.h index d246e97f..768dcaf4 100644 --- a/src/ckpool.h +++ b/src/ckpool.h @@ -172,6 +172,9 @@ struct ckpool_instance { char **proxyauth; char **proxypass; server_instance_t *btcdbackup; + + /* Private data for each process */ + void *data; }; #ifdef USE_CKDB diff --git a/src/generator.c b/src/generator.c index f9791257..d7e238e6 100644 --- a/src/generator.c +++ b/src/generator.c @@ -122,9 +122,13 @@ struct proxy_instance { typedef struct proxy_instance proxy_instance_t; -static int proxy_notify_id; // Globally increasing notify id +/* Private data for the generator */ +struct generator_data { + int proxy_notify_id; // Globally increasing notify id + ckmsgq_t *srvchk; // Server check message queue +}; -static ckmsgq_t *srvchk; // Server check message queue +typedef struct generator_data gdata_t; static bool server_alive(ckpool_t *ckp, server_instance_t *si, bool pinging) { @@ -238,6 +242,7 @@ static int gen_loop(proc_instance_t *pi) bool reconnecting = false; unixsock_t *us = &pi->us; ckpool_t *ckp = pi->ckp; + gdata_t *gdata = ckp->data; bool started = false; char *buf = NULL; connsock_t *cs; @@ -261,7 +266,7 @@ reconnect: } retry: - ckmsgq_add(srvchk, si); + ckmsgq_add(gdata->srvchk, si); do { selret = wait_read_select(us->sockd, 5); @@ -681,6 +686,7 @@ static bool parse_notify(proxy_instance_t *proxi, json_t *val) { const char *prev_hash, *bbversion, *nbit, *ntime; char *job_id, *coinbase1, *coinbase2; + gdata_t *gdata = proxi->ckp->data; bool clean, ret = false; notify_instance_t *ni; int merkles, i; @@ -740,7 +746,7 @@ static bool parse_notify(proxy_instance_t *proxi, json_t *val) ni->notify_time = time(NULL); mutex_lock(&proxi->notify_lock); - ni->id = proxy_notify_id++; + ni->id = gdata->proxy_notify_id++; HASH_ADD_INT(proxi->notify_instances, id, ni); proxi->current_notify = ni; mutex_unlock(&proxi->notify_lock); @@ -1448,6 +1454,7 @@ static int proxy_loop(proc_instance_t *pi) bool reconnecting = false; unixsock_t *us = &pi->us; ckpool_t *ckp = pi->ckp; + gdata_t *gdata = ckp->data; char *buf = NULL; reconnect: @@ -1474,7 +1481,7 @@ reconnect: } retry: - ckmsgq_add(srvchk, proxi->si); + ckmsgq_add(gdata->srvchk, proxi->si); do { selret = wait_read_select(us->sockd, 5); @@ -1725,11 +1732,13 @@ static void server_watchdog(ckpool_t *ckp, server_instance_t *cursi) int generator(proc_instance_t *pi) { ckpool_t *ckp = pi->ckp; + gdata_t *gdata; int ret; LOGWARNING("%s generator starting", ckp->name); - - srvchk = create_ckmsgq(ckp, "srvchk", &server_watchdog); + gdata = ckzalloc(sizeof(gdata_t)); + ckp->data = gdata; + gdata->srvchk = create_ckmsgq(ckp, "srvchk", &server_watchdog); if (ckp->proxy) ret = proxy_mode(ckp, pi); From 3ddef2d66fa9fb857bc8c6cb08f1b4406d808ebf Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Fri, 7 Nov 2014 14:27:25 +1100 Subject: [PATCH 05/47] Privatise all the connector specific data --- src/connector.c | 245 ++++++++++++++++++++++++------------------------ src/generator.c | 1 + 2 files changed, 126 insertions(+), 120 deletions(-) diff --git a/src/connector.c b/src/connector.c index 401f70a4..8c098b96 100644 --- a/src/connector.c +++ b/src/connector.c @@ -24,18 +24,6 @@ #define MAX_MSGSIZE 1024 #define SOI (sizeof(int)) -struct connector_instance { - cklock_t lock; - proc_instance_t *pi; - int serverfd; - int nfds; - bool accept; - pthread_t pth_sender; - pthread_t pth_receiver; -}; - -typedef struct connector_instance conn_instance_t; - struct client_instance { /* For clients hashtable */ UT_hash_handle hh; @@ -56,13 +44,6 @@ struct client_instance { typedef struct client_instance client_instance_t; -/* For the hashtable of all clients */ -static client_instance_t *clients; -/* Linked list of dead clients no longer in use but may still have references */ -static client_instance_t *dead_clients; - -static int64_t client_id = 1; - struct sender_send { struct sender_send *next; struct sender_send *prev; @@ -74,27 +55,48 @@ struct sender_send { typedef struct sender_send sender_send_t; -/* For the linked list of pending sends */ -static sender_send_t *sender_sends; -static sender_send_t *delayed_sends; +/* Private data for the connector */ +struct connector_data { + ckpool_t *ckp; + cklock_t lock; + proc_instance_t *pi; + int serverfd; + int nfds; + bool accept; + pthread_t pth_sender; + pthread_t pth_receiver; + + /* For the hashtable of all clients */ + client_instance_t *clients; + /* Linked list of dead clients no longer in use but may still have references */ + client_instance_t *dead_clients; + + int64_t client_id; + + /* For the linked list of pending sends */ + sender_send_t *sender_sends; + sender_send_t *delayed_sends; + + /* For protecting the pending sends list */ + pthread_mutex_t sender_lock; + pthread_cond_t sender_cond; +}; -/* For protecting the pending sends list */ -static pthread_mutex_t sender_lock; -static pthread_cond_t sender_cond; +typedef struct connector_data cdata_t; /* Accepts incoming connections on the server socket and generates client * instances */ -static int accept_client(conn_instance_t *ci, int epfd) +static int accept_client(cdata_t *cdata, int epfd) { - ckpool_t *ckp = ci->pi->ckp; + ckpool_t *ckp = cdata->ckp; client_instance_t *client; struct epoll_event event; int fd, port, no_clients; socklen_t address_len; - ck_rlock(&ci->lock); - no_clients = HASH_COUNT(clients); - ck_runlock(&ci->lock); + ck_rlock(&cdata->lock); + no_clients = HASH_COUNT(cdata->clients); + ck_runlock(&cdata->lock); if (unlikely(ckp->maxclients && no_clients >= ckp->maxclients)) { LOGWARNING("Server full with %d clients", no_clients); @@ -103,7 +105,7 @@ static int accept_client(conn_instance_t *ci, int epfd) client = ckzalloc(sizeof(client_instance_t)); address_len = sizeof(client->address); - fd = accept(ci->serverfd, &client->address, &address_len); + fd = accept(cdata->serverfd, &client->address, &address_len); if (unlikely(fd < 0)) { /* Handle these errors gracefully should we ever share this * socket */ @@ -111,7 +113,7 @@ static int accept_client(conn_instance_t *ci, int epfd) LOGERR("Recoverable error on accept in accept_client"); return 0; } - LOGERR("Failed to accept on socket %d in acceptor", ci->serverfd); + LOGERR("Failed to accept on socket %d in acceptor", cdata->serverfd); dealloc(client); return -1; } @@ -132,7 +134,7 @@ static int accept_client(conn_instance_t *ci, int epfd) break; default: LOGWARNING("Unknown INET type for client %d on socket %d", - ci->nfds, fd); + cdata->nfds, fd); Close(fd); free(client); return 0; @@ -142,7 +144,7 @@ static int accept_client(conn_instance_t *ci, int epfd) nolinger_socket(fd); LOGINFO("Connected new client %d on socket %d to %d active clients from %s:%d", - ci->nfds, fd, no_clients, client->address_name, port); + cdata->nfds, fd, no_clients, client->address_name, port); client->fd = fd; event.data.ptr = client; @@ -153,27 +155,27 @@ static int accept_client(conn_instance_t *ci, int epfd) return 0; } - ck_wlock(&ci->lock); - client->id = client_id++; - HASH_ADD_I64(clients, id, client); - ci->nfds++; - ck_wunlock(&ci->lock); + ck_wlock(&cdata->lock); + client->id = cdata->client_id++; + HASH_ADD_I64(cdata->clients, id, client); + cdata->nfds++; + ck_wunlock(&cdata->lock); return 1; } -static int drop_client(conn_instance_t *ci, client_instance_t *client) +static int drop_client(cdata_t *cdata, client_instance_t *client) { int fd; - ck_wlock(&ci->lock); + ck_wlock(&cdata->lock); fd = client->fd; if (fd != -1) { Close(client->fd); - HASH_DEL(clients, client); - LL_PREPEND(dead_clients, client); + HASH_DEL(cdata->clients, client); + LL_PREPEND(cdata->dead_clients, client); } - ck_wunlock(&ci->lock); + ck_wunlock(&cdata->lock); if (fd > -1) LOGINFO("Connector dropped client %d fd %d", client->id, fd); @@ -192,26 +194,26 @@ static void stratifier_drop_client(ckpool_t *ckp, int64_t id) /* Invalidate this instance. Remove them from the hashtables we look up * regularly but keep the instances in a linked list indefinitely in case we * still reference any of its members. */ -static void invalidate_client(ckpool_t *ckp, conn_instance_t *ci, client_instance_t *client) +static void invalidate_client(ckpool_t *ckp, cdata_t *cdata, client_instance_t *client) { - drop_client(ci, client); + drop_client(cdata, client); if (ckp->passthrough) return; stratifier_drop_client(ckp, client->id); } -static void send_client(conn_instance_t *ci, int64_t id, char *buf); +static void send_client(cdata_t *cdata, int64_t id, char *buf); -static void parse_client_msg(conn_instance_t *ci, client_instance_t *client) +static void parse_client_msg(cdata_t *cdata, client_instance_t *client) { int buflen, ret, selfail = 0; - ckpool_t *ckp = ci->pi->ckp; + ckpool_t *ckp = cdata->ckp; char msg[PAGESIZE], *eol; json_t *val; retry: /* Select should always return positive after poll unless we have - * been disconnected. On retries, decide whether we should do further + * been disconnected. On retries, decdatade whether we should do further * reads based on select readiness and only fail if we get an error. */ ret = wait_read_select(client->fd, 0); if (ret < 1) { @@ -219,7 +221,7 @@ retry: return; LOGINFO("Client fd %d disconnected - select fail with bufofs %d ret %d errno %d %s", client->fd, client->bufofs, ret, errno, ret && errno ? strerror(errno) : ""); - invalidate_client(ckp, ci, client); + invalidate_client(ckp, cdata, client); return; } selfail = -1; @@ -231,7 +233,7 @@ retry: * client has disconnected. */ LOGINFO("Client fd %d disconnected - recv fail with bufofs %d ret %d errno %d %s", client->fd, client->bufofs, ret, errno, ret && errno ? strerror(errno) : ""); - invalidate_client(ckp, ci, client); + invalidate_client(ckp, cdata, client); return; } client->bufofs += ret; @@ -240,7 +242,7 @@ reparse: if (!eol) { if (unlikely(client->bufofs > MAX_MSGSIZE)) { LOGWARNING("Client fd %d overloaded buffer without EOL, disconnecting", client->fd); - invalidate_client(ckp, ci, client); + invalidate_client(ckp, cdata, client); return; } goto retry; @@ -250,7 +252,7 @@ reparse: buflen = eol - client->buf + 1; if (unlikely(buflen > MAX_MSGSIZE)) { LOGWARNING("Client fd %d message oversize, disconnecting", client->fd); - invalidate_client(ckp, ci, client); + invalidate_client(ckp, cdata, client); return; } memcpy(msg, client->buf, buflen); @@ -262,8 +264,8 @@ reparse: char *buf = strdup("Invalid JSON, disconnecting\n"); LOGINFO("Client id %d sent invalid json message %s", client->id, msg); - send_client(ci, client->id, buf); - invalidate_client(ckp, ci, client); + send_client(cdata, client->id, buf); + invalidate_client(ckp, cdata, client); return; } else { int64_t passthrough_id; @@ -295,7 +297,7 @@ reparse: * handles the incoming messages */ void *receiver(void *arg) { - conn_instance_t *ci = (conn_instance_t *)arg; + cdata_t *cdata = (cdata_t *)arg; struct epoll_event event; bool maxconn = true; int ret, epfd; @@ -307,9 +309,9 @@ void *receiver(void *arg) LOGEMERG("FATAL: Failed to create epoll in receiver"); return NULL; } - event.data.fd = ci->serverfd; + event.data.fd = cdata->serverfd; event.events = EPOLLIN; - ret = epoll_ctl(epfd, EPOLL_CTL_ADD, ci->serverfd, &event); + ret = epoll_ctl(epfd, EPOLL_CTL_ADD, cdata->serverfd, &event); if (ret < 0) { LOGEMERG("FATAL: Failed to add epfd %d to epoll_ctl", epfd); return NULL; @@ -318,7 +320,7 @@ void *receiver(void *arg) while (42) { client_instance_t *client; - while (unlikely(!ci->accept)) + while (unlikely(!cdata->accept)) cksleep_ms(100); ret = epoll_wait(epfd, &event, 1, 1000); if (unlikely(ret == -1)) { @@ -333,12 +335,12 @@ void *receiver(void *arg) * can receive connections. */ LOGDEBUG("Dropping listen backlog to 0"); maxconn = false; - listen(ci->serverfd, 0); + listen(cdata->serverfd, 0); } continue; } - if (event.data.fd == ci->serverfd) { - ret = accept_client(ci, epfd); + if (event.data.fd == cdata->serverfd) { + ret = accept_client(cdata, epfd); if (unlikely(ret < 0)) { LOGEMERG("FATAL: Failed to accept_client in receiver"); break; @@ -349,10 +351,10 @@ void *receiver(void *arg) if ((event.events & EPOLLERR) || (event.events & EPOLLHUP)) { /* Client disconnected */ LOGDEBUG("Client fd %d HUP in epoll", client->fd); - invalidate_client(ci->pi->ckp, ci, client); + invalidate_client(cdata->pi->ckp, cdata, client); continue; } - parse_client_msg(ci, client); + parse_client_msg(cdata, client); } return NULL; } @@ -361,8 +363,8 @@ void *receiver(void *arg) * ready for writing immediately to not delay other messages. */ void *sender(void *arg) { - conn_instance_t *ci = (conn_instance_t *)arg; - ckpool_t *ckp = ci->pi->ckp; + cdata_t *cdata = (cdata_t *)arg; + ckpool_t *ckp = cdata->ckp; bool sent = false; rename_proc("csender"); @@ -372,23 +374,23 @@ void *sender(void *arg) client_instance_t *client; int ret, fd, ofs = 0; - mutex_lock(&sender_lock); + mutex_lock(&cdata->sender_lock); /* Poll every 100ms if there are no new sends. Re-examine * delayed sends immediately after a successful send in case * endless new sends more frequently end up starving the * delayed sends. */ - if (!sender_sends && !sent) { + if (!cdata->sender_sends && !sent) { const ts_t polltime = {0, 100000000}; ts_t timeout_ts; ts_realtime(&timeout_ts); timeraddspec(&timeout_ts, &polltime); - pthread_cond_timedwait(&sender_cond, &sender_lock, &timeout_ts); + pthread_cond_timedwait(&cdata->sender_cond, &cdata->sender_lock, &timeout_ts); } - sender_send = sender_sends; + sender_send = cdata->sender_sends; if (sender_send) - DL_DELETE(sender_sends, sender_send); - mutex_unlock(&sender_lock); + DL_DELETE(cdata->sender_sends, sender_send); + mutex_unlock(&cdata->sender_lock); sent = false; @@ -396,17 +398,17 @@ void *sender(void *arg) * conditional with no new sends appearing or have just * serviced another message successfully. */ if (!sender_send) { - if (!delayed_sends) + if (!cdata->delayed_sends) continue; - sender_send = delayed_sends; - DL_DELETE(delayed_sends, sender_send); + sender_send = cdata->delayed_sends; + DL_DELETE(cdata->delayed_sends, sender_send); } client = sender_send->client; - ck_rlock(&ci->lock); + ck_rlock(&cdata->lock); fd = client->fd; - ck_runlock(&ci->lock); + ck_runlock(&cdata->lock); if (fd == -1) { LOGDEBUG("Discarding message sent to invalidated client"); @@ -422,7 +424,7 @@ void *sender(void *arg) if (ret < 1) { if (ret < 0) { LOGINFO("Client id %d fd %d interrupted", client->id, fd); - invalidate_client(ckp, ci, client); + invalidate_client(ckp, cdata, client); free(sender_send->buf); free(sender_send); continue; @@ -432,7 +434,7 @@ void *sender(void *arg) /* Append it to the tail of the delayed sends list. * This is the only function that alters it so no * locking is required. */ - DL_APPEND(delayed_sends, sender_send); + DL_APPEND(cdata->delayed_sends, sender_send); continue; } sent = true; @@ -440,7 +442,7 @@ void *sender(void *arg) ret = send(fd, sender_send->buf + ofs, sender_send->len , 0); if (unlikely(ret < 0)) { LOGINFO("Client id %d fd %d disconnected", client->id, fd); - invalidate_client(ckp, ci, client); + invalidate_client(ckp, cdata, client); break; } ofs += ret; @@ -455,7 +457,7 @@ void *sender(void *arg) /* Send a client by id a heap allocated buffer, allowing this function to * free the ram. */ -static void send_client(conn_instance_t *ci, int64_t id, char *buf) +static void send_client(cdata_t *cdata, int64_t id, char *buf) { sender_send_t *sender_send; client_instance_t *client; @@ -472,19 +474,19 @@ static void send_client(conn_instance_t *ci, int64_t id, char *buf) return; } - ck_rlock(&ci->lock); - HASH_FIND_I64(clients, &id, client); + ck_rlock(&cdata->lock); + HASH_FIND_I64(cdata->clients, &id, client); if (likely(client)) fd = client->fd; - ck_runlock(&ci->lock); + ck_runlock(&cdata->lock); if (unlikely(fd == -1)) { - ckpool_t *ckp = ci->pi->ckp; + ckpool_t *ckp = cdata->ckp; if (client) { /* This shouldn't happen */ LOGWARNING("Client id %ld disconnected but fd already invalidated!", id); - invalidate_client(ckp, ci, client); + invalidate_client(ckp, cdata, client); } else { LOGINFO("Connector failed to find client id %ld to send to", id); stratifier_drop_client(ckp, id); @@ -498,34 +500,34 @@ static void send_client(conn_instance_t *ci, int64_t id, char *buf) sender_send->buf = buf; sender_send->len = len; - mutex_lock(&sender_lock); - DL_APPEND(sender_sends, sender_send); - pthread_cond_signal(&sender_cond); - mutex_unlock(&sender_lock); + mutex_lock(&cdata->sender_lock); + DL_APPEND(cdata->sender_sends, sender_send); + pthread_cond_signal(&cdata->sender_cond); + mutex_unlock(&cdata->sender_lock); } -static client_instance_t *client_by_id(conn_instance_t *ci, int64_t id) +static client_instance_t *client_by_id(cdata_t *cdata, int64_t id) { client_instance_t *client; - ck_rlock(&ci->lock); - HASH_FIND_I64(clients, &id, client); - ck_runlock(&ci->lock); + ck_rlock(&cdata->lock); + HASH_FIND_I64(cdata->clients, &id, client); + ck_runlock(&cdata->lock); return client; } -static void passthrough_client(conn_instance_t *ci, client_instance_t *client) +static void passthrough_client(cdata_t *cdata, client_instance_t *client) { char *buf; LOGINFO("Connector adding passthrough client %d", client->id); client->passthrough = true; ASPRINTF(&buf, "{\"result\": true}\n"); - send_client(ci, client->id, buf); + send_client(cdata, client->id, buf); } -static int connector_loop(proc_instance_t *pi, conn_instance_t *ci) +static int connector_loop(proc_instance_t *pi, cdata_t *cdata) { int sockd = -1, ret = 0, selret; int64_t client_id64, client_id; @@ -546,12 +548,12 @@ static int connector_loop(proc_instance_t *pi, conn_instance_t *ci) LOGWARNING("%s connector ready", ckp->name); retry: - if (unlikely(!pthread_tryjoin_np(ci->pth_sender, NULL))) { + if (unlikely(!pthread_tryjoin_np(cdata->pth_sender, NULL))) { LOGEMERG("Connector sender thread shutdown, exiting"); ret = 1; goto out; } - if (unlikely(!pthread_tryjoin_np(ci->pth_receiver, NULL))) { + if (unlikely(!pthread_tryjoin_np(cdata->pth_receiver, NULL))) { LOGEMERG("Connector receiver thread shutdown, exiting"); ret = 1; goto out; @@ -579,12 +581,12 @@ retry: } if (cmdmatch(buf, "accept")) { LOGDEBUG("Connector received accept signal"); - ci->accept = true; + cdata->accept = true; goto retry; } if (cmdmatch(buf, "reject")) { LOGDEBUG("Connector received reject signal"); - ci->accept = false; + cdata->accept = false; goto retry; } if (cmdmatch(buf, "loglevel")) { @@ -603,12 +605,12 @@ retry: goto retry; } client_id = client_id64 & 0xffffffffll; - client = client_by_id(ci, client_id); + client = client_by_id(cdata, client_id); if (unlikely(!client)) { LOGINFO("Connector failed to find client id %ld to drop", client_id); goto retry; } - ret = drop_client(ci, client); + ret = drop_client(cdata, client); if (ret >= 0) LOGINFO("Connector dropped client id: %ld", client_id); goto retry; @@ -621,16 +623,16 @@ retry: LOGDEBUG("Connector failed to parse passthrough command: %s", buf); goto retry; } - client = client_by_id(ci, client_id); + client = client_by_id(cdata, client_id); if (unlikely(!client)) { LOGINFO("Connector failed to find client id %ld to pass through", client_id); goto retry; } - passthrough_client(ci, client); + passthrough_client(cdata, client); goto retry; } if (cmdmatch(buf, "getfd")) { - send_fd(ci->serverfd, sockd); + send_fd(cdata->serverfd, sockd); goto retry; } @@ -655,7 +657,7 @@ retry: dealloc(buf); buf = json_dumps(json_msg, 0); realloc_strcat(&buf, "\n"); - send_client(ci, client_id, buf); + send_client(cdata, client_id, buf); json_decref(json_msg); buf = NULL; @@ -668,14 +670,16 @@ out: int connector(proc_instance_t *pi) { + cdata_t *cdata = ckzalloc(sizeof(cdata_t)); char *url = NULL, *port = NULL; ckpool_t *ckp = pi->ckp; int sockd, ret = 0; - conn_instance_t ci; const int on = 1; int tries = 0; LOGWARNING("%s connector starting", ckp->name); + ckp->data = cdata; + cdata->ckp = ckp; if (ckp->oldconnfd > 0) { sockd = ckp->oldconnfd; @@ -737,17 +741,18 @@ int connector(proc_instance_t *pi) goto out; } - cklock_init(&ci.lock); - memset(&ci, 0, sizeof(ci)); - ci.pi = pi; - ci.serverfd = sockd; - ci.nfds = 0; - mutex_init(&sender_lock); - cond_init(&sender_cond); - create_pthread(&ci.pth_sender, sender, &ci); - create_pthread(&ci.pth_receiver, receiver, &ci); - - ret = connector_loop(pi, &ci); + cklock_init(&cdata->lock); + cdata->pi = pi; + cdata->serverfd = sockd; + cdata->nfds = 0; + cdata->client_id = 1; + mutex_init(&cdata->sender_lock); + cond_init(&cdata->sender_cond); + create_pthread(&cdata->pth_sender, sender, &cdata); + create_pthread(&cdata->pth_receiver, receiver, &cdata); + + ret = connector_loop(pi, cdata); out: + dealloc(ckp->data); return process_exit(ckp, pi, ret); } diff --git a/src/generator.c b/src/generator.c index d7e238e6..524b1ec6 100644 --- a/src/generator.c +++ b/src/generator.c @@ -1745,5 +1745,6 @@ int generator(proc_instance_t *pi) else ret = server_mode(ckp, pi); + dealloc(ckp->data); return process_exit(ckp, pi, ret); } From 1e4030b9ed90322a14ebbb8b34578895da04c5e6 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Fri, 7 Nov 2014 15:26:09 +1100 Subject: [PATCH 06/47] Fix wrong pointer in connector --- src/connector.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/connector.c b/src/connector.c index 8c098b96..58281bf4 100644 --- a/src/connector.c +++ b/src/connector.c @@ -748,8 +748,8 @@ int connector(proc_instance_t *pi) cdata->client_id = 1; mutex_init(&cdata->sender_lock); cond_init(&cdata->sender_cond); - create_pthread(&cdata->pth_sender, sender, &cdata); - create_pthread(&cdata->pth_receiver, receiver, &cdata); + create_pthread(&cdata->pth_sender, sender, cdata); + create_pthread(&cdata->pth_receiver, receiver, cdata); ret = connector_loop(pi, cdata); out: From b7785415e65ce50773e90e2615dd415f76b5e34d Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Fri, 7 Nov 2014 15:46:05 +1100 Subject: [PATCH 07/47] Privatise all the data in the stratifier --- src/stratifier.c | 907 +++++++++++++++++++++++++---------------------- 1 file changed, 476 insertions(+), 431 deletions(-) diff --git a/src/stratifier.c b/src/stratifier.c index d6279b58..4cfcfe29 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -28,14 +28,11 @@ #include "uthash.h" #include "utlist.h" +/* Consistent across all pool instances */ static const char *workpadding = "000000800000000000000000000000000000000000000000000000000000000000000000000000000000000080020000"; - static const char *scriptsig_header = "01000000010000000000000000000000000000000000000000000000000000000000000000ffffffff"; static uchar scriptsig_header_bin[41]; -static char pubkeytxnbin[25]; -static char donkeytxnbin[25]; - /* Add unaccounted shares when they arrive, remove them with each update of * rolling stats. */ struct pool_stats { @@ -76,21 +73,6 @@ struct pool_stats { typedef struct pool_stats pool_stats_t; -static pool_stats_t stats; - -/* Protects changes to pool stats */ -static pthread_mutex_t stats_lock; - -/* Serialises sends/receives to ckdb if possible */ -static pthread_mutex_t ckdb_lock; - -static union { - uint64_t u64; - uint32_t u32; - uint16_t u16; - uint8_t u8; -} enonce1u; - struct workbase { /* Hash table data */ UT_hash_handle hh; @@ -148,31 +130,6 @@ struct workbase { typedef struct workbase workbase_t; -/* For protecting the hashtable data */ -static cklock_t workbase_lock; - -/* For the hashtable of all workbases */ -static workbase_t *workbases; -static workbase_t *current_workbase; - -static struct { - double diff; - - char enonce1[32]; - uchar enonce1bin[16]; - int enonce1constlen; - int enonce1varlen; - - int nonce2len; - int enonce2varlen; - - bool subscribed; -} proxy_base; - -static int64_t workbase_id; -static int64_t blockchange_id; -static char lasthash[68], lastswaphash[68]; - struct json_params { json_t *method; json_t *params; @@ -192,15 +149,6 @@ struct smsg { typedef struct smsg smsg_t; -static ckmsgq_t *ssends; // Stratum sends -static ckmsgq_t *srecvs; // Stratum receives -static ckmsgq_t *ckdbq; // ckdb -static ckmsgq_t *sshareq; // Stratum share sends -static ckmsgq_t *sauthq; // Stratum authorisations -static ckmsgq_t *stxnq; // Transaction requests - -static int64_t user_instance_id; - struct user_instance; struct worker_instance; struct stratum_instance; @@ -232,8 +180,6 @@ struct user_instance { tv_t last_share; }; -static user_instance_t *user_instances; - /* Combined data from workers with the same workername */ struct worker_instance { user_instance_t *instance; @@ -305,14 +251,6 @@ struct stratum_instance { int64_t suggest_diff; /* Stratum client suggested diff */ }; -/* Stratum_instances hashlist is stored by id, whereas disconnected_instances - * is sorted by enonce1_64. */ -static stratum_instance_t *stratum_instances; -static stratum_instance_t *disconnected_instances; - -/* Protects both stratum and user instances */ -static cklock_t instance_lock; - struct share { UT_hash_handle hh; uchar hash[32]; @@ -321,17 +259,84 @@ struct share { typedef struct share share_t; -static share_t *shares; +struct stratifier_data { + char pubkeytxnbin[25]; + char donkeytxnbin[25]; + + pool_stats_t stats; + /* Protects changes to pool stats */ + pthread_mutex_t stats_lock; -static cklock_t share_lock; + /* Serialises sends/receives to ckdb if possible */ + pthread_mutex_t ckdb_lock; -/* Linked list of block solves, added to during submission, removed on - * accept/reject. It is likely we only ever have one solve on here but you - * never know... */ -static pthread_mutex_t block_lock; -static ckmsg_t *block_solves; + /* Variable length enonce1 always refers back to a u64 */ + union { + uint64_t u64; + uint32_t u32; + uint16_t u16; + uint8_t u8; + } enonce1u; -static int gen_priority; + /* For protecting the hashtable data */ + cklock_t workbase_lock; + + /* For the hashtable of all workbases */ + workbase_t *workbases; + workbase_t *current_workbase; + + int64_t workbase_id; + int64_t blockchange_id; + char lasthash[68]; + char lastswaphash[68]; + + ckmsgq_t *ssends; // Stratum sends + ckmsgq_t *srecvs; // Stratum receives + ckmsgq_t *ckdbq; // ckdb + ckmsgq_t *sshareq; // Stratum share sends + ckmsgq_t *sauthq; // Stratum authorisations + ckmsgq_t *stxnq; // Transaction requests + + int64_t user_instance_id; + + /* Stratum_instances hashlist is stored by id, whereas disconnected_instances + * is sorted by enonce1_64. */ + stratum_instance_t *stratum_instances; + stratum_instance_t *disconnected_instances; + + user_instance_t *user_instances; + + /* Protects both stratum and user instances */ + cklock_t instance_lock; + + share_t *shares; + cklock_t share_lock; + + /* Linked list of block solves, added to during submission, removed on + * accept/reject. It is likely we only ever have one solve on here but + * you never know... */ + pthread_mutex_t block_lock; + ckmsg_t *block_solves; + + /* Generator message priority */ + int gen_priority; + + struct { + double diff; + + char enonce1[32]; + uchar enonce1bin[16]; + int enonce1constlen; + int enonce1varlen; + + int nonce2len; + int enonce2varlen; + + bool subscribed; + } proxy_base; +}; + +typedef struct stratifier_data sdata_t; /* Priority levels for generator messages */ #define GEN_LAX 0 @@ -365,6 +370,7 @@ static const char *ckdb_ids[] = { static void generate_coinbase(ckpool_t *ckp, workbase_t *wb) { uint64_t *u64, g64, d64 = 0; + sdata_t *sdata = ckp->data; char header[228]; int len, ofs = 0; ts_t now; @@ -448,7 +454,7 @@ static void generate_coinbase(ckpool_t *ckp, workbase_t *wb) wb->coinb2len += 8; wb->coinb2bin[wb->coinb2len++] = 25; - memcpy(wb->coinb2bin + wb->coinb2len, pubkeytxnbin, 25); + memcpy(wb->coinb2bin + wb->coinb2len, sdata->pubkeytxnbin, 25); wb->coinb2len += 25; if (ckp->donvalid) { @@ -457,7 +463,7 @@ static void generate_coinbase(ckpool_t *ckp, workbase_t *wb) wb->coinb2len += 8; wb->coinb2bin[wb->coinb2len++] = 25; - memcpy(wb->coinb2bin + wb->coinb2len, donkeytxnbin, 25); + memcpy(wb->coinb2bin + wb->coinb2len, sdata->donkeytxnbin, 25); wb->coinb2len += 25; } @@ -477,7 +483,7 @@ static void generate_coinbase(ckpool_t *ckp, workbase_t *wb) hex2bin(wb->headerbin, header, 112); } -static void stratum_broadcast_update(bool clean); +static void stratum_broadcast_update(sdata_t *sdata, bool clean); static void clear_workbase(workbase_t *wb) { @@ -493,20 +499,20 @@ static void clear_workbase(workbase_t *wb) free(wb); } -static void purge_share_hashtable(int64_t wb_id) +static void purge_share_hashtable(sdata_t *sdata, int64_t wb_id) { share_t *share, *tmp; int purged = 0; - ck_wlock(&share_lock); - HASH_ITER(hh, shares, share, tmp) { + ck_wlock(&sdata->share_lock); + HASH_ITER(hh, sdata->shares, share, tmp) { if (share->workbase_id < wb_id) { - HASH_DEL(shares, share); + HASH_DEL(sdata->shares, share); free(share); purged++; } } - ck_wunlock(&share_lock); + ck_wunlock(&sdata->share_lock); if (purged) LOGINFO("Cleared %d shares from share hashtable", purged); @@ -537,6 +543,7 @@ static void _ckdbq_add(ckpool_t *ckp, const int idtype, json_t *val, const char const char *func, const int line) { static time_t time_counter; + sdata_t *sdata = ckp->data; static int counter = 0; char *json_msg; time_t now_t; @@ -565,7 +572,7 @@ static void _ckdbq_add(ckpool_t *ckp, const int idtype, json_t *val, const char return; } - ckmsgq_add(ckdbq, json_msg); + ckmsgq_add(sdata->ckdbq, json_msg); } #define ckdbq_add(ckp, idtype, val) _ckdbq_add(ckp, idtype, val, __FILE__, __func__, __LINE__) @@ -618,6 +625,7 @@ static void send_ageworkinfo(ckpool_t *ckp, int64_t id) static void add_base(ckpool_t *ckp, workbase_t *wb, bool *new_block) { workbase_t *tmp, *tmpa, *aged = NULL; + sdata_t *sdata = ckp->data; int len, ret; ts_realtime(&wb->gentime); @@ -629,21 +637,21 @@ static void add_base(ckpool_t *ckp, workbase_t *wb, bool *new_block) /* In proxy mode, the wb->id is received in the notify update and * we set workbase_id from it. In server mode the stratifier is * setting the workbase_id */ - ck_wlock(&workbase_lock); + ck_wlock(&sdata->workbase_lock); if (!ckp->proxy) - wb->id = workbase_id++; + wb->id = sdata->workbase_id++; else - workbase_id = wb->id; - if (strncmp(wb->prevhash, lasthash, 64)) { + sdata->workbase_id = wb->id; + if (strncmp(wb->prevhash, sdata->lasthash, 64)) { char bin[32], swap[32]; *new_block = true; - memcpy(lasthash, wb->prevhash, 65); - hex2bin(bin, lasthash, 32); + memcpy(sdata->lasthash, wb->prevhash, 65); + hex2bin(bin, sdata->lasthash, 32); swap_256(swap, bin); - __bin2hex(lastswaphash, swap, 32); - LOGNOTICE("Block hash changed to %s", lastswaphash); - blockchange_id = wb->id; + __bin2hex(sdata->lastswaphash, swap, 32); + LOGNOTICE("Block hash changed to %s", sdata->lastswaphash); + sdata->blockchange_id = wb->id; } if (*new_block && ckp->logshares) { sprintf(wb->logdir, "%s%08x/", ckp->logdir, wb->height); @@ -655,22 +663,22 @@ static void add_base(ckpool_t *ckp, workbase_t *wb, bool *new_block) if (ckp->logshares) sprintf(wb->logdir, "%s%08x/%s", ckp->logdir, wb->height, wb->idstring); - HASH_ITER(hh, workbases, tmp, tmpa) { - if (HASH_COUNT(workbases) < 3) + HASH_ITER(hh, sdata->workbases, tmp, tmpa) { + if (HASH_COUNT(sdata->workbases) < 3) break; /* Age old workbases older than 10 minutes old */ if (tmp->gentime.tv_sec < wb->gentime.tv_sec - 600) { - HASH_DEL(workbases, tmp); + HASH_DEL(sdata->workbases, tmp); aged = tmp; break; } } - HASH_ADD_I64(workbases, id, wb); - current_workbase = wb; - ck_wunlock(&workbase_lock); + HASH_ADD_I64(sdata->workbases, id, wb); + sdata->current_workbase = wb; + ck_wunlock(&sdata->workbase_lock); if (*new_block) - purge_share_hashtable(wb->id); + purge_share_hashtable(sdata, wb->id); send_workinfo(ckp, wb); @@ -687,17 +695,18 @@ static void add_base(ckpool_t *ckp, workbase_t *wb, bool *new_block) * read the wrong priority but occasional wrong values are harmless. */ static char *__send_recv_generator(ckpool_t *ckp, const char *msg, int prio) { + sdata_t *sdata = ckp->data; char *buf = NULL; bool set; - if (prio > gen_priority) { - gen_priority = prio; + if (prio > sdata->gen_priority) { + sdata->gen_priority = prio; set = true; } else set = false; buf = send_recv_proc(ckp->generator, msg); if (set) - gen_priority = 0; + sdata->gen_priority = 0; return buf; } @@ -706,25 +715,27 @@ static char *__send_recv_generator(ckpool_t *ckp, const char *msg, int prio) * any currently being serviced. */ static char *send_recv_generator(ckpool_t *ckp, const char *msg, int prio) { + sdata_t *sdata = ckp->data; char *buf = NULL; - if (prio >= gen_priority) + if (prio >= sdata->gen_priority) buf = __send_recv_generator(ckp, msg, prio); return buf; } static void send_generator(ckpool_t *ckp, const char *msg, int prio) { + sdata_t *sdata = ckp->data; bool set; - if (prio > gen_priority) { - gen_priority = prio; + if (prio > sdata->gen_priority) { + sdata->gen_priority = prio; set = true; } else set = false; send_proc(ckp->generator, msg); if (set) - gen_priority = 0; + sdata->gen_priority = 0; } struct update_req { @@ -733,7 +744,7 @@ struct update_req { int prio; }; -static void broadcast_ping(void); +static void broadcast_ping(sdata_t *sdata); /* This function assumes it will only receive a valid json gbt base template * since checking should have been done earlier, and creates the base template @@ -742,6 +753,7 @@ static void *do_update(void *arg) { struct update_req *ur = (struct update_req *)arg; ckpool_t *ckp = ur->ckp; + sdata_t *sdata = ckp->data; bool new_block = false; int prio = ur->prio; bool ret = false; @@ -803,7 +815,7 @@ static void *do_update(void *arg) add_base(ckp, wb, &new_block); - stratum_broadcast_update(new_block); + stratum_broadcast_update(sdata, new_block); ret = true; LOGINFO("Broadcast updated stratum base"); out: @@ -811,7 +823,7 @@ out: * connected while bitcoind recovers(?) */ if (!ret) { LOGWARNING("Broadcast ping due to failed stratum base update"); - broadcast_ping(); + broadcast_ping(sdata); } free(ur->pth); free(ur); @@ -832,22 +844,24 @@ static void update_base(ckpool_t *ckp, int prio) static void drop_allclients(ckpool_t *ckp) { stratum_instance_t *client, *tmp; + sdata_t *sdata = ckp->data; char buf[128]; - ck_wlock(&instance_lock); - HASH_ITER(hh, stratum_instances, client, tmp) { - HASH_DEL(stratum_instances, client); + ck_wlock(&sdata->instance_lock); + HASH_ITER(hh, sdata->stratum_instances, client, tmp) { + HASH_DEL(sdata->stratum_instances, client); sprintf(buf, "dropclient=%ld", client->id); send_proc(ckp->connector, buf); } - HASH_ITER(hh, disconnected_instances, client, tmp) - HASH_DEL(disconnected_instances, client); - stats.users = stats.workers = 0; - ck_wunlock(&instance_lock); + HASH_ITER(hh, sdata->disconnected_instances, client, tmp) + HASH_DEL(sdata->disconnected_instances, client); + sdata->stats.users = sdata->stats.workers = 0; + ck_wunlock(&sdata->instance_lock); } static void update_subscribe(ckpool_t *ckp) { + sdata_t *sdata = ckp->data; json_t *val; char *buf; @@ -861,22 +875,22 @@ static void update_subscribe(ckpool_t *ckp) val = json_loads(buf, 0, NULL); free(buf); - ck_wlock(&workbase_lock); - proxy_base.subscribed = true; - proxy_base.diff = ckp->startdiff; + ck_wlock(&sdata->workbase_lock); + sdata->proxy_base.subscribed = true; + sdata->proxy_base.diff = ckp->startdiff; /* Length is checked by generator */ - strcpy(proxy_base.enonce1, json_string_value(json_object_get(val, "enonce1"))); - proxy_base.enonce1constlen = strlen(proxy_base.enonce1) / 2; - hex2bin(proxy_base.enonce1bin, proxy_base.enonce1, proxy_base.enonce1constlen); - proxy_base.nonce2len = json_integer_value(json_object_get(val, "nonce2len")); - if (proxy_base.nonce2len > 7) - proxy_base.enonce1varlen = 4; - else if (proxy_base.nonce2len > 5) - proxy_base.enonce1varlen = 2; + strcpy(sdata->proxy_base.enonce1, json_string_value(json_object_get(val, "enonce1"))); + sdata->proxy_base.enonce1constlen = strlen(sdata->proxy_base.enonce1) / 2; + hex2bin(sdata->proxy_base.enonce1bin, sdata->proxy_base.enonce1, sdata->proxy_base.enonce1constlen); + sdata->proxy_base.nonce2len = json_integer_value(json_object_get(val, "nonce2len")); + if (sdata->proxy_base.nonce2len > 7) + sdata->proxy_base.enonce1varlen = 4; + else if (sdata->proxy_base.nonce2len > 5) + sdata->proxy_base.enonce1varlen = 2; else - proxy_base.enonce1varlen = 1; - proxy_base.enonce2varlen = proxy_base.nonce2len - proxy_base.enonce1varlen; - ck_wunlock(&workbase_lock); + sdata->proxy_base.enonce1varlen = 1; + sdata->proxy_base.enonce2varlen = sdata->proxy_base.nonce2len - sdata->proxy_base.enonce1varlen; + ck_wunlock(&sdata->workbase_lock); json_decref(val); drop_allclients(ckp); @@ -887,6 +901,7 @@ static void update_diff(ckpool_t *ckp); static void update_notify(ckpool_t *ckp) { bool new_block = false, clean; + sdata_t *sdata = ckp->data; char header[228]; workbase_t *wb; json_t *val; @@ -899,7 +914,7 @@ static void update_notify(ckpool_t *ckp) return; } - if (unlikely(!proxy_base.subscribed)) { + if (unlikely(!sdata->proxy_base.subscribed)) { LOGINFO("No valid proxy subscription to update notify yet"); return; } @@ -949,30 +964,31 @@ static void update_notify(ckpool_t *ckp) /* Check diff on each notify */ update_diff(ckp); - ck_rlock(&workbase_lock); - strcpy(wb->enonce1const, proxy_base.enonce1); - wb->enonce1constlen = proxy_base.enonce1constlen; - memcpy(wb->enonce1constbin, proxy_base.enonce1bin, wb->enonce1constlen); - wb->enonce1varlen = proxy_base.enonce1varlen; - wb->enonce2varlen = proxy_base.enonce2varlen; - wb->diff = proxy_base.diff; - ck_runlock(&workbase_lock); + ck_rlock(&sdata->workbase_lock); + strcpy(wb->enonce1const, sdata->proxy_base.enonce1); + wb->enonce1constlen = sdata->proxy_base.enonce1constlen; + memcpy(wb->enonce1constbin, sdata->proxy_base.enonce1bin, wb->enonce1constlen); + wb->enonce1varlen = sdata->proxy_base.enonce1varlen; + wb->enonce2varlen = sdata->proxy_base.enonce2varlen; + wb->diff = sdata->proxy_base.diff; + ck_runlock(&sdata->workbase_lock); add_base(ckp, wb, &new_block); - stratum_broadcast_update(new_block | clean); + stratum_broadcast_update(sdata, new_block | clean); } -static void stratum_send_diff(stratum_instance_t *client); +static void stratum_send_diff(sdata_t *sdata, stratum_instance_t *client); static void update_diff(ckpool_t *ckp) { stratum_instance_t *client; + sdata_t *sdata = ckp->data; double old_diff, diff; json_t *val; char *buf; - if (unlikely(!current_workbase)) { + if (unlikely(!sdata->current_workbase)) { LOGINFO("No current workbase to update diff yet"); return; } @@ -994,32 +1010,32 @@ static void update_diff(ckpool_t *ckp) if (unlikely(diff < 1)) diff = 1; - ck_wlock(&workbase_lock); - old_diff = proxy_base.diff; - current_workbase->diff = proxy_base.diff = diff; - ck_wunlock(&workbase_lock); + ck_wlock(&sdata->workbase_lock); + old_diff = sdata->proxy_base.diff; + sdata->current_workbase->diff = sdata->proxy_base.diff = diff; + ck_wunlock(&sdata->workbase_lock); if (old_diff < diff) return; /* If the diff has dropped, iterate over all the clients and check * they're at or below the new diff, and update it if not. */ - ck_rlock(&instance_lock); - for (client = stratum_instances; client != NULL; client = client->hh.next) { + ck_rlock(&sdata->instance_lock); + for (client = sdata->stratum_instances; client != NULL; client = client->hh.next) { if (client->diff > diff) { client->diff = diff; - stratum_send_diff(client); + stratum_send_diff(sdata, client); } } - ck_runlock(&instance_lock); + ck_runlock(&sdata->instance_lock); } /* Enter with instance_lock held */ -static stratum_instance_t *__instance_by_id(int64_t id) +static stratum_instance_t *__instance_by_id(sdata_t *sdata, int64_t id) { stratum_instance_t *instance; - HASH_FIND_I64(stratum_instances, &id, instance); + HASH_FIND_I64(sdata->stratum_instances, &id, instance); return instance; } @@ -1027,18 +1043,19 @@ static stratum_instance_t *__instance_by_id(int64_t id) static stratum_instance_t *__stratum_add_instance(ckpool_t *ckp, int64_t id) { stratum_instance_t *instance = ckzalloc(sizeof(stratum_instance_t)); + sdata_t *sdata = ckp->data; instance->id = id; instance->diff = instance->old_diff = ckp->startdiff; instance->ckp = ckp; tv_time(&instance->ldc); LOGINFO("Added instance %ld", id); - HASH_ADD_I64(stratum_instances, id, instance); + HASH_ADD_I64(sdata->stratum_instances, id, instance); return instance; } /* Only supports a full ckpool instance sessionid with an 8 byte sessionid */ -static bool disconnected_sessionid_exists(const char *sessionid, int64_t id) +static bool disconnected_sessionid_exists(sdata_t *sdata, const char *sessionid, int64_t id) { stratum_instance_t *instance, *tmp; uint64_t session64; @@ -1051,8 +1068,8 @@ static bool disconnected_sessionid_exists(const char *sessionid, int64_t id) /* Number is in BE but we don't swap either of them */ hex2bin(&session64, sessionid, 8); - ck_rlock(&instance_lock); - HASH_ITER(hh, stratum_instances, instance, tmp) { + ck_rlock(&sdata->instance_lock); + HASH_ITER(hh, sdata->stratum_instances, instance, tmp) { if (instance->id == id) continue; if (instance->enonce1_64 == session64) { @@ -1061,11 +1078,11 @@ static bool disconnected_sessionid_exists(const char *sessionid, int64_t id) } } instance = NULL; - HASH_FIND(hh, disconnected_instances, &session64, sizeof(uint64_t), instance); + HASH_FIND(hh, sdata->disconnected_instances, &session64, sizeof(uint64_t), instance); if (instance) ret = true; out_unlock: - ck_runlock(&instance_lock); + ck_runlock(&sdata->instance_lock); out: return ret; } @@ -1073,7 +1090,7 @@ out: /* For creating a list of sends without locking that can then be concatenated * to the stratum_sends list. Minimises locking and avoids taking recursive * locks. */ -static void stratum_broadcast(json_t *val) +static void stratum_broadcast(sdata_t *sdata, json_t *val) { stratum_instance_t *instance, *tmp; ckmsg_t *bulk_send = NULL; @@ -1083,8 +1100,8 @@ static void stratum_broadcast(json_t *val) return; } - ck_rlock(&instance_lock); - HASH_ITER(hh, stratum_instances, instance, tmp) { + ck_rlock(&sdata->instance_lock); + HASH_ITER(hh, sdata->stratum_instances, instance, tmp) { ckmsg_t *client_msg; smsg_t *msg; @@ -1097,59 +1114,63 @@ static void stratum_broadcast(json_t *val) client_msg->data = msg; DL_APPEND(bulk_send, client_msg); } - ck_runlock(&instance_lock); + ck_runlock(&sdata->instance_lock); json_decref(val); if (!bulk_send) return; - mutex_lock(ssends->lock); - if (ssends->msgs) - DL_CONCAT(ssends->msgs, bulk_send); + mutex_lock(sdata->ssends->lock); + if (sdata->ssends->msgs) + DL_CONCAT(sdata->ssends->msgs, bulk_send); else - ssends->msgs = bulk_send; - pthread_cond_signal(ssends->cond); - mutex_unlock(ssends->lock); + sdata->ssends->msgs = bulk_send; + pthread_cond_signal(sdata->ssends->cond); + mutex_unlock(sdata->ssends->lock); } -static void stratum_add_send(json_t *val, int64_t client_id) +static void stratum_add_send(sdata_t *sdata, json_t *val, int64_t client_id) { smsg_t *msg; msg = ckzalloc(sizeof(smsg_t)); msg->json_msg = val; msg->client_id = client_id; - ckmsgq_add(ssends, msg); + ckmsgq_add(sdata->ssends, msg); } -static void inc_worker(user_instance_t *instance) +static void inc_worker(ckpool_t *ckp, user_instance_t *instance) { - mutex_lock(&stats_lock); - stats.workers++; + sdata_t *sdata = ckp->data; + + mutex_lock(&sdata->stats_lock); + sdata->stats.workers++; if (!instance->workers++) - stats.users++; - mutex_unlock(&stats_lock); + sdata->stats.users++; + mutex_unlock(&sdata->stats_lock); } -static void dec_worker(user_instance_t *instance) +static void dec_worker(ckpool_t *ckp, user_instance_t *instance) { - mutex_lock(&stats_lock); - stats.workers--; + sdata_t *sdata = ckp->data; + + mutex_lock(&sdata->stats_lock); + sdata->stats.workers--; if (!--instance->workers) - stats.users--; - mutex_unlock(&stats_lock); + sdata->stats.users--; + mutex_unlock(&sdata->stats_lock); } -static void drop_client(int64_t id) +static void drop_client(sdata_t *sdata, int64_t id) { stratum_instance_t *client = NULL; bool dec = false; LOGINFO("Stratifier dropping client %ld", id); - ck_wlock(&instance_lock); - client = __instance_by_id(id); + ck_wlock(&sdata->instance_lock); + client = __instance_by_id(sdata, id); if (client) { stratum_instance_t *old_client = NULL; @@ -1158,30 +1179,30 @@ static void drop_client(int64_t id) client->authorised = false; } - HASH_DEL(stratum_instances, client); - HASH_FIND(hh, disconnected_instances, &client->enonce1_64, sizeof(uint64_t), old_client); + HASH_DEL(sdata->stratum_instances, client); + HASH_FIND(hh, sdata->disconnected_instances, &client->enonce1_64, sizeof(uint64_t), old_client); /* Only keep around one copy of the old client in server mode */ if (!client->ckp->proxy && !old_client && client->enonce1_64) - HASH_ADD(hh, disconnected_instances, enonce1_64, sizeof(uint64_t), client); + HASH_ADD(hh, sdata->disconnected_instances, enonce1_64, sizeof(uint64_t), client); } - ck_wunlock(&instance_lock); + ck_wunlock(&sdata->instance_lock); if (dec) - dec_worker(client->user_instance); + dec_worker(client->ckp, client->user_instance); } -static void stratum_broadcast_message(const char *msg) +static void stratum_broadcast_message(sdata_t *sdata, const char *msg) { json_t *json_msg; JSON_CPACK(json_msg, "{sosss[s]}", "id", json_null(), "method", "client.show_message", "params", msg); - stratum_broadcast(json_msg); + stratum_broadcast(sdata, json_msg); } /* Send a generic reconnect to all clients without parameters to make them * reconnect to the same server. */ -static void reconnect_clients(const char *cmd) +static void reconnect_clients(sdata_t *sdata, const char *cmd) { char *port = strdupa(cmd), *url = NULL; json_t *json_msg; @@ -1198,12 +1219,13 @@ static void reconnect_clients(const char *cmd) } else JSON_CPACK(json_msg, "{sosss[]}", "id", json_null(), "method", "client.reconnect", "params"); - stratum_broadcast(json_msg); + stratum_broadcast(sdata, json_msg); } static void block_solve(ckpool_t *ckp, const char *blockhash) { ckmsg_t *block, *tmp, *found = NULL; + sdata_t *sdata = ckp->data; char cdfield[64]; int height = 0; ts_t ts_now; @@ -1215,8 +1237,8 @@ static void block_solve(ckpool_t *ckp, const char *blockhash) ts_realtime(&ts_now); sprintf(cdfield, "%lu,%lu", ts_now.tv_sec, ts_now.tv_nsec); - mutex_lock(&block_lock); - DL_FOREACH_SAFE(block_solves, block, tmp) { + mutex_lock(&sdata->block_lock); + DL_FOREACH_SAFE(sdata->block_solves, block, tmp) { val = block->data; char *solvehash; @@ -1228,12 +1250,12 @@ static void block_solve(ckpool_t *ckp, const char *blockhash) if (!strcmp(solvehash, blockhash)) { dealloc(solvehash); found = block; - DL_DELETE(block_solves, block); + DL_DELETE(sdata->block_solves, block); break; } dealloc(solvehash); } - mutex_unlock(&block_lock); + mutex_unlock(&sdata->block_lock); if (unlikely(!found)) { LOGERR("Failed to find blockhash %s in block_solve!", blockhash); @@ -1249,20 +1271,20 @@ static void block_solve(ckpool_t *ckp, const char *blockhash) free(found); ASPRINTF(&msg, "Block %d solved by %s!", height, ckp->name); - stratum_broadcast_message(msg); + stratum_broadcast_message(sdata, msg); free(msg); LOGWARNING("Solved and confirmed block %d", height); } -static void block_reject(const char *blockhash) +static void block_reject(sdata_t *sdata, const char *blockhash) { ckmsg_t *block, *tmp, *found = NULL; int height = 0; json_t *val; - mutex_lock(&block_lock); - DL_FOREACH_SAFE(block_solves, block, tmp) { + mutex_lock(&sdata->block_lock); + DL_FOREACH_SAFE(sdata->block_solves, block, tmp) { val = block->data; char *solvehash; @@ -1274,12 +1296,12 @@ static void block_reject(const char *blockhash) if (!strcmp(solvehash, blockhash)) { dealloc(solvehash); found = block; - DL_DELETE(block_solves, block); + DL_DELETE(sdata->block_solves, block); break; } dealloc(solvehash); } - mutex_unlock(&block_lock); + mutex_unlock(&sdata->block_lock); if (unlikely(!found)) { LOGERR("Failed to find blockhash %s in block_reject!", blockhash); @@ -1296,7 +1318,7 @@ static void block_reject(const char *blockhash) /* Some upstream pools (like p2pool) don't update stratum often enough and * miners disconnect if they don't receive regular communication so send them * a ping at regular intervals */ -static void broadcast_ping(void) +static void broadcast_ping(sdata_t *sdata) { json_t *json_msg; @@ -1305,12 +1327,13 @@ static void broadcast_ping(void) "id", 42, "method", "mining.ping"); - stratum_broadcast(json_msg); + stratum_broadcast(sdata, json_msg); } static int stratum_loop(ckpool_t *ckp, proc_instance_t *pi) { int sockd, ret = 0, selret = 0; + sdata_t *sdata = ckp->data; unixsock_t *us = &pi->us; tv_t start_tv = {0, 0}; char *buf = NULL; @@ -1331,7 +1354,7 @@ retry: } else { LOGDEBUG("%ds elapsed in strat_loop, pinging miners", ckp->update_interval); - broadcast_ping(); + broadcast_ping(sdata); } continue; } @@ -1386,20 +1409,20 @@ retry: if (ret < 0) LOGDEBUG("Stratifier failed to parse dropclient command: %s", buf); else - drop_client(client_id); + drop_client(sdata, client_id); } else if (cmdmatch(buf, "dropall")) { drop_allclients(ckp); } else if (cmdmatch(buf, "block")) { block_solve(ckp, buf + 6); } else if (cmdmatch(buf, "noblock")) { - block_reject(buf + 8); + block_reject(sdata, buf + 8); } else if (cmdmatch(buf, "reconnect")) { - reconnect_clients(buf); + reconnect_clients(sdata, buf); } else if (cmdmatch(buf, "loglevel")) { sscanf(buf, "loglevel=%d", &ckp->loglevel); } else { /* The srecv_process frees the buf heap ram */ - ckmsgq_add(srecvs, buf); + ckmsgq_add(sdata->srecvs, buf); buf = NULL; } goto retry; @@ -1412,6 +1435,7 @@ out: static void *blockupdate(void *arg) { ckpool_t *ckp = (ckpool_t *)arg; + sdata_t *sdata = ckp->data; char *buf = NULL; char request[8]; @@ -1428,7 +1452,7 @@ static void *blockupdate(void *arg) buf = send_recv_generator(ckp, request, GEN_LAX); if (buf && cmdmatch(buf, "notify")) cksleep_ms(5000); - else if (buf && strcmp(buf, lastswaphash) && !cmdmatch(buf, "failed")) + else if (buf && strcmp(buf, sdata->lastswaphash) && !cmdmatch(buf, "failed")) update_base(ckp, GEN_PRIORITY); else cksleep_ms(ckp->blockpoll); @@ -1436,7 +1460,7 @@ static void *blockupdate(void *arg) return NULL; } -static inline bool enonce1_free(uint64_t enonce1) +static inline bool enonce1_free(sdata_t *sdata, uint64_t enonce1) { stratum_instance_t *client, *tmp; bool ret = true; @@ -1445,7 +1469,7 @@ static inline bool enonce1_free(uint64_t enonce1) ret = false; goto out; } - HASH_ITER(hh, stratum_instances, client, tmp) { + HASH_ITER(hh, sdata->stratum_instances, client, tmp) { if (client->enonce1_64 == enonce1) { ret = false; break; @@ -1461,46 +1485,47 @@ out: * unused enonce1 value and reject clients instead if there is no space left */ static bool new_enonce1(stratum_instance_t *client) { + sdata_t *sdata = client->ckp->data; bool ret = false; workbase_t *wb; int i; - ck_wlock(&workbase_lock); - wb = current_workbase; + ck_wlock(&sdata->workbase_lock); + wb = sdata->current_workbase; switch(wb->enonce1varlen) { case 8: - enonce1u.u64++; + sdata->enonce1u.u64++; ret = true; break; case 4: - enonce1u.u32++; + sdata->enonce1u.u32++; ret = true; break; case 2: for (i = 0; i < 65536; i++) { - enonce1u.u16++; - ret = enonce1_free(enonce1u.u64); + sdata->enonce1u.u16++; + ret = enonce1_free(sdata, sdata->enonce1u.u64); if (ret) break; } break; case 1: for (i = 0; i < 256; i++) { - enonce1u.u8++; - ret = enonce1_free(enonce1u.u64); + sdata->enonce1u.u8++; + ret = enonce1_free(sdata, sdata->enonce1u.u64); if (ret) break; } break; } if (ret) - client->enonce1_64 = enonce1u.u64; + client->enonce1_64 = sdata->enonce1u.u64; if (wb->enonce1constlen) memcpy(client->enonce1bin, wb->enonce1constbin, wb->enonce1constlen); memcpy(client->enonce1bin + wb->enonce1constlen, &client->enonce1_64, wb->enonce1varlen); __bin2hex(client->enonce1var, &client->enonce1_64, wb->enonce1varlen); __bin2hex(client->enonce1, client->enonce1bin, wb->enonce1constlen + wb->enonce1varlen); - ck_wunlock(&workbase_lock); + ck_wunlock(&sdata->workbase_lock); if (unlikely(!ret)) LOGWARNING("Enonce1 space exhausted! Proxy rejecting clients"); @@ -1508,23 +1533,24 @@ static bool new_enonce1(stratum_instance_t *client) return ret; } -static void stratum_send_message(stratum_instance_t *client, const char *msg); +static void stratum_send_message(sdata_t *sdata, stratum_instance_t *client, const char *msg); /* Extranonce1 must be set here */ static json_t *parse_subscribe(stratum_instance_t *client, int64_t client_id, json_t *params_val) { + sdata_t *sdata = client->ckp->data; bool old_match = false; int arr_size; json_t *ret; int n2len; if (unlikely(!json_is_array(params_val))) { - stratum_send_message(client, "Invalid json: params not an array"); + stratum_send_message(sdata, client, "Invalid json: params not an array"); return json_string("params not an array"); } - if (unlikely(!current_workbase)) { - stratum_send_message(client, "Pool Initialising"); + if (unlikely(!sdata->current_workbase)) { + stratum_send_message(sdata, client, "Pool Initialising"); return json_string("Initialising"); } @@ -1543,7 +1569,7 @@ static json_t *parse_subscribe(stratum_instance_t *client, int64_t client_id, js buf = json_string_value(json_array_get(params_val, 1)); LOGDEBUG("Found old session id %s", buf); /* Add matching here */ - if (disconnected_sessionid_exists(buf, client_id)) { + if (disconnected_sessionid_exists(sdata, buf, client_id)) { sprintf(client->enonce1, "%016lx", client->enonce1_64); old_match = true; } @@ -1553,7 +1579,7 @@ static json_t *parse_subscribe(stratum_instance_t *client, int64_t client_id, js if (!old_match) { /* Create a new extranonce1 based on a uint64_t pointer */ if (!new_enonce1(client)) { - stratum_send_message(client, "Pool full of clients"); + stratum_send_message(sdata, client, "Pool full of clients"); client->reject = 2; return json_string("proxy full"); } @@ -1564,14 +1590,14 @@ static json_t *parse_subscribe(stratum_instance_t *client, int64_t client_id, js client->enonce1); } - ck_rlock(&workbase_lock); - if (likely(workbases)) - n2len = workbases->enonce2varlen; + ck_rlock(&sdata->workbase_lock); + if (likely(sdata->workbases)) + n2len = sdata->workbases->enonce2varlen; else n2len = 8; JSON_CPACK(ret, "[[[s,s]],s,i]", "mining.notify", client->enonce1, client->enonce1, n2len); - ck_runlock(&workbase_lock); + ck_runlock(&sdata->workbase_lock); client->subscribed = true; @@ -1708,6 +1734,7 @@ static user_instance_t *generate_user(ckpool_t *ckp, stratum_instance_t *client, const char *workername) { char *base_username = strdupa(workername), *username; + sdata_t *sdata = ckp->data; user_instance_t *instance; stratum_instance_t *tmp; bool new = false; @@ -1720,16 +1747,16 @@ static user_instance_t *generate_user(ckpool_t *ckp, stratum_instance_t *client, if (unlikely(len > 127)) username[127] = '\0'; - ck_wlock(&instance_lock); - HASH_FIND_STR(user_instances, username, instance); + ck_wlock(&sdata->instance_lock); + HASH_FIND_STR(sdata->user_instances, username, instance); if (!instance) { /* New user instance. Secondary user id will be NULL */ instance = ckzalloc(sizeof(user_instance_t)); strcpy(instance->username, username); new = true; - instance->id = user_instance_id++; - HASH_ADD_STR(user_instances, username, instance); + instance->id = sdata->user_instance_id++; + HASH_ADD_STR(sdata->user_instances, username, instance); read_userstats(ckp, instance); } DL_FOREACH(instance->instances, tmp) { @@ -1751,7 +1778,7 @@ static user_instance_t *generate_user(ckpool_t *ckp, stratum_instance_t *client, client->worker_instance = worker; } DL_APPEND(instance->instances, client); - ck_wunlock(&instance_lock); + ck_wunlock(&sdata->instance_lock); if (new && !ckp->proxy) { /* Is this a btc address based username? */ @@ -1765,13 +1792,14 @@ static user_instance_t *generate_user(ckpool_t *ckp, stratum_instance_t *client, } /* Send this to the database and parse the response to authorise a user - * and get SUID parameters back. We don't add these requests to the ckdbqueue + * and get SUID parameters back. We don't add these requests to the sdata->ckdbqueue * since we have to wait for the response but this is done from the authoriser * thread so it won't hold anything up but other authorisations. */ static int send_recv_auth(stratum_instance_t *client) { user_instance_t *user_instance = client->user_instance; ckpool_t *ckp = client->ckp; + sdata_t *sdata = ckp->data; char *buf = NULL, *json_msg; char cdfield[64]; int ret = 1; @@ -1805,9 +1833,9 @@ static int send_recv_auth(stratum_instance_t *client) /* We want responses from ckdb serialised and not interleaved with * other requests. Wait up to 3 seconds for exclusive access to ckdb * and if we don't receive it treat it as a delayed auth if possible */ - if (likely(!mutex_timedlock(&ckdb_lock, 3))) { + if (likely(!mutex_timedlock(&sdata->ckdb_lock, 3))) { buf = ckdb_msg_call(ckp, json_msg); - mutex_unlock(&ckdb_lock); + mutex_unlock(&sdata->ckdb_lock); } free(json_msg); @@ -1883,6 +1911,7 @@ static json_t *parse_authorise(stratum_instance_t *client, json_t *params_val, j const char *address, int *errnum) { user_instance_t *user_instance; + ckpool_t *ckp = client->ckp; bool ret = false; const char *buf; int arr_size; @@ -1918,7 +1947,7 @@ static json_t *parse_authorise(stratum_instance_t *client, json_t *params_val, j *err_val = json_string("Invalid character in username"); goto out; } - user_instance = client->user_instance = generate_user(client->ckp, client, buf); + user_instance = client->user_instance = generate_user(ckp, client, buf); client->user_id = user_instance->id; ts_realtime(&now); client->start_time = now.tv_sec; @@ -1927,7 +1956,7 @@ static json_t *parse_authorise(stratum_instance_t *client, json_t *params_val, j LOGNOTICE("Authorised client %ld worker %s as user %s", client->id, buf, user_instance->username); client->workername = strdup(buf); - if (CKP_STANDALONE(client->ckp)) + if (CKP_STANDALONE(ckp)) ret = true; else { *errnum = send_recv_auth(client); @@ -1943,27 +1972,27 @@ static json_t *parse_authorise(stratum_instance_t *client, json_t *params_val, j } client->authorised = ret; if (client->authorised) - inc_worker(user_instance); + inc_worker(ckp, user_instance); out: return json_boolean(ret); } -static void stratum_send_diff(stratum_instance_t *client) +static void stratum_send_diff(sdata_t *sdata, stratum_instance_t *client) { json_t *json_msg; JSON_CPACK(json_msg, "{s[I]soss}", "params", client->diff, "id", json_null(), "method", "mining.set_difficulty"); - stratum_add_send(json_msg, client->id); + stratum_add_send(sdata, json_msg, client->id); } -static void stratum_send_message(stratum_instance_t *client, const char *msg) +static void stratum_send_message(sdata_t *sdata, stratum_instance_t *client, const char *msg) { json_t *json_msg; JSON_CPACK(json_msg, "{sosss[s]}", "id", json_null(), "method", "client.show_message", "params", msg); - stratum_add_send(json_msg, client->id); + stratum_add_send(sdata, json_msg, client->id); } static double time_bias(double tdiff, double period) @@ -1993,15 +2022,16 @@ static void add_submit(ckpool_t *ckp, stratum_instance_t *client, int diff, bool double tdiff, bdiff, dsps, drr, network_diff, bias; user_instance_t *instance = client->user_instance; int64_t next_blockid, optimal; + sdata_t *sdata = ckp->data; tv_t now_t; - mutex_lock(&stats_lock); + mutex_lock(&sdata->stats_lock); if (valid) { - stats.unaccounted_shares++; - stats.unaccounted_diff_shares += diff; + sdata->stats.unaccounted_shares++; + sdata->stats.unaccounted_diff_shares += diff; } else - stats.unaccounted_rejects += diff; - mutex_unlock(&stats_lock); + sdata->stats.unaccounted_rejects += diff; + mutex_unlock(&sdata->stats_lock); /* Count only accepted and stale rejects in diff calculation. */ if (!valid && !submit) @@ -2009,13 +2039,13 @@ static void add_submit(ckpool_t *ckp, stratum_instance_t *client, int diff, bool tv_time(&now_t); - ck_rlock(&workbase_lock); - next_blockid = workbase_id + 1; + ck_rlock(&sdata->workbase_lock); + next_blockid = sdata->workbase_id + 1; if (ckp->proxy) - network_diff = current_workbase->diff; + network_diff = sdata->current_workbase->diff; else - network_diff = current_workbase->network_diff; - ck_runlock(&workbase_lock); + network_diff = sdata->current_workbase->network_diff; + ck_runlock(&sdata->workbase_lock); if (unlikely(!client->first_share.tv_sec)) { copy_tv(&client->first_share, &now_t); @@ -2111,7 +2141,7 @@ static void add_submit(ckpool_t *ckp, stratum_instance_t *client, int diff, bool client->diff_change_job_id = next_blockid; client->old_diff = client->diff; client->diff = optimal; - stratum_send_diff(client); + stratum_send_diff(sdata, client); } /* We should already be holding the workbase_lock */ @@ -2123,14 +2153,15 @@ test_blocksolve(stratum_instance_t *client, workbase_t *wb, const uchar *data, c char hexcoinbase[1024], blockhash[68]; json_t *val = NULL, *val_copy; char *gbt_block, varint[12]; + ckpool_t *ckp = wb->ckp; + sdata_t *sdata = ckp->data; ckmsg_t *block_ckmsg; char cdfield[64]; uchar swap[32]; - ckpool_t *ckp; ts_t ts_now; /* Submit anything over 99% of the diff in case of rounding errors */ - if (diff < current_workbase->network_diff * 0.99) + if (diff < sdata->current_workbase->network_diff * 0.99) return; LOGWARNING("Possible block solve diff %f !", diff); @@ -2168,7 +2199,6 @@ test_blocksolve(stratum_instance_t *client, workbase_t *wb, const uchar *data, c strcat(gbt_block, hexcoinbase); if (wb->transactions) realloc_strcat(&gbt_block, wb->txn_data); - ckp = wb->ckp; send_generator(ckp, gbt_block, GEN_PRIORITY); free(gbt_block); @@ -2192,9 +2222,9 @@ test_blocksolve(stratum_instance_t *client, workbase_t *wb, const uchar *data, c block_ckmsg = ckalloc(sizeof(ckmsg_t)); block_ckmsg->data = val_copy; - mutex_lock(&block_lock); - DL_APPEND(block_solves, block_ckmsg); - mutex_unlock(&block_lock); + mutex_lock(&sdata->block_lock); + DL_APPEND(sdata->block_solves, block_ckmsg); + mutex_unlock(&sdata->block_lock); ckdbq_add(ckp, ID_BLOCK, val); } @@ -2259,22 +2289,22 @@ static double submission_diff(stratum_instance_t *client, workbase_t *wb, const return ret; } -static bool new_share(const uchar *hash, int64_t wb_id) +static bool new_share(sdata_t *sdata, const uchar *hash, int64_t wb_id) { share_t *share, *match = NULL; bool ret = false; - ck_wlock(&share_lock); - HASH_FIND(hh, shares, hash, 32, match); + ck_wlock(&sdata->share_lock); + HASH_FIND(hh, sdata->shares, hash, 32, match); if (match) goto out_unlock; share = ckzalloc(sizeof(share_t)); memcpy(share->hash, hash, 32); share->workbase_id = wb_id; - HASH_ADD(hh, shares, hash, 32, share); + HASH_ADD(hh, sdata->shares, hash, 32, share); ret = true; out_unlock: - ck_wunlock(&share_lock); + ck_wunlock(&sdata->share_lock); return ret; } @@ -2311,6 +2341,7 @@ static json_t *parse_submit(stratum_instance_t *client, json_t *json_msg, char *fname = NULL, *s, *nonce2; enum share_err err = SE_NONE; ckpool_t *ckp = client->ckp; + sdata_t *sdata = ckp->data; char idstring[20] = {}; workbase_t *wb = NULL; uint32_t ntime32; @@ -2376,14 +2407,14 @@ static json_t *parse_submit(stratum_instance_t *client, json_t *json_msg, share = true; - ck_rlock(&workbase_lock); - HASH_FIND_I64(workbases, &id, wb); + ck_rlock(&sdata->workbase_lock); + HASH_FIND_I64(sdata->workbases, &id, wb); if (unlikely(!wb)) { - id = current_workbase->id; + id = sdata->current_workbase->id; err = SE_INVALID_JOBID; json_set_string(json_msg, "reject-reason", SHARE_ERR(err)); strncpy(idstring, job_id, 19); - ASPRINTF(&fname, "%s.sharelog", current_workbase->logdir); + ASPRINTF(&fname, "%s.sharelog", sdata->current_workbase->logdir); goto out_unlock; } wdiff = wb->diff; @@ -2407,7 +2438,7 @@ static json_t *parse_submit(stratum_instance_t *client, json_t *json_msg, bswap_256(sharehash, hash); __bin2hex(hexhash, sharehash, 32); - if (id < blockchange_id) { + if (id < sdata->blockchange_id) { err = SE_STALE; json_set_string(json_msg, "reject-reason", SHARE_ERR(err)); goto out_submit; @@ -2423,7 +2454,7 @@ out_submit: if (sdiff >= wdiff) submit = true; out_unlock: - ck_runlock(&workbase_lock); + ck_runlock(&sdata->workbase_lock); /* Accept the lower of new and old diffs until the next update */ if (id < client->diff_change_job_id && client->old_diff < client->diff) @@ -2433,7 +2464,7 @@ out_unlock: suffix_string(wdiff, wdiffsuffix, 16, 0); if (sdiff >= diff) { - if (new_share(hash, id)) { + if (new_share(sdata, hash, id)) { LOGINFO("Accepted client %ld share diff %.1f/%.0f/%s: %s", client->id, sdiff, diff, wdiffsuffix, hexhash); result = true; @@ -2508,12 +2539,12 @@ out: client->first_invalid = now_t; else if (client->first_invalid && client->first_invalid < now_t - 120) { LOGNOTICE("Client %d rejecting for 120s, disconnecting", client->id); - stratum_send_message(client, "Disconnecting for continuous invalid shares"); + stratum_send_message(sdata, client, "Disconnecting for continuous invalid shares"); client->reject = 2; } else if (client->first_invalid && client->first_invalid < now_t - 60) { if (!client->reject) { LOGINFO("Client %d rejecting for 60s, sending diff", client->id); - stratum_send_diff(client); + stratum_send_diff(sdata, client); client->reject = 1; } } @@ -2527,7 +2558,7 @@ out: "clientid", client->id, "secondaryuserid", user_instance->secondaryuserid, "enonce1", client->enonce1, - "workinfoid", current_workbase->id, + "workinfoid", sdata->current_workbase->id, "workername", client->workername, "username", user_instance->username, "error", json_copy(*err_val), @@ -2544,61 +2575,61 @@ out: } /* Must enter with workbase_lock held */ -static json_t *__stratum_notify(bool clean) +static json_t *__stratum_notify(sdata_t *sdata, bool clean) { json_t *val; JSON_CPACK(val, "{s:[ssssosssb],s:o,s:s}", "params", - current_workbase->idstring, - current_workbase->prevhash, - current_workbase->coinb1, - current_workbase->coinb2, - json_deep_copy(current_workbase->merkle_array), - current_workbase->bbversion, - current_workbase->nbit, - current_workbase->ntime, + sdata->current_workbase->idstring, + sdata->current_workbase->prevhash, + sdata->current_workbase->coinb1, + sdata->current_workbase->coinb2, + json_deep_copy(sdata->current_workbase->merkle_array), + sdata->current_workbase->bbversion, + sdata->current_workbase->nbit, + sdata->current_workbase->ntime, clean, "id", json_null(), "method", "mining.notify"); return val; } -static void stratum_broadcast_update(bool clean) +static void stratum_broadcast_update(sdata_t *sdata, bool clean) { json_t *json_msg; - ck_rlock(&workbase_lock); - json_msg = __stratum_notify(clean); - ck_runlock(&workbase_lock); + ck_rlock(&sdata->workbase_lock); + json_msg = __stratum_notify(sdata, clean); + ck_runlock(&sdata->workbase_lock); - stratum_broadcast(json_msg); + stratum_broadcast(sdata, json_msg); } /* For sending a single stratum template update */ -static void stratum_send_update(int64_t client_id, bool clean) +static void stratum_send_update(sdata_t *sdata, int64_t client_id, bool clean) { json_t *json_msg; - ck_rlock(&workbase_lock); - json_msg = __stratum_notify(clean); - ck_runlock(&workbase_lock); + ck_rlock(&sdata->workbase_lock); + json_msg = __stratum_notify(sdata, clean); + ck_runlock(&sdata->workbase_lock); - stratum_add_send(json_msg, client_id); + stratum_add_send(sdata, json_msg, client_id); } -static void send_json_err(int64_t client_id, json_t *id_val, const char *err_msg) +static void send_json_err(sdata_t *sdata, int64_t client_id, json_t *id_val, const char *err_msg) { json_t *val; JSON_CPACK(val, "{soss}", "id", json_copy(id_val), "error", err_msg); - stratum_add_send(val, client_id); + stratum_add_send(sdata, val, client_id); } -static void update_client(stratum_instance_t *client, const int64_t client_id) +static void update_client(sdata_t *sdata, stratum_instance_t *client, const int64_t client_id) { - stratum_send_update(client_id, true); - stratum_send_diff(client); + stratum_send_update(sdata, client_id, true); + stratum_send_diff(sdata, client); } static json_params_t @@ -2621,28 +2652,29 @@ static void set_worker_mindiff(ckpool_t *ckp, const char *workername, int mindif char *username = strdupa(workername), *ignore; user_instance_t *instance = NULL; stratum_instance_t *client; + sdata_t *sdata = ckp->data; ignore = username; strsep(&ignore, "._"); /* Find the user first */ - ck_rlock(&instance_lock); - HASH_FIND_STR(user_instances, username, instance); - ck_runlock(&instance_lock); + ck_rlock(&sdata->instance_lock); + HASH_FIND_STR(sdata->user_instances, username, instance); + ck_runlock(&sdata->instance_lock); /* They may just have not connected yet */ if (!instance) return LOGINFO("Failed to find user %s in set_worker_mindiff", username); /* Then find the matching worker instance */ - ck_rlock(&instance_lock); + ck_rlock(&sdata->instance_lock); DL_FOREACH(instance->worker_instances, tmp) { if (!safecmp(workername, tmp->workername)) { worker = tmp; break; } } - ck_runlock(&instance_lock); + ck_runlock(&sdata->instance_lock); /* They may just not be connected at the moment */ if (!worker) @@ -2660,7 +2692,7 @@ static void set_worker_mindiff(ckpool_t *ckp, const char *workername, int mindif * matching worker that are currently live and send them a new diff * if we can. Otherwise it will only act as a clamp on next share * submission. */ - ck_rlock(&instance_lock); + ck_rlock(&sdata->instance_lock); DL_FOREACH(instance->instances, client) { if (client->worker_instance != worker) continue; @@ -2670,9 +2702,9 @@ static void set_worker_mindiff(ckpool_t *ckp, const char *workername, int mindif if (mindiff == client->diff) continue; client->diff = mindiff; - stratum_send_diff(client); + stratum_send_diff(sdata, client); } - ck_runlock(&instance_lock); + ck_runlock(&sdata->instance_lock); } /* Implement support for the diff in the params as well as the originally @@ -2680,6 +2712,7 @@ static void set_worker_mindiff(ckpool_t *ckp, const char *workername, int mindif static void suggest_diff(stratum_instance_t *client, const char *method, json_t *params_val) { json_t *arr_val = json_array_get(params_val, 0); + sdata_t *sdata = client->ckp->data; int64_t sdiff; if (unlikely(!client->authorised)) @@ -2697,19 +2730,19 @@ static void suggest_diff(stratum_instance_t *client, const char *method, json_t client->diff = client->ckp->mindiff; else client->diff = sdiff; - stratum_send_diff(client); + stratum_send_diff(sdata, client); } -static void parse_method(const int64_t client_id, json_t *id_val, json_t *method_val, - json_t *params_val, char *address) +static void parse_method(sdata_t *sdata, const int64_t client_id, json_t *id_val, + json_t *method_val, json_t *params_val, char *address) { stratum_instance_t *client; const char *method; char buf[256]; - ck_rlock(&instance_lock); - client = __instance_by_id(client_id); - ck_runlock(&instance_lock); + ck_rlock(&sdata->instance_lock); + client = __instance_by_id(sdata, client_id); + ck_runlock(&sdata->instance_lock); if (unlikely(!client)) { LOGINFO("Failed to find client id %ld in hashtable!", client_id); @@ -2738,9 +2771,9 @@ static void parse_method(const int64_t client_id, json_t *id_val, json_t *method json_object_set_new_nocheck(val, "result", result_val); json_object_set_nocheck(val, "id", id_val); json_object_set_new_nocheck(val, "error", json_null()); - stratum_add_send(val, client_id); + stratum_add_send(sdata, val, client_id); if (likely(client->subscribed)) - update_client(client, client_id); + update_client(sdata, client, client_id); return; } @@ -2749,9 +2782,9 @@ static void parse_method(const int64_t client_id, json_t *id_val, json_t *method * is a passthrough and to manage its messages accordingly. * Remove this instance since the client id may well be * reused */ - ck_wlock(&instance_lock); - HASH_DEL(stratum_instances, client); - ck_wunlock(&instance_lock); + ck_wlock(&sdata->instance_lock); + HASH_DEL(sdata->stratum_instances, client); + ck_wunlock(&sdata->instance_lock); LOGINFO("Adding passthrough client %ld", client->id); snprintf(buf, 255, "passthrough=%ld", client->id); @@ -2763,7 +2796,7 @@ static void parse_method(const int64_t client_id, json_t *id_val, json_t *method if (cmdmatch(method, "mining.auth") && client->subscribed) { json_params_t *jp = create_json_params(client_id, method_val, params_val, id_val, address); - ckmsgq_add(sauthq, jp); + ckmsgq_add(sdata->sauthq, jp); return; } @@ -2781,7 +2814,7 @@ static void parse_method(const int64_t client_id, json_t *id_val, json_t *method if (cmdmatch(method, "mining.submit")) { json_params_t *jp = create_json_params(client_id, method_val, params_val, id_val, address); - ckmsgq_add(sshareq, jp); + ckmsgq_add(sdata->sshareq, jp); return; } @@ -2794,13 +2827,13 @@ static void parse_method(const int64_t client_id, json_t *id_val, json_t *method if (cmdmatch(method, "mining.get")) { json_params_t *jp = create_json_params(client_id, method_val, params_val, id_val, address); - ckmsgq_add(stxnq, jp); + ckmsgq_add(sdata->stxnq, jp); return; } /* Unhandled message here */ } -static void parse_instance_msg(smsg_t *msg) +static void parse_instance_msg(sdata_t *sdata, smsg_t *msg) { json_t *val = msg->json_msg, *id_val, *method, *params; int64_t client_id = msg->client_id; @@ -2819,19 +2852,19 @@ static void parse_instance_msg(smsg_t *msg) LOGDEBUG("Received spurious response %s", result ? result : ""); goto out; } - send_json_err(client_id, id_val, "-3:method not found"); + send_json_err(sdata, client_id, id_val, "-3:method not found"); goto out; } if (unlikely(!json_is_string(method))) { - send_json_err(client_id, id_val, "-1:method is not string"); + send_json_err(sdata, client_id, id_val, "-1:method is not string"); goto out; } params = json_object_get(val, "params"); if (unlikely(!params)) { - send_json_err(client_id, id_val, "-1:params not found"); + send_json_err(sdata, client_id, id_val, "-1:params not found"); goto out; } - parse_method(client_id, id_val, method, params, msg->address); + parse_method(sdata, client_id, id_val, method, params, msg->address); out: json_decref(val); free(msg); @@ -2840,6 +2873,7 @@ out: static void srecv_process(ckpool_t *ckp, char *buf) { stratum_instance_t *instance; + sdata_t *sdata = ckp->data; smsg_t *msg; json_t *val; @@ -2872,15 +2906,15 @@ static void srecv_process(ckpool_t *ckp, char *buf) json_object_clear(val); /* Parse the message here */ - ck_wlock(&instance_lock); - instance = __instance_by_id(msg->client_id); + ck_wlock(&sdata->instance_lock); + instance = __instance_by_id(sdata, msg->client_id); if (!instance) { /* client_id instance doesn't exist yet, create one */ instance = __stratum_add_instance(ckp, msg->client_id); } - ck_wunlock(&instance_lock); + ck_wunlock(&sdata->instance_lock); - parse_instance_msg(msg); + parse_instance_msg(sdata, msg); out: free(buf); } @@ -2920,17 +2954,18 @@ static void discard_json_params(json_params_t **jp) *jp = NULL; } -static void sshare_process(ckpool_t __maybe_unused *ckp, json_params_t *jp) +static void sshare_process(ckpool_t *ckp, json_params_t *jp) { json_t *result_val, *json_msg, *err_val = NULL; stratum_instance_t *client; + sdata_t *sdata = ckp->data; int64_t client_id; client_id = jp->client_id; - ck_rlock(&instance_lock); - client = __instance_by_id(client_id); - ck_runlock(&instance_lock); + ck_rlock(&sdata->instance_lock); + client = __instance_by_id(sdata, client_id); + ck_runlock(&sdata->instance_lock); if (unlikely(!client)) { LOGINFO("Share processor failed to find client id %ld in hashtable!", client_id); @@ -2945,7 +2980,7 @@ static void sshare_process(ckpool_t __maybe_unused *ckp, json_params_t *jp) json_object_set_new_nocheck(json_msg, "result", result_val); json_object_set_new_nocheck(json_msg, "error", err_val ? err_val : json_null()); json_object_set_nocheck(json_msg, "id", jp->id_val); - stratum_add_send(json_msg, client_id); + stratum_add_send(sdata, json_msg, client_id); out: discard_json_params(&jp); } @@ -2954,14 +2989,15 @@ static void sauth_process(ckpool_t *ckp, json_params_t *jp) { json_t *result_val, *json_msg, *err_val = NULL; stratum_instance_t *client; + sdata_t *sdata = ckp->data; int mindiff, errnum = 0; int64_t client_id; client_id = jp->client_id; - ck_rlock(&instance_lock); - client = __instance_by_id(client_id); - ck_runlock(&instance_lock); + ck_rlock(&sdata->instance_lock); + client = __instance_by_id(sdata, client_id); + ck_runlock(&sdata->instance_lock); if (unlikely(!client)) { LOGINFO("Authoriser failed to find client id %ld in hashtable!", client_id); @@ -2973,18 +3009,18 @@ static void sauth_process(ckpool_t *ckp, json_params_t *jp) ASPRINTF(&buf, "Authorised, welcome to %s %s!", ckp->name, client->user_instance->username); - stratum_send_message(client, buf); + stratum_send_message(sdata, client, buf); } else { if (errnum < 0) - stratum_send_message(client, "Authorisations temporarily offline :("); + stratum_send_message(sdata, client, "Authorisations temporarily offline :("); else - stratum_send_message(client, "Failed authorisation :("); + stratum_send_message(sdata, client, "Failed authorisation :("); } json_msg = json_object(); json_object_set_new_nocheck(json_msg, "result", result_val); json_object_set_new_nocheck(json_msg, "error", err_val ? err_val : json_null()); json_object_set_nocheck(json_msg, "id", jp->id_val); - stratum_add_send(json_msg, client_id); + stratum_add_send(sdata, json_msg, client_id); if (!json_is_true(result_val) || !client->suggest_diff) goto out; @@ -2994,7 +3030,7 @@ static void sauth_process(ckpool_t *ckp, json_params_t *jp) mindiff = MAX(ckp->mindiff, client->suggest_diff); if (mindiff != client->diff) { client->diff = mindiff; - stratum_send_diff(client); + stratum_send_diff(sdata, client); } out: discard_json_params(&jp); @@ -3030,12 +3066,13 @@ static void parse_ckdb_cmd(ckpool_t __maybe_unused *ckp, const char *cmd) static void ckdbq_process(ckpool_t *ckp, char *msg) { static bool failed = false; + sdata_t *sdata = ckp->data; char *buf = NULL; while (!buf) { - mutex_lock(&ckdb_lock); + mutex_lock(&sdata->ckdb_lock); buf = ckdb_msg_call(ckp, msg); - mutex_unlock(&ckdb_lock); + mutex_unlock(&sdata->ckdb_lock); if (unlikely(!buf)) { if (!failed) { @@ -3072,30 +3109,30 @@ static void ckdbq_process(ckpool_t *ckp, char *msg) } } -static int transactions_by_jobid(int64_t id) +static int transactions_by_jobid(sdata_t *sdata, int64_t id) { workbase_t *wb; int ret = -1; - ck_rlock(&workbase_lock); - HASH_FIND_I64(workbases, &id, wb); + ck_rlock(&sdata->workbase_lock); + HASH_FIND_I64(sdata->workbases, &id, wb); if (wb) ret = wb->transactions; - ck_runlock(&workbase_lock); + ck_runlock(&sdata->workbase_lock); return ret; } -static json_t *txnhashes_by_jobid(int64_t id) +static json_t *txnhashes_by_jobid(sdata_t *sdata, int64_t id) { json_t *ret = NULL; workbase_t *wb; - ck_rlock(&workbase_lock); - HASH_FIND_I64(workbases, &id, wb); + ck_rlock(&sdata->workbase_lock); + HASH_FIND_I64(sdata->workbases, &id, wb); if (wb) ret = json_string(wb->txn_hashes); - ck_runlock(&workbase_lock); + ck_runlock(&sdata->workbase_lock); return ret; } @@ -3105,6 +3142,7 @@ static void send_transactions(ckpool_t *ckp, json_params_t *jp) const char *msg = json_string_value(jp->method), *params = json_string_value(json_array_get(jp->params, 0)); stratum_instance_t *client; + sdata_t *sdata = ckp->data; json_t *val, *hashes; int64_t job_id = 0; time_t now_t; @@ -3126,7 +3164,7 @@ static void send_transactions(ckpool_t *ckp, json_params_t *jp) sscanf(params, "%lx", &job_id); else sscanf(msg, "mining.get_transactions(%lx", &job_id); - txns = transactions_by_jobid(job_id); + txns = transactions_by_jobid(sdata, job_id); if (txns != -1) { json_set_int(val, "result", txns); json_object_set_new_nocheck(val, "error", json_null()); @@ -3140,9 +3178,9 @@ static void send_transactions(ckpool_t *ckp, json_params_t *jp) goto out_send; } - ck_rlock(&instance_lock); - client = __instance_by_id(jp->client_id); - ck_runlock(&instance_lock); + ck_rlock(&sdata->instance_lock); + client = __instance_by_id(sdata, jp->client_id); + ck_runlock(&sdata->instance_lock); if (unlikely(!client)) { LOGINFO("send_transactions failed to find client id %ld in hashtable!", @@ -3162,14 +3200,14 @@ static void send_transactions(ckpool_t *ckp, json_params_t *jp) goto out_send; } sscanf(params, "%lx", &job_id); - hashes = txnhashes_by_jobid(job_id); + hashes = txnhashes_by_jobid(sdata, job_id); if (hashes) { json_object_set_new_nocheck(val, "result", hashes); json_object_set_new_nocheck(val, "error", json_null()); } else json_set_string(val, "error", "Invalid job_id"); out_send: - stratum_add_send(val, jp->client_id); + stratum_add_send(sdata, val, jp->client_id); out: discard_json_params(&jp); } @@ -3177,29 +3215,29 @@ out: /* Called every 20 seconds, we send the updated stats to ckdb of those users * who have gone 10 minutes between updates. This ends up staggering stats to * avoid floods of stat data coming at once. */ -static void update_workerstats(ckpool_t *ckp) +static void update_workerstats(ckpool_t *ckp, sdata_t *sdata) { user_instance_t *user, *tmp; char cdfield[64]; time_t now_t; ts_t ts_now; - if (++stats.userstats_cycle > 0x1f) - stats.userstats_cycle = 0; + if (++sdata->stats.userstats_cycle > 0x1f) + sdata->stats.userstats_cycle = 0; ts_realtime(&ts_now); sprintf(cdfield, "%lu,%lu", ts_now.tv_sec, ts_now.tv_nsec); now_t = ts_now.tv_sec; - ck_rlock(&instance_lock); - HASH_ITER(hh, user_instances, user, tmp) { + ck_rlock(&sdata->instance_lock); + HASH_ITER(hh, sdata->user_instances, user, tmp) { worker_instance_t *worker; uint8_t cycle_mask; /* Select users using a mask to return each user's stats once * every ~10 minutes */ cycle_mask = user->id & 0x1f; - if (cycle_mask != stats.userstats_cycle) + if (cycle_mask != sdata->stats.userstats_cycle) continue; DL_FOREACH(user->worker_instances, worker) { double ghs1, ghs5, ghs60, ghs1440; @@ -3234,18 +3272,20 @@ static void update_workerstats(ckpool_t *ckp) ckdbq_add(ckp, ID_WORKERSTATS, val); } } - ck_runlock(&instance_lock); + ck_runlock(&sdata->instance_lock); } static void *statsupdate(void *arg) { ckpool_t *ckp = (ckpool_t *)arg; + sdata_t *sdata = ckp->data; + pool_stats_t *stats = &sdata->stats; pthread_detach(pthread_self()); rename_proc("statsupdate"); - tv_time(&stats.start_time); - cksleep_prepare_r(&stats.last_update); + tv_time(&stats->start_time); + cksleep_prepare_r(&stats->last_update); sleep(1); while (42) { @@ -3266,38 +3306,38 @@ static void *statsupdate(void *arg) int i; tv_time(&now); - timersub(&now, &stats.start_time, &diff); + timersub(&now, &stats->start_time, &diff); tdiff = diff.tv_sec + (double)diff.tv_usec / 1000000; - ghs1 = stats.dsps1 * nonces; + ghs1 = stats->dsps1 * nonces; suffix_string(ghs1, suffix1, 16, 0); - sps1 = stats.sps1; + sps1 = stats->sps1; bias5 = time_bias(tdiff, 300); - ghs5 = stats.dsps5 * nonces / bias5; + ghs5 = stats->dsps5 * nonces / bias5; suffix_string(ghs5, suffix5, 16, 0); - sps5 = stats.sps5 / bias5; + sps5 = stats->sps5 / bias5; bias = time_bias(tdiff, 900); - ghs15 = stats.dsps15 * nonces / bias; + ghs15 = stats->dsps15 * nonces / bias; suffix_string(ghs15, suffix15, 16, 0); - sps15 = stats.sps15 / bias; + sps15 = stats->sps15 / bias; bias60 = time_bias(tdiff, 3600); - ghs60 = stats.dsps60 * nonces / bias60; + ghs60 = stats->dsps60 * nonces / bias60; suffix_string(ghs60, suffix60, 16, 0); - sps60 = stats.sps60 / bias60; + sps60 = stats->sps60 / bias60; bias = time_bias(tdiff, 21600); - ghs360 = stats.dsps360 * nonces / bias; + ghs360 = stats->dsps360 * nonces / bias; suffix_string(ghs360, suffix360, 16, 0); bias1440 = time_bias(tdiff, 86400); - ghs1440 = stats.dsps1440 * nonces / bias1440; + ghs1440 = stats->dsps1440 * nonces / bias1440; suffix_string(ghs1440, suffix1440, 16, 0); bias = time_bias(tdiff, 604800); - ghs10080 = stats.dsps10080 * nonces / bias; + ghs10080 = stats->dsps10080 * nonces / bias; suffix_string(ghs10080, suffix10080, 16, 0); snprintf(fname, 511, "%s/pool/pool.status", ckp->logdir); @@ -3307,8 +3347,8 @@ static void *statsupdate(void *arg) JSON_CPACK(val, "{si,si,si}", "runtime", diff.tv_sec, - "Users", stats.users, - "Workers", stats.workers); + "Users", stats->users, + "Workers", stats->workers); s = json_dumps(val, JSON_NO_UTF8 | JSON_PRESERVE_ORDER); json_decref(val); LOGNOTICE("Pool:%s", s); @@ -3341,8 +3381,8 @@ static void *statsupdate(void *arg) dealloc(s); fclose(fp); - ck_rlock(&instance_lock); - HASH_ITER(hh, stratum_instances, client, tmp) { + ck_rlock(&sdata->instance_lock); + HASH_ITER(hh, sdata->stratum_instances, client, tmp) { if (!client->authorised) continue; @@ -3361,7 +3401,7 @@ static void *statsupdate(void *arg) } } - HASH_ITER(hh, user_instances, instance, tmpuser) { + HASH_ITER(hh, sdata->user_instances, instance, tmpuser) { worker_instance_t *worker; bool idle = false; @@ -3453,15 +3493,15 @@ static void *statsupdate(void *arg) json_decref(val); fclose(fp); } - ck_runlock(&instance_lock); + ck_runlock(&sdata->instance_lock); ts_realtime(&ts_now); sprintf(cdfield, "%lu,%lu", ts_now.tv_sec, ts_now.tv_nsec); JSON_CPACK(val, "{ss,si,si,si,sf,sf,sf,sf,ss,ss,ss,ss}", "poolinstance", ckp->name, "elapsed", diff.tv_sec, - "users", stats.users, - "workers", stats.workers, + "users", stats->users, + "workers", stats->workers, "hashrate", ghs1, "hashrate5m", ghs5, "hashrate1hr", ghs60, @@ -3475,32 +3515,32 @@ static void *statsupdate(void *arg) /* Update stats 3 times per minute for smooth values, displaying * status every minute. */ for (i = 0; i < 3; i++) { - cksleep_ms_r(&stats.last_update, 20000); - cksleep_prepare_r(&stats.last_update); - update_workerstats(ckp); - - mutex_lock(&stats_lock); - stats.accounted_shares += stats.unaccounted_shares; - stats.accounted_diff_shares += stats.unaccounted_diff_shares; - stats.accounted_rejects += stats.unaccounted_rejects; - - decay_time(&stats.sps1, stats.unaccounted_shares, 20, 60); - decay_time(&stats.sps5, stats.unaccounted_shares, 20, 300); - decay_time(&stats.sps15, stats.unaccounted_shares, 20, 900); - decay_time(&stats.sps60, stats.unaccounted_shares, 20, 3600); - - decay_time(&stats.dsps1, stats.unaccounted_diff_shares, 20, 60); - decay_time(&stats.dsps5, stats.unaccounted_diff_shares, 20, 300); - decay_time(&stats.dsps15, stats.unaccounted_diff_shares, 20, 900); - decay_time(&stats.dsps60, stats.unaccounted_diff_shares, 20, 3600); - decay_time(&stats.dsps360, stats.unaccounted_diff_shares, 20, 21600); - decay_time(&stats.dsps1440, stats.unaccounted_diff_shares, 20, 86400); - decay_time(&stats.dsps10080, stats.unaccounted_diff_shares, 20, 604800); - - stats.unaccounted_shares = - stats.unaccounted_diff_shares = - stats.unaccounted_rejects = 0; - mutex_unlock(&stats_lock); + cksleep_ms_r(&stats->last_update, 20000); + cksleep_prepare_r(&stats->last_update); + update_workerstats(ckp, sdata); + + mutex_lock(&sdata->stats_lock); + stats->accounted_shares += stats->unaccounted_shares; + stats->accounted_diff_shares += stats->unaccounted_diff_shares; + stats->accounted_rejects += stats->unaccounted_rejects; + + decay_time(&stats->sps1, stats->unaccounted_shares, 20, 60); + decay_time(&stats->sps5, stats->unaccounted_shares, 20, 300); + decay_time(&stats->sps15, stats->unaccounted_shares, 20, 900); + decay_time(&stats->sps60, stats->unaccounted_shares, 20, 3600); + + decay_time(&stats->dsps1, stats->unaccounted_diff_shares, 20, 60); + decay_time(&stats->dsps5, stats->unaccounted_diff_shares, 20, 300); + decay_time(&stats->dsps15, stats->unaccounted_diff_shares, 20, 900); + decay_time(&stats->dsps60, stats->unaccounted_diff_shares, 20, 3600); + decay_time(&stats->dsps360, stats->unaccounted_diff_shares, 20, 21600); + decay_time(&stats->dsps1440, stats->unaccounted_diff_shares, 20, 86400); + decay_time(&stats->dsps10080, stats->unaccounted_diff_shares, 20, 604800); + + stats->unaccounted_shares = + stats->unaccounted_diff_shares = + stats->unaccounted_rejects = 0; + mutex_unlock(&sdata->stats_lock); } } @@ -3513,6 +3553,7 @@ static void *statsupdate(void *arg) static void *ckdb_heartbeat(void *arg) { ckpool_t *ckp = (ckpool_t *)arg; + sdata_t *sdata = ckp->data; pthread_detach(pthread_self()); rename_proc("heartbeat"); @@ -3523,7 +3564,7 @@ static void *ckdb_heartbeat(void *arg) json_t *val; cksleep_ms(1000); - if (unlikely(!ckmsgq_empty(ckdbq))) { + if (unlikely(!ckmsgq_empty(sdata->ckdbq))) { LOGDEBUG("Witholding heartbeat due to ckdb messages being queued"); continue; } @@ -3544,9 +3585,12 @@ int stratifier(proc_instance_t *pi) pthread_t pth_blockupdate, pth_statsupdate, pth_heartbeat; ckpool_t *ckp = pi->ckp; int ret = 1, threads; + sdata_t *sdata; char *buf; LOGWARNING("%s stratifier starting", ckp->name); + sdata = ckzalloc(sizeof(sdata_t)); + ckp->data = sdata; /* Wait for the generator to have something for us */ do { @@ -3565,52 +3609,53 @@ int stratifier(proc_instance_t *pi) /* Store this for use elsewhere */ hex2bin(scriptsig_header_bin, scriptsig_header, 41); - address_to_pubkeytxn(pubkeytxnbin, ckp->btcaddress); + address_to_pubkeytxn(sdata->pubkeytxnbin, ckp->btcaddress); if (test_address(ckp, ckp->donaddress)) { ckp->donvalid = true; - address_to_pubkeytxn(donkeytxnbin, ckp->donaddress); + address_to_pubkeytxn(sdata->donkeytxnbin, ckp->donaddress); } } /* Set the initial id to time as high bits so as to not send the same * id on restarts */ if (!ckp->proxy) - blockchange_id = workbase_id = ((int64_t)time(NULL)) << 32; + sdata->blockchange_id = sdata->workbase_id = ((int64_t)time(NULL)) << 32; dealloc(buf); if (!ckp->serverurl) ckp->serverurl = "127.0.0.1"; - cklock_init(&instance_lock); + cklock_init(&sdata->instance_lock); - mutex_init(&ckdb_lock); - ssends = create_ckmsgq(ckp, "ssender", &ssend_process); + mutex_init(&sdata->ckdb_lock); + sdata->ssends = create_ckmsgq(ckp, "ssender", &ssend_process); /* Create half as many share processing threads as there are CPUs */ threads = sysconf(_SC_NPROCESSORS_ONLN) / 2 ? : 1; - sshareq = create_ckmsgqs(ckp, "sprocessor", &sshare_process, threads); + sdata->sshareq = create_ckmsgqs(ckp, "sprocessor", &sshare_process, threads); /* Create 1/4 as many stratum processing threads as there are CPUs */ threads = threads / 2 ? : 1; - srecvs = create_ckmsgqs(ckp, "sreceiver", &srecv_process, threads); - sauthq = create_ckmsgq(ckp, "authoriser", &sauth_process); - ckdbq = create_ckmsgq(ckp, "ckdbqueue", &ckdbq_process); - stxnq = create_ckmsgq(ckp, "stxnq", &send_transactions); + sdata->srecvs = create_ckmsgqs(ckp, "sreceiver", &srecv_process, threads); + sdata->sauthq = create_ckmsgq(ckp, "authoriser", &sauth_process); + sdata->ckdbq = create_ckmsgq(ckp, "ckdbqueue", &ckdbq_process); + sdata->stxnq = create_ckmsgq(ckp, "stxnq", &send_transactions); if (!CKP_STANDALONE(ckp)) create_pthread(&pth_heartbeat, ckdb_heartbeat, ckp); - cklock_init(&workbase_lock); + cklock_init(&sdata->workbase_lock); if (!ckp->proxy) create_pthread(&pth_blockupdate, blockupdate, ckp); - mutex_init(&stats_lock); + mutex_init(&sdata->stats_lock); create_pthread(&pth_statsupdate, statsupdate, ckp); - cklock_init(&share_lock); - mutex_init(&block_lock); + cklock_init(&sdata->share_lock); + mutex_init(&sdata->block_lock); LOGWARNING("%s stratifier ready", ckp->name); ret = stratum_loop(ckp, pi); out: + dealloc(ckp->data); return process_exit(ckp, pi, ret); } From d86c0cd2c0ac2c570d790e0f741d5866c06826d3 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Fri, 7 Nov 2014 17:28:11 +1100 Subject: [PATCH 08/47] Differentiate failed getbase requests that occur for priority reasons from actual comms failures and clarify and demote the messages accordingly --- src/stratifier.c | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/src/stratifier.c b/src/stratifier.c index 4cfcfe29..dcd2e30f 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -705,6 +705,8 @@ static char *__send_recv_generator(ckpool_t *ckp, const char *msg, int prio) } else set = false; buf = send_recv_proc(ckp->generator, msg); + if (unlikely(!buf)) + buf = strdup("failed"); if (set) sdata->gen_priority = 0; @@ -712,7 +714,8 @@ static char *__send_recv_generator(ckpool_t *ckp, const char *msg, int prio) } /* Conditionally send_recv a message only if it's equal or higher priority than - * any currently being serviced. */ + * any currently being serviced. NULL is returned if the request is not + * processed for priority reasons, "failed" for an actual failure. */ static char *send_recv_generator(ckpool_t *ckp, const char *msg, int prio) { sdata_t *sdata = ckp->data; @@ -766,7 +769,7 @@ static void *do_update(void *arg) buf = send_recv_generator(ckp, "getbase", prio); if (unlikely(!buf)) { - LOGWARNING("Failed to get base from generator in update_base"); + LOGNOTICE("Get base in update_base delayed due to higher priority request"); goto out; } if (unlikely(cmdmatch(buf, "failed"))) { @@ -777,7 +780,6 @@ static void *do_update(void *arg) wb = ckzalloc(sizeof(workbase_t)); wb->ckp = ckp; val = json_loads(buf, 0, NULL); - dealloc(buf); json_strcpy(wb->target, val, "target"); json_dblcpy(&wb->diff, val, "diff"); @@ -821,10 +823,11 @@ static void *do_update(void *arg) out: /* Send a ping to miners if we fail to get a base to keep them * connected while bitcoind recovers(?) */ - if (!ret) { - LOGWARNING("Broadcast ping due to failed stratum base update"); + if (unlikely(!ret)) { + LOGINFO("Broadcast ping due to failed stratum base update"); broadcast_ping(sdata); } + dealloc(buf); free(ur->pth); free(ur); return NULL; From f5f5c6a6784f3f756c657d9e2e4f33efcfe1d1a1 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Fri, 7 Nov 2014 17:38:46 +1100 Subject: [PATCH 09/47] Only read off the user and workerstats from previous startup and bias pool stats in standalone mode --- src/stratifier.c | 30 ++++++++++++++++-------------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/src/stratifier.c b/src/stratifier.c index dcd2e30f..e6292f3a 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -1760,7 +1760,8 @@ static user_instance_t *generate_user(ckpool_t *ckp, stratum_instance_t *client, instance->id = sdata->user_instance_id++; HASH_ADD_STR(sdata->user_instances, username, instance); - read_userstats(ckp, instance); + if (CKP_STANDALONE(ckp)) + read_userstats(ckp, instance); } DL_FOREACH(instance->instances, tmp) { if (!safecmp(workername, tmp->workername)) { @@ -1776,7 +1777,8 @@ static user_instance_t *generate_user(ckpool_t *ckp, stratum_instance_t *client, worker->workername = strdup(workername); worker->instance = instance; DL_APPEND(instance->worker_instances, worker); - read_workerstats(ckp, worker); + if (CKP_STANDALONE(ckp)) + read_workerstats(ckp, worker); worker->start_time = time(NULL); client->worker_instance = worker; } @@ -3293,7 +3295,7 @@ static void *statsupdate(void *arg) while (42) { double ghs, ghs1, ghs5, ghs15, ghs60, ghs360, ghs1440, ghs10080; - double bias, bias5, bias60, bias1440; + double bias; double tdiff, per_tdiff; char suffix1[16], suffix5[16], suffix15[16], suffix60[16], cdfield[64]; char suffix360[16], suffix1440[16], suffix10080[16]; @@ -3316,30 +3318,30 @@ static void *statsupdate(void *arg) suffix_string(ghs1, suffix1, 16, 0); sps1 = stats->sps1; - bias5 = time_bias(tdiff, 300); - ghs5 = stats->dsps5 * nonces / bias5; + bias = !CKP_STANDALONE(ckp) ? 1.0 : time_bias(tdiff, 300); + ghs5 = stats->dsps5 * nonces / bias; + sps5 = stats->sps5 / bias; suffix_string(ghs5, suffix5, 16, 0); - sps5 = stats->sps5 / bias5; - bias = time_bias(tdiff, 900); + bias = !CKP_STANDALONE(ckp) ? 1.0 : time_bias(tdiff, 900); ghs15 = stats->dsps15 * nonces / bias; suffix_string(ghs15, suffix15, 16, 0); sps15 = stats->sps15 / bias; - bias60 = time_bias(tdiff, 3600); - ghs60 = stats->dsps60 * nonces / bias60; + bias = !CKP_STANDALONE(ckp) ? 1.0 : time_bias(tdiff, 3600); + ghs60 = stats->dsps60 * nonces / bias; + sps60 = stats->sps60 / bias; suffix_string(ghs60, suffix60, 16, 0); - sps60 = stats->sps60 / bias60; - bias = time_bias(tdiff, 21600); + bias = !CKP_STANDALONE(ckp) ? 1.0 : time_bias(tdiff, 21600); ghs360 = stats->dsps360 * nonces / bias; suffix_string(ghs360, suffix360, 16, 0); - bias1440 = time_bias(tdiff, 86400); - ghs1440 = stats->dsps1440 * nonces / bias1440; + bias = !CKP_STANDALONE(ckp) ? 1.0 : time_bias(tdiff, 86400); + ghs1440 = stats->dsps1440 * nonces / bias; suffix_string(ghs1440, suffix1440, 16, 0); - bias = time_bias(tdiff, 604800); + bias = !CKP_STANDALONE(ckp) ? 1.0 : time_bias(tdiff, 604800); ghs10080 = stats->dsps10080 * nonces / bias; suffix_string(ghs10080, suffix10080, 16, 0); From c33ae04ad889de165630378fb3521d3ca8ded8cb Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Fri, 7 Nov 2014 21:54:55 +1100 Subject: [PATCH 10/47] Handle mining.ping in proxy mode --- src/generator.c | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/src/generator.c b/src/generator.c index 524b1ec6..6257b338 100644 --- a/src/generator.c +++ b/src/generator.c @@ -792,6 +792,19 @@ static bool show_message(json_t *val) return true; } +static bool send_pong(proxy_instance_t *proxi, json_t *val) +{ + json_t *json_msg, *id_val = json_object_dup(val, "id"); + connsock_t *cs = proxi->cs; + bool ret; + + JSON_CPACK(json_msg, "{sossso}", "id", id_val, "result", "pong", + "error", json_null()); + ret = send_json_msg(cs, json_msg); + json_decref(json_msg); + return ret; +} + static bool parse_reconnect(proxy_instance_t *proxi, json_t *val) { server_instance_t *newsi, *si = proxi->si; @@ -924,6 +937,11 @@ static bool parse_method(proxy_instance_t *proxi, const char *msg) ret = show_message(params); goto out; } + + if (cmdmatch(buf, "mining.ping")) { + ret = send_pong(proxi, val); + goto out; + } out: if (val) json_decref(val); From 0941d5a7516f6fa38000ceb3da16459ed9cb26c7 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Fri, 14 Nov 2014 21:41:54 +1100 Subject: [PATCH 11/47] Add disconnected stratum clients to a linked list and use reference counting to know when they are no longer being referenced and can have their data freed. --- src/stratifier.c | 154 +++++++++++++++++++++++++++++++++++------------ 1 file changed, 115 insertions(+), 39 deletions(-) diff --git a/src/stratifier.c b/src/stratifier.c index e6292f3a..f48f7833 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -209,6 +209,10 @@ struct stratum_instance { stratum_instance_t *next; stratum_instance_t *prev; + /* Reference count for when this instance is used outside of the + * instance_lock */ + int ref; + char enonce1[32]; uchar enonce1bin[16]; char enonce1var[12]; @@ -303,6 +307,7 @@ struct stratifier_data { * is sorted by enonce1_64. */ stratum_instance_t *stratum_instances; stratum_instance_t *disconnected_instances; + stratum_instance_t *dead_instances; user_instance_t *user_instances; @@ -1042,6 +1047,41 @@ static stratum_instance_t *__instance_by_id(sdata_t *sdata, int64_t id) return instance; } +/* Increase the reference count of instance */ +static void __inc_instance_ref(stratum_instance_t *instance) +{ + instance->ref++; +} + +/* Find an __instance_by_id and increase its reference count allowing us to + * use this instance outside of instance_lock without fear of it being + * dereferenced. */ +static stratum_instance_t *ref_instance_by_id(sdata_t *sdata, int64_t id) +{ + stratum_instance_t *instance; + + ck_wlock(&sdata->instance_lock); + instance = __instance_by_id(sdata, id); + if (instance) + __inc_instance_ref(instance); + ck_wunlock(&sdata->instance_lock); + + return instance; +} + +/* Decrease the reference count of instance. */ +static void __dec_instance_ref(stratum_instance_t *instance) +{ + instance->ref--; +} + +static void dec_instance_ref(sdata_t *sdata, stratum_instance_t *instance) +{ + ck_wlock(&sdata->instance_lock); + __dec_instance_ref(instance); + ck_wunlock(&sdata->instance_lock); +} + /* Enter with write instance_lock held */ static stratum_instance_t *__stratum_add_instance(ckpool_t *ckp, int64_t id) { @@ -1167,7 +1207,7 @@ static void dec_worker(ckpool_t *ckp, user_instance_t *instance) static void drop_client(sdata_t *sdata, int64_t id) { - stratum_instance_t *client = NULL; + stratum_instance_t *client, *tmp; bool dec = false; LOGINFO("Stratifier dropping client %ld", id); @@ -1177,6 +1217,7 @@ static void drop_client(sdata_t *sdata, int64_t id) if (client) { stratum_instance_t *old_client = NULL; + __inc_instance_ref(client); if (client->authorised) { dec = true; client->authorised = false; @@ -1187,11 +1228,34 @@ static void drop_client(sdata_t *sdata, int64_t id) /* Only keep around one copy of the old client in server mode */ if (!client->ckp->proxy && !old_client && client->enonce1_64) HASH_ADD(hh, sdata->disconnected_instances, enonce1_64, sizeof(uint64_t), client); + else { + if (client->user_instance) + DL_DELETE(client->user_instance->instances, client); + DL_APPEND(sdata->dead_instances, client); + } } ck_wunlock(&sdata->instance_lock); + /* Decrease worker count outside of instance_lock to avoid recursive + * locking */ if (dec) dec_worker(client->ckp, client->user_instance); + + /* Cull old unused clients lazily when there are no more reference + * counts for them. */ + ck_wlock(&sdata->instance_lock); + if (client) + __dec_instance_ref(client); + DL_FOREACH_SAFE(sdata->dead_instances, client, tmp) { + if (!client->ref) { + LOGINFO("Stratifier discarding instance %ld", client->id); + DL_DELETE(sdata->dead_instances, client); + free(client->workername); + free(client->useragent); + free(client); + } + } + ck_wunlock(&sdata->instance_lock); } static void stratum_broadcast_message(sdata_t *sdata, const char *msg) @@ -1472,12 +1536,15 @@ static inline bool enonce1_free(sdata_t *sdata, uint64_t enonce1) ret = false; goto out; } + + ck_rlock(&sdata->instance_lock); HASH_ITER(hh, sdata->stratum_instances, client, tmp) { if (client->enonce1_64 == enonce1) { ret = false; break; } } + ck_runlock(&sdata->instance_lock); out: return ret; } @@ -1485,7 +1552,8 @@ out: /* Create a new enonce1 from the 64 bit enonce1_64 value, using only the number * of bytes we have to work with when we are proxying with a split nonce2. * When the proxy space is less than 32 bits to work with, we look for an - * unused enonce1 value and reject clients instead if there is no space left */ + * unused enonce1 value and reject clients instead if there is no space left. + * Needs to be entered with client holding a ref count. */ static bool new_enonce1(stratum_instance_t *client) { sdata_t *sdata = client->ckp->data; @@ -1538,7 +1606,8 @@ static bool new_enonce1(stratum_instance_t *client) static void stratum_send_message(sdata_t *sdata, stratum_instance_t *client, const char *msg); -/* Extranonce1 must be set here */ +/* Extranonce1 must be set here. Needs to be entered with client holding a ref + * count. */ static json_t *parse_subscribe(stratum_instance_t *client, int64_t client_id, json_t *params_val) { sdata_t *sdata = client->ckp->data; @@ -1732,7 +1801,8 @@ static void read_workerstats(ckpool_t *ckp, worker_instance_t *worker) /* This simply strips off the first part of the workername and matches it to a - * user or creates a new one. */ + * user or creates a new one. Needs to be entered with client holding a ref + * count. */ static user_instance_t *generate_user(ckpool_t *ckp, stratum_instance_t *client, const char *workername) { @@ -1799,7 +1869,8 @@ static user_instance_t *generate_user(ckpool_t *ckp, stratum_instance_t *client, /* Send this to the database and parse the response to authorise a user * and get SUID parameters back. We don't add these requests to the sdata->ckdbqueue * since we have to wait for the response but this is done from the authoriser - * thread so it won't hold anything up but other authorisations. */ + * thread so it won't hold anything up but other authorisations. Needs to be + * entered with client holding a ref count. */ static int send_recv_auth(stratum_instance_t *client) { user_instance_t *user_instance = client->user_instance; @@ -1886,7 +1957,8 @@ static int send_recv_auth(stratum_instance_t *client) /* For sending auths to ckdb after we've already decided we can authorise * these clients while ckdb is offline, based on an existing client of the - * same username already having been authorised. */ + * same username already having been authorised. Needs to be entered with + * client holding a ref count. */ static void queue_delayed_auth(stratum_instance_t *client) { ckpool_t *ckp = client->ckp; @@ -1912,6 +1984,7 @@ static void queue_delayed_auth(stratum_instance_t *client) ckdbq_add(ckp, ID_AUTH, val); } +/* Needs to be entered with client holding a ref count. */ static json_t *parse_authorise(stratum_instance_t *client, json_t *params_val, json_t **err_val, const char *address, int *errnum) { @@ -1982,6 +2055,7 @@ out: return json_boolean(ret); } +/* Needs to be entered with client holding a ref count. */ static void stratum_send_diff(sdata_t *sdata, stratum_instance_t *client) { json_t *json_msg; @@ -1991,6 +2065,7 @@ static void stratum_send_diff(sdata_t *sdata, stratum_instance_t *client) stratum_add_send(sdata, json_msg, client->id); } +/* Needs to be entered with client holding a ref count. */ static void stratum_send_message(sdata_t *sdata, stratum_instance_t *client, const char *msg) { json_t *json_msg; @@ -2020,6 +2095,7 @@ static double sane_tdiff(tv_t *end, tv_t *start) return tdiff; } +/* Needs to be entered with client holding a ref count. */ static void add_submit(ckpool_t *ckp, stratum_instance_t *client, int diff, bool valid, bool submit) { @@ -2149,7 +2225,8 @@ static void add_submit(ckpool_t *ckp, stratum_instance_t *client, int diff, bool stratum_send_diff(sdata, client); } -/* We should already be holding the workbase_lock */ +/* We should already be holding the workbase_lock. Needs to be entered with + * client holding a ref count. */ static void test_blocksolve(stratum_instance_t *client, workbase_t *wb, const uchar *data, const uchar *hash, double diff, const char *coinbase, int cblen, const char *nonce2, const char *nonce) @@ -2234,6 +2311,7 @@ test_blocksolve(stratum_instance_t *client, workbase_t *wb, const uchar *data, c ckdbq_add(ckp, ID_BLOCK, val); } +/* Needs to be entered with client holding a ref count. */ static double submission_diff(stratum_instance_t *client, workbase_t *wb, const char *nonce2, uint32_t ntime32, const char *nonce, uchar *hash) { @@ -2314,7 +2392,8 @@ out_unlock: return ret; } -/* Submit a share in proxy mode to the parent pool. workbase_lock is held */ +/* Submit a share in proxy mode to the parent pool. workbase_lock is held. + * Needs to be entered with client holding a ref count. */ static void submit_share(stratum_instance_t *client, int64_t jobid, const char *nonce2, const char *ntime, const char *nonce, int msg_id) { @@ -2335,6 +2414,7 @@ static void submit_share(stratum_instance_t *client, int64_t jobid, const char * #define JSON_ERR(err) json_string(SHARE_ERR(err)) +/* Needs to be entered with client holding a ref count. */ static json_t *parse_submit(stratum_instance_t *client, json_t *json_msg, json_t *params_val, json_t **err_val) { @@ -2631,6 +2711,7 @@ static void send_json_err(sdata_t *sdata, int64_t client_id, json_t *id_val, con stratum_add_send(sdata, val, client_id); } +/* Needs to be entered with client holding a ref count. */ static void update_client(sdata_t *sdata, stratum_instance_t *client, const int64_t client_id) { stratum_send_update(sdata, client_id, true); @@ -2713,7 +2794,8 @@ static void set_worker_mindiff(ckpool_t *ckp, const char *workername, int mindif } /* Implement support for the diff in the params as well as the originally - * documented form of placing diff within the method. */ + * documented form of placing diff within the method. Needs to be entered with + * client holding a ref count. */ static void suggest_diff(stratum_instance_t *client, const char *method, json_t *params_val) { json_t *arr_val = json_array_get(params_val, 0); @@ -2745,10 +2827,7 @@ static void parse_method(sdata_t *sdata, const int64_t client_id, json_t *id_val const char *method; char buf[256]; - ck_rlock(&sdata->instance_lock); - client = __instance_by_id(sdata, client_id); - ck_runlock(&sdata->instance_lock); - + client = ref_instance_by_id(sdata, client_id); if (unlikely(!client)) { LOGINFO("Failed to find client id %ld in hashtable!", client_id); return; @@ -2758,7 +2837,7 @@ static void parse_method(sdata_t *sdata, const int64_t client_id, json_t *id_val LOGINFO("Dropping client %d tagged for lazy invalidation", client_id); snprintf(buf, 255, "dropclient=%ld", client->id); send_proc(client->ckp->connector, buf); - return; + goto out; } /* Random broken clients send something not an integer as the id so we copy @@ -2779,7 +2858,7 @@ static void parse_method(sdata_t *sdata, const int64_t client_id, json_t *id_val stratum_add_send(sdata, val, client_id); if (likely(client->subscribed)) update_client(sdata, client, client_id); - return; + goto out; } if (unlikely(cmdmatch(method, "mining.passthrough"))) { @@ -2795,14 +2874,14 @@ static void parse_method(sdata_t *sdata, const int64_t client_id, json_t *id_val snprintf(buf, 255, "passthrough=%ld", client->id); send_proc(client->ckp->connector, buf); free(client); - return; + goto out; } if (cmdmatch(method, "mining.auth") && client->subscribed) { json_params_t *jp = create_json_params(client_id, method_val, params_val, id_val, address); ckmsgq_add(sdata->sauthq, jp); - return; + goto out; } /* We should only accept authorised requests from here on */ @@ -2813,14 +2892,14 @@ static void parse_method(sdata_t *sdata, const int64_t client_id, json_t *id_val LOGINFO("Dropping unauthorised client %ld", client->id); snprintf(buf, 255, "dropclient=%ld", client->id); send_proc(client->ckp->connector, buf); - return; + goto out; } if (cmdmatch(method, "mining.submit")) { json_params_t *jp = create_json_params(client_id, method_val, params_val, id_val, address); ckmsgq_add(sdata->sshareq, jp); - return; + goto out; } if (cmdmatch(method, "mining.suggest")) { @@ -2833,9 +2912,11 @@ static void parse_method(sdata_t *sdata, const int64_t client_id, json_t *id_val json_params_t *jp = create_json_params(client_id, method_val, params_val, id_val, address); ckmsgq_add(sdata->stxnq, jp); - return; + goto out; } /* Unhandled message here */ +out: + dec_instance_ref(sdata, client); } static void parse_instance_msg(sdata_t *sdata, smsg_t *msg) @@ -2877,7 +2958,6 @@ out: static void srecv_process(ckpool_t *ckp, char *buf) { - stratum_instance_t *instance; sdata_t *sdata = ckp->data; smsg_t *msg; json_t *val; @@ -2912,11 +2992,9 @@ static void srecv_process(ckpool_t *ckp, char *buf) /* Parse the message here */ ck_wlock(&sdata->instance_lock); - instance = __instance_by_id(sdata, msg->client_id); - if (!instance) { - /* client_id instance doesn't exist yet, create one */ - instance = __stratum_add_instance(ckp, msg->client_id); - } + /* client_id instance doesn't exist yet, create one */ + if (!__instance_by_id(sdata, msg->client_id)) + __stratum_add_instance(ckp, msg->client_id); ck_wunlock(&sdata->instance_lock); parse_instance_msg(sdata, msg); @@ -2968,17 +3046,14 @@ static void sshare_process(ckpool_t *ckp, json_params_t *jp) client_id = jp->client_id; - ck_rlock(&sdata->instance_lock); - client = __instance_by_id(sdata, client_id); - ck_runlock(&sdata->instance_lock); - + client = ref_instance_by_id(sdata, client_id); if (unlikely(!client)) { LOGINFO("Share processor failed to find client id %ld in hashtable!", client_id); goto out; } if (unlikely(!client->authorised)) { LOGDEBUG("Client %ld no longer authorised to submit shares", client_id); - goto out; + goto out_decref; } json_msg = json_object(); result_val = parse_submit(client, json_msg, jp->params, &err_val); @@ -2986,6 +3061,8 @@ static void sshare_process(ckpool_t *ckp, json_params_t *jp) json_object_set_new_nocheck(json_msg, "error", err_val ? err_val : json_null()); json_object_set_nocheck(json_msg, "id", jp->id_val); stratum_add_send(sdata, json_msg, client_id); +out_decref: + dec_instance_ref(sdata, client); out: discard_json_params(&jp); } @@ -3000,9 +3077,7 @@ static void sauth_process(ckpool_t *ckp, json_params_t *jp) client_id = jp->client_id; - ck_rlock(&sdata->instance_lock); - client = __instance_by_id(sdata, client_id); - ck_runlock(&sdata->instance_lock); + client = ref_instance_by_id(sdata, client_id); if (unlikely(!client)) { LOGINFO("Authoriser failed to find client id %ld in hashtable!", client_id); @@ -3038,6 +3113,8 @@ static void sauth_process(ckpool_t *ckp, json_params_t *jp) stratum_send_diff(sdata, client); } out: + if (client) + dec_instance_ref(sdata, client); discard_json_params(&jp); } @@ -3146,7 +3223,7 @@ static void send_transactions(ckpool_t *ckp, json_params_t *jp) { const char *msg = json_string_value(jp->method), *params = json_string_value(json_array_get(jp->params, 0)); - stratum_instance_t *client; + stratum_instance_t *client = NULL; sdata_t *sdata = ckp->data; json_t *val, *hashes; int64_t job_id = 0; @@ -3183,10 +3260,7 @@ static void send_transactions(ckpool_t *ckp, json_params_t *jp) goto out_send; } - ck_rlock(&sdata->instance_lock); - client = __instance_by_id(sdata, jp->client_id); - ck_runlock(&sdata->instance_lock); - + client = ref_instance_by_id(sdata, jp->client_id); if (unlikely(!client)) { LOGINFO("send_transactions failed to find client id %ld in hashtable!", jp->client_id); @@ -3215,6 +3289,8 @@ out_send: stratum_add_send(sdata, val, jp->client_id); out: discard_json_params(&jp); + if (client) + dec_instance_ref(sdata, client); } /* Called every 20 seconds, we send the updated stats to ckdb of those users From ef649813f8268ba8ae59c9b0ac1398090938de4a Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Sat, 15 Nov 2014 10:25:37 +1100 Subject: [PATCH 12/47] Dead stratum instances need not be in a doubly linked list --- src/stratifier.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/stratifier.c b/src/stratifier.c index f48f7833..be5b316c 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -1231,7 +1231,7 @@ static void drop_client(sdata_t *sdata, int64_t id) else { if (client->user_instance) DL_DELETE(client->user_instance->instances, client); - DL_APPEND(sdata->dead_instances, client); + LL_PREPEND(sdata->dead_instances, client); } } ck_wunlock(&sdata->instance_lock); @@ -1249,7 +1249,7 @@ static void drop_client(sdata_t *sdata, int64_t id) DL_FOREACH_SAFE(sdata->dead_instances, client, tmp) { if (!client->ref) { LOGINFO("Stratifier discarding instance %ld", client->id); - DL_DELETE(sdata->dead_instances, client); + LL_DELETE(sdata->dead_instances, client); free(client->workername); free(client->useragent); free(client); From 6c47d61bb1563f6ef96f761098d9cf95efbf4952 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Sat, 15 Nov 2014 11:31:47 +1100 Subject: [PATCH 13/47] Use reference counting to know when we can drop client structures in the connector --- src/connector.c | 93 ++++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 76 insertions(+), 17 deletions(-) diff --git a/src/connector.c b/src/connector.c index 58281bf4..64b77afd 100644 --- a/src/connector.c +++ b/src/connector.c @@ -30,6 +30,10 @@ struct client_instance { int64_t id; int fd; + /* Reference count for when this instance is used outside of the + * connector_data lock */ + int ref; + /* For dead_clients list */ struct client_instance *next; @@ -84,6 +88,25 @@ struct connector_data { typedef struct connector_data cdata_t; +/* Increase the reference count of instance */ +static void __inc_instance_ref(client_instance_t *client) +{ + client->ref++; +} + +/* Increase the reference count of instance */ +static void __dec_instance_ref(client_instance_t *client) +{ + client->ref--; +} + +static void dec_instance_ref(cdata_t *cdata, client_instance_t *client) +{ + ck_wlock(&cdata->lock); + __dec_instance_ref(client); + ck_wunlock(&cdata->lock); +} + /* Accepts incoming connections on the server socket and generates client * instances */ static int accept_client(cdata_t *cdata, int epfd) @@ -155,6 +178,11 @@ static int accept_client(cdata_t *cdata, int epfd) return 0; } + /* We increase the ref count on this client as epoll creates a pointer + * to it. We drop that reference when the socket is closed which + * removes it automatically from the epoll list. */ + __inc_instance_ref(client); + ck_wlock(&cdata->lock); client->id = cdata->client_id++; HASH_ADD_I64(cdata->clients, id, client); @@ -164,6 +192,7 @@ static int accept_client(cdata_t *cdata, int epfd) return 1; } +/* Client must hold a reference count */ static int drop_client(cdata_t *cdata, client_instance_t *client) { int fd; @@ -174,6 +203,9 @@ static int drop_client(cdata_t *cdata, client_instance_t *client) Close(client->fd); HASH_DEL(cdata->clients, client); LL_PREPEND(cdata->dead_clients, client); + /* This is the reference to this client's presence in the + * epoll list. */ + __dec_instance_ref(client); } ck_wunlock(&cdata->lock); @@ -192,18 +224,34 @@ static void stratifier_drop_client(ckpool_t *ckp, int64_t id) } /* Invalidate this instance. Remove them from the hashtables we look up - * regularly but keep the instances in a linked list indefinitely in case we - * still reference any of its members. */ + * regularly but keep the instances in a linked list until their ref count + * drops to zero when we can remove them lazily. Client must hold a reference + * count. */ static void invalidate_client(ckpool_t *ckp, cdata_t *cdata, client_instance_t *client) { + client_instance_t *tmp; + drop_client(cdata, client); if (ckp->passthrough) return; stratifier_drop_client(ckp, client->id); + + /* Cull old unused clients lazily when there are no more reference + * counts for them. */ + ck_wlock(&cdata->lock); + LL_FOREACH_SAFE(cdata->dead_clients, client, tmp) { + if (!client->ref) { + LL_DELETE(cdata->dead_clients, client); + LOGINFO("Connector discarding client %ld", client->id); + free(client); + } + } + ck_wunlock(&cdata->lock); } static void send_client(cdata_t *cdata, int64_t id, char *buf); +/* Client is holding a reference count from being on the epoll list */ static void parse_client_msg(cdata_t *cdata, client_instance_t *client) { int buflen, ret, selfail = 0; @@ -412,9 +460,7 @@ void *sender(void *arg) if (fd == -1) { LOGDEBUG("Discarding message sent to invalidated client"); - free(sender_send->buf); - free(sender_send); - continue; + goto contfree; } /* If this socket is not ready to receive data from us, put the * send back on the tail of the list and decrease the timeout @@ -425,15 +471,13 @@ void *sender(void *arg) if (ret < 0) { LOGINFO("Client id %d fd %d interrupted", client->id, fd); invalidate_client(ckp, cdata, client); - free(sender_send->buf); - free(sender_send); - continue; + goto contfree; } LOGDEBUG("Client %d not ready for writes", client->id); /* Append it to the tail of the delayed sends list. * This is the only function that alters it so no - * locking is required. */ + * locking is required. Keep the client ref. */ DL_APPEND(cdata->delayed_sends, sender_send); continue; } @@ -448,8 +492,10 @@ void *sender(void *arg) ofs += ret; sender_send->len -= ret; } +contfree: free(sender_send->buf); free(sender_send); + dec_instance_ref(cdata, client); } return NULL; @@ -474,11 +520,17 @@ static void send_client(cdata_t *cdata, int64_t id, char *buf) return; } - ck_rlock(&cdata->lock); + ck_ilock(&cdata->lock); HASH_FIND_I64(cdata->clients, &id, client); - if (likely(client)) + if (likely(client)) { + ck_ulock(&cdata->lock); fd = client->fd; - ck_runlock(&cdata->lock); + /* Grab a reference to this client until the sender_send has + * completed processing. */ + __inc_instance_ref(client); + ck_dwilock(&cdata->lock); + } + ck_uilock(&cdata->lock); if (unlikely(fd == -1)) { ckpool_t *ckp = cdata->ckp; @@ -506,13 +558,18 @@ static void send_client(cdata_t *cdata, int64_t id, char *buf) mutex_unlock(&cdata->sender_lock); } -static client_instance_t *client_by_id(cdata_t *cdata, int64_t id) +static client_instance_t *ref_client_by_id(cdata_t *cdata, int64_t id) { client_instance_t *client; - ck_rlock(&cdata->lock); + ck_ilock(&cdata->lock); HASH_FIND_I64(cdata->clients, &id, client); - ck_runlock(&cdata->lock); + if (client) { + ck_ulock(&cdata->lock); + __inc_instance_ref(client); + ck_dwilock(&cdata->lock); + } + ck_uilock(&cdata->lock); return client; } @@ -605,12 +662,13 @@ retry: goto retry; } client_id = client_id64 & 0xffffffffll; - client = client_by_id(cdata, client_id); + client = ref_client_by_id(cdata, client_id); if (unlikely(!client)) { LOGINFO("Connector failed to find client id %ld to drop", client_id); goto retry; } ret = drop_client(cdata, client); + dec_instance_ref(cdata, client); if (ret >= 0) LOGINFO("Connector dropped client id: %ld", client_id); goto retry; @@ -623,12 +681,13 @@ retry: LOGDEBUG("Connector failed to parse passthrough command: %s", buf); goto retry; } - client = client_by_id(cdata, client_id); + client = ref_client_by_id(cdata, client_id); if (unlikely(!client)) { LOGINFO("Connector failed to find client id %ld to pass through", client_id); goto retry; } passthrough_client(cdata, client); + dec_instance_ref(cdata, client); goto retry; } if (cmdmatch(buf, "getfd")) { From ab91682e22cdb4b1914f82e19bdb03799c33b3ca Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Sat, 15 Nov 2014 15:32:01 +1100 Subject: [PATCH 14/47] Add support for custom extranonce1 lengths --- README | 3 +++ ckpool.conf | 1 + src/ckpool.c | 5 +++++ src/ckpool.h | 1 + src/stratifier.c | 10 +++++++--- 5 files changed, 17 insertions(+), 3 deletions(-) diff --git a/README b/README index 3d8c90ea..735bcbb0 100644 --- a/README +++ b/README @@ -240,6 +240,9 @@ new network blocks and is 100 by default. It is intended to be a backup only for when the notifier is not set up and only polls if the "notify" field is not set on a btcd. +'noncelength" : This is optional allowing the extranonce1 length to be chosen +from 2 to 8. Default 8 + "update_interval" : This is the frequency that stratum updates are sent out to miners and is set to 30 seconds by default to help perpetuate transactions for the health of the bitcoin network. diff --git a/ckpool.conf b/ckpool.conf index 340556bd..f9f767da 100644 --- a/ckpool.conf +++ b/ckpool.conf @@ -16,6 +16,7 @@ "btcaddress" : "14BMjogz69qe8hk9thyzbmR5pg34mVKB1e", "btcsig" : "/mined by ck/", "blockpoll" : 100, +"noncelength" : 8, "update_interval" : 30, "serverurl" : "ckpool.org:3333", "mindiff" : 1, diff --git a/src/ckpool.c b/src/ckpool.c index e1d1186b..ee528880 100644 --- a/src/ckpool.c +++ b/src/ckpool.c @@ -1017,6 +1017,7 @@ static void parse_config(ckpool_t *ckp) ckp->btcsig[38] = '\0'; } json_get_int(&ckp->blockpoll, json_conf, "blockpoll"); + json_get_int(&ckp->noncelength, json_conf, "noncelength"); json_get_int(&ckp->update_interval, json_conf, "update_interval"); json_get_string(&ckp->serverurl, json_conf, "serverurl"); json_get_int64(&ckp->mindiff, json_conf, "mindiff"); @@ -1311,6 +1312,10 @@ int main(int argc, char **argv) ckp.btcaddress = ckp.donaddress; if (!ckp.blockpoll) ckp.blockpoll = 100; + if (!ckp.noncelength) + ckp.noncelength = 8; + else if (ckp.noncelength < 2 || ckp.noncelength > 8) + quit(0, "Invalid noncelength %d specified, must be 2~8", ckp.noncelength); if (!ckp.update_interval) ckp.update_interval = 30; if (!ckp.mindiff) diff --git a/src/ckpool.h b/src/ckpool.h index 768dcaf4..5d5f48a1 100644 --- a/src/ckpool.h +++ b/src/ckpool.h @@ -148,6 +148,7 @@ struct ckpool_instance { char **btcdpass; bool *btcdnotify; int blockpoll; // How frequently in ms to poll bitcoind for block updates + int noncelength; // Extranonce1 length /* Difficulty settings */ int64_t mindiff; // Default 1 diff --git a/src/stratifier.c b/src/stratifier.c index be5b316c..f20db9d8 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -410,9 +410,7 @@ static void generate_coinbase(ckpool_t *ckp, workbase_t *wb) len = ser_number(wb->coinb1bin + ofs, now.tv_nsec); ofs += len; - /* Leave enonce1/2varlen constant at 8 bytes for bitcoind sources */ - wb->enonce1varlen = 8; - wb->enonce2varlen = 8; + wb->enonce1varlen = wb->enonce2varlen = ckp->noncelength; wb->coinb1bin[ofs++] = wb->enonce1varlen + wb->enonce2varlen; wb->coinb1len = ofs; @@ -1568,10 +1566,14 @@ static bool new_enonce1(stratum_instance_t *client) sdata->enonce1u.u64++; ret = true; break; + case 7: + case 6: + case 5: case 4: sdata->enonce1u.u32++; ret = true; break; + case 3: case 2: for (i = 0; i < 65536; i++) { sdata->enonce1u.u16++; @@ -1588,6 +1590,8 @@ static bool new_enonce1(stratum_instance_t *client) break; } break; + default: + quit(0, "Invalid enonce1varlen %d", wb->enonce1varlen); } if (ret) client->enonce1_64 = sdata->enonce1u.u64; From 721d6d22ab62fc4314947713ec1588eb5a07c527 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Mon, 17 Nov 2014 13:57:40 +1100 Subject: [PATCH 15/47] Set the initial enonce1 to a value that will differ on every restart --- src/stratifier.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/stratifier.c b/src/stratifier.c index f20db9d8..c25f1e50 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -3670,6 +3670,7 @@ int stratifier(proc_instance_t *pi) pthread_t pth_blockupdate, pth_statsupdate, pth_heartbeat; ckpool_t *ckp = pi->ckp; int ret = 1, threads; + int64_t randomiser; sdata_t *sdata; char *buf; @@ -3702,10 +3703,12 @@ int stratifier(proc_instance_t *pi) } } + randomiser = ((int64_t)time(NULL)) << 32; /* Set the initial id to time as high bits so as to not send the same * id on restarts */ if (!ckp->proxy) - sdata->blockchange_id = sdata->workbase_id = ((int64_t)time(NULL)) << 32; + sdata->blockchange_id = sdata->workbase_id = randomiser; + sdata->enonce1u.u64 = htobe64(randomiser); dealloc(buf); From 0709058b25a40190e9fdf8ec22b38b26f749796b Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Mon, 17 Nov 2014 14:01:54 +1100 Subject: [PATCH 16/47] Allow nonce1 and nonce2 lengths to be specified separately --- README | 5 ++++- ckpool.conf | 3 ++- src/ckpool.c | 15 ++++++++++----- src/ckpool.h | 3 ++- src/stratifier.c | 3 ++- 5 files changed, 20 insertions(+), 9 deletions(-) diff --git a/README b/README index 735bcbb0..e879c954 100644 --- a/README +++ b/README @@ -240,7 +240,10 @@ new network blocks and is 100 by default. It is intended to be a backup only for when the notifier is not set up and only polls if the "notify" field is not set on a btcd. -'noncelength" : This is optional allowing the extranonce1 length to be chosen +"nonce1length" : This is optional allowing the extranonce1 length to be chosen +from 2 to 8. Default 8 + +"nonce2length" : This is optional allowing the extranonce2 length to be chosen from 2 to 8. Default 8 "update_interval" : This is the frequency that stratum updates are sent out to diff --git a/ckpool.conf b/ckpool.conf index f9f767da..cf2dcc4e 100644 --- a/ckpool.conf +++ b/ckpool.conf @@ -16,7 +16,8 @@ "btcaddress" : "14BMjogz69qe8hk9thyzbmR5pg34mVKB1e", "btcsig" : "/mined by ck/", "blockpoll" : 100, -"noncelength" : 8, +"nonce1length" : 8, +"nonce2length" : 8, "update_interval" : 30, "serverurl" : "ckpool.org:3333", "mindiff" : 1, diff --git a/src/ckpool.c b/src/ckpool.c index ee528880..630a7721 100644 --- a/src/ckpool.c +++ b/src/ckpool.c @@ -1017,7 +1017,8 @@ static void parse_config(ckpool_t *ckp) ckp->btcsig[38] = '\0'; } json_get_int(&ckp->blockpoll, json_conf, "blockpoll"); - json_get_int(&ckp->noncelength, json_conf, "noncelength"); + json_get_int(&ckp->nonce1length, json_conf, "nonce1length"); + json_get_int(&ckp->nonce2length, json_conf, "nonce2length"); json_get_int(&ckp->update_interval, json_conf, "update_interval"); json_get_string(&ckp->serverurl, json_conf, "serverurl"); json_get_int64(&ckp->mindiff, json_conf, "mindiff"); @@ -1312,10 +1313,14 @@ int main(int argc, char **argv) ckp.btcaddress = ckp.donaddress; if (!ckp.blockpoll) ckp.blockpoll = 100; - if (!ckp.noncelength) - ckp.noncelength = 8; - else if (ckp.noncelength < 2 || ckp.noncelength > 8) - quit(0, "Invalid noncelength %d specified, must be 2~8", ckp.noncelength); + if (!ckp.nonce1length) + ckp.nonce1length = 8; + else if (ckp.nonce1length < 2 || ckp.nonce1length > 8) + quit(0, "Invalid nonce1length %d specified, must be 2~8", ckp.nonce1length); + if (!ckp.nonce2length) + ckp.nonce2length = 8; + else if (ckp.nonce2length < 2 || ckp.nonce2length > 8) + quit(0, "Invalid nonce2length %d specified, must be 2~8", ckp.nonce2length); if (!ckp.update_interval) ckp.update_interval = 30; if (!ckp.mindiff) diff --git a/src/ckpool.h b/src/ckpool.h index 5d5f48a1..3660d50b 100644 --- a/src/ckpool.h +++ b/src/ckpool.h @@ -148,7 +148,8 @@ struct ckpool_instance { char **btcdpass; bool *btcdnotify; int blockpoll; // How frequently in ms to poll bitcoind for block updates - int noncelength; // Extranonce1 length + int nonce1length; // Extranonce1 length + int nonce2length; // Extranonce2 length /* Difficulty settings */ int64_t mindiff; // Default 1 diff --git a/src/stratifier.c b/src/stratifier.c index c25f1e50..9efecb29 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -410,7 +410,8 @@ static void generate_coinbase(ckpool_t *ckp, workbase_t *wb) len = ser_number(wb->coinb1bin + ofs, now.tv_nsec); ofs += len; - wb->enonce1varlen = wb->enonce2varlen = ckp->noncelength; + wb->enonce1varlen = ckp->nonce1length; + wb->enonce2varlen = ckp->nonce2length; wb->coinb1bin[ofs++] = wb->enonce1varlen + wb->enonce2varlen; wb->coinb1len = ofs; From 480c705e96fc25cf3933c831e7fbc5ac297d307f Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Wed, 19 Nov 2014 21:22:31 +1100 Subject: [PATCH 17/47] Handle invalidated socket fds with an error in the wait read/write select functions --- src/libckpool.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/libckpool.c b/src/libckpool.c index 8fb55173..747ac02d 100644 --- a/src/libckpool.c +++ b/src/libckpool.c @@ -711,6 +711,8 @@ int wait_read_select(int sockd, int timeout) { struct pollfd sfd; + if (unlikely(sockd < 0)) + return -1; sfd.fd = sockd; sfd.events = POLLIN; sfd.revents = 0; @@ -784,6 +786,8 @@ int wait_write_select(int sockd, int timeout) { struct pollfd sfd; + if (unlikely(sockd < 0)) + return -1; sfd.fd = sockd; sfd.events = POLLOUT; sfd.revents = 0; From 615e9e8b22b3cd93f35c7dca7d01dbfe2c1c8924 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Wed, 19 Nov 2014 21:25:33 +1100 Subject: [PATCH 18/47] Handle read and write_length functions receiving an invalidated fd --- src/libckpool.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/libckpool.c b/src/libckpool.c index 747ac02d..860b537e 100644 --- a/src/libckpool.c +++ b/src/libckpool.c @@ -728,6 +728,8 @@ int read_length(int sockd, void *buf, int len) LOGWARNING("Invalid read length of %d requested in read_length", len); return -1; } + if (unlikely(sockd < 0)) + return -1; while (len) { ret = recv(sockd, buf + ofs, len, MSG_WAITALL); if (unlikely(ret < 1)) @@ -803,6 +805,8 @@ int write_length(int sockd, const void *buf, int len) LOGWARNING("Invalid write length of %d requested in write_length", len); return -1; } + if (unlikely(sockd < 0)) + return -1; while (len) { ret = write(sockd, buf + ofs, len); if (unlikely(ret < 0)) From fe7f015ec62d71ba778dc47cd808abaaecb700ea Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Wed, 19 Nov 2014 21:29:18 +1100 Subject: [PATCH 19/47] Empty buffers of servers and proxy when killing them --- src/ckpool.h | 1 + src/generator.c | 2 ++ 2 files changed, 3 insertions(+) diff --git a/src/ckpool.h b/src/ckpool.h index 3660d50b..d06820a0 100644 --- a/src/ckpool.h +++ b/src/ckpool.h @@ -193,6 +193,7 @@ bool ckmsgq_empty(ckmsgq_t *ckmsgq); ckpool_t *global_ckp; bool ping_main(ckpool_t *ckp); +void empty_buffer(connsock_t *cs); int read_socket_line(connsock_t *cs, int timeout); bool _send_proc(proc_instance_t *pi, const char *msg, const char *file, const char *func, const int line); #define send_proc(pi, msg) _send_proc(pi, msg, __FILE__, __func__, __LINE__) diff --git a/src/generator.c b/src/generator.c index 6257b338..6e00669e 100644 --- a/src/generator.c +++ b/src/generator.c @@ -229,6 +229,7 @@ static void kill_server(server_instance_t *si) LOGNOTICE("Killing server"); cs = &si->cs; Close(cs->fd); + empty_buffer(cs); dealloc(cs->url); dealloc(cs->port); dealloc(cs->auth); @@ -1452,6 +1453,7 @@ static void kill_proxy(ckpool_t *ckp, proxy_instance_t *proxi) LOGNOTICE("Killing proxy"); cs = proxi->cs; Close(cs->fd); + empty_buffer(cs); /* All our notify data is invalid if we reconnect so discard them */ mutex_lock(&proxi->notify_lock); From bbfadc3e47f9d22195045c6cd9c658deed22c8ed Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Wed, 19 Nov 2014 21:34:40 +1100 Subject: [PATCH 20/47] Store the fd in read_socket_line to not have it change under us, and empty the cs buffer on failure --- src/ckpool.c | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/ckpool.c b/src/ckpool.c index 630a7721..c7ff6fee 100644 --- a/src/ckpool.c +++ b/src/ckpool.c @@ -367,11 +367,11 @@ void empty_buffer(connsock_t *cs) * of the buffer for use on the next receive. */ int read_socket_line(connsock_t *cs, int timeout) { + int fd = cs->fd, ret = -1; char *eom = NULL; size_t buflen; - int ret = -1; - if (unlikely(cs->fd < 0)) + if (unlikely(fd < 0)) goto out; if (unlikely(!cs->buf)) @@ -388,7 +388,7 @@ int read_socket_line(connsock_t *cs, int timeout) while (42) { char readbuf[PAGESIZE] = {}; - ret = wait_read_select(cs->fd, eom ? 0 : timeout); + ret = wait_read_select(fd, eom ? 0 : timeout); if (eom && !ret) break; if (ret < 1) { @@ -398,7 +398,7 @@ int read_socket_line(connsock_t *cs, int timeout) LOGERR("Select failed in read_socket_line"); goto out; } - ret = recv(cs->fd, readbuf, PAGESIZE - 4, 0); + ret = recv(fd, readbuf, PAGESIZE - 4, 0); if (ret < 1) { LOGERR("Failed to recv in read_socket_line"); ret = -1; @@ -423,6 +423,7 @@ int read_socket_line(connsock_t *cs, int timeout) *eom = '\0'; out: if (ret < 0) { + empty_buffer(cs); dealloc(cs->buf); Close(cs->fd); } From 00fa820c3b272df21fde06c4e3ea37e97cfdac61 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Fri, 21 Nov 2014 10:39:04 +1100 Subject: [PATCH 21/47] Make the json_get functions return boolean for success or failure --- src/ckpool.c | 30 +++++++++++++++++++++--------- src/ckpool.h | 4 ++-- 2 files changed, 23 insertions(+), 11 deletions(-) diff --git a/src/ckpool.c b/src/ckpool.c index c7ff6fee..0a87c430 100644 --- a/src/ckpool.c +++ b/src/ckpool.c @@ -890,23 +890,27 @@ static void sighandler(int sig) exit(0); } -void json_get_string(char **store, json_t *val, const char *res) +bool json_get_string(char **store, json_t *val, const char *res) { json_t *entry = json_object_get(val, res); + bool ret = false; const char *buf; *store = NULL; if (!entry || json_is_null(entry)) { LOGDEBUG("Json did not find entry %s", res); - return; + goto out; } if (!json_is_string(entry)) { LOGWARNING("Json entry %s is not a string", res); - return; + goto out; } buf = json_string_value(entry); LOGDEBUG("Json found entry %s: %s", res, buf); *store = strdup(buf); + ret = true; +out: + return ret; } static void json_get_int64(int64_t *store, json_t *val, const char *res) @@ -925,36 +929,44 @@ static void json_get_int64(int64_t *store, json_t *val, const char *res) LOGDEBUG("Json found entry %s: %ld", res, *store); } -void json_get_int(int *store, json_t *val, const char *res) +bool json_get_int(int *store, json_t *val, const char *res) { json_t *entry = json_object_get(val, res); + bool ret = false; if (!entry) { LOGDEBUG("Json did not find entry %s", res); - return; + goto out; } if (!json_is_integer(entry)) { LOGWARNING("Json entry %s is not an integer", res); - return; + goto out; } *store = json_integer_value(entry); LOGDEBUG("Json found entry %s: %d", res, *store); + ret = true; +out: + return ret; } -static void json_get_bool(bool *store, json_t *val, const char *res) +static bool json_get_bool(bool *store, json_t *val, const char *res) { json_t *entry = json_object_get(val, res); + bool ret = false; if (!entry) { LOGDEBUG("Json did not find entry %s", res); - return; + goto out; } if (!json_is_boolean(entry)) { LOGWARNING("Json entry %s is not a boolean", res); - return; + goto out; } *store = json_is_true(entry); LOGDEBUG("Json found entry %s: %s", res, *store ? "true" : "false"); + ret = true; +out: + return ret; } static void parse_btcds(ckpool_t *ckp, json_t *arr_val, int arr_size) diff --git a/src/ckpool.h b/src/ckpool.h index d06820a0..f7bcc85a 100644 --- a/src/ckpool.h +++ b/src/ckpool.h @@ -208,7 +208,7 @@ char *_ckdb_msg_call(const ckpool_t *ckp, char *msg, const char *file, const ch json_t *json_rpc_call(connsock_t *cs, const char *rpc_req); int process_exit(ckpool_t *ckp, proc_instance_t *pi, int ret); -void json_get_string(char **store, json_t *val, const char *res); -void json_get_int(int *store, json_t *val, const char *res); +bool json_get_string(char **store, json_t *val, const char *res); +bool json_get_int(int *store, json_t *val, const char *res); #endif /* CKPOOL_H */ From 5dd076cfbeda82f35396908afd9fa688ca291dfb Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Fri, 21 Nov 2014 15:51:16 +1100 Subject: [PATCH 22/47] Allow serverurl to take an array of entries for future binding to multiple interfaces/ports --- src/ckpool.c | 57 ++++++++++++++++++++++++++++++++++++++++++------ src/ckpool.h | 3 ++- src/connector.c | 6 ++--- src/stratifier.c | 20 +++++++++-------- 4 files changed, 66 insertions(+), 20 deletions(-) diff --git a/src/ckpool.c b/src/ckpool.c index 0a87c430..252c9a7f 100644 --- a/src/ckpool.c +++ b/src/ckpool.c @@ -890,9 +890,8 @@ static void sighandler(int sig) exit(0); } -bool json_get_string(char **store, json_t *val, const char *res) +static bool _json_get_string(char **store, json_t *entry, const char *res) { - json_t *entry = json_object_get(val, res); bool ret = false; const char *buf; @@ -913,6 +912,11 @@ out: return ret; } +bool json_get_string(char **store, json_t *val, const char *res) +{ + return _json_get_string(store, json_object_get(val, res), res); +} + static void json_get_int64(int64_t *store, json_t *val, const char *res) { json_t *entry = json_object_get(val, res); @@ -1005,10 +1009,41 @@ static void parse_proxies(ckpool_t *ckp, json_t *arr_val, int arr_size) } } +static bool parse_serverurls(ckpool_t *ckp, json_t *arr_val) +{ + bool ret = false; + int arr_size, i; + + if (!arr_val) + goto out; + if (!json_is_array(arr_val)) { + LOGWARNING("Unable to parse serverurl entries as an array"); + goto out; + } + arr_size = json_array_size(arr_val); + if (!arr_size) { + LOGWARNING("Serverurl array empty"); + goto out; + } + ckp->serverurls = arr_size; + ckp->serverurl = ckalloc(sizeof(char *) * arr_size); + for (i = 0; i < arr_size; i++) { + json_t *val = json_array_get(arr_val, i); + + if (!_json_get_string(&ckp->serverurl[i], val, "serverurl")) + LOGWARNING("Invalid serverurl entry number %d", i); + } + ret = true; +out: + return ret; +} + static void parse_config(ckpool_t *ckp) { json_t *json_conf, *arr_val; json_error_t err_val; + int arr_size; + char *url; json_conf = json_load_file(ckp->config, JSON_DISABLE_EOF_CHECK, &err_val); if (!json_conf) { @@ -1018,8 +1053,7 @@ static void parse_config(ckpool_t *ckp) } arr_val = json_object_get(json_conf, "btcd"); if (arr_val && json_is_array(arr_val)) { - int arr_size = json_array_size(arr_val); - + arr_size = json_array_size(arr_val); if (arr_size) parse_btcds(ckp, arr_val, arr_size); } @@ -1033,7 +1067,15 @@ static void parse_config(ckpool_t *ckp) json_get_int(&ckp->nonce1length, json_conf, "nonce1length"); json_get_int(&ckp->nonce2length, json_conf, "nonce2length"); json_get_int(&ckp->update_interval, json_conf, "update_interval"); - json_get_string(&ckp->serverurl, json_conf, "serverurl"); + /* Look for an array first and then a single entry */ + arr_val = json_object_get(json_conf, "serverurl"); + if (!parse_serverurls(ckp, arr_val)) { + if (json_get_string(&url, json_conf, "serverurl")) { + ckp->serverurl = ckalloc(sizeof(char *)); + ckp->serverurl[0] = url; + ckp->serverurls = 1; + } + } json_get_int64(&ckp->mindiff, json_conf, "mindiff"); json_get_int64(&ckp->startdiff, json_conf, "startdiff"); json_get_int64(&ckp->maxdiff, json_conf, "maxdiff"); @@ -1041,8 +1083,7 @@ static void parse_config(ckpool_t *ckp) json_get_int(&ckp->maxclients, json_conf, "maxclients"); arr_val = json_object_get(json_conf, "proxy"); if (arr_val && json_is_array(arr_val)) { - int arr_size = json_array_size(arr_val); - + arr_size = json_array_size(arr_val); if (arr_size) parse_proxies(ckp, arr_val, arr_size); } @@ -1342,6 +1383,8 @@ int main(int argc, char **argv) ckp.startdiff = 42; if (!ckp.logdir) ckp.logdir = strdup("logs"); + if (!ckp.serverurls) + ckp.serverurl = ckzalloc(sizeof(char *)); if (ckp.proxy && !ckp.proxies) quit(0, "No proxy entries found in config file %s", ckp.config); diff --git a/src/ckpool.h b/src/ckpool.h index f7bcc85a..b495becf 100644 --- a/src/ckpool.h +++ b/src/ckpool.h @@ -164,7 +164,8 @@ struct ckpool_instance { /* Stratum options */ server_instance_t **servers; - char *serverurl; // URL to bind our server/proxy to + char **serverurl; // Array of URLs to bind our server/proxy to + int serverurls; // Number of server bindings int update_interval; // Seconds between stratum updates int chosen_server; // Chosen server for next connection diff --git a/src/connector.c b/src/connector.c index 64b77afd..eba91ca0 100644 --- a/src/connector.c +++ b/src/connector.c @@ -742,9 +742,9 @@ int connector(proc_instance_t *pi) if (ckp->oldconnfd > 0) { sockd = ckp->oldconnfd; - } else if (ckp->serverurl) { - if (!extract_sockaddr(ckp->serverurl, &url, &port)) { - LOGWARNING("Failed to extract server address from %s", ckp->serverurl); + } else if (ckp->serverurls && ckp->serverurl[0]) { + if (!extract_sockaddr(ckp->serverurl[0], &url, &port)) { + LOGWARNING("Failed to extract server address from %s", ckp->serverurl[0]); ret = 1; goto out; } diff --git a/src/stratifier.c b/src/stratifier.c index 9efecb29..94e9c546 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -603,7 +603,7 @@ static void send_workinfo(ckpool_t *ckp, workbase_t *wb) "createdate", cdfield, "createby", "code", "createcode", __func__, - "createinet", ckp->serverurl); + "createinet", ckp->serverurl[0]); ckdbq_add(ckp, ID_WORKINFO, val); } @@ -622,7 +622,7 @@ static void send_ageworkinfo(ckpool_t *ckp, int64_t id) "createdate", cdfield, "createby", "code", "createcode", __func__, - "createinet", ckp->serverurl); + "createinet", ckp->serverurl[0]); ckdbq_add(ckp, ID_AGEWORKINFO, val); } @@ -2304,7 +2304,7 @@ test_blocksolve(stratum_instance_t *client, workbase_t *wb, const uchar *data, c "createdate", cdfield, "createby", "code", "createcode", __func__, - "createinet", ckp->serverurl); + "createinet", ckp->serverurl[0]); val_copy = json_deep_copy(val); block_ckmsg = ckalloc(sizeof(ckmsg_t)); block_ckmsg->data = val_copy; @@ -2604,7 +2604,7 @@ out_unlock: json_set_string(val, "createdate", cdfield); json_set_string(val, "createby", "code"); json_set_string(val, "createcode", __func__); - json_set_string(val, "createinet", ckp->serverurl); + json_set_string(val, "createinet", ckp->serverurl[0]); json_set_string(val, "workername", client->workername); json_set_string(val, "username", user_instance->username); @@ -2656,7 +2656,7 @@ out: "createdate", cdfield, "createby", "code", "createcode", __func__, - "createinet", ckp->serverurl); + "createinet", ckp->serverurl[0]); ckdbq_add(ckp, ID_SHAREERR, val); LOGINFO("Invalid share from client %ld: %s", client->id, client->workername); } @@ -3595,7 +3595,7 @@ static void *statsupdate(void *arg) "createdate", cdfield, "createby", "code", "createcode", __func__, - "createinet", ckp->serverurl); + "createinet", ckp->serverurl[0]); ckdbq_add(ckp, ID_POOLSTATS, val); /* Update stats 3 times per minute for smooth values, displaying @@ -3660,7 +3660,7 @@ static void *ckdb_heartbeat(void *arg) "createdate", cdfield, "createby", "code", "createcode", __func__, - "createinet", ckp->serverurl); + "createinet", ckp->serverurl[0]); ckdbq_add(ckp, ID_HEARTBEAT, val); } return NULL; @@ -3713,8 +3713,10 @@ int stratifier(proc_instance_t *pi) dealloc(buf); - if (!ckp->serverurl) - ckp->serverurl = "127.0.0.1"; + if (!ckp->serverurls) { + ckp->serverurl[0] = "127.0.0.1"; + ckp->serverurls = 1; + } cklock_init(&sdata->instance_lock); mutex_init(&sdata->ckdb_lock); From b1cd770c8ae7f4ff0a2993c745038947bb5aa8d6 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Fri, 21 Nov 2014 15:53:19 +1100 Subject: [PATCH 23/47] Update ckpool and ckproxy example configurations --- ckpool.conf | 6 +++++- ckproxy.conf | 5 ++++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/ckpool.conf b/ckpool.conf index cf2dcc4e..f857443d 100644 --- a/ckpool.conf +++ b/ckpool.conf @@ -19,7 +19,11 @@ "nonce1length" : 8, "nonce2length" : 8, "update_interval" : 30, -"serverurl" : "ckpool.org:3333", +"serverurl" : [ + "ckpool.org:3333", + "node.ckpool.org:3333", + "node.ckpool.org:80" + ], "mindiff" : 1, "startdiff" : 42, "maxdiff" : 0, diff --git a/ckproxy.conf b/ckproxy.conf index 6754198b..fec5783a 100644 --- a/ckproxy.conf +++ b/ckproxy.conf @@ -12,7 +12,10 @@ } ], "update_interval" : 30, -"serverurl" : "192.168.1.100:3334", +"serverurl" : [ + "192.168.1.100:3334", + "127.0.0.1:3334" + ], "mindiff" : 1, "startdiff" : 42, "maxdiff" : 0, From 0f7b6c5a68cc8e82bb45cabb2f029784a0977d91 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Fri, 21 Nov 2014 17:36:04 +1100 Subject: [PATCH 24/47] Make the connector bind to multiple IPs and Ports specified in the configuration, handling incoming connections --- README | 6 ++- src/connector.c | 123 ++++++++++++++++++++++++++++++------------------ 2 files changed, 82 insertions(+), 47 deletions(-) diff --git a/README b/README index e879c954..a5f0661a 100644 --- a/README +++ b/README @@ -250,9 +250,11 @@ from 2 to 8. Default 8 miners and is set to 30 seconds by default to help perpetuate transactions for the health of the bitcoin network. -"serverurl" : This is the IP to try to bind ckpool uniquely to, otherwise it +"serverurl" : This is the IP(s) to try to bind ckpool uniquely to, otherwise it will attempt to bind to all interfaces in port 3333 by default in pool mode -and 3334 in proxy mode. +and 3334 in proxy mode. Multiple entries can be specified as an array by +either IP or resolvable domain name but the executable must be able to bind to +all of them and ports up to 1024 usually require privileged access. "mindiff" : Minimum diff that vardiff will allow miners to drop to. Default 1 diff --git a/src/connector.c b/src/connector.c index eba91ca0..dcf5e48c 100644 --- a/src/connector.c +++ b/src/connector.c @@ -64,8 +64,14 @@ struct connector_data { ckpool_t *ckp; cklock_t lock; proc_instance_t *pi; - int serverfd; + + /* Array of server fds */ + int *serverfd; + /* Number of server fds */ + int serverfds; + /* All time count of clients connected */ int nfds; + bool accept; pthread_t pth_sender; pthread_t pth_receiver; @@ -109,7 +115,7 @@ static void dec_instance_ref(cdata_t *cdata, client_instance_t *client) /* Accepts incoming connections on the server socket and generates client * instances */ -static int accept_client(cdata_t *cdata, int epfd) +static int accept_client(cdata_t *cdata, const int epfd, const int sockd) { ckpool_t *ckp = cdata->ckp; client_instance_t *client; @@ -128,7 +134,7 @@ static int accept_client(cdata_t *cdata, int epfd) client = ckzalloc(sizeof(client_instance_t)); address_len = sizeof(client->address); - fd = accept(cdata->serverfd, &client->address, &address_len); + fd = accept(sockd, &client->address, &address_len); if (unlikely(fd < 0)) { /* Handle these errors gracefully should we ever share this * socket */ @@ -136,7 +142,7 @@ static int accept_client(cdata_t *cdata, int epfd) LOGERR("Recoverable error on accept in accept_client"); return 0; } - LOGERR("Failed to accept on socket %d in acceptor", cdata->serverfd); + LOGERR("Failed to accept on socket %d in acceptor", sockd); dealloc(client); return -1; } @@ -348,7 +354,7 @@ void *receiver(void *arg) cdata_t *cdata = (cdata_t *)arg; struct epoll_event event; bool maxconn = true; - int ret, epfd; + int ret, epfd, i; rename_proc("creceiver"); @@ -357,12 +363,17 @@ void *receiver(void *arg) LOGEMERG("FATAL: Failed to create epoll in receiver"); return NULL; } - event.data.fd = cdata->serverfd; - event.events = EPOLLIN; - ret = epoll_ctl(epfd, EPOLL_CTL_ADD, cdata->serverfd, &event); - if (ret < 0) { - LOGEMERG("FATAL: Failed to add epfd %d to epoll_ctl", epfd); - return NULL; + /* Add all the serverfds to the epoll */ + for (i = 0; i < cdata->serverfds; i++) { + /* The small values will be easily identifiable compared to + * pointers */ + event.data.u64 = i; + event.events = EPOLLIN; + ret = epoll_ctl(epfd, EPOLL_CTL_ADD, cdata->serverfd[i], &event); + if (ret < 0) { + LOGEMERG("FATAL: Failed to add epfd %d to epoll_ctl", epfd); + return NULL; + } } while (42) { @@ -383,12 +394,13 @@ void *receiver(void *arg) * can receive connections. */ LOGDEBUG("Dropping listen backlog to 0"); maxconn = false; - listen(cdata->serverfd, 0); + for (i = 0; i < cdata->serverfds; i++) + listen(cdata->serverfd[i], 0); } continue; } - if (event.data.fd == cdata->serverfd) { - ret = accept_client(cdata, epfd); + if (event.data.u64 < (uint64_t)cdata->serverfds) { + ret = accept_client(cdata, epfd, (int)cdata->serverfd[event.data.u64]); if (unlikely(ret < 0)) { LOGEMERG("FATAL: Failed to accept_client in receiver"); break; @@ -691,7 +703,7 @@ retry: goto retry; } if (cmdmatch(buf, "getfd")) { - send_fd(cdata->serverfd, sockd); + send_fd(cdata->serverfd[0], sockd); goto retry; } @@ -740,32 +752,23 @@ int connector(proc_instance_t *pi) ckp->data = cdata; cdata->ckp = ckp; + if (!ckp->serverurls) + cdata->serverfd = ckalloc(sizeof(int *)); + else + cdata->serverfd = ckalloc(sizeof(int *) * ckp->serverurls); + if (ckp->oldconnfd > 0) { - sockd = ckp->oldconnfd; - } else if (ckp->serverurls && ckp->serverurl[0]) { - if (!extract_sockaddr(ckp->serverurl[0], &url, &port)) { - LOGWARNING("Failed to extract server address from %s", ckp->serverurl[0]); - ret = 1; - goto out; - } - do { - sockd = bind_socket(url, port); - if (sockd > 0) - break; - LOGWARNING("Connector failed to bind to socket, retrying in 5s"); - sleep(5); - } while (++tries < 25); + /* Only handing over the first interface socket for now */ + cdata->serverfd[0] = ckp->oldconnfd; + cdata->serverfds++; + } - dealloc(url); - dealloc(port); - if (sockd < 0) { - LOGERR("Connector failed to bind to socket for 2 minutes"); - ret = 1; - goto out; - } - } else { + if (!cdata->serverfds && !ckp->serverurls) { + /* No serverurls have been specified and no sockets have been + * inherited. Bind to all interfaces on default sockets. */ struct sockaddr_in serv_addr; + cdata->serverfds = 1; sockd = socket(AF_INET, SOCK_STREAM, 0); if (sockd < 0) { LOGERR("Connector failed to open socket"); @@ -789,20 +792,50 @@ int connector(proc_instance_t *pi) Close(sockd); goto out; } + if (listen(sockd, SOMAXCONN) < 0) { + LOGERR("Connector failed to listen on socket"); + Close(sockd); + goto out; + } + cdata->serverfd[0] = sockd; + } else { + for ( ; cdata->serverfds < ckp->serverurls; cdata->serverfds++) { + char *serverurl = ckp->serverurl[cdata->serverfds]; + + if (!extract_sockaddr(serverurl, &url, &port)) { + LOGWARNING("Failed to extract server address from %s", serverurl); + ret = 1; + goto out; + } + do { + sockd = bind_socket(url, port); + if (sockd > 0) + break; + LOGWARNING("Connector failed to bind to socket, retrying in 5s"); + sleep(5); + } while (++tries < 25); + + dealloc(url); + dealloc(port); + if (sockd < 0) { + LOGERR("Connector failed to bind to socket for 2 minutes"); + ret = 1; + goto out; + } + if (listen(sockd, SOMAXCONN) < 0) { + LOGERR("Connector failed to listen on socket"); + Close(sockd); + goto out; + } + cdata->serverfd[cdata->serverfds] = sockd; + } } + if (tries) LOGWARNING("Connector successfully bound to socket"); - ret = listen(sockd, SOMAXCONN); - if (ret < 0) { - LOGERR("Connector failed to listen on socket"); - Close(sockd); - goto out; - } - cklock_init(&cdata->lock); cdata->pi = pi; - cdata->serverfd = sockd; cdata->nfds = 0; cdata->client_id = 1; mutex_init(&cdata->sender_lock); From 8d4a0f3fa772928a8a62cedfffbaaa4142230d4c Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Fri, 21 Nov 2014 20:02:37 +1100 Subject: [PATCH 25/47] Fix pointer --- src/stratifier.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/stratifier.c b/src/stratifier.c index 94e9c546..f08e185d 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -3353,7 +3353,7 @@ static void update_workerstats(ckpool_t *ckp, sdata_t *sdata) "createdate", cdfield, "createby", "code", "createcode", __func__, - "createinet", ckp->serverurl); + "createinet", ckp->serverurl[0]); worker->notified_idle = worker->idle; ckdbq_add(ckp, ID_WORKERSTATS, val); } From b8e125a1f7c4186383d0553e0bb3c57b5965d715 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Fri, 21 Nov 2014 20:27:52 +1100 Subject: [PATCH 26/47] Store which serverurl each client is bound to in the connector and pass the information to the stratifier --- src/connector.c | 12 +++++++++--- src/stratifier.c | 19 ++++++++++++++++--- 2 files changed, 25 insertions(+), 6 deletions(-) diff --git a/src/connector.c b/src/connector.c index dcf5e48c..4759ec07 100644 --- a/src/connector.c +++ b/src/connector.c @@ -40,6 +40,9 @@ struct client_instance { struct sockaddr address; char address_name[INET6_ADDRSTRLEN]; + /* Which serverurl is this instance connected to */ + int server; + char buf[PAGESIZE]; int bufofs; @@ -115,12 +118,12 @@ static void dec_instance_ref(cdata_t *cdata, client_instance_t *client) /* Accepts incoming connections on the server socket and generates client * instances */ -static int accept_client(cdata_t *cdata, const int epfd, const int sockd) +static int accept_client(cdata_t *cdata, const int epfd, const uint64_t server) { + int fd, port, no_clients, sockd; ckpool_t *ckp = cdata->ckp; client_instance_t *client; struct epoll_event event; - int fd, port, no_clients; socklen_t address_len; ck_rlock(&cdata->lock); @@ -132,7 +135,9 @@ static int accept_client(cdata_t *cdata, const int epfd, const int sockd) return 0; } + sockd = cdata->serverfd[server]; client = ckzalloc(sizeof(client_instance_t)); + client->server = server; address_len = sizeof(client->address); fd = accept(sockd, &client->address, &address_len); if (unlikely(fd < 0)) { @@ -333,6 +338,7 @@ reparse: } else json_object_set_new_nocheck(val, "client_id", json_integer(client->id)); json_object_set_new_nocheck(val, "address", json_string(client->address_name)); + json_object_set_new_nocheck(val, "server", json_integer(client->server)); s = json_dumps(val, 0); if (ckp->passthrough) send_proc(ckp->generator, s); @@ -400,7 +406,7 @@ void *receiver(void *arg) continue; } if (event.data.u64 < (uint64_t)cdata->serverfds) { - ret = accept_client(cdata, epfd, (int)cdata->serverfd[event.data.u64]); + ret = accept_client(cdata, epfd, event.data.u64); if (unlikely(ret < 0)) { LOGEMERG("FATAL: Failed to accept_client in receiver"); break; diff --git a/src/stratifier.c b/src/stratifier.c index f08e185d..ee6af47c 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -247,6 +247,7 @@ struct stratum_instance { char *useragent; char *workername; int64_t user_id; + int server; /* Which server is this instance bound to */ ckpool_t *ckp; @@ -1082,16 +1083,17 @@ static void dec_instance_ref(sdata_t *sdata, stratum_instance_t *instance) } /* Enter with write instance_lock held */ -static stratum_instance_t *__stratum_add_instance(ckpool_t *ckp, int64_t id) +static stratum_instance_t *__stratum_add_instance(ckpool_t *ckp, int64_t id, int server) { stratum_instance_t *instance = ckzalloc(sizeof(stratum_instance_t)); sdata_t *sdata = ckp->data; instance->id = id; + instance->server = server; instance->diff = instance->old_diff = ckp->startdiff; instance->ckp = ckp; tv_time(&instance->ldc); - LOGINFO("Added instance %ld", id); + LOGINFO("Stratifier added instance %ld server %d", id, server); HASH_ADD_I64(sdata->stratum_instances, id, instance); return instance; } @@ -2966,6 +2968,7 @@ static void srecv_process(ckpool_t *ckp, char *buf) sdata_t *sdata = ckp->data; smsg_t *msg; json_t *val; + int server; val = json_loads(buf, 0, NULL); if (unlikely(!val)) { @@ -2995,11 +2998,21 @@ static void srecv_process(ckpool_t *ckp, char *buf) strcpy(msg->address, json_string_value(val)); json_object_clear(val); + val = json_object_get(msg->json_msg, "server"); + if (unlikely(!val)) { + LOGWARNING("Failed to extract server from connector json smsg %s", buf); + json_decref(msg->json_msg); + free(msg); + goto out; + } + server = json_integer_value(val); + json_object_clear(val); + /* Parse the message here */ ck_wlock(&sdata->instance_lock); /* client_id instance doesn't exist yet, create one */ if (!__instance_by_id(sdata, msg->client_id)) - __stratum_add_instance(ckp, msg->client_id); + __stratum_add_instance(ckp, msg->client_id, server); ck_wunlock(&sdata->instance_lock); parse_instance_msg(sdata, msg); From 6ab9129bd6143b4659fd4eaedb51ae490d549565 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Fri, 21 Nov 2014 20:31:18 +1100 Subject: [PATCH 27/47] Send the associated serverurl with client based data sent to ckdb --- src/stratifier.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/stratifier.c b/src/stratifier.c index ee6af47c..af3fb00c 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -2306,7 +2306,7 @@ test_blocksolve(stratum_instance_t *client, workbase_t *wb, const uchar *data, c "createdate", cdfield, "createby", "code", "createcode", __func__, - "createinet", ckp->serverurl[0]); + "createinet", ckp->serverurl[client->server]); val_copy = json_deep_copy(val); block_ckmsg = ckalloc(sizeof(ckmsg_t)); block_ckmsg->data = val_copy; @@ -2606,7 +2606,7 @@ out_unlock: json_set_string(val, "createdate", cdfield); json_set_string(val, "createby", "code"); json_set_string(val, "createcode", __func__); - json_set_string(val, "createinet", ckp->serverurl[0]); + json_set_string(val, "createinet", ckp->serverurl[client->server]); json_set_string(val, "workername", client->workername); json_set_string(val, "username", user_instance->username); @@ -2658,7 +2658,7 @@ out: "createdate", cdfield, "createby", "code", "createcode", __func__, - "createinet", ckp->serverurl[0]); + "createinet", ckp->serverurl[client->server]); ckdbq_add(ckp, ID_SHAREERR, val); LOGINFO("Invalid share from client %ld: %s", client->id, client->workername); } From 6a4da2814ce53d2e13a7a0ab789dd595358d141c Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Thu, 27 Nov 2014 15:11:35 +1100 Subject: [PATCH 28/47] Look for received json messages in the stratifier first since they'll be the most common --- src/stratifier.c | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/src/stratifier.c b/src/stratifier.c index 18e30b63..1813aab1 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -1443,11 +1443,20 @@ retry: dealloc(buf); buf = recv_unix_msg(sockd); - if (!buf) { + if (unlikely(!buf)) { Close(sockd); LOGWARNING("Failed to get message in stratum_loop"); goto retry; } + if (likely(buf[0] == '{')) { + /* The bulk of the messages will be received json from the + * connector so look for this first. The srecv_process frees + * the buf heap ram */ + ckmsgq_add(sdata->srecvs, buf); + Close(sockd); + buf = NULL; + goto retry; + } if (cmdmatch(buf, "ping")) { LOGDEBUG("Stratifier received ping request"); send_unix_msg(sockd, "pong"); @@ -1488,11 +1497,8 @@ retry: reconnect_clients(sdata, buf); } else if (cmdmatch(buf, "loglevel")) { sscanf(buf, "loglevel=%d", &ckp->loglevel); - } else { - /* The srecv_process frees the buf heap ram */ - ckmsgq_add(sdata->srecvs, buf); - buf = NULL; - } + } else + LOGWARNING("Unhandled stratifier message: %s", buf); goto retry; out: From 2dc67ccc3561a68e6c748f6103b3a04553f0a42f Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Thu, 27 Nov 2014 19:33:30 +1100 Subject: [PATCH 29/47] Drop listen backlog to zero as soon as we start polling --- src/connector.c | 20 +++++++------------- 1 file changed, 7 insertions(+), 13 deletions(-) diff --git a/src/connector.c b/src/connector.c index 4759ec07..83c23517 100644 --- a/src/connector.c +++ b/src/connector.c @@ -359,7 +359,6 @@ void *receiver(void *arg) { cdata_t *cdata = (cdata_t *)arg; struct epoll_event event; - bool maxconn = true; int ret, epfd, i; rename_proc("creceiver"); @@ -380,6 +379,12 @@ void *receiver(void *arg) LOGEMERG("FATAL: Failed to add epfd %d to epoll_ctl", epfd); return NULL; } + /* When we first start we listen to as many connections as + * possible. Once we start polling we drop the listen to the + * minimum to effectively ratelimit how fast we can receive + * connections. */ + LOGDEBUG("Dropping listen backlog to 0"); + listen(cdata->serverfd[i], 0); } while (42) { @@ -392,19 +397,8 @@ void *receiver(void *arg) LOGEMERG("FATAL: Failed to epoll_wait in receiver"); break; } - if (unlikely(!ret)) { - if (unlikely(maxconn)) { - /* When we first start we listen to as many connections as - * possible. Once we stop receiving connections we drop the - * listen to the minimum to effectively ratelimit how fast we - * can receive connections. */ - LOGDEBUG("Dropping listen backlog to 0"); - maxconn = false; - for (i = 0; i < cdata->serverfds; i++) - listen(cdata->serverfd[i], 0); - } + if (unlikely(!ret)) continue; - } if (event.data.u64 < (uint64_t)cdata->serverfds) { ret = accept_client(cdata, epfd, event.data.u64); if (unlikely(ret < 0)) { From 8d1af2b7fe1b6088811aafbdad127133457028fd Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Thu, 27 Nov 2014 21:48:02 +1100 Subject: [PATCH 30/47] Check for presence of main process before trying to ping it in case the socket is pointing a new process --- src/ckpool.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/ckpool.c b/src/ckpool.c index 252c9a7f..dc5feab6 100644 --- a/src/ckpool.c +++ b/src/ckpool.c @@ -350,8 +350,10 @@ bool ping_main(ckpool_t *ckp) { char *buf; + if (unlikely(kill_pid(ckp->main.pid, 0))) + return false; buf = send_recv_proc(&ckp->main, "ping"); - if (!buf) + if (unlikely(!buf)) return false; free(buf); return true; From 19bf803b99e68928bea0c386442207a612ee8bc9 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Thu, 27 Nov 2014 22:14:16 +1100 Subject: [PATCH 31/47] Clean up shutting down of old processes, polling for when they exit for faster handover --- src/ckpool.c | 41 +++++++++++++++++++++++++++++++++++------ 1 file changed, 35 insertions(+), 6 deletions(-) diff --git a/src/ckpool.c b/src/ckpool.c index dc5feab6..0308c6ce 100644 --- a/src/ckpool.c +++ b/src/ckpool.c @@ -221,6 +221,21 @@ static int kill_pid(int pid, int sig) return kill(pid, sig); } +static int pid_wait(pid_t pid, int ms) +{ + tv_t start, now; + int ret; + + tv_time(&start); + do { + ret = kill_pid(pid, 0); + if (ret) + break; + tv_time(&now); + } while (ms_tvdiff(&now, &start) < ms); + return ret; +} + static int send_procmsg(proc_instance_t *pi, const char *buf) { char *path = pi->us.path; @@ -676,18 +691,35 @@ static bool write_pid(ckpool_t *ckp, const char *path, pid_t pid) ret = fscanf(fp, "%d", &oldpid); fclose(fp); if (ret == 1 && !(kill_pid(oldpid, 0))) { + if (ckp->handover) { + if (pid_wait(oldpid, 500)) + goto out; + LOGWARNING("Old process pid %d failed to shutdown cleanly, terminating"); + } if (!ckp->killold) { LOGEMERG("Process %s pid %d still exists, start ckpool with -k if you wish to kill it", path, oldpid); return false; } + if (kill_pid(oldpid, 15)) { + LOGEMERG("Unable to kill old process %s pid %d", path, oldpid); + return false; + } + LOGWARNING("Terminating old process %s pid %d", path, oldpid); + if (pid_wait(oldpid, 500)) + goto out; if (kill_pid(oldpid, 9)) { LOGEMERG("Unable to kill old process %s pid %d", path, oldpid); return false; } - LOGWARNING("Killing off old process %s pid %d", path, oldpid); + LOGWARNING("Unable to terminate old process %s pid %d, killing", path, oldpid); + if (!pid_wait(oldpid, 500)) { + LOGEMERG("Unable to kill old process %s pid %d", path, oldpid); + return false; + } } } +out: fp = fopen(path, "we"); if (!fp) { LOGERR("Failed to open file %s", path); @@ -749,11 +781,9 @@ static void childsighandler(int sig) signal(sig, SIG_IGN); signal(SIGTERM, SIG_IGN); if (sig != SIGUSR1) { - pid_t ppid = getppid(); - LOGWARNING("Child process received signal %d, forwarding signal to %s main process", - sig, global_ckp->name); - kill_pid(ppid, sig); + sig, global_ckp->name); + kill_pid(global_ckp->main.pid, sig); } exit(0); } @@ -1436,7 +1466,6 @@ int main(int argc, char **argv) send_recv_path(ckp.main.us.path, "reject"); send_recv_path(ckp.main.us.path, "reconnect"); send_recv_path(ckp.main.us.path, "shutdown"); - cksleep_ms(500); if (ckp.oldconnfd > 0) LOGWARNING("Inherited old socket with new file descriptor %d!", ckp.oldconnfd); From 65a6f4be5f6e286fbb77448aba94b9d75958f0ec Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Thu, 27 Nov 2014 22:45:23 +1100 Subject: [PATCH 32/47] Look for json messages for clients and process them first in the connector --- src/connector.c | 92 +++++++++++++++++++++++-------------------------- 1 file changed, 43 insertions(+), 49 deletions(-) diff --git a/src/connector.c b/src/connector.c index 83c23517..aebc27f9 100644 --- a/src/connector.c +++ b/src/connector.c @@ -596,6 +596,35 @@ static void passthrough_client(cdata_t *cdata, client_instance_t *client) send_client(cdata, client->id, buf); } +static void process_client_msg(cdata_t *cdata, const char *buf) +{ + int64_t client_id64, client_id; + json_t *json_msg; + char *msg; + + json_msg = json_loads(buf, 0, NULL); + if (unlikely(!json_msg)) { + LOGWARNING("Invalid json message: %s", buf); + return; + } + + /* Extract the client id from the json message and remove its entry */ + client_id64 = json_integer_value(json_object_get(json_msg, "client_id")); + json_object_del(json_msg, "client_id"); + if (client_id64 > 0xffffffffll) { + int64_t passthrough_id; + + passthrough_id = client_id64 & 0xffffffffll; + client_id = client_id64 >> 32; + json_object_set_new_nocheck(json_msg, "client_id", json_integer(passthrough_id)); + } else + client_id = client_id64; + msg = json_dumps(json_msg, 0); + realloc_strcat(&msg, "\n"); + send_client(cdata, client_id, msg); + json_decref(json_msg); +} + static int connector_loop(proc_instance_t *pi, cdata_t *cdata) { int sockd = -1, ret = 0, selret; @@ -603,7 +632,6 @@ static int connector_loop(proc_instance_t *pi, cdata_t *cdata) unixsock_t *us = &pi->us; ckpool_t *ckp = pi->ckp; char *buf = NULL; - json_t *json_msg; do { selret = wait_read_select(us->sockd, 5); @@ -642,30 +670,26 @@ retry: LOGWARNING("Failed to get message in connector_loop"); goto retry; } + LOGDEBUG("Connector received message: %s", buf); - if (cmdmatch(buf, "ping")) { + /* The bulk of the messages will be json messages to send to clients + * so look for them first. */ + if (likely(buf[0] == '{')) { + process_client_msg(cdata, buf); + } else if (cmdmatch(buf, "ping")) { LOGDEBUG("Connector received ping request"); send_unix_msg(sockd, "pong"); - goto retry; - } - if (cmdmatch(buf, "accept")) { + } else if (cmdmatch(buf, "accept")) { LOGDEBUG("Connector received accept signal"); cdata->accept = true; - goto retry; - } - if (cmdmatch(buf, "reject")) { + } else if (cmdmatch(buf, "reject")) { LOGDEBUG("Connector received reject signal"); cdata->accept = false; - goto retry; - } - if (cmdmatch(buf, "loglevel")) { + } else if (cmdmatch(buf, "loglevel")) { sscanf(buf, "loglevel=%d", &ckp->loglevel); - goto retry; - } - - if (cmdmatch(buf, "shutdown")) + } else if (cmdmatch(buf, "shutdown")) { goto out; - if (cmdmatch(buf, "dropclient")) { + } else if (cmdmatch(buf, "dropclient")) { client_instance_t *client; ret = sscanf(buf, "dropclient=%ld", &client_id64); @@ -683,9 +707,7 @@ retry: dec_instance_ref(cdata, client); if (ret >= 0) LOGINFO("Connector dropped client id: %ld", client_id); - goto retry; - } - if (cmdmatch(buf, "passthrough")) { + } else if (cmdmatch(buf, "passthrough")) { client_instance_t *client; ret = sscanf(buf, "passthrough=%ld", &client_id); @@ -700,38 +722,10 @@ retry: } passthrough_client(cdata, client); dec_instance_ref(cdata, client); - goto retry; - } - if (cmdmatch(buf, "getfd")) { + } else if (cmdmatch(buf, "getfd")) { send_fd(cdata->serverfd[0], sockd); - goto retry; - } - - /* Anything else should be a json message to send to a client */ - json_msg = json_loads(buf, 0, NULL); - if (unlikely(!json_msg)) { - LOGWARNING("Invalid json message: %s", buf); - goto retry; - } - - /* Extract the client id from the json message and remove its entry */ - client_id64 = json_integer_value(json_object_get(json_msg, "client_id")); - json_object_del(json_msg, "client_id"); - if (client_id64 > 0xffffffffll) { - int64_t passthrough_id; - - passthrough_id = client_id64 & 0xffffffffll; - client_id = client_id64 >> 32; - json_object_set_new_nocheck(json_msg, "client_id", json_integer(passthrough_id)); } else - client_id = client_id64; - dealloc(buf); - buf = json_dumps(json_msg, 0); - realloc_strcat(&buf, "\n"); - send_client(cdata, client_id, buf); - json_decref(json_msg); - buf = NULL; - + LOGWARNING("Unhandled connector message: %s", buf); goto retry; out: Close(sockd); From 00f9cf297d2b0e422cb01e815683a33f1cd570d1 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Thu, 27 Nov 2014 23:33:13 +1100 Subject: [PATCH 33/47] Hand over multiple sockets if we can --- src/ckpool.c | 51 ++++++++++++++++++++++++++++++++----------------- src/ckpool.h | 4 ++-- src/connector.c | 17 +++++++++++------ 3 files changed, 47 insertions(+), 25 deletions(-) diff --git a/src/ckpool.c b/src/ckpool.c index 0308c6ce..e089ace7 100644 --- a/src/ckpool.c +++ b/src/ckpool.c @@ -312,8 +312,8 @@ retry: broadcast_proc(ckp, buf); send_unix_msg(sockd, "success"); } - } else if (cmdmatch(buf, "getfd")) { - int connfd = send_procmsg(ckp->connector, "getfd"); + } else if (cmdmatch(buf, "getxfd")) { + int connfd = send_procmsg(ckp->connector, buf); if (connfd > 0) { int newfd = get_fd(connfd); @@ -1221,16 +1221,21 @@ static struct option long_options[] = { }; #endif -static void send_recv_path(const char *path, const char *msg) +static bool send_recv_path(const char *path, const char *msg) { int sockd = open_unix_client(path); + bool ret = false; char *response; send_unix_msg(sockd, msg); response = recv_unix_msg(sockd); - LOGWARNING("Received: %s in response to %s request", response, msg); - dealloc(response); + if (response) { + ret = true; + LOGWARNING("Received: %s in response to %s request", response, msg); + dealloc(response); + } Close(sockd); + return ret; } int main(int argc, char **argv) @@ -1456,19 +1461,31 @@ int main(int argc, char **argv) ckp.main.processname = strdup("main"); ckp.main.sockname = strdup("listener"); name_process_sockname(&ckp.main.us, &ckp.main); + ckp.oldconnfd = ckzalloc(sizeof(int *) * ckp.serverurls); if (ckp.handover) { - int sockd = open_unix_client(ckp.main.us.path); - - if (sockd > 0 && send_unix_msg(sockd, "getfd")) { - ckp.oldconnfd = get_fd(sockd); - Close(sockd); - - send_recv_path(ckp.main.us.path, "reject"); - send_recv_path(ckp.main.us.path, "reconnect"); - send_recv_path(ckp.main.us.path, "shutdown"); - - if (ckp.oldconnfd > 0) - LOGWARNING("Inherited old socket with new file descriptor %d!", ckp.oldconnfd); + const char *path = ckp.main.us.path; + + if (send_recv_path(path, "ping")) { + for (i = 0; i < ckp.serverurls; i++) { + char getfd[16]; + int sockd; + + snprintf(getfd, 15, "getxfd%d", i); + sockd = open_unix_client(path); + if (sockd < 1) + break; + if (!send_unix_msg(sockd, getfd)) + break; + ckp.oldconnfd[i] = get_fd(sockd); + Close(sockd); + if (!ckp.oldconnfd[i]) + break; + LOGWARNING("Inherited old server socket %d with new file descriptor %d!", + i, ckp.oldconnfd[i]); + } + send_recv_path(path, "reject"); + send_recv_path(path, "reconnect"); + send_recv_path(path, "shutdown"); } } diff --git a/src/ckpool.h b/src/ckpool.h index b495becf..fc534068 100644 --- a/src/ckpool.h +++ b/src/ckpool.h @@ -109,8 +109,8 @@ struct ckpool_instance { /* Logfile */ FILE *logfp; int logfd; - /* Connector fd if we inherited it from a running process */ - int oldconnfd; + /* Connector fds if we inherit them from a running process */ + int *oldconnfd; /* Should we inherit a running instance's socket and shut it down */ bool handover; /* How many clients maximum to accept before rejecting further */ diff --git a/src/connector.c b/src/connector.c index aebc27f9..571f9975 100644 --- a/src/connector.c +++ b/src/connector.c @@ -722,8 +722,12 @@ retry: } passthrough_client(cdata, client); dec_instance_ref(cdata, client); - } else if (cmdmatch(buf, "getfd")) { - send_fd(cdata->serverfd[0], sockd); + } else if (cmdmatch(buf, "getxfd")) { + int fdno = -1; + + sscanf(buf, "getxfd%d", &fdno); + if (fdno > -1 && fdno < cdata->serverfds) + send_fd(cdata->serverfd[fdno], sockd); } else LOGWARNING("Unhandled connector message: %s", buf); goto retry; @@ -738,7 +742,7 @@ int connector(proc_instance_t *pi) cdata_t *cdata = ckzalloc(sizeof(cdata_t)); char *url = NULL, *port = NULL; ckpool_t *ckp = pi->ckp; - int sockd, ret = 0; + int sockd, ret = 0, i; const int on = 1; int tries = 0; @@ -751,9 +755,10 @@ int connector(proc_instance_t *pi) else cdata->serverfd = ckalloc(sizeof(int *) * ckp->serverurls); - if (ckp->oldconnfd > 0) { - /* Only handing over the first interface socket for now */ - cdata->serverfd[0] = ckp->oldconnfd; + for (i = 0; i < ckp->serverurls; i++) { + if (!ckp->oldconnfd[i]) + break; + cdata->serverfd[i] = ckp->oldconnfd[i]; cdata->serverfds++; } From 6d406473341758ca02e6ba77d8cfbdf1303a0469 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Fri, 28 Nov 2014 00:24:58 +1100 Subject: [PATCH 34/47] Add a helper function for converting a struct sockaddr into a url and port --- src/libckpool.c | 25 +++++++++++++++++++++++++ src/libckpool.h | 1 + 2 files changed, 26 insertions(+) diff --git a/src/libckpool.c b/src/libckpool.c index 860b537e..499e66be 100644 --- a/src/libckpool.c +++ b/src/libckpool.c @@ -30,6 +30,7 @@ #include #include #include +#include #include "libckpool.h" #include "sha2.h" @@ -406,6 +407,30 @@ bool extract_sockaddr(char *url, char **sockaddr_url, char **sockaddr_port) return true; } +/* Convert a sockaddr structure into a url and port. URL should be a string of + * INET6_ADDRSTRLEN size */ +bool url_from_sockaddr(const struct sockaddr *addr, char *url, int *port_no) +{ + switch(addr->sa_family) { + const struct sockaddr_in *inet4_in; + const struct sockaddr_in6 *inet6_in; + + case AF_INET: + inet4_in = (struct sockaddr_in *)url; + inet_ntop(AF_INET, &inet4_in->sin_addr, url, INET6_ADDRSTRLEN); + *port_no = htons(inet4_in->sin_port); + break; + case AF_INET6: + inet6_in = (struct sockaddr_in6 *)url; + inet_ntop(AF_INET6, &inet6_in->sin6_addr, url, INET6_ADDRSTRLEN); + *port_no = htons(inet6_in->sin6_port); + break; + default: + return false; + } + return true; +} + void keep_sockalive(int fd) { const int tcp_one = 1; diff --git a/src/libckpool.h b/src/libckpool.h index b6d3c8f7..fba59562 100644 --- a/src/libckpool.h +++ b/src/libckpool.h @@ -422,6 +422,7 @@ static inline bool sock_timeout(void) } bool extract_sockaddr(char *url, char **sockaddr_url, char **sockaddr_port); +bool url_from_sockaddr(const struct sockaddr *addr, char *url, int *port_no); void keep_sockalive(int fd); void nolinger_socket(int fd); void noblock_socket(int fd); From e5f26443afc24ea6eee1d60c8bc1f689c34283b7 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Fri, 28 Nov 2014 00:31:57 +1100 Subject: [PATCH 35/47] Make url from sockaddr take port as a char --- src/libckpool.c | 11 +++++++---- src/libckpool.h | 2 +- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/src/libckpool.c b/src/libckpool.c index 499e66be..7f3f350f 100644 --- a/src/libckpool.c +++ b/src/libckpool.c @@ -408,9 +408,11 @@ bool extract_sockaddr(char *url, char **sockaddr_url, char **sockaddr_port) } /* Convert a sockaddr structure into a url and port. URL should be a string of - * INET6_ADDRSTRLEN size */ -bool url_from_sockaddr(const struct sockaddr *addr, char *url, int *port_no) + * INET6_ADDRSTRLEN size, port at least a string of 6 bytes */ +bool url_from_sockaddr(const struct sockaddr *addr, char *url, char *port) { + int port_no = 0; + switch(addr->sa_family) { const struct sockaddr_in *inet4_in; const struct sockaddr_in6 *inet6_in; @@ -418,16 +420,17 @@ bool url_from_sockaddr(const struct sockaddr *addr, char *url, int *port_no) case AF_INET: inet4_in = (struct sockaddr_in *)url; inet_ntop(AF_INET, &inet4_in->sin_addr, url, INET6_ADDRSTRLEN); - *port_no = htons(inet4_in->sin_port); + port_no = htons(inet4_in->sin_port); break; case AF_INET6: inet6_in = (struct sockaddr_in6 *)url; inet_ntop(AF_INET6, &inet6_in->sin6_addr, url, INET6_ADDRSTRLEN); - *port_no = htons(inet6_in->sin6_port); + port_no = htons(inet6_in->sin6_port); break; default: return false; } + sprintf(port, "%d", port_no); return true; } diff --git a/src/libckpool.h b/src/libckpool.h index fba59562..ecd9465d 100644 --- a/src/libckpool.h +++ b/src/libckpool.h @@ -422,7 +422,7 @@ static inline bool sock_timeout(void) } bool extract_sockaddr(char *url, char **sockaddr_url, char **sockaddr_port); -bool url_from_sockaddr(const struct sockaddr *addr, char *url, int *port_no); +bool url_from_sockaddr(const struct sockaddr *addr, char *url, char *port); void keep_sockalive(int fd); void nolinger_socket(int fd); void noblock_socket(int fd); From 02dfda681449c635467497e74c448ecdea9a9d71 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Fri, 28 Nov 2014 00:41:18 +1100 Subject: [PATCH 36/47] Add a helper function for getting an addinfo from a url and port --- src/libckpool.c | 17 +++++++++++++++++ src/libckpool.h | 2 ++ 2 files changed, 19 insertions(+) diff --git a/src/libckpool.c b/src/libckpool.c index 7f3f350f..f4a28271 100644 --- a/src/libckpool.c +++ b/src/libckpool.c @@ -434,6 +434,23 @@ bool url_from_sockaddr(const struct sockaddr *addr, char *url, char *port) return true; } +bool addrinfo_from_url(const char *url, const char *port, struct addrinfo *addrinfo) +{ + struct addrinfo *servinfo, hints; + + memset(&hints, 0, sizeof(struct addrinfo)); + hints.ai_family = AF_UNSPEC; + hints.ai_socktype = SOCK_STREAM; + servinfo = addrinfo; + if (getaddrinfo(url, port, &hints, &servinfo) != 0) + return false; + if (!servinfo) + return false; + memcpy(addrinfo, servinfo->ai_addr, sizeof(struct addrinfo)); + freeaddrinfo(servinfo); + return true; +} + void keep_sockalive(int fd) { const int tcp_one = 1; diff --git a/src/libckpool.h b/src/libckpool.h index ecd9465d..26994aa7 100644 --- a/src/libckpool.h +++ b/src/libckpool.h @@ -14,6 +14,7 @@ #include #include +#include #include #include #include @@ -423,6 +424,7 @@ static inline bool sock_timeout(void) bool extract_sockaddr(char *url, char **sockaddr_url, char **sockaddr_port); bool url_from_sockaddr(const struct sockaddr *addr, char *url, char *port); +bool addrinfo_from_url(const char *url, const char *port, struct addrinfo *addrinfo); void keep_sockalive(int fd); void nolinger_socket(int fd); void noblock_socket(int fd); From 540db4f5a99ac3f36797e39f54e32f773ac669eb Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Fri, 28 Nov 2014 00:42:37 +1100 Subject: [PATCH 37/47] Free the addrinfo allocated in bind_socket --- src/libckpool.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/libckpool.c b/src/libckpool.c index f4a28271..aa2ab7c4 100644 --- a/src/libckpool.c +++ b/src/libckpool.c @@ -509,7 +509,7 @@ int bind_socket(char *url, char *port) if (getaddrinfo(url, port, &hints, &servinfo) != 0) { LOGWARNING("Failed to resolve (?wrong URL) %s:%s", url, port); - goto out; + return sockd; } for (p = servinfo; p != NULL; p = p->ai_next) { sockd = socket(p->ai_family, p->ai_socktype, p->ai_protocol); @@ -529,6 +529,7 @@ int bind_socket(char *url, char *port) } out: + freeaddrinfo(servinfo); return sockd; } From be87144fe1580cb299fe30dc8439136cf6d3b76d Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Fri, 28 Nov 2014 00:57:26 +1100 Subject: [PATCH 38/47] Provide a helper function for turning a socket into a url and port --- src/libckpool.c | 17 +++++++++++++++++ src/libckpool.h | 2 ++ 2 files changed, 19 insertions(+) diff --git a/src/libckpool.c b/src/libckpool.c index aa2ab7c4..913f8d3e 100644 --- a/src/libckpool.c +++ b/src/libckpool.c @@ -451,6 +451,23 @@ bool addrinfo_from_url(const char *url, const char *port, struct addrinfo *addri return true; } +/* Convert a socket into a url and port. URL should be a string of + * INET6_ADDRSTRLEN size, port at least a string of 6 bytes */ +bool url_from_socket(const int sockd, char *url, char *port) +{ + socklen_t addrlen = sizeof(struct sockaddr); + struct sockaddr addr; + + if (sockd < 1) + return false; + if (getsockname(sockd, &addr, &addrlen)) + return false; + if (!url_from_sockaddr(&addr, url, port)) + return false; + return true; +} + + void keep_sockalive(int fd) { const int tcp_one = 1; diff --git a/src/libckpool.h b/src/libckpool.h index 26994aa7..9b1b3a98 100644 --- a/src/libckpool.h +++ b/src/libckpool.h @@ -425,6 +425,8 @@ static inline bool sock_timeout(void) bool extract_sockaddr(char *url, char **sockaddr_url, char **sockaddr_port); bool url_from_sockaddr(const struct sockaddr *addr, char *url, char *port); bool addrinfo_from_url(const char *url, const char *port, struct addrinfo *addrinfo); +bool url_from_socket(const int sockd, char *url, char *port); + void keep_sockalive(int fd); void nolinger_socket(int fd); void noblock_socket(int fd); From 65f52112ba0b3ce32521abbf3fe8a21c5a17d963 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Fri, 28 Nov 2014 10:12:32 +1100 Subject: [PATCH 39/47] Confirm all handed over sockets match the configured sockets --- src/connector.c | 53 +++++++++++++++++++++++++------------------------ src/libckpool.c | 36 ++++++++++++++++++++++++++++++--- src/libckpool.h | 1 + 3 files changed, 61 insertions(+), 29 deletions(-) diff --git a/src/connector.c b/src/connector.c index 571f9975..15905a18 100644 --- a/src/connector.c +++ b/src/connector.c @@ -70,8 +70,6 @@ struct connector_data { /* Array of server fds */ int *serverfd; - /* Number of server fds */ - int serverfds; /* All time count of clients connected */ int nfds; @@ -358,8 +356,8 @@ reparse: void *receiver(void *arg) { cdata_t *cdata = (cdata_t *)arg; + int ret, epfd, i, serverfds; struct epoll_event event; - int ret, epfd, i; rename_proc("creceiver"); @@ -368,8 +366,9 @@ void *receiver(void *arg) LOGEMERG("FATAL: Failed to create epoll in receiver"); return NULL; } + serverfds = cdata->ckp->serverurls; /* Add all the serverfds to the epoll */ - for (i = 0; i < cdata->serverfds; i++) { + for (i = 0; i < serverfds; i++) { /* The small values will be easily identifiable compared to * pointers */ event.data.u64 = i; @@ -399,7 +398,7 @@ void *receiver(void *arg) } if (unlikely(!ret)) continue; - if (event.data.u64 < (uint64_t)cdata->serverfds) { + if (event.data.u64 < (uint64_t)serverfds) { ret = accept_client(cdata, epfd, event.data.u64); if (unlikely(ret < 0)) { LOGEMERG("FATAL: Failed to accept_client in receiver"); @@ -726,7 +725,7 @@ retry: int fdno = -1; sscanf(buf, "getxfd%d", &fdno); - if (fdno > -1 && fdno < cdata->serverfds) + if (fdno > -1 && fdno < ckp->serverurls) send_fd(cdata->serverfd[fdno], sockd); } else LOGWARNING("Unhandled connector message: %s", buf); @@ -740,7 +739,6 @@ out: int connector(proc_instance_t *pi) { cdata_t *cdata = ckzalloc(sizeof(cdata_t)); - char *url = NULL, *port = NULL; ckpool_t *ckp = pi->ckp; int sockd, ret = 0, i; const int on = 1; @@ -755,19 +753,11 @@ int connector(proc_instance_t *pi) else cdata->serverfd = ckalloc(sizeof(int *) * ckp->serverurls); - for (i = 0; i < ckp->serverurls; i++) { - if (!ckp->oldconnfd[i]) - break; - cdata->serverfd[i] = ckp->oldconnfd[i]; - cdata->serverfds++; - } - - if (!cdata->serverfds && !ckp->serverurls) { - /* No serverurls have been specified and no sockets have been - * inherited. Bind to all interfaces on default sockets. */ + if (!ckp->serverurls) { + /* No serverurls have been specified. Bind to all interfaces + * on default sockets. */ struct sockaddr_in serv_addr; - cdata->serverfds = 1; sockd = socket(AF_INET, SOCK_STREAM, 0); if (sockd < 0) { LOGERR("Connector failed to open socket"); @@ -798,24 +788,35 @@ int connector(proc_instance_t *pi) } cdata->serverfd[0] = sockd; } else { - for ( ; cdata->serverfds < ckp->serverurls; cdata->serverfds++) { - char *serverurl = ckp->serverurl[cdata->serverfds]; + for (i = 0; i < ckp->serverurls; i++) { + char oldurl[INET6_ADDRSTRLEN], oldport[8]; + char newurl[INET6_ADDRSTRLEN], newport[8]; + char *serverurl = ckp->serverurl[i]; - if (!extract_sockaddr(serverurl, &url, &port)) { - LOGWARNING("Failed to extract server address from %s", serverurl); + if (!url_from_serverurl(serverurl, newurl, newport)) { + LOGWARNING("Failed to extract resolved url from %s", serverurl); ret = 1; goto out; } + sockd = ckp->oldconnfd[i]; + if (url_from_socket(sockd, oldurl, oldport)) { + if (strcmp(newurl, oldurl) || strcmp(newport, oldport)) { + LOGWARNING("Handed over socket url %s:%s does not match config %s:%s, creating new socket", + oldurl, oldport, newurl, newport); + Close(sockd); + } + } + do { - sockd = bind_socket(url, port); + if (sockd > 0) + break; + sockd = bind_socket(newurl, newport); if (sockd > 0) break; LOGWARNING("Connector failed to bind to socket, retrying in 5s"); sleep(5); } while (++tries < 25); - dealloc(url); - dealloc(port); if (sockd < 0) { LOGERR("Connector failed to bind to socket for 2 minutes"); ret = 1; @@ -826,7 +827,7 @@ int connector(proc_instance_t *pi) Close(sockd); goto out; } - cdata->serverfd[cdata->serverfds] = sockd; + cdata->serverfd[i] = sockd; } } diff --git a/src/libckpool.c b/src/libckpool.c index 913f8d3e..6a643f7b 100644 --- a/src/libckpool.c +++ b/src/libckpool.c @@ -342,6 +342,8 @@ void cksem_destroy(sem_t *sem) sem_destroy(sem); } +/* Extract just the url and port information from a url string, allocating + * heap memory for sockaddr_url and sockaddr_port. */ bool extract_sockaddr(char *url, char **sockaddr_url, char **sockaddr_port) { char *url_begin, *url_end, *ipv6_begin, *ipv6_end, *port_start = NULL; @@ -418,12 +420,12 @@ bool url_from_sockaddr(const struct sockaddr *addr, char *url, char *port) const struct sockaddr_in6 *inet6_in; case AF_INET: - inet4_in = (struct sockaddr_in *)url; + inet4_in = (struct sockaddr_in *)addr; inet_ntop(AF_INET, &inet4_in->sin_addr, url, INET6_ADDRSTRLEN); port_no = htons(inet4_in->sin_port); break; case AF_INET6: - inet6_in = (struct sockaddr_in6 *)url; + inet6_in = (struct sockaddr_in6 *)addr; inet_ntop(AF_INET6, &inet6_in->sin6_addr, url, INET6_ADDRSTRLEN); port_no = htons(inet6_in->sin6_port); break; @@ -446,11 +448,39 @@ bool addrinfo_from_url(const char *url, const char *port, struct addrinfo *addri return false; if (!servinfo) return false; - memcpy(addrinfo, servinfo->ai_addr, sizeof(struct addrinfo)); + memcpy(addrinfo, servinfo->ai_addr, servinfo->ai_addrlen); freeaddrinfo(servinfo); return true; } +/* Extract a resolved url and port from a serverurl string. newurl must be + * a string of at least INET6_ADDRSTRLEN and newport at least 6 bytes. */ +bool url_from_serverurl(char *serverurl, char *newurl, char *newport) +{ + char *url = NULL, *port = NULL; + struct addrinfo addrinfo; + bool ret = false; + + if (!extract_sockaddr(serverurl, &url, &port)) { + LOGWARNING("Failed to extract server address from %s", serverurl); + goto out; + } + if (!addrinfo_from_url(url, port, &addrinfo)) { + LOGWARNING("Failed to extract addrinfo from url %s:%s", url, port); + goto out; + } + if (!url_from_sockaddr((const struct sockaddr *)&addrinfo, newurl, newport)) { + LOGWARNING("Failed to extract url from sockaddr for original url: %s:%s", + url, port); + goto out; + } + ret = true; +out: + dealloc(url); + dealloc(port); + return ret; +} + /* Convert a socket into a url and port. URL should be a string of * INET6_ADDRSTRLEN size, port at least a string of 6 bytes */ bool url_from_socket(const int sockd, char *url, char *port) diff --git a/src/libckpool.h b/src/libckpool.h index 9b1b3a98..5836a06d 100644 --- a/src/libckpool.h +++ b/src/libckpool.h @@ -425,6 +425,7 @@ static inline bool sock_timeout(void) bool extract_sockaddr(char *url, char **sockaddr_url, char **sockaddr_port); bool url_from_sockaddr(const struct sockaddr *addr, char *url, char *port); bool addrinfo_from_url(const char *url, const char *port, struct addrinfo *addrinfo); +bool url_from_serverurl(char *serverurl, char *newurl, char *newport); bool url_from_socket(const int sockd, char *url, char *port); void keep_sockalive(int fd); From 649ad7953f3550d218c1f46c0fec893ce7a528e1 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Fri, 28 Nov 2014 10:24:32 +1100 Subject: [PATCH 40/47] Fix potential socket leak in gen_loop --- src/generator.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/generator.c b/src/generator.c index 6e00669e..c1804029 100644 --- a/src/generator.c +++ b/src/generator.c @@ -251,6 +251,7 @@ static int gen_loop(proc_instance_t *pi) char hash[68]; reconnect: + Close(sockd); if (si) { kill_server(si); reconnecting = true; @@ -267,6 +268,7 @@ reconnect: } retry: + Close(sockd); ckmsgq_add(gdata->srvchk, si); do { @@ -294,7 +296,6 @@ retry: buf = recv_unix_msg(sockd); if (!buf) { LOGWARNING("Failed to get message in gen_loop"); - Close(sockd); goto retry; } LOGDEBUG("Generator received request: %s", buf); @@ -374,7 +375,6 @@ retry: LOGDEBUG("Generator received ping request"); send_unix_msg(sockd, "pong"); } - Close(sockd); goto retry; out: From 343699a7676a72a628aa97c46aeb0706cc4d54ec Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Fri, 28 Nov 2014 11:19:20 +1100 Subject: [PATCH 41/47] Add sanity checks in pthread cancels/joins --- src/ckpool.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/ckpool.c b/src/ckpool.c index e089ace7..13d10479 100644 --- a/src/ckpool.c +++ b/src/ckpool.c @@ -865,7 +865,7 @@ static void clean_up(ckpool_t *ckp) static void cancel_join_pthread(pthread_t *pth) { - if (!*pth) + if (!pth || !*pth) return; pthread_cancel(*pth); join_pthread(*pth); @@ -874,7 +874,7 @@ static void cancel_join_pthread(pthread_t *pth) static void cancel_pthread(pthread_t *pth) { - if (!*pth) + if (!pth || !*pth) return; pthread_cancel(*pth); pth = NULL; From 92fb508c4463a5c5cbbe052fc636b135a0d80bd4 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Fri, 28 Nov 2014 12:06:59 +1100 Subject: [PATCH 42/47] Handle SIGUSR1 in the child sighandler --- src/ckpool.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/ckpool.c b/src/ckpool.c index 13d10479..ef888c07 100644 --- a/src/ckpool.c +++ b/src/ckpool.c @@ -814,6 +814,7 @@ static void launch_process(proc_instance_t *pi) handler.sa_handler = &childsighandler; handler.sa_flags = 0; sigemptyset(&handler.sa_mask); + sigaction(SIGUSR1, &handler, NULL); sigaction(SIGTERM, &handler, NULL); signal(SIGINT, SIG_IGN); signal(SIGQUIT, SIG_IGN); From 8fb14c8c9b08cb9835e45609d1053632abd406ed Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Fri, 28 Nov 2014 12:18:33 +1100 Subject: [PATCH 43/47] Use SIGUSR1 to shut down children only --- src/ckpool.c | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/src/ckpool.c b/src/ckpool.c index ef888c07..3e17d5e6 100644 --- a/src/ckpool.c +++ b/src/ckpool.c @@ -881,7 +881,7 @@ static void cancel_pthread(pthread_t *pth) pth = NULL; } -static void __shutdown_children(ckpool_t *ckp, int sig) +static void __shutdown_children(ckpool_t *ckp) { int i; @@ -894,15 +894,15 @@ static void __shutdown_children(ckpool_t *ckp, int sig) for (i = 0; i < ckp->proc_instances; i++) { pid_t pid = ckp->children[i]->pid; if (!kill_pid(pid, 0)) - kill_pid(pid, sig); + kill_pid(pid, SIGUSR1); } } -static void shutdown_children(ckpool_t *ckp, int sig) +static void shutdown_children(ckpool_t *ckp) { cancel_join_pthread(&ckp->pth_watchdog); - __shutdown_children(ckp, sig); + __shutdown_children(ckp); } static void sighandler(int sig) @@ -915,10 +915,7 @@ static void sighandler(int sig) ckp->name, sig); cancel_join_pthread(&ckp->pth_watchdog); - __shutdown_children(ckp, SIGUSR1); - /* Wait, then send SIGKILL */ - cksleep_ms(100); - __shutdown_children(ckp, SIGKILL); + __shutdown_children(ckp); cancel_pthread(&ckp->pth_listener); exit(0); } @@ -1516,7 +1513,7 @@ int main(int argc, char **argv) if (ckp.pth_listener) join_pthread(ckp.pth_listener); - shutdown_children(&ckp, SIGTERM); + shutdown_children(&ckp); clean_up(&ckp); return 0; From 3e335c3b385b784e202386a3555f92fa17b8260a Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Sat, 29 Nov 2014 11:07:07 +1100 Subject: [PATCH 44/47] Abort any functions on signal handling, cleaning up cksem functions --- src/connector.c | 2 +- src/libckpool.c | 27 ++++++--------------------- src/libckpool.h | 12 ++++++------ 3 files changed, 13 insertions(+), 28 deletions(-) diff --git a/src/connector.c b/src/connector.c index 15905a18..9b5e626c 100644 --- a/src/connector.c +++ b/src/connector.c @@ -141,7 +141,7 @@ static int accept_client(cdata_t *cdata, const int epfd, const uint64_t server) if (unlikely(fd < 0)) { /* Handle these errors gracefully should we ever share this * socket */ - if (errno == EAGAIN || errno == EWOULDBLOCK || errno == ECONNABORTED || errno == EINTR) { + if (errno == EAGAIN || errno == EWOULDBLOCK || errno == ECONNABORTED) { LOGERR("Recoverable error on accept in accept_client"); return 0; } diff --git a/src/libckpool.c b/src/libckpool.c index 6a643f7b..7c50b4ff 100644 --- a/src/libckpool.c +++ b/src/libckpool.c @@ -295,10 +295,9 @@ void _cksem_post(sem_t *sem, const char *file, const char *func, const int line) void _cksem_wait(sem_t *sem, const char *file, const char *func, const int line) { -retry: if (unlikely(sem_wait(sem))) { if (errno == EINTR) - goto retry; + return; quitfrom(1, file, func, line, "Failed to sem_wait errno=%d sem=0x%p", errno, sem); } } @@ -312,7 +311,6 @@ int _cksem_mswait(sem_t *sem, int ms, const char *file, const char *func, const tv_time(&tv_now); tv_to_ts(&ts_now, &tv_now); ms_to_ts(&abs_timeout, ms); -retry: timeraddspec(&abs_timeout, &ts_now); ret = sem_timedwait(sem, &abs_timeout); @@ -320,26 +318,17 @@ retry: if (likely(errno == ETIMEDOUT)) return ETIMEDOUT; if (errno == EINTR) - goto retry; + return EINTR; quitfrom(1, file, func, line, "Failed to sem_timedwait errno=%d sem=0x%p", errno, sem); } return 0; } -void cksem_reset(sem_t *sem) +void _cksem_destroy(sem_t *sem, const char *file, const char *func, const int line) { - int ret; - - do { - ret = sem_trywait(sem); - if (unlikely(ret < 0 && (errno == EINTR))) - ret = 0; - } while (!ret); -} -void cksem_destroy(sem_t *sem) -{ - sem_destroy(sem); + if (unlikely(sem_destroy(sem))) + quitfrom(1, file, func, line, "Failed to sem_destroy errno=%d sem=0x%p", errno, sem); } /* Extract just the url and port information from a url string, allocating @@ -1587,11 +1576,7 @@ void cksleep_prepare_r(ts_t *ts) void nanosleep_abstime(ts_t *ts_end) { - int ret; - - do { - ret = clock_nanosleep(CLOCK_MONOTONIC, TIMER_ABSTIME, ts_end, NULL); - } while (ret == EINTR); + clock_nanosleep(CLOCK_MONOTONIC, TIMER_ABSTIME, ts_end, NULL); } void timeraddspec(ts_t *a, const ts_t *b) diff --git a/src/libckpool.h b/src/libckpool.h index 5836a06d..b98c8c2c 100644 --- a/src/libckpool.h +++ b/src/libckpool.h @@ -399,13 +399,13 @@ void _cksem_init(sem_t *sem, const char *file, const char *func, const int line) void _cksem_post(sem_t *sem, const char *file, const char *func, const int line); void _cksem_wait(sem_t *sem, const char *file, const char *func, const int line); int _cksem_mswait(sem_t *sem, int ms, const char *file, const char *func, const int line); -void cksem_reset(sem_t *sem); -void cksem_destroy(sem_t *sem); +void _cksem_destroy(sem_t *sem, const char *file, const char *func, const int line); -#define cksem_init(_sem) _cksem_init(_sem, __FILE__, __func__, __LINE__) -#define cksem_post(_sem) _cksem_post(_sem, __FILE__, __func__, __LINE__) -#define cksem_wait(_sem) _cksem_wait(_sem, __FILE__, __func__, __LINE__) -#define cksem_mswait(_sem, _timeout) _cksem_mswait(_sem, _timeout, __FILE__, __func__, __LINE__) +#define cksem_init(SEM) _cksem_init(SEM, __FILE__, __func__, __LINE__) +#define cksem_post(SEM) _cksem_post(SEM, __FILE__, __func__, __LINE__) +#define cksem_wait(SEM) _cksem_wait(SEM, __FILE__, __func__, __LINE__) +#define cksem_mswait(SEM, _timeout) _cksem_mswait(SEM, _timeout, __FILE__, __func__, __LINE__) +#define cksem_destroy(SEM) _cksem_destroy(SEM, __FILE__, __func__, __LINE__) static inline bool sock_connecting(void) { From 098930afd5f6f25909d49402d801fbd0a68cab6f Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Sat, 29 Nov 2014 11:34:07 +1100 Subject: [PATCH 45/47] Add a completion timeout helper function --- src/libckpool.c | 37 +++++++++++++++++++++++++++++++++++++ src/libckpool.h | 1 + 2 files changed, 38 insertions(+) diff --git a/src/libckpool.c b/src/libckpool.c index 7c50b4ff..5d6cbb98 100644 --- a/src/libckpool.c +++ b/src/libckpool.c @@ -78,6 +78,43 @@ void join_pthread(pthread_t thread) pthread_join(thread, NULL); } +struct ck_completion { + sem_t sem; + void (*fn)(void *fnarg); + void *fnarg; +}; + +static void *completion_thread(void *arg) +{ + struct ck_completion *ckc = (struct ck_completion *)arg; + + ckc->fn(ckc->fnarg); + cksem_post(&ckc->sem); + + return NULL; +} + +bool ck_completion_timeout(void *fn, void *fnarg, int timeout) +{ + struct ck_completion ckc; + pthread_t pthread; + bool ret = false; + + cksem_init(&ckc.sem); + ckc.fn = fn; + ckc.fnarg = fnarg; + + pthread_create(&pthread, NULL, completion_thread, (void *)&ckc); + + ret = cksem_mswait(&ckc.sem, timeout); + if (!ret) + pthread_join(pthread, NULL); + else + pthread_cancel(pthread); + return !ret; +} + + /* Place holders for when we add lock debugging */ #define GETLOCK(_lock, _file, _func, _line) #define GOTLOCK(_lock, _file, _func, _line) diff --git a/src/libckpool.h b/src/libckpool.h index b98c8c2c..3401e11a 100644 --- a/src/libckpool.h +++ b/src/libckpool.h @@ -362,6 +362,7 @@ static inline void _json_set_bool(json_t *val, const char *key, bool boolean, void rename_proc(const char *name); void create_pthread(pthread_t *thread, void *(*start_routine)(void *), void *arg); void join_pthread(pthread_t thread); +bool ck_completion_timeout(void *fn, void *fnarg, int timeout); void _mutex_lock(pthread_mutex_t *lock, const char *file, const char *func, const int line); void _mutex_unlock_noyield(pthread_mutex_t *lock, const char *file, const char *func, const int line); From c019a746f5f342f1f7de3bd3c1f8b07f4daeacce Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Sat, 29 Nov 2014 11:40:57 +1100 Subject: [PATCH 46/47] Use a completion timeout to wait for children to exit and kill them if they don't --- src/ckpool.c | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/src/ckpool.c b/src/ckpool.c index 3e17d5e6..66054670 100644 --- a/src/ckpool.c +++ b/src/ckpool.c @@ -881,6 +881,15 @@ static void cancel_pthread(pthread_t *pth) pth = NULL; } +static void wait_child(pid_t *pid) +{ + int ret; + + do { + ret = waitpid(*pid, NULL, 0); + } while (ret != *pid); +} + static void __shutdown_children(ckpool_t *ckp) { int i; @@ -891,10 +900,14 @@ static void __shutdown_children(ckpool_t *ckp) if (!ckp->children) return; + /* Send the children a SIGUSR1 for them to shutdown gracefully, then + * wait for them to exit and kill them if they don't for 500ms. */ for (i = 0; i < ckp->proc_instances; i++) { pid_t pid = ckp->children[i]->pid; - if (!kill_pid(pid, 0)) - kill_pid(pid, SIGUSR1); + + kill_pid(pid, SIGUSR1); + if (!ck_completion_timeout(&wait_child, (void *)&pid, 500)) + kill_pid(pid, SIGKILL); } } From 1df924e73df54fa4006935c25ed9b95fe4463cf6 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Sat, 29 Nov 2014 18:26:42 +1100 Subject: [PATCH 47/47] Provide an option to daemonise ckpool --- src/ckpool.c | 21 ++++++++++++++++++++- src/ckpool.h | 3 +++ 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/src/ckpool.c b/src/ckpool.c index 66054670..078cd1c3 100644 --- a/src/ckpool.c +++ b/src/ckpool.c @@ -1201,6 +1201,7 @@ static void *watchdog(void *arg) static struct option long_options[] = { {"standalone", no_argument, 0, 'A'}, {"config", required_argument, 0, 'c'}, + {"daemonise", no_argument, 0, 'D'}, {"ckdb-name", required_argument, 0, 'd'}, {"group", required_argument, 0, 'g'}, {"handover", no_argument, 0, 'H'}, @@ -1218,6 +1219,7 @@ static struct option long_options[] = { #else static struct option long_options[] = { {"config", required_argument, 0, 'c'}, + {"daemonise", no_argument, 0, 'D'}, {"group", required_argument, 0, 'g'}, {"handover", no_argument, 0, 'H'}, {"help", no_argument, 0, 'h'}, @@ -1267,7 +1269,7 @@ int main(int argc, char **argv) ckp.initial_args[ckp.args] = strdup(argv[ckp.args]); ckp.initial_args[ckp.args] = NULL; - while ((c = getopt_long(argc, argv, "Ac:d:g:HhkLl:n:PpS:s:", long_options, &i)) != -1) { + while ((c = getopt_long(argc, argv, "Ac:Dd:g:HhkLl:n:PpS:s:", long_options, &i)) != -1) { switch (c) { case 'A': ckp.standalone = true; @@ -1275,6 +1277,9 @@ int main(int argc, char **argv) case 'c': ckp.config = optarg; break; + case 'D': + ckp.daemon = true; + break; case 'd': ckp.ckdb_name = optarg; break; @@ -1500,6 +1505,20 @@ int main(int argc, char **argv) } } + if (ckp.daemon) { + int fd; + + if (fork()) + exit(0); + setsid(); + fd = open("/dev/null",O_RDWR, 0); + if (fd != -1) { + dup2(fd, STDIN_FILENO); + dup2(fd, STDOUT_FILENO); + dup2(fd, STDERR_FILENO); + } + } + write_namepid(&ckp.main); open_process_sock(&ckp, &ckp.main, &ckp.main.us); launch_logger(&ckp.main); diff --git a/src/ckpool.h b/src/ckpool.h index fc534068..b8293474 100644 --- a/src/ckpool.h +++ b/src/ckpool.h @@ -141,6 +141,9 @@ struct ckpool_instance { /* Are we running without ckdb */ bool standalone; + /* Should we daemonise the ckpool process */ + bool daemon; + /* Bitcoind data */ int btcds; char **btcdurl;