diff --git a/README b/README index 3d8c90ea..a5f0661a 100644 --- a/README +++ b/README @@ -240,13 +240,21 @@ new network blocks and is 100 by default. It is intended to be a backup only for when the notifier is not set up and only polls if the "notify" field is not set on a btcd. +"nonce1length" : This is optional allowing the extranonce1 length to be chosen +from 2 to 8. Default 8 + +"nonce2length" : This is optional allowing the extranonce2 length to be chosen +from 2 to 8. Default 8 + "update_interval" : This is the frequency that stratum updates are sent out to miners and is set to 30 seconds by default to help perpetuate transactions for the health of the bitcoin network. -"serverurl" : This is the IP to try to bind ckpool uniquely to, otherwise it +"serverurl" : This is the IP(s) to try to bind ckpool uniquely to, otherwise it will attempt to bind to all interfaces in port 3333 by default in pool mode -and 3334 in proxy mode. +and 3334 in proxy mode. Multiple entries can be specified as an array by +either IP or resolvable domain name but the executable must be able to bind to +all of them and ports up to 1024 usually require privileged access. "mindiff" : Minimum diff that vardiff will allow miners to drop to. Default 1 diff --git a/ckpool.conf b/ckpool.conf index 340556bd..f857443d 100644 --- a/ckpool.conf +++ b/ckpool.conf @@ -16,8 +16,14 @@ "btcaddress" : "14BMjogz69qe8hk9thyzbmR5pg34mVKB1e", "btcsig" : "/mined by ck/", "blockpoll" : 100, +"nonce1length" : 8, +"nonce2length" : 8, "update_interval" : 30, -"serverurl" : "ckpool.org:3333", +"serverurl" : [ + "ckpool.org:3333", + "node.ckpool.org:3333", + "node.ckpool.org:80" + ], "mindiff" : 1, "startdiff" : 42, "maxdiff" : 0, diff --git a/ckproxy.conf b/ckproxy.conf index 6754198b..fec5783a 100644 --- a/ckproxy.conf +++ b/ckproxy.conf @@ -12,7 +12,10 @@ } ], "update_interval" : 30, -"serverurl" : "192.168.1.100:3334", +"serverurl" : [ + "192.168.1.100:3334", + "127.0.0.1:3334" + ], "mindiff" : 1, "startdiff" : 42, "maxdiff" : 0, diff --git a/src/ckpool.c b/src/ckpool.c index 54ef0f8e..46ee4edf 100644 --- a/src/ckpool.c +++ b/src/ckpool.c @@ -221,6 +221,21 @@ static int kill_pid(int pid, int sig) return kill(pid, sig); } +static int pid_wait(pid_t pid, int ms) +{ + tv_t start, now; + int ret; + + tv_time(&start); + do { + ret = kill_pid(pid, 0); + if (ret) + break; + tv_time(&now); + } while (ms_tvdiff(&now, &start) < ms); + return ret; +} + static int send_procmsg(proc_instance_t *pi, const char *buf) { char *path = pi->us.path; @@ -297,8 +312,8 @@ retry: broadcast_proc(ckp, buf); send_unix_msg(sockd, "success"); } - } else if (cmdmatch(buf, "getfd")) { - int connfd = send_procmsg(ckp->connector, "getfd"); + } else if (cmdmatch(buf, "getxfd")) { + int connfd = send_procmsg(ckp->connector, buf); if (connfd > 0) { int newfd = get_fd(connfd); @@ -350,8 +365,10 @@ bool ping_main(ckpool_t *ckp) { char *buf; + if (unlikely(kill_pid(ckp->main.pid, 0))) + return false; buf = send_recv_proc(&ckp->main, "ping"); - if (!buf) + if (unlikely(!buf)) return false; free(buf); return true; @@ -674,18 +691,35 @@ static bool write_pid(ckpool_t *ckp, const char *path, pid_t pid) ret = fscanf(fp, "%d", &oldpid); fclose(fp); if (ret == 1 && !(kill_pid(oldpid, 0))) { + if (ckp->handover) { + if (pid_wait(oldpid, 500)) + goto out; + LOGWARNING("Old process pid %d failed to shutdown cleanly, terminating", oldpid); + } if (!ckp->killold) { LOGEMERG("Process %s pid %d still exists, start ckpool with -k if you wish to kill it", path, oldpid); return false; } + if (kill_pid(oldpid, 15)) { + LOGEMERG("Unable to kill old process %s pid %d", path, oldpid); + return false; + } + LOGWARNING("Terminating old process %s pid %d", path, oldpid); + if (pid_wait(oldpid, 500)) + goto out; if (kill_pid(oldpid, 9)) { LOGEMERG("Unable to kill old process %s pid %d", path, oldpid); return false; } - LOGWARNING("Killing off old process %s pid %d", path, oldpid); + LOGWARNING("Unable to terminate old process %s pid %d, killing", path, oldpid); + if (!pid_wait(oldpid, 500)) { + LOGEMERG("Unable to kill old process %s pid %d", path, oldpid); + return false; + } } } +out: fp = fopen(path, "we"); if (!fp) { LOGERR("Failed to open file %s", path); @@ -747,11 +781,9 @@ static void childsighandler(int sig) signal(sig, SIG_IGN); signal(SIGTERM, SIG_IGN); if (sig != SIGUSR1) { - pid_t ppid = getppid(); - LOGWARNING("Child process received signal %d, forwarding signal to %s main process", - sig, global_ckp->name); - kill_pid(ppid, sig); + sig, global_ckp->name); + kill_pid(global_ckp->main.pid, sig); } exit(0); } @@ -834,7 +866,7 @@ static void clean_up(ckpool_t *ckp) static void cancel_join_pthread(pthread_t *pth) { - if (!*pth) + if (!pth || !*pth) return; pthread_cancel(*pth); join_pthread(*pth); @@ -843,13 +875,22 @@ static void cancel_join_pthread(pthread_t *pth) static void cancel_pthread(pthread_t *pth) { - if (!*pth) + if (!pth || !*pth) return; pthread_cancel(*pth); pth = NULL; } -static void __shutdown_children(ckpool_t *ckp, int sig) +static void wait_child(pid_t *pid) +{ + int ret; + + do { + ret = waitpid(*pid, NULL, 0); + } while (ret != *pid); +} + +static void __shutdown_children(ckpool_t *ckp) { int i; @@ -859,18 +900,22 @@ static void __shutdown_children(ckpool_t *ckp, int sig) if (!ckp->children) return; + /* Send the children a SIGUSR1 for them to shutdown gracefully, then + * wait for them to exit and kill them if they don't for 500ms. */ for (i = 0; i < ckp->proc_instances; i++) { pid_t pid = ckp->children[i]->pid; - if (!kill_pid(pid, 0)) - kill_pid(pid, sig); + + kill_pid(pid, SIGUSR1); + if (!ck_completion_timeout(&wait_child, (void *)&pid, 500)) + kill_pid(pid, SIGKILL); } } -static void shutdown_children(ckpool_t *ckp, int sig) +static void shutdown_children(ckpool_t *ckp) { cancel_join_pthread(&ckp->pth_watchdog); - __shutdown_children(ckp, sig); + __shutdown_children(ckp); } static void sighandler(int sig) @@ -883,31 +928,36 @@ static void sighandler(int sig) ckp->name, sig); cancel_join_pthread(&ckp->pth_watchdog); - __shutdown_children(ckp, SIGUSR1); - /* Wait, then send SIGKILL */ - cksleep_ms(100); - __shutdown_children(ckp, SIGKILL); + __shutdown_children(ckp); cancel_pthread(&ckp->pth_listener); exit(0); } -void json_get_string(char **store, json_t *val, const char *res) +static bool _json_get_string(char **store, json_t *entry, const char *res) { - json_t *entry = json_object_get(val, res); + bool ret = false; const char *buf; *store = NULL; if (!entry || json_is_null(entry)) { LOGDEBUG("Json did not find entry %s", res); - return; + goto out; } if (!json_is_string(entry)) { LOGWARNING("Json entry %s is not a string", res); - return; + goto out; } buf = json_string_value(entry); LOGDEBUG("Json found entry %s: %s", res, buf); *store = strdup(buf); + ret = true; +out: + return ret; +} + +bool json_get_string(char **store, json_t *val, const char *res) +{ + return _json_get_string(store, json_object_get(val, res), res); } static void json_get_int64(int64_t *store, json_t *val, const char *res) @@ -926,36 +976,44 @@ static void json_get_int64(int64_t *store, json_t *val, const char *res) LOGDEBUG("Json found entry %s: %ld", res, *store); } -void json_get_int(int *store, json_t *val, const char *res) +bool json_get_int(int *store, json_t *val, const char *res) { json_t *entry = json_object_get(val, res); + bool ret = false; if (!entry) { LOGDEBUG("Json did not find entry %s", res); - return; + goto out; } if (!json_is_integer(entry)) { LOGWARNING("Json entry %s is not an integer", res); - return; + goto out; } *store = json_integer_value(entry); LOGDEBUG("Json found entry %s: %d", res, *store); + ret = true; +out: + return ret; } -static void json_get_bool(bool *store, json_t *val, const char *res) +static bool json_get_bool(bool *store, json_t *val, const char *res) { json_t *entry = json_object_get(val, res); + bool ret = false; if (!entry) { LOGDEBUG("Json did not find entry %s", res); - return; + goto out; } if (!json_is_boolean(entry)) { LOGWARNING("Json entry %s is not a boolean", res); - return; + goto out; } *store = json_is_true(entry); LOGDEBUG("Json found entry %s: %s", res, *store ? "true" : "false"); + ret = true; +out: + return ret; } static void parse_btcds(ckpool_t *ckp, json_t *arr_val, int arr_size) @@ -994,10 +1052,41 @@ static void parse_proxies(ckpool_t *ckp, json_t *arr_val, int arr_size) } } +static bool parse_serverurls(ckpool_t *ckp, json_t *arr_val) +{ + bool ret = false; + int arr_size, i; + + if (!arr_val) + goto out; + if (!json_is_array(arr_val)) { + LOGWARNING("Unable to parse serverurl entries as an array"); + goto out; + } + arr_size = json_array_size(arr_val); + if (!arr_size) { + LOGWARNING("Serverurl array empty"); + goto out; + } + ckp->serverurls = arr_size; + ckp->serverurl = ckalloc(sizeof(char *) * arr_size); + for (i = 0; i < arr_size; i++) { + json_t *val = json_array_get(arr_val, i); + + if (!_json_get_string(&ckp->serverurl[i], val, "serverurl")) + LOGWARNING("Invalid serverurl entry number %d", i); + } + ret = true; +out: + return ret; +} + static void parse_config(ckpool_t *ckp) { json_t *json_conf, *arr_val; json_error_t err_val; + int arr_size; + char *url; json_conf = json_load_file(ckp->config, JSON_DISABLE_EOF_CHECK, &err_val); if (!json_conf) { @@ -1007,8 +1096,7 @@ static void parse_config(ckpool_t *ckp) } arr_val = json_object_get(json_conf, "btcd"); if (arr_val && json_is_array(arr_val)) { - int arr_size = json_array_size(arr_val); - + arr_size = json_array_size(arr_val); if (arr_size) parse_btcds(ckp, arr_val, arr_size); } @@ -1019,8 +1107,18 @@ static void parse_config(ckpool_t *ckp) ckp->btcsig[38] = '\0'; } json_get_int(&ckp->blockpoll, json_conf, "blockpoll"); + json_get_int(&ckp->nonce1length, json_conf, "nonce1length"); + json_get_int(&ckp->nonce2length, json_conf, "nonce2length"); json_get_int(&ckp->update_interval, json_conf, "update_interval"); - json_get_string(&ckp->serverurl, json_conf, "serverurl"); + /* Look for an array first and then a single entry */ + arr_val = json_object_get(json_conf, "serverurl"); + if (!parse_serverurls(ckp, arr_val)) { + if (json_get_string(&url, json_conf, "serverurl")) { + ckp->serverurl = ckalloc(sizeof(char *)); + ckp->serverurl[0] = url; + ckp->serverurls = 1; + } + } json_get_int64(&ckp->mindiff, json_conf, "mindiff"); json_get_int64(&ckp->startdiff, json_conf, "startdiff"); json_get_int64(&ckp->maxdiff, json_conf, "maxdiff"); @@ -1028,8 +1126,7 @@ static void parse_config(ckpool_t *ckp) json_get_int(&ckp->maxclients, json_conf, "maxclients"); arr_val = json_object_get(json_conf, "proxy"); if (arr_val && json_is_array(arr_val)) { - int arr_size = json_array_size(arr_val); - + arr_size = json_array_size(arr_val); if (arr_size) parse_proxies(ckp, arr_val, arr_size); } @@ -1104,6 +1201,7 @@ static void *watchdog(void *arg) static struct option long_options[] = { {"standalone", no_argument, 0, 'A'}, {"config", required_argument, 0, 'c'}, + {"daemonise", no_argument, 0, 'D'}, {"ckdb-name", required_argument, 0, 'd'}, {"group", required_argument, 0, 'g'}, {"handover", no_argument, 0, 'H'}, @@ -1121,6 +1219,7 @@ static struct option long_options[] = { #else static struct option long_options[] = { {"config", required_argument, 0, 'c'}, + {"daemonise", no_argument, 0, 'D'}, {"group", required_argument, 0, 'g'}, {"handover", no_argument, 0, 'H'}, {"help", no_argument, 0, 'h'}, @@ -1135,16 +1234,21 @@ static struct option long_options[] = { }; #endif -static void send_recv_path(const char *path, const char *msg) +static bool send_recv_path(const char *path, const char *msg) { int sockd = open_unix_client(path); + bool ret = false; char *response; send_unix_msg(sockd, msg); response = recv_unix_msg(sockd); - LOGWARNING("Received: %s in response to %s request", response, msg); - dealloc(response); + if (response) { + ret = true; + LOGWARNING("Received: %s in response to %s request", response, msg); + dealloc(response); + } Close(sockd); + return ret; } int main(int argc, char **argv) @@ -1165,7 +1269,7 @@ int main(int argc, char **argv) ckp.initial_args[ckp.args] = strdup(argv[ckp.args]); ckp.initial_args[ckp.args] = NULL; - while ((c = getopt_long(argc, argv, "Ac:d:g:HhkLl:n:PpS:s:", long_options, &i)) != -1) { + while ((c = getopt_long(argc, argv, "Ac:Dd:g:HhkLl:n:PpS:s:", long_options, &i)) != -1) { switch (c) { case 'A': ckp.standalone = true; @@ -1173,6 +1277,9 @@ int main(int argc, char **argv) case 'c': ckp.config = optarg; break; + case 'D': + ckp.daemon = true; + break; case 'd': ckp.ckdb_name = optarg; break; @@ -1313,6 +1420,14 @@ int main(int argc, char **argv) ckp.btcaddress = ckp.donaddress; if (!ckp.blockpoll) ckp.blockpoll = 100; + if (!ckp.nonce1length) + ckp.nonce1length = 8; + else if (ckp.nonce1length < 2 || ckp.nonce1length > 8) + quit(0, "Invalid nonce1length %d specified, must be 2~8", ckp.nonce1length); + if (!ckp.nonce2length) + ckp.nonce2length = 8; + else if (ckp.nonce2length < 2 || ckp.nonce2length > 8) + quit(0, "Invalid nonce2length %d specified, must be 2~8", ckp.nonce2length); if (!ckp.update_interval) ckp.update_interval = 30; if (!ckp.mindiff) @@ -1321,6 +1436,8 @@ int main(int argc, char **argv) ckp.startdiff = 42; if (!ckp.logdir) ckp.logdir = strdup("logs"); + if (!ckp.serverurls) + ckp.serverurl = ckzalloc(sizeof(char *)); if (ckp.proxy && !ckp.proxies) quit(0, "No proxy entries found in config file %s", ckp.config); @@ -1360,20 +1477,45 @@ int main(int argc, char **argv) ckp.main.processname = strdup("main"); ckp.main.sockname = strdup("listener"); name_process_sockname(&ckp.main.us, &ckp.main); + ckp.oldconnfd = ckzalloc(sizeof(int *) * ckp.serverurls); if (ckp.handover) { - int sockd = open_unix_client(ckp.main.us.path); - - if (sockd > 0 && send_unix_msg(sockd, "getfd")) { - ckp.oldconnfd = get_fd(sockd); - Close(sockd); + const char *path = ckp.main.us.path; + + if (send_recv_path(path, "ping")) { + for (i = 0; i < ckp.serverurls; i++) { + char getfd[16]; + int sockd; + + snprintf(getfd, 15, "getxfd%d", i); + sockd = open_unix_client(path); + if (sockd < 1) + break; + if (!send_unix_msg(sockd, getfd)) + break; + ckp.oldconnfd[i] = get_fd(sockd); + Close(sockd); + if (!ckp.oldconnfd[i]) + break; + LOGWARNING("Inherited old server socket %d with new file descriptor %d!", + i, ckp.oldconnfd[i]); + } + send_recv_path(path, "reject"); + send_recv_path(path, "reconnect"); + send_recv_path(path, "shutdown"); + } + } - send_recv_path(ckp.main.us.path, "reject"); - send_recv_path(ckp.main.us.path, "reconnect"); - send_recv_path(ckp.main.us.path, "shutdown"); - cksleep_ms(500); + if (ckp.daemon) { + int fd; - if (ckp.oldconnfd > 0) - LOGWARNING("Inherited old socket with new file descriptor %d!", ckp.oldconnfd); + if (fork()) + exit(0); + setsid(); + fd = open("/dev/null",O_RDWR, 0); + if (fd != -1) { + dup2(fd, STDIN_FILENO); + dup2(fd, STDOUT_FILENO); + dup2(fd, STDERR_FILENO); } } @@ -1403,7 +1545,7 @@ int main(int argc, char **argv) if (ckp.pth_listener) join_pthread(ckp.pth_listener); - shutdown_children(&ckp, SIGTERM); + shutdown_children(&ckp); clean_up(&ckp); return 0; diff --git a/src/ckpool.h b/src/ckpool.h index 6c7743a2..b8293474 100644 --- a/src/ckpool.h +++ b/src/ckpool.h @@ -109,8 +109,8 @@ struct ckpool_instance { /* Logfile */ FILE *logfp; int logfd; - /* Connector fd if we inherited it from a running process */ - int oldconnfd; + /* Connector fds if we inherit them from a running process */ + int *oldconnfd; /* Should we inherit a running instance's socket and shut it down */ bool handover; /* How many clients maximum to accept before rejecting further */ @@ -141,6 +141,9 @@ struct ckpool_instance { /* Are we running without ckdb */ bool standalone; + /* Should we daemonise the ckpool process */ + bool daemon; + /* Bitcoind data */ int btcds; char **btcdurl; @@ -148,6 +151,8 @@ struct ckpool_instance { char **btcdpass; bool *btcdnotify; int blockpoll; // How frequently in ms to poll bitcoind for block updates + int nonce1length; // Extranonce1 length + int nonce2length; // Extranonce2 length /* Difficulty settings */ int64_t mindiff; // Default 1 @@ -162,7 +167,8 @@ struct ckpool_instance { /* Stratum options */ server_instance_t **servers; - char *serverurl; // URL to bind our server/proxy to + char **serverurl; // Array of URLs to bind our server/proxy to + int serverurls; // Number of server bindings int update_interval; // Seconds between stratum updates int chosen_server; // Chosen server for next connection @@ -172,6 +178,9 @@ struct ckpool_instance { char **proxyauth; char **proxypass; server_instance_t *btcdbackup; + + /* Private data for each process */ + void *data; }; #ifdef USE_CKDB @@ -203,7 +212,7 @@ char *_ckdb_msg_call(const ckpool_t *ckp, char *msg, const char *file, const ch json_t *json_rpc_call(connsock_t *cs, const char *rpc_req); int process_exit(ckpool_t *ckp, proc_instance_t *pi, int ret); -void json_get_string(char **store, json_t *val, const char *res); -void json_get_int(int *store, json_t *val, const char *res); +bool json_get_string(char **store, json_t *val, const char *res); +bool json_get_int(int *store, json_t *val, const char *res); #endif /* CKPOOL_H */ diff --git a/src/connector.c b/src/connector.c index 93ed41e1..7355f9aa 100644 --- a/src/connector.c +++ b/src/connector.c @@ -24,30 +24,25 @@ #define MAX_MSGSIZE 1024 #define SOI (sizeof(int)) -struct connector_instance { - cklock_t lock; - proc_instance_t *pi; - int serverfd; - int nfds; - bool accept; - pthread_t pth_sender; - pthread_t pth_receiver; -}; - -typedef struct connector_instance conn_instance_t; - struct client_instance { /* For clients hashtable */ UT_hash_handle hh; int64_t id; int fd; + /* Reference count for when this instance is used outside of the + * connector_data lock */ + int ref; + /* For dead_clients list */ struct client_instance *next; struct sockaddr address; char address_name[INET6_ADDRSTRLEN]; + /* Which serverurl is this instance connected to */ + int server; + char buf[PAGESIZE]; int bufofs; @@ -56,13 +51,6 @@ struct client_instance { typedef struct client_instance client_instance_t; -/* For the hashtable of all clients */ -static client_instance_t *clients; -/* Linked list of dead clients no longer in use but may still have references */ -static client_instance_t *dead_clients; - -static int64_t client_id = 1; - struct sender_send { struct sender_send *next; struct sender_send *prev; @@ -74,44 +62,90 @@ struct sender_send { typedef struct sender_send sender_send_t; -/* For the linked list of pending sends */ -static sender_send_t *sender_sends; -static sender_send_t *delayed_sends; +/* Private data for the connector */ +struct connector_data { + ckpool_t *ckp; + cklock_t lock; + proc_instance_t *pi; + + /* Array of server fds */ + int *serverfd; + /* All time count of clients connected */ + int nfds; -/* For protecting the pending sends list */ -static pthread_mutex_t sender_lock; -static pthread_cond_t sender_cond; + bool accept; + pthread_t pth_sender; + pthread_t pth_receiver; + + /* For the hashtable of all clients */ + client_instance_t *clients; + /* Linked list of dead clients no longer in use but may still have references */ + client_instance_t *dead_clients; + + int64_t client_id; + + /* For the linked list of pending sends */ + sender_send_t *sender_sends; + sender_send_t *delayed_sends; + + /* For protecting the pending sends list */ + pthread_mutex_t sender_lock; + pthread_cond_t sender_cond; +}; + +typedef struct connector_data cdata_t; + +/* Increase the reference count of instance */ +static void __inc_instance_ref(client_instance_t *client) +{ + client->ref++; +} + +/* Increase the reference count of instance */ +static void __dec_instance_ref(client_instance_t *client) +{ + client->ref--; +} + +static void dec_instance_ref(cdata_t *cdata, client_instance_t *client) +{ + ck_wlock(&cdata->lock); + __dec_instance_ref(client); + ck_wunlock(&cdata->lock); +} /* Accepts incoming connections on the server socket and generates client * instances */ -static int accept_client(conn_instance_t *ci, int epfd) +static int accept_client(cdata_t *cdata, const int epfd, const uint64_t server) { - ckpool_t *ckp = ci->pi->ckp; + int fd, port, no_clients, sockd; + ckpool_t *ckp = cdata->ckp; client_instance_t *client; struct epoll_event event; - int fd, port, no_clients; socklen_t address_len; - ck_rlock(&ci->lock); - no_clients = HASH_COUNT(clients); - ck_runlock(&ci->lock); + ck_rlock(&cdata->lock); + no_clients = HASH_COUNT(cdata->clients); + ck_runlock(&cdata->lock); if (unlikely(ckp->maxclients && no_clients >= ckp->maxclients)) { LOGWARNING("Server full with %d clients", no_clients); return 0; } + sockd = cdata->serverfd[server]; client = ckzalloc(sizeof(client_instance_t)); + client->server = server; address_len = sizeof(client->address); - fd = accept(ci->serverfd, &client->address, &address_len); + fd = accept(sockd, &client->address, &address_len); if (unlikely(fd < 0)) { /* Handle these errors gracefully should we ever share this * socket */ - if (errno == EAGAIN || errno == EWOULDBLOCK || errno == ECONNABORTED || errno == EINTR) { + if (errno == EAGAIN || errno == EWOULDBLOCK || errno == ECONNABORTED) { LOGERR("Recoverable error on accept in accept_client"); return 0; } - LOGERR("Failed to accept on socket %d in acceptor", ci->serverfd); + LOGERR("Failed to accept on socket %d in acceptor", sockd); dealloc(client); return -1; } @@ -132,7 +166,7 @@ static int accept_client(conn_instance_t *ci, int epfd) break; default: LOGWARNING("Unknown INET type for client %d on socket %d", - ci->nfds, fd); + cdata->nfds, fd); Close(fd); free(client); return 0; @@ -142,7 +176,7 @@ static int accept_client(conn_instance_t *ci, int epfd) nolinger_socket(fd); LOGINFO("Connected new client %d on socket %d to %d active clients from %s:%d", - ci->nfds, fd, no_clients, client->address_name, port); + cdata->nfds, fd, no_clients, client->address_name, port); client->fd = fd; event.data.ptr = client; @@ -153,27 +187,36 @@ static int accept_client(conn_instance_t *ci, int epfd) return 0; } - ck_wlock(&ci->lock); - client->id = client_id++; - HASH_ADD_I64(clients, id, client); - ci->nfds++; - ck_wunlock(&ci->lock); + /* We increase the ref count on this client as epoll creates a pointer + * to it. We drop that reference when the socket is closed which + * removes it automatically from the epoll list. */ + __inc_instance_ref(client); + + ck_wlock(&cdata->lock); + client->id = cdata->client_id++; + HASH_ADD_I64(cdata->clients, id, client); + cdata->nfds++; + ck_wunlock(&cdata->lock); return 1; } -static int drop_client(conn_instance_t *ci, client_instance_t *client) +/* Client must hold a reference count */ +static int drop_client(cdata_t *cdata, client_instance_t *client) { int fd; - ck_wlock(&ci->lock); + ck_wlock(&cdata->lock); fd = client->fd; if (fd != -1) { Close(client->fd); - HASH_DEL(clients, client); - LL_PREPEND(dead_clients, client); + HASH_DEL(cdata->clients, client); + LL_PREPEND(cdata->dead_clients, client); + /* This is the reference to this client's presence in the + * epoll list. */ + __dec_instance_ref(client); } - ck_wunlock(&ci->lock); + ck_wunlock(&cdata->lock); if (fd > -1) LOGINFO("Connector dropped client %"PRId64" fd %d", client->id, fd); @@ -190,28 +233,44 @@ static void stratifier_drop_client(ckpool_t *ckp, int64_t id) } /* Invalidate this instance. Remove them from the hashtables we look up - * regularly but keep the instances in a linked list indefinitely in case we - * still reference any of its members. */ -static void invalidate_client(ckpool_t *ckp, conn_instance_t *ci, client_instance_t *client) + * regularly but keep the instances in a linked list until their ref count + * drops to zero when we can remove them lazily. Client must hold a reference + * count. */ +static void invalidate_client(ckpool_t *ckp, cdata_t *cdata, client_instance_t *client) { - drop_client(ci, client); + client_instance_t *tmp; + + drop_client(cdata, client); if (ckp->passthrough) return; stratifier_drop_client(ckp, client->id); + + /* Cull old unused clients lazily when there are no more reference + * counts for them. */ + ck_wlock(&cdata->lock); + LL_FOREACH_SAFE(cdata->dead_clients, client, tmp) { + if (!client->ref) { + LL_DELETE(cdata->dead_clients, client); + LOGINFO("Connector discarding client %ld", client->id); + free(client); + } + } + ck_wunlock(&cdata->lock); } -static void send_client(conn_instance_t *ci, int64_t id, char *buf); +static void send_client(cdata_t *cdata, int64_t id, char *buf); -static void parse_client_msg(conn_instance_t *ci, client_instance_t *client) +/* Client is holding a reference count from being on the epoll list */ +static void parse_client_msg(cdata_t *cdata, client_instance_t *client) { int buflen, ret, selfail = 0; - ckpool_t *ckp = ci->pi->ckp; + ckpool_t *ckp = cdata->ckp; char msg[PAGESIZE], *eol; json_t *val; retry: /* Select should always return positive after poll unless we have - * been disconnected. On retries, decide whether we should do further + * been disconnected. On retries, decdatade whether we should do further * reads based on select readiness and only fail if we get an error. */ ret = wait_read_select(client->fd, 0); if (ret < 1) { @@ -219,7 +278,7 @@ retry: return; LOGINFO("Client fd %d disconnected - select fail with bufofs %d ret %d errno %d %s", client->fd, client->bufofs, ret, errno, ret && errno ? strerror(errno) : ""); - invalidate_client(ckp, ci, client); + invalidate_client(ckp, cdata, client); return; } selfail = -1; @@ -231,7 +290,7 @@ retry: * client has disconnected. */ LOGINFO("Client fd %d disconnected - recv fail with bufofs %d ret %d errno %d %s", client->fd, client->bufofs, ret, errno, ret && errno ? strerror(errno) : ""); - invalidate_client(ckp, ci, client); + invalidate_client(ckp, cdata, client); return; } client->bufofs += ret; @@ -240,7 +299,7 @@ reparse: if (!eol) { if (unlikely(client->bufofs > MAX_MSGSIZE)) { LOGWARNING("Client fd %d overloaded buffer without EOL, disconnecting", client->fd); - invalidate_client(ckp, ci, client); + invalidate_client(ckp, cdata, client); return; } goto retry; @@ -250,7 +309,7 @@ reparse: buflen = eol - client->buf + 1; if (unlikely(buflen > MAX_MSGSIZE)) { LOGWARNING("Client fd %d message oversize, disconnecting", client->fd); - invalidate_client(ckp, ci, client); + invalidate_client(ckp, cdata, client); return; } memcpy(msg, client->buf, buflen); @@ -261,9 +320,9 @@ reparse: if (!(val = json_loads(msg, 0, NULL))) { char *buf = strdup("Invalid JSON, disconnecting\n"); - LOGINFO("Client id %"PRId64" sent invalid json message %s", client->id, msg); - send_client(ci, client->id, buf); - invalidate_client(ckp, ci, client); + LOGINFO("Client id %ld sent invalid json message %s", client->id, msg); + send_client(cdata, client->id, buf); + invalidate_client(ckp, cdata, client); return; } else { int64_t passthrough_id; @@ -277,6 +336,7 @@ reparse: } else json_object_set_new_nocheck(val, "client_id", json_integer(client->id)); json_object_set_new_nocheck(val, "address", json_string(client->address_name)); + json_object_set_new_nocheck(val, "server", json_integer(client->server)); s = json_dumps(val, 0); if (ckp->passthrough) send_proc(ckp->generator, s); @@ -295,10 +355,9 @@ reparse: * handles the incoming messages */ void *receiver(void *arg) { - conn_instance_t *ci = (conn_instance_t *)arg; + cdata_t *cdata = (cdata_t *)arg; + int ret, epfd, i, serverfds; struct epoll_event event; - bool maxconn = true; - int ret, epfd; rename_proc("creceiver"); @@ -307,38 +366,40 @@ void *receiver(void *arg) LOGEMERG("FATAL: Failed to create epoll in receiver"); return NULL; } - event.data.fd = ci->serverfd; - event.events = EPOLLIN; - ret = epoll_ctl(epfd, EPOLL_CTL_ADD, ci->serverfd, &event); - if (ret < 0) { - LOGEMERG("FATAL: Failed to add epfd %d to epoll_ctl", epfd); - return NULL; + serverfds = cdata->ckp->serverurls; + /* Add all the serverfds to the epoll */ + for (i = 0; i < serverfds; i++) { + /* The small values will be easily identifiable compared to + * pointers */ + event.data.u64 = i; + event.events = EPOLLIN; + ret = epoll_ctl(epfd, EPOLL_CTL_ADD, cdata->serverfd[i], &event); + if (ret < 0) { + LOGEMERG("FATAL: Failed to add epfd %d to epoll_ctl", epfd); + return NULL; + } + /* When we first start we listen to as many connections as + * possible. Once we start polling we drop the listen to the + * minimum to effectively ratelimit how fast we can receive + * connections. */ + LOGDEBUG("Dropping listen backlog to 0"); + listen(cdata->serverfd[i], 0); } while (42) { client_instance_t *client; - while (unlikely(!ci->accept)) + while (unlikely(!cdata->accept)) cksleep_ms(100); ret = epoll_wait(epfd, &event, 1, 1000); if (unlikely(ret == -1)) { LOGEMERG("FATAL: Failed to epoll_wait in receiver"); break; } - if (unlikely(!ret)) { - if (unlikely(maxconn)) { - /* When we first start we listen to as many connections as - * possible. Once we stop receiving connections we drop the - * listen to the minimum to effectively ratelimit how fast we - * can receive connections. */ - LOGDEBUG("Dropping listen backlog to 0"); - maxconn = false; - listen(ci->serverfd, 0); - } + if (unlikely(!ret)) continue; - } - if (event.data.fd == ci->serverfd) { - ret = accept_client(ci, epfd); + if (event.data.u64 < (uint64_t)serverfds) { + ret = accept_client(cdata, epfd, event.data.u64); if (unlikely(ret < 0)) { LOGEMERG("FATAL: Failed to accept_client in receiver"); break; @@ -349,10 +410,10 @@ void *receiver(void *arg) if ((event.events & EPOLLERR) || (event.events & EPOLLHUP)) { /* Client disconnected */ LOGDEBUG("Client fd %d HUP in epoll", client->fd); - invalidate_client(ci->pi->ckp, ci, client); + invalidate_client(cdata->pi->ckp, cdata, client); continue; } - parse_client_msg(ci, client); + parse_client_msg(cdata, client); } return NULL; } @@ -361,8 +422,8 @@ void *receiver(void *arg) * ready for writing immediately to not delay other messages. */ void *sender(void *arg) { - conn_instance_t *ci = (conn_instance_t *)arg; - ckpool_t *ckp = ci->pi->ckp; + cdata_t *cdata = (cdata_t *)arg; + ckpool_t *ckp = cdata->ckp; bool sent = false; rename_proc("csender"); @@ -372,23 +433,23 @@ void *sender(void *arg) client_instance_t *client; int ret, fd, ofs = 0; - mutex_lock(&sender_lock); + mutex_lock(&cdata->sender_lock); /* Poll every 100ms if there are no new sends. Re-examine * delayed sends immediately after a successful send in case * endless new sends more frequently end up starving the * delayed sends. */ - if (!sender_sends && !sent) { + if (!cdata->sender_sends && !sent) { const ts_t polltime = {0, 100000000}; ts_t timeout_ts; ts_realtime(&timeout_ts); timeraddspec(&timeout_ts, &polltime); - pthread_cond_timedwait(&sender_cond, &sender_lock, &timeout_ts); + pthread_cond_timedwait(&cdata->sender_cond, &cdata->sender_lock, &timeout_ts); } - sender_send = sender_sends; + sender_send = cdata->sender_sends; if (sender_send) - DL_DELETE(sender_sends, sender_send); - mutex_unlock(&sender_lock); + DL_DELETE(cdata->sender_sends, sender_send); + mutex_unlock(&cdata->sender_lock); sent = false; @@ -396,23 +457,21 @@ void *sender(void *arg) * conditional with no new sends appearing or have just * serviced another message successfully. */ if (!sender_send) { - if (!delayed_sends) + if (!cdata->delayed_sends) continue; - sender_send = delayed_sends; - DL_DELETE(delayed_sends, sender_send); + sender_send = cdata->delayed_sends; + DL_DELETE(cdata->delayed_sends, sender_send); } client = sender_send->client; - ck_rlock(&ci->lock); + ck_rlock(&cdata->lock); fd = client->fd; - ck_runlock(&ci->lock); + ck_runlock(&cdata->lock); if (fd == -1) { LOGDEBUG("Discarding message sent to invalidated client"); - free(sender_send->buf); - free(sender_send); - continue; + goto contfree; } /* If this socket is not ready to receive data from us, put the * send back on the tail of the list and decrease the timeout @@ -421,33 +480,33 @@ void *sender(void *arg) ret = wait_write_select(fd, 0); if (ret < 1) { if (ret < 0) { - LOGINFO("Client id %"PRId64" fd %d interrupted", client->id, fd); - invalidate_client(ckp, ci, client); - free(sender_send->buf); - free(sender_send); - continue; + LOGINFO("Client id %ld fd %d interrupted", client->id, fd); + invalidate_client(ckp, cdata, client); + goto contfree; } LOGDEBUG("Client %"PRId64" not ready for writes", client->id); /* Append it to the tail of the delayed sends list. * This is the only function that alters it so no - * locking is required. */ - DL_APPEND(delayed_sends, sender_send); + * locking is required. Keep the client ref. */ + DL_APPEND(cdata->delayed_sends, sender_send); continue; } sent = true; while (sender_send->len) { ret = send(fd, sender_send->buf + ofs, sender_send->len , 0); if (unlikely(ret < 0)) { - LOGINFO("Client id %"PRId64" fd %d disconnected", client->id, fd); - invalidate_client(ckp, ci, client); + LOGINFO("Client id %ld fd %d disconnected", client->id, fd); + invalidate_client(ckp, cdata, client); break; } ofs += ret; sender_send->len -= ret; } +contfree: free(sender_send->buf); free(sender_send); + dec_instance_ref(cdata, client); } return NULL; @@ -455,7 +514,7 @@ void *sender(void *arg) /* Send a client by id a heap allocated buffer, allowing this function to * free the ram. */ -static void send_client(conn_instance_t *ci, int64_t id, char *buf) +static void send_client(cdata_t *cdata, int64_t id, char *buf) { sender_send_t *sender_send; client_instance_t *client; @@ -472,19 +531,25 @@ static void send_client(conn_instance_t *ci, int64_t id, char *buf) return; } - ck_rlock(&ci->lock); - HASH_FIND_I64(clients, &id, client); - if (likely(client)) + ck_ilock(&cdata->lock); + HASH_FIND_I64(cdata->clients, &id, client); + if (likely(client)) { + ck_ulock(&cdata->lock); fd = client->fd; - ck_runlock(&ci->lock); + /* Grab a reference to this client until the sender_send has + * completed processing. */ + __inc_instance_ref(client); + ck_dwilock(&cdata->lock); + } + ck_uilock(&cdata->lock); if (unlikely(fd == -1)) { - ckpool_t *ckp = ci->pi->ckp; + ckpool_t *ckp = cdata->ckp; if (client) { /* This shouldn't happen */ LOGWARNING("Client id %ld disconnected but fd already invalidated!", id); - invalidate_client(ckp, ci, client); + invalidate_client(ckp, cdata, client); } else { LOGINFO("Connector failed to find client id %ld to send to", id); stratifier_drop_client(ckp, id); @@ -498,41 +563,74 @@ static void send_client(conn_instance_t *ci, int64_t id, char *buf) sender_send->buf = buf; sender_send->len = len; - mutex_lock(&sender_lock); - DL_APPEND(sender_sends, sender_send); - pthread_cond_signal(&sender_cond); - mutex_unlock(&sender_lock); + mutex_lock(&cdata->sender_lock); + DL_APPEND(cdata->sender_sends, sender_send); + pthread_cond_signal(&cdata->sender_cond); + mutex_unlock(&cdata->sender_lock); } -static client_instance_t *client_by_id(conn_instance_t *ci, int64_t id) +static client_instance_t *ref_client_by_id(cdata_t *cdata, int64_t id) { client_instance_t *client; - ck_rlock(&ci->lock); - HASH_FIND_I64(clients, &id, client); - ck_runlock(&ci->lock); + ck_ilock(&cdata->lock); + HASH_FIND_I64(cdata->clients, &id, client); + if (client) { + ck_ulock(&cdata->lock); + __inc_instance_ref(client); + ck_dwilock(&cdata->lock); + } + ck_uilock(&cdata->lock); return client; } -static void passthrough_client(conn_instance_t *ci, client_instance_t *client) +static void passthrough_client(cdata_t *cdata, client_instance_t *client) { char *buf; LOGINFO("Connector adding passthrough client %"PRId64, client->id); client->passthrough = true; ASPRINTF(&buf, "{\"result\": true}\n"); - send_client(ci, client->id, buf); + send_client(cdata, client->id, buf); } -static int connector_loop(proc_instance_t *pi, conn_instance_t *ci) +static void process_client_msg(cdata_t *cdata, const char *buf) +{ + int64_t client_id64, client_id; + json_t *json_msg; + char *msg; + + json_msg = json_loads(buf, 0, NULL); + if (unlikely(!json_msg)) { + LOGWARNING("Invalid json message: %s", buf); + return; + } + + /* Extract the client id from the json message and remove its entry */ + client_id64 = json_integer_value(json_object_get(json_msg, "client_id")); + json_object_del(json_msg, "client_id"); + if (client_id64 > 0xffffffffll) { + int64_t passthrough_id; + + passthrough_id = client_id64 & 0xffffffffll; + client_id = client_id64 >> 32; + json_object_set_new_nocheck(json_msg, "client_id", json_integer(passthrough_id)); + } else + client_id = client_id64; + msg = json_dumps(json_msg, 0); + realloc_strcat(&msg, "\n"); + send_client(cdata, client_id, msg); + json_decref(json_msg); +} + +static int connector_loop(proc_instance_t *pi, cdata_t *cdata) { int sockd = -1, ret = 0, selret; int64_t client_id64, client_id; unixsock_t *us = &pi->us; ckpool_t *ckp = pi->ckp; char *buf = NULL; - json_t *json_msg; do { selret = wait_read_select(us->sockd, 5); @@ -546,12 +644,12 @@ static int connector_loop(proc_instance_t *pi, conn_instance_t *ci) LOGWARNING("%s connector ready", ckp->name); retry: - if (unlikely(!pthread_tryjoin_np(ci->pth_sender, NULL))) { + if (unlikely(!pthread_tryjoin_np(cdata->pth_sender, NULL))) { LOGEMERG("Connector sender thread shutdown, exiting"); ret = 1; goto out; } - if (unlikely(!pthread_tryjoin_np(ci->pth_receiver, NULL))) { + if (unlikely(!pthread_tryjoin_np(cdata->pth_receiver, NULL))) { LOGEMERG("Connector receiver thread shutdown, exiting"); ret = 1; goto out; @@ -571,30 +669,26 @@ retry: LOGWARNING("Failed to get message in connector_loop"); goto retry; } + LOGDEBUG("Connector received message: %s", buf); - if (cmdmatch(buf, "ping")) { + /* The bulk of the messages will be json messages to send to clients + * so look for them first. */ + if (likely(buf[0] == '{')) { + process_client_msg(cdata, buf); + } else if (cmdmatch(buf, "ping")) { LOGDEBUG("Connector received ping request"); send_unix_msg(sockd, "pong"); - goto retry; - } - if (cmdmatch(buf, "accept")) { + } else if (cmdmatch(buf, "accept")) { LOGDEBUG("Connector received accept signal"); - ci->accept = true; - goto retry; - } - if (cmdmatch(buf, "reject")) { + cdata->accept = true; + } else if (cmdmatch(buf, "reject")) { LOGDEBUG("Connector received reject signal"); - ci->accept = false; - goto retry; - } - if (cmdmatch(buf, "loglevel")) { + cdata->accept = false; + } else if (cmdmatch(buf, "loglevel")) { sscanf(buf, "loglevel=%d", &ckp->loglevel); - goto retry; - } - - if (cmdmatch(buf, "shutdown")) + } else if (cmdmatch(buf, "shutdown")) { goto out; - if (cmdmatch(buf, "dropclient")) { + } else if (cmdmatch(buf, "dropclient")) { client_instance_t *client; ret = sscanf(buf, "dropclient=%ld", &client_id64); @@ -603,17 +697,16 @@ retry: goto retry; } client_id = client_id64 & 0xffffffffll; - client = client_by_id(ci, client_id); + client = ref_client_by_id(cdata, client_id); if (unlikely(!client)) { LOGINFO("Connector failed to find client id %ld to drop", client_id); goto retry; } - ret = drop_client(ci, client); + ret = drop_client(cdata, client); + dec_instance_ref(cdata, client); if (ret >= 0) LOGINFO("Connector dropped client id: %ld", client_id); - goto retry; - } - if (cmdmatch(buf, "passthrough")) { + } else if (cmdmatch(buf, "passthrough")) { client_instance_t *client; ret = sscanf(buf, "passthrough=%ld", &client_id); @@ -621,44 +714,21 @@ retry: LOGDEBUG("Connector failed to parse passthrough command: %s", buf); goto retry; } - client = client_by_id(ci, client_id); + client = ref_client_by_id(cdata, client_id); if (unlikely(!client)) { LOGINFO("Connector failed to find client id %ld to pass through", client_id); goto retry; } - passthrough_client(ci, client); - goto retry; - } - if (cmdmatch(buf, "getfd")) { - send_fd(ci->serverfd, sockd); - goto retry; - } - - /* Anything else should be a json message to send to a client */ - json_msg = json_loads(buf, 0, NULL); - if (unlikely(!json_msg)) { - LOGWARNING("Invalid json message: %s", buf); - goto retry; - } - - /* Extract the client id from the json message and remove its entry */ - client_id64 = json_integer_value(json_object_get(json_msg, "client_id")); - json_object_del(json_msg, "client_id"); - if (client_id64 > 0xffffffffll) { - int64_t passthrough_id; - - passthrough_id = client_id64 & 0xffffffffll; - client_id = client_id64 >> 32; - json_object_set_new_nocheck(json_msg, "client_id", json_integer(passthrough_id)); + passthrough_client(cdata, client); + dec_instance_ref(cdata, client); + } else if (cmdmatch(buf, "getxfd")) { + int fdno = -1; + + sscanf(buf, "getxfd%d", &fdno); + if (fdno > -1 && fdno < ckp->serverurls) + send_fd(cdata->serverfd[fdno], sockd); } else - client_id = client_id64; - dealloc(buf); - buf = json_dumps(json_msg, 0); - realloc_strcat(&buf, "\n"); - send_client(ci, client_id, buf); - json_decref(json_msg); - buf = NULL; - + LOGWARNING("Unhandled connector message: %s", buf); goto retry; out: Close(sockd); @@ -668,39 +738,24 @@ out: int connector(proc_instance_t *pi) { - char *url = NULL, *port = NULL; + cdata_t *cdata = ckzalloc(sizeof(cdata_t)); ckpool_t *ckp = pi->ckp; - int sockd, ret = 0; - conn_instance_t ci; + int sockd, ret = 0, i; const int on = 1; int tries = 0; LOGWARNING("%s connector starting", ckp->name); + ckp->data = cdata; + cdata->ckp = ckp; - if (ckp->oldconnfd > 0) { - sockd = ckp->oldconnfd; - } else if (ckp->serverurl) { - if (!extract_sockaddr(ckp->serverurl, &url, &port)) { - LOGWARNING("Failed to extract server address from %s", ckp->serverurl); - ret = 1; - goto out; - } - do { - sockd = bind_socket(url, port); - if (sockd > 0) - break; - LOGWARNING("Connector failed to bind to socket, retrying in 5s"); - sleep(5); - } while (++tries < 25); + if (!ckp->serverurls) + cdata->serverfd = ckalloc(sizeof(int *)); + else + cdata->serverfd = ckalloc(sizeof(int *) * ckp->serverurls); - dealloc(url); - dealloc(port); - if (sockd < 0) { - LOGERR("Connector failed to bind to socket for 2 minutes"); - ret = 1; - goto out; - } - } else { + if (!ckp->serverurls) { + /* No serverurls have been specified. Bind to all interfaces + * on default sockets. */ struct sockaddr_in serv_addr; sockd = socket(AF_INET, SOCK_STREAM, 0); @@ -726,28 +781,70 @@ int connector(proc_instance_t *pi) Close(sockd); goto out; } + if (listen(sockd, SOMAXCONN) < 0) { + LOGERR("Connector failed to listen on socket"); + Close(sockd); + goto out; + } + cdata->serverfd[0] = sockd; + } else { + for (i = 0; i < ckp->serverurls; i++) { + char oldurl[INET6_ADDRSTRLEN], oldport[8]; + char newurl[INET6_ADDRSTRLEN], newport[8]; + char *serverurl = ckp->serverurl[i]; + + if (!url_from_serverurl(serverurl, newurl, newport)) { + LOGWARNING("Failed to extract resolved url from %s", serverurl); + ret = 1; + goto out; + } + sockd = ckp->oldconnfd[i]; + if (url_from_socket(sockd, oldurl, oldport)) { + if (strcmp(newurl, oldurl) || strcmp(newport, oldport)) { + LOGWARNING("Handed over socket url %s:%s does not match config %s:%s, creating new socket", + oldurl, oldport, newurl, newport); + Close(sockd); + } + } + + do { + if (sockd > 0) + break; + sockd = bind_socket(newurl, newport); + if (sockd > 0) + break; + LOGWARNING("Connector failed to bind to socket, retrying in 5s"); + sleep(5); + } while (++tries < 25); + + if (sockd < 0) { + LOGERR("Connector failed to bind to socket for 2 minutes"); + ret = 1; + goto out; + } + if (listen(sockd, SOMAXCONN) < 0) { + LOGERR("Connector failed to listen on socket"); + Close(sockd); + goto out; + } + cdata->serverfd[i] = sockd; + } } + if (tries) LOGWARNING("Connector successfully bound to socket"); - ret = listen(sockd, SOMAXCONN); - if (ret < 0) { - LOGERR("Connector failed to listen on socket"); - Close(sockd); - goto out; - } + cklock_init(&cdata->lock); + cdata->pi = pi; + cdata->nfds = 0; + cdata->client_id = 1; + mutex_init(&cdata->sender_lock); + cond_init(&cdata->sender_cond); + create_pthread(&cdata->pth_sender, sender, cdata); + create_pthread(&cdata->pth_receiver, receiver, cdata); - cklock_init(&ci.lock); - memset(&ci, 0, sizeof(ci)); - ci.pi = pi; - ci.serverfd = sockd; - ci.nfds = 0; - mutex_init(&sender_lock); - cond_init(&sender_cond); - create_pthread(&ci.pth_sender, sender, &ci); - create_pthread(&ci.pth_receiver, receiver, &ci); - - ret = connector_loop(pi, &ci); + ret = connector_loop(pi, cdata); out: + dealloc(ckp->data); return process_exit(ckp, pi, ret); } diff --git a/src/generator.c b/src/generator.c index 07fc5d40..c1804029 100644 --- a/src/generator.c +++ b/src/generator.c @@ -122,9 +122,13 @@ struct proxy_instance { typedef struct proxy_instance proxy_instance_t; -static int proxy_notify_id; // Globally increasing notify id +/* Private data for the generator */ +struct generator_data { + int proxy_notify_id; // Globally increasing notify id + ckmsgq_t *srvchk; // Server check message queue +}; -static ckmsgq_t *srvchk; // Server check message queue +typedef struct generator_data gdata_t; static bool server_alive(ckpool_t *ckp, server_instance_t *si, bool pinging) { @@ -239,6 +243,7 @@ static int gen_loop(proc_instance_t *pi) bool reconnecting = false; unixsock_t *us = &pi->us; ckpool_t *ckp = pi->ckp; + gdata_t *gdata = ckp->data; bool started = false; char *buf = NULL; connsock_t *cs; @@ -264,7 +269,7 @@ reconnect: retry: Close(sockd); - ckmsgq_add(srvchk, si); + ckmsgq_add(gdata->srvchk, si); do { selret = wait_read_select(us->sockd, 5); @@ -682,6 +687,7 @@ static bool parse_notify(proxy_instance_t *proxi, json_t *val) { const char *prev_hash, *bbversion, *nbit, *ntime; char *job_id, *coinbase1, *coinbase2; + gdata_t *gdata = proxi->ckp->data; bool clean, ret = false; notify_instance_t *ni; int merkles, i; @@ -741,7 +747,7 @@ static bool parse_notify(proxy_instance_t *proxi, json_t *val) ni->notify_time = time(NULL); mutex_lock(&proxi->notify_lock); - ni->id = proxy_notify_id++; + ni->id = gdata->proxy_notify_id++; HASH_ADD_INT(proxi->notify_instances, id, ni); proxi->current_notify = ni; mutex_unlock(&proxi->notify_lock); @@ -787,6 +793,19 @@ static bool show_message(json_t *val) return true; } +static bool send_pong(proxy_instance_t *proxi, json_t *val) +{ + json_t *json_msg, *id_val = json_object_dup(val, "id"); + connsock_t *cs = proxi->cs; + bool ret; + + JSON_CPACK(json_msg, "{sossso}", "id", id_val, "result", "pong", + "error", json_null()); + ret = send_json_msg(cs, json_msg); + json_decref(json_msg); + return ret; +} + static bool parse_reconnect(proxy_instance_t *proxi, json_t *val) { server_instance_t *newsi, *si = proxi->si; @@ -919,6 +938,11 @@ static bool parse_method(proxy_instance_t *proxi, const char *msg) ret = show_message(params); goto out; } + + if (cmdmatch(buf, "mining.ping")) { + ret = send_pong(proxi, val); + goto out; + } out: if (val) json_decref(val); @@ -1450,6 +1474,7 @@ static int proxy_loop(proc_instance_t *pi) bool reconnecting = false; unixsock_t *us = &pi->us; ckpool_t *ckp = pi->ckp; + gdata_t *gdata = ckp->data; char *buf = NULL; reconnect: @@ -1476,7 +1501,7 @@ reconnect: } retry: - ckmsgq_add(srvchk, proxi->si); + ckmsgq_add(gdata->srvchk, proxi->si); do { selret = wait_read_select(us->sockd, 5); @@ -1727,16 +1752,19 @@ static void server_watchdog(ckpool_t *ckp, server_instance_t *cursi) int generator(proc_instance_t *pi) { ckpool_t *ckp = pi->ckp; + gdata_t *gdata; int ret; LOGWARNING("%s generator starting", ckp->name); - - srvchk = create_ckmsgq(ckp, "srvchk", &server_watchdog); + gdata = ckzalloc(sizeof(gdata_t)); + ckp->data = gdata; + gdata->srvchk = create_ckmsgq(ckp, "srvchk", &server_watchdog); if (ckp->proxy) ret = proxy_mode(ckp, pi); else ret = server_mode(ckp, pi); + dealloc(ckp->data); return process_exit(ckp, pi, ret); } diff --git a/src/libckpool.c b/src/libckpool.c index 860b537e..5d6cbb98 100644 --- a/src/libckpool.c +++ b/src/libckpool.c @@ -30,6 +30,7 @@ #include #include #include +#include #include "libckpool.h" #include "sha2.h" @@ -77,6 +78,43 @@ void join_pthread(pthread_t thread) pthread_join(thread, NULL); } +struct ck_completion { + sem_t sem; + void (*fn)(void *fnarg); + void *fnarg; +}; + +static void *completion_thread(void *arg) +{ + struct ck_completion *ckc = (struct ck_completion *)arg; + + ckc->fn(ckc->fnarg); + cksem_post(&ckc->sem); + + return NULL; +} + +bool ck_completion_timeout(void *fn, void *fnarg, int timeout) +{ + struct ck_completion ckc; + pthread_t pthread; + bool ret = false; + + cksem_init(&ckc.sem); + ckc.fn = fn; + ckc.fnarg = fnarg; + + pthread_create(&pthread, NULL, completion_thread, (void *)&ckc); + + ret = cksem_mswait(&ckc.sem, timeout); + if (!ret) + pthread_join(pthread, NULL); + else + pthread_cancel(pthread); + return !ret; +} + + /* Place holders for when we add lock debugging */ #define GETLOCK(_lock, _file, _func, _line) #define GOTLOCK(_lock, _file, _func, _line) @@ -294,10 +332,9 @@ void _cksem_post(sem_t *sem, const char *file, const char *func, const int line) void _cksem_wait(sem_t *sem, const char *file, const char *func, const int line) { -retry: if (unlikely(sem_wait(sem))) { if (errno == EINTR) - goto retry; + return; quitfrom(1, file, func, line, "Failed to sem_wait errno=%d sem=0x%p", errno, sem); } } @@ -311,7 +348,6 @@ int _cksem_mswait(sem_t *sem, int ms, const char *file, const char *func, const tv_time(&tv_now); tv_to_ts(&ts_now, &tv_now); ms_to_ts(&abs_timeout, ms); -retry: timeraddspec(&abs_timeout, &ts_now); ret = sem_timedwait(sem, &abs_timeout); @@ -319,28 +355,21 @@ retry: if (likely(errno == ETIMEDOUT)) return ETIMEDOUT; if (errno == EINTR) - goto retry; + return EINTR; quitfrom(1, file, func, line, "Failed to sem_timedwait errno=%d sem=0x%p", errno, sem); } return 0; } -void cksem_reset(sem_t *sem) +void _cksem_destroy(sem_t *sem, const char *file, const char *func, const int line) { - int ret; - do { - ret = sem_trywait(sem); - if (unlikely(ret < 0 && (errno == EINTR))) - ret = 0; - } while (!ret); -} - -void cksem_destroy(sem_t *sem) -{ - sem_destroy(sem); + if (unlikely(sem_destroy(sem))) + quitfrom(1, file, func, line, "Failed to sem_destroy errno=%d sem=0x%p", errno, sem); } +/* Extract just the url and port information from a url string, allocating + * heap memory for sockaddr_url and sockaddr_port. */ bool extract_sockaddr(char *url, char **sockaddr_url, char **sockaddr_port) { char *url_begin, *url_end, *ipv6_begin, *ipv6_end, *port_start = NULL; @@ -406,6 +435,95 @@ bool extract_sockaddr(char *url, char **sockaddr_url, char **sockaddr_port) return true; } +/* Convert a sockaddr structure into a url and port. URL should be a string of + * INET6_ADDRSTRLEN size, port at least a string of 6 bytes */ +bool url_from_sockaddr(const struct sockaddr *addr, char *url, char *port) +{ + int port_no = 0; + + switch(addr->sa_family) { + const struct sockaddr_in *inet4_in; + const struct sockaddr_in6 *inet6_in; + + case AF_INET: + inet4_in = (struct sockaddr_in *)addr; + inet_ntop(AF_INET, &inet4_in->sin_addr, url, INET6_ADDRSTRLEN); + port_no = htons(inet4_in->sin_port); + break; + case AF_INET6: + inet6_in = (struct sockaddr_in6 *)addr; + inet_ntop(AF_INET6, &inet6_in->sin6_addr, url, INET6_ADDRSTRLEN); + port_no = htons(inet6_in->sin6_port); + break; + default: + return false; + } + sprintf(port, "%d", port_no); + return true; +} + +bool addrinfo_from_url(const char *url, const char *port, struct addrinfo *addrinfo) +{ + struct addrinfo *servinfo, hints; + + memset(&hints, 0, sizeof(struct addrinfo)); + hints.ai_family = AF_UNSPEC; + hints.ai_socktype = SOCK_STREAM; + servinfo = addrinfo; + if (getaddrinfo(url, port, &hints, &servinfo) != 0) + return false; + if (!servinfo) + return false; + memcpy(addrinfo, servinfo->ai_addr, servinfo->ai_addrlen); + freeaddrinfo(servinfo); + return true; +} + +/* Extract a resolved url and port from a serverurl string. newurl must be + * a string of at least INET6_ADDRSTRLEN and newport at least 6 bytes. */ +bool url_from_serverurl(char *serverurl, char *newurl, char *newport) +{ + char *url = NULL, *port = NULL; + struct addrinfo addrinfo; + bool ret = false; + + if (!extract_sockaddr(serverurl, &url, &port)) { + LOGWARNING("Failed to extract server address from %s", serverurl); + goto out; + } + if (!addrinfo_from_url(url, port, &addrinfo)) { + LOGWARNING("Failed to extract addrinfo from url %s:%s", url, port); + goto out; + } + if (!url_from_sockaddr((const struct sockaddr *)&addrinfo, newurl, newport)) { + LOGWARNING("Failed to extract url from sockaddr for original url: %s:%s", + url, port); + goto out; + } + ret = true; +out: + dealloc(url); + dealloc(port); + return ret; +} + +/* Convert a socket into a url and port. URL should be a string of + * INET6_ADDRSTRLEN size, port at least a string of 6 bytes */ +bool url_from_socket(const int sockd, char *url, char *port) +{ + socklen_t addrlen = sizeof(struct sockaddr); + struct sockaddr addr; + + if (sockd < 1) + return false; + if (getsockname(sockd, &addr, &addrlen)) + return false; + if (!url_from_sockaddr(&addr, url, port)) + return false; + return true; +} + + void keep_sockalive(int fd) { const int tcp_one = 1; @@ -464,7 +582,7 @@ int bind_socket(char *url, char *port) if (getaddrinfo(url, port, &hints, &servinfo) != 0) { LOGWARNING("Failed to resolve (?wrong URL) %s:%s", url, port); - goto out; + return sockd; } for (p = servinfo; p != NULL; p = p->ai_next) { sockd = socket(p->ai_family, p->ai_socktype, p->ai_protocol); @@ -484,6 +602,7 @@ int bind_socket(char *url, char *port) } out: + freeaddrinfo(servinfo); return sockd; } @@ -1494,11 +1613,7 @@ void cksleep_prepare_r(ts_t *ts) void nanosleep_abstime(ts_t *ts_end) { - int ret; - - do { - ret = clock_nanosleep(CLOCK_MONOTONIC, TIMER_ABSTIME, ts_end, NULL); - } while (ret == EINTR); + clock_nanosleep(CLOCK_MONOTONIC, TIMER_ABSTIME, ts_end, NULL); } void timeraddspec(ts_t *a, const ts_t *b) diff --git a/src/libckpool.h b/src/libckpool.h index 154cec75..ac865046 100644 --- a/src/libckpool.h +++ b/src/libckpool.h @@ -14,6 +14,7 @@ #include #include +#include #include #include #include @@ -372,6 +373,7 @@ static inline void _json_set_bool(json_t *val, const char *key, bool boolean, void rename_proc(const char *name); void create_pthread(pthread_t *thread, void *(*start_routine)(void *), void *arg); void join_pthread(pthread_t thread); +bool ck_completion_timeout(void *fn, void *fnarg, int timeout); void _mutex_lock(pthread_mutex_t *lock, const char *file, const char *func, const int line); void _mutex_unlock_noyield(pthread_mutex_t *lock, const char *file, const char *func, const int line); @@ -409,13 +411,13 @@ void _cksem_init(sem_t *sem, const char *file, const char *func, const int line) void _cksem_post(sem_t *sem, const char *file, const char *func, const int line); void _cksem_wait(sem_t *sem, const char *file, const char *func, const int line); int _cksem_mswait(sem_t *sem, int ms, const char *file, const char *func, const int line); -void cksem_reset(sem_t *sem); -void cksem_destroy(sem_t *sem); +void _cksem_destroy(sem_t *sem, const char *file, const char *func, const int line); -#define cksem_init(_sem) _cksem_init(_sem, __FILE__, __func__, __LINE__) -#define cksem_post(_sem) _cksem_post(_sem, __FILE__, __func__, __LINE__) -#define cksem_wait(_sem) _cksem_wait(_sem, __FILE__, __func__, __LINE__) -#define cksem_mswait(_sem, _timeout) _cksem_mswait(_sem, _timeout, __FILE__, __func__, __LINE__) +#define cksem_init(SEM) _cksem_init(SEM, __FILE__, __func__, __LINE__) +#define cksem_post(SEM) _cksem_post(SEM, __FILE__, __func__, __LINE__) +#define cksem_wait(SEM) _cksem_wait(SEM, __FILE__, __func__, __LINE__) +#define cksem_mswait(SEM, _timeout) _cksem_mswait(SEM, _timeout, __FILE__, __func__, __LINE__) +#define cksem_destroy(SEM) _cksem_destroy(SEM, __FILE__, __func__, __LINE__) static inline bool sock_connecting(void) { @@ -433,6 +435,11 @@ static inline bool sock_timeout(void) } bool extract_sockaddr(char *url, char **sockaddr_url, char **sockaddr_port); +bool url_from_sockaddr(const struct sockaddr *addr, char *url, char *port); +bool addrinfo_from_url(const char *url, const char *port, struct addrinfo *addrinfo); +bool url_from_serverurl(char *serverurl, char *newurl, char *newport); +bool url_from_socket(const int sockd, char *url, char *port); + void keep_sockalive(int fd); void nolinger_socket(int fd); void noblock_socket(int fd); diff --git a/src/stratifier.c b/src/stratifier.c index a47e801d..4d34a3a1 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -28,14 +28,11 @@ #include "uthash.h" #include "utlist.h" +/* Consistent across all pool instances */ static const char *workpadding = "000000800000000000000000000000000000000000000000000000000000000000000000000000000000000080020000"; - static const char *scriptsig_header = "01000000010000000000000000000000000000000000000000000000000000000000000000ffffffff"; static uchar scriptsig_header_bin[41]; -static char pubkeytxnbin[25]; -static char donkeytxnbin[25]; - /* Add unaccounted shares when they arrive, remove them with each update of * rolling stats. */ struct pool_stats { @@ -76,21 +73,6 @@ struct pool_stats { typedef struct pool_stats pool_stats_t; -static pool_stats_t stats; - -/* Protects changes to pool stats */ -static pthread_mutex_t stats_lock; - -/* Serialises sends/receives to ckdb if possible */ -static pthread_mutex_t ckdb_lock; - -static union { - uint64_t u64; - uint32_t u32; - uint16_t u16; - uint8_t u8; -} enonce1u; - struct workbase { /* Hash table data */ UT_hash_handle hh; @@ -148,31 +130,6 @@ struct workbase { typedef struct workbase workbase_t; -/* For protecting the hashtable data */ -static cklock_t workbase_lock; - -/* For the hashtable of all workbases */ -static workbase_t *workbases; -static workbase_t *current_workbase; - -static struct { - double diff; - - char enonce1[32]; - uchar enonce1bin[16]; - int enonce1constlen; - int enonce1varlen; - - int nonce2len; - int enonce2varlen; - - bool subscribed; -} proxy_base; - -static int64_t workbase_id; -static int64_t blockchange_id; -static char lasthash[68], lastswaphash[68]; - struct json_params { json_t *method; json_t *params; @@ -192,15 +149,6 @@ struct smsg { typedef struct smsg smsg_t; -static ckmsgq_t *ssends; // Stratum sends -static ckmsgq_t *srecvs; // Stratum receives -static ckmsgq_t *ckdbq; // ckdb -static ckmsgq_t *sshareq; // Stratum share sends -static ckmsgq_t *sauthq; // Stratum authorisations -static ckmsgq_t *stxnq; // Transaction requests - -static int64_t user_instance_id; - struct user_instance; struct worker_instance; struct stratum_instance; @@ -232,8 +180,6 @@ struct user_instance { tv_t last_share; }; -static user_instance_t *user_instances; - /* Combined data from workers with the same workername */ struct worker_instance { user_instance_t *instance; @@ -247,8 +193,12 @@ struct worker_instance { double dsps60; double dsps1440; tv_t last_share; + time_t start_time; int mindiff; /* User chosen mindiff */ + + bool idle; + bool notified_idle; }; /* Per client stratum instance == workers */ @@ -259,6 +209,10 @@ struct stratum_instance { stratum_instance_t *next; stratum_instance_t *prev; + /* Reference count for when this instance is used outside of the + * instance_lock */ + int ref; + char enonce1[32]; uchar enonce1bin[16]; char enonce1var[12]; @@ -283,7 +237,6 @@ struct stratum_instance { bool subscribed; bool authorised; bool idle; - bool notified_idle; int reject; /* Indicator that this client is having a run of rejects * or other problem and should be dropped lazily if * this is set to 2 */ @@ -294,6 +247,7 @@ struct stratum_instance { char *useragent; char *workername; int64_t user_id; + int server; /* Which server is this instance bound to */ ckpool_t *ckp; @@ -302,14 +256,6 @@ struct stratum_instance { int64_t suggest_diff; /* Stratum client suggested diff */ }; -/* Stratum_instances hashlist is stored by id, whereas disconnected_instances - * is sorted by enonce1_64. */ -static stratum_instance_t *stratum_instances; -static stratum_instance_t *disconnected_instances; - -/* Protects both stratum and user instances */ -static cklock_t instance_lock; - struct share { UT_hash_handle hh; uchar hash[32]; @@ -318,17 +264,85 @@ struct share { typedef struct share share_t; -static share_t *shares; +struct stratifier_data { + char pubkeytxnbin[25]; + char donkeytxnbin[25]; + + pool_stats_t stats; + /* Protects changes to pool stats */ + pthread_mutex_t stats_lock; + + /* Serialises sends/receives to ckdb if possible */ + pthread_mutex_t ckdb_lock; + + /* Variable length enonce1 always refers back to a u64 */ + union { + uint64_t u64; + uint32_t u32; + uint16_t u16; + uint8_t u8; + } enonce1u; + + /* For protecting the hashtable data */ + cklock_t workbase_lock; + + /* For the hashtable of all workbases */ + workbase_t *workbases; + workbase_t *current_workbase; + + int64_t workbase_id; + int64_t blockchange_id; + char lasthash[68]; + char lastswaphash[68]; + + ckmsgq_t *ssends; // Stratum sends + ckmsgq_t *srecvs; // Stratum receives + ckmsgq_t *ckdbq; // ckdb + ckmsgq_t *sshareq; // Stratum share sends + ckmsgq_t *sauthq; // Stratum authorisations + ckmsgq_t *stxnq; // Transaction requests + + int64_t user_instance_id; + + /* Stratum_instances hashlist is stored by id, whereas disconnected_instances + * is sorted by enonce1_64. */ + stratum_instance_t *stratum_instances; + stratum_instance_t *disconnected_instances; + stratum_instance_t *dead_instances; + + user_instance_t *user_instances; + + /* Protects both stratum and user instances */ + cklock_t instance_lock; + + share_t *shares; + cklock_t share_lock; + + /* Linked list of block solves, added to during submission, removed on + * accept/reject. It is likely we only ever have one solve on here but + * you never know... */ + pthread_mutex_t block_lock; + ckmsg_t *block_solves; + + /* Generator message priority */ + int gen_priority; -static cklock_t share_lock; + struct { + double diff; -/* Linked list of block solves, added to during submission, removed on - * accept/reject. It is likely we only ever have one solve on here but you - * never know... */ -static pthread_mutex_t block_lock; -static ckmsg_t *block_solves; + char enonce1[32]; + uchar enonce1bin[16]; + int enonce1constlen; + int enonce1varlen; -static int gen_priority; + int nonce2len; + int enonce2varlen; + + bool subscribed; + } proxy_base; +}; + +typedef struct stratifier_data sdata_t; /* Priority levels for generator messages */ #define GEN_LAX 0 @@ -341,7 +355,7 @@ static int gen_priority; #define ID_SHARES 3 #define ID_SHAREERR 4 #define ID_POOLSTATS 5 -#define ID_USERSTATS 6 +#define ID_WORKERSTATS 6 #define ID_BLOCK 7 #define ID_ADDRAUTH 8 #define ID_HEARTBEAT 9 @@ -353,7 +367,7 @@ static const char *ckdb_ids[] = { "shares", "shareerror", "poolstats", - "userstats", + "workerstats", "block", "addrauth", "heartbeat" @@ -362,6 +376,7 @@ static const char *ckdb_ids[] = { static void generate_coinbase(ckpool_t *ckp, workbase_t *wb) { uint64_t *u64, g64, d64 = 0; + sdata_t *sdata = ckp->data; char header[228]; int len, ofs = 0; ts_t now; @@ -396,9 +411,8 @@ static void generate_coinbase(ckpool_t *ckp, workbase_t *wb) len = ser_number(wb->coinb1bin + ofs, now.tv_nsec); ofs += len; - /* Leave enonce1/2varlen constant at 8 bytes for bitcoind sources */ - wb->enonce1varlen = 8; - wb->enonce2varlen = 8; + wb->enonce1varlen = ckp->nonce1length; + wb->enonce2varlen = ckp->nonce2length; wb->coinb1bin[ofs++] = wb->enonce1varlen + wb->enonce2varlen; wb->coinb1len = ofs; @@ -445,7 +459,7 @@ static void generate_coinbase(ckpool_t *ckp, workbase_t *wb) wb->coinb2len += 8; wb->coinb2bin[wb->coinb2len++] = 25; - memcpy(wb->coinb2bin + wb->coinb2len, pubkeytxnbin, 25); + memcpy(wb->coinb2bin + wb->coinb2len, sdata->pubkeytxnbin, 25); wb->coinb2len += 25; if (ckp->donvalid) { @@ -454,7 +468,7 @@ static void generate_coinbase(ckpool_t *ckp, workbase_t *wb) wb->coinb2len += 8; wb->coinb2bin[wb->coinb2len++] = 25; - memcpy(wb->coinb2bin + wb->coinb2len, donkeytxnbin, 25); + memcpy(wb->coinb2bin + wb->coinb2len, sdata->donkeytxnbin, 25); wb->coinb2len += 25; } @@ -474,7 +488,7 @@ static void generate_coinbase(ckpool_t *ckp, workbase_t *wb) hex2bin(wb->headerbin, header, 112); } -static void stratum_broadcast_update(bool clean); +static void stratum_broadcast_update(sdata_t *sdata, bool clean); static void clear_workbase(workbase_t *wb) { @@ -490,20 +504,20 @@ static void clear_workbase(workbase_t *wb) free(wb); } -static void purge_share_hashtable(int64_t wb_id) +static void purge_share_hashtable(sdata_t *sdata, int64_t wb_id) { share_t *share, *tmp; int purged = 0; - ck_wlock(&share_lock); - HASH_ITER(hh, shares, share, tmp) { + ck_wlock(&sdata->share_lock); + HASH_ITER(hh, sdata->shares, share, tmp) { if (share->workbase_id < wb_id) { - HASH_DEL(shares, share); + HASH_DEL(sdata->shares, share); free(share); purged++; } } - ck_wunlock(&share_lock); + ck_wunlock(&sdata->share_lock); if (purged) LOGINFO("Cleared %d shares from share hashtable", purged); @@ -534,6 +548,7 @@ static void _ckdbq_add(ckpool_t *ckp, const int idtype, json_t *val, const char const char *func, const int line) { static time_t time_counter; + sdata_t *sdata = ckp->data; static int counter = 0; char *json_msg; time_t now_t; @@ -562,7 +577,7 @@ static void _ckdbq_add(ckpool_t *ckp, const int idtype, json_t *val, const char return; } - ckmsgq_add(ckdbq, json_msg); + ckmsgq_add(sdata->ckdbq, json_msg); } #define ckdbq_add(ckp, idtype, val) _ckdbq_add(ckp, idtype, val, __FILE__, __func__, __LINE__) @@ -589,7 +604,7 @@ static void send_workinfo(ckpool_t *ckp, workbase_t *wb) "createdate", cdfield, "createby", "code", "createcode", __func__, - "createinet", ckp->serverurl); + "createinet", ckp->serverurl[0]); ckdbq_add(ckp, ID_WORKINFO, val); } @@ -608,13 +623,14 @@ static void send_ageworkinfo(ckpool_t *ckp, int64_t id) "createdate", cdfield, "createby", "code", "createcode", __func__, - "createinet", ckp->serverurl); + "createinet", ckp->serverurl[0]); ckdbq_add(ckp, ID_AGEWORKINFO, val); } static void add_base(ckpool_t *ckp, workbase_t *wb, bool *new_block) { workbase_t *tmp, *tmpa, *aged = NULL; + sdata_t *sdata = ckp->data; int len, ret; ts_realtime(&wb->gentime); @@ -626,21 +642,21 @@ static void add_base(ckpool_t *ckp, workbase_t *wb, bool *new_block) /* In proxy mode, the wb->id is received in the notify update and * we set workbase_id from it. In server mode the stratifier is * setting the workbase_id */ - ck_wlock(&workbase_lock); + ck_wlock(&sdata->workbase_lock); if (!ckp->proxy) - wb->id = workbase_id++; + wb->id = sdata->workbase_id++; else - workbase_id = wb->id; - if (strncmp(wb->prevhash, lasthash, 64)) { + sdata->workbase_id = wb->id; + if (strncmp(wb->prevhash, sdata->lasthash, 64)) { char bin[32], swap[32]; *new_block = true; - memcpy(lasthash, wb->prevhash, 65); - hex2bin(bin, lasthash, 32); + memcpy(sdata->lasthash, wb->prevhash, 65); + hex2bin(bin, sdata->lasthash, 32); swap_256(swap, bin); - __bin2hex(lastswaphash, swap, 32); - LOGNOTICE("Block hash changed to %s", lastswaphash); - blockchange_id = wb->id; + __bin2hex(sdata->lastswaphash, swap, 32); + LOGNOTICE("Block hash changed to %s", sdata->lastswaphash); + sdata->blockchange_id = wb->id; } if (*new_block && ckp->logshares) { sprintf(wb->logdir, "%s%08x/", ckp->logdir, wb->height); @@ -652,22 +668,22 @@ static void add_base(ckpool_t *ckp, workbase_t *wb, bool *new_block) if (ckp->logshares) sprintf(wb->logdir, "%s%08x/%s", ckp->logdir, wb->height, wb->idstring); - HASH_ITER(hh, workbases, tmp, tmpa) { - if (HASH_COUNT(workbases) < 3) + HASH_ITER(hh, sdata->workbases, tmp, tmpa) { + if (HASH_COUNT(sdata->workbases) < 3) break; /* Age old workbases older than 10 minutes old */ if (tmp->gentime.tv_sec < wb->gentime.tv_sec - 600) { - HASH_DEL(workbases, tmp); + HASH_DEL(sdata->workbases, tmp); aged = tmp; break; } } - HASH_ADD_I64(workbases, id, wb); - current_workbase = wb; - ck_wunlock(&workbase_lock); + HASH_ADD_I64(sdata->workbases, id, wb); + sdata->current_workbase = wb; + ck_wunlock(&sdata->workbase_lock); if (*new_block) - purge_share_hashtable(wb->id); + purge_share_hashtable(sdata, wb->id); send_workinfo(ckp, wb); @@ -684,44 +700,50 @@ static void add_base(ckpool_t *ckp, workbase_t *wb, bool *new_block) * read the wrong priority but occasional wrong values are harmless. */ static char *__send_recv_generator(ckpool_t *ckp, const char *msg, int prio) { + sdata_t *sdata = ckp->data; char *buf = NULL; bool set; - if (prio > gen_priority) { - gen_priority = prio; + if (prio > sdata->gen_priority) { + sdata->gen_priority = prio; set = true; } else set = false; buf = send_recv_proc(ckp->generator, msg); + if (unlikely(!buf)) + buf = strdup("failed"); if (set) - gen_priority = 0; + sdata->gen_priority = 0; return buf; } /* Conditionally send_recv a message only if it's equal or higher priority than - * any currently being serviced. */ + * any currently being serviced. NULL is returned if the request is not + * processed for priority reasons, "failed" for an actual failure. */ static char *send_recv_generator(ckpool_t *ckp, const char *msg, int prio) { + sdata_t *sdata = ckp->data; char *buf = NULL; - if (prio >= gen_priority) + if (prio >= sdata->gen_priority) buf = __send_recv_generator(ckp, msg, prio); return buf; } static void send_generator(ckpool_t *ckp, const char *msg, int prio) { + sdata_t *sdata = ckp->data; bool set; - if (prio > gen_priority) { - gen_priority = prio; + if (prio > sdata->gen_priority) { + sdata->gen_priority = prio; set = true; } else set = false; send_proc(ckp->generator, msg); if (set) - gen_priority = 0; + sdata->gen_priority = 0; } struct update_req { @@ -730,7 +752,7 @@ struct update_req { int prio; }; -static void broadcast_ping(void); +static void broadcast_ping(sdata_t *sdata); /* This function assumes it will only receive a valid json gbt base template * since checking should have been done earlier, and creates the base template @@ -739,6 +761,7 @@ static void *do_update(void *arg) { struct update_req *ur = (struct update_req *)arg; ckpool_t *ckp = ur->ckp; + sdata_t *sdata = ckp->data; bool new_block = false; int prio = ur->prio; bool ret = false; @@ -751,7 +774,7 @@ static void *do_update(void *arg) buf = send_recv_generator(ckp, "getbase", prio); if (unlikely(!buf)) { - LOGWARNING("Failed to get base from generator in update_base"); + LOGNOTICE("Get base in update_base delayed due to higher priority request"); goto out; } if (unlikely(cmdmatch(buf, "failed"))) { @@ -762,7 +785,6 @@ static void *do_update(void *arg) wb = ckzalloc(sizeof(workbase_t)); wb->ckp = ckp; val = json_loads(buf, 0, NULL); - dealloc(buf); json_strcpy(wb->target, val, "target"); json_dblcpy(&wb->diff, val, "diff"); @@ -800,16 +822,17 @@ static void *do_update(void *arg) add_base(ckp, wb, &new_block); - stratum_broadcast_update(new_block); + stratum_broadcast_update(sdata, new_block); ret = true; LOGINFO("Broadcast updated stratum base"); out: /* Send a ping to miners if we fail to get a base to keep them * connected while bitcoind recovers(?) */ - if (!ret) { - LOGWARNING("Broadcast ping due to failed stratum base update"); - broadcast_ping(); + if (unlikely(!ret)) { + LOGINFO("Broadcast ping due to failed stratum base update"); + broadcast_ping(sdata); } + dealloc(buf); free(ur->pth); free(ur); return NULL; @@ -829,22 +852,24 @@ static void update_base(ckpool_t *ckp, int prio) static void drop_allclients(ckpool_t *ckp) { stratum_instance_t *client, *tmp; + sdata_t *sdata = ckp->data; char buf[128]; - ck_wlock(&instance_lock); - HASH_ITER(hh, stratum_instances, client, tmp) { - HASH_DEL(stratum_instances, client); + ck_wlock(&sdata->instance_lock); + HASH_ITER(hh, sdata->stratum_instances, client, tmp) { + HASH_DEL(sdata->stratum_instances, client); sprintf(buf, "dropclient=%ld", client->id); send_proc(ckp->connector, buf); } - HASH_ITER(hh, disconnected_instances, client, tmp) - HASH_DEL(disconnected_instances, client); - stats.users = stats.workers = 0; - ck_wunlock(&instance_lock); + HASH_ITER(hh, sdata->disconnected_instances, client, tmp) + HASH_DEL(sdata->disconnected_instances, client); + sdata->stats.users = sdata->stats.workers = 0; + ck_wunlock(&sdata->instance_lock); } static void update_subscribe(ckpool_t *ckp) { + sdata_t *sdata = ckp->data; json_t *val; char *buf; @@ -858,22 +883,22 @@ static void update_subscribe(ckpool_t *ckp) val = json_loads(buf, 0, NULL); free(buf); - ck_wlock(&workbase_lock); - proxy_base.subscribed = true; - proxy_base.diff = ckp->startdiff; + ck_wlock(&sdata->workbase_lock); + sdata->proxy_base.subscribed = true; + sdata->proxy_base.diff = ckp->startdiff; /* Length is checked by generator */ - strcpy(proxy_base.enonce1, json_string_value(json_object_get(val, "enonce1"))); - proxy_base.enonce1constlen = strlen(proxy_base.enonce1) / 2; - hex2bin(proxy_base.enonce1bin, proxy_base.enonce1, proxy_base.enonce1constlen); - proxy_base.nonce2len = json_integer_value(json_object_get(val, "nonce2len")); - if (proxy_base.nonce2len > 7) - proxy_base.enonce1varlen = 4; - else if (proxy_base.nonce2len > 5) - proxy_base.enonce1varlen = 2; + strcpy(sdata->proxy_base.enonce1, json_string_value(json_object_get(val, "enonce1"))); + sdata->proxy_base.enonce1constlen = strlen(sdata->proxy_base.enonce1) / 2; + hex2bin(sdata->proxy_base.enonce1bin, sdata->proxy_base.enonce1, sdata->proxy_base.enonce1constlen); + sdata->proxy_base.nonce2len = json_integer_value(json_object_get(val, "nonce2len")); + if (sdata->proxy_base.nonce2len > 7) + sdata->proxy_base.enonce1varlen = 4; + else if (sdata->proxy_base.nonce2len > 5) + sdata->proxy_base.enonce1varlen = 2; else - proxy_base.enonce1varlen = 1; - proxy_base.enonce2varlen = proxy_base.nonce2len - proxy_base.enonce1varlen; - ck_wunlock(&workbase_lock); + sdata->proxy_base.enonce1varlen = 1; + sdata->proxy_base.enonce2varlen = sdata->proxy_base.nonce2len - sdata->proxy_base.enonce1varlen; + ck_wunlock(&sdata->workbase_lock); json_decref(val); drop_allclients(ckp); @@ -884,6 +909,7 @@ static void update_diff(ckpool_t *ckp); static void update_notify(ckpool_t *ckp) { bool new_block = false, clean; + sdata_t *sdata = ckp->data; char header[228]; workbase_t *wb; json_t *val; @@ -896,7 +922,7 @@ static void update_notify(ckpool_t *ckp) return; } - if (unlikely(!proxy_base.subscribed)) { + if (unlikely(!sdata->proxy_base.subscribed)) { LOGINFO("No valid proxy subscription to update notify yet"); return; } @@ -946,30 +972,31 @@ static void update_notify(ckpool_t *ckp) /* Check diff on each notify */ update_diff(ckp); - ck_rlock(&workbase_lock); - strcpy(wb->enonce1const, proxy_base.enonce1); - wb->enonce1constlen = proxy_base.enonce1constlen; - memcpy(wb->enonce1constbin, proxy_base.enonce1bin, wb->enonce1constlen); - wb->enonce1varlen = proxy_base.enonce1varlen; - wb->enonce2varlen = proxy_base.enonce2varlen; - wb->diff = proxy_base.diff; - ck_runlock(&workbase_lock); + ck_rlock(&sdata->workbase_lock); + strcpy(wb->enonce1const, sdata->proxy_base.enonce1); + wb->enonce1constlen = sdata->proxy_base.enonce1constlen; + memcpy(wb->enonce1constbin, sdata->proxy_base.enonce1bin, wb->enonce1constlen); + wb->enonce1varlen = sdata->proxy_base.enonce1varlen; + wb->enonce2varlen = sdata->proxy_base.enonce2varlen; + wb->diff = sdata->proxy_base.diff; + ck_runlock(&sdata->workbase_lock); add_base(ckp, wb, &new_block); - stratum_broadcast_update(new_block | clean); + stratum_broadcast_update(sdata, new_block | clean); } -static void stratum_send_diff(stratum_instance_t *client); +static void stratum_send_diff(sdata_t *sdata, stratum_instance_t *client); static void update_diff(ckpool_t *ckp) { stratum_instance_t *client; + sdata_t *sdata = ckp->data; double old_diff, diff; json_t *val; char *buf; - if (unlikely(!current_workbase)) { + if (unlikely(!sdata->current_workbase)) { LOGINFO("No current workbase to update diff yet"); return; } @@ -991,51 +1018,88 @@ static void update_diff(ckpool_t *ckp) if (unlikely(diff < 1)) diff = 1; - ck_wlock(&workbase_lock); - old_diff = proxy_base.diff; - current_workbase->diff = proxy_base.diff = diff; - ck_wunlock(&workbase_lock); + ck_wlock(&sdata->workbase_lock); + old_diff = sdata->proxy_base.diff; + sdata->current_workbase->diff = sdata->proxy_base.diff = diff; + ck_wunlock(&sdata->workbase_lock); if (old_diff < diff) return; /* If the diff has dropped, iterate over all the clients and check * they're at or below the new diff, and update it if not. */ - ck_rlock(&instance_lock); - for (client = stratum_instances; client != NULL; client = client->hh.next) { + ck_rlock(&sdata->instance_lock); + for (client = sdata->stratum_instances; client != NULL; client = client->hh.next) { if (client->diff > diff) { client->diff = diff; - stratum_send_diff(client); + stratum_send_diff(sdata, client); } } - ck_runlock(&instance_lock); + ck_runlock(&sdata->instance_lock); } /* Enter with instance_lock held */ -static stratum_instance_t *__instance_by_id(int64_t id) +static stratum_instance_t *__instance_by_id(sdata_t *sdata, int64_t id) +{ + stratum_instance_t *instance; + + HASH_FIND_I64(sdata->stratum_instances, &id, instance); + return instance; +} + +/* Increase the reference count of instance */ +static void __inc_instance_ref(stratum_instance_t *instance) +{ + instance->ref++; +} + +/* Find an __instance_by_id and increase its reference count allowing us to + * use this instance outside of instance_lock without fear of it being + * dereferenced. */ +static stratum_instance_t *ref_instance_by_id(sdata_t *sdata, int64_t id) { stratum_instance_t *instance; - HASH_FIND_I64(stratum_instances, &id, instance); + ck_wlock(&sdata->instance_lock); + instance = __instance_by_id(sdata, id); + if (instance) + __inc_instance_ref(instance); + ck_wunlock(&sdata->instance_lock); + return instance; } +/* Decrease the reference count of instance. */ +static void __dec_instance_ref(stratum_instance_t *instance) +{ + instance->ref--; +} + +static void dec_instance_ref(sdata_t *sdata, stratum_instance_t *instance) +{ + ck_wlock(&sdata->instance_lock); + __dec_instance_ref(instance); + ck_wunlock(&sdata->instance_lock); +} + /* Enter with write instance_lock held */ -static stratum_instance_t *__stratum_add_instance(ckpool_t *ckp, int64_t id) +static stratum_instance_t *__stratum_add_instance(ckpool_t *ckp, int64_t id, int server) { stratum_instance_t *instance = ckzalloc(sizeof(stratum_instance_t)); + sdata_t *sdata = ckp->data; instance->id = id; + instance->server = server; instance->diff = instance->old_diff = ckp->startdiff; instance->ckp = ckp; tv_time(&instance->ldc); - LOGINFO("Added instance %ld", id); - HASH_ADD_I64(stratum_instances, id, instance); + LOGINFO("Stratifier added instance %ld server %d", id, server); + HASH_ADD_I64(sdata->stratum_instances, id, instance); return instance; } /* Only supports a full ckpool instance sessionid with an 8 byte sessionid */ -static bool disconnected_sessionid_exists(const char *sessionid, int64_t id) +static bool disconnected_sessionid_exists(sdata_t *sdata, const char *sessionid, int64_t id) { stratum_instance_t *instance, *tmp; uint64_t session64; @@ -1048,8 +1112,8 @@ static bool disconnected_sessionid_exists(const char *sessionid, int64_t id) /* Number is in BE but we don't swap either of them */ hex2bin(&session64, sessionid, 8); - ck_rlock(&instance_lock); - HASH_ITER(hh, stratum_instances, instance, tmp) { + ck_rlock(&sdata->instance_lock); + HASH_ITER(hh, sdata->stratum_instances, instance, tmp) { if (instance->id == id) continue; if (instance->enonce1_64 == session64) { @@ -1058,11 +1122,11 @@ static bool disconnected_sessionid_exists(const char *sessionid, int64_t id) } } instance = NULL; - HASH_FIND(hh, disconnected_instances, &session64, sizeof(uint64_t), instance); + HASH_FIND(hh, sdata->disconnected_instances, &session64, sizeof(uint64_t), instance); if (instance) ret = true; out_unlock: - ck_runlock(&instance_lock); + ck_runlock(&sdata->instance_lock); out: return ret; } @@ -1070,7 +1134,7 @@ out: /* For creating a list of sends without locking that can then be concatenated * to the stratum_sends list. Minimises locking and avoids taking recursive * locks. */ -static void stratum_broadcast(json_t *val) +static void stratum_broadcast(sdata_t *sdata, json_t *val) { stratum_instance_t *instance, *tmp; ckmsg_t *bulk_send = NULL; @@ -1080,8 +1144,8 @@ static void stratum_broadcast(json_t *val) return; } - ck_rlock(&instance_lock); - HASH_ITER(hh, stratum_instances, instance, tmp) { + ck_rlock(&sdata->instance_lock); + HASH_ITER(hh, sdata->stratum_instances, instance, tmp) { ckmsg_t *client_msg; smsg_t *msg; @@ -1094,91 +1158,119 @@ static void stratum_broadcast(json_t *val) client_msg->data = msg; DL_APPEND(bulk_send, client_msg); } - ck_runlock(&instance_lock); + ck_runlock(&sdata->instance_lock); json_decref(val); if (!bulk_send) return; - mutex_lock(ssends->lock); - if (ssends->msgs) - DL_CONCAT(ssends->msgs, bulk_send); + mutex_lock(sdata->ssends->lock); + if (sdata->ssends->msgs) + DL_CONCAT(sdata->ssends->msgs, bulk_send); else - ssends->msgs = bulk_send; - pthread_cond_signal(ssends->cond); - mutex_unlock(ssends->lock); + sdata->ssends->msgs = bulk_send; + pthread_cond_signal(sdata->ssends->cond); + mutex_unlock(sdata->ssends->lock); } -static void stratum_add_send(json_t *val, int64_t client_id) +static void stratum_add_send(sdata_t *sdata, json_t *val, int64_t client_id) { smsg_t *msg; msg = ckzalloc(sizeof(smsg_t)); msg->json_msg = val; msg->client_id = client_id; - ckmsgq_add(ssends, msg); + ckmsgq_add(sdata->ssends, msg); } -static void inc_worker(user_instance_t *instance) +static void inc_worker(ckpool_t *ckp, user_instance_t *instance) { - mutex_lock(&stats_lock); - stats.workers++; + sdata_t *sdata = ckp->data; + + mutex_lock(&sdata->stats_lock); + sdata->stats.workers++; if (!instance->workers++) - stats.users++; - mutex_unlock(&stats_lock); + sdata->stats.users++; + mutex_unlock(&sdata->stats_lock); } -static void dec_worker(user_instance_t *instance) +static void dec_worker(ckpool_t *ckp, user_instance_t *instance) { - mutex_lock(&stats_lock); - stats.workers--; + sdata_t *sdata = ckp->data; + + mutex_lock(&sdata->stats_lock); + sdata->stats.workers--; if (!--instance->workers) - stats.users--; - mutex_unlock(&stats_lock); + sdata->stats.users--; + mutex_unlock(&sdata->stats_lock); } -static void drop_client(int64_t id) +static void drop_client(sdata_t *sdata, int64_t id) { - stratum_instance_t *client = NULL; + stratum_instance_t *client, *tmp; bool dec = false; LOGINFO("Stratifier dropping client %ld", id); - ck_wlock(&instance_lock); - client = __instance_by_id(id); + ck_wlock(&sdata->instance_lock); + client = __instance_by_id(sdata, id); if (client) { stratum_instance_t *old_client = NULL; + __inc_instance_ref(client); if (client->authorised) { dec = true; client->authorised = false; } - HASH_DEL(stratum_instances, client); - HASH_FIND(hh, disconnected_instances, &client->enonce1_64, sizeof(uint64_t), old_client); + HASH_DEL(sdata->stratum_instances, client); + HASH_FIND(hh, sdata->disconnected_instances, &client->enonce1_64, sizeof(uint64_t), old_client); /* Only keep around one copy of the old client in server mode */ if (!client->ckp->proxy && !old_client && client->enonce1_64) - HASH_ADD(hh, disconnected_instances, enonce1_64, sizeof(uint64_t), client); + HASH_ADD(hh, sdata->disconnected_instances, enonce1_64, sizeof(uint64_t), client); + else { + if (client->user_instance) + DL_DELETE(client->user_instance->instances, client); + LL_PREPEND(sdata->dead_instances, client); + } } - ck_wunlock(&instance_lock); + ck_wunlock(&sdata->instance_lock); + /* Decrease worker count outside of instance_lock to avoid recursive + * locking */ if (dec) - dec_worker(client->user_instance); + dec_worker(client->ckp, client->user_instance); + + /* Cull old unused clients lazily when there are no more reference + * counts for them. */ + ck_wlock(&sdata->instance_lock); + if (client) + __dec_instance_ref(client); + DL_FOREACH_SAFE(sdata->dead_instances, client, tmp) { + if (!client->ref) { + LOGINFO("Stratifier discarding instance %ld", client->id); + LL_DELETE(sdata->dead_instances, client); + free(client->workername); + free(client->useragent); + free(client); + } + } + ck_wunlock(&sdata->instance_lock); } -static void stratum_broadcast_message(const char *msg) +static void stratum_broadcast_message(sdata_t *sdata, const char *msg) { json_t *json_msg; JSON_CPACK(json_msg, "{sosss[s]}", "id", json_null(), "method", "client.show_message", "params", msg); - stratum_broadcast(json_msg); + stratum_broadcast(sdata, json_msg); } /* Send a generic reconnect to all clients without parameters to make them * reconnect to the same server. */ -static void reconnect_clients(const char *cmd) +static void reconnect_clients(sdata_t *sdata, const char *cmd) { char *port = strdupa(cmd), *url = NULL; json_t *json_msg; @@ -1195,12 +1287,13 @@ static void reconnect_clients(const char *cmd) } else JSON_CPACK(json_msg, "{sosss[]}", "id", json_null(), "method", "client.reconnect", "params"); - stratum_broadcast(json_msg); + stratum_broadcast(sdata, json_msg); } static void block_solve(ckpool_t *ckp, const char *blockhash) { ckmsg_t *block, *tmp, *found = NULL; + sdata_t *sdata = ckp->data; char cdfield[64]; int height = 0; ts_t ts_now; @@ -1212,8 +1305,8 @@ static void block_solve(ckpool_t *ckp, const char *blockhash) ts_realtime(&ts_now); sprintf(cdfield, "%lu,%lu", ts_now.tv_sec, ts_now.tv_nsec); - mutex_lock(&block_lock); - DL_FOREACH_SAFE(block_solves, block, tmp) { + mutex_lock(&sdata->block_lock); + DL_FOREACH_SAFE(sdata->block_solves, block, tmp) { val = block->data; char *solvehash; @@ -1225,12 +1318,12 @@ static void block_solve(ckpool_t *ckp, const char *blockhash) if (!strcmp(solvehash, blockhash)) { dealloc(solvehash); found = block; - DL_DELETE(block_solves, block); + DL_DELETE(sdata->block_solves, block); break; } dealloc(solvehash); } - mutex_unlock(&block_lock); + mutex_unlock(&sdata->block_lock); if (unlikely(!found)) { LOGERR("Failed to find blockhash %s in block_solve!", blockhash); @@ -1246,20 +1339,20 @@ static void block_solve(ckpool_t *ckp, const char *blockhash) free(found); ASPRINTF(&msg, "Block %d solved by %s!", height, ckp->name); - stratum_broadcast_message(msg); + stratum_broadcast_message(sdata, msg); free(msg); LOGWARNING("Solved and confirmed block %d", height); } -static void block_reject(const char *blockhash) +static void block_reject(sdata_t *sdata, const char *blockhash) { ckmsg_t *block, *tmp, *found = NULL; int height = 0; json_t *val; - mutex_lock(&block_lock); - DL_FOREACH_SAFE(block_solves, block, tmp) { + mutex_lock(&sdata->block_lock); + DL_FOREACH_SAFE(sdata->block_solves, block, tmp) { val = block->data; char *solvehash; @@ -1271,12 +1364,12 @@ static void block_reject(const char *blockhash) if (!strcmp(solvehash, blockhash)) { dealloc(solvehash); found = block; - DL_DELETE(block_solves, block); + DL_DELETE(sdata->block_solves, block); break; } dealloc(solvehash); } - mutex_unlock(&block_lock); + mutex_unlock(&sdata->block_lock); if (unlikely(!found)) { LOGERR("Failed to find blockhash %s in block_reject!", blockhash); @@ -1293,7 +1386,7 @@ static void block_reject(const char *blockhash) /* Some upstream pools (like p2pool) don't update stratum often enough and * miners disconnect if they don't receive regular communication so send them * a ping at regular intervals */ -static void broadcast_ping(void) +static void broadcast_ping(sdata_t *sdata) { json_t *json_msg; @@ -1302,12 +1395,13 @@ static void broadcast_ping(void) "id", 42, "method", "mining.ping"); - stratum_broadcast(json_msg); + stratum_broadcast(sdata, json_msg); } static int stratum_loop(ckpool_t *ckp, proc_instance_t *pi) { int sockd, ret = 0, selret = 0; + sdata_t *sdata = ckp->data; unixsock_t *us = &pi->us; tv_t start_tv = {0, 0}; char *buf = NULL; @@ -1328,7 +1422,7 @@ retry: } else { LOGDEBUG("%ds elapsed in strat_loop, pinging miners", ckp->update_interval); - broadcast_ping(); + broadcast_ping(sdata); } continue; } @@ -1349,11 +1443,20 @@ retry: dealloc(buf); buf = recv_unix_msg(sockd); - if (!buf) { + if (unlikely(!buf)) { Close(sockd); LOGWARNING("Failed to get message in stratum_loop"); goto retry; } + if (likely(buf[0] == '{')) { + /* The bulk of the messages will be received json from the + * connector so look for this first. The srecv_process frees + * the buf heap ram */ + ckmsgq_add(sdata->srecvs, buf); + Close(sockd); + buf = NULL; + goto retry; + } if (cmdmatch(buf, "ping")) { LOGDEBUG("Stratifier received ping request"); send_unix_msg(sockd, "pong"); @@ -1383,22 +1486,19 @@ retry: if (ret < 0) LOGDEBUG("Stratifier failed to parse dropclient command: %s", buf); else - drop_client(client_id); + drop_client(sdata, client_id); } else if (cmdmatch(buf, "dropall")) { drop_allclients(ckp); } else if (cmdmatch(buf, "block")) { block_solve(ckp, buf + 6); } else if (cmdmatch(buf, "noblock")) { - block_reject(buf + 8); + block_reject(sdata, buf + 8); } else if (cmdmatch(buf, "reconnect")) { - reconnect_clients(buf); + reconnect_clients(sdata, buf); } else if (cmdmatch(buf, "loglevel")) { sscanf(buf, "loglevel=%d", &ckp->loglevel); - } else { - /* The srecv_process frees the buf heap ram */ - ckmsgq_add(srecvs, buf); - buf = NULL; - } + } else + LOGWARNING("Unhandled stratifier message: %s", buf); goto retry; out: @@ -1409,6 +1509,7 @@ out: static void *blockupdate(void *arg) { ckpool_t *ckp = (ckpool_t *)arg; + sdata_t *sdata = ckp->data; char *buf = NULL; char request[8]; @@ -1425,7 +1526,7 @@ static void *blockupdate(void *arg) buf = send_recv_generator(ckp, request, GEN_LAX); if (buf && cmdmatch(buf, "notify")) cksleep_ms(5000); - else if (buf && strcmp(buf, lastswaphash) && !cmdmatch(buf, "failed")) + else if (buf && strcmp(buf, sdata->lastswaphash) && !cmdmatch(buf, "failed")) update_base(ckp, GEN_PRIORITY); else cksleep_ms(ckp->blockpoll); @@ -1433,7 +1534,7 @@ static void *blockupdate(void *arg) return NULL; } -static inline bool enonce1_free(uint64_t enonce1) +static inline bool enonce1_free(sdata_t *sdata, uint64_t enonce1) { stratum_instance_t *client, *tmp; bool ret = true; @@ -1442,12 +1543,15 @@ static inline bool enonce1_free(uint64_t enonce1) ret = false; goto out; } - HASH_ITER(hh, stratum_instances, client, tmp) { + + ck_rlock(&sdata->instance_lock); + HASH_ITER(hh, sdata->stratum_instances, client, tmp) { if (client->enonce1_64 == enonce1) { ret = false; break; } } + ck_runlock(&sdata->instance_lock); out: return ret; } @@ -1455,49 +1559,57 @@ out: /* Create a new enonce1 from the 64 bit enonce1_64 value, using only the number * of bytes we have to work with when we are proxying with a split nonce2. * When the proxy space is less than 32 bits to work with, we look for an - * unused enonce1 value and reject clients instead if there is no space left */ + * unused enonce1 value and reject clients instead if there is no space left. + * Needs to be entered with client holding a ref count. */ static bool new_enonce1(stratum_instance_t *client) { + sdata_t *sdata = client->ckp->data; bool ret = false; workbase_t *wb; int i; - ck_wlock(&workbase_lock); - wb = current_workbase; + ck_wlock(&sdata->workbase_lock); + wb = sdata->current_workbase; switch(wb->enonce1varlen) { case 8: - enonce1u.u64++; + sdata->enonce1u.u64++; ret = true; break; + case 7: + case 6: + case 5: case 4: - enonce1u.u32++; + sdata->enonce1u.u32++; ret = true; break; + case 3: case 2: for (i = 0; i < 65536; i++) { - enonce1u.u16++; - ret = enonce1_free(enonce1u.u64); + sdata->enonce1u.u16++; + ret = enonce1_free(sdata, sdata->enonce1u.u64); if (ret) break; } break; case 1: for (i = 0; i < 256; i++) { - enonce1u.u8++; - ret = enonce1_free(enonce1u.u64); + sdata->enonce1u.u8++; + ret = enonce1_free(sdata, sdata->enonce1u.u64); if (ret) break; } break; + default: + quit(0, "Invalid enonce1varlen %d", wb->enonce1varlen); } if (ret) - client->enonce1_64 = enonce1u.u64; + client->enonce1_64 = sdata->enonce1u.u64; if (wb->enonce1constlen) memcpy(client->enonce1bin, wb->enonce1constbin, wb->enonce1constlen); memcpy(client->enonce1bin + wb->enonce1constlen, &client->enonce1_64, wb->enonce1varlen); __bin2hex(client->enonce1var, &client->enonce1_64, wb->enonce1varlen); __bin2hex(client->enonce1, client->enonce1bin, wb->enonce1constlen + wb->enonce1varlen); - ck_wunlock(&workbase_lock); + ck_wunlock(&sdata->workbase_lock); if (unlikely(!ret)) LOGWARNING("Enonce1 space exhausted! Proxy rejecting clients"); @@ -1505,23 +1617,25 @@ static bool new_enonce1(stratum_instance_t *client) return ret; } -static void stratum_send_message(stratum_instance_t *client, const char *msg); +static void stratum_send_message(sdata_t *sdata, stratum_instance_t *client, const char *msg); -/* Extranonce1 must be set here */ +/* Extranonce1 must be set here. Needs to be entered with client holding a ref + * count. */ static json_t *parse_subscribe(stratum_instance_t *client, int64_t client_id, json_t *params_val) { + sdata_t *sdata = client->ckp->data; bool old_match = false; int arr_size; json_t *ret; int n2len; if (unlikely(!json_is_array(params_val))) { - stratum_send_message(client, "Invalid json: params not an array"); + stratum_send_message(sdata, client, "Invalid json: params not an array"); return json_string("params not an array"); } - if (unlikely(!current_workbase)) { - stratum_send_message(client, "Pool Initialising"); + if (unlikely(!sdata->current_workbase)) { + stratum_send_message(sdata, client, "Pool Initialising"); return json_string("Initialising"); } @@ -1540,7 +1654,7 @@ static json_t *parse_subscribe(stratum_instance_t *client, int64_t client_id, js buf = json_string_value(json_array_get(params_val, 1)); LOGDEBUG("Found old session id %s", buf); /* Add matching here */ - if (disconnected_sessionid_exists(buf, client_id)) { + if (disconnected_sessionid_exists(sdata, buf, client_id)) { sprintf(client->enonce1, "%016lx", client->enonce1_64); old_match = true; } @@ -1550,7 +1664,7 @@ static json_t *parse_subscribe(stratum_instance_t *client, int64_t client_id, js if (!old_match) { /* Create a new extranonce1 based on a uint64_t pointer */ if (!new_enonce1(client)) { - stratum_send_message(client, "Pool full of clients"); + stratum_send_message(sdata, client, "Pool full of clients"); client->reject = 2; return json_string("proxy full"); } @@ -1561,14 +1675,14 @@ static json_t *parse_subscribe(stratum_instance_t *client, int64_t client_id, js client->enonce1); } - ck_rlock(&workbase_lock); - if (likely(workbases)) - n2len = workbases->enonce2varlen; + ck_rlock(&sdata->workbase_lock); + if (likely(sdata->workbases)) + n2len = sdata->workbases->enonce2varlen; else n2len = 8; JSON_CPACK(ret, "[[[s,s]],s,i]", "mining.notify", client->enonce1, client->enonce1, n2len); - ck_runlock(&workbase_lock); + ck_runlock(&sdata->workbase_lock); client->subscribed = true; @@ -1700,11 +1814,13 @@ static void read_workerstats(ckpool_t *ckp, worker_instance_t *worker) /* This simply strips off the first part of the workername and matches it to a - * user or creates a new one. */ + * user or creates a new one. Needs to be entered with client holding a ref + * count. */ static user_instance_t *generate_user(ckpool_t *ckp, stratum_instance_t *client, const char *workername) { char *base_username = strdupa(workername), *username; + sdata_t *sdata = ckp->data; user_instance_t *instance; stratum_instance_t *tmp; bool new = false; @@ -1717,17 +1833,18 @@ static user_instance_t *generate_user(ckpool_t *ckp, stratum_instance_t *client, if (unlikely(len > 127)) username[127] = '\0'; - ck_wlock(&instance_lock); - HASH_FIND_STR(user_instances, username, instance); + ck_wlock(&sdata->instance_lock); + HASH_FIND_STR(sdata->user_instances, username, instance); if (!instance) { /* New user instance. Secondary user id will be NULL */ instance = ckzalloc(sizeof(user_instance_t)); strcpy(instance->username, username); new = true; - instance->id = user_instance_id++; - HASH_ADD_STR(user_instances, username, instance); - read_userstats(ckp, instance); + instance->id = sdata->user_instance_id++; + HASH_ADD_STR(sdata->user_instances, username, instance); + if (CKP_STANDALONE(ckp)) + read_userstats(ckp, instance); } DL_FOREACH(instance->instances, tmp) { if (!safecmp(workername, tmp->workername)) { @@ -1738,14 +1855,18 @@ static user_instance_t *generate_user(ckpool_t *ckp, stratum_instance_t *client, /* Create one worker instance for combined data from workers of the * same name */ if (!client->worker_instance) { - client->worker_instance = ckzalloc(sizeof(worker_instance_t)); - client->worker_instance->workername = strdup(workername); - client->worker_instance->instance = instance; - DL_APPEND(instance->worker_instances, client->worker_instance); - read_workerstats(ckp, client->worker_instance); + worker_instance_t *worker = ckzalloc(sizeof(worker_instance_t)); + + worker->workername = strdup(workername); + worker->instance = instance; + DL_APPEND(instance->worker_instances, worker); + if (CKP_STANDALONE(ckp)) + read_workerstats(ckp, worker); + worker->start_time = time(NULL); + client->worker_instance = worker; } DL_APPEND(instance->instances, client); - ck_wunlock(&instance_lock); + ck_wunlock(&sdata->instance_lock); if (new && !ckp->proxy) { /* Is this a btc address based username? */ @@ -1759,13 +1880,15 @@ static user_instance_t *generate_user(ckpool_t *ckp, stratum_instance_t *client, } /* Send this to the database and parse the response to authorise a user - * and get SUID parameters back. We don't add these requests to the ckdbqueue + * and get SUID parameters back. We don't add these requests to the sdata->ckdbqueue * since we have to wait for the response but this is done from the authoriser - * thread so it won't hold anything up but other authorisations. */ + * thread so it won't hold anything up but other authorisations. Needs to be + * entered with client holding a ref count. */ static int send_recv_auth(stratum_instance_t *client) { user_instance_t *user_instance = client->user_instance; ckpool_t *ckp = client->ckp; + sdata_t *sdata = ckp->data; char *buf = NULL, *json_msg; char cdfield[64]; int ret = 1; @@ -1799,9 +1922,9 @@ static int send_recv_auth(stratum_instance_t *client) /* We want responses from ckdb serialised and not interleaved with * other requests. Wait up to 3 seconds for exclusive access to ckdb * and if we don't receive it treat it as a delayed auth if possible */ - if (likely(!mutex_timedlock(&ckdb_lock, 3))) { + if (likely(!mutex_timedlock(&sdata->ckdb_lock, 3))) { buf = ckdb_msg_call(ckp, json_msg); - mutex_unlock(&ckdb_lock); + mutex_unlock(&sdata->ckdb_lock); } free(json_msg); @@ -1847,7 +1970,8 @@ static int send_recv_auth(stratum_instance_t *client) /* For sending auths to ckdb after we've already decided we can authorise * these clients while ckdb is offline, based on an existing client of the - * same username already having been authorised. */ + * same username already having been authorised. Needs to be entered with + * client holding a ref count. */ static void queue_delayed_auth(stratum_instance_t *client) { ckpool_t *ckp = client->ckp; @@ -1873,10 +1997,12 @@ static void queue_delayed_auth(stratum_instance_t *client) ckdbq_add(ckp, ID_AUTH, val); } +/* Needs to be entered with client holding a ref count. */ static json_t *parse_authorise(stratum_instance_t *client, json_t *params_val, json_t **err_val, const char *address, int *errnum) { user_instance_t *user_instance; + ckpool_t *ckp = client->ckp; bool ret = false; const char *buf; int arr_size; @@ -1912,7 +2038,7 @@ static json_t *parse_authorise(stratum_instance_t *client, json_t *params_val, j *err_val = json_string("Invalid character in username"); goto out; } - user_instance = client->user_instance = generate_user(client->ckp, client, buf); + user_instance = client->user_instance = generate_user(ckp, client, buf); client->user_id = user_instance->id; ts_realtime(&now); client->start_time = now.tv_sec; @@ -1921,7 +2047,7 @@ static json_t *parse_authorise(stratum_instance_t *client, json_t *params_val, j LOGNOTICE("Authorised client %ld worker %s as user %s", client->id, buf, user_instance->username); client->workername = strdup(buf); - if (CKP_STANDALONE(client->ckp)) + if (CKP_STANDALONE(ckp)) ret = true; else { *errnum = send_recv_auth(client); @@ -1937,27 +2063,29 @@ static json_t *parse_authorise(stratum_instance_t *client, json_t *params_val, j } client->authorised = ret; if (client->authorised) - inc_worker(user_instance); + inc_worker(ckp, user_instance); out: return json_boolean(ret); } -static void stratum_send_diff(stratum_instance_t *client) +/* Needs to be entered with client holding a ref count. */ +static void stratum_send_diff(sdata_t *sdata, stratum_instance_t *client) { json_t *json_msg; JSON_CPACK(json_msg, "{s[I]soss}", "params", client->diff, "id", json_null(), "method", "mining.set_difficulty"); - stratum_add_send(json_msg, client->id); + stratum_add_send(sdata, json_msg, client->id); } -static void stratum_send_message(stratum_instance_t *client, const char *msg) +/* Needs to be entered with client holding a ref count. */ +static void stratum_send_message(sdata_t *sdata, stratum_instance_t *client, const char *msg) { json_t *json_msg; JSON_CPACK(json_msg, "{sosss[s]}", "id", json_null(), "method", "client.show_message", "params", msg); - stratum_add_send(json_msg, client->id); + stratum_add_send(sdata, json_msg, client->id); } static double time_bias(double tdiff, double period) @@ -1980,6 +2108,7 @@ static double sane_tdiff(tv_t *end, tv_t *start) return tdiff; } +/* Needs to be entered with client holding a ref count. */ static void add_submit(ckpool_t *ckp, stratum_instance_t *client, int diff, bool valid, bool submit) { @@ -1987,15 +2116,16 @@ static void add_submit(ckpool_t *ckp, stratum_instance_t *client, int diff, bool double tdiff, bdiff, dsps, drr, network_diff, bias; user_instance_t *instance = client->user_instance; int64_t next_blockid, optimal; + sdata_t *sdata = ckp->data; tv_t now_t; - mutex_lock(&stats_lock); + mutex_lock(&sdata->stats_lock); if (valid) { - stats.unaccounted_shares++; - stats.unaccounted_diff_shares += diff; + sdata->stats.unaccounted_shares++; + sdata->stats.unaccounted_diff_shares += diff; } else - stats.unaccounted_rejects += diff; - mutex_unlock(&stats_lock); + sdata->stats.unaccounted_rejects += diff; + mutex_unlock(&sdata->stats_lock); /* Count only accepted and stale rejects in diff calculation. */ if (!valid && !submit) @@ -2003,13 +2133,13 @@ static void add_submit(ckpool_t *ckp, stratum_instance_t *client, int diff, bool tv_time(&now_t); - ck_rlock(&workbase_lock); - next_blockid = workbase_id + 1; + ck_rlock(&sdata->workbase_lock); + next_blockid = sdata->workbase_id + 1; if (ckp->proxy) - network_diff = current_workbase->diff; + network_diff = sdata->current_workbase->diff; else - network_diff = current_workbase->network_diff; - ck_runlock(&workbase_lock); + network_diff = sdata->current_workbase->network_diff; + ck_runlock(&sdata->workbase_lock); if (unlikely(!client->first_share.tv_sec)) { copy_tv(&client->first_share, &now_t); @@ -2030,6 +2160,7 @@ static void add_submit(ckpool_t *ckp, stratum_instance_t *client, int diff, bool decay_time(&worker->dsps60, diff, tdiff, 3600); decay_time(&worker->dsps1440, diff, tdiff, 86400); copy_tv(&worker->last_share, &now_t); + worker->idle = false; tdiff = sane_tdiff(&now_t, &instance->last_share); decay_time(&instance->dsps1, diff, tdiff, 60); @@ -2104,10 +2235,11 @@ static void add_submit(ckpool_t *ckp, stratum_instance_t *client, int diff, bool client->diff_change_job_id = next_blockid; client->old_diff = client->diff; client->diff = optimal; - stratum_send_diff(client); + stratum_send_diff(sdata, client); } -/* We should already be holding the workbase_lock */ +/* We should already be holding the workbase_lock. Needs to be entered with + * client holding a ref count. */ static void test_blocksolve(stratum_instance_t *client, workbase_t *wb, const uchar *data, const uchar *hash, double diff, const char *coinbase, int cblen, const char *nonce2, const char *nonce) @@ -2116,14 +2248,15 @@ test_blocksolve(stratum_instance_t *client, workbase_t *wb, const uchar *data, c char hexcoinbase[1024], blockhash[68]; json_t *val = NULL, *val_copy; char *gbt_block, varint[12]; + ckpool_t *ckp = wb->ckp; + sdata_t *sdata = ckp->data; ckmsg_t *block_ckmsg; char cdfield[64]; uchar swap[32]; - ckpool_t *ckp; ts_t ts_now; /* Submit anything over 99% of the diff in case of rounding errors */ - if (diff < current_workbase->network_diff * 0.99) + if (diff < sdata->current_workbase->network_diff * 0.99) return; LOGWARNING("Possible block solve diff %f !", diff); @@ -2161,7 +2294,6 @@ test_blocksolve(stratum_instance_t *client, workbase_t *wb, const uchar *data, c strcat(gbt_block, hexcoinbase); if (wb->transactions) realloc_strcat(&gbt_block, wb->txn_data); - ckp = wb->ckp; send_generator(ckp, gbt_block, GEN_PRIORITY); free(gbt_block); @@ -2180,18 +2312,19 @@ test_blocksolve(stratum_instance_t *client, workbase_t *wb, const uchar *data, c "createdate", cdfield, "createby", "code", "createcode", __func__, - "createinet", ckp->serverurl); + "createinet", ckp->serverurl[client->server]); val_copy = json_deep_copy(val); block_ckmsg = ckalloc(sizeof(ckmsg_t)); block_ckmsg->data = val_copy; - mutex_lock(&block_lock); - DL_APPEND(block_solves, block_ckmsg); - mutex_unlock(&block_lock); + mutex_lock(&sdata->block_lock); + DL_APPEND(sdata->block_solves, block_ckmsg); + mutex_unlock(&sdata->block_lock); ckdbq_add(ckp, ID_BLOCK, val); } +/* Needs to be entered with client holding a ref count. */ static double submission_diff(stratum_instance_t *client, workbase_t *wb, const char *nonce2, uint32_t ntime32, const char *nonce, uchar *hash) { @@ -2252,27 +2385,28 @@ static double submission_diff(stratum_instance_t *client, workbase_t *wb, const return ret; } -static bool new_share(const uchar *hash, int64_t wb_id) +static bool new_share(sdata_t *sdata, const uchar *hash, int64_t wb_id) { share_t *share, *match = NULL; bool ret = false; - ck_wlock(&share_lock); - HASH_FIND(hh, shares, hash, 32, match); + ck_wlock(&sdata->share_lock); + HASH_FIND(hh, sdata->shares, hash, 32, match); if (match) goto out_unlock; share = ckzalloc(sizeof(share_t)); memcpy(share->hash, hash, 32); share->workbase_id = wb_id; - HASH_ADD(hh, shares, hash, 32, share); + HASH_ADD(hh, sdata->shares, hash, 32, share); ret = true; out_unlock: - ck_wunlock(&share_lock); + ck_wunlock(&sdata->share_lock); return ret; } -/* Submit a share in proxy mode to the parent pool. workbase_lock is held */ +/* Submit a share in proxy mode to the parent pool. workbase_lock is held. + * Needs to be entered with client holding a ref count. */ static void submit_share(stratum_instance_t *client, int64_t jobid, const char *nonce2, const char *ntime, const char *nonce, int msg_id) { @@ -2293,6 +2427,7 @@ static void submit_share(stratum_instance_t *client, int64_t jobid, const char * #define JSON_ERR(err) json_string(SHARE_ERR(err)) +/* Needs to be entered with client holding a ref count. */ static json_t *parse_submit(stratum_instance_t *client, json_t *json_msg, json_t *params_val, json_t **err_val) { @@ -2304,6 +2439,7 @@ static json_t *parse_submit(stratum_instance_t *client, json_t *json_msg, char *fname = NULL, *s, *nonce2; enum share_err err = SE_NONE; ckpool_t *ckp = client->ckp; + sdata_t *sdata = ckp->data; char idstring[20] = {}; workbase_t *wb = NULL; uint32_t ntime32; @@ -2369,14 +2505,14 @@ static json_t *parse_submit(stratum_instance_t *client, json_t *json_msg, share = true; - ck_rlock(&workbase_lock); - HASH_FIND_I64(workbases, &id, wb); + ck_rlock(&sdata->workbase_lock); + HASH_FIND_I64(sdata->workbases, &id, wb); if (unlikely(!wb)) { - id = current_workbase->id; + id = sdata->current_workbase->id; err = SE_INVALID_JOBID; json_set_string(json_msg, "reject-reason", SHARE_ERR(err)); strncpy(idstring, job_id, 19); - ASPRINTF(&fname, "%s.sharelog", current_workbase->logdir); + ASPRINTF(&fname, "%s.sharelog", sdata->current_workbase->logdir); goto out_unlock; } wdiff = wb->diff; @@ -2400,7 +2536,7 @@ static json_t *parse_submit(stratum_instance_t *client, json_t *json_msg, bswap_256(sharehash, hash); __bin2hex(hexhash, sharehash, 32); - if (id < blockchange_id) { + if (id < sdata->blockchange_id) { err = SE_STALE; json_set_string(json_msg, "reject-reason", SHARE_ERR(err)); goto out_submit; @@ -2416,7 +2552,7 @@ out_submit: if (sdiff >= wdiff) submit = true; out_unlock: - ck_runlock(&workbase_lock); + ck_runlock(&sdata->workbase_lock); /* Accept the lower of new and old diffs until the next update */ if (id < client->diff_change_job_id && client->old_diff < client->diff) @@ -2426,7 +2562,7 @@ out_unlock: suffix_string(wdiff, wdiffsuffix, 16, 0); if (sdiff >= diff) { - if (new_share(hash, id)) { + if (new_share(sdata, hash, id)) { LOGINFO("Accepted client %ld share diff %.1f/%.0f/%s: %s", client->id, sdiff, diff, wdiffsuffix, hexhash); result = true; @@ -2476,7 +2612,7 @@ out_unlock: json_set_string(val, "createdate", cdfield); json_set_string(val, "createby", "code"); json_set_string(val, "createcode", __func__); - json_set_string(val, "createinet", ckp->serverurl); + json_set_string(val, "createinet", ckp->serverurl[client->server]); json_set_string(val, "workername", client->workername); json_set_string(val, "username", user_instance->username); @@ -2500,13 +2636,13 @@ out: if (client->first_invalid < client->last_share.tv_sec || !client->first_invalid) client->first_invalid = now_t; else if (client->first_invalid && client->first_invalid < now_t - 120) { - LOGNOTICE("Client %"PRId64" rejecting for 120s, disconnecting", client->id); - stratum_send_message(client, "Disconnecting for continuous invalid shares"); + LOGNOTICE("Client %ld rejecting for 120s, disconnecting", client->id); + stratum_send_message(sdata, client, "Disconnecting for continuous invalid shares"); client->reject = 2; } else if (client->first_invalid && client->first_invalid < now_t - 60) { if (!client->reject) { - LOGINFO("Client %"PRId64" rejecting for 60s, sending diff", client->id); - stratum_send_diff(client); + LOGINFO("Client %ld rejecting for 60s, sending diff", client->id); + stratum_send_diff(sdata, client); client->reject = 1; } } @@ -2521,7 +2657,7 @@ out: if (!CKP_STANDALONE(ckp)) json_set_string(val, "secondaryuserid", user_instance->secondaryuserid); json_set_string(val, "enonce1", client->enonce1); - json_set_int(val, "workinfoid", current_workbase->id); + json_set_int(val, "workinfoid", sdata->current_workbase->id); json_set_string(val, "workername", client->workername); json_set_string(val, "username", user_instance->username); json_object_set(val, "error", *err_val); @@ -2529,7 +2665,7 @@ out: json_set_string(val, "createdate", cdfield); json_set_string(val, "createby", "code"); json_set_string(val, "createcode", __func__); - json_set_string(val, "createinet", ckp->serverurl); + json_set_string(val, "createinet", ckp->serverurl[client->server]); ckdbq_add(ckp, ID_SHAREERR, val); LOGINFO("Invalid share from client %ld: %s", client->id, client->workername); } @@ -2538,61 +2674,62 @@ out: } /* Must enter with workbase_lock held */ -static json_t *__stratum_notify(bool clean) +static json_t *__stratum_notify(sdata_t *sdata, bool clean) { json_t *val; JSON_CPACK(val, "{s:[ssssosssb],s:o,s:s}", "params", - current_workbase->idstring, - current_workbase->prevhash, - current_workbase->coinb1, - current_workbase->coinb2, - json_deep_copy(current_workbase->merkle_array), - current_workbase->bbversion, - current_workbase->nbit, - current_workbase->ntime, + sdata->current_workbase->idstring, + sdata->current_workbase->prevhash, + sdata->current_workbase->coinb1, + sdata->current_workbase->coinb2, + json_deep_copy(sdata->current_workbase->merkle_array), + sdata->current_workbase->bbversion, + sdata->current_workbase->nbit, + sdata->current_workbase->ntime, clean, "id", json_null(), "method", "mining.notify"); return val; } -static void stratum_broadcast_update(bool clean) +static void stratum_broadcast_update(sdata_t *sdata, bool clean) { json_t *json_msg; - ck_rlock(&workbase_lock); - json_msg = __stratum_notify(clean); - ck_runlock(&workbase_lock); + ck_rlock(&sdata->workbase_lock); + json_msg = __stratum_notify(sdata, clean); + ck_runlock(&sdata->workbase_lock); - stratum_broadcast(json_msg); + stratum_broadcast(sdata, json_msg); } /* For sending a single stratum template update */ -static void stratum_send_update(int64_t client_id, bool clean) +static void stratum_send_update(sdata_t *sdata, int64_t client_id, bool clean) { json_t *json_msg; - ck_rlock(&workbase_lock); - json_msg = __stratum_notify(clean); - ck_runlock(&workbase_lock); + ck_rlock(&sdata->workbase_lock); + json_msg = __stratum_notify(sdata, clean); + ck_runlock(&sdata->workbase_lock); - stratum_add_send(json_msg, client_id); + stratum_add_send(sdata, json_msg, client_id); } -static void send_json_err(int64_t client_id, json_t *id_val, const char *err_msg) +static void send_json_err(sdata_t *sdata, int64_t client_id, json_t *id_val, const char *err_msg) { json_t *val; JSON_CPACK(val, "{soss}", "id", json_copy(id_val), "error", err_msg); - stratum_add_send(val, client_id); + stratum_add_send(sdata, val, client_id); } -static void update_client(stratum_instance_t *client, const int64_t client_id) +/* Needs to be entered with client holding a ref count. */ +static void update_client(sdata_t *sdata, stratum_instance_t *client, const int64_t client_id) { - stratum_send_update(client_id, true); - stratum_send_diff(client); + stratum_send_update(sdata, client_id, true); + stratum_send_diff(sdata, client); } static json_params_t @@ -2615,14 +2752,15 @@ static void set_worker_mindiff(ckpool_t *ckp, const char *workername, int mindif char *username = strdupa(workername), *ignore; user_instance_t *instance = NULL; stratum_instance_t *client; + sdata_t *sdata = ckp->data; ignore = username; strsep(&ignore, "._"); /* Find the user first */ - ck_rlock(&instance_lock); - HASH_FIND_STR(user_instances, username, instance); - ck_runlock(&instance_lock); + ck_rlock(&sdata->instance_lock); + HASH_FIND_STR(sdata->user_instances, username, instance); + ck_runlock(&sdata->instance_lock); /* They may just have not connected yet */ if (!instance) { @@ -2631,14 +2769,14 @@ static void set_worker_mindiff(ckpool_t *ckp, const char *workername, int mindif } /* Then find the matching worker instance */ - ck_rlock(&instance_lock); + ck_rlock(&sdata->instance_lock); DL_FOREACH(instance->worker_instances, tmp) { if (!safecmp(workername, tmp->workername)) { worker = tmp; break; } } - ck_runlock(&instance_lock); + ck_runlock(&sdata->instance_lock); /* They may just not be connected at the moment */ if (!worker) { @@ -2660,7 +2798,7 @@ static void set_worker_mindiff(ckpool_t *ckp, const char *workername, int mindif * matching worker that are currently live and send them a new diff * if we can. Otherwise it will only act as a clamp on next share * submission. */ - ck_rlock(&instance_lock); + ck_rlock(&sdata->instance_lock); DL_FOREACH(instance->instances, client) { if (client->worker_instance != worker) continue; @@ -2670,16 +2808,18 @@ static void set_worker_mindiff(ckpool_t *ckp, const char *workername, int mindif if (mindiff == client->diff) continue; client->diff = mindiff; - stratum_send_diff(client); + stratum_send_diff(sdata, client); } - ck_runlock(&instance_lock); + ck_runlock(&sdata->instance_lock); } /* Implement support for the diff in the params as well as the originally - * documented form of placing diff within the method. */ + * documented form of placing diff within the method. Needs to be entered with + * client holding a ref count. */ static void suggest_diff(stratum_instance_t *client, const char *method, json_t *params_val) { json_t *arr_val = json_array_get(params_val, 0); + sdata_t *sdata = client->ckp->data; int64_t sdiff; if (unlikely(!client->authorised)) { @@ -2701,20 +2841,17 @@ static void suggest_diff(stratum_instance_t *client, const char *method, json_t client->diff = client->ckp->mindiff; else client->diff = sdiff; - stratum_send_diff(client); + stratum_send_diff(sdata, client); } -static void parse_method(const int64_t client_id, json_t *id_val, json_t *method_val, - json_t *params_val, char *address) +static void parse_method(sdata_t *sdata, const int64_t client_id, json_t *id_val, + json_t *method_val, json_t *params_val, char *address) { stratum_instance_t *client; const char *method; char buf[256]; - ck_rlock(&instance_lock); - client = __instance_by_id(client_id); - ck_runlock(&instance_lock); - + client = ref_instance_by_id(sdata, client_id); if (unlikely(!client)) { LOGINFO("Failed to find client id %ld in hashtable!", client_id); return; @@ -2724,7 +2861,7 @@ static void parse_method(const int64_t client_id, json_t *id_val, json_t *method LOGINFO("Dropping client %"PRId64" tagged for lazy invalidation", client_id); snprintf(buf, 255, "dropclient=%ld", client->id); send_proc(client->ckp->connector, buf); - return; + goto out; } /* Random broken clients send something not an integer as the id so we copy @@ -2742,10 +2879,10 @@ static void parse_method(const int64_t client_id, json_t *id_val, json_t *method json_object_set_new_nocheck(val, "result", result_val); json_object_set_nocheck(val, "id", id_val); json_object_set_new_nocheck(val, "error", json_null()); - stratum_add_send(val, client_id); + stratum_add_send(sdata, val, client_id); if (likely(client->subscribed)) - update_client(client, client_id); - return; + update_client(sdata, client, client_id); + goto out; } if (unlikely(cmdmatch(method, "mining.passthrough"))) { @@ -2753,22 +2890,22 @@ static void parse_method(const int64_t client_id, json_t *id_val, json_t *method * is a passthrough and to manage its messages accordingly. * Remove this instance since the client id may well be * reused */ - ck_wlock(&instance_lock); - HASH_DEL(stratum_instances, client); - ck_wunlock(&instance_lock); + ck_wlock(&sdata->instance_lock); + HASH_DEL(sdata->stratum_instances, client); + ck_wunlock(&sdata->instance_lock); LOGINFO("Adding passthrough client %ld", client->id); snprintf(buf, 255, "passthrough=%ld", client->id); send_proc(client->ckp->connector, buf); free(client); - return; + goto out; } if (cmdmatch(method, "mining.auth") && client->subscribed) { json_params_t *jp = create_json_params(client_id, method_val, params_val, id_val, address); - ckmsgq_add(sauthq, jp); - return; + ckmsgq_add(sdata->sauthq, jp); + goto out; } /* We should only accept authorised requests from here on */ @@ -2779,14 +2916,14 @@ static void parse_method(const int64_t client_id, json_t *id_val, json_t *method LOGINFO("Dropping unauthorised client %ld", client->id); snprintf(buf, 255, "dropclient=%ld", client->id); send_proc(client->ckp->connector, buf); - return; + goto out; } if (cmdmatch(method, "mining.submit")) { json_params_t *jp = create_json_params(client_id, method_val, params_val, id_val, address); - ckmsgq_add(sshareq, jp); - return; + ckmsgq_add(sdata->sshareq, jp); + goto out; } if (cmdmatch(method, "mining.suggest")) { @@ -2798,13 +2935,15 @@ static void parse_method(const int64_t client_id, json_t *id_val, json_t *method if (cmdmatch(method, "mining.get")) { json_params_t *jp = create_json_params(client_id, method_val, params_val, id_val, address); - ckmsgq_add(stxnq, jp); - return; + ckmsgq_add(sdata->stxnq, jp); + goto out; } /* Unhandled message here */ +out: + dec_instance_ref(sdata, client); } -static void parse_instance_msg(smsg_t *msg) +static void parse_instance_msg(sdata_t *sdata, smsg_t *msg) { json_t *val = msg->json_msg, *id_val, *method, *params; int64_t client_id = msg->client_id; @@ -2823,19 +2962,19 @@ static void parse_instance_msg(smsg_t *msg) LOGDEBUG("Received spurious response %s", result ? result : ""); goto out; } - send_json_err(client_id, id_val, "-3:method not found"); + send_json_err(sdata, client_id, id_val, "-3:method not found"); goto out; } if (unlikely(!json_is_string(method))) { - send_json_err(client_id, id_val, "-1:method is not string"); + send_json_err(sdata, client_id, id_val, "-1:method is not string"); goto out; } params = json_object_get(val, "params"); if (unlikely(!params)) { - send_json_err(client_id, id_val, "-1:params not found"); + send_json_err(sdata, client_id, id_val, "-1:params not found"); goto out; } - parse_method(client_id, id_val, method, params, msg->address); + parse_method(sdata, client_id, id_val, method, params, msg->address); out: json_decref(val); free(msg); @@ -2843,9 +2982,10 @@ out: static void srecv_process(ckpool_t *ckp, char *buf) { - stratum_instance_t *instance; + sdata_t *sdata = ckp->data; smsg_t *msg; json_t *val; + int server; val = json_loads(buf, 0, NULL); if (unlikely(!val)) { @@ -2875,16 +3015,24 @@ static void srecv_process(ckpool_t *ckp, char *buf) strcpy(msg->address, json_string_value(val)); json_object_clear(val); - /* Parse the message here */ - ck_wlock(&instance_lock); - instance = __instance_by_id(msg->client_id); - if (!instance) { - /* client_id instance doesn't exist yet, create one */ - instance = __stratum_add_instance(ckp, msg->client_id); + val = json_object_get(msg->json_msg, "server"); + if (unlikely(!val)) { + LOGWARNING("Failed to extract server from connector json smsg %s", buf); + json_decref(msg->json_msg); + free(msg); + goto out; } - ck_wunlock(&instance_lock); + server = json_integer_value(val); + json_object_clear(val); + + /* Parse the message here */ + ck_wlock(&sdata->instance_lock); + /* client_id instance doesn't exist yet, create one */ + if (!__instance_by_id(sdata, msg->client_id)) + __stratum_add_instance(ckp, msg->client_id, server); + ck_wunlock(&sdata->instance_lock); - parse_instance_msg(msg); + parse_instance_msg(sdata, msg); out: free(buf); } @@ -2924,32 +3072,32 @@ static void discard_json_params(json_params_t **jp) *jp = NULL; } -static void sshare_process(ckpool_t __maybe_unused *ckp, json_params_t *jp) +static void sshare_process(ckpool_t *ckp, json_params_t *jp) { json_t *result_val, *json_msg, *err_val = NULL; stratum_instance_t *client; + sdata_t *sdata = ckp->data; int64_t client_id; client_id = jp->client_id; - ck_rlock(&instance_lock); - client = __instance_by_id(client_id); - ck_runlock(&instance_lock); - + client = ref_instance_by_id(sdata, client_id); if (unlikely(!client)) { LOGINFO("Share processor failed to find client id %ld in hashtable!", client_id); goto out; } if (unlikely(!client->authorised)) { LOGDEBUG("Client %ld no longer authorised to submit shares", client_id); - goto out; + goto out_decref; } json_msg = json_object(); result_val = parse_submit(client, json_msg, jp->params, &err_val); json_object_set_new_nocheck(json_msg, "result", result_val); json_object_set_new_nocheck(json_msg, "error", err_val ? err_val : json_null()); json_object_set_nocheck(json_msg, "id", jp->id_val); - stratum_add_send(json_msg, client_id); + stratum_add_send(sdata, json_msg, client_id); +out_decref: + dec_instance_ref(sdata, client); out: discard_json_params(&jp); } @@ -2958,14 +3106,13 @@ static void sauth_process(ckpool_t *ckp, json_params_t *jp) { json_t *result_val, *json_msg, *err_val = NULL; stratum_instance_t *client; + sdata_t *sdata = ckp->data; int mindiff, errnum = 0; int64_t client_id; client_id = jp->client_id; - ck_rlock(&instance_lock); - client = __instance_by_id(client_id); - ck_runlock(&instance_lock); + client = ref_instance_by_id(sdata, client_id); if (unlikely(!client)) { LOGINFO("Authoriser failed to find client id %ld in hashtable!", client_id); @@ -2977,18 +3124,18 @@ static void sauth_process(ckpool_t *ckp, json_params_t *jp) ASPRINTF(&buf, "Authorised, welcome to %s %s!", ckp->name, client->user_instance->username); - stratum_send_message(client, buf); + stratum_send_message(sdata, client, buf); } else { if (errnum < 0) - stratum_send_message(client, "Authorisations temporarily offline :("); + stratum_send_message(sdata, client, "Authorisations temporarily offline :("); else - stratum_send_message(client, "Failed authorisation :("); + stratum_send_message(sdata, client, "Failed authorisation :("); } json_msg = json_object(); json_object_set_new_nocheck(json_msg, "result", result_val); json_object_set_new_nocheck(json_msg, "error", err_val ? err_val : json_null()); json_object_set_nocheck(json_msg, "id", jp->id_val); - stratum_add_send(json_msg, client_id); + stratum_add_send(sdata, json_msg, client_id); if (!json_is_true(result_val) || !client->suggest_diff) goto out; @@ -2998,9 +3145,11 @@ static void sauth_process(ckpool_t *ckp, json_params_t *jp) mindiff = MAX(ckp->mindiff, client->suggest_diff); if (mindiff != client->diff) { client->diff = mindiff; - stratum_send_diff(client); + stratum_send_diff(sdata, client); } out: + if (client) + dec_instance_ref(sdata, client); discard_json_params(&jp); } @@ -3034,12 +3183,13 @@ static void parse_ckdb_cmd(ckpool_t __maybe_unused *ckp, const char *cmd) static void ckdbq_process(ckpool_t *ckp, char *msg) { static bool failed = false; + sdata_t *sdata = ckp->data; char *buf = NULL; while (!buf) { - mutex_lock(&ckdb_lock); + mutex_lock(&sdata->ckdb_lock); buf = ckdb_msg_call(ckp, msg); - mutex_unlock(&ckdb_lock); + mutex_unlock(&sdata->ckdb_lock); if (unlikely(!buf)) { if (!failed) { @@ -3076,30 +3226,30 @@ static void ckdbq_process(ckpool_t *ckp, char *msg) } } -static int transactions_by_jobid(int64_t id) +static int transactions_by_jobid(sdata_t *sdata, int64_t id) { workbase_t *wb; int ret = -1; - ck_rlock(&workbase_lock); - HASH_FIND_I64(workbases, &id, wb); + ck_rlock(&sdata->workbase_lock); + HASH_FIND_I64(sdata->workbases, &id, wb); if (wb) ret = wb->transactions; - ck_runlock(&workbase_lock); + ck_runlock(&sdata->workbase_lock); return ret; } -static json_t *txnhashes_by_jobid(int64_t id) +static json_t *txnhashes_by_jobid(sdata_t *sdata, int64_t id) { json_t *ret = NULL; workbase_t *wb; - ck_rlock(&workbase_lock); - HASH_FIND_I64(workbases, &id, wb); + ck_rlock(&sdata->workbase_lock); + HASH_FIND_I64(sdata->workbases, &id, wb); if (wb) ret = json_string(wb->txn_hashes); - ck_runlock(&workbase_lock); + ck_runlock(&sdata->workbase_lock); return ret; } @@ -3108,7 +3258,8 @@ static void send_transactions(ckpool_t *ckp, json_params_t *jp) { const char *msg = json_string_value(jp->method), *params = json_string_value(json_array_get(jp->params, 0)); - stratum_instance_t *client; + stratum_instance_t *client = NULL; + sdata_t *sdata = ckp->data; json_t *val, *hashes; int64_t job_id = 0; time_t now_t; @@ -3130,7 +3281,7 @@ static void send_transactions(ckpool_t *ckp, json_params_t *jp) sscanf(params, "%lx", &job_id); else sscanf(msg, "mining.get_transactions(%lx", &job_id); - txns = transactions_by_jobid(job_id); + txns = transactions_by_jobid(sdata, job_id); if (txns != -1) { json_set_int(val, "result", txns); json_object_set_new_nocheck(val, "error", json_null()); @@ -3144,10 +3295,7 @@ static void send_transactions(ckpool_t *ckp, json_params_t *jp) goto out_send; } - ck_rlock(&instance_lock); - client = __instance_by_id(jp->client_id); - ck_runlock(&instance_lock); - + client = ref_instance_by_id(sdata, jp->client_id); if (unlikely(!client)) { LOGINFO("send_transactions failed to find client id %ld in hashtable!", jp->client_id); @@ -3166,105 +3314,99 @@ static void send_transactions(ckpool_t *ckp, json_params_t *jp) goto out_send; } sscanf(params, "%lx", &job_id); - hashes = txnhashes_by_jobid(job_id); + hashes = txnhashes_by_jobid(sdata, job_id); if (hashes) { json_object_set_new_nocheck(val, "result", hashes); json_object_set_new_nocheck(val, "error", json_null()); } else json_set_string(val, "error", "Invalid job_id"); out_send: - stratum_add_send(val, jp->client_id); + stratum_add_send(sdata, val, jp->client_id); out: discard_json_params(&jp); + if (client) + dec_instance_ref(sdata, client); } /* Called every 20 seconds, we send the updated stats to ckdb of those users * who have gone 10 minutes between updates. This ends up staggering stats to * avoid floods of stat data coming at once. */ -static void update_userstats(ckpool_t *ckp) +static void update_workerstats(ckpool_t *ckp, sdata_t *sdata) { - stratum_instance_t *client, *tmp; - json_t *val = NULL; + user_instance_t *user, *tmp; char cdfield[64]; time_t now_t; ts_t ts_now; - if (++stats.userstats_cycle > 0x1f) - stats.userstats_cycle = 0; + if (++sdata->stats.userstats_cycle > 0x1f) + sdata->stats.userstats_cycle = 0; ts_realtime(&ts_now); sprintf(cdfield, "%lu,%lu", ts_now.tv_sec, ts_now.tv_nsec); now_t = ts_now.tv_sec; - ck_rlock(&instance_lock); - HASH_ITER(hh, stratum_instances, client, tmp) { - double ghs1, ghs5, ghs60, ghs1440; + ck_rlock(&sdata->instance_lock); + HASH_ITER(hh, sdata->user_instances, user, tmp) { + worker_instance_t *worker; uint8_t cycle_mask; - int elapsed; - - if (!client->authorised) - continue; - /* Send one lot of stats once the client is idle if they have submitted - * no shares in the last 10 minutes with the idle bool set. */ - if (client->idle && client->notified_idle) - continue; - /* Select clients using a mask to return each user's stats once + /* Select users using a mask to return each user's stats once * every ~10 minutes */ - cycle_mask = client->user_id & 0x1f; - if (cycle_mask != stats.userstats_cycle) + cycle_mask = user->id & 0x1f; + if (cycle_mask != sdata->stats.userstats_cycle) continue; - - if (val) { - json_set_bool(val,"eos", false); - ckdbq_add(ckp, ID_USERSTATS, val); - val = NULL; + DL_FOREACH(user->worker_instances, worker) { + double ghs1, ghs5, ghs60, ghs1440; + int elapsed; + json_t *val; + + /* Send one lot of stats once the worker is idle if + * they have submitted no shares in the last 10 minutes + * with the idle bool set. */ + if (worker->idle && worker->notified_idle) + continue; + elapsed = now_t - worker->start_time; + ghs1 = worker->dsps1 * nonces; + ghs5 = worker->dsps5 * nonces; + ghs60 = worker->dsps60 * nonces; + ghs1440 = worker->dsps1440 * nonces; + JSON_CPACK(val, "{ss,si,ss,ss,sf,sf,sf,sf,sb,ss,ss,ss,ss}", + "poolinstance", ckp->name, + "elapsed", elapsed, + "username", user->username, + "workername", worker->workername, + "hashrate", ghs1, + "hashrate5m", ghs5, + "hashrate1hr", ghs60, + "hashrate24hr", ghs1440, + "idle", worker->idle, + "createdate", cdfield, + "createby", "code", + "createcode", __func__, + "createinet", ckp->serverurl[0]); + worker->notified_idle = worker->idle; + ckdbq_add(ckp, ID_WORKERSTATS, val); } - elapsed = now_t - client->start_time; - ghs1 = client->dsps1 * nonces; - ghs5 = client->dsps5 * nonces; - ghs60 = client->dsps60 * nonces; - ghs1440 = client->dsps1440 * nonces; - JSON_CPACK(val, "{ss,sI,si,ss,ss,sf,sf,sf,sf,sb,ss,ss,ss,ss}", - "poolinstance", ckp->name, - "instanceid", client->id, - "elapsed", elapsed, - "username", client->user_instance->username, - "workername", client->workername, - "hashrate", ghs1, - "hashrate5m", ghs5, - "hashrate1hr", ghs60, - "hashrate24hr", ghs1440, - "idle", client->idle, - "createdate", cdfield, - "createby", "code", - "createcode", __func__, - "createinet", ckp->serverurl); - client->notified_idle = client->idle; } - /* Mark the last userstats sent on this pass of stats with an end of - * stats marker. */ - if (val) { - json_set_bool(val,"eos", true); - ckdbq_add(ckp, ID_USERSTATS, val); - } - ck_runlock(&instance_lock); + ck_runlock(&sdata->instance_lock); } static void *statsupdate(void *arg) { ckpool_t *ckp = (ckpool_t *)arg; + sdata_t *sdata = ckp->data; + pool_stats_t *stats = &sdata->stats; pthread_detach(pthread_self()); rename_proc("statsupdate"); - tv_time(&stats.start_time); - cksleep_prepare_r(&stats.last_update); + tv_time(&stats->start_time); + cksleep_prepare_r(&stats->last_update); sleep(1); while (42) { double ghs, ghs1, ghs5, ghs15, ghs60, ghs360, ghs1440, ghs10080; - double bias, bias5, bias60, bias1440; + double bias; double tdiff, per_tdiff; char suffix1[16], suffix5[16], suffix15[16], suffix60[16], cdfield[64]; char suffix360[16], suffix1440[16], suffix10080[16]; @@ -3280,38 +3422,38 @@ static void *statsupdate(void *arg) int i; tv_time(&now); - timersub(&now, &stats.start_time, &diff); + timersub(&now, &stats->start_time, &diff); tdiff = diff.tv_sec + (double)diff.tv_usec / 1000000; - ghs1 = stats.dsps1 * nonces; + ghs1 = stats->dsps1 * nonces; suffix_string(ghs1, suffix1, 16, 0); - sps1 = stats.sps1; + sps1 = stats->sps1; - bias5 = time_bias(tdiff, 300); - ghs5 = stats.dsps5 * nonces / bias5; + bias = !CKP_STANDALONE(ckp) ? 1.0 : time_bias(tdiff, 300); + ghs5 = stats->dsps5 * nonces / bias; + sps5 = stats->sps5 / bias; suffix_string(ghs5, suffix5, 16, 0); - sps5 = stats.sps5 / bias5; - bias = time_bias(tdiff, 900); - ghs15 = stats.dsps15 * nonces / bias; + bias = !CKP_STANDALONE(ckp) ? 1.0 : time_bias(tdiff, 900); + ghs15 = stats->dsps15 * nonces / bias; suffix_string(ghs15, suffix15, 16, 0); - sps15 = stats.sps15 / bias; + sps15 = stats->sps15 / bias; - bias60 = time_bias(tdiff, 3600); - ghs60 = stats.dsps60 * nonces / bias60; + bias = !CKP_STANDALONE(ckp) ? 1.0 : time_bias(tdiff, 3600); + ghs60 = stats->dsps60 * nonces / bias; + sps60 = stats->sps60 / bias; suffix_string(ghs60, suffix60, 16, 0); - sps60 = stats.sps60 / bias60; - bias = time_bias(tdiff, 21600); - ghs360 = stats.dsps360 * nonces / bias; + bias = !CKP_STANDALONE(ckp) ? 1.0 : time_bias(tdiff, 21600); + ghs360 = stats->dsps360 * nonces / bias; suffix_string(ghs360, suffix360, 16, 0); - bias1440 = time_bias(tdiff, 86400); - ghs1440 = stats.dsps1440 * nonces / bias1440; + bias = !CKP_STANDALONE(ckp) ? 1.0 : time_bias(tdiff, 86400); + ghs1440 = stats->dsps1440 * nonces / bias; suffix_string(ghs1440, suffix1440, 16, 0); - bias = time_bias(tdiff, 604800); - ghs10080 = stats.dsps10080 * nonces / bias; + bias = !CKP_STANDALONE(ckp) ? 1.0 : time_bias(tdiff, 604800); + ghs10080 = stats->dsps10080 * nonces / bias; suffix_string(ghs10080, suffix10080, 16, 0); snprintf(fname, 511, "%s/pool/pool.status", ckp->logdir); @@ -3321,8 +3463,8 @@ static void *statsupdate(void *arg) JSON_CPACK(val, "{si,si,si}", "runtime", diff.tv_sec, - "Users", stats.users, - "Workers", stats.workers); + "Users", stats->users, + "Workers", stats->workers); s = json_dumps(val, JSON_NO_UTF8 | JSON_PRESERVE_ORDER); json_decref(val); LOGNOTICE("Pool:%s", s); @@ -3355,8 +3497,8 @@ static void *statsupdate(void *arg) dealloc(s); fclose(fp); - ck_rlock(&instance_lock); - HASH_ITER(hh, stratum_instances, client, tmp) { + ck_rlock(&sdata->instance_lock); + HASH_ITER(hh, sdata->stratum_instances, client, tmp) { if (!client->authorised) continue; @@ -3375,7 +3517,7 @@ static void *statsupdate(void *arg) } } - HASH_ITER(hh, user_instances, instance, tmpuser) { + HASH_ITER(hh, sdata->user_instances, instance, tmpuser) { worker_instance_t *worker; bool idle = false; @@ -3387,6 +3529,7 @@ static void *statsupdate(void *arg) decay_time(&worker->dsps5, 0, per_tdiff, 300); decay_time(&worker->dsps60, 0, per_tdiff, 3600); decay_time(&worker->dsps1440, 0, per_tdiff, 86400); + worker->idle = true; } ghs = worker->dsps1 * nonces; suffix_string(ghs, suffix1, 16, 0); @@ -3466,15 +3609,15 @@ static void *statsupdate(void *arg) json_decref(val); fclose(fp); } - ck_runlock(&instance_lock); + ck_runlock(&sdata->instance_lock); ts_realtime(&ts_now); sprintf(cdfield, "%lu,%lu", ts_now.tv_sec, ts_now.tv_nsec); JSON_CPACK(val, "{ss,si,si,si,sf,sf,sf,sf,ss,ss,ss,ss}", "poolinstance", ckp->name, "elapsed", diff.tv_sec, - "users", stats.users, - "workers", stats.workers, + "users", stats->users, + "workers", stats->workers, "hashrate", ghs1, "hashrate5m", ghs5, "hashrate1hr", ghs60, @@ -3482,38 +3625,38 @@ static void *statsupdate(void *arg) "createdate", cdfield, "createby", "code", "createcode", __func__, - "createinet", ckp->serverurl); + "createinet", ckp->serverurl[0]); ckdbq_add(ckp, ID_POOLSTATS, val); /* Update stats 3 times per minute for smooth values, displaying * status every minute. */ for (i = 0; i < 3; i++) { - cksleep_ms_r(&stats.last_update, 20000); - cksleep_prepare_r(&stats.last_update); - update_userstats(ckp); - - mutex_lock(&stats_lock); - stats.accounted_shares += stats.unaccounted_shares; - stats.accounted_diff_shares += stats.unaccounted_diff_shares; - stats.accounted_rejects += stats.unaccounted_rejects; - - decay_time(&stats.sps1, stats.unaccounted_shares, 20, 60); - decay_time(&stats.sps5, stats.unaccounted_shares, 20, 300); - decay_time(&stats.sps15, stats.unaccounted_shares, 20, 900); - decay_time(&stats.sps60, stats.unaccounted_shares, 20, 3600); - - decay_time(&stats.dsps1, stats.unaccounted_diff_shares, 20, 60); - decay_time(&stats.dsps5, stats.unaccounted_diff_shares, 20, 300); - decay_time(&stats.dsps15, stats.unaccounted_diff_shares, 20, 900); - decay_time(&stats.dsps60, stats.unaccounted_diff_shares, 20, 3600); - decay_time(&stats.dsps360, stats.unaccounted_diff_shares, 20, 21600); - decay_time(&stats.dsps1440, stats.unaccounted_diff_shares, 20, 86400); - decay_time(&stats.dsps10080, stats.unaccounted_diff_shares, 20, 604800); - - stats.unaccounted_shares = - stats.unaccounted_diff_shares = - stats.unaccounted_rejects = 0; - mutex_unlock(&stats_lock); + cksleep_ms_r(&stats->last_update, 20000); + cksleep_prepare_r(&stats->last_update); + update_workerstats(ckp, sdata); + + mutex_lock(&sdata->stats_lock); + stats->accounted_shares += stats->unaccounted_shares; + stats->accounted_diff_shares += stats->unaccounted_diff_shares; + stats->accounted_rejects += stats->unaccounted_rejects; + + decay_time(&stats->sps1, stats->unaccounted_shares, 20, 60); + decay_time(&stats->sps5, stats->unaccounted_shares, 20, 300); + decay_time(&stats->sps15, stats->unaccounted_shares, 20, 900); + decay_time(&stats->sps60, stats->unaccounted_shares, 20, 3600); + + decay_time(&stats->dsps1, stats->unaccounted_diff_shares, 20, 60); + decay_time(&stats->dsps5, stats->unaccounted_diff_shares, 20, 300); + decay_time(&stats->dsps15, stats->unaccounted_diff_shares, 20, 900); + decay_time(&stats->dsps60, stats->unaccounted_diff_shares, 20, 3600); + decay_time(&stats->dsps360, stats->unaccounted_diff_shares, 20, 21600); + decay_time(&stats->dsps1440, stats->unaccounted_diff_shares, 20, 86400); + decay_time(&stats->dsps10080, stats->unaccounted_diff_shares, 20, 604800); + + stats->unaccounted_shares = + stats->unaccounted_diff_shares = + stats->unaccounted_rejects = 0; + mutex_unlock(&sdata->stats_lock); } } @@ -3526,6 +3669,7 @@ static void *statsupdate(void *arg) static void *ckdb_heartbeat(void *arg) { ckpool_t *ckp = (ckpool_t *)arg; + sdata_t *sdata = ckp->data; pthread_detach(pthread_self()); rename_proc("heartbeat"); @@ -3536,7 +3680,7 @@ static void *ckdb_heartbeat(void *arg) json_t *val; cksleep_ms(1000); - if (unlikely(!ckmsgq_empty(ckdbq))) { + if (unlikely(!ckmsgq_empty(sdata->ckdbq))) { LOGDEBUG("Witholding heartbeat due to ckdb messages being queued"); continue; } @@ -3546,7 +3690,7 @@ static void *ckdb_heartbeat(void *arg) "createdate", cdfield, "createby", "code", "createcode", __func__, - "createinet", ckp->serverurl); + "createinet", ckp->serverurl[0]); ckdbq_add(ckp, ID_HEARTBEAT, val); } return NULL; @@ -3557,9 +3701,13 @@ int stratifier(proc_instance_t *pi) pthread_t pth_blockupdate, pth_statsupdate, pth_heartbeat; ckpool_t *ckp = pi->ckp; int ret = 1, threads; + int64_t randomiser; + sdata_t *sdata; char *buf; LOGWARNING("%s stratifier starting", ckp->name); + sdata = ckzalloc(sizeof(sdata_t)); + ckp->data = sdata; /* Wait for the generator to have something for us */ do { @@ -3578,52 +3726,57 @@ int stratifier(proc_instance_t *pi) /* Store this for use elsewhere */ hex2bin(scriptsig_header_bin, scriptsig_header, 41); - address_to_pubkeytxn(pubkeytxnbin, ckp->btcaddress); + address_to_pubkeytxn(sdata->pubkeytxnbin, ckp->btcaddress); if (test_address(ckp, ckp->donaddress)) { ckp->donvalid = true; - address_to_pubkeytxn(donkeytxnbin, ckp->donaddress); + address_to_pubkeytxn(sdata->donkeytxnbin, ckp->donaddress); } } + randomiser = ((int64_t)time(NULL)) << 32; /* Set the initial id to time as high bits so as to not send the same * id on restarts */ if (!ckp->proxy) - blockchange_id = workbase_id = ((int64_t)time(NULL)) << 32; + sdata->blockchange_id = sdata->workbase_id = randomiser; + sdata->enonce1u.u64 = htobe64(randomiser); dealloc(buf); - if (!ckp->serverurl) - ckp->serverurl = "127.0.0.1"; - cklock_init(&instance_lock); + if (!ckp->serverurls) { + ckp->serverurl[0] = "127.0.0.1"; + ckp->serverurls = 1; + } + cklock_init(&sdata->instance_lock); - mutex_init(&ckdb_lock); - ssends = create_ckmsgq(ckp, "ssender", &ssend_process); + mutex_init(&sdata->ckdb_lock); + sdata->ssends = create_ckmsgq(ckp, "ssender", &ssend_process); /* Create half as many share processing threads as there are CPUs */ threads = sysconf(_SC_NPROCESSORS_ONLN) / 2 ? : 1; - sshareq = create_ckmsgqs(ckp, "sprocessor", &sshare_process, threads); + sdata->sshareq = create_ckmsgqs(ckp, "sprocessor", &sshare_process, threads); /* Create 1/4 as many stratum processing threads as there are CPUs */ threads = threads / 2 ? : 1; - srecvs = create_ckmsgqs(ckp, "sreceiver", &srecv_process, threads); - sauthq = create_ckmsgq(ckp, "authoriser", &sauth_process); - ckdbq = create_ckmsgq(ckp, "ckdbqueue", &ckdbq_process); - stxnq = create_ckmsgq(ckp, "stxnq", &send_transactions); + sdata->srecvs = create_ckmsgqs(ckp, "sreceiver", &srecv_process, threads); + sdata->sauthq = create_ckmsgq(ckp, "authoriser", &sauth_process); + sdata->ckdbq = create_ckmsgq(ckp, "ckdbqueue", &ckdbq_process); + sdata->stxnq = create_ckmsgq(ckp, "stxnq", &send_transactions); if (!CKP_STANDALONE(ckp)) create_pthread(&pth_heartbeat, ckdb_heartbeat, ckp); - cklock_init(&workbase_lock); + cklock_init(&sdata->workbase_lock); if (!ckp->proxy) create_pthread(&pth_blockupdate, blockupdate, ckp); - mutex_init(&stats_lock); + mutex_init(&sdata->stats_lock); create_pthread(&pth_statsupdate, statsupdate, ckp); - cklock_init(&share_lock); - mutex_init(&block_lock); + cklock_init(&sdata->share_lock); + mutex_init(&sdata->block_lock); LOGWARNING("%s stratifier ready", ckp->name); ret = stratum_loop(ckp, pi); out: + dealloc(ckp->data); return process_exit(ckp, pi, ret); }