From 15351e5ddef3624fa03aa4dd0b1f5901f6a2a4eb Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Fri, 6 Feb 2015 12:00:34 +1100 Subject: [PATCH 001/544] Proxy msg_id is not used in any meaningful way so remove it --- src/generator.c | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/generator.c b/src/generator.c index 9c23e5c3..e5e82962 100644 --- a/src/generator.c +++ b/src/generator.c @@ -98,7 +98,6 @@ struct proxy_instance { double diff; tv_t last_share; - int msg_id; /* Message id for sending stratum messages */ bool no_sessionid; /* Doesn't support session id resume on subscribe */ bool no_params; /* Doesn't want any parameters on subscribe */ @@ -660,19 +659,19 @@ retry: /* Attempt to reconnect if the pool supports resuming */ if (proxi->sessionid) { JSON_CPACK(req, "{s:i,s:s,s:[s,s]}", - "id", proxi->msg_id++, + "id", 0, "method", "mining.subscribe", "params", PACKAGE"/"VERSION, proxi->sessionid); /* Then attempt to connect with just the client description */ } else if (!proxi->no_params) { JSON_CPACK(req, "{s:i,s:s,s:[s]}", - "id", proxi->msg_id++, + "id", 0, "method", "mining.subscribe", "params", PACKAGE"/"VERSION); /* Then try without any parameters */ } else { JSON_CPACK(req, "{s:i,s:s,s:[]}", - "id", proxi->msg_id++, + "id", 0, "method", "mining.subscribe", "params"); } @@ -1042,7 +1041,7 @@ static bool auth_stratum(connsock_t *cs, proxy_instance_t *proxi) bool ret; JSON_CPACK(req, "{s:i,s:s,s:[s,s]}", - "id", proxi->msg_id++, + "id", 42, "method", "mining.authorize", "params", proxi->auth, proxi->pass); ret = send_json_msg(cs, req); From 622bd1541f2017212ab17ba99838bebe8575e0dd Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Fri, 6 Feb 2015 13:45:41 +1100 Subject: [PATCH 002/544] Make stratifier data contain a pointer to the proxy data and add a hashlist to be able to add multiple proxies, allocating only one for now --- src/stratifier.c | 105 +++++++++++++++++++++++++++++------------------ 1 file changed, 65 insertions(+), 40 deletions(-) diff --git a/src/stratifier.c b/src/stratifier.c index 0f01e42e..3a80ff6c 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -279,6 +279,25 @@ struct share { typedef struct share share_t; +struct proxy_base { + UT_hash_handle hh; + int id; + + double diff; + + char enonce1[32]; + uchar enonce1bin[16]; + int enonce1constlen; + int enonce1varlen; + + int nonce2len; + int enonce2varlen; + + bool subscribed; +}; + +typedef struct proxy_base proxy_t; + struct stratifier_data { char pubkeytxnbin[25]; char donkeytxnbin[25]; @@ -349,19 +368,9 @@ struct stratifier_data { /* Generator message priority */ int gen_priority; - struct { - double diff; - - char enonce1[32]; - uchar enonce1bin[16]; - int enonce1constlen; - int enonce1varlen; - - int nonce2len; - int enonce2varlen; - - bool subscribed; - } proxy_base; + proxy_t *proxy; /* Current proxy in use */ + proxy_t *proxies; /* Hashlist of all proxies */ + pthread_mutex_t proxy_lock; /* Protects all proxy data */ }; typedef struct stratifier_data sdata_t; @@ -978,33 +987,33 @@ static void update_subscribe(ckpool_t *ckp) free(buf); ck_wlock(&sdata->workbase_lock); - sdata->proxy_base.subscribed = true; - sdata->proxy_base.diff = ckp->startdiff; + sdata->proxy->subscribed = true; + sdata->proxy->diff = ckp->startdiff; /* Length is checked by generator */ - strcpy(sdata->proxy_base.enonce1, json_string_value(json_object_get(val, "enonce1"))); - sdata->proxy_base.enonce1constlen = strlen(sdata->proxy_base.enonce1) / 2; - hex2bin(sdata->proxy_base.enonce1bin, sdata->proxy_base.enonce1, sdata->proxy_base.enonce1constlen); - sdata->proxy_base.nonce2len = json_integer_value(json_object_get(val, "nonce2len")); + strcpy(sdata->proxy->enonce1, json_string_value(json_object_get(val, "enonce1"))); + sdata->proxy->enonce1constlen = strlen(sdata->proxy->enonce1) / 2; + hex2bin(sdata->proxy->enonce1bin, sdata->proxy->enonce1, sdata->proxy->enonce1constlen); + sdata->proxy->nonce2len = json_integer_value(json_object_get(val, "nonce2len")); if (ckp->clientsvspeed) { - if (sdata->proxy_base.nonce2len > 5) - sdata->proxy_base.enonce1varlen = 4; - else if (sdata->proxy_base.nonce2len > 3) - sdata->proxy_base.enonce1varlen = 2; + if (sdata->proxy->nonce2len > 5) + sdata->proxy->enonce1varlen = 4; + else if (sdata->proxy->nonce2len > 3) + sdata->proxy->enonce1varlen = 2; else - sdata->proxy_base.enonce1varlen = 1; + sdata->proxy->enonce1varlen = 1; } else { - if (sdata->proxy_base.nonce2len > 7) - sdata->proxy_base.enonce1varlen = 4; - else if (sdata->proxy_base.nonce2len > 5) - sdata->proxy_base.enonce1varlen = 2; + if (sdata->proxy->nonce2len > 7) + sdata->proxy->enonce1varlen = 4; + else if (sdata->proxy->nonce2len > 5) + sdata->proxy->enonce1varlen = 2; else - sdata->proxy_base.enonce1varlen = 1; + sdata->proxy->enonce1varlen = 1; } - sdata->proxy_base.enonce2varlen = sdata->proxy_base.nonce2len - sdata->proxy_base.enonce1varlen; + sdata->proxy->enonce2varlen = sdata->proxy->nonce2len - sdata->proxy->enonce1varlen; ck_wunlock(&sdata->workbase_lock); LOGNOTICE("Upstream pool extranonce2 length %d, max proxy clients %lld", - sdata->proxy_base.nonce2len, 1ll << (sdata->proxy_base.enonce1varlen * 8)); + sdata->proxy->nonce2len, 1ll << (sdata->proxy->enonce1varlen * 8)); json_decref(val); drop_allclients(ckp); @@ -1028,7 +1037,7 @@ static void update_notify(ckpool_t *ckp) return; } - if (unlikely(!sdata->proxy_base.subscribed)) { + if (unlikely(!sdata->proxy->subscribed)) { LOGINFO("No valid proxy subscription to update notify yet"); return; } @@ -1079,12 +1088,12 @@ static void update_notify(ckpool_t *ckp) update_diff(ckp); ck_rlock(&sdata->workbase_lock); - strcpy(wb->enonce1const, sdata->proxy_base.enonce1); - wb->enonce1constlen = sdata->proxy_base.enonce1constlen; - memcpy(wb->enonce1constbin, sdata->proxy_base.enonce1bin, wb->enonce1constlen); - wb->enonce1varlen = sdata->proxy_base.enonce1varlen; - wb->enonce2varlen = sdata->proxy_base.enonce2varlen; - wb->diff = sdata->proxy_base.diff; + strcpy(wb->enonce1const, sdata->proxy->enonce1); + wb->enonce1constlen = sdata->proxy->enonce1constlen; + memcpy(wb->enonce1constbin, sdata->proxy->enonce1bin, wb->enonce1constlen); + wb->enonce1varlen = sdata->proxy->enonce1varlen; + wb->enonce2varlen = sdata->proxy->enonce2varlen; + wb->diff = sdata->proxy->diff; ck_runlock(&sdata->workbase_lock); add_base(ckp, wb, &new_block); @@ -1125,8 +1134,8 @@ static void update_diff(ckpool_t *ckp) diff = 1; ck_wlock(&sdata->workbase_lock); - old_diff = sdata->proxy_base.diff; - sdata->current_workbase->diff = sdata->proxy_base.diff = diff; + old_diff = sdata->proxy->diff; + sdata->current_workbase->diff = sdata->proxy->diff = diff; ck_wunlock(&sdata->workbase_lock); if (old_diff < diff) @@ -4330,6 +4339,7 @@ static void read_poolstats(ckpool_t *ckp) int stratifier(proc_instance_t *pi) { pthread_t pth_blockupdate, pth_statsupdate, pth_heartbeat; + proxy_t *proxy, *tmpproxy; ckpool_t *ckp = pi->ckp; int ret = 1, threads; int64_t randomiser; @@ -4399,6 +4409,13 @@ int stratifier(proc_instance_t *pi) cklock_init(&sdata->workbase_lock); if (!ckp->proxy) create_pthread(&pth_blockupdate, blockupdate, ckp); + else { + /* Generate one proxy for now */ + proxy = ckzalloc(sizeof(proxy_t)); + mutex_init(&sdata->proxy_lock); + sdata->proxy = proxy; + HASH_ADD_INT(sdata->proxies, id, proxy); + } mutex_init(&sdata->stats_lock); create_pthread(&pth_statsupdate, statsupdate, ckp); @@ -4410,6 +4427,14 @@ int stratifier(proc_instance_t *pi) ret = stratum_loop(ckp, pi); out: + if (ckp->proxy) { + mutex_lock(&sdata->proxy_lock); + HASH_ITER(hh, sdata->proxies, proxy, tmpproxy) { + HASH_DEL(sdata->proxies, proxy); + dealloc(proxy); + } + mutex_unlock(&sdata->proxy_lock); + } dealloc(ckp->data); return process_exit(ckp, pi, ret); } From 52cd0665637eb22ef3bf48100be855d0b9c4949b Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Fri, 6 Feb 2015 14:40:22 +1100 Subject: [PATCH 003/544] Add proxies to the stratifier as its notified of their existence by the generator and issue reconnects without rejecting connections when a new subscribe is discovered --- src/generator.c | 15 +++-- src/stratifier.c | 142 ++++++++++++++++++++++++++++++++++------------- 2 files changed, 109 insertions(+), 48 deletions(-) diff --git a/src/generator.c b/src/generator.c index e5e82962..77782403 100644 --- a/src/generator.c +++ b/src/generator.c @@ -1113,7 +1113,9 @@ static void send_subscribe(proxy_instance_t *proxi, int *sockd) json_t *json_msg; char *msg; - JSON_CPACK(json_msg, "{sssi}", "enonce1", proxi->enonce1, + JSON_CPACK(json_msg, "{sisssi}", + "proxy", proxi->id, + "enonce1", proxi->enonce1, "nonce2len", proxi->nonce2len); msg = json_dumps(json_msg, JSON_NO_UTF8); json_decref(json_msg); @@ -1140,7 +1142,7 @@ static void send_notify(proxy_instance_t *proxi, int *sockd) for (i = 0; i < ni->merkles; i++) json_array_append_new(merkle_arr, json_string(&ni->merklehash[i][0])); /* Use our own jobid instead of the server's one for easy lookup */ - JSON_CPACK(json_msg, "{si,ss,si,ss,ss,so,ss,ss,ss,sb}", + JSON_CPACK(json_msg, "{si,si,ss,si,ss,ss,so,ss,ss,ss,sb}", "proxy", proxi->id, "jobid", ni->id, "prevhash", ni->prevhash, "coinb1len", ni->coinb1len, "coinbase1", ni->coinbase1, "coinbase2", ni->coinbase2, "merklehash", merkle_arr, "bbversion", ni->bbversion, @@ -1546,14 +1548,11 @@ out: return alive; } -static void kill_proxy(ckpool_t *ckp, proxy_instance_t *proxi) +static void kill_proxy(proxy_instance_t *proxi) { notify_instance_t *ni, *tmp; connsock_t *cs; - send_proc(ckp->stratifier, "reconnect"); - send_proc(ckp->connector, "reject"); - if (!proxi) // This shouldn't happen return; @@ -1586,7 +1585,7 @@ static int proxy_loop(proc_instance_t *pi) reconnect: if (proxi) { - kill_proxy(ckp, proxi); + kill_proxy(proxi); reconnecting = true; } proxi = live_proxy(ckp); @@ -1671,7 +1670,7 @@ retry: Close(sockd); goto retry; out: - kill_proxy(ckp, proxi); + kill_proxy(proxi); Close(sockd); return ret; } diff --git a/src/stratifier.c b/src/stratifier.c index 3a80ff6c..dba09bc2 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -937,12 +937,20 @@ static void __del_client(sdata_t *sdata, stratum_instance_t *client, user_instan DL_DELETE(user->clients, client); } +static void connector_drop_client(ckpool_t *ckp, const int64_t id) +{ + char buf[256]; + + LOGWARNING("Stratifier requesting connector drop client %"PRId64, id); + snprintf(buf, 255, "dropclient=%"PRId64, id); + send_proc(ckp->connector, buf); +} + static void drop_allclients(ckpool_t *ckp) { stratum_instance_t *client, *tmp; int disconnects = 0, kills = 0; sdata_t *sdata = ckp->data; - char buf[128]; ck_wlock(&sdata->instance_lock); HASH_ITER(hh, sdata->stratum_instances, client, tmp) { @@ -954,8 +962,7 @@ static void drop_allclients(ckpool_t *ckp) } else client->dropped = true; kills++; - sprintf(buf, "dropclient=%"PRId64, client_id); - send_proc(ckp->connector, buf); + connector_drop_client(ckp, client_id); } HASH_ITER(hh, sdata->disconnected_instances, client, tmp) { disconnects++; @@ -970,62 +977,119 @@ static void drop_allclients(ckpool_t *ckp) LOGNOTICE("Dropped %d instances", kills); } +static proxy_t *__generate_proxy(sdata_t *sdata, const int id) +{ + proxy_t *proxy = ckzalloc(sizeof(proxy_t)); + + proxy->id = id; + HASH_ADD_INT(sdata->proxies, id, proxy); + return proxy; +} + +/* Find proxy by id number, generate one if none exist yet by that id */ +static proxy_t *proxy_by_id(sdata_t *sdata, const int id) +{ + bool new_proxy = false; + proxy_t *proxy; + + mutex_lock(&sdata->proxy_lock); + HASH_FIND_INT(sdata->proxies, &id, proxy); + if (unlikely(!proxy)) { + new_proxy = true; + proxy = __generate_proxy(sdata, id); + } + mutex_unlock(&sdata->proxy_lock); + + if (unlikely(new_proxy)) + LOGINFO("Stratifier added new proxy %d", id); + + return proxy; +} + +static void reconnect_clients(sdata_t *sdata, const char *cmd); + static void update_subscribe(ckpool_t *ckp) { sdata_t *sdata = ckp->data; + proxy_t *proxy; json_t *val; + int id = 0; char *buf; buf = send_recv_proc(ckp->generator, "getsubscribe"); if (unlikely(!buf)) { - LOGWARNING("Failed to get subscribe from generator in update_notify"); + LOGWARNING("Failed to get subscribe from generator in update_subscribe"); drop_allclients(ckp); return; } LOGDEBUG("Update subscribe: %s", buf); val = json_loads(buf, 0, NULL); free(buf); + if (unlikely(!val)) { + LOGWARNING("Failed to json decode getsubscribe response in update_subscribe"); + return; + } + + json_get_int(&id, val, "proxy"); + proxy = proxy_by_id(sdata, id); + + mutex_lock(&sdata->proxy_lock); + if (sdata->proxy != proxy) + sdata->proxy = proxy; + mutex_unlock(&sdata->proxy_lock); ck_wlock(&sdata->workbase_lock); - sdata->proxy->subscribed = true; - sdata->proxy->diff = ckp->startdiff; + proxy->subscribed = true; + proxy->diff = ckp->startdiff; /* Length is checked by generator */ - strcpy(sdata->proxy->enonce1, json_string_value(json_object_get(val, "enonce1"))); - sdata->proxy->enonce1constlen = strlen(sdata->proxy->enonce1) / 2; - hex2bin(sdata->proxy->enonce1bin, sdata->proxy->enonce1, sdata->proxy->enonce1constlen); - sdata->proxy->nonce2len = json_integer_value(json_object_get(val, "nonce2len")); + strcpy(proxy->enonce1, json_string_value(json_object_get(val, "enonce1"))); + proxy->enonce1constlen = strlen(proxy->enonce1) / 2; + hex2bin(proxy->enonce1bin, proxy->enonce1, proxy->enonce1constlen); + proxy->nonce2len = json_integer_value(json_object_get(val, "nonce2len")); if (ckp->clientsvspeed) { - if (sdata->proxy->nonce2len > 5) - sdata->proxy->enonce1varlen = 4; - else if (sdata->proxy->nonce2len > 3) - sdata->proxy->enonce1varlen = 2; + if (proxy->nonce2len > 5) + proxy->enonce1varlen = 4; + else if (proxy->nonce2len > 3) + proxy->enonce1varlen = 2; else - sdata->proxy->enonce1varlen = 1; + proxy->enonce1varlen = 1; } else { - if (sdata->proxy->nonce2len > 7) - sdata->proxy->enonce1varlen = 4; - else if (sdata->proxy->nonce2len > 5) - sdata->proxy->enonce1varlen = 2; + if (proxy->nonce2len > 7) + proxy->enonce1varlen = 4; + else if (proxy->nonce2len > 5) + proxy->enonce1varlen = 2; else - sdata->proxy->enonce1varlen = 1; + proxy->enonce1varlen = 1; } - sdata->proxy->enonce2varlen = sdata->proxy->nonce2len - sdata->proxy->enonce1varlen; + proxy->enonce2varlen = proxy->nonce2len - proxy->enonce1varlen; ck_wunlock(&sdata->workbase_lock); LOGNOTICE("Upstream pool extranonce2 length %d, max proxy clients %lld", - sdata->proxy->nonce2len, 1ll << (sdata->proxy->enonce1varlen * 8)); + proxy->nonce2len, 1ll << (proxy->enonce1varlen * 8)); json_decref(val); - drop_allclients(ckp); + reconnect_clients(sdata, ""); } static void update_diff(ckpool_t *ckp); +static proxy_t *current_proxy(sdata_t *sdata) +{ + proxy_t *proxy; + + mutex_lock(&sdata->proxy_lock); + proxy = sdata->proxy; + mutex_unlock(&sdata->proxy_lock); + + return proxy; +} + static void update_notify(ckpool_t *ckp) { bool new_block = false, clean; sdata_t *sdata = ckp->data; char header[228]; + proxy_t *proxy; workbase_t *wb; json_t *val; char *buf; @@ -1037,7 +1101,8 @@ static void update_notify(ckpool_t *ckp) return; } - if (unlikely(!sdata->proxy->subscribed)) { + proxy = current_proxy(sdata); + if (unlikely(!proxy || !proxy->subscribed)) { LOGINFO("No valid proxy subscription to update notify yet"); return; } @@ -1088,12 +1153,12 @@ static void update_notify(ckpool_t *ckp) update_diff(ckp); ck_rlock(&sdata->workbase_lock); - strcpy(wb->enonce1const, sdata->proxy->enonce1); - wb->enonce1constlen = sdata->proxy->enonce1constlen; - memcpy(wb->enonce1constbin, sdata->proxy->enonce1bin, wb->enonce1constlen); - wb->enonce1varlen = sdata->proxy->enonce1varlen; - wb->enonce2varlen = sdata->proxy->enonce2varlen; - wb->diff = sdata->proxy->diff; + strcpy(wb->enonce1const, proxy->enonce1); + wb->enonce1constlen = proxy->enonce1constlen; + memcpy(wb->enonce1constbin, proxy->enonce1bin, wb->enonce1constlen); + wb->enonce1varlen = proxy->enonce1varlen; + wb->enonce2varlen = proxy->enonce2varlen; + wb->diff = proxy->diff; ck_runlock(&sdata->workbase_lock); add_base(ckp, wb, &new_block); @@ -3291,7 +3356,6 @@ static void parse_method(sdata_t *sdata, stratum_instance_t *client, const int64 json_t *id_val, json_t *method_val, json_t *params_val, const char *address) { const char *method; - char buf[256]; /* Random broken clients send something not an integer as the id so we * copy the json item for id_val as is for the response. By far the @@ -3328,6 +3392,8 @@ static void parse_method(sdata_t *sdata, stratum_instance_t *client, const int64 } if (unlikely(cmdmatch(method, "mining.passthrough"))) { + char buf[256]; + LOGNOTICE("Adding passthrough client %"PRId64, client_id); /* We need to inform the connector process that this client * is a passthrough and to manage its messages accordingly. @@ -3342,8 +3408,7 @@ static void parse_method(sdata_t *sdata, stratum_instance_t *client, const int64 /* We should only accept subscribed requests from here on */ if (!client->subscribed) { LOGINFO("Dropping unsubscribed client %"PRId64, client_id); - snprintf(buf, 255, "dropclient=%"PRId64, client_id); - send_proc(client->ckp->connector, buf); + connector_drop_client(client->ckp, client_id); return; } @@ -3365,8 +3430,7 @@ static void parse_method(sdata_t *sdata, stratum_instance_t *client, const int64 * stratifier process to restart since it will have lost all * the stratum instance data. Clients will just reconnect. */ LOGINFO("Dropping unauthorised client %"PRId64, client_id); - snprintf(buf, 255, "dropclient=%"PRId64, client_id); - send_proc(client->ckp->connector, buf); + connector_drop_client(client->ckp, client_id); return; } @@ -3507,6 +3571,7 @@ static void srecv_process(ckpool_t *ckp, char *buf) /* Client may be NULL here */ LOGNOTICE("Stratifier skipped dropped instance %"PRId64" message from server %d", msg->client_id, server); + connector_drop_client(ckp, msg->client_id); free_smsg(msg); goto out; } @@ -4339,7 +4404,6 @@ static void read_poolstats(ckpool_t *ckp) int stratifier(proc_instance_t *pi) { pthread_t pth_blockupdate, pth_statsupdate, pth_heartbeat; - proxy_t *proxy, *tmpproxy; ckpool_t *ckp = pi->ckp; int ret = 1, threads; int64_t randomiser; @@ -4410,11 +4474,7 @@ int stratifier(proc_instance_t *pi) if (!ckp->proxy) create_pthread(&pth_blockupdate, blockupdate, ckp); else { - /* Generate one proxy for now */ - proxy = ckzalloc(sizeof(proxy_t)); mutex_init(&sdata->proxy_lock); - sdata->proxy = proxy; - HASH_ADD_INT(sdata->proxies, id, proxy); } mutex_init(&sdata->stats_lock); @@ -4428,6 +4488,8 @@ int stratifier(proc_instance_t *pi) ret = stratum_loop(ckp, pi); out: if (ckp->proxy) { + proxy_t *proxy, *tmpproxy; + mutex_lock(&sdata->proxy_lock); HASH_ITER(hh, sdata->proxies, proxy, tmpproxy) { HASH_DEL(sdata->proxies, proxy); From b7e71e1be781e1bfee9f0636eaeaea022426b0ac Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Fri, 6 Feb 2015 17:03:14 +1100 Subject: [PATCH 004/544] Attach to every upstream pool in proxy mode all the time and fail over and back as needed --- src/generator.c | 456 ++++++++++++++++++++++------------------------- src/stratifier.c | 3 + 2 files changed, 213 insertions(+), 246 deletions(-) diff --git a/src/generator.c b/src/generator.c index 77782403..d932826e 100644 --- a/src/generator.c +++ b/src/generator.c @@ -75,8 +75,7 @@ typedef struct proxy_instance proxy_instance_t; /* Per proxied pool instance data */ struct proxy_instance { - proxy_instance_t *next; - proxy_instance_t *prev; + UT_hash_handle hh; ckpool_t *ckp; connsock_t *cs; @@ -104,6 +103,7 @@ struct proxy_instance { bool notified; /* Received new template for work */ bool diffed; /* Received new diff */ bool reconnect; /* We need to drop and reconnect */ + bool alive; pthread_mutex_t notify_lock; notify_instance_t *notify_instances; @@ -128,7 +128,7 @@ struct proxy_instance { /* Private data for the generator */ struct generator_data { pthread_mutex_t lock; /* Lock protecting linked lists */ - proxy_instance_t *proxy_list; /* Linked list of all active proxies */ + proxy_instance_t *proxies; /* Hash list of all proxies */ int proxy_notify_id; // Globally increasing notify id ckmsgq_t *srvchk; // Server check message queue }; @@ -878,6 +878,8 @@ static bool send_pong(proxy_instance_t *proxi, json_t *val) return ret; } +static void prepare_proxy(proxy_instance_t *proxi); + static bool parse_reconnect(proxy_instance_t *proxi, json_t *val) { server_instance_t *newsi, *si = proxi->si; @@ -930,6 +932,7 @@ static bool parse_reconnect(proxy_instance_t *proxi, json_t *val) newsi = ckzalloc(sizeof(server_instance_t)); mutex_lock(&gdata->lock); + HASH_DEL(gdata->proxies, proxi); newsi->id = si->id; /* Inherit the old connection's id */ si->id = ckp->proxies++; /* Give the old connection the lowest id */ ckp->servers = realloc(ckp->servers, sizeof(server_instance_t *) * ckp->proxies); @@ -946,7 +949,12 @@ static bool parse_reconnect(proxy_instance_t *proxi, json_t *val) newproxi->si = newsi; newproxi->ckp = ckp; newproxi->cs = &newsi->cs; + newproxi->id = newsi->id; + HASH_ADD_INT(gdata->proxies, id, proxi); + HASH_ADD_INT(gdata->proxies, id, newproxi); mutex_unlock(&gdata->lock); + + prepare_proxy(newproxi); out: return ret; } @@ -1248,85 +1256,6 @@ out: return ret; } -static void *proxy_recv(void *arg) -{ - proxy_instance_t *proxi = (proxy_instance_t *)arg; - connsock_t *cs = proxi->cs; - ckpool_t *ckp = proxi->ckp; - - rename_proc("proxyrecv"); - - while (42) { - notify_instance_t *ni, *tmp; - share_msg_t *share, *tmpshare; - int retries = 0, ret; - time_t now; - - now = time(NULL); - - /* Age old notifications older than 10 mins old */ - mutex_lock(&proxi->notify_lock); - HASH_ITER(hh, proxi->notify_instances, ni, tmp) { - if (HASH_COUNT(proxi->notify_instances) < 3) - break; - if (ni->notify_time < now - 600) { - HASH_DEL(proxi->notify_instances, ni); - clear_notify(ni); - } - } - mutex_unlock(&proxi->notify_lock); - - /* Similary with shares older than 2 mins without response */ - mutex_lock(&proxi->share_lock); - HASH_ITER(hh, proxi->shares, share, tmpshare) { - if (share->submit_time < now - 120) { - HASH_DEL(proxi->shares, share); - } - } - mutex_unlock(&proxi->share_lock); - - /* If we don't get an update within 10 minutes the upstream pool - * has likely stopped responding. */ - do { - if (cs->fd == -1) { - ret = -1; - break; - } - ret = read_socket_line(cs, 5); - } while (ret == 0 && ++retries < 120); - - if (ret < 1) { - /* Send ourselves a reconnect message */ - LOGWARNING("Failed to read_socket_line in proxy_recv, attempting reconnect"); - send_proc(ckp->generator, "reconnect"); - break; - } - if (parse_method(proxi, cs->buf)) { - if (proxi->notified) { - send_proc(ckp->stratifier, "notify"); - proxi->notified = false; - } - if (proxi->diffed) { - send_proc(ckp->stratifier, "diff"); - proxi->diffed = false; - } - if (proxi->reconnect) { - proxi->reconnect = false; - LOGWARNING("Reconnect issue, dropping existing connection"); - send_proc(ckp->generator, "reconnect"); - break; - } - continue; - } - if (parse_share(proxi, cs->buf)) { - continue; - } - /* If it's not a method it should be a share result */ - LOGWARNING("Unhandled stratum message: %s", cs->buf); - } - return NULL; -} - /* For processing and sending shares */ static void *proxy_send(void *arg) { @@ -1384,36 +1313,6 @@ static void *proxy_send(void *arg) return NULL; } -/* For receiving messages from an upstream pool to pass downstream */ -static void *passthrough_recv(void *arg) -{ - proxy_instance_t *proxi = (proxy_instance_t *)arg; - connsock_t *cs = proxi->cs; - ckpool_t *ckp = proxi->ckp; - - rename_proc("passrecv"); - - while (42) { - int ret; - - do { - ret = read_socket_line(cs, 60); - } while (ret == 0); - - if (ret < 1) { - /* Send ourselves a reconnect message */ - LOGWARNING("Failed to read_socket_line in proxy_recv, attempting reconnect"); - send_proc(ckp->generator, "reconnect"); - break; - } - /* Simply forward the message on, as is, to the connector to - * process. Possibly parse parameters sent by upstream pool - * here */ - send_proc(ckp->connector, cs->buf); - } - return NULL; -} - static void passthrough_send(ckpool_t __maybe_unused *ckp, pass_msg_t *pm) { int len, sent; @@ -1491,124 +1390,238 @@ out: return ret; } -/* Cycle through the available proxies and find the first alive one */ -static proxy_instance_t *live_proxy(ckpool_t *ckp) +/* For receiving messages from an upstream pool to pass downstream. Responsible + * for setting up the connection and testing pool is live. */ +static void *passthrough_recv(void *arg) { - proxy_instance_t *alive = NULL; - gdata_t *gdata = ckp->data; - connsock_t *cs; - int i, srvs; + proxy_instance_t *proxi = (proxy_instance_t *)arg; + server_instance_t *si = proxi->si; + connsock_t *cs = proxi->cs; + ckpool_t *ckp = proxi->ckp; - LOGDEBUG("Attempting to connect to proxy"); -retry: - if (!ping_main(ckp)) - goto out; + rename_proc("passrecv"); - mutex_lock(&gdata->lock); - srvs = ckp->proxies; - mutex_unlock(&gdata->lock); + if (proxy_alive(ckp, si, proxi, cs, false)) { + proxi->alive = true; + send_proc(ckp->generator, "reconnect"); + LOGWARNING("Proxy %d:%s connection established", + proxi->id, proxi->si->url); + } - for (i = 0; i < srvs; i++) { - proxy_instance_t *proxi; - server_instance_t *si; + while (42) { + int ret; - mutex_lock(&gdata->lock); - si = ckp->servers[i]; - proxi = si->data; - mutex_unlock(&gdata->lock); + while (!proxy_alive(ckp, si, proxi, cs, true)) { + if (proxi->alive) { + proxi->alive = false; + send_proc(ckp->generator, "reconnect"); + } + sleep(5); + } + if (!proxi->alive) { + proxi->alive = true; + send_proc(ckp->generator, "reconnect"); + } - cs = proxi->cs; - if (proxy_alive(ckp, si, proxi, cs, false)) { - alive = proxi; - break; + do { + ret = read_socket_line(cs, 60); + } while (ret == 0); + + if (ret < 1) { + LOGWARNING("Failed to read_socket_line in proxy_recv, attempting reconnect"); + continue; } + /* Simply forward the message on, as is, to the connector to + * process. Possibly parse parameters sent by upstream pool + * here */ + send_proc(ckp->connector, cs->buf); } - if (!alive) { - send_proc(ckp->connector, "reject"); - send_proc(ckp->stratifier, "dropall"); - LOGWARNING("Failed to connect to any servers as proxy, retrying in 5s!"); - sleep(5); - goto retry; + return NULL; +} + +/* For receiving messages from the upstream proxy, also responsible for setting + * up the connection and testing it's alive. */ +static void *proxy_recv(void *arg) +{ + proxy_instance_t *proxi = (proxy_instance_t *)arg; + server_instance_t *si = proxi->si; + connsock_t *cs = proxi->cs; + ckpool_t *ckp = proxi->ckp; + + rename_proc("proxyrecv"); + + if (proxy_alive(ckp, si, proxi, cs, false)) { + proxi->alive = true; + send_proc(ckp->generator, "reconnect"); + LOGWARNING("Proxy %d:%s connection established", + proxi->id, proxi->si->url); } - cs = alive->cs; - LOGNOTICE("Connected to upstream server %s:%s as proxy%s", cs->url, cs->port, - ckp->passthrough ? " in passthrough mode" : ""); - if (ckp->passthrough) { - create_pthread(&alive->pth_precv, passthrough_recv, alive); - alive->passsends = create_ckmsgq(ckp, "passsend", &passthrough_send); - } else { - create_pthread(&alive->pth_precv, proxy_recv, alive); - mutex_init(&alive->psend_lock); - cond_init(&alive->psend_cond); - create_pthread(&alive->pth_psend, proxy_send, alive); + while (42) { + notify_instance_t *ni, *tmp; + share_msg_t *share, *tmpshare; + int retries = 0, ret; + time_t now; + + while (!proxy_alive(ckp, si, proxi, cs, true)) { + if (proxi->alive) { + proxi->alive = false; + send_proc(ckp->generator, "reconnect"); + } + sleep(5); + } + if (!proxi->alive) { + proxi->alive = true; + send_proc(ckp->generator, "reconnect"); + } + + now = time(NULL); + + /* Age old notifications older than 10 mins old */ + mutex_lock(&proxi->notify_lock); + HASH_ITER(hh, proxi->notify_instances, ni, tmp) { + if (HASH_COUNT(proxi->notify_instances) < 3) + break; + if (ni->notify_time < now - 600) { + HASH_DEL(proxi->notify_instances, ni); + clear_notify(ni); + } + } + mutex_unlock(&proxi->notify_lock); + + /* Similary with shares older than 2 mins without response */ + mutex_lock(&proxi->share_lock); + HASH_ITER(hh, proxi->shares, share, tmpshare) { + if (share->submit_time < now - 120) { + HASH_DEL(proxi->shares, share); + } + } + mutex_unlock(&proxi->share_lock); + + /* If we don't get an update within 10 minutes the upstream pool + * has likely stopped responding. */ + do { + if (cs->fd == -1) { + ret = -1; + break; + } + ret = read_socket_line(cs, 5); + } while (ret == 0 && ++retries < 120); + + if (ret < 1) { + /* Send ourselves a reconnect message */ + LOGWARNING("Failed to read_socket_line in proxy_recv, attempting reconnect"); + continue; + } + if (parse_method(proxi, cs->buf)) { + if (proxi->notified) { + send_proc(ckp->stratifier, "notify"); + proxi->notified = false; + } + if (proxi->diffed) { + send_proc(ckp->stratifier, "diff"); + proxi->diffed = false; + } + if (proxi->reconnect) { + proxi->reconnect = false; + LOGWARNING("Proxy %d:%s reconnect issue, dropping existing connection", + proxi->id, proxi->si->url); + send_proc(ckp->generator, "reconnect"); + break; + } + continue; + } + if (parse_share(proxi, cs->buf)) { + continue; + } + /* If it's not a method it should be a share result */ + LOGWARNING("Unhandled stratum message: %s", cs->buf); } -out: - send_proc(ckp->connector, alive ? "accept" : "reject"); - return alive; + return NULL; } -static void kill_proxy(proxy_instance_t *proxi) +static void prepare_proxy(proxy_instance_t *proxi) { - notify_instance_t *ni, *tmp; - connsock_t *cs; + mutex_init(&proxi->psend_lock); + cond_init(&proxi->psend_cond); + create_pthread(&proxi->pth_psend, proxy_send, proxi); + create_pthread(&proxi->pth_precv, proxy_recv, proxi); +} - if (!proxi) // This shouldn't happen - return; +static void setup_proxies(ckpool_t *ckp, gdata_t *gdata) +{ + int i; - LOGNOTICE("Killing proxy connection to %s", proxi->si->url); - cs = proxi->cs; - Close(cs->fd); - empty_buffer(cs); + for (i = 0; i < ckp->proxies; i++) { + proxy_instance_t *proxi; + server_instance_t *si; - /* All our notify data is invalid if we reconnect so discard them */ - mutex_lock(&proxi->notify_lock); - HASH_ITER(hh, proxi->notify_instances, ni, tmp) { - HASH_DEL(proxi->notify_instances, ni); - clear_notify(ni); + si = ckp->servers[i]; + proxi = si->data; + proxi->id = i; + HASH_ADD_INT(gdata->proxies, id, proxi); + if (ckp->passthrough) { + create_pthread(&proxi->pth_precv, passthrough_recv, proxi); + proxi->passsends = create_ckmsgq(ckp, "passsend", &passthrough_send); + } else { + prepare_proxy(proxi); + } } - mutex_unlock(&proxi->notify_lock); +} + +static proxy_instance_t *current_proxy(ckpool_t *ckp, gdata_t *gdata) +{ + proxy_instance_t *ret = NULL, *proxi, *tmp; - pthread_cancel(proxi->pth_precv); - pthread_cancel(proxi->pth_psend); + while (42) { + if (!ping_main(ckp)) + break; + HASH_ITER(hh, gdata->proxies, proxi, tmp) { + if (proxi->alive) { + if (!ret) { + ret = proxi; + continue; + } + if (proxi->id < ret->id) + ret = proxi; + } + } + if (ret) + break; + sleep(1); + } + send_proc(ckp->connector, ret ? "accept" : "reject"); + return ret; } static int proxy_loop(proc_instance_t *pi) { + proxy_instance_t *proxi = NULL, *cproxy; int sockd = -1, ret = 0, selret; - proxy_instance_t *proxi = NULL; - bool reconnecting = false; unixsock_t *us = &pi->us; ckpool_t *ckp = pi->ckp; gdata_t *gdata = ckp->data; char *buf = NULL; + setup_proxies(ckp, gdata); reconnect: - if (proxi) { - kill_proxy(proxi); - reconnecting = true; - } - proxi = live_proxy(ckp); - if (!proxi) + /* This does not necessarily mean we reconnect, but a change has + * occurred and we need to reexamine the proxies. */ + cproxy = current_proxy(ckp, gdata); + if (!cproxy) goto out; - if (reconnecting) { - reconnecting = false; - connsock_t *cs = proxi->cs; - LOGWARNING("Successfully reconnected to %s:%s as proxy", - cs->url, cs->port); - } - - /* We've just subscribed and authorised so tell the stratifier to - * retrieve the first subscription. */ - if (!ckp->passthrough) { - send_proc(ckp->stratifier, "subscribe"); - send_proc(ckp->stratifier, "notify"); - proxi->notified = false; + if (proxi != cproxy) { + proxi = cproxy; + if (!ckp->passthrough) { + connsock_t *cs = proxi->cs; + LOGWARNING("Successfully connected to %s:%s as proxy", + cs->url, cs->port); + /* Sending subscribe implies stratifier will also do a notify */ + send_proc(ckp->stratifier, "subscribe"); + proxi->notified = false; + } } - retry: - ckmsgq_add(gdata->srvchk, proxi->si); - do { selret = wait_read_select(us->sockd, 5); if (!selret && !ping_main(ckp)) { @@ -1670,7 +1683,6 @@ retry: Close(sockd); goto retry; out: - kill_proxy(proxi); Close(sockd); return ret; } @@ -1793,53 +1805,6 @@ static void server_watchdog(ckpool_t *ckp, server_instance_t *cursi) send_proc(ckp->generator, "reconnect"); } -static void proxy_watchdog(ckpool_t *ckp, server_instance_t *cursi) -{ - gdata_t *gdata = ckp->data; - static time_t last_t = 0; - bool alive = false; - time_t now_t; - int i, srvs; - - /* Rate limit to checking only once every 5 seconds */ - now_t = time(NULL); - if (now_t <= last_t + 5) - return; - - last_t = now_t; - - /* Is this the highest priority server already? */ - if (!cursi->id) - return; - - mutex_lock(&gdata->lock); - srvs = ckp->proxies; - mutex_unlock(&gdata->lock); - - for (i = 0; i < srvs; i++) { - proxy_instance_t *proxi; - server_instance_t *si; - connsock_t *cs; - - mutex_lock(&gdata->lock); - si = ckp->servers[i]; - proxi = si->data; - mutex_unlock(&gdata->lock); - - /* Have we reached the current server? */ - if (si == cursi) - return; - - cs = proxi->cs; - alive = proxy_alive(ckp, si, proxi, cs, true); - if (alive) - break; - } - if (alive) - send_proc(ckp->generator, "reconnect"); -} - - int generator(proc_instance_t *pi) { ckpool_t *ckp = pi->ckp; @@ -1850,7 +1815,6 @@ int generator(proc_instance_t *pi) gdata = ckzalloc(sizeof(gdata_t)); ckp->data = gdata; if (ckp->proxy) { - gdata->srvchk = create_ckmsgq(ckp, "prxchk", &proxy_watchdog); ret = proxy_mode(ckp, pi); } else { gdata->srvchk = create_ckmsgq(ckp, "srvchk", &server_watchdog); diff --git a/src/stratifier.c b/src/stratifier.c index dba09bc2..b0b0ebc1 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -1006,6 +1006,7 @@ static proxy_t *proxy_by_id(sdata_t *sdata, const int id) return proxy; } +static void update_notify(ckpool_t *ckp); static void reconnect_clients(sdata_t *sdata, const char *cmd); static void update_subscribe(ckpool_t *ckp) @@ -1068,6 +1069,8 @@ static void update_subscribe(ckpool_t *ckp) proxy->nonce2len, 1ll << (proxy->enonce1varlen * 8)); json_decref(val); + /* Notify implied required now too */ + update_notify(ckp); reconnect_clients(sdata, ""); } From d474047d42560b013f18ff62ae95ecea2855beb7 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Fri, 6 Feb 2015 18:14:08 +1100 Subject: [PATCH 005/544] Drop proxy reference when socket invalidated --- src/generator.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/generator.c b/src/generator.c index d932826e..a50a40ae 100644 --- a/src/generator.c +++ b/src/generator.c @@ -1633,6 +1633,7 @@ retry: if (unlikely(proxi->cs->fd < 0)) { LOGWARNING("Upstream socket invalidated, will attempt failover"); + proxi = NULL; goto reconnect; } From f02a35dfcfff8e562541f4bba598e1983444b52b Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Fri, 6 Feb 2015 18:20:18 +1100 Subject: [PATCH 006/544] Send notready message when notify data is not ready in the generator instead of no response --- src/generator.c | 5 +++-- src/stratifier.c | 4 ++++ 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/src/generator.c b/src/generator.c index a50a40ae..75e55dff 100644 --- a/src/generator.c +++ b/src/generator.c @@ -1145,7 +1145,8 @@ static void send_notify(proxy_instance_t *proxi, int *sockd) ni = proxi->current_notify; if (unlikely(!ni)) { mutex_unlock(&proxi->notify_lock); - goto out_close; + ASPRINTF(&msg, "notready"); + goto out_send; } for (i = 0; i < ni->merkles; i++) json_array_append_new(merkle_arr, json_string(&ni->merklehash[i][0])); @@ -1160,9 +1161,9 @@ static void send_notify(proxy_instance_t *proxi, int *sockd) msg = json_dumps(json_msg, JSON_NO_UTF8); json_decref(json_msg); +out_send: send_unix_msg(*sockd, msg); free(msg); -out_close: _Close(sockd); } diff --git a/src/stratifier.c b/src/stratifier.c index b0b0ebc1..bd194998 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -1103,6 +1103,10 @@ static void update_notify(ckpool_t *ckp) LOGWARNING("Failed to get notify from generator in update_notify"); return; } + if (unlikely(!safecmp(buf, "notready"))) { + LOGNOTICE("Generator not ready to send notify to stratifier"); + return; + } proxy = current_proxy(sdata); if (unlikely(!proxy || !proxy->subscribed)) { From 520b36cc707b1c93fc93dfd30751e48467e80b37 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Fri, 6 Feb 2015 19:17:45 +1100 Subject: [PATCH 007/544] Send which proxy we wish to get a subscribe or notify to/from the connector to stratifier and whether to reconnect --- src/ckpool.c | 2 +- src/ckpool.h | 1 + src/generator.c | 46 +++++++++++++++++++++++++++++++------- src/stratifier.c | 58 ++++++++++++++++++++++++++++++++---------------- 4 files changed, 79 insertions(+), 28 deletions(-) diff --git a/src/ckpool.c b/src/ckpool.c index cdbfbe39..7f2f0ff5 100644 --- a/src/ckpool.c +++ b/src/ckpool.c @@ -1024,7 +1024,7 @@ out: return ret; } -static bool json_get_bool(bool *store, const json_t *val, const char *res) +bool json_get_bool(bool *store, const json_t *val, const char *res) { json_t *entry = json_object_get(val, res); bool ret = false; diff --git a/src/ckpool.h b/src/ckpool.h index b2f59333..f2d8bb3b 100644 --- a/src/ckpool.h +++ b/src/ckpool.h @@ -229,5 +229,6 @@ bool json_get_string(char **store, const json_t *val, const char *res); bool json_get_int64(int64_t *store, const json_t *val, const char *res); bool json_get_int(int *store, const json_t *val, const char *res); bool json_get_double(double *store, const json_t *val, const char *res); +bool json_get_bool(bool *store, const json_t *val, const char *res); #endif /* CKPOOL_H */ diff --git a/src/generator.c b/src/generator.c index 75e55dff..53908353 100644 --- a/src/generator.c +++ b/src/generator.c @@ -1116,29 +1116,57 @@ out: return ret; } -static void send_subscribe(proxy_instance_t *proxi, int *sockd) +static proxy_instance_t *proxy_by_id(gdata_t *gdata, const int id) { + proxy_instance_t *proxi; + + mutex_lock(&gdata->lock); + HASH_FIND_INT(gdata->proxies, &id, proxi); + mutex_unlock(&gdata->lock); + + return proxi; +} + +static void send_subscribe(gdata_t *gdata, proxy_instance_t *cproxy, int *sockd, const char *buf) +{ + proxy_instance_t *proxi; json_t *json_msg; + int id = 0; char *msg; - JSON_CPACK(json_msg, "{sisssi}", + sscanf(buf, "getsubscribe=%d", &id); + proxi = proxy_by_id(gdata, id); + if (unlikely(!proxi || !proxi->nonce2len)) { + ASPRINTF(&msg, "notready"); + goto out_send; + } + JSON_CPACK(json_msg, "{sisssisb}", "proxy", proxi->id, "enonce1", proxi->enonce1, - "nonce2len", proxi->nonce2len); + "nonce2len", proxi->nonce2len, + "reconnect", proxi == cproxy ? true : false); msg = json_dumps(json_msg, JSON_NO_UTF8); json_decref(json_msg); +out_send: send_unix_msg(*sockd, msg); free(msg); _Close(sockd); } -static void send_notify(proxy_instance_t *proxi, int *sockd) +static void send_notify(gdata_t *gdata, int *sockd, const char *buf) { json_t *json_msg, *merkle_arr; + proxy_instance_t *proxi; notify_instance_t *ni; + int i, id = 0; char *msg; - int i; + sscanf(buf, "getnotify=%d", &id); + proxi = proxy_by_id(gdata, id); + if (unlikely(!proxi)) { + ASPRINTF(&msg, "notready"); + goto out_send; + } merkle_arr = json_array(); mutex_lock(&proxi->notify_lock); @@ -1618,7 +1646,9 @@ reconnect: LOGWARNING("Successfully connected to %s:%s as proxy", cs->url, cs->port); /* Sending subscribe implies stratifier will also do a notify */ - send_proc(ckp->stratifier, "subscribe"); + dealloc(buf); + ASPRINTF(&buf, "subscribe=%d", proxi->id); + send_proc(ckp->stratifier, buf); proxi->notified = false; } } @@ -1656,9 +1686,9 @@ retry: ret = 0; goto out; } else if (cmdmatch(buf, "getsubscribe")) { - send_subscribe(proxi, &sockd); + send_subscribe(gdata, proxi, &sockd, buf); } else if (cmdmatch(buf, "getnotify")) { - send_notify(proxi, &sockd); + send_notify(gdata, &sockd, buf); } else if (cmdmatch(buf, "getdiff")) { send_diff(proxi, &sockd); } else if (cmdmatch(buf, "reconnect")) { diff --git a/src/stratifier.c b/src/stratifier.c index bd194998..199895b8 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -1006,24 +1006,32 @@ static proxy_t *proxy_by_id(sdata_t *sdata, const int id) return proxy; } -static void update_notify(ckpool_t *ckp); static void reconnect_clients(sdata_t *sdata, const char *cmd); +static void _update_notify(ckpool_t *ckp, const int id); -static void update_subscribe(ckpool_t *ckp) +static void update_subscribe(ckpool_t *ckp, const char *cmd) { sdata_t *sdata = ckp->data; + bool reconnect = true; + char *buf, *msg; proxy_t *proxy; json_t *val; int id = 0; - char *buf; - buf = send_recv_proc(ckp->generator, "getsubscribe"); + sscanf(cmd, "subscribe=%d", &id); + ASPRINTF(&msg, "getsubscribe=%d", id); + buf = send_recv_proc(ckp->generator, msg); + dealloc(msg); if (unlikely(!buf)) { LOGWARNING("Failed to get subscribe from generator in update_subscribe"); drop_allclients(ckp); return; } LOGDEBUG("Update subscribe: %s", buf); + if (unlikely(!safecmp(buf, "notready"))) { + LOGNOTICE("Generator not ready to send subscribe for proxy %d", id); + return; + } val = json_loads(buf, 0, NULL); free(buf); if (unlikely(!val)) { @@ -1031,8 +1039,8 @@ static void update_subscribe(ckpool_t *ckp) return; } - json_get_int(&id, val, "proxy"); proxy = proxy_by_id(sdata, id); + json_get_bool(&reconnect, val, "reconnect"); mutex_lock(&sdata->proxy_lock); if (sdata->proxy != proxy) @@ -1070,12 +1078,13 @@ static void update_subscribe(ckpool_t *ckp) json_decref(val); /* Notify implied required now too */ - update_notify(ckp); - reconnect_clients(sdata, ""); + _update_notify(ckp, id); + if (reconnect) + reconnect_clients(sdata, ""); } static void update_diff(ckpool_t *ckp); - +#if 0 static proxy_t *current_proxy(sdata_t *sdata) { proxy_t *proxy; @@ -1086,19 +1095,28 @@ static proxy_t *current_proxy(sdata_t *sdata) return proxy; } +#endif -static void update_notify(ckpool_t *ckp) +static void _update_notify(ckpool_t *ckp, const int id) { bool new_block = false, clean; sdata_t *sdata = ckp->data; char header[228]; + char *buf, *msg; proxy_t *proxy; workbase_t *wb; json_t *val; - char *buf; int i; - buf = send_recv_proc(ckp->generator, "getnotify"); + proxy = proxy_by_id(sdata, id); + if (unlikely(!proxy->subscribed)) { + LOGINFO("No valid proxy subscription to update notify yet"); + return; + } + + ASPRINTF(&msg, "getnotify=%d", id); + buf = send_recv_proc(ckp->generator, msg); + dealloc(msg); if (unlikely(!buf)) { LOGWARNING("Failed to get notify from generator in update_notify"); return; @@ -1108,12 +1126,6 @@ static void update_notify(ckpool_t *ckp) return; } - proxy = current_proxy(sdata); - if (unlikely(!proxy || !proxy->subscribed)) { - LOGINFO("No valid proxy subscription to update notify yet"); - return; - } - LOGDEBUG("Update notify: %s", buf); wb = ckzalloc(sizeof(workbase_t)); val = json_loads(buf, 0, NULL); @@ -1173,6 +1185,14 @@ static void update_notify(ckpool_t *ckp) stratum_broadcast_update(sdata, new_block | clean); } +static void update_notify(ckpool_t *ckp, const char *cmd) +{ + int id = 0; + + sscanf(cmd, "notify=%d", &id); + _update_notify(ckp, id); +} + static void stratum_send_diff(sdata_t *sdata, const stratum_instance_t *client); static void update_diff(ckpool_t *ckp) @@ -1866,10 +1886,10 @@ retry: update_base(ckp, GEN_PRIORITY); } else if (cmdmatch(buf, "subscribe")) { /* Proxifier has a new subscription */ - update_subscribe(ckp); + update_subscribe(ckp, buf); } else if (cmdmatch(buf, "notify")) { /* Proxifier has a new notify ready */ - update_notify(ckp); + update_notify(ckp, buf); } else if (cmdmatch(buf, "diff")) { update_diff(ckp); } else if (cmdmatch(buf, "dropclient")) { From d02862d1fe3c9de82b9aa0ddff2d08b069cf5370 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Fri, 6 Feb 2015 20:24:53 +1100 Subject: [PATCH 008/544] Update subscriptions and notifications for proxies that aren't the current proxy for faster switching --- src/generator.c | 29 ++++++++++++++++++++++++++++- src/stratifier.c | 5 ++++- 2 files changed, 32 insertions(+), 2 deletions(-) diff --git a/src/generator.c b/src/generator.c index 53908353..afc295e6 100644 --- a/src/generator.c +++ b/src/generator.c @@ -129,6 +129,7 @@ struct proxy_instance { struct generator_data { pthread_mutex_t lock; /* Lock protecting linked lists */ proxy_instance_t *proxies; /* Hash list of all proxies */ + proxy_instance_t *proxy; /* Current proxy */ int proxy_notify_id; // Globally increasing notify id ckmsgq_t *srvchk; // Server check message queue }; @@ -1366,6 +1367,18 @@ static void passthrough_add_send(proxy_instance_t *proxi, const char *msg) ckmsgq_add(proxi->passsends, pm); } +static int current_proxy_id(gdata_t *gdata) +{ + int ret = 0; + + mutex_lock(&gdata->lock); + if (gdata->proxy) + ret = gdata->proxy->id; + mutex_unlock(&gdata->lock); + + return ret; +} + static bool proxy_alive(ckpool_t *ckp, server_instance_t *si, proxy_instance_t *proxi, connsock_t *cs, bool pinging) { @@ -1414,8 +1427,19 @@ out: if (!ret) { /* Close and invalidate the file handle */ Close(cs->fd); - } else + } else { keep_sockalive(cs->fd); + /* If this isn't a new higher priority proxy we won't be + * issuing a reconnect so tell the stratifier to get the + * subscription and notification data without reconnecting. */ + if (proxi->id > current_proxy_id(ckp->data)) { + char *msg; + + ASPRINTF(&msg, "subscribe=%d", proxi->id); + send_proc(ckp->stratifier, msg); + free(msg); + } + } return ret; } @@ -1605,6 +1629,7 @@ static proxy_instance_t *current_proxy(ckpool_t *ckp, gdata_t *gdata) while (42) { if (!ping_main(ckp)) break; + mutex_lock(&gdata->lock); HASH_ITER(hh, gdata->proxies, proxi, tmp) { if (proxi->alive) { if (!ret) { @@ -1615,6 +1640,8 @@ static proxy_instance_t *current_proxy(ckpool_t *ckp, gdata_t *gdata) ret = proxi; } } + gdata->proxy = ret; + mutex_unlock(&gdata->lock); if (ret) break; sleep(1); diff --git a/src/stratifier.c b/src/stratifier.c index 199895b8..b60ec695 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -1039,6 +1039,8 @@ static void update_subscribe(ckpool_t *ckp, const char *cmd) return; } + LOGNOTICE("Got updated subscribe for proxy %d", id); + proxy = proxy_by_id(sdata, id); json_get_bool(&reconnect, val, "reconnect"); @@ -1121,12 +1123,13 @@ static void _update_notify(ckpool_t *ckp, const int id) LOGWARNING("Failed to get notify from generator in update_notify"); return; } + LOGDEBUG("Update notify: %s", buf); if (unlikely(!safecmp(buf, "notready"))) { LOGNOTICE("Generator not ready to send notify to stratifier"); return; } - LOGDEBUG("Update notify: %s", buf); + LOGINFO("Got updated notify for proxy %d", id); wb = ckzalloc(sizeof(workbase_t)); val = json_loads(buf, 0, NULL); dealloc(buf); From fea1ea29531cf29c900a2ea8a7c9a8660ed5eb9a Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Fri, 6 Feb 2015 20:33:56 +1100 Subject: [PATCH 009/544] Always update the subscription and notification details for every proxy updated, sending a reconnect instead when the proxy changes --- src/generator.c | 33 ++++++--------------------------- 1 file changed, 6 insertions(+), 27 deletions(-) diff --git a/src/generator.c b/src/generator.c index afc295e6..608771b2 100644 --- a/src/generator.c +++ b/src/generator.c @@ -1367,18 +1367,6 @@ static void passthrough_add_send(proxy_instance_t *proxi, const char *msg) ckmsgq_add(proxi->passsends, pm); } -static int current_proxy_id(gdata_t *gdata) -{ - int ret = 0; - - mutex_lock(&gdata->lock); - if (gdata->proxy) - ret = gdata->proxy->id; - mutex_unlock(&gdata->lock); - - return ret; -} - static bool proxy_alive(ckpool_t *ckp, server_instance_t *si, proxy_instance_t *proxi, connsock_t *cs, bool pinging) { @@ -1428,17 +1416,12 @@ out: /* Close and invalidate the file handle */ Close(cs->fd); } else { + char msg[128]; + keep_sockalive(cs->fd); - /* If this isn't a new higher priority proxy we won't be - * issuing a reconnect so tell the stratifier to get the - * subscription and notification data without reconnecting. */ - if (proxi->id > current_proxy_id(ckp->data)) { - char *msg; - - ASPRINTF(&msg, "subscribe=%d", proxi->id); - send_proc(ckp->stratifier, msg); - free(msg); - } + snprintf(msg, 127, "subscribe=%d", proxi->id); + send_proc(ckp->stratifier, msg); + proxi->notified = false; } return ret; } @@ -1672,11 +1655,7 @@ reconnect: connsock_t *cs = proxi->cs; LOGWARNING("Successfully connected to %s:%s as proxy", cs->url, cs->port); - /* Sending subscribe implies stratifier will also do a notify */ - dealloc(buf); - ASPRINTF(&buf, "subscribe=%d", proxi->id); - send_proc(ckp->stratifier, buf); - proxi->notified = false; + send_proc(ckp->stratifier, "reconnect"); } } retry: From 16fbfee0681e3707c73fcf4d737b4648901182fa Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Fri, 6 Feb 2015 20:52:40 +1100 Subject: [PATCH 010/544] Wiat 90 seconds before potentially failing over to a pool that has just recovered --- src/generator.c | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/src/generator.c b/src/generator.c index 608771b2..73a73194 100644 --- a/src/generator.c +++ b/src/generator.c @@ -123,6 +123,8 @@ struct proxy_instance { ckmsgq_t *passsends; // passthrough sends char_entry_t *recvd_lines; /* Linked list of unprocessed messages */ + + time_t reconnect_time; }; /* Private data for the generator */ @@ -1475,6 +1477,8 @@ static void *passthrough_recv(void *arg) return NULL; } +static proxy_instance_t *current_proxy(ckpool_t *ckp, gdata_t *gdata); + /* For receiving messages from the upstream proxy, also responsible for setting * up the connection and testing it's alive. */ static void *proxy_recv(void *arg) @@ -1497,6 +1501,7 @@ static void *proxy_recv(void *arg) notify_instance_t *ni, *tmp; share_msg_t *share, *tmpshare; int retries = 0, ret; + char buf[128]; time_t now; while (!proxy_alive(ckp, si, proxi, cs, true)) { @@ -1505,9 +1510,14 @@ static void *proxy_recv(void *arg) send_proc(ckp->generator, "reconnect"); } sleep(5); + proxi->reconnect_time = time(NULL); } - if (!proxi->alive) { + /* Wait 90 seconds before declaring this upstream pool alive + * to prevent switching to unstable pools. */ + if (!proxi->alive && (!current_proxy(ckp, ckp->data) || + time(NULL) - proxi->reconnect_time > 90)) { proxi->alive = true; + proxi->reconnect_time = 0; send_proc(ckp->generator, "reconnect"); } @@ -1551,7 +1561,8 @@ static void *proxy_recv(void *arg) } if (parse_method(proxi, cs->buf)) { if (proxi->notified) { - send_proc(ckp->stratifier, "notify"); + snprintf(buf, 127, "notify=%d", proxi->id); + send_proc(ckp->stratifier, buf); proxi->notified = false; } if (proxi->diffed) { @@ -1612,6 +1623,7 @@ static proxy_instance_t *current_proxy(ckpool_t *ckp, gdata_t *gdata) while (42) { if (!ping_main(ckp)) break; + mutex_lock(&gdata->lock); HASH_ITER(hh, gdata->proxies, proxi, tmp) { if (proxi->alive) { @@ -1625,6 +1637,7 @@ static proxy_instance_t *current_proxy(ckpool_t *ckp, gdata_t *gdata) } gdata->proxy = ret; mutex_unlock(&gdata->lock); + if (ret) break; sleep(1); From d8d9388b6c7327a0c2cb152c7cd0863d729852df Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Fri, 6 Feb 2015 20:55:08 +1100 Subject: [PATCH 011/544] Add more information and demote messages for proxy send fails --- src/generator.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/generator.c b/src/generator.c index 73a73194..a265b41e 100644 --- a/src/generator.c +++ b/src/generator.c @@ -1334,11 +1334,13 @@ static void *proxy_send(void *arg) ret = send_json_msg(cs, val); json_decref(val); } else - LOGWARNING("Failed to find matching jobid in proxysend"); + LOGNOTICE("Proxy %d:%s failed to find matching jobid in proxysend", + proxi->id, proxi->si->url); json_decref(msg->json_msg); free(msg); if (!ret && cs->fd > 0) { - LOGWARNING("Failed to send msg in proxy_send, dropping to reconnect"); + LOGWARNING("Proxy %d:%s failed to send msg in proxy_send, dropping to reconnect", + proxi->id, proxi->si->url); Close(cs->fd); } } From 288e4b67bba2f6c6e45d3184acf44a2a44da0d8c Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Fri, 6 Feb 2015 21:07:47 +1100 Subject: [PATCH 012/544] Demote various messages in proxy mode but add more information to warnings --- src/ckpool.c | 13 ++++++++++--- src/ckpool.h | 2 +- src/generator.c | 8 ++++++-- 3 files changed, 17 insertions(+), 6 deletions(-) diff --git a/src/ckpool.c b/src/ckpool.c index 7f2f0ff5..3d122a61 100644 --- a/src/ckpool.c +++ b/src/ckpool.c @@ -424,8 +424,12 @@ int read_socket_line(connsock_t *cs, const int timeout) if (ret < 1) { if (!ret) LOGDEBUG("Select timed out in read_socket_line"); - else - LOGERR("Select failed in read_socket_line"); + else { + if (cs->ckp->proxy) + LOGNOTICE("Select failed in read_socket_line"); + else + LOGERR("Select failed in read_socket_line"); + } goto out; } ret = recv(fd, readbuf, PAGESIZE - 4, 0); @@ -433,7 +437,10 @@ int read_socket_line(connsock_t *cs, const int timeout) /* Closed socket after valid message */ if (!ret && eom) break; - LOGERR("Failed to recv in read_socket_line"); + if (cs->ckp->proxy) + LOGNOTICE("Failed to recv in read_socket_line"); + else + LOGERR("Failed to recv in read_socket_line"); ret = -1; goto out; } diff --git a/src/ckpool.h b/src/ckpool.h index f2d8bb3b..3f6935ae 100644 --- a/src/ckpool.h +++ b/src/ckpool.h @@ -19,7 +19,6 @@ #include "libckpool.h" #include "uthash.h" -struct ckpool_instance; typedef struct ckpool_instance ckpool_t; struct ckmsg { @@ -61,6 +60,7 @@ struct connsock { char *buf; int bufofs; int buflen; + ckpool_t *ckp; }; typedef struct connsock connsock_t; diff --git a/src/generator.c b/src/generator.c index a265b41e..7d472fdf 100644 --- a/src/generator.c +++ b/src/generator.c @@ -952,6 +952,7 @@ static bool parse_reconnect(proxy_instance_t *proxi, json_t *val) newproxi->si = newsi; newproxi->ckp = ckp; newproxi->cs = &newsi->cs; + newproxi->cs->ckp = ckp; newproxi->id = newsi->id; HASH_ADD_INT(gdata->proxies, id, proxi); HASH_ADD_INT(gdata->proxies, id, newproxi); @@ -1468,7 +1469,8 @@ static void *passthrough_recv(void *arg) } while (ret == 0); if (ret < 1) { - LOGWARNING("Failed to read_socket_line in proxy_recv, attempting reconnect"); + LOGWARNING("Proxy %d:%s failed to read_socket_line in proxy_recv, attempting reconnect", + proxi->id, proxi->si->url); continue; } /* Simply forward the message on, as is, to the connector to @@ -1558,7 +1560,8 @@ static void *proxy_recv(void *arg) if (ret < 1) { /* Send ourselves a reconnect message */ - LOGWARNING("Failed to read_socket_line in proxy_recv, attempting reconnect"); + LOGWARNING("Proxy %d:%s failed to read_socket_line in proxy_recv, attempting reconnect", + proxi->id, proxi->si->url); continue; } if (parse_method(proxi, cs->buf)) { @@ -1791,6 +1794,7 @@ static int proxy_mode(ckpool_t *ckp, proc_instance_t *pi) proxi->si = si; proxi->ckp = ckp; proxi->cs = &si->cs; + proxi->cs->ckp = ckp; mutex_init(&proxi->notify_lock); mutex_init(&proxi->share_lock); } From 57e7916296f73c012ee594d2e3c8e7dad60faf44 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Fri, 6 Feb 2015 21:08:29 +1100 Subject: [PATCH 013/544] Demote unnecessary warning in stratifier to debug --- src/stratifier.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/stratifier.c b/src/stratifier.c index b60ec695..a6431251 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -941,7 +941,7 @@ static void connector_drop_client(ckpool_t *ckp, const int64_t id) { char buf[256]; - LOGWARNING("Stratifier requesting connector drop client %"PRId64, id); + LOGDEBUG("Stratifier requesting connector drop client %"PRId64, id); snprintf(buf, 255, "dropclient=%"PRId64, id); send_proc(ckp->connector, buf); } From 5a19bd0e4aab0c486e464474ffebe5f1af5d81c8 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Fri, 6 Feb 2015 21:19:21 +1100 Subject: [PATCH 014/544] Send a reconnect if we detect a changed subscription on the current proxy --- src/stratifier.c | 25 ++++++++++++------------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/src/stratifier.c b/src/stratifier.c index a6431251..b8639a32 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -1009,6 +1009,17 @@ static proxy_t *proxy_by_id(sdata_t *sdata, const int id) static void reconnect_clients(sdata_t *sdata, const char *cmd); static void _update_notify(ckpool_t *ckp, const int id); +static proxy_t *current_proxy(sdata_t *sdata) +{ + proxy_t *proxy; + + mutex_lock(&sdata->proxy_lock); + proxy = sdata->proxy; + mutex_unlock(&sdata->proxy_lock); + + return proxy; +} + static void update_subscribe(ckpool_t *ckp, const char *cmd) { sdata_t *sdata = ckp->data; @@ -1081,23 +1092,11 @@ static void update_subscribe(ckpool_t *ckp, const char *cmd) json_decref(val); /* Notify implied required now too */ _update_notify(ckp, id); - if (reconnect) + if (reconnect || proxy == current_proxy(sdata)) reconnect_clients(sdata, ""); } static void update_diff(ckpool_t *ckp); -#if 0 -static proxy_t *current_proxy(sdata_t *sdata) -{ - proxy_t *proxy; - - mutex_lock(&sdata->proxy_lock); - proxy = sdata->proxy; - mutex_unlock(&sdata->proxy_lock); - - return proxy; -} -#endif static void _update_notify(ckpool_t *ckp, const int id) { From 14e3ed70c856a574be9579ea9de21105ef94a93c Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Fri, 6 Feb 2015 21:35:48 +1100 Subject: [PATCH 015/544] Don't try to act on inactive clients --- src/stratifier.c | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/src/stratifier.c b/src/stratifier.c index b8639a32..bc1e6945 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -1434,6 +1434,11 @@ out: return ret; } +static inline bool client_active(stratum_instance_t *client) +{ + return (client->authorised && !client->dropped); +} + /* For creating a list of sends without locking that can then be concatenated * to the stratum_sends list. Minimises locking and avoids taking recursive * locks. */ @@ -1453,7 +1458,7 @@ static void stratum_broadcast(sdata_t *sdata, json_t *val) ckmsg_t *client_msg; smsg_t *msg; - if (!client->authorised) + if (!client_active(client)) continue; client_msg = ckalloc(sizeof(ckmsg_t)); msg = ckzalloc(sizeof(smsg_t)); @@ -3358,7 +3363,7 @@ static void suggest_diff(stratum_instance_t *client, const char *method, const j sdata_t *sdata = client->ckp->data; int64_t sdiff; - if (unlikely(!client->authorised)) { + if (unlikely(!client_active(client))) { LOGWARNING("Attempted to suggest diff on unauthorised client %"PRId64, client->id); return; } @@ -4057,7 +4062,7 @@ static void *statsupdate(void *arg) ck_rlock(&sdata->instance_lock); HASH_ITER(hh, sdata->stratum_instances, client, tmp) { - if (!client->authorised) + if (!client_active(client)) continue; per_tdiff = tvdiff(&now, &client->last_share); From cdc8efb0c4333180d631b8646fd65cfaf81102a5 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Fri, 6 Feb 2015 21:41:00 +1100 Subject: [PATCH 016/544] Add message when proxy recovers --- src/generator.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/generator.c b/src/generator.c index 7d472fdf..4dcf78ef 100644 --- a/src/generator.c +++ b/src/generator.c @@ -1520,6 +1520,7 @@ static void *proxy_recv(void *arg) * to prevent switching to unstable pools. */ if (!proxi->alive && (!current_proxy(ckp, ckp->data) || time(NULL) - proxi->reconnect_time > 90)) { + LOGWARNING("Proxy %d:%s recovered", proxi->id, proxi->si->url); proxi->alive = true; proxi->reconnect_time = 0; send_proc(ckp->generator, "reconnect"); From c2fe4f30e161c7ca3e53e0fc99e3c9a69ac28c68 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Fri, 6 Feb 2015 21:56:48 +1100 Subject: [PATCH 017/544] Set proxies to not alive when their sockets invalidate and close sockets on reconnect being issued --- src/generator.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/generator.c b/src/generator.c index 4dcf78ef..e5ba4b59 100644 --- a/src/generator.c +++ b/src/generator.c @@ -1471,6 +1471,7 @@ static void *passthrough_recv(void *arg) if (ret < 1) { LOGWARNING("Proxy %d:%s failed to read_socket_line in proxy_recv, attempting reconnect", proxi->id, proxi->si->url); + proxi->alive = false; continue; } /* Simply forward the message on, as is, to the connector to @@ -1576,9 +1577,11 @@ static void *proxy_recv(void *arg) proxi->diffed = false; } if (proxi->reconnect) { + proxi->alive = false; proxi->reconnect = false; LOGWARNING("Proxy %d:%s reconnect issue, dropping existing connection", proxi->id, proxi->si->url); + Close(cs->fd); send_proc(ckp->generator, "reconnect"); break; } @@ -1689,6 +1692,7 @@ retry: if (unlikely(proxi->cs->fd < 0)) { LOGWARNING("Upstream socket invalidated, will attempt failover"); + proxi->alive = false; proxi = NULL; goto reconnect; } From 32b88de36c661681aede890cb7383820742af652 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Fri, 6 Feb 2015 22:35:26 +1100 Subject: [PATCH 018/544] Explicitly set the proxy in the stratifier when we switch and avoid creating workbases from backup proxies --- src/generator.c | 8 ++++---- src/stratifier.c | 48 +++++++++++++++++++++++++++++++++++------------- 2 files changed, 39 insertions(+), 17 deletions(-) diff --git a/src/generator.c b/src/generator.c index e5ba4b59..c1286974 100644 --- a/src/generator.c +++ b/src/generator.c @@ -1472,6 +1472,7 @@ static void *passthrough_recv(void *arg) LOGWARNING("Proxy %d:%s failed to read_socket_line in proxy_recv, attempting reconnect", proxi->id, proxi->si->url); proxi->alive = false; + send_proc(ckp->generator, "reconnect"); continue; } /* Simply forward the message on, as is, to the connector to @@ -1561,7 +1562,6 @@ static void *proxy_recv(void *arg) } while (ret == 0 && ++retries < 120); if (ret < 1) { - /* Send ourselves a reconnect message */ LOGWARNING("Proxy %d:%s failed to read_socket_line in proxy_recv, attempting reconnect", proxi->id, proxi->si->url); continue; @@ -1577,12 +1577,10 @@ static void *proxy_recv(void *arg) proxi->diffed = false; } if (proxi->reconnect) { - proxi->alive = false; proxi->reconnect = false; LOGWARNING("Proxy %d:%s reconnect issue, dropping existing connection", proxi->id, proxi->si->url); Close(cs->fd); - send_proc(ckp->generator, "reconnect"); break; } continue; @@ -1677,7 +1675,9 @@ reconnect: connsock_t *cs = proxi->cs; LOGWARNING("Successfully connected to %s:%s as proxy", cs->url, cs->port); - send_proc(ckp->stratifier, "reconnect"); + dealloc(buf); + ASPRINTF(&buf, "proxy=%d", proxi->id); + send_proc(ckp->stratifier, buf); } } retry: diff --git a/src/stratifier.c b/src/stratifier.c index bc1e6945..d8a0df36 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -987,21 +987,26 @@ static proxy_t *__generate_proxy(sdata_t *sdata, const int id) } /* Find proxy by id number, generate one if none exist yet by that id */ -static proxy_t *proxy_by_id(sdata_t *sdata, const int id) +static proxy_t *__proxy_by_id(sdata_t *sdata, const int id) { - bool new_proxy = false; proxy_t *proxy; - mutex_lock(&sdata->proxy_lock); HASH_FIND_INT(sdata->proxies, &id, proxy); if (unlikely(!proxy)) { - new_proxy = true; proxy = __generate_proxy(sdata, id); + LOGINFO("Stratifier added new proxy %d", id); } - mutex_unlock(&sdata->proxy_lock); - if (unlikely(new_proxy)) - LOGINFO("Stratifier added new proxy %d", id); + return proxy; +} + +static proxy_t *proxy_by_id(sdata_t *sdata, const int id) +{ + proxy_t *proxy; + + mutex_lock(&sdata->proxy_lock); + proxy = __proxy_by_id(sdata, id); + mutex_unlock(&sdata->proxy_lock); return proxy; } @@ -1055,11 +1060,6 @@ static void update_subscribe(ckpool_t *ckp, const char *cmd) proxy = proxy_by_id(sdata, id); json_get_bool(&reconnect, val, "reconnect"); - mutex_lock(&sdata->proxy_lock); - if (sdata->proxy != proxy) - sdata->proxy = proxy; - mutex_unlock(&sdata->proxy_lock); - ck_wlock(&sdata->workbase_lock); proxy->subscribed = true; proxy->diff = ckp->startdiff; @@ -1092,7 +1092,7 @@ static void update_subscribe(ckpool_t *ckp, const char *cmd) json_decref(val); /* Notify implied required now too */ _update_notify(ckp, id); - if (reconnect || proxy == current_proxy(sdata)) + if (reconnect) reconnect_clients(sdata, ""); } @@ -1114,6 +1114,10 @@ static void _update_notify(ckpool_t *ckp, const int id) LOGINFO("No valid proxy subscription to update notify yet"); return; } + if (proxy != current_proxy(sdata)) { + LOGINFO("Notify from backup proxy"); + return; + } ASPRINTF(&msg, "getnotify=%d", id); buf = send_recv_proc(ckp->generator, msg); @@ -1809,6 +1813,22 @@ static char *stratifier_stats(ckpool_t *ckp, sdata_t *sdata) return buf; } +/* Sets the currently active proxy */ +static void set_proxy(sdata_t *sdata, const char *buf) +{ + proxy_t *proxy; + int id = 0; + + sscanf(buf, "proxy=%d", &id); + + mutex_lock(&sdata->proxy_lock); + proxy = __proxy_by_id(sdata, id); + sdata->proxy = proxy; + mutex_unlock(&sdata->proxy_lock); + + reconnect_clients(sdata, ""); +} + static int stratum_loop(ckpool_t *ckp, proc_instance_t *pi) { int sockd, ret = 0, selret = 0; @@ -1915,6 +1935,8 @@ retry: block_reject(sdata, buf + 8); } else if (cmdmatch(buf, "reconnect")) { reconnect_clients(sdata, buf); + } else if (cmdmatch(buf, "proxy")) { + set_proxy(sdata, buf); } else if (cmdmatch(buf, "loglevel")) { sscanf(buf, "loglevel=%d", &ckp->loglevel); } else From c5b856ac7c1ac2e516c07382ace4103251fe66dc Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Fri, 6 Feb 2015 22:36:43 +1100 Subject: [PATCH 019/544] Update notify after setting proxy before reconnecting clients --- src/stratifier.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/stratifier.c b/src/stratifier.c index d8a0df36..28ce0ad2 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -1814,7 +1814,7 @@ static char *stratifier_stats(ckpool_t *ckp, sdata_t *sdata) } /* Sets the currently active proxy */ -static void set_proxy(sdata_t *sdata, const char *buf) +static void set_proxy(ckpool_t *ckp, sdata_t *sdata, const char *buf) { proxy_t *proxy; int id = 0; @@ -1826,6 +1826,7 @@ static void set_proxy(sdata_t *sdata, const char *buf) sdata->proxy = proxy; mutex_unlock(&sdata->proxy_lock); + _update_notify(ckp, id); reconnect_clients(sdata, ""); } @@ -1936,7 +1937,7 @@ retry: } else if (cmdmatch(buf, "reconnect")) { reconnect_clients(sdata, buf); } else if (cmdmatch(buf, "proxy")) { - set_proxy(sdata, buf); + set_proxy(ckp, sdata, buf); } else if (cmdmatch(buf, "loglevel")) { sscanf(buf, "loglevel=%d", &ckp->loglevel); } else From 8825c7c901a3eb92d69c131066aa31f0040982ae Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Fri, 6 Feb 2015 22:42:41 +1100 Subject: [PATCH 020/544] Only give read socket line warning if proxy was considered alive at the time --- src/generator.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/generator.c b/src/generator.c index c1286974..4e7d92c1 100644 --- a/src/generator.c +++ b/src/generator.c @@ -1562,8 +1562,10 @@ static void *proxy_recv(void *arg) } while (ret == 0 && ++retries < 120); if (ret < 1) { - LOGWARNING("Proxy %d:%s failed to read_socket_line in proxy_recv, attempting reconnect", - proxi->id, proxi->si->url); + if (proxi->alive) { + LOGWARNING("Proxy %d:%s failed to read_socket_line in proxy_recv, attempting reconnect", + proxi->id, proxi->si->url); + } continue; } if (parse_method(proxi, cs->buf)) { From 37afa5fd02199b269132ab265ac00ec29957f146 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Fri, 6 Feb 2015 22:47:54 +1100 Subject: [PATCH 021/544] Rename current_proxy in generator to best_proxy for clarity --- src/generator.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/generator.c b/src/generator.c index 4e7d92c1..89736610 100644 --- a/src/generator.c +++ b/src/generator.c @@ -1483,7 +1483,7 @@ static void *passthrough_recv(void *arg) return NULL; } -static proxy_instance_t *current_proxy(ckpool_t *ckp, gdata_t *gdata); +static proxy_instance_t *best_proxy(ckpool_t *ckp, gdata_t *gdata); /* For receiving messages from the upstream proxy, also responsible for setting * up the connection and testing it's alive. */ @@ -1520,7 +1520,7 @@ static void *proxy_recv(void *arg) } /* Wait 90 seconds before declaring this upstream pool alive * to prevent switching to unstable pools. */ - if (!proxi->alive && (!current_proxy(ckp, ckp->data) || + if (!proxi->alive && (!best_proxy(ckp, ckp->data) || time(NULL) - proxi->reconnect_time > 90)) { LOGWARNING("Proxy %d:%s recovered", proxi->id, proxi->si->url); proxi->alive = true; @@ -1625,7 +1625,7 @@ static void setup_proxies(ckpool_t *ckp, gdata_t *gdata) } } -static proxy_instance_t *current_proxy(ckpool_t *ckp, gdata_t *gdata) +static proxy_instance_t *best_proxy(ckpool_t *ckp, gdata_t *gdata) { proxy_instance_t *ret = NULL, *proxi, *tmp; @@ -1668,7 +1668,7 @@ static int proxy_loop(proc_instance_t *pi) reconnect: /* This does not necessarily mean we reconnect, but a change has * occurred and we need to reexamine the proxies. */ - cproxy = current_proxy(ckp, gdata); + cproxy = best_proxy(ckp, gdata); if (!cproxy) goto out; if (proxi != cproxy) { From 84a399adf058fb5dfa4a325cbe0fd40f5d577b27 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Fri, 6 Feb 2015 22:54:46 +1100 Subject: [PATCH 022/544] Only send a notify request to the stratifier if it's from the current proxy --- src/generator.c | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/src/generator.c b/src/generator.c index 89736610..3a95c7f1 100644 --- a/src/generator.c +++ b/src/generator.c @@ -1485,6 +1485,17 @@ static void *passthrough_recv(void *arg) static proxy_instance_t *best_proxy(ckpool_t *ckp, gdata_t *gdata); +static proxy_instance_t *current_proxy(gdata_t *gdata) +{ + proxy_instance_t *ret; + + mutex_lock(&gdata->lock); + ret = gdata->proxy; + mutex_unlock(&gdata->lock); + + return ret; +} + /* For receiving messages from the upstream proxy, also responsible for setting * up the connection and testing it's alive. */ static void *proxy_recv(void *arg) @@ -1493,6 +1504,7 @@ static void *proxy_recv(void *arg) server_instance_t *si = proxi->si; connsock_t *cs = proxi->cs; ckpool_t *ckp = proxi->ckp; + gdata_t *gdata = ckp->data; rename_proc("proxyrecv"); @@ -1520,7 +1532,7 @@ static void *proxy_recv(void *arg) } /* Wait 90 seconds before declaring this upstream pool alive * to prevent switching to unstable pools. */ - if (!proxi->alive && (!best_proxy(ckp, ckp->data) || + if (!proxi->alive && (!best_proxy(ckp, gdata) || time(NULL) - proxi->reconnect_time > 90)) { LOGWARNING("Proxy %d:%s recovered", proxi->id, proxi->si->url); proxi->alive = true; @@ -1569,7 +1581,7 @@ static void *proxy_recv(void *arg) continue; } if (parse_method(proxi, cs->buf)) { - if (proxi->notified) { + if (proxi->notified && proxi == current_proxy(gdata)) { snprintf(buf, 127, "notify=%d", proxi->id); send_proc(ckp->stratifier, buf); proxi->notified = false; From 289caa273263258a18a062e105f14a3f47305673 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Sat, 7 Feb 2015 11:29:28 +1100 Subject: [PATCH 023/544] Issue reconnect to generator immediately upon a reconnect request --- src/generator.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/generator.c b/src/generator.c index 3a95c7f1..c449d2e6 100644 --- a/src/generator.c +++ b/src/generator.c @@ -1591,7 +1591,12 @@ static void *proxy_recv(void *arg) proxi->diffed = false; } if (proxi->reconnect) { + /* Call this proxy dead to allow us to fail + * over to a backup pool until the reconnect + * pool is up */ proxi->reconnect = false; + proxi->alive = false; + send_proc(ckp->generator, "reconnect"); LOGWARNING("Proxy %d:%s reconnect issue, dropping existing connection", proxi->id, proxi->si->url); Close(cs->fd); From f0f628df8442d72b9211a3d5e7397a1411cfeb13 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Sat, 7 Feb 2015 11:41:02 +1100 Subject: [PATCH 024/544] Reuse known ckp variable in drop_client --- src/stratifier.c | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/stratifier.c b/src/stratifier.c index 28ce0ad2..9d64f774 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -1521,13 +1521,12 @@ static void dec_worker(ckpool_t *ckp, user_instance_t *instance) mutex_unlock(&sdata->stats_lock); } -static void drop_client(sdata_t *sdata, const int64_t id) +static void drop_client(ckpool_t *ckp, sdata_t *sdata, const int64_t id) { stratum_instance_t *client, *tmp; user_instance_t *user = NULL; int dropped = 0, aged = 0; time_t now_t = time(NULL); - ckpool_t *ckp = NULL; LOGINFO("Stratifier asked to drop client %"PRId64, id); @@ -1537,7 +1536,6 @@ static void drop_client(sdata_t *sdata, const int64_t id) ck_ulock(&sdata->instance_lock); if (client && !client->dropped) { user = client->user_instance; - ckp = client->ckp; /* If the client is still holding a reference, don't drop them * now but wait till the reference is dropped */ if (!client->ref) @@ -1927,7 +1925,7 @@ retry: if (ret < 0) LOGDEBUG("Stratifier failed to parse dropclient command: %s", buf); else - drop_client(sdata, client_id); + drop_client(ckp, sdata, client_id); } else if (cmdmatch(buf, "dropall")) { drop_allclients(ckp); } else if (cmdmatch(buf, "block")) { From a80f9aa49227801ad66b4b019c23987ad1adf224 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Sat, 7 Feb 2015 11:49:14 +1100 Subject: [PATCH 025/544] Don't miss dropping stratifier clients whose dropped flag is set when no reference count is held --- src/stratifier.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/stratifier.c b/src/stratifier.c index 9d64f774..c0fad849 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -1534,13 +1534,13 @@ static void drop_client(ckpool_t *ckp, sdata_t *sdata, const int64_t id) client = __instance_by_id(sdata, id); /* Upgrade to write lock */ ck_ulock(&sdata->instance_lock); - if (client && !client->dropped) { - user = client->user_instance; + if (client) { /* If the client is still holding a reference, don't drop them * now but wait till the reference is dropped */ - if (!client->ref) + if (!client->ref) { + user = client->user_instance; dropped = __drop_client(sdata, client, user); - else + } else client->dropped = true; } From 5665d5454575a28614333793dbfbe13dcd545015 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Sat, 7 Feb 2015 13:15:06 +1100 Subject: [PATCH 026/544] Test without dropping listen backlog --- src/connector.c | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/src/connector.c b/src/connector.c index 950dd26c..028e6f9e 100644 --- a/src/connector.c +++ b/src/connector.c @@ -380,7 +380,6 @@ reparse: void *receiver(void *arg) { cdata_t *cdata = (cdata_t *)arg; - bool dropped_backlog = false; struct epoll_event event; uint64_t serverfds, i; time_t start_t; @@ -414,16 +413,6 @@ void *receiver(void *arg) while (42) { client_instance_t *client; - if (unlikely(!dropped_backlog && time(NULL) - start_t > 90)) { - /* When we first start we listen to as many connections - * as possible. After the first minute we drop the - * listen to the minimum to effectively ratelimit how - * fast we can receive new connections. */ - dropped_backlog = true; - LOGNOTICE("Dropping server listen backlog to 0"); - for (i = 0; i < serverfds; i++) - listen(cdata->serverfd[i], 0); - } while (unlikely(!cdata->accept)) cksleep_ms(10); ret = epoll_wait(epfd, &event, 1, 1000); From db9a777767b08733191bea3bcb8c48a6669ca7d0 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Sat, 7 Feb 2015 14:46:43 +1100 Subject: [PATCH 027/544] Reconnect clients lazily on proxy switch once we receive another message from them and drop them if they don't disconnect on their own within a minute --- src/stratifier.c | 53 ++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 42 insertions(+), 11 deletions(-) diff --git a/src/stratifier.c b/src/stratifier.c index c0fad849..ec0238a4 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -254,6 +254,9 @@ struct stratum_instance { * or other problem and should be dropped lazily if * this is set to 2 */ + bool reconnect; /* Do we need to send this client a reconnect message */ + time_t reconnect_request; /* The time we sent a reconnect message */ + user_instance_t *user_instance; worker_instance_t *worker_instance; @@ -1011,7 +1014,19 @@ static proxy_t *proxy_by_id(sdata_t *sdata, const int id) return proxy; } -static void reconnect_clients(sdata_t *sdata, const char *cmd); +/* Iterates over all clients and sets the reconnect bool for the message + * to be sent lazily next time they speak to us. */ +static void reconnect_clients(sdata_t *sdata) +{ + stratum_instance_t *client, *tmp; + + ck_rlock(&sdata->instance_lock); + HASH_ITER(hh, sdata->stratum_instances, client, tmp) { + client->reconnect = true; + } + ck_runlock(&sdata->instance_lock); +} + static void _update_notify(ckpool_t *ckp, const int id); static proxy_t *current_proxy(sdata_t *sdata) @@ -1093,7 +1108,7 @@ static void update_subscribe(ckpool_t *ckp, const char *cmd) /* Notify implied required now too */ _update_notify(ckp, id); if (reconnect) - reconnect_clients(sdata, ""); + reconnect_clients(sdata); } static void update_diff(ckpool_t *ckp); @@ -1576,7 +1591,7 @@ static void stratum_broadcast_message(sdata_t *sdata, const char *msg) /* Send a generic reconnect to all clients without parameters to make them * reconnect to the same server. */ -static void reconnect_clients(sdata_t *sdata, const char *cmd) +static void request_reconnect(sdata_t *sdata, const char *cmd) { char *port = strdupa(cmd), *url = NULL; stratum_instance_t *client, *tmp; @@ -1825,7 +1840,7 @@ static void set_proxy(ckpool_t *ckp, sdata_t *sdata, const char *buf) mutex_unlock(&sdata->proxy_lock); _update_notify(ckp, id); - reconnect_clients(sdata, ""); + reconnect_clients(sdata); } static int stratum_loop(ckpool_t *ckp, proc_instance_t *pi) @@ -1933,7 +1948,7 @@ retry: } else if (cmdmatch(buf, "noblock")) { block_reject(sdata, buf + 8); } else if (cmdmatch(buf, "reconnect")) { - reconnect_clients(sdata, buf); + request_reconnect(sdata, buf); } else if (cmdmatch(buf, "proxy")) { set_proxy(ckp, sdata, buf); } else if (cmdmatch(buf, "loglevel")) { @@ -3513,16 +3528,14 @@ static void free_smsg(smsg_t *msg) } /* Entered with client holding ref count */ -static void parse_instance_msg(sdata_t *sdata, smsg_t *msg, stratum_instance_t *client) +static void parse_instance_msg(ckpool_t *ckp, sdata_t *sdata, smsg_t *msg, stratum_instance_t *client) { json_t *val = msg->json_msg, *id_val, *method, *params; int64_t client_id = msg->client_id; - char buf[256]; - if (unlikely(client->reject == 2)) { + if (client->reject == 2 || (client->reconnect_request && time(NULL) - client->reconnect_request > 60)) { LOGINFO("Dropping client %"PRId64" tagged for lazy invalidation", client_id); - snprintf(buf, 255, "dropclient=%"PRId64, client_id); - send_proc(client->ckp->connector, buf); + connector_drop_client(ckp, client_id); goto out; } @@ -3557,6 +3570,20 @@ out: free_smsg(msg); } +/* Send a single client a reconnect request, setting the time we sent the + * request so we can drop the client lazily if it hasn't reconnected on its + * own one minute later */ +static void reconnect_client(sdata_t *sdata, stratum_instance_t *client) +{ + json_t *json_msg; + + client->reconnect = false; + client->reconnect_request = time(NULL); + JSON_CPACK(json_msg, "{sosss[]}", "id", json_null(), "method", "client.reconnect", + "params"); + stratum_add_send(sdata, json_msg, client->id); +} + static void srecv_process(ckpool_t *ckp, char *buf) { bool noid = false, dropped = false; @@ -3633,7 +3660,11 @@ static void srecv_process(ckpool_t *ckp, char *buf) if (unlikely(noid)) LOGINFO("Stratifier added instance %"PRId64" server %d", client->id, server); - parse_instance_msg(sdata, msg, client); + parse_instance_msg(ckp, sdata, msg, client); + /* The client is still active but has been issued a reconnect request + * so use this opportunity to send it a reconnect message */ + if (unlikely(client->reconnect)) + reconnect_client(sdata, client); dec_instance_ref(sdata, client); out: free(buf); From 25d8c015e56059d57b38468db9d19533b1d14a21 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Sat, 7 Feb 2015 15:06:12 +1100 Subject: [PATCH 028/544] Add information about which proxy's socket is invalidated --- src/generator.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/generator.c b/src/generator.c index c449d2e6..b0d4c378 100644 --- a/src/generator.c +++ b/src/generator.c @@ -1692,8 +1692,8 @@ reconnect: proxi = cproxy; if (!ckp->passthrough) { connsock_t *cs = proxi->cs; - LOGWARNING("Successfully connected to %s:%s as proxy", - cs->url, cs->port); + LOGWARNING("Successfully connected to proxy %d %s:%s as proxy", + proxi->id, cs->url, cs->port); dealloc(buf); ASPRINTF(&buf, "proxy=%d", proxi->id); send_proc(ckp->stratifier, buf); @@ -1710,7 +1710,8 @@ retry: } while (selret < 1); if (unlikely(proxi->cs->fd < 0)) { - LOGWARNING("Upstream socket invalidated, will attempt failover"); + LOGWARNING("Upstream proxy %d:%s socket invalidated, will attempt failover", + proxi->id, proxi->cs->url); proxi->alive = false; proxi = NULL; goto reconnect; From 836f2f3ff2560faebc16d6a2e5bb2e7a157b4ab1 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Sat, 7 Feb 2015 17:41:47 +1100 Subject: [PATCH 029/544] Avoid extra call to get subscription from generator by pushing it with the subscribe message to the stratifier --- src/generator.c | 31 ++++++++++++------------------- src/stratifier.c | 16 ++++++---------- 2 files changed, 18 insertions(+), 29 deletions(-) diff --git a/src/generator.c b/src/generator.c index b0d4c378..df8d9999 100644 --- a/src/generator.c +++ b/src/generator.c @@ -1131,30 +1131,22 @@ static proxy_instance_t *proxy_by_id(gdata_t *gdata, const int id) return proxi; } -static void send_subscribe(gdata_t *gdata, proxy_instance_t *cproxy, int *sockd, const char *buf) +static proxy_instance_t *current_proxy(gdata_t *gdata); + +static char *get_subscribe(ckpool_t *ckp, proxy_instance_t *proxi) { - proxy_instance_t *proxi; + gdata_t *gdata = ckp->data; json_t *json_msg; - int id = 0; char *msg; - sscanf(buf, "getsubscribe=%d", &id); - proxi = proxy_by_id(gdata, id); - if (unlikely(!proxi || !proxi->nonce2len)) { - ASPRINTF(&msg, "notready"); - goto out_send; - } JSON_CPACK(json_msg, "{sisssisb}", "proxy", proxi->id, "enonce1", proxi->enonce1, "nonce2len", proxi->nonce2len, - "reconnect", proxi == cproxy ? true : false); + "reconnect", proxi == current_proxy(gdata) ? true : false); msg = json_dumps(json_msg, JSON_NO_UTF8); json_decref(json_msg); -out_send: - send_unix_msg(*sockd, msg); - free(msg); - _Close(sockd); + return msg; } static void send_notify(gdata_t *gdata, int *sockd, const char *buf) @@ -1421,11 +1413,14 @@ out: /* Close and invalidate the file handle */ Close(cs->fd); } else { - char msg[128]; + char *msg, *buf; keep_sockalive(cs->fd); - snprintf(msg, 127, "subscribe=%d", proxi->id); - send_proc(ckp->stratifier, msg); + msg = get_subscribe(ckp, proxi); + ASPRINTF(&buf, "subscribe=%s", msg); + free(msg); + send_proc(ckp->stratifier, buf); + free(buf); proxi->notified = false; } return ret; @@ -1734,8 +1729,6 @@ retry: if (cmdmatch(buf, "shutdown")) { ret = 0; goto out; - } else if (cmdmatch(buf, "getsubscribe")) { - send_subscribe(gdata, proxi, &sockd, buf); } else if (cmdmatch(buf, "getnotify")) { send_notify(gdata, &sockd, buf); } else if (cmdmatch(buf, "getdiff")) { diff --git a/src/stratifier.c b/src/stratifier.c index ec0238a4..06f92b77 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -1044,31 +1044,27 @@ static void update_subscribe(ckpool_t *ckp, const char *cmd) { sdata_t *sdata = ckp->data; bool reconnect = true; - char *buf, *msg; + const char *buf; proxy_t *proxy; json_t *val; int id = 0; - sscanf(cmd, "subscribe=%d", &id); - ASPRINTF(&msg, "getsubscribe=%d", id); - buf = send_recv_proc(ckp->generator, msg); - dealloc(msg); - if (unlikely(!buf)) { - LOGWARNING("Failed to get subscribe from generator in update_subscribe"); - drop_allclients(ckp); + if (unlikely(strlen(cmd) < 11)) { + LOGWARNING("Received zero length string for subscribe in update_subscribe"); return; } + buf = cmd + 10; LOGDEBUG("Update subscribe: %s", buf); if (unlikely(!safecmp(buf, "notready"))) { LOGNOTICE("Generator not ready to send subscribe for proxy %d", id); return; } val = json_loads(buf, 0, NULL); - free(buf); if (unlikely(!val)) { - LOGWARNING("Failed to json decode getsubscribe response in update_subscribe"); + LOGWARNING("Failed to json decode subscribe response in update_subscribe"); return; } + json_get_int(&id, val, "proxy"); LOGNOTICE("Got updated subscribe for proxy %d", id); From 1d69aa0ae9d489f0973a6546515be98c58072f57 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Sat, 7 Feb 2015 17:45:25 +1100 Subject: [PATCH 030/544] Simplify send_subscribe code --- src/generator.c | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/src/generator.c b/src/generator.c index df8d9999..4c39f8ef 100644 --- a/src/generator.c +++ b/src/generator.c @@ -1133,11 +1133,11 @@ static proxy_instance_t *proxy_by_id(gdata_t *gdata, const int id) static proxy_instance_t *current_proxy(gdata_t *gdata); -static char *get_subscribe(ckpool_t *ckp, proxy_instance_t *proxi) +static void send_subscribe(ckpool_t *ckp, proxy_instance_t *proxi) { gdata_t *gdata = ckp->data; json_t *json_msg; - char *msg; + char *msg, *buf; JSON_CPACK(json_msg, "{sisssisb}", "proxy", proxi->id, @@ -1146,7 +1146,10 @@ static char *get_subscribe(ckpool_t *ckp, proxy_instance_t *proxi) "reconnect", proxi == current_proxy(gdata) ? true : false); msg = json_dumps(json_msg, JSON_NO_UTF8); json_decref(json_msg); - return msg; + ASPRINTF(&buf, "subscribe=%s", msg); + free(msg); + send_proc(ckp->stratifier, buf); + free(buf); } static void send_notify(gdata_t *gdata, int *sockd, const char *buf) @@ -1413,14 +1416,8 @@ out: /* Close and invalidate the file handle */ Close(cs->fd); } else { - char *msg, *buf; - keep_sockalive(cs->fd); - msg = get_subscribe(ckp, proxi); - ASPRINTF(&buf, "subscribe=%s", msg); - free(msg); - send_proc(ckp->stratifier, buf); - free(buf); + send_subscribe(ckp, proxi); proxi->notified = false; } return ret; From 64a3703f09378b32e58ec69e1b9fb26ce16cb28b Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Sat, 7 Feb 2015 18:21:08 +1100 Subject: [PATCH 031/544] Move to pushing all notify data from the generator to avoid an extra message and reconnect clients once the current proxy has its first notification data --- src/generator.c | 42 ++++++++++--------------- src/stratifier.c | 81 ++++++++++++++++++++---------------------------- 2 files changed, 50 insertions(+), 73 deletions(-) diff --git a/src/generator.c b/src/generator.c index 4c39f8ef..9de90674 100644 --- a/src/generator.c +++ b/src/generator.c @@ -1120,6 +1120,7 @@ out: return ret; } +#if 0 static proxy_instance_t *proxy_by_id(gdata_t *gdata, const int id) { proxy_instance_t *proxi; @@ -1130,20 +1131,17 @@ static proxy_instance_t *proxy_by_id(gdata_t *gdata, const int id) return proxi; } - -static proxy_instance_t *current_proxy(gdata_t *gdata); +#endif static void send_subscribe(ckpool_t *ckp, proxy_instance_t *proxi) { - gdata_t *gdata = ckp->data; json_t *json_msg; char *msg, *buf; - JSON_CPACK(json_msg, "{sisssisb}", + JSON_CPACK(json_msg, "{sisssi}", "proxy", proxi->id, "enonce1", proxi->enonce1, - "nonce2len", proxi->nonce2len, - "reconnect", proxi == current_proxy(gdata) ? true : false); + "nonce2len", proxi->nonce2len); msg = json_dumps(json_msg, JSON_NO_UTF8); json_decref(json_msg); ASPRINTF(&buf, "subscribe=%s", msg); @@ -1152,28 +1150,21 @@ static void send_subscribe(ckpool_t *ckp, proxy_instance_t *proxi) free(buf); } -static void send_notify(gdata_t *gdata, int *sockd, const char *buf) +static void send_notify(ckpool_t *ckp, proxy_instance_t *proxi) { json_t *json_msg, *merkle_arr; - proxy_instance_t *proxi; notify_instance_t *ni; - int i, id = 0; - char *msg; + char *msg, *buf; + int i; - sscanf(buf, "getnotify=%d", &id); - proxi = proxy_by_id(gdata, id); - if (unlikely(!proxi)) { - ASPRINTF(&msg, "notready"); - goto out_send; - } merkle_arr = json_array(); mutex_lock(&proxi->notify_lock); ni = proxi->current_notify; if (unlikely(!ni)) { mutex_unlock(&proxi->notify_lock); - ASPRINTF(&msg, "notready"); - goto out_send; + LOGNOTICE("Proxi %d not ready to send notify", proxi->id); + return; } for (i = 0; i < ni->merkles; i++) json_array_append_new(merkle_arr, json_string(&ni->merklehash[i][0])); @@ -1188,10 +1179,10 @@ static void send_notify(gdata_t *gdata, int *sockd, const char *buf) msg = json_dumps(json_msg, JSON_NO_UTF8); json_decref(json_msg); -out_send: - send_unix_msg(*sockd, msg); + ASPRINTF(&buf, "notify=%s", msg); free(msg); - _Close(sockd); + send_proc(ckp->stratifier, buf); + free(buf); } static void send_diff(proxy_instance_t *proxi, int *sockd) @@ -1511,7 +1502,6 @@ static void *proxy_recv(void *arg) notify_instance_t *ni, *tmp; share_msg_t *share, *tmpshare; int retries = 0, ret; - char buf[128]; time_t now; while (!proxy_alive(ckp, si, proxi, cs, true)) { @@ -1574,8 +1564,7 @@ static void *proxy_recv(void *arg) } if (parse_method(proxi, cs->buf)) { if (proxi->notified && proxi == current_proxy(gdata)) { - snprintf(buf, 127, "notify=%d", proxi->id); - send_proc(ckp->stratifier, buf); + send_notify(ckp, proxi); proxi->notified = false; } if (proxi->diffed) { @@ -1689,6 +1678,9 @@ reconnect: dealloc(buf); ASPRINTF(&buf, "proxy=%d", proxi->id); send_proc(ckp->stratifier, buf); + /* Send a notify for the new chosen proxy or the + * stratifier won't be able to switch. */ + send_notify(ckp, proxi); } } retry: @@ -1726,8 +1718,6 @@ retry: if (cmdmatch(buf, "shutdown")) { ret = 0; goto out; - } else if (cmdmatch(buf, "getnotify")) { - send_notify(gdata, &sockd, buf); } else if (cmdmatch(buf, "getdiff")) { send_diff(proxi, &sockd); } else if (cmdmatch(buf, "reconnect")) { diff --git a/src/stratifier.c b/src/stratifier.c index 06f92b77..1e165940 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -297,6 +297,7 @@ struct proxy_base { int enonce2varlen; bool subscribed; + bool notified; }; typedef struct proxy_base proxy_t; @@ -1027,8 +1028,6 @@ static void reconnect_clients(sdata_t *sdata) ck_runlock(&sdata->instance_lock); } -static void _update_notify(ckpool_t *ckp, const int id); - static proxy_t *current_proxy(sdata_t *sdata) { proxy_t *proxy; @@ -1043,7 +1042,6 @@ static proxy_t *current_proxy(sdata_t *sdata) static void update_subscribe(ckpool_t *ckp, const char *cmd) { sdata_t *sdata = ckp->data; - bool reconnect = true; const char *buf; proxy_t *proxy; json_t *val; @@ -1055,10 +1053,6 @@ static void update_subscribe(ckpool_t *ckp, const char *cmd) } buf = cmd + 10; LOGDEBUG("Update subscribe: %s", buf); - if (unlikely(!safecmp(buf, "notready"))) { - LOGNOTICE("Generator not ready to send subscribe for proxy %d", id); - return; - } val = json_loads(buf, 0, NULL); if (unlikely(!val)) { LOGWARNING("Failed to json decode subscribe response in update_subscribe"); @@ -1069,7 +1063,7 @@ static void update_subscribe(ckpool_t *ckp, const char *cmd) LOGNOTICE("Got updated subscribe for proxy %d", id); proxy = proxy_by_id(sdata, id); - json_get_bool(&reconnect, val, "reconnect"); + proxy->notified = false; /* Reset this */ ck_wlock(&sdata->workbase_lock); proxy->subscribed = true; @@ -1101,52 +1095,54 @@ static void update_subscribe(ckpool_t *ckp, const char *cmd) proxy->nonce2len, 1ll << (proxy->enonce1varlen * 8)); json_decref(val); - /* Notify implied required now too */ - _update_notify(ckp, id); - if (reconnect) - reconnect_clients(sdata); } static void update_diff(ckpool_t *ckp); -static void _update_notify(ckpool_t *ckp, const int id) +static void update_notify(ckpool_t *ckp, const char *cmd) { bool new_block = false, clean; sdata_t *sdata = ckp->data; char header[228]; - char *buf, *msg; + const char *buf; proxy_t *proxy; workbase_t *wb; + int i, id = 0; json_t *val; - int i; + if (unlikely(strlen(cmd) < 8)) { + LOGWARNING("Zero length string passed to update_notify"); + return; + } + buf = cmd + 7; + LOGDEBUG("Update notify: %s", buf); + + val = json_loads(buf, 0, NULL); + if (unlikely(!val)) { + LOGWARNING("Failed to json decode in update_notify"); + return; + } + json_get_int(&id, val, "proxy"); proxy = proxy_by_id(sdata, id); if (unlikely(!proxy->subscribed)) { - LOGINFO("No valid proxy subscription to update notify yet"); - return; + LOGNOTICE("No valid proxy %d subscription to update notify yet", id); + goto out; } + LOGNOTICE("Got updated notify for proxy %d", id); if (proxy != current_proxy(sdata)) { LOGINFO("Notify from backup proxy"); - return; + goto out; } - ASPRINTF(&msg, "getnotify=%d", id); - buf = send_recv_proc(ckp->generator, msg); - dealloc(msg); - if (unlikely(!buf)) { - LOGWARNING("Failed to get notify from generator in update_notify"); - return; - } - LOGDEBUG("Update notify: %s", buf); - if (unlikely(!safecmp(buf, "notready"))) { - LOGNOTICE("Generator not ready to send notify to stratifier"); - return; + if (!proxy->notified) { + /* This is the first notification from the current proxy, tell + * clients now to reconnect since we have enough information to + * switch. */ + proxy->notified = true; + reconnect_clients(sdata); } - LOGINFO("Got updated notify for proxy %d", id); wb = ckzalloc(sizeof(workbase_t)); - val = json_loads(buf, 0, NULL); - dealloc(buf); wb->ckp = ckp; wb->proxy = true; @@ -1173,7 +1169,6 @@ static void _update_notify(ckpool_t *ckp, const int id) json_strcpy(wb->ntime, val, "ntime"); sscanf(wb->ntime, "%x", &wb->ntime32); clean = json_is_true(json_object_get(val, "clean")); - json_decref(val); ts_realtime(&wb->gentime); snprintf(header, 225, "%s%s%s%s%s%s%s", wb->bbversion, wb->prevhash, @@ -1200,14 +1195,8 @@ static void _update_notify(ckpool_t *ckp, const int id) add_base(ckp, wb, &new_block); stratum_broadcast_update(sdata, new_block | clean); -} - -static void update_notify(ckpool_t *ckp, const char *cmd) -{ - int id = 0; - - sscanf(cmd, "notify=%d", &id); - _update_notify(ckp, id); +out: + json_decref(val); } static void stratum_send_diff(sdata_t *sdata, const stratum_instance_t *client); @@ -1822,8 +1811,9 @@ static char *stratifier_stats(ckpool_t *ckp, sdata_t *sdata) return buf; } -/* Sets the currently active proxy */ -static void set_proxy(ckpool_t *ckp, sdata_t *sdata, const char *buf) +/* Sets the currently active proxy. Clients will be told to reconnect once the + * first notify data comes from this proxy. */ +static void set_proxy(sdata_t *sdata, const char *buf) { proxy_t *proxy; int id = 0; @@ -1834,9 +1824,6 @@ static void set_proxy(ckpool_t *ckp, sdata_t *sdata, const char *buf) proxy = __proxy_by_id(sdata, id); sdata->proxy = proxy; mutex_unlock(&sdata->proxy_lock); - - _update_notify(ckp, id); - reconnect_clients(sdata); } static int stratum_loop(ckpool_t *ckp, proc_instance_t *pi) @@ -1946,7 +1933,7 @@ retry: } else if (cmdmatch(buf, "reconnect")) { request_reconnect(sdata, buf); } else if (cmdmatch(buf, "proxy")) { - set_proxy(ckp, sdata, buf); + set_proxy(sdata, buf); } else if (cmdmatch(buf, "loglevel")) { sscanf(buf, "loglevel=%d", &ckp->loglevel); } else From ab4d3a672f930d59506d9fee50468a48442e32db Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Sat, 7 Feb 2015 18:24:47 +1100 Subject: [PATCH 032/544] Set notified flag to false immediately upon setting a new proxy --- src/stratifier.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/stratifier.c b/src/stratifier.c index 1e165940..da2759aa 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -1824,6 +1824,10 @@ static void set_proxy(sdata_t *sdata, const char *buf) proxy = __proxy_by_id(sdata, id); sdata->proxy = proxy; mutex_unlock(&sdata->proxy_lock); + + /* We will receive a notification immediately after this and it should + * be the flag to reconnect clients. */ + proxy->notified = false; } static int stratum_loop(ckpool_t *ckp, proc_instance_t *pi) From 71be1210831ca65425f33cc85ecc1ece2c6717ba Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Sun, 8 Feb 2015 08:38:10 +1100 Subject: [PATCH 033/544] Microoptimise --- src/generator.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/generator.c b/src/generator.c index 9de90674..e2d0460a 100644 --- a/src/generator.c +++ b/src/generator.c @@ -1736,8 +1736,8 @@ retry: /* Anything remaining should be share submissions */ json_t *val = json_loads(buf, 0, NULL); - if (!val) - LOGWARNING("Received unrecognised message: %s", buf); + if (unlikely(!val)) + LOGWARNING("Generator received unrecognised message: %s", buf); else submit_share(proxi, val); } From 26c890c291424c992d109722c6553fcad0634f5a Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Sun, 8 Feb 2015 09:02:22 +1100 Subject: [PATCH 034/544] Send the diff from the proxy instance as soon as we get it and differentiate which proxy it comes from --- src/generator.c | 51 +++++++++++++++++++++++------------------------- src/stratifier.c | 36 +++++++++++++++++++++------------- 2 files changed, 46 insertions(+), 41 deletions(-) diff --git a/src/generator.c b/src/generator.c index e2d0460a..e2c58fb8 100644 --- a/src/generator.c +++ b/src/generator.c @@ -101,7 +101,6 @@ struct proxy_instance { bool no_params; /* Doesn't want any parameters on subscribe */ bool notified; /* Received new template for work */ - bool diffed; /* Received new diff */ bool reconnect; /* We need to drop and reconnect */ bool alive; @@ -838,7 +837,6 @@ static bool parse_diff(proxy_instance_t *proxi, json_t *val) if (diff == 0 || diff == proxi->diff) return true; proxi->diff = diff; - proxi->diffed = true; return true; } @@ -963,7 +961,23 @@ out: return ret; } -static bool parse_method(proxy_instance_t *proxi, const char *msg) +static void send_diff(ckpool_t *ckp, proxy_instance_t *proxi) +{ + json_t *json_msg; + char *msg, *buf; + + JSON_CPACK(json_msg, "{sisf}", + "proxy", proxi->id, + "diff", proxi->diff); + msg = json_dumps(json_msg, JSON_NO_UTF8); + json_decref(json_msg); + ASPRINTF(&buf, "diff=%s", msg); + free(msg); + send_proc(ckp->stratifier, buf); + free(buf); +} + +static bool parse_method(ckpool_t *ckp, proxy_instance_t *proxi, const char *msg) { json_t *val = NULL, *method, *err_val, *params; json_error_t err; @@ -1018,6 +1032,8 @@ static bool parse_method(proxy_instance_t *proxi, const char *msg) if (cmdmatch(buf, "mining.set_difficulty")) { ret = parse_diff(proxi, params); + if (likely(ret)) + send_diff(ckp, proxi); goto out; } @@ -1046,7 +1062,7 @@ out: return ret; } -static bool auth_stratum(connsock_t *cs, proxy_instance_t *proxi) +static bool auth_stratum(ckpool_t *ckp, connsock_t *cs, proxy_instance_t *proxi) { json_t *val = NULL, *res_val, *req, *err_val; char *buf = NULL; @@ -1076,7 +1092,7 @@ static bool auth_stratum(connsock_t *cs, proxy_instance_t *proxi) ret = false; goto out; } - ret = parse_method(proxi, buf); + ret = parse_method(ckp, proxi, buf); } while (ret); val = json_msg_result(buf, &res_val, &err_val); @@ -1114,7 +1130,7 @@ out: buf = cached_proxy_line(proxi); if (!buf) break; - parse_method(proxi, buf); + parse_method(ckp, proxi, buf); }; } return ret; @@ -1185,19 +1201,6 @@ static void send_notify(ckpool_t *ckp, proxy_instance_t *proxi) free(buf); } -static void send_diff(proxy_instance_t *proxi, int *sockd) -{ - json_t *json_msg; - char *msg; - - JSON_CPACK(json_msg, "{sf}", "diff", proxi->diff); - msg = json_dumps(json_msg, JSON_NO_UTF8); - json_decref(json_msg); - send_unix_msg(*sockd, msg); - free(msg); - _Close(sockd); -} - static void submit_share(proxy_instance_t *proxi, json_t *val) { stratum_msg_t *msg; @@ -1394,7 +1397,7 @@ static bool proxy_alive(ckpool_t *ckp, server_instance_t *si, proxy_instance_t * } goto out; } - if (!auth_stratum(cs, proxi)) { + if (!auth_stratum(ckp, cs, proxi)) { if (!pinging) { LOGWARNING("Failed initial authorise to %s:%s with %s:%s !", cs->url, cs->port, si->auth, si->pass); @@ -1562,15 +1565,11 @@ static void *proxy_recv(void *arg) } continue; } - if (parse_method(proxi, cs->buf)) { + if (parse_method(ckp, proxi, cs->buf)) { if (proxi->notified && proxi == current_proxy(gdata)) { send_notify(ckp, proxi); proxi->notified = false; } - if (proxi->diffed) { - send_proc(ckp->stratifier, "diff"); - proxi->diffed = false; - } if (proxi->reconnect) { /* Call this proxy dead to allow us to fail * over to a backup pool until the reconnect @@ -1718,8 +1717,6 @@ retry: if (cmdmatch(buf, "shutdown")) { ret = 0; goto out; - } else if (cmdmatch(buf, "getdiff")) { - send_diff(proxi, &sockd); } else if (cmdmatch(buf, "reconnect")) { goto reconnect; } else if (cmdmatch(buf, "submitblock:")) { diff --git a/src/stratifier.c b/src/stratifier.c index da2759aa..bf819694 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -1097,8 +1097,6 @@ static void update_subscribe(ckpool_t *ckp, const char *cmd) json_decref(val); } -static void update_diff(ckpool_t *ckp); - static void update_notify(ckpool_t *ckp, const char *cmd) { bool new_block = false, clean; @@ -1114,7 +1112,7 @@ static void update_notify(ckpool_t *ckp, const char *cmd) LOGWARNING("Zero length string passed to update_notify"); return; } - buf = cmd + 7; + buf = cmd + 7; /* "notify=" */ LOGDEBUG("Update notify: %s", buf); val = json_loads(buf, 0, NULL); @@ -1180,9 +1178,6 @@ static void update_notify(ckpool_t *ckp, const char *cmd) hex2bin(wb->headerbin, header, 112); wb->txn_hashes = ckzalloc(1); - /* Check diff on each notify */ - update_diff(ckp); - ck_rlock(&sdata->workbase_lock); strcpy(wb->enonce1const, proxy->enonce1); wb->enonce1constlen = proxy->enonce1constlen; @@ -1201,31 +1196,44 @@ out: static void stratum_send_diff(sdata_t *sdata, const stratum_instance_t *client); -static void update_diff(ckpool_t *ckp) +static void update_diff(ckpool_t *ckp, const char *cmd) { stratum_instance_t *client, *tmp; sdata_t *sdata = ckp->data; double old_diff, diff; + const char *buf; + proxy_t *proxy; json_t *val; - char *buf; + int id = 0; if (unlikely(!sdata->current_workbase)) { LOGINFO("No current workbase to update diff yet"); return; } - buf = send_recv_proc(ckp->generator, "getdiff"); - if (unlikely(!buf)) { - LOGWARNING("Failed to get diff from generator in update_diff"); + if (unlikely(strlen(cmd) < 6)) { + LOGWARNING("Zero length string passed to update_diff"); return; } - + buf = cmd + 5; /* "diff=" */ LOGDEBUG("Update diff: %s", buf); + val = json_loads(buf, 0, NULL); - dealloc(buf); + if (unlikely(!val)) { + LOGWARNING("Failed to json decode in update_diff"); + return; + } + json_get_int(&id, val, "proxy"); json_dblcpy(&diff, val, "diff"); json_decref(val); + LOGNOTICE("Got updated diff for proxy %d", id); + proxy = proxy_by_id(sdata, id); + if (proxy != current_proxy(sdata)) { + LOGINFO("Diff from backup proxy"); + return; + } + /* We only really care about integer diffs so clamp the lower limit to * 1 or it will round down to zero. */ if (unlikely(diff < 1)) @@ -1919,7 +1927,7 @@ retry: /* Proxifier has a new notify ready */ update_notify(ckp, buf); } else if (cmdmatch(buf, "diff")) { - update_diff(ckp); + update_diff(ckp, buf); } else if (cmdmatch(buf, "dropclient")) { int64_t client_id; From 5bff2819da694dc30721dee52e6c00c5ac3a2eba Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Sun, 8 Feb 2015 09:05:40 +1100 Subject: [PATCH 035/544] Send notify from each proxy as soon as we receive it --- src/generator.c | 85 +++++++++++++++++++++++-------------------------- 1 file changed, 40 insertions(+), 45 deletions(-) diff --git a/src/generator.c b/src/generator.c index e2c58fb8..4a7bd6d4 100644 --- a/src/generator.c +++ b/src/generator.c @@ -100,7 +100,6 @@ struct proxy_instance { bool no_sessionid; /* Doesn't support session id resume on subscribe */ bool no_params; /* Doesn't want any parameters on subscribe */ - bool notified; /* Received new template for work */ bool reconnect; /* We need to drop and reconnect */ bool alive; @@ -977,6 +976,41 @@ static void send_diff(ckpool_t *ckp, proxy_instance_t *proxi) free(buf); } +static void send_notify(ckpool_t *ckp, proxy_instance_t *proxi) +{ + json_t *json_msg, *merkle_arr; + notify_instance_t *ni; + char *msg, *buf; + int i; + + merkle_arr = json_array(); + + mutex_lock(&proxi->notify_lock); + ni = proxi->current_notify; + if (unlikely(!ni)) { + mutex_unlock(&proxi->notify_lock); + LOGNOTICE("Proxi %d not ready to send notify", proxi->id); + return; + } + for (i = 0; i < ni->merkles; i++) + json_array_append_new(merkle_arr, json_string(&ni->merklehash[i][0])); + /* Use our own jobid instead of the server's one for easy lookup */ + JSON_CPACK(json_msg, "{si,si,ss,si,ss,ss,so,ss,ss,ss,sb}", "proxy", proxi->id, + "jobid", ni->id, "prevhash", ni->prevhash, "coinb1len", ni->coinb1len, + "coinbase1", ni->coinbase1, "coinbase2", ni->coinbase2, + "merklehash", merkle_arr, "bbversion", ni->bbversion, + "nbit", ni->nbit, "ntime", ni->ntime, + "clean", ni->clean); + mutex_unlock(&proxi->notify_lock); + + msg = json_dumps(json_msg, JSON_NO_UTF8); + json_decref(json_msg); + ASPRINTF(&buf, "notify=%s", msg); + free(msg); + send_proc(ckp->stratifier, buf); + free(buf); +} + static bool parse_method(ckpool_t *ckp, proxy_instance_t *proxi, const char *msg) { json_t *val = NULL, *method, *err_val, *params; @@ -1023,10 +1057,9 @@ static bool parse_method(ckpool_t *ckp, proxy_instance_t *proxi, const char *msg } if (cmdmatch(buf, "mining.notify")) { - if (parse_notify(proxi, params)) - proxi->notified = ret = true; - else - proxi->notified = ret = false; + ret = parse_notify(proxi, params); + if (ret) + send_notify(ckp, proxi); goto out; } @@ -1166,41 +1199,6 @@ static void send_subscribe(ckpool_t *ckp, proxy_instance_t *proxi) free(buf); } -static void send_notify(ckpool_t *ckp, proxy_instance_t *proxi) -{ - json_t *json_msg, *merkle_arr; - notify_instance_t *ni; - char *msg, *buf; - int i; - - merkle_arr = json_array(); - - mutex_lock(&proxi->notify_lock); - ni = proxi->current_notify; - if (unlikely(!ni)) { - mutex_unlock(&proxi->notify_lock); - LOGNOTICE("Proxi %d not ready to send notify", proxi->id); - return; - } - for (i = 0; i < ni->merkles; i++) - json_array_append_new(merkle_arr, json_string(&ni->merklehash[i][0])); - /* Use our own jobid instead of the server's one for easy lookup */ - JSON_CPACK(json_msg, "{si,si,ss,si,ss,ss,so,ss,ss,ss,sb}", "proxy", proxi->id, - "jobid", ni->id, "prevhash", ni->prevhash, "coinb1len", ni->coinb1len, - "coinbase1", ni->coinbase1, "coinbase2", ni->coinbase2, - "merklehash", merkle_arr, "bbversion", ni->bbversion, - "nbit", ni->nbit, "ntime", ni->ntime, - "clean", ni->clean); - mutex_unlock(&proxi->notify_lock); - - msg = json_dumps(json_msg, JSON_NO_UTF8); - json_decref(json_msg); - ASPRINTF(&buf, "notify=%s", msg); - free(msg); - send_proc(ckp->stratifier, buf); - free(buf); -} - static void submit_share(proxy_instance_t *proxi, json_t *val) { stratum_msg_t *msg; @@ -1412,7 +1410,6 @@ out: } else { keep_sockalive(cs->fd); send_subscribe(ckp, proxi); - proxi->notified = false; } return ret; } @@ -1471,6 +1468,7 @@ static void *passthrough_recv(void *arg) static proxy_instance_t *best_proxy(ckpool_t *ckp, gdata_t *gdata); +#if 0 static proxy_instance_t *current_proxy(gdata_t *gdata) { proxy_instance_t *ret; @@ -1481,6 +1479,7 @@ static proxy_instance_t *current_proxy(gdata_t *gdata) return ret; } +#endif /* For receiving messages from the upstream proxy, also responsible for setting * up the connection and testing it's alive. */ @@ -1566,10 +1565,6 @@ static void *proxy_recv(void *arg) continue; } if (parse_method(ckp, proxi, cs->buf)) { - if (proxi->notified && proxi == current_proxy(gdata)) { - send_notify(ckp, proxi); - proxi->notified = false; - } if (proxi->reconnect) { /* Call this proxy dead to allow us to fail * over to a backup pool until the reconnect From b637efa17919dfa14bbbeeb946aece1a6dde725f Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Sun, 8 Feb 2015 10:12:04 +1100 Subject: [PATCH 036/544] Handle other forms of read_socket_line ending after message complete as not a failure --- src/ckpool.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/ckpool.c b/src/ckpool.c index 3d122a61..bf9b6286 100644 --- a/src/ckpool.c +++ b/src/ckpool.c @@ -419,9 +419,9 @@ int read_socket_line(connsock_t *cs, const int timeout) char *newbuf; ret = wait_read_select(fd, eom ? 0 : timeout); - if (eom && !ret) - break; if (ret < 1) { + if (eom) + break; if (!ret) LOGDEBUG("Select timed out in read_socket_line"); else { @@ -435,7 +435,7 @@ int read_socket_line(connsock_t *cs, const int timeout) ret = recv(fd, readbuf, PAGESIZE - 4, 0); if (ret < 1) { /* Closed socket after valid message */ - if (!ret && eom) + if (eom) break; if (cs->ckp->proxy) LOGNOTICE("Failed to recv in read_socket_line"); From 1c1cb445efee115815568bb15453cd025a64de8b Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Sun, 8 Feb 2015 10:26:33 +1100 Subject: [PATCH 037/544] Don't send subscriptions to the stratifier in passthrough mode --- src/generator.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/generator.c b/src/generator.c index 4a7bd6d4..1fd029f1 100644 --- a/src/generator.c +++ b/src/generator.c @@ -1409,7 +1409,8 @@ out: Close(cs->fd); } else { keep_sockalive(cs->fd); - send_subscribe(ckp, proxi); + if (!ckp->passthrough) + send_subscribe(ckp, proxi); } return ret; } From 83208ebc7254aad5c28195f827fa7e845be9b6c2 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Sun, 8 Feb 2015 13:02:50 +1100 Subject: [PATCH 038/544] fd being invalidated is checked for in wait_write_select so we don't need to handle it twice --- src/connector.c | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/connector.c b/src/connector.c index 028e6f9e..ca19a692 100644 --- a/src/connector.c +++ b/src/connector.c @@ -495,10 +495,6 @@ void *sender(void *arg) fd = client->fd; ck_runlock(&cdata->lock); - if (fd == -1) { - LOGDEBUG("Discarding message sent to invalidated client"); - goto contfree; - } /* If this socket is not ready to receive data from us, put the * send back on the tail of the list and decrease the timeout * to poll to either look for a client that is ready or poll From 0e7bc51541b25676fac194184119d6d1df95481c Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Sun, 8 Feb 2015 13:27:33 +1100 Subject: [PATCH 039/544] Check all delayed clients for a serviceable one in the connector when we can and consider dropping a client servicing one to not potentially create delayed sends faster than we service them --- src/connector.c | 38 ++++++++++++++++++++++++-------------- 1 file changed, 24 insertions(+), 14 deletions(-) diff --git a/src/connector.c b/src/connector.c index ca19a692..00e480b8 100644 --- a/src/connector.c +++ b/src/connector.c @@ -455,17 +455,17 @@ void *sender(void *arg) rename_proc("csender"); while (42) { - sender_send_t *sender_send; + sender_send_t *sender_send, *delayed; client_instance_t *client; - int ret, fd, ofs = 0; + int ret = 0, fd, ofs = 0; mutex_lock(&cdata->sender_lock); - /* Poll every 100ms if there are no new sends. Re-examine + /* Poll every 10ms if there are no new sends. Re-examine * delayed sends immediately after a successful send in case * endless new sends more frequently end up starving the * delayed sends. */ if (!cdata->sender_sends && !sent) { - const ts_t polltime = {0, 100000000}; + const ts_t polltime = {0, 10000000}; ts_t timeout_ts; ts_realtime(&timeout_ts); @@ -483,23 +483,33 @@ void *sender(void *arg) * conditional with no new sends appearing or have just * serviced another message successfully. */ if (!sender_send) { - if (!cdata->delayed_sends) + /* Find a delayed client that needs servicing and set + * ret accordingly. We do not need to use FOREACH_SAFE + * as we break out of the loop as soon as we manipuate + * the list. */ + DL_FOREACH(cdata->delayed_sends, delayed) { + if ((ret = wait_write_select(delayed->client->fd, 0))) { + sender_send = cdata->delayed_sends; + DL_DELETE(cdata->delayed_sends, sender_send); + break; + } + } + /* None found ? */ + if (!sender_send) continue; - sender_send = cdata->delayed_sends; - DL_DELETE(cdata->delayed_sends, sender_send); } - client = sender_send->client; - ck_rlock(&cdata->lock); - fd = client->fd; - ck_runlock(&cdata->lock); - /* If this socket is not ready to receive data from us, put the * send back on the tail of the list and decrease the timeout * to poll to either look for a client that is ready or poll * select on this one */ - ret = wait_write_select(fd, 0); + ck_rlock(&cdata->lock); + fd = client->fd; + if (!ret) + ret = wait_write_select(fd, 0); + ck_runlock(&cdata->lock); + if (ret < 1) { if (ret < 0) { LOGINFO("Client id %"PRId64" fd %d interrupted", client->id, fd); @@ -515,7 +525,6 @@ void *sender(void *arg) cdata->sends_delayed++; continue; } - sent = true; while (sender_send->len) { ret = send(fd, sender_send->buf + ofs, sender_send->len , 0); if (unlikely(ret < 0)) { @@ -527,6 +536,7 @@ void *sender(void *arg) sender_send->len -= ret; } contfree: + sent = true; free(sender_send->buf); free(sender_send); dec_instance_ref(cdata, client); From fe4a25f861986dfe099572c773ec50b41f07f30c Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Mon, 9 Feb 2015 15:10:14 +1100 Subject: [PATCH 040/544] Create rudimentary subproxy structures, moving proxy receives to epoll in order to receive from multiple connections upstream --- src/generator.c | 94 +++++++++++++++++++++++++++++++++++++------------ 1 file changed, 72 insertions(+), 22 deletions(-) diff --git a/src/generator.c b/src/generator.c index 1fd029f1..acd61fcf 100644 --- a/src/generator.c +++ b/src/generator.c @@ -9,6 +9,7 @@ #include "config.h" +#include #include #include #include @@ -81,7 +82,7 @@ struct proxy_instance { connsock_t *cs; server_instance_t *si; bool passthrough; - int id; /* Proxy server id */ + int id; /* Proxy server id, or subproxy id if this is a subproxy */ const char *auth; const char *pass; @@ -123,6 +124,13 @@ struct proxy_instance { char_entry_t *recvd_lines; /* Linked list of unprocessed messages */ time_t reconnect_time; + + pthread_mutex_t proxy_lock; /* Lock protecting hashlist of proxies */ + int64_t clients_per_proxy; /* How many clients can connect to each subproxy */ + int64_t client_headroom; /* How many more clients can we connect */ + proxy_instance_t *proxy; /* Parent proxy of subproxies */ + proxy_instance_t *subproxies; /* Hashlist of subproxies of this proxy */ + int subproxy_count; /* Number of subproxies */ }; /* Private data for the generator */ @@ -630,15 +638,18 @@ retry: LOGWARNING("Invalid nonce2len %d in parse_subscribe", size); goto out; } - if (size == 3 || (size == 4 && proxi->ckp->clientsvspeed)) - LOGWARNING("Proxy %d:%s Nonce2 length %d means proxied clients can't be >5TH each", - proxi->id, proxi->si->url, size); - else if (size < 3) { + if (size < 3) { LOGWARNING("Proxy %d:%s Nonce2 length %d too small to be able to proxy", proxi->id, proxi->si->url, size); goto out; } proxi->nonce2len = size; + if (!proxi->proxy) { + /* Set the number of clients per proxy on the parent proxy */ + proxi->clients_per_proxy = 1ll << ((size - 3) * 8); + LOGNOTICE("Proxy %d:%s clients per proxy: %"PRId64, proxi->id, proxi->si->url, + proxi->clients_per_proxy); + } LOGINFO("Found notify with enonce %s nonce2len %d", proxi->enonce1, proxi->nonce2len); @@ -1360,8 +1371,9 @@ static void passthrough_add_send(proxy_instance_t *proxi, const char *msg) } static bool proxy_alive(ckpool_t *ckp, server_instance_t *si, proxy_instance_t *proxi, - connsock_t *cs, bool pinging) + connsock_t *cs, bool pinging, int epfd) { + struct epoll_event event; bool ret = false; /* Has this proxy already been reconnected? */ @@ -1411,6 +1423,11 @@ out: keep_sockalive(cs->fd); if (!ckp->passthrough) send_subscribe(ckp, proxi); + event.events = EPOLLIN; + event.data.ptr = proxi; + /* Add this connsock_t to the epoll list */ + if (unlikely(epoll_ctl(epfd, EPOLL_CTL_ADD, cs->fd, &event) == -1)) + quit(1, "FATAL: Failed to add epfd to epoll_ctl in proxy_alive"); } return ret; } @@ -1423,10 +1440,18 @@ static void *passthrough_recv(void *arg) server_instance_t *si = proxi->si; connsock_t *cs = proxi->cs; ckpool_t *ckp = proxi->ckp; + struct epoll_event event; + int epfd; rename_proc("passrecv"); - if (proxy_alive(ckp, si, proxi, cs, false)) { + epfd = epoll_create1(EPOLL_CLOEXEC); + if (epfd < 0){ + LOGEMERG("FATAL: Failed to create epoll in passrecv"); + return NULL; + } + + if (proxy_alive(ckp, si, proxi, cs, false, epfd)) { proxi->alive = true; send_proc(ckp->generator, "reconnect"); LOGWARNING("Proxy %d:%s connection established", @@ -1436,7 +1461,7 @@ static void *passthrough_recv(void *arg) while (42) { int ret; - while (!proxy_alive(ckp, si, proxi, cs, true)) { + while (!proxy_alive(ckp, si, proxi, cs, true, epfd)) { if (proxi->alive) { proxi->alive = false; send_proc(ckp->generator, "reconnect"); @@ -1448,10 +1473,10 @@ static void *passthrough_recv(void *arg) send_proc(ckp->generator, "reconnect"); } - do { + /* Make sure we receive a line within 90 seconds */ + ret = epoll_wait(epfd, &event, 1, 90000); + if (likely(ret > 0)) ret = read_socket_line(cs, 60); - } while (ret == 0); - if (ret < 1) { LOGWARNING("Proxy %d:%s failed to read_socket_line in proxy_recv, attempting reconnect", proxi->id, proxi->si->url); @@ -1491,10 +1516,18 @@ static void *proxy_recv(void *arg) connsock_t *cs = proxi->cs; ckpool_t *ckp = proxi->ckp; gdata_t *gdata = ckp->data; + struct epoll_event event; + int epfd; rename_proc("proxyrecv"); - if (proxy_alive(ckp, si, proxi, cs, false)) { + epfd = epoll_create1(EPOLL_CLOEXEC); + if (epfd < 0){ + LOGEMERG("FATAL: Failed to create epoll in proxyrecv"); + return NULL; + } + + if (proxy_alive(ckp, si, proxi, cs, false, epfd)) { proxi->alive = true; send_proc(ckp->generator, "reconnect"); LOGWARNING("Proxy %d:%s connection established", @@ -1504,10 +1537,10 @@ static void *proxy_recv(void *arg) while (42) { notify_instance_t *ni, *tmp; share_msg_t *share, *tmpshare; - int retries = 0, ret; time_t now; + int ret; - while (!proxy_alive(ckp, si, proxi, cs, true)) { + while (!proxy_alive(ckp, si, proxi, cs, true, epfd)) { if (proxi->alive) { proxi->alive = false; send_proc(ckp->generator, "reconnect"); @@ -1550,17 +1583,12 @@ static void *proxy_recv(void *arg) /* If we don't get an update within 10 minutes the upstream pool * has likely stopped responding. */ - do { - if (cs->fd == -1) { - ret = -1; - break; - } + ret = epoll_wait(epfd, &event, 1, 600000); + if (likely(ret > 0)) ret = read_socket_line(cs, 5); - } while (ret == 0 && ++retries < 120); - if (ret < 1) { if (proxi->alive) { - LOGWARNING("Proxy %d:%s failed to read_socket_line in proxy_recv, attempting reconnect", + LOGWARNING("Proxy %d:%s failed to epoll/read_socket_line in proxy_recv, attempting reconnect", proxi->id, proxi->si->url); } continue; @@ -1589,8 +1617,30 @@ static void *proxy_recv(void *arg) return NULL; } +/* Creates a duplicate instance or proxi to be used as a subproxy, ignoring + * fields we don't use in the subproxy. */ +static proxy_instance_t *create_subproxy(proxy_instance_t *proxi) +{ + proxy_instance_t *subproxy = ckzalloc(sizeof(proxy_instance_t)); + + subproxy->ckp = proxi->ckp; + subproxy->cs = ckzalloc(sizeof(connsock_t)); + subproxy->si = proxi->si; + subproxy->id = proxi->subproxy_count++; + subproxy->auth = proxi->auth; + subproxy->pass = proxi->pass; + subproxy->proxy = proxi; + return subproxy; +} + +/* Create a single subproxy instance immediately to be the first used + * by the stratifier. To be used in future code */ static void prepare_proxy(proxy_instance_t *proxi) { + proxy_instance_t *subproxy = create_subproxy(proxi); + + mutex_init(&proxi->proxy_lock); + HASH_ADD_INT(proxi->subproxies, id, subproxy); mutex_init(&proxi->psend_lock); cond_init(&proxi->psend_cond); create_pthread(&proxi->pth_psend, proxy_send, proxi); From 04d43b3afba280649c292b9c9d241c55901efabc Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Mon, 9 Feb 2015 15:45:26 +1100 Subject: [PATCH 041/544] Recruit extra subproxies to ensure at least 42 client headroom, receiving data from correct socket in proxyrecv --- src/generator.c | 81 ++++++++++++++++++++++++++++++++++--------------- 1 file changed, 57 insertions(+), 24 deletions(-) diff --git a/src/generator.c b/src/generator.c index acd61fcf..8cba9777 100644 --- a/src/generator.c +++ b/src/generator.c @@ -644,7 +644,7 @@ retry: goto out; } proxi->nonce2len = size; - if (!proxi->proxy) { + if (proxi->proxy == proxi) { /* Set the number of clients per proxy on the parent proxy */ proxi->clients_per_proxy = 1ll << ((size - 3) * 8); LOGNOTICE("Proxy %d:%s clients per proxy: %"PRId64, proxi->id, proxi->si->url, @@ -1370,6 +1370,8 @@ static void passthrough_add_send(proxy_instance_t *proxi, const char *msg) ckmsgq_add(proxi->passsends, pm); } +static bool recruit_subproxy(proxy_instance_t *proxi, int epfd); + static bool proxy_alive(ckpool_t *ckp, server_instance_t *si, proxy_instance_t *proxi, connsock_t *cs, bool pinging, int epfd) { @@ -1428,10 +1430,57 @@ out: /* Add this connsock_t to the epoll list */ if (unlikely(epoll_ctl(epfd, EPOLL_CTL_ADD, cs->fd, &event) == -1)) quit(1, "FATAL: Failed to add epfd to epoll_ctl in proxy_alive"); + if (!ckp->passthrough && proxi->proxy == proxi) { + while (proxi->client_headroom < 42) { + /* Note recursive call of proxy_alive here */ + if (!recruit_subproxy(proxi, epfd)) { + LOGWARNING("Unable to recruit extra subproxies after just %"PRId64, + proxi->client_headroom); + break; + } + LOGWARNING("Proxy %d:%s recruited extra subproxy!", + proxi->id, cs->url); + } + } } return ret; } +/* Creates a duplicate instance or proxi to be used as a subproxy, ignoring + * fields we don't use in the subproxy. */ +static proxy_instance_t *create_subproxy(proxy_instance_t *proxi) +{ + proxy_instance_t *subproxy = ckzalloc(sizeof(proxy_instance_t)); + + subproxy->ckp = proxi->ckp; + subproxy->cs = ckzalloc(sizeof(connsock_t)); + subproxy->si = proxi->si; + subproxy->id = proxi->subproxy_count++; + subproxy->auth = proxi->auth; + subproxy->pass = proxi->pass; + subproxy->proxy = proxi; + return subproxy; +} + +static bool recruit_subproxy(proxy_instance_t *proxi, int epfd) +{ + proxy_instance_t *subproxy = create_subproxy(proxi); + + if (!proxy_alive(subproxy->ckp, subproxy->si, subproxy, subproxy->cs, false, epfd)) { + LOGNOTICE("Subproxy failed proxy_alive testing"); + free(subproxy->cs); + free(subproxy); + return false; + } + + mutex_lock(&proxi->proxy_lock); + HASH_ADD_INT(proxi->subproxies, id, subproxy); + proxi->client_headroom += proxi->clients_per_proxy; + mutex_unlock(&proxi->proxy_lock); + + return true; +} + /* For receiving messages from an upstream pool to pass downstream. Responsible * for setting up the connection and testing pool is live. */ static void *passthrough_recv(void *arg) @@ -1537,10 +1586,11 @@ static void *proxy_recv(void *arg) while (42) { notify_instance_t *ni, *tmp; share_msg_t *share, *tmpshare; + proxy_instance_t *subproxy; time_t now; int ret; - while (!proxy_alive(ckp, si, proxi, cs, true, epfd)) { + while (!proxy_alive(ckp, si, proxi, proxi->cs, true, epfd)) { if (proxi->alive) { proxi->alive = false; send_proc(ckp->generator, "reconnect"); @@ -1584,8 +1634,11 @@ static void *proxy_recv(void *arg) /* If we don't get an update within 10 minutes the upstream pool * has likely stopped responding. */ ret = epoll_wait(epfd, &event, 1, 600000); - if (likely(ret > 0)) + if (likely(ret > 0)) { + subproxy = event.data.ptr; + cs = subproxy->cs; ret = read_socket_line(cs, 5); + } if (ret < 1) { if (proxi->alive) { LOGWARNING("Proxy %d:%s failed to epoll/read_socket_line in proxy_recv, attempting reconnect", @@ -1617,30 +1670,10 @@ static void *proxy_recv(void *arg) return NULL; } -/* Creates a duplicate instance or proxi to be used as a subproxy, ignoring - * fields we don't use in the subproxy. */ -static proxy_instance_t *create_subproxy(proxy_instance_t *proxi) -{ - proxy_instance_t *subproxy = ckzalloc(sizeof(proxy_instance_t)); - - subproxy->ckp = proxi->ckp; - subproxy->cs = ckzalloc(sizeof(connsock_t)); - subproxy->si = proxi->si; - subproxy->id = proxi->subproxy_count++; - subproxy->auth = proxi->auth; - subproxy->pass = proxi->pass; - subproxy->proxy = proxi; - return subproxy; -} - -/* Create a single subproxy instance immediately to be the first used - * by the stratifier. To be used in future code */ static void prepare_proxy(proxy_instance_t *proxi) { - proxy_instance_t *subproxy = create_subproxy(proxi); - + proxi->proxy = proxi; mutex_init(&proxi->proxy_lock); - HASH_ADD_INT(proxi->subproxies, id, subproxy); mutex_init(&proxi->psend_lock); cond_init(&proxi->psend_cond); create_pthread(&proxi->pth_psend, proxy_send, proxi); From a006dab4ae809635edff213bd5eba20ac981aee0 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Mon, 9 Feb 2015 16:15:59 +1100 Subject: [PATCH 042/544] Do not use master proxy for work, using the first subproxy instance for now, sending the id with it to the stratifier --- src/generator.c | 48 ++++++++++++++++++++++++++++++------------------ 1 file changed, 30 insertions(+), 18 deletions(-) diff --git a/src/generator.c b/src/generator.c index 8cba9777..3b85f0c4 100644 --- a/src/generator.c +++ b/src/generator.c @@ -976,8 +976,13 @@ static void send_diff(ckpool_t *ckp, proxy_instance_t *proxi) json_t *json_msg; char *msg, *buf; - JSON_CPACK(json_msg, "{sisf}", - "proxy", proxi->id, + /* Master proxy, we don't use this for work */ + if (proxi == proxi->proxy) + return; + + JSON_CPACK(json_msg, "{sisisf}", + "proxy", proxi->proxy->id, + "subproxy", proxi->id, "diff", proxi->diff); msg = json_dumps(json_msg, JSON_NO_UTF8); json_decref(json_msg); @@ -994,6 +999,10 @@ static void send_notify(ckpool_t *ckp, proxy_instance_t *proxi) char *msg, *buf; int i; + /* Master proxy, we don't use this for work */ + if (proxi == proxi->proxy) + return; + merkle_arr = json_array(); mutex_lock(&proxi->notify_lock); @@ -1006,7 +1015,8 @@ static void send_notify(ckpool_t *ckp, proxy_instance_t *proxi) for (i = 0; i < ni->merkles; i++) json_array_append_new(merkle_arr, json_string(&ni->merklehash[i][0])); /* Use our own jobid instead of the server's one for easy lookup */ - JSON_CPACK(json_msg, "{si,si,ss,si,ss,ss,so,ss,ss,ss,sb}", "proxy", proxi->id, + JSON_CPACK(json_msg, "{sisisisssisssssosssssssb}", + "proxy", proxi->proxy->id, "subproxy", proxi->id, "jobid", ni->id, "prevhash", ni->prevhash, "coinb1len", ni->coinb1len, "coinbase1", ni->coinbase1, "coinbase2", ni->coinbase2, "merklehash", merkle_arr, "bbversion", ni->bbversion, @@ -1198,8 +1208,13 @@ static void send_subscribe(ckpool_t *ckp, proxy_instance_t *proxi) json_t *json_msg; char *msg, *buf; - JSON_CPACK(json_msg, "{sisssi}", - "proxy", proxi->id, + /* Master proxy, we don't use this for work */ + if (proxi == proxi->proxy) + return; + + JSON_CPACK(json_msg, "{sisisssi}", + "proxy", proxi->proxy->id, + "subproxy", proxi->id, "enonce1", proxi->enonce1, "nonce2len", proxi->nonce2len); msg = json_dumps(json_msg, JSON_NO_UTF8); @@ -1584,9 +1599,9 @@ static void *proxy_recv(void *arg) } while (42) { - notify_instance_t *ni, *tmp; + proxy_instance_t *subproxy = proxi; share_msg_t *share, *tmpshare; - proxy_instance_t *subproxy; + notify_instance_t *ni, *tmp; time_t now; int ret; @@ -1640,28 +1655,28 @@ static void *proxy_recv(void *arg) ret = read_socket_line(cs, 5); } if (ret < 1) { - if (proxi->alive) { + if (subproxy->alive) { LOGWARNING("Proxy %d:%s failed to epoll/read_socket_line in proxy_recv, attempting reconnect", - proxi->id, proxi->si->url); + subproxy->id, subproxy->si->url); } continue; } - if (parse_method(ckp, proxi, cs->buf)) { - if (proxi->reconnect) { + if (parse_method(ckp, subproxy, cs->buf)) { + if (subproxy->reconnect) { /* Call this proxy dead to allow us to fail * over to a backup pool until the reconnect * pool is up */ - proxi->reconnect = false; - proxi->alive = false; + subproxy->reconnect = false; + subproxy->alive = false; send_proc(ckp->generator, "reconnect"); LOGWARNING("Proxy %d:%s reconnect issue, dropping existing connection", - proxi->id, proxi->si->url); + subproxy->id, subproxy->si->url); Close(cs->fd); break; } continue; } - if (parse_share(proxi, cs->buf)) { + if (parse_share(subproxy, cs->buf)) { continue; } /* If it's not a method it should be a share result */ @@ -1756,9 +1771,6 @@ reconnect: dealloc(buf); ASPRINTF(&buf, "proxy=%d", proxi->id); send_proc(ckp->stratifier, buf); - /* Send a notify for the new chosen proxy or the - * stratifier won't be able to switch. */ - send_notify(ckp, proxi); } } retry: From abde70a8d9425cfcdc16edaf9e8267c4ce3bc8fb Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Mon, 9 Feb 2015 16:27:25 +1100 Subject: [PATCH 043/544] Store the id and subid of the proxy in use by each client, submitting the value with its shares to the generator --- src/stratifier.c | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/src/stratifier.c b/src/stratifier.c index bf819694..c0b44a86 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -272,6 +272,9 @@ struct stratum_instance { int64_t suggest_diff; /* Stratum client suggested diff */ double best_diff; /* Best share found by this instance */ + + int proxyid; /* Which proxy this is bound to in proxy mode */ + int subproxyid; /* Which subproxy */ }; struct share { @@ -285,6 +288,7 @@ typedef struct share share_t; struct proxy_base { UT_hash_handle hh; int id; + int subid; double diff; @@ -2021,12 +2025,19 @@ static void __fill_enonce1data(const workbase_t *wb, stratum_instance_t *client) * When the proxy space is less than 32 bits to work with, we look for an * unused enonce1 value and reject clients instead if there is no space left. * Needs to be entered with client holding a ref count. */ -static bool new_enonce1(stratum_instance_t *client) +static bool new_enonce1(sdata_t *sdata, stratum_instance_t *client) { - sdata_t *sdata = client->ckp->data; int enonce1varlen, i; bool ret = false; + if (!sdata->proxy) + return false; + + mutex_lock(&sdata->proxy_lock); + client->proxyid = sdata->proxy->id; + client->subproxyid = sdata->proxy->subid; + mutex_unlock(&sdata->proxy_lock); + /* Extract the enonce1varlen from the current workbase which may be * a different workbase to when we __fill_enonce1data but the value * will not change and this avoids grabbing recursive locks */ @@ -2134,7 +2145,7 @@ static json_t *parse_subscribe(stratum_instance_t *client, const int64_t client_ client->useragent = ckzalloc(1); if (!old_match) { /* Create a new extranonce1 based on a uint64_t pointer */ - if (!new_enonce1(client)) { + if (!new_enonce1(sdata, client)) { stratum_send_message(sdata, client, "Pool full of clients"); client->reject = 2; return json_string("proxy full"); @@ -2977,9 +2988,10 @@ static void submit_share(stratum_instance_t *client, const int64_t jobid, const char *msg; sprintf(enonce2, "%s%s", client->enonce1var, nonce2); - JSON_CPACK(json_msg, "{sisssssssIsi}", "jobid", jobid, "nonce2", enonce2, + JSON_CPACK(json_msg, "{sisssssssIsisisi}", "jobid", jobid, "nonce2", enonce2, "ntime", ntime, "nonce", nonce, "client_id", client->id, - "msg_id", msg_id); + "msg_id", msg_id, "proxy", client->proxyid, + "subproxy", client->subproxyid); msg = json_dumps(json_msg, 0); json_decref(json_msg); send_generator(ckp, msg, GEN_LAX); From 3b40b805bd60489ee928fae3eef439f7bead72f7 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Mon, 9 Feb 2015 16:55:00 +1100 Subject: [PATCH 044/544] Store the proxy and subproxy id work is generated from --- src/ckpool.c | 1 - src/ckpool.h | 3 --- src/stratifier.c | 30 +++++++++++++----------------- 3 files changed, 13 insertions(+), 21 deletions(-) diff --git a/src/ckpool.c b/src/ckpool.c index bf9b6286..95490c5e 100644 --- a/src/ckpool.c +++ b/src/ckpool.c @@ -1165,7 +1165,6 @@ static void parse_config(ckpool_t *ckp) if (arr_size) parse_proxies(ckp, arr_val, arr_size); } - json_get_bool(&ckp->clientsvspeed, json_conf, "clientsvspeed"); json_decref(json_conf); } diff --git a/src/ckpool.h b/src/ckpool.h index 3f6935ae..9b2eb5f9 100644 --- a/src/ckpool.h +++ b/src/ckpool.h @@ -148,9 +148,6 @@ struct ckpool_instance { /* Are we running as a proxy */ bool proxy; - /* Do we prefer more proxy clients over support for >5TH clients */ - bool clientsvspeed; - /* Are we running without ckdb */ bool standalone; diff --git a/src/stratifier.c b/src/stratifier.c index c0b44a86..a984a155 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -126,7 +126,9 @@ struct workbase { char *logdir; ckpool_t *ckp; - bool proxy; + bool proxy; /* This workbase is proxied work */ + int proxyid; /* The id of the proxy the workbase is from */ + int subproxyid; /* The id of the subproxy the worbase is from */ }; typedef struct workbase workbase_t; @@ -1077,21 +1079,12 @@ static void update_subscribe(ckpool_t *ckp, const char *cmd) proxy->enonce1constlen = strlen(proxy->enonce1) / 2; hex2bin(proxy->enonce1bin, proxy->enonce1, proxy->enonce1constlen); proxy->nonce2len = json_integer_value(json_object_get(val, "nonce2len")); - if (ckp->clientsvspeed) { - if (proxy->nonce2len > 5) - proxy->enonce1varlen = 4; - else if (proxy->nonce2len > 3) - proxy->enonce1varlen = 2; - else - proxy->enonce1varlen = 1; - } else { - if (proxy->nonce2len > 7) - proxy->enonce1varlen = 4; - else if (proxy->nonce2len > 5) - proxy->enonce1varlen = 2; - else - proxy->enonce1varlen = 1; - } + if (proxy->nonce2len > 7) + proxy->enonce1varlen = 4; + else if (proxy->nonce2len > 5) + proxy->enonce1varlen = 2; + else + proxy->enonce1varlen = 1; proxy->enonce2varlen = proxy->nonce2len - proxy->enonce1varlen; ck_wunlock(&sdata->workbase_lock); @@ -1105,11 +1098,11 @@ static void update_notify(ckpool_t *ckp, const char *cmd) { bool new_block = false, clean; sdata_t *sdata = ckp->data; + int i, id = 0, subid = 0; char header[228]; const char *buf; proxy_t *proxy; workbase_t *wb; - int i, id = 0; json_t *val; if (unlikely(strlen(cmd) < 8)) { @@ -1125,6 +1118,7 @@ static void update_notify(ckpool_t *ckp, const char *cmd) return; } json_get_int(&id, val, "proxy"); + json_get_int(&subid, val, "subproxy"); proxy = proxy_by_id(sdata, id); if (unlikely(!proxy->subscribed)) { LOGNOTICE("No valid proxy %d subscription to update notify yet", id); @@ -1147,6 +1141,8 @@ static void update_notify(ckpool_t *ckp, const char *cmd) wb = ckzalloc(sizeof(workbase_t)); wb->ckp = ckp; wb->proxy = true; + wb->proxyid = id; + wb->subproxyid = subid; json_int64cpy(&wb->id, val, "jobid"); json_strcpy(wb->prevhash, val, "prevhash"); From 575ac70de8f5e0590745b457419f5f689d74f535 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Mon, 9 Feb 2015 17:18:52 +1100 Subject: [PATCH 045/544] Store all notifies in the generator in the parent proxy list and extract the subproxy details from share submission to submit upstream to the right connection --- src/generator.c | 85 ++++++++++++++++++++++++++++++++----------------- 1 file changed, 56 insertions(+), 29 deletions(-) diff --git a/src/generator.c b/src/generator.c index 3b85f0c4..9fd26cce 100644 --- a/src/generator.c +++ b/src/generator.c @@ -770,6 +770,7 @@ out: static bool parse_notify(proxy_instance_t *proxi, json_t *val) { const char *prev_hash, *bbversion, *nbit, *ntime; + proxy_instance_t *proxy = proxi->proxy; char *job_id, *coinbase1, *coinbase2; gdata_t *gdata = proxi->ckp->data; bool clean, ret = false; @@ -830,11 +831,13 @@ static bool parse_notify(proxy_instance_t *proxi, json_t *val) ret = true; ni->notify_time = time(NULL); - mutex_lock(&proxi->notify_lock); + /* Add the notify instance to the parent proxy list, not the subproxy */ + mutex_lock(&proxy->notify_lock); ni->id = gdata->proxy_notify_id++; - HASH_ADD_INT(proxi->notify_instances, id, ni); + HASH_ADD_INT(proxy->notify_instances, id, ni); + /* Now set the subproxy's current notify to this */ proxi->current_notify = ni; - mutex_unlock(&proxi->notify_lock); + mutex_unlock(&proxy->notify_lock); out: return ret; @@ -973,15 +976,16 @@ out: static void send_diff(ckpool_t *ckp, proxy_instance_t *proxi) { + proxy_instance_t *proxy = proxi->proxy; json_t *json_msg; char *msg, *buf; /* Master proxy, we don't use this for work */ - if (proxi == proxi->proxy) + if (proxi == proxy) return; JSON_CPACK(json_msg, "{sisisf}", - "proxy", proxi->proxy->id, + "proxy", proxy->id, "subproxy", proxi->id, "diff", proxi->diff); msg = json_dumps(json_msg, JSON_NO_UTF8); @@ -994,21 +998,22 @@ static void send_diff(ckpool_t *ckp, proxy_instance_t *proxi) static void send_notify(ckpool_t *ckp, proxy_instance_t *proxi) { + proxy_instance_t *proxy = proxi->proxy; json_t *json_msg, *merkle_arr; notify_instance_t *ni; char *msg, *buf; int i; /* Master proxy, we don't use this for work */ - if (proxi == proxi->proxy) + if (proxi == proxy) return; merkle_arr = json_array(); - mutex_lock(&proxi->notify_lock); + mutex_lock(&proxy->notify_lock); ni = proxi->current_notify; if (unlikely(!ni)) { - mutex_unlock(&proxi->notify_lock); + mutex_unlock(&proxy->notify_lock); LOGNOTICE("Proxi %d not ready to send notify", proxi->id); return; } @@ -1016,13 +1021,13 @@ static void send_notify(ckpool_t *ckp, proxy_instance_t *proxi) json_array_append_new(merkle_arr, json_string(&ni->merklehash[i][0])); /* Use our own jobid instead of the server's one for easy lookup */ JSON_CPACK(json_msg, "{sisisisssisssssosssssssb}", - "proxy", proxi->proxy->id, "subproxy", proxi->id, + "proxy", proxy->id, "subproxy", proxi->id, "jobid", ni->id, "prevhash", ni->prevhash, "coinb1len", ni->coinb1len, "coinbase1", ni->coinbase1, "coinbase2", ni->coinbase2, "merklehash", merkle_arr, "bbversion", ni->bbversion, "nbit", ni->nbit, "ntime", ni->ntime, "clean", ni->clean); - mutex_unlock(&proxi->notify_lock); + mutex_unlock(&proxy->notify_lock); msg = json_dumps(json_msg, JSON_NO_UTF8); json_decref(json_msg); @@ -1030,6 +1035,10 @@ static void send_notify(ckpool_t *ckp, proxy_instance_t *proxi) free(msg); send_proc(ckp->stratifier, buf); free(buf); + + /* Send diff now as stratifier will not accept diff till it has a + * valid workbase */ + send_diff(ckp, proxi); } static bool parse_method(ckpool_t *ckp, proxy_instance_t *proxi, const char *msg) @@ -1302,59 +1311,77 @@ out: return ret; } -/* For processing and sending shares */ +static proxy_instance_t *subproxy_by_id(proxy_instance_t *proxy, const int id) +{ + proxy_instance_t *subproxy; + + mutex_lock(&proxy->proxy_lock); + HASH_FIND_INT(proxy->subproxies, &id, subproxy); + mutex_unlock(&proxy->proxy_lock); + + return subproxy; +} + +/* For processing and sending shares. proxy refers to parent proxy here */ static void *proxy_send(void *arg) { - proxy_instance_t *proxi = (proxy_instance_t *)arg; - connsock_t *cs = proxi->cs; + proxy_instance_t *proxy = (proxy_instance_t *)arg; + connsock_t *cs = proxy->cs; rename_proc("proxysend"); while (42) { + proxy_instance_t *subproxy; notify_instance_t *ni; stratum_msg_t *msg; char *jobid = NULL; bool ret = true; + int subid = 0; json_t *val; uint32_t id; - mutex_lock(&proxi->psend_lock); - if (!proxi->psends) - pthread_cond_wait(&proxi->psend_cond, &proxi->psend_lock); - msg = proxi->psends; + mutex_lock(&proxy->psend_lock); + if (!proxy->psends) + pthread_cond_wait(&proxy->psend_cond, &proxy->psend_lock); + msg = proxy->psends; if (likely(msg)) - DL_DELETE(proxi->psends, msg); - mutex_unlock(&proxi->psend_lock); + DL_DELETE(proxy->psends, msg); + mutex_unlock(&proxy->psend_lock); if (unlikely(!msg)) continue; + json_get_int(&subid, msg->json_msg, "subproxy"); json_uintcpy(&id, msg->json_msg, "jobid"); - mutex_lock(&proxi->notify_lock); - HASH_FIND_INT(proxi->notify_instances, &id, ni); + mutex_lock(&proxy->notify_lock); + HASH_FIND_INT(proxy->notify_instances, &id, ni); if (ni) jobid = strdup(ni->jobid); - mutex_unlock(&proxi->notify_lock); + mutex_unlock(&proxy->notify_lock); + + subproxy = subproxy_by_id(proxy, subid); - if (jobid) { - JSON_CPACK(val, "{s[ssooo]soss}", "params", proxi->auth, jobid, + if (jobid && subproxy) { + cs = subproxy->cs; + JSON_CPACK(val, "{s[ssooo]soss}", "params", proxy->auth, jobid, json_object_dup(msg->json_msg, "nonce2"), json_object_dup(msg->json_msg, "ntime"), json_object_dup(msg->json_msg, "nonce"), "id", json_object_dup(msg->json_msg, "id"), "method", "mining.submit"); - free(jobid); ret = send_json_msg(cs, val); json_decref(val); - } else + } else { LOGNOTICE("Proxy %d:%s failed to find matching jobid in proxysend", - proxi->id, proxi->si->url); + proxy->id, proxy->si->url); + } + free(jobid); json_decref(msg->json_msg); free(msg); - if (!ret && cs->fd > 0) { + if (!ret && subproxy && cs->fd > 0) { LOGWARNING("Proxy %d:%s failed to send msg in proxy_send, dropping to reconnect", - proxi->id, proxi->si->url); + proxy->id, proxy->si->url); Close(cs->fd); } } From c92cbd6f62653f32b247c63ff109a0b160991f60 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Mon, 9 Feb 2015 17:37:31 +1100 Subject: [PATCH 046/544] Handle submitting upstream shares properly with subproxies having their own share tables --- src/generator.c | 46 +++++++++++++++++++++++++++++++--------------- 1 file changed, 31 insertions(+), 15 deletions(-) diff --git a/src/generator.c b/src/generator.c index 9fd26cce..dd8182ec 100644 --- a/src/generator.c +++ b/src/generator.c @@ -1199,7 +1199,6 @@ out: return ret; } -#if 0 static proxy_instance_t *proxy_by_id(gdata_t *gdata, const int id) { proxy_instance_t *proxi; @@ -1210,7 +1209,6 @@ static proxy_instance_t *proxy_by_id(gdata_t *gdata, const int id) return proxi; } -#endif static void send_subscribe(ckpool_t *ckp, proxy_instance_t *proxi) { @@ -1234,10 +1232,38 @@ static void send_subscribe(ckpool_t *ckp, proxy_instance_t *proxi) free(buf); } -static void submit_share(proxy_instance_t *proxi, json_t *val) +static proxy_instance_t *subproxy_by_id(proxy_instance_t *proxy, const int id) +{ + proxy_instance_t *subproxy; + + mutex_lock(&proxy->proxy_lock); + HASH_FIND_INT(proxy->subproxies, &id, subproxy); + mutex_unlock(&proxy->proxy_lock); + + return subproxy; +} + +static void submit_share(gdata_t *gdata, json_t *val) { + proxy_instance_t *proxy, *proxi; stratum_msg_t *msg; share_msg_t *share; + int id, subid; + + json_get_int(&id, val, "proxy"); + json_object_del(val, "proxy"); + proxy = proxy_by_id(gdata, id); + if (unlikely(!proxy)) { + LOGWARNING("Failed to find proxy %d to send share to", id); + return json_decref(val); + } + json_get_int(&subid, val, "subproxy"); + json_object_del(val, "subproxy"); + proxi = subproxy_by_id(proxy, subid); + if (unlikely(!proxi)) { + LOGWARNING("Failed to find subproxy %d to send share to", subid); + return json_decref(val); + } msg = ckzalloc(sizeof(stratum_msg_t)); share = ckzalloc(sizeof(share_msg_t)); @@ -1311,17 +1337,6 @@ out: return ret; } -static proxy_instance_t *subproxy_by_id(proxy_instance_t *proxy, const int id) -{ - proxy_instance_t *subproxy; - - mutex_lock(&proxy->proxy_lock); - HASH_FIND_INT(proxy->subproxies, &id, subproxy); - mutex_unlock(&proxy->proxy_lock); - - return subproxy; -} - /* For processing and sending shares. proxy refers to parent proxy here */ static void *proxy_send(void *arg) { @@ -1501,6 +1516,7 @@ static proxy_instance_t *create_subproxy(proxy_instance_t *proxi) subproxy->auth = proxi->auth; subproxy->pass = proxi->pass; subproxy->proxy = proxi; + mutex_init(&subproxy->share_lock); return subproxy; } @@ -1854,7 +1870,7 @@ retry: if (unlikely(!val)) LOGWARNING("Generator received unrecognised message: %s", buf); else - submit_share(proxi, val); + submit_share(gdata, val); } Close(sockd); goto retry; From e986c04c6895c0f64074cba0439f5716ba9e8c14 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Mon, 9 Feb 2015 17:38:01 +1100 Subject: [PATCH 047/544] Fix non proxy mode breakage --- src/stratifier.c | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/src/stratifier.c b/src/stratifier.c index a984a155..c0a59bba 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -2026,13 +2026,14 @@ static bool new_enonce1(sdata_t *sdata, stratum_instance_t *client) int enonce1varlen, i; bool ret = false; - if (!sdata->proxy) - return false; - - mutex_lock(&sdata->proxy_lock); - client->proxyid = sdata->proxy->id; - client->subproxyid = sdata->proxy->subid; - mutex_unlock(&sdata->proxy_lock); + if (client->ckp->proxy) { + if (!sdata->proxy) + return false; + mutex_lock(&sdata->proxy_lock); + client->proxyid = sdata->proxy->id; + client->subproxyid = sdata->proxy->subid; + mutex_unlock(&sdata->proxy_lock); + } /* Extract the enonce1varlen from the current workbase which may be * a different workbase to when we __fill_enonce1data but the value From 08e47d28cf788bb9cab51a5d5ae324dc743e358b Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Mon, 9 Feb 2015 21:37:06 +1100 Subject: [PATCH 048/544] Give each proxy an enonce1 union and cap the handout of enonce1 values without repeating, assuming we will recruit more proxied connections upstream to compensate --- src/stratifier.c | 109 ++++++++++++++++++++++++----------------------- 1 file changed, 55 insertions(+), 54 deletions(-) diff --git a/src/stratifier.c b/src/stratifier.c index c0a59bba..802b9d58 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -287,6 +287,14 @@ struct share { typedef struct share share_t; +/* Variable length enonce1 always refers back to a u64 */ +typedef union { + uint64_t u64; + uint32_t u32; + uint16_t u16; + uint8_t u8; +} enonce1_t; + struct proxy_base { UT_hash_handle hh; int id; @@ -304,6 +312,10 @@ struct proxy_base { bool subscribed; bool notified; + + int64_t clients; + int64_t max_clients; + enonce1_t enonce1u; }; typedef struct proxy_base proxy_t; @@ -321,13 +333,7 @@ struct stratifier_data { bool ckdb_offline; - /* Variable length enonce1 always refers back to a u64 */ - union { - uint64_t u64; - uint32_t u32; - uint16_t u16; - uint8_t u8; - } enonce1u; + enonce1_t enonce1u; /* For protecting the hashtable data */ cklock_t workbase_lock; @@ -1083,13 +1089,16 @@ static void update_subscribe(ckpool_t *ckp, const char *cmd) proxy->enonce1varlen = 4; else if (proxy->nonce2len > 5) proxy->enonce1varlen = 2; - else + else if (proxy->nonce2len > 3) proxy->enonce1varlen = 1; + else + proxy->enonce1varlen = 0; proxy->enonce2varlen = proxy->nonce2len - proxy->enonce1varlen; + proxy->max_clients = 1ll << (proxy->enonce1varlen * 8); ck_wunlock(&sdata->workbase_lock); - LOGNOTICE("Upstream pool extranonce2 length %d, max proxy clients %lld", - proxy->nonce2len, 1ll << (proxy->enonce1varlen * 8)); + LOGNOTICE("Upstream pool extranonce2 length %d, max proxy clients %"PRId64, + proxy->nonce2len, proxy->max_clients); json_decref(val); } @@ -1985,34 +1994,15 @@ static void *blockupdate(void *arg) return NULL; } -/* Enter holding instance_lock */ -static bool __enonce1_free(sdata_t *sdata, const uint64_t enonce1) -{ - stratum_instance_t *client, *tmp; - bool ret = true; - - if (unlikely(!enonce1)) { - ret = false; - goto out; - } - - HASH_ITER(hh, sdata->stratum_instances, client, tmp) { - if (client->enonce1_64 == enonce1) { - ret = false; - break; - } - } -out: - return ret; -} - /* Enter holding workbase_lock and client a ref count. */ static void __fill_enonce1data(const workbase_t *wb, stratum_instance_t *client) { if (wb->enonce1constlen) memcpy(client->enonce1bin, wb->enonce1constbin, wb->enonce1constlen); - memcpy(client->enonce1bin + wb->enonce1constlen, &client->enonce1_64, wb->enonce1varlen); - __bin2hex(client->enonce1var, &client->enonce1_64, wb->enonce1varlen); + if (wb->enonce1varlen) { + memcpy(client->enonce1bin + wb->enonce1constlen, &client->enonce1_64, wb->enonce1varlen); + __bin2hex(client->enonce1var, &client->enonce1_64, wb->enonce1varlen); + } __bin2hex(client->enonce1, client->enonce1bin, wb->enonce1constlen + wb->enonce1varlen); } @@ -2023,17 +2013,28 @@ static void __fill_enonce1data(const workbase_t *wb, stratum_instance_t *client) * Needs to be entered with client holding a ref count. */ static bool new_enonce1(sdata_t *sdata, stratum_instance_t *client) { - int enonce1varlen, i; + proxy_t *proxy = NULL; + enonce1_t *enonce1u; + int enonce1varlen; bool ret = false; if (client->ckp->proxy) { if (!sdata->proxy) return false; + mutex_lock(&sdata->proxy_lock); - client->proxyid = sdata->proxy->id; - client->subproxyid = sdata->proxy->subid; + proxy = sdata->proxy; + enonce1u = &proxy->enonce1u; + client->proxyid = proxy->id; + client->subproxyid = proxy->subid; mutex_unlock(&sdata->proxy_lock); - } + + if (proxy->clients >= proxy->max_clients) { + LOGWARNING("Proxy reached max clients %"PRId64, proxy->max_clients); + return false; + } + } else + enonce1u = &sdata->enonce1u; /* Extract the enonce1varlen from the current workbase which may be * a different workbase to when we __fill_enonce1data but the value @@ -2042,42 +2043,42 @@ static bool new_enonce1(sdata_t *sdata, stratum_instance_t *client) enonce1varlen = sdata->current_workbase->enonce1varlen; ck_runlock(&sdata->workbase_lock); - /* instance_lock protects sdata->enonce1u */ + /* instance_lock protects enonce1u. Recruiting extra proxies should + * prevent these ever locking out.*/ ck_wlock(&sdata->instance_lock); switch(enonce1varlen) { case 8: - sdata->enonce1u.u64++; + enonce1u->u64++; ret = true; break; case 7: case 6: case 5: case 4: - sdata->enonce1u.u32++; + enonce1u->u32++; ret = true; break; case 3: case 2: - for (i = 0; i < 65536; i++) { - sdata->enonce1u.u16++; - ret = __enonce1_free(sdata, sdata->enonce1u.u64); - if (ret) - break; - } + enonce1u->u16++; + ret = true; break; case 1: - for (i = 0; i < 256; i++) { - sdata->enonce1u.u8++; - ret = __enonce1_free(sdata, sdata->enonce1u.u64); - if (ret) - break; - } + enonce1u->u8++; + ret = true; + break; + case 0: + /* Only one client/enonce1 used on this proxy */ + ret = true; break; default: quit(0, "Invalid enonce1varlen %d", enonce1varlen); } - if (ret) - client->enonce1_64 = sdata->enonce1u.u64; + if (ret) { + if (proxy) + proxy->clients++; + client->enonce1_64 = enonce1u->u64; + } ck_wunlock(&sdata->instance_lock); ck_rlock(&sdata->workbase_lock); From d05db855ff152e7db84c17539de5e4970e8491ad Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Tue, 10 Feb 2015 16:36:12 +1100 Subject: [PATCH 049/544] Create infrastructure for a list of subproxies per proxy and generating them --- src/stratifier.c | 37 +++++++++++++++++++++++++++++++++++-- 1 file changed, 35 insertions(+), 2 deletions(-) diff --git a/src/stratifier.c b/src/stratifier.c index 802b9d58..6d15aa9b 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -295,6 +295,8 @@ typedef union { uint8_t u8; } enonce1_t; +typedef struct proxy_base proxy_t; + struct proxy_base { UT_hash_handle hh; int id; @@ -316,9 +318,9 @@ struct proxy_base { int64_t clients; int64_t max_clients; enonce1_t enonce1u; -}; -typedef struct proxy_base proxy_t; + proxy_t *subproxies; /* Hashlist of subproxies sorted by subid */ +}; struct stratifier_data { char pubkeytxnbin[25]; @@ -1002,6 +1004,15 @@ static proxy_t *__generate_proxy(sdata_t *sdata, const int id) return proxy; } +static proxy_t *__generate_subproxy(proxy_t *proxy, const int id) +{ + proxy_t *subproxy = ckzalloc(sizeof(proxy_t)); + + subproxy->id = id; + HASH_ADD_INT(proxy->subproxies, id, subproxy); + return subproxy; +} + /* Find proxy by id number, generate one if none exist yet by that id */ static proxy_t *__proxy_by_id(sdata_t *sdata, const int id) { @@ -1016,6 +1027,16 @@ static proxy_t *__proxy_by_id(sdata_t *sdata, const int id) return proxy; } +static proxy_t *__subproxy_by_id(proxy_t *proxy, const int id) +{ + proxy_t *subproxy; + + HASH_FIND_INT(proxy->subproxies, &id, subproxy); + if (!subproxy) + subproxy = __generate_subproxy(proxy, id); + return subproxy; +} + static proxy_t *proxy_by_id(sdata_t *sdata, const int id) { proxy_t *proxy; @@ -1027,6 +1048,18 @@ static proxy_t *proxy_by_id(sdata_t *sdata, const int id) return proxy; } +static proxy_t *subproxy_by_id(sdata_t *sdata, const int id, const int subid) +{ + proxy_t *proxy, *subproxy; + + mutex_lock(&sdata->proxy_lock); + proxy = __proxy_by_id(sdata, id); + subproxy = __subproxy_by_id(proxy, subid); + mutex_unlock(&sdata->proxy_lock); + + return subproxy; +} + /* Iterates over all clients and sets the reconnect bool for the message * to be sent lazily next time they speak to us. */ static void reconnect_clients(sdata_t *sdata) From b77027b09c95bdc72ba16dad9a42220431e81bb1 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Tue, 10 Feb 2015 18:22:52 +1100 Subject: [PATCH 050/544] Create a unique stratifer data structure per subproxy --- src/stratifier.c | 32 +++++++++++++++++++++++++------- 1 file changed, 25 insertions(+), 7 deletions(-) diff --git a/src/stratifier.c b/src/stratifier.c index 6d15aa9b..c5569562 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -297,6 +297,8 @@ typedef union { typedef struct proxy_base proxy_t; +typedef struct stratifier_data sdata_t; + struct proxy_base { UT_hash_handle hh; int id; @@ -320,6 +322,7 @@ struct proxy_base { enonce1_t enonce1u; proxy_t *subproxies; /* Hashlist of subproxies sorted by subid */ + sdata_t *sdata; /* Unique stratifer data for each subproxy */ }; struct stratifier_data { @@ -391,8 +394,6 @@ struct stratifier_data { pthread_mutex_t proxy_lock; /* Protects all proxy data */ }; -typedef struct stratifier_data sdata_t; - typedef struct json_entry json_entry_t; struct json_entry { @@ -995,12 +996,26 @@ static void drop_allclients(ckpool_t *ckp) LOGNOTICE("Dropped %d instances", kills); } +/* Copy only the relevant parts of the master sdata for each subproxy */ +static sdata_t *duplicate_sdata(const sdata_t *sdata) +{ + sdata_t *dsdata = ckzalloc(sizeof(sdata_t)); + int64_t randomiser; + + memcpy(dsdata->pubkeytxnbin, sdata->pubkeytxnbin, 25); + memcpy(dsdata->donkeytxnbin, sdata->donkeytxnbin, 25); + randomiser = ((int64_t)time(NULL)) << 32; + dsdata->blockchange_id = dsdata->workbase_id = randomiser; + return dsdata; +} + static proxy_t *__generate_proxy(sdata_t *sdata, const int id) { proxy_t *proxy = ckzalloc(sizeof(proxy_t)); proxy->id = id; HASH_ADD_INT(sdata->proxies, id, proxy); + proxy->sdata = duplicate_sdata(sdata); return proxy; } @@ -1010,6 +1025,7 @@ static proxy_t *__generate_subproxy(proxy_t *proxy, const int id) subproxy->id = id; HASH_ADD_INT(proxy->subproxies, id, subproxy); + subproxy->sdata = proxy->sdata; return subproxy; } @@ -1060,15 +1076,17 @@ static proxy_t *subproxy_by_id(sdata_t *sdata, const int id, const int subid) return subproxy; } -/* Iterates over all clients and sets the reconnect bool for the message - * to be sent lazily next time they speak to us. */ -static void reconnect_clients(sdata_t *sdata) +/* Iterates over all clients in proxy mode and sets the reconnect bool for the + * message to be sent lazily next time they speak to us if they're not bound + * to the requested proxy id */ +static void reconnect_clients_to(sdata_t *sdata, const int id) { stratum_instance_t *client, *tmp; ck_rlock(&sdata->instance_lock); HASH_ITER(hh, sdata->stratum_instances, client, tmp) { - client->reconnect = true; + if (client->proxyid != id) + client->reconnect = true; } ck_runlock(&sdata->instance_lock); } @@ -1177,7 +1195,7 @@ static void update_notify(ckpool_t *ckp, const char *cmd) * clients now to reconnect since we have enough information to * switch. */ proxy->notified = true; - reconnect_clients(sdata); + reconnect_clients_to(sdata, id); } wb = ckzalloc(sizeof(workbase_t)); From 1f83be877d9bf301502673beb18712e5fd612059 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Tue, 10 Feb 2015 21:16:38 +1100 Subject: [PATCH 051/544] Send subproxy id when setting proxy at stratifier and set the current proxy to the subproxy instead of the master proxy --- src/generator.c | 7 +++++-- src/stratifier.c | 18 +++++++++++------- 2 files changed, 16 insertions(+), 9 deletions(-) diff --git a/src/generator.c b/src/generator.c index dd8182ec..a0a60718 100644 --- a/src/generator.c +++ b/src/generator.c @@ -1512,7 +1512,7 @@ static proxy_instance_t *create_subproxy(proxy_instance_t *proxi) subproxy->ckp = proxi->ckp; subproxy->cs = ckzalloc(sizeof(connsock_t)); subproxy->si = proxi->si; - subproxy->id = proxi->subproxy_count++; + subproxy->id = proxi->subproxy_count; subproxy->auth = proxi->auth; subproxy->pass = proxi->pass; subproxy->proxy = proxi; @@ -1532,6 +1532,7 @@ static bool recruit_subproxy(proxy_instance_t *proxi, int epfd) } mutex_lock(&proxi->proxy_lock); + proxi->subproxy_count++; HASH_ADD_INT(proxi->subproxies, id, subproxy); proxi->client_headroom += proxi->clients_per_proxy; mutex_unlock(&proxi->proxy_lock); @@ -1808,11 +1809,13 @@ reconnect: if (proxi != cproxy) { proxi = cproxy; if (!ckp->passthrough) { + proxy_instance_t *proxy = proxi->proxy; + connsock_t *cs = proxi->cs; LOGWARNING("Successfully connected to proxy %d %s:%s as proxy", proxi->id, cs->url, cs->port); dealloc(buf); - ASPRINTF(&buf, "proxy=%d", proxi->id); + ASPRINTF(&buf, "proxy=%d:%d", proxy->id, proxi->id); send_proc(ckp->stratifier, buf); } } diff --git a/src/stratifier.c b/src/stratifier.c index c5569562..df2f15be 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -1006,6 +1006,7 @@ static sdata_t *duplicate_sdata(const sdata_t *sdata) memcpy(dsdata->donkeytxnbin, sdata->donkeytxnbin, 25); randomiser = ((int64_t)time(NULL)) << 32; dsdata->blockchange_id = dsdata->workbase_id = randomiser; + cklock_init(&dsdata->workbase_lock); return dsdata; } @@ -1105,10 +1106,10 @@ static proxy_t *current_proxy(sdata_t *sdata) static void update_subscribe(ckpool_t *ckp, const char *cmd) { sdata_t *sdata = ckp->data; + int id = 0, subid = 0; const char *buf; proxy_t *proxy; json_t *val; - int id = 0; if (unlikely(strlen(cmd) < 11)) { LOGWARNING("Received zero length string for subscribe in update_subscribe"); @@ -1122,11 +1123,13 @@ static void update_subscribe(ckpool_t *ckp, const char *cmd) return; } json_get_int(&id, val, "proxy"); + json_get_int(&subid, val, "subproxy"); - LOGNOTICE("Got updated subscribe for proxy %d", id); + LOGNOTICE("Got updated subscribe for proxy %d:%d", id, subid); - proxy = proxy_by_id(sdata, id); + proxy = subproxy_by_id(sdata, id, subid); proxy->notified = false; /* Reset this */ + sdata = proxy->sdata; ck_wlock(&sdata->workbase_lock); proxy->subscribed = true; @@ -1883,14 +1886,15 @@ static char *stratifier_stats(ckpool_t *ckp, sdata_t *sdata) * first notify data comes from this proxy. */ static void set_proxy(sdata_t *sdata, const char *buf) { - proxy_t *proxy; - int id = 0; + proxy_t *proxy, *subproxy; + int id = 0, subid = 0; - sscanf(buf, "proxy=%d", &id); + sscanf(buf, "proxy=%d:%d", &id, &subid); mutex_lock(&sdata->proxy_lock); proxy = __proxy_by_id(sdata, id); - sdata->proxy = proxy; + subproxy = __subproxy_by_id(proxy, subid); + sdata->proxy = subproxy; mutex_unlock(&sdata->proxy_lock); /* We will receive a notification immediately after this and it should From ba910251da161b5d0e93c56f55f2af43d4892512 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Tue, 10 Feb 2015 21:34:22 +1100 Subject: [PATCH 052/544] Choose the sdata to use by subproxy --- src/stratifier.c | 48 +++++++++++++++++++++++++++++------------------- 1 file changed, 29 insertions(+), 19 deletions(-) diff --git a/src/stratifier.c b/src/stratifier.c index df2f15be..dada019f 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -321,6 +321,7 @@ struct proxy_base { int64_t max_clients; enonce1_t enonce1u; + proxy_t *parent; /* Parent proxy - set to NULL on parent itself */ proxy_t *subproxies; /* Hashlist of subproxies sorted by subid */ sdata_t *sdata; /* Unique stratifer data for each subproxy */ }; @@ -1027,6 +1028,7 @@ static proxy_t *__generate_subproxy(proxy_t *proxy, const int id) subproxy->id = id; HASH_ADD_INT(proxy->subproxies, id, subproxy); subproxy->sdata = proxy->sdata; + subproxy->parent = proxy; return subproxy; } @@ -1105,7 +1107,7 @@ static proxy_t *current_proxy(sdata_t *sdata) static void update_subscribe(ckpool_t *ckp, const char *cmd) { - sdata_t *sdata = ckp->data; + sdata_t *sdata = ckp->data, *dsdata; int id = 0, subid = 0; const char *buf; proxy_t *proxy; @@ -1129,9 +1131,9 @@ static void update_subscribe(ckpool_t *ckp, const char *cmd) proxy = subproxy_by_id(sdata, id, subid); proxy->notified = false; /* Reset this */ - sdata = proxy->sdata; + dsdata = proxy->sdata; - ck_wlock(&sdata->workbase_lock); + ck_wlock(&dsdata->workbase_lock); proxy->subscribed = true; proxy->diff = ckp->startdiff; /* Length is checked by generator */ @@ -1149,7 +1151,7 @@ static void update_subscribe(ckpool_t *ckp, const char *cmd) proxy->enonce1varlen = 0; proxy->enonce2varlen = proxy->nonce2len - proxy->enonce1varlen; proxy->max_clients = 1ll << (proxy->enonce1varlen * 8); - ck_wunlock(&sdata->workbase_lock); + ck_wunlock(&dsdata->workbase_lock); LOGNOTICE("Upstream pool extranonce2 length %d, max proxy clients %"PRId64, proxy->nonce2len, proxy->max_clients); @@ -1159,8 +1161,8 @@ static void update_subscribe(ckpool_t *ckp, const char *cmd) static void update_notify(ckpool_t *ckp, const char *cmd) { + sdata_t *sdata = ckp->data, *dsdata; bool new_block = false, clean; - sdata_t *sdata = ckp->data; int i, id = 0, subid = 0; char header[228]; const char *buf; @@ -1182,13 +1184,13 @@ static void update_notify(ckpool_t *ckp, const char *cmd) } json_get_int(&id, val, "proxy"); json_get_int(&subid, val, "subproxy"); - proxy = proxy_by_id(sdata, id); + proxy = subproxy_by_id(sdata, id, subid); if (unlikely(!proxy->subscribed)) { - LOGNOTICE("No valid proxy %d subscription to update notify yet", id); + LOGNOTICE("No valid proxy %d:%d subscription to update notify yet", id, subid); goto out; } LOGNOTICE("Got updated notify for proxy %d", id); - if (proxy != current_proxy(sdata)) { + if (proxy->parent != current_proxy(sdata)) { LOGINFO("Notify from backup proxy"); goto out; } @@ -1241,17 +1243,20 @@ static void update_notify(ckpool_t *ckp, const char *cmd) hex2bin(wb->headerbin, header, 112); wb->txn_hashes = ckzalloc(1); - ck_rlock(&sdata->workbase_lock); + dsdata = proxy->sdata; + + ck_rlock(&dsdata->workbase_lock); strcpy(wb->enonce1const, proxy->enonce1); wb->enonce1constlen = proxy->enonce1constlen; memcpy(wb->enonce1constbin, proxy->enonce1bin, wb->enonce1constlen); wb->enonce1varlen = proxy->enonce1varlen; wb->enonce2varlen = proxy->enonce2varlen; wb->diff = proxy->diff; - ck_runlock(&sdata->workbase_lock); + ck_runlock(&dsdata->workbase_lock); add_base(ckp, wb, &new_block); + /* FIXME: Goes to everyone, separate by proxy only */ stratum_broadcast_update(sdata, new_block | clean); out: json_decref(val); @@ -1261,13 +1266,13 @@ static void stratum_send_diff(sdata_t *sdata, const stratum_instance_t *client); static void update_diff(ckpool_t *ckp, const char *cmd) { + sdata_t *sdata = ckp->data, *dsdata; stratum_instance_t *client, *tmp; - sdata_t *sdata = ckp->data; double old_diff, diff; + int id = 0, subid = 0; const char *buf; proxy_t *proxy; json_t *val; - int id = 0; if (unlikely(!sdata->current_workbase)) { LOGINFO("No current workbase to update diff yet"); @@ -1287,12 +1292,13 @@ static void update_diff(ckpool_t *ckp, const char *cmd) return; } json_get_int(&id, val, "proxy"); + json_get_int(&subid, val, "subproxy"); json_dblcpy(&diff, val, "diff"); json_decref(val); LOGNOTICE("Got updated diff for proxy %d", id); - proxy = proxy_by_id(sdata, id); - if (proxy != current_proxy(sdata)) { + proxy = subproxy_by_id(sdata, id, subid); + if (proxy->parent != current_proxy(sdata)) { LOGINFO("Diff from backup proxy"); return; } @@ -1302,10 +1308,12 @@ static void update_diff(ckpool_t *ckp, const char *cmd) if (unlikely(diff < 1)) diff = 1; - ck_wlock(&sdata->workbase_lock); - old_diff = sdata->proxy->diff; - sdata->current_workbase->diff = sdata->proxy->diff = diff; - ck_wunlock(&sdata->workbase_lock); + dsdata = proxy->sdata; + + ck_wlock(&dsdata->workbase_lock); + old_diff = proxy->diff; + dsdata->current_workbase->diff = dsdata->proxy->diff = diff; + ck_wunlock(&dsdata->workbase_lock); if (old_diff < diff) return; @@ -1314,6 +1322,8 @@ static void update_diff(ckpool_t *ckp, const char *cmd) * they're at or below the new diff, and update it if not. */ ck_rlock(&sdata->instance_lock); HASH_ITER(hh, sdata->stratum_instances, client, tmp) { + if (client->proxyid != id) + continue; if (client->diff > diff) { client->diff = diff; stratum_send_diff(sdata, client); @@ -1894,7 +1904,7 @@ static void set_proxy(sdata_t *sdata, const char *buf) mutex_lock(&sdata->proxy_lock); proxy = __proxy_by_id(sdata, id); subproxy = __subproxy_by_id(proxy, subid); - sdata->proxy = subproxy; + sdata->proxy = proxy; mutex_unlock(&sdata->proxy_lock); /* We will receive a notification immediately after this and it should From e360a9842bc1496aaebded1d332ff49df6c20d02 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Tue, 10 Feb 2015 21:39:42 +1100 Subject: [PATCH 053/544] Add the workbase to each unique sdata as required --- src/stratifier.c | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/src/stratifier.c b/src/stratifier.c index dada019f..b8dff8d8 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -707,10 +707,12 @@ static void send_ageworkinfo(ckpool_t *ckp, const int64_t id) ckdbq_add(ckp, ID_AGEWORKINFO, val); } -static void add_base(ckpool_t *ckp, workbase_t *wb, bool *new_block) +/* Add a new workbase to the table of workbases. Sdata is the global data in + * pool mode but unique to each subproxy in proxy mode */ +static void add_base(ckpool_t *ckp, sdata_t *sdata, workbase_t *wb, bool *new_block) { workbase_t *tmp, *tmpa, *aged = NULL; - sdata_t *sdata = ckp->data; + sdata_t *ckp_sdata = ckp->data; int len, ret; ts_realtime(&wb->gentime); @@ -723,7 +725,7 @@ static void add_base(ckpool_t *ckp, workbase_t *wb, bool *new_block) * we set workbase_id from it. In server mode the stratifier is * setting the workbase_id */ ck_wlock(&sdata->workbase_lock); - sdata->workbases_generated++; + ckp_sdata->workbases_generated++; if (!ckp->proxy) wb->id = sdata->workbase_id++; else @@ -905,7 +907,7 @@ static void *do_update(void *arg) json_decref(val); generate_coinbase(ckp, wb); - add_base(ckp, wb, &new_block); + add_base(ckp, sdata, wb, &new_block); stratum_broadcast_update(sdata, new_block); ret = true; @@ -1254,7 +1256,7 @@ static void update_notify(ckpool_t *ckp, const char *cmd) wb->diff = proxy->diff; ck_runlock(&dsdata->workbase_lock); - add_base(ckp, wb, &new_block); + add_base(ckp, dsdata, wb, &new_block); /* FIXME: Goes to everyone, separate by proxy only */ stratum_broadcast_update(sdata, new_block | clean); From b1f92414c5924315136775038aa9256bd1121f4e Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Fri, 13 Feb 2015 09:05:04 +1100 Subject: [PATCH 054/544] Store which sdata each client is bound to --- src/stratifier.c | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/src/stratifier.c b/src/stratifier.c index b8dff8d8..f198c3f6 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -214,6 +214,8 @@ struct worker_instance { bool notified_idle; }; +typedef struct stratifier_data sdata_t; + /* Per client stratum instance == workers */ struct stratum_instance { UT_hash_handle hh; @@ -275,6 +277,7 @@ struct stratum_instance { int64_t suggest_diff; /* Stratum client suggested diff */ double best_diff; /* Best share found by this instance */ + sdata_t *sdata; /* Which sdata this client is bound to */ int proxyid; /* Which proxy this is bound to in proxy mode */ int subproxyid; /* Which subproxy */ }; @@ -297,8 +300,6 @@ typedef union { typedef struct proxy_base proxy_t; -typedef struct stratifier_data sdata_t; - struct proxy_base { UT_hash_handle hh; int id; @@ -1456,7 +1457,8 @@ static void _dec_instance_ref(sdata_t *sdata, stratum_instance_t *client, const #define dec_instance_ref(sdata, instance) _dec_instance_ref(sdata, instance, __FILE__, __func__, __LINE__) /* Enter with write instance_lock held */ -static stratum_instance_t *__stratum_add_instance(ckpool_t *ckp, const int64_t id, const int server) +static stratum_instance_t *__stratum_add_instance(ckpool_t *ckp, const int64_t id, + const int server) { stratum_instance_t *client = ckzalloc(sizeof(stratum_instance_t)); sdata_t *sdata = ckp->data; @@ -1468,6 +1470,9 @@ static stratum_instance_t *__stratum_add_instance(ckpool_t *ckp, const int64_t i client->ckp = ckp; tv_time(&client->ldc); HASH_ADD_I64(sdata->stratum_instances, id, client); + /* Points to ckp sdata in ckpool mode, but is changed later in proxy + * mode . */ + client->sdata = sdata; return client; } From 03712fe2d7789d4effc58d817c317c29245066b0 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Fri, 13 Feb 2015 09:13:38 +1100 Subject: [PATCH 055/544] Broadcast data according to the sdata it belongs to --- src/stratifier.c | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/src/stratifier.c b/src/stratifier.c index f198c3f6..401e201f 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -1006,8 +1006,20 @@ static sdata_t *duplicate_sdata(const sdata_t *sdata) sdata_t *dsdata = ckzalloc(sizeof(sdata_t)); int64_t randomiser; + /* Copy the transaction binaries for workbase creation */ memcpy(dsdata->pubkeytxnbin, sdata->pubkeytxnbin, 25); memcpy(dsdata->donkeytxnbin, sdata->donkeytxnbin, 25); + + /* Use the same work queues for all subproxies */ + dsdata->ssends = sdata->ssends; + dsdata->srecvs = sdata->srecvs; + dsdata->ckdbq = sdata->ckdbq; + dsdata->sshareq = sdata->sshareq; + dsdata->sauthq = sdata->sauthq; + dsdata->stxnq = sdata->stxnq; + + /* Give the sbuproxy a unique workbase id and its own workbase list + * and lock */ randomiser = ((int64_t)time(NULL)) << 32; dsdata->blockchange_id = dsdata->workbase_id = randomiser; cklock_init(&dsdata->workbase_lock); @@ -1533,7 +1545,7 @@ static inline bool client_active(stratum_instance_t *client) /* For creating a list of sends without locking that can then be concatenated * to the stratum_sends list. Minimises locking and avoids taking recursive - * locks. */ + * locks. Sends only to sdata bound clients (everyone in ckpool) */ static void stratum_broadcast(sdata_t *sdata, json_t *val) { stratum_instance_t *client, *tmp; @@ -1550,6 +1562,8 @@ static void stratum_broadcast(sdata_t *sdata, json_t *val) ckmsg_t *client_msg; smsg_t *msg; + if (client->sdata != sdata) + continue; if (!client_active(client)) continue; client_msg = ckalloc(sizeof(ckmsg_t)); From d62483fbd2691341e9d30f17ba48317f012e44cf Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Fri, 13 Feb 2015 09:25:42 +1100 Subject: [PATCH 056/544] Workbases always exists by the time we reach the point we can set the enonce2varlen --- src/stratifier.c | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/stratifier.c b/src/stratifier.c index 401e201f..a8bc5774 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -2241,11 +2241,9 @@ static json_t *parse_subscribe(stratum_instance_t *client, const int64_t client_ client->id, client->enonce1_64, client->enonce1); } + /* Workbases will exist if sdata->current_workbase is not NULL */ ck_rlock(&sdata->workbase_lock); - if (likely(sdata->workbases)) - n2len = sdata->workbases->enonce2varlen; - else - n2len = 8; + n2len = sdata->workbases->enonce2varlen; JSON_CPACK(ret, "[[[s,s]],s,i]", "mining.notify", client->enonce1, client->enonce1, n2len); ck_runlock(&sdata->workbase_lock); From 68222826721a4c269c2de133d7046f783163386e Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Fri, 13 Feb 2015 09:53:19 +1100 Subject: [PATCH 057/544] Generate stratifier data per subproxy instead of per proxy since they'll have unique workbases --- src/stratifier.c | 37 ++++++++++++++++++++++++++----------- 1 file changed, 26 insertions(+), 11 deletions(-) diff --git a/src/stratifier.c b/src/stratifier.c index a8bc5774..484a6fe0 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -1032,17 +1032,16 @@ static proxy_t *__generate_proxy(sdata_t *sdata, const int id) proxy->id = id; HASH_ADD_INT(sdata->proxies, id, proxy); - proxy->sdata = duplicate_sdata(sdata); return proxy; } -static proxy_t *__generate_subproxy(proxy_t *proxy, const int id) +static proxy_t *__generate_subproxy(sdata_t *sdata, proxy_t *proxy, const int id) { proxy_t *subproxy = ckzalloc(sizeof(proxy_t)); subproxy->id = id; HASH_ADD_INT(proxy->subproxies, id, subproxy); - subproxy->sdata = proxy->sdata; + proxy->sdata = duplicate_sdata(sdata); subproxy->parent = proxy; return subproxy; } @@ -1061,13 +1060,13 @@ static proxy_t *__proxy_by_id(sdata_t *sdata, const int id) return proxy; } -static proxy_t *__subproxy_by_id(proxy_t *proxy, const int id) +static proxy_t *__subproxy_by_id(sdata_t *sdata, proxy_t *proxy, const int id) { proxy_t *subproxy; HASH_FIND_INT(proxy->subproxies, &id, subproxy); if (!subproxy) - subproxy = __generate_subproxy(proxy, id); + subproxy = __generate_subproxy(sdata, proxy, id); return subproxy; } @@ -1088,7 +1087,7 @@ static proxy_t *subproxy_by_id(sdata_t *sdata, const int id, const int subid) mutex_lock(&sdata->proxy_lock); proxy = __proxy_by_id(sdata, id); - subproxy = __subproxy_by_id(proxy, subid); + subproxy = __subproxy_by_id(sdata, proxy, subid); mutex_unlock(&sdata->proxy_lock); return subproxy; @@ -1924,7 +1923,7 @@ static void set_proxy(sdata_t *sdata, const char *buf) mutex_lock(&sdata->proxy_lock); proxy = __proxy_by_id(sdata, id); - subproxy = __subproxy_by_id(proxy, subid); + subproxy = __subproxy_by_id(sdata, proxy, subid); sdata->proxy = proxy; mutex_unlock(&sdata->proxy_lock); @@ -2179,23 +2178,39 @@ static bool new_enonce1(sdata_t *sdata, stratum_instance_t *client) static void stratum_send_message(sdata_t *sdata, const stratum_instance_t *client, const char *msg); +/* Choose the stratifier data for a new client. Use the main ckp_sdata except + * in proxy mode where we find a subproxy based on the current proxy with room + * for more clients. Signal the generator to recruit more subproxies if we are + * running out of room. */ +static sdata_t *select_sdata(const ckpool_t *ckp, sdata_t *ckp_sdata) +{ + if (!ckp->proxy || ckp->passthrough) + return ckp_sdata; + if (!ckp_sdata->proxy) + return NULL; + /* FIXME: Choose a subproxy, not the parent proxy */ + return ckp_sdata->proxy->sdata; +} + /* Extranonce1 must be set here. Needs to be entered with client holding a ref * count. */ static json_t *parse_subscribe(stratum_instance_t *client, const int64_t client_id, const json_t *params_val) { - sdata_t *sdata = client->ckp->data; + ckpool_t *ckp = client->ckp; + sdata_t *sdata, *ckp_sdata = ckp->data; bool old_match = false; int arr_size; json_t *ret; int n2len; if (unlikely(!json_is_array(params_val))) { - stratum_send_message(sdata, client, "Invalid json: params not an array"); + stratum_send_message(ckp_sdata, client, "Invalid json: params not an array"); return json_string("params not an array"); } - if (unlikely(!sdata->current_workbase)) { - stratum_send_message(sdata, client, "Pool Initialising"); + sdata = select_sdata(ckp, ckp_sdata); + if (unlikely(!sdata || !sdata->current_workbase)) { + stratum_send_message(ckp_sdata, client, "Pool Initialising"); return json_string("Initialising"); } From 7763b49c2e9b2edeabfa69318772406e343eeaa5 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Fri, 13 Feb 2015 10:15:48 +1100 Subject: [PATCH 058/544] Allow stratifier to choose the best subproxy to attach new users to and request recruitment of more subproxies when the headroom for more clients is low --- src/generator.c | 15 +++++++++++---- src/stratifier.c | 35 +++++++++++++++++++++++++++++++---- 2 files changed, 42 insertions(+), 8 deletions(-) diff --git a/src/generator.c b/src/generator.c index a0a60718..baeae613 100644 --- a/src/generator.c +++ b/src/generator.c @@ -125,6 +125,8 @@ struct proxy_instance { time_t reconnect_time; + int epfd; /* Epoll fd used by the parent proxy */ + pthread_mutex_t proxy_lock; /* Lock protecting hashlist of proxies */ int64_t clients_per_proxy; /* How many clients can connect to each subproxy */ int64_t client_headroom; /* How many more clients can we connect */ @@ -1427,7 +1429,7 @@ static void passthrough_add_send(proxy_instance_t *proxi, const char *msg) ckmsgq_add(proxi->passsends, pm); } -static bool recruit_subproxy(proxy_instance_t *proxi, int epfd); +static bool recruit_subproxy(proxy_instance_t *proxi); static bool proxy_alive(ckpool_t *ckp, server_instance_t *si, proxy_instance_t *proxi, connsock_t *cs, bool pinging, int epfd) @@ -1488,9 +1490,11 @@ out: if (unlikely(epoll_ctl(epfd, EPOLL_CTL_ADD, cs->fd, &event) == -1)) quit(1, "FATAL: Failed to add epfd to epoll_ctl in proxy_alive"); if (!ckp->passthrough && proxi->proxy == proxi) { + /* We recruit enough proxies to begin with and then + * recruit extra when asked by the stratifier. */ while (proxi->client_headroom < 42) { /* Note recursive call of proxy_alive here */ - if (!recruit_subproxy(proxi, epfd)) { + if (!recruit_subproxy(proxi)) { LOGWARNING("Unable to recruit extra subproxies after just %"PRId64, proxi->client_headroom); break; @@ -1520,9 +1524,10 @@ static proxy_instance_t *create_subproxy(proxy_instance_t *proxi) return subproxy; } -static bool recruit_subproxy(proxy_instance_t *proxi, int epfd) +static bool recruit_subproxy(proxy_instance_t *proxi) { proxy_instance_t *subproxy = create_subproxy(proxi); + int epfd = proxi->epfd; if (!proxy_alive(subproxy->ckp, subproxy->si, subproxy, subproxy->cs, false, epfd)) { LOGNOTICE("Subproxy failed proxy_alive testing"); @@ -1553,7 +1558,7 @@ static void *passthrough_recv(void *arg) rename_proc("passrecv"); - epfd = epoll_create1(EPOLL_CLOEXEC); + proxi->epfd = epfd = epoll_create1(EPOLL_CLOEXEC); if (epfd < 0){ LOGEMERG("FATAL: Failed to create epoll in passrecv"); return NULL; @@ -1863,6 +1868,8 @@ retry: } else if (cmdmatch(buf, "ping")) { LOGDEBUG("Proxy received ping request"); send_unix_msg(sockd, "pong"); + } else if (cmdmatch(buf, "recruit")) { + recruit_subproxy(proxi); } else if (ckp->passthrough) { /* Anything remaining should be stratum messages */ passthrough_add_send(proxi, buf); diff --git a/src/stratifier.c b/src/stratifier.c index 484a6fe0..648755d1 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -819,7 +819,7 @@ static char *send_recv_generator(ckpool_t *ckp, const char *msg, const int prio) return buf; } -static void send_generator(ckpool_t *ckp, const char *msg, const int prio) +static void send_generator(const ckpool_t *ckp, const char *msg, const int prio) { sdata_t *sdata = ckp->data; bool set; @@ -1070,6 +1070,7 @@ static proxy_t *__subproxy_by_id(sdata_t *sdata, proxy_t *proxy, const int id) return subproxy; } +#if 0 static proxy_t *proxy_by_id(sdata_t *sdata, const int id) { proxy_t *proxy; @@ -1080,6 +1081,7 @@ static proxy_t *proxy_by_id(sdata_t *sdata, const int id) return proxy; } +#endif static proxy_t *subproxy_by_id(sdata_t *sdata, const int id, const int subid) { @@ -2184,12 +2186,37 @@ static void stratum_send_message(sdata_t *sdata, const stratum_instance_t *clien * running out of room. */ static sdata_t *select_sdata(const ckpool_t *ckp, sdata_t *ckp_sdata) { + proxy_t *proxy, *subproxy, *best = NULL, *tmp; + int64_t headroom = 0, most_headroom = 0; + if (!ckp->proxy || ckp->passthrough) return ckp_sdata; - if (!ckp_sdata->proxy) + proxy = ckp_sdata->proxy; + if (!proxy) { + LOGWARNING("No proxy available yet to generate subscribes"); return NULL; - /* FIXME: Choose a subproxy, not the parent proxy */ - return ckp_sdata->proxy->sdata; + } + mutex_lock(&ckp_sdata->proxy_lock); + HASH_ITER(hh, proxy->subproxies, subproxy, tmp) { + int64_t subproxy_headroom = subproxy->max_clients - subproxy->clients; + + headroom += subproxy_headroom; + if (subproxy_headroom > most_headroom) { + best = subproxy; + most_headroom = subproxy_headroom; + } + } + mutex_unlock(&ckp_sdata->proxy_lock); + + if (headroom < 42) { + LOGNOTICE("Stratifer requesting more proxies from generator"); + send_generator(ckp, "recruit", GEN_PRIORITY); + } + if (!best) { + LOGWARNING("Insufficient subproxies to accept more clients"); + return NULL; + } + return best->sdata; } /* Extranonce1 must be set here. Needs to be entered with client holding a ref From dd1dae4ae396cb73525d485a87d17a2cc9a696fa Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Fri, 13 Feb 2015 10:25:54 +1100 Subject: [PATCH 059/544] Send the relevant sdata to be used by new_enonce1 --- src/stratifier.c | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/src/stratifier.c b/src/stratifier.c index 648755d1..64e47bf3 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -2098,23 +2098,23 @@ static void __fill_enonce1data(const workbase_t *wb, stratum_instance_t *client) * When the proxy space is less than 32 bits to work with, we look for an * unused enonce1 value and reject clients instead if there is no space left. * Needs to be entered with client holding a ref count. */ -static bool new_enonce1(sdata_t *sdata, stratum_instance_t *client) +static bool new_enonce1(ckpool_t *ckp, sdata_t *ckp_sdata, sdata_t *sdata, stratum_instance_t *client) { proxy_t *proxy = NULL; enonce1_t *enonce1u; int enonce1varlen; bool ret = false; - if (client->ckp->proxy) { - if (!sdata->proxy) + if (ckp->proxy) { + if (!ckp_sdata->proxy) return false; - mutex_lock(&sdata->proxy_lock); - proxy = sdata->proxy; + mutex_lock(&ckp_sdata->proxy_lock); + proxy = ckp_sdata->proxy; enonce1u = &proxy->enonce1u; client->proxyid = proxy->id; client->subproxyid = proxy->subid; - mutex_unlock(&sdata->proxy_lock); + mutex_unlock(&ckp_sdata->proxy_lock); if (proxy->clients >= proxy->max_clients) { LOGWARNING("Proxy reached max clients %"PRId64, proxy->max_clients); @@ -2132,7 +2132,7 @@ static bool new_enonce1(sdata_t *sdata, stratum_instance_t *client) /* instance_lock protects enonce1u. Recruiting extra proxies should * prevent these ever locking out.*/ - ck_wlock(&sdata->instance_lock); + ck_wlock(&ckp_sdata->instance_lock); switch(enonce1varlen) { case 8: enonce1u->u64++; @@ -2166,7 +2166,7 @@ static bool new_enonce1(sdata_t *sdata, stratum_instance_t *client) proxy->clients++; client->enonce1_64 = enonce1u->u64; } - ck_wunlock(&sdata->instance_lock); + ck_wunlock(&ckp_sdata->instance_lock); ck_rlock(&sdata->workbase_lock); __fill_enonce1data(sdata->current_workbase, client); @@ -2271,7 +2271,7 @@ static json_t *parse_subscribe(stratum_instance_t *client, const int64_t client_ client->useragent = ckzalloc(1); if (!old_match) { /* Create a new extranonce1 based on a uint64_t pointer */ - if (!new_enonce1(sdata, client)) { + if (!new_enonce1(ckp, ckp_sdata, sdata, client)) { stratum_send_message(sdata, client, "Pool full of clients"); client->reject = 2; return json_string("proxy full"); From dc293beb9c1f5bada8d9a10834ae50a0f83c93f3 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Fri, 13 Feb 2015 10:37:56 +1100 Subject: [PATCH 060/544] Separate delivery of messages by ckpool sdata and per proxy sdata --- src/stratifier.c | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/stratifier.c b/src/stratifier.c index 64e47bf3..ca2bb775 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -328,6 +328,8 @@ struct proxy_base { }; struct stratifier_data { + ckpool_t *ckp; + char pubkeytxnbin[25]; char donkeytxnbin[25]; @@ -1006,6 +1008,8 @@ static sdata_t *duplicate_sdata(const sdata_t *sdata) sdata_t *dsdata = ckzalloc(sizeof(sdata_t)); int64_t randomiser; + dsdata->ckp = sdata->ckp; + /* Copy the transaction binaries for workbase creation */ memcpy(dsdata->pubkeytxnbin, sdata->pubkeytxnbin, 25); memcpy(dsdata->donkeytxnbin, sdata->donkeytxnbin, 25); @@ -1272,7 +1276,6 @@ static void update_notify(ckpool_t *ckp, const char *cmd) add_base(ckp, dsdata, wb, &new_block); - /* FIXME: Goes to everyone, separate by proxy only */ stratum_broadcast_update(sdata, new_block | clean); out: json_decref(val); @@ -1549,6 +1552,7 @@ static inline bool client_active(stratum_instance_t *client) * locks. Sends only to sdata bound clients (everyone in ckpool) */ static void stratum_broadcast(sdata_t *sdata, json_t *val) { + sdata_t *ckp_sdata = sdata->ckp->data; stratum_instance_t *client, *tmp; ckmsg_t *bulk_send = NULL; ckmsgq_t *ssends; @@ -1563,7 +1567,7 @@ static void stratum_broadcast(sdata_t *sdata, json_t *val) ckmsg_t *client_msg; smsg_t *msg; - if (client->sdata != sdata) + if (sdata != ckp_sdata && client->sdata != sdata) continue; if (!client_active(client)) continue; @@ -4630,6 +4634,7 @@ int stratifier(proc_instance_t *pi) LOGWARNING("%s stratifier starting", ckp->name); sdata = ckzalloc(sizeof(sdata_t)); ckp->data = sdata; + sdata->ckp = ckp; /* Wait for the generator to have something for us */ do { From c872ce9de8a814e5bf7dc2ac73f0291fd164bb05 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Fri, 13 Feb 2015 12:02:15 +1100 Subject: [PATCH 061/544] Store the proxy and subproxy id only in clients and sort subproxies by subid to avoid confusion --- src/stratifier.c | 45 ++++++++++++++++++--------------------------- 1 file changed, 18 insertions(+), 27 deletions(-) diff --git a/src/stratifier.c b/src/stratifier.c index ca2bb775..2e1accbc 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -127,8 +127,6 @@ struct workbase { ckpool_t *ckp; bool proxy; /* This workbase is proxied work */ - int proxyid; /* The id of the proxy the workbase is from */ - int subproxyid; /* The id of the subproxy the worbase is from */ }; typedef struct workbase workbase_t; @@ -1039,12 +1037,13 @@ static proxy_t *__generate_proxy(sdata_t *sdata, const int id) return proxy; } -static proxy_t *__generate_subproxy(sdata_t *sdata, proxy_t *proxy, const int id) +static proxy_t *__generate_subproxy(sdata_t *sdata, proxy_t *proxy, const int subid) { proxy_t *subproxy = ckzalloc(sizeof(proxy_t)); - subproxy->id = id; - HASH_ADD_INT(proxy->subproxies, id, subproxy); + subproxy->id = proxy->id; + subproxy->subid = subid; + HASH_ADD_INT(proxy->subproxies, subid, subproxy); proxy->sdata = duplicate_sdata(sdata); subproxy->parent = proxy; return subproxy; @@ -1064,13 +1063,13 @@ static proxy_t *__proxy_by_id(sdata_t *sdata, const int id) return proxy; } -static proxy_t *__subproxy_by_id(sdata_t *sdata, proxy_t *proxy, const int id) +static proxy_t *__subproxy_by_id(sdata_t *sdata, proxy_t *proxy, const int subid) { proxy_t *subproxy; - HASH_FIND_INT(proxy->subproxies, &id, subproxy); + HASH_FIND_INT(proxy->subproxies, &subid, subproxy); if (!subproxy) - subproxy = __generate_subproxy(sdata, proxy, id); + subproxy = __generate_subproxy(sdata, proxy, subid); return subproxy; } @@ -1210,24 +1209,10 @@ static void update_notify(ckpool_t *ckp, const char *cmd) goto out; } LOGNOTICE("Got updated notify for proxy %d", id); - if (proxy->parent != current_proxy(sdata)) { - LOGINFO("Notify from backup proxy"); - goto out; - } - - if (!proxy->notified) { - /* This is the first notification from the current proxy, tell - * clients now to reconnect since we have enough information to - * switch. */ - proxy->notified = true; - reconnect_clients_to(sdata, id); - } wb = ckzalloc(sizeof(workbase_t)); wb->ckp = ckp; wb->proxy = true; - wb->proxyid = id; - wb->subproxyid = subid; json_int64cpy(&wb->id, val, "jobid"); json_strcpy(wb->prevhash, val, "prevhash"); @@ -1276,7 +1261,14 @@ static void update_notify(ckpool_t *ckp, const char *cmd) add_base(ckp, dsdata, wb, &new_block); - stratum_broadcast_update(sdata, new_block | clean); + if (proxy->parent == current_proxy(sdata) && !proxy->notified) { + /* This is the first notification from the current proxy, tell + * clients now to reconnect since we have enough information to + * switch. */ + proxy->notified = true; + reconnect_clients_to(sdata, id); + } + stratum_broadcast_update(dsdata, new_block | clean); out: json_decref(val); } @@ -1922,14 +1914,13 @@ static char *stratifier_stats(ckpool_t *ckp, sdata_t *sdata) * first notify data comes from this proxy. */ static void set_proxy(sdata_t *sdata, const char *buf) { - proxy_t *proxy, *subproxy; - int id = 0, subid = 0; + proxy_t *proxy; + int id = 0; - sscanf(buf, "proxy=%d:%d", &id, &subid); + sscanf(buf, "proxy=%d", &id); mutex_lock(&sdata->proxy_lock); proxy = __proxy_by_id(sdata, id); - subproxy = __subproxy_by_id(sdata, proxy, subid); sdata->proxy = proxy; mutex_unlock(&sdata->proxy_lock); From bcd499bf310cf0d8d7f5cadac61d9c77f9a5fc27 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Fri, 13 Feb 2015 12:07:07 +1100 Subject: [PATCH 062/544] Assign the client subproxy data in new_enonce1 --- src/stratifier.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/stratifier.c b/src/stratifier.c index 2e1accbc..458ffc37 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -394,6 +394,7 @@ struct stratifier_data { proxy_t *proxy; /* Current proxy in use */ proxy_t *proxies; /* Hashlist of all proxies */ pthread_mutex_t proxy_lock; /* Protects all proxy data */ + proxy_t *subproxy; /* Which subproxy this sdata belongs to in proxy mode */ }; typedef struct json_entry json_entry_t; @@ -1028,6 +1029,7 @@ static sdata_t *duplicate_sdata(const sdata_t *sdata) return dsdata; } +/* Note that proxies don't have unique sdata, only the subproxies do */ static proxy_t *__generate_proxy(sdata_t *sdata, const int id) { proxy_t *proxy = ckzalloc(sizeof(proxy_t)); @@ -1044,7 +1046,8 @@ static proxy_t *__generate_subproxy(sdata_t *sdata, proxy_t *proxy, const int su subproxy->id = proxy->id; subproxy->subid = subid; HASH_ADD_INT(proxy->subproxies, subid, subproxy); - proxy->sdata = duplicate_sdata(sdata); + subproxy->sdata = duplicate_sdata(sdata); + subproxy->sdata->subproxy = subproxy; subproxy->parent = proxy; return subproxy; } @@ -2105,7 +2108,7 @@ static bool new_enonce1(ckpool_t *ckp, sdata_t *ckp_sdata, sdata_t *sdata, strat return false; mutex_lock(&ckp_sdata->proxy_lock); - proxy = ckp_sdata->proxy; + proxy = sdata->subproxy; enonce1u = &proxy->enonce1u; client->proxyid = proxy->id; client->subproxyid = proxy->subid; From 319d0943fc8e2a060295c5fa508b967f388ced26 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Fri, 13 Feb 2015 12:14:31 +1100 Subject: [PATCH 063/544] Update diff on all stratifier data but send diff only to clients bound to the current subproxy --- src/stratifier.c | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/stratifier.c b/src/stratifier.c index 458ffc37..e089d621 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -1312,10 +1312,6 @@ static void update_diff(ckpool_t *ckp, const char *cmd) LOGNOTICE("Got updated diff for proxy %d", id); proxy = subproxy_by_id(sdata, id, subid); - if (proxy->parent != current_proxy(sdata)) { - LOGINFO("Diff from backup proxy"); - return; - } /* We only really care about integer diffs so clamp the lower limit to * 1 or it will round down to zero. */ @@ -1338,6 +1334,8 @@ static void update_diff(ckpool_t *ckp, const char *cmd) HASH_ITER(hh, sdata->stratum_instances, client, tmp) { if (client->proxyid != id) continue; + if (client->subproxyid != subid) + continue; if (client->diff > diff) { client->diff = diff; stratum_send_diff(sdata, client); From 27faf4d1c4531bbef51c86cd7818ed25ed3dc8f6 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Fri, 13 Feb 2015 12:45:05 +1100 Subject: [PATCH 064/544] Fix proxy epfd not being set --- src/generator.c | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/generator.c b/src/generator.c index baeae613..ddf66d4f 100644 --- a/src/generator.c +++ b/src/generator.c @@ -1487,8 +1487,11 @@ out: event.events = EPOLLIN; event.data.ptr = proxi; /* Add this connsock_t to the epoll list */ - if (unlikely(epoll_ctl(epfd, EPOLL_CTL_ADD, cs->fd, &event) == -1)) - quit(1, "FATAL: Failed to add epfd to epoll_ctl in proxy_alive"); + if (unlikely(epoll_ctl(epfd, EPOLL_CTL_ADD, cs->fd, &event) == -1)) { + LOGERR("Failed to add fd %d to epfd %d to epoll_ctl in proxy_alive", + cs->fd, epfd); + return false; + } if (!ckp->passthrough && proxi->proxy == proxi) { /* We recruit enough proxies to begin with and then * recruit extra when asked by the stratifier. */ @@ -1520,6 +1523,7 @@ static proxy_instance_t *create_subproxy(proxy_instance_t *proxi) subproxy->auth = proxi->auth; subproxy->pass = proxi->pass; subproxy->proxy = proxi; + subproxy->epfd = proxi->epfd; mutex_init(&subproxy->share_lock); return subproxy; } @@ -1634,7 +1638,7 @@ static void *proxy_recv(void *arg) rename_proc("proxyrecv"); - epfd = epoll_create1(EPOLL_CLOEXEC); + proxi->epfd = epfd = epoll_create1(EPOLL_CLOEXEC); if (epfd < 0){ LOGEMERG("FATAL: Failed to create epoll in proxyrecv"); return NULL; From 6a21a67c1e1f0c03ce9faf18ba689486820981ba Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Fri, 13 Feb 2015 12:51:35 +1100 Subject: [PATCH 065/544] Use client sdata with update_client call --- src/stratifier.c | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/src/stratifier.c b/src/stratifier.c index e089d621..f1added9 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -2236,6 +2236,7 @@ static json_t *parse_subscribe(stratum_instance_t *client, const int64_t client_ stratum_send_message(ckp_sdata, client, "Pool Initialising"); return json_string("Initialising"); } + client->sdata = sdata; arr_size = json_array_size(params_val); /* NOTE useragent is NULL prior to this so should not be used in code @@ -3095,7 +3096,7 @@ static bool new_share(sdata_t *sdata, const uchar *hash, const int64_t wb_id) return ret; } -static void update_client(sdata_t *sdata, const stratum_instance_t *client, const int64_t client_id); +static void update_client(const stratum_instance_t *client, const int64_t client_id); /* Submit a share in proxy mode to the parent pool. workbase_lock is held. * Needs to be entered with client holding a ref count. */ @@ -3346,7 +3347,7 @@ out: } else if (client->first_invalid && client->first_invalid < now_t - 60) { if (!client->reject) { LOGINFO("Client %"PRId64" rejecting for 60s, sending update", client->id); - update_client(sdata, client, client->id); + update_client(client, client->id); client->reject = 1; } } @@ -3414,6 +3415,11 @@ static void stratum_send_update(sdata_t *sdata, const int64_t client_id, const b { json_t *json_msg; + if (unlikely(!sdata->current_workbase)) { + LOGWARNING("No current workbase to send stratum update"); + return; + } + ck_rlock(&sdata->workbase_lock); json_msg = __stratum_notify(sdata->current_workbase, clean); ck_runlock(&sdata->workbase_lock); @@ -3430,8 +3436,10 @@ static void send_json_err(sdata_t *sdata, const int64_t client_id, json_t *id_va } /* Needs to be entered with client holding a ref count. */ -static void update_client(sdata_t *sdata, const stratum_instance_t *client, const int64_t client_id) +static void update_client(const stratum_instance_t *client, const int64_t client_id) { + sdata_t *sdata = client->sdata; + stratum_send_update(sdata, client_id, true); stratum_send_diff(sdata, client); } @@ -3584,7 +3592,7 @@ static void parse_method(sdata_t *sdata, stratum_instance_t *client, const int64 json_object_set_new_nocheck(val, "error", json_null()); stratum_add_send(sdata, val, client_id); if (likely(client->subscribed)) - update_client(sdata, client, client_id); + update_client(client, client_id); return; } From 955bbe69771aea7d29740cec03968a208c7ab09b Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Fri, 13 Feb 2015 14:05:39 +1100 Subject: [PATCH 066/544] Use the client's sdata to associate correct share submissions with the upstream proxy --- src/stratifier.c | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/stratifier.c b/src/stratifier.c index f1added9..e9775259 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -2802,8 +2802,8 @@ static void add_submit(ckpool_t *ckp, stratum_instance_t *client, const int diff worker_instance_t *worker = client->worker_instance; double tdiff, bdiff, dsps, drr, network_diff, bias; user_instance_t *user = client->user_instance; + sdata_t *sdata = client->sdata; int64_t next_blockid, optimal; - sdata_t *sdata = ckp->data; tv_t now_t; mutex_lock(&sdata->stats_lock); @@ -2934,10 +2934,10 @@ test_blocksolve(const stratum_instance_t *client, const workbase_t *wb, const uc { int transactions = wb->transactions + 1; char hexcoinbase[1024], blockhash[68]; + sdata_t *sdata = client->sdata; json_t *val = NULL, *val_copy; char *gbt_block, varint[12]; ckpool_t *ckp = wb->ckp; - sdata_t *sdata = ckp->data; ckmsg_t *block_ckmsg; char cdfield[64]; uchar swap[32]; @@ -3131,9 +3131,9 @@ static json_t *parse_submit(stratum_instance_t *client, json_t *json_msg, char hexhash[68] = {}, sharehash[32], cdfield[64]; const char *workername, *job_id, *ntime, *nonce; char *fname = NULL, *s, *nonce2; + sdata_t *sdata = client->sdata; enum share_err err = SE_NONE; ckpool_t *ckp = client->ckp; - sdata_t *sdata = ckp->data; char idstring[20] = {}; workbase_t *wb = NULL; uint32_t ntime32; @@ -3202,6 +3202,8 @@ static json_t *parse_submit(stratum_instance_t *client, json_t *json_msg, ck_rlock(&sdata->workbase_lock); HASH_FIND_I64(sdata->workbases, &id, wb); if (unlikely(!wb)) { + if (!sdata->current_workbase) + return json_boolean(false); id = sdata->current_workbase->id; err = SE_INVALID_JOBID; json_set_string(json_msg, "reject-reason", SHARE_ERR(err)); From f15fbdbc705480fae65a7f84b67cb41d102c5103 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Fri, 13 Feb 2015 14:18:30 +1100 Subject: [PATCH 067/544] Receive diff from the upstream proxy and add share submission to the correct psend list --- src/generator.c | 8 ++++---- src/stratifier.c | 10 +++++----- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/generator.c b/src/generator.c index ddf66d4f..2491cc1d 100644 --- a/src/generator.c +++ b/src/generator.c @@ -1285,10 +1285,10 @@ static void submit_share(gdata_t *gdata, json_t *val) json_object_set_nocheck(val, "id", json_integer(share->id)); /* Add the new message to the psend list */ - mutex_lock(&proxi->psend_lock); - DL_APPEND(proxi->psends, msg); - pthread_cond_signal(&proxi->psend_cond); - mutex_unlock(&proxi->psend_lock); + mutex_lock(&proxy->psend_lock); + DL_APPEND(proxy->psends, msg); + pthread_cond_signal(&proxy->psend_cond); + mutex_unlock(&proxy->psend_lock); } static void clear_notify(notify_instance_t *ni) diff --git a/src/stratifier.c b/src/stratifier.c index e9775259..d3682c86 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -1288,11 +1288,6 @@ static void update_diff(ckpool_t *ckp, const char *cmd) proxy_t *proxy; json_t *val; - if (unlikely(!sdata->current_workbase)) { - LOGINFO("No current workbase to update diff yet"); - return; - } - if (unlikely(strlen(cmd) < 6)) { LOGWARNING("Zero length string passed to update_diff"); return; @@ -1320,6 +1315,11 @@ static void update_diff(ckpool_t *ckp, const char *cmd) dsdata = proxy->sdata; + if (unlikely(!dsdata->current_workbase)) { + LOGINFO("No current workbase to update diff yet"); + return; + } + ck_wlock(&dsdata->workbase_lock); old_diff = proxy->diff; dsdata->current_workbase->diff = dsdata->proxy->diff = diff; From 9b04f81666ec177098c11319433b390906d3daeb Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Fri, 13 Feb 2015 14:22:20 +1100 Subject: [PATCH 068/544] Set correct variable in update_diff --- src/stratifier.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/stratifier.c b/src/stratifier.c index d3682c86..0e6bd623 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -1322,7 +1322,7 @@ static void update_diff(ckpool_t *ckp, const char *cmd) ck_wlock(&dsdata->workbase_lock); old_diff = proxy->diff; - dsdata->current_workbase->diff = dsdata->proxy->diff = diff; + dsdata->current_workbase->diff = proxy->diff = diff; ck_wunlock(&dsdata->workbase_lock); if (old_diff < diff) From e287feede0204db3dd86e1d305a50023d3770f84 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Fri, 13 Feb 2015 14:25:47 +1100 Subject: [PATCH 069/544] Send only the parent proxy id from the generator to the stratifier --- src/generator.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/generator.c b/src/generator.c index 2491cc1d..b9e84e7c 100644 --- a/src/generator.c +++ b/src/generator.c @@ -1818,13 +1818,11 @@ reconnect: if (proxi != cproxy) { proxi = cproxy; if (!ckp->passthrough) { - proxy_instance_t *proxy = proxi->proxy; - connsock_t *cs = proxi->cs; LOGWARNING("Successfully connected to proxy %d %s:%s as proxy", proxi->id, cs->url, cs->port); dealloc(buf); - ASPRINTF(&buf, "proxy=%d:%d", proxy->id, proxi->id); + ASPRINTF(&buf, "proxy=%d", proxi->id); send_proc(ckp->stratifier, buf); } } From b13446e387a334dbbed828ee02e5648f3768d5f5 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Fri, 13 Feb 2015 15:02:04 +1100 Subject: [PATCH 070/544] Send proxy subscription to stratifier as soon as we have it --- src/generator.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/generator.c b/src/generator.c index b9e84e7c..ec558eb5 100644 --- a/src/generator.c +++ b/src/generator.c @@ -1468,6 +1468,8 @@ static bool proxy_alive(ckpool_t *ckp, server_instance_t *si, proxy_instance_t * } goto out; } + if (!ckp->passthrough) + send_subscribe(ckp, proxi); if (!auth_stratum(ckp, cs, proxi)) { if (!pinging) { LOGWARNING("Failed initial authorise to %s:%s with %s:%s !", @@ -1482,8 +1484,6 @@ out: Close(cs->fd); } else { keep_sockalive(cs->fd); - if (!ckp->passthrough) - send_subscribe(ckp, proxi); event.events = EPOLLIN; event.data.ptr = proxi; /* Add this connsock_t to the epoll list */ From 9edfbc8790309b0038cc56944ce9e542ed10e0d7 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Fri, 13 Feb 2015 15:08:25 +1100 Subject: [PATCH 071/544] Don't send proxy diff till it's set yet --- src/generator.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/generator.c b/src/generator.c index ec558eb5..ca56ab5b 100644 --- a/src/generator.c +++ b/src/generator.c @@ -986,6 +986,10 @@ static void send_diff(ckpool_t *ckp, proxy_instance_t *proxi) if (proxi == proxy) return; + /* Not set yet */ + if (!proxi->diff) + return; + JSON_CPACK(json_msg, "{sisisf}", "proxy", proxy->id, "subproxy", proxi->id, From 338cc49c7e364dbf6afccf16f51b4c0e3e1e3c6f Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Fri, 13 Feb 2015 15:14:21 +1100 Subject: [PATCH 072/544] Add verbose notify message --- src/stratifier.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/stratifier.c b/src/stratifier.c index 0e6bd623..c1f2c0b0 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -1271,6 +1271,7 @@ static void update_notify(ckpool_t *ckp, const char *cmd) proxy->notified = true; reconnect_clients_to(sdata, id); } + LOGINFO("Broadcast updated stratum notify"); stratum_broadcast_update(dsdata, new_block | clean); out: json_decref(val); From c43d89275a102383e4382e9d7aeb07163998f801 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Fri, 13 Feb 2015 15:52:52 +1100 Subject: [PATCH 073/544] Fix various logic errors dealing with upstream proxy disconnects --- src/generator.c | 32 +++++++++++++++++--------------- 1 file changed, 17 insertions(+), 15 deletions(-) diff --git a/src/generator.c b/src/generator.c index ca56ab5b..484e9460 100644 --- a/src/generator.c +++ b/src/generator.c @@ -1511,6 +1511,7 @@ out: } } } + proxi->alive = ret; return ret; } @@ -1520,8 +1521,8 @@ static proxy_instance_t *create_subproxy(proxy_instance_t *proxi) { proxy_instance_t *subproxy = ckzalloc(sizeof(proxy_instance_t)); - subproxy->ckp = proxi->ckp; subproxy->cs = ckzalloc(sizeof(connsock_t)); + subproxy->cs->ckp = subproxy->ckp = proxi->ckp; subproxy->si = proxi->si; subproxy->id = proxi->subproxy_count; subproxy->auth = proxi->auth; @@ -1562,6 +1563,7 @@ static void *passthrough_recv(void *arg) connsock_t *cs = proxi->cs; ckpool_t *ckp = proxi->ckp; struct epoll_event event; + bool alive; int epfd; rename_proc("passrecv"); @@ -1573,26 +1575,24 @@ static void *passthrough_recv(void *arg) } if (proxy_alive(ckp, si, proxi, cs, false, epfd)) { - proxi->alive = true; send_proc(ckp->generator, "reconnect"); LOGWARNING("Proxy %d:%s connection established", proxi->id, proxi->si->url); } + alive = proxi->alive; while (42) { int ret; while (!proxy_alive(ckp, si, proxi, cs, true, epfd)) { - if (proxi->alive) { - proxi->alive = false; + if (alive) { + alive = false; send_proc(ckp->generator, "reconnect"); } sleep(5); } - if (!proxi->alive) { - proxi->alive = true; + if (!alive) send_proc(ckp->generator, "reconnect"); - } /* Make sure we receive a line within 90 seconds */ ret = epoll_wait(epfd, &event, 1, 90000); @@ -1601,7 +1601,7 @@ static void *passthrough_recv(void *arg) if (ret < 1) { LOGWARNING("Proxy %d:%s failed to read_socket_line in proxy_recv, attempting reconnect", proxi->id, proxi->si->url); - proxi->alive = false; + alive = proxi->alive = false; send_proc(ckp->generator, "reconnect"); continue; } @@ -1638,6 +1638,7 @@ static void *proxy_recv(void *arg) ckpool_t *ckp = proxi->ckp; gdata_t *gdata = ckp->data; struct epoll_event event; + bool alive; int epfd; rename_proc("proxyrecv"); @@ -1649,11 +1650,11 @@ static void *proxy_recv(void *arg) } if (proxy_alive(ckp, si, proxi, cs, false, epfd)) { - proxi->alive = true; send_proc(ckp->generator, "reconnect"); LOGWARNING("Proxy %d:%s connection established", proxi->id, proxi->si->url); } + alive = proxi->alive; while (42) { proxy_instance_t *subproxy = proxi; @@ -1663,8 +1664,8 @@ static void *proxy_recv(void *arg) int ret; while (!proxy_alive(ckp, si, proxi, proxi->cs, true, epfd)) { - if (proxi->alive) { - proxi->alive = false; + if (alive) { + alive = false; send_proc(ckp->generator, "reconnect"); } sleep(5); @@ -1672,13 +1673,13 @@ static void *proxy_recv(void *arg) } /* Wait 90 seconds before declaring this upstream pool alive * to prevent switching to unstable pools. */ - if (!proxi->alive && (!best_proxy(ckp, gdata) || + if (!alive && (!best_proxy(ckp, gdata) || time(NULL) - proxi->reconnect_time > 90)) { LOGWARNING("Proxy %d:%s recovered", proxi->id, proxi->si->url); - proxi->alive = true; proxi->reconnect_time = 0; send_proc(ckp->generator, "reconnect"); } + alive = true; now = time(NULL); @@ -1712,7 +1713,8 @@ static void *proxy_recv(void *arg) ret = read_socket_line(cs, 5); } if (ret < 1) { - if (subproxy->alive) { + if (alive) { + alive = false; LOGWARNING("Proxy %d:%s failed to epoll/read_socket_line in proxy_recv, attempting reconnect", subproxy->id, subproxy->si->url); } @@ -1724,7 +1726,7 @@ static void *proxy_recv(void *arg) * over to a backup pool until the reconnect * pool is up */ subproxy->reconnect = false; - subproxy->alive = false; + alive = subproxy->alive = false; send_proc(ckp->generator, "reconnect"); LOGWARNING("Proxy %d:%s reconnect issue, dropping existing connection", subproxy->id, subproxy->si->url); From 2d35d0c679e7143d3cc6f8d2151993e5c34df718 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Fri, 13 Feb 2015 16:11:11 +1100 Subject: [PATCH 074/544] Parse reconnect differently for subproxies than master proxies --- src/generator.c | 23 ++++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/src/generator.c b/src/generator.c index 484e9460..1e7f7965 100644 --- a/src/generator.c +++ b/src/generator.c @@ -895,6 +895,16 @@ static bool send_pong(proxy_instance_t *proxi, json_t *val) } static void prepare_proxy(proxy_instance_t *proxi); +static proxy_instance_t *create_subproxy(proxy_instance_t *proxi); + +static void add_subproxy(proxy_instance_t *proxi, proxy_instance_t *subproxy) +{ + mutex_lock(&proxi->proxy_lock); + proxi->subproxy_count++; + HASH_ADD_INT(proxi->subproxies, id, subproxy); + proxi->client_headroom += proxi->clients_per_proxy; + mutex_unlock(&proxi->proxy_lock); +} static bool parse_reconnect(proxy_instance_t *proxi, json_t *val) { @@ -945,6 +955,13 @@ static bool parse_reconnect(proxy_instance_t *proxi, json_t *val) LOGINFO("Processing reconnect request to %s", url); ret = true; + /* If this isn't a parent proxy, add a new subproxy to the parent */ + if (proxi != proxi->proxy) { + newproxi = create_subproxy(proxi); + add_subproxy(proxi, newproxi); + goto out; + } + newsi = ckzalloc(sizeof(server_instance_t)); mutex_lock(&gdata->lock); @@ -1545,11 +1562,7 @@ static bool recruit_subproxy(proxy_instance_t *proxi) return false; } - mutex_lock(&proxi->proxy_lock); - proxi->subproxy_count++; - HASH_ADD_INT(proxi->subproxies, id, subproxy); - proxi->client_headroom += proxi->clients_per_proxy; - mutex_unlock(&proxi->proxy_lock); + add_subproxy(proxi, subproxy); return true; } From 31a5e3d5cec22e8495a6d3e0993b9241a129a930 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Fri, 13 Feb 2015 16:28:59 +1100 Subject: [PATCH 075/544] Delete the subproxy entry in the share after sending it to the proxy send thread --- src/generator.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/generator.c b/src/generator.c index 1e7f7965..340e9a88 100644 --- a/src/generator.c +++ b/src/generator.c @@ -1281,7 +1281,6 @@ static void submit_share(gdata_t *gdata, json_t *val) return json_decref(val); } json_get_int(&subid, val, "subproxy"); - json_object_del(val, "subproxy"); proxi = subproxy_by_id(proxy, subid); if (unlikely(!proxi)) { LOGWARNING("Failed to find subproxy %d to send share to", subid); @@ -1390,6 +1389,7 @@ static void *proxy_send(void *arg) continue; json_get_int(&subid, msg->json_msg, "subproxy"); + json_object_del(msg->json_msg, "subproxy"); json_uintcpy(&id, msg->json_msg, "jobid"); mutex_lock(&proxy->notify_lock); From 40b10ee2ba9e582cb4b144ea3c3915ec496b04f2 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Sat, 14 Feb 2015 11:10:03 +1100 Subject: [PATCH 076/544] Add helper to set a json integer and then delete its entry from the json --- src/ckpool.c | 10 ++++++++++ src/ckpool.h | 1 + 2 files changed, 11 insertions(+) diff --git a/src/ckpool.c b/src/ckpool.c index 95490c5e..06fddd1e 100644 --- a/src/ckpool.c +++ b/src/ckpool.c @@ -1051,6 +1051,16 @@ out: return ret; } +bool json_getdel_int(int *store, json_t *val, const char *res) +{ + bool ret; + + ret = json_get_int(store, val, res); + if (ret) + json_object_del(val, res); + return ret; +} + static void parse_btcds(ckpool_t *ckp, const json_t *arr_val, const int arr_size) { json_t *val; diff --git a/src/ckpool.h b/src/ckpool.h index 9b2eb5f9..97fb27f7 100644 --- a/src/ckpool.h +++ b/src/ckpool.h @@ -227,5 +227,6 @@ bool json_get_int64(int64_t *store, const json_t *val, const char *res); bool json_get_int(int *store, const json_t *val, const char *res); bool json_get_double(double *store, const json_t *val, const char *res); bool json_get_bool(bool *store, const json_t *val, const char *res); +bool json_getdel_int(int *store, json_t *val, const char *res); #endif /* CKPOOL_H */ From ed0d20d216ecc567a04520159fbacc344c42f61e Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Sat, 14 Feb 2015 11:12:01 +1100 Subject: [PATCH 077/544] Use json_getdel_int helper in generator --- src/generator.c | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/src/generator.c b/src/generator.c index 340e9a88..e8eec196 100644 --- a/src/generator.c +++ b/src/generator.c @@ -1273,8 +1273,7 @@ static void submit_share(gdata_t *gdata, json_t *val) share_msg_t *share; int id, subid; - json_get_int(&id, val, "proxy"); - json_object_del(val, "proxy"); + json_getdel_int(&id, val, "proxy"); proxy = proxy_by_id(gdata, id); if (unlikely(!proxy)) { LOGWARNING("Failed to find proxy %d to send share to", id); @@ -1290,10 +1289,8 @@ static void submit_share(gdata_t *gdata, json_t *val) msg = ckzalloc(sizeof(stratum_msg_t)); share = ckzalloc(sizeof(share_msg_t)); share->submit_time = time(NULL); - share->client_id = json_integer_value(json_object_get(val, "client_id")); - share->msg_id = json_integer_value(json_object_get(val, "msg_id")); - json_object_del(val, "client_id"); - json_object_del(val, "msg_id"); + json_getdel_int(&share->client_id, val, "client_id"); + json_getdel_int(&share->msg_id, val, "msg_id"); msg->json_msg = val; /* Add new share entry to the share hashtable */ @@ -1388,8 +1385,7 @@ static void *proxy_send(void *arg) if (unlikely(!msg)) continue; - json_get_int(&subid, msg->json_msg, "subproxy"); - json_object_del(msg->json_msg, "subproxy"); + json_getdel_int(&subid, msg->json_msg, "subproxy"); json_uintcpy(&id, msg->json_msg, "jobid"); mutex_lock(&proxy->notify_lock); From 48f7702bd6d91ecafb34e10bd3f79e517c1b7458 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Sat, 14 Feb 2015 11:15:45 +1100 Subject: [PATCH 078/544] Add json_getdel_int64 helper --- src/ckpool.c | 10 ++++++++++ src/ckpool.h | 1 + 2 files changed, 11 insertions(+) diff --git a/src/ckpool.c b/src/ckpool.c index 06fddd1e..1e51d664 100644 --- a/src/ckpool.c +++ b/src/ckpool.c @@ -1061,6 +1061,16 @@ bool json_getdel_int(int *store, json_t *val, const char *res) return ret; } +bool json_getdel_int64(int64_t *store, json_t *val, const char *res) +{ + bool ret; + + ret = json_get_int64(store, val, res); + if (ret) + json_object_del(val, res); + return ret; +} + static void parse_btcds(ckpool_t *ckp, const json_t *arr_val, const int arr_size) { json_t *val; diff --git a/src/ckpool.h b/src/ckpool.h index 97fb27f7..48687646 100644 --- a/src/ckpool.h +++ b/src/ckpool.h @@ -228,5 +228,6 @@ bool json_get_int(int *store, const json_t *val, const char *res); bool json_get_double(double *store, const json_t *val, const char *res); bool json_get_bool(bool *store, const json_t *val, const char *res); bool json_getdel_int(int *store, json_t *val, const char *res); +bool json_getdel_int64(int64_t *store, json_t *val, const char *res); #endif /* CKPOOL_H */ From 8ebd5de4200c9670a36dd83df04686ac54910812 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Sat, 14 Feb 2015 11:16:01 +1100 Subject: [PATCH 079/544] Use json_getdel_int64 helper in the connector --- src/connector.c | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/connector.c b/src/connector.c index 00e480b8..73d1865b 100644 --- a/src/connector.c +++ b/src/connector.c @@ -346,8 +346,7 @@ reparse: char *s; if (client->passthrough) { - passthrough_id = json_integer_value(json_object_get(val, "client_id")); - json_object_del(val, "client_id"); + json_getdel_int64(&passthrough_id, val, "client_id"); passthrough_id = (client->id << 32) | passthrough_id; json_object_set_new_nocheck(val, "client_id", json_integer(passthrough_id)); } else @@ -641,8 +640,7 @@ static void process_client_msg(cdata_t *cdata, const char *buf) } /* Extract the client id from the json message and remove its entry */ - client_id64 = json_integer_value(json_object_get(json_msg, "client_id")); - json_object_del(json_msg, "client_id"); + json_getdel_int64(&client_id64, json_msg, "client_id"); if (client_id64 > 0xffffffffll) { int64_t passthrough_id; From df1465f27546ac3e558dde07047b4c51d6f01cd9 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Sat, 14 Feb 2015 11:20:45 +1100 Subject: [PATCH 080/544] Use msg_id variable for clarity --- src/stratifier.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/stratifier.c b/src/stratifier.c index c1f2c0b0..b28e61a9 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -3294,8 +3294,11 @@ out_unlock: /* Submit share to upstream pool in proxy mode. We submit valid and * stale shares and filter out the rest. */ if (wb && wb->proxy && submit) { + int msg_id = 0; + + json_get_int(&msg_id, json_msg, "id"); LOGINFO("Submitting share upstream: %s", hexhash); - submit_share(client, id, nonce2, ntime, nonce, json_integer_value(json_object_get(json_msg, "id"))); + submit_share(client, id, nonce2, ntime, nonce, msg_id); } add_submit(ckp, client, diff, result, submit); From aa153ba7425acf66b0e02f3c5fa786b2f2f1d53e Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Sat, 14 Feb 2015 11:25:33 +1100 Subject: [PATCH 081/544] Differentiate pong from other spurious messages from clients --- src/stratifier.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/stratifier.c b/src/stratifier.c index b28e61a9..a78be6c7 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -3691,7 +3691,11 @@ static void parse_instance_msg(ckpool_t *ckp, sdata_t *sdata, smsg_t *msg, strat if (res_val) { const char *result = json_string_value(res_val); - LOGDEBUG("Received spurious response %s", result ? result : ""); + if (!safecmp(result, "pong")) + LOGDEBUG("Received pong from client %"PRId64, client_id); + else + LOGDEBUG("Received spurious response %s from client %"PRId64, + result ? result : "", client_id); goto out; } send_json_err(sdata, client_id, id_val, "-3:method not found"); From b59760bb4a56da7cbf63e4b748dabd29893361e2 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Sat, 14 Feb 2015 12:03:41 +1100 Subject: [PATCH 082/544] Disable subproxies as they die, moving them to a dead list instead of trying to reuse them and recruit fresh proxies, disconnecting clients connected to them --- src/generator.c | 70 ++++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 64 insertions(+), 6 deletions(-) diff --git a/src/generator.c b/src/generator.c index e8eec196..83a813e9 100644 --- a/src/generator.c +++ b/src/generator.c @@ -77,6 +77,7 @@ typedef struct proxy_instance proxy_instance_t; /* Per proxied pool instance data */ struct proxy_instance { UT_hash_handle hh; + proxy_instance_t *next; /* For dead proxy list */ ckpool_t *ckp; connsock_t *cs; @@ -101,6 +102,7 @@ struct proxy_instance { bool no_sessionid; /* Doesn't support session id resume on subscribe */ bool no_params; /* Doesn't want any parameters on subscribe */ + bool disabled; /* Subproxy no longer to be used */ bool reconnect; /* We need to drop and reconnect */ bool alive; @@ -137,9 +139,11 @@ struct proxy_instance { /* Private data for the generator */ struct generator_data { + ckpool_t *ckp; pthread_mutex_t lock; /* Lock protecting linked lists */ proxy_instance_t *proxies; /* Hash list of all proxies */ proxy_instance_t *proxy; /* Current proxy */ + proxy_instance_t *dead_proxies; /* Disabled proxies */ int proxy_notify_id; // Globally increasing notify id ckmsgq_t *srvchk; // Server check message queue }; @@ -410,7 +414,7 @@ static bool send_json_msg(connsock_t *cs, json_t *json_msg) sent = write_socket(cs->fd, s, len); dealloc(s); if (sent != len) { - LOGWARNING("Failed to send %d bytes sent %d in send_json_msg", len, sent); + LOGNOTICE("Failed to send %d bytes sent %d in send_json_msg", len, sent); return false; } return true; @@ -896,6 +900,7 @@ static bool send_pong(proxy_instance_t *proxi, json_t *val) static void prepare_proxy(proxy_instance_t *proxi); static proxy_instance_t *create_subproxy(proxy_instance_t *proxi); +static bool recruit_subproxy(proxy_instance_t *proxi); static void add_subproxy(proxy_instance_t *proxi, proxy_instance_t *subproxy) { @@ -906,6 +911,32 @@ static void add_subproxy(proxy_instance_t *proxi, proxy_instance_t *subproxy) mutex_unlock(&proxi->proxy_lock); } +static proxy_instance_t *__subproxy_by_id(proxy_instance_t *proxy, const int id) +{ + proxy_instance_t *subproxy; + + HASH_FIND_INT(proxy->subproxies, &id, subproxy); + return subproxy; +} + +/* Remove the subproxy from the proxi list. Do not remove its ram in case of + * dangling references */ +static void disable_subproxy(gdata_t *gdata, proxy_instance_t *proxi, proxy_instance_t *subproxy) +{ + mutex_lock(&proxi->proxy_lock); + subproxy->disabled = true; + /* Make sure subproxy is still in the list */ + if (__subproxy_by_id(proxi, subproxy->id)) { + HASH_DEL(proxi->subproxies, subproxy); + proxi->client_headroom -= proxi->clients_per_proxy; + LL_PREPEND(gdata->dead_proxies, subproxy); + } + mutex_unlock(&proxi->proxy_lock); + + if (proxi->client_headroom < 42 && proxi->alive) + recruit_subproxy(proxi); +} + static bool parse_reconnect(proxy_instance_t *proxi, json_t *val) { server_instance_t *newsi, *si = proxi->si; @@ -1260,36 +1291,59 @@ static proxy_instance_t *subproxy_by_id(proxy_instance_t *proxy, const int id) proxy_instance_t *subproxy; mutex_lock(&proxy->proxy_lock); - HASH_FIND_INT(proxy->subproxies, &id, subproxy); + subproxy = __subproxy_by_id(proxy, id); + if (subproxy && subproxy->disabled) + subproxy = NULL; mutex_unlock(&proxy->proxy_lock); return subproxy; } +static void stratifier_drop_client(ckpool_t *ckp, int64_t id) +{ + char buf[256]; + + sprintf(buf, "dropclient=%"PRId64, id); + send_proc(ckp->stratifier, buf); +} + static void submit_share(gdata_t *gdata, json_t *val) { proxy_instance_t *proxy, *proxi; + ckpool_t *ckp = gdata->ckp; stratum_msg_t *msg; share_msg_t *share; + int64_t client_id; int id, subid; + /* Get the client id so we can tell the stratifier to drop it if the + * proxy it's bound to is not functional */ + json_getdel_int64(&client_id, val, "client_id"); json_getdel_int(&id, val, "proxy"); proxy = proxy_by_id(gdata, id); if (unlikely(!proxy)) { LOGWARNING("Failed to find proxy %d to send share to", id); + stratifier_drop_client(ckp, client_id); return json_decref(val); } json_get_int(&subid, val, "subproxy"); proxi = subproxy_by_id(proxy, subid); if (unlikely(!proxi)) { - LOGWARNING("Failed to find subproxy %d to send share to", subid); + LOGNOTICE("Failed to find proxy %d:%d to send share to", id, subid); + stratifier_drop_client(ckp, client_id); + return json_decref(val); + } + if (!proxi->alive) { + LOGNOTICE("Client %"PRId64" attempting to send shares to dead proxy %d, dropping", + client_id, id); + stratifier_drop_client(ckp, client_id); return json_decref(val); } msg = ckzalloc(sizeof(stratum_msg_t)); share = ckzalloc(sizeof(share_msg_t)); share->submit_time = time(NULL); - json_getdel_int(&share->client_id, val, "client_id"); + share->client_id = client_id; json_getdel_int(&share->msg_id, val, "msg_id"); msg->json_msg = val; @@ -1361,6 +1415,7 @@ static void *proxy_send(void *arg) { proxy_instance_t *proxy = (proxy_instance_t *)arg; connsock_t *cs = proxy->cs; + gdata_t *gdata = cs->ckp->data; rename_proc("proxysend"); @@ -1405,6 +1460,10 @@ static void *proxy_send(void *arg) "id", json_object_dup(msg->json_msg, "id"), "method", "mining.submit"); ret = send_json_msg(cs, val); + if (unlikely(!ret && !subproxy->disabled)) { + LOGWARNING("Proxy %d:%d dead, disabling", proxy->id, subid); + disable_subproxy(gdata, proxy, subproxy); + } json_decref(val); } else { LOGNOTICE("Proxy %d:%s failed to find matching jobid in proxysend", @@ -1446,8 +1505,6 @@ static void passthrough_add_send(proxy_instance_t *proxi, const char *msg) ckmsgq_add(proxi->passsends, pm); } -static bool recruit_subproxy(proxy_instance_t *proxi); - static bool proxy_alive(ckpool_t *ckp, server_instance_t *si, proxy_instance_t *proxi, connsock_t *cs, bool pinging, int epfd) { @@ -2034,6 +2091,7 @@ int generator(proc_instance_t *pi) LOGWARNING("%s generator starting", ckp->name); gdata = ckzalloc(sizeof(gdata_t)); ckp->data = gdata; + gdata->ckp = ckp; if (ckp->proxy) { ret = proxy_mode(ckp, pi); } else { From 3d44a13470fc172c368150125c7ce37d08141847 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Sat, 14 Feb 2015 12:32:02 +1100 Subject: [PATCH 083/544] Store dead proxy instances to be recycled if possible --- src/generator.c | 55 ++++++++++++++++++++++++++++++++++--------------- 1 file changed, 38 insertions(+), 17 deletions(-) diff --git a/src/generator.c b/src/generator.c index 83a813e9..ae1a6c56 100644 --- a/src/generator.c +++ b/src/generator.c @@ -78,6 +78,7 @@ typedef struct proxy_instance proxy_instance_t; struct proxy_instance { UT_hash_handle hh; proxy_instance_t *next; /* For dead proxy list */ + proxy_instance_t *prev; /* For dead proxy list */ ckpool_t *ckp; connsock_t *cs; @@ -899,8 +900,8 @@ static bool send_pong(proxy_instance_t *proxi, json_t *val) } static void prepare_proxy(proxy_instance_t *proxi); -static proxy_instance_t *create_subproxy(proxy_instance_t *proxi); -static bool recruit_subproxy(proxy_instance_t *proxi); +static proxy_instance_t *create_subproxy(gdata_t *gdata, proxy_instance_t *proxi); +static bool recruit_subproxy(gdata_t *gdata, proxy_instance_t *proxi); static void add_subproxy(proxy_instance_t *proxi, proxy_instance_t *subproxy) { @@ -919,22 +920,32 @@ static proxy_instance_t *__subproxy_by_id(proxy_instance_t *proxy, const int id) return subproxy; } -/* Remove the subproxy from the proxi list. Do not remove its ram in case of - * dangling references */ +/* Add to the dead list to be recycled if possible */ +static void store_proxy(gdata_t *gdata, proxy_instance_t *proxy) +{ + mutex_lock(&gdata->lock); + DL_APPEND(gdata->dead_proxies, proxy); + mutex_unlock(&gdata->lock); +} + +/* Remove the subproxy from the proxi list and put it on the dead list */ static void disable_subproxy(gdata_t *gdata, proxy_instance_t *proxi, proxy_instance_t *subproxy) { mutex_lock(&proxi->proxy_lock); subproxy->disabled = true; /* Make sure subproxy is still in the list */ - if (__subproxy_by_id(proxi, subproxy->id)) { + subproxy = __subproxy_by_id(proxi, subproxy->id); + if (subproxy) { HASH_DEL(proxi->subproxies, subproxy); proxi->client_headroom -= proxi->clients_per_proxy; - LL_PREPEND(gdata->dead_proxies, subproxy); } mutex_unlock(&proxi->proxy_lock); + if (subproxy) + store_proxy(gdata, subproxy); + if (proxi->client_headroom < 42 && proxi->alive) - recruit_subproxy(proxi); + recruit_subproxy(gdata, proxi); } static bool parse_reconnect(proxy_instance_t *proxi, json_t *val) @@ -988,7 +999,7 @@ static bool parse_reconnect(proxy_instance_t *proxi, json_t *val) ret = true; /* If this isn't a parent proxy, add a new subproxy to the parent */ if (proxi != proxi->proxy) { - newproxi = create_subproxy(proxi); + newproxi = create_subproxy(gdata, proxi); add_subproxy(proxi, newproxi); goto out; } @@ -1508,6 +1519,7 @@ static void passthrough_add_send(proxy_instance_t *proxi, const char *msg) static bool proxy_alive(ckpool_t *ckp, server_instance_t *si, proxy_instance_t *proxi, connsock_t *cs, bool pinging, int epfd) { + gdata_t *gdata = ckp->data; struct epoll_event event; bool ret = false; @@ -1571,7 +1583,7 @@ out: * recruit extra when asked by the stratifier. */ while (proxi->client_headroom < 42) { /* Note recursive call of proxy_alive here */ - if (!recruit_subproxy(proxi)) { + if (!recruit_subproxy(gdata, proxi)) { LOGWARNING("Unable to recruit extra subproxies after just %"PRId64, proxi->client_headroom); break; @@ -1587,11 +1599,21 @@ out: /* Creates a duplicate instance or proxi to be used as a subproxy, ignoring * fields we don't use in the subproxy. */ -static proxy_instance_t *create_subproxy(proxy_instance_t *proxi) +static proxy_instance_t *create_subproxy(gdata_t *gdata, proxy_instance_t *proxi) { - proxy_instance_t *subproxy = ckzalloc(sizeof(proxy_instance_t)); + proxy_instance_t *subproxy; + + mutex_lock(&gdata->lock); + if (gdata->dead_proxies) { + /* Recycle an old proxy instance if one exists */ + subproxy = gdata->dead_proxies; + DL_DELETE(gdata->dead_proxies, subproxy); + } else { + subproxy = ckzalloc(sizeof(proxy_instance_t)); + subproxy->cs = ckzalloc(sizeof(connsock_t)); + } + mutex_unlock(&gdata->lock); - subproxy->cs = ckzalloc(sizeof(connsock_t)); subproxy->cs->ckp = subproxy->ckp = proxi->ckp; subproxy->si = proxi->si; subproxy->id = proxi->subproxy_count; @@ -1603,15 +1625,14 @@ static proxy_instance_t *create_subproxy(proxy_instance_t *proxi) return subproxy; } -static bool recruit_subproxy(proxy_instance_t *proxi) +static bool recruit_subproxy(gdata_t *gdata, proxy_instance_t *proxi) { - proxy_instance_t *subproxy = create_subproxy(proxi); + proxy_instance_t *subproxy = create_subproxy(gdata, proxi); int epfd = proxi->epfd; if (!proxy_alive(subproxy->ckp, subproxy->si, subproxy, subproxy->cs, false, epfd)) { LOGNOTICE("Subproxy failed proxy_alive testing"); - free(subproxy->cs); - free(subproxy); + store_proxy(gdata, subproxy); return false; } @@ -1943,7 +1964,7 @@ retry: LOGDEBUG("Proxy received ping request"); send_unix_msg(sockd, "pong"); } else if (cmdmatch(buf, "recruit")) { - recruit_subproxy(proxi); + recruit_subproxy(gdata, proxi); } else if (ckp->passthrough) { /* Anything remaining should be stratum messages */ passthrough_add_send(proxi, buf); From 81370d2c87d4214ee8ed28735b16a3b30032a4a7 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Sat, 14 Feb 2015 12:41:48 +1100 Subject: [PATCH 084/544] Reset variables that will affect functioning of recycled proxies --- src/generator.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/generator.c b/src/generator.c index ae1a6c56..dad0448a 100644 --- a/src/generator.c +++ b/src/generator.c @@ -1608,9 +1608,11 @@ static proxy_instance_t *create_subproxy(gdata_t *gdata, proxy_instance_t *proxi /* Recycle an old proxy instance if one exists */ subproxy = gdata->dead_proxies; DL_DELETE(gdata->dead_proxies, subproxy); + subproxy->disabled = false; } else { subproxy = ckzalloc(sizeof(proxy_instance_t)); subproxy->cs = ckzalloc(sizeof(connsock_t)); + mutex_init(&subproxy->share_lock); } mutex_unlock(&gdata->lock); @@ -1621,7 +1623,6 @@ static proxy_instance_t *create_subproxy(gdata_t *gdata, proxy_instance_t *proxi subproxy->pass = proxi->pass; subproxy->proxy = proxi; subproxy->epfd = proxi->epfd; - mutex_init(&subproxy->share_lock); return subproxy; } From 0092e0982c83a545ee7b51b1a65cbfb07de365b7 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Sat, 14 Feb 2015 12:54:31 +1100 Subject: [PATCH 085/544] Create a helper function for determining parent proxy --- src/generator.c | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/src/generator.c b/src/generator.c index dad0448a..443d5635 100644 --- a/src/generator.c +++ b/src/generator.c @@ -558,6 +558,11 @@ out: return buf; } +static inline bool parent_proxy(proxy_instance_t *proxy) +{ + return (proxy->proxy == proxy); +} + static bool parse_subscribe(connsock_t *cs, proxy_instance_t *proxi) { json_t *val = NULL, *res_val, *notify_val, *tmp; @@ -651,7 +656,7 @@ retry: goto out; } proxi->nonce2len = size; - if (proxi->proxy == proxi) { + if (parent_proxy(proxi)) { /* Set the number of clients per proxy on the parent proxy */ proxi->clients_per_proxy = 1ll << ((size - 3) * 8); LOGNOTICE("Proxy %d:%s clients per proxy: %"PRId64, proxi->id, proxi->si->url, @@ -1042,7 +1047,7 @@ static void send_diff(ckpool_t *ckp, proxy_instance_t *proxi) char *msg, *buf; /* Master proxy, we don't use this for work */ - if (proxi == proxy) + if (parent_proxy(proxi)) return; /* Not set yet */ @@ -1070,7 +1075,7 @@ static void send_notify(ckpool_t *ckp, proxy_instance_t *proxi) int i; /* Master proxy, we don't use this for work */ - if (proxi == proxy) + if (parent_proxy(proxi)) return; merkle_arr = json_array(); @@ -1281,7 +1286,7 @@ static void send_subscribe(ckpool_t *ckp, proxy_instance_t *proxi) char *msg, *buf; /* Master proxy, we don't use this for work */ - if (proxi == proxi->proxy) + if (parent_proxy(proxi)) return; JSON_CPACK(json_msg, "{sisisssi}", @@ -1578,7 +1583,7 @@ out: cs->fd, epfd); return false; } - if (!ckp->passthrough && proxi->proxy == proxi) { + if (!ckp->passthrough && parent_proxy(proxi)) { /* We recruit enough proxies to begin with and then * recruit extra when asked by the stratifier. */ while (proxi->client_headroom < 42) { From 5ce95d99a724df30e9f288d6fb9b55071d1de608 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Sat, 14 Feb 2015 14:21:20 +1100 Subject: [PATCH 086/544] Use the parent proxy as one of the subproxies as upstream pools may disconnect idle clients and the parent determines the children's activities --- src/generator.c | 20 ++++---------------- 1 file changed, 4 insertions(+), 16 deletions(-) diff --git a/src/generator.c b/src/generator.c index 443d5635..340b67e0 100644 --- a/src/generator.c +++ b/src/generator.c @@ -659,6 +659,7 @@ retry: if (parent_proxy(proxi)) { /* Set the number of clients per proxy on the parent proxy */ proxi->clients_per_proxy = 1ll << ((size - 3) * 8); + proxi->client_headroom = proxi->clients_per_proxy; LOGNOTICE("Proxy %d:%s clients per proxy: %"PRId64, proxi->id, proxi->si->url, proxi->clients_per_proxy); } @@ -1046,10 +1047,6 @@ static void send_diff(ckpool_t *ckp, proxy_instance_t *proxi) json_t *json_msg; char *msg, *buf; - /* Master proxy, we don't use this for work */ - if (parent_proxy(proxi)) - return; - /* Not set yet */ if (!proxi->diff) return; @@ -1074,10 +1071,6 @@ static void send_notify(ckpool_t *ckp, proxy_instance_t *proxi) char *msg, *buf; int i; - /* Master proxy, we don't use this for work */ - if (parent_proxy(proxi)) - return; - merkle_arr = json_array(); mutex_lock(&proxy->notify_lock); @@ -1285,10 +1278,6 @@ static void send_subscribe(ckpool_t *ckp, proxy_instance_t *proxi) json_t *json_msg; char *msg, *buf; - /* Master proxy, we don't use this for work */ - if (parent_proxy(proxi)) - return; - JSON_CPACK(json_msg, "{sisisssi}", "proxy", proxi->proxy->id, "subproxy", proxi->id, @@ -1476,10 +1465,6 @@ static void *proxy_send(void *arg) "id", json_object_dup(msg->json_msg, "id"), "method", "mining.submit"); ret = send_json_msg(cs, val); - if (unlikely(!ret && !subproxy->disabled)) { - LOGWARNING("Proxy %d:%d dead, disabling", proxy->id, subid); - disable_subproxy(gdata, proxy, subproxy); - } json_decref(val); } else { LOGNOTICE("Proxy %d:%s failed to find matching jobid in proxysend", @@ -1492,6 +1477,8 @@ static void *proxy_send(void *arg) LOGWARNING("Proxy %d:%s failed to send msg in proxy_send, dropping to reconnect", proxy->id, proxy->si->url); Close(cs->fd); + if (!parent_proxy && !subproxy->disabled) + disable_subproxy(gdata, proxy, subproxy); } } return NULL; @@ -1841,6 +1828,7 @@ static void prepare_proxy(proxy_instance_t *proxi) { proxi->proxy = proxi; mutex_init(&proxi->proxy_lock); + add_subproxy(proxi, proxi); mutex_init(&proxi->psend_lock); cond_init(&proxi->psend_cond); create_pthread(&proxi->pth_psend, proxy_send, proxi); From 1e7a78691595e121215a64ed4f302a3e139b4367 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Sat, 14 Feb 2015 14:29:02 +1100 Subject: [PATCH 087/544] Fix misuse of function --- src/generator.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/generator.c b/src/generator.c index 340b67e0..b3a6072a 100644 --- a/src/generator.c +++ b/src/generator.c @@ -1477,7 +1477,7 @@ static void *proxy_send(void *arg) LOGWARNING("Proxy %d:%s failed to send msg in proxy_send, dropping to reconnect", proxy->id, proxy->si->url); Close(cs->fd); - if (!parent_proxy && !subproxy->disabled) + if (!parent_proxy(subproxy) && !subproxy->disabled) disable_subproxy(gdata, proxy, subproxy); } } From d0f557bbf89f60dfe9c6a45807954fe495c78ba8 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Sat, 14 Feb 2015 14:44:33 +1100 Subject: [PATCH 088/544] Handle reconnect message differently for parent proxy than child subproxies --- src/generator.c | 20 +++++++++++++------- src/stratifier.c | 13 ++++++------- 2 files changed, 19 insertions(+), 14 deletions(-) diff --git a/src/generator.c b/src/generator.c index b3a6072a..fecf0895 100644 --- a/src/generator.c +++ b/src/generator.c @@ -105,6 +105,7 @@ struct proxy_instance { bool disabled; /* Subproxy no longer to be used */ bool reconnect; /* We need to drop and reconnect */ + bool reconnecting; /* Parsed reconnect, need to reconnect clients */ bool alive; pthread_mutex_t notify_lock; @@ -1805,13 +1806,17 @@ static void *proxy_recv(void *arg) /* Call this proxy dead to allow us to fail * over to a backup pool until the reconnect * pool is up */ - subproxy->reconnect = false; - alive = subproxy->alive = false; - send_proc(ckp->generator, "reconnect"); - LOGWARNING("Proxy %d:%s reconnect issue, dropping existing connection", - subproxy->id, subproxy->si->url); Close(cs->fd); - break; + subproxy->reconnect = false; + subproxy->reconnecting = true; + subproxy->alive = false; + if (parent_proxy(subproxy)) { + alive = false; + send_proc(ckp->generator, "reconnect"); + LOGWARNING("Proxy %d:%s reconnect issue, dropping existing connection", + subproxy->id, subproxy->si->url); + break; + } } continue; } @@ -1902,7 +1907,8 @@ reconnect: cproxy = best_proxy(ckp, gdata); if (!cproxy) goto out; - if (proxi != cproxy) { + if (proxi != cproxy || cproxy->reconnecting) { + cproxy->reconnecting = false; proxi = cproxy; if (!ckp->passthrough) { connsock_t *cs = proxi->cs; diff --git a/src/stratifier.c b/src/stratifier.c index a78be6c7..76e86c2a 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -1102,16 +1102,14 @@ static proxy_t *subproxy_by_id(sdata_t *sdata, const int id, const int subid) } /* Iterates over all clients in proxy mode and sets the reconnect bool for the - * message to be sent lazily next time they speak to us if they're not bound - * to the requested proxy id */ -static void reconnect_clients_to(sdata_t *sdata, const int id) + * message to be sent lazily next time they speak to us */ +static void reconnect_clients(sdata_t *sdata) { stratum_instance_t *client, *tmp; ck_rlock(&sdata->instance_lock); HASH_ITER(hh, sdata->stratum_instances, client, tmp) { - if (client->proxyid != id) - client->reconnect = true; + client->reconnect = true; } ck_runlock(&sdata->instance_lock); } @@ -1269,7 +1267,7 @@ static void update_notify(ckpool_t *ckp, const char *cmd) * clients now to reconnect since we have enough information to * switch. */ proxy->notified = true; - reconnect_clients_to(sdata, id); + reconnect_clients(sdata); } LOGINFO("Broadcast updated stratum notify"); stratum_broadcast_update(dsdata, new_block | clean); @@ -1913,7 +1911,8 @@ static char *stratifier_stats(ckpool_t *ckp, sdata_t *sdata) } /* Sets the currently active proxy. Clients will be told to reconnect once the - * first notify data comes from this proxy. */ + * first notify data comes from this proxy. Even if we are already bound to + * this proxy we are only given this message if all clients must move. */ static void set_proxy(sdata_t *sdata, const char *buf) { proxy_t *proxy; From ca8d50280d8f9046788268befc516dfe187466f0 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Sat, 14 Feb 2015 14:51:37 +1100 Subject: [PATCH 089/544] Only set alive to false on parent proxy failing --- src/generator.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/generator.c b/src/generator.c index fecf0895..3a769e14 100644 --- a/src/generator.c +++ b/src/generator.c @@ -1794,10 +1794,11 @@ static void *proxy_recv(void *arg) ret = read_socket_line(cs, 5); } if (ret < 1) { - if (alive) { + if (alive && parent_proxy(subproxy)) { alive = false; LOGWARNING("Proxy %d:%s failed to epoll/read_socket_line in proxy_recv, attempting reconnect", subproxy->id, subproxy->si->url); + send_proc(ckp->generator, "reconnect"); } continue; } From bef6bd57db738d82207707ce8bfc19e0e4529407 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Sat, 14 Feb 2015 14:59:21 +1100 Subject: [PATCH 090/544] Make best_proxy non blocking compared to wait_best_proxy --- src/generator.c | 27 ++++++++++++++++++++++++--- 1 file changed, 24 insertions(+), 3 deletions(-) diff --git a/src/generator.c b/src/generator.c index 3a769e14..08c551dc 100644 --- a/src/generator.c +++ b/src/generator.c @@ -1694,7 +1694,27 @@ static void *passthrough_recv(void *arg) return NULL; } -static proxy_instance_t *best_proxy(ckpool_t *ckp, gdata_t *gdata); +static proxy_instance_t *best_proxy(ckpool_t *ckp, gdata_t *gdata) +{ + proxy_instance_t *ret = NULL, *proxi, *tmp; + + mutex_lock(&gdata->lock); + HASH_ITER(hh, gdata->proxies, proxi, tmp) { + if (proxi->alive) { + if (!ret) { + ret = proxi; + continue; + } + if (proxi->id < ret->id) + ret = proxi; + } + } + gdata->proxy = ret; + mutex_unlock(&gdata->lock); + + send_proc(ckp->connector, ret ? "accept" : "reject"); + return ret; +} #if 0 static proxy_instance_t *current_proxy(gdata_t *gdata) @@ -1862,7 +1882,7 @@ static void setup_proxies(ckpool_t *ckp, gdata_t *gdata) } } -static proxy_instance_t *best_proxy(ckpool_t *ckp, gdata_t *gdata) +static proxy_instance_t *wait_best_proxy(ckpool_t *ckp, gdata_t *gdata) { proxy_instance_t *ret = NULL, *proxi, *tmp; @@ -1886,6 +1906,7 @@ static proxy_instance_t *best_proxy(ckpool_t *ckp, gdata_t *gdata) if (ret) break; + send_proc(ckp->connector, "reject"); sleep(1); } send_proc(ckp->connector, ret ? "accept" : "reject"); @@ -1905,7 +1926,7 @@ static int proxy_loop(proc_instance_t *pi) reconnect: /* This does not necessarily mean we reconnect, but a change has * occurred and we need to reexamine the proxies. */ - cproxy = best_proxy(ckp, gdata); + cproxy = wait_best_proxy(ckp, gdata); if (!cproxy) goto out; if (proxi != cproxy || cproxy->reconnecting) { From 008c9a32d93add3c36ca26699b7181cc2c3806e1 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Sat, 14 Feb 2015 15:57:12 +1100 Subject: [PATCH 091/544] Give subproxies their own hashlist to allow them to be on two hashtables --- src/generator.c | 25 ++++++++++--------------- 1 file changed, 10 insertions(+), 15 deletions(-) diff --git a/src/generator.c b/src/generator.c index 08c551dc..227214da 100644 --- a/src/generator.c +++ b/src/generator.c @@ -76,7 +76,8 @@ typedef struct proxy_instance proxy_instance_t; /* Per proxied pool instance data */ struct proxy_instance { - UT_hash_handle hh; + UT_hash_handle hh; /* Proxy list */ + UT_hash_handle sh; /* Subproxy list */ proxy_instance_t *next; /* For dead proxy list */ proxy_instance_t *prev; /* For dead proxy list */ @@ -84,7 +85,8 @@ struct proxy_instance { connsock_t *cs; server_instance_t *si; bool passthrough; - int id; /* Proxy server id, or subproxy id if this is a subproxy */ + int id; /* Proxy server id*/ + int subid; /* Subproxy id */ const char *auth; const char *pass; @@ -914,16 +916,16 @@ static void add_subproxy(proxy_instance_t *proxi, proxy_instance_t *subproxy) { mutex_lock(&proxi->proxy_lock); proxi->subproxy_count++; - HASH_ADD_INT(proxi->subproxies, id, subproxy); + HASH_ADD(sh, proxi->subproxies, subid, sizeof(int), subproxy); proxi->client_headroom += proxi->clients_per_proxy; mutex_unlock(&proxi->proxy_lock); } -static proxy_instance_t *__subproxy_by_id(proxy_instance_t *proxy, const int id) +static proxy_instance_t *__subproxy_by_id(proxy_instance_t *proxy, const int subid) { proxy_instance_t *subproxy; - HASH_FIND_INT(proxy->subproxies, &id, subproxy); + HASH_FIND(sh, proxy->subproxies, &subid, sizeof(int), subproxy); return subproxy; } @@ -1005,7 +1007,7 @@ static bool parse_reconnect(proxy_instance_t *proxi, json_t *val) ret = true; /* If this isn't a parent proxy, add a new subproxy to the parent */ - if (proxi != proxi->proxy) { + if (!parent_proxy(proxi)) { newproxi = create_subproxy(gdata, proxi); add_subproxy(proxi, newproxi); goto out; @@ -1016,8 +1018,6 @@ static bool parse_reconnect(proxy_instance_t *proxi, json_t *val) mutex_lock(&gdata->lock); HASH_DEL(gdata->proxies, proxi); newsi->id = si->id; /* Inherit the old connection's id */ - si->id = ckp->proxies++; /* Give the old connection the lowest id */ - ckp->servers = realloc(ckp->servers, sizeof(server_instance_t *) * ckp->proxies); ckp->servers[newsi->id] = newsi; newsi->url = url; newsi->auth = strdup(si->auth); @@ -1033,7 +1033,6 @@ static bool parse_reconnect(proxy_instance_t *proxi, json_t *val) newproxi->cs = &newsi->cs; newproxi->cs->ckp = ckp; newproxi->id = newsi->id; - HASH_ADD_INT(gdata->proxies, id, proxi); HASH_ADD_INT(gdata->proxies, id, newproxi); mutex_unlock(&gdata->lock); @@ -1611,7 +1610,7 @@ static proxy_instance_t *create_subproxy(gdata_t *gdata, proxy_instance_t *proxi subproxy->cs->ckp = subproxy->ckp = proxi->ckp; subproxy->si = proxi->si; - subproxy->id = proxi->subproxy_count; + subproxy->subid = proxi->subproxy_count; subproxy->auth = proxi->auth; subproxy->pass = proxi->pass; subproxy->proxy = proxi; @@ -1893,11 +1892,7 @@ static proxy_instance_t *wait_best_proxy(ckpool_t *ckp, gdata_t *gdata) mutex_lock(&gdata->lock); HASH_ITER(hh, gdata->proxies, proxi, tmp) { if (proxi->alive) { - if (!ret) { - ret = proxi; - continue; - } - if (proxi->id < ret->id) + if (!ret || proxi->id < ret->id) ret = proxi; } } From 31b0e4df67f64da86385344823dd4caffaccd6e8 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Sat, 14 Feb 2015 15:59:37 +1100 Subject: [PATCH 092/544] Only check for existence of current proxy when switching, not best which may have already been set to the alive proxy --- src/generator.c | 27 +-------------------------- 1 file changed, 1 insertion(+), 26 deletions(-) diff --git a/src/generator.c b/src/generator.c index 227214da..557b0c27 100644 --- a/src/generator.c +++ b/src/generator.c @@ -1693,29 +1693,6 @@ static void *passthrough_recv(void *arg) return NULL; } -static proxy_instance_t *best_proxy(ckpool_t *ckp, gdata_t *gdata) -{ - proxy_instance_t *ret = NULL, *proxi, *tmp; - - mutex_lock(&gdata->lock); - HASH_ITER(hh, gdata->proxies, proxi, tmp) { - if (proxi->alive) { - if (!ret) { - ret = proxi; - continue; - } - if (proxi->id < ret->id) - ret = proxi; - } - } - gdata->proxy = ret; - mutex_unlock(&gdata->lock); - - send_proc(ckp->connector, ret ? "accept" : "reject"); - return ret; -} - -#if 0 static proxy_instance_t *current_proxy(gdata_t *gdata) { proxy_instance_t *ret; @@ -1726,7 +1703,6 @@ static proxy_instance_t *current_proxy(gdata_t *gdata) return ret; } -#endif /* For receiving messages from the upstream proxy, also responsible for setting * up the connection and testing it's alive. */ @@ -1773,8 +1749,7 @@ static void *proxy_recv(void *arg) } /* Wait 90 seconds before declaring this upstream pool alive * to prevent switching to unstable pools. */ - if (!alive && (!best_proxy(ckp, gdata) || - time(NULL) - proxi->reconnect_time > 90)) { + if (!alive && (!current_proxy(gdata) || time(NULL) - proxi->reconnect_time > 90)) { LOGWARNING("Proxy %d:%s recovered", proxi->id, proxi->si->url); proxi->reconnect_time = 0; send_proc(ckp->generator, "reconnect"); From d9ea42bdc7f7ec57d41885c36171da8b5a4adbf6 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Sat, 14 Feb 2015 16:05:06 +1100 Subject: [PATCH 093/544] Disable subproxies that are given a reconnect message, allowing more subproxies to be recruited by parent's reconnect if need be instead --- src/generator.c | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/src/generator.c b/src/generator.c index 557b0c27..fc1ad22b 100644 --- a/src/generator.c +++ b/src/generator.c @@ -1006,12 +1006,12 @@ static bool parse_reconnect(proxy_instance_t *proxi, json_t *val) LOGINFO("Processing reconnect request to %s", url); ret = true; - /* If this isn't a parent proxy, add a new subproxy to the parent */ - if (!parent_proxy(proxi)) { - newproxi = create_subproxy(gdata, proxi); - add_subproxy(proxi, newproxi); + proxi->reconnect = true; + + /* If this isn't a parent proxy, simply set the reconnect bool allowing + * it to be disabled. More will be recruited if necessary */ + if (!parent_proxy(proxi)) goto out; - } newsi = ckzalloc(sizeof(server_instance_t)); @@ -1811,7 +1811,8 @@ static void *proxy_recv(void *arg) LOGWARNING("Proxy %d:%s reconnect issue, dropping existing connection", subproxy->id, subproxy->si->url); break; - } + } else + disable_subproxy(gdata, proxi, subproxy); } continue; } From 8476d128762720493e4ad81060c9a21c047e03ca Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Sat, 14 Feb 2015 16:21:27 +1100 Subject: [PATCH 094/544] Fix failover of parent proxy --- src/generator.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/generator.c b/src/generator.c index fc1ad22b..c23b7da8 100644 --- a/src/generator.c +++ b/src/generator.c @@ -1753,8 +1753,8 @@ static void *proxy_recv(void *arg) LOGWARNING("Proxy %d:%s recovered", proxi->id, proxi->si->url); proxi->reconnect_time = 0; send_proc(ckp->generator, "reconnect"); + alive = true; } - alive = true; now = time(NULL); @@ -1788,7 +1788,7 @@ static void *proxy_recv(void *arg) ret = read_socket_line(cs, 5); } if (ret < 1) { - if (alive && parent_proxy(subproxy)) { + if (parent_proxy(subproxy)) { alive = false; LOGWARNING("Proxy %d:%s failed to epoll/read_socket_line in proxy_recv, attempting reconnect", subproxy->id, subproxy->si->url); From 37b6406e0ed69d44ee621db696795fefa1e1c022 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Sat, 14 Feb 2015 16:45:46 +1100 Subject: [PATCH 095/544] Set share response to true even if we don't find the matching share --- src/generator.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/generator.c b/src/generator.c index c23b7da8..f8301d21 100644 --- a/src/generator.c +++ b/src/generator.c @@ -1401,11 +1401,13 @@ static bool parse_share(proxy_instance_t *proxi, const char *buf) HASH_DEL(proxi->shares, share); mutex_unlock(&proxi->share_lock); + /* We set response to true even if we don't find the matching share, + * so long as we recognised it as a share response */ + ret = true; if (!share) { LOGINFO("Failed to find matching share to result: %s", buf); goto out; } - ret = true; LOGDEBUG("Found share from client %d with msg_id %d", share->client_id, share->msg_id); free(share); From 413c73bc338467c5c39a8dcff3536af398ba0bf7 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Sat, 14 Feb 2015 16:59:59 +1100 Subject: [PATCH 096/544] Drop the child subproxies if the parent proxy is not alive --- src/generator.c | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/src/generator.c b/src/generator.c index f8301d21..d57d9fad 100644 --- a/src/generator.c +++ b/src/generator.c @@ -945,6 +945,7 @@ static void disable_subproxy(gdata_t *gdata, proxy_instance_t *proxi, proxy_inst /* Make sure subproxy is still in the list */ subproxy = __subproxy_by_id(proxi, subproxy->id); if (subproxy) { + Close(subproxy->cs->fd); HASH_DEL(proxi->subproxies, subproxy); proxi->client_headroom -= proxi->clients_per_proxy; } @@ -1510,6 +1511,20 @@ static void passthrough_add_send(proxy_instance_t *proxi, const char *msg) ckmsgq_add(proxi->passsends, pm); } +/* If the parent is dead, we shouldn't use any of the child subproxies to be + * used. */ +static void drop_subproxies(proxy_instance_t *proxi) +{ + proxy_instance_t *subproxy, *tmp; + + mutex_lock(&proxi->proxy_lock); + HASH_ITER(sh, proxi->subproxies, subproxy, tmp) { + if (!parent_proxy(subproxy)) + Close(subproxy->cs->fd); + } + mutex_unlock(&proxi->proxy_lock); +} + static bool proxy_alive(ckpool_t *ckp, server_instance_t *si, proxy_instance_t *proxi, connsock_t *cs, bool pinging, int epfd) { @@ -1588,6 +1603,8 @@ out: } } proxi->alive = ret; + if (!ret && parent_proxy(proxi)) + drop_subproxies(proxi); return ret; } From 371b59f452e2f24befaf30e357fbea6de864248f Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Sat, 14 Feb 2015 17:09:07 +1100 Subject: [PATCH 097/544] Use a different hashlist for the subproxies in the stratifier to avoid hashlist clash --- src/stratifier.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/stratifier.c b/src/stratifier.c index 76e86c2a..f1bccbad 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -300,6 +300,7 @@ typedef struct proxy_base proxy_t; struct proxy_base { UT_hash_handle hh; + UT_hash_handle sh; /* For subproxy hashlist */ int id; int subid; @@ -1045,7 +1046,7 @@ static proxy_t *__generate_subproxy(sdata_t *sdata, proxy_t *proxy, const int su subproxy->id = proxy->id; subproxy->subid = subid; - HASH_ADD_INT(proxy->subproxies, subid, subproxy); + HASH_ADD(sh, proxy->subproxies, subid, sizeof(int), subproxy); subproxy->sdata = duplicate_sdata(sdata); subproxy->sdata->subproxy = subproxy; subproxy->parent = proxy; @@ -1070,7 +1071,7 @@ static proxy_t *__subproxy_by_id(sdata_t *sdata, proxy_t *proxy, const int subid { proxy_t *subproxy; - HASH_FIND_INT(proxy->subproxies, &subid, subproxy); + HASH_FIND(sh, proxy->subproxies, &subid, sizeof(int), subproxy); if (!subproxy) subproxy = __generate_subproxy(sdata, proxy, subid); return subproxy; @@ -2193,7 +2194,7 @@ static sdata_t *select_sdata(const ckpool_t *ckp, sdata_t *ckp_sdata) return NULL; } mutex_lock(&ckp_sdata->proxy_lock); - HASH_ITER(hh, proxy->subproxies, subproxy, tmp) { + HASH_ITER(sh, proxy->subproxies, subproxy, tmp) { int64_t subproxy_headroom = subproxy->max_clients - subproxy->clients; headroom += subproxy_headroom; From c8025958f962382e01251e126c195076178ff67f Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Sat, 14 Feb 2015 17:15:23 +1100 Subject: [PATCH 098/544] Add notify and diff subid details --- src/stratifier.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/stratifier.c b/src/stratifier.c index f1bccbad..3af9c63c 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -1210,7 +1210,7 @@ static void update_notify(ckpool_t *ckp, const char *cmd) LOGNOTICE("No valid proxy %d:%d subscription to update notify yet", id, subid); goto out; } - LOGNOTICE("Got updated notify for proxy %d", id); + LOGNOTICE("Got updated notify for proxy %d:%d", id, subid); wb = ckzalloc(sizeof(workbase_t)); wb->ckp = ckp; @@ -1305,7 +1305,7 @@ static void update_diff(ckpool_t *ckp, const char *cmd) json_dblcpy(&diff, val, "diff"); json_decref(val); - LOGNOTICE("Got updated diff for proxy %d", id); + LOGNOTICE("Got updated diff for proxy %d:%d", id, subid); proxy = subproxy_by_id(sdata, id, subid); /* We only really care about integer diffs so clamp the lower limit to From 566d6c656e0a686c261170e1f66706e0850e0568 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Sat, 14 Feb 2015 17:20:16 +1100 Subject: [PATCH 099/544] Send correct subproxy id to stratifier --- src/generator.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/generator.c b/src/generator.c index d57d9fad..93a9390e 100644 --- a/src/generator.c +++ b/src/generator.c @@ -1054,7 +1054,7 @@ static void send_diff(ckpool_t *ckp, proxy_instance_t *proxi) JSON_CPACK(json_msg, "{sisisf}", "proxy", proxy->id, - "subproxy", proxi->id, + "subproxy", proxi->subid, "diff", proxi->diff); msg = json_dumps(json_msg, JSON_NO_UTF8); json_decref(json_msg); @@ -1085,7 +1085,7 @@ static void send_notify(ckpool_t *ckp, proxy_instance_t *proxi) json_array_append_new(merkle_arr, json_string(&ni->merklehash[i][0])); /* Use our own jobid instead of the server's one for easy lookup */ JSON_CPACK(json_msg, "{sisisisssisssssosssssssb}", - "proxy", proxy->id, "subproxy", proxi->id, + "proxy", proxy->id, "subproxy", proxi->subid, "jobid", ni->id, "prevhash", ni->prevhash, "coinb1len", ni->coinb1len, "coinbase1", ni->coinbase1, "coinbase2", ni->coinbase2, "merklehash", merkle_arr, "bbversion", ni->bbversion, @@ -1281,7 +1281,7 @@ static void send_subscribe(ckpool_t *ckp, proxy_instance_t *proxi) JSON_CPACK(json_msg, "{sisisssi}", "proxy", proxi->proxy->id, - "subproxy", proxi->id, + "subproxy", proxi->subid, "enonce1", proxi->enonce1, "nonce2len", proxi->nonce2len); msg = json_dumps(json_msg, JSON_NO_UTF8); From 5cb12347a7ea79827c6ce2b84e1de7a5eedf86f6 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Sat, 14 Feb 2015 17:36:21 +1100 Subject: [PATCH 100/544] Add parent proxy as a subproxy in stratifier --- src/stratifier.c | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/src/stratifier.c b/src/stratifier.c index 3af9c63c..12d0d4f3 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -321,7 +321,7 @@ struct proxy_base { int64_t max_clients; enonce1_t enonce1u; - proxy_t *parent; /* Parent proxy - set to NULL on parent itself */ + proxy_t *parent; /* Parent proxy - set to self on parent itself */ proxy_t *subproxies; /* Hashlist of subproxies sorted by subid */ sdata_t *sdata; /* Unique stratifer data for each subproxy */ }; @@ -1030,12 +1030,16 @@ static sdata_t *duplicate_sdata(const sdata_t *sdata) return dsdata; } -/* Note that proxies don't have unique sdata, only the subproxies do */ static proxy_t *__generate_proxy(sdata_t *sdata, const int id) { proxy_t *proxy = ckzalloc(sizeof(proxy_t)); proxy->id = id; + proxy->sdata = duplicate_sdata(sdata); + proxy->sdata->subproxy = proxy; + proxy->parent = proxy; + /* subid == 0 on parent proxy */ + HASH_ADD(sh, proxy->subproxies, subid, sizeof(int), proxy); HASH_ADD_INT(sdata->proxies, id, proxy); return proxy; } @@ -1061,7 +1065,7 @@ static proxy_t *__proxy_by_id(sdata_t *sdata, const int id) HASH_FIND_INT(sdata->proxies, &id, proxy); if (unlikely(!proxy)) { proxy = __generate_proxy(sdata, id); - LOGINFO("Stratifier added new proxy %d", id); + LOGNOTICE("Stratifier added new proxy %d", id); } return proxy; @@ -1072,8 +1076,10 @@ static proxy_t *__subproxy_by_id(sdata_t *sdata, proxy_t *proxy, const int subid proxy_t *subproxy; HASH_FIND(sh, proxy->subproxies, &subid, sizeof(int), subproxy); - if (!subproxy) + if (!subproxy) { subproxy = __generate_subproxy(sdata, proxy, subid); + LOGINFO("Stratifier added new subproxy %d:%d", proxy->id, subid); + } return subproxy; } @@ -1926,6 +1932,8 @@ static void set_proxy(sdata_t *sdata, const char *buf) sdata->proxy = proxy; mutex_unlock(&sdata->proxy_lock); + LOGNOTICE("Stratifier setting active proxy to %d", id); + /* We will receive a notification immediately after this and it should * be the flag to reconnect clients. */ proxy->notified = false; From 5dc3f9cb22a8564a55c08322869c826e9ed41667 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Sat, 14 Feb 2015 18:21:57 +1100 Subject: [PATCH 101/544] Only issue reconnect when the parent pool has set its notified flag --- src/stratifier.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/stratifier.c b/src/stratifier.c index 12d0d4f3..a2f11b41 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -1269,7 +1269,7 @@ static void update_notify(ckpool_t *ckp, const char *cmd) add_base(ckp, dsdata, wb, &new_block); - if (proxy->parent == current_proxy(sdata) && !proxy->notified) { + if (proxy == current_proxy(sdata) && !proxy->notified) { /* This is the first notification from the current proxy, tell * clients now to reconnect since we have enough information to * switch. */ From a7da93981e812c06f65c0a41086277b27221299a Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Sun, 15 Feb 2015 10:45:04 +1100 Subject: [PATCH 102/544] Determine whether we need to reconnect clients based on the parent proxy's notify id which is reset on each subscribe --- src/stratifier.c | 27 +++++++++++++++++---------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/src/stratifier.c b/src/stratifier.c index a2f11b41..ab86244e 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -278,6 +278,7 @@ struct stratum_instance { sdata_t *sdata; /* Which sdata this client is bound to */ int proxyid; /* Which proxy this is bound to in proxy mode */ int subproxyid; /* Which subproxy */ + int64_t notify_id; /* Which notify_id from the subproxy did we join */ }; struct share { @@ -316,6 +317,7 @@ struct proxy_base { bool subscribed; bool notified; + int64_t notify_id; /* What ID was the first notify from this proxy */ int64_t clients; int64_t max_clients; @@ -1110,13 +1112,14 @@ static proxy_t *subproxy_by_id(sdata_t *sdata, const int id, const int subid) /* Iterates over all clients in proxy mode and sets the reconnect bool for the * message to be sent lazily next time they speak to us */ -static void reconnect_clients(sdata_t *sdata) +static void reconnect_clients(sdata_t *sdata, const int proxyid, const int64_t notify_id) { stratum_instance_t *client, *tmp; ck_rlock(&sdata->instance_lock); HASH_ITER(hh, sdata->stratum_instances, client, tmp) { - client->reconnect = true; + if (client->proxyid != proxyid || client->notify_id != notify_id) + client->reconnect = true; } ck_runlock(&sdata->instance_lock); } @@ -1157,7 +1160,7 @@ static void update_subscribe(ckpool_t *ckp, const char *cmd) LOGNOTICE("Got updated subscribe for proxy %d:%d", id, subid); proxy = subproxy_by_id(sdata, id, subid); - proxy->notified = false; /* Reset this */ + proxy->notify_id = -1; /* Reset this */ dsdata = proxy->sdata; ck_wlock(&dsdata->workbase_lock); @@ -1186,6 +1189,11 @@ static void update_subscribe(ckpool_t *ckp, const char *cmd) json_decref(val); } +static inline bool parent_proxy(const proxy_t *proxy) +{ + return (proxy->parent == proxy); +} + static void update_notify(ckpool_t *ckp, const char *cmd) { sdata_t *sdata = ckp->data, *dsdata; @@ -1269,12 +1277,14 @@ static void update_notify(ckpool_t *ckp, const char *cmd) add_base(ckp, dsdata, wb, &new_block); - if (proxy == current_proxy(sdata) && !proxy->notified) { + if (parent_proxy(proxy) && proxy->notify_id == -1) { /* This is the first notification from the current proxy, tell * clients now to reconnect since we have enough information to * switch. */ - proxy->notified = true; - reconnect_clients(sdata); + proxy->notify_id = wb->id; + LOGINFO("Setting reconnect to proxy %d notifyid %"PRId64, proxy->id, proxy->notify_id); + if (proxy == current_proxy(sdata)) + reconnect_clients(sdata, proxy->id, proxy->notify_id); } LOGINFO("Broadcast updated stratum notify"); stratum_broadcast_update(dsdata, new_block | clean); @@ -1933,10 +1943,6 @@ static void set_proxy(sdata_t *sdata, const char *buf) mutex_unlock(&sdata->proxy_lock); LOGNOTICE("Stratifier setting active proxy to %d", id); - - /* We will receive a notification immediately after this and it should - * be the flag to reconnect clients. */ - proxy->notified = false; } static int stratum_loop(ckpool_t *ckp, proc_instance_t *pi) @@ -2119,6 +2125,7 @@ static bool new_enonce1(ckpool_t *ckp, sdata_t *ckp_sdata, sdata_t *sdata, strat enonce1u = &proxy->enonce1u; client->proxyid = proxy->id; client->subproxyid = proxy->subid; + client->notify_id = proxy->parent->notify_id; mutex_unlock(&ckp_sdata->proxy_lock); if (proxy->clients >= proxy->max_clients) { From 26ca36feb1eaa03bdb85b6b06e2ed9de39e0b3a4 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Sun, 15 Feb 2015 10:57:08 +1100 Subject: [PATCH 103/544] Set reconnect on both notify id fix and setproxy when we have enough information --- src/stratifier.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/stratifier.c b/src/stratifier.c index ab86244e..8c45262b 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -1116,6 +1116,8 @@ static void reconnect_clients(sdata_t *sdata, const int proxyid, const int64_t n { stratum_instance_t *client, *tmp; + LOGINFO("Setting reconnect to proxy %d notifyid %"PRId64, proxyid, notify_id); + ck_rlock(&sdata->instance_lock); HASH_ITER(hh, sdata->stratum_instances, client, tmp) { if (client->proxyid != proxyid || client->notify_id != notify_id) @@ -1282,7 +1284,6 @@ static void update_notify(ckpool_t *ckp, const char *cmd) * clients now to reconnect since we have enough information to * switch. */ proxy->notify_id = wb->id; - LOGINFO("Setting reconnect to proxy %d notifyid %"PRId64, proxy->id, proxy->notify_id); if (proxy == current_proxy(sdata)) reconnect_clients(sdata, proxy->id, proxy->notify_id); } @@ -1943,6 +1944,8 @@ static void set_proxy(sdata_t *sdata, const char *buf) mutex_unlock(&sdata->proxy_lock); LOGNOTICE("Stratifier setting active proxy to %d", id); + if (proxy->notify_id != -1) + reconnect_clients(sdata, proxy->id, proxy->notify_id); } static int stratum_loop(ckpool_t *ckp, proc_instance_t *pi) From 051ee7278a910ac2f43869289fc70a0fab926ebf Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Sun, 15 Feb 2015 11:13:10 +1100 Subject: [PATCH 104/544] Iterate over correct hashtable with ckp sdata for stratum broadcasts --- src/stratifier.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/stratifier.c b/src/stratifier.c index 8c45262b..6a230a83 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -1572,8 +1572,8 @@ static void stratum_broadcast(sdata_t *sdata, json_t *val) return; } - ck_rlock(&sdata->instance_lock); - HASH_ITER(hh, sdata->stratum_instances, client, tmp) { + ck_rlock(&ckp_sdata->instance_lock); + HASH_ITER(hh, ckp_sdata->stratum_instances, client, tmp) { ckmsg_t *client_msg; smsg_t *msg; @@ -1588,7 +1588,7 @@ static void stratum_broadcast(sdata_t *sdata, json_t *val) client_msg->data = msg; DL_APPEND(bulk_send, client_msg); } - ck_runlock(&sdata->instance_lock); + ck_runlock(&ckp_sdata->instance_lock); json_decref(val); From 395d1277a7248596faaed573d0befcfc1483e5af Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Sun, 15 Feb 2015 11:50:13 +1100 Subject: [PATCH 105/544] Set notify_id on all proxies in case we switch later to them with setproxy --- src/stratifier.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/stratifier.c b/src/stratifier.c index 6a230a83..40d4a8ca 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -1279,12 +1279,12 @@ static void update_notify(ckpool_t *ckp, const char *cmd) add_base(ckp, dsdata, wb, &new_block); - if (parent_proxy(proxy) && proxy->notify_id == -1) { + if (proxy->notify_id == -1) { /* This is the first notification from the current proxy, tell * clients now to reconnect since we have enough information to * switch. */ proxy->notify_id = wb->id; - if (proxy == current_proxy(sdata)) + if (parent_proxy(proxy) && proxy == current_proxy(sdata)) reconnect_clients(sdata, proxy->id, proxy->notify_id); } LOGINFO("Broadcast updated stratum notify"); From 7af4b89138900553bf60e6e38dd77d013dbe6f3c Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Sun, 15 Feb 2015 13:37:44 +1100 Subject: [PATCH 106/544] Demote json_result decode failed message to notice level --- src/generator.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/generator.c b/src/generator.c index 93a9390e..062ed6d8 100644 --- a/src/generator.c +++ b/src/generator.c @@ -456,7 +456,7 @@ static json_t *json_result(json_t *val) else ss = strdup("(unknown reason)"); - LOGWARNING("JSON-RPC decode failed: %s", ss); + LOGNOTICE("JSON-RPC decode of json_result failed: %s", ss); free(ss); } return res_val; From 0fc4507bf3612980d118aa717131e3cd43d14642 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Sun, 15 Feb 2015 13:42:16 +1100 Subject: [PATCH 107/544] Add more info to failed to authorise --- src/generator.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/generator.c b/src/generator.c index 062ed6d8..ed6e69b4 100644 --- a/src/generator.c +++ b/src/generator.c @@ -1237,8 +1237,8 @@ static bool auth_stratum(ckpool_t *ckp, connsock_t *cs, proxy_instance_t *proxi) if (res_val) { ret = json_is_true(res_val); if (!ret) { - LOGWARNING("Proxy %d:%s failed to authorise in auth_stratum", - proxi->id, proxi->si->url); + LOGWARNING("Proxy %d:%s failed to authorise in auth_stratum, got: %s", + proxi->id, proxi->si->url, buf); goto out; } } else { From b7680263aa5e5e66ad5d045a468ea3297235a849 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Sun, 15 Feb 2015 14:11:30 +1100 Subject: [PATCH 108/544] Set subproxies to disabled when dropping them --- src/generator.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/generator.c b/src/generator.c index ed6e69b4..0c96851a 100644 --- a/src/generator.c +++ b/src/generator.c @@ -1519,8 +1519,10 @@ static void drop_subproxies(proxy_instance_t *proxi) mutex_lock(&proxi->proxy_lock); HASH_ITER(sh, proxi->subproxies, subproxy, tmp) { - if (!parent_proxy(subproxy)) + if (!parent_proxy(subproxy)) { + subproxy->disabled = true; Close(subproxy->cs->fd); + } } mutex_unlock(&proxi->proxy_lock); } From 4d5ef2a76a6686995e2884c03ab3f8166bd5cc84 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Sun, 15 Feb 2015 14:18:35 +1100 Subject: [PATCH 109/544] Don't try to recruit extra subproxies on initial connect --- src/generator.c | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/src/generator.c b/src/generator.c index 0c96851a..4a58371c 100644 --- a/src/generator.c +++ b/src/generator.c @@ -1530,7 +1530,6 @@ static void drop_subproxies(proxy_instance_t *proxi) static bool proxy_alive(ckpool_t *ckp, server_instance_t *si, proxy_instance_t *proxi, connsock_t *cs, bool pinging, int epfd) { - gdata_t *gdata = ckp->data; struct epoll_event event; bool ret = false; @@ -1589,20 +1588,6 @@ out: cs->fd, epfd); return false; } - if (!ckp->passthrough && parent_proxy(proxi)) { - /* We recruit enough proxies to begin with and then - * recruit extra when asked by the stratifier. */ - while (proxi->client_headroom < 42) { - /* Note recursive call of proxy_alive here */ - if (!recruit_subproxy(gdata, proxi)) { - LOGWARNING("Unable to recruit extra subproxies after just %"PRId64, - proxi->client_headroom); - break; - } - LOGWARNING("Proxy %d:%s recruited extra subproxy!", - proxi->id, cs->url); - } - } } proxi->alive = ret; if (!ret && parent_proxy(proxi)) From ffdac57b0e78d705857c3dd99a71415b90054c62 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Sun, 15 Feb 2015 14:20:05 +1100 Subject: [PATCH 110/544] Decrease failback time to 30 seconds on proxy --- src/generator.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/generator.c b/src/generator.c index 4a58371c..ab9384ba 100644 --- a/src/generator.c +++ b/src/generator.c @@ -1753,9 +1753,9 @@ static void *proxy_recv(void *arg) sleep(5); proxi->reconnect_time = time(NULL); } - /* Wait 90 seconds before declaring this upstream pool alive + /* Wait 30 seconds before declaring this upstream pool alive * to prevent switching to unstable pools. */ - if (!alive && (!current_proxy(gdata) || time(NULL) - proxi->reconnect_time > 90)) { + if (!alive && (!current_proxy(gdata) || time(NULL) - proxi->reconnect_time > 30)) { LOGWARNING("Proxy %d:%s recovered", proxi->id, proxi->si->url); proxi->reconnect_time = 0; send_proc(ckp->generator, "reconnect"); From 835ff395c2230e73e0fdd0e7be6a478101f54d9a Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Sun, 15 Feb 2015 14:46:36 +1100 Subject: [PATCH 111/544] Reset client and enonce1u count on resubscribe --- src/stratifier.c | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/stratifier.c b/src/stratifier.c index 40d4a8ca..36a5a3c4 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -1183,6 +1183,11 @@ static void update_subscribe(ckpool_t *ckp, const char *cmd) proxy->enonce1varlen = 0; proxy->enonce2varlen = proxy->nonce2len - proxy->enonce1varlen; proxy->max_clients = 1ll << (proxy->enonce1varlen * 8); + /* Reset the enonce1u in case this is a resubscribe of an existing + * parent proxy. All clients previously bound will be disconnected so + * we can start with a fresh count. */ + proxy->clients = 0; + proxy->enonce1u.u64 = 0; ck_wunlock(&dsdata->workbase_lock); LOGNOTICE("Upstream pool extranonce2 length %d, max proxy clients %"PRId64, @@ -2228,7 +2233,8 @@ static sdata_t *select_sdata(const ckpool_t *ckp, sdata_t *ckp_sdata) send_generator(ckp, "recruit", GEN_PRIORITY); } if (!best) { - LOGWARNING("Insufficient subproxies to accept more clients"); + LOGNOTICE("Temporarily insufficient subproxies of proxy %d to accept more clients", + proxy->id); return NULL; } return best->sdata; From a2131e438deb4db74f52e19fd91d6d588983eb71 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Sun, 15 Feb 2015 14:49:12 +1100 Subject: [PATCH 112/544] Drop subproxies when parent proxy is invalidated --- src/generator.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/generator.c b/src/generator.c index ab9384ba..15053041 100644 --- a/src/generator.c +++ b/src/generator.c @@ -1931,6 +1931,7 @@ retry: if (unlikely(proxi->cs->fd < 0)) { LOGWARNING("Upstream proxy %d:%s socket invalidated, will attempt failover", proxi->id, proxi->cs->url); + drop_subproxies(proxi); proxi->alive = false; proxi = NULL; goto reconnect; From b4bf8672695cc44e5e3b4a3ba2287159aa7f8d35 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Sun, 15 Feb 2015 14:52:39 +1100 Subject: [PATCH 113/544] Always disable subproxies on failure to send message --- src/generator.c | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/generator.c b/src/generator.c index 15053041..489ac3ab 100644 --- a/src/generator.c +++ b/src/generator.c @@ -1476,10 +1476,12 @@ static void *proxy_send(void *arg) free(jobid); json_decref(msg->json_msg); free(msg); - if (!ret && subproxy && cs->fd > 0) { - LOGWARNING("Proxy %d:%s failed to send msg in proxy_send, dropping to reconnect", - proxy->id, proxy->si->url); - Close(cs->fd); + if (!ret && subproxy) { + if (cs->fd > 0) { + LOGWARNING("Proxy %d:%s failed to send msg in proxy_send, dropping to reconnect", + proxy->id, proxy->si->url); + Close(cs->fd); + } if (!parent_proxy(subproxy) && !subproxy->disabled) disable_subproxy(gdata, proxy, subproxy); } From 02f31fccc32fbb7ee6dbd5825484477ea9cf6852 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Sun, 15 Feb 2015 15:04:40 +1100 Subject: [PATCH 114/544] Add reconnect proxies to the hashlist using hash_replace in case it has already been removed, and add it to its own subproxy list --- src/generator.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/generator.c b/src/generator.c index 489ac3ab..bea00a68 100644 --- a/src/generator.c +++ b/src/generator.c @@ -932,6 +932,8 @@ static proxy_instance_t *__subproxy_by_id(proxy_instance_t *proxy, const int sub /* Add to the dead list to be recycled if possible */ static void store_proxy(gdata_t *gdata, proxy_instance_t *proxy) { + LOGINFO("Recycling data from proxy %d:%d", proxy->id, proxy->subid); + mutex_lock(&gdata->lock); DL_APPEND(gdata->dead_proxies, proxy); mutex_unlock(&gdata->lock); @@ -1017,7 +1019,6 @@ static bool parse_reconnect(proxy_instance_t *proxi, json_t *val) newsi = ckzalloc(sizeof(server_instance_t)); mutex_lock(&gdata->lock); - HASH_DEL(gdata->proxies, proxi); newsi->id = si->id; /* Inherit the old connection's id */ ckp->servers[newsi->id] = newsi; newsi->url = url; @@ -1034,7 +1035,8 @@ static bool parse_reconnect(proxy_instance_t *proxi, json_t *val) newproxi->cs = &newsi->cs; newproxi->cs->ckp = ckp; newproxi->id = newsi->id; - HASH_ADD_INT(gdata->proxies, id, newproxi); + HASH_REPLACE_INT(gdata->proxies, id, newproxi, proxi); + HASH_ADD(sh, newproxi->subproxies, subid, sizeof(int), newproxi); mutex_unlock(&gdata->lock); prepare_proxy(newproxi); @@ -1830,6 +1832,8 @@ static void *proxy_recv(void *arg) /* If it's not a method it should be a share result */ LOGWARNING("Unhandled stratum message: %s", cs->buf); } + store_proxy(gdata, proxi); + return NULL; } From e24b6846d862211849b8af9842033b367493f9a8 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Sun, 15 Feb 2015 15:52:15 +1100 Subject: [PATCH 115/544] Count stats from ckp sdata to show proper proxy hashrate --- src/stratifier.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/stratifier.c b/src/stratifier.c index 36a5a3c4..f95d14f8 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -2824,20 +2824,20 @@ static double sane_tdiff(tv_t *end, tv_t *start) static void add_submit(ckpool_t *ckp, stratum_instance_t *client, const int diff, const bool valid, const bool submit) { + sdata_t *ckp_sdata = ckp->data, *sdata = client->sdata; worker_instance_t *worker = client->worker_instance; double tdiff, bdiff, dsps, drr, network_diff, bias; user_instance_t *user = client->user_instance; - sdata_t *sdata = client->sdata; int64_t next_blockid, optimal; tv_t now_t; - mutex_lock(&sdata->stats_lock); + mutex_lock(&ckp_sdata->stats_lock); if (valid) { - sdata->stats.unaccounted_shares++; - sdata->stats.unaccounted_diff_shares += diff; + ckp_sdata->stats.unaccounted_shares++; + ckp_sdata->stats.unaccounted_diff_shares += diff; } else - sdata->stats.unaccounted_rejects += diff; - mutex_unlock(&sdata->stats_lock); + ckp_sdata->stats.unaccounted_rejects += diff; + mutex_unlock(&ckp_sdata->stats_lock); /* Count only accepted and stale rejects in diff calculation. */ if (!valid && !submit) From 39f283925225fd9f2fe6922b5c80433c82d820b5 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Sun, 15 Feb 2015 20:19:41 +1100 Subject: [PATCH 116/544] Send notify as soon as a proxy receives it instead of trying to look it up later --- src/generator.c | 20 ++++++-------------- 1 file changed, 6 insertions(+), 14 deletions(-) diff --git a/src/generator.c b/src/generator.c index bea00a68..808a408d 100644 --- a/src/generator.c +++ b/src/generator.c @@ -783,7 +783,9 @@ out: return ret; } -static bool parse_notify(proxy_instance_t *proxi, json_t *val) +static void send_notify(ckpool_t *ckp, proxy_instance_t *proxi, notify_instance_t *ni); + +static bool parse_notify(ckpool_t *ckp, proxy_instance_t *proxi, json_t *val) { const char *prev_hash, *bbversion, *nbit, *ntime; proxy_instance_t *proxy = proxi->proxy; @@ -855,6 +857,7 @@ static bool parse_notify(proxy_instance_t *proxi, json_t *val) proxi->current_notify = ni; mutex_unlock(&proxy->notify_lock); + send_notify(ckp, proxi, ni); out: return ret; } @@ -1066,23 +1069,15 @@ static void send_diff(ckpool_t *ckp, proxy_instance_t *proxi) free(buf); } -static void send_notify(ckpool_t *ckp, proxy_instance_t *proxi) +static void send_notify(ckpool_t *ckp, proxy_instance_t *proxi, notify_instance_t *ni) { proxy_instance_t *proxy = proxi->proxy; json_t *json_msg, *merkle_arr; - notify_instance_t *ni; char *msg, *buf; int i; merkle_arr = json_array(); - mutex_lock(&proxy->notify_lock); - ni = proxi->current_notify; - if (unlikely(!ni)) { - mutex_unlock(&proxy->notify_lock); - LOGNOTICE("Proxi %d not ready to send notify", proxi->id); - return; - } for (i = 0; i < ni->merkles; i++) json_array_append_new(merkle_arr, json_string(&ni->merklehash[i][0])); /* Use our own jobid instead of the server's one for easy lookup */ @@ -1093,7 +1088,6 @@ static void send_notify(ckpool_t *ckp, proxy_instance_t *proxi) "merklehash", merkle_arr, "bbversion", ni->bbversion, "nbit", ni->nbit, "ntime", ni->ntime, "clean", ni->clean); - mutex_unlock(&proxy->notify_lock); msg = json_dumps(json_msg, JSON_NO_UTF8); json_decref(json_msg); @@ -1153,9 +1147,7 @@ static bool parse_method(ckpool_t *ckp, proxy_instance_t *proxi, const char *msg } if (cmdmatch(buf, "mining.notify")) { - ret = parse_notify(proxi, params); - if (ret) - send_notify(ckp, proxi); + ret = parse_notify(ckp, proxi, params); goto out; } From fe6fe5bac2b7a90ae0a7c379a9dfecbfd7292fed Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Sun, 15 Feb 2015 20:22:53 +1100 Subject: [PATCH 117/544] Remove the now unused current_notify --- src/generator.c | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/generator.c b/src/generator.c index 808a408d..b4eb73fc 100644 --- a/src/generator.c +++ b/src/generator.c @@ -112,7 +112,6 @@ struct proxy_instance { pthread_mutex_t notify_lock; notify_instance_t *notify_instances; - notify_instance_t *current_notify; pthread_t pth_precv; pthread_t pth_psend; @@ -853,8 +852,6 @@ static bool parse_notify(ckpool_t *ckp, proxy_instance_t *proxi, json_t *val) mutex_lock(&proxy->notify_lock); ni->id = gdata->proxy_notify_id++; HASH_ADD_INT(proxy->notify_instances, id, ni); - /* Now set the subproxy's current notify to this */ - proxi->current_notify = ni; mutex_unlock(&proxy->notify_lock); send_notify(ckp, proxi, ni); From 5fb4e8d545d10c813d7836866ff1b2b336d2b611 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Sun, 15 Feb 2015 21:41:49 +1100 Subject: [PATCH 118/544] Set the parent proxy to the new instance on a reconnect --- src/generator.c | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/src/generator.c b/src/generator.c index b4eb73fc..abcbd008 100644 --- a/src/generator.c +++ b/src/generator.c @@ -135,7 +135,7 @@ struct proxy_instance { pthread_mutex_t proxy_lock; /* Lock protecting hashlist of proxies */ int64_t clients_per_proxy; /* How many clients can connect to each subproxy */ int64_t client_headroom; /* How many more clients can we connect */ - proxy_instance_t *proxy; /* Parent proxy of subproxies */ + proxy_instance_t *parent; /* Parent proxy of subproxies */ proxy_instance_t *subproxies; /* Hashlist of subproxies of this proxy */ int subproxy_count; /* Number of subproxies */ }; @@ -562,7 +562,7 @@ out: static inline bool parent_proxy(proxy_instance_t *proxy) { - return (proxy->proxy == proxy); + return (proxy->parent == proxy); } static bool parse_subscribe(connsock_t *cs, proxy_instance_t *proxi) @@ -787,7 +787,7 @@ static void send_notify(ckpool_t *ckp, proxy_instance_t *proxi, notify_instance_ static bool parse_notify(ckpool_t *ckp, proxy_instance_t *proxi, json_t *val) { const char *prev_hash, *bbversion, *nbit, *ntime; - proxy_instance_t *proxy = proxi->proxy; + proxy_instance_t *proxy = proxi->parent; char *job_id, *coinbase1, *coinbase2; gdata_t *gdata = proxi->ckp->data; bool clean, ret = false; @@ -1035,6 +1035,7 @@ static bool parse_reconnect(proxy_instance_t *proxi, json_t *val) newproxi->cs = &newsi->cs; newproxi->cs->ckp = ckp; newproxi->id = newsi->id; + newproxi->parent = newproxi; HASH_REPLACE_INT(gdata->proxies, id, newproxi, proxi); HASH_ADD(sh, newproxi->subproxies, subid, sizeof(int), newproxi); mutex_unlock(&gdata->lock); @@ -1046,7 +1047,7 @@ out: static void send_diff(ckpool_t *ckp, proxy_instance_t *proxi) { - proxy_instance_t *proxy = proxi->proxy; + proxy_instance_t *proxy = proxi->parent; json_t *json_msg; char *msg, *buf; @@ -1068,7 +1069,7 @@ static void send_diff(ckpool_t *ckp, proxy_instance_t *proxi) static void send_notify(ckpool_t *ckp, proxy_instance_t *proxi, notify_instance_t *ni) { - proxy_instance_t *proxy = proxi->proxy; + proxy_instance_t *proxy = proxi->parent; json_t *json_msg, *merkle_arr; char *msg, *buf; int i; @@ -1271,7 +1272,7 @@ static void send_subscribe(ckpool_t *ckp, proxy_instance_t *proxi) char *msg, *buf; JSON_CPACK(json_msg, "{sisisssi}", - "proxy", proxi->proxy->id, + "proxy", proxi->parent->id, "subproxy", proxi->subid, "enonce1", proxi->enonce1, "nonce2len", proxi->nonce2len); @@ -1612,7 +1613,7 @@ static proxy_instance_t *create_subproxy(gdata_t *gdata, proxy_instance_t *proxi subproxy->subid = proxi->subproxy_count; subproxy->auth = proxi->auth; subproxy->pass = proxi->pass; - subproxy->proxy = proxi; + subproxy->parent = proxi; subproxy->epfd = proxi->epfd; return subproxy; } @@ -1828,7 +1829,7 @@ static void *proxy_recv(void *arg) static void prepare_proxy(proxy_instance_t *proxi) { - proxi->proxy = proxi; + proxi->parent = proxi; mutex_init(&proxi->proxy_lock); add_subproxy(proxi, proxi); mutex_init(&proxi->psend_lock); From edbe863a262a95a2c6bc3dd5925b215653f34a59 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Sun, 15 Feb 2015 21:52:32 +1100 Subject: [PATCH 119/544] Only add the parent proxy to the subproxy list once --- src/generator.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/generator.c b/src/generator.c index abcbd008..07a7fc1b 100644 --- a/src/generator.c +++ b/src/generator.c @@ -1035,9 +1035,7 @@ static bool parse_reconnect(proxy_instance_t *proxi, json_t *val) newproxi->cs = &newsi->cs; newproxi->cs->ckp = ckp; newproxi->id = newsi->id; - newproxi->parent = newproxi; HASH_REPLACE_INT(gdata->proxies, id, newproxi, proxi); - HASH_ADD(sh, newproxi->subproxies, subid, sizeof(int), newproxi); mutex_unlock(&gdata->lock); prepare_proxy(newproxi); From 7adf0f1c7e4351f180e449ea6c0440215bce38b4 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Sun, 15 Feb 2015 22:08:32 +1100 Subject: [PATCH 120/544] All reconnects will discard the original proxy instance data so shut down the associated send thread as well as the receive and remove unnecessary extra flags --- src/generator.c | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/src/generator.c b/src/generator.c index 07a7fc1b..8307ab39 100644 --- a/src/generator.c +++ b/src/generator.c @@ -107,7 +107,6 @@ struct proxy_instance { bool disabled; /* Subproxy no longer to be used */ bool reconnect; /* We need to drop and reconnect */ - bool reconnecting; /* Parsed reconnect, need to reconnect clients */ bool alive; pthread_mutex_t notify_lock; @@ -1024,7 +1023,6 @@ static bool parse_reconnect(proxy_instance_t *proxi, json_t *val) newsi->url = url; newsi->auth = strdup(si->auth); newsi->pass = strdup(si->pass); - proxi->reconnect = true; newproxi = ckzalloc(sizeof(proxy_instance_t)); newsi->data = newproxi; @@ -1426,16 +1424,28 @@ static void *proxy_send(void *arg) int subid = 0; json_t *val; uint32_t id; + tv_t now; + ts_t abs; + + if (unlikely(proxy->reconnect)) { + LOGINFO("Shutting down proxy_send thread for proxy %d to reconnect", + proxy->id); + break; + } + + tv_time(&now); + tv_to_ts(&abs, &now); + abs.tv_sec++; mutex_lock(&proxy->psend_lock); if (!proxy->psends) - pthread_cond_wait(&proxy->psend_cond, &proxy->psend_lock); + pthread_cond_timedwait(&proxy->psend_cond, &proxy->psend_lock, &abs); msg = proxy->psends; if (likely(msg)) DL_DELETE(proxy->psends, msg); mutex_unlock(&proxy->psend_lock); - if (unlikely(!msg)) + if (!msg) continue; json_getdel_int(&subid, msg->json_msg, "subproxy"); @@ -1800,11 +1810,8 @@ static void *proxy_recv(void *arg) * over to a backup pool until the reconnect * pool is up */ Close(cs->fd); - subproxy->reconnect = false; - subproxy->reconnecting = true; subproxy->alive = false; if (parent_proxy(subproxy)) { - alive = false; send_proc(ckp->generator, "reconnect"); LOGWARNING("Proxy %d:%s reconnect issue, dropping existing connection", subproxy->id, subproxy->si->url); @@ -1820,7 +1827,6 @@ static void *proxy_recv(void *arg) /* If it's not a method it should be a share result */ LOGWARNING("Unhandled stratum message: %s", cs->buf); } - store_proxy(gdata, proxi); return NULL; } @@ -1900,8 +1906,7 @@ reconnect: cproxy = wait_best_proxy(ckp, gdata); if (!cproxy) goto out; - if (proxi != cproxy || cproxy->reconnecting) { - cproxy->reconnecting = false; + if (proxi != cproxy) { proxi = cproxy; if (!ckp->passthrough) { connsock_t *cs = proxi->cs; From d61ba33e69efeaa4476c93dc952e50ef86728a25 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Sun, 15 Feb 2015 22:44:24 +1100 Subject: [PATCH 121/544] Set subproxy count and subid under lock to prevent clashes and ensure the subproxy inherits the parent's id --- src/generator.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/generator.c b/src/generator.c index 8307ab39..1f44fbad 100644 --- a/src/generator.c +++ b/src/generator.c @@ -914,7 +914,6 @@ static bool recruit_subproxy(gdata_t *gdata, proxy_instance_t *proxi); static void add_subproxy(proxy_instance_t *proxi, proxy_instance_t *subproxy) { mutex_lock(&proxi->proxy_lock); - proxi->subproxy_count++; HASH_ADD(sh, proxi->subproxies, subid, sizeof(int), subproxy); proxi->client_headroom += proxi->clients_per_proxy; mutex_unlock(&proxi->proxy_lock); @@ -1618,7 +1617,12 @@ static proxy_instance_t *create_subproxy(gdata_t *gdata, proxy_instance_t *proxi subproxy->cs->ckp = subproxy->ckp = proxi->ckp; subproxy->si = proxi->si; - subproxy->subid = proxi->subproxy_count; + + mutex_lock(&proxi->proxy_lock); + subproxy->subid = ++proxi->subproxy_count; + mutex_unlock(&proxi->proxy_lock); + + subproxy->id = proxi->id; subproxy->auth = proxi->auth; subproxy->pass = proxi->pass; subproxy->parent = proxi; From a98d510f9fba380a946002dfdaf42077a967358b Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Sun, 15 Feb 2015 22:55:24 +1100 Subject: [PATCH 122/544] Add subproxy information to generator messages where appropriate --- src/generator.c | 65 ++++++++++++++++++++++++++----------------------- 1 file changed, 34 insertions(+), 31 deletions(-) diff --git a/src/generator.c b/src/generator.c index 1f44fbad..25e9a687 100644 --- a/src/generator.c +++ b/src/generator.c @@ -575,8 +575,8 @@ static bool parse_subscribe(connsock_t *cs, proxy_instance_t *proxi) retry: parsed = true; if (!(buf = new_proxy_line(cs))) { - LOGNOTICE("Proxy %d:%s failed to receive line in parse_subscribe", - proxi->id, proxi->si->url); + LOGNOTICE("Proxy %d:%d %s failed to receive line in parse_subscribe", + proxi->id, proxi->subid, proxi->si->url); goto out; } LOGDEBUG("parse_subscribe received %s", buf); @@ -608,8 +608,8 @@ retry: buf = NULL; goto retry; } - LOGNOTICE("Proxy %d:%s failed to parse subscribe response in parse_subscribe", - proxi->id, proxi->si->url); + LOGNOTICE("Proxy %d:%d %s failed to parse subscribe response in parse_subscribe", + proxi->id, proxi->subid, proxi->si->url); goto out; } @@ -662,7 +662,7 @@ retry: proxi->clients_per_proxy = 1ll << ((size - 3) * 8); proxi->client_headroom = proxi->clients_per_proxy; LOGNOTICE("Proxy %d:%s clients per proxy: %"PRId64, proxi->id, proxi->si->url, - proxi->clients_per_proxy); + proxi->clients_per_proxy); } LOGINFO("Found notify with enonce %s nonce2len %d", proxi->enonce1, @@ -704,8 +704,8 @@ retry: ret = send_json_msg(cs, req); json_decref(req); if (!ret) { - LOGNOTICE("Proxy %d:%s failed to send message in subscribe_stratum", - proxi->id, proxi->si->url); + LOGNOTICE("Proxy %d:%d %s failed to send message in subscribe_stratum", + proxi->id, proxi->subid, proxi->si->url); goto out; } ret = parse_subscribe(cs, proxi); @@ -713,24 +713,24 @@ retry: goto out; if (proxi->no_params) { - LOGNOTICE("Proxy %d:%s failed all subscription options in subscribe_stratum", - proxi->id, proxi->si->url); + LOGNOTICE("Proxy %d:%d %s failed all subscription options in subscribe_stratum", + proxi->id, proxi->subid, proxi->si->url); goto out; } if (proxi->sessionid) { - LOGINFO("Proxy %d:%s failed sessionid reconnect in subscribe_stratum, retrying without", - proxi->id, proxi->si->url); + LOGINFO("Proxy %d:%d %s failed sessionid reconnect in subscribe_stratum, retrying without", + proxi->id, proxi->subid, proxi->si->url); proxi->no_sessionid = true; dealloc(proxi->sessionid); } else { - LOGINFO("Proxy %d:%s failed connecting with parameters in subscribe_stratum, retrying without", - proxi->id, proxi->si->url); + LOGINFO("Proxy %d:%d %s failed connecting with parameters in subscribe_stratum, retrying without", + proxi->id, proxi->subid, proxi->si->url); proxi->no_params = true; } ret = connect_proxy(cs); if (!ret) { - LOGNOTICE("Proxy %d:%s failed to reconnect in subscribe_stratum", - proxi->id, proxi->si->url); + LOGNOTICE("Proxy %d:%d %s failed to reconnect in subscribe_stratum", + proxi->id, proxi->subid, proxi->si->url); goto out; } goto retry; @@ -1189,8 +1189,8 @@ static bool auth_stratum(ckpool_t *ckp, connsock_t *cs, proxy_instance_t *proxi) ret = send_json_msg(cs, req); json_decref(req); if (!ret) { - LOGNOTICE("Proxy %d:%s failed to send message in auth_stratum", - proxi->id, proxi->si->url); + LOGNOTICE("Proxy %d:%d %s failed to send message in auth_stratum", + proxi->id, proxi->subid, proxi->si->url); Close(cs->fd); goto out; } @@ -1201,8 +1201,8 @@ static bool auth_stratum(ckpool_t *ckp, connsock_t *cs, proxy_instance_t *proxi) free(buf); buf = next_proxy_line(cs, proxi); if (!buf) { - LOGNOTICE("Proxy %d:%s failed to receive line in auth_stratum", - proxi->id, proxi->si->url); + LOGNOTICE("Proxy %d:%d %s failed to receive line in auth_stratum", + proxi->id, proxi->subid, proxi->si->url); ret = false; goto out; } @@ -1211,28 +1211,28 @@ static bool auth_stratum(ckpool_t *ckp, connsock_t *cs, proxy_instance_t *proxi) val = json_msg_result(buf, &res_val, &err_val); if (!val) { - LOGWARNING("Proxy %d:%s failed to get a json result in auth_stratum, got: %s", - proxi->id, proxi->si->url, buf); + LOGWARNING("Proxy %d:%d %s failed to get a json result in auth_stratum, got: %s", + proxi->id, proxi->subid, proxi->si->url, buf); goto out; } if (err_val && !json_is_null(err_val)) { - LOGWARNING("Proxy %d:%s failed to authorise in auth_stratum due to err_val, got: %s", - proxi->id, proxi->si->url, buf); + LOGWARNING("Proxy %d:%d %s failed to authorise in auth_stratum due to err_val, got: %s", + proxi->id, proxi->subid, proxi->si->url, buf); goto out; } if (res_val) { ret = json_is_true(res_val); if (!ret) { - LOGWARNING("Proxy %d:%s failed to authorise in auth_stratum, got: %s", - proxi->id, proxi->si->url, buf); + LOGWARNING("Proxy %d:%d %s failed to authorise in auth_stratum, got: %s", + proxi->id, proxi->subid, proxi->si->url, buf); goto out; } } else { /* No result and no error but successful val means auth success */ ret = true; } - LOGINFO("Proxy %d:%s auth success in auth_stratum", proxi->id, proxi->si->url); + LOGINFO("Proxy %d:%d %s auth success in auth_stratum", proxi->id, proxi->subid, proxi->si->url); out: if (val) json_decref(val); @@ -1327,8 +1327,8 @@ static void submit_share(gdata_t *gdata, json_t *val) return json_decref(val); } if (!proxi->alive) { - LOGNOTICE("Client %"PRId64" attempting to send shares to dead proxy %d, dropping", - client_id, id); + LOGNOTICE("Client %"PRId64" attempting to send shares to dead proxy %d:%d, dropping", + client_id, id, subid); stratifier_drop_client(ckp, client_id); return json_decref(val); } @@ -1468,17 +1468,20 @@ static void *proxy_send(void *arg) "method", "mining.submit"); ret = send_json_msg(cs, val); json_decref(val); - } else { + } else if (!jobid) { LOGNOTICE("Proxy %d:%s failed to find matching jobid in proxysend", proxy->id, proxy->si->url); + } else { + LOGNOTICE("Failed to find subproxy %d:%d to send message to", + proxy->id, subid); } free(jobid); json_decref(msg->json_msg); free(msg); if (!ret && subproxy) { if (cs->fd > 0) { - LOGWARNING("Proxy %d:%s failed to send msg in proxy_send, dropping to reconnect", - proxy->id, proxy->si->url); + LOGWARNING("Proxy %d:%d %s failed to send msg in proxy_send, dropping to reconnect", + proxy->id, proxy->subid, proxy->si->url); Close(cs->fd); } if (!parent_proxy(subproxy) && !subproxy->disabled) From 14a7ec05e2ef44a1033217a1d635d21a89758021 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Mon, 16 Feb 2015 00:16:29 +1100 Subject: [PATCH 123/544] Use the subid when disabling a subproxy instead of the id --- src/generator.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/generator.c b/src/generator.c index 25e9a687..519551fd 100644 --- a/src/generator.c +++ b/src/generator.c @@ -943,7 +943,7 @@ static void disable_subproxy(gdata_t *gdata, proxy_instance_t *proxi, proxy_inst mutex_lock(&proxi->proxy_lock); subproxy->disabled = true; /* Make sure subproxy is still in the list */ - subproxy = __subproxy_by_id(proxi, subproxy->id); + subproxy = __subproxy_by_id(proxi, subproxy->subid); if (subproxy) { Close(subproxy->cs->fd); HASH_DEL(proxi->subproxies, subproxy); @@ -1279,12 +1279,12 @@ static void send_subscribe(ckpool_t *ckp, proxy_instance_t *proxi) free(buf); } -static proxy_instance_t *subproxy_by_id(proxy_instance_t *proxy, const int id) +static proxy_instance_t *subproxy_by_id(proxy_instance_t *proxy, const int subid) { proxy_instance_t *subproxy; mutex_lock(&proxy->proxy_lock); - subproxy = __subproxy_by_id(proxy, id); + subproxy = __subproxy_by_id(proxy, subid); if (subproxy && subproxy->disabled) subproxy = NULL; mutex_unlock(&proxy->proxy_lock); From 50ebbe808675a5920ed32eed8e7e60a1e49e8e71 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Mon, 16 Feb 2015 01:09:14 +1100 Subject: [PATCH 124/544] Make reconnect proxies inherit the subproxy count of the original for the stratifier to know they're unique --- src/generator.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/generator.c b/src/generator.c index 519551fd..46adf243 100644 --- a/src/generator.c +++ b/src/generator.c @@ -1032,6 +1032,7 @@ static bool parse_reconnect(proxy_instance_t *proxi, json_t *val) newproxi->cs = &newsi->cs; newproxi->cs->ckp = ckp; newproxi->id = newsi->id; + newproxi->subproxy_count = ++proxi->subproxy_count; HASH_REPLACE_INT(gdata->proxies, id, newproxi, proxi); mutex_unlock(&gdata->lock); From 4db64cd883c0755a62d9093883cd845f06f1f250 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Mon, 16 Feb 2015 01:16:37 +1100 Subject: [PATCH 125/544] Allow but warn about ultra small nonce2 size upstream pools --- src/generator.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/generator.c b/src/generator.c index 46adf243..cda1a686 100644 --- a/src/generator.c +++ b/src/generator.c @@ -652,9 +652,8 @@ retry: goto out; } if (size < 3) { - LOGWARNING("Proxy %d:%s Nonce2 length %d too small to be able to proxy", - proxi->id, proxi->si->url, size); - goto out; + LOGWARNING("Proxy %d:%d %s Nonce2 length %d too small for fast miners", + proxi->id, proxi->subid, proxi->si->url, size); } proxi->nonce2len = size; if (parent_proxy(proxi)) { From e2826e3c08f878e923a74be0663a9323dbc3c1ab Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Mon, 16 Feb 2015 20:59:34 +1100 Subject: [PATCH 126/544] Add minor comment about memory leak --- src/generator.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/generator.c b/src/generator.c index cda1a686..36fd8218 100644 --- a/src/generator.c +++ b/src/generator.c @@ -1035,6 +1035,7 @@ static bool parse_reconnect(proxy_instance_t *proxi, json_t *val) HASH_REPLACE_INT(gdata->proxies, id, newproxi, proxi); mutex_unlock(&gdata->lock); + /* Old proxy memory is basically lost here */ prepare_proxy(newproxi); out: return ret; From c295d6a8d4f2ec7b2d964a9862f848c885015b92 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Mon, 16 Feb 2015 21:46:44 +1100 Subject: [PATCH 127/544] Do a soft failover to backup proxies, not disconnecting them unless the upstream subproxy no longer exists, but a hard failover to higher priority proxies --- src/generator.c | 46 +++++++++++++++++++-------------------- src/stratifier.c | 56 ++++++++++++++++++++++++++++++++++-------------- 2 files changed, 62 insertions(+), 40 deletions(-) diff --git a/src/generator.c b/src/generator.c index 36fd8218..7612caa3 100644 --- a/src/generator.c +++ b/src/generator.c @@ -957,6 +957,22 @@ static void disable_subproxy(gdata_t *gdata, proxy_instance_t *proxi, proxy_inst recruit_subproxy(gdata, proxi); } +/* If the parent is no longer in use due to reconnect, we shouldn't use any of + * the child subproxies. */ +static void drop_subproxies(proxy_instance_t *proxi) +{ + proxy_instance_t *subproxy, *tmp; + + mutex_lock(&proxi->proxy_lock); + HASH_ITER(sh, proxi->subproxies, subproxy, tmp) { + if (!parent_proxy(subproxy)) { + subproxy->disabled = true; + Close(subproxy->cs->fd); + } + } + mutex_unlock(&proxi->proxy_lock); +} + static bool parse_reconnect(proxy_instance_t *proxi, json_t *val) { server_instance_t *newsi, *si = proxi->si; @@ -1037,6 +1053,7 @@ static bool parse_reconnect(proxy_instance_t *proxi, json_t *val) /* Old proxy memory is basically lost here */ prepare_proxy(newproxi); + drop_subproxies(proxi); out: return ret; } @@ -1293,11 +1310,11 @@ static proxy_instance_t *subproxy_by_id(proxy_instance_t *proxy, const int subid return subproxy; } -static void stratifier_drop_client(ckpool_t *ckp, int64_t id) +static void stratifier_reconnect_client(ckpool_t *ckp, int64_t id) { char buf[256]; - sprintf(buf, "dropclient=%"PRId64, id); + sprintf(buf, "reconnclient=%"PRId64, id); send_proc(ckp->stratifier, buf); } @@ -1317,20 +1334,20 @@ static void submit_share(gdata_t *gdata, json_t *val) proxy = proxy_by_id(gdata, id); if (unlikely(!proxy)) { LOGWARNING("Failed to find proxy %d to send share to", id); - stratifier_drop_client(ckp, client_id); + stratifier_reconnect_client(ckp, client_id); return json_decref(val); } json_get_int(&subid, val, "subproxy"); proxi = subproxy_by_id(proxy, subid); if (unlikely(!proxi)) { LOGNOTICE("Failed to find proxy %d:%d to send share to", id, subid); - stratifier_drop_client(ckp, client_id); + stratifier_reconnect_client(ckp, client_id); return json_decref(val); } if (!proxi->alive) { LOGNOTICE("Client %"PRId64" attempting to send shares to dead proxy %d:%d, dropping", client_id, id, subid); - stratifier_drop_client(ckp, client_id); + stratifier_reconnect_client(ckp, client_id); return json_decref(val); } @@ -1516,22 +1533,6 @@ static void passthrough_add_send(proxy_instance_t *proxi, const char *msg) ckmsgq_add(proxi->passsends, pm); } -/* If the parent is dead, we shouldn't use any of the child subproxies to be - * used. */ -static void drop_subproxies(proxy_instance_t *proxi) -{ - proxy_instance_t *subproxy, *tmp; - - mutex_lock(&proxi->proxy_lock); - HASH_ITER(sh, proxi->subproxies, subproxy, tmp) { - if (!parent_proxy(subproxy)) { - subproxy->disabled = true; - Close(subproxy->cs->fd); - } - } - mutex_unlock(&proxi->proxy_lock); -} - static bool proxy_alive(ckpool_t *ckp, server_instance_t *si, proxy_instance_t *proxi, connsock_t *cs, bool pinging, int epfd) { @@ -1595,8 +1596,6 @@ out: } } proxi->alive = ret; - if (!ret && parent_proxy(proxi)) - drop_subproxies(proxi); return ret; } @@ -1938,7 +1937,6 @@ retry: if (unlikely(proxi->cs->fd < 0)) { LOGWARNING("Upstream proxy %d:%s socket invalidated, will attempt failover", proxi->id, proxi->cs->url); - drop_subproxies(proxi); proxi->alive = false; proxi = NULL; goto reconnect; diff --git a/src/stratifier.c b/src/stratifier.c index f95d14f8..1efb0862 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -1111,7 +1111,9 @@ static proxy_t *subproxy_by_id(sdata_t *sdata, const int id, const int subid) } /* Iterates over all clients in proxy mode and sets the reconnect bool for the - * message to be sent lazily next time they speak to us */ + * message to be sent lazily next time they speak to us only if the proxy is + * higher priority than the one they're currently connected to or the notify_id + * on their proxy has changed indicating a new subscription. */ static void reconnect_clients(sdata_t *sdata, const int proxyid, const int64_t notify_id) { stratum_instance_t *client, *tmp; @@ -1120,7 +1122,7 @@ static void reconnect_clients(sdata_t *sdata, const int proxyid, const int64_t n ck_rlock(&sdata->instance_lock); HASH_ITER(hh, sdata->stratum_instances, client, tmp) { - if (client->proxyid != proxyid || client->notify_id != notify_id) + if (client->proxyid > proxyid || (client->proxyid == proxyid && client->notify_id != notify_id)) client->reconnect = true; } ck_runlock(&sdata->instance_lock); @@ -1953,6 +1955,34 @@ static void set_proxy(sdata_t *sdata, const char *buf) reconnect_clients(sdata, proxy->id, proxy->notify_id); } +/* Send a single client a reconnect request, setting the time we sent the + * request so we can drop the client lazily if it hasn't reconnected on its + * own one minute later */ +static void reconnect_client(sdata_t *sdata, stratum_instance_t *client) +{ + json_t *json_msg; + + client->reconnect = false; + client->reconnect_request = time(NULL); + JSON_CPACK(json_msg, "{sosss[]}", "id", json_null(), "method", "client.reconnect", + "params"); + stratum_add_send(sdata, json_msg, client->id); +} + +static void reconnect_client_id(sdata_t *sdata, const int64_t client_id) +{ + stratum_instance_t *client; + + client = ref_instance_by_id(sdata, client_id); + if (!client) { + LOGINFO("reconnect_client_id failed to find client %"PRId64, client_id); + return; + } + LOGNOTICE("Reconnecting client %"PRId64, client_id); + reconnect_client(sdata, client); + dec_instance_ref(sdata, client); +} + static int stratum_loop(ckpool_t *ckp, proc_instance_t *pi) { int sockd, ret = 0, selret = 0; @@ -2051,6 +2081,14 @@ retry: LOGDEBUG("Stratifier failed to parse dropclient command: %s", buf); else drop_client(ckp, sdata, client_id); + } else if (cmdmatch(buf, "reconnclient")) { + int64_t client_id; + + ret = sscanf(buf, "reconnclient=%"PRId64, &client_id); + if (ret < 0) + LOGDEBUG("Stratifier failed to parse reconnclient command: %s", buf); + else + reconnect_client_id(sdata, client_id); } else if (cmdmatch(buf, "dropall")) { drop_allclients(ckp); } else if (cmdmatch(buf, "block")) { @@ -3739,20 +3777,6 @@ out: free_smsg(msg); } -/* Send a single client a reconnect request, setting the time we sent the - * request so we can drop the client lazily if it hasn't reconnected on its - * own one minute later */ -static void reconnect_client(sdata_t *sdata, stratum_instance_t *client) -{ - json_t *json_msg; - - client->reconnect = false; - client->reconnect_request = time(NULL); - JSON_CPACK(json_msg, "{sosss[]}", "id", json_null(), "method", "client.reconnect", - "params"); - stratum_add_send(sdata, json_msg, client->id); -} - static void srecv_process(ckpool_t *ckp, char *buf) { bool noid = false, dropped = false; From 40ebe7d1b0b71a705f8b23ba9c379eabdd38ca6c Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Mon, 16 Feb 2015 23:02:25 +1100 Subject: [PATCH 128/544] Delete subproxies from the correct hashlist --- src/generator.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/generator.c b/src/generator.c index 7612caa3..e32b0423 100644 --- a/src/generator.c +++ b/src/generator.c @@ -945,7 +945,7 @@ static void disable_subproxy(gdata_t *gdata, proxy_instance_t *proxi, proxy_inst subproxy = __subproxy_by_id(proxi, subproxy->subid); if (subproxy) { Close(subproxy->cs->fd); - HASH_DEL(proxi->subproxies, subproxy); + HASH_DELETE(sh, proxi->subproxies, subproxy); proxi->client_headroom -= proxi->clients_per_proxy; } mutex_unlock(&proxi->proxy_lock); From a1eabf7305bb78de5f5416e2ad6f955b8d798159 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Tue, 17 Feb 2015 00:09:28 +1100 Subject: [PATCH 129/544] Send a dead proxy notification to the stratifier when we disable a subproxy to switch clients on it --- src/generator.c | 7 ++++++- src/stratifier.c | 40 ++++++++++++++++++++++++++++++---------- 2 files changed, 36 insertions(+), 11 deletions(-) diff --git a/src/generator.c b/src/generator.c index e32b0423..427d41b5 100644 --- a/src/generator.c +++ b/src/generator.c @@ -950,8 +950,13 @@ static void disable_subproxy(gdata_t *gdata, proxy_instance_t *proxi, proxy_inst } mutex_unlock(&proxi->proxy_lock); - if (subproxy) + if (subproxy) { + char buf[256]; + + sprintf(buf, "deadproxy=%d:%d", subproxy->id, subproxy->subid); + send_proc(gdata->ckp->stratifier, buf); store_proxy(gdata, subproxy); + } if (proxi->client_headroom < 42 && proxi->alive) recruit_subproxy(gdata, proxi); diff --git a/src/stratifier.c b/src/stratifier.c index 1efb0862..52abe3f4 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -1935,6 +1935,20 @@ static char *stratifier_stats(ckpool_t *ckp, sdata_t *sdata) return buf; } +/* Send a single client a reconnect request, setting the time we sent the + * request so we can drop the client lazily if it hasn't reconnected on its + * own one minute later */ +static void reconnect_client(sdata_t *sdata, stratum_instance_t *client) +{ + json_t *json_msg; + + client->reconnect = false; + client->reconnect_request = time(NULL); + JSON_CPACK(json_msg, "{sosss[]}", "id", json_null(), "method", "client.reconnect", + "params"); + stratum_add_send(sdata, json_msg, client->id); +} + /* Sets the currently active proxy. Clients will be told to reconnect once the * first notify data comes from this proxy. Even if we are already bound to * this proxy we are only given this message if all clients must move. */ @@ -1955,18 +1969,22 @@ static void set_proxy(sdata_t *sdata, const char *buf) reconnect_clients(sdata, proxy->id, proxy->notify_id); } -/* Send a single client a reconnect request, setting the time we sent the - * request so we can drop the client lazily if it hasn't reconnected on its - * own one minute later */ -static void reconnect_client(sdata_t *sdata, stratum_instance_t *client) +static void dead_proxy(sdata_t *sdata, const char *buf) { - json_t *json_msg; + stratum_instance_t *client, *tmp; + int id = 0, subid = 0; + proxy_t *proxy; - client->reconnect = false; - client->reconnect_request = time(NULL); - JSON_CPACK(json_msg, "{sosss[]}", "id", json_null(), "method", "client.reconnect", - "params"); - stratum_add_send(sdata, json_msg, client->id); + sscanf(buf, "deadproxy=%d:%d", &id, &subid); + proxy = subproxy_by_id(sdata, id, subid); + LOGNOTICE("Stratifier dropping clients from proxy %d:%d", id, subid); + + ck_rlock(&sdata->instance_lock); + HASH_ITER(hh, sdata->stratum_instances, client, tmp) { + if (client->proxyid == id && client->subproxyid == subid) + reconnect_client(sdata, client); + } + ck_runlock(&sdata->instance_lock); } static void reconnect_client_id(sdata_t *sdata, const int64_t client_id) @@ -2099,6 +2117,8 @@ retry: request_reconnect(sdata, buf); } else if (cmdmatch(buf, "proxy")) { set_proxy(sdata, buf); + } else if (cmdmatch(buf, "deadproxy")) { + dead_proxy(sdata, buf); } else if (cmdmatch(buf, "loglevel")) { sscanf(buf, "loglevel=%d", &ckp->loglevel); } else From cb1c58846bb350ccbbf27267262d35fa9fe2b4af Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Tue, 17 Feb 2015 00:18:28 +1100 Subject: [PATCH 130/544] Demote update notices for subproxies --- src/stratifier.c | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/src/stratifier.c b/src/stratifier.c index 52abe3f4..9697e190 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -1161,7 +1161,10 @@ static void update_subscribe(ckpool_t *ckp, const char *cmd) json_get_int(&id, val, "proxy"); json_get_int(&subid, val, "subproxy"); - LOGNOTICE("Got updated subscribe for proxy %d:%d", id, subid); + if (!subid) + LOGNOTICE("Got updated subscribe for proxy %d", id); + else + LOGINFO("Got updated subscribe for proxy %d:%d", id, subid); proxy = subproxy_by_id(sdata, id, subid); proxy->notify_id = -1; /* Reset this */ @@ -1233,7 +1236,10 @@ static void update_notify(ckpool_t *ckp, const char *cmd) LOGNOTICE("No valid proxy %d:%d subscription to update notify yet", id, subid); goto out; } - LOGNOTICE("Got updated notify for proxy %d:%d", id, subid); + if (!subid) + LOGNOTICE("Got updated notify for proxy %d", id); + else + LOGINFO("Got updated notify for proxy %d:%d", id, subid); wb = ckzalloc(sizeof(workbase_t)); wb->ckp = ckp; @@ -1329,7 +1335,10 @@ static void update_diff(ckpool_t *ckp, const char *cmd) json_dblcpy(&diff, val, "diff"); json_decref(val); - LOGNOTICE("Got updated diff for proxy %d:%d", id, subid); + if (!subid) + LOGNOTICE("Got updated diff for proxy %d", id); + else + LOGINFO("Got updated diff for proxy %d:%d", id, subid); proxy = subproxy_by_id(sdata, id, subid); /* We only really care about integer diffs so clamp the lower limit to From 328f1ef5db92abd703c622790537d4d4d113abdb Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Tue, 17 Feb 2015 10:11:54 +1100 Subject: [PATCH 131/544] Demote select failed warning to notice level --- src/libckpool.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libckpool.c b/src/libckpool.c index bed7a816..9d730e26 100644 --- a/src/libckpool.c +++ b/src/libckpool.c @@ -682,12 +682,12 @@ int write_socket(int fd, const void *buf, size_t nbyte) if (!ret) LOGNOTICE("Select timed out in write_socket"); else - LOGERR("Select failed in write_socket"); + LOGNOTICE("Select failed in write_socket"); goto out; } ret = write_length(fd, buf, nbyte); if (ret < 0) - LOGWARNING("Failed to write in write_socket"); + LOGNOTICE("Failed to write in write_socket"); out: return ret; } From 2562068163e1a3503336ecde8f6de3b32b2e28b3 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Tue, 17 Feb 2015 11:00:37 +1100 Subject: [PATCH 132/544] Remove any concept of clients per proxy from the generator and simply display how many on connecting the parent' --- src/generator.c | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/src/generator.c b/src/generator.c index 427d41b5..00f1f6f5 100644 --- a/src/generator.c +++ b/src/generator.c @@ -132,8 +132,6 @@ struct proxy_instance { int epfd; /* Epoll fd used by the parent proxy */ pthread_mutex_t proxy_lock; /* Lock protecting hashlist of proxies */ - int64_t clients_per_proxy; /* How many clients can connect to each subproxy */ - int64_t client_headroom; /* How many more clients can we connect */ proxy_instance_t *parent; /* Parent proxy of subproxies */ proxy_instance_t *subproxies; /* Hashlist of subproxies of this proxy */ int subproxy_count; /* Number of subproxies */ @@ -658,10 +656,10 @@ retry: proxi->nonce2len = size; if (parent_proxy(proxi)) { /* Set the number of clients per proxy on the parent proxy */ - proxi->clients_per_proxy = 1ll << ((size - 3) * 8); - proxi->client_headroom = proxi->clients_per_proxy; + int64_t clients_per_proxy = 1ll << ((size - 3) * 8); + LOGNOTICE("Proxy %d:%s clients per proxy: %"PRId64, proxi->id, proxi->si->url, - proxi->clients_per_proxy); + clients_per_proxy); } LOGINFO("Found notify with enonce %s nonce2len %d", proxi->enonce1, @@ -914,7 +912,6 @@ static void add_subproxy(proxy_instance_t *proxi, proxy_instance_t *subproxy) { mutex_lock(&proxi->proxy_lock); HASH_ADD(sh, proxi->subproxies, subid, sizeof(int), subproxy); - proxi->client_headroom += proxi->clients_per_proxy; mutex_unlock(&proxi->proxy_lock); } @@ -946,7 +943,6 @@ static void disable_subproxy(gdata_t *gdata, proxy_instance_t *proxi, proxy_inst if (subproxy) { Close(subproxy->cs->fd); HASH_DELETE(sh, proxi->subproxies, subproxy); - proxi->client_headroom -= proxi->clients_per_proxy; } mutex_unlock(&proxi->proxy_lock); @@ -957,9 +953,6 @@ static void disable_subproxy(gdata_t *gdata, proxy_instance_t *proxi, proxy_inst send_proc(gdata->ckp->stratifier, buf); store_proxy(gdata, subproxy); } - - if (proxi->client_headroom < 42 && proxi->alive) - recruit_subproxy(gdata, proxi); } /* If the parent is no longer in use due to reconnect, we shouldn't use any of From f68e398958dd60108e8a9850b04011735d02945a Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Tue, 17 Feb 2015 12:26:27 +1100 Subject: [PATCH 133/544] Add more debug info for proxies --- src/generator.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/generator.c b/src/generator.c index 00f1f6f5..bf850d8e 100644 --- a/src/generator.c +++ b/src/generator.c @@ -814,7 +814,7 @@ static bool parse_notify(ckpool_t *ckp, proxy_instance_t *proxi, json_t *val) goto out; } - LOGDEBUG("New notify"); + LOGDEBUG("Received new notify from proxy %d:%d", proxi->id, proxi->subid); ni = ckzalloc(sizeof(notify_instance_t)); ni->jobid = job_id; LOGDEBUG("Job ID %s", job_id); @@ -1155,6 +1155,7 @@ static bool parse_method(ckpool_t *ckp, proxy_instance_t *proxi, const char *msg goto out; } + LOGDEBUG("Proxy %d:%d received method %s", proxi->id, proxi->subid, buf); if (cmdmatch(buf, "mining.notify")) { ret = parse_notify(ckp, proxi, params); goto out; From 3ba0aeee69def1cc2b18b0dbc819c3363164ade9 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Tue, 17 Feb 2015 13:16:56 +1100 Subject: [PATCH 134/544] Disable and remove the parent proxy from the subproxy list on reconnect as well --- src/generator.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/generator.c b/src/generator.c index bf850d8e..9eb577ea 100644 --- a/src/generator.c +++ b/src/generator.c @@ -951,7 +951,8 @@ static void disable_subproxy(gdata_t *gdata, proxy_instance_t *proxi, proxy_inst sprintf(buf, "deadproxy=%d:%d", subproxy->id, subproxy->subid); send_proc(gdata->ckp->stratifier, buf); - store_proxy(gdata, subproxy); + if (!parent_proxy(subproxy)) + store_proxy(gdata, subproxy); } } @@ -1815,15 +1816,14 @@ static void *proxy_recv(void *arg) /* Call this proxy dead to allow us to fail * over to a backup pool until the reconnect * pool is up */ - Close(cs->fd); subproxy->alive = false; + disable_subproxy(gdata, proxi, subproxy); if (parent_proxy(subproxy)) { send_proc(ckp->generator, "reconnect"); LOGWARNING("Proxy %d:%s reconnect issue, dropping existing connection", subproxy->id, subproxy->si->url); break; - } else - disable_subproxy(gdata, proxi, subproxy); + } } continue; } From c33362c8ad4b2e7fa87cb060468f880fbe7a1951 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Tue, 17 Feb 2015 13:27:47 +1100 Subject: [PATCH 135/544] Make subproxy recruiting asynchronous to not delay other proxy loop functions --- src/generator.c | 30 +++++++++++++++++++----------- 1 file changed, 19 insertions(+), 11 deletions(-) diff --git a/src/generator.c b/src/generator.c index 9eb577ea..6cb4ab2d 100644 --- a/src/generator.c +++ b/src/generator.c @@ -906,7 +906,6 @@ static bool send_pong(proxy_instance_t *proxi, json_t *val) static void prepare_proxy(proxy_instance_t *proxi); static proxy_instance_t *create_subproxy(gdata_t *gdata, proxy_instance_t *proxi); -static bool recruit_subproxy(gdata_t *gdata, proxy_instance_t *proxi); static void add_subproxy(proxy_instance_t *proxi, proxy_instance_t *subproxy) { @@ -1633,20 +1632,28 @@ static proxy_instance_t *create_subproxy(gdata_t *gdata, proxy_instance_t *proxi return subproxy; } -static bool recruit_subproxy(gdata_t *gdata, proxy_instance_t *proxi) +static void *proxy_recruit(void *arg) { - proxy_instance_t *subproxy = create_subproxy(gdata, proxi); - int epfd = proxi->epfd; + proxy_instance_t *proxy = (proxy_instance_t *)arg, *parent = proxy->parent; + ckpool_t *ckp = proxy->ckp; + gdata_t *gdata = ckp->data; + + pthread_detach(pthread_self()); - if (!proxy_alive(subproxy->ckp, subproxy->si, subproxy, subproxy->cs, false, epfd)) { + if (!proxy_alive(ckp, proxy->si, proxy, proxy->cs, false, parent->epfd)) { LOGNOTICE("Subproxy failed proxy_alive testing"); - store_proxy(gdata, subproxy); - return false; - } + store_proxy(gdata, proxy); + } else + add_subproxy(parent, proxy); + return NULL; +} - add_subproxy(proxi, subproxy); +static void recruit_subproxy(gdata_t *gdata, proxy_instance_t *proxi) +{ + proxy_instance_t *subproxy = create_subproxy(gdata, proxi); + pthread_t pth; - return true; + create_pthread(&pth, proxy_recruit, subproxy); } /* For receiving messages from an upstream pool to pass downstream. Responsible @@ -1823,7 +1830,8 @@ static void *proxy_recv(void *arg) LOGWARNING("Proxy %d:%s reconnect issue, dropping existing connection", subproxy->id, subproxy->si->url); break; - } + } else + recruit_subproxy(gdata, proxi); } continue; } From eed7df18262f5e8894ad0042b2b4c9599e84a4d6 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Tue, 17 Feb 2015 13:30:36 +1100 Subject: [PATCH 136/544] Do all the proxy recruitment within the new thread --- src/generator.c | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/generator.c b/src/generator.c index 6cb4ab2d..12925be0 100644 --- a/src/generator.c +++ b/src/generator.c @@ -1634,12 +1634,13 @@ static proxy_instance_t *create_subproxy(gdata_t *gdata, proxy_instance_t *proxi static void *proxy_recruit(void *arg) { - proxy_instance_t *proxy = (proxy_instance_t *)arg, *parent = proxy->parent; - ckpool_t *ckp = proxy->ckp; + proxy_instance_t *proxy, *parent = (proxy_instance_t *)arg; + ckpool_t *ckp = parent->ckp; gdata_t *gdata = ckp->data; pthread_detach(pthread_self()); + proxy = create_subproxy(gdata, parent); if (!proxy_alive(ckp, proxy->si, proxy, proxy->cs, false, parent->epfd)) { LOGNOTICE("Subproxy failed proxy_alive testing"); store_proxy(gdata, proxy); @@ -1648,12 +1649,11 @@ static void *proxy_recruit(void *arg) return NULL; } -static void recruit_subproxy(gdata_t *gdata, proxy_instance_t *proxi) +static void recruit_subproxy(proxy_instance_t *proxi) { - proxy_instance_t *subproxy = create_subproxy(gdata, proxi); pthread_t pth; - create_pthread(&pth, proxy_recruit, subproxy); + create_pthread(&pth, proxy_recruit, proxi); } /* For receiving messages from an upstream pool to pass downstream. Responsible @@ -1831,7 +1831,7 @@ static void *proxy_recv(void *arg) subproxy->id, subproxy->si->url); break; } else - recruit_subproxy(gdata, proxi); + recruit_subproxy(proxi); } continue; } @@ -1976,7 +1976,7 @@ retry: LOGDEBUG("Proxy received ping request"); send_unix_msg(sockd, "pong"); } else if (cmdmatch(buf, "recruit")) { - recruit_subproxy(gdata, proxi); + recruit_subproxy(proxi); } else if (ckp->passthrough) { /* Anything remaining should be stratum messages */ passthrough_add_send(proxi, buf); From b96106a41429fa35551d758f11667ae2140b4b32 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Tue, 17 Feb 2015 13:36:55 +1100 Subject: [PATCH 137/544] Inform stratifier of deadproxy on every failed proxy_alive call --- src/generator.c | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/src/generator.c b/src/generator.c index 12925be0..45c609fd 100644 --- a/src/generator.c +++ b/src/generator.c @@ -932,6 +932,14 @@ static void store_proxy(gdata_t *gdata, proxy_instance_t *proxy) mutex_unlock(&gdata->lock); } +static void send_stratifier_deadproxy(ckpool_t *ckp, const int id, const int subid) +{ + char buf[256]; + + sprintf(buf, "deadproxy=%d:%d", id, subid); + send_proc(ckp->stratifier, buf); +} + /* Remove the subproxy from the proxi list and put it on the dead list */ static void disable_subproxy(gdata_t *gdata, proxy_instance_t *proxi, proxy_instance_t *subproxy) { @@ -946,10 +954,7 @@ static void disable_subproxy(gdata_t *gdata, proxy_instance_t *proxi, proxy_inst mutex_unlock(&proxi->proxy_lock); if (subproxy) { - char buf[256]; - - sprintf(buf, "deadproxy=%d:%d", subproxy->id, subproxy->subid); - send_proc(gdata->ckp->stratifier, buf); + send_stratifier_deadproxy(gdata->ckp, subproxy->id, subproxy->subid); if (!parent_proxy(subproxy)) store_proxy(gdata, subproxy); } @@ -1581,6 +1586,7 @@ static bool proxy_alive(ckpool_t *ckp, server_instance_t *si, proxy_instance_t * ret = true; out: if (!ret) { + send_stratifier_deadproxy(ckp, proxi->id, proxi->subid); /* Close and invalidate the file handle */ Close(cs->fd); } else { From 79cca5a6d56ff126e1d4e764a3bf5b36e1c13a1d Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Tue, 17 Feb 2015 14:37:11 +1100 Subject: [PATCH 138/544] Remove replaced proxies from the main hashlist and store them in an old proxy hashlist upon a new subscription --- src/stratifier.c | 25 +++++++++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/src/stratifier.c b/src/stratifier.c index 9697e190..8509c6f4 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -396,6 +396,7 @@ struct stratifier_data { proxy_t *proxy; /* Current proxy in use */ proxy_t *proxies; /* Hashlist of all proxies */ + proxy_t *old_proxies; /* Hashlist of proxies now no longer in user */ pthread_mutex_t proxy_lock; /* Protects all proxy data */ proxy_t *subproxy; /* Which subproxy this sdata belongs to in proxy mode */ }; @@ -1139,6 +1140,25 @@ static proxy_t *current_proxy(sdata_t *sdata) return proxy; } +static void new_proxy(sdata_t *sdata, const int id) +{ + bool exists = false; + proxy_t *proxy; + + mutex_lock(&sdata->proxy_lock); + HASH_FIND_INT(sdata->proxies, &id, proxy); + if (proxy) { + exists = true; + HASH_DEL(sdata->proxies, proxy); + HASH_ADD_INT(sdata->old_proxies, id, proxy); + } + proxy = __generate_proxy(sdata, id); + mutex_unlock(&sdata->proxy_lock); + + if (exists) + LOGNOTICE("Stratifier replaced old proxy instance %d", id); +} + static void update_subscribe(ckpool_t *ckp, const char *cmd) { sdata_t *sdata = ckp->data, *dsdata; @@ -1161,9 +1181,10 @@ static void update_subscribe(ckpool_t *ckp, const char *cmd) json_get_int(&id, val, "proxy"); json_get_int(&subid, val, "subproxy"); - if (!subid) + if (!subid) { + new_proxy(sdata, id); LOGNOTICE("Got updated subscribe for proxy %d", id); - else + } else LOGINFO("Got updated subscribe for proxy %d:%d", id, subid); proxy = subproxy_by_id(sdata, id, subid); From 4fe614ea7d2fd37d4ccea5fbfc22f1ee2801ee26 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Tue, 17 Feb 2015 14:40:16 +1100 Subject: [PATCH 139/544] Don't try to allocate clients to dead subproxies in the stratifier --- src/stratifier.c | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/stratifier.c b/src/stratifier.c index 8509c6f4..0eecad81 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -326,6 +326,7 @@ struct proxy_base { proxy_t *parent; /* Parent proxy - set to self on parent itself */ proxy_t *subproxies; /* Hashlist of subproxies sorted by subid */ sdata_t *sdata; /* Unique stratifer data for each subproxy */ + bool dead; }; struct stratifier_data { @@ -2007,6 +2008,7 @@ static void dead_proxy(sdata_t *sdata, const char *buf) sscanf(buf, "deadproxy=%d:%d", &id, &subid); proxy = subproxy_by_id(sdata, id, subid); + proxy->dead = true; LOGNOTICE("Stratifier dropping clients from proxy %d:%d", id, subid); ck_rlock(&sdata->instance_lock); @@ -2306,7 +2308,11 @@ static sdata_t *select_sdata(const ckpool_t *ckp, sdata_t *ckp_sdata) } mutex_lock(&ckp_sdata->proxy_lock); HASH_ITER(sh, proxy->subproxies, subproxy, tmp) { - int64_t subproxy_headroom = subproxy->max_clients - subproxy->clients; + int64_t subproxy_headroom; + + if (subproxy->dead) + continue; + subproxy_headroom = subproxy->max_clients - subproxy->clients; headroom += subproxy_headroom; if (subproxy_headroom > most_headroom) { From 2a809faeb4b0dba6255190569a436d32a3f37b00 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Tue, 17 Feb 2015 15:26:48 +1100 Subject: [PATCH 140/544] Change block hash update message according to proxy/pool --- src/stratifier.c | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/src/stratifier.c b/src/stratifier.c index 0eecad81..9731d572 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -343,6 +343,7 @@ struct stratifier_data { pthread_mutex_t ckdb_lock; bool ckdb_offline; + bool verbose; enonce1_t enonce1u; @@ -773,10 +774,8 @@ static void add_base(ckpool_t *ckp, sdata_t *sdata, workbase_t *wb, bool *new_bl sdata->current_workbase = wb; ck_wunlock(&sdata->workbase_lock); - if (*new_block) { - LOGNOTICE("Block hash changed to %s", sdata->lastswaphash); + if (*new_block) purge_share_hashtable(sdata, wb->id); - } send_workinfo(ckp, wb); @@ -915,6 +914,8 @@ static void *do_update(void *arg) generate_coinbase(ckp, wb); add_base(ckp, sdata, wb, &new_block); + if (new_block) + LOGNOTICE("Block hash changed to %s", sdata->lastswaphash); stratum_broadcast_update(sdata, new_block); ret = true; @@ -1041,6 +1042,7 @@ static proxy_t *__generate_proxy(sdata_t *sdata, const int id) proxy->id = id; proxy->sdata = duplicate_sdata(sdata); proxy->sdata->subproxy = proxy; + proxy->sdata->verbose = true; proxy->parent = proxy; /* subid == 0 on parent proxy */ HASH_ADD(sh, proxy->subproxies, subid, sizeof(int), proxy); @@ -1313,6 +1315,12 @@ static void update_notify(ckpool_t *ckp, const char *cmd) ck_runlock(&dsdata->workbase_lock); add_base(ckp, dsdata, wb, &new_block); + if (new_block) { + if (subid) + LOGINFO("Block hash on proxy %d:%d changed to %s", id, subid, dsdata->lastswaphash); + else + LOGNOTICE("Block hash on proxy %d changed to %s", id, dsdata->lastswaphash); + } if (proxy->notify_id == -1) { /* This is the first notification from the current proxy, tell @@ -4749,6 +4757,7 @@ int stratifier(proc_instance_t *pi) sdata = ckzalloc(sizeof(sdata_t)); ckp->data = sdata; sdata->ckp = ckp; + sdata->verbose = true; /* Wait for the generator to have something for us */ do { From fc26ec2ddc50087ced63567295b312356bcd9dc6 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Tue, 17 Feb 2015 15:42:02 +1100 Subject: [PATCH 141/544] Differentiate proxy_by_id for when we only want existing entries --- src/stratifier.c | 34 +++++++++++++++++++++++++++++++--- 1 file changed, 31 insertions(+), 3 deletions(-) diff --git a/src/stratifier.c b/src/stratifier.c index 9731d572..8aa36fb8 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -1063,12 +1063,19 @@ static proxy_t *__generate_subproxy(sdata_t *sdata, proxy_t *proxy, const int su return subproxy; } -/* Find proxy by id number, generate one if none exist yet by that id */ -static proxy_t *__proxy_by_id(sdata_t *sdata, const int id) +static proxy_t *__existing_proxy(const sdata_t *sdata, const int id) { proxy_t *proxy; HASH_FIND_INT(sdata->proxies, &id, proxy); + return proxy; +} + +/* Find proxy by id number, generate one if none exist yet by that id */ +static proxy_t *__proxy_by_id(sdata_t *sdata, const int id) +{ + proxy_t *proxy = __existing_proxy(sdata, id); + if (unlikely(!proxy)) { proxy = __generate_proxy(sdata, id); LOGNOTICE("Stratifier added new proxy %d", id); @@ -1077,11 +1084,18 @@ static proxy_t *__proxy_by_id(sdata_t *sdata, const int id) return proxy; } -static proxy_t *__subproxy_by_id(sdata_t *sdata, proxy_t *proxy, const int subid) +static proxy_t *__existing_subproxy(proxy_t *proxy, const int subid) { proxy_t *subproxy; HASH_FIND(sh, proxy->subproxies, &subid, sizeof(int), subproxy); + return subproxy; +} + +static proxy_t *__subproxy_by_id(sdata_t *sdata, proxy_t *proxy, const int subid) +{ + proxy_t *subproxy = __existing_subproxy(proxy, subid); + if (!subproxy) { subproxy = __generate_subproxy(sdata, proxy, subid); LOGINFO("Stratifier added new subproxy %d:%d", proxy->id, subid); @@ -1114,6 +1128,20 @@ static proxy_t *subproxy_by_id(sdata_t *sdata, const int id, const int subid) return subproxy; } +static proxy_t *existing_subproxy(sdata_t *sdata, const int id, const int subid) +{ + proxy_t *proxy, *subproxy = NULL; + + mutex_lock(&sdata->proxy_lock); + proxy = __existing_proxy(sdata, id); + if (proxy) + subproxy = __existing_subproxy(proxy, subid); + mutex_unlock(&sdata->proxy_lock); + + return subproxy; +} + + /* Iterates over all clients in proxy mode and sets the reconnect bool for the * message to be sent lazily next time they speak to us only if the proxy is * higher priority than the one they're currently connected to or the notify_id From d1fe21601e47bfc9687f08202fb9ce44d4d2d24f Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Tue, 17 Feb 2015 15:45:57 +1100 Subject: [PATCH 142/544] Don't create subproxies except when appropriate --- src/stratifier.c | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/src/stratifier.c b/src/stratifier.c index 8aa36fb8..7e8b4aa6 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -1283,9 +1283,9 @@ static void update_notify(ckpool_t *ckp, const char *cmd) } json_get_int(&id, val, "proxy"); json_get_int(&subid, val, "subproxy"); - proxy = subproxy_by_id(sdata, id, subid); - if (unlikely(!proxy->subscribed)) { - LOGNOTICE("No valid proxy %d:%d subscription to update notify yet", id, subid); + proxy = existing_subproxy(sdata, id, subid); + if (unlikely(!proxy || !proxy->subscribed)) { + LOGINFO("No valid proxy %d:%d subscription to update notify yet", id, subid); goto out; } if (!subid) @@ -1397,7 +1397,11 @@ static void update_diff(ckpool_t *ckp, const char *cmd) LOGNOTICE("Got updated diff for proxy %d", id); else LOGINFO("Got updated diff for proxy %d:%d", id, subid); - proxy = subproxy_by_id(sdata, id, subid); + proxy = existing_subproxy(sdata, id, subid); + if (!proxy) { + LOGINFO("No existing subproxy %d:%d to update diff", id, subid); + return; + } /* We only really care about integer diffs so clamp the lower limit to * 1 or it will round down to zero. */ @@ -2043,7 +2047,9 @@ static void dead_proxy(sdata_t *sdata, const char *buf) proxy_t *proxy; sscanf(buf, "deadproxy=%d:%d", &id, &subid); - proxy = subproxy_by_id(sdata, id, subid); + proxy = existing_subproxy(sdata, id, subid); + if (!proxy) + return; proxy->dead = true; LOGNOTICE("Stratifier dropping clients from proxy %d:%d", id, subid); From 9dad2175624d5648e8f214d34040c259e3f9caec Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Tue, 17 Feb 2015 15:48:21 +1100 Subject: [PATCH 143/544] Switch the current proxy if we generate a replacement one --- src/stratifier.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/stratifier.c b/src/stratifier.c index 7e8b4aa6..d8c28882 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -1173,7 +1173,7 @@ static proxy_t *current_proxy(sdata_t *sdata) static void new_proxy(sdata_t *sdata, const int id) { - bool exists = false; + bool exists = false, current = false; proxy_t *proxy; mutex_lock(&sdata->proxy_lock); @@ -1182,8 +1182,12 @@ static void new_proxy(sdata_t *sdata, const int id) exists = true; HASH_DEL(sdata->proxies, proxy); HASH_ADD_INT(sdata->old_proxies, id, proxy); + if (proxy == sdata->proxy) + current = true; } proxy = __generate_proxy(sdata, id); + if (current) + sdata->proxy = proxy; mutex_unlock(&sdata->proxy_lock); if (exists) From eab6c1ba6439fdc8b5b10d51b81409660998d422 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Tue, 17 Feb 2015 15:52:14 +1100 Subject: [PATCH 144/544] Add more info to missing jobid notice --- src/generator.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/generator.c b/src/generator.c index 45c609fd..a3f48388 100644 --- a/src/generator.c +++ b/src/generator.c @@ -1491,8 +1491,8 @@ static void *proxy_send(void *arg) ret = send_json_msg(cs, val); json_decref(val); } else if (!jobid) { - LOGNOTICE("Proxy %d:%s failed to find matching jobid in proxysend", - proxy->id, proxy->si->url); + LOGNOTICE("Proxy %d:%s failed to find matching jobid for %sknown subproxy in proxysend", + proxy->id, proxy->si->url, subproxy ? "" : "un"); } else { LOGNOTICE("Failed to find subproxy %d:%d to send message to", proxy->id, subid); From dba4f02fae49ac56d591b0d372aba79de93e8663 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Tue, 17 Feb 2015 16:59:30 +1100 Subject: [PATCH 145/544] Find any subproxy with headroom to take more clients if we do not have enough upstream connections yet to service new clients --- src/stratifier.c | 61 ++++++++++++++++++++++++------------------------ 1 file changed, 31 insertions(+), 30 deletions(-) diff --git a/src/stratifier.c b/src/stratifier.c index d8c28882..37b68f4b 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -321,6 +321,7 @@ struct proxy_base { int64_t clients; int64_t max_clients; + int64_t headroom; enonce1_t enonce1u; proxy_t *parent; /* Parent proxy - set to self on parent itself */ @@ -396,6 +397,7 @@ struct stratifier_data { /* Generator message priority */ int gen_priority; + int proxy_count; proxy_t *proxy; /* Current proxy in use */ proxy_t *proxies; /* Hashlist of all proxies */ proxy_t *old_proxies; /* Hashlist of proxies now no longer in user */ @@ -1047,6 +1049,7 @@ static proxy_t *__generate_proxy(sdata_t *sdata, const int id) /* subid == 0 on parent proxy */ HASH_ADD(sh, proxy->subproxies, subid, sizeof(int), proxy); HASH_ADD_INT(sdata->proxies, id, proxy); + sdata->proxy_count++; return proxy; } @@ -1103,19 +1106,6 @@ static proxy_t *__subproxy_by_id(sdata_t *sdata, proxy_t *proxy, const int subid return subproxy; } -#if 0 -static proxy_t *proxy_by_id(sdata_t *sdata, const int id) -{ - proxy_t *proxy; - - mutex_lock(&sdata->proxy_lock); - proxy = __proxy_by_id(sdata, id); - mutex_unlock(&sdata->proxy_lock); - - return proxy; -} -#endif - static proxy_t *subproxy_by_id(sdata_t *sdata, const int id, const int subid) { proxy_t *proxy, *subproxy; @@ -2342,41 +2332,52 @@ static void stratum_send_message(sdata_t *sdata, const stratum_instance_t *clien * running out of room. */ static sdata_t *select_sdata(const ckpool_t *ckp, sdata_t *ckp_sdata) { - proxy_t *proxy, *subproxy, *best = NULL, *tmp; - int64_t headroom = 0, most_headroom = 0; + proxy_t *current, *proxy, *subproxy, *best = NULL, *tmp, *tmpsub; + int best_id, best_subid = 0; if (!ckp->proxy || ckp->passthrough) return ckp_sdata; - proxy = ckp_sdata->proxy; - if (!proxy) { + current = ckp_sdata->proxy; + if (!current) { LOGWARNING("No proxy available yet to generate subscribes"); return NULL; } + best_id = ckp_sdata->proxy_count; + mutex_lock(&ckp_sdata->proxy_lock); - HASH_ITER(sh, proxy->subproxies, subproxy, tmp) { - int64_t subproxy_headroom; + HASH_ITER(hh, ckp_sdata->proxies, proxy, tmp) { + int most_headroom; - if (subproxy->dead) - continue; - subproxy_headroom = subproxy->max_clients - subproxy->clients; + proxy->headroom = most_headroom = 0; + HASH_ITER(sh, proxy->subproxies, subproxy, tmpsub) { + int64_t subproxy_headroom; - headroom += subproxy_headroom; - if (subproxy_headroom > most_headroom) { - best = subproxy; - most_headroom = subproxy_headroom; + if (subproxy->dead) + continue; + subproxy_headroom = subproxy->max_clients - subproxy->clients; + + proxy->headroom += subproxy_headroom; + if (subproxy_headroom > most_headroom) { + best = subproxy; + most_headroom = subproxy_headroom; + } + } + if (best && best->id < best_id) { + best_id = best->id; + best_subid = best->subid; } } mutex_unlock(&ckp_sdata->proxy_lock); - if (headroom < 42) { + if (best_id != current->id || current->headroom < 42) { LOGNOTICE("Stratifer requesting more proxies from generator"); send_generator(ckp, "recruit", GEN_PRIORITY); } - if (!best) { - LOGNOTICE("Temporarily insufficient subproxies of proxy %d to accept more clients", - proxy->id); + if (best_id == ckp_sdata->proxy_count) { + LOGNOTICE("Temporarily insufficient subproxies to accept more clients"); return NULL; } + best = subproxy_by_id(ckp_sdata, best_id, best_subid); return best->sdata; } From d6f0cfb210d4a2e2bd337c3c45b9a1af320aac1e Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Tue, 17 Feb 2015 17:26:02 +1100 Subject: [PATCH 146/544] Break out of searching for a proxy if the best has headroom --- src/stratifier.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/stratifier.c b/src/stratifier.c index 37b68f4b..7deba67a 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -2348,6 +2348,7 @@ static sdata_t *select_sdata(const ckpool_t *ckp, sdata_t *ckp_sdata) HASH_ITER(hh, ckp_sdata->proxies, proxy, tmp) { int most_headroom; + best = NULL; proxy->headroom = most_headroom = 0; HASH_ITER(sh, proxy->subproxies, subproxy, tmpsub) { int64_t subproxy_headroom; @@ -2365,6 +2366,8 @@ static sdata_t *select_sdata(const ckpool_t *ckp, sdata_t *ckp_sdata) if (best && best->id < best_id) { best_id = best->id; best_subid = best->subid; + if (best == current) + break; } } mutex_unlock(&ckp_sdata->proxy_lock); From 14805ce8fab733d1f784df6168499df3b7073767 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Tue, 17 Feb 2015 17:27:02 +1100 Subject: [PATCH 147/544] Send stratifier deadproxy message if it's trying to send shares to it --- src/generator.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/generator.c b/src/generator.c index a3f48388..745c89e2 100644 --- a/src/generator.c +++ b/src/generator.c @@ -1351,6 +1351,7 @@ static void submit_share(gdata_t *gdata, json_t *val) if (!proxi->alive) { LOGNOTICE("Client %"PRId64" attempting to send shares to dead proxy %d:%d, dropping", client_id, id, subid); + send_stratifier_deadproxy(ckp, id, subid); stratifier_reconnect_client(ckp, client_id); return json_decref(val); } From 9cbe225c4d5a33c3884a67807e0a7643aa2f524a Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Tue, 17 Feb 2015 18:03:50 +1100 Subject: [PATCH 148/544] Reconnect backup clients on stratum notify if they're not already on the current proxy --- src/stratifier.c | 35 ++++++++++++++++++++++++++++++++++- 1 file changed, 34 insertions(+), 1 deletion(-) diff --git a/src/stratifier.c b/src/stratifier.c index 7deba67a..3d61f85c 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -1252,6 +1252,38 @@ static inline bool parent_proxy(const proxy_t *proxy) return (proxy->parent == proxy); } +/* Find how much headroom we have and connect up to that many clients that are + * not currently on this pool */ +static void reconnect_backup_clients(sdata_t *sdata) +{ + stratum_instance_t *client, *tmpclient; + proxy_t *proxy, *subproxy, *tmp; + int64_t headroom = 0; + int reconnects = 0; + + mutex_lock(&sdata->proxy_lock); + proxy = sdata->proxy; + HASH_ITER(sh, proxy->subproxies, subproxy, tmp) { + if (subproxy->dead) + continue; + headroom += subproxy->max_clients - subproxy->clients; + } + mutex_unlock(&sdata->proxy_lock); + + ck_rlock(&sdata->instance_lock); + HASH_ITER(hh, sdata->stratum_instances, client, tmpclient) { + if (reconnects >= headroom) + break; + if (client->proxyid == proxy->id) + continue; + if (client->reconnect) + continue; + client->reconnect = true; + reconnects++; + } + ck_runlock(&sdata->instance_lock); +} + static void update_notify(ckpool_t *ckp, const char *cmd) { sdata_t *sdata = ckp->data, *dsdata; @@ -1351,7 +1383,8 @@ static void update_notify(ckpool_t *ckp, const char *cmd) proxy->notify_id = wb->id; if (parent_proxy(proxy) && proxy == current_proxy(sdata)) reconnect_clients(sdata, proxy->id, proxy->notify_id); - } + } else if (parent_proxy(proxy) && proxy == current_proxy(sdata)) + reconnect_backup_clients(sdata); LOGINFO("Broadcast updated stratum notify"); stratum_broadcast_update(dsdata, new_block | clean); out: From 865cd3a74141fe41f1a7dd6740605bb99163370d Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Thu, 19 Feb 2015 13:21:11 +1100 Subject: [PATCH 149/544] Remove unused variable --- src/connector.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/connector.c b/src/connector.c index 73d1865b..ba4bf416 100644 --- a/src/connector.c +++ b/src/connector.c @@ -381,7 +381,6 @@ void *receiver(void *arg) cdata_t *cdata = (cdata_t *)arg; struct epoll_event event; uint64_t serverfds, i; - time_t start_t; int ret, epfd; rename_proc("creceiver"); @@ -407,7 +406,6 @@ void *receiver(void *arg) while (!cdata->accept) cksleep_ms(1); - start_t = time(NULL); while (42) { client_instance_t *client; From c093b4a06f0f5c9776c2e11395efdb74ea3d1f97 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Thu, 19 Feb 2015 21:04:37 +1100 Subject: [PATCH 150/544] Keep track of the number of clients bound to each proxy --- src/stratifier.c | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/src/stratifier.c b/src/stratifier.c index 2c30fe53..6784cb99 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -214,6 +214,8 @@ struct worker_instance { typedef struct stratifier_data sdata_t; +typedef struct proxy_base proxy_t; + /* Per client stratum instance == workers */ struct stratum_instance { UT_hash_handle hh; @@ -276,7 +278,8 @@ struct stratum_instance { double best_diff; /* Best share found by this instance */ sdata_t *sdata; /* Which sdata this client is bound to */ - int proxyid; /* Which proxy this is bound to in proxy mode */ + proxy_t *proxy; /* Proxy this is bound to in proxy mode */ + int proxyid; /* Which proxy id */ int subproxyid; /* Which subproxy */ int64_t notify_id; /* Which notify_id from the subproxy did we join */ }; @@ -297,8 +300,6 @@ typedef union { uint8_t u8; } enonce1_t; -typedef struct proxy_base proxy_t; - struct proxy_base { UT_hash_handle hh; UT_hash_handle sh; /* For subproxy hashlist */ @@ -321,6 +322,7 @@ struct proxy_base { int64_t clients; int64_t max_clients; + int64_t bound_clients; int64_t headroom; enonce1_t enonce1u; @@ -948,6 +950,8 @@ static void update_base(ckpool_t *ckp, const int prio) static void __kill_instance(stratum_instance_t *client) { + if (client->proxy) + client->proxy->bound_clients--; free(client->workername); free(client->useragent); free(client); @@ -2341,8 +2345,11 @@ static bool new_enonce1(ckpool_t *ckp, sdata_t *ckp_sdata, sdata_t *sdata, strat quit(0, "Invalid enonce1varlen %d", enonce1varlen); } if (ret) { - if (proxy) + if (proxy) { + client->proxy = proxy; proxy->clients++; + proxy->bound_clients++; + } client->enonce1_64 = enonce1u->u64; } ck_wunlock(&ckp_sdata->instance_lock); From 631467ff707fcc3ce9691ba6228ca0d3e29d65cf Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Thu, 19 Feb 2015 22:34:53 +1100 Subject: [PATCH 151/544] Use a linked list for retired proxies in case they have the same id as a previously retired one --- src/stratifier.c | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/stratifier.c b/src/stratifier.c index 6784cb99..6a72b799 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -303,6 +303,8 @@ typedef union { struct proxy_base { UT_hash_handle hh; UT_hash_handle sh; /* For subproxy hashlist */ + proxy_t *next; /* For retired subproxies */ + proxy_t *prev; int id; int subid; @@ -399,10 +401,10 @@ struct stratifier_data { /* Generator message priority */ int gen_priority; - int proxy_count; + int proxy_count; /* Total proxies generated (not necessarily still alive) */ proxy_t *proxy; /* Current proxy in use */ proxy_t *proxies; /* Hashlist of all proxies */ - proxy_t *old_proxies; /* Hashlist of proxies now no longer in user */ + proxy_t *retired_proxies; /* Hashlist of proxies now no longer in user */ mutex_t proxy_lock; /* Protects all proxy data */ proxy_t *subproxy; /* Which subproxy this sdata belongs to in proxy mode */ }; @@ -1175,7 +1177,7 @@ static void new_proxy(sdata_t *sdata, const int id) if (proxy) { exists = true; HASH_DEL(sdata->proxies, proxy); - HASH_ADD_INT(sdata->old_proxies, id, proxy); + DL_APPEND(sdata->retired_proxies, proxy); if (proxy == sdata->proxy) current = true; } From e9af372a5609db82fc9d1836abc8d89f48fa9d17 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Fri, 20 Feb 2015 11:09:56 +1100 Subject: [PATCH 152/544] Reap old subproxies when they're no longer in use --- src/generator.c | 26 +++++++++++++++++++++++++ src/stratifier.c | 49 +++++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 74 insertions(+), 1 deletion(-) diff --git a/src/generator.c b/src/generator.c index 73dd4830..deef63b0 100644 --- a/src/generator.c +++ b/src/generator.c @@ -1314,6 +1314,30 @@ static proxy_instance_t *subproxy_by_id(proxy_instance_t *proxy, const int subid return subproxy; } +static void drop_proxy(gdata_t *gdata, const char *buf) +{ + proxy_instance_t *proxy, *subproxy; + int id = 0, subid = 0; + + sscanf(buf, "dropproxy=%d:%d", &id, &subid); + if (unlikely(!subid)) { + LOGWARNING("Generator asked to drop parent proxy %d", id); + return; + } + proxy = proxy_by_id(gdata, id); + if (unlikely(!proxy)) { + LOGINFO("Generator asked to drop subproxy from non-existent parent %d", id); + return; + } + subproxy = subproxy_by_id(proxy, subid); + if (!subproxy) { + LOGINFO("Generator asked to drop non-existent subproxy %d:%d", id, subid); + return; + } + LOGNOTICE("Generator asked to drop proxy %d:%d", id, subid); + disable_subproxy(gdata, proxy, subproxy); +} + static void stratifier_reconnect_client(ckpool_t *ckp, int64_t id) { char buf[256]; @@ -1984,6 +2008,8 @@ retry: send_unix_msg(sockd, "pong"); } else if (cmdmatch(buf, "recruit")) { recruit_subproxy(proxi); + } else if (cmdmatch(buf, "dropproxy")) { + drop_proxy(gdata, buf); } else if (ckp->passthrough) { /* Anything remaining should be stratum messages */ passthrough_add_send(proxi, buf); diff --git a/src/stratifier.c b/src/stratifier.c index 6a72b799..d5e2fc77 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -1472,6 +1472,48 @@ static void update_diff(ckpool_t *ckp, const char *cmd) ck_runlock(&sdata->instance_lock); } +static void generator_drop_proxy(ckpool_t *ckp, const int id, const int subid) +{ + char msg[256]; + + sprintf(msg, "dropproxy=%d:%d", id, subid); + send_generator(ckp, msg, GEN_LAX); +} + +/* Remove subproxies that are flagged dead or have used up their quota of + * clients and inform the generator if it is still alive */ +static void reap_proxies(ckpool_t *ckp, sdata_t *sdata) +{ + proxy_t *proxy, *proxytmp, *subproxy, *subtmp; + + if (!ckp->proxy) + return; + + mutex_lock(&sdata->proxy_lock); + HASH_ITER(hh, sdata->proxies, proxy, proxytmp) { + HASH_ITER(sh, proxy->subproxies, subproxy, subtmp) { + if (proxy == subproxy) + continue; + if (subproxy->bound_clients) + continue; + if (!subproxy->dead) { + if (subproxy->clients < subproxy->max_clients) + continue; + generator_drop_proxy(ckp, subproxy->id, subproxy->subid); + LOGNOTICE("Stratifier discarding used proxy %d:%d", + subproxy->id, subproxy->subid); + } else { + LOGNOTICE("Stratifier discarding dead proxy %d:%d", + subproxy->id, subproxy->subid); + } + HASH_DELETE(sh, proxy->subproxies, subproxy); + free(subproxy->sdata); + free(subproxy); + } + } + mutex_unlock(&sdata->proxy_lock); +} + /* Enter with instance_lock held */ static stratum_instance_t *__instance_by_id(sdata_t *sdata, const int64_t id) { @@ -1589,6 +1631,9 @@ static void _dec_instance_ref(sdata_t *sdata, stratum_instance_t *client, const /* This should never happen */ if (unlikely(ref < 0)) LOGERR("Instance ref count dropped below zero from %s %s:%d", file, func, line); + + if (dropped) + reap_proxies(sdata->ckp, sdata); } #define dec_instance_ref(sdata, instance) _dec_instance_ref(sdata, instance, __FILE__, __func__, __LINE__) @@ -1786,11 +1831,13 @@ static void drop_client(ckpool_t *ckp, sdata_t *sdata, const int64_t id) client_drop_message(id, dropped, false); if (aged) LOGINFO("Aged %d disconnected instances to dead", aged); - /* Decrease worker count outside of instance_lock to avoid recursive * locking */ if (user) dec_worker(ckp, user); + + if (aged || dropped) + reap_proxies(ckp, sdata); } static void stratum_broadcast_message(sdata_t *sdata, const char *msg) From 68da0e07fabb822909468fa2d17e24fa29870e48 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Fri, 20 Feb 2015 11:18:11 +1100 Subject: [PATCH 153/544] Reap any retired proxies that no longer have any subproxies --- src/stratifier.c | 23 ++++++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/src/stratifier.c b/src/stratifier.c index d5e2fc77..c23c2135 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -1480,8 +1480,16 @@ static void generator_drop_proxy(ckpool_t *ckp, const int id, const int subid) send_generator(ckp, msg, GEN_LAX); } +static void free_proxy(proxy_t *proxy) +{ + free(proxy->sdata); + free(proxy); +} + /* Remove subproxies that are flagged dead or have used up their quota of - * clients and inform the generator if it is still alive */ + * clients and inform the generator if it is still alive. Then see if there + * are any retired proxies that no longer have any other subproxies and reap + * those. */ static void reap_proxies(ckpool_t *ckp, sdata_t *sdata) { proxy_t *proxy, *proxytmp, *subproxy, *subtmp; @@ -1507,8 +1515,17 @@ static void reap_proxies(ckpool_t *ckp, sdata_t *sdata) subproxy->id, subproxy->subid); } HASH_DELETE(sh, proxy->subproxies, subproxy); - free(subproxy->sdata); - free(subproxy); + free_proxy(subproxy); + } + } + + DL_FOREACH_SAFE(sdata->retired_proxies, proxy, proxytmp) { + if (unlikely(proxy->bound_clients)) + continue; + if (HASH_CNT(sh, proxy->subproxies) == 1) { + LOGNOTICE("Stratifier discarding retired proxy %d", proxy->id); + DL_DELETE(sdata->retired_proxies, proxy); + free_proxy(proxy); } } mutex_unlock(&sdata->proxy_lock); From 29ebae7410d3683191593e634ca13d8127fc08fb Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Fri, 20 Feb 2015 11:22:51 +1100 Subject: [PATCH 154/544] Fix incorrect cs being used in proxy_send --- src/generator.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/generator.c b/src/generator.c index deef63b0..c4e56113 100644 --- a/src/generator.c +++ b/src/generator.c @@ -1504,9 +1504,9 @@ static void *proxy_send(void *arg) mutex_unlock(&proxy->notify_lock); subproxy = subproxy_by_id(proxy, subid); - - if (jobid && subproxy) { + if (subproxy) cs = subproxy->cs; + if (jobid && subproxy) { JSON_CPACK(val, "{s[ssooo]soss}", "params", proxy->auth, jobid, json_object_dup(msg->json_msg, "nonce2"), json_object_dup(msg->json_msg, "ntime"), From 5a7febc821575bf77f06f0fb486c6eef7b53c58b Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Fri, 20 Feb 2015 12:20:08 +1100 Subject: [PATCH 155/544] Workbase id and blockchange id are determined by the generator in proxy mode and should not be set per proxy --- src/stratifier.c | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/src/stratifier.c b/src/stratifier.c index c23c2135..e252e281 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -1019,7 +1019,6 @@ static void drop_allclients(ckpool_t *ckp) static sdata_t *duplicate_sdata(const sdata_t *sdata) { sdata_t *dsdata = ckzalloc(sizeof(sdata_t)); - int64_t randomiser; dsdata->ckp = sdata->ckp; @@ -1035,10 +1034,7 @@ static sdata_t *duplicate_sdata(const sdata_t *sdata) dsdata->sauthq = sdata->sauthq; dsdata->stxnq = sdata->stxnq; - /* Give the sbuproxy a unique workbase id and its own workbase list - * and lock */ - randomiser = ((int64_t)time(NULL)) << 32; - dsdata->blockchange_id = dsdata->workbase_id = randomiser; + /* Give the sbuproxy its own workbase list and lock */ cklock_init(&dsdata->workbase_lock); return dsdata; } @@ -1770,7 +1766,7 @@ static void stratum_broadcast(sdata_t *sdata, json_t *val) ssends = sdata->ssends; - mutex_lock(sdata->ssends->lock); + mutex_lock(ssends->lock); if (ssends->msgs) DL_CONCAT(ssends->msgs, bulk_send); else From 6e9901f9fe39715be2d8714b53c87aebea84f5d1 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Fri, 20 Feb 2015 15:57:55 +1100 Subject: [PATCH 156/544] Shutdown instead of closing a socket after sending a unix message allowing the receiving end to close the socket after receiving the data --- src/ckpool.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ckpool.c b/src/ckpool.c index 4a0130c7..ec7cb517 100644 --- a/src/ckpool.c +++ b/src/ckpool.c @@ -532,7 +532,7 @@ bool _send_proc(proc_instance_t *pi, const char *msg, const char *file, const ch LOGWARNING("Failed to send %s to socket %s", msg, path); else ret = true; - Close(sockd); + shutdown(sockd, SHUT_WR); out: if (unlikely(!ret)) { LOGERR("Failure in send_proc from %s %s:%d", file, func, line); From ea56de3bf64c92629f0bca375f8bacaa26beb82f Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Fri, 20 Feb 2015 17:11:38 +1100 Subject: [PATCH 157/544] Check for pid in send_recv_proc as well --- src/ckpool.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/ckpool.c b/src/ckpool.c index ec7cb517..d15afab1 100644 --- a/src/ckpool.c +++ b/src/ckpool.c @@ -556,6 +556,8 @@ char *_send_recv_proc(proc_instance_t *pi, const char *msg, const char *file, co LOGERR("Attempted to send null message to socket %s in send_proc", path); goto out; } + if (unlikely(!pi->pid)) + pi->pid = get_proc_pid(pi); if (unlikely(kill_pid(pi->pid, 0))) { LOGALERT("Attempting to send message %s to dead process %s", msg, pi->processname); goto out; From 33e195511cb076a1f513022f5fde38245d3763a4 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Fri, 20 Feb 2015 17:49:56 +1100 Subject: [PATCH 158/544] Wait for the other end to close a unix socket to ensure the message has gone through --- src/ckpool.c | 2 ++ src/libckpool.c | 14 ++++++++++++++ src/libckpool.h | 1 + 3 files changed, 17 insertions(+) diff --git a/src/ckpool.c b/src/ckpool.c index d15afab1..e4db077a 100644 --- a/src/ckpool.c +++ b/src/ckpool.c @@ -533,6 +533,8 @@ bool _send_proc(proc_instance_t *pi, const char *msg, const char *file, const ch else ret = true; shutdown(sockd, SHUT_WR); + if (!wait_close(sockd, 5)) + LOGWARNING("send_proc did not close from %s %s:%d", file, func, line); out: if (unlikely(!ret)) { LOGERR("Failure in send_proc from %s %s:%d", file, func, line); diff --git a/src/libckpool.c b/src/libckpool.c index 10e5b8bc..4a9675e8 100644 --- a/src/libckpool.c +++ b/src/libckpool.c @@ -910,6 +910,20 @@ out: return sockd; } +/* Wait till a socket has been closed at the other end */ +int wait_close(int sockd, int timeout) +{ + struct pollfd sfd; + + if (unlikely(sockd < 0)) + return -1; + sfd.fd = sockd; + sfd.events = POLLHUP; + sfd.revents = 0; + timeout *= 1000; + return poll(&sfd, 1, timeout); +} + /* Emulate a select read wait for high fds that select doesn't support */ int wait_read_select(int sockd, int timeout) { diff --git a/src/libckpool.h b/src/libckpool.h index e21acc96..79f0f8fd 100644 --- a/src/libckpool.h +++ b/src/libckpool.h @@ -493,6 +493,7 @@ int _open_unix_server(const char *server_path, const char *file, const char *fun #define open_unix_server(server_path) _open_unix_server(server_path, __FILE__, __func__, __LINE__) int _open_unix_client(const char *server_path, const char *file, const char *func, const int line); #define open_unix_client(server_path) _open_unix_client(server_path, __FILE__, __func__, __LINE__) +int wait_close(int sockd, int timeout); int wait_read_select(int sockd, int timeout); int read_length(int sockd, void *buf, int len); char *_recv_unix_msg(int sockd, int timeout1, int timeout2, const char *file, const char *func, const int line); From 9de62bdea90ac934571edee11d2593bb9db639e5 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Fri, 20 Feb 2015 17:50:24 +1100 Subject: [PATCH 159/544] Reverse the waiting order between generator and stratifier in proxy mode --- src/generator.c | 12 +++++++++++- src/stratifier.c | 5 +++-- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/src/generator.c b/src/generator.c index c4e56113..c0abfb42 100644 --- a/src/generator.c +++ b/src/generator.c @@ -2159,12 +2159,22 @@ int generator(proc_instance_t *pi) ckp->data = gdata; gdata->ckp = ckp; if (ckp->proxy) { + char *buf; + + /* Wait for the stratifier to be ready for us */ + do { + if (!ping_main(ckp)) { + ret = 1; + goto out; + } + buf = send_recv_proc(ckp->stratifier, "ping"); + } while (!buf); ret = proxy_mode(ckp, pi); } else { gdata->srvchk = create_ckmsgq(ckp, "srvchk", &server_watchdog); ret = server_mode(ckp, pi); } - +out: dealloc(ckp->data); return process_exit(ckp, pi, ret); } diff --git a/src/stratifier.c b/src/stratifier.c index e252e281..be4b6dcc 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -4909,8 +4909,11 @@ int stratifier(proc_instance_t *pi) ret = 1; goto out; } + if (ckp->proxy) + break; buf = send_recv_proc(ckp->generator, "ping"); } while (!buf); + dealloc(buf); if (!ckp->proxy) { if (!test_address(ckp, ckp->btcaddress)) { @@ -4935,8 +4938,6 @@ int stratifier(proc_instance_t *pi) sdata->blockchange_id = sdata->workbase_id = randomiser; sdata->enonce1u.u64 = htobe64(randomiser); - dealloc(buf); - if (!ckp->serverurls) { ckp->serverurl[0] = "127.0.0.1"; ckp->serverurls = 1; From 8dd1754426901ebb62c1275b7179929260c40676 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Fri, 20 Feb 2015 17:56:03 +1100 Subject: [PATCH 160/544] Close socket reliably in generator --- src/generator.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/generator.c b/src/generator.c index c0abfb42..b2b5539d 100644 --- a/src/generator.c +++ b/src/generator.c @@ -1946,6 +1946,7 @@ static int proxy_loop(proc_instance_t *pi) setup_proxies(ckp, gdata); reconnect: + Close(sockd); /* This does not necessarily mean we reconnect, but a change has * occurred and we need to reexamine the proxies. */ cproxy = wait_best_proxy(ckp, gdata); @@ -1963,6 +1964,7 @@ reconnect: } } retry: + Close(sockd); do { selret = wait_read_select(us->sockd, 5); if (!selret && !ping_main(ckp)) { @@ -1990,7 +1992,6 @@ retry: buf = recv_unix_msg(sockd); if (!buf) { LOGWARNING("Failed to get message in proxy_loop"); - Close(sockd); goto retry; } LOGDEBUG("Proxy received request: %s", buf); @@ -2022,7 +2023,6 @@ retry: else submit_share(gdata, val); } - Close(sockd); goto retry; out: Close(sockd); From 33f8c4275d4ad4ca0aacddf409f77488dca46581 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Fri, 20 Feb 2015 20:04:03 +1100 Subject: [PATCH 161/544] Close our end of the socket in send_proc --- src/ckpool.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ckpool.c b/src/ckpool.c index e4db077a..545bdf6b 100644 --- a/src/ckpool.c +++ b/src/ckpool.c @@ -532,9 +532,9 @@ bool _send_proc(proc_instance_t *pi, const char *msg, const char *file, const ch LOGWARNING("Failed to send %s to socket %s", msg, path); else ret = true; - shutdown(sockd, SHUT_WR); if (!wait_close(sockd, 5)) LOGWARNING("send_proc did not close from %s %s:%d", file, func, line); + Close(sockd); out: if (unlikely(!ret)) { LOGERR("Failure in send_proc from %s %s:%d", file, func, line); From 41f042c3e518a54f5ff4d374c9522f7a9a50e86c Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Fri, 20 Feb 2015 21:14:20 +1100 Subject: [PATCH 162/544] Duplicate the actual jobid json object to return when submitting shares and use an integer consistently as the internal jobid --- src/generator.c | 42 +++++++++++++++++++++++------------------- 1 file changed, 23 insertions(+), 19 deletions(-) diff --git a/src/generator.c b/src/generator.c index b2b5539d..421fa6d3 100644 --- a/src/generator.c +++ b/src/generator.c @@ -28,7 +28,7 @@ struct notify_instance { int id; char prevhash[68]; - char *jobid; + json_t *jobid; char *coinbase1; char *coinbase2; int coinb1len; @@ -403,7 +403,7 @@ out: return ret; } -static bool send_json_msg(connsock_t *cs, json_t *json_msg) +static bool send_json_msg(connsock_t *cs, const json_t *json_msg) { int len, sent; char *s; @@ -784,19 +784,19 @@ static bool parse_notify(ckpool_t *ckp, proxy_instance_t *proxi, json_t *val) { const char *prev_hash, *bbversion, *nbit, *ntime; proxy_instance_t *proxy = proxi->parent; - char *job_id, *coinbase1, *coinbase2; gdata_t *gdata = proxi->ckp->data; + char *coinbase1, *coinbase2; bool clean, ret = false; notify_instance_t *ni; + json_t *arr, *job_id; int merkles, i; - json_t *arr; arr = json_array_get(val, 4); if (!arr || !json_is_array(arr)) goto out; merkles = json_array_size(arr); - job_id = json_array_string(val, 0); + job_id = json_copy(json_array_get(val, 0)); prev_hash = __json_array_string(val, 1); coinbase1 = json_array_string(val, 2); coinbase2 = json_array_string(val, 3); @@ -806,7 +806,7 @@ static bool parse_notify(ckpool_t *ckp, proxy_instance_t *proxi, json_t *val) clean = json_is_true(json_array_get(val, 8)); if (!job_id || !prev_hash || !coinbase1 || !coinbase2 || !bbversion || !nbit || !ntime) { if (job_id) - free(job_id); + json_decref(job_id); if (coinbase1) free(coinbase1); if (coinbase2) @@ -817,7 +817,6 @@ static bool parse_notify(ckpool_t *ckp, proxy_instance_t *proxi, json_t *val) LOGDEBUG("Received new notify from proxy %d:%d", proxi->id, proxi->subid); ni = ckzalloc(sizeof(notify_instance_t)); ni->jobid = job_id; - LOGDEBUG("Job ID %s", job_id); ni->coinbase1 = coinbase1; LOGDEBUG("Coinbase1 %s", coinbase1); ni->coinb1len = strlen(coinbase1) / 2; @@ -1357,8 +1356,8 @@ static void submit_share(gdata_t *gdata, json_t *val) /* Get the client id so we can tell the stratifier to drop it if the * proxy it's bound to is not functional */ - json_getdel_int64(&client_id, val, "client_id"); - json_getdel_int(&id, val, "proxy"); + json_get_int64(&client_id, val, "client_id"); + json_get_int(&id, val, "proxy"); proxy = proxy_by_id(gdata, id); if (unlikely(!proxy)) { LOGWARNING("Failed to find proxy %d to send share to", id); @@ -1384,7 +1383,7 @@ static void submit_share(gdata_t *gdata, json_t *val) share = ckzalloc(sizeof(share_msg_t)); share->submit_time = time(NULL); share->client_id = client_id; - json_getdel_int(&share->msg_id, val, "msg_id"); + json_get_int(&share->msg_id, val, "msg_id"); msg->json_msg = val; /* Add new share entry to the share hashtable */ @@ -1404,7 +1403,8 @@ static void submit_share(gdata_t *gdata, json_t *val) static void clear_notify(notify_instance_t *ni) { - free(ni->jobid); + if (ni->jobid) + json_decref(ni->jobid); free(ni->coinbase1); free(ni->coinbase2); free(ni); @@ -1463,13 +1463,13 @@ static void *proxy_send(void *arg) while (42) { proxy_instance_t *subproxy; + int proxyid = 0, subid = 0; notify_instance_t *ni; + json_t *jobid = NULL; stratum_msg_t *msg; - char *jobid = NULL; bool ret = true; - int subid = 0; json_t *val; - uint32_t id; + int id; tv_t now; ts_t abs; @@ -1494,20 +1494,25 @@ static void *proxy_send(void *arg) if (!msg) continue; - json_getdel_int(&subid, msg->json_msg, "subproxy"); - json_uintcpy(&id, msg->json_msg, "jobid"); + json_get_int(&subid, msg->json_msg, "subproxy"); + json_get_int(&id, msg->json_msg, "jobid"); + json_get_int(&proxyid, msg->json_msg, "proxy"); + if (unlikely(proxyid != proxy->id)) { + LOGWARNING("Proxysend for proxy %d got message for proxy %d!", + proxy->id, proxyid); + } mutex_lock(&proxy->notify_lock); HASH_FIND_INT(proxy->notify_instances, &id, ni); if (ni) - jobid = strdup(ni->jobid); + jobid = json_copy(ni->jobid); mutex_unlock(&proxy->notify_lock); subproxy = subproxy_by_id(proxy, subid); if (subproxy) cs = subproxy->cs; if (jobid && subproxy) { - JSON_CPACK(val, "{s[ssooo]soss}", "params", proxy->auth, jobid, + JSON_CPACK(val, "{s[soooo]soss}", "params", proxy->auth, jobid, json_object_dup(msg->json_msg, "nonce2"), json_object_dup(msg->json_msg, "ntime"), json_object_dup(msg->json_msg, "nonce"), @@ -1522,7 +1527,6 @@ static void *proxy_send(void *arg) LOGNOTICE("Failed to find subproxy %d:%d to send message to", proxy->id, subid); } - free(jobid); json_decref(msg->json_msg); free(msg); if (!ret && subproxy) { From 2055f0e2e87e3b05a601e8438bfb1ff753eb766f Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Fri, 20 Feb 2015 21:30:28 +1100 Subject: [PATCH 163/544] Demote nonce length warning for subproxies --- src/generator.c | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/generator.c b/src/generator.c index 421fa6d3..8bd3ea57 100644 --- a/src/generator.c +++ b/src/generator.c @@ -650,8 +650,13 @@ retry: goto out; } if (size < 3) { - LOGWARNING("Proxy %d:%d %s Nonce2 length %d too small for fast miners", - proxi->id, proxi->subid, proxi->si->url, size); + if (!proxi->subid) { + LOGWARNING("Proxy %d %s Nonce2 length %d too small for fast miners", + proxi->id, proxi->si->url, size); + } else { + LOGNOTICE("Proxy %d:%d Nonce2 length %d too small for fast miners", + proxi->id, proxi->subid, size); + } } proxi->nonce2len = size; if (parent_proxy(proxi)) { From ad5b2a089b4943fb5ea44411e05e9a3d75cad85b Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Fri, 20 Feb 2015 22:34:29 +1100 Subject: [PATCH 164/544] Add warning about reason for failed subscription --- src/stratifier.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/stratifier.c b/src/stratifier.c index be4b6dcc..0b3af579 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -2504,6 +2504,7 @@ static json_t *parse_subscribe(stratum_instance_t *client, const int64_t client_ sdata = select_sdata(ckp, ckp_sdata); if (unlikely(!sdata || !sdata->current_workbase)) { + LOGWARNING("Failed to provide subscription due to no %s", sdata ? "current workbase" : "sdata"); stratum_send_message(ckp_sdata, client, "Pool Initialising"); return json_string("Initialising"); } From 8c07e6eda7646c86dd8797a1916d2ba6ed2c56b1 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Fri, 20 Feb 2015 22:37:54 +1100 Subject: [PATCH 165/544] Give extra info for extranonce size in update subscribe --- src/stratifier.c | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/stratifier.c b/src/stratifier.c index 0b3af579..69845ba4 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -1243,8 +1243,13 @@ static void update_subscribe(ckpool_t *ckp, const char *cmd) proxy->enonce1u.u64 = 0; ck_wunlock(&dsdata->workbase_lock); - LOGNOTICE("Upstream pool extranonce2 length %d, max proxy clients %"PRId64, - proxy->nonce2len, proxy->max_clients); + if (subid) { + LOGINFO("Upstream pool %d:%d extranonce2 length %d, max proxy clients %"PRId64, + id, subid, proxy->nonce2len, proxy->max_clients); + } else { + LOGNOTICE("Upstream pool %d extranonce2 length %d, max proxy clients %"PRId64, + id, proxy->nonce2len, proxy->max_clients); + } json_decref(val); } From 4cd554a23817605fd8d3562cac7cb9dd767abd02 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Fri, 20 Feb 2015 22:48:21 +1100 Subject: [PATCH 166/544] Set the reconnect time on a proxy when a read socket detects a disconnection --- src/generator.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/generator.c b/src/generator.c index 8bd3ea57..7ab04a84 100644 --- a/src/generator.c +++ b/src/generator.c @@ -1851,6 +1851,7 @@ static void *proxy_recv(void *arg) } if (ret < 1) { if (parent_proxy(subproxy)) { + proxi->reconnect_time = time(NULL); alive = false; LOGWARNING("Proxy %d:%s failed to epoll/read_socket_line in proxy_recv, attempting reconnect", subproxy->id, subproxy->si->url); From 111a10bf9d0d26cc382a106878e501340cd0c9ee Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Fri, 20 Feb 2015 23:08:43 +1100 Subject: [PATCH 167/544] Send clients a reconnect even if we're switching to a backup pool --- src/stratifier.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/stratifier.c b/src/stratifier.c index 69845ba4..736a5e3f 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -1146,7 +1146,7 @@ static void reconnect_clients(sdata_t *sdata, const int proxyid, const int64_t n ck_rlock(&sdata->instance_lock); HASH_ITER(hh, sdata->stratum_instances, client, tmp) { - if (client->proxyid > proxyid || (client->proxyid == proxyid && client->notify_id != notify_id)) + if (client->proxyid != proxyid || client->notify_id != notify_id) client->reconnect = true; } ck_runlock(&sdata->instance_lock); From 65d9abab0ba981bebaccdcc4e3ecba7110747d9f Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Fri, 20 Feb 2015 23:26:38 +1100 Subject: [PATCH 168/544] Skip subproxies that don't yet have a current workbase when selecting a subscription --- src/stratifier.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/stratifier.c b/src/stratifier.c index 736a5e3f..99d7afc3 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -2462,6 +2462,8 @@ static sdata_t *select_sdata(const ckpool_t *ckp, sdata_t *ckp_sdata) if (subproxy->dead) continue; + if (!subproxy->sdata->current_workbase) + continue; subproxy_headroom = subproxy->max_clients - subproxy->clients; proxy->headroom += subproxy_headroom; From 1ff651971ffd2d8ad7cf746b4bfe3d700ab59c3b Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Fri, 20 Feb 2015 23:54:57 +1100 Subject: [PATCH 169/544] Close socket before adding message to recvs --- src/stratifier.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/stratifier.c b/src/stratifier.c index 99d7afc3..a52977ab 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -2227,8 +2227,8 @@ retry: /* The bulk of the messages will be received json from the * connector so look for this first. The srecv_process frees * the buf heap ram */ - ckmsgq_add(sdata->srecvs, buf); Close(sockd); + ckmsgq_add(sdata->srecvs, buf); buf = NULL; goto retry; } From 235cdf3c964f8ba74a7d0396e89198bfae568974 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Fri, 20 Feb 2015 23:59:05 +1100 Subject: [PATCH 170/544] Flag proxies as dead if they fail connect_proxy --- src/generator.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/generator.c b/src/generator.c index 7ab04a84..f988111e 100644 --- a/src/generator.c +++ b/src/generator.c @@ -1589,7 +1589,7 @@ static bool proxy_alive(ckpool_t *ckp, server_instance_t *si, proxy_instance_t * LOGINFO("Failed to connect to %s:%s in proxy_mode!", cs->url, cs->port); } - return ret; + goto out; } if (ckp->passthrough) { if (!passthrough_stratum(cs, proxi)) { From cef4a2cd36583931a7b44c60b48481422f12e902 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Fri, 20 Feb 2015 23:59:42 +1100 Subject: [PATCH 171/544] Show message associated with no close fd detection --- src/ckpool.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ckpool.c b/src/ckpool.c index 545bdf6b..5902c443 100644 --- a/src/ckpool.c +++ b/src/ckpool.c @@ -533,7 +533,7 @@ bool _send_proc(proc_instance_t *pi, const char *msg, const char *file, const ch else ret = true; if (!wait_close(sockd, 5)) - LOGWARNING("send_proc did not close from %s %s:%d", file, func, line); + LOGWARNING("send_proc %s did not detect close from %s %s:%d", msg, file, func, line); Close(sockd); out: if (unlikely(!ret)) { From 7fb37b788669a93c52e9ab80cf5ef5d1d27c0713 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Sat, 21 Feb 2015 00:02:25 +1100 Subject: [PATCH 172/544] Do not loop after updating immediately in the strat loop --- src/stratifier.c | 1 - 1 file changed, 1 deletion(-) diff --git a/src/stratifier.c b/src/stratifier.c index a52977ab..26565f14 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -2199,7 +2199,6 @@ retry: ckp->update_interval); broadcast_ping(sdata); } - continue; } selret = wait_read_select(us->sockd, 5); if (!selret && !ping_main(ckp)) { From f37279da12e6595400026cd51e5ce6fe9182bd53 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Sat, 21 Feb 2015 00:34:48 +1100 Subject: [PATCH 173/544] Check for correct condition in wait_close --- src/libckpool.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/libckpool.c b/src/libckpool.c index 4a9675e8..6b47502e 100644 --- a/src/libckpool.c +++ b/src/libckpool.c @@ -914,14 +914,18 @@ out: int wait_close(int sockd, int timeout) { struct pollfd sfd; + int ret; if (unlikely(sockd < 0)) return -1; sfd.fd = sockd; - sfd.events = POLLHUP; + sfd.events = POLLIN; sfd.revents = 0; timeout *= 1000; - return poll(&sfd, 1, timeout); + ret = poll(&sfd, 1, timeout); + if (ret < 1) + return 0; + return sfd.revents & POLLHUP; } /* Emulate a select read wait for high fds that select doesn't support */ From 47fa24dca04307d4606e04abb8cc30fb75037d60 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Sat, 21 Feb 2015 00:40:09 +1100 Subject: [PATCH 174/544] Move reap messages out of lock --- src/stratifier.c | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/src/stratifier.c b/src/stratifier.c index 26565f14..a7b4ee96 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -1494,6 +1494,7 @@ static void free_proxy(proxy_t *proxy) static void reap_proxies(ckpool_t *ckp, sdata_t *sdata) { proxy_t *proxy, *proxytmp, *subproxy, *subtmp; + int used = 0, dead = 0, retired = 0; if (!ckp->proxy) return; @@ -1509,11 +1510,9 @@ static void reap_proxies(ckpool_t *ckp, sdata_t *sdata) if (subproxy->clients < subproxy->max_clients) continue; generator_drop_proxy(ckp, subproxy->id, subproxy->subid); - LOGNOTICE("Stratifier discarding used proxy %d:%d", - subproxy->id, subproxy->subid); + used++; } else { - LOGNOTICE("Stratifier discarding dead proxy %d:%d", - subproxy->id, subproxy->subid); + dead++; } HASH_DELETE(sh, proxy->subproxies, subproxy); free_proxy(subproxy); @@ -1524,12 +1523,17 @@ static void reap_proxies(ckpool_t *ckp, sdata_t *sdata) if (unlikely(proxy->bound_clients)) continue; if (HASH_CNT(sh, proxy->subproxies) == 1) { - LOGNOTICE("Stratifier discarding retired proxy %d", proxy->id); + retired++; DL_DELETE(sdata->retired_proxies, proxy); free_proxy(proxy); } } mutex_unlock(&sdata->proxy_lock); + + if (used || dead || retired) { + LOGNOTICE("Stratifier discarded %d used, %d dead and %d retired proxies", + used, dead, retired); + } } /* Enter with instance_lock held */ From 9842659e394884378695ba5644712b8a067be026 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Sat, 21 Feb 2015 01:08:26 +1100 Subject: [PATCH 175/544] Fix buf dereference error --- src/stratifier.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/stratifier.c b/src/stratifier.c index a7b4ee96..29944db5 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -4905,8 +4905,8 @@ int stratifier(proc_instance_t *pi) ckpool_t *ckp = pi->ckp; int ret = 1, threads; int64_t randomiser; + char *buf = NULL; sdata_t *sdata; - char *buf; LOGWARNING("%s stratifier starting", ckp->name); sdata = ckzalloc(sizeof(sdata_t)); From dffc938519101aa64e5eb9b9f8b24eae12883a73 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Sat, 21 Feb 2015 01:09:22 +1100 Subject: [PATCH 176/544] Free buffer used in generator --- src/generator.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/generator.c b/src/generator.c index f988111e..ea44c7a9 100644 --- a/src/generator.c +++ b/src/generator.c @@ -2169,7 +2169,7 @@ int generator(proc_instance_t *pi) ckp->data = gdata; gdata->ckp = ckp; if (ckp->proxy) { - char *buf; + char *buf = NULL; /* Wait for the stratifier to be ready for us */ do { @@ -2179,6 +2179,7 @@ int generator(proc_instance_t *pi) } buf = send_recv_proc(ckp->stratifier, "ping"); } while (!buf); + dealloc(buf); ret = proxy_mode(ckp, pi); } else { gdata->srvchk = create_ckmsgq(ckp, "srvchk", &server_watchdog); From b0060079db711a224b9f24ce2ef1e253c0302c58 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Sat, 21 Feb 2015 01:16:05 +1100 Subject: [PATCH 177/544] Return value of send_proc is never used --- src/ckpool.c | 3 +-- src/ckpool.h | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/src/ckpool.c b/src/ckpool.c index 5902c443..4206c615 100644 --- a/src/ckpool.c +++ b/src/ckpool.c @@ -501,7 +501,7 @@ out: /* Send a single message to a process instance when there will be no response, * closing the socket immediately. */ -bool _send_proc(proc_instance_t *pi, const char *msg, const char *file, const char *func, const int line) +void _send_proc(proc_instance_t *pi, const char *msg, const char *file, const char *func, const int line) { char *path = pi->us.path; bool ret = false; @@ -540,7 +540,6 @@ out: LOGERR("Failure in send_proc from %s %s:%d", file, func, line); childsighandler(15); } - return ret; } /* Send a single message to a process instance and retrieve the response, then diff --git a/src/ckpool.h b/src/ckpool.h index a46889a4..e0e0f947 100644 --- a/src/ckpool.h +++ b/src/ckpool.h @@ -209,7 +209,7 @@ ckpool_t *global_ckp; bool ping_main(ckpool_t *ckp); void empty_buffer(connsock_t *cs); int read_socket_line(connsock_t *cs, const int timeout); -bool _send_proc(proc_instance_t *pi, const char *msg, const char *file, const char *func, const int line); +void _send_proc(proc_instance_t *pi, const char *msg, const char *file, const char *func, const int line); #define send_proc(pi, msg) _send_proc(pi, msg, __FILE__, __func__, __LINE__) char *_send_recv_proc(proc_instance_t *pi, const char *msg, const char *file, const char *func, const int line); #define send_recv_proc(pi, msg) _send_recv_proc(pi, msg, __FILE__, __func__, __LINE__) From 9e5d725e68ebfbe538badc7fbeabaf6dab204f29 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Sat, 21 Feb 2015 01:49:24 +1100 Subject: [PATCH 178/544] Make all one way send_procs asynchronous to avoid message response deadlocks --- src/ckpool.c | 42 +++++++++++++++++++++++++++++++++++++++--- 1 file changed, 39 insertions(+), 3 deletions(-) diff --git a/src/ckpool.c b/src/ckpool.c index 4206c615..89a81e34 100644 --- a/src/ckpool.c +++ b/src/ckpool.c @@ -499,14 +499,32 @@ out: return pid; } -/* Send a single message to a process instance when there will be no response, - * closing the socket immediately. */ -void _send_proc(proc_instance_t *pi, const char *msg, const char *file, const char *func, const int line) +struct proc_message { + proc_instance_t *pi; + char *msg; + const char *file; + const char *func; + int line; +}; + +/* Send all one way messages asynchronously so we can wait till the receiving + * end closes the socket to ensure all messages are received but no deadlocks + * can occur with 2 processes waiting for each other's socket closure. */ +void *async_send_proc(void *arg) { + struct proc_message *pm = (struct proc_message *)arg; + proc_instance_t *pi = pm->pi; + char *msg = pm->msg; + const char *file = pm->file; + const char *func = pm->func; + int line = pm->line; + char *path = pi->us.path; bool ret = false; int sockd; + pthread_detach(pthread_self()); + if (unlikely(!path || !strlen(path))) { LOGERR("Attempted to send message %s to null path in send_proc", msg ? msg : ""); goto out; @@ -540,6 +558,24 @@ out: LOGERR("Failure in send_proc from %s %s:%d", file, func, line); childsighandler(15); } + free(msg); + free(pm); + return NULL; +} + +/* Send a single message to a process instance when there will be no response, + * closing the socket immediately. */ +void _send_proc(proc_instance_t *pi, const char *msg, const char *file, const char *func, const int line) +{ + struct proc_message *pm = ckalloc(sizeof(struct proc_message)); + pthread_t pth; + + pm->pi = pi; + pm->msg = strdup(msg); + pm->file = file; + pm->func = func; + pm->line = line; + create_pthread(&pth, async_send_proc, pm); } /* Send a single message to a process instance and retrieve the response, then From 765c3a050d9c035d6e83844685807f350bb61fdd Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Sat, 21 Feb 2015 09:47:37 +1100 Subject: [PATCH 179/544] Reset the pi pid after a failure to find the process alive so we can look it up again in case it has changed --- src/ckpool.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/ckpool.c b/src/ckpool.c index 89a81e34..10c1548c 100644 --- a/src/ckpool.c +++ b/src/ckpool.c @@ -596,6 +596,9 @@ char *_send_recv_proc(proc_instance_t *pi, const char *msg, const char *file, co if (unlikely(!pi->pid)) pi->pid = get_proc_pid(pi); if (unlikely(kill_pid(pi->pid, 0))) { + /* Reset the pid value in case we are still looking for an old + * process */ + pi->pid = 0; LOGALERT("Attempting to send message %s to dead process %s", msg, pi->processname); goto out; } From 0c185fbd2a3d4c0ad5851cf696cd6782b6981c19 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Sat, 21 Feb 2015 09:58:36 +1100 Subject: [PATCH 180/544] Max sure max headroom variable is 64 bit --- src/stratifier.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/stratifier.c b/src/stratifier.c index 29944db5..be6c49f5 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -2456,10 +2456,10 @@ static sdata_t *select_sdata(const ckpool_t *ckp, sdata_t *ckp_sdata) mutex_lock(&ckp_sdata->proxy_lock); HASH_ITER(hh, ckp_sdata->proxies, proxy, tmp) { - int most_headroom; + int64_t max_headroom; best = NULL; - proxy->headroom = most_headroom = 0; + proxy->headroom = max_headroom = 0; HASH_ITER(sh, proxy->subproxies, subproxy, tmpsub) { int64_t subproxy_headroom; @@ -2470,9 +2470,9 @@ static sdata_t *select_sdata(const ckpool_t *ckp, sdata_t *ckp_sdata) subproxy_headroom = subproxy->max_clients - subproxy->clients; proxy->headroom += subproxy_headroom; - if (subproxy_headroom > most_headroom) { + if (subproxy_headroom > max_headroom) { best = subproxy; - most_headroom = subproxy_headroom; + max_headroom = subproxy_headroom; } } if (best && best->id < best_id) { From 51337105ac32d4990d349a0080226990aee37270 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Sat, 21 Feb 2015 10:08:54 +1100 Subject: [PATCH 181/544] Count existing reconnects in headroom count --- src/stratifier.c | 30 ++++++++++++++++++++---------- 1 file changed, 20 insertions(+), 10 deletions(-) diff --git a/src/stratifier.c b/src/stratifier.c index be6c49f5..b9e97b15 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -1133,6 +1133,23 @@ static proxy_t *existing_subproxy(sdata_t *sdata, const int id, const int subid) return subproxy; } +/* Set proxy to the current proxy and calculate how much headroom it has */ +static int64_t current_headroom(sdata_t *sdata, proxy_t **proxy) +{ + proxy_t *subproxy, *tmp; + int64_t headroom = 0; + + mutex_lock(&sdata->proxy_lock); + *proxy = sdata->proxy; + HASH_ITER(sh, (*proxy)->subproxies, subproxy, tmp) { + if (subproxy->dead) + continue; + headroom += subproxy->max_clients - subproxy->clients; + } + mutex_unlock(&sdata->proxy_lock); + + return headroom; +} /* Iterates over all clients in proxy mode and sets the reconnect bool for the * message to be sent lazily next time they speak to us only if the proxy is @@ -1264,18 +1281,11 @@ static inline bool parent_proxy(const proxy_t *proxy) static void reconnect_backup_clients(sdata_t *sdata) { stratum_instance_t *client, *tmpclient; - proxy_t *proxy, *subproxy, *tmp; int64_t headroom = 0; int reconnects = 0; + proxy_t *proxy; - mutex_lock(&sdata->proxy_lock); - proxy = sdata->proxy; - HASH_ITER(sh, proxy->subproxies, subproxy, tmp) { - if (subproxy->dead) - continue; - headroom += subproxy->max_clients - subproxy->clients; - } - mutex_unlock(&sdata->proxy_lock); + headroom = current_headroom(sdata, &proxy); ck_rlock(&sdata->instance_lock); HASH_ITER(hh, sdata->stratum_instances, client, tmpclient) { @@ -1283,10 +1293,10 @@ static void reconnect_backup_clients(sdata_t *sdata) break; if (client->proxyid == proxy->id) continue; + reconnects++; if (client->reconnect) continue; client->reconnect = true; - reconnects++; } ck_runlock(&sdata->instance_lock); } From cffc412427fb96b24fccea02d5d7446ee315fa08 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Sat, 21 Feb 2015 10:29:29 +1100 Subject: [PATCH 182/544] Combine the reconnect proxy functions, switching as many clients as there is headroom and flagging the rest for lazy reconnection --- src/stratifier.c | 76 ++++++++++++++++++++---------------------------- 1 file changed, 32 insertions(+), 44 deletions(-) diff --git a/src/stratifier.c b/src/stratifier.c index b9e97b15..22675833 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -1151,22 +1151,40 @@ static int64_t current_headroom(sdata_t *sdata, proxy_t **proxy) return headroom; } -/* Iterates over all clients in proxy mode and sets the reconnect bool for the - * message to be sent lazily next time they speak to us only if the proxy is - * higher priority than the one they're currently connected to or the notify_id - * on their proxy has changed indicating a new subscription. */ -static void reconnect_clients(sdata_t *sdata, const int proxyid, const int64_t notify_id) +static void reconnect_client(sdata_t *sdata, stratum_instance_t *client); + +/* Find how much headroom we have and connect up to that many clients that are + * not currently on this pool, setting the reconnect for the remainder to be + * switched lazily. */ +static void reconnect_clients(sdata_t *sdata) { - stratum_instance_t *client, *tmp; + stratum_instance_t *client, *tmpclient; + int reconnects = 0, flagged = 0; + int64_t headroom; + proxy_t *proxy; - LOGINFO("Setting reconnect to proxy %d notifyid %"PRId64, proxyid, notify_id); + headroom = current_headroom(sdata, &proxy); ck_rlock(&sdata->instance_lock); - HASH_ITER(hh, sdata->stratum_instances, client, tmp) { - if (client->proxyid != proxyid || client->notify_id != notify_id) - client->reconnect = true; + HASH_ITER(hh, sdata->stratum_instances, client, tmpclient) { + if (client->proxyid == proxy->id && client->notify_id == proxy->parent->notify_id) + continue; + if (reconnects >= headroom) { + if (!client->reconnect) { + client->reconnect = true; + flagged++; + } + continue; + } + reconnects++; + reconnect_client(sdata, client); } ck_runlock(&sdata->instance_lock); + + if (reconnects || flagged) { + LOGNOTICE("Reconnected %d clients, flagged %d for proxy %d", reconnects, + flagged, proxy->id); + } } static proxy_t *current_proxy(sdata_t *sdata) @@ -1276,31 +1294,6 @@ static inline bool parent_proxy(const proxy_t *proxy) return (proxy->parent == proxy); } -/* Find how much headroom we have and connect up to that many clients that are - * not currently on this pool */ -static void reconnect_backup_clients(sdata_t *sdata) -{ - stratum_instance_t *client, *tmpclient; - int64_t headroom = 0; - int reconnects = 0; - proxy_t *proxy; - - headroom = current_headroom(sdata, &proxy); - - ck_rlock(&sdata->instance_lock); - HASH_ITER(hh, sdata->stratum_instances, client, tmpclient) { - if (reconnects >= headroom) - break; - if (client->proxyid == proxy->id) - continue; - reconnects++; - if (client->reconnect) - continue; - client->reconnect = true; - } - ck_runlock(&sdata->instance_lock); -} - static void update_notify(ckpool_t *ckp, const char *cmd) { sdata_t *sdata = ckp->data, *dsdata; @@ -1393,15 +1386,10 @@ static void update_notify(ckpool_t *ckp, const char *cmd) LOGNOTICE("Block hash on proxy %d changed to %s", id, dsdata->lastswaphash); } - if (proxy->notify_id == -1) { - /* This is the first notification from the current proxy, tell - * clients now to reconnect since we have enough information to - * switch. */ + if (proxy->notify_id == -1) proxy->notify_id = wb->id; - if (parent_proxy(proxy) && proxy == current_proxy(sdata)) - reconnect_clients(sdata, proxy->id, proxy->notify_id); - } else if (parent_proxy(proxy) && proxy == current_proxy(sdata)) - reconnect_backup_clients(sdata); + if (parent_proxy(proxy) && proxy == current_proxy(sdata)) + reconnect_clients(sdata); LOGINFO("Broadcast updated stratum notify"); stratum_broadcast_update(dsdata, new_block | clean); out: @@ -2149,7 +2137,7 @@ static void set_proxy(sdata_t *sdata, const char *buf) LOGNOTICE("Stratifier setting active proxy to %d", id); if (proxy->notify_id != -1) - reconnect_clients(sdata, proxy->id, proxy->notify_id); + reconnect_clients(sdata); } static void dead_proxy(sdata_t *sdata, const char *buf) From 73e6824c008a41c13f3a5ec54edd50912ca7d41e Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Sat, 21 Feb 2015 10:39:41 +1100 Subject: [PATCH 183/544] Recruit more proxies when we have flagged clients in reconnect_clients --- src/stratifier.c | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/src/stratifier.c b/src/stratifier.c index 22675833..b1bfd37f 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -1153,6 +1153,12 @@ static int64_t current_headroom(sdata_t *sdata, proxy_t **proxy) static void reconnect_client(sdata_t *sdata, stratum_instance_t *client); +static void generator_recruit(const ckpool_t *ckp) +{ + LOGINFO("Stratifer requesting more proxies from generator"); + send_generator(ckp, "recruit", GEN_PRIORITY); +} + /* Find how much headroom we have and connect up to that many clients that are * not currently on this pool, setting the reconnect for the remainder to be * switched lazily. */ @@ -1185,6 +1191,8 @@ static void reconnect_clients(sdata_t *sdata) LOGNOTICE("Reconnected %d clients, flagged %d for proxy %d", reconnects, flagged, proxy->id); } + if (flagged) + generator_recruit(client->ckp); } static proxy_t *current_proxy(sdata_t *sdata) @@ -2482,10 +2490,8 @@ static sdata_t *select_sdata(const ckpool_t *ckp, sdata_t *ckp_sdata) } mutex_unlock(&ckp_sdata->proxy_lock); - if (best_id != current->id || current->headroom < 42) { - LOGNOTICE("Stratifer requesting more proxies from generator"); - send_generator(ckp, "recruit", GEN_PRIORITY); - } + if (best_id != current->id || current->headroom < 42) + generator_recruit(ckp); if (best_id == ckp_sdata->proxy_count) { LOGNOTICE("Temporarily insufficient subproxies to accept more clients"); return NULL; From 08d8644d1a49d48484317b3c401df7673a872697 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Sat, 21 Feb 2015 10:42:55 +1100 Subject: [PATCH 184/544] Still drop clients from dead proxies even if we don't have their data any more --- src/stratifier.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/stratifier.c b/src/stratifier.c index b1bfd37f..165dfd9c 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -2156,9 +2156,8 @@ static void dead_proxy(sdata_t *sdata, const char *buf) sscanf(buf, "deadproxy=%d:%d", &id, &subid); proxy = existing_subproxy(sdata, id, subid); - if (!proxy) - return; - proxy->dead = true; + if (proxy) + proxy->dead = true; LOGNOTICE("Stratifier dropping clients from proxy %d:%d", id, subid); ck_rlock(&sdata->instance_lock); From a470ca6431ecbed34aaef2d2d6be89bceff60987 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Sat, 21 Feb 2015 11:33:49 +1100 Subject: [PATCH 185/544] Only reconnect clients when there is room for them on the new proxy, tagging the rest and recruiting as needed --- src/stratifier.c | 65 +++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 59 insertions(+), 6 deletions(-) diff --git a/src/stratifier.c b/src/stratifier.c index 165dfd9c..9cbc14b4 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -1141,11 +1141,14 @@ static int64_t current_headroom(sdata_t *sdata, proxy_t **proxy) mutex_lock(&sdata->proxy_lock); *proxy = sdata->proxy; + if (!*proxy) + goto out_unlock; HASH_ITER(sh, (*proxy)->subproxies, subproxy, tmp) { if (subproxy->dead) continue; headroom += subproxy->max_clients - subproxy->clients; } +out_unlock: mutex_unlock(&sdata->proxy_lock); return headroom; @@ -1170,6 +1173,8 @@ static void reconnect_clients(sdata_t *sdata) proxy_t *proxy; headroom = current_headroom(sdata, &proxy); + if (!proxy) + return; ck_rlock(&sdata->instance_lock); HASH_ITER(hh, sdata->stratum_instances, client, tmpclient) { @@ -1192,7 +1197,7 @@ static void reconnect_clients(sdata_t *sdata) flagged, proxy->id); } if (flagged) - generator_recruit(client->ckp); + generator_recruit(sdata->ckp); } static proxy_t *current_proxy(sdata_t *sdata) @@ -2121,6 +2126,12 @@ static void reconnect_client(sdata_t *sdata, stratum_instance_t *client) { json_t *json_msg; + /* Already requested? */ + if (client->reconnect_request) { + connector_drop_client(sdata->ckp, client->id); + client->dropped = true; + return; + } client->reconnect = false; client->reconnect_request = time(NULL); JSON_CPACK(json_msg, "{sosss[]}", "id", json_null(), "method", "client.reconnect", @@ -2151,7 +2162,9 @@ static void set_proxy(sdata_t *sdata, const char *buf) static void dead_proxy(sdata_t *sdata, const char *buf) { stratum_instance_t *client, *tmp; + int reconnects = 0, flagged = 0; int id = 0, subid = 0; + int64_t headroom; proxy_t *proxy; sscanf(buf, "deadproxy=%d:%d", &id, &subid); @@ -2159,13 +2172,54 @@ static void dead_proxy(sdata_t *sdata, const char *buf) if (proxy) proxy->dead = true; LOGNOTICE("Stratifier dropping clients from proxy %d:%d", id, subid); + headroom = current_headroom(sdata, &proxy); + if (!proxy) + return; ck_rlock(&sdata->instance_lock); HASH_ITER(hh, sdata->stratum_instances, client, tmp) { - if (client->proxyid == id && client->subproxyid == subid) - reconnect_client(sdata, client); + if (client->proxyid != id || client->subproxyid != subid) + continue; + if (reconnects >= headroom) { + if (!client->reconnect) { + client->reconnect = true; + flagged++; + } + continue; + } + client->reconnect = true; + reconnects++; + reconnect_client(sdata, client); } ck_runlock(&sdata->instance_lock); + + if (reconnects || flagged) { + LOGNOTICE("Reconnected %d clients, flagged %d from dead proxy %d:%d", reconnects, + flagged, id, subid); + } + if (flagged) + generator_recruit(sdata->ckp); +} + +/* Must hold a reference */ +static void lazy_reconnect_client(sdata_t *sdata, stratum_instance_t *client) +{ + int64_t headroom; + proxy_t *proxy; + + headroom = current_headroom(sdata, &proxy); + if (!proxy) + return; + if (headroom > 0) { + LOGNOTICE("Reconnecting client %"PRId64, client->id); + reconnect_client(sdata, client); + } else { + generator_recruit(sdata->ckp); + if (!client->reconnect) { + LOGNOTICE("Flagging client %"PRId64, client->id); + client->reconnect = true; + } + } } static void reconnect_client_id(sdata_t *sdata, const int64_t client_id) @@ -2177,8 +2231,7 @@ static void reconnect_client_id(sdata_t *sdata, const int64_t client_id) LOGINFO("reconnect_client_id failed to find client %"PRId64, client_id); return; } - LOGNOTICE("Reconnecting client %"PRId64, client_id); - reconnect_client(sdata, client); + lazy_reconnect_client(sdata, client); dec_instance_ref(sdata, client); } @@ -4079,7 +4132,7 @@ static void srecv_process(ckpool_t *ckp, char *buf) /* The client is still active but has been issued a reconnect request * so use this opportunity to send it a reconnect message */ if (unlikely(client->reconnect)) - reconnect_client(sdata, client); + lazy_reconnect_client(sdata, client); dec_instance_ref(sdata, client); out: free(buf); From 8dbf39dae5a52d482c9b523e5a96755a1ba2d0b1 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Sat, 21 Feb 2015 11:38:04 +1100 Subject: [PATCH 186/544] Reconnect clients that are sending shares that can't be processed --- src/generator.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/generator.c b/src/generator.c index ea44c7a9..995b554d 100644 --- a/src/generator.c +++ b/src/generator.c @@ -1462,7 +1462,8 @@ static void *proxy_send(void *arg) { proxy_instance_t *proxy = (proxy_instance_t *)arg; connsock_t *cs = proxy->cs; - gdata_t *gdata = cs->ckp->data; + ckpool_t *ckp = cs->ckp; + gdata_t *gdata = ckp->data; rename_proc("proxysend"); @@ -1470,6 +1471,7 @@ static void *proxy_send(void *arg) proxy_instance_t *subproxy; int proxyid = 0, subid = 0; notify_instance_t *ni; + int64_t client_id = 0; json_t *jobid = NULL; stratum_msg_t *msg; bool ret = true; @@ -1502,6 +1504,7 @@ static void *proxy_send(void *arg) json_get_int(&subid, msg->json_msg, "subproxy"); json_get_int(&id, msg->json_msg, "jobid"); json_get_int(&proxyid, msg->json_msg, "proxy"); + json_get_int64(&client_id, val, "client_id"); if (unlikely(proxyid != proxy->id)) { LOGWARNING("Proxysend for proxy %d got message for proxy %d!", proxy->id, proxyid); @@ -1526,9 +1529,11 @@ static void *proxy_send(void *arg) ret = send_json_msg(cs, val); json_decref(val); } else if (!jobid) { + stratifier_reconnect_client(ckp, client_id); LOGNOTICE("Proxy %d:%s failed to find matching jobid for %sknown subproxy in proxysend", proxy->id, proxy->si->url, subproxy ? "" : "un"); } else { + stratifier_reconnect_client(ckp, client_id); LOGNOTICE("Failed to find subproxy %d:%d to send message to", proxy->id, subid); } From 396b656c82881cd568785212f9cc683438fe76b2 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Sat, 21 Feb 2015 11:39:42 +1100 Subject: [PATCH 187/544] Use the correct json --- src/generator.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/generator.c b/src/generator.c index 995b554d..73503436 100644 --- a/src/generator.c +++ b/src/generator.c @@ -1504,7 +1504,7 @@ static void *proxy_send(void *arg) json_get_int(&subid, msg->json_msg, "subproxy"); json_get_int(&id, msg->json_msg, "jobid"); json_get_int(&proxyid, msg->json_msg, "proxy"); - json_get_int64(&client_id, val, "client_id"); + json_get_int64(&client_id, msg->json_msg, "client_id"); if (unlikely(proxyid != proxy->id)) { LOGWARNING("Proxysend for proxy %d got message for proxy %d!", proxy->id, proxyid); From 0a165bfc66fb7b3e7980e4f00e36dc71e6511f81 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Sat, 21 Feb 2015 12:18:23 +1100 Subject: [PATCH 188/544] Revert to lazily reconnecting all clients to prevent floods --- src/stratifier.c | 41 ++++++++++++++++------------------------- 1 file changed, 16 insertions(+), 25 deletions(-) diff --git a/src/stratifier.c b/src/stratifier.c index 9cbc14b4..54a31a9f 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -1168,7 +1168,7 @@ static void generator_recruit(const ckpool_t *ckp) static void reconnect_clients(sdata_t *sdata) { stratum_instance_t *client, *tmpclient; - int reconnects = 0, flagged = 0; + int reconnects = 0; int64_t headroom; proxy_t *proxy; @@ -1180,23 +1180,19 @@ static void reconnect_clients(sdata_t *sdata) HASH_ITER(hh, sdata->stratum_instances, client, tmpclient) { if (client->proxyid == proxy->id && client->notify_id == proxy->parent->notify_id) continue; - if (reconnects >= headroom) { - if (!client->reconnect) { - client->reconnect = true; - flagged++; - } + if (client->reconnect) continue; - } + headroom--; reconnects++; - reconnect_client(sdata, client); + client->reconnect = true; } ck_runlock(&sdata->instance_lock); - if (reconnects || flagged) { - LOGNOTICE("Reconnected %d clients, flagged %d for proxy %d", reconnects, - flagged, proxy->id); + if (reconnects) { + LOGNOTICE("Flagged %d clients for reconnect to proxy %d", reconnects, + proxy->id); } - if (flagged) + if (headroom < 42) generator_recruit(sdata->ckp); } @@ -2162,8 +2158,8 @@ static void set_proxy(sdata_t *sdata, const char *buf) static void dead_proxy(sdata_t *sdata, const char *buf) { stratum_instance_t *client, *tmp; - int reconnects = 0, flagged = 0; int id = 0, subid = 0; + int reconnects = 0; int64_t headroom; proxy_t *proxy; @@ -2180,24 +2176,19 @@ static void dead_proxy(sdata_t *sdata, const char *buf) HASH_ITER(hh, sdata->stratum_instances, client, tmp) { if (client->proxyid != id || client->subproxyid != subid) continue; - if (reconnects >= headroom) { - if (!client->reconnect) { - client->reconnect = true; - flagged++; - } + if (client->reconnect) continue; - } - client->reconnect = true; + headroom--; reconnects++; - reconnect_client(sdata, client); + client->reconnect = true; } ck_runlock(&sdata->instance_lock); - if (reconnects || flagged) { - LOGNOTICE("Reconnected %d clients, flagged %d from dead proxy %d:%d", reconnects, - flagged, id, subid); + if (reconnects) { + LOGNOTICE("Flagged %d clients to reconnect from dead proxy %d:%d", reconnects, + id, subid); } - if (flagged) + if (headroom < 42) generator_recruit(sdata->ckp); } From 1bdf57e21da9f5b22cd12b037cd5e79da7cbe9b1 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Sat, 21 Feb 2015 17:50:50 +1100 Subject: [PATCH 189/544] Cope with unknown pids in various send msg commands without terminal failure --- src/ckpool.c | 60 +++++++++++++++++++++++++++++++++------------------- 1 file changed, 38 insertions(+), 22 deletions(-) diff --git a/src/ckpool.c b/src/ckpool.c index 10c1548c..69afbf80 100644 --- a/src/ckpool.c +++ b/src/ckpool.c @@ -237,7 +237,25 @@ static int pid_wait(const pid_t pid, const int ms) return ret; } -static int send_procmsg(const proc_instance_t *pi, const char *buf) +static int get_proc_pid(const proc_instance_t *pi) +{ + int ret, pid = 0; + char path[256]; + FILE *fp; + + sprintf(path, "%s%s.pid", pi->ckp->socket_dir, pi->processname); + fp = fopen(path, "re"); + if (!fp) + goto out; + ret = fscanf(fp, "%d", &pid); + if (ret < 1) + pid = 0; + fclose(fp); +out: + return pid; +} + +static int send_procmsg(proc_instance_t *pi, const char *buf) { char *path = pi->us.path; int ret = -1; @@ -251,6 +269,12 @@ static int send_procmsg(const proc_instance_t *pi, const char *buf) LOGERR("Attempted to send null message to socket %s in send_proc", path); goto out; } + if (unlikely(!pi->pid)) { + pi->pid = get_proc_pid(pi); + if (!pi->pid) + goto out; + + } if (unlikely(kill_pid(pi->pid, 0))) { LOGALERT("Attempting to send message %s to dead process %s", buf, pi->processname); goto out; @@ -481,24 +505,6 @@ out: static void childsighandler(const int sig); -static int get_proc_pid(const proc_instance_t *pi) -{ - int ret, pid = 0; - char path[256]; - FILE *fp; - - sprintf(path, "%s%s.pid", pi->ckp->socket_dir, pi->processname); - fp = fopen(path, "re"); - if (!fp) - goto out; - ret = fscanf(fp, "%d", &pid); - if (ret < 1) - pid = 0; - fclose(fp); -out: - return pid; -} - struct proc_message { proc_instance_t *pi; char *msg; @@ -535,10 +541,16 @@ void *async_send_proc(void *arg) } /* At startup the pid fields are not set up before some processes are * forked so they never inherit them. */ - if (unlikely(!pi->pid)) + if (unlikely(!pi->pid)) { pi->pid = get_proc_pid(pi); + if (!pi->pid) { + LOGALERT("Attempting to send message %s to non existent process %s", msg, pi->processname); + goto out_nofail; + } + } if (unlikely(kill_pid(pi->pid, 0))) { - LOGALERT("Attempting to send message %s to non existent process %s", msg, pi->processname); + LOGALERT("Attempting to send message %s to non existent process %s pid %d", + msg, pi->processname, pi->pid); goto out; } sockd = open_unix_client(path); @@ -558,6 +570,7 @@ out: LOGERR("Failure in send_proc from %s %s:%d", file, func, line); childsighandler(15); } +out_nofail: free(msg); free(pm); return NULL; @@ -593,8 +606,11 @@ char *_send_recv_proc(proc_instance_t *pi, const char *msg, const char *file, co LOGERR("Attempted to send null message to socket %s in send_proc", path); goto out; } - if (unlikely(!pi->pid)) + if (unlikely(!pi->pid)) { pi->pid = get_proc_pid(pi); + if (!pi->pid) + goto out; + } if (unlikely(kill_pid(pi->pid, 0))) { /* Reset the pid value in case we are still looking for an old * process */ From 6971bf45cd1e399fe68957af04428ced3bfc941c Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Sat, 21 Feb 2015 17:59:59 +1100 Subject: [PATCH 190/544] Remove the old pid file per process when preparing the new child processes --- src/ckpool.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/ckpool.c b/src/ckpool.c index 69afbf80..484eda48 100644 --- a/src/ckpool.c +++ b/src/ckpool.c @@ -1286,6 +1286,8 @@ static proc_instance_t *prepare_child(ckpool_t *ckp, int (*process)(), char *nam pi->process = process; create_process_unixsock(pi); manage_old_child(ckp, pi); + /* Remove the old pid file if we've succeeded in coming this far */ + rm_namepid(pi); return pi; } From ac8bb3259dd37038eb2754dcfa0404e167cdc26b Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Sat, 21 Feb 2015 18:16:17 +1100 Subject: [PATCH 191/544] Send reconnect to clients immediately upon detecting they're sending to a dead proxy instead of flagging them --- src/stratifier.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/stratifier.c b/src/stratifier.c index 54a31a9f..b1e7b366 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -2222,7 +2222,7 @@ static void reconnect_client_id(sdata_t *sdata, const int64_t client_id) LOGINFO("reconnect_client_id failed to find client %"PRId64, client_id); return; } - lazy_reconnect_client(sdata, client); + reconnect_client(sdata, client); dec_instance_ref(sdata, client); } From 0afbf3eff2748f60c0492ece54aabb2af1aa6575 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Sat, 21 Feb 2015 21:47:35 +1100 Subject: [PATCH 192/544] Only consider a proxy dead if all subproxy connections are also dead --- src/generator.c | 97 ++++++++++++++++++++++++++++++++++--------------- 1 file changed, 67 insertions(+), 30 deletions(-) diff --git a/src/generator.c b/src/generator.c index 73503436..867bf5d2 100644 --- a/src/generator.c +++ b/src/generator.c @@ -1701,6 +1701,31 @@ static void recruit_subproxy(proxy_instance_t *proxi) create_pthread(&pth, proxy_recruit, proxi); } +static void *proxy_reconnect(void *arg) +{ + proxy_instance_t *proxy = (proxy_instance_t *)arg; + server_instance_t *si = proxy->si; + connsock_t *cs = proxy->cs; + ckpool_t *ckp = proxy->ckp; + + pthread_detach(pthread_self()); + proxy_alive(ckp, si, proxy, cs, true, proxy->epfd); + return NULL; +} + +/* For reconnecting the parent proxy instance async */ +static void reconnect_proxy(proxy_instance_t *proxi) +{ + pthread_t pth; + + create_pthread(&pth, proxy_reconnect, proxi); +} + +static void reconnect_generator(const ckpool_t *ckp) +{ + send_proc(ckp->generator, "reconnect"); +} + /* For receiving messages from an upstream pool to pass downstream. Responsible * for setting up the connection and testing pool is live. */ static void *passthrough_recv(void *arg) @@ -1722,7 +1747,7 @@ static void *passthrough_recv(void *arg) } if (proxy_alive(ckp, si, proxi, cs, false, epfd)) { - send_proc(ckp->generator, "reconnect"); + reconnect_generator(ckp); LOGWARNING("Proxy %d:%s connection established", proxi->id, proxi->si->url); } @@ -1734,12 +1759,12 @@ static void *passthrough_recv(void *arg) while (!proxy_alive(ckp, si, proxi, cs, true, epfd)) { if (alive) { alive = false; - send_proc(ckp->generator, "reconnect"); + reconnect_generator(ckp); } sleep(5); } if (!alive) - send_proc(ckp->generator, "reconnect"); + reconnect_generator(ckp); /* Make sure we receive a line within 90 seconds */ ret = epoll_wait(epfd, &event, 1, 90000); @@ -1749,7 +1774,7 @@ static void *passthrough_recv(void *arg) LOGWARNING("Proxy %d:%s failed to read_socket_line in proxy_recv, attempting reconnect", proxi->id, proxi->si->url); alive = proxi->alive = false; - send_proc(ckp->generator, "reconnect"); + reconnect_generator(ckp); continue; } /* Simply forward the message on, as is, to the connector to @@ -1771,6 +1796,23 @@ static proxy_instance_t *current_proxy(gdata_t *gdata) return ret; } +static bool subproxies_alive(proxy_instance_t *proxy) +{ + proxy_instance_t *subproxy, *tmp; + bool ret = false; + + mutex_lock(&proxy->proxy_lock); + HASH_ITER(sh, proxy->subproxies, subproxy, tmp) { + if (subproxy->alive) { + ret = true; + break; + } + } + mutex_unlock(&proxy->proxy_lock); + + return ret; +} + /* For receiving messages from the upstream proxy, also responsible for setting * up the connection and testing it's alive. */ static void *proxy_recv(void *arg) @@ -1793,7 +1835,7 @@ static void *proxy_recv(void *arg) } if (proxy_alive(ckp, si, proxi, cs, false, epfd)) { - send_proc(ckp->generator, "reconnect"); + reconnect_generator(ckp); LOGWARNING("Proxy %d:%s connection established", proxi->id, proxi->si->url); } @@ -1806,20 +1848,25 @@ static void *proxy_recv(void *arg) time_t now; int ret; - while (!proxy_alive(ckp, si, proxi, proxi->cs, true, epfd)) { - if (alive) { - alive = false; - send_proc(ckp->generator, "reconnect"); + if (!proxi->alive) { + while (!subproxies_alive(proxi)) { + reconnect_proxy(proxi); + if (alive) { + LOGWARNING("Proxy %d:%s failed, attempting reconnect", + proxi->id, proxi->si->url); + alive = false; + reconnect_generator(ckp); + } + sleep(5); + proxi->reconnect_time = time(NULL); } - sleep(5); - proxi->reconnect_time = time(NULL); } /* Wait 30 seconds before declaring this upstream pool alive * to prevent switching to unstable pools. */ if (!alive && (!current_proxy(gdata) || time(NULL) - proxi->reconnect_time > 30)) { LOGWARNING("Proxy %d:%s recovered", proxi->id, proxi->si->url); proxi->reconnect_time = 0; - send_proc(ckp->generator, "reconnect"); + reconnect_generator(ckp); alive = true; } @@ -1855,13 +1902,11 @@ static void *proxy_recv(void *arg) ret = read_socket_line(cs, 5); } if (ret < 1) { - if (parent_proxy(subproxy)) { - proxi->reconnect_time = time(NULL); - alive = false; - LOGWARNING("Proxy %d:%s failed to epoll/read_socket_line in proxy_recv, attempting reconnect", - subproxy->id, subproxy->si->url); - send_proc(ckp->generator, "reconnect"); - } + subproxy->alive = false; + if (!parent_proxy(subproxy)) + recruit_subproxy(proxi); + LOGNOTICE("Proxy %d:%d %s failed to epoll/read_socket_line in proxy_recv, attempting reconnect", + proxi->id, subproxy->subid, subproxy->si->url); continue; } if (parse_method(ckp, subproxy, cs->buf)) { @@ -1872,7 +1917,7 @@ static void *proxy_recv(void *arg) subproxy->alive = false; disable_subproxy(gdata, proxi, subproxy); if (parent_proxy(subproxy)) { - send_proc(ckp->generator, "reconnect"); + reconnect_generator(ckp); LOGWARNING("Proxy %d:%s reconnect issue, dropping existing connection", subproxy->id, subproxy->si->url); break; @@ -1933,7 +1978,7 @@ static proxy_instance_t *wait_best_proxy(ckpool_t *ckp, gdata_t *gdata) mutex_lock(&gdata->lock); HASH_ITER(hh, gdata->proxies, proxi, tmp) { - if (proxi->alive) { + if (proxi->alive || subproxies_alive(proxi)) { if (!ret || proxi->id < ret->id) ret = proxi; } @@ -1989,14 +2034,6 @@ retry: } } while (selret < 1); - if (unlikely(proxi->cs->fd < 0)) { - LOGWARNING("Upstream proxy %d:%s socket invalidated, will attempt failover", - proxi->id, proxi->cs->url); - proxi->alive = false; - proxi = NULL; - goto reconnect; - } - sockd = accept(us->sockd, NULL, NULL); if (sockd < 0) { LOGEMERG("Failed to accept on proxy socket"); @@ -2160,7 +2197,7 @@ static void server_watchdog(ckpool_t *ckp, server_instance_t *cursi) break; } if (alive) - send_proc(ckp->generator, "reconnect"); + reconnect_generator(ckp); } int generator(proc_instance_t *pi) From 0d39d22c16a0f9cdc21b2ebe9be43e8d46c5d46d Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Sat, 21 Feb 2015 22:28:57 +1100 Subject: [PATCH 193/544] Use the disable_subproxy function for all proxies including the parent and ensure we message the stratifier about all dead proxies, leaving recruitment only to on demand --- src/generator.c | 34 ++++++++++++++++------------------ 1 file changed, 16 insertions(+), 18 deletions(-) diff --git a/src/generator.c b/src/generator.c index 867bf5d2..8236d609 100644 --- a/src/generator.c +++ b/src/generator.c @@ -944,9 +944,16 @@ static void send_stratifier_deadproxy(ckpool_t *ckp, const int id, const int sub send_proc(ckp->stratifier, buf); } -/* Remove the subproxy from the proxi list and put it on the dead list */ +/* Remove the subproxy from the proxi list and put it on the dead list. + * Further use of the subproxy pointer may point to a new proxy but will not + * dereference */ static void disable_subproxy(gdata_t *gdata, proxy_instance_t *proxi, proxy_instance_t *subproxy) { + subproxy->alive = false; + send_stratifier_deadproxy(gdata->ckp, subproxy->id, subproxy->subid); + if (parent_proxy(subproxy)) + return; + mutex_lock(&proxi->proxy_lock); subproxy->disabled = true; /* Make sure subproxy is still in the list */ @@ -957,11 +964,8 @@ static void disable_subproxy(gdata_t *gdata, proxy_instance_t *proxi, proxy_inst } mutex_unlock(&proxi->proxy_lock); - if (subproxy) { - send_stratifier_deadproxy(gdata->ckp, subproxy->id, subproxy->subid); - if (!parent_proxy(subproxy)) - store_proxy(gdata, subproxy); - } + if (subproxy) + store_proxy(gdata, subproxy); } /* If the parent is no longer in use due to reconnect, we shouldn't use any of @@ -973,6 +977,7 @@ static void drop_subproxies(proxy_instance_t *proxi) mutex_lock(&proxi->proxy_lock); HASH_ITER(sh, proxi->subproxies, subproxy, tmp) { if (!parent_proxy(subproxy)) { + send_stratifier_deadproxy(proxi->ckp, proxi->id, subproxy->subid); subproxy->disabled = true; Close(subproxy->cs->fd); } @@ -1540,13 +1545,9 @@ static void *proxy_send(void *arg) json_decref(msg->json_msg); free(msg); if (!ret && subproxy) { - if (cs->fd > 0) { - LOGWARNING("Proxy %d:%d %s failed to send msg in proxy_send, dropping to reconnect", - proxy->id, proxy->subid, proxy->si->url); - Close(cs->fd); - } - if (!parent_proxy(subproxy) && !subproxy->disabled) - disable_subproxy(gdata, proxy, subproxy); + LOGNOTICE("Proxy %d:%d %s failed to send msg in proxy_send, dropping to reconnect", + proxy->id, proxy->subid, proxy->si->url); + disable_subproxy(gdata, proxy, subproxy); } } return NULL; @@ -1902,11 +1903,9 @@ static void *proxy_recv(void *arg) ret = read_socket_line(cs, 5); } if (ret < 1) { - subproxy->alive = false; - if (!parent_proxy(subproxy)) - recruit_subproxy(proxi); LOGNOTICE("Proxy %d:%d %s failed to epoll/read_socket_line in proxy_recv, attempting reconnect", proxi->id, subproxy->subid, subproxy->si->url); + disable_subproxy(gdata, proxi, subproxy); continue; } if (parse_method(ckp, subproxy, cs->buf)) { @@ -1914,12 +1913,11 @@ static void *proxy_recv(void *arg) /* Call this proxy dead to allow us to fail * over to a backup pool until the reconnect * pool is up */ - subproxy->alive = false; disable_subproxy(gdata, proxi, subproxy); if (parent_proxy(subproxy)) { reconnect_generator(ckp); LOGWARNING("Proxy %d:%s reconnect issue, dropping existing connection", - subproxy->id, subproxy->si->url); + subproxy->id, subproxy->si->url); break; } else recruit_subproxy(proxi); From 77e68e8db1e488d8aafeda6c3d569844d55502ec Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Sun, 22 Feb 2015 23:49:18 +1100 Subject: [PATCH 194/544] Revert "Send reconnect to clients immediately upon detecting they're sending to a dead proxy instead of flagging them" This reverts commit ac8bb3259dd37038eb2754dcfa0404e167cdc26b. --- src/stratifier.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/stratifier.c b/src/stratifier.c index b1e7b366..54a31a9f 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -2222,7 +2222,7 @@ static void reconnect_client_id(sdata_t *sdata, const int64_t client_id) LOGINFO("reconnect_client_id failed to find client %"PRId64, client_id); return; } - reconnect_client(sdata, client); + lazy_reconnect_client(sdata, client); dec_instance_ref(sdata, client); } From 234f76f8e5910714a9008a7ef44b20786b24adac Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Mon, 23 Feb 2015 00:03:37 +1100 Subject: [PATCH 195/544] Force a reconnect send to clients that have been given the reconnect flag already and are requested to reconnect again --- src/generator.c | 1 - src/stratifier.c | 3 ++- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/generator.c b/src/generator.c index 8236d609..62e4ed89 100644 --- a/src/generator.c +++ b/src/generator.c @@ -1384,7 +1384,6 @@ static void submit_share(gdata_t *gdata, json_t *val) if (!proxi->alive) { LOGNOTICE("Client %"PRId64" attempting to send shares to dead proxy %d:%d, dropping", client_id, id, subid); - send_stratifier_deadproxy(ckp, id, subid); stratifier_reconnect_client(ckp, client_id); return json_decref(val); } diff --git a/src/stratifier.c b/src/stratifier.c index 54a31a9f..89d2d852 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -2209,7 +2209,8 @@ static void lazy_reconnect_client(sdata_t *sdata, stratum_instance_t *client) if (!client->reconnect) { LOGNOTICE("Flagging client %"PRId64, client->id); client->reconnect = true; - } + } else /* Already been flagged, force the send */ + reconnect_client(sdata, client); } } From 59951a81929acbdfb2010ff3bd246cfd02d3d2f5 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Mon, 23 Feb 2015 12:01:15 +1100 Subject: [PATCH 196/544] Use async send proc in the stratifier --- src/stratifier.c | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/src/stratifier.c b/src/stratifier.c index 0397e639..3e8760d8 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -808,7 +808,7 @@ static void send_generator(ckpool_t *ckp, const char *msg, const int prio) set = true; } else set = false; - send_proc(ckp->generator, msg); + async_send_proc(ckp, ckp->generator, msg); if (set) sdata->gen_priority = 0; } @@ -929,7 +929,7 @@ static void connector_drop_client(ckpool_t *ckp, const int64_t id) LOGDEBUG("Stratifier requesting connector drop client %"PRId64, id); snprintf(buf, 255, "dropclient=%"PRId64, id); - send_proc(ckp->connector, buf); + async_send_proc(ckp, ckp->connector, buf); } static void drop_allclients(ckpool_t *ckp) @@ -3421,6 +3421,7 @@ static void send_transactions(ckpool_t *ckp, json_params_t *jp); static void parse_method(sdata_t *sdata, stratum_instance_t *client, const int64_t client_id, json_t *id_val, json_t *method_val, json_t *params_val, const char *address) { + ckpool_t *ckp = client->ckp; const char *method; /* Random broken clients send something not an integer as the id so we @@ -3467,14 +3468,14 @@ static void parse_method(sdata_t *sdata, stratum_instance_t *client, const int64 * to it since it's unauthorised. Set the flag just in case. */ client->authorised = false; snprintf(buf, 255, "passthrough=%"PRId64, client_id); - send_proc(client->ckp->connector, buf); + async_send_proc(ckp, ckp->connector, buf); return; } /* We should only accept subscribed requests from here on */ if (!client->subscribed) { LOGINFO("Dropping unsubscribed client %"PRId64, client_id); - connector_drop_client(client->ckp, client_id); + connector_drop_client(ckp, client_id); return; } @@ -3496,7 +3497,7 @@ static void parse_method(sdata_t *sdata, stratum_instance_t *client, const int64 * stratifier process to restart since it will have lost all * the stratum instance data. Clients will just reconnect. */ LOGINFO("Dropping unauthorised client %"PRId64, client_id); - connector_drop_client(client->ckp, client_id); + connector_drop_client(ckp, client_id); return; } @@ -3684,7 +3685,7 @@ static void ssend_process(ckpool_t *ckp, smsg_t *msg) * connector process to be delivered */ json_object_set_new_nocheck(msg->json_msg, "client_id", json_integer(msg->client_id)); s = json_dumps(msg->json_msg, 0); - send_proc(ckp->connector, s); + async_send_proc(ckp, ckp->connector, s); free(s); free_smsg(msg); } From 5b41b1cad6c9441917c1b6657685df26bbbb0fcb Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Mon, 23 Feb 2015 12:01:40 +1100 Subject: [PATCH 197/544] Use async send proc in the generator --- src/generator.c | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/src/generator.c b/src/generator.c index 1eb7e8fb..a47dfe74 100644 --- a/src/generator.c +++ b/src/generator.c @@ -221,7 +221,7 @@ retry: cs = &alive->cs; LOGINFO("Connected to live server %s:%s", cs->url, cs->port); out: - send_proc(ckp->connector, alive ? "accept" : "reject"); + async_send_proc(ckp, ckp->connector, alive ? "accept" : "reject"); return alive; } @@ -367,7 +367,7 @@ retry: ret = submit_block(cs, buf + 12 + 64 + 1); memset(buf + 12 + 64, 0, 1); sprintf(blockmsg, "%sblock:%s", ret ? "" : "no", buf + 12); - send_proc(ckp->stratifier, blockmsg); + async_send_proc(ckp, ckp->stratifier, blockmsg); } else if (cmdmatch(buf, "checkaddr:")) { if (validate_address(cs, buf + 10)) send_unix_msg(sockd, "true"); @@ -972,7 +972,7 @@ static void send_diff(ckpool_t *ckp, proxy_instance_t *proxi) json_decref(json_msg); ASPRINTF(&buf, "diff=%s", msg); free(msg); - send_proc(ckp->stratifier, buf); + async_send_proc(ckp, ckp->stratifier, buf); free(buf); } @@ -1007,7 +1007,7 @@ static void send_notify(ckpool_t *ckp, proxy_instance_t *proxi) json_decref(json_msg); ASPRINTF(&buf, "notify=%s", msg); free(msg); - send_proc(ckp->stratifier, buf); + async_send_proc(ckp, ckp->stratifier, buf); free(buf); } @@ -1195,7 +1195,7 @@ static void send_subscribe(ckpool_t *ckp, proxy_instance_t *proxi) json_decref(json_msg); ASPRINTF(&buf, "subscribe=%s", msg); free(msg); - send_proc(ckp->stratifier, buf); + async_send_proc(ckp, ckp->stratifier, buf); free(buf); } @@ -1428,7 +1428,7 @@ static void *passthrough_recv(void *arg) if (proxy_alive(ckp, si, proxi, cs, false)) { proxi->alive = true; - send_proc(ckp->generator, "reconnect"); + async_send_proc(ckp, ckp->generator, "reconnect"); LOGWARNING("Proxy %d:%s connection established", proxi->id, proxi->si->url); } @@ -1439,13 +1439,13 @@ static void *passthrough_recv(void *arg) while (!proxy_alive(ckp, si, proxi, cs, true)) { if (proxi->alive) { proxi->alive = false; - send_proc(ckp->generator, "reconnect"); + async_send_proc(ckp, ckp->generator, "reconnect"); } sleep(5); } if (!proxi->alive) { proxi->alive = true; - send_proc(ckp->generator, "reconnect"); + async_send_proc(ckp, ckp->generator, "reconnect"); } do { @@ -1456,13 +1456,13 @@ static void *passthrough_recv(void *arg) LOGWARNING("Proxy %d:%s failed to read_socket_line in proxy_recv, attempting reconnect", proxi->id, proxi->si->url); proxi->alive = false; - send_proc(ckp->generator, "reconnect"); + async_send_proc(ckp, ckp->generator, "reconnect"); continue; } /* Simply forward the message on, as is, to the connector to * process. Possibly parse parameters sent by upstream pool * here */ - send_proc(ckp->connector, cs->buf); + async_send_proc(ckp, ckp->connector, cs->buf); } return NULL; } @@ -1496,7 +1496,7 @@ static void *proxy_recv(void *arg) if (proxy_alive(ckp, si, proxi, cs, false)) { proxi->alive = true; - send_proc(ckp->generator, "reconnect"); + async_send_proc(ckp, ckp->generator, "reconnect"); LOGWARNING("Proxy %d:%s connection established", proxi->id, proxi->si->url); } @@ -1510,7 +1510,7 @@ static void *proxy_recv(void *arg) while (!proxy_alive(ckp, si, proxi, cs, true)) { if (proxi->alive) { proxi->alive = false; - send_proc(ckp->generator, "reconnect"); + async_send_proc(ckp, ckp->generator, "reconnect"); } sleep(5); proxi->reconnect_time = time(NULL); @@ -1522,7 +1522,7 @@ static void *proxy_recv(void *arg) LOGWARNING("Proxy %d:%s recovered", proxi->id, proxi->si->url); proxi->alive = true; proxi->reconnect_time = 0; - send_proc(ckp->generator, "reconnect"); + async_send_proc(ckp, ckp->generator, "reconnect"); } now = time(NULL); @@ -1572,7 +1572,7 @@ static void *proxy_recv(void *arg) * pool is up */ proxi->reconnect = false; proxi->alive = false; - send_proc(ckp->generator, "reconnect"); + async_send_proc(ckp, ckp->generator, "reconnect"); LOGWARNING("Proxy %d:%s reconnect issue, dropping existing connection", proxi->id, proxi->si->url); Close(cs->fd); @@ -1644,7 +1644,7 @@ static proxy_instance_t *best_proxy(ckpool_t *ckp, gdata_t *gdata) break; sleep(1); } - send_proc(ckp->connector, ret ? "accept" : "reject"); + async_send_proc(ckp, ckp->connector, ret ? "accept" : "reject"); return ret; } @@ -1672,7 +1672,7 @@ reconnect: proxi->id, cs->url, cs->port); dealloc(buf); ASPRINTF(&buf, "proxy=%d", proxi->id); - send_proc(ckp->stratifier, buf); + async_send_proc(ckp, ckp->stratifier, buf); /* Send a notify for the new chosen proxy or the * stratifier won't be able to switch. */ send_notify(ckp, proxi); @@ -1857,7 +1857,7 @@ static void server_watchdog(ckpool_t *ckp, server_instance_t *cursi) break; } if (alive) - send_proc(ckp->generator, "reconnect"); + async_send_proc(ckp, ckp->generator, "reconnect"); } int generator(proc_instance_t *pi) From adec278e7cf1a97c3114d698675058b37faca6d4 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Mon, 23 Feb 2015 09:54:39 +1100 Subject: [PATCH 198/544] Create generic workqueue function and message receiving and parsing helpers --- src/ckpool.c | 72 +++++++++++++++++++++++++++++++++++++++++++++++++++- src/ckpool.h | 29 ++++++++++++++++++--- 2 files changed, 97 insertions(+), 4 deletions(-) diff --git a/src/ckpool.c b/src/ckpool.c index 484eda48..cb5849d5 100644 --- a/src/ckpool.c +++ b/src/ckpool.c @@ -134,6 +134,39 @@ static void *ckmsg_queue(void *arg) return NULL; } +/* Generic workqueue function and message receiving and parsing thread */ +static void *ckwq_queue(void *arg) +{ + ckwq_t *ckmsgq = (ckwq_t *)arg; + ckpool_t *ckp = ckmsgq->ckp; + + pthread_detach(pthread_self()); + rename_proc(ckmsgq->name); + + while (42) { + ckwqmsg_t *wqmsg; + tv_t now; + ts_t abs; + + mutex_lock(ckmsgq->lock); + tv_time(&now); + tv_to_ts(&abs, &now); + abs.tv_sec++; + if (!ckmsgq->wqmsgs) + cond_timedwait(ckmsgq->cond, ckmsgq->lock, &abs); + wqmsg = ckmsgq->wqmsgs; + if (wqmsg) + DL_DELETE(ckmsgq->wqmsgs, wqmsg); + mutex_unlock(ckmsgq->lock); + + if (!wqmsg) + continue; + wqmsg->func(ckp, wqmsg->data); + free(wqmsg); + } + return NULL; +} + ckmsgq_t *create_ckmsgq(ckpool_t *ckp, const char *name, const void *func) { ckmsgq_t *ckmsgq = ckzalloc(sizeof(ckmsgq_t)); @@ -174,6 +207,29 @@ ckmsgq_t *create_ckmsgqs(ckpool_t *ckp, const char *name, const void *func, cons return ckmsgq; } +ckwq_t *create_ckwqs(ckpool_t *ckp, const char *name, const int count) +{ + ckwq_t *ckwq = ckzalloc(sizeof(ckmsgq_t) * count); + mutex_t *lock; + pthread_cond_t *cond; + int i; + + lock = ckalloc(sizeof(mutex_t)); + cond = ckalloc(sizeof(pthread_cond_t)); + mutex_init(lock); + cond_init(cond); + + for (i = 0; i < count; i++) { + snprintf(ckwq[i].name, 15, "%.6swq%d", name, i); + ckwq[i].ckp = ckp; + ckwq[i].lock = lock; + ckwq[i].cond = cond; + create_pthread(&ckwq[i].pth, ckwq_queue, &ckwq[i]); + } + + return ckwq; +} + /* Generic function for adding messages to a ckmsgq linked list and signal the * ckmsgq parsing thread to wake up and process it. */ void ckmsgq_add(ckmsgq_t *ckmsgq, void *data) @@ -185,10 +241,24 @@ void ckmsgq_add(ckmsgq_t *ckmsgq, void *data) mutex_lock(ckmsgq->lock); ckmsgq->messages++; DL_APPEND(ckmsgq->msgs, msg); - pthread_cond_signal(ckmsgq->cond); + pthread_cond_broadcast(ckmsgq->cond); mutex_unlock(ckmsgq->lock); } +void ckwq_add(ckwq_t *ckwq, const void *func, void *data) +{ + ckwqmsg_t *wqmsg = ckalloc(sizeof(ckwqmsg_t)); + + wqmsg->func = func; + wqmsg->data = data; + + mutex_lock(ckwq->lock); + ckwq->messages++; + DL_APPEND(ckwq->wqmsgs, wqmsg); + pthread_cond_broadcast(ckwq->cond); + mutex_unlock(ckwq->lock); +} + /* Return whether there are any messages queued in the ckmsgq linked list. */ bool ckmsgq_empty(ckmsgq_t *ckmsgq) { diff --git a/src/ckpool.h b/src/ckpool.h index e0e0f947..d840528e 100644 --- a/src/ckpool.h +++ b/src/ckpool.h @@ -21,13 +21,22 @@ typedef struct ckpool_instance ckpool_t; +typedef struct ckmsg ckmsg_t; + struct ckmsg { - struct ckmsg *next; - struct ckmsg *prev; + ckmsg_t *next; + ckmsg_t *prev; void *data; }; -typedef struct ckmsg ckmsg_t; +typedef struct ckwqmsg ckwqmsg_t; + +struct ckwqmsg { + ckwqmsg_t *next; + ckwqmsg_t *prev; + void *data; + void (*func)(ckpool_t *, void *); +}; struct ckmsgq { ckpool_t *ckp; @@ -42,6 +51,18 @@ struct ckmsgq { typedef struct ckmsgq ckmsgq_t; +struct ckwq { + ckpool_t *ckp; + char name[16]; + pthread_t pth; + mutex_t *lock; + pthread_cond_t *cond; + ckwqmsg_t *wqmsgs; + int64_t messages; +}; + +typedef struct ckwq ckwq_t; + struct proc_instance { ckpool_t *ckp; unixsock_t us; @@ -201,7 +222,9 @@ struct ckpool_instance { ckmsgq_t *create_ckmsgq(ckpool_t *ckp, const char *name, const void *func); ckmsgq_t *create_ckmsgqs(ckpool_t *ckp, const char *name, const void *func, const int count); +ckwq_t *create_ckwqs(ckpool_t *ckp, const char *name, const int count); void ckmsgq_add(ckmsgq_t *ckmsgq, void *data); +void ckwq_add(ckwq_t *ckwq, const void *func, void *data); bool ckmsgq_empty(ckmsgq_t *ckmsgq); ckpool_t *global_ckp; From e40d560f573d16e76a36236506784fb598d0e922 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Mon, 23 Feb 2015 10:17:17 +1100 Subject: [PATCH 199/544] Create a pool of workqueue threads for use by the stratifier using them for share processing, stratum receiving and transaction processing Conflicts: src/stratifier.c --- src/stratifier.c | 46 +++++++++++++++++++++++++++++----------------- 1 file changed, 29 insertions(+), 17 deletions(-) diff --git a/src/stratifier.c b/src/stratifier.c index 89d2d852..1cc76e6f 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -365,12 +365,10 @@ struct stratifier_data { char lasthash[68]; char lastswaphash[68]; + ckwq_t *ckwqs; // Generic workqueues ckmsgq_t *ssends; // Stratum sends - ckmsgq_t *srecvs; // Stratum receives ckmsgq_t *ckdbq; // ckdb - ckmsgq_t *sshareq; // Stratum share sends ckmsgq_t *sauthq; // Stratum authorisations - ckmsgq_t *stxnq; // Transaction requests int64_t user_instance_id; @@ -2053,6 +2051,21 @@ static void ckmsgq_stats(ckmsgq_t *ckmsgq, const int size, json_t **val) JSON_CPACK(*val, "{si,si,si}", "count", objects, "memory", memsize, "generated", generated); } +static void ckwq_stats(ckwq_t *ckwq, const int size, json_t **val) +{ + int objects, generated; + int64_t memsize; + ckwqmsg_t *wqmsg; + + mutex_lock(ckwq->lock); + DL_COUNT(ckwq->wqmsgs, wqmsg, objects); + generated = ckwq->messages; + mutex_unlock(ckwq->lock); + + memsize = (sizeof(ckwqmsg_t) + size) * objects; + JSON_CPACK(*val, "{si,si,si}", "count", objects, "memory", memsize, "generated", generated); +} + static char *stratifier_stats(ckpool_t *ckp, sdata_t *sdata) { json_t *val = json_object(), *subval; @@ -2099,15 +2112,14 @@ static char *stratifier_stats(ckpool_t *ckp, sdata_t *sdata) ckmsgq_stats(sdata->ssends, sizeof(smsg_t), &subval); json_set_object(val, "ssends", subval); + /* Don't know exactly how big the string is so just count the pointer for now */ - ckmsgq_stats(sdata->srecvs, sizeof(char *), &subval); - json_set_object(val, "srecvs", subval); + ckwq_stats(sdata->ckwqs, sizeof(char *) + sizeof(void *), &subval); + json_set_object(val, "ckwqs", subval); if (!CKP_STANDALONE(ckp)) { ckmsgq_stats(sdata->ckdbq, sizeof(char *), &subval); json_set_object(val, "ckdbq", subval); } - ckmsgq_stats(sdata->stxnq, sizeof(json_params_t), &subval); - json_set_object(val, "stxnq", subval); buf = json_dumps(val, JSON_NO_UTF8 | JSON_PRESERVE_ORDER); json_decref(val); @@ -2226,6 +2238,7 @@ static void reconnect_client_id(sdata_t *sdata, const int64_t client_id) lazy_reconnect_client(sdata, client); dec_instance_ref(sdata, client); } +static void srecv_process(ckpool_t *ckp, char *buf); static int stratum_loop(ckpool_t *ckp, proc_instance_t *pi) { @@ -2281,7 +2294,7 @@ retry: * connector so look for this first. The srecv_process frees * the buf heap ram */ Close(sockd); - ckmsgq_add(sdata->srecvs, buf); + ckwq_add(sdata->ckwqs, &srecv_process, buf); buf = NULL; goto retry; } @@ -3891,6 +3904,9 @@ static void suggest_diff(stratum_instance_t *client, const char *method, const j stratum_send_diff(sdata, client); } +static void sshare_process(ckpool_t *ckp, json_params_t *jp); +static void send_transactions(ckpool_t *ckp, json_params_t *jp); + /* Enter with client holding ref count */ static void parse_method(sdata_t *sdata, stratum_instance_t *client, const int64_t client_id, json_t *id_val, json_t *method_val, json_t *params_val, const char *address) @@ -3904,7 +3920,7 @@ static void parse_method(sdata_t *sdata, stratum_instance_t *client, const int64 if (likely(cmdmatch(method, "mining.submit") && client->authorised)) { json_params_t *jp = create_json_params(client_id, method_val, params_val, id_val, address); - ckmsgq_add(sdata->sshareq, jp); + ckwq_add(sdata->ckwqs, &sshare_process, jp); return; } @@ -3983,7 +3999,7 @@ static void parse_method(sdata_t *sdata, stratum_instance_t *client, const int64 if (cmdmatch(method, "mining.get")) { json_params_t *jp = create_json_params(client_id, method_val, params_val, id_val, address); - ckmsgq_add(sdata->stxnq, jp); + ckwq_add(sdata->ckwqs, &send_transactions, jp); return; } /* Unhandled message here */ @@ -5005,14 +5021,10 @@ int stratifier(proc_instance_t *pi) mutex_init(&sdata->ckdb_lock); sdata->ssends = create_ckmsgq(ckp, "ssender", &ssend_process); - /* Create half as many share processing threads as there are CPUs */ - threads = sysconf(_SC_NPROCESSORS_ONLN) / 2 ? : 1; - sdata->sshareq = create_ckmsgqs(ckp, "sprocessor", &sshare_process, threads); - /* Create 1/4 as many stratum processing threads as there are CPUs */ - threads = threads / 2 ? : 1; - sdata->srecvs = create_ckmsgqs(ckp, "sreceiver", &srecv_process, threads); + /* Create as many generic workqueue threads as there are CPUs */ + threads = sysconf(_SC_NPROCESSORS_ONLN); + sdata->ckwqs = create_ckwqs(ckp, "strat", threads); sdata->sauthq = create_ckmsgq(ckp, "authoriser", &sauth_process); - sdata->stxnq = create_ckmsgq(ckp, "stxnq", &send_transactions); if (!CKP_STANDALONE(ckp)) { sdata->ckdbq = create_ckmsgq(ckp, "ckdbqueue", &ckdbq_process); create_pthread(&pth_heartbeat, ckdb_heartbeat, ckp); From 6c75760502afbbfffb4a3d11f09e10439511d4d2 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Mon, 23 Feb 2015 10:27:22 +1100 Subject: [PATCH 200/544] Use the generic workqueues for do_update --- src/stratifier.c | 32 ++++++++------------------------ 1 file changed, 8 insertions(+), 24 deletions(-) diff --git a/src/stratifier.c b/src/stratifier.c index 1cc76e6f..501aabd4 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -843,33 +843,21 @@ static void send_generator(const ckpool_t *ckp, const char *msg, const int prio) sdata->gen_priority = 0; } -struct update_req { - pthread_t *pth; - ckpool_t *ckp; - int prio; -}; - static void broadcast_ping(sdata_t *sdata); /* This function assumes it will only receive a valid json gbt base template * since checking should have been done earlier, and creates the base template * for generating work templates. */ -static void *do_update(void *arg) +static void do_update(ckpool_t *ckp, int *prio) { - struct update_req *ur = (struct update_req *)arg; - ckpool_t *ckp = ur->ckp; sdata_t *sdata = ckp->data; bool new_block = false; - int prio = ur->prio; bool ret = false; workbase_t *wb; json_t *val; char *buf; - pthread_detach(pthread_self()); - rename_proc("updater"); - - buf = send_recv_generator(ckp, "getbase", prio); + buf = send_recv_generator(ckp, "getbase", *prio); if (unlikely(!buf)) { LOGNOTICE("Get base in update_base delayed due to higher priority request"); goto out; @@ -931,21 +919,17 @@ out: LOGINFO("Broadcast ping due to failed stratum base update"); broadcast_ping(sdata); } - dealloc(buf); - free(ur->pth); - free(ur); - return NULL; + free(buf); + free(prio); } static void update_base(ckpool_t *ckp, const int prio) { - struct update_req *ur = ckalloc(sizeof(struct update_req)); - pthread_t *pth = ckalloc(sizeof(pthread_t)); + int *pprio = ckalloc(sizeof(int)); + sdata_t *sdata = ckp->data; - ur->pth = pth; - ur->ckp = ckp; - ur->prio = prio; - create_pthread(pth, do_update, ur); + *pprio = prio; + ckwq_add(sdata->ckwqs, &do_update, pprio); } static void __kill_instance(stratum_instance_t *client) From fc34318ea8469d8600922b6ce1c584cb3d11eb6d Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Mon, 23 Feb 2015 11:11:00 +1100 Subject: [PATCH 201/544] Revert to synchronous proc messages in anticipation of new async functions --- src/ckpool.c | 55 ++++++++-------------------------------------------- 1 file changed, 8 insertions(+), 47 deletions(-) diff --git a/src/ckpool.c b/src/ckpool.c index cb5849d5..893c026a 100644 --- a/src/ckpool.c +++ b/src/ckpool.c @@ -575,32 +575,14 @@ out: static void childsighandler(const int sig); -struct proc_message { - proc_instance_t *pi; - char *msg; - const char *file; - const char *func; - int line; -}; - -/* Send all one way messages asynchronously so we can wait till the receiving - * end closes the socket to ensure all messages are received but no deadlocks - * can occur with 2 processes waiting for each other's socket closure. */ -void *async_send_proc(void *arg) +/* Send a single message to a process instance when there will be no response, + * closing the socket immediately. */ +void _send_proc(proc_instance_t *pi, const char *msg, const char *file, const char *func, const int line) { - struct proc_message *pm = (struct proc_message *)arg; - proc_instance_t *pi = pm->pi; - char *msg = pm->msg; - const char *file = pm->file; - const char *func = pm->func; - int line = pm->line; - char *path = pi->us.path; bool ret = false; int sockd; - pthread_detach(pthread_self()); - if (unlikely(!path || !strlen(path))) { LOGERR("Attempted to send message %s to null path in send_proc", msg ? msg : ""); goto out; @@ -611,16 +593,14 @@ void *async_send_proc(void *arg) } /* At startup the pid fields are not set up before some processes are * forked so they never inherit them. */ - if (unlikely(!pi->pid)) { + if (unlikely(!pi->pid)) pi->pid = get_proc_pid(pi); - if (!pi->pid) { - LOGALERT("Attempting to send message %s to non existent process %s", msg, pi->processname); - goto out_nofail; - } + if (!pi->pid) { + LOGALERT("Attempting to send message %s to non existent process %s", msg, pi->processname); + return; } if (unlikely(kill_pid(pi->pid, 0))) { - LOGALERT("Attempting to send message %s to non existent process %s pid %d", - msg, pi->processname, pi->pid); + LOGALERT("Attempting to send message %s to non existent process %s", msg, pi->processname); goto out; } sockd = open_unix_client(path); @@ -640,25 +620,6 @@ out: LOGERR("Failure in send_proc from %s %s:%d", file, func, line); childsighandler(15); } -out_nofail: - free(msg); - free(pm); - return NULL; -} - -/* Send a single message to a process instance when there will be no response, - * closing the socket immediately. */ -void _send_proc(proc_instance_t *pi, const char *msg, const char *file, const char *func, const int line) -{ - struct proc_message *pm = ckalloc(sizeof(struct proc_message)); - pthread_t pth; - - pm->pi = pi; - pm->msg = strdup(msg); - pm->file = file; - pm->func = func; - pm->line = line; - create_pthread(&pth, async_send_proc, pm); } /* Send a single message to a process instance and retrieve the response, then From 97e2b3179b1944a6aa470be5b951681e85d86120 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Mon, 23 Feb 2015 11:15:44 +1100 Subject: [PATCH 202/544] Keep track of per process ckwqs in the ckpool structure --- src/ckpool.h | 2 ++ src/stratifier.c | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/ckpool.h b/src/ckpool.h index d840528e..a36be659 100644 --- a/src/ckpool.h +++ b/src/ckpool.h @@ -210,6 +210,8 @@ struct ckpool_instance { /* Private data for each process */ void *data; + /* Private generic workqueues if this process has them */ + ckwq_t *ckwqs; }; #ifdef USE_CKDB diff --git a/src/stratifier.c b/src/stratifier.c index 501aabd4..324ddc2a 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -5007,7 +5007,7 @@ int stratifier(proc_instance_t *pi) sdata->ssends = create_ckmsgq(ckp, "ssender", &ssend_process); /* Create as many generic workqueue threads as there are CPUs */ threads = sysconf(_SC_NPROCESSORS_ONLN); - sdata->ckwqs = create_ckwqs(ckp, "strat", threads); + ckp->ckwqs = sdata->ckwqs = create_ckwqs(ckp, "strat", threads); sdata->sauthq = create_ckmsgq(ckp, "authoriser", &sauth_process); if (!CKP_STANDALONE(ckp)) { sdata->ckdbq = create_ckmsgq(ckp, "ckdbqueue", &ckdbq_process); From 9378f77fd999c372a0184f48b6347b21570d7c49 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Mon, 23 Feb 2015 11:26:10 +1100 Subject: [PATCH 203/544] Add an asynchronous send proc function which uses each process' generic workqueues if they exist --- src/ckpool.c | 35 +++++++++++++++++++++++++++++++++++ src/ckpool.h | 2 ++ 2 files changed, 37 insertions(+) diff --git a/src/ckpool.c b/src/ckpool.c index 893c026a..cf445281 100644 --- a/src/ckpool.c +++ b/src/ckpool.c @@ -622,6 +622,41 @@ out: } } +struct proc_message { + proc_instance_t *pi; + char *msg; + const char *file; + const char *func; + int line; +}; + +static void asp_send(ckpool_t __maybe_unused *ckp, struct proc_message *pm) +{ + _send_proc(pm->pi, pm->msg, pm->file, pm->func, pm->line); + free(pm->msg); + free(pm); +} + +/* Fore sending asynchronous messages to another process, the sending process + * must have ckwqs of its own, referenced in the ckpool structure */ +void _async_send_proc(ckpool_t *ckp, proc_instance_t *pi, const char *msg, const char *file, const char *func, const int line) +{ + struct proc_message *pm; + + if (unlikely(!ckp->ckwqs)) { + LOGALERT("Workqueues not set up in async_send_proc!"); + _send_proc(pi, msg, file, func, line); + return; + } + pm = ckzalloc(sizeof(struct proc_message)); + pm->pi = pi; + pm->msg = strdup(msg); + pm->file = file; + pm->func = func; + pm->line = line; + ckwq_add(ckp->ckwqs, &asp_send, pm); +} + /* Send a single message to a process instance and retrieve the response, then * close the socket. */ char *_send_recv_proc(proc_instance_t *pi, const char *msg, const char *file, const char *func, const int line) diff --git a/src/ckpool.h b/src/ckpool.h index a36be659..e88aa71e 100644 --- a/src/ckpool.h +++ b/src/ckpool.h @@ -236,6 +236,8 @@ void empty_buffer(connsock_t *cs); int read_socket_line(connsock_t *cs, const int timeout); void _send_proc(proc_instance_t *pi, const char *msg, const char *file, const char *func, const int line); #define send_proc(pi, msg) _send_proc(pi, msg, __FILE__, __func__, __LINE__) +void _async_send_proc(ckpool_t *ckp, proc_instance_t *pi, const char *msg, const char *file, const char *func, const int line); +#define async_send_proc(ckp, pi, msg) _async_send_proc(ckp, pi, msg, __FILE__, __func__, __LINE__) char *_send_recv_proc(proc_instance_t *pi, const char *msg, const char *file, const char *func, const int line); #define send_recv_proc(pi, msg) _send_recv_proc(pi, msg, __FILE__, __func__, __LINE__) char *_send_recv_ckdb(const ckpool_t *ckp, const char *msg, const char *file, const char *func, const int line); From 8708b36b8da77a3465d8a8a96d94fb3e9275a6ee Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Mon, 23 Feb 2015 11:40:02 +1100 Subject: [PATCH 204/544] Use async send proc in the connector --- src/connector.c | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/connector.c b/src/connector.c index d79d441a..6c0bf43b 100644 --- a/src/connector.c +++ b/src/connector.c @@ -97,6 +97,8 @@ struct connector_data { /* For protecting the pending sends list */ mutex_t sender_lock; pthread_cond_t sender_cond; + + ckwq_t *ckwqs; }; typedef struct connector_data cdata_t; @@ -242,7 +244,7 @@ static void stratifier_drop_client(ckpool_t *ckp, int64_t id) char buf[256]; sprintf(buf, "dropclient=%"PRId64, id); - send_proc(ckp->stratifier, buf); + async_send_proc(ckp, ckp->stratifier, buf); } /* Invalidate this instance. Remove them from the hashtables we look up @@ -360,9 +362,9 @@ reparse: * filtered by the stratifier. */ if (likely(client->fd != -1)) { if (ckp->passthrough) - send_proc(ckp->generator, s); + async_send_proc(ckp, ckp->generator, s); else - send_proc(ckp->stratifier, s); + async_send_proc(ckp, ckp->stratifier, s); } free(s); @@ -846,6 +848,8 @@ int connector(proc_instance_t *pi) LOGWARNING("%s connector starting", ckp->name); ckp->data = cdata; cdata->ckp = ckp; + /* Connector only requires one work queue */ + ckp->ckwqs = cdata->ckwqs = create_ckwqs(ckp, "conn", 1); if (!ckp->serverurls) cdata->serverfd = ckalloc(sizeof(int *)); From 8291489a5b3cb4aacfd8af6c58adf67b390698f7 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Mon, 23 Feb 2015 12:11:13 +1100 Subject: [PATCH 205/544] Use async send proc in the stratifier --- src/stratifier.c | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/src/stratifier.c b/src/stratifier.c index 324ddc2a..227feb4b 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -828,7 +828,7 @@ static char *send_recv_generator(ckpool_t *ckp, const char *msg, const int prio) return buf; } -static void send_generator(const ckpool_t *ckp, const char *msg, const int prio) +static void send_generator(ckpool_t *ckp, const char *msg, const int prio) { sdata_t *sdata = ckp->data; bool set; @@ -838,7 +838,7 @@ static void send_generator(const ckpool_t *ckp, const char *msg, const int prio) set = true; } else set = false; - send_proc(ckp->generator, msg); + async_send_proc(ckp, ckp->generator, msg); if (set) sdata->gen_priority = 0; } @@ -963,7 +963,7 @@ static void connector_drop_client(ckpool_t *ckp, const int64_t id) LOGDEBUG("Stratifier requesting connector drop client %"PRId64, id); snprintf(buf, 255, "dropclient=%"PRId64, id); - send_proc(ckp->connector, buf); + async_send_proc(ckp, ckp->connector, buf); } static void drop_allclients(ckpool_t *ckp) @@ -1009,12 +1009,10 @@ static sdata_t *duplicate_sdata(const sdata_t *sdata) memcpy(dsdata->donkeytxnbin, sdata->donkeytxnbin, 25); /* Use the same work queues for all subproxies */ + dsdata->ckwqs = sdata->ckwqs; dsdata->ssends = sdata->ssends; - dsdata->srecvs = sdata->srecvs; dsdata->ckdbq = sdata->ckdbq; - dsdata->sshareq = sdata->sshareq; dsdata->sauthq = sdata->sauthq; - dsdata->stxnq = sdata->stxnq; /* Give the sbuproxy its own workbase list and lock */ cklock_init(&dsdata->workbase_lock); @@ -1138,7 +1136,7 @@ out_unlock: static void reconnect_client(sdata_t *sdata, stratum_instance_t *client); -static void generator_recruit(const ckpool_t *ckp) +static void generator_recruit(ckpool_t *ckp) { LOGINFO("Stratifer requesting more proxies from generator"); send_generator(ckp, "recruit", GEN_PRIORITY); @@ -2487,7 +2485,7 @@ static void stratum_send_message(sdata_t *sdata, const stratum_instance_t *clien * in proxy mode where we find a subproxy based on the current proxy with room * for more clients. Signal the generator to recruit more subproxies if we are * running out of room. */ -static sdata_t *select_sdata(const ckpool_t *ckp, sdata_t *ckp_sdata) +static sdata_t *select_sdata(ckpool_t *ckp, sdata_t *ckp_sdata) { proxy_t *current, *proxy, *subproxy, *best = NULL, *tmp, *tmpsub; int best_id, best_subid = 0; @@ -3895,6 +3893,7 @@ static void send_transactions(ckpool_t *ckp, json_params_t *jp); static void parse_method(sdata_t *sdata, stratum_instance_t *client, const int64_t client_id, json_t *id_val, json_t *method_val, json_t *params_val, const char *address) { + ckpool_t *ckp = client->ckp; const char *method; /* Random broken clients send something not an integer as the id so we @@ -3941,14 +3940,14 @@ static void parse_method(sdata_t *sdata, stratum_instance_t *client, const int64 * to it since it's unauthorised. Set the flag just in case. */ client->authorised = false; snprintf(buf, 255, "passthrough=%"PRId64, client_id); - send_proc(client->ckp->connector, buf); + async_send_proc(ckp, ckp->connector, buf); return; } /* We should only accept subscribed requests from here on */ if (!client->subscribed) { LOGINFO("Dropping unsubscribed client %"PRId64, client_id); - connector_drop_client(client->ckp, client_id); + connector_drop_client(ckp, client_id); return; } @@ -3970,7 +3969,7 @@ static void parse_method(sdata_t *sdata, stratum_instance_t *client, const int64 * stratifier process to restart since it will have lost all * the stratum instance data. Clients will just reconnect. */ LOGINFO("Dropping unauthorised client %"PRId64, client_id); - connector_drop_client(client->ckp, client_id); + connector_drop_client(ckp, client_id); return; } @@ -4144,7 +4143,7 @@ static void ssend_process(ckpool_t *ckp, smsg_t *msg) * connector process to be delivered */ json_object_set_new_nocheck(msg->json_msg, "client_id", json_integer(msg->client_id)); s = json_dumps(msg->json_msg, 0); - send_proc(ckp->connector, s); + async_send_proc(ckp, ckp->connector, s); free(s); free_smsg(msg); } From 3f103ef67e8428e7737a99e85c0591c29b894fe5 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Mon, 23 Feb 2015 12:12:02 +1100 Subject: [PATCH 206/544] Use async send proc in the generator --- src/generator.c | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/src/generator.c b/src/generator.c index 62e4ed89..fe907444 100644 --- a/src/generator.c +++ b/src/generator.c @@ -235,7 +235,7 @@ retry: cs = &alive->cs; LOGINFO("Connected to live server %s:%s", cs->url, cs->port); out: - send_proc(ckp->connector, alive ? "accept" : "reject"); + async_send_proc(ckp, ckp->connector, alive ? "accept" : "reject"); return alive; } @@ -381,7 +381,7 @@ retry: ret = submit_block(cs, buf + 12 + 64 + 1); memset(buf + 12 + 64, 0, 1); sprintf(blockmsg, "%sblock:%s", ret ? "" : "no", buf + 12); - send_proc(ckp->stratifier, blockmsg); + async_send_proc(ckp, ckp->stratifier, blockmsg); } else if (cmdmatch(buf, "checkaddr:")) { if (validate_address(cs, buf + 10)) send_unix_msg(sockd, "true"); @@ -941,7 +941,7 @@ static void send_stratifier_deadproxy(ckpool_t *ckp, const int id, const int sub char buf[256]; sprintf(buf, "deadproxy=%d:%d", id, subid); - send_proc(ckp->stratifier, buf); + async_send_proc(ckp, ckp->stratifier, buf); } /* Remove the subproxy from the proxi list and put it on the dead list. @@ -1088,7 +1088,7 @@ static void send_diff(ckpool_t *ckp, proxy_instance_t *proxi) json_decref(json_msg); ASPRINTF(&buf, "diff=%s", msg); free(msg); - send_proc(ckp->stratifier, buf); + async_send_proc(ckp, ckp->stratifier, buf); free(buf); } @@ -1116,7 +1116,7 @@ static void send_notify(ckpool_t *ckp, proxy_instance_t *proxi, notify_instance_ json_decref(json_msg); ASPRINTF(&buf, "notify=%s", msg); free(msg); - send_proc(ckp->stratifier, buf); + async_send_proc(ckp, ckp->stratifier, buf); free(buf); /* Send diff now as stratifier will not accept diff till it has a @@ -1306,7 +1306,7 @@ static void send_subscribe(ckpool_t *ckp, proxy_instance_t *proxi) json_decref(json_msg); ASPRINTF(&buf, "subscribe=%s", msg); free(msg); - send_proc(ckp->stratifier, buf); + async_send_proc(ckp, ckp->stratifier, buf); free(buf); } @@ -1352,7 +1352,7 @@ static void stratifier_reconnect_client(ckpool_t *ckp, int64_t id) char buf[256]; sprintf(buf, "reconnclient=%"PRId64, id); - send_proc(ckp->stratifier, buf); + async_send_proc(ckp, ckp->stratifier, buf); } static void submit_share(gdata_t *gdata, json_t *val) @@ -1721,9 +1721,9 @@ static void reconnect_proxy(proxy_instance_t *proxi) create_pthread(&pth, proxy_reconnect, proxi); } -static void reconnect_generator(const ckpool_t *ckp) +static void reconnect_generator(ckpool_t *ckp) { - send_proc(ckp->generator, "reconnect"); + async_send_proc(ckp, ckp->generator, "reconnect"); } /* For receiving messages from an upstream pool to pass downstream. Responsible @@ -1780,7 +1780,7 @@ static void *passthrough_recv(void *arg) /* Simply forward the message on, as is, to the connector to * process. Possibly parse parameters sent by upstream pool * here */ - send_proc(ckp->connector, cs->buf); + async_send_proc(ckp, ckp->connector, cs->buf); } return NULL; } @@ -1985,10 +1985,10 @@ static proxy_instance_t *wait_best_proxy(ckpool_t *ckp, gdata_t *gdata) if (ret) break; - send_proc(ckp->connector, "reject"); + async_send_proc(ckp, ckp->connector, "reject"); sleep(1); } - send_proc(ckp->connector, ret ? "accept" : "reject"); + async_send_proc(ckp, ckp->connector, ret ? "accept" : "reject"); return ret; } @@ -2017,7 +2017,7 @@ reconnect: proxi->id, cs->url, cs->port); dealloc(buf); ASPRINTF(&buf, "proxy=%d", proxi->id); - send_proc(ckp->stratifier, buf); + async_send_proc(ckp, ckp->stratifier, buf); } } retry: From 23aa6a623af2990f436fbbd1507b9d8cd59d05b5 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Mon, 23 Feb 2015 12:18:42 +1100 Subject: [PATCH 207/544] Create two workqueues for the generator --- src/generator.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/generator.c b/src/generator.c index fe907444..bcf7ab78 100644 --- a/src/generator.c +++ b/src/generator.c @@ -146,6 +146,7 @@ struct generator_data { proxy_instance_t *dead_proxies; /* Disabled proxies */ int proxy_notify_id; // Globally increasing notify id ckmsgq_t *srvchk; // Server check message queue + ckwq_t *ckwqs; }; typedef struct generator_data gdata_t; @@ -2207,6 +2208,7 @@ int generator(proc_instance_t *pi) gdata = ckzalloc(sizeof(gdata_t)); ckp->data = gdata; gdata->ckp = ckp; + ckp->ckwqs = gdata->ckwqs = create_ckwqs(ckp, "gen", 2); if (ckp->proxy) { char *buf = NULL; From 0ecd5b5098c076d57f58f46df963b946e87eb063 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Mon, 23 Feb 2015 12:22:21 +1100 Subject: [PATCH 208/544] Use generic workqueues for the recruit subproxy function --- src/generator.c | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/src/generator.c b/src/generator.c index bcf7ab78..851d8697 100644 --- a/src/generator.c +++ b/src/generator.c @@ -1678,13 +1678,10 @@ static proxy_instance_t *create_subproxy(gdata_t *gdata, proxy_instance_t *proxi return subproxy; } -static void *proxy_recruit(void *arg) +static void proxy_recruit(ckpool_t *ckp, proxy_instance_t *parent) { - proxy_instance_t *proxy, *parent = (proxy_instance_t *)arg; - ckpool_t *ckp = parent->ckp; gdata_t *gdata = ckp->data; - - pthread_detach(pthread_self()); + proxy_instance_t *proxy; proxy = create_subproxy(gdata, parent); if (!proxy_alive(ckp, proxy->si, proxy, proxy->cs, false, parent->epfd)) { @@ -1692,14 +1689,11 @@ static void *proxy_recruit(void *arg) store_proxy(gdata, proxy); } else add_subproxy(parent, proxy); - return NULL; } -static void recruit_subproxy(proxy_instance_t *proxi) +static void recruit_subproxy(gdata_t *gdata, proxy_instance_t *proxi) { - pthread_t pth; - - create_pthread(&pth, proxy_recruit, proxi); + ckwq_add(gdata->ckwqs, &proxy_recruit, proxi); } static void *proxy_reconnect(void *arg) @@ -1920,7 +1914,7 @@ static void *proxy_recv(void *arg) subproxy->id, subproxy->si->url); break; } else - recruit_subproxy(proxi); + recruit_subproxy(gdata, proxi); } continue; } @@ -2058,7 +2052,7 @@ retry: LOGDEBUG("Proxy received ping request"); send_unix_msg(sockd, "pong"); } else if (cmdmatch(buf, "recruit")) { - recruit_subproxy(proxi); + recruit_subproxy(gdata, proxi); } else if (cmdmatch(buf, "dropproxy")) { drop_proxy(gdata, buf); } else if (ckp->passthrough) { From e6ce49d2effd23e0a4fde24d55444548b03a21ea Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Mon, 23 Feb 2015 12:24:07 +1100 Subject: [PATCH 209/544] Use generic workqueues for the proxy reconnect function --- src/generator.c | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/src/generator.c b/src/generator.c index 851d8697..f5a6f4c5 100644 --- a/src/generator.c +++ b/src/generator.c @@ -1696,24 +1696,18 @@ static void recruit_subproxy(gdata_t *gdata, proxy_instance_t *proxi) ckwq_add(gdata->ckwqs, &proxy_recruit, proxi); } -static void *proxy_reconnect(void *arg) +static void proxy_reconnect(ckpool_t *ckp, proxy_instance_t *proxy) { - proxy_instance_t *proxy = (proxy_instance_t *)arg; server_instance_t *si = proxy->si; connsock_t *cs = proxy->cs; - ckpool_t *ckp = proxy->ckp; - pthread_detach(pthread_self()); proxy_alive(ckp, si, proxy, cs, true, proxy->epfd); - return NULL; } /* For reconnecting the parent proxy instance async */ -static void reconnect_proxy(proxy_instance_t *proxi) +static void reconnect_proxy(gdata_t *gdata, proxy_instance_t *proxi) { - pthread_t pth; - - create_pthread(&pth, proxy_reconnect, proxi); + ckwq_add(gdata->ckwqs, &proxy_reconnect, proxi); } static void reconnect_generator(ckpool_t *ckp) @@ -1845,7 +1839,7 @@ static void *proxy_recv(void *arg) if (!proxi->alive) { while (!subproxies_alive(proxi)) { - reconnect_proxy(proxi); + reconnect_proxy(gdata, proxi); if (alive) { LOGWARNING("Proxy %d:%s failed, attempting reconnect", proxi->id, proxi->si->url); From 6887f778e9e88238ed34d8addfe190410dc14d3f Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Mon, 23 Feb 2015 12:42:25 +1100 Subject: [PATCH 210/544] Send reconnect to all clients that have been asked to reconnect already and don't unset the bool to be able to trim them --- src/stratifier.c | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/src/stratifier.c b/src/stratifier.c index 227feb4b..9d332f11 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -1160,11 +1160,12 @@ static void reconnect_clients(sdata_t *sdata) HASH_ITER(hh, sdata->stratum_instances, client, tmpclient) { if (client->proxyid == proxy->id && client->notify_id == proxy->parent->notify_id) continue; - if (client->reconnect) - continue; headroom--; reconnects++; - client->reconnect = true; + if (client->reconnect) + reconnect_client(sdata, client); + else + client->reconnect = true; } ck_runlock(&sdata->instance_lock); @@ -2122,7 +2123,6 @@ static void reconnect_client(sdata_t *sdata, stratum_instance_t *client) client->dropped = true; return; } - client->reconnect = false; client->reconnect_request = time(NULL); JSON_CPACK(json_msg, "{sosss[]}", "id", json_null(), "method", "client.reconnect", "params"); @@ -2170,11 +2170,12 @@ static void dead_proxy(sdata_t *sdata, const char *buf) HASH_ITER(hh, sdata->stratum_instances, client, tmp) { if (client->proxyid != id || client->subproxyid != subid) continue; - if (client->reconnect) - continue; headroom--; reconnects++; - client->reconnect = true; + if (client->reconnect) + reconnect_client(sdata, client); + else + client->reconnect = true; } ck_runlock(&sdata->instance_lock); From 352c8a4eb72e6f0f2b1009bd1d65f98571ee037c Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Mon, 23 Feb 2015 13:37:40 +1100 Subject: [PATCH 211/544] Set client start time when the instance is added --- src/stratifier.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/stratifier.c b/src/stratifier.c index 9d332f11..1c10ce5f 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -1655,6 +1655,7 @@ static stratum_instance_t *__stratum_add_instance(ckpool_t *ckp, const int64_t i stratum_instance_t *client = ckzalloc(sizeof(stratum_instance_t)); sdata_t *sdata = ckp->data; + client->start_time = time(NULL); sdata->stratum_generated++; client->id = id; client->server = server; @@ -2987,7 +2988,6 @@ static json_t *parse_authorise(stratum_instance_t *client, const json_t *params_ bool ret = false; const char *buf; int arr_size; - ts_t now; if (unlikely(!json_is_array(params_val))) { *err_val = json_string("params not an array"); @@ -3021,8 +3021,6 @@ static json_t *parse_authorise(stratum_instance_t *client, const json_t *params_ } user = generate_user(ckp, client, buf); client->user_id = user->id; - ts_realtime(&now); - client->start_time = now.tv_sec; strcpy(client->address, address); /* NOTE workername is NULL prior to this so should not be used in code * till after this point */ From 6b911d71d664f52faa08d30eb6294386d7f0c7a6 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Mon, 23 Feb 2015 13:39:11 +1100 Subject: [PATCH 212/544] Send stratifier generic dead proxy message on submit share fail to drop all clients --- src/generator.c | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/generator.c b/src/generator.c index f5a6f4c5..ce0a574c 100644 --- a/src/generator.c +++ b/src/generator.c @@ -1369,23 +1369,24 @@ static void submit_share(gdata_t *gdata, json_t *val) * proxy it's bound to is not functional */ json_get_int64(&client_id, val, "client_id"); json_get_int(&id, val, "proxy"); + json_get_int(&subid, val, "subproxy"); + proxy = proxy_by_id(gdata, id); if (unlikely(!proxy)) { LOGWARNING("Failed to find proxy %d to send share to", id); - stratifier_reconnect_client(ckp, client_id); + send_stratifier_deadproxy(ckp, id, subid); return json_decref(val); } - json_get_int(&subid, val, "subproxy"); proxi = subproxy_by_id(proxy, subid); if (unlikely(!proxi)) { LOGNOTICE("Failed to find proxy %d:%d to send share to", id, subid); - stratifier_reconnect_client(ckp, client_id); + send_stratifier_deadproxy(ckp, id, subid); return json_decref(val); } if (!proxi->alive) { LOGNOTICE("Client %"PRId64" attempting to send shares to dead proxy %d:%d, dropping", client_id, id, subid); - stratifier_reconnect_client(ckp, client_id); + send_stratifier_deadproxy(ckp, id, subid); return json_decref(val); } From 41388a6e8c2a7b491e279a10d18a181438fd82e9 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Mon, 23 Feb 2015 14:57:59 +1100 Subject: [PATCH 213/544] Add proxy details to notify logging --- src/generator.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/generator.c b/src/generator.c index ce0a574c..b2c3465b 100644 --- a/src/generator.c +++ b/src/generator.c @@ -668,8 +668,8 @@ retry: clients_per_proxy); } - LOGINFO("Found notify with enonce %s nonce2len %d", proxi->enonce1, - proxi->nonce2len); + LOGINFO("Found notify for proxy %d:%d with enonce %s nonce2len %d", proxi->id, + proxi->subid, proxi->enonce1, proxi->nonce2len); ret = true; out: From 4ff3972ef0991c24715537b499fcc0df5be1ccc9 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Mon, 23 Feb 2015 15:17:34 +1100 Subject: [PATCH 214/544] Look for other clients that should have been dropped in the stratifier and inform or query the connector about them --- src/connector.c | 26 ++++++++++++++++++++++++-- src/stratifier.c | 29 +++++++++++++++++++++++++++++ 2 files changed, 53 insertions(+), 2 deletions(-) diff --git a/src/connector.c b/src/connector.c index 6c0bf43b..3e568d36 100644 --- a/src/connector.c +++ b/src/connector.c @@ -239,7 +239,7 @@ static int drop_client(cdata_t *cdata, client_instance_t *client) return fd; } -static void stratifier_drop_client(ckpool_t *ckp, int64_t id) +static void stratifier_drop_client(ckpool_t *ckp, const int64_t id) { char buf[256]; @@ -601,6 +601,17 @@ static void send_client(cdata_t *cdata, int64_t id, char *buf) mutex_unlock(&cdata->sender_lock); } +static bool client_exists(cdata_t *cdata, const int64_t id) +{ + client_instance_t *client; + + ck_rlock(&cdata->lock); + HASH_FIND_I64(cdata->clients, &id, client); + ck_runlock(&cdata->lock); + + return !!client; +} + static client_instance_t *ref_client_by_id(cdata_t *cdata, int64_t id) { client_instance_t *client; @@ -774,7 +785,7 @@ retry: client_instance_t *client; ret = sscanf(buf, "dropclient=%"PRId64, &client_id64); - if (ret < 0) { + if (unlikely(ret < 0)) { LOGDEBUG("Connector failed to parse dropclient command: %s", buf); goto retry; } @@ -788,6 +799,17 @@ retry: dec_instance_ref(cdata, client); if (ret >= 0) LOGINFO("Connector dropped client id: %"PRId64, client_id); + } else if (cmdmatch(buf, "testclient")) { + ret = sscanf(buf, "testclient=%"PRId64, &client_id64); + if (unlikely(ret < 0)) { + LOGDEBUG("Connector failed to parse testclient command: %s", buf); + goto retry; + } + client_id = client_id64 & 0xffffffffll; + if (client_exists(cdata, client_id)) + goto retry; + LOGINFO("Connector detected non-existent client id: %"PRId64, client_id); + stratifier_drop_client(ckp, client_id); } else if (cmdmatch(buf, "ping")) { LOGDEBUG("Connector received ping request"); send_unix_msg(sockd, "pong"); diff --git a/src/stratifier.c b/src/stratifier.c index 1c10ce5f..e262c7e1 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -1805,6 +1805,17 @@ static void dec_worker(ckpool_t *ckp, user_instance_t *instance) mutex_unlock(&sdata->stats_lock); } +/* Ask the connector asynchronously to send us dropclient commands if this + * client no longer exists. */ +static void connector_test_client(ckpool_t *ckp, const int64_t id) +{ + char buf[256]; + + LOGDEBUG("Stratifier requesting connector test client %"PRId64, id); + snprintf(buf, 255, "testclient=%"PRId64, id); + async_send_proc(ckp, ckp->connector, buf); +} + static void drop_client(ckpool_t *ckp, sdata_t *sdata, const int64_t id) { stratum_instance_t *client, *tmp; @@ -1814,7 +1825,25 @@ static void drop_client(ckpool_t *ckp, sdata_t *sdata, const int64_t id) LOGINFO("Stratifier asked to drop client %"PRId64, id); + /* Use this locking as an opportunity to test other clients. */ ck_ilock(&sdata->instance_lock); + /* Test for clients that haven't authed in over a minute and drop them */ + HASH_ITER(hh, sdata->stratum_instances, client, tmp) { + if (client->authorised) + continue; + if (now_t > client->start_time + 60) { + client->dropped = true; + connector_drop_client(ckp, client->id); + } + } + + /* Look for clients that may have been dropped which the stratifer has + * not been informed about and ask the connector of they still exist */ + HASH_ITER(hh, sdata->stratum_instances, client, tmp) { + if (client->dropped || client->reconnect) + connector_test_client(ckp, client->id); + } + client = __instance_by_id(sdata, id); /* Upgrade to write lock */ ck_ulock(&sdata->instance_lock); From c55cf08d6a742de0512b550d3ae0c6ea47050423 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Mon, 23 Feb 2015 15:21:30 +1100 Subject: [PATCH 215/544] Check for failure to decode proxy/subproxy in update_subscribe --- src/stratifier.c | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/src/stratifier.c b/src/stratifier.c index e262c7e1..9053f134 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -1227,11 +1227,17 @@ static void update_subscribe(ckpool_t *ckp, const char *cmd) LOGDEBUG("Update subscribe: %s", buf); val = json_loads(buf, 0, NULL); if (unlikely(!val)) { - LOGWARNING("Failed to json decode subscribe response in update_subscribe"); + LOGWARNING("Failed to json decode subscribe response in update_subscribe %s", buf); + return; + } + if (unlikely(!json_get_int(&id, val, "proxy"))) { + LOGWARNING("Failed to json decode proxy value in update_subscribe %s", buf); + return; + } + if (unlikely(!json_get_int(&subid, val, "subproxy"))) { + LOGWARNING("Failed to json decode subproxy value in update_subscribe %s", buf); return; } - json_get_int(&id, val, "proxy"); - json_get_int(&subid, val, "subproxy"); if (!subid) { new_proxy(sdata, id); From fe7ea6ae89375b64750c93f393a5e6e80966894f Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Mon, 23 Feb 2015 15:22:18 +1100 Subject: [PATCH 216/544] Demote updated diff message --- src/stratifier.c | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/stratifier.c b/src/stratifier.c index 9053f134..570feec0 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -1421,10 +1421,7 @@ static void update_diff(ckpool_t *ckp, const char *cmd) json_dblcpy(&diff, val, "diff"); json_decref(val); - if (!subid) - LOGNOTICE("Got updated diff for proxy %d", id); - else - LOGINFO("Got updated diff for proxy %d:%d", id, subid); + LOGINFO("Got updated diff for proxy %d:%d", id, subid); proxy = existing_subproxy(sdata, id, subid); if (!proxy) { LOGINFO("No existing subproxy %d:%d to update diff", id, subid); From 2ff9d7f0f6acf57f41c43ff653357fb765641da0 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Mon, 23 Feb 2015 16:25:10 +1100 Subject: [PATCH 217/544] Drop clients bound to a subproxy even if the proxy no longer exists in dead_proxy --- src/stratifier.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/stratifier.c b/src/stratifier.c index 570feec0..aa543ef5 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -2196,8 +2196,6 @@ static void dead_proxy(sdata_t *sdata, const char *buf) proxy->dead = true; LOGNOTICE("Stratifier dropping clients from proxy %d:%d", id, subid); headroom = current_headroom(sdata, &proxy); - if (!proxy) - return; ck_rlock(&sdata->instance_lock); HASH_ITER(hh, sdata->stratum_instances, client, tmp) { From 2347097476911bdc404ecfe89ac6a384593c80b6 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Mon, 23 Feb 2015 16:48:31 +1100 Subject: [PATCH 218/544] Move client culling and testing to the broadcast to avoid cycles of drop/test --- src/stratifier.c | 61 ++++++++++++++++++++++++------------------------ 1 file changed, 31 insertions(+), 30 deletions(-) diff --git a/src/stratifier.c b/src/stratifier.c index aa543ef5..3bfaba2e 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -1727,14 +1727,27 @@ static inline bool client_active(stratum_instance_t *client) return (client->authorised && !client->dropped); } +/* Ask the connector asynchronously to send us dropclient commands if this + * client no longer exists. */ +static void connector_test_client(ckpool_t *ckp, const int64_t id) +{ + char buf[256]; + + LOGDEBUG("Stratifier requesting connector test client %"PRId64, id); + snprintf(buf, 255, "testclient=%"PRId64, id); + async_send_proc(ckp, ckp->connector, buf); +} + /* For creating a list of sends without locking that can then be concatenated * to the stratum_sends list. Minimises locking and avoids taking recursive * locks. Sends only to sdata bound clients (everyone in ckpool) */ static void stratum_broadcast(sdata_t *sdata, json_t *val) { - sdata_t *ckp_sdata = sdata->ckp->data; + ckpool_t *ckp = sdata->ckp; + sdata_t *ckp_sdata = ckp->data; stratum_instance_t *client, *tmp; ckmsg_t *bulk_send = NULL; + time_t now_t = time(NULL); ckmsgq_t *ssends; if (unlikely(!val)) { @@ -1742,6 +1755,7 @@ static void stratum_broadcast(sdata_t *sdata, json_t *val) return; } + /* Use this locking as an opportunity to test other clients. */ ck_rlock(&ckp_sdata->instance_lock); HASH_ITER(hh, ckp_sdata->stratum_instances, client, tmp) { ckmsg_t *client_msg; @@ -1749,6 +1763,22 @@ static void stratum_broadcast(sdata_t *sdata, json_t *val) if (sdata != ckp_sdata && client->sdata != sdata) continue; + + /* Test for clients that haven't authed in over a minute and drop them */ + if (!client->authorised) { + if (now_t > client->start_time + 60) { + client->dropped = true; + connector_drop_client(ckp, client->id); + } + continue; + } + /* Look for clients that may have been dropped which the stratifer has + * not been informed about and ask the connector of they still exist */ + if (client->dropped || client->reconnect) { + connector_test_client(ckp, client->id); + continue; + } + if (!client_active(client)) continue; client_msg = ckalloc(sizeof(ckmsg_t)); @@ -1808,17 +1838,6 @@ static void dec_worker(ckpool_t *ckp, user_instance_t *instance) mutex_unlock(&sdata->stats_lock); } -/* Ask the connector asynchronously to send us dropclient commands if this - * client no longer exists. */ -static void connector_test_client(ckpool_t *ckp, const int64_t id) -{ - char buf[256]; - - LOGDEBUG("Stratifier requesting connector test client %"PRId64, id); - snprintf(buf, 255, "testclient=%"PRId64, id); - async_send_proc(ckp, ckp->connector, buf); -} - static void drop_client(ckpool_t *ckp, sdata_t *sdata, const int64_t id) { stratum_instance_t *client, *tmp; @@ -1828,25 +1847,7 @@ static void drop_client(ckpool_t *ckp, sdata_t *sdata, const int64_t id) LOGINFO("Stratifier asked to drop client %"PRId64, id); - /* Use this locking as an opportunity to test other clients. */ ck_ilock(&sdata->instance_lock); - /* Test for clients that haven't authed in over a minute and drop them */ - HASH_ITER(hh, sdata->stratum_instances, client, tmp) { - if (client->authorised) - continue; - if (now_t > client->start_time + 60) { - client->dropped = true; - connector_drop_client(ckp, client->id); - } - } - - /* Look for clients that may have been dropped which the stratifer has - * not been informed about and ask the connector of they still exist */ - HASH_ITER(hh, sdata->stratum_instances, client, tmp) { - if (client->dropped || client->reconnect) - connector_test_client(ckp, client->id); - } - client = __instance_by_id(sdata, id); /* Upgrade to write lock */ ck_ulock(&sdata->instance_lock); From eef959f5d8b00f4dfe9178233b119ead7719be4a Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Mon, 23 Feb 2015 17:07:34 +1100 Subject: [PATCH 219/544] Fix use of ckwq variables --- src/ckpool.c | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/src/ckpool.c b/src/ckpool.c index cf445281..ff303c50 100644 --- a/src/ckpool.c +++ b/src/ckpool.c @@ -137,27 +137,27 @@ static void *ckmsg_queue(void *arg) /* Generic workqueue function and message receiving and parsing thread */ static void *ckwq_queue(void *arg) { - ckwq_t *ckmsgq = (ckwq_t *)arg; - ckpool_t *ckp = ckmsgq->ckp; + ckwq_t *ckwq = (ckwq_t *)arg; + ckpool_t *ckp = ckwq->ckp; pthread_detach(pthread_self()); - rename_proc(ckmsgq->name); + rename_proc(ckwq->name); while (42) { ckwqmsg_t *wqmsg; tv_t now; ts_t abs; - mutex_lock(ckmsgq->lock); + mutex_lock(ckwq->lock); tv_time(&now); tv_to_ts(&abs, &now); abs.tv_sec++; - if (!ckmsgq->wqmsgs) - cond_timedwait(ckmsgq->cond, ckmsgq->lock, &abs); - wqmsg = ckmsgq->wqmsgs; + if (!ckwq->wqmsgs) + cond_timedwait(ckwq->cond, ckwq->lock, &abs); + wqmsg = ckwq->wqmsgs; if (wqmsg) - DL_DELETE(ckmsgq->wqmsgs, wqmsg); - mutex_unlock(ckmsgq->lock); + DL_DELETE(ckwq->wqmsgs, wqmsg); + mutex_unlock(ckwq->lock); if (!wqmsg) continue; @@ -209,7 +209,7 @@ ckmsgq_t *create_ckmsgqs(ckpool_t *ckp, const char *name, const void *func, cons ckwq_t *create_ckwqs(ckpool_t *ckp, const char *name, const int count) { - ckwq_t *ckwq = ckzalloc(sizeof(ckmsgq_t) * count); + ckwq_t *ckwq = ckzalloc(sizeof(ckwq_t) * count); mutex_t *lock; pthread_cond_t *cond; int i; From b153be29fa416e767a8d65be9120f51003ebe480 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Mon, 23 Feb 2015 17:36:27 +1100 Subject: [PATCH 220/544] Revert "Use generic workqueues for the proxy reconnect function" This reverts commit e6ce49d2effd23e0a4fde24d55444548b03a21ea. Holds up workqueues too long. --- src/generator.c | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/src/generator.c b/src/generator.c index b2c3465b..da3a11c4 100644 --- a/src/generator.c +++ b/src/generator.c @@ -1697,18 +1697,24 @@ static void recruit_subproxy(gdata_t *gdata, proxy_instance_t *proxi) ckwq_add(gdata->ckwqs, &proxy_recruit, proxi); } -static void proxy_reconnect(ckpool_t *ckp, proxy_instance_t *proxy) +static void *proxy_reconnect(void *arg) { + proxy_instance_t *proxy = (proxy_instance_t *)arg; server_instance_t *si = proxy->si; connsock_t *cs = proxy->cs; + ckpool_t *ckp = proxy->ckp; + pthread_detach(pthread_self()); proxy_alive(ckp, si, proxy, cs, true, proxy->epfd); + return NULL; } /* For reconnecting the parent proxy instance async */ -static void reconnect_proxy(gdata_t *gdata, proxy_instance_t *proxi) +static void reconnect_proxy(proxy_instance_t *proxi) { - ckwq_add(gdata->ckwqs, &proxy_reconnect, proxi); + pthread_t pth; + + create_pthread(&pth, proxy_reconnect, proxi); } static void reconnect_generator(ckpool_t *ckp) @@ -1840,7 +1846,7 @@ static void *proxy_recv(void *arg) if (!proxi->alive) { while (!subproxies_alive(proxi)) { - reconnect_proxy(gdata, proxi); + reconnect_proxy(proxi); if (alive) { LOGWARNING("Proxy %d:%s failed, attempting reconnect", proxi->id, proxi->si->url); From 6e95cd1f669e05903320d6dd147bbe12211a66d7 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Mon, 23 Feb 2015 17:37:21 +1100 Subject: [PATCH 221/544] Revert "Use generic workqueues for the recruit subproxy function" This reverts commit 0ecd5b5098c076d57f58f46df963b946e87eb063. --- src/generator.c | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/src/generator.c b/src/generator.c index da3a11c4..08dda589 100644 --- a/src/generator.c +++ b/src/generator.c @@ -1679,10 +1679,13 @@ static proxy_instance_t *create_subproxy(gdata_t *gdata, proxy_instance_t *proxi return subproxy; } -static void proxy_recruit(ckpool_t *ckp, proxy_instance_t *parent) +static void *proxy_recruit(void *arg) { + proxy_instance_t *proxy, *parent = (proxy_instance_t *)arg; + ckpool_t *ckp = parent->ckp; gdata_t *gdata = ckp->data; - proxy_instance_t *proxy; + + pthread_detach(pthread_self()); proxy = create_subproxy(gdata, parent); if (!proxy_alive(ckp, proxy->si, proxy, proxy->cs, false, parent->epfd)) { @@ -1690,11 +1693,14 @@ static void proxy_recruit(ckpool_t *ckp, proxy_instance_t *parent) store_proxy(gdata, proxy); } else add_subproxy(parent, proxy); + return NULL; } -static void recruit_subproxy(gdata_t *gdata, proxy_instance_t *proxi) +static void recruit_subproxy(proxy_instance_t *proxi) { - ckwq_add(gdata->ckwqs, &proxy_recruit, proxi); + pthread_t pth; + + create_pthread(&pth, proxy_recruit, proxi); } static void *proxy_reconnect(void *arg) @@ -1915,7 +1921,7 @@ static void *proxy_recv(void *arg) subproxy->id, subproxy->si->url); break; } else - recruit_subproxy(gdata, proxi); + recruit_subproxy(proxi); } continue; } @@ -2053,7 +2059,7 @@ retry: LOGDEBUG("Proxy received ping request"); send_unix_msg(sockd, "pong"); } else if (cmdmatch(buf, "recruit")) { - recruit_subproxy(gdata, proxi); + recruit_subproxy(proxi); } else if (cmdmatch(buf, "dropproxy")) { drop_proxy(gdata, buf); } else if (ckp->passthrough) { From 2cc67dab43d1a141708feb0fc781c5a7d8e51274 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Mon, 23 Feb 2015 17:37:58 +1100 Subject: [PATCH 222/544] Only one workqueue will suffice in the generator when there's no conflict over resources --- src/generator.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/generator.c b/src/generator.c index 08dda589..c9fa76b9 100644 --- a/src/generator.c +++ b/src/generator.c @@ -2209,7 +2209,7 @@ int generator(proc_instance_t *pi) gdata = ckzalloc(sizeof(gdata_t)); ckp->data = gdata; gdata->ckp = ckp; - ckp->ckwqs = gdata->ckwqs = create_ckwqs(ckp, "gen", 2); + ckp->ckwqs = gdata->ckwqs = create_ckwqs(ckp, "gen", 1); if (ckp->proxy) { char *buf = NULL; From 401b18adbdd9c40e2c4e4dd370559096a55a0587 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Mon, 23 Feb 2015 17:07:34 +1100 Subject: [PATCH 223/544] Fix use of ckwq variables --- src/ckpool.c | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/src/ckpool.c b/src/ckpool.c index 90f4a98f..68cf93fd 100644 --- a/src/ckpool.c +++ b/src/ckpool.c @@ -137,27 +137,27 @@ static void *ckmsg_queue(void *arg) /* Generic workqueue function and message receiving and parsing thread */ static void *ckwq_queue(void *arg) { - ckwq_t *ckmsgq = (ckwq_t *)arg; - ckpool_t *ckp = ckmsgq->ckp; + ckwq_t *ckwq = (ckwq_t *)arg; + ckpool_t *ckp = ckwq->ckp; pthread_detach(pthread_self()); - rename_proc(ckmsgq->name); + rename_proc(ckwq->name); while (42) { ckwqmsg_t *wqmsg; tv_t now; ts_t abs; - mutex_lock(ckmsgq->lock); + mutex_lock(ckwq->lock); tv_time(&now); tv_to_ts(&abs, &now); abs.tv_sec++; - if (!ckmsgq->wqmsgs) - cond_timedwait(ckmsgq->cond, ckmsgq->lock, &abs); - wqmsg = ckmsgq->wqmsgs; + if (!ckwq->wqmsgs) + cond_timedwait(ckwq->cond, ckwq->lock, &abs); + wqmsg = ckwq->wqmsgs; if (wqmsg) - DL_DELETE(ckmsgq->wqmsgs, wqmsg); - mutex_unlock(ckmsgq->lock); + DL_DELETE(ckwq->wqmsgs, wqmsg); + mutex_unlock(ckwq->lock); if (!wqmsg) continue; @@ -209,7 +209,7 @@ ckmsgq_t *create_ckmsgqs(ckpool_t *ckp, const char *name, const void *func, cons ckwq_t *create_ckwqs(ckpool_t *ckp, const char *name, const int count) { - ckwq_t *ckwq = ckzalloc(sizeof(ckmsgq_t) * count); + ckwq_t *ckwq = ckzalloc(sizeof(ckwq_t) * count); mutex_t *lock; pthread_cond_t *cond; int i; From 4d25a197e78644ee25dae1178e103263e1b5d902 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Mon, 23 Feb 2015 18:04:45 +1100 Subject: [PATCH 224/544] Only have one thread trying to reconnect a proxy at a time --- src/generator.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/generator.c b/src/generator.c index c9fa76b9..19dcb47f 100644 --- a/src/generator.c +++ b/src/generator.c @@ -107,6 +107,7 @@ struct proxy_instance { bool disabled; /* Subproxy no longer to be used */ bool reconnect; /* We need to drop and reconnect */ + bool reconnecting; /* Testing in progress */ bool alive; mutex_t notify_lock; @@ -1712,6 +1713,7 @@ static void *proxy_reconnect(void *arg) pthread_detach(pthread_self()); proxy_alive(ckp, si, proxy, cs, true, proxy->epfd); + proxy->reconnecting = false; return NULL; } @@ -1720,6 +1722,9 @@ static void reconnect_proxy(proxy_instance_t *proxi) { pthread_t pth; + if (proxi->reconnecting) + return; + proxi->reconnecting = true; create_pthread(&pth, proxy_reconnect, proxi); } From f78f72dc4f87113c78c78209a4506bc454a36abe Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Mon, 23 Feb 2015 18:25:30 +1100 Subject: [PATCH 225/544] Make a new proxy inherit the existing subproxies of the old one --- src/stratifier.c | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/src/stratifier.c b/src/stratifier.c index 3bfaba2e..8cd7223f 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -1191,7 +1191,7 @@ static proxy_t *current_proxy(sdata_t *sdata) static void new_proxy(sdata_t *sdata, const int id) { bool exists = false, current = false; - proxy_t *proxy; + proxy_t *proxy, *proxy_list = NULL; mutex_lock(&sdata->proxy_lock); HASH_FIND_INT(sdata->proxies, &id, proxy); @@ -1201,10 +1201,20 @@ static void new_proxy(sdata_t *sdata, const int id) DL_APPEND(sdata->retired_proxies, proxy); if (proxy == sdata->proxy) current = true; + proxy_list = proxy->subproxies; + HASH_DELETE(sh, proxy_list, proxy); + proxy->subproxies = NULL; } proxy = __generate_proxy(sdata, id); if (current) sdata->proxy = proxy; + /* The old proxy had subproxies on its list so steal its list and add + * ourselves to it. */ + if (proxy_list) { + HASH_DELETE(sh, proxy->subproxies, proxy); + proxy->subproxies = proxy_list; + HASH_ADD(sh, proxy->subproxies, subid, sizeof(int), proxy); + } mutex_unlock(&sdata->proxy_lock); if (exists) From 3e760f1993281c22d7f06922b475bc4daec8ab79 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Mon, 23 Feb 2015 18:37:11 +1100 Subject: [PATCH 226/544] Reconnect parent proxy even if subproxies are alive --- src/generator.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/generator.c b/src/generator.c index 19dcb47f..aca93e84 100644 --- a/src/generator.c +++ b/src/generator.c @@ -1856,6 +1856,7 @@ static void *proxy_recv(void *arg) int ret; if (!proxi->alive) { + reconnect_proxy(proxi); while (!subproxies_alive(proxi)) { reconnect_proxy(proxi); if (alive) { From 36b217a45dd4bd832b90c17faaf4b0bccd264cfd Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Mon, 23 Feb 2015 20:21:05 +1100 Subject: [PATCH 227/544] Count dropped workers in _dec_instance_ref --- src/stratifier.c | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/stratifier.c b/src/stratifier.c index 8cd7223f..ba439561 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -1634,10 +1634,13 @@ static void client_drop_message(const int64_t client_id, const int dropped, cons } } +static void dec_worker(ckpool_t *ckp, user_instance_t *instance); + /* Decrease the reference count of instance. */ static void _dec_instance_ref(sdata_t *sdata, stratum_instance_t *client, const char *file, const char *func, const int line) { + user_instance_t *user = client->user_instance; int64_t client_id = client->id; int dropped = 0, ref; @@ -1646,7 +1649,7 @@ static void _dec_instance_ref(sdata_t *sdata, stratum_instance_t *client, const /* See if there are any instances that were dropped that could not be * moved due to holding a reference and drop them now. */ if (unlikely(client->dropped && !ref)) - dropped = __drop_client(sdata, client, client->user_instance); + dropped = __drop_client(sdata, client, user); ck_wunlock(&sdata->instance_lock); client_drop_message(client_id, dropped, true); @@ -1655,8 +1658,10 @@ static void _dec_instance_ref(sdata_t *sdata, stratum_instance_t *client, const if (unlikely(ref < 0)) LOGERR("Instance ref count dropped below zero from %s %s:%d", file, func, line); - if (dropped) + if (dropped) { + dec_worker(sdata->ckp, user); reap_proxies(sdata->ckp, sdata); + } } #define dec_instance_ref(sdata, instance) _dec_instance_ref(sdata, instance, __FILE__, __func__, __LINE__) From 4e5401a99157d1269923fecad20155796eef2ecb Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Mon, 23 Feb 2015 20:30:26 +1100 Subject: [PATCH 228/544] Reconnect clients connected to the parent proxy when it's replaced --- src/stratifier.c | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/src/stratifier.c b/src/stratifier.c index ba439561..e66ee30c 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -1188,6 +1188,29 @@ static proxy_t *current_proxy(sdata_t *sdata) return proxy; } +static void dead_parent_proxy(sdata_t *sdata, const int id) +{ + stratum_instance_t *client, *tmp; + int reconnects = 0; + + ck_rlock(&sdata->instance_lock); + HASH_ITER(hh, sdata->stratum_instances, client, tmp) { + if (client->proxyid != id || client->subproxyid) + continue; + reconnects++; + if (client->reconnect) + reconnect_client(sdata, client); + else + client->reconnect = true; + } + ck_runlock(&sdata->instance_lock); + + if (reconnects) { + LOGNOTICE("Flagged %d clients to reconnect from dead proxy %d", + reconnects, id); + } +} + static void new_proxy(sdata_t *sdata, const int id) { bool exists = false, current = false; @@ -1217,8 +1240,10 @@ static void new_proxy(sdata_t *sdata, const int id) } mutex_unlock(&sdata->proxy_lock); - if (exists) + if (exists) { LOGNOTICE("Stratifier replaced old proxy instance %d", id); + dead_parent_proxy(sdata, id); + } } static void update_subscribe(ckpool_t *ckp, const char *cmd) From 11cc87393a6a18d85ba195c8272f14dd00487b48 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Mon, 23 Feb 2015 20:51:33 +1100 Subject: [PATCH 229/544] Reconnect instead of testing clients that have their reconnect flag set on a stratum broadcast --- src/stratifier.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/stratifier.c b/src/stratifier.c index e66ee30c..9172cf5e 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -1814,10 +1814,14 @@ static void stratum_broadcast(sdata_t *sdata, json_t *val) } /* Look for clients that may have been dropped which the stratifer has * not been informed about and ask the connector of they still exist */ - if (client->dropped || client->reconnect) { + if (client->dropped) { connector_test_client(ckp, client->id); continue; } + if (client->reconnect) { + reconnect_client(ckp_sdata, client); + continue; + } if (!client_active(client)) continue; From 5123b2e0ecb3fcce75f070ed4f7d82e1b3e64b79 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Mon, 23 Feb 2015 21:10:54 +1100 Subject: [PATCH 230/544] Only trigger connection recruitment if we're reconnecting clients --- src/stratifier.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/stratifier.c b/src/stratifier.c index 9172cf5e..ddfb9c03 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -1172,9 +1172,9 @@ static void reconnect_clients(sdata_t *sdata) if (reconnects) { LOGNOTICE("Flagged %d clients for reconnect to proxy %d", reconnects, proxy->id); + if (headroom < 42) + generator_recruit(sdata->ckp); } - if (headroom < 42) - generator_recruit(sdata->ckp); } static proxy_t *current_proxy(sdata_t *sdata) @@ -2258,9 +2258,9 @@ static void dead_proxy(sdata_t *sdata, const char *buf) if (reconnects) { LOGNOTICE("Flagged %d clients to reconnect from dead proxy %d:%d", reconnects, id, subid); + if (headroom < 42) + generator_recruit(sdata->ckp); } - if (headroom < 42) - generator_recruit(sdata->ckp); } /* Must hold a reference */ From 7e343244ea464768e125b0b70b11cf3c1b0cee3e Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Mon, 23 Feb 2015 21:15:32 +1100 Subject: [PATCH 231/544] Change message slightly since we don't necessarily reconnect --- src/generator.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/generator.c b/src/generator.c index aca93e84..7622d8a4 100644 --- a/src/generator.c +++ b/src/generator.c @@ -1910,7 +1910,7 @@ static void *proxy_recv(void *arg) ret = read_socket_line(cs, 5); } if (ret < 1) { - LOGNOTICE("Proxy %d:%d %s failed to epoll/read_socket_line in proxy_recv, attempting reconnect", + LOGNOTICE("Proxy %d:%d %s failed to epoll/read_socket_line in proxy_recv", proxi->id, subproxy->subid, subproxy->si->url); disable_subproxy(gdata, proxi, subproxy); continue; From af21689d6797eff3037fb1c2228225fb59da0f40 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Mon, 23 Feb 2015 21:19:18 +1100 Subject: [PATCH 232/544] Don't reconnect generator immediately after the reconnect message, waiting for the first proxy_alive test on the new instance --- src/generator.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/generator.c b/src/generator.c index 7622d8a4..bd8db15a 100644 --- a/src/generator.c +++ b/src/generator.c @@ -1842,11 +1842,11 @@ static void *proxy_recv(void *arg) } if (proxy_alive(ckp, si, proxi, cs, false, epfd)) { - reconnect_generator(ckp); LOGWARNING("Proxy %d:%s connection established", proxi->id, proxi->si->url); } alive = proxi->alive; + reconnect_generator(ckp); while (42) { proxy_instance_t *subproxy = proxi; @@ -1922,7 +1922,6 @@ static void *proxy_recv(void *arg) * pool is up */ disable_subproxy(gdata, proxi, subproxy); if (parent_proxy(subproxy)) { - reconnect_generator(ckp); LOGWARNING("Proxy %d:%s reconnect issue, dropping existing connection", subproxy->id, subproxy->si->url); break; From d9b27c612ca089e065c8ea449c4f60562760f500 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Mon, 23 Feb 2015 21:47:27 +1100 Subject: [PATCH 233/544] Add received proxy share result to verbose logging output --- src/generator.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/generator.c b/src/generator.c index bd8db15a..851e9372 100644 --- a/src/generator.c +++ b/src/generator.c @@ -1455,8 +1455,8 @@ static bool parse_share(proxy_instance_t *proxi, const char *buf) LOGINFO("Failed to find matching share to result: %s", buf); goto out; } - LOGDEBUG("Found share from client %d with msg_id %d", share->client_id, - share->msg_id); + LOGINFO("Found share result %s from client %d with msg_id %d", buf, share->client_id, + share->msg_id); free(share); out: if (val) From b3589a71f3f8ac66b24bded664880be1cf4f9bb1 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Mon, 23 Feb 2015 22:26:57 +1100 Subject: [PATCH 234/544] Add proxy info to verbose share result logging --- src/generator.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/generator.c b/src/generator.c index 851e9372..0c2656b5 100644 --- a/src/generator.c +++ b/src/generator.c @@ -1452,11 +1452,12 @@ static bool parse_share(proxy_instance_t *proxi, const char *buf) * so long as we recognised it as a share response */ ret = true; if (!share) { - LOGINFO("Failed to find matching share to result: %s", buf); + LOGINFO("Proxy %d:%d failed to find matching share to result: %s", + proxi->id, proxi->subid, buf); goto out; } - LOGINFO("Found share result %s from client %d with msg_id %d", buf, share->client_id, - share->msg_id); + LOGINFO("Proxy %d:%d share result %s from client %d with msg_id %d", proxi->id, + proxi->subid, buf, share->client_id, share->msg_id); free(share); out: if (val) From 7f04ba2e03dbba53eeb676acba7245f335c232bb Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Mon, 23 Feb 2015 23:02:45 +1100 Subject: [PATCH 235/544] Transmit the actual workbase we want broadcast with stratum_broadcast_update and add more info the verbose logging --- src/stratifier.c | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/src/stratifier.c b/src/stratifier.c index ddfb9c03..db04833a 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -559,7 +559,7 @@ static void generate_coinbase(const ckpool_t *ckp, workbase_t *wb) hex2bin(wb->headerbin, header, 112); } -static void stratum_broadcast_update(sdata_t *sdata, bool clean); +static void stratum_broadcast_update(sdata_t *sdata, const workbase_t *wb, bool clean); static void clear_workbase(workbase_t *wb) { @@ -762,6 +762,9 @@ static void add_base(ckpool_t *ckp, sdata_t *sdata, workbase_t *wb, bool *new_bl if (ckp->logshares) sprintf(wb->logdir, "%s%08x/%s", ckp->logdir, wb->height, wb->idstring); + /* Is this long enough to ensure we don't dereference a workbase + * immediately? Should be unless clock changes 10 minutes so we use + * ts_realtime */ HASH_ITER(hh, sdata->workbases, tmp, tmpa) { if (HASH_COUNT(sdata->workbases) < 3) break; @@ -909,7 +912,7 @@ static void do_update(ckpool_t *ckp, int *prio) if (new_block) LOGNOTICE("Block hash changed to %s", sdata->lastswaphash); - stratum_broadcast_update(sdata, new_block); + stratum_broadcast_update(sdata, wb, new_block); ret = true; LOGINFO("Broadcast updated stratum base"); out: @@ -1421,8 +1424,10 @@ static void update_notify(ckpool_t *ckp, const char *cmd) proxy->notify_id = wb->id; if (parent_proxy(proxy) && proxy == current_proxy(sdata)) reconnect_clients(sdata); - LOGINFO("Broadcast updated stratum notify"); - stratum_broadcast_update(dsdata, new_block | clean); + clean |= new_block; + LOGINFO("Proxy %d:%d broadcast updated stratum notify with%s clean", id, + subid, clean ? "" : "out"); + stratum_broadcast_update(dsdata, wb, clean); out: json_decref(val); } @@ -3803,12 +3808,12 @@ static json_t *__stratum_notify(const workbase_t *wb, const bool clean) return val; } -static void stratum_broadcast_update(sdata_t *sdata, const bool clean) +static void stratum_broadcast_update(sdata_t *sdata, const workbase_t *wb, const bool clean) { json_t *json_msg; ck_rlock(&sdata->workbase_lock); - json_msg = __stratum_notify(sdata->current_workbase, clean); + json_msg = __stratum_notify(wb, clean); ck_runlock(&sdata->workbase_lock); stratum_broadcast(sdata, json_msg); From df7d17f3a97f4176d25533d39574ea3dd7cf93c7 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Tue, 24 Feb 2015 00:00:49 +1100 Subject: [PATCH 236/544] Ensure we use int64 for internal jobid throughout and add the string to debug message from the upstream pool --- src/generator.c | 8 +++++--- src/stratifier.c | 4 ++-- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/src/generator.c b/src/generator.c index 0c2656b5..e649d8c3 100644 --- a/src/generator.c +++ b/src/generator.c @@ -792,7 +792,7 @@ static bool parse_notify(ckpool_t *ckp, proxy_instance_t *proxi, json_t *val) const char *prev_hash, *bbversion, *nbit, *ntime; proxy_instance_t *proxy = proxi->parent; gdata_t *gdata = proxi->ckp->data; - char *coinbase1, *coinbase2; + char *coinbase1, *coinbase2, *buf; bool clean, ret = false; notify_instance_t *ni; json_t *arr, *job_id; @@ -824,6 +824,8 @@ static bool parse_notify(ckpool_t *ckp, proxy_instance_t *proxi, json_t *val) LOGDEBUG("Received new notify from proxy %d:%d", proxi->id, proxi->subid); ni = ckzalloc(sizeof(notify_instance_t)); ni->jobid = job_id; + buf = json_string_value(job_id); + LOGDEBUG("JobID %s", buf); ni->coinbase1 = coinbase1; LOGDEBUG("Coinbase1 %s", coinbase1); ni->coinb1len = strlen(coinbase1) / 2; @@ -1484,7 +1486,7 @@ static void *proxy_send(void *arg) stratum_msg_t *msg; bool ret = true; json_t *val; - int id; + int64_t id; tv_t now; ts_t abs; @@ -1510,7 +1512,7 @@ static void *proxy_send(void *arg) continue; json_get_int(&subid, msg->json_msg, "subproxy"); - json_get_int(&id, msg->json_msg, "jobid"); + json_get_int64(&id, msg->json_msg, "jobid"); json_get_int(&proxyid, msg->json_msg, "proxy"); json_get_int64(&client_id, msg->json_msg, "client_id"); if (unlikely(proxyid != proxy->id)) { diff --git a/src/stratifier.c b/src/stratifier.c index db04833a..19e2896f 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -1367,7 +1367,7 @@ static void update_notify(ckpool_t *ckp, const char *cmd) wb->ckp = ckp; wb->proxy = true; - json_int64cpy(&wb->id, val, "jobid"); + json_get_int64(&wb->id, val, "jobid"); json_strcpy(wb->prevhash, val, "prevhash"); json_intcpy(&wb->coinb1len, val, "coinb1len"); wb->coinb1bin = ckalloc(wb->coinb1len); @@ -3513,7 +3513,7 @@ static void submit_share(stratum_instance_t *client, const int64_t jobid, const char *msg; sprintf(enonce2, "%s%s", client->enonce1var, nonce2); - JSON_CPACK(json_msg, "{sisssssssIsisisi}", "jobid", jobid, "nonce2", enonce2, + JSON_CPACK(json_msg, "{sIsssssssIsisisi}", "jobid", jobid, "nonce2", enonce2, "ntime", ntime, "nonce", nonce, "client_id", client->id, "msg_id", msg_id, "proxy", client->proxyid, "subproxy", client->subproxyid); From 0dece89fa35663d7779d25adf210877eadd6f5eb Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Tue, 24 Feb 2015 00:02:59 +1100 Subject: [PATCH 237/544] Fix const warning --- src/generator.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/generator.c b/src/generator.c index e649d8c3..1a6f8ee4 100644 --- a/src/generator.c +++ b/src/generator.c @@ -792,7 +792,8 @@ static bool parse_notify(ckpool_t *ckp, proxy_instance_t *proxi, json_t *val) const char *prev_hash, *bbversion, *nbit, *ntime; proxy_instance_t *proxy = proxi->parent; gdata_t *gdata = proxi->ckp->data; - char *coinbase1, *coinbase2, *buf; + char *coinbase1, *coinbase2; + const char *jobidbuf; bool clean, ret = false; notify_instance_t *ni; json_t *arr, *job_id; @@ -824,8 +825,8 @@ static bool parse_notify(ckpool_t *ckp, proxy_instance_t *proxi, json_t *val) LOGDEBUG("Received new notify from proxy %d:%d", proxi->id, proxi->subid); ni = ckzalloc(sizeof(notify_instance_t)); ni->jobid = job_id; - buf = json_string_value(job_id); - LOGDEBUG("JobID %s", buf); + jobidbuf = json_string_value(job_id); + LOGDEBUG("JobID %s", jobidbuf); ni->coinbase1 = coinbase1; LOGDEBUG("Coinbase1 %s", coinbase1); ni->coinb1len = strlen(coinbase1) / 2; From 08a997e19047ea60180ad81de3ad83fca2414613 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Tue, 24 Feb 2015 01:03:07 +1100 Subject: [PATCH 238/544] Remove the unused msg_id field --- src/generator.c | 6 ++---- src/stratifier.c | 12 ++++-------- 2 files changed, 6 insertions(+), 12 deletions(-) diff --git a/src/generator.c b/src/generator.c index 1a6f8ee4..859a7fa3 100644 --- a/src/generator.c +++ b/src/generator.c @@ -49,7 +49,6 @@ struct share_msg { int64_t id; // Our own id for submitting upstream int client_id; - int msg_id; // Stratum message id from client time_t submit_time; }; @@ -1398,7 +1397,6 @@ static void submit_share(gdata_t *gdata, json_t *val) share = ckzalloc(sizeof(share_msg_t)); share->submit_time = time(NULL); share->client_id = client_id; - json_get_int(&share->msg_id, val, "msg_id"); msg->json_msg = val; /* Add new share entry to the share hashtable */ @@ -1459,8 +1457,8 @@ static bool parse_share(proxy_instance_t *proxi, const char *buf) proxi->id, proxi->subid, buf); goto out; } - LOGINFO("Proxy %d:%d share result %s from client %d with msg_id %d", proxi->id, - proxi->subid, buf, share->client_id, share->msg_id); + LOGINFO("Proxy %d:%d share result %s from client %d", proxi->id, proxi->subid, + buf, share->client_id); free(share); out: if (val) diff --git a/src/stratifier.c b/src/stratifier.c index 19e2896f..8a21a058 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -3505,7 +3505,7 @@ static void update_client(const stratum_instance_t *client, const int64_t client /* Submit a share in proxy mode to the parent pool. workbase_lock is held. * Needs to be entered with client holding a ref count. */ static void submit_share(stratum_instance_t *client, const int64_t jobid, const char *nonce2, - const char *ntime, const char *nonce, const int msg_id) + const char *ntime, const char *nonce) { ckpool_t *ckp = client->ckp; json_t *json_msg; @@ -3513,10 +3513,9 @@ static void submit_share(stratum_instance_t *client, const int64_t jobid, const char *msg; sprintf(enonce2, "%s%s", client->enonce1var, nonce2); - JSON_CPACK(json_msg, "{sIsssssssIsisisi}", "jobid", jobid, "nonce2", enonce2, + JSON_CPACK(json_msg, "{sIsssssssIsisi}", "jobid", jobid, "nonce2", enonce2, "ntime", ntime, "nonce", nonce, "client_id", client->id, - "msg_id", msg_id, "proxy", client->proxyid, - "subproxy", client->subproxyid); + "proxy", client->proxyid, "subproxy", client->subproxyid); msg = json_dumps(json_msg, 0); json_decref(json_msg); send_generator(ckp, msg, GEN_LAX); @@ -3697,11 +3696,8 @@ out_unlock: /* Submit share to upstream pool in proxy mode. We submit valid and * stale shares and filter out the rest. */ if (wb && wb->proxy && submit) { - int msg_id = 0; - - json_get_int(&msg_id, json_msg, "id"); LOGINFO("Submitting share upstream: %s", hexhash); - submit_share(client, id, nonce2, ntime, nonce, msg_id); + submit_share(client, id, nonce2, ntime, nonce); } add_submit(ckp, client, diff, result, submit); From 7de43b1c6e1c939484b8cea60b1dfa345773e220 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Tue, 24 Feb 2015 09:01:20 +1100 Subject: [PATCH 239/544] Use async send proc as a separate thread from workqueues --- src/ckpool.c | 70 +++++++++++++++++++++++++----------------------- src/ckpool.h | 4 --- src/connector.c | 10 +++---- src/generator.c | 28 +++++++++---------- src/stratifier.c | 25 +++++++++-------- 5 files changed, 65 insertions(+), 72 deletions(-) diff --git a/src/ckpool.c b/src/ckpool.c index ff303c50..63593b21 100644 --- a/src/ckpool.c +++ b/src/ckpool.c @@ -575,14 +575,32 @@ out: static void childsighandler(const int sig); -/* Send a single message to a process instance when there will be no response, - * closing the socket immediately. */ -void _send_proc(proc_instance_t *pi, const char *msg, const char *file, const char *func, const int line) +struct proc_message { + proc_instance_t *pi; + char *msg; + const char *file; + const char *func; + int line; +}; + +/* Send all one way messages asynchronously so we can wait till the receiving + * end closes the socket to ensure all messages are received but no deadlocks + * can occur with 2 processes waiting for each other's socket closure. */ +void *async_send_proc(void *arg) { + struct proc_message *pm = (struct proc_message *)arg; + proc_instance_t *pi = pm->pi; + char *msg = pm->msg; + const char *file = pm->file; + const char *func = pm->func; + int line = pm->line; + char *path = pi->us.path; bool ret = false; int sockd; + pthread_detach(pthread_self()); + if (unlikely(!path || !strlen(path))) { LOGERR("Attempted to send message %s to null path in send_proc", msg ? msg : ""); goto out; @@ -593,14 +611,16 @@ void _send_proc(proc_instance_t *pi, const char *msg, const char *file, const ch } /* At startup the pid fields are not set up before some processes are * forked so they never inherit them. */ - if (unlikely(!pi->pid)) + if (unlikely(!pi->pid)) { pi->pid = get_proc_pid(pi); - if (!pi->pid) { - LOGALERT("Attempting to send message %s to non existent process %s", msg, pi->processname); - return; + if (!pi->pid) { + LOGALERT("Attempting to send message %s to non existent process %s", msg, pi->processname); + goto out_nofail; + } } if (unlikely(kill_pid(pi->pid, 0))) { - LOGALERT("Attempting to send message %s to non existent process %s", msg, pi->processname); + LOGALERT("Attempting to send message %s to non existent process %s pid %d", + msg, pi->processname, pi->pid); goto out; } sockd = open_unix_client(path); @@ -620,41 +640,25 @@ out: LOGERR("Failure in send_proc from %s %s:%d", file, func, line); childsighandler(15); } -} - -struct proc_message { - proc_instance_t *pi; - char *msg; - const char *file; - const char *func; - int line; -}; - -static void asp_send(ckpool_t __maybe_unused *ckp, struct proc_message *pm) -{ - _send_proc(pm->pi, pm->msg, pm->file, pm->func, pm->line); - free(pm->msg); +out_nofail: + free(msg); free(pm); + return NULL; } -/* Fore sending asynchronous messages to another process, the sending process - * must have ckwqs of its own, referenced in the ckpool structure */ -void _async_send_proc(ckpool_t *ckp, proc_instance_t *pi, const char *msg, const char *file, const char *func, const int line) +/* Send a single message to a process instance when there will be no response, + * closing the socket immediately. */ +void _send_proc(proc_instance_t *pi, const char *msg, const char *file, const char *func, const int line) { - struct proc_message *pm; + struct proc_message *pm = ckalloc(sizeof(struct proc_message)); + pthread_t pth; - if (unlikely(!ckp->ckwqs)) { - LOGALERT("Workqueues not set up in async_send_proc!"); - _send_proc(pi, msg, file, func, line); - return; - } - pm = ckzalloc(sizeof(struct proc_message)); pm->pi = pi; pm->msg = strdup(msg); pm->file = file; pm->func = func; pm->line = line; - ckwq_add(ckp->ckwqs, &asp_send, pm); + create_pthread(&pth, async_send_proc, pm); } /* Send a single message to a process instance and retrieve the response, then diff --git a/src/ckpool.h b/src/ckpool.h index e88aa71e..d840528e 100644 --- a/src/ckpool.h +++ b/src/ckpool.h @@ -210,8 +210,6 @@ struct ckpool_instance { /* Private data for each process */ void *data; - /* Private generic workqueues if this process has them */ - ckwq_t *ckwqs; }; #ifdef USE_CKDB @@ -236,8 +234,6 @@ void empty_buffer(connsock_t *cs); int read_socket_line(connsock_t *cs, const int timeout); void _send_proc(proc_instance_t *pi, const char *msg, const char *file, const char *func, const int line); #define send_proc(pi, msg) _send_proc(pi, msg, __FILE__, __func__, __LINE__) -void _async_send_proc(ckpool_t *ckp, proc_instance_t *pi, const char *msg, const char *file, const char *func, const int line); -#define async_send_proc(ckp, pi, msg) _async_send_proc(ckp, pi, msg, __FILE__, __func__, __LINE__) char *_send_recv_proc(proc_instance_t *pi, const char *msg, const char *file, const char *func, const int line); #define send_recv_proc(pi, msg) _send_recv_proc(pi, msg, __FILE__, __func__, __LINE__) char *_send_recv_ckdb(const ckpool_t *ckp, const char *msg, const char *file, const char *func, const int line); diff --git a/src/connector.c b/src/connector.c index 3e568d36..ee6cf809 100644 --- a/src/connector.c +++ b/src/connector.c @@ -97,8 +97,6 @@ struct connector_data { /* For protecting the pending sends list */ mutex_t sender_lock; pthread_cond_t sender_cond; - - ckwq_t *ckwqs; }; typedef struct connector_data cdata_t; @@ -244,7 +242,7 @@ static void stratifier_drop_client(ckpool_t *ckp, const int64_t id) char buf[256]; sprintf(buf, "dropclient=%"PRId64, id); - async_send_proc(ckp, ckp->stratifier, buf); + send_proc(ckp->stratifier, buf); } /* Invalidate this instance. Remove them from the hashtables we look up @@ -362,9 +360,9 @@ reparse: * filtered by the stratifier. */ if (likely(client->fd != -1)) { if (ckp->passthrough) - async_send_proc(ckp, ckp->generator, s); + send_proc(ckp->generator, s); else - async_send_proc(ckp, ckp->stratifier, s); + send_proc(ckp->stratifier, s); } free(s); @@ -870,8 +868,6 @@ int connector(proc_instance_t *pi) LOGWARNING("%s connector starting", ckp->name); ckp->data = cdata; cdata->ckp = ckp; - /* Connector only requires one work queue */ - ckp->ckwqs = cdata->ckwqs = create_ckwqs(ckp, "conn", 1); if (!ckp->serverurls) cdata->serverfd = ckalloc(sizeof(int *)); diff --git a/src/generator.c b/src/generator.c index 859a7fa3..f1a0b97f 100644 --- a/src/generator.c +++ b/src/generator.c @@ -146,7 +146,6 @@ struct generator_data { proxy_instance_t *dead_proxies; /* Disabled proxies */ int proxy_notify_id; // Globally increasing notify id ckmsgq_t *srvchk; // Server check message queue - ckwq_t *ckwqs; }; typedef struct generator_data gdata_t; @@ -236,7 +235,7 @@ retry: cs = &alive->cs; LOGINFO("Connected to live server %s:%s", cs->url, cs->port); out: - async_send_proc(ckp, ckp->connector, alive ? "accept" : "reject"); + send_proc(ckp->connector, alive ? "accept" : "reject"); return alive; } @@ -382,7 +381,7 @@ retry: ret = submit_block(cs, buf + 12 + 64 + 1); memset(buf + 12 + 64, 0, 1); sprintf(blockmsg, "%sblock:%s", ret ? "" : "no", buf + 12); - async_send_proc(ckp, ckp->stratifier, blockmsg); + send_proc(ckp->stratifier, blockmsg); } else if (cmdmatch(buf, "checkaddr:")) { if (validate_address(cs, buf + 10)) send_unix_msg(sockd, "true"); @@ -945,7 +944,7 @@ static void send_stratifier_deadproxy(ckpool_t *ckp, const int id, const int sub char buf[256]; sprintf(buf, "deadproxy=%d:%d", id, subid); - async_send_proc(ckp, ckp->stratifier, buf); + send_proc(ckp->stratifier, buf); } /* Remove the subproxy from the proxi list and put it on the dead list. @@ -1092,7 +1091,7 @@ static void send_diff(ckpool_t *ckp, proxy_instance_t *proxi) json_decref(json_msg); ASPRINTF(&buf, "diff=%s", msg); free(msg); - async_send_proc(ckp, ckp->stratifier, buf); + send_proc(ckp->stratifier, buf); free(buf); } @@ -1120,7 +1119,7 @@ static void send_notify(ckpool_t *ckp, proxy_instance_t *proxi, notify_instance_ json_decref(json_msg); ASPRINTF(&buf, "notify=%s", msg); free(msg); - async_send_proc(ckp, ckp->stratifier, buf); + send_proc(ckp->stratifier, buf); free(buf); /* Send diff now as stratifier will not accept diff till it has a @@ -1310,7 +1309,7 @@ static void send_subscribe(ckpool_t *ckp, proxy_instance_t *proxi) json_decref(json_msg); ASPRINTF(&buf, "subscribe=%s", msg); free(msg); - async_send_proc(ckp, ckp->stratifier, buf); + send_proc(ckp->stratifier, buf); free(buf); } @@ -1356,7 +1355,7 @@ static void stratifier_reconnect_client(ckpool_t *ckp, int64_t id) char buf[256]; sprintf(buf, "reconnclient=%"PRId64, id); - async_send_proc(ckp, ckp->stratifier, buf); + send_proc(ckp->stratifier, buf); } static void submit_share(gdata_t *gdata, json_t *val) @@ -1730,9 +1729,9 @@ static void reconnect_proxy(proxy_instance_t *proxi) create_pthread(&pth, proxy_reconnect, proxi); } -static void reconnect_generator(ckpool_t *ckp) +static void reconnect_generator(const ckpool_t *ckp) { - async_send_proc(ckp, ckp->generator, "reconnect"); + send_proc(ckp->generator, "reconnect"); } /* For receiving messages from an upstream pool to pass downstream. Responsible @@ -1789,7 +1788,7 @@ static void *passthrough_recv(void *arg) /* Simply forward the message on, as is, to the connector to * process. Possibly parse parameters sent by upstream pool * here */ - async_send_proc(ckp, ckp->connector, cs->buf); + send_proc(ckp->connector, cs->buf); } return NULL; } @@ -1994,10 +1993,10 @@ static proxy_instance_t *wait_best_proxy(ckpool_t *ckp, gdata_t *gdata) if (ret) break; - async_send_proc(ckp, ckp->connector, "reject"); + send_proc(ckp->connector, "reject"); sleep(1); } - async_send_proc(ckp, ckp->connector, ret ? "accept" : "reject"); + send_proc(ckp->connector, ret ? "accept" : "reject"); return ret; } @@ -2026,7 +2025,7 @@ reconnect: proxi->id, cs->url, cs->port); dealloc(buf); ASPRINTF(&buf, "proxy=%d", proxi->id); - async_send_proc(ckp, ckp->stratifier, buf); + send_proc(ckp->stratifier, buf); } } retry: @@ -2216,7 +2215,6 @@ int generator(proc_instance_t *pi) gdata = ckzalloc(sizeof(gdata_t)); ckp->data = gdata; gdata->ckp = ckp; - ckp->ckwqs = gdata->ckwqs = create_ckwqs(ckp, "gen", 1); if (ckp->proxy) { char *buf = NULL; diff --git a/src/stratifier.c b/src/stratifier.c index 8a21a058..afd5796f 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -831,7 +831,7 @@ static char *send_recv_generator(ckpool_t *ckp, const char *msg, const int prio) return buf; } -static void send_generator(ckpool_t *ckp, const char *msg, const int prio) +static void send_generator(const ckpool_t *ckp, const char *msg, const int prio) { sdata_t *sdata = ckp->data; bool set; @@ -841,7 +841,7 @@ static void send_generator(ckpool_t *ckp, const char *msg, const int prio) set = true; } else set = false; - async_send_proc(ckp, ckp->generator, msg); + send_proc(ckp->generator, msg); if (set) sdata->gen_priority = 0; } @@ -966,7 +966,7 @@ static void connector_drop_client(ckpool_t *ckp, const int64_t id) LOGDEBUG("Stratifier requesting connector drop client %"PRId64, id); snprintf(buf, 255, "dropclient=%"PRId64, id); - async_send_proc(ckp, ckp->connector, buf); + send_proc(ckp->connector, buf); } static void drop_allclients(ckpool_t *ckp) @@ -1012,8 +1012,8 @@ static sdata_t *duplicate_sdata(const sdata_t *sdata) memcpy(dsdata->donkeytxnbin, sdata->donkeytxnbin, 25); /* Use the same work queues for all subproxies */ - dsdata->ckwqs = sdata->ckwqs; dsdata->ssends = sdata->ssends; + dsdata->ckwqs = sdata->ckwqs; dsdata->ckdbq = sdata->ckdbq; dsdata->sauthq = sdata->sauthq; @@ -1139,7 +1139,7 @@ out_unlock: static void reconnect_client(sdata_t *sdata, stratum_instance_t *client); -static void generator_recruit(ckpool_t *ckp) +static void generator_recruit(const ckpool_t *ckp) { LOGINFO("Stratifer requesting more proxies from generator"); send_generator(ckp, "recruit", GEN_PRIORITY); @@ -1780,7 +1780,7 @@ static void connector_test_client(ckpool_t *ckp, const int64_t id) LOGDEBUG("Stratifier requesting connector test client %"PRId64, id); snprintf(buf, 255, "testclient=%"PRId64, id); - async_send_proc(ckp, ckp->connector, buf); + send_proc(ckp->connector, buf); } /* For creating a list of sends without locking that can then be concatenated @@ -2567,7 +2567,7 @@ static void stratum_send_message(sdata_t *sdata, const stratum_instance_t *clien * in proxy mode where we find a subproxy based on the current proxy with room * for more clients. Signal the generator to recruit more subproxies if we are * running out of room. */ -static sdata_t *select_sdata(ckpool_t *ckp, sdata_t *ckp_sdata) +static sdata_t *select_sdata(const ckpool_t *ckp, sdata_t *ckp_sdata) { proxy_t *current, *proxy, *subproxy, *best = NULL, *tmp, *tmpsub; int best_id, best_subid = 0; @@ -3968,7 +3968,6 @@ static void send_transactions(ckpool_t *ckp, json_params_t *jp); static void parse_method(sdata_t *sdata, stratum_instance_t *client, const int64_t client_id, json_t *id_val, json_t *method_val, json_t *params_val, const char *address) { - ckpool_t *ckp = client->ckp; const char *method; /* Random broken clients send something not an integer as the id so we @@ -4015,14 +4014,14 @@ static void parse_method(sdata_t *sdata, stratum_instance_t *client, const int64 * to it since it's unauthorised. Set the flag just in case. */ client->authorised = false; snprintf(buf, 255, "passthrough=%"PRId64, client_id); - async_send_proc(ckp, ckp->connector, buf); + send_proc(client->ckp->connector, buf); return; } /* We should only accept subscribed requests from here on */ if (!client->subscribed) { LOGINFO("Dropping unsubscribed client %"PRId64, client_id); - connector_drop_client(ckp, client_id); + connector_drop_client(client->ckp, client_id); return; } @@ -4044,7 +4043,7 @@ static void parse_method(sdata_t *sdata, stratum_instance_t *client, const int64 * stratifier process to restart since it will have lost all * the stratum instance data. Clients will just reconnect. */ LOGINFO("Dropping unauthorised client %"PRId64, client_id); - connector_drop_client(ckp, client_id); + connector_drop_client(client->ckp, client_id); return; } @@ -4218,7 +4217,7 @@ static void ssend_process(ckpool_t *ckp, smsg_t *msg) * connector process to be delivered */ json_object_set_new_nocheck(msg->json_msg, "client_id", json_integer(msg->client_id)); s = json_dumps(msg->json_msg, 0); - async_send_proc(ckp, ckp->connector, s); + send_proc(ckp->connector, s); free(s); free_smsg(msg); } @@ -5081,7 +5080,7 @@ int stratifier(proc_instance_t *pi) sdata->ssends = create_ckmsgq(ckp, "ssender", &ssend_process); /* Create as many generic workqueue threads as there are CPUs */ threads = sysconf(_SC_NPROCESSORS_ONLN); - ckp->ckwqs = sdata->ckwqs = create_ckwqs(ckp, "strat", threads); + sdata->ckwqs = create_ckwqs(ckp, "strat", threads); sdata->sauthq = create_ckmsgq(ckp, "authoriser", &sauth_process); if (!CKP_STANDALONE(ckp)) { sdata->ckdbq = create_ckmsgq(ckp, "ckdbqueue", &ckdbq_process); From 4a3907d4e18b15ddf9740d87b0c930da23c530e3 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Tue, 24 Feb 2015 09:14:10 +1100 Subject: [PATCH 240/544] Remove use of unreliable notify_id to determine if clients are on the current pool with the more extensive switching in place --- src/stratifier.c | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/stratifier.c b/src/stratifier.c index afd5796f..9e2011ec 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -1161,7 +1161,7 @@ static void reconnect_clients(sdata_t *sdata) ck_rlock(&sdata->instance_lock); HASH_ITER(hh, sdata->stratum_instances, client, tmpclient) { - if (client->proxyid == proxy->id && client->notify_id == proxy->parent->notify_id) + if (client->proxyid == proxy->id) continue; headroom--; reconnects++; @@ -1284,7 +1284,6 @@ static void update_subscribe(ckpool_t *ckp, const char *cmd) LOGINFO("Got updated subscribe for proxy %d:%d", id, subid); proxy = subproxy_by_id(sdata, id, subid); - proxy->notify_id = -1; /* Reset this */ dsdata = proxy->sdata; ck_wlock(&dsdata->workbase_lock); @@ -1420,8 +1419,6 @@ static void update_notify(ckpool_t *ckp, const char *cmd) LOGNOTICE("Block hash on proxy %d changed to %s", id, dsdata->lastswaphash); } - if (proxy->notify_id == -1) - proxy->notify_id = wb->id; if (parent_proxy(proxy) && proxy == current_proxy(sdata)) reconnect_clients(sdata); clean |= new_block; @@ -2493,7 +2490,6 @@ static bool new_enonce1(ckpool_t *ckp, sdata_t *ckp_sdata, sdata_t *sdata, strat enonce1u = &proxy->enonce1u; client->proxyid = proxy->id; client->subproxyid = proxy->subid; - client->notify_id = proxy->parent->notify_id; mutex_unlock(&ckp_sdata->proxy_lock); if (proxy->clients >= proxy->max_clients) { From 8502036c5d5fefa0d003873740170056b2268faa Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Tue, 24 Feb 2015 10:32:08 +1100 Subject: [PATCH 241/544] Only dec worker if user is set --- src/stratifier.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/stratifier.c b/src/stratifier.c index 9e2011ec..dd129bf7 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -1686,7 +1686,8 @@ static void _dec_instance_ref(sdata_t *sdata, stratum_instance_t *client, const LOGERR("Instance ref count dropped below zero from %s %s:%d", file, func, line); if (dropped) { - dec_worker(sdata->ckp, user); + if (user) + dec_worker(sdata->ckp, user); reap_proxies(sdata->ckp, sdata); } } From 51670edb611b5390abfbd0e8a0e200848e8f261f Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Tue, 24 Feb 2015 10:38:00 +1100 Subject: [PATCH 242/544] Move to only reconnect the client sending shares to the dead proxy to smooth out reconnects --- src/generator.c | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/src/generator.c b/src/generator.c index f1a0b97f..8e270d1f 100644 --- a/src/generator.c +++ b/src/generator.c @@ -1350,7 +1350,7 @@ static void drop_proxy(gdata_t *gdata, const char *buf) disable_subproxy(gdata, proxy, subproxy); } -static void stratifier_reconnect_client(ckpool_t *ckp, int64_t id) +static void stratifier_reconnect_client(ckpool_t *ckp, const int64_t id) { char buf[256]; @@ -1375,20 +1375,22 @@ static void submit_share(gdata_t *gdata, json_t *val) proxy = proxy_by_id(gdata, id); if (unlikely(!proxy)) { - LOGWARNING("Failed to find proxy %d to send share to", id); - send_stratifier_deadproxy(ckp, id, subid); + LOGNOTICE("Client %"PRId64" sending shares to non existent proxy %d, dropping", + client_id, id); + stratifier_reconnect_client(ckp, client_id); return json_decref(val); } proxi = subproxy_by_id(proxy, subid); if (unlikely(!proxi)) { - LOGNOTICE("Failed to find proxy %d:%d to send share to", id, subid); - send_stratifier_deadproxy(ckp, id, subid); + LOGNOTICE("Client %"PRId64" sending shares to non existent subproxy %d:%d, dropping", + client_id, id, subid); + stratifier_reconnect_client(ckp, client_id); return json_decref(val); } if (!proxi->alive) { - LOGNOTICE("Client %"PRId64" attempting to send shares to dead proxy %d:%d, dropping", + LOGNOTICE("Client %"PRId64" sending shares to dead subproxy %d:%d, dropping", client_id, id, subid); - send_stratifier_deadproxy(ckp, id, subid); + stratifier_reconnect_client(ckp, client_id); return json_decref(val); } From 6b3cd82525709b6007b7570077808da32f9384d3 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Tue, 24 Feb 2015 10:38:16 +1100 Subject: [PATCH 243/544] Revert "Use async send proc as a separate thread from workqueues" This reverts commit 7de43b1c6e1c939484b8cea60b1dfa345773e220. --- src/ckpool.c | 70 +++++++++++++++++++++++------------------------- src/ckpool.h | 4 +++ src/connector.c | 10 ++++--- src/generator.c | 28 ++++++++++--------- src/stratifier.c | 25 ++++++++--------- 5 files changed, 72 insertions(+), 65 deletions(-) diff --git a/src/ckpool.c b/src/ckpool.c index 63593b21..ff303c50 100644 --- a/src/ckpool.c +++ b/src/ckpool.c @@ -575,32 +575,14 @@ out: static void childsighandler(const int sig); -struct proc_message { - proc_instance_t *pi; - char *msg; - const char *file; - const char *func; - int line; -}; - -/* Send all one way messages asynchronously so we can wait till the receiving - * end closes the socket to ensure all messages are received but no deadlocks - * can occur with 2 processes waiting for each other's socket closure. */ -void *async_send_proc(void *arg) +/* Send a single message to a process instance when there will be no response, + * closing the socket immediately. */ +void _send_proc(proc_instance_t *pi, const char *msg, const char *file, const char *func, const int line) { - struct proc_message *pm = (struct proc_message *)arg; - proc_instance_t *pi = pm->pi; - char *msg = pm->msg; - const char *file = pm->file; - const char *func = pm->func; - int line = pm->line; - char *path = pi->us.path; bool ret = false; int sockd; - pthread_detach(pthread_self()); - if (unlikely(!path || !strlen(path))) { LOGERR("Attempted to send message %s to null path in send_proc", msg ? msg : ""); goto out; @@ -611,16 +593,14 @@ void *async_send_proc(void *arg) } /* At startup the pid fields are not set up before some processes are * forked so they never inherit them. */ - if (unlikely(!pi->pid)) { + if (unlikely(!pi->pid)) pi->pid = get_proc_pid(pi); - if (!pi->pid) { - LOGALERT("Attempting to send message %s to non existent process %s", msg, pi->processname); - goto out_nofail; - } + if (!pi->pid) { + LOGALERT("Attempting to send message %s to non existent process %s", msg, pi->processname); + return; } if (unlikely(kill_pid(pi->pid, 0))) { - LOGALERT("Attempting to send message %s to non existent process %s pid %d", - msg, pi->processname, pi->pid); + LOGALERT("Attempting to send message %s to non existent process %s", msg, pi->processname); goto out; } sockd = open_unix_client(path); @@ -640,25 +620,41 @@ out: LOGERR("Failure in send_proc from %s %s:%d", file, func, line); childsighandler(15); } -out_nofail: - free(msg); +} + +struct proc_message { + proc_instance_t *pi; + char *msg; + const char *file; + const char *func; + int line; +}; + +static void asp_send(ckpool_t __maybe_unused *ckp, struct proc_message *pm) +{ + _send_proc(pm->pi, pm->msg, pm->file, pm->func, pm->line); + free(pm->msg); free(pm); - return NULL; } -/* Send a single message to a process instance when there will be no response, - * closing the socket immediately. */ -void _send_proc(proc_instance_t *pi, const char *msg, const char *file, const char *func, const int line) +/* Fore sending asynchronous messages to another process, the sending process + * must have ckwqs of its own, referenced in the ckpool structure */ +void _async_send_proc(ckpool_t *ckp, proc_instance_t *pi, const char *msg, const char *file, const char *func, const int line) { - struct proc_message *pm = ckalloc(sizeof(struct proc_message)); - pthread_t pth; + struct proc_message *pm; + if (unlikely(!ckp->ckwqs)) { + LOGALERT("Workqueues not set up in async_send_proc!"); + _send_proc(pi, msg, file, func, line); + return; + } + pm = ckzalloc(sizeof(struct proc_message)); pm->pi = pi; pm->msg = strdup(msg); pm->file = file; pm->func = func; pm->line = line; - create_pthread(&pth, async_send_proc, pm); + ckwq_add(ckp->ckwqs, &asp_send, pm); } /* Send a single message to a process instance and retrieve the response, then diff --git a/src/ckpool.h b/src/ckpool.h index d840528e..e88aa71e 100644 --- a/src/ckpool.h +++ b/src/ckpool.h @@ -210,6 +210,8 @@ struct ckpool_instance { /* Private data for each process */ void *data; + /* Private generic workqueues if this process has them */ + ckwq_t *ckwqs; }; #ifdef USE_CKDB @@ -234,6 +236,8 @@ void empty_buffer(connsock_t *cs); int read_socket_line(connsock_t *cs, const int timeout); void _send_proc(proc_instance_t *pi, const char *msg, const char *file, const char *func, const int line); #define send_proc(pi, msg) _send_proc(pi, msg, __FILE__, __func__, __LINE__) +void _async_send_proc(ckpool_t *ckp, proc_instance_t *pi, const char *msg, const char *file, const char *func, const int line); +#define async_send_proc(ckp, pi, msg) _async_send_proc(ckp, pi, msg, __FILE__, __func__, __LINE__) char *_send_recv_proc(proc_instance_t *pi, const char *msg, const char *file, const char *func, const int line); #define send_recv_proc(pi, msg) _send_recv_proc(pi, msg, __FILE__, __func__, __LINE__) char *_send_recv_ckdb(const ckpool_t *ckp, const char *msg, const char *file, const char *func, const int line); diff --git a/src/connector.c b/src/connector.c index ee6cf809..3e568d36 100644 --- a/src/connector.c +++ b/src/connector.c @@ -97,6 +97,8 @@ struct connector_data { /* For protecting the pending sends list */ mutex_t sender_lock; pthread_cond_t sender_cond; + + ckwq_t *ckwqs; }; typedef struct connector_data cdata_t; @@ -242,7 +244,7 @@ static void stratifier_drop_client(ckpool_t *ckp, const int64_t id) char buf[256]; sprintf(buf, "dropclient=%"PRId64, id); - send_proc(ckp->stratifier, buf); + async_send_proc(ckp, ckp->stratifier, buf); } /* Invalidate this instance. Remove them from the hashtables we look up @@ -360,9 +362,9 @@ reparse: * filtered by the stratifier. */ if (likely(client->fd != -1)) { if (ckp->passthrough) - send_proc(ckp->generator, s); + async_send_proc(ckp, ckp->generator, s); else - send_proc(ckp->stratifier, s); + async_send_proc(ckp, ckp->stratifier, s); } free(s); @@ -868,6 +870,8 @@ int connector(proc_instance_t *pi) LOGWARNING("%s connector starting", ckp->name); ckp->data = cdata; cdata->ckp = ckp; + /* Connector only requires one work queue */ + ckp->ckwqs = cdata->ckwqs = create_ckwqs(ckp, "conn", 1); if (!ckp->serverurls) cdata->serverfd = ckalloc(sizeof(int *)); diff --git a/src/generator.c b/src/generator.c index 8e270d1f..d76f31f3 100644 --- a/src/generator.c +++ b/src/generator.c @@ -146,6 +146,7 @@ struct generator_data { proxy_instance_t *dead_proxies; /* Disabled proxies */ int proxy_notify_id; // Globally increasing notify id ckmsgq_t *srvchk; // Server check message queue + ckwq_t *ckwqs; }; typedef struct generator_data gdata_t; @@ -235,7 +236,7 @@ retry: cs = &alive->cs; LOGINFO("Connected to live server %s:%s", cs->url, cs->port); out: - send_proc(ckp->connector, alive ? "accept" : "reject"); + async_send_proc(ckp, ckp->connector, alive ? "accept" : "reject"); return alive; } @@ -381,7 +382,7 @@ retry: ret = submit_block(cs, buf + 12 + 64 + 1); memset(buf + 12 + 64, 0, 1); sprintf(blockmsg, "%sblock:%s", ret ? "" : "no", buf + 12); - send_proc(ckp->stratifier, blockmsg); + async_send_proc(ckp, ckp->stratifier, blockmsg); } else if (cmdmatch(buf, "checkaddr:")) { if (validate_address(cs, buf + 10)) send_unix_msg(sockd, "true"); @@ -944,7 +945,7 @@ static void send_stratifier_deadproxy(ckpool_t *ckp, const int id, const int sub char buf[256]; sprintf(buf, "deadproxy=%d:%d", id, subid); - send_proc(ckp->stratifier, buf); + async_send_proc(ckp, ckp->stratifier, buf); } /* Remove the subproxy from the proxi list and put it on the dead list. @@ -1091,7 +1092,7 @@ static void send_diff(ckpool_t *ckp, proxy_instance_t *proxi) json_decref(json_msg); ASPRINTF(&buf, "diff=%s", msg); free(msg); - send_proc(ckp->stratifier, buf); + async_send_proc(ckp, ckp->stratifier, buf); free(buf); } @@ -1119,7 +1120,7 @@ static void send_notify(ckpool_t *ckp, proxy_instance_t *proxi, notify_instance_ json_decref(json_msg); ASPRINTF(&buf, "notify=%s", msg); free(msg); - send_proc(ckp->stratifier, buf); + async_send_proc(ckp, ckp->stratifier, buf); free(buf); /* Send diff now as stratifier will not accept diff till it has a @@ -1309,7 +1310,7 @@ static void send_subscribe(ckpool_t *ckp, proxy_instance_t *proxi) json_decref(json_msg); ASPRINTF(&buf, "subscribe=%s", msg); free(msg); - send_proc(ckp->stratifier, buf); + async_send_proc(ckp, ckp->stratifier, buf); free(buf); } @@ -1355,7 +1356,7 @@ static void stratifier_reconnect_client(ckpool_t *ckp, const int64_t id) char buf[256]; sprintf(buf, "reconnclient=%"PRId64, id); - send_proc(ckp->stratifier, buf); + async_send_proc(ckp, ckp->stratifier, buf); } static void submit_share(gdata_t *gdata, json_t *val) @@ -1731,9 +1732,9 @@ static void reconnect_proxy(proxy_instance_t *proxi) create_pthread(&pth, proxy_reconnect, proxi); } -static void reconnect_generator(const ckpool_t *ckp) +static void reconnect_generator(ckpool_t *ckp) { - send_proc(ckp->generator, "reconnect"); + async_send_proc(ckp, ckp->generator, "reconnect"); } /* For receiving messages from an upstream pool to pass downstream. Responsible @@ -1790,7 +1791,7 @@ static void *passthrough_recv(void *arg) /* Simply forward the message on, as is, to the connector to * process. Possibly parse parameters sent by upstream pool * here */ - send_proc(ckp->connector, cs->buf); + async_send_proc(ckp, ckp->connector, cs->buf); } return NULL; } @@ -1995,10 +1996,10 @@ static proxy_instance_t *wait_best_proxy(ckpool_t *ckp, gdata_t *gdata) if (ret) break; - send_proc(ckp->connector, "reject"); + async_send_proc(ckp, ckp->connector, "reject"); sleep(1); } - send_proc(ckp->connector, ret ? "accept" : "reject"); + async_send_proc(ckp, ckp->connector, ret ? "accept" : "reject"); return ret; } @@ -2027,7 +2028,7 @@ reconnect: proxi->id, cs->url, cs->port); dealloc(buf); ASPRINTF(&buf, "proxy=%d", proxi->id); - send_proc(ckp->stratifier, buf); + async_send_proc(ckp, ckp->stratifier, buf); } } retry: @@ -2217,6 +2218,7 @@ int generator(proc_instance_t *pi) gdata = ckzalloc(sizeof(gdata_t)); ckp->data = gdata; gdata->ckp = ckp; + ckp->ckwqs = gdata->ckwqs = create_ckwqs(ckp, "gen", 1); if (ckp->proxy) { char *buf = NULL; diff --git a/src/stratifier.c b/src/stratifier.c index dd129bf7..a96b1526 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -831,7 +831,7 @@ static char *send_recv_generator(ckpool_t *ckp, const char *msg, const int prio) return buf; } -static void send_generator(const ckpool_t *ckp, const char *msg, const int prio) +static void send_generator(ckpool_t *ckp, const char *msg, const int prio) { sdata_t *sdata = ckp->data; bool set; @@ -841,7 +841,7 @@ static void send_generator(const ckpool_t *ckp, const char *msg, const int prio) set = true; } else set = false; - send_proc(ckp->generator, msg); + async_send_proc(ckp, ckp->generator, msg); if (set) sdata->gen_priority = 0; } @@ -966,7 +966,7 @@ static void connector_drop_client(ckpool_t *ckp, const int64_t id) LOGDEBUG("Stratifier requesting connector drop client %"PRId64, id); snprintf(buf, 255, "dropclient=%"PRId64, id); - send_proc(ckp->connector, buf); + async_send_proc(ckp, ckp->connector, buf); } static void drop_allclients(ckpool_t *ckp) @@ -1012,8 +1012,8 @@ static sdata_t *duplicate_sdata(const sdata_t *sdata) memcpy(dsdata->donkeytxnbin, sdata->donkeytxnbin, 25); /* Use the same work queues for all subproxies */ - dsdata->ssends = sdata->ssends; dsdata->ckwqs = sdata->ckwqs; + dsdata->ssends = sdata->ssends; dsdata->ckdbq = sdata->ckdbq; dsdata->sauthq = sdata->sauthq; @@ -1139,7 +1139,7 @@ out_unlock: static void reconnect_client(sdata_t *sdata, stratum_instance_t *client); -static void generator_recruit(const ckpool_t *ckp) +static void generator_recruit(ckpool_t *ckp) { LOGINFO("Stratifer requesting more proxies from generator"); send_generator(ckp, "recruit", GEN_PRIORITY); @@ -1778,7 +1778,7 @@ static void connector_test_client(ckpool_t *ckp, const int64_t id) LOGDEBUG("Stratifier requesting connector test client %"PRId64, id); snprintf(buf, 255, "testclient=%"PRId64, id); - send_proc(ckp->connector, buf); + async_send_proc(ckp, ckp->connector, buf); } /* For creating a list of sends without locking that can then be concatenated @@ -2564,7 +2564,7 @@ static void stratum_send_message(sdata_t *sdata, const stratum_instance_t *clien * in proxy mode where we find a subproxy based on the current proxy with room * for more clients. Signal the generator to recruit more subproxies if we are * running out of room. */ -static sdata_t *select_sdata(const ckpool_t *ckp, sdata_t *ckp_sdata) +static sdata_t *select_sdata(ckpool_t *ckp, sdata_t *ckp_sdata) { proxy_t *current, *proxy, *subproxy, *best = NULL, *tmp, *tmpsub; int best_id, best_subid = 0; @@ -3965,6 +3965,7 @@ static void send_transactions(ckpool_t *ckp, json_params_t *jp); static void parse_method(sdata_t *sdata, stratum_instance_t *client, const int64_t client_id, json_t *id_val, json_t *method_val, json_t *params_val, const char *address) { + ckpool_t *ckp = client->ckp; const char *method; /* Random broken clients send something not an integer as the id so we @@ -4011,14 +4012,14 @@ static void parse_method(sdata_t *sdata, stratum_instance_t *client, const int64 * to it since it's unauthorised. Set the flag just in case. */ client->authorised = false; snprintf(buf, 255, "passthrough=%"PRId64, client_id); - send_proc(client->ckp->connector, buf); + async_send_proc(ckp, ckp->connector, buf); return; } /* We should only accept subscribed requests from here on */ if (!client->subscribed) { LOGINFO("Dropping unsubscribed client %"PRId64, client_id); - connector_drop_client(client->ckp, client_id); + connector_drop_client(ckp, client_id); return; } @@ -4040,7 +4041,7 @@ static void parse_method(sdata_t *sdata, stratum_instance_t *client, const int64 * stratifier process to restart since it will have lost all * the stratum instance data. Clients will just reconnect. */ LOGINFO("Dropping unauthorised client %"PRId64, client_id); - connector_drop_client(client->ckp, client_id); + connector_drop_client(ckp, client_id); return; } @@ -4214,7 +4215,7 @@ static void ssend_process(ckpool_t *ckp, smsg_t *msg) * connector process to be delivered */ json_object_set_new_nocheck(msg->json_msg, "client_id", json_integer(msg->client_id)); s = json_dumps(msg->json_msg, 0); - send_proc(ckp->connector, s); + async_send_proc(ckp, ckp->connector, s); free(s); free_smsg(msg); } @@ -5077,7 +5078,7 @@ int stratifier(proc_instance_t *pi) sdata->ssends = create_ckmsgq(ckp, "ssender", &ssend_process); /* Create as many generic workqueue threads as there are CPUs */ threads = sysconf(_SC_NPROCESSORS_ONLN); - sdata->ckwqs = create_ckwqs(ckp, "strat", threads); + ckp->ckwqs = sdata->ckwqs = create_ckwqs(ckp, "strat", threads); sdata->sauthq = create_ckmsgq(ckp, "authoriser", &sauth_process); if (!CKP_STANDALONE(ckp)) { sdata->ckdbq = create_ckmsgq(ckp, "ckdbqueue", &ckdbq_process); From d8eaa0b4e9a22c11976c100a60211e41b7a707a0 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Tue, 24 Feb 2015 10:45:44 +1100 Subject: [PATCH 244/544] Rate limit proxy recruiting to one thread at a time --- src/generator.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/generator.c b/src/generator.c index d76f31f3..475339dc 100644 --- a/src/generator.c +++ b/src/generator.c @@ -107,6 +107,7 @@ struct proxy_instance { bool disabled; /* Subproxy no longer to be used */ bool reconnect; /* We need to drop and reconnect */ bool reconnecting; /* Testing in progress */ + bool recruiting; /* Recruiting in progress */ bool alive; mutex_t notify_lock; @@ -1698,6 +1699,7 @@ static void *proxy_recruit(void *arg) store_proxy(gdata, proxy); } else add_subproxy(parent, proxy); + parent->recruiting = false; return NULL; } @@ -1705,6 +1707,9 @@ static void recruit_subproxy(proxy_instance_t *proxi) { pthread_t pth; + if (proxi->recruiting) + return; + proxi->recruiting = true; create_pthread(&pth, proxy_recruit, proxi); } From fc75956823871f3c4aa1fa2a6188c3cf984c1b62 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Tue, 24 Feb 2015 10:57:33 +1100 Subject: [PATCH 245/544] Replace the parent of existing proxies with the new one --- src/stratifier.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/stratifier.c b/src/stratifier.c index a96b1526..0caeab51 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -1216,8 +1216,8 @@ static void dead_parent_proxy(sdata_t *sdata, const int id) static void new_proxy(sdata_t *sdata, const int id) { + proxy_t *proxy, *subproxy, *tmp, *proxy_list = NULL; bool exists = false, current = false; - proxy_t *proxy, *proxy_list = NULL; mutex_lock(&sdata->proxy_lock); HASH_FIND_INT(sdata->proxies, &id, proxy); @@ -1240,6 +1240,8 @@ static void new_proxy(sdata_t *sdata, const int id) HASH_DELETE(sh, proxy->subproxies, proxy); proxy->subproxies = proxy_list; HASH_ADD(sh, proxy->subproxies, subid, sizeof(int), proxy); + HASH_ITER(sh, proxy->subproxies, subproxy, tmp) + subproxy->parent = proxy; } mutex_unlock(&sdata->proxy_lock); From dd22d73145142925300ef4b51344844d2169e82f Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Tue, 24 Feb 2015 11:25:01 +1100 Subject: [PATCH 246/544] Do not switch more clients than we have headroom --- src/stratifier.c | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/stratifier.c b/src/stratifier.c index 0caeab51..8e9f2343 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -1163,7 +1163,8 @@ static void reconnect_clients(sdata_t *sdata) HASH_ITER(hh, sdata->stratum_instances, client, tmpclient) { if (client->proxyid == proxy->id) continue; - headroom--; + if (--headroom < 0) + break; reconnects++; if (client->reconnect) reconnect_client(sdata, client); @@ -1177,7 +1178,8 @@ static void reconnect_clients(sdata_t *sdata) proxy->id); if (headroom < 42) generator_recruit(sdata->ckp); - } + } else if (headroom < 0) + generator_recruit(sdata->ckp); } static proxy_t *current_proxy(sdata_t *sdata) @@ -2251,7 +2253,8 @@ static void dead_proxy(sdata_t *sdata, const char *buf) HASH_ITER(hh, sdata->stratum_instances, client, tmp) { if (client->proxyid != id || client->subproxyid != subid) continue; - headroom--; + if (--headroom < 0) + break; reconnects++; if (client->reconnect) reconnect_client(sdata, client); @@ -2265,7 +2268,8 @@ static void dead_proxy(sdata_t *sdata, const char *buf) id, subid); if (headroom < 42) generator_recruit(sdata->ckp); - } + } else if (headroom < 0) + generator_recruit(sdata->ckp); } /* Must hold a reference */ From 922855b17193df73cf312293892c38bbaa658d57 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Tue, 24 Feb 2015 11:38:26 +1100 Subject: [PATCH 247/544] Fine tune proxy reconnects --- src/stratifier.c | 33 +++++++++++++++++++-------------- 1 file changed, 19 insertions(+), 14 deletions(-) diff --git a/src/stratifier.c b/src/stratifier.c index 8e9f2343..2705fd77 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -1163,10 +1163,9 @@ static void reconnect_clients(sdata_t *sdata) HASH_ITER(hh, sdata->stratum_instances, client, tmpclient) { if (client->proxyid == proxy->id) continue; - if (--headroom < 0) - break; + headroom--; reconnects++; - if (client->reconnect) + if (client->reconnect && headroom > 0) reconnect_client(sdata, client); else client->reconnect = true; @@ -1174,12 +1173,11 @@ static void reconnect_clients(sdata_t *sdata) ck_runlock(&sdata->instance_lock); if (reconnects) { - LOGNOTICE("Flagged %d clients for reconnect to proxy %d", reconnects, + LOGNOTICE("%d clients flagged for reconnect to proxy %d", reconnects, proxy->id); if (headroom < 42) generator_recruit(sdata->ckp); - } else if (headroom < 0) - generator_recruit(sdata->ckp); + } } static proxy_t *current_proxy(sdata_t *sdata) @@ -1197,13 +1195,20 @@ static void dead_parent_proxy(sdata_t *sdata, const int id) { stratum_instance_t *client, *tmp; int reconnects = 0; + int64_t headroom; + proxy_t *proxy; + + headroom = current_headroom(sdata, &proxy); + if (!proxy) + return; ck_rlock(&sdata->instance_lock); HASH_ITER(hh, sdata->stratum_instances, client, tmp) { if (client->proxyid != id || client->subproxyid) continue; + headroom--; reconnects++; - if (client->reconnect) + if (client->reconnect && headroom > 0) reconnect_client(sdata, client); else client->reconnect = true; @@ -1211,8 +1216,10 @@ static void dead_parent_proxy(sdata_t *sdata, const int id) ck_runlock(&sdata->instance_lock); if (reconnects) { - LOGNOTICE("Flagged %d clients to reconnect from dead proxy %d", + LOGNOTICE("%d clients flagged to reconnect from dead proxy %d", reconnects, id); + if (headroom < 42) + generator_recruit(sdata->ckp); } } @@ -2253,10 +2260,9 @@ static void dead_proxy(sdata_t *sdata, const char *buf) HASH_ITER(hh, sdata->stratum_instances, client, tmp) { if (client->proxyid != id || client->subproxyid != subid) continue; - if (--headroom < 0) - break; + headroom--; reconnects++; - if (client->reconnect) + if (client->reconnect && headroom > 0) reconnect_client(sdata, client); else client->reconnect = true; @@ -2264,12 +2270,11 @@ static void dead_proxy(sdata_t *sdata, const char *buf) ck_runlock(&sdata->instance_lock); if (reconnects) { - LOGNOTICE("Flagged %d clients to reconnect from dead proxy %d:%d", reconnects, + LOGNOTICE("%d clients flagged to reconnect from dead proxy %d:%d", reconnects, id, subid); if (headroom < 42) generator_recruit(sdata->ckp); - } else if (headroom < 0) - generator_recruit(sdata->ckp); + } } /* Must hold a reference */ From 80c0ddb346dc488be6c89daefdfb419e97dab5c2 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Tue, 24 Feb 2015 11:38:54 +1100 Subject: [PATCH 248/544] Revert "Rate limit proxy recruiting to one thread at a time" This reverts commit d8eaa0b4e9a22c11976c100a60211e41b7a707a0. --- src/generator.c | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/generator.c b/src/generator.c index 475339dc..d76f31f3 100644 --- a/src/generator.c +++ b/src/generator.c @@ -107,7 +107,6 @@ struct proxy_instance { bool disabled; /* Subproxy no longer to be used */ bool reconnect; /* We need to drop and reconnect */ bool reconnecting; /* Testing in progress */ - bool recruiting; /* Recruiting in progress */ bool alive; mutex_t notify_lock; @@ -1699,7 +1698,6 @@ static void *proxy_recruit(void *arg) store_proxy(gdata, proxy); } else add_subproxy(parent, proxy); - parent->recruiting = false; return NULL; } @@ -1707,9 +1705,6 @@ static void recruit_subproxy(proxy_instance_t *proxi) { pthread_t pth; - if (proxi->recruiting) - return; - proxi->recruiting = true; create_pthread(&pth, proxy_recruit, proxi); } From 7f129e93c8546b92f68cbc4c8a304104bb24ce3b Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Tue, 24 Feb 2015 11:39:13 +1100 Subject: [PATCH 249/544] Revert "Revert "Rate limit proxy recruiting to one thread at a time"" This reverts commit 80c0ddb346dc488be6c89daefdfb419e97dab5c2. Wrong revert. --- src/generator.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/generator.c b/src/generator.c index d76f31f3..475339dc 100644 --- a/src/generator.c +++ b/src/generator.c @@ -107,6 +107,7 @@ struct proxy_instance { bool disabled; /* Subproxy no longer to be used */ bool reconnect; /* We need to drop and reconnect */ bool reconnecting; /* Testing in progress */ + bool recruiting; /* Recruiting in progress */ bool alive; mutex_t notify_lock; @@ -1698,6 +1699,7 @@ static void *proxy_recruit(void *arg) store_proxy(gdata, proxy); } else add_subproxy(parent, proxy); + parent->recruiting = false; return NULL; } @@ -1705,6 +1707,9 @@ static void recruit_subproxy(proxy_instance_t *proxi) { pthread_t pth; + if (proxi->recruiting) + return; + proxi->recruiting = true; create_pthread(&pth, proxy_recruit, proxi); } From ac86d9b736bc4293959770e0d81bfb881c6562fb Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Tue, 24 Feb 2015 12:43:10 +1100 Subject: [PATCH 250/544] Add an HASH_REPLACE_I64 helper macro --- src/uthash.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/uthash.h b/src/uthash.h index 10c9b6b7..31db3fa8 100644 --- a/src/uthash.h +++ b/src/uthash.h @@ -265,6 +265,8 @@ do { HASH_FIND(hh,head,findint,sizeof(int64_t),out) #define HASH_ADD_I64(head,intfield,add) \ HASH_ADD(hh,head,intfield,sizeof(int64_t),add) +#define HASH_REPLACE_I64(head,intfield,add,replaced) \ + HASH_REPLACE(hh,head,intfield,sizeof(int64_t),add,replaced) #define HASH_FIND_PTR(head,findptr,out) \ HASH_FIND(hh,head,findptr,sizeof(void *),out) #define HASH_ADD_PTR(head,ptrfield,add) \ From 14873b44b70ac84b0b688a2bc80c0c5315f5863f Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Tue, 24 Feb 2015 13:05:36 +1100 Subject: [PATCH 251/544] Convert proxy id to int64 --- src/generator.c | 126 +++++++++++++++++++++++------------------------ src/stratifier.c | 98 ++++++++++++++++++------------------ 2 files changed, 114 insertions(+), 110 deletions(-) diff --git a/src/generator.c b/src/generator.c index 475339dc..008adf16 100644 --- a/src/generator.c +++ b/src/generator.c @@ -84,7 +84,7 @@ struct proxy_instance { connsock_t *cs; server_instance_t *si; bool passthrough; - int id; /* Proxy server id*/ + int64_t id; /* Proxy server id*/ int subid; /* Subproxy id */ const char *auth; @@ -575,7 +575,7 @@ static bool parse_subscribe(connsock_t *cs, proxy_instance_t *proxi) retry: parsed = true; if (!(buf = new_proxy_line(cs))) { - LOGNOTICE("Proxy %d:%d %s failed to receive line in parse_subscribe", + LOGNOTICE("Proxy %ld:%d %s failed to receive line in parse_subscribe", proxi->id, proxi->subid, proxi->si->url); goto out; } @@ -608,7 +608,7 @@ retry: buf = NULL; goto retry; } - LOGNOTICE("Proxy %d:%d %s failed to parse subscribe response in parse_subscribe", + LOGNOTICE("Proxy %ld:%d %s failed to parse subscribe response in parse_subscribe", proxi->id, proxi->subid, proxi->si->url); goto out; } @@ -653,10 +653,10 @@ retry: } if (size < 3) { if (!proxi->subid) { - LOGWARNING("Proxy %d %s Nonce2 length %d too small for fast miners", + LOGWARNING("Proxy %ld %s Nonce2 length %d too small for fast miners", proxi->id, proxi->si->url, size); } else { - LOGNOTICE("Proxy %d:%d Nonce2 length %d too small for fast miners", + LOGNOTICE("Proxy %ld:%d Nonce2 length %d too small for fast miners", proxi->id, proxi->subid, size); } } @@ -665,11 +665,11 @@ retry: /* Set the number of clients per proxy on the parent proxy */ int64_t clients_per_proxy = 1ll << ((size - 3) * 8); - LOGNOTICE("Proxy %d:%s clients per proxy: %"PRId64, proxi->id, proxi->si->url, + LOGNOTICE("Proxy %ld:%s clients per proxy: %"PRId64, proxi->id, proxi->si->url, clients_per_proxy); } - LOGINFO("Found notify for proxy %d:%d with enonce %s nonce2len %d", proxi->id, + LOGINFO("Found notify for proxy %ld:%d with enonce %s nonce2len %d", proxi->id, proxi->subid, proxi->enonce1, proxi->nonce2len); ret = true; @@ -708,7 +708,7 @@ retry: ret = send_json_msg(cs, req); json_decref(req); if (!ret) { - LOGNOTICE("Proxy %d:%d %s failed to send message in subscribe_stratum", + LOGNOTICE("Proxy %ld:%d %s failed to send message in subscribe_stratum", proxi->id, proxi->subid, proxi->si->url); goto out; } @@ -717,23 +717,23 @@ retry: goto out; if (proxi->no_params) { - LOGNOTICE("Proxy %d:%d %s failed all subscription options in subscribe_stratum", + LOGNOTICE("Proxy %ld:%d %s failed all subscription options in subscribe_stratum", proxi->id, proxi->subid, proxi->si->url); goto out; } if (proxi->sessionid) { - LOGINFO("Proxy %d:%d %s failed sessionid reconnect in subscribe_stratum, retrying without", + LOGINFO("Proxy %ld:%d %s failed sessionid reconnect in subscribe_stratum, retrying without", proxi->id, proxi->subid, proxi->si->url); proxi->no_sessionid = true; dealloc(proxi->sessionid); } else { - LOGINFO("Proxy %d:%d %s failed connecting with parameters in subscribe_stratum, retrying without", + LOGINFO("Proxy %ld:%d %s failed connecting with parameters in subscribe_stratum, retrying without", proxi->id, proxi->subid, proxi->si->url); proxi->no_params = true; } ret = connect_proxy(cs); if (!ret) { - LOGNOTICE("Proxy %d:%d %s failed to reconnect in subscribe_stratum", + LOGNOTICE("Proxy %ld:%d %s failed to reconnect in subscribe_stratum", proxi->id, proxi->subid, proxi->si->url); goto out; } @@ -822,7 +822,7 @@ static bool parse_notify(ckpool_t *ckp, proxy_instance_t *proxi, json_t *val) goto out; } - LOGDEBUG("Received new notify from proxy %d:%d", proxi->id, proxi->subid); + LOGDEBUG("Received new notify from proxy %ld:%d", proxi->id, proxi->subid); ni = ckzalloc(sizeof(notify_instance_t)); ni->jobid = job_id; jobidbuf = json_string_value(job_id); @@ -934,18 +934,18 @@ static proxy_instance_t *__subproxy_by_id(proxy_instance_t *proxy, const int sub /* Add to the dead list to be recycled if possible */ static void store_proxy(gdata_t *gdata, proxy_instance_t *proxy) { - LOGINFO("Recycling data from proxy %d:%d", proxy->id, proxy->subid); + LOGINFO("Recycling data from proxy %ld:%d", proxy->id, proxy->subid); mutex_lock(&gdata->lock); DL_APPEND(gdata->dead_proxies, proxy); mutex_unlock(&gdata->lock); } -static void send_stratifier_deadproxy(ckpool_t *ckp, const int id, const int subid) +static void send_stratifier_deadproxy(ckpool_t *ckp, const int64_t id, const int subid) { char buf[256]; - sprintf(buf, "deadproxy=%d:%d", id, subid); + sprintf(buf, "deadproxy=%ld:%d", id, subid); async_send_proc(ckp, ckp->stratifier, buf); } @@ -1065,7 +1065,7 @@ static bool parse_reconnect(proxy_instance_t *proxi, json_t *val) newproxi->cs->ckp = ckp; newproxi->id = newsi->id; newproxi->subproxy_count = ++proxi->subproxy_count; - HASH_REPLACE_INT(gdata->proxies, id, newproxi, proxi); + HASH_REPLACE_I64(gdata->proxies, id, newproxi, proxi); mutex_unlock(&gdata->lock); /* Old proxy memory is basically lost here */ @@ -1085,7 +1085,7 @@ static void send_diff(ckpool_t *ckp, proxy_instance_t *proxi) if (!proxi->diff) return; - JSON_CPACK(json_msg, "{sisisf}", + JSON_CPACK(json_msg, "{sIsisf}", "proxy", proxy->id, "subproxy", proxi->subid, "diff", proxi->diff); @@ -1109,7 +1109,7 @@ static void send_notify(ckpool_t *ckp, proxy_instance_t *proxi, notify_instance_ for (i = 0; i < ni->merkles; i++) json_array_append_new(merkle_arr, json_string(&ni->merklehash[i][0])); /* Use our own jobid instead of the server's one for easy lookup */ - JSON_CPACK(json_msg, "{sisisisssisssssosssssssb}", + JSON_CPACK(json_msg, "{sIsisisssisssssosssssssb}", "proxy", proxy->id, "subproxy", proxi->subid, "jobid", ni->id, "prevhash", ni->prevhash, "coinb1len", ni->coinb1len, "coinbase1", ni->coinbase1, "coinbase2", ni->coinbase2, @@ -1174,7 +1174,7 @@ static bool parse_method(ckpool_t *ckp, proxy_instance_t *proxi, const char *msg goto out; } - LOGDEBUG("Proxy %d:%d received method %s", proxi->id, proxi->subid, buf); + LOGDEBUG("Proxy %ld:%d received method %s", proxi->id, proxi->subid, buf); if (cmdmatch(buf, "mining.notify")) { ret = parse_notify(ckp, proxi, params); goto out; @@ -1225,7 +1225,7 @@ static bool auth_stratum(ckpool_t *ckp, connsock_t *cs, proxy_instance_t *proxi) ret = send_json_msg(cs, req); json_decref(req); if (!ret) { - LOGNOTICE("Proxy %d:%d %s failed to send message in auth_stratum", + LOGNOTICE("Proxy %ld:%d %s failed to send message in auth_stratum", proxi->id, proxi->subid, proxi->si->url); Close(cs->fd); goto out; @@ -1237,7 +1237,7 @@ static bool auth_stratum(ckpool_t *ckp, connsock_t *cs, proxy_instance_t *proxi) free(buf); buf = next_proxy_line(cs, proxi); if (!buf) { - LOGNOTICE("Proxy %d:%d %s failed to receive line in auth_stratum", + LOGNOTICE("Proxy %ld:%d %s failed to receive line in auth_stratum", proxi->id, proxi->subid, proxi->si->url); ret = false; goto out; @@ -1247,20 +1247,20 @@ static bool auth_stratum(ckpool_t *ckp, connsock_t *cs, proxy_instance_t *proxi) val = json_msg_result(buf, &res_val, &err_val); if (!val) { - LOGWARNING("Proxy %d:%d %s failed to get a json result in auth_stratum, got: %s", + LOGWARNING("Proxy %ld:%d %s failed to get a json result in auth_stratum, got: %s", proxi->id, proxi->subid, proxi->si->url, buf); goto out; } if (err_val && !json_is_null(err_val)) { - LOGWARNING("Proxy %d:%d %s failed to authorise in auth_stratum due to err_val, got: %s", + LOGWARNING("Proxy %ld:%d %s failed to authorise in auth_stratum due to err_val, got: %s", proxi->id, proxi->subid, proxi->si->url, buf); goto out; } if (res_val) { ret = json_is_true(res_val); if (!ret) { - LOGWARNING("Proxy %d:%d %s failed to authorise in auth_stratum, got: %s", + LOGWARNING("Proxy %ld:%d %s failed to authorise in auth_stratum, got: %s", proxi->id, proxi->subid, proxi->si->url, buf); goto out; } @@ -1268,7 +1268,7 @@ static bool auth_stratum(ckpool_t *ckp, connsock_t *cs, proxy_instance_t *proxi) /* No result and no error but successful val means auth success */ ret = true; } - LOGINFO("Proxy %d:%d %s auth success in auth_stratum", proxi->id, proxi->subid, proxi->si->url); + LOGINFO("Proxy %ld:%d %s auth success in auth_stratum", proxi->id, proxi->subid, proxi->si->url); out: if (val) json_decref(val); @@ -1286,12 +1286,12 @@ out: return ret; } -static proxy_instance_t *proxy_by_id(gdata_t *gdata, const int id) +static proxy_instance_t *proxy_by_id(gdata_t *gdata, const int64_t id) { proxy_instance_t *proxi; mutex_lock(&gdata->lock); - HASH_FIND_INT(gdata->proxies, &id, proxi); + HASH_FIND_I64(gdata->proxies, &id, proxi); mutex_unlock(&gdata->lock); return proxi; @@ -1302,8 +1302,8 @@ static void send_subscribe(ckpool_t *ckp, proxy_instance_t *proxi) json_t *json_msg; char *msg, *buf; - JSON_CPACK(json_msg, "{sisisssi}", - "proxy", proxi->parent->id, + JSON_CPACK(json_msg, "{sIsisssi}", + "proxy", proxi->id, "subproxy", proxi->subid, "enonce1", proxi->enonce1, "nonce2len", proxi->nonce2len); @@ -1331,24 +1331,25 @@ static proxy_instance_t *subproxy_by_id(proxy_instance_t *proxy, const int subid static void drop_proxy(gdata_t *gdata, const char *buf) { proxy_instance_t *proxy, *subproxy; - int id = 0, subid = 0; + int64_t id = 0; + int subid = 0; - sscanf(buf, "dropproxy=%d:%d", &id, &subid); + sscanf(buf, "dropproxy=%ld:%d", &id, &subid); if (unlikely(!subid)) { - LOGWARNING("Generator asked to drop parent proxy %d", id); + LOGWARNING("Generator asked to drop parent proxy %ld", id); return; } proxy = proxy_by_id(gdata, id); if (unlikely(!proxy)) { - LOGINFO("Generator asked to drop subproxy from non-existent parent %d", id); + LOGINFO("Generator asked to drop subproxy from non-existent parent %ld", id); return; } subproxy = subproxy_by_id(proxy, subid); if (!subproxy) { - LOGINFO("Generator asked to drop non-existent subproxy %d:%d", id, subid); + LOGINFO("Generator asked to drop non-existent subproxy %ld:%d", id, subid); return; } - LOGNOTICE("Generator asked to drop proxy %d:%d", id, subid); + LOGNOTICE("Generator asked to drop proxy %ld:%d", id, subid); disable_subproxy(gdata, proxy, subproxy); } @@ -1364,33 +1365,33 @@ static void submit_share(gdata_t *gdata, json_t *val) { proxy_instance_t *proxy, *proxi; ckpool_t *ckp = gdata->ckp; + int64_t client_id, id; stratum_msg_t *msg; share_msg_t *share; - int64_t client_id; - int id, subid; + int subid; /* Get the client id so we can tell the stratifier to drop it if the * proxy it's bound to is not functional */ json_get_int64(&client_id, val, "client_id"); - json_get_int(&id, val, "proxy"); + json_get_int64(&id, val, "proxy"); json_get_int(&subid, val, "subproxy"); proxy = proxy_by_id(gdata, id); if (unlikely(!proxy)) { - LOGNOTICE("Client %"PRId64" sending shares to non existent proxy %d, dropping", + LOGNOTICE("Client %"PRId64" sending shares to non existent proxy %ld, dropping", client_id, id); stratifier_reconnect_client(ckp, client_id); return json_decref(val); } proxi = subproxy_by_id(proxy, subid); if (unlikely(!proxi)) { - LOGNOTICE("Client %"PRId64" sending shares to non existent subproxy %d:%d, dropping", + LOGNOTICE("Client %"PRId64" sending shares to non existent subproxy %ld:%d, dropping", client_id, id, subid); stratifier_reconnect_client(ckp, client_id); return json_decref(val); } if (!proxi->alive) { - LOGNOTICE("Client %"PRId64" sending shares to dead subproxy %d:%d, dropping", + LOGNOTICE("Client %"PRId64" sending shares to dead subproxy %ld:%d, dropping", client_id, id, subid); stratifier_reconnect_client(ckp, client_id); return json_decref(val); @@ -1456,11 +1457,11 @@ static bool parse_share(proxy_instance_t *proxi, const char *buf) * so long as we recognised it as a share response */ ret = true; if (!share) { - LOGINFO("Proxy %d:%d failed to find matching share to result: %s", + LOGINFO("Proxy %ld:%d failed to find matching share to result: %s", proxi->id, proxi->subid, buf); goto out; } - LOGINFO("Proxy %d:%d share result %s from client %d", proxi->id, proxi->subid, + LOGINFO("Proxy %ld:%d share result %s from client %d", proxi->id, proxi->subid, buf, share->client_id); free(share); out: @@ -1480,20 +1481,19 @@ static void *proxy_send(void *arg) rename_proc("proxysend"); while (42) { + int64_t proxyid = 0, client_id = 0, id; proxy_instance_t *subproxy; - int proxyid = 0, subid = 0; notify_instance_t *ni; - int64_t client_id = 0; json_t *jobid = NULL; stratum_msg_t *msg; bool ret = true; + int subid = 0; json_t *val; - int64_t id; tv_t now; ts_t abs; if (unlikely(proxy->reconnect)) { - LOGINFO("Shutting down proxy_send thread for proxy %d to reconnect", + LOGINFO("Shutting down proxy_send thread for proxy %ld to reconnect", proxy->id); break; } @@ -1515,10 +1515,10 @@ static void *proxy_send(void *arg) json_get_int(&subid, msg->json_msg, "subproxy"); json_get_int64(&id, msg->json_msg, "jobid"); - json_get_int(&proxyid, msg->json_msg, "proxy"); + json_get_int64(&proxyid, msg->json_msg, "proxy"); json_get_int64(&client_id, msg->json_msg, "client_id"); if (unlikely(proxyid != proxy->id)) { - LOGWARNING("Proxysend for proxy %d got message for proxy %d!", + LOGWARNING("Proxysend for proxy %ld got message for proxy %ld!", proxy->id, proxyid); } @@ -1542,17 +1542,17 @@ static void *proxy_send(void *arg) json_decref(val); } else if (!jobid) { stratifier_reconnect_client(ckp, client_id); - LOGNOTICE("Proxy %d:%s failed to find matching jobid for %sknown subproxy in proxysend", + LOGNOTICE("Proxy %ld:%s failed to find matching jobid for %sknown subproxy in proxysend", proxy->id, proxy->si->url, subproxy ? "" : "un"); } else { stratifier_reconnect_client(ckp, client_id); - LOGNOTICE("Failed to find subproxy %d:%d to send message to", + LOGNOTICE("Failed to find subproxy %ld:%d to send message to", proxy->id, subid); } json_decref(msg->json_msg); free(msg); if (!ret && subproxy) { - LOGNOTICE("Proxy %d:%d %s failed to send msg in proxy_send, dropping to reconnect", + LOGNOTICE("Proxy %ld:%d %s failed to send msg in proxy_send, dropping to reconnect", proxy->id, proxy->subid, proxy->si->url); disable_subproxy(gdata, proxy, subproxy); } @@ -1764,7 +1764,7 @@ static void *passthrough_recv(void *arg) if (proxy_alive(ckp, si, proxi, cs, false, epfd)) { reconnect_generator(ckp); - LOGWARNING("Proxy %d:%s connection established", + LOGWARNING("Proxy %ld:%s connection established", proxi->id, proxi->si->url); } alive = proxi->alive; @@ -1787,7 +1787,7 @@ static void *passthrough_recv(void *arg) if (likely(ret > 0)) ret = read_socket_line(cs, 60); if (ret < 1) { - LOGWARNING("Proxy %d:%s failed to read_socket_line in proxy_recv, attempting reconnect", + LOGWARNING("Proxy %ld:%s failed to read_socket_line in proxy_recv, attempting reconnect", proxi->id, proxi->si->url); alive = proxi->alive = false; reconnect_generator(ckp); @@ -1851,7 +1851,7 @@ static void *proxy_recv(void *arg) } if (proxy_alive(ckp, si, proxi, cs, false, epfd)) { - LOGWARNING("Proxy %d:%s connection established", + LOGWARNING("Proxy %ld:%s connection established", proxi->id, proxi->si->url); } alive = proxi->alive; @@ -1869,7 +1869,7 @@ static void *proxy_recv(void *arg) while (!subproxies_alive(proxi)) { reconnect_proxy(proxi); if (alive) { - LOGWARNING("Proxy %d:%s failed, attempting reconnect", + LOGWARNING("Proxy %ld:%s failed, attempting reconnect", proxi->id, proxi->si->url); alive = false; reconnect_generator(ckp); @@ -1881,7 +1881,7 @@ static void *proxy_recv(void *arg) /* Wait 30 seconds before declaring this upstream pool alive * to prevent switching to unstable pools. */ if (!alive && (!current_proxy(gdata) || time(NULL) - proxi->reconnect_time > 30)) { - LOGWARNING("Proxy %d:%s recovered", proxi->id, proxi->si->url); + LOGWARNING("Proxy %ld:%s recovered", proxi->id, proxi->si->url); proxi->reconnect_time = 0; reconnect_generator(ckp); alive = true; @@ -1919,7 +1919,7 @@ static void *proxy_recv(void *arg) ret = read_socket_line(cs, 5); } if (ret < 1) { - LOGNOTICE("Proxy %d:%d %s failed to epoll/read_socket_line in proxy_recv", + LOGNOTICE("Proxy %ld:%d %s failed to epoll/read_socket_line in proxy_recv", proxi->id, subproxy->subid, subproxy->si->url); disable_subproxy(gdata, proxi, subproxy); continue; @@ -1931,7 +1931,7 @@ static void *proxy_recv(void *arg) * pool is up */ disable_subproxy(gdata, proxi, subproxy); if (parent_proxy(subproxy)) { - LOGWARNING("Proxy %d:%s reconnect issue, dropping existing connection", + LOGWARNING("Proxy %ld:%s reconnect issue, dropping existing connection", subproxy->id, subproxy->si->url); break; } else @@ -1971,7 +1971,7 @@ static void setup_proxies(ckpool_t *ckp, gdata_t *gdata) si = ckp->servers[i]; proxi = si->data; proxi->id = i; - HASH_ADD_INT(gdata->proxies, id, proxi); + HASH_ADD_I64(gdata->proxies, id, proxi); if (ckp->passthrough) { create_pthread(&proxi->pth_precv, passthrough_recv, proxi); proxi->passsends = create_ckmsgq(ckp, "passsend", &passthrough_send); @@ -2029,10 +2029,10 @@ reconnect: proxi = cproxy; if (!ckp->passthrough) { connsock_t *cs = proxi->cs; - LOGWARNING("Successfully connected to proxy %d %s:%s as proxy", + LOGWARNING("Successfully connected to proxy %ld %s:%s as proxy", proxi->id, cs->url, cs->port); dealloc(buf); - ASPRINTF(&buf, "proxy=%d", proxi->id); + ASPRINTF(&buf, "proxy=%ld", proxi->id); async_send_proc(ckp, ckp->stratifier, buf); } } diff --git a/src/stratifier.c b/src/stratifier.c index 2705fd77..172d5d19 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -279,7 +279,7 @@ struct stratum_instance { sdata_t *sdata; /* Which sdata this client is bound to */ proxy_t *proxy; /* Proxy this is bound to in proxy mode */ - int proxyid; /* Which proxy id */ + int64_t proxyid; /* Which proxy id */ int subproxyid; /* Which subproxy */ int64_t notify_id; /* Which notify_id from the subproxy did we join */ }; @@ -305,7 +305,7 @@ struct proxy_base { UT_hash_handle sh; /* For subproxy hashlist */ proxy_t *next; /* For retired subproxies */ proxy_t *prev; - int id; + int64_t id; int subid; double diff; @@ -1022,7 +1022,7 @@ static sdata_t *duplicate_sdata(const sdata_t *sdata) return dsdata; } -static proxy_t *__generate_proxy(sdata_t *sdata, const int id) +static proxy_t *__generate_proxy(sdata_t *sdata, const int64_t id) { proxy_t *proxy = ckzalloc(sizeof(proxy_t)); @@ -1033,7 +1033,7 @@ static proxy_t *__generate_proxy(sdata_t *sdata, const int id) proxy->parent = proxy; /* subid == 0 on parent proxy */ HASH_ADD(sh, proxy->subproxies, subid, sizeof(int), proxy); - HASH_ADD_INT(sdata->proxies, id, proxy); + HASH_ADD_I64(sdata->proxies, id, proxy); sdata->proxy_count++; return proxy; } @@ -1051,22 +1051,22 @@ static proxy_t *__generate_subproxy(sdata_t *sdata, proxy_t *proxy, const int su return subproxy; } -static proxy_t *__existing_proxy(const sdata_t *sdata, const int id) +static proxy_t *__existing_proxy(const sdata_t *sdata, const int64_t id) { proxy_t *proxy; - HASH_FIND_INT(sdata->proxies, &id, proxy); + HASH_FIND_I64(sdata->proxies, &id, proxy); return proxy; } /* Find proxy by id number, generate one if none exist yet by that id */ -static proxy_t *__proxy_by_id(sdata_t *sdata, const int id) +static proxy_t *__proxy_by_id(sdata_t *sdata, const int64_t id) { proxy_t *proxy = __existing_proxy(sdata, id); if (unlikely(!proxy)) { proxy = __generate_proxy(sdata, id); - LOGNOTICE("Stratifier added new proxy %d", id); + LOGNOTICE("Stratifier added new proxy %ld", id); } return proxy; @@ -1086,12 +1086,12 @@ static proxy_t *__subproxy_by_id(sdata_t *sdata, proxy_t *proxy, const int subid if (!subproxy) { subproxy = __generate_subproxy(sdata, proxy, subid); - LOGINFO("Stratifier added new subproxy %d:%d", proxy->id, subid); + LOGINFO("Stratifier added new subproxy %ld:%d", proxy->id, subid); } return subproxy; } -static proxy_t *subproxy_by_id(sdata_t *sdata, const int id, const int subid) +static proxy_t *subproxy_by_id(sdata_t *sdata, const int64_t id, const int subid) { proxy_t *proxy, *subproxy; @@ -1103,7 +1103,7 @@ static proxy_t *subproxy_by_id(sdata_t *sdata, const int id, const int subid) return subproxy; } -static proxy_t *existing_subproxy(sdata_t *sdata, const int id, const int subid) +static proxy_t *existing_subproxy(sdata_t *sdata, const int64_t id, const int subid) { proxy_t *proxy, *subproxy = NULL; @@ -1173,7 +1173,7 @@ static void reconnect_clients(sdata_t *sdata) ck_runlock(&sdata->instance_lock); if (reconnects) { - LOGNOTICE("%d clients flagged for reconnect to proxy %d", reconnects, + LOGNOTICE("%d clients flagged for reconnect to proxy %ld", reconnects, proxy->id); if (headroom < 42) generator_recruit(sdata->ckp); @@ -1191,7 +1191,7 @@ static proxy_t *current_proxy(sdata_t *sdata) return proxy; } -static void dead_parent_proxy(sdata_t *sdata, const int id) +static void dead_parent_proxy(sdata_t *sdata, const int64_t id) { stratum_instance_t *client, *tmp; int reconnects = 0; @@ -1216,20 +1216,20 @@ static void dead_parent_proxy(sdata_t *sdata, const int id) ck_runlock(&sdata->instance_lock); if (reconnects) { - LOGNOTICE("%d clients flagged to reconnect from dead proxy %d", + LOGNOTICE("%d clients flagged to reconnect from dead proxy %ld", reconnects, id); if (headroom < 42) generator_recruit(sdata->ckp); } } -static void new_proxy(sdata_t *sdata, const int id) +static void new_proxy(sdata_t *sdata, const int64_t id) { proxy_t *proxy, *subproxy, *tmp, *proxy_list = NULL; bool exists = false, current = false; mutex_lock(&sdata->proxy_lock); - HASH_FIND_INT(sdata->proxies, &id, proxy); + HASH_FIND_I64(sdata->proxies, &id, proxy); if (proxy) { exists = true; HASH_DEL(sdata->proxies, proxy); @@ -1255,7 +1255,7 @@ static void new_proxy(sdata_t *sdata, const int id) mutex_unlock(&sdata->proxy_lock); if (exists) { - LOGNOTICE("Stratifier replaced old proxy instance %d", id); + LOGNOTICE("Stratifier replaced old proxy %ld", id); dead_parent_proxy(sdata, id); } } @@ -1263,9 +1263,10 @@ static void new_proxy(sdata_t *sdata, const int id) static void update_subscribe(ckpool_t *ckp, const char *cmd) { sdata_t *sdata = ckp->data, *dsdata; - int id = 0, subid = 0; const char *buf; proxy_t *proxy; + int64_t id = 0; + int subid = 0; json_t *val; if (unlikely(strlen(cmd) < 11)) { @@ -1279,7 +1280,7 @@ static void update_subscribe(ckpool_t *ckp, const char *cmd) LOGWARNING("Failed to json decode subscribe response in update_subscribe %s", buf); return; } - if (unlikely(!json_get_int(&id, val, "proxy"))) { + if (unlikely(!json_get_int64(&id, val, "proxy"))) { LOGWARNING("Failed to json decode proxy value in update_subscribe %s", buf); return; } @@ -1290,9 +1291,9 @@ static void update_subscribe(ckpool_t *ckp, const char *cmd) if (!subid) { new_proxy(sdata, id); - LOGNOTICE("Got updated subscribe for proxy %d", id); + LOGNOTICE("Got updated subscribe for proxy %ld", id); } else - LOGINFO("Got updated subscribe for proxy %d:%d", id, subid); + LOGINFO("Got updated subscribe for proxy %ld:%d", id, subid); proxy = subproxy_by_id(sdata, id, subid); dsdata = proxy->sdata; @@ -1323,10 +1324,10 @@ static void update_subscribe(ckpool_t *ckp, const char *cmd) ck_wunlock(&dsdata->workbase_lock); if (subid) { - LOGINFO("Upstream pool %d:%d extranonce2 length %d, max proxy clients %"PRId64, + LOGINFO("Upstream pool %ld:%d extranonce2 length %d, max proxy clients %"PRId64, id, subid, proxy->nonce2len, proxy->max_clients); } else { - LOGNOTICE("Upstream pool %d extranonce2 length %d, max proxy clients %"PRId64, + LOGNOTICE("Upstream pool %ld extranonce2 length %d, max proxy clients %"PRId64, id, proxy->nonce2len, proxy->max_clients); } @@ -1342,9 +1343,10 @@ static void update_notify(ckpool_t *ckp, const char *cmd) { sdata_t *sdata = ckp->data, *dsdata; bool new_block = false, clean; - int i, id = 0, subid = 0; + int i, subid = 0; char header[228]; const char *buf; + int64_t id = 0; proxy_t *proxy; workbase_t *wb; json_t *val; @@ -1361,17 +1363,17 @@ static void update_notify(ckpool_t *ckp, const char *cmd) LOGWARNING("Failed to json decode in update_notify"); return; } - json_get_int(&id, val, "proxy"); + json_get_int64(&id, val, "proxy"); json_get_int(&subid, val, "subproxy"); proxy = existing_subproxy(sdata, id, subid); if (unlikely(!proxy || !proxy->subscribed)) { - LOGINFO("No valid proxy %d:%d subscription to update notify yet", id, subid); + LOGINFO("No valid proxy %ld:%d subscription to update notify yet", id, subid); goto out; } if (!subid) - LOGNOTICE("Got updated notify for proxy %d", id); + LOGNOTICE("Got updated notify for proxy %ld", id); else - LOGINFO("Got updated notify for proxy %d:%d", id, subid); + LOGINFO("Got updated notify for proxy %ld:%d", id, subid); wb = ckzalloc(sizeof(workbase_t)); wb->ckp = ckp; @@ -1425,15 +1427,15 @@ static void update_notify(ckpool_t *ckp, const char *cmd) add_base(ckp, dsdata, wb, &new_block); if (new_block) { if (subid) - LOGINFO("Block hash on proxy %d:%d changed to %s", id, subid, dsdata->lastswaphash); + LOGINFO("Block hash on proxy %ld:%d changed to %s", id, subid, dsdata->lastswaphash); else - LOGNOTICE("Block hash on proxy %d changed to %s", id, dsdata->lastswaphash); + LOGNOTICE("Block hash on proxy %ld changed to %s", id, dsdata->lastswaphash); } if (parent_proxy(proxy) && proxy == current_proxy(sdata)) reconnect_clients(sdata); clean |= new_block; - LOGINFO("Proxy %d:%d broadcast updated stratum notify with%s clean", id, + LOGINFO("Proxy %ld:%d broadcast updated stratum notify with%s clean", id, subid, clean ? "" : "out"); stratum_broadcast_update(dsdata, wb, clean); out: @@ -1447,9 +1449,10 @@ static void update_diff(ckpool_t *ckp, const char *cmd) sdata_t *sdata = ckp->data, *dsdata; stratum_instance_t *client, *tmp; double old_diff, diff; - int id = 0, subid = 0; const char *buf; + int64_t id = 0; proxy_t *proxy; + int subid = 0; json_t *val; if (unlikely(strlen(cmd) < 6)) { @@ -1464,15 +1467,15 @@ static void update_diff(ckpool_t *ckp, const char *cmd) LOGWARNING("Failed to json decode in update_diff"); return; } - json_get_int(&id, val, "proxy"); + json_get_int64(&id, val, "proxy"); json_get_int(&subid, val, "subproxy"); json_dblcpy(&diff, val, "diff"); json_decref(val); - LOGINFO("Got updated diff for proxy %d:%d", id, subid); + LOGINFO("Got updated diff for proxy %ld:%d", id, subid); proxy = existing_subproxy(sdata, id, subid); if (!proxy) { - LOGINFO("No existing subproxy %d:%d to update diff", id, subid); + LOGINFO("No existing subproxy %ld:%d to update diff", id, subid); return; } @@ -1512,11 +1515,11 @@ static void update_diff(ckpool_t *ckp, const char *cmd) ck_runlock(&sdata->instance_lock); } -static void generator_drop_proxy(ckpool_t *ckp, const int id, const int subid) +static void generator_drop_proxy(ckpool_t *ckp, const int64_t id, const int subid) { char msg[256]; - sprintf(msg, "dropproxy=%d:%d", id, subid); + sprintf(msg, "dropproxy=%ld:%d", id, subid); send_generator(ckp, msg, GEN_LAX); } @@ -2226,17 +2229,17 @@ static void reconnect_client(sdata_t *sdata, stratum_instance_t *client) * this proxy we are only given this message if all clients must move. */ static void set_proxy(sdata_t *sdata, const char *buf) { + int64_t id = 0; proxy_t *proxy; - int id = 0; - sscanf(buf, "proxy=%d", &id); + sscanf(buf, "proxy=%ld", &id); mutex_lock(&sdata->proxy_lock); proxy = __proxy_by_id(sdata, id); sdata->proxy = proxy; mutex_unlock(&sdata->proxy_lock); - LOGNOTICE("Stratifier setting active proxy to %d", id); + LOGNOTICE("Stratifier setting active proxy to %ld", id); if (proxy->notify_id != -1) reconnect_clients(sdata); } @@ -2244,16 +2247,16 @@ static void set_proxy(sdata_t *sdata, const char *buf) static void dead_proxy(sdata_t *sdata, const char *buf) { stratum_instance_t *client, *tmp; - int id = 0, subid = 0; + int64_t headroom, id = 0; int reconnects = 0; - int64_t headroom; proxy_t *proxy; + int subid = 0; - sscanf(buf, "deadproxy=%d:%d", &id, &subid); + sscanf(buf, "deadproxy=%ld:%d", &id, &subid); proxy = existing_subproxy(sdata, id, subid); if (proxy) proxy->dead = true; - LOGNOTICE("Stratifier dropping clients from proxy %d:%d", id, subid); + LOGNOTICE("Stratifier dropping clients from proxy %ld:%d", id, subid); headroom = current_headroom(sdata, &proxy); ck_rlock(&sdata->instance_lock); @@ -2270,7 +2273,7 @@ static void dead_proxy(sdata_t *sdata, const char *buf) ck_runlock(&sdata->instance_lock); if (reconnects) { - LOGNOTICE("%d clients flagged to reconnect from dead proxy %d:%d", reconnects, + LOGNOTICE("%d clients flagged to reconnect from dead proxy %ld:%d", reconnects, id, subid); if (headroom < 42) generator_recruit(sdata->ckp); @@ -2578,7 +2581,8 @@ static void stratum_send_message(sdata_t *sdata, const stratum_instance_t *clien static sdata_t *select_sdata(ckpool_t *ckp, sdata_t *ckp_sdata) { proxy_t *current, *proxy, *subproxy, *best = NULL, *tmp, *tmpsub; - int best_id, best_subid = 0; + int best_subid = 0; + int64_t best_id; if (!ckp->proxy || ckp->passthrough) return ckp_sdata; @@ -3521,7 +3525,7 @@ static void submit_share(stratum_instance_t *client, const int64_t jobid, const char *msg; sprintf(enonce2, "%s%s", client->enonce1var, nonce2); - JSON_CPACK(json_msg, "{sIsssssssIsisi}", "jobid", jobid, "nonce2", enonce2, + JSON_CPACK(json_msg, "{sIsssssssIsIsi}", "jobid", jobid, "nonce2", enonce2, "ntime", ntime, "nonce", nonce, "client_id", client->id, "proxy", client->proxyid, "subproxy", client->subproxyid); msg = json_dumps(json_msg, 0); From 2f2d454763a77f032fddc0171abe9a9cc74d1a24 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Tue, 24 Feb 2015 14:01:37 +1100 Subject: [PATCH 252/544] Create new instances of proxies on reconnect instead of trying to overwrite old ones on reconnect --- src/generator.c | 66 +++++++++++++---------------- src/stratifier.c | 106 +++++++++++++---------------------------------- 2 files changed, 57 insertions(+), 115 deletions(-) diff --git a/src/generator.c b/src/generator.c index 008adf16..38e020d7 100644 --- a/src/generator.c +++ b/src/generator.c @@ -85,6 +85,7 @@ struct proxy_instance { server_instance_t *si; bool passthrough; int64_t id; /* Proxy server id*/ + int low_id; /* Low bits of id */ int subid; /* Subproxy id */ const char *auth; @@ -653,8 +654,8 @@ retry: } if (size < 3) { if (!proxi->subid) { - LOGWARNING("Proxy %ld %s Nonce2 length %d too small for fast miners", - proxi->id, proxi->si->url, size); + LOGWARNING("Proxy %d %s Nonce2 length %d too small for fast miners", + proxi->low_id, proxi->si->url, size); } else { LOGNOTICE("Proxy %ld:%d Nonce2 length %d too small for fast miners", proxi->id, proxi->subid, size); @@ -973,26 +974,10 @@ static void disable_subproxy(gdata_t *gdata, proxy_instance_t *proxi, proxy_inst store_proxy(gdata, subproxy); } -/* If the parent is no longer in use due to reconnect, we shouldn't use any of - * the child subproxies. */ -static void drop_subproxies(proxy_instance_t *proxi) -{ - proxy_instance_t *subproxy, *tmp; - - mutex_lock(&proxi->proxy_lock); - HASH_ITER(sh, proxi->subproxies, subproxy, tmp) { - if (!parent_proxy(subproxy)) { - send_stratifier_deadproxy(proxi->ckp, proxi->id, subproxy->subid); - subproxy->disabled = true; - Close(subproxy->cs->fd); - } - } - mutex_unlock(&proxi->proxy_lock); -} - static bool parse_reconnect(proxy_instance_t *proxi, json_t *val) { server_instance_t *newsi, *si = proxi->si; + int64_t high_id, low_id, new_id; proxy_instance_t *newproxi; ckpool_t *ckp = proxi->ckp; gdata_t *gdata = ckp->data; @@ -1049,8 +1034,12 @@ static bool parse_reconnect(proxy_instance_t *proxi, json_t *val) newsi = ckzalloc(sizeof(server_instance_t)); mutex_lock(&gdata->lock); - newsi->id = si->id; /* Inherit the old connection's id */ - ckp->servers[newsi->id] = newsi; + high_id = proxi->id >> 32; /* Use the high bits for the reconnect id */ + high_id++; + high_id <<= 32; + low_id = proxi->id & 0x00000000FFFFFFFFll; /* Use the low bits for the master id */ + new_id = high_id | low_id; + ckp->servers[low_id] = newsi; newsi->url = url; newsi->auth = strdup(si->auth); newsi->pass = strdup(si->pass); @@ -1063,14 +1052,14 @@ static bool parse_reconnect(proxy_instance_t *proxi, json_t *val) newproxi->ckp = ckp; newproxi->cs = &newsi->cs; newproxi->cs->ckp = ckp; - newproxi->id = newsi->id; + newproxi->low_id = low_id; + newproxi->id = new_id; newproxi->subproxy_count = ++proxi->subproxy_count; - HASH_REPLACE_I64(gdata->proxies, id, newproxi, proxi); + HASH_ADD_I64(gdata->proxies, id, newproxi); mutex_unlock(&gdata->lock); - /* Old proxy memory is basically lost here */ + proxi->disabled = true; prepare_proxy(newproxi); - drop_subproxies(proxi); out: return ret; } @@ -1247,21 +1236,21 @@ static bool auth_stratum(ckpool_t *ckp, connsock_t *cs, proxy_instance_t *proxi) val = json_msg_result(buf, &res_val, &err_val); if (!val) { - LOGWARNING("Proxy %ld:%d %s failed to get a json result in auth_stratum, got: %s", - proxi->id, proxi->subid, proxi->si->url, buf); + LOGWARNING("Proxy %d:%d %s failed to get a json result in auth_stratum, got: %s", + proxi->low_id, proxi->subid, proxi->si->url, buf); goto out; } if (err_val && !json_is_null(err_val)) { - LOGWARNING("Proxy %ld:%d %s failed to authorise in auth_stratum due to err_val, got: %s", - proxi->id, proxi->subid, proxi->si->url, buf); + LOGWARNING("Proxy %d:%d %s failed to authorise in auth_stratum due to err_val, got: %s", + proxi->low_id, proxi->subid, proxi->si->url, buf); goto out; } if (res_val) { ret = json_is_true(res_val); if (!ret) { - LOGWARNING("Proxy %ld:%d %s failed to authorise in auth_stratum, got: %s", - proxi->id, proxi->subid, proxi->si->url, buf); + LOGWARNING("Proxy %d:%d %s failed to authorise in auth_stratum, got: %s", + proxi->low_id, proxi->subid, proxi->si->url, buf); goto out; } } else { @@ -1764,8 +1753,8 @@ static void *passthrough_recv(void *arg) if (proxy_alive(ckp, si, proxi, cs, false, epfd)) { reconnect_generator(ckp); - LOGWARNING("Proxy %ld:%s connection established", - proxi->id, proxi->si->url); + LOGWARNING("Proxy %d:%s connection established", + proxi->low_id, proxi->si->url); } alive = proxi->alive; @@ -1970,7 +1959,7 @@ static void setup_proxies(ckpool_t *ckp, gdata_t *gdata) si = ckp->servers[i]; proxi = si->data; - proxi->id = i; + proxi->id = proxi->low_id = i; HASH_ADD_I64(gdata->proxies, id, proxi); if (ckp->passthrough) { create_pthread(&proxi->pth_precv, passthrough_recv, proxi); @@ -1991,8 +1980,10 @@ static proxy_instance_t *wait_best_proxy(ckpool_t *ckp, gdata_t *gdata) mutex_lock(&gdata->lock); HASH_ITER(hh, gdata->proxies, proxi, tmp) { + if (proxi->disabled) + continue; if (proxi->alive || subproxies_alive(proxi)) { - if (!ret || proxi->id < ret->id) + if (!ret || proxi->low_id < ret->low_id) ret = proxi; } } @@ -2029,8 +2020,8 @@ reconnect: proxi = cproxy; if (!ckp->passthrough) { connsock_t *cs = proxi->cs; - LOGWARNING("Successfully connected to proxy %ld %s:%s as proxy", - proxi->id, cs->url, cs->port); + LOGWARNING("Successfully connected to proxy %d %s:%s as proxy", + proxi->low_id, cs->url, cs->port); dealloc(buf); ASPRINTF(&buf, "proxy=%ld", proxi->id); async_send_proc(ckp, ckp->stratifier, buf); @@ -2134,7 +2125,6 @@ static int proxy_mode(ckpool_t *ckp, proc_instance_t *pi) for (i = 0; i < ckp->proxies; i++) { ckp->servers[i] = ckzalloc(sizeof(server_instance_t)); si = ckp->servers[i]; - si->id = i; si->url = strdup(ckp->proxyurl[i]); si->auth = strdup(ckp->proxyauth[i]); si->pass = strdup(ckp->proxypass[i]); diff --git a/src/stratifier.c b/src/stratifier.c index 172d5d19..2afbd530 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -306,6 +306,7 @@ struct proxy_base { proxy_t *next; /* For retired subproxies */ proxy_t *prev; int64_t id; + int low_id; int subid; double diff; @@ -1027,6 +1028,7 @@ static proxy_t *__generate_proxy(sdata_t *sdata, const int64_t id) proxy_t *proxy = ckzalloc(sizeof(proxy_t)); proxy->id = id; + proxy->low_id = id & 0xFFFFFFFF; proxy->sdata = duplicate_sdata(sdata); proxy->sdata->subproxy = proxy; proxy->sdata->verbose = true; @@ -1191,20 +1193,22 @@ static proxy_t *current_proxy(sdata_t *sdata) return proxy; } -static void dead_parent_proxy(sdata_t *sdata, const int64_t id) +static void dead_proxyid(sdata_t *sdata, const int64_t id, const int subid) { stratum_instance_t *client, *tmp; int reconnects = 0; int64_t headroom; proxy_t *proxy; + proxy = existing_subproxy(sdata, id, subid); + if (proxy) + proxy->dead = true; + LOGINFO("Stratifier dropping clients from proxy %ld:%d", id, subid); headroom = current_headroom(sdata, &proxy); - if (!proxy) - return; ck_rlock(&sdata->instance_lock); HASH_ITER(hh, sdata->stratum_instances, client, tmp) { - if (client->proxyid != id || client->subproxyid) + if (client->proxyid != id || client->subproxyid != subid) continue; headroom--; reconnects++; @@ -1216,55 +1220,18 @@ static void dead_parent_proxy(sdata_t *sdata, const int64_t id) ck_runlock(&sdata->instance_lock); if (reconnects) { - LOGNOTICE("%d clients flagged to reconnect from dead proxy %ld", - reconnects, id); + LOGNOTICE("%d clients flagged to reconnect from dead proxy %ld:%d", reconnects, + id, subid); if (headroom < 42) generator_recruit(sdata->ckp); } } -static void new_proxy(sdata_t *sdata, const int64_t id) -{ - proxy_t *proxy, *subproxy, *tmp, *proxy_list = NULL; - bool exists = false, current = false; - - mutex_lock(&sdata->proxy_lock); - HASH_FIND_I64(sdata->proxies, &id, proxy); - if (proxy) { - exists = true; - HASH_DEL(sdata->proxies, proxy); - DL_APPEND(sdata->retired_proxies, proxy); - if (proxy == sdata->proxy) - current = true; - proxy_list = proxy->subproxies; - HASH_DELETE(sh, proxy_list, proxy); - proxy->subproxies = NULL; - } - proxy = __generate_proxy(sdata, id); - if (current) - sdata->proxy = proxy; - /* The old proxy had subproxies on its list so steal its list and add - * ourselves to it. */ - if (proxy_list) { - HASH_DELETE(sh, proxy->subproxies, proxy); - proxy->subproxies = proxy_list; - HASH_ADD(sh, proxy->subproxies, subid, sizeof(int), proxy); - HASH_ITER(sh, proxy->subproxies, subproxy, tmp) - subproxy->parent = proxy; - } - mutex_unlock(&sdata->proxy_lock); - - if (exists) { - LOGNOTICE("Stratifier replaced old proxy %ld", id); - dead_parent_proxy(sdata, id); - } -} - static void update_subscribe(ckpool_t *ckp, const char *cmd) { sdata_t *sdata = ckp->data, *dsdata; + proxy_t *proxy, *old = NULL; const char *buf; - proxy_t *proxy; int64_t id = 0; int subid = 0; json_t *val; @@ -1289,13 +1256,19 @@ static void update_subscribe(ckpool_t *ckp, const char *cmd) return; } - if (!subid) { - new_proxy(sdata, id); + if (!subid) LOGNOTICE("Got updated subscribe for proxy %ld", id); - } else + else LOGINFO("Got updated subscribe for proxy %ld:%d", id, subid); - proxy = subproxy_by_id(sdata, id, subid); + /* Is this a replacement for an existing proxy id? */ + old = existing_subproxy(sdata, id, subid); + if (old) { + dead_proxyid(sdata, id, subid); + proxy = old; + proxy->dead = false; + } else + proxy = subproxy_by_id(sdata, id, subid); dsdata = proxy->sdata; ck_wlock(&dsdata->workbase_lock); @@ -1323,6 +1296,12 @@ static void update_subscribe(ckpool_t *ckp, const char *cmd) proxy->enonce1u.u64 = 0; ck_wunlock(&dsdata->workbase_lock); + /* Is this a replacement proxy for the current one */ + mutex_lock(&sdata->proxy_lock); + if (sdata->proxy && sdata->proxy->low_id == proxy->low_id) + sdata->proxy = proxy; + mutex_unlock(&sdata->proxy_lock); + if (subid) { LOGINFO("Upstream pool %ld:%d extranonce2 length %d, max proxy clients %"PRId64, id, subid, proxy->nonce2len, proxy->max_clients); @@ -2246,38 +2225,11 @@ static void set_proxy(sdata_t *sdata, const char *buf) static void dead_proxy(sdata_t *sdata, const char *buf) { - stratum_instance_t *client, *tmp; - int64_t headroom, id = 0; - int reconnects = 0; - proxy_t *proxy; + int64_t id = 0; int subid = 0; sscanf(buf, "deadproxy=%ld:%d", &id, &subid); - proxy = existing_subproxy(sdata, id, subid); - if (proxy) - proxy->dead = true; - LOGNOTICE("Stratifier dropping clients from proxy %ld:%d", id, subid); - headroom = current_headroom(sdata, &proxy); - - ck_rlock(&sdata->instance_lock); - HASH_ITER(hh, sdata->stratum_instances, client, tmp) { - if (client->proxyid != id || client->subproxyid != subid) - continue; - headroom--; - reconnects++; - if (client->reconnect && headroom > 0) - reconnect_client(sdata, client); - else - client->reconnect = true; - } - ck_runlock(&sdata->instance_lock); - - if (reconnects) { - LOGNOTICE("%d clients flagged to reconnect from dead proxy %ld:%d", reconnects, - id, subid); - if (headroom < 42) - generator_recruit(sdata->ckp); - } + dead_proxyid(sdata, id, subid); } /* Must hold a reference */ From 2b6b3fdbd7498a1a1cd1c92b773f145cc6887e3f Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Tue, 24 Feb 2015 14:14:45 +1100 Subject: [PATCH 253/544] Look for replacement proxies in wait_best_proxy as well --- src/generator.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/generator.c b/src/generator.c index 38e020d7..690d8ce2 100644 --- a/src/generator.c +++ b/src/generator.c @@ -1983,7 +1983,9 @@ static proxy_instance_t *wait_best_proxy(ckpool_t *ckp, gdata_t *gdata) if (proxi->disabled) continue; if (proxi->alive || subproxies_alive(proxi)) { - if (!ret || proxi->low_id < ret->low_id) + if ((!ret) || + (proxi->low_id < ret->low_id) || + (proxi->low_id == ret->low_id && proxi->id > ret->id)) ret = proxi; } } From 668d96710e1673edeed5ff918677c25291a98921 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Tue, 24 Feb 2015 14:35:41 +1100 Subject: [PATCH 254/544] Check lowids in selecting stratum data --- src/stratifier.c | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/stratifier.c b/src/stratifier.c index 2afbd530..6297b5e9 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -2533,8 +2533,8 @@ static void stratum_send_message(sdata_t *sdata, const stratum_instance_t *clien static sdata_t *select_sdata(ckpool_t *ckp, sdata_t *ckp_sdata) { proxy_t *current, *proxy, *subproxy, *best = NULL, *tmp, *tmpsub; - int best_subid = 0; - int64_t best_id; + int best_subid = 0, best_lowid; + int64_t best_id = 0; if (!ckp->proxy || ckp->passthrough) return ckp_sdata; @@ -2543,7 +2543,7 @@ static sdata_t *select_sdata(ckpool_t *ckp, sdata_t *ckp_sdata) LOGWARNING("No proxy available yet to generate subscribes"); return NULL; } - best_id = ckp_sdata->proxy_count; + best_lowid = ckp_sdata->proxy_count; mutex_lock(&ckp_sdata->proxy_lock); HASH_ITER(hh, ckp_sdata->proxies, proxy, tmp) { @@ -2566,8 +2566,9 @@ static sdata_t *select_sdata(ckpool_t *ckp, sdata_t *ckp_sdata) max_headroom = subproxy_headroom; } } - if (best && best->id < best_id) { + if (best && (best->low_id < best_lowid || (best->low_id == best_lowid && best->id > best_id))) { best_id = best->id; + best_lowid = best->low_id; best_subid = best->subid; if (best == current) break; From 9234ced6f29fcfdae347a9a8fb8157ae0ebfd91b Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Tue, 24 Feb 2015 14:39:28 +1100 Subject: [PATCH 255/544] Give subproxy the lowid of the parent proxy --- src/stratifier.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/stratifier.c b/src/stratifier.c index 6297b5e9..e3c431e1 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -1045,6 +1045,7 @@ static proxy_t *__generate_subproxy(sdata_t *sdata, proxy_t *proxy, const int su proxy_t *subproxy = ckzalloc(sizeof(proxy_t)); subproxy->id = proxy->id; + subproxy->low_id = proxy->low_id; subproxy->subid = subid; HASH_ADD(sh, proxy->subproxies, subid, sizeof(int), subproxy); subproxy->sdata = duplicate_sdata(sdata); From 73dab076522e521aed2dd33cd1466768fcaa1ecf Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Tue, 24 Feb 2015 15:32:42 +1100 Subject: [PATCH 256/544] Add proxy selection info --- src/stratifier.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/stratifier.c b/src/stratifier.c index e3c431e1..48fe404b 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -2609,6 +2609,10 @@ static json_t *parse_subscribe(stratum_instance_t *client, const int64_t client_ stratum_send_message(ckp_sdata, client, "Pool Initialising"); return json_string("Initialising"); } + if (ckp->proxy) { + LOGINFO("Current %ld, selecting proxy %ld:%d for client %"PRId64, ckp_sdata->proxy->id, + sdata->subproxy->id, sdata->subproxy->subid, client->id); + } client->sdata = sdata; arr_size = json_array_size(params_val); From 7009fb10772758a74a7e56aa1c1eee658fc28fdd Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Tue, 24 Feb 2015 15:37:48 +1100 Subject: [PATCH 257/544] Reconnect clients whenever we detect a new notify from a subproxy of the current proxy --- src/stratifier.c | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/src/stratifier.c b/src/stratifier.c index 48fe404b..67a0f884 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -281,7 +281,6 @@ struct stratum_instance { proxy_t *proxy; /* Proxy this is bound to in proxy mode */ int64_t proxyid; /* Which proxy id */ int subproxyid; /* Which subproxy */ - int64_t notify_id; /* Which notify_id from the subproxy did we join */ }; struct share { @@ -1271,6 +1270,7 @@ static void update_subscribe(ckpool_t *ckp, const char *cmd) } else proxy = subproxy_by_id(sdata, id, subid); dsdata = proxy->sdata; + proxy->notify_id = -1; /* Reset this so we can detect its first notify */ ck_wlock(&dsdata->workbase_lock); proxy->subscribed = true; @@ -1412,8 +1412,14 @@ static void update_notify(ckpool_t *ckp, const char *cmd) LOGNOTICE("Block hash on proxy %ld changed to %s", id, dsdata->lastswaphash); } - if (parent_proxy(proxy) && proxy == current_proxy(sdata)) - reconnect_clients(sdata); + /* Is this the first notify from this proxy? If so, we know we have + * more room and can reconnect clients if we're a subproxy of the + * current proxy. */ + if (proxy->notify_id == -1) { + proxy->notify_id = wb->id; + if (proxy->parent == current_proxy(sdata)) + reconnect_clients(sdata); + } clean |= new_block; LOGINFO("Proxy %ld:%d broadcast updated stratum notify with%s clean", id, subid, clean ? "" : "out"); From 484bdbaad21a4ca82b75cbbae293d22b04b4d73e Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Tue, 24 Feb 2015 15:39:50 +1100 Subject: [PATCH 258/544] Cosmetic --- src/generator.c | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/generator.c b/src/generator.c index 690d8ce2..5678241f 100644 --- a/src/generator.c +++ b/src/generator.c @@ -1840,8 +1840,8 @@ static void *proxy_recv(void *arg) } if (proxy_alive(ckp, si, proxi, cs, false, epfd)) { - LOGWARNING("Proxy %ld:%s connection established", - proxi->id, proxi->si->url); + LOGWARNING("Proxy %d:%s connection established", + proxi->low_id, proxi->si->url); } alive = proxi->alive; reconnect_generator(ckp); @@ -1858,8 +1858,8 @@ static void *proxy_recv(void *arg) while (!subproxies_alive(proxi)) { reconnect_proxy(proxi); if (alive) { - LOGWARNING("Proxy %ld:%s failed, attempting reconnect", - proxi->id, proxi->si->url); + LOGWARNING("Proxy %d:%s failed, attempting reconnect", + proxi->low_id, proxi->si->url); alive = false; reconnect_generator(ckp); } @@ -1870,7 +1870,7 @@ static void *proxy_recv(void *arg) /* Wait 30 seconds before declaring this upstream pool alive * to prevent switching to unstable pools. */ if (!alive && (!current_proxy(gdata) || time(NULL) - proxi->reconnect_time > 30)) { - LOGWARNING("Proxy %ld:%s recovered", proxi->id, proxi->si->url); + LOGWARNING("Proxy %d:%s recovered", proxi->low_id, proxi->si->url); proxi->reconnect_time = 0; reconnect_generator(ckp); alive = true; @@ -1920,8 +1920,8 @@ static void *proxy_recv(void *arg) * pool is up */ disable_subproxy(gdata, proxi, subproxy); if (parent_proxy(subproxy)) { - LOGWARNING("Proxy %ld:%s reconnect issue, dropping existing connection", - subproxy->id, subproxy->si->url); + LOGWARNING("Proxy %d:%s reconnect issue, dropping existing connection", + subproxy->low_id, subproxy->si->url); break; } else recruit_subproxy(proxi); From 75b9a4f1405d3e73f2982d2f1c7886633f02fe12 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Tue, 24 Feb 2015 15:49:09 +1100 Subject: [PATCH 259/544] Reconnect clients on every notify from a subproxy --- src/stratifier.c | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/src/stratifier.c b/src/stratifier.c index 67a0f884..aa5c4295 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -320,7 +320,6 @@ struct proxy_base { bool subscribed; bool notified; - int64_t notify_id; /* What ID was the first notify from this proxy */ int64_t clients; int64_t max_clients; @@ -1270,7 +1269,6 @@ static void update_subscribe(ckpool_t *ckp, const char *cmd) } else proxy = subproxy_by_id(sdata, id, subid); dsdata = proxy->sdata; - proxy->notify_id = -1; /* Reset this so we can detect its first notify */ ck_wlock(&dsdata->workbase_lock); proxy->subscribed = true; @@ -1412,14 +1410,8 @@ static void update_notify(ckpool_t *ckp, const char *cmd) LOGNOTICE("Block hash on proxy %ld changed to %s", id, dsdata->lastswaphash); } - /* Is this the first notify from this proxy? If so, we know we have - * more room and can reconnect clients if we're a subproxy of the - * current proxy. */ - if (proxy->notify_id == -1) { - proxy->notify_id = wb->id; - if (proxy->parent == current_proxy(sdata)) - reconnect_clients(sdata); - } + if (proxy->parent == current_proxy(sdata)) + reconnect_clients(sdata); clean |= new_block; LOGINFO("Proxy %ld:%d broadcast updated stratum notify with%s clean", id, subid, clean ? "" : "out"); @@ -2226,8 +2218,6 @@ static void set_proxy(sdata_t *sdata, const char *buf) mutex_unlock(&sdata->proxy_lock); LOGNOTICE("Stratifier setting active proxy to %ld", id); - if (proxy->notify_id != -1) - reconnect_clients(sdata); } static void dead_proxy(sdata_t *sdata, const char *buf) From 55d616817bc2be21b020eadb3d9c9d52be441009 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Tue, 24 Feb 2015 16:14:19 +1100 Subject: [PATCH 260/544] Use the proxy id to determine if we should try reconnecting clients on notify --- src/stratifier.c | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/src/stratifier.c b/src/stratifier.c index aa5c4295..375acb4b 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -327,7 +327,6 @@ struct proxy_base { int64_t headroom; enonce1_t enonce1u; - proxy_t *parent; /* Parent proxy - set to self on parent itself */ proxy_t *subproxies; /* Hashlist of subproxies sorted by subid */ sdata_t *sdata; /* Unique stratifer data for each subproxy */ bool dead; @@ -1030,7 +1029,6 @@ static proxy_t *__generate_proxy(sdata_t *sdata, const int64_t id) proxy->sdata = duplicate_sdata(sdata); proxy->sdata->subproxy = proxy; proxy->sdata->verbose = true; - proxy->parent = proxy; /* subid == 0 on parent proxy */ HASH_ADD(sh, proxy->subproxies, subid, sizeof(int), proxy); HASH_ADD_I64(sdata->proxies, id, proxy); @@ -1048,7 +1046,6 @@ static proxy_t *__generate_subproxy(sdata_t *sdata, proxy_t *proxy, const int su HASH_ADD(sh, proxy->subproxies, subid, sizeof(int), subproxy); subproxy->sdata = duplicate_sdata(sdata); subproxy->sdata->subproxy = subproxy; - subproxy->parent = proxy; return subproxy; } @@ -1181,6 +1178,7 @@ static void reconnect_clients(sdata_t *sdata) } } +#if 0 static proxy_t *current_proxy(sdata_t *sdata) { proxy_t *proxy; @@ -1191,6 +1189,7 @@ static proxy_t *current_proxy(sdata_t *sdata) return proxy; } +#endif static void dead_proxyid(sdata_t *sdata, const int64_t id, const int subid) { @@ -1312,20 +1311,16 @@ static void update_subscribe(ckpool_t *ckp, const char *cmd) json_decref(val); } -static inline bool parent_proxy(const proxy_t *proxy) -{ - return (proxy->parent == proxy); -} - static void update_notify(ckpool_t *ckp, const char *cmd) { sdata_t *sdata = ckp->data, *dsdata; bool new_block = false, clean; + proxy_t *proxy, *current; + int64_t current_id; int i, subid = 0; char header[228]; const char *buf; int64_t id = 0; - proxy_t *proxy; workbase_t *wb; json_t *val; @@ -1410,7 +1405,14 @@ static void update_notify(ckpool_t *ckp, const char *cmd) LOGNOTICE("Block hash on proxy %ld changed to %s", id, dsdata->lastswaphash); } - if (proxy->parent == current_proxy(sdata)) + mutex_lock(&sdata->proxy_lock); + current = sdata->proxy; + if (unlikely(!current)) + current = sdata->proxy = proxy; + current_id = current->id; + mutex_unlock(&sdata->proxy_lock); + + if (proxy->id == current_id) reconnect_clients(sdata); clean |= new_block; LOGINFO("Proxy %ld:%d broadcast updated stratum notify with%s clean", id, From dd4b563cb990f2e67f68af32a7076f0e87deba20 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Tue, 24 Feb 2015 16:49:17 +1100 Subject: [PATCH 261/544] Clean up and test return values of share decoding in submit_share --- src/generator.c | 31 ++++++++++++++++++++++++------- 1 file changed, 24 insertions(+), 7 deletions(-) diff --git a/src/generator.c b/src/generator.c index 5678241f..19177d32 100644 --- a/src/generator.c +++ b/src/generator.c @@ -1355,37 +1355,50 @@ static void submit_share(gdata_t *gdata, json_t *val) proxy_instance_t *proxy, *proxi; ckpool_t *ckp = gdata->ckp; int64_t client_id, id; + bool success = false; stratum_msg_t *msg; share_msg_t *share; int subid; /* Get the client id so we can tell the stratifier to drop it if the * proxy it's bound to is not functional */ - json_get_int64(&client_id, val, "client_id"); - json_get_int64(&id, val, "proxy"); - json_get_int(&subid, val, "subproxy"); - + if (unlikely(!json_get_int64(&client_id, val, "client_id"))) { + LOGWARNING("Got no client_id in share"); + goto out; + } + if (unlikely(!json_get_int64(&id, val, "proxy"))) { + LOGWARNING("Got no proxy in share"); + goto out; + } + if (unlikely(!json_get_int(&subid, val, "subproxy"))) { + LOGWARNING("Got no subproxy in share"); + goto out; + } proxy = proxy_by_id(gdata, id); if (unlikely(!proxy)) { LOGNOTICE("Client %"PRId64" sending shares to non existent proxy %ld, dropping", client_id, id); + send_stratifier_deadproxy(ckp, id, subid); stratifier_reconnect_client(ckp, client_id); - return json_decref(val); + goto out; } proxi = subproxy_by_id(proxy, subid); if (unlikely(!proxi)) { LOGNOTICE("Client %"PRId64" sending shares to non existent subproxy %ld:%d, dropping", client_id, id, subid); + send_stratifier_deadproxy(ckp, id, subid); stratifier_reconnect_client(ckp, client_id); - return json_decref(val); + goto out; } if (!proxi->alive) { LOGNOTICE("Client %"PRId64" sending shares to dead subproxy %ld:%d, dropping", client_id, id, subid); + send_stratifier_deadproxy(ckp, id, subid); stratifier_reconnect_client(ckp, client_id); - return json_decref(val); + goto out; } + success = true; msg = ckzalloc(sizeof(stratum_msg_t)); share = ckzalloc(sizeof(share_msg_t)); share->submit_time = time(NULL); @@ -1405,6 +1418,10 @@ static void submit_share(gdata_t *gdata, json_t *val) DL_APPEND(proxy->psends, msg); pthread_cond_signal(&proxy->psend_cond); mutex_unlock(&proxy->psend_lock); + +out: + if (!success) + json_decref(val); } static void clear_notify(notify_instance_t *ni) From 2054f116d7e137aec7d42d5e34fc87b8eb0cf14c Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Tue, 24 Feb 2015 16:56:33 +1100 Subject: [PATCH 262/544] Only reconnect with the parent proxy notify --- src/stratifier.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/stratifier.c b/src/stratifier.c index 375acb4b..d669ffc5 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -1412,7 +1412,7 @@ static void update_notify(ckpool_t *ckp, const char *cmd) current_id = current->id; mutex_unlock(&sdata->proxy_lock); - if (proxy->id == current_id) + if (!proxy->subid && proxy->id == current_id) reconnect_clients(sdata); clean |= new_block; LOGINFO("Proxy %ld:%d broadcast updated stratum notify with%s clean", id, From 0e70b8bf196fb3f6b9038f67173995300b14408b Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Tue, 24 Feb 2015 17:16:13 +1100 Subject: [PATCH 263/544] Reuse proxies that are still alive and no longer have any clients bound to them --- src/stratifier.c | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/src/stratifier.c b/src/stratifier.c index d669ffc5..ceff4d3b 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -1495,6 +1495,7 @@ static void update_diff(ckpool_t *ckp, const char *cmd) ck_runlock(&sdata->instance_lock); } +#if 0 static void generator_drop_proxy(ckpool_t *ckp, const int64_t id, const int subid) { char msg[256]; @@ -1502,6 +1503,7 @@ static void generator_drop_proxy(ckpool_t *ckp, const int64_t id, const int subi sprintf(msg, "dropproxy=%ld:%d", id, subid); send_generator(ckp, msg, GEN_LAX); } +#endif static void free_proxy(proxy_t *proxy) { @@ -1509,14 +1511,13 @@ static void free_proxy(proxy_t *proxy) free(proxy); } -/* Remove subproxies that are flagged dead or have used up their quota of - * clients and inform the generator if it is still alive. Then see if there +/* Remove subproxies that are flagged dead. Then see if there * are any retired proxies that no longer have any other subproxies and reap * those. */ static void reap_proxies(ckpool_t *ckp, sdata_t *sdata) { proxy_t *proxy, *proxytmp, *subproxy, *subtmp; - int used = 0, dead = 0, retired = 0; + int dead = 0, retired = 0; if (!ckp->proxy) return; @@ -1529,10 +1530,9 @@ static void reap_proxies(ckpool_t *ckp, sdata_t *sdata) if (subproxy->bound_clients) continue; if (!subproxy->dead) { - if (subproxy->clients < subproxy->max_clients) - continue; - generator_drop_proxy(ckp, subproxy->id, subproxy->subid); - used++; + /* Reset the counter to reuse this proxy */ + subproxy->clients = 0; + continue; } else { dead++; } @@ -1552,9 +1552,9 @@ static void reap_proxies(ckpool_t *ckp, sdata_t *sdata) } mutex_unlock(&sdata->proxy_lock); - if (used || dead || retired) { - LOGNOTICE("Stratifier discarded %d used, %d dead and %d retired proxies", - used, dead, retired); + if (dead || retired) { + LOGNOTICE("Stratifier discarded %d dead and %d retired proxies", + dead, retired); } } From 7a4cab1a88c24e4558fcdd0717cbd9a3199f70b3 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Tue, 24 Feb 2015 17:38:32 +1100 Subject: [PATCH 264/544] Reset parent proxy as well in reap_proxies and add some sanity checks, removing now unused retired proxies list --- src/stratifier.c | 44 +++++++++++++++++++++----------------------- 1 file changed, 21 insertions(+), 23 deletions(-) diff --git a/src/stratifier.c b/src/stratifier.c index ceff4d3b..dcec3ec8 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -400,7 +400,6 @@ struct stratifier_data { int proxy_count; /* Total proxies generated (not necessarily still alive) */ proxy_t *proxy; /* Current proxy in use */ proxy_t *proxies; /* Hashlist of all proxies */ - proxy_t *retired_proxies; /* Hashlist of proxies now no longer in user */ mutex_t proxy_lock; /* Protects all proxy data */ proxy_t *subproxy; /* Which subproxy this sdata belongs to in proxy mode */ }; @@ -1288,8 +1287,7 @@ static void update_subscribe(ckpool_t *ckp, const char *cmd) proxy->enonce2varlen = proxy->nonce2len - proxy->enonce1varlen; proxy->max_clients = 1ll << (proxy->enonce1varlen * 8); /* Reset the enonce1u in case this is a resubscribe of an existing - * parent proxy. All clients previously bound will be disconnected so - * we can start with a fresh count. */ + * parent proxy. */ proxy->clients = 0; proxy->enonce1u.u64 = 0; ck_wunlock(&dsdata->workbase_lock); @@ -1517,7 +1515,7 @@ static void free_proxy(proxy_t *proxy) static void reap_proxies(ckpool_t *ckp, sdata_t *sdata) { proxy_t *proxy, *proxytmp, *subproxy, *subtmp; - int dead = 0, retired = 0; + int dead = 0; if (!ckp->proxy) return; @@ -1525,37 +1523,37 @@ static void reap_proxies(ckpool_t *ckp, sdata_t *sdata) mutex_lock(&sdata->proxy_lock); HASH_ITER(hh, sdata->proxies, proxy, proxytmp) { HASH_ITER(sh, proxy->subproxies, subproxy, subtmp) { + if (!subproxy->bound_clients && !subproxy->dead) { + /* Reset the counter to reuse this proxy */ + subproxy->clients = 0; + subproxy->enonce1u.u64 = 0; + continue; + } if (proxy == subproxy) continue; if (subproxy->bound_clients) continue; - if (!subproxy->dead) { - /* Reset the counter to reuse this proxy */ - subproxy->clients = 0; + if (!subproxy->dead) + continue; + if (unlikely(!subproxy->subid)) { + LOGWARNING("Unexepectedly found proxy %ld:%d as subproxy of %ld:%d", + subproxy->id, subproxy->subid, proxy->id, proxy->subid); continue; - } else { - dead++; } + if (unlikely(subproxy == sdata->proxy)) { + LOGWARNING("Unexepectedly found proxy %ld:%d as current", + subproxy->id, subproxy->subid); + continue; + } + dead++; HASH_DELETE(sh, proxy->subproxies, subproxy); free_proxy(subproxy); } } - - DL_FOREACH_SAFE(sdata->retired_proxies, proxy, proxytmp) { - if (unlikely(proxy->bound_clients)) - continue; - if (HASH_CNT(sh, proxy->subproxies) == 1) { - retired++; - DL_DELETE(sdata->retired_proxies, proxy); - free_proxy(proxy); - } - } mutex_unlock(&sdata->proxy_lock); - if (dead || retired) { - LOGNOTICE("Stratifier discarded %d dead and %d retired proxies", - dead, retired); - } + if (dead) + LOGNOTICE("Stratifier discarded %d dead proxies", dead); } /* Enter with instance_lock held */ From 95102da85492a0f0d13e1bdfb8c08fee9b63b00b Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Tue, 24 Feb 2015 17:50:38 +1100 Subject: [PATCH 265/544] Only set current proxy to a parent proxy --- src/stratifier.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/stratifier.c b/src/stratifier.c index dcec3ec8..aa29cf4a 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -1294,7 +1294,7 @@ static void update_subscribe(ckpool_t *ckp, const char *cmd) /* Is this a replacement proxy for the current one */ mutex_lock(&sdata->proxy_lock); - if (sdata->proxy && sdata->proxy->low_id == proxy->low_id) + if (sdata->proxy && sdata->proxy->low_id == proxy->low_id && !proxy->subid) sdata->proxy = proxy; mutex_unlock(&sdata->proxy_lock); From 6f08c301e9a87fcfe48f27b110d80a3055bae9ed Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Tue, 24 Feb 2015 18:18:22 +1100 Subject: [PATCH 266/544] Tweak reconnects --- src/stratifier.c | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/src/stratifier.c b/src/stratifier.c index aa29cf4a..7f2a28c4 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -1160,9 +1160,11 @@ static void reconnect_clients(sdata_t *sdata) HASH_ITER(hh, sdata->stratum_instances, client, tmpclient) { if (client->proxyid == proxy->id) continue; + if (headroom < 1) + break; headroom--; reconnects++; - if (client->reconnect && headroom > 0) + if (client->reconnect) reconnect_client(sdata, client); else client->reconnect = true; @@ -1172,9 +1174,9 @@ static void reconnect_clients(sdata_t *sdata) if (reconnects) { LOGNOTICE("%d clients flagged for reconnect to proxy %ld", reconnects, proxy->id); - if (headroom < 42) - generator_recruit(sdata->ckp); } + if (headroom < 42) + generator_recruit(sdata->ckp); } #if 0 @@ -1207,9 +1209,11 @@ static void dead_proxyid(sdata_t *sdata, const int64_t id, const int subid) HASH_ITER(hh, sdata->stratum_instances, client, tmp) { if (client->proxyid != id || client->subproxyid != subid) continue; + if (headroom < 1) + break; headroom--; reconnects++; - if (client->reconnect && headroom > 0) + if (client->reconnect) reconnect_client(sdata, client); else client->reconnect = true; @@ -1219,9 +1223,9 @@ static void dead_proxyid(sdata_t *sdata, const int64_t id, const int subid) if (reconnects) { LOGNOTICE("%d clients flagged to reconnect from dead proxy %ld:%d", reconnects, id, subid); - if (headroom < 42) - generator_recruit(sdata->ckp); } + if (headroom < 42) + generator_recruit(sdata->ckp); } static void update_subscribe(ckpool_t *ckp, const char *cmd) From 565d253e3f273eb402e8c8f1ee0e1697e9734c5f Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Tue, 24 Feb 2015 18:28:20 +1100 Subject: [PATCH 267/544] Reassess on all notifies --- src/stratifier.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/stratifier.c b/src/stratifier.c index 7f2a28c4..017e356e 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -1414,7 +1414,7 @@ static void update_notify(ckpool_t *ckp, const char *cmd) current_id = current->id; mutex_unlock(&sdata->proxy_lock); - if (!proxy->subid && proxy->id == current_id) + if (proxy->id == current_id) reconnect_clients(sdata); clean |= new_block; LOGINFO("Proxy %ld:%d broadcast updated stratum notify with%s clean", id, From ad5def42d2d51424bfc43852408f0279701d123c Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Tue, 24 Feb 2015 19:58:48 +1100 Subject: [PATCH 268/544] Reassess headroom and recruit extra proxies upon receiving each new subscribe --- src/stratifier.c | 37 ++++++++++++++++++++++++++++++++++--- 1 file changed, 34 insertions(+), 3 deletions(-) diff --git a/src/stratifier.c b/src/stratifier.c index 017e356e..602fe6d0 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -1228,10 +1228,38 @@ static void dead_proxyid(sdata_t *sdata, const int64_t id, const int subid) generator_recruit(sdata->ckp); } +static void reassess_headroom(sdata_t *sdata, const proxy_t *proxy) +{ + stratum_instance_t *client, *tmpclient; + proxy_t *subproxy, *tmp; + int64_t headroom = 0; + + mutex_lock(&sdata->proxy_lock); + HASH_ITER(sh, proxy->subproxies, subproxy, tmp) { + if (subproxy->dead) + continue; + headroom += subproxy->max_clients - subproxy->clients; + } + mutex_unlock(&sdata->proxy_lock); + + ck_rlock(&sdata->instance_lock); + HASH_ITER(hh, sdata->stratum_instances, client, tmpclient) { + if (client->dropped || client->reconnect) + continue; + if (client->proxyid != proxy->id) + headroom--; + } + ck_runlock(&sdata->instance_lock); + + if (headroom < 0) + generator_recruit(sdata->ckp); +} + static void update_subscribe(ckpool_t *ckp, const char *cmd) { sdata_t *sdata = ckp->data, *dsdata; proxy_t *proxy, *old = NULL; + bool current = false; const char *buf; int64_t id = 0; int subid = 0; @@ -1298,10 +1326,14 @@ static void update_subscribe(ckpool_t *ckp, const char *cmd) /* Is this a replacement proxy for the current one */ mutex_lock(&sdata->proxy_lock); - if (sdata->proxy && sdata->proxy->low_id == proxy->low_id && !proxy->subid) + if (sdata->proxy && sdata->proxy->low_id == proxy->low_id && !proxy->subid) { + current = true; sdata->proxy = proxy; + } mutex_unlock(&sdata->proxy_lock); + if (current) + reassess_headroom(sdata, proxy); if (subid) { LOGINFO("Upstream pool %ld:%d extranonce2 length %d, max proxy clients %"PRId64, id, subid, proxy->nonce2len, proxy->max_clients); @@ -1309,7 +1341,6 @@ static void update_subscribe(ckpool_t *ckp, const char *cmd) LOGNOTICE("Upstream pool %ld extranonce2 length %d, max proxy clients %"PRId64, id, proxy->nonce2len, proxy->max_clients); } - json_decref(val); } @@ -1414,7 +1445,7 @@ static void update_notify(ckpool_t *ckp, const char *cmd) current_id = current->id; mutex_unlock(&sdata->proxy_lock); - if (proxy->id == current_id) + if (!proxy->subid && proxy->id == current_id) reconnect_clients(sdata); clean |= new_block; LOGINFO("Proxy %ld:%d broadcast updated stratum notify with%s clean", id, From 89f22deaff21dc337edb66f8b95f81bc9371dd66 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Tue, 24 Feb 2015 20:32:57 +1100 Subject: [PATCH 269/544] Remove fds from the proxy recv epoll when they're being closed --- src/generator.c | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/src/generator.c b/src/generator.c index 19177d32..06111319 100644 --- a/src/generator.c +++ b/src/generator.c @@ -741,8 +741,10 @@ retry: goto retry; out: - if (!ret) + if (!ret && cs->fd > 0) { + epoll_ctl(proxi->epfd, EPOLL_CTL_DEL, cs->fd, NULL); Close(cs->fd); + } return ret; } @@ -965,7 +967,10 @@ static void disable_subproxy(gdata_t *gdata, proxy_instance_t *proxi, proxy_inst /* Make sure subproxy is still in the list */ subproxy = __subproxy_by_id(proxi, subproxy->subid); if (subproxy) { - Close(subproxy->cs->fd); + if (subproxy->cs->fd > 0) { + epoll_ctl(proxi->epfd, EPOLL_CTL_DEL, subproxy->cs->fd, NULL); + Close(subproxy->cs->fd); + } HASH_DELETE(sh, proxi->subproxies, subproxy); } mutex_unlock(&proxi->proxy_lock); @@ -1216,7 +1221,10 @@ static bool auth_stratum(ckpool_t *ckp, connsock_t *cs, proxy_instance_t *proxi) if (!ret) { LOGNOTICE("Proxy %ld:%d %s failed to send message in auth_stratum", proxi->id, proxi->subid, proxi->si->url); - Close(cs->fd); + if (cs->fd > 0) { + epoll_ctl(proxi->epfd, EPOLL_CTL_DEL, cs->fd, NULL); + Close(cs->fd); + } goto out; } @@ -1641,7 +1649,10 @@ out: if (!ret) { send_stratifier_deadproxy(ckp, proxi->id, proxi->subid); /* Close and invalidate the file handle */ - Close(cs->fd); + if (cs->fd > 0) { + epoll_ctl(proxi->epfd, EPOLL_CTL_DEL, cs->fd, NULL); + Close(cs->fd); + } } else { keep_sockalive(cs->fd); event.events = EPOLLIN; From cd342551a876dacd02a9847315ab9932501f6d9e Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Tue, 24 Feb 2015 20:38:44 +1100 Subject: [PATCH 270/544] Count reconnecting clients as part of the headroom consumed --- src/stratifier.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/stratifier.c b/src/stratifier.c index 602fe6d0..f31dac4f 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -1244,9 +1244,9 @@ static void reassess_headroom(sdata_t *sdata, const proxy_t *proxy) ck_rlock(&sdata->instance_lock); HASH_ITER(hh, sdata->stratum_instances, client, tmpclient) { - if (client->dropped || client->reconnect) + if (client->dropped) continue; - if (client->proxyid != proxy->id) + if (client->reconnect || client->proxyid != proxy->id) headroom--; } ck_runlock(&sdata->instance_lock); From daa478f1f5e506b60cec2c1391610e6281caaa09 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Tue, 24 Feb 2015 20:51:16 +1100 Subject: [PATCH 271/544] Off by one on headroom --- src/stratifier.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/stratifier.c b/src/stratifier.c index f31dac4f..c154713e 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -1251,7 +1251,7 @@ static void reassess_headroom(sdata_t *sdata, const proxy_t *proxy) } ck_runlock(&sdata->instance_lock); - if (headroom < 0) + if (headroom < 1) generator_recruit(sdata->ckp); } From a6ac868cd305490f10f982d556f53f72e1b50b5e Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Tue, 24 Feb 2015 21:16:37 +1100 Subject: [PATCH 272/544] Rework subproxy recruitment to allow requests to stack but abandon them if the subproxy is not alive --- src/generator.c | 35 +++++++++++++++++++++++++------- src/stratifier.c | 52 +++++++++--------------------------------------- 2 files changed, 37 insertions(+), 50 deletions(-) diff --git a/src/generator.c b/src/generator.c index 06111319..a873eb08 100644 --- a/src/generator.c +++ b/src/generator.c @@ -108,7 +108,7 @@ struct proxy_instance { bool disabled; /* Subproxy no longer to be used */ bool reconnect; /* We need to drop and reconnect */ bool reconnecting; /* Testing in progress */ - bool recruiting; /* Recruiting in progress */ + int recruit; /* Recruiting in progress */ bool alive; mutex_t notify_lock; @@ -1707,27 +1707,48 @@ static void *proxy_recruit(void *arg) proxy_instance_t *proxy, *parent = (proxy_instance_t *)arg; ckpool_t *ckp = parent->ckp; gdata_t *gdata = ckp->data; + bool recruit, alive; pthread_detach(pthread_self()); +retry: + recruit = false; proxy = create_subproxy(gdata, parent); - if (!proxy_alive(ckp, proxy->si, proxy, proxy->cs, false, parent->epfd)) { + alive = proxy_alive(ckp, proxy->si, proxy, proxy->cs, false, parent->epfd); + if (!alive) { LOGNOTICE("Subproxy failed proxy_alive testing"); store_proxy(gdata, proxy); } else add_subproxy(parent, proxy); - parent->recruiting = false; + + mutex_lock(&parent->proxy_lock); + if (alive) { + if (--parent->recruit > 0) + recruit = true; + } + mutex_unlock(&parent->proxy_lock); + + if (recruit) + goto retry; + return NULL; } +/* Allow up to 42 recruit requests to accumulate */ static void recruit_subproxy(proxy_instance_t *proxi) { + bool recruit = false; pthread_t pth; - if (proxi->recruiting) - return; - proxi->recruiting = true; - create_pthread(&pth, proxy_recruit, proxi); + mutex_lock(&proxi->proxy_lock); + if (!proxi->recruit++ > 0) + recruit = true; + else if (proxi->recruit > 42) + proxi->recruit = 42; + mutex_unlock(&proxi->proxy_lock); + + if (recruit) + create_pthread(&pth, proxy_recruit, proxi); } static void *proxy_reconnect(void *arg) diff --git a/src/stratifier.c b/src/stratifier.c index c154713e..12e1a8a5 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -1160,10 +1160,9 @@ static void reconnect_clients(sdata_t *sdata) HASH_ITER(hh, sdata->stratum_instances, client, tmpclient) { if (client->proxyid == proxy->id) continue; - if (headroom < 1) - break; - headroom--; reconnects++; + if (headroom-- < 1) + break; if (client->reconnect) reconnect_client(sdata, client); else @@ -1174,9 +1173,9 @@ static void reconnect_clients(sdata_t *sdata) if (reconnects) { LOGNOTICE("%d clients flagged for reconnect to proxy %ld", reconnects, proxy->id); + if (headroom < 42) + generator_recruit(sdata->ckp); } - if (headroom < 42) - generator_recruit(sdata->ckp); } #if 0 @@ -1209,10 +1208,9 @@ static void dead_proxyid(sdata_t *sdata, const int64_t id, const int subid) HASH_ITER(hh, sdata->stratum_instances, client, tmp) { if (client->proxyid != id || client->subproxyid != subid) continue; - if (headroom < 1) - break; - headroom--; reconnects++; + if (headroom-- < 1) + break; if (client->reconnect) reconnect_client(sdata, client); else @@ -1223,43 +1221,15 @@ static void dead_proxyid(sdata_t *sdata, const int64_t id, const int subid) if (reconnects) { LOGNOTICE("%d clients flagged to reconnect from dead proxy %ld:%d", reconnects, id, subid); + if (headroom < 42) + generator_recruit(sdata->ckp); } - if (headroom < 42) - generator_recruit(sdata->ckp); -} - -static void reassess_headroom(sdata_t *sdata, const proxy_t *proxy) -{ - stratum_instance_t *client, *tmpclient; - proxy_t *subproxy, *tmp; - int64_t headroom = 0; - - mutex_lock(&sdata->proxy_lock); - HASH_ITER(sh, proxy->subproxies, subproxy, tmp) { - if (subproxy->dead) - continue; - headroom += subproxy->max_clients - subproxy->clients; - } - mutex_unlock(&sdata->proxy_lock); - - ck_rlock(&sdata->instance_lock); - HASH_ITER(hh, sdata->stratum_instances, client, tmpclient) { - if (client->dropped) - continue; - if (client->reconnect || client->proxyid != proxy->id) - headroom--; - } - ck_runlock(&sdata->instance_lock); - - if (headroom < 1) - generator_recruit(sdata->ckp); } static void update_subscribe(ckpool_t *ckp, const char *cmd) { sdata_t *sdata = ckp->data, *dsdata; proxy_t *proxy, *old = NULL; - bool current = false; const char *buf; int64_t id = 0; int subid = 0; @@ -1326,14 +1296,10 @@ static void update_subscribe(ckpool_t *ckp, const char *cmd) /* Is this a replacement proxy for the current one */ mutex_lock(&sdata->proxy_lock); - if (sdata->proxy && sdata->proxy->low_id == proxy->low_id && !proxy->subid) { - current = true; + if (sdata->proxy && sdata->proxy->low_id == proxy->low_id && !proxy->subid) sdata->proxy = proxy; - } mutex_unlock(&sdata->proxy_lock); - if (current) - reassess_headroom(sdata, proxy); if (subid) { LOGINFO("Upstream pool %ld:%d extranonce2 length %d, max proxy clients %"PRId64, id, subid, proxy->nonce2len, proxy->max_clients); From 918a6ebe0c84e56963359c07322ba81d3a7c16f2 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Tue, 24 Feb 2015 21:57:27 +1100 Subject: [PATCH 273/544] Make the stratifier tell the generator precisely how many extra connections we need --- src/generator.c | 18 ++++++++++-------- src/stratifier.c | 33 ++++++++++++++++++--------------- 2 files changed, 28 insertions(+), 23 deletions(-) diff --git a/src/generator.c b/src/generator.c index a873eb08..829feac1 100644 --- a/src/generator.c +++ b/src/generator.c @@ -1734,17 +1734,20 @@ retry: return NULL; } -/* Allow up to 42 recruit requests to accumulate */ -static void recruit_subproxy(proxy_instance_t *proxi) +/* Queue up to the requested amount */ +static void recruit_subproxy(proxy_instance_t *proxi, const char *buf) { bool recruit = false; + int recruits = 1; pthread_t pth; + sscanf(buf, "recruit=%d", &recruits); + mutex_lock(&proxi->proxy_lock); - if (!proxi->recruit++ > 0) + if (!proxi->recruit) recruit = true; - else if (proxi->recruit > 42) - proxi->recruit = 42; + if (proxi->recruit < recruits) + proxi->recruit = recruits; mutex_unlock(&proxi->proxy_lock); if (recruit) @@ -1972,8 +1975,7 @@ static void *proxy_recv(void *arg) LOGWARNING("Proxy %d:%s reconnect issue, dropping existing connection", subproxy->low_id, subproxy->si->url); break; - } else - recruit_subproxy(proxi); + } } continue; } @@ -2115,7 +2117,7 @@ retry: LOGDEBUG("Proxy received ping request"); send_unix_msg(sockd, "pong"); } else if (cmdmatch(buf, "recruit")) { - recruit_subproxy(proxi); + recruit_subproxy(proxi, buf); } else if (cmdmatch(buf, "dropproxy")) { drop_proxy(gdata, buf); } else if (ckp->passthrough) { diff --git a/src/stratifier.c b/src/stratifier.c index 12e1a8a5..87464e6c 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -1136,10 +1136,13 @@ out_unlock: static void reconnect_client(sdata_t *sdata, stratum_instance_t *client); -static void generator_recruit(ckpool_t *ckp) +static void generator_recruit(ckpool_t *ckp, const int recruits) { - LOGINFO("Stratifer requesting more proxies from generator"); - send_generator(ckp, "recruit", GEN_PRIORITY); + char buf[256]; + + sprintf(buf, "recruit=%d", recruits); + LOGINFO("Stratifer requesting %d more proxies from generator", recruits); + send_generator(ckp, buf, GEN_PRIORITY); } /* Find how much headroom we have and connect up to that many clients that are @@ -1160,9 +1163,9 @@ static void reconnect_clients(sdata_t *sdata) HASH_ITER(hh, sdata->stratum_instances, client, tmpclient) { if (client->proxyid == proxy->id) continue; - reconnects++; if (headroom-- < 1) - break; + continue; + reconnects++; if (client->reconnect) reconnect_client(sdata, client); else @@ -1173,9 +1176,9 @@ static void reconnect_clients(sdata_t *sdata) if (reconnects) { LOGNOTICE("%d clients flagged for reconnect to proxy %ld", reconnects, proxy->id); - if (headroom < 42) - generator_recruit(sdata->ckp); } + if (headroom < 0) + generator_recruit(sdata->ckp, -headroom); } #if 0 @@ -1208,9 +1211,9 @@ static void dead_proxyid(sdata_t *sdata, const int64_t id, const int subid) HASH_ITER(hh, sdata->stratum_instances, client, tmp) { if (client->proxyid != id || client->subproxyid != subid) continue; - reconnects++; if (headroom-- < 1) - break; + continue; + reconnects++; if (client->reconnect) reconnect_client(sdata, client); else @@ -1221,9 +1224,9 @@ static void dead_proxyid(sdata_t *sdata, const int64_t id, const int subid) if (reconnects) { LOGNOTICE("%d clients flagged to reconnect from dead proxy %ld:%d", reconnects, id, subid); - if (headroom < 42) - generator_recruit(sdata->ckp); } + if (headroom < 0) + generator_recruit(sdata->ckp, -headroom); } static void update_subscribe(ckpool_t *ckp, const char *cmd) @@ -2239,11 +2242,11 @@ static void lazy_reconnect_client(sdata_t *sdata, stratum_instance_t *client) headroom = current_headroom(sdata, &proxy); if (!proxy) return; - if (headroom > 0) { + if (headroom-- > 0) { LOGNOTICE("Reconnecting client %"PRId64, client->id); reconnect_client(sdata, client); } else { - generator_recruit(sdata->ckp); + generator_recruit(sdata->ckp, -headroom); if (!client->reconnect) { LOGNOTICE("Flagging client %"PRId64, client->id); client->reconnect = true; @@ -2574,8 +2577,8 @@ static sdata_t *select_sdata(ckpool_t *ckp, sdata_t *ckp_sdata) } mutex_unlock(&ckp_sdata->proxy_lock); - if (best_id != current->id || current->headroom < 42) - generator_recruit(ckp); + if (best_id != current->id || current->headroom < 2) + generator_recruit(ckp, 1); if (best_id == ckp_sdata->proxy_count) { LOGNOTICE("Temporarily insufficient subproxies to accept more clients"); return NULL; From d745e7135d1c2ab4ca1709310d3a6c56f24d4c39 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Tue, 24 Feb 2015 22:00:38 +1100 Subject: [PATCH 274/544] Remove the old proxy on a reconnect so we don't send shares nowhere --- src/generator.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/generator.c b/src/generator.c index 829feac1..7b4e5141 100644 --- a/src/generator.c +++ b/src/generator.c @@ -1039,6 +1039,7 @@ static bool parse_reconnect(proxy_instance_t *proxi, json_t *val) newsi = ckzalloc(sizeof(server_instance_t)); mutex_lock(&gdata->lock); + HASH_DEL(gdata->proxies, proxi); high_id = proxi->id >> 32; /* Use the high bits for the reconnect id */ high_id++; high_id <<= 32; From d75dd5543c24ae6b1aeb593beb78e67846019a8b Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Tue, 24 Feb 2015 22:26:37 +1100 Subject: [PATCH 275/544] Try async messages again --- src/ckpool.c | 70 +++++++++++++++++++++++++----------------------- src/ckpool.h | 4 --- src/connector.c | 10 +++---- src/generator.c | 28 +++++++++---------- src/stratifier.c | 25 +++++++++-------- 5 files changed, 65 insertions(+), 72 deletions(-) diff --git a/src/ckpool.c b/src/ckpool.c index ff303c50..63593b21 100644 --- a/src/ckpool.c +++ b/src/ckpool.c @@ -575,14 +575,32 @@ out: static void childsighandler(const int sig); -/* Send a single message to a process instance when there will be no response, - * closing the socket immediately. */ -void _send_proc(proc_instance_t *pi, const char *msg, const char *file, const char *func, const int line) +struct proc_message { + proc_instance_t *pi; + char *msg; + const char *file; + const char *func; + int line; +}; + +/* Send all one way messages asynchronously so we can wait till the receiving + * end closes the socket to ensure all messages are received but no deadlocks + * can occur with 2 processes waiting for each other's socket closure. */ +void *async_send_proc(void *arg) { + struct proc_message *pm = (struct proc_message *)arg; + proc_instance_t *pi = pm->pi; + char *msg = pm->msg; + const char *file = pm->file; + const char *func = pm->func; + int line = pm->line; + char *path = pi->us.path; bool ret = false; int sockd; + pthread_detach(pthread_self()); + if (unlikely(!path || !strlen(path))) { LOGERR("Attempted to send message %s to null path in send_proc", msg ? msg : ""); goto out; @@ -593,14 +611,16 @@ void _send_proc(proc_instance_t *pi, const char *msg, const char *file, const ch } /* At startup the pid fields are not set up before some processes are * forked so they never inherit them. */ - if (unlikely(!pi->pid)) + if (unlikely(!pi->pid)) { pi->pid = get_proc_pid(pi); - if (!pi->pid) { - LOGALERT("Attempting to send message %s to non existent process %s", msg, pi->processname); - return; + if (!pi->pid) { + LOGALERT("Attempting to send message %s to non existent process %s", msg, pi->processname); + goto out_nofail; + } } if (unlikely(kill_pid(pi->pid, 0))) { - LOGALERT("Attempting to send message %s to non existent process %s", msg, pi->processname); + LOGALERT("Attempting to send message %s to non existent process %s pid %d", + msg, pi->processname, pi->pid); goto out; } sockd = open_unix_client(path); @@ -620,41 +640,25 @@ out: LOGERR("Failure in send_proc from %s %s:%d", file, func, line); childsighandler(15); } -} - -struct proc_message { - proc_instance_t *pi; - char *msg; - const char *file; - const char *func; - int line; -}; - -static void asp_send(ckpool_t __maybe_unused *ckp, struct proc_message *pm) -{ - _send_proc(pm->pi, pm->msg, pm->file, pm->func, pm->line); - free(pm->msg); +out_nofail: + free(msg); free(pm); + return NULL; } -/* Fore sending asynchronous messages to another process, the sending process - * must have ckwqs of its own, referenced in the ckpool structure */ -void _async_send_proc(ckpool_t *ckp, proc_instance_t *pi, const char *msg, const char *file, const char *func, const int line) +/* Send a single message to a process instance when there will be no response, + * closing the socket immediately. */ +void _send_proc(proc_instance_t *pi, const char *msg, const char *file, const char *func, const int line) { - struct proc_message *pm; + struct proc_message *pm = ckalloc(sizeof(struct proc_message)); + pthread_t pth; - if (unlikely(!ckp->ckwqs)) { - LOGALERT("Workqueues not set up in async_send_proc!"); - _send_proc(pi, msg, file, func, line); - return; - } - pm = ckzalloc(sizeof(struct proc_message)); pm->pi = pi; pm->msg = strdup(msg); pm->file = file; pm->func = func; pm->line = line; - ckwq_add(ckp->ckwqs, &asp_send, pm); + create_pthread(&pth, async_send_proc, pm); } /* Send a single message to a process instance and retrieve the response, then diff --git a/src/ckpool.h b/src/ckpool.h index e88aa71e..d840528e 100644 --- a/src/ckpool.h +++ b/src/ckpool.h @@ -210,8 +210,6 @@ struct ckpool_instance { /* Private data for each process */ void *data; - /* Private generic workqueues if this process has them */ - ckwq_t *ckwqs; }; #ifdef USE_CKDB @@ -236,8 +234,6 @@ void empty_buffer(connsock_t *cs); int read_socket_line(connsock_t *cs, const int timeout); void _send_proc(proc_instance_t *pi, const char *msg, const char *file, const char *func, const int line); #define send_proc(pi, msg) _send_proc(pi, msg, __FILE__, __func__, __LINE__) -void _async_send_proc(ckpool_t *ckp, proc_instance_t *pi, const char *msg, const char *file, const char *func, const int line); -#define async_send_proc(ckp, pi, msg) _async_send_proc(ckp, pi, msg, __FILE__, __func__, __LINE__) char *_send_recv_proc(proc_instance_t *pi, const char *msg, const char *file, const char *func, const int line); #define send_recv_proc(pi, msg) _send_recv_proc(pi, msg, __FILE__, __func__, __LINE__) char *_send_recv_ckdb(const ckpool_t *ckp, const char *msg, const char *file, const char *func, const int line); diff --git a/src/connector.c b/src/connector.c index 3e568d36..ee6cf809 100644 --- a/src/connector.c +++ b/src/connector.c @@ -97,8 +97,6 @@ struct connector_data { /* For protecting the pending sends list */ mutex_t sender_lock; pthread_cond_t sender_cond; - - ckwq_t *ckwqs; }; typedef struct connector_data cdata_t; @@ -244,7 +242,7 @@ static void stratifier_drop_client(ckpool_t *ckp, const int64_t id) char buf[256]; sprintf(buf, "dropclient=%"PRId64, id); - async_send_proc(ckp, ckp->stratifier, buf); + send_proc(ckp->stratifier, buf); } /* Invalidate this instance. Remove them from the hashtables we look up @@ -362,9 +360,9 @@ reparse: * filtered by the stratifier. */ if (likely(client->fd != -1)) { if (ckp->passthrough) - async_send_proc(ckp, ckp->generator, s); + send_proc(ckp->generator, s); else - async_send_proc(ckp, ckp->stratifier, s); + send_proc(ckp->stratifier, s); } free(s); @@ -870,8 +868,6 @@ int connector(proc_instance_t *pi) LOGWARNING("%s connector starting", ckp->name); ckp->data = cdata; cdata->ckp = ckp; - /* Connector only requires one work queue */ - ckp->ckwqs = cdata->ckwqs = create_ckwqs(ckp, "conn", 1); if (!ckp->serverurls) cdata->serverfd = ckalloc(sizeof(int *)); diff --git a/src/generator.c b/src/generator.c index 7b4e5141..98e9a00d 100644 --- a/src/generator.c +++ b/src/generator.c @@ -148,7 +148,6 @@ struct generator_data { proxy_instance_t *dead_proxies; /* Disabled proxies */ int proxy_notify_id; // Globally increasing notify id ckmsgq_t *srvchk; // Server check message queue - ckwq_t *ckwqs; }; typedef struct generator_data gdata_t; @@ -238,7 +237,7 @@ retry: cs = &alive->cs; LOGINFO("Connected to live server %s:%s", cs->url, cs->port); out: - async_send_proc(ckp, ckp->connector, alive ? "accept" : "reject"); + send_proc(ckp->connector, alive ? "accept" : "reject"); return alive; } @@ -384,7 +383,7 @@ retry: ret = submit_block(cs, buf + 12 + 64 + 1); memset(buf + 12 + 64, 0, 1); sprintf(blockmsg, "%sblock:%s", ret ? "" : "no", buf + 12); - async_send_proc(ckp, ckp->stratifier, blockmsg); + send_proc(ckp->stratifier, blockmsg); } else if (cmdmatch(buf, "checkaddr:")) { if (validate_address(cs, buf + 10)) send_unix_msg(sockd, "true"); @@ -949,7 +948,7 @@ static void send_stratifier_deadproxy(ckpool_t *ckp, const int64_t id, const int char buf[256]; sprintf(buf, "deadproxy=%ld:%d", id, subid); - async_send_proc(ckp, ckp->stratifier, buf); + send_proc(ckp->stratifier, buf); } /* Remove the subproxy from the proxi list and put it on the dead list. @@ -1088,7 +1087,7 @@ static void send_diff(ckpool_t *ckp, proxy_instance_t *proxi) json_decref(json_msg); ASPRINTF(&buf, "diff=%s", msg); free(msg); - async_send_proc(ckp, ckp->stratifier, buf); + send_proc(ckp->stratifier, buf); free(buf); } @@ -1116,7 +1115,7 @@ static void send_notify(ckpool_t *ckp, proxy_instance_t *proxi, notify_instance_ json_decref(json_msg); ASPRINTF(&buf, "notify=%s", msg); free(msg); - async_send_proc(ckp, ckp->stratifier, buf); + send_proc(ckp->stratifier, buf); free(buf); /* Send diff now as stratifier will not accept diff till it has a @@ -1309,7 +1308,7 @@ static void send_subscribe(ckpool_t *ckp, proxy_instance_t *proxi) json_decref(json_msg); ASPRINTF(&buf, "subscribe=%s", msg); free(msg); - async_send_proc(ckp, ckp->stratifier, buf); + send_proc(ckp->stratifier, buf); free(buf); } @@ -1356,7 +1355,7 @@ static void stratifier_reconnect_client(ckpool_t *ckp, const int64_t id) char buf[256]; sprintf(buf, "reconnclient=%"PRId64, id); - async_send_proc(ckp, ckp->stratifier, buf); + send_proc(ckp->stratifier, buf); } static void submit_share(gdata_t *gdata, json_t *val) @@ -1779,9 +1778,9 @@ static void reconnect_proxy(proxy_instance_t *proxi) create_pthread(&pth, proxy_reconnect, proxi); } -static void reconnect_generator(ckpool_t *ckp) +static void reconnect_generator(const ckpool_t *ckp) { - async_send_proc(ckp, ckp->generator, "reconnect"); + send_proc(ckp->generator, "reconnect"); } /* For receiving messages from an upstream pool to pass downstream. Responsible @@ -1838,7 +1837,7 @@ static void *passthrough_recv(void *arg) /* Simply forward the message on, as is, to the connector to * process. Possibly parse parameters sent by upstream pool * here */ - async_send_proc(ckp, ckp->connector, cs->buf); + send_proc(ckp->connector, cs->buf); } return NULL; } @@ -2046,10 +2045,10 @@ static proxy_instance_t *wait_best_proxy(ckpool_t *ckp, gdata_t *gdata) if (ret) break; - async_send_proc(ckp, ckp->connector, "reject"); + send_proc(ckp->connector, "reject"); sleep(1); } - async_send_proc(ckp, ckp->connector, ret ? "accept" : "reject"); + send_proc(ckp->connector, ret ? "accept" : "reject"); return ret; } @@ -2078,7 +2077,7 @@ reconnect: proxi->low_id, cs->url, cs->port); dealloc(buf); ASPRINTF(&buf, "proxy=%ld", proxi->id); - async_send_proc(ckp, ckp->stratifier, buf); + send_proc(ckp->stratifier, buf); } } retry: @@ -2267,7 +2266,6 @@ int generator(proc_instance_t *pi) gdata = ckzalloc(sizeof(gdata_t)); ckp->data = gdata; gdata->ckp = ckp; - ckp->ckwqs = gdata->ckwqs = create_ckwqs(ckp, "gen", 1); if (ckp->proxy) { char *buf = NULL; diff --git a/src/stratifier.c b/src/stratifier.c index 87464e6c..f09494ad 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -828,7 +828,7 @@ static char *send_recv_generator(ckpool_t *ckp, const char *msg, const int prio) return buf; } -static void send_generator(ckpool_t *ckp, const char *msg, const int prio) +static void send_generator(const ckpool_t *ckp, const char *msg, const int prio) { sdata_t *sdata = ckp->data; bool set; @@ -838,7 +838,7 @@ static void send_generator(ckpool_t *ckp, const char *msg, const int prio) set = true; } else set = false; - async_send_proc(ckp, ckp->generator, msg); + send_proc(ckp->generator, msg); if (set) sdata->gen_priority = 0; } @@ -963,7 +963,7 @@ static void connector_drop_client(ckpool_t *ckp, const int64_t id) LOGDEBUG("Stratifier requesting connector drop client %"PRId64, id); snprintf(buf, 255, "dropclient=%"PRId64, id); - async_send_proc(ckp, ckp->connector, buf); + send_proc(ckp->connector, buf); } static void drop_allclients(ckpool_t *ckp) @@ -1009,8 +1009,8 @@ static sdata_t *duplicate_sdata(const sdata_t *sdata) memcpy(dsdata->donkeytxnbin, sdata->donkeytxnbin, 25); /* Use the same work queues for all subproxies */ - dsdata->ckwqs = sdata->ckwqs; dsdata->ssends = sdata->ssends; + dsdata->ckwqs = sdata->ckwqs; dsdata->ckdbq = sdata->ckdbq; dsdata->sauthq = sdata->sauthq; @@ -1136,7 +1136,7 @@ out_unlock: static void reconnect_client(sdata_t *sdata, stratum_instance_t *client); -static void generator_recruit(ckpool_t *ckp, const int recruits) +static void generator_recruit(const ckpool_t *ckp, const int recruits) { char buf[256]; @@ -1774,7 +1774,7 @@ static void connector_test_client(ckpool_t *ckp, const int64_t id) LOGDEBUG("Stratifier requesting connector test client %"PRId64, id); snprintf(buf, 255, "testclient=%"PRId64, id); - async_send_proc(ckp, ckp->connector, buf); + send_proc(ckp->connector, buf); } /* For creating a list of sends without locking that can then be concatenated @@ -2531,7 +2531,7 @@ static void stratum_send_message(sdata_t *sdata, const stratum_instance_t *clien * in proxy mode where we find a subproxy based on the current proxy with room * for more clients. Signal the generator to recruit more subproxies if we are * running out of room. */ -static sdata_t *select_sdata(ckpool_t *ckp, sdata_t *ckp_sdata) +static sdata_t *select_sdata(const ckpool_t *ckp, sdata_t *ckp_sdata) { proxy_t *current, *proxy, *subproxy, *best = NULL, *tmp, *tmpsub; int best_subid = 0, best_lowid; @@ -3938,7 +3938,6 @@ static void send_transactions(ckpool_t *ckp, json_params_t *jp); static void parse_method(sdata_t *sdata, stratum_instance_t *client, const int64_t client_id, json_t *id_val, json_t *method_val, json_t *params_val, const char *address) { - ckpool_t *ckp = client->ckp; const char *method; /* Random broken clients send something not an integer as the id so we @@ -3985,14 +3984,14 @@ static void parse_method(sdata_t *sdata, stratum_instance_t *client, const int64 * to it since it's unauthorised. Set the flag just in case. */ client->authorised = false; snprintf(buf, 255, "passthrough=%"PRId64, client_id); - async_send_proc(ckp, ckp->connector, buf); + send_proc(client->ckp->connector, buf); return; } /* We should only accept subscribed requests from here on */ if (!client->subscribed) { LOGINFO("Dropping unsubscribed client %"PRId64, client_id); - connector_drop_client(ckp, client_id); + connector_drop_client(client->ckp, client_id); return; } @@ -4014,7 +4013,7 @@ static void parse_method(sdata_t *sdata, stratum_instance_t *client, const int64 * stratifier process to restart since it will have lost all * the stratum instance data. Clients will just reconnect. */ LOGINFO("Dropping unauthorised client %"PRId64, client_id); - connector_drop_client(ckp, client_id); + connector_drop_client(client->ckp, client_id); return; } @@ -4188,7 +4187,7 @@ static void ssend_process(ckpool_t *ckp, smsg_t *msg) * connector process to be delivered */ json_object_set_new_nocheck(msg->json_msg, "client_id", json_integer(msg->client_id)); s = json_dumps(msg->json_msg, 0); - async_send_proc(ckp, ckp->connector, s); + send_proc(ckp->connector, s); free(s); free_smsg(msg); } @@ -5051,7 +5050,7 @@ int stratifier(proc_instance_t *pi) sdata->ssends = create_ckmsgq(ckp, "ssender", &ssend_process); /* Create as many generic workqueue threads as there are CPUs */ threads = sysconf(_SC_NPROCESSORS_ONLN); - ckp->ckwqs = sdata->ckwqs = create_ckwqs(ckp, "strat", threads); + sdata->ckwqs = create_ckwqs(ckp, "strat", threads); sdata->sauthq = create_ckmsgq(ckp, "authoriser", &sauth_process); if (!CKP_STANDALONE(ckp)) { sdata->ckdbq = create_ckmsgq(ckp, "ckdbqueue", &ckdbq_process); From 8386d8aede7cf29131de46beca946aaeb56825aa Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Tue, 24 Feb 2015 22:46:01 +1100 Subject: [PATCH 276/544] Reset the recruit count in the event of failed proxy_alive test --- src/generator.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/generator.c b/src/generator.c index 98e9a00d..912aa696 100644 --- a/src/generator.c +++ b/src/generator.c @@ -1725,7 +1725,8 @@ retry: if (alive) { if (--parent->recruit > 0) recruit = true; - } + } else /* Reset so the next request will try again */ + parent->recruit = 0; mutex_unlock(&parent->proxy_lock); if (recruit) From 228c2cf19b9f19d4f86f602667b28ae9b46bc718 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Tue, 24 Feb 2015 23:30:35 +1100 Subject: [PATCH 277/544] These weren't the droids we were looking for --- src/ckpool.c | 72 +----------------------------------------- src/ckpool.h | 29 ++--------------- src/stratifier.c | 82 ++++++++++++++++++++++++++---------------------- 3 files changed, 48 insertions(+), 135 deletions(-) diff --git a/src/ckpool.c b/src/ckpool.c index 63593b21..484eda48 100644 --- a/src/ckpool.c +++ b/src/ckpool.c @@ -134,39 +134,6 @@ static void *ckmsg_queue(void *arg) return NULL; } -/* Generic workqueue function and message receiving and parsing thread */ -static void *ckwq_queue(void *arg) -{ - ckwq_t *ckwq = (ckwq_t *)arg; - ckpool_t *ckp = ckwq->ckp; - - pthread_detach(pthread_self()); - rename_proc(ckwq->name); - - while (42) { - ckwqmsg_t *wqmsg; - tv_t now; - ts_t abs; - - mutex_lock(ckwq->lock); - tv_time(&now); - tv_to_ts(&abs, &now); - abs.tv_sec++; - if (!ckwq->wqmsgs) - cond_timedwait(ckwq->cond, ckwq->lock, &abs); - wqmsg = ckwq->wqmsgs; - if (wqmsg) - DL_DELETE(ckwq->wqmsgs, wqmsg); - mutex_unlock(ckwq->lock); - - if (!wqmsg) - continue; - wqmsg->func(ckp, wqmsg->data); - free(wqmsg); - } - return NULL; -} - ckmsgq_t *create_ckmsgq(ckpool_t *ckp, const char *name, const void *func) { ckmsgq_t *ckmsgq = ckzalloc(sizeof(ckmsgq_t)); @@ -207,29 +174,6 @@ ckmsgq_t *create_ckmsgqs(ckpool_t *ckp, const char *name, const void *func, cons return ckmsgq; } -ckwq_t *create_ckwqs(ckpool_t *ckp, const char *name, const int count) -{ - ckwq_t *ckwq = ckzalloc(sizeof(ckwq_t) * count); - mutex_t *lock; - pthread_cond_t *cond; - int i; - - lock = ckalloc(sizeof(mutex_t)); - cond = ckalloc(sizeof(pthread_cond_t)); - mutex_init(lock); - cond_init(cond); - - for (i = 0; i < count; i++) { - snprintf(ckwq[i].name, 15, "%.6swq%d", name, i); - ckwq[i].ckp = ckp; - ckwq[i].lock = lock; - ckwq[i].cond = cond; - create_pthread(&ckwq[i].pth, ckwq_queue, &ckwq[i]); - } - - return ckwq; -} - /* Generic function for adding messages to a ckmsgq linked list and signal the * ckmsgq parsing thread to wake up and process it. */ void ckmsgq_add(ckmsgq_t *ckmsgq, void *data) @@ -241,24 +185,10 @@ void ckmsgq_add(ckmsgq_t *ckmsgq, void *data) mutex_lock(ckmsgq->lock); ckmsgq->messages++; DL_APPEND(ckmsgq->msgs, msg); - pthread_cond_broadcast(ckmsgq->cond); + pthread_cond_signal(ckmsgq->cond); mutex_unlock(ckmsgq->lock); } -void ckwq_add(ckwq_t *ckwq, const void *func, void *data) -{ - ckwqmsg_t *wqmsg = ckalloc(sizeof(ckwqmsg_t)); - - wqmsg->func = func; - wqmsg->data = data; - - mutex_lock(ckwq->lock); - ckwq->messages++; - DL_APPEND(ckwq->wqmsgs, wqmsg); - pthread_cond_broadcast(ckwq->cond); - mutex_unlock(ckwq->lock); -} - /* Return whether there are any messages queued in the ckmsgq linked list. */ bool ckmsgq_empty(ckmsgq_t *ckmsgq) { diff --git a/src/ckpool.h b/src/ckpool.h index d840528e..e0e0f947 100644 --- a/src/ckpool.h +++ b/src/ckpool.h @@ -21,22 +21,13 @@ typedef struct ckpool_instance ckpool_t; -typedef struct ckmsg ckmsg_t; - struct ckmsg { - ckmsg_t *next; - ckmsg_t *prev; + struct ckmsg *next; + struct ckmsg *prev; void *data; }; -typedef struct ckwqmsg ckwqmsg_t; - -struct ckwqmsg { - ckwqmsg_t *next; - ckwqmsg_t *prev; - void *data; - void (*func)(ckpool_t *, void *); -}; +typedef struct ckmsg ckmsg_t; struct ckmsgq { ckpool_t *ckp; @@ -51,18 +42,6 @@ struct ckmsgq { typedef struct ckmsgq ckmsgq_t; -struct ckwq { - ckpool_t *ckp; - char name[16]; - pthread_t pth; - mutex_t *lock; - pthread_cond_t *cond; - ckwqmsg_t *wqmsgs; - int64_t messages; -}; - -typedef struct ckwq ckwq_t; - struct proc_instance { ckpool_t *ckp; unixsock_t us; @@ -222,9 +201,7 @@ struct ckpool_instance { ckmsgq_t *create_ckmsgq(ckpool_t *ckp, const char *name, const void *func); ckmsgq_t *create_ckmsgqs(ckpool_t *ckp, const char *name, const void *func, const int count); -ckwq_t *create_ckwqs(ckpool_t *ckp, const char *name, const int count); void ckmsgq_add(ckmsgq_t *ckmsgq, void *data); -void ckwq_add(ckwq_t *ckwq, const void *func, void *data); bool ckmsgq_empty(ckmsgq_t *ckmsgq); ckpool_t *global_ckp; diff --git a/src/stratifier.c b/src/stratifier.c index f09494ad..ee29c978 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -363,10 +363,12 @@ struct stratifier_data { char lasthash[68]; char lastswaphash[68]; - ckwq_t *ckwqs; // Generic workqueues ckmsgq_t *ssends; // Stratum sends + ckmsgq_t *srecvs; // Stratum receives ckmsgq_t *ckdbq; // ckdb + ckmsgq_t *sshareq; // Stratum share sends ckmsgq_t *sauthq; // Stratum authorisations + ckmsgq_t *stxnq; // Transaction requests int64_t user_instance_id; @@ -843,21 +845,33 @@ static void send_generator(const ckpool_t *ckp, const char *msg, const int prio) sdata->gen_priority = 0; } +struct update_req { + pthread_t *pth; + ckpool_t *ckp; + int prio; +}; + static void broadcast_ping(sdata_t *sdata); /* This function assumes it will only receive a valid json gbt base template * since checking should have been done earlier, and creates the base template * for generating work templates. */ -static void do_update(ckpool_t *ckp, int *prio) +static void *do_update(void *arg) { + struct update_req *ur = (struct update_req *)arg; + ckpool_t *ckp = ur->ckp; sdata_t *sdata = ckp->data; bool new_block = false; + int prio = ur->prio; bool ret = false; workbase_t *wb; json_t *val; char *buf; - buf = send_recv_generator(ckp, "getbase", *prio); + pthread_detach(pthread_self()); + rename_proc("updater"); + + buf = send_recv_generator(ckp, "getbase", prio); if (unlikely(!buf)) { LOGNOTICE("Get base in update_base delayed due to higher priority request"); goto out; @@ -919,17 +933,21 @@ out: LOGINFO("Broadcast ping due to failed stratum base update"); broadcast_ping(sdata); } - free(buf); - free(prio); + dealloc(buf); + free(ur->pth); + free(ur); + return NULL; } static void update_base(ckpool_t *ckp, const int prio) { - int *pprio = ckalloc(sizeof(int)); - sdata_t *sdata = ckp->data; + struct update_req *ur = ckalloc(sizeof(struct update_req)); + pthread_t *pth = ckalloc(sizeof(pthread_t)); - *pprio = prio; - ckwq_add(sdata->ckwqs, &do_update, pprio); + ur->pth = pth; + ur->ckp = ckp; + ur->prio = prio; + create_pthread(pth, do_update, ur); } static void __kill_instance(stratum_instance_t *client) @@ -1010,9 +1028,11 @@ static sdata_t *duplicate_sdata(const sdata_t *sdata) /* Use the same work queues for all subproxies */ dsdata->ssends = sdata->ssends; - dsdata->ckwqs = sdata->ckwqs; + dsdata->srecvs = sdata->srecvs; dsdata->ckdbq = sdata->ckdbq; + dsdata->sshareq = sdata->sshareq; dsdata->sauthq = sdata->sauthq; + dsdata->stxnq = sdata->stxnq; /* Give the sbuproxy its own workbase list and lock */ cklock_init(&dsdata->workbase_lock); @@ -2111,21 +2131,6 @@ static void ckmsgq_stats(ckmsgq_t *ckmsgq, const int size, json_t **val) JSON_CPACK(*val, "{si,si,si}", "count", objects, "memory", memsize, "generated", generated); } -static void ckwq_stats(ckwq_t *ckwq, const int size, json_t **val) -{ - int objects, generated; - int64_t memsize; - ckwqmsg_t *wqmsg; - - mutex_lock(ckwq->lock); - DL_COUNT(ckwq->wqmsgs, wqmsg, objects); - generated = ckwq->messages; - mutex_unlock(ckwq->lock); - - memsize = (sizeof(ckwqmsg_t) + size) * objects; - JSON_CPACK(*val, "{si,si,si}", "count", objects, "memory", memsize, "generated", generated); -} - static char *stratifier_stats(ckpool_t *ckp, sdata_t *sdata) { json_t *val = json_object(), *subval; @@ -2172,14 +2177,15 @@ static char *stratifier_stats(ckpool_t *ckp, sdata_t *sdata) ckmsgq_stats(sdata->ssends, sizeof(smsg_t), &subval); json_set_object(val, "ssends", subval); - /* Don't know exactly how big the string is so just count the pointer for now */ - ckwq_stats(sdata->ckwqs, sizeof(char *) + sizeof(void *), &subval); - json_set_object(val, "ckwqs", subval); + ckmsgq_stats(sdata->srecvs, sizeof(char *), &subval); + json_set_object(val, "srecvs", subval); if (!CKP_STANDALONE(ckp)) { ckmsgq_stats(sdata->ckdbq, sizeof(char *), &subval); json_set_object(val, "ckdbq", subval); } + ckmsgq_stats(sdata->stxnq, sizeof(json_params_t), &subval); + json_set_object(val, "stxnq", subval); buf = json_dumps(val, JSON_NO_UTF8 | JSON_PRESERVE_ORDER); json_decref(val); @@ -2267,7 +2273,6 @@ static void reconnect_client_id(sdata_t *sdata, const int64_t client_id) lazy_reconnect_client(sdata, client); dec_instance_ref(sdata, client); } -static void srecv_process(ckpool_t *ckp, char *buf); static int stratum_loop(ckpool_t *ckp, proc_instance_t *pi) { @@ -2323,7 +2328,7 @@ retry: * connector so look for this first. The srecv_process frees * the buf heap ram */ Close(sockd); - ckwq_add(sdata->ckwqs, &srecv_process, buf); + ckmsgq_add(sdata->srecvs, buf); buf = NULL; goto retry; } @@ -3931,9 +3936,6 @@ static void suggest_diff(stratum_instance_t *client, const char *method, const j stratum_send_diff(sdata, client); } -static void sshare_process(ckpool_t *ckp, json_params_t *jp); -static void send_transactions(ckpool_t *ckp, json_params_t *jp); - /* Enter with client holding ref count */ static void parse_method(sdata_t *sdata, stratum_instance_t *client, const int64_t client_id, json_t *id_val, json_t *method_val, json_t *params_val, const char *address) @@ -3947,7 +3949,7 @@ static void parse_method(sdata_t *sdata, stratum_instance_t *client, const int64 if (likely(cmdmatch(method, "mining.submit") && client->authorised)) { json_params_t *jp = create_json_params(client_id, method_val, params_val, id_val, address); - ckwq_add(sdata->ckwqs, &sshare_process, jp); + ckmsgq_add(sdata->sshareq, jp); return; } @@ -4026,7 +4028,7 @@ static void parse_method(sdata_t *sdata, stratum_instance_t *client, const int64 if (cmdmatch(method, "mining.get")) { json_params_t *jp = create_json_params(client_id, method_val, params_val, id_val, address); - ckwq_add(sdata->ckwqs, &send_transactions, jp); + ckmsgq_add(sdata->stxnq, jp); return; } /* Unhandled message here */ @@ -5048,10 +5050,14 @@ int stratifier(proc_instance_t *pi) mutex_init(&sdata->ckdb_lock); sdata->ssends = create_ckmsgq(ckp, "ssender", &ssend_process); - /* Create as many generic workqueue threads as there are CPUs */ - threads = sysconf(_SC_NPROCESSORS_ONLN); - sdata->ckwqs = create_ckwqs(ckp, "strat", threads); + /* Create half as many share processing threads as there are CPUs */ + threads = sysconf(_SC_NPROCESSORS_ONLN) / 2 ? : 1; + sdata->sshareq = create_ckmsgqs(ckp, "sprocessor", &sshare_process, threads); + /* Create 1/4 as many stratum processing threads as there are CPUs */ + threads = threads / 2 ? : 1; + sdata->srecvs = create_ckmsgqs(ckp, "sreceiver", &srecv_process, threads); sdata->sauthq = create_ckmsgq(ckp, "authoriser", &sauth_process); + sdata->stxnq = create_ckmsgq(ckp, "stxnq", &send_transactions); if (!CKP_STANDALONE(ckp)) { sdata->ckdbq = create_ckmsgq(ckp, "ckdbqueue", &ckdbq_process); create_pthread(&pth_heartbeat, ckdb_heartbeat, ckp); From f027885ff6162aaf550b8f9141986a1d2146f41b Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Tue, 24 Feb 2015 23:41:00 +1100 Subject: [PATCH 278/544] These weren't the droids we were looking for. --- src/ckpool.c | 90 +++++++++++++++++++++++++----------------------- src/ckpool.h | 4 --- src/generator.c | 34 +++++++++--------- src/stratifier.c | 90 +++++++++++++++++++++++++----------------------- 4 files changed, 111 insertions(+), 107 deletions(-) diff --git a/src/ckpool.c b/src/ckpool.c index 68cf93fd..b2b6eb00 100644 --- a/src/ckpool.c +++ b/src/ckpool.c @@ -137,27 +137,27 @@ static void *ckmsg_queue(void *arg) /* Generic workqueue function and message receiving and parsing thread */ static void *ckwq_queue(void *arg) { - ckwq_t *ckwq = (ckwq_t *)arg; - ckpool_t *ckp = ckwq->ckp; + ckwq_t *ckmsgq = (ckwq_t *)arg; + ckpool_t *ckp = ckmsgq->ckp; pthread_detach(pthread_self()); - rename_proc(ckwq->name); + rename_proc(ckmsgq->name); while (42) { ckwqmsg_t *wqmsg; tv_t now; ts_t abs; - mutex_lock(ckwq->lock); + mutex_lock(ckmsgq->lock); tv_time(&now); tv_to_ts(&abs, &now); abs.tv_sec++; - if (!ckwq->wqmsgs) - cond_timedwait(ckwq->cond, ckwq->lock, &abs); - wqmsg = ckwq->wqmsgs; + if (!ckmsgq->wqmsgs) + cond_timedwait(ckmsgq->cond, ckmsgq->lock, &abs); + wqmsg = ckmsgq->wqmsgs; if (wqmsg) - DL_DELETE(ckwq->wqmsgs, wqmsg); - mutex_unlock(ckwq->lock); + DL_DELETE(ckmsgq->wqmsgs, wqmsg); + mutex_unlock(ckmsgq->lock); if (!wqmsg) continue; @@ -209,7 +209,7 @@ ckmsgq_t *create_ckmsgqs(ckpool_t *ckp, const char *name, const void *func, cons ckwq_t *create_ckwqs(ckpool_t *ckp, const char *name, const int count) { - ckwq_t *ckwq = ckzalloc(sizeof(ckwq_t) * count); + ckwq_t *ckwq = ckzalloc(sizeof(ckmsgq_t) * count); mutex_t *lock; pthread_cond_t *cond; int i; @@ -575,14 +575,32 @@ out: static void childsighandler(const int sig); -/* Send a single message to a process instance when there will be no response, - * closing the socket immediately. */ -void _send_proc(proc_instance_t *pi, const char *msg, const char *file, const char *func, const int line) +struct proc_message { + proc_instance_t *pi; + char *msg; + const char *file; + const char *func; + int line; +}; + +/* Send all one way messages asynchronously so we can wait till the receiving + * end closes the socket to ensure all messages are received but no deadlocks + * can occur with 2 processes waiting for each other's socket closure. */ +void *async_send_proc(void *arg) { + struct proc_message *pm = (struct proc_message *)arg; + proc_instance_t *pi = pm->pi; + char *msg = pm->msg; + const char *file = pm->file; + const char *func = pm->func; + int line = pm->line; + char *path = pi->us.path; bool ret = false; int sockd; + pthread_detach(pthread_self()); + if (unlikely(!path || !strlen(path))) { LOGERR("Attempted to send message %s to null path in send_proc", msg ? msg : ""); goto out; @@ -593,14 +611,16 @@ void _send_proc(proc_instance_t *pi, const char *msg, const char *file, const ch } /* At startup the pid fields are not set up before some processes are * forked so they never inherit them. */ - if (unlikely(!pi->pid)) + if (unlikely(!pi->pid)) { pi->pid = get_proc_pid(pi); - if (!pi->pid) { - LOGALERT("Attempting to send message %s to non existent process %s", msg, pi->processname); - return; + if (!pi->pid) { + LOGALERT("Attempting to send message %s to non existent process %s", msg, pi->processname); + goto out_nofail; + } } if (unlikely(kill_pid(pi->pid, 0))) { - LOGALERT("Attempting to send message %s to non existent process %s", msg, pi->processname); + LOGALERT("Attempting to send message %s to non existent process %s pid %d", + msg, pi->processname, pi->pid); goto out; } sockd = open_unix_client(path); @@ -620,41 +640,25 @@ out: LOGERR("Failure in send_proc from %s %s:%d", file, func, line); childsighandler(15); } -} - -struct proc_message { - proc_instance_t *pi; - char *msg; - const char *file; - const char *func; - int line; -}; - -static void asp_send(ckpool_t __maybe_unused *ckp, struct proc_message *pm) -{ - _send_proc(pm->pi, pm->msg, pm->file, pm->func, pm->line); - free(pm->msg); +out_nofail: + free(msg); free(pm); + return NULL; } -/* Fore sending asynchronous messages to another process, the sending process - * must have ckwqs of its own, referenced in the ckpool structure */ -void _async_send_proc(ckpool_t *ckp, proc_instance_t *pi, const char *msg, const char *file, const char *func, const int line) +/* Send a single message to a process instance when there will be no response, + * closing the socket immediately. */ +void _send_proc(proc_instance_t *pi, const char *msg, const char *file, const char *func, const int line) { - struct proc_message *pm; + struct proc_message *pm = ckalloc(sizeof(struct proc_message)); + pthread_t pth; - if (unlikely(!ckp->ckwqs)) { - LOGALERT("Workqueues not set up in async_send_proc!"); - _send_proc(pi, msg, file, func, line); - return; - } - pm = ckzalloc(sizeof(struct proc_message)); pm->pi = pi; pm->msg = strdup(msg); pm->file = file; pm->func = func; pm->line = line; - ckwq_add(ckp->ckwqs, &asp_send, pm); + create_pthread(&pth, async_send_proc, pm); } /* Send a single message to a process instance and retrieve the response, then diff --git a/src/ckpool.h b/src/ckpool.h index 3193db89..2c3bcb0b 100644 --- a/src/ckpool.h +++ b/src/ckpool.h @@ -213,8 +213,6 @@ struct ckpool_instance { /* Private data for each process */ void *data; - /* Private generic workqueues if this process has them */ - ckwq_t *ckwqs; }; #ifdef USE_CKDB @@ -239,8 +237,6 @@ void empty_buffer(connsock_t *cs); int read_socket_line(connsock_t *cs, const int timeout); void _send_proc(proc_instance_t *pi, const char *msg, const char *file, const char *func, const int line); #define send_proc(pi, msg) _send_proc(pi, msg, __FILE__, __func__, __LINE__) -void _async_send_proc(ckpool_t *ckp, proc_instance_t *pi, const char *msg, const char *file, const char *func, const int line); -#define async_send_proc(ckp, pi, msg) _async_send_proc(ckp, pi, msg, __FILE__, __func__, __LINE__) char *_send_recv_proc(proc_instance_t *pi, const char *msg, const char *file, const char *func, const int line); #define send_recv_proc(pi, msg) _send_recv_proc(pi, msg, __FILE__, __func__, __LINE__) char *_send_recv_ckdb(const ckpool_t *ckp, const char *msg, const char *file, const char *func, const int line); diff --git a/src/generator.c b/src/generator.c index a47dfe74..1eb7e8fb 100644 --- a/src/generator.c +++ b/src/generator.c @@ -221,7 +221,7 @@ retry: cs = &alive->cs; LOGINFO("Connected to live server %s:%s", cs->url, cs->port); out: - async_send_proc(ckp, ckp->connector, alive ? "accept" : "reject"); + send_proc(ckp->connector, alive ? "accept" : "reject"); return alive; } @@ -367,7 +367,7 @@ retry: ret = submit_block(cs, buf + 12 + 64 + 1); memset(buf + 12 + 64, 0, 1); sprintf(blockmsg, "%sblock:%s", ret ? "" : "no", buf + 12); - async_send_proc(ckp, ckp->stratifier, blockmsg); + send_proc(ckp->stratifier, blockmsg); } else if (cmdmatch(buf, "checkaddr:")) { if (validate_address(cs, buf + 10)) send_unix_msg(sockd, "true"); @@ -972,7 +972,7 @@ static void send_diff(ckpool_t *ckp, proxy_instance_t *proxi) json_decref(json_msg); ASPRINTF(&buf, "diff=%s", msg); free(msg); - async_send_proc(ckp, ckp->stratifier, buf); + send_proc(ckp->stratifier, buf); free(buf); } @@ -1007,7 +1007,7 @@ static void send_notify(ckpool_t *ckp, proxy_instance_t *proxi) json_decref(json_msg); ASPRINTF(&buf, "notify=%s", msg); free(msg); - async_send_proc(ckp, ckp->stratifier, buf); + send_proc(ckp->stratifier, buf); free(buf); } @@ -1195,7 +1195,7 @@ static void send_subscribe(ckpool_t *ckp, proxy_instance_t *proxi) json_decref(json_msg); ASPRINTF(&buf, "subscribe=%s", msg); free(msg); - async_send_proc(ckp, ckp->stratifier, buf); + send_proc(ckp->stratifier, buf); free(buf); } @@ -1428,7 +1428,7 @@ static void *passthrough_recv(void *arg) if (proxy_alive(ckp, si, proxi, cs, false)) { proxi->alive = true; - async_send_proc(ckp, ckp->generator, "reconnect"); + send_proc(ckp->generator, "reconnect"); LOGWARNING("Proxy %d:%s connection established", proxi->id, proxi->si->url); } @@ -1439,13 +1439,13 @@ static void *passthrough_recv(void *arg) while (!proxy_alive(ckp, si, proxi, cs, true)) { if (proxi->alive) { proxi->alive = false; - async_send_proc(ckp, ckp->generator, "reconnect"); + send_proc(ckp->generator, "reconnect"); } sleep(5); } if (!proxi->alive) { proxi->alive = true; - async_send_proc(ckp, ckp->generator, "reconnect"); + send_proc(ckp->generator, "reconnect"); } do { @@ -1456,13 +1456,13 @@ static void *passthrough_recv(void *arg) LOGWARNING("Proxy %d:%s failed to read_socket_line in proxy_recv, attempting reconnect", proxi->id, proxi->si->url); proxi->alive = false; - async_send_proc(ckp, ckp->generator, "reconnect"); + send_proc(ckp->generator, "reconnect"); continue; } /* Simply forward the message on, as is, to the connector to * process. Possibly parse parameters sent by upstream pool * here */ - async_send_proc(ckp, ckp->connector, cs->buf); + send_proc(ckp->connector, cs->buf); } return NULL; } @@ -1496,7 +1496,7 @@ static void *proxy_recv(void *arg) if (proxy_alive(ckp, si, proxi, cs, false)) { proxi->alive = true; - async_send_proc(ckp, ckp->generator, "reconnect"); + send_proc(ckp->generator, "reconnect"); LOGWARNING("Proxy %d:%s connection established", proxi->id, proxi->si->url); } @@ -1510,7 +1510,7 @@ static void *proxy_recv(void *arg) while (!proxy_alive(ckp, si, proxi, cs, true)) { if (proxi->alive) { proxi->alive = false; - async_send_proc(ckp, ckp->generator, "reconnect"); + send_proc(ckp->generator, "reconnect"); } sleep(5); proxi->reconnect_time = time(NULL); @@ -1522,7 +1522,7 @@ static void *proxy_recv(void *arg) LOGWARNING("Proxy %d:%s recovered", proxi->id, proxi->si->url); proxi->alive = true; proxi->reconnect_time = 0; - async_send_proc(ckp, ckp->generator, "reconnect"); + send_proc(ckp->generator, "reconnect"); } now = time(NULL); @@ -1572,7 +1572,7 @@ static void *proxy_recv(void *arg) * pool is up */ proxi->reconnect = false; proxi->alive = false; - async_send_proc(ckp, ckp->generator, "reconnect"); + send_proc(ckp->generator, "reconnect"); LOGWARNING("Proxy %d:%s reconnect issue, dropping existing connection", proxi->id, proxi->si->url); Close(cs->fd); @@ -1644,7 +1644,7 @@ static proxy_instance_t *best_proxy(ckpool_t *ckp, gdata_t *gdata) break; sleep(1); } - async_send_proc(ckp, ckp->connector, ret ? "accept" : "reject"); + send_proc(ckp->connector, ret ? "accept" : "reject"); return ret; } @@ -1672,7 +1672,7 @@ reconnect: proxi->id, cs->url, cs->port); dealloc(buf); ASPRINTF(&buf, "proxy=%d", proxi->id); - async_send_proc(ckp, ckp->stratifier, buf); + send_proc(ckp->stratifier, buf); /* Send a notify for the new chosen proxy or the * stratifier won't be able to switch. */ send_notify(ckp, proxi); @@ -1857,7 +1857,7 @@ static void server_watchdog(ckpool_t *ckp, server_instance_t *cursi) break; } if (alive) - async_send_proc(ckp, ckp->generator, "reconnect"); + send_proc(ckp->generator, "reconnect"); } int generator(proc_instance_t *pi) diff --git a/src/stratifier.c b/src/stratifier.c index 3e8760d8..792972fb 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -336,10 +336,12 @@ struct stratifier_data { char lasthash[68]; char lastswaphash[68]; - ckwq_t *ckwqs; // Generic workqueues ckmsgq_t *ssends; // Stratum sends + ckmsgq_t *srecvs; // Stratum receives ckmsgq_t *ckdbq; // ckdb + ckmsgq_t *sshareq; // Stratum share sends ckmsgq_t *sauthq; // Stratum authorisations + ckmsgq_t *stxnq; // Transaction requests int64_t user_instance_id; @@ -808,26 +810,38 @@ static void send_generator(ckpool_t *ckp, const char *msg, const int prio) set = true; } else set = false; - async_send_proc(ckp, ckp->generator, msg); + send_proc(ckp->generator, msg); if (set) sdata->gen_priority = 0; } +struct update_req { + pthread_t *pth; + ckpool_t *ckp; + int prio; +}; + static void broadcast_ping(sdata_t *sdata); /* This function assumes it will only receive a valid json gbt base template * since checking should have been done earlier, and creates the base template * for generating work templates. */ -static void do_update(ckpool_t *ckp, int *prio) +static void *do_update(void *arg) { + struct update_req *ur = (struct update_req *)arg; + ckpool_t *ckp = ur->ckp; sdata_t *sdata = ckp->data; bool new_block = false; + int prio = ur->prio; bool ret = false; workbase_t *wb; json_t *val; char *buf; - buf = send_recv_generator(ckp, "getbase", *prio); + pthread_detach(pthread_self()); + rename_proc("updater"); + + buf = send_recv_generator(ckp, "getbase", prio); if (unlikely(!buf)) { LOGNOTICE("Get base in update_base delayed due to higher priority request"); goto out; @@ -887,17 +901,21 @@ out: LOGINFO("Broadcast ping due to failed stratum base update"); broadcast_ping(sdata); } - free(buf); - free(prio); + dealloc(buf); + free(ur->pth); + free(ur); + return NULL; } static void update_base(ckpool_t *ckp, const int prio) { - int *pprio = ckalloc(sizeof(int)); - sdata_t *sdata = ckp->data; + struct update_req *ur = ckalloc(sizeof(struct update_req)); + pthread_t *pth = ckalloc(sizeof(pthread_t)); - *pprio = prio; - ckwq_add(sdata->ckwqs, &do_update, pprio); + ur->pth = pth; + ur->ckp = ckp; + ur->prio = prio; + create_pthread(pth, do_update, ur); } static void __kill_instance(stratum_instance_t *client) @@ -929,7 +947,7 @@ static void connector_drop_client(ckpool_t *ckp, const int64_t id) LOGDEBUG("Stratifier requesting connector drop client %"PRId64, id); snprintf(buf, 255, "dropclient=%"PRId64, id); - async_send_proc(ckp, ckp->connector, buf); + send_proc(ckp->connector, buf); } static void drop_allclients(ckpool_t *ckp) @@ -1739,21 +1757,6 @@ static void ckmsgq_stats(ckmsgq_t *ckmsgq, const int size, json_t **val) JSON_CPACK(*val, "{si,si,si}", "count", objects, "memory", memsize, "generated", generated); } -static void ckwq_stats(ckwq_t *ckwq, const int size, json_t **val) -{ - int objects, generated; - int64_t memsize; - ckwqmsg_t *wqmsg; - - mutex_lock(ckwq->lock); - DL_COUNT(ckwq->wqmsgs, wqmsg, objects); - generated = ckwq->messages; - mutex_unlock(ckwq->lock); - - memsize = (sizeof(ckwqmsg_t) + size) * objects; - JSON_CPACK(*val, "{si,si,si}", "count", objects, "memory", memsize, "generated", generated); -} - static char *stratifier_stats(ckpool_t *ckp, sdata_t *sdata) { json_t *val = json_object(), *subval; @@ -1800,14 +1803,15 @@ static char *stratifier_stats(ckpool_t *ckp, sdata_t *sdata) ckmsgq_stats(sdata->ssends, sizeof(smsg_t), &subval); json_set_object(val, "ssends", subval); - /* Don't know exactly how big the string is so just count the pointer for now */ - ckwq_stats(sdata->ckwqs, sizeof(char *) + sizeof(void *), &subval); - json_set_object(val, "ckwqs", subval); + ckmsgq_stats(sdata->srecvs, sizeof(char *), &subval); + json_set_object(val, "srecvs", subval); if (!CKP_STANDALONE(ckp)) { ckmsgq_stats(sdata->ckdbq, sizeof(char *), &subval); json_set_object(val, "ckdbq", subval); } + ckmsgq_stats(sdata->stxnq, sizeof(json_params_t), &subval); + json_set_object(val, "stxnq", subval); buf = json_dumps(val, JSON_NO_UTF8 | JSON_PRESERVE_ORDER); json_decref(val); @@ -1890,7 +1894,7 @@ retry: /* The bulk of the messages will be received json from the * connector so look for this first. The srecv_process frees * the buf heap ram */ - ckwq_add(sdata->ckwqs, &srecv_process, buf); + ckmsgq_add(sdata->srecvs, buf); Close(sockd); buf = NULL; goto retry; @@ -3414,14 +3418,10 @@ static void suggest_diff(stratum_instance_t *client, const char *method, const j stratum_send_diff(sdata, client); } -static void sshare_process(ckpool_t *ckp, json_params_t *jp); -static void send_transactions(ckpool_t *ckp, json_params_t *jp); - /* Enter with client holding ref count */ static void parse_method(sdata_t *sdata, stratum_instance_t *client, const int64_t client_id, json_t *id_val, json_t *method_val, json_t *params_val, const char *address) { - ckpool_t *ckp = client->ckp; const char *method; /* Random broken clients send something not an integer as the id so we @@ -3431,7 +3431,7 @@ static void parse_method(sdata_t *sdata, stratum_instance_t *client, const int64 if (likely(cmdmatch(method, "mining.submit") && client->authorised)) { json_params_t *jp = create_json_params(client_id, method_val, params_val, id_val, address); - ckwq_add(sdata->ckwqs, &sshare_process, jp); + ckmsgq_add(sdata->sshareq, jp); return; } @@ -3468,14 +3468,14 @@ static void parse_method(sdata_t *sdata, stratum_instance_t *client, const int64 * to it since it's unauthorised. Set the flag just in case. */ client->authorised = false; snprintf(buf, 255, "passthrough=%"PRId64, client_id); - async_send_proc(ckp, ckp->connector, buf); + send_proc(client->ckp->connector, buf); return; } /* We should only accept subscribed requests from here on */ if (!client->subscribed) { LOGINFO("Dropping unsubscribed client %"PRId64, client_id); - connector_drop_client(ckp, client_id); + connector_drop_client(client->ckp, client_id); return; } @@ -3497,7 +3497,7 @@ static void parse_method(sdata_t *sdata, stratum_instance_t *client, const int64 * stratifier process to restart since it will have lost all * the stratum instance data. Clients will just reconnect. */ LOGINFO("Dropping unauthorised client %"PRId64, client_id); - connector_drop_client(ckp, client_id); + connector_drop_client(client->ckp, client_id); return; } @@ -3510,7 +3510,7 @@ static void parse_method(sdata_t *sdata, stratum_instance_t *client, const int64 if (cmdmatch(method, "mining.get")) { json_params_t *jp = create_json_params(client_id, method_val, params_val, id_val, address); - ckwq_add(sdata->ckwqs, &send_transactions, jp); + ckmsgq_add(sdata->stxnq, jp); return; } /* Unhandled message here */ @@ -3685,7 +3685,7 @@ static void ssend_process(ckpool_t *ckp, smsg_t *msg) * connector process to be delivered */ json_object_set_new_nocheck(msg->json_msg, "client_id", json_integer(msg->client_id)); s = json_dumps(msg->json_msg, 0); - async_send_proc(ckp, ckp->connector, s); + send_proc(ckp->connector, s); free(s); free_smsg(msg); } @@ -4543,10 +4543,14 @@ int stratifier(proc_instance_t *pi) mutex_init(&sdata->ckdb_lock); sdata->ssends = create_ckmsgq(ckp, "ssender", &ssend_process); - /* Create as many generic workqueue threads as there are CPUs */ - threads = sysconf(_SC_NPROCESSORS_ONLN); - ckp->ckwqs = sdata->ckwqs = create_ckwqs(ckp, "strat", threads); + /* Create half as many share processing threads as there are CPUs */ + threads = sysconf(_SC_NPROCESSORS_ONLN) / 2 ? : 1; + sdata->sshareq = create_ckmsgqs(ckp, "sprocessor", &sshare_process, threads); + /* Create 1/4 as many stratum processing threads as there are CPUs */ + threads = threads / 2 ? : 1; + sdata->srecvs = create_ckmsgqs(ckp, "sreceiver", &srecv_process, threads); sdata->sauthq = create_ckmsgq(ckp, "authoriser", &sauth_process); + sdata->stxnq = create_ckmsgq(ckp, "stxnq", &send_transactions); if (!CKP_STANDALONE(ckp)) { sdata->ckdbq = create_ckmsgq(ckp, "ckdbqueue", &ckdbq_process); create_pthread(&pth_heartbeat, ckdb_heartbeat, ckp); From 222880d23401cb342f24b18aa4e43d0d28852d19 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Tue, 24 Feb 2015 23:41:32 +1100 Subject: [PATCH 279/544] Revert "Create generic workqueue function and message receiving and parsing helpers" This reverts commit d594f86520731384278ba32f3190bef6ba64ea9f. --- src/ckpool.c | 72 +--------------------------------------------------- src/ckpool.h | 29 +++------------------ 2 files changed, 4 insertions(+), 97 deletions(-) diff --git a/src/ckpool.c b/src/ckpool.c index b2b6eb00..bcf92ad0 100644 --- a/src/ckpool.c +++ b/src/ckpool.c @@ -134,39 +134,6 @@ static void *ckmsg_queue(void *arg) return NULL; } -/* Generic workqueue function and message receiving and parsing thread */ -static void *ckwq_queue(void *arg) -{ - ckwq_t *ckmsgq = (ckwq_t *)arg; - ckpool_t *ckp = ckmsgq->ckp; - - pthread_detach(pthread_self()); - rename_proc(ckmsgq->name); - - while (42) { - ckwqmsg_t *wqmsg; - tv_t now; - ts_t abs; - - mutex_lock(ckmsgq->lock); - tv_time(&now); - tv_to_ts(&abs, &now); - abs.tv_sec++; - if (!ckmsgq->wqmsgs) - cond_timedwait(ckmsgq->cond, ckmsgq->lock, &abs); - wqmsg = ckmsgq->wqmsgs; - if (wqmsg) - DL_DELETE(ckmsgq->wqmsgs, wqmsg); - mutex_unlock(ckmsgq->lock); - - if (!wqmsg) - continue; - wqmsg->func(ckp, wqmsg->data); - free(wqmsg); - } - return NULL; -} - ckmsgq_t *create_ckmsgq(ckpool_t *ckp, const char *name, const void *func) { ckmsgq_t *ckmsgq = ckzalloc(sizeof(ckmsgq_t)); @@ -207,29 +174,6 @@ ckmsgq_t *create_ckmsgqs(ckpool_t *ckp, const char *name, const void *func, cons return ckmsgq; } -ckwq_t *create_ckwqs(ckpool_t *ckp, const char *name, const int count) -{ - ckwq_t *ckwq = ckzalloc(sizeof(ckmsgq_t) * count); - mutex_t *lock; - pthread_cond_t *cond; - int i; - - lock = ckalloc(sizeof(mutex_t)); - cond = ckalloc(sizeof(pthread_cond_t)); - mutex_init(lock); - cond_init(cond); - - for (i = 0; i < count; i++) { - snprintf(ckwq[i].name, 15, "%.6swq%d", name, i); - ckwq[i].ckp = ckp; - ckwq[i].lock = lock; - ckwq[i].cond = cond; - create_pthread(&ckwq[i].pth, ckwq_queue, &ckwq[i]); - } - - return ckwq; -} - /* Generic function for adding messages to a ckmsgq linked list and signal the * ckmsgq parsing thread to wake up and process it. */ void ckmsgq_add(ckmsgq_t *ckmsgq, void *data) @@ -241,24 +185,10 @@ void ckmsgq_add(ckmsgq_t *ckmsgq, void *data) mutex_lock(ckmsgq->lock); ckmsgq->messages++; DL_APPEND(ckmsgq->msgs, msg); - pthread_cond_broadcast(ckmsgq->cond); + pthread_cond_signal(ckmsgq->cond); mutex_unlock(ckmsgq->lock); } -void ckwq_add(ckwq_t *ckwq, const void *func, void *data) -{ - ckwqmsg_t *wqmsg = ckalloc(sizeof(ckwqmsg_t)); - - wqmsg->func = func; - wqmsg->data = data; - - mutex_lock(ckwq->lock); - ckwq->messages++; - DL_APPEND(ckwq->wqmsgs, wqmsg); - pthread_cond_broadcast(ckwq->cond); - mutex_unlock(ckwq->lock); -} - /* Return whether there are any messages queued in the ckmsgq linked list. */ bool ckmsgq_empty(ckmsgq_t *ckmsgq) { diff --git a/src/ckpool.h b/src/ckpool.h index 2c3bcb0b..29e1d269 100644 --- a/src/ckpool.h +++ b/src/ckpool.h @@ -21,22 +21,13 @@ typedef struct ckpool_instance ckpool_t; -typedef struct ckmsg ckmsg_t; - struct ckmsg { - ckmsg_t *next; - ckmsg_t *prev; + struct ckmsg *next; + struct ckmsg *prev; void *data; }; -typedef struct ckwqmsg ckwqmsg_t; - -struct ckwqmsg { - ckwqmsg_t *next; - ckwqmsg_t *prev; - void *data; - void (*func)(ckpool_t *, void *); -}; +typedef struct ckmsg ckmsg_t; struct ckmsgq { ckpool_t *ckp; @@ -51,18 +42,6 @@ struct ckmsgq { typedef struct ckmsgq ckmsgq_t; -struct ckwq { - ckpool_t *ckp; - char name[16]; - pthread_t pth; - mutex_t *lock; - pthread_cond_t *cond; - ckwqmsg_t *wqmsgs; - int64_t messages; -}; - -typedef struct ckwq ckwq_t; - struct proc_instance { ckpool_t *ckp; unixsock_t us; @@ -225,9 +204,7 @@ struct ckpool_instance { ckmsgq_t *create_ckmsgq(ckpool_t *ckp, const char *name, const void *func); ckmsgq_t *create_ckmsgqs(ckpool_t *ckp, const char *name, const void *func, const int count); -ckwq_t *create_ckwqs(ckpool_t *ckp, const char *name, const int count); void ckmsgq_add(ckmsgq_t *ckmsgq, void *data); -void ckwq_add(ckwq_t *ckwq, const void *func, void *data); bool ckmsgq_empty(ckmsgq_t *ckmsgq); ckpool_t *global_ckp; From 66e6b1fa7632d897f005f7ace3dda488e86a88c7 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Tue, 24 Feb 2015 23:42:16 +1100 Subject: [PATCH 280/544] Unused variable --- src/connector.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/connector.c b/src/connector.c index f567369a..8b3cadfc 100644 --- a/src/connector.c +++ b/src/connector.c @@ -382,7 +382,6 @@ void *receiver(void *arg) cdata_t *cdata = (cdata_t *)arg; struct epoll_event event; uint64_t serverfds, i; - time_t start_t; int ret, epfd; rename_proc("creceiver"); @@ -408,7 +407,6 @@ void *receiver(void *arg) while (!cdata->accept) cksleep_ms(1); - start_t = time(NULL); while (42) { client_instance_t *client; From c6336cdc26cae3f965f2ce743c20fb170576917e Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Mon, 23 Feb 2015 20:21:05 +1100 Subject: [PATCH 281/544] Count dropped workers in _dec_instance_ref Conflicts: src/stratifier.c --- src/stratifier.c | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/stratifier.c b/src/stratifier.c index 792972fb..ca07a717 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -1356,11 +1356,15 @@ static void client_drop_message(const int64_t client_id, const int dropped, cons } } +static void dec_worker(ckpool_t *ckp, user_instance_t *instance); + /* Decrease the reference count of instance. */ static void _dec_instance_ref(sdata_t *sdata, stratum_instance_t *client, const char *file, const char *func, const int line) { + user_instance_t *user = client->user_instance; int64_t client_id = client->id; + ckpool_t *ckp = client->ckp; int dropped = 0, ref; ck_wlock(&sdata->instance_lock); @@ -1368,7 +1372,7 @@ static void _dec_instance_ref(sdata_t *sdata, stratum_instance_t *client, const /* See if there are any instances that were dropped that could not be * moved due to holding a reference and drop them now. */ if (unlikely(client->dropped && !ref)) - dropped = __drop_client(sdata, client, client->user_instance); + dropped = __drop_client(sdata, client, user); ck_wunlock(&sdata->instance_lock); client_drop_message(client_id, dropped, true); @@ -1376,6 +1380,9 @@ static void _dec_instance_ref(sdata_t *sdata, stratum_instance_t *client, const /* This should never happen */ if (unlikely(ref < 0)) LOGERR("Instance ref count dropped below zero from %s %s:%d", file, func, line); + + if (dropped) + dec_worker(ckp, user); } #define dec_instance_ref(sdata, instance) _dec_instance_ref(sdata, instance, __FILE__, __func__, __LINE__) From ddfc0dbfd4354d0869658634ed741e11aa54c846 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Tue, 24 Feb 2015 10:32:51 +1100 Subject: [PATCH 282/544] Only dec worker if user exists --- src/stratifier.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/stratifier.c b/src/stratifier.c index ca07a717..a4d20b90 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -1381,7 +1381,7 @@ static void _dec_instance_ref(sdata_t *sdata, stratum_instance_t *client, const if (unlikely(ref < 0)) LOGERR("Instance ref count dropped below zero from %s %s:%d", file, func, line); - if (dropped) + if (dropped && user) dec_worker(ckp, user); } From 3fd9ead54a7e731b8bbc41cd08a21ecd3ea15080 Mon Sep 17 00:00:00 2001 From: ckolivas Date: Wed, 25 Feb 2015 08:40:19 +1100 Subject: [PATCH 283/544] Parse any extra data already in the cs buffer in proxy_recv before returning to the epoll loop --- src/generator.c | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/src/generator.c b/src/generator.c index 912aa696..32824ef5 100644 --- a/src/generator.c +++ b/src/generator.c @@ -1966,25 +1966,25 @@ static void *proxy_recv(void *arg) disable_subproxy(gdata, proxi, subproxy); continue; } - if (parse_method(ckp, subproxy, cs->buf)) { - if (subproxy->reconnect) { - /* Call this proxy dead to allow us to fail - * over to a backup pool until the reconnect - * pool is up */ - disable_subproxy(gdata, proxi, subproxy); - if (parent_proxy(subproxy)) { - LOGWARNING("Proxy %d:%s reconnect issue, dropping existing connection", - subproxy->low_id, subproxy->si->url); - break; + do { + if (parse_method(ckp, subproxy, cs->buf)) { + if (subproxy->reconnect) { + /* Call this proxy dead to allow us to fail + * over to a backup pool until the reconnect + * pool is up */ + disable_subproxy(gdata, proxi, subproxy); + if (parent_proxy(subproxy)) { + LOGWARNING("Proxy %d:%s reconnect issue, dropping existing connection", + subproxy->low_id, subproxy->si->url); + break; + } } + continue; } - continue; - } - if (parse_share(subproxy, cs->buf)) { - continue; - } - /* If it's not a method it should be a share result */ - LOGWARNING("Unhandled stratum message: %s", cs->buf); + /* If it's not a method it should be a share result */ + if (!parse_share(subproxy, cs->buf)) + LOGWARNING("Unhandled stratum message: %s", cs->buf); + } while (read_socket_line(cs, 0) > 0); } return NULL; From e6b79715cb590b953ad22be8223bebac16168aa1 Mon Sep 17 00:00:00 2001 From: ckolivas Date: Wed, 25 Feb 2015 09:57:26 +1100 Subject: [PATCH 284/544] Abstract out recruit_subproxies function --- src/generator.c | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/src/generator.c b/src/generator.c index 32824ef5..bea23bf2 100644 --- a/src/generator.c +++ b/src/generator.c @@ -1735,15 +1735,11 @@ retry: return NULL; } -/* Queue up to the requested amount */ -static void recruit_subproxy(proxy_instance_t *proxi, const char *buf) +static void recruit_subproxies(proxy_instance_t *proxi, const int recruits) { bool recruit = false; - int recruits = 1; pthread_t pth; - sscanf(buf, "recruit=%d", &recruits); - mutex_lock(&proxi->proxy_lock); if (!proxi->recruit) recruit = true; @@ -1755,6 +1751,15 @@ static void recruit_subproxy(proxy_instance_t *proxi, const char *buf) create_pthread(&pth, proxy_recruit, proxi); } +/* Queue up to the requested amount */ +static void recruit_subproxy(proxy_instance_t *proxi, const char *buf) +{ + int recruits = 1; + + sscanf(buf, "recruit=%d", &recruits); + recruit_subproxies(proxi, recruits); +} + static void *proxy_reconnect(void *arg) { proxy_instance_t *proxy = (proxy_instance_t *)arg; From bc7d59ad6617252795df519f641b51b3e03cac92 Mon Sep 17 00:00:00 2001 From: ckolivas Date: Wed, 25 Feb 2015 10:02:59 +1100 Subject: [PATCH 285/544] Recruit extra subproxies on a new parent with a small nonce size, being more verbose about proxy recruitment --- src/generator.c | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/generator.c b/src/generator.c index bea23bf2..629f4643 100644 --- a/src/generator.c +++ b/src/generator.c @@ -136,6 +136,7 @@ struct proxy_instance { mutex_t proxy_lock; /* Lock protecting hashlist of proxies */ proxy_instance_t *parent; /* Parent proxy of subproxies */ proxy_instance_t *subproxies; /* Hashlist of subproxies of this proxy */ + int64_t clients_per_proxy; /* Max number of clients of this proxy */ int subproxy_count; /* Number of subproxies */ }; @@ -564,6 +565,8 @@ static inline bool parent_proxy(proxy_instance_t *proxy) return (proxy->parent == proxy); } +static void recruit_subproxies(proxy_instance_t *proxi, const int recruits); + static bool parse_subscribe(connsock_t *cs, proxy_instance_t *proxi) { json_t *val = NULL, *res_val, *notify_val, *tmp; @@ -663,13 +666,14 @@ retry: proxi->nonce2len = size; if (parent_proxy(proxi)) { /* Set the number of clients per proxy on the parent proxy */ - int64_t clients_per_proxy = 1ll << ((size - 3) * 8); - + proxi->clients_per_proxy = 1ll << ((size - 3) * 8); LOGNOTICE("Proxy %ld:%s clients per proxy: %"PRId64, proxi->id, proxi->si->url, - clients_per_proxy); + proxi->clients_per_proxy); + if (proxi->clients_per_proxy == 1) + recruit_subproxies(proxi, 41); } - LOGINFO("Found notify for proxy %ld:%d with enonce %s nonce2len %d", proxi->id, + LOGNOTICE("Found notify for new proxy %ld:%d with enonce %s nonce2len %d", proxi->id, proxi->subid, proxi->enonce1, proxi->nonce2len); ret = true; From 576ac8f71af07bba8f659b6bffa1b709edb99cc4 Mon Sep 17 00:00:00 2001 From: ckolivas Date: Wed, 25 Feb 2015 10:12:09 +1100 Subject: [PATCH 286/544] Check return value of subsequent read_socket_line, disabling a subproxy if it has failed and recruit more subproxies if required --- src/generator.c | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/generator.c b/src/generator.c index 629f4643..4fec657b 100644 --- a/src/generator.c +++ b/src/generator.c @@ -965,6 +965,9 @@ static void disable_subproxy(gdata_t *gdata, proxy_instance_t *proxi, proxy_inst if (parent_proxy(subproxy)) return; + if (proxi->alive && proxi->clients_per_proxy == 1 && HASH_CNT(sh, proxi->subproxies) < 42) + recruit_subproxies(proxi, 1); + mutex_lock(&proxi->proxy_lock); subproxy->disabled = true; /* Make sure subproxy is still in the list */ @@ -1993,7 +1996,12 @@ static void *proxy_recv(void *arg) /* If it's not a method it should be a share result */ if (!parse_share(subproxy, cs->buf)) LOGWARNING("Unhandled stratum message: %s", cs->buf); - } while (read_socket_line(cs, 0) > 0); + } while ((ret = read_socket_line(cs, 0)) > 0); + if (ret < 0) { + LOGNOTICE("Proxy %ld:%d %s failed to epoll/read_socket_line in proxy_recv", + proxi->id, subproxy->subid, subproxy->si->url); + disable_subproxy(gdata, proxi, subproxy); + } } return NULL; From 30297c3ba2df317a8efae708a10d39f9b15ba9ba Mon Sep 17 00:00:00 2001 From: ckolivas Date: Wed, 25 Feb 2015 10:35:13 +1100 Subject: [PATCH 287/544] Revert ret < 0 check but keep recruiting of new subproxies --- src/generator.c | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/generator.c b/src/generator.c index 4fec657b..e4bd72be 100644 --- a/src/generator.c +++ b/src/generator.c @@ -1997,11 +1997,6 @@ static void *proxy_recv(void *arg) if (!parse_share(subproxy, cs->buf)) LOGWARNING("Unhandled stratum message: %s", cs->buf); } while ((ret = read_socket_line(cs, 0)) > 0); - if (ret < 0) { - LOGNOTICE("Proxy %ld:%d %s failed to epoll/read_socket_line in proxy_recv", - proxi->id, subproxy->subid, subproxy->si->url); - disable_subproxy(gdata, proxi, subproxy); - } } return NULL; From d103ea97df331793f158c4b6572769ebb252b7bb Mon Sep 17 00:00:00 2001 From: ckolivas Date: Wed, 25 Feb 2015 10:57:56 +1100 Subject: [PATCH 288/544] Only kill off proxy if it's actually dead --- src/stratifier.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/stratifier.c b/src/stratifier.c index ee29c978..8adb95e2 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -1286,7 +1286,8 @@ static void update_subscribe(ckpool_t *ckp, const char *cmd) /* Is this a replacement for an existing proxy id? */ old = existing_subproxy(sdata, id, subid); if (old) { - dead_proxyid(sdata, id, subid); + if (old->dead) + dead_proxyid(sdata, id, subid); proxy = old; proxy->dead = false; } else From 0fa55f311f700270eaacf4f4b092eb64a7b9c881 Mon Sep 17 00:00:00 2001 From: ckolivas Date: Wed, 25 Feb 2015 11:10:59 +1100 Subject: [PATCH 289/544] Finesse what messages to send --- src/generator.c | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/generator.c b/src/generator.c index e4bd72be..1ff796f5 100644 --- a/src/generator.c +++ b/src/generator.c @@ -1393,7 +1393,6 @@ static void submit_share(gdata_t *gdata, json_t *val) if (unlikely(!proxy)) { LOGNOTICE("Client %"PRId64" sending shares to non existent proxy %ld, dropping", client_id, id); - send_stratifier_deadproxy(ckp, id, subid); stratifier_reconnect_client(ckp, client_id); goto out; } @@ -1401,14 +1400,12 @@ static void submit_share(gdata_t *gdata, json_t *val) if (unlikely(!proxi)) { LOGNOTICE("Client %"PRId64" sending shares to non existent subproxy %ld:%d, dropping", client_id, id, subid); - send_stratifier_deadproxy(ckp, id, subid); stratifier_reconnect_client(ckp, client_id); goto out; } if (!proxi->alive) { LOGNOTICE("Client %"PRId64" sending shares to dead subproxy %ld:%d, dropping", client_id, id, subid); - send_stratifier_deadproxy(ckp, id, subid); stratifier_reconnect_client(ckp, client_id); goto out; } From 90ab75ce24aae1ac4b28cb03056b90c70d64eb1d Mon Sep 17 00:00:00 2001 From: ckolivas Date: Wed, 25 Feb 2015 11:26:52 +1100 Subject: [PATCH 290/544] Drop subproxies when the parent proxy thread exits --- src/generator.c | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/src/generator.c b/src/generator.c index 1ff796f5..98dd8124 100644 --- a/src/generator.c +++ b/src/generator.c @@ -1885,6 +1885,7 @@ static bool subproxies_alive(proxy_instance_t *proxy) static void *proxy_recv(void *arg) { proxy_instance_t *proxi = (proxy_instance_t *)arg; + proxy_instance_t *subproxy, *tmp; server_instance_t *si = proxi->si; connsock_t *cs = proxi->cs; ckpool_t *ckp = proxi->ckp; @@ -1909,12 +1910,12 @@ static void *proxy_recv(void *arg) reconnect_generator(ckp); while (42) { - proxy_instance_t *subproxy = proxi; share_msg_t *share, *tmpshare; notify_instance_t *ni, *tmp; time_t now; int ret; + subproxy = proxi; if (!proxi->alive) { reconnect_proxy(proxi); while (!subproxies_alive(proxi)) { @@ -1996,6 +1997,18 @@ static void *proxy_recv(void *arg) } while ((ret = read_socket_line(cs, 0)) > 0); } + mutex_lock(&proxi->proxy_lock); + HASH_ITER(sh, proxi->subproxies, subproxy, tmp) { + subproxy->disabled = true; + send_stratifier_deadproxy(ckp, subproxy->id, subproxy->subid); + if (subproxy->cs->fd > 0) { + epoll_ctl(epfd, EPOLL_CTL_DEL, subproxy->cs->fd, NULL); + Close(subproxy->cs->fd); + } + HASH_DELETE(sh, proxi->subproxies, subproxy); + } + mutex_unlock(&proxi->proxy_lock); + return NULL; } From 16bd2fc35230c050e06745d06318e322257d5460 Mon Sep 17 00:00:00 2001 From: ckolivas Date: Wed, 25 Feb 2015 13:36:04 +1100 Subject: [PATCH 291/544] Redirect from existing parent when child gets a reconnect --- src/generator.c | 26 ++++++++++++++++++-------- 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/src/generator.c b/src/generator.c index 98dd8124..2bcaf385 100644 --- a/src/generator.c +++ b/src/generator.c @@ -108,6 +108,7 @@ struct proxy_instance { bool disabled; /* Subproxy no longer to be used */ bool reconnect; /* We need to drop and reconnect */ bool reconnecting; /* Testing in progress */ + bool redirecting; /* Children have received a reconnect */ int recruit; /* Recruiting in progress */ bool alive; @@ -988,8 +989,8 @@ static void disable_subproxy(gdata_t *gdata, proxy_instance_t *proxi, proxy_inst static bool parse_reconnect(proxy_instance_t *proxi, json_t *val) { server_instance_t *newsi, *si = proxi->si; + proxy_instance_t *parent, *newproxi; int64_t high_id, low_id, new_id; - proxy_instance_t *newproxi; ckpool_t *ckp = proxi->ckp; gdata_t *gdata = ckp->data; const char *new_url; @@ -1035,17 +1036,22 @@ static bool parse_reconnect(proxy_instance_t *proxi, json_t *val) LOGINFO("Processing reconnect request to %s", url); ret = true; - proxi->reconnect = true; - /* If this isn't a parent proxy, simply set the reconnect bool allowing - * it to be disabled. More will be recruited if necessary */ - if (!parent_proxy(proxi)) - goto out; + /* If this isn't a parent proxy, recruit a new parent! */ + parent = proxi->parent; + if (parent != proxi) { + proxi->reconnect = true; + /* Do we already know this proxy is redirecting? */ + if (parent->redirecting) + goto out; + proxi = parent; + proxi->redirecting = true; + } else + proxi->redirecting = false; newsi = ckzalloc(sizeof(server_instance_t)); mutex_lock(&gdata->lock); - HASH_DEL(gdata->proxies, proxi); high_id = proxi->id >> 32; /* Use the high bits for the reconnect id */ high_id++; high_id <<= 32; @@ -1068,9 +1074,13 @@ static bool parse_reconnect(proxy_instance_t *proxi, json_t *val) newproxi->id = new_id; newproxi->subproxy_count = ++proxi->subproxy_count; HASH_ADD_I64(gdata->proxies, id, newproxi); + if (!proxi->redirecting) { + proxi->disabled = true; + HASH_DEL(gdata->proxies, proxi); + proxi->reconnect = true; + } mutex_unlock(&gdata->lock); - proxi->disabled = true; prepare_proxy(newproxi); out: return ret; From 47be81bc04f88b80ed1380c33333386bc3878a16 Mon Sep 17 00:00:00 2001 From: ckolivas Date: Wed, 25 Feb 2015 15:08:41 +1100 Subject: [PATCH 292/544] Cope with null message in send_proc --- src/ckpool.c | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/ckpool.c b/src/ckpool.c index 484eda48..21f3da4b 100644 --- a/src/ckpool.c +++ b/src/ckpool.c @@ -535,10 +535,6 @@ void *async_send_proc(void *arg) LOGERR("Attempted to send message %s to null path in send_proc", msg ? msg : ""); goto out; } - if (unlikely(!msg || !strlen(msg))) { - LOGERR("Attempted to send null message to socket %s in send_proc", path); - goto out; - } /* At startup the pid fields are not set up before some processes are * forked so they never inherit them. */ if (unlikely(!pi->pid)) { @@ -580,9 +576,14 @@ out_nofail: * closing the socket immediately. */ void _send_proc(proc_instance_t *pi, const char *msg, const char *file, const char *func, const int line) { - struct proc_message *pm = ckalloc(sizeof(struct proc_message)); + struct proc_message *pm; pthread_t pth; + if (unlikely(!msg || !strlen(msg))) { + LOGERR("Attempted to send null message to %s in send_proc", pi->processname); + return; + } + pm = ckalloc(sizeof(struct proc_message)); pm->pi = pi; pm->msg = strdup(msg); pm->file = file; From e3e83df196698fb9666ab7ebfdbc8cd0306058cb Mon Sep 17 00:00:00 2001 From: ckolivas Date: Wed, 25 Feb 2015 15:29:39 +1100 Subject: [PATCH 293/544] Add a small sleep between attempts at pinging the stratifier at startup to avoid floods of messages --- src/generator.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/generator.c b/src/generator.c index 2bcaf385..0cec54b0 100644 --- a/src/generator.c +++ b/src/generator.c @@ -2308,6 +2308,7 @@ int generator(proc_instance_t *pi) ret = 1; goto out; } + cksleep_ms(10); buf = send_recv_proc(ckp->stratifier, "ping"); } while (!buf); dealloc(buf); From 93a1414e29a6ef8b8f70276c458fe318a01170bd Mon Sep 17 00:00:00 2001 From: ckolivas Date: Wed, 25 Feb 2015 15:53:29 +1100 Subject: [PATCH 294/544] Explicitly check for HUP in proxy_recv, avoiding an attempt at reading a line --- src/generator.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/generator.c b/src/generator.c index 0cec54b0..1a13a423 100644 --- a/src/generator.c +++ b/src/generator.c @@ -1978,7 +1978,10 @@ static void *proxy_recv(void *arg) if (likely(ret > 0)) { subproxy = event.data.ptr; cs = subproxy->cs; - ret = read_socket_line(cs, 5); + if (event.events & EPOLLHUP) + ret = -1; + else + ret = read_socket_line(cs, 5); } if (ret < 1) { LOGNOTICE("Proxy %ld:%d %s failed to epoll/read_socket_line in proxy_recv", From 01c00ed3c264fe2b6f4dd30db1748eb054a8685b Mon Sep 17 00:00:00 2001 From: ckolivas Date: Wed, 25 Feb 2015 16:16:34 +1100 Subject: [PATCH 295/544] Too much recruiting --- src/generator.c | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/generator.c b/src/generator.c index 1a13a423..58d51f08 100644 --- a/src/generator.c +++ b/src/generator.c @@ -671,7 +671,7 @@ retry: LOGNOTICE("Proxy %ld:%s clients per proxy: %"PRId64, proxi->id, proxi->si->url, proxi->clients_per_proxy); if (proxi->clients_per_proxy == 1) - recruit_subproxies(proxi, 41); + recruit_subproxies(proxi, 1); } LOGNOTICE("Found notify for new proxy %ld:%d with enonce %s nonce2len %d", proxi->id, @@ -966,9 +966,6 @@ static void disable_subproxy(gdata_t *gdata, proxy_instance_t *proxi, proxy_inst if (parent_proxy(subproxy)) return; - if (proxi->alive && proxi->clients_per_proxy == 1 && HASH_CNT(sh, proxi->subproxies) < 42) - recruit_subproxies(proxi, 1); - mutex_lock(&proxi->proxy_lock); subproxy->disabled = true; /* Make sure subproxy is still in the list */ From 794770422daf7fc448b510acec07b09facf62da1 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Wed, 25 Feb 2015 18:10:14 +1100 Subject: [PATCH 296/544] Count effective number recruited and subtract that from the recruit requests count --- src/generator.c | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/src/generator.c b/src/generator.c index 58d51f08..7a1992de 100644 --- a/src/generator.c +++ b/src/generator.c @@ -109,7 +109,7 @@ struct proxy_instance { bool reconnect; /* We need to drop and reconnect */ bool reconnecting; /* Testing in progress */ bool redirecting; /* Children have received a reconnect */ - int recruit; /* Recruiting in progress */ + int64_t recruit; /* No of recruiting requests in progress */ bool alive; mutex_t notify_lock; @@ -572,6 +572,7 @@ static bool parse_subscribe(connsock_t *cs, proxy_instance_t *proxi) { json_t *val = NULL, *res_val, *notify_val, *tmp; bool parsed, ret = false; + proxy_instance_t *parent; int retries = 0, size; const char *string; char *buf, *old; @@ -665,14 +666,14 @@ retry: } } proxi->nonce2len = size; - if (parent_proxy(proxi)) { - /* Set the number of clients per proxy on the parent proxy */ - proxi->clients_per_proxy = 1ll << ((size - 3) * 8); - LOGNOTICE("Proxy %ld:%s clients per proxy: %"PRId64, proxi->id, proxi->si->url, - proxi->clients_per_proxy); - if (proxi->clients_per_proxy == 1) - recruit_subproxies(proxi, 1); - } + proxi->clients_per_proxy = 1ll << ((size - 3) * 8); + parent = proxi->parent; + + mutex_lock(&parent->proxy_lock); + parent->recruit -= proxi->clients_per_proxy; + if (parent->recruit < 0) + parent->recruit = 0; + mutex_unlock(&parent->proxy_lock); LOGNOTICE("Found notify for new proxy %ld:%d with enonce %s nonce2len %d", proxi->id, proxi->subid, proxi->enonce1, proxi->nonce2len); @@ -1733,10 +1734,9 @@ retry: add_subproxy(parent, proxy); mutex_lock(&parent->proxy_lock); - if (alive) { - if (--parent->recruit > 0) - recruit = true; - } else /* Reset so the next request will try again */ + if (alive && parent->recruit > 0) + recruit = true; + else /* Reset so the next request will try again */ parent->recruit = 0; mutex_unlock(&parent->proxy_lock); From ecc4482b79a2a3ae8dae197c8d9db382e1077432 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Wed, 25 Feb 2015 19:42:17 +1100 Subject: [PATCH 297/544] Test for dropped clients before not authorised ones --- src/stratifier.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/stratifier.c b/src/stratifier.c index 8adb95e2..99b18242 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -1824,6 +1824,12 @@ static void stratum_broadcast(sdata_t *sdata, json_t *val) if (sdata != ckp_sdata && client->sdata != sdata) continue; + /* Look for clients that may have been dropped which the stratifer has + * not been informed about and ask the connector of they still exist */ + if (client->dropped) { + connector_test_client(ckp, client->id); + continue; + } /* Test for clients that haven't authed in over a minute and drop them */ if (!client->authorised) { if (now_t > client->start_time + 60) { @@ -1832,12 +1838,6 @@ static void stratum_broadcast(sdata_t *sdata, json_t *val) } continue; } - /* Look for clients that may have been dropped which the stratifer has - * not been informed about and ask the connector of they still exist */ - if (client->dropped) { - connector_test_client(ckp, client->id); - continue; - } if (client->reconnect) { reconnect_client(ckp_sdata, client); continue; From 8231c5359276d36fb5662a15bb004764a2c3ef7a Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Wed, 25 Feb 2015 19:51:07 +1100 Subject: [PATCH 298/544] Don't set up a new parent if a reconnect is to the same URL --- src/generator.c | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/src/generator.c b/src/generator.c index 7a1992de..6e966e52 100644 --- a/src/generator.c +++ b/src/generator.c @@ -988,11 +988,11 @@ static bool parse_reconnect(proxy_instance_t *proxi, json_t *val) { server_instance_t *newsi, *si = proxi->si; proxy_instance_t *parent, *newproxi; + bool sameurl = false, ret = false; int64_t high_id, low_id, new_id; ckpool_t *ckp = proxi->ckp; gdata_t *gdata = ckp->data; const char *new_url; - bool ret = false; int new_port; char *url; @@ -1029,14 +1029,23 @@ static bool parse_reconnect(proxy_instance_t *proxi, json_t *val) goto out; } ASPRINTF(&url, "%s:%d", new_url, new_port); - } else + } else { url = strdup(si->url); + sameurl = true; + } LOGINFO("Processing reconnect request to %s", url); ret = true; + parent = proxi->parent; + /* If this is the same url we don't need to replace any existing + * parents, just drop the connection and allow a new one to be + * recruited. */ + if (sameurl) { + disable_subproxy(gdata, parent, proxi); + goto out; + } /* If this isn't a parent proxy, recruit a new parent! */ - parent = proxi->parent; if (parent != proxi) { proxi->reconnect = true; /* Do we already know this proxy is redirecting? */ From a7252c83d538ef360ef8bcc2a865e6b1aeadfd0d Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Wed, 25 Feb 2015 20:51:08 +1100 Subject: [PATCH 299/544] Limit reconnects sent concurrently --- src/stratifier.c | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/src/stratifier.c b/src/stratifier.c index 99b18242..b139ac67 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -1171,7 +1171,7 @@ static void generator_recruit(const ckpool_t *ckp, const int recruits) static void reconnect_clients(sdata_t *sdata) { stratum_instance_t *client, *tmpclient; - int reconnects = 0; + int reconnects = 0, hard = 0; int64_t headroom; proxy_t *proxy; @@ -1186,9 +1186,12 @@ static void reconnect_clients(sdata_t *sdata) if (headroom-- < 1) continue; reconnects++; - if (client->reconnect) + /* Limit reconnects sent concurrently to prevent a flood of new + * connections */ + if (client->reconnect && hard <= SOMAXCONN / 2) { + hard++; reconnect_client(sdata, client); - else + } else client->reconnect = true; } ck_runlock(&sdata->instance_lock); @@ -1217,7 +1220,7 @@ static proxy_t *current_proxy(sdata_t *sdata) static void dead_proxyid(sdata_t *sdata, const int64_t id, const int subid) { stratum_instance_t *client, *tmp; - int reconnects = 0; + int reconnects = 0, hard = 0; int64_t headroom; proxy_t *proxy; @@ -1234,9 +1237,10 @@ static void dead_proxyid(sdata_t *sdata, const int64_t id, const int subid) if (headroom-- < 1) continue; reconnects++; - if (client->reconnect) + if (client->reconnect && hard <= SOMAXCONN / 2) { + hard++; reconnect_client(sdata, client); - else + } else client->reconnect = true; } ck_runlock(&sdata->instance_lock); @@ -1808,6 +1812,7 @@ static void stratum_broadcast(sdata_t *sdata, json_t *val) stratum_instance_t *client, *tmp; ckmsg_t *bulk_send = NULL; time_t now_t = time(NULL); + int hard_reconnect = 0; ckmsgq_t *ssends; if (unlikely(!val)) { @@ -1839,7 +1844,10 @@ static void stratum_broadcast(sdata_t *sdata, json_t *val) continue; } if (client->reconnect) { - reconnect_client(ckp_sdata, client); + if (hard_reconnect <= SOMAXCONN / 2) { + hard_reconnect++; + reconnect_client(ckp_sdata, client); + } continue; } From 13546f54bbab26568266b6cf51c1775c5292d6fc Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Wed, 25 Feb 2015 21:10:26 +1100 Subject: [PATCH 300/544] Close all fds in disable_subproxy, including the parent proxy --- src/generator.c | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/src/generator.c b/src/generator.c index 6e966e52..a676bab3 100644 --- a/src/generator.c +++ b/src/generator.c @@ -964,6 +964,10 @@ static void disable_subproxy(gdata_t *gdata, proxy_instance_t *proxi, proxy_inst { subproxy->alive = false; send_stratifier_deadproxy(gdata->ckp, subproxy->id, subproxy->subid); + if (subproxy->cs->fd > 0) { + epoll_ctl(proxi->epfd, EPOLL_CTL_DEL, subproxy->cs->fd, NULL); + Close(subproxy->cs->fd); + } if (parent_proxy(subproxy)) return; @@ -971,13 +975,8 @@ static void disable_subproxy(gdata_t *gdata, proxy_instance_t *proxi, proxy_inst subproxy->disabled = true; /* Make sure subproxy is still in the list */ subproxy = __subproxy_by_id(proxi, subproxy->subid); - if (subproxy) { - if (subproxy->cs->fd > 0) { - epoll_ctl(proxi->epfd, EPOLL_CTL_DEL, subproxy->cs->fd, NULL); - Close(subproxy->cs->fd); - } + if (likely(subproxy)) HASH_DELETE(sh, proxi->subproxies, subproxy); - } mutex_unlock(&proxi->proxy_lock); if (subproxy) From 523f7236ef288e506d746ab307c18e6bd67d0cb0 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Wed, 25 Feb 2015 21:23:13 +1100 Subject: [PATCH 301/544] Break out of both loops on reconnect issue --- src/generator.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/generator.c b/src/generator.c index a676bab3..512dac29 100644 --- a/src/generator.c +++ b/src/generator.c @@ -2004,7 +2004,7 @@ static void *proxy_recv(void *arg) if (parent_proxy(subproxy)) { LOGWARNING("Proxy %d:%s reconnect issue, dropping existing connection", subproxy->low_id, subproxy->si->url); - break; + goto out; } } continue; @@ -2014,7 +2014,7 @@ static void *proxy_recv(void *arg) LOGWARNING("Unhandled stratum message: %s", cs->buf); } while ((ret = read_socket_line(cs, 0)) > 0); } - +out: mutex_lock(&proxi->proxy_lock); HASH_ITER(sh, proxi->subproxies, subproxy, tmp) { subproxy->disabled = true; From 5688225711620cd754b54e7d7c153d654d6f682b Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Wed, 25 Feb 2015 21:44:17 +1100 Subject: [PATCH 302/544] Reconnect generator after a parent proxy has died or it has received its first notify only --- src/generator.c | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/src/generator.c b/src/generator.c index 512dac29..1634b68f 100644 --- a/src/generator.c +++ b/src/generator.c @@ -105,6 +105,7 @@ struct proxy_instance { bool no_sessionid; /* Doesn't support session id resume on subscribe */ bool no_params; /* Doesn't want any parameters on subscribe */ + bool notified; /* Has this proxy received any notifies yet */ bool disabled; /* Subproxy no longer to be used */ bool reconnect; /* We need to drop and reconnect */ bool reconnecting; /* Testing in progress */ @@ -795,6 +796,11 @@ out: static void send_notify(ckpool_t *ckp, proxy_instance_t *proxi, notify_instance_t *ni); +static void reconnect_generator(const ckpool_t *ckp) +{ + send_proc(ckp->generator, "reconnect"); +} + static bool parse_notify(ckpool_t *ckp, proxy_instance_t *proxi, json_t *val) { const char *prev_hash, *bbversion, *nbit, *ntime; @@ -868,6 +874,12 @@ static bool parse_notify(ckpool_t *ckp, proxy_instance_t *proxi, json_t *val) mutex_unlock(&proxy->notify_lock); send_notify(ckp, proxi, ni); + /* We have all the ingredients necessary to switch to this proxy if + * it's the best one so reassess now. */ + if (unlikely(parent_proxy(proxi) && !proxi->notified)) { + proxi->notified = true; + reconnect_generator(ckp); + } out: return ret; } @@ -1629,7 +1641,7 @@ static bool proxy_alive(ckpool_t *ckp, server_instance_t *si, proxy_instance_t * return true; if (!extract_sockaddr(si->url, &cs->url, &cs->port)) { LOGWARNING("Failed to extract address from %s", si->url); - return ret; + goto out; } if (!connect_proxy(cs)) { if (!pinging) { @@ -1673,6 +1685,8 @@ out: epoll_ctl(proxi->epfd, EPOLL_CTL_DEL, cs->fd, NULL); Close(cs->fd); } + if (parent_proxy(proxi)) + reconnect_generator(ckp); } else { keep_sockalive(cs->fd); event.events = EPOLLIN; @@ -1803,11 +1817,6 @@ static void reconnect_proxy(proxy_instance_t *proxi) create_pthread(&pth, proxy_reconnect, proxi); } -static void reconnect_generator(const ckpool_t *ckp) -{ - send_proc(ckp->generator, "reconnect"); -} - /* For receiving messages from an upstream pool to pass downstream. Responsible * for setting up the connection and testing pool is live. */ static void *passthrough_recv(void *arg) @@ -1922,7 +1931,6 @@ static void *proxy_recv(void *arg) proxi->low_id, proxi->si->url); } alive = proxi->alive; - reconnect_generator(ckp); while (42) { share_msg_t *share, *tmpshare; @@ -1939,7 +1947,6 @@ static void *proxy_recv(void *arg) LOGWARNING("Proxy %d:%s failed, attempting reconnect", proxi->low_id, proxi->si->url); alive = false; - reconnect_generator(ckp); } sleep(5); proxi->reconnect_time = time(NULL); @@ -1950,7 +1957,6 @@ static void *proxy_recv(void *arg) if (!alive && (!current_proxy(gdata) || time(NULL) - proxi->reconnect_time > 30)) { LOGWARNING("Proxy %d:%s recovered", proxi->low_id, proxi->si->url); proxi->reconnect_time = 0; - reconnect_generator(ckp); alive = true; } From b8fb469e07324149af2c57764d75ba10a504051f Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Wed, 25 Feb 2015 23:29:56 +1100 Subject: [PATCH 303/544] Display correct subproxy details in proxysend notice and check for valid values --- src/generator.c | 31 +++++++++++++++++++++++-------- 1 file changed, 23 insertions(+), 8 deletions(-) diff --git a/src/generator.c b/src/generator.c index 1634b68f..c1ec4518 100644 --- a/src/generator.c +++ b/src/generator.c @@ -1523,6 +1523,7 @@ static void *proxy_send(void *arg) connsock_t *cs = proxy->cs; ckpool_t *ckp = cs->ckp; gdata_t *gdata = ckp->data; + stratum_msg_t *msg = NULL; rename_proc("proxysend"); @@ -1531,7 +1532,6 @@ static void *proxy_send(void *arg) proxy_instance_t *subproxy; notify_instance_t *ni; json_t *jobid = NULL; - stratum_msg_t *msg; bool ret = true; int subid = 0; json_t *val; @@ -1548,6 +1548,11 @@ static void *proxy_send(void *arg) tv_to_ts(&abs, &now); abs.tv_sec++; + if (unlikely(msg)) { + json_decref(msg->json_msg); + free(msg); + } + mutex_lock(&proxy->psend_lock); if (!proxy->psends) cond_timedwait(&proxy->psend_cond, &proxy->psend_lock, &abs); @@ -1559,10 +1564,22 @@ static void *proxy_send(void *arg) if (!msg) continue; - json_get_int(&subid, msg->json_msg, "subproxy"); - json_get_int64(&id, msg->json_msg, "jobid"); - json_get_int64(&proxyid, msg->json_msg, "proxy"); - json_get_int64(&client_id, msg->json_msg, "client_id"); + if (unlikely(!json_get_int(&subid, msg->json_msg, "subproxy"))) { + LOGWARNING("Failed to find subproxy in proxy_send msg"); + continue; + } + if (unlikely(!json_get_int64(&id, msg->json_msg, "jobid"))) { + LOGWARNING("Failed to find jobid in proxy_send msg"); + continue; + } + if (unlikely(!json_get_int64(&proxyid, msg->json_msg, "proxy"))) { + LOGWARNING("Failed to find proxy in proxy_send msg"); + continue; + } + if (unlikely(!json_get_int64(&client_id, msg->json_msg, "client_id"))) { + LOGWARNING("Failed to find client_id in proxy_send msg"); + continue; + } if (unlikely(proxyid != proxy->id)) { LOGWARNING("Proxysend for proxy %ld got message for proxy %ld!", proxy->id, proxyid); @@ -1595,11 +1612,9 @@ static void *proxy_send(void *arg) LOGNOTICE("Failed to find subproxy %ld:%d to send message to", proxy->id, subid); } - json_decref(msg->json_msg); - free(msg); if (!ret && subproxy) { LOGNOTICE("Proxy %ld:%d %s failed to send msg in proxy_send, dropping to reconnect", - proxy->id, proxy->subid, proxy->si->url); + id, subid, proxy->si->url); disable_subproxy(gdata, proxy, subproxy); } } From 955e413bea01403fd2cbee8856f6f8a9eaa47fae Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Wed, 25 Feb 2015 23:33:39 +1100 Subject: [PATCH 304/544] Issue reconnect when an upstream pool recovers --- src/generator.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/generator.c b/src/generator.c index c1ec4518..cc3657de 100644 --- a/src/generator.c +++ b/src/generator.c @@ -1970,6 +1970,7 @@ static void *proxy_recv(void *arg) /* Wait 30 seconds before declaring this upstream pool alive * to prevent switching to unstable pools. */ if (!alive && (!current_proxy(gdata) || time(NULL) - proxi->reconnect_time > 30)) { + reconnect_generator(ckp); LOGWARNING("Proxy %d:%s recovered", proxi->low_id, proxi->si->url); proxi->reconnect_time = 0; alive = true; From c28a8fa202815a3f6447780a55ed3021917b4e26 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Wed, 25 Feb 2015 23:48:16 +1100 Subject: [PATCH 305/544] Avoid any attempt at using sessionid in proxy mode --- src/generator.c | 36 +++++------------------------------- src/stratifier.c | 2 +- 2 files changed, 6 insertions(+), 32 deletions(-) diff --git a/src/generator.c b/src/generator.c index cc3657de..6ca9d575 100644 --- a/src/generator.c +++ b/src/generator.c @@ -94,7 +94,6 @@ struct proxy_instance { char *enonce1; char *enonce1bin; int nonce1len; - char *sessionid; int nonce2len; tv_t last_message; @@ -102,7 +101,6 @@ struct proxy_instance { double diff; tv_t last_share; - bool no_sessionid; /* Doesn't support session id resume on subscribe */ bool no_params; /* Doesn't want any parameters on subscribe */ bool notified; /* Has this proxy received any notifies yet */ @@ -619,16 +617,6 @@ retry: goto out; } - /* Free up old data in place if we are re-subscribing */ - old = proxi->sessionid; - proxi->sessionid = NULL; - if (!proxi->no_params && !proxi->no_sessionid && json_array_size(notify_val) > 1) { - /* Copy the session id if one exists. */ - string = json_string_value(json_array_get(notify_val, 1)); - if (string) - proxi->sessionid = strdup(string); - } - free(old); tmp = json_array_get(res_val, 1); if (!tmp || !json_is_string(tmp)) { LOGWARNING("Failed to parse enonce1 in parse_subscribe"); @@ -693,14 +681,8 @@ static bool subscribe_stratum(connsock_t *cs, proxy_instance_t *proxi) json_t *req; retry: - /* Attempt to reconnect if the pool supports resuming */ - if (proxi->sessionid) { - JSON_CPACK(req, "{s:i,s:s,s:[s,s]}", - "id", 0, - "method", "mining.subscribe", - "params", PACKAGE"/"VERSION, proxi->sessionid); - /* Then attempt to connect with just the client description */ - } else if (!proxi->no_params) { + /* Attempt to connect with the client description */ + if (!proxi->no_params) { JSON_CPACK(req, "{s:i,s:s,s:[s]}", "id", 0, "method", "mining.subscribe", @@ -728,16 +710,9 @@ retry: proxi->id, proxi->subid, proxi->si->url); goto out; } - if (proxi->sessionid) { - LOGINFO("Proxy %ld:%d %s failed sessionid reconnect in subscribe_stratum, retrying without", - proxi->id, proxi->subid, proxi->si->url); - proxi->no_sessionid = true; - dealloc(proxi->sessionid); - } else { - LOGINFO("Proxy %ld:%d %s failed connecting with parameters in subscribe_stratum, retrying without", - proxi->id, proxi->subid, proxi->si->url); - proxi->no_params = true; - } + LOGINFO("Proxy %ld:%d %s failed connecting with parameters in subscribe_stratum, retrying without", + proxi->id, proxi->subid, proxi->si->url); + proxi->no_params = true; ret = connect_proxy(cs); if (!ret) { LOGNOTICE("Proxy %ld:%d %s failed to reconnect in subscribe_stratum", @@ -2267,7 +2242,6 @@ static int proxy_mode(ckpool_t *ckp, proc_instance_t *pi) proxi = si->data; free(proxi->enonce1); free(proxi->enonce1bin); - free(proxi->sessionid); pthread_cancel(proxi->pth_psend); pthread_cancel(proxi->pth_precv); join_pthread(proxi->pth_psend); diff --git a/src/stratifier.c b/src/stratifier.c index b139ac67..251b9319 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -2640,7 +2640,7 @@ static json_t *parse_subscribe(stratum_instance_t *client, const int64_t client_ client->useragent = strdup(buf); else client->useragent = ckzalloc(1); // Set to "" - if (arr_size > 1) { + if (arr_size > 1 && !ckp->proxy) { /* This would be the session id for reconnect, it will * not work for clients on a proxied connection. */ buf = json_string_value(json_array_get(params_val, 1)); From 656da05f83f7f5761018740c4cf22716ce71e40a Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Wed, 25 Feb 2015 23:56:21 +1100 Subject: [PATCH 306/544] Add proxy info to client auth message --- src/stratifier.c | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/stratifier.c b/src/stratifier.c index 251b9319..2795f01a 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -3125,8 +3125,13 @@ static json_t *parse_authorise(stratum_instance_t *client, const json_t *params_ if (ret) { client->authorised = ret; user->authorised = ret; - LOGNOTICE("Authorised client %"PRId64" worker %s as user %s", client->id, buf, - user->username); + if (ckp->proxy) { + LOGNOTICE("Authorised client %"PRId64" to proxy %ld:%d, worker %s as user %s", + client->id, client->proxyid, client->subproxyid, buf, user->username); + } else { + LOGNOTICE("Authorised client %"PRId64" worker %s as user %s", + client->id, buf, user->username); + } user->auth_backoff = 3; /* Reset auth backoff time */ } else { LOGNOTICE("Client %"PRId64" worker %s failed to authorise as user %s", client->id, buf, From 8cd36b225bb425ad1eceeeea30cf92d468ea43e2 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Thu, 26 Feb 2015 00:02:34 +1100 Subject: [PATCH 307/544] We must flag all clients to switch on a deadproxy regardless of whether there is room on the current proxy or not --- src/stratifier.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/stratifier.c b/src/stratifier.c index 2795f01a..694b47c9 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -1234,10 +1234,8 @@ static void dead_proxyid(sdata_t *sdata, const int64_t id, const int subid) HASH_ITER(hh, sdata->stratum_instances, client, tmp) { if (client->proxyid != id || client->subproxyid != subid) continue; - if (headroom-- < 1) - continue; reconnects++; - if (client->reconnect && hard <= SOMAXCONN / 2) { + if (headroom-- > 0 && client->reconnect && hard <= SOMAXCONN / 2) { hard++; reconnect_client(sdata, client); } else From 7ca5f9688febc2e055306dba8115acecaebf1815 Mon Sep 17 00:00:00 2001 From: ckolivas Date: Thu, 26 Feb 2015 09:34:56 +1100 Subject: [PATCH 308/544] Use the same enonce1u across all proxies to avoid clients thinking they have resumed with the same sessionid and enonce1u --- src/stratifier.c | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/src/stratifier.c b/src/stratifier.c index 694b47c9..647c4df2 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -325,7 +325,6 @@ struct proxy_base { int64_t max_clients; int64_t bound_clients; int64_t headroom; - enonce1_t enonce1u; proxy_t *subproxies; /* Hashlist of subproxies sorted by subid */ sdata_t *sdata; /* Unique stratifer data for each subproxy */ @@ -1314,10 +1313,7 @@ static void update_subscribe(ckpool_t *ckp, const char *cmd) proxy->enonce1varlen = 0; proxy->enonce2varlen = proxy->nonce2len - proxy->enonce1varlen; proxy->max_clients = 1ll << (proxy->enonce1varlen * 8); - /* Reset the enonce1u in case this is a resubscribe of an existing - * parent proxy. */ proxy->clients = 0; - proxy->enonce1u.u64 = 0; ck_wunlock(&dsdata->workbase_lock); /* Is this a replacement proxy for the current one */ @@ -1553,7 +1549,6 @@ static void reap_proxies(ckpool_t *ckp, sdata_t *sdata) if (!subproxy->bound_clients && !subproxy->dead) { /* Reset the counter to reuse this proxy */ subproxy->clients = 0; - subproxy->enonce1u.u64 = 0; continue; } if (proxy == subproxy) @@ -2467,7 +2462,6 @@ static bool new_enonce1(ckpool_t *ckp, sdata_t *ckp_sdata, sdata_t *sdata, strat mutex_lock(&ckp_sdata->proxy_lock); proxy = sdata->subproxy; - enonce1u = &proxy->enonce1u; client->proxyid = proxy->id; client->subproxyid = proxy->subid; mutex_unlock(&ckp_sdata->proxy_lock); @@ -2476,8 +2470,8 @@ static bool new_enonce1(ckpool_t *ckp, sdata_t *ckp_sdata, sdata_t *sdata, strat LOGWARNING("Proxy reached max clients %"PRId64, proxy->max_clients); return false; } - } else - enonce1u = &sdata->enonce1u; + } + enonce1u = &ckp_sdata->enonce1u; /* Extract the enonce1varlen from the current workbase which may be * a different workbase to when we __fill_enonce1data but the value @@ -2487,7 +2481,7 @@ static bool new_enonce1(ckpool_t *ckp, sdata_t *ckp_sdata, sdata_t *sdata, strat ck_runlock(&sdata->workbase_lock); /* instance_lock protects enonce1u. Recruiting extra proxies should - * prevent these ever locking out.*/ + * prevent these ever running out.*/ ck_wlock(&ckp_sdata->instance_lock); switch(enonce1varlen) { case 8: From 0a4ce58b3f2709b1168cbd7e4c2caa3286d9c075 Mon Sep 17 00:00:00 2001 From: ckolivas Date: Thu, 26 Feb 2015 13:11:20 +1100 Subject: [PATCH 309/544] Dramatically simplify the setting of enonce1 to just an LE encoded 64 bit --- src/stratifier.c | 80 ++++++++++-------------------------------------- 1 file changed, 17 insertions(+), 63 deletions(-) diff --git a/src/stratifier.c b/src/stratifier.c index 647c4df2..baccdc7f 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -291,14 +291,6 @@ struct share { typedef struct share share_t; -/* Variable length enonce1 always refers back to a u64 */ -typedef union { - uint64_t u64; - uint32_t u32; - uint16_t u16; - uint8_t u8; -} enonce1_t; - struct proxy_base { UT_hash_handle hh; UT_hash_handle sh; /* For subproxy hashlist */ @@ -347,7 +339,7 @@ struct stratifier_data { bool ckdb_offline; bool verbose; - enonce1_t enonce1u; + uint64_t enonce1_64; /* For protecting the hashtable data */ cklock_t workbase_lock; @@ -2452,9 +2444,7 @@ static void __fill_enonce1data(const workbase_t *wb, stratum_instance_t *client) static bool new_enonce1(ckpool_t *ckp, sdata_t *ckp_sdata, sdata_t *sdata, stratum_instance_t *client) { proxy_t *proxy = NULL; - enonce1_t *enonce1u; - int enonce1varlen; - bool ret = false; + uint64_t enonce1; if (ckp->proxy) { if (!ckp_sdata->proxy) @@ -2471,53 +2461,19 @@ static bool new_enonce1(ckpool_t *ckp, sdata_t *ckp_sdata, sdata_t *sdata, strat return false; } } - enonce1u = &ckp_sdata->enonce1u; - - /* Extract the enonce1varlen from the current workbase which may be - * a different workbase to when we __fill_enonce1data but the value - * will not change and this avoids grabbing recursive locks */ - ck_rlock(&sdata->workbase_lock); - enonce1varlen = sdata->current_workbase->enonce1varlen; - ck_runlock(&sdata->workbase_lock); - /* instance_lock protects enonce1u. Recruiting extra proxies should - * prevent these ever running out.*/ + /* instance_lock protects enonce1_64. Incrementing a little endian 64bit + * number ensures that no matter how many of the bits we take from the + * left depending on nonce2 length, we'll always get a changing value + * for every next client.*/ ck_wlock(&ckp_sdata->instance_lock); - switch(enonce1varlen) { - case 8: - enonce1u->u64++; - ret = true; - break; - case 7: - case 6: - case 5: - case 4: - enonce1u->u32++; - ret = true; - break; - case 3: - case 2: - enonce1u->u16++; - ret = true; - break; - case 1: - enonce1u->u8++; - ret = true; - break; - case 0: - /* Only one client/enonce1 used on this proxy */ - ret = true; - break; - default: - quit(0, "Invalid enonce1varlen %d", enonce1varlen); - } - if (ret) { - if (proxy) { - client->proxy = proxy; - proxy->clients++; - proxy->bound_clients++; - } - client->enonce1_64 = enonce1u->u64; + enonce1 = htole64(ckp_sdata->enonce1_64); + enonce1++; + client->enonce1_64 = ckp_sdata->enonce1_64 = le64toh(enonce1); + if (proxy) { + client->proxy = proxy; + proxy->clients++; + proxy->bound_clients++; } ck_wunlock(&ckp_sdata->instance_lock); @@ -2525,10 +2481,7 @@ static bool new_enonce1(ckpool_t *ckp, sdata_t *ckp_sdata, sdata_t *sdata, strat __fill_enonce1data(sdata->current_workbase, client); ck_runlock(&sdata->workbase_lock); - if (unlikely(!ret)) - LOGWARNING("Enonce1 space exhausted! Proxy rejecting clients"); - - return ret; + return true; } static void stratum_send_message(sdata_t *sdata, const stratum_instance_t *client, const char *msg); @@ -5041,12 +4994,13 @@ int stratifier(proc_instance_t *pi) } } - randomiser = ((int64_t)time(NULL)) << 32; + randomiser = time(NULL); + sdata->enonce1_64 = htole64(randomiser); /* Set the initial id to time as high bits so as to not send the same * id on restarts */ + randomiser <<= 32; if (!ckp->proxy) sdata->blockchange_id = sdata->workbase_id = randomiser; - sdata->enonce1u.u64 = htobe64(randomiser); if (!ckp->serverurls) { ckp->serverurl[0] = "127.0.0.1"; From d5dedfe4a905a65b6d4675f0a53541ccb875dabc Mon Sep 17 00:00:00 2001 From: ckolivas Date: Thu, 26 Feb 2015 15:01:30 +1100 Subject: [PATCH 310/544] Demote discarded message --- src/stratifier.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/stratifier.c b/src/stratifier.c index baccdc7f..b78e6609 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -1567,7 +1567,7 @@ static void reap_proxies(ckpool_t *ckp, sdata_t *sdata) mutex_unlock(&sdata->proxy_lock); if (dead) - LOGNOTICE("Stratifier discarded %d dead proxies", dead); + LOGINFO("Stratifier discarded %d dead proxies", dead); } /* Enter with instance_lock held */ From 45e80f09ffcf46ef4a79f33b2b6d5513cf13e7b4 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Fri, 27 Feb 2015 11:39:12 +1100 Subject: [PATCH 311/544] Send a blank sessionid in proxy mode so clients don't think we have resumed if enonce1 ends up matching on reconnect --- src/stratifier.c | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/src/stratifier.c b/src/stratifier.c index 694b47c9..50aa5cfb 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -2672,10 +2672,18 @@ static json_t *parse_subscribe(stratum_instance_t *client, const int64_t client_ /* Workbases will exist if sdata->current_workbase is not NULL */ ck_rlock(&sdata->workbase_lock); n2len = sdata->workbases->enonce2varlen; - JSON_CPACK(ret, "[[[s,s]],s,i]", "mining.notify", client->enonce1, client->enonce1, - n2len); ck_runlock(&sdata->workbase_lock); + /* Send a blank sessionid in proxy mode so clients don't think we have + * resumed if enonce1 ends up matching on reconnect. */ + if (ckp->proxy) { + JSON_CPACK(ret, "[[[s,s]],s,i]", "mining.notify", "", client->enonce1, + n2len); + } else { + JSON_CPACK(ret, "[[[s,s]],s,i]", "mining.notify", client->enonce1, client->enonce1, + n2len); + } + client->subscribed = true; return ret; From 64f4abd302f92a7eedf0bace475b4bc8badd1d75 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Fri, 27 Feb 2015 11:53:43 +1100 Subject: [PATCH 312/544] Count combined clients per parent proxy --- src/stratifier.c | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/src/stratifier.c b/src/stratifier.c index a1a6691e..7a244f78 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -313,11 +313,13 @@ struct proxy_base { bool subscribed; bool notified; - int64_t clients; - int64_t max_clients; - int64_t bound_clients; - int64_t headroom; + int64_t clients; /* Incrementing client count */ + int64_t max_clients; /* Maximum number of clients per subproxy */ + int64_t bound_clients; /* Currently actively bound clients */ + int64_t combined_clients; /* Total clients of all subproxies of a parent proxy */ + int64_t headroom; /* Temporary variable when calculating how many more clients can bind */ + proxy_t *parent; /* Parent proxy of each subproxy */ proxy_t *subproxies; /* Hashlist of subproxies sorted by subid */ sdata_t *sdata; /* Unique stratifer data for each subproxy */ bool dead; @@ -943,8 +945,10 @@ static void update_base(ckpool_t *ckp, const int prio) static void __kill_instance(stratum_instance_t *client) { - if (client->proxy) + if (client->proxy) { client->proxy->bound_clients--; + client->proxy->parent->combined_clients--; + } free(client->workername); free(client->useragent); free(client); @@ -1034,6 +1038,7 @@ static proxy_t *__generate_proxy(sdata_t *sdata, const int64_t id) { proxy_t *proxy = ckzalloc(sizeof(proxy_t)); + proxy->parent = proxy; proxy->id = id; proxy->low_id = id & 0xFFFFFFFF; proxy->sdata = duplicate_sdata(sdata); @@ -1050,6 +1055,7 @@ static proxy_t *__generate_subproxy(sdata_t *sdata, proxy_t *proxy, const int su { proxy_t *subproxy = ckzalloc(sizeof(proxy_t)); + subproxy->parent = proxy; subproxy->id = proxy->id; subproxy->low_id = proxy->low_id; subproxy->subid = subid; @@ -2474,6 +2480,7 @@ static bool new_enonce1(ckpool_t *ckp, sdata_t *ckp_sdata, sdata_t *sdata, strat client->proxy = proxy; proxy->clients++; proxy->bound_clients++; + proxy->parent->combined_clients++; } ck_wunlock(&ckp_sdata->instance_lock); From d80393930d0929b549d1dd5629543cb3d59d68af Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Fri, 27 Feb 2015 11:58:42 +1100 Subject: [PATCH 313/544] Store a count of the number of subproxies per proxy --- src/stratifier.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/stratifier.c b/src/stratifier.c index 7a244f78..09a2621c 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -319,6 +319,7 @@ struct proxy_base { int64_t combined_clients; /* Total clients of all subproxies of a parent proxy */ int64_t headroom; /* Temporary variable when calculating how many more clients can bind */ + int subproxy_count; /* Number of subproxies */ proxy_t *parent; /* Parent proxy of each subproxy */ proxy_t *subproxies; /* Hashlist of subproxies sorted by subid */ sdata_t *sdata; /* Unique stratifer data for each subproxy */ @@ -1046,6 +1047,7 @@ static proxy_t *__generate_proxy(sdata_t *sdata, const int64_t id) proxy->sdata->verbose = true; /* subid == 0 on parent proxy */ HASH_ADD(sh, proxy->subproxies, subid, sizeof(int), proxy); + proxy->subproxy_count++; HASH_ADD_I64(sdata->proxies, id, proxy); sdata->proxy_count++; return proxy; @@ -1060,6 +1062,7 @@ static proxy_t *__generate_subproxy(sdata_t *sdata, proxy_t *proxy, const int su subproxy->low_id = proxy->low_id; subproxy->subid = subid; HASH_ADD(sh, proxy->subproxies, subid, sizeof(int), subproxy); + proxy->subproxy_count++; subproxy->sdata = duplicate_sdata(sdata); subproxy->sdata->subproxy = subproxy; return subproxy; @@ -1567,6 +1570,7 @@ static void reap_proxies(ckpool_t *ckp, sdata_t *sdata) } dead++; HASH_DELETE(sh, proxy->subproxies, subproxy); + proxy->subproxy_count--; free_proxy(subproxy); } } From ede31398436c5c89f876d26c1af80de1e4075d2b Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Fri, 27 Feb 2015 12:36:32 +1100 Subject: [PATCH 314/544] Add proxy statistics to logging --- src/stratifier.c | 45 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/src/stratifier.c b/src/stratifier.c index 09a2621c..9d4282ee 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -4796,6 +4796,51 @@ static void *statsupdate(void *arg) dealloc(s); fclose(fp); + if (ckp->proxy && sdata->proxy) { + proxy_t *proxy, *proxytmp, *subproxy, *subtmp; + + mutex_lock(&sdata->proxy_lock); + JSON_CPACK(val, "{sI,si,si}", + "current", sdata->proxy->id, + "active", HASH_COUNT(sdata->proxies), + "total", sdata->proxy_count); + mutex_unlock(&sdata->proxy_lock); + + s = json_dumps(val, JSON_NO_UTF8 | JSON_PRESERVE_ORDER); + json_decref(val); + LOGNOTICE("Proxy:%s", s); + dealloc(s); + + mutex_lock(&sdata->proxy_lock); + HASH_ITER(hh, sdata->proxies, proxy, proxytmp) { + JSON_CPACK(val, "{sI,si,si,sI,sb}", + "id", proxy->id, + "priority", proxy->low_id, + "subproxies", proxy->subproxy_count, + "clients", proxy->combined_clients, + "alive", !proxy->dead); + s = json_dumps(val, JSON_NO_UTF8 | JSON_PRESERVE_ORDER); + json_decref(val); + LOGNOTICE("Proxies:%s", s); + dealloc(s); + HASH_ITER(sh, proxy->subproxies, subproxy, subtmp) { + JSON_CPACK(val, "{sI,si,si,sI,sI,sf,sb}", + "id", subproxy->id, + "subid", subproxy->subid, + "nonce2len", subproxy->nonce2len, + "clients", subproxy->bound_clients, + "maxclients", subproxy->max_clients, + "diff", subproxy->diff, + "alive", !subproxy->dead); + s = json_dumps(val, JSON_NO_UTF8 | JSON_PRESERVE_ORDER); + json_decref(val); + LOGNOTICE("subproxies:%s", s); + dealloc(s); + } + } + mutex_unlock(&sdata->proxy_lock); + } + ts_realtime(&ts_now); sprintf(cdfield, "%lu,%lu", ts_now.tv_sec, ts_now.tv_nsec); JSON_CPACK(val, "{ss,si,si,si,sf,sf,sf,sf,ss,ss,ss,ss}", From 1479303138721f46b9dfadb7a8846b4c7394b1e3 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Sat, 28 Feb 2015 11:53:48 +1100 Subject: [PATCH 315/544] Add helper functions for storing messages to be dumped outside of other locks --- src/stratifier.c | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/src/stratifier.c b/src/stratifier.c index d9e646bb..cead06a9 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -437,6 +437,29 @@ static const char *ckdb_ids[] = { "heartbeat" }; +/* For storing a set of messages within another lock, allowing us to dump them + * to the log outside of lock */ +static void add_msg_entry(char_entry_t **entries, char **buf) +{ + char_entry_t *entry = ckalloc(sizeof(char_entry_t)); + + entry->buf = *buf; + *buf = NULL; + DL_APPEND(*entries, entry); +} + +static void notice_msg_entries(char_entry_t **entries) +{ + char_entry_t *entry, *tmpentry; + + DL_FOREACH_SAFE(*entries, entry, tmpentry) { + DL_DELETE(*entries, entry); + LOGNOTICE("%s", entry->buf); + free(entry->buf); + free(entry); + } +} + static void generate_coinbase(const ckpool_t *ckp, workbase_t *wb) { uint64_t *u64, g64, d64 = 0; From 343e5922e77e2ac0177ea17db21f734c5a38f009 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Sat, 28 Feb 2015 11:58:32 +1100 Subject: [PATCH 316/544] Use the message entry helper functions for user stats logging --- src/stratifier.c | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/src/stratifier.c b/src/stratifier.c index cead06a9..293915c2 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -4634,10 +4634,10 @@ static void *statsupdate(void *arg) double ghs, ghs1, ghs5, ghs15, ghs60, ghs360, ghs1440, ghs10080, per_tdiff; char suffix1[16], suffix5[16], suffix15[16], suffix60[16], cdfield[64]; char suffix360[16], suffix1440[16], suffix10080[16]; - char_entry_t *char_list = NULL, *char_t, *chartmp_t; stratum_instance_t *client, *tmp; log_entry_t *log_entries = NULL; user_instance_t *user, *tmpuser; + char_entry_t *char_list = NULL; int idle_workers = 0; char *fname, *s; tv_t now, diff; @@ -4756,10 +4756,12 @@ static void *statsupdate(void *arg) s = json_dumps(val, JSON_NO_UTF8 | JSON_PRESERVE_ORDER | JSON_EOL); add_log_entry(&log_entries, &fname, &s); if (!idle) { - char_t = ckalloc(sizeof(char_entry_t)); + char *sp; + s = json_dumps(val, JSON_NO_UTF8 | JSON_PRESERVE_ORDER); - ASPRINTF(&char_t->buf, "User %s:%s", user->username, s); - DL_APPEND(char_list, char_t); + ASPRINTF(&sp, "User %s:%s", user->username, s); + dealloc(s); + add_msg_entry(&char_list, &sp); } json_decref(val); } @@ -4767,13 +4769,7 @@ static void *statsupdate(void *arg) /* Dump log entries out of instance_lock */ dump_log_entries(&log_entries); - - DL_FOREACH_SAFE(char_list, char_t, chartmp_t) { - LOGNOTICE("%s", char_t->buf); - DL_DELETE(char_list, char_t); - free(char_t->buf); - dealloc(char_t); - } + notice_msg_entries(&char_list); ghs1 = stats->dsps1 * nonces; suffix_string(ghs1, suffix1, 16, 0); From 75cc43664a6f62c69ec6817ec8cc181722043bb2 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Sat, 28 Feb 2015 12:06:11 +1100 Subject: [PATCH 317/544] Use the msg entry helpers for proxy logging --- src/stratifier.c | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/src/stratifier.c b/src/stratifier.c index 293915c2..dc51331e 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -4639,7 +4639,7 @@ static void *statsupdate(void *arg) user_instance_t *user, *tmpuser; char_entry_t *char_list = NULL; int idle_workers = 0; - char *fname, *s; + char *fname, *s, *sp; tv_t now, diff; ts_t ts_now; json_t *val; @@ -4756,8 +4756,6 @@ static void *statsupdate(void *arg) s = json_dumps(val, JSON_NO_UTF8 | JSON_PRESERVE_ORDER | JSON_EOL); add_log_entry(&log_entries, &fname, &s); if (!idle) { - char *sp; - s = json_dumps(val, JSON_NO_UTF8 | JSON_PRESERVE_ORDER); ASPRINTF(&sp, "User %s:%s", user->username, s); dealloc(s); @@ -4862,8 +4860,9 @@ static void *statsupdate(void *arg) "alive", !proxy->dead); s = json_dumps(val, JSON_NO_UTF8 | JSON_PRESERVE_ORDER); json_decref(val); - LOGNOTICE("Proxies:%s", s); + ASPRINTF(&sp, "Proxies:%s", s); dealloc(s); + add_msg_entry(&char_list, &sp); HASH_ITER(sh, proxy->subproxies, subproxy, subtmp) { JSON_CPACK(val, "{sI,si,si,sI,sI,sf,sb}", "id", subproxy->id, @@ -4873,13 +4872,15 @@ static void *statsupdate(void *arg) "maxclients", subproxy->max_clients, "diff", subproxy->diff, "alive", !subproxy->dead); - s = json_dumps(val, JSON_NO_UTF8 | JSON_PRESERVE_ORDER); - json_decref(val); - LOGNOTICE("subproxies:%s", s); - dealloc(s); + s = json_dumps(val, JSON_NO_UTF8 | JSON_PRESERVE_ORDER); + json_decref(val); + ASPRINTF(&sp, "Subproxies:%s", s); + dealloc(s); + add_msg_entry(&char_list, &sp); } } mutex_unlock(&sdata->proxy_lock); + notice_msg_entries(&char_list); } ts_realtime(&ts_now); From 992fd54b9d9be8e47e0208e7b582bacf82b7e983 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Wed, 4 Mar 2015 21:24:37 +1100 Subject: [PATCH 318/544] Choose best proxy when there is any adequately sized subproxy matching the current one --- src/stratifier.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/stratifier.c b/src/stratifier.c index dc51331e..cf37916c 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -2565,7 +2565,7 @@ static sdata_t *select_sdata(const ckpool_t *ckp, sdata_t *ckp_sdata) best_id = best->id; best_lowid = best->low_id; best_subid = best->subid; - if (best == current) + if (proxy == current) break; } } From f6d87f1569a56e4eaaf85bf75557d2247963125d Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Sun, 15 Mar 2015 09:50:12 +1100 Subject: [PATCH 319/544] Create a basic handler of json messages on the listener unix socket as RPC API commands with a ckmsgq --- src/Makefile.am | 2 +- src/ckpool.c | 16 ++++++++++++++++ src/ckpool.h | 3 +++ 3 files changed, 20 insertions(+), 1 deletion(-) diff --git a/src/Makefile.am b/src/Makefile.am index 61d13a6d..de543330 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -10,7 +10,7 @@ libckpool_la_LIBADD = @PTHREAD_LIBS@ @MATH_LIBS@ @RT_LIBS@ bin_PROGRAMS = ckpool ckpmsg notifier ckpool_SOURCES = ckpool.c ckpool.h generator.c generator.h bitcoin.c bitcoin.h \ stratifier.c stratifier.h connector.c connector.h uthash.h \ - utlist.h + utlist.h api.c api.h ckpool_LDADD = libckpool.la @JANSSON_LIBS@ ckpmsg_SOURCES = ckpmsg.c diff --git a/src/ckpool.c b/src/ckpool.c index 21f3da4b..8b45df8b 100644 --- a/src/ckpool.c +++ b/src/ckpool.c @@ -31,6 +31,7 @@ #include "generator.h" #include "stratifier.h" #include "connector.h" +#include "api.h" ckpool_t *global_ckp; @@ -294,6 +295,17 @@ out: return ret; } +static void api_message(ckpool_t *ckp, char **buf, int *sockd) +{ + apimsg_t *apimsg = ckalloc(sizeof(apimsg_t)); + + apimsg->buf = *buf; + *buf = NULL; + apimsg->sockd = *sockd; + *sockd = -1; + ckmsgq_add(ckp->ckpapi, apimsg); +} + /* Listen for incoming global requests. Always returns a response if possible */ static void *listener(void *arg) { @@ -316,6 +328,9 @@ retry: if (!buf) { LOGWARNING("Failed to get message in listener"); send_unix_msg(sockd, "failed"); + } else if (buf[0] == '{') { + /* Any JSON messages received are for the RPC API to handle */ + api_message(ckp, &buf, &sockd); } else if (cmdmatch(buf, "shutdown")) { LOGWARNING("Listener received shutdown message, terminating ckpool"); send_unix_msg(sockd, "exiting"); @@ -1684,6 +1699,7 @@ int main(int argc, char **argv) launch_logger(&ckp.main); ckp.logfd = fileno(ckp.logfp); + ckp.ckpapi = create_ckmsgq(&ckp, "api", &ckpool_api); create_pthread(&ckp.pth_listener, listener, &ckp.main); /* Launch separate processes from here */ diff --git a/src/ckpool.h b/src/ckpool.h index 64abe7b8..e9bc4793 100644 --- a/src/ckpool.h +++ b/src/ckpool.h @@ -135,6 +135,9 @@ struct ckpool_instance { /* How many clients maximum to accept before rejecting further */ int maxclients; + /* API message queue */ + ckmsgq_t *ckpapi; + /* Logger message queue NOTE: Unique per process */ ckmsgq_t *logger; /* Process instance data of parent/child processes */ From a6682d96bcd15459a54841952246986fd034c088 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Sun, 15 Mar 2015 10:03:53 +1100 Subject: [PATCH 320/544] Look preferentially for stratum messages in the proxy loop --- src/generator.c | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/src/generator.c b/src/generator.c index 6ca9d575..115eee50 100644 --- a/src/generator.c +++ b/src/generator.c @@ -2142,7 +2142,19 @@ retry: goto retry; } LOGDEBUG("Proxy received request: %s", buf); - if (cmdmatch(buf, "shutdown")) { + if (likely(buf[0] == '{')) { + if (ckp->passthrough) + passthrough_add_send(proxi, buf); + else { + /* Anything remaining should be share submissions */ + json_t *val = json_loads(buf, 0, NULL); + + if (unlikely(!val)) + LOGWARNING("Generator received invalid json message: %s", buf); + else + submit_share(gdata, val); + } + } else if (cmdmatch(buf, "shutdown")) { ret = 0; goto out; } else if (cmdmatch(buf, "reconnect")) { @@ -2162,13 +2174,7 @@ retry: /* Anything remaining should be stratum messages */ passthrough_add_send(proxi, buf); } else { - /* Anything remaining should be share submissions */ - json_t *val = json_loads(buf, 0, NULL); - - if (unlikely(!val)) - LOGWARNING("Generator received unrecognised message: %s", buf); - else - submit_share(gdata, val); + LOGWARNING("Generator received unrecognised message: %s", buf); } goto retry; out: From 5c7059492ad85088c82bbe0abdafd93d7301fbb0 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Sun, 15 Mar 2015 16:52:39 +1100 Subject: [PATCH 324/544] Make add_proxy a separate function --- src/generator.c | 59 +++++++++++++++++++++++++++---------------------- 1 file changed, 33 insertions(+), 26 deletions(-) diff --git a/src/generator.c b/src/generator.c index 115eee50..2b5ff784 100644 --- a/src/generator.c +++ b/src/generator.c @@ -2208,34 +2208,41 @@ static int server_mode(ckpool_t *ckp, proc_instance_t *pi) return ret; } +static void add_proxy(ckpool_t *ckp, const int num) +{ + proxy_instance_t *proxy; + server_instance_t *si; + + ckp->servers = realloc(ckp->servers, sizeof(server_instance_t *) * (num + 1)); + ckp->servers[num] = ckzalloc(sizeof(server_instance_t)); + si = ckp->servers[num]; + si->url = strdup(ckp->proxyurl[num]); + si->auth = strdup(ckp->proxyauth[num]); + si->pass = strdup(ckp->proxypass[num]); + proxy = ckzalloc(sizeof(proxy_instance_t)); + si->data = proxy; + proxy->auth = si->auth; + proxy->pass = si->pass; + proxy->si = si; + proxy->ckp = ckp; + proxy->cs = &si->cs; + proxy->cs->ckp = ckp; + mutex_init(&proxy->notify_lock); + mutex_init(&proxy->share_lock); +} + static int proxy_mode(ckpool_t *ckp, proc_instance_t *pi) { gdata_t *gdata = ckp->data; - proxy_instance_t *proxi; + proxy_instance_t *proxy; server_instance_t *si; int i, ret; mutex_init(&gdata->lock); /* Create all our proxy structures and pointers */ - ckp->servers = ckalloc(sizeof(server_instance_t *) * ckp->proxies); - for (i = 0; i < ckp->proxies; i++) { - ckp->servers[i] = ckzalloc(sizeof(server_instance_t)); - si = ckp->servers[i]; - si->url = strdup(ckp->proxyurl[i]); - si->auth = strdup(ckp->proxyauth[i]); - si->pass = strdup(ckp->proxypass[i]); - proxi = ckzalloc(sizeof(proxy_instance_t)); - si->data = proxi; - proxi->auth = si->auth; - proxi->pass = si->pass; - proxi->si = si; - proxi->ckp = ckp; - proxi->cs = &si->cs; - proxi->cs->ckp = ckp; - mutex_init(&proxi->notify_lock); - mutex_init(&proxi->share_lock); - } + for (i = 0; i < ckp->proxies; i++) + add_proxy(ckp, i); LOGWARNING("%s generator ready", ckp->name); @@ -2245,13 +2252,13 @@ static int proxy_mode(ckpool_t *ckp, proc_instance_t *pi) for (i = 0; i < ckp->proxies; i++) { si = ckp->servers[i]; Close(si->cs.fd); - proxi = si->data; - free(proxi->enonce1); - free(proxi->enonce1bin); - pthread_cancel(proxi->pth_psend); - pthread_cancel(proxi->pth_precv); - join_pthread(proxi->pth_psend); - join_pthread(proxi->pth_precv); + proxy = si->data; + free(proxy->enonce1); + free(proxy->enonce1bin); + pthread_cancel(proxy->pth_psend); + pthread_cancel(proxy->pth_precv); + join_pthread(proxy->pth_psend); + join_pthread(proxy->pth_precv); dealloc(si->data); dealloc(si->url); dealloc(si->auth); From 78af906c14b59f177b216b71981c8ef3051e1b10 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Sun, 15 Mar 2015 17:37:02 +1100 Subject: [PATCH 325/544] Add command proxy.list to get a list of current proxies, ensuring we return valid json --- src/generator.c | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/src/generator.c b/src/generator.c index 2b5ff784..9ed24ab8 100644 --- a/src/generator.c +++ b/src/generator.c @@ -2090,6 +2090,38 @@ static proxy_instance_t *wait_best_proxy(ckpool_t *ckp, gdata_t *gdata) return ret; } +static void send_list(gdata_t *gdata, const int sockd) +{ + proxy_instance_t *proxy, *tmp; + json_t *val, *array_val; + char *response; + + array_val = json_array(); + + mutex_lock(&gdata->lock); + HASH_ITER(hh, gdata->proxies, proxy, tmp) { + JSON_CPACK(val, "{sI,si,ss,ss,sf,sb,sb,sb,si}", + "id", proxy->id, "low_id", proxy->low_id, + "auth", proxy->auth, "pass", proxy->pass, + "diff", proxy->diff, "notified", proxy->notified, + "disabled", proxy->disabled, "alive", proxy->alive, + "subproxies", proxy->subproxy_count); + if (proxy->enonce1) { + json_set_string(val, "enonce1", proxy->enonce1); + json_set_int(val, "nonce1len", proxy->nonce1len); + json_set_int(val, "nonce2len", proxy->nonce2len); + } + json_array_append_new(array_val, val); + } + mutex_unlock(&gdata->lock); + + JSON_CPACK(val, "{so}", "proxies", array_val); + response = json_dumps(val, JSON_NO_UTF8 | JSON_PRESERVE_ORDER); + json_decref(val); + send_unix_msg(sockd, response); + free(response); +} + static int proxy_loop(proc_instance_t *pi) { proxy_instance_t *proxi = NULL, *cproxy; @@ -2154,6 +2186,8 @@ retry: else submit_share(gdata, val); } + } else if (cmdmatch(buf, "list")) { + send_list(gdata, sockd); } else if (cmdmatch(buf, "shutdown")) { ret = 0; goto out; From 5a6b1ee7175b0cc9afc7e0842f3070189ba2d5d2 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Sun, 15 Mar 2015 19:15:12 +1100 Subject: [PATCH 326/544] Add an API command to return the subproxy list by proxy id --- src/generator.c | 56 ++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 53 insertions(+), 3 deletions(-) diff --git a/src/generator.c b/src/generator.c index 9ed24ab8..66293593 100644 --- a/src/generator.c +++ b/src/generator.c @@ -2122,6 +2122,57 @@ static void send_list(gdata_t *gdata, const int sockd) free(response); } +static void send_sublist(gdata_t *gdata, const int sockd, const char *buf) +{ + proxy_instance_t *proxy, *subproxy, *tmp; + json_t *val = NULL, *array_val; + json_error_t err_val; + char *response; + int64_t id; + + array_val = json_array(); + + val = json_loads(buf, 0, NULL); + if (unlikely(!val)) { + LOGWARNING("Failed to JSON decode sublist \"%s\" (%d):%s", buf, + err_val.line, err_val.text); + goto out; + } + if (unlikely(!json_get_int64(&id, val, "id"))) { + LOGWARNING("Failed to get ID in send_sublist JSON: %s", buf); + goto out; + } + proxy = proxy_by_id(gdata, id); + if (unlikely(!proxy)) { + LOGWARNING("Failed to find proxy %"PRId64" in send_sublist", id); + goto out; + } + + mutex_lock(&gdata->lock); + HASH_ITER(sh, proxy->subproxies, subproxy, tmp) { + JSON_CPACK(val, "{si,ss,ss,sf,sb,sb,sb}", + "subid", subproxy->id, + "auth", subproxy->auth, "pass", subproxy->pass, + "diff", subproxy->diff, "notified", subproxy->notified, + "disabled", subproxy->disabled, "alive", subproxy->alive); + if (subproxy->enonce1) { + json_set_string(val, "enonce1", subproxy->enonce1); + json_set_int(val, "nonce1len", subproxy->nonce1len); + json_set_int(val, "nonce2len", subproxy->nonce2len); + } + json_array_append_new(array_val, val); + } + mutex_unlock(&gdata->lock); + +out: + JSON_CPACK(val, "{so}", "subproxies", array_val); + response = json_dumps(val, JSON_NO_UTF8 | JSON_PRESERVE_ORDER); + if (val) + json_decref(val); + send_unix_msg(sockd, response); + free(response); +} + static int proxy_loop(proc_instance_t *pi) { proxy_instance_t *proxi = NULL, *cproxy; @@ -2188,6 +2239,8 @@ retry: } } else if (cmdmatch(buf, "list")) { send_list(gdata, sockd); + } else if (cmdmatch(buf, "sublist")) { + send_sublist(gdata, sockd, buf + 8); } else if (cmdmatch(buf, "shutdown")) { ret = 0; goto out; @@ -2204,9 +2257,6 @@ retry: recruit_subproxy(proxi, buf); } else if (cmdmatch(buf, "dropproxy")) { drop_proxy(gdata, buf); - } else if (ckp->passthrough) { - /* Anything remaining should be stratum messages */ - passthrough_add_send(proxi, buf); } else { LOGWARNING("Generator received unrecognised message: %s", buf); } From 24762bf57638c5f631a2bbe554ada73a01e57ef8 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Sun, 15 Mar 2015 20:40:15 +1100 Subject: [PATCH 327/544] Provide means for adding proxies to the global list --- src/generator.c | 47 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/src/generator.c b/src/generator.c index 66293593..f03eef90 100644 --- a/src/generator.c +++ b/src/generator.c @@ -2173,6 +2173,51 @@ out: free(response); } +static void add_proxy(ckpool_t *ckp, const int num); + +static void parse_addproxy(ckpool_t *ckp, gdata_t *gdata, const int sockd, const char *buf) +{ + char *url = NULL, *auth = NULL, *pass = NULL, *response; + proxy_instance_t *proxy; + json_error_t err_val; + json_t *val = NULL; + int id; + + val = json_loads(buf, 0, NULL); + if (unlikely(!val)) { + LOGWARNING("Failed to JSON decode addproxy \"%s\" (%d):%s", buf, + err_val.line, err_val.text); + goto out; + } + json_get_string(&url, val, "url"); + json_get_string(&auth, val, "auth"); + json_get_string(&pass, val, "pass"); + json_decref(val); + if (unlikely(!url || !auth || !pass)) { + LOGWARNING("Failed to decode url/auth/pass in addproxy %s", buf); + goto out; + } + + mutex_lock(&gdata->lock); + id = ckp->proxies++; + ckp->proxyurl[id] = strdup(url); + ckp->proxyauth[id] = strdup(auth); + ckp->proxypass[id] = strdup(pass); + add_proxy(ckp, id); + proxy = ckp->servers[id]->data; + proxy->id = proxy->low_id = id; + HASH_ADD_I64(gdata->proxies, id, proxy); + mutex_unlock(&gdata->lock); + + prepare_proxy(proxy); + JSON_CPACK(val, "{sI,ss,ss,ss}", + "id", proxy->id, "url", url, "auth", auth, "pass", pass); +out: + response = json_dumps(val, JSON_NO_UTF8 | JSON_PRESERVE_ORDER); + send_unix_msg(sockd, response); + free(response); +} + static int proxy_loop(proc_instance_t *pi) { proxy_instance_t *proxi = NULL, *cproxy; @@ -2241,6 +2286,8 @@ retry: send_list(gdata, sockd); } else if (cmdmatch(buf, "sublist")) { send_sublist(gdata, sockd, buf + 8); + } else if (cmdmatch(buf, "addproxy")) { + parse_addproxy(ckp, gdata, sockd, buf + 9); } else if (cmdmatch(buf, "shutdown")) { ret = 0; goto out; From 9dabedde7d22ef5a0ac1b9e84921d5919366a04e Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Sun, 15 Mar 2015 21:39:54 +1100 Subject: [PATCH 328/544] Encode error message in failed API responses --- src/generator.c | 71 ++++++++++++++++++++++++++++++++++--------------- 1 file changed, 49 insertions(+), 22 deletions(-) diff --git a/src/generator.c b/src/generator.c index f03eef90..412d9632 100644 --- a/src/generator.c +++ b/src/generator.c @@ -12,6 +12,7 @@ #include #include #include +#include #include #include @@ -2090,11 +2091,22 @@ static proxy_instance_t *wait_best_proxy(ckpool_t *ckp, gdata_t *gdata) return ret; } +static void send_json_response(json_t *val, const int sockd) +{ + char *response; + + response = json_dumps(val, JSON_NO_UTF8 | JSON_PRESERVE_ORDER); + if (!response) + return; + json_decref(val); + send_unix_msg(sockd, response); + free(response); +} + static void send_list(gdata_t *gdata, const int sockd) { proxy_instance_t *proxy, *tmp; json_t *val, *array_val; - char *response; array_val = json_array(); @@ -2116,10 +2128,34 @@ static void send_list(gdata_t *gdata, const int sockd) mutex_unlock(&gdata->lock); JSON_CPACK(val, "{so}", "proxies", array_val); - response = json_dumps(val, JSON_NO_UTF8 | JSON_PRESERVE_ORDER); - json_decref(val); - send_unix_msg(sockd, response); - free(response); + send_json_response(val, sockd); +} + +static json_t *_json_encode_errormsg(json_error_t *err_val, const char *func) +{ + json_t *ret; + char *buf; + + ASPRINTF(&buf, "Failed to JSON decode in %s (%d):%s", func, err_val->line, err_val->text); + JSON_CPACK(ret, "{ss}", "errormsg", buf); + free(buf); + return ret; +} + +#define json_encode_errormsg(err_val) _json_encode_errormsg(err_val, __func__) + +static json_t *json_errormsg(const char *fmt, ...) +{ + char *buf = NULL; + json_t *ret; + va_list ap; + + va_start(ap, fmt); + VASPRINTF(&buf, fmt, ap); + va_end(ap); + JSON_CPACK(ret, "{ss}", "errormsg", buf); + free(buf); + return ret; } static void send_sublist(gdata_t *gdata, const int sockd, const char *buf) @@ -2127,24 +2163,22 @@ static void send_sublist(gdata_t *gdata, const int sockd, const char *buf) proxy_instance_t *proxy, *subproxy, *tmp; json_t *val = NULL, *array_val; json_error_t err_val; - char *response; int64_t id; array_val = json_array(); val = json_loads(buf, 0, NULL); if (unlikely(!val)) { - LOGWARNING("Failed to JSON decode sublist \"%s\" (%d):%s", buf, - err_val.line, err_val.text); + val = json_encode_errormsg(&err_val); goto out; } if (unlikely(!json_get_int64(&id, val, "id"))) { - LOGWARNING("Failed to get ID in send_sublist JSON: %s", buf); + val = json_errormsg("Failed to get ID in send_sublist JSON: %s", buf); goto out; } proxy = proxy_by_id(gdata, id); if (unlikely(!proxy)) { - LOGWARNING("Failed to find proxy %"PRId64" in send_sublist", id); + val = json_errormsg("Failed to find proxy %"PRId64" in send_sublist", id); goto out; } @@ -2164,20 +2198,16 @@ static void send_sublist(gdata_t *gdata, const int sockd, const char *buf) } mutex_unlock(&gdata->lock); -out: JSON_CPACK(val, "{so}", "subproxies", array_val); - response = json_dumps(val, JSON_NO_UTF8 | JSON_PRESERVE_ORDER); - if (val) - json_decref(val); - send_unix_msg(sockd, response); - free(response); +out: + send_json_response(val, sockd); } static void add_proxy(ckpool_t *ckp, const int num); static void parse_addproxy(ckpool_t *ckp, gdata_t *gdata, const int sockd, const char *buf) { - char *url = NULL, *auth = NULL, *pass = NULL, *response; + char *url = NULL, *auth = NULL, *pass = NULL; proxy_instance_t *proxy; json_error_t err_val; json_t *val = NULL; @@ -2185,8 +2215,7 @@ static void parse_addproxy(ckpool_t *ckp, gdata_t *gdata, const int sockd, const val = json_loads(buf, 0, NULL); if (unlikely(!val)) { - LOGWARNING("Failed to JSON decode addproxy \"%s\" (%d):%s", buf, - err_val.line, err_val.text); + val = json_encode_errormsg(&err_val); goto out; } json_get_string(&url, val, "url"); @@ -2213,9 +2242,7 @@ static void parse_addproxy(ckpool_t *ckp, gdata_t *gdata, const int sockd, const JSON_CPACK(val, "{sI,ss,ss,ss}", "id", proxy->id, "url", url, "auth", auth, "pass", pass); out: - response = json_dumps(val, JSON_NO_UTF8 | JSON_PRESERVE_ORDER); - send_unix_msg(sockd, response); - free(response); + send_json_response(val, sockd); } static int proxy_loop(proc_instance_t *pi) From 788918460e8337982727e4aee939a7993f5c3c97 Mon Sep 17 00:00:00 2001 From: ckolivas Date: Mon, 16 Mar 2015 10:02:55 +1100 Subject: [PATCH 329/544] Make api responses use a common function in api.c --- src/generator.c | 19 ++++--------------- 1 file changed, 4 insertions(+), 15 deletions(-) diff --git a/src/generator.c b/src/generator.c index 412d9632..97199976 100644 --- a/src/generator.c +++ b/src/generator.c @@ -22,6 +22,7 @@ #include "bitcoin.h" #include "uthash.h" #include "utlist.h" +#include "api.h" struct notify_instance { /* Hash table data */ @@ -2091,18 +2092,6 @@ static proxy_instance_t *wait_best_proxy(ckpool_t *ckp, gdata_t *gdata) return ret; } -static void send_json_response(json_t *val, const int sockd) -{ - char *response; - - response = json_dumps(val, JSON_NO_UTF8 | JSON_PRESERVE_ORDER); - if (!response) - return; - json_decref(val); - send_unix_msg(sockd, response); - free(response); -} - static void send_list(gdata_t *gdata, const int sockd) { proxy_instance_t *proxy, *tmp; @@ -2128,7 +2117,7 @@ static void send_list(gdata_t *gdata, const int sockd) mutex_unlock(&gdata->lock); JSON_CPACK(val, "{so}", "proxies", array_val); - send_json_response(val, sockd); + send_api_response(val, sockd); } static json_t *_json_encode_errormsg(json_error_t *err_val, const char *func) @@ -2200,7 +2189,7 @@ static void send_sublist(gdata_t *gdata, const int sockd, const char *buf) JSON_CPACK(val, "{so}", "subproxies", array_val); out: - send_json_response(val, sockd); + send_api_response(val, sockd); } static void add_proxy(ckpool_t *ckp, const int num); @@ -2242,7 +2231,7 @@ static void parse_addproxy(ckpool_t *ckp, gdata_t *gdata, const int sockd, const JSON_CPACK(val, "{sI,ss,ss,ss}", "id", proxy->id, "url", url, "auth", auth, "pass", pass); out: - send_json_response(val, sockd); + send_api_response(val, sockd); } static int proxy_loop(proc_instance_t *pi) From 80be061c3d914a15faa004e7f56d7c12bc27a0a5 Mon Sep 17 00:00:00 2001 From: ckolivas Date: Mon, 16 Mar 2015 10:07:26 +1100 Subject: [PATCH 330/544] Make json encoded errormsgs part of api.c for general use --- src/generator.c | 28 ---------------------------- 1 file changed, 28 deletions(-) diff --git a/src/generator.c b/src/generator.c index 97199976..ad8fd17d 100644 --- a/src/generator.c +++ b/src/generator.c @@ -12,7 +12,6 @@ #include #include #include -#include #include #include @@ -2120,33 +2119,6 @@ static void send_list(gdata_t *gdata, const int sockd) send_api_response(val, sockd); } -static json_t *_json_encode_errormsg(json_error_t *err_val, const char *func) -{ - json_t *ret; - char *buf; - - ASPRINTF(&buf, "Failed to JSON decode in %s (%d):%s", func, err_val->line, err_val->text); - JSON_CPACK(ret, "{ss}", "errormsg", buf); - free(buf); - return ret; -} - -#define json_encode_errormsg(err_val) _json_encode_errormsg(err_val, __func__) - -static json_t *json_errormsg(const char *fmt, ...) -{ - char *buf = NULL; - json_t *ret; - va_list ap; - - va_start(ap, fmt); - VASPRINTF(&buf, fmt, ap); - va_end(ap); - JSON_CPACK(ret, "{ss}", "errormsg", buf); - free(buf); - return ret; -} - static void send_sublist(gdata_t *gdata, const int sockd, const char *buf) { proxy_instance_t *proxy, *subproxy, *tmp; From 2122b0f68918b07d818d2841e4dfd2b7572ebe5d Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Mon, 16 Mar 2015 22:27:22 +1100 Subject: [PATCH 333/544] Remove use of server_instances by proxies in the generator --- src/generator.c | 228 ++++++++++++++++++++---------------------------- 1 file changed, 94 insertions(+), 134 deletions(-) diff --git a/src/generator.c b/src/generator.c index ad8fd17d..fd90be00 100644 --- a/src/generator.c +++ b/src/generator.c @@ -82,15 +82,15 @@ struct proxy_instance { proxy_instance_t *prev; /* For dead proxy list */ ckpool_t *ckp; - connsock_t *cs; - server_instance_t *si; + connsock_t cs; bool passthrough; int64_t id; /* Proxy server id*/ int low_id; /* Low bits of id */ int subid; /* Subproxy id */ - const char *auth; - const char *pass; + char *url; + char *auth; + char *pass; char *enonce1; char *enonce1bin; @@ -581,7 +581,7 @@ retry: parsed = true; if (!(buf = new_proxy_line(cs))) { LOGNOTICE("Proxy %ld:%d %s failed to receive line in parse_subscribe", - proxi->id, proxi->subid, proxi->si->url); + proxi->id, proxi->subid, proxi->url); goto out; } LOGDEBUG("parse_subscribe received %s", buf); @@ -614,7 +614,7 @@ retry: goto retry; } LOGNOTICE("Proxy %ld:%d %s failed to parse subscribe response in parse_subscribe", - proxi->id, proxi->subid, proxi->si->url); + proxi->id, proxi->subid, proxi->url); goto out; } @@ -649,7 +649,7 @@ retry: if (size < 3) { if (!proxi->subid) { LOGWARNING("Proxy %d %s Nonce2 length %d too small for fast miners", - proxi->low_id, proxi->si->url, size); + proxi->low_id, proxi->url, size); } else { LOGNOTICE("Proxy %ld:%d Nonce2 length %d too small for fast miners", proxi->id, proxi->subid, size); @@ -699,7 +699,7 @@ retry: json_decref(req); if (!ret) { LOGNOTICE("Proxy %ld:%d %s failed to send message in subscribe_stratum", - proxi->id, proxi->subid, proxi->si->url); + proxi->id, proxi->subid, proxi->url); goto out; } ret = parse_subscribe(cs, proxi); @@ -708,16 +708,16 @@ retry: if (proxi->no_params) { LOGNOTICE("Proxy %ld:%d %s failed all subscription options in subscribe_stratum", - proxi->id, proxi->subid, proxi->si->url); + proxi->id, proxi->subid, proxi->url); goto out; } LOGINFO("Proxy %ld:%d %s failed connecting with parameters in subscribe_stratum, retrying without", - proxi->id, proxi->subid, proxi->si->url); + proxi->id, proxi->subid, proxi->url); proxi->no_params = true; ret = connect_proxy(cs); if (!ret) { LOGNOTICE("Proxy %ld:%d %s failed to reconnect in subscribe_stratum", - proxi->id, proxi->subid, proxi->si->url); + proxi->id, proxi->subid, proxi->url); goto out; } goto retry; @@ -873,12 +873,11 @@ static bool parse_diff(proxy_instance_t *proxi, json_t *val) static bool send_version(proxy_instance_t *proxi, json_t *val) { json_t *json_msg, *id_val = json_object_dup(val, "id"); - connsock_t *cs = proxi->cs; bool ret; JSON_CPACK(json_msg, "{sossso}", "id", id_val, "result", PACKAGE"/"VERSION, "error", json_null()); - ret = send_json_msg(cs, json_msg); + ret = send_json_msg(&proxi->cs, json_msg); json_decref(json_msg); return ret; } @@ -899,12 +898,11 @@ static bool show_message(json_t *val) static bool send_pong(proxy_instance_t *proxi, json_t *val) { json_t *json_msg, *id_val = json_object_dup(val, "id"); - connsock_t *cs = proxi->cs; bool ret; JSON_CPACK(json_msg, "{sossso}", "id", id_val, "result", "pong", "error", json_null()); - ret = send_json_msg(cs, json_msg); + ret = send_json_msg(&proxi->cs, json_msg); json_decref(json_msg); return ret; } @@ -933,6 +931,9 @@ static void store_proxy(gdata_t *gdata, proxy_instance_t *proxy) LOGINFO("Recycling data from proxy %ld:%d", proxy->id, proxy->subid); mutex_lock(&gdata->lock); + dealloc(proxy->url); + dealloc(proxy->auth); + dealloc(proxy->pass); DL_APPEND(gdata->dead_proxies, proxy); mutex_unlock(&gdata->lock); } @@ -952,9 +953,9 @@ static void disable_subproxy(gdata_t *gdata, proxy_instance_t *proxi, proxy_inst { subproxy->alive = false; send_stratifier_deadproxy(gdata->ckp, subproxy->id, subproxy->subid); - if (subproxy->cs->fd > 0) { - epoll_ctl(proxi->epfd, EPOLL_CTL_DEL, subproxy->cs->fd, NULL); - Close(subproxy->cs->fd); + if (subproxy->cs.fd > 0) { + epoll_ctl(proxi->epfd, EPOLL_CTL_DEL, subproxy->cs.fd, NULL); + Close(subproxy->cs.fd); } if (parent_proxy(subproxy)) return; @@ -973,7 +974,6 @@ static void disable_subproxy(gdata_t *gdata, proxy_instance_t *proxi, proxy_inst static bool parse_reconnect(proxy_instance_t *proxi, json_t *val) { - server_instance_t *newsi, *si = proxi->si; proxy_instance_t *parent, *newproxi; bool sameurl = false, ret = false; int64_t high_id, low_id, new_id; @@ -997,10 +997,10 @@ static bool parse_reconnect(proxy_instance_t *proxi, json_t *val) char *dot_pool, *dot_reconnect; int len; - dot_pool = strchr(si->url, '.'); + dot_pool = strchr(proxi->url, '.'); if (!dot_pool) { LOGWARNING("Denied stratum reconnect request from server without domain %s", - si->url); + proxi->url); goto out; } dot_reconnect = strchr(new_url, '.'); @@ -1012,12 +1012,12 @@ static bool parse_reconnect(proxy_instance_t *proxi, json_t *val) len = strlen(dot_reconnect); if (strncmp(dot_pool, dot_reconnect, len)) { LOGWARNING("Denied stratum reconnect request from %s to non-matching domain %s", - si->url, new_url); + proxi->url, new_url); goto out; } ASPRINTF(&url, "%s:%d", new_url, new_port); } else { - url = strdup(si->url); + url = strdup(proxi->url); sameurl = true; } LOGINFO("Processing reconnect request to %s", url); @@ -1043,27 +1043,18 @@ static bool parse_reconnect(proxy_instance_t *proxi, json_t *val) } else proxi->redirecting = false; - newsi = ckzalloc(sizeof(server_instance_t)); - mutex_lock(&gdata->lock); high_id = proxi->id >> 32; /* Use the high bits for the reconnect id */ high_id++; high_id <<= 32; low_id = proxi->id & 0x00000000FFFFFFFFll; /* Use the low bits for the master id */ new_id = high_id | low_id; - ckp->servers[low_id] = newsi; - newsi->url = url; - newsi->auth = strdup(si->auth); - newsi->pass = strdup(si->pass); - newproxi = ckzalloc(sizeof(proxy_instance_t)); - newsi->data = newproxi; - newproxi->auth = newsi->auth; - newproxi->pass = newsi->pass; - newproxi->si = newsi; + newproxi->url = url; + newproxi->auth = strdup(proxi->auth); + newproxi->pass = strdup(proxi->pass); newproxi->ckp = ckp; - newproxi->cs = &newsi->cs; - newproxi->cs->ckp = ckp; + newproxi->cs.ckp = ckp; newproxi->low_id = low_id; newproxi->id = new_id; newproxi->subproxy_count = ++proxi->subproxy_count; @@ -1231,7 +1222,7 @@ static bool auth_stratum(ckpool_t *ckp, connsock_t *cs, proxy_instance_t *proxi) json_decref(req); if (!ret) { LOGNOTICE("Proxy %ld:%d %s failed to send message in auth_stratum", - proxi->id, proxi->subid, proxi->si->url); + proxi->id, proxi->subid, proxi->url); if (cs->fd > 0) { epoll_ctl(proxi->epfd, EPOLL_CTL_DEL, cs->fd, NULL); Close(cs->fd); @@ -1246,7 +1237,7 @@ static bool auth_stratum(ckpool_t *ckp, connsock_t *cs, proxy_instance_t *proxi) buf = next_proxy_line(cs, proxi); if (!buf) { LOGNOTICE("Proxy %ld:%d %s failed to receive line in auth_stratum", - proxi->id, proxi->subid, proxi->si->url); + proxi->id, proxi->subid, proxi->url); ret = false; goto out; } @@ -1256,27 +1247,27 @@ static bool auth_stratum(ckpool_t *ckp, connsock_t *cs, proxy_instance_t *proxi) val = json_msg_result(buf, &res_val, &err_val); if (!val) { LOGWARNING("Proxy %d:%d %s failed to get a json result in auth_stratum, got: %s", - proxi->low_id, proxi->subid, proxi->si->url, buf); + proxi->low_id, proxi->subid, proxi->url, buf); goto out; } if (err_val && !json_is_null(err_val)) { LOGWARNING("Proxy %d:%d %s failed to authorise in auth_stratum due to err_val, got: %s", - proxi->low_id, proxi->subid, proxi->si->url, buf); + proxi->low_id, proxi->subid, proxi->url, buf); goto out; } if (res_val) { ret = json_is_true(res_val); if (!ret) { LOGWARNING("Proxy %d:%d %s failed to authorise in auth_stratum, got: %s", - proxi->low_id, proxi->subid, proxi->si->url, buf); + proxi->low_id, proxi->subid, proxi->url, buf); goto out; } } else { /* No result and no error but successful val means auth success */ ret = true; } - LOGINFO("Proxy %ld:%d %s auth success in auth_stratum", proxi->id, proxi->subid, proxi->si->url); + LOGINFO("Proxy %ld:%d %s auth success in auth_stratum", proxi->id, proxi->subid, proxi->url); out: if (val) json_decref(val); @@ -1496,7 +1487,7 @@ out: static void *proxy_send(void *arg) { proxy_instance_t *proxy = (proxy_instance_t *)arg; - connsock_t *cs = proxy->cs; + connsock_t *cs = &proxy->cs; ckpool_t *ckp = cs->ckp; gdata_t *gdata = ckp->data; stratum_msg_t *msg = NULL; @@ -1569,7 +1560,7 @@ static void *proxy_send(void *arg) subproxy = subproxy_by_id(proxy, subid); if (subproxy) - cs = subproxy->cs; + cs = &subproxy->cs; if (jobid && subproxy) { JSON_CPACK(val, "{s[soooo]soss}", "params", proxy->auth, jobid, json_object_dup(msg->json_msg, "nonce2"), @@ -1582,7 +1573,7 @@ static void *proxy_send(void *arg) } else if (!jobid) { stratifier_reconnect_client(ckp, client_id); LOGNOTICE("Proxy %ld:%s failed to find matching jobid for %sknown subproxy in proxysend", - proxy->id, proxy->si->url, subproxy ? "" : "un"); + proxy->id, proxy->url, subproxy ? "" : "un"); } else { stratifier_reconnect_client(ckp, client_id); LOGNOTICE("Failed to find subproxy %ld:%d to send message to", @@ -1590,7 +1581,7 @@ static void *proxy_send(void *arg) } if (!ret && subproxy) { LOGNOTICE("Proxy %ld:%d %s failed to send msg in proxy_send, dropping to reconnect", - id, subid, proxy->si->url); + id, subid, proxy->url); disable_subproxy(gdata, proxy, subproxy); } } @@ -1616,13 +1607,13 @@ static void passthrough_add_send(proxy_instance_t *proxi, const char *msg) { pass_msg_t *pm = ckzalloc(sizeof(pass_msg_t)); - pm->cs = proxi->cs; + pm->cs = &proxi->cs; ASPRINTF(&pm->msg, "%s\n", msg); ckmsgq_add(proxi->passsends, pm); } -static bool proxy_alive(ckpool_t *ckp, server_instance_t *si, proxy_instance_t *proxi, - connsock_t *cs, bool pinging, int epfd) +static bool proxy_alive(ckpool_t *ckp, proxy_instance_t *proxi, connsock_t *cs, + bool pinging, int epfd) { struct epoll_event event; bool ret = false; @@ -1630,8 +1621,8 @@ static bool proxy_alive(ckpool_t *ckp, server_instance_t *si, proxy_instance_t * /* Has this proxy already been reconnected? */ if (cs->fd > 0) return true; - if (!extract_sockaddr(si->url, &cs->url, &cs->port)) { - LOGWARNING("Failed to extract address from %s", si->url); + if (!extract_sockaddr(proxi->url, &cs->url, &cs->port)) { + LOGWARNING("Failed to extract address from %s", proxi->url); goto out; } if (!connect_proxy(cs)) { @@ -1663,7 +1654,7 @@ static bool proxy_alive(ckpool_t *ckp, server_instance_t *si, proxy_instance_t * if (!auth_stratum(ckp, cs, proxi)) { if (!pinging) { LOGWARNING("Failed initial authorise to %s:%s with %s:%s !", - cs->url, cs->port, si->auth, si->pass); + cs->url, cs->port, proxi->auth, proxi->pass); } goto out; } @@ -1707,21 +1698,20 @@ static proxy_instance_t *create_subproxy(gdata_t *gdata, proxy_instance_t *proxi subproxy->disabled = false; } else { subproxy = ckzalloc(sizeof(proxy_instance_t)); - subproxy->cs = ckzalloc(sizeof(connsock_t)); mutex_init(&subproxy->share_lock); } mutex_unlock(&gdata->lock); - subproxy->cs->ckp = subproxy->ckp = proxi->ckp; - subproxy->si = proxi->si; + subproxy->cs.ckp = subproxy->ckp = proxi->ckp; mutex_lock(&proxi->proxy_lock); subproxy->subid = ++proxi->subproxy_count; mutex_unlock(&proxi->proxy_lock); subproxy->id = proxi->id; - subproxy->auth = proxi->auth; - subproxy->pass = proxi->pass; + subproxy->url = strdup(proxi->url); + subproxy->auth = strdup(proxi->auth); + subproxy->pass = strdup(proxi->pass); subproxy->parent = proxi; subproxy->epfd = proxi->epfd; return subproxy; @@ -1739,7 +1729,7 @@ static void *proxy_recruit(void *arg) retry: recruit = false; proxy = create_subproxy(gdata, parent); - alive = proxy_alive(ckp, proxy->si, proxy, proxy->cs, false, parent->epfd); + alive = proxy_alive(ckp, proxy, &proxy->cs, false, parent->epfd); if (!alive) { LOGNOTICE("Subproxy failed proxy_alive testing"); store_proxy(gdata, proxy); @@ -1787,12 +1777,11 @@ static void recruit_subproxy(proxy_instance_t *proxi, const char *buf) static void *proxy_reconnect(void *arg) { proxy_instance_t *proxy = (proxy_instance_t *)arg; - server_instance_t *si = proxy->si; - connsock_t *cs = proxy->cs; + connsock_t *cs = &proxy->cs; ckpool_t *ckp = proxy->ckp; pthread_detach(pthread_self()); - proxy_alive(ckp, si, proxy, cs, true, proxy->epfd); + proxy_alive(ckp, proxy, cs, true, proxy->epfd); proxy->reconnecting = false; return NULL; } @@ -1813,8 +1802,7 @@ static void reconnect_proxy(proxy_instance_t *proxi) static void *passthrough_recv(void *arg) { proxy_instance_t *proxi = (proxy_instance_t *)arg; - server_instance_t *si = proxi->si; - connsock_t *cs = proxi->cs; + connsock_t *cs = &proxi->cs; ckpool_t *ckp = proxi->ckp; struct epoll_event event; bool alive; @@ -1828,17 +1816,17 @@ static void *passthrough_recv(void *arg) return NULL; } - if (proxy_alive(ckp, si, proxi, cs, false, epfd)) { + if (proxy_alive(ckp, proxi, cs, false, epfd)) { reconnect_generator(ckp); LOGWARNING("Proxy %d:%s connection established", - proxi->low_id, proxi->si->url); + proxi->low_id, proxi->url); } alive = proxi->alive; while (42) { int ret; - while (!proxy_alive(ckp, si, proxi, cs, true, epfd)) { + while (!proxy_alive(ckp, proxi, cs, true, epfd)) { if (alive) { alive = false; reconnect_generator(ckp); @@ -1854,7 +1842,7 @@ static void *passthrough_recv(void *arg) ret = read_socket_line(cs, 60); if (ret < 1) { LOGWARNING("Proxy %ld:%s failed to read_socket_line in proxy_recv, attempting reconnect", - proxi->id, proxi->si->url); + proxi->id, proxi->url); alive = proxi->alive = false; reconnect_generator(ckp); continue; @@ -1901,8 +1889,7 @@ static void *proxy_recv(void *arg) { proxy_instance_t *proxi = (proxy_instance_t *)arg; proxy_instance_t *subproxy, *tmp; - server_instance_t *si = proxi->si; - connsock_t *cs = proxi->cs; + connsock_t *cs = &proxi->cs; ckpool_t *ckp = proxi->ckp; gdata_t *gdata = ckp->data; struct epoll_event event; @@ -1917,9 +1904,9 @@ static void *proxy_recv(void *arg) return NULL; } - if (proxy_alive(ckp, si, proxi, cs, false, epfd)) { + if (proxy_alive(ckp, proxi, cs, false, epfd)) { LOGWARNING("Proxy %d:%s connection established", - proxi->low_id, proxi->si->url); + proxi->low_id, proxi->url); } alive = proxi->alive; @@ -1936,7 +1923,7 @@ static void *proxy_recv(void *arg) reconnect_proxy(proxi); if (alive) { LOGWARNING("Proxy %d:%s failed, attempting reconnect", - proxi->low_id, proxi->si->url); + proxi->low_id, proxi->url); alive = false; } sleep(5); @@ -1947,7 +1934,7 @@ static void *proxy_recv(void *arg) * to prevent switching to unstable pools. */ if (!alive && (!current_proxy(gdata) || time(NULL) - proxi->reconnect_time > 30)) { reconnect_generator(ckp); - LOGWARNING("Proxy %d:%s recovered", proxi->low_id, proxi->si->url); + LOGWARNING("Proxy %d:%s recovered", proxi->low_id, proxi->url); proxi->reconnect_time = 0; alive = true; } @@ -1980,7 +1967,7 @@ static void *proxy_recv(void *arg) ret = epoll_wait(epfd, &event, 1, 600000); if (likely(ret > 0)) { subproxy = event.data.ptr; - cs = subproxy->cs; + cs = &subproxy->cs; if (event.events & EPOLLHUP) ret = -1; else @@ -1988,7 +1975,7 @@ static void *proxy_recv(void *arg) } if (ret < 1) { LOGNOTICE("Proxy %ld:%d %s failed to epoll/read_socket_line in proxy_recv", - proxi->id, subproxy->subid, subproxy->si->url); + proxi->id, subproxy->subid, subproxy->url); disable_subproxy(gdata, proxi, subproxy); continue; } @@ -2001,7 +1988,7 @@ static void *proxy_recv(void *arg) disable_subproxy(gdata, proxi, subproxy); if (parent_proxy(subproxy)) { LOGWARNING("Proxy %d:%s reconnect issue, dropping existing connection", - subproxy->low_id, subproxy->si->url); + subproxy->low_id, subproxy->url); goto out; } } @@ -2017,9 +2004,9 @@ out: HASH_ITER(sh, proxi->subproxies, subproxy, tmp) { subproxy->disabled = true; send_stratifier_deadproxy(ckp, subproxy->id, subproxy->subid); - if (subproxy->cs->fd > 0) { - epoll_ctl(epfd, EPOLL_CTL_DEL, subproxy->cs->fd, NULL); - Close(subproxy->cs->fd); + if (subproxy->cs.fd > 0) { + epoll_ctl(epfd, EPOLL_CTL_DEL, subproxy->cs.fd, NULL); + Close(subproxy->cs.fd); } HASH_DELETE(sh, proxi->subproxies, subproxy); } @@ -2039,27 +2026,6 @@ static void prepare_proxy(proxy_instance_t *proxi) create_pthread(&proxi->pth_precv, proxy_recv, proxi); } -static void setup_proxies(ckpool_t *ckp, gdata_t *gdata) -{ - int i; - - for (i = 0; i < ckp->proxies; i++) { - proxy_instance_t *proxi; - server_instance_t *si; - - si = ckp->servers[i]; - proxi = si->data; - proxi->id = proxi->low_id = i; - HASH_ADD_I64(gdata->proxies, id, proxi); - if (ckp->passthrough) { - create_pthread(&proxi->pth_precv, passthrough_recv, proxi); - proxi->passsends = create_ckmsgq(ckp, "passsend", &passthrough_send); - } else { - prepare_proxy(proxi); - } - } -} - static proxy_instance_t *wait_best_proxy(ckpool_t *ckp, gdata_t *gdata) { proxy_instance_t *ret = NULL, *proxi, *tmp; @@ -2164,7 +2130,7 @@ out: send_api_response(val, sockd); } -static void add_proxy(ckpool_t *ckp, const int num); +static proxy_instance_t *__add_proxy(ckpool_t *ckp, gdata_t *gdata, const int num); static void parse_addproxy(ckpool_t *ckp, gdata_t *gdata, const int sockd, const char *buf) { @@ -2190,13 +2156,13 @@ static void parse_addproxy(ckpool_t *ckp, gdata_t *gdata, const int sockd, const mutex_lock(&gdata->lock); id = ckp->proxies++; + ckp->proxyurl = realloc(ckp->proxyurl, sizeof(char **) * ckp->proxies); + ckp->proxyauth = realloc(ckp->proxyauth, sizeof(char **) * ckp->proxies); + ckp->proxypass = realloc(ckp->proxypass, sizeof(char **) * ckp->proxies); ckp->proxyurl[id] = strdup(url); ckp->proxyauth[id] = strdup(auth); ckp->proxypass[id] = strdup(pass); - add_proxy(ckp, id); - proxy = ckp->servers[id]->data; - proxy->id = proxy->low_id = id; - HASH_ADD_I64(gdata->proxies, id, proxy); + proxy = __add_proxy(ckp, gdata, id); mutex_unlock(&gdata->lock); prepare_proxy(proxy); @@ -2215,7 +2181,6 @@ static int proxy_loop(proc_instance_t *pi) gdata_t *gdata = ckp->data; char *buf = NULL; - setup_proxies(ckp, gdata); reconnect: Close(sockd); /* This does not necessarily mean we reconnect, but a change has @@ -2226,9 +2191,8 @@ reconnect: if (proxi != cproxy) { proxi = cproxy; if (!ckp->passthrough) { - connsock_t *cs = proxi->cs; LOGWARNING("Successfully connected to proxy %d %s:%s as proxy", - proxi->low_id, cs->url, cs->port); + proxi->low_id, proxi->url, proxi->cs.port); dealloc(buf); ASPRINTF(&buf, "proxy=%ld", proxi->id); send_proc(ckp->stratifier, buf); @@ -2327,41 +2291,41 @@ static int server_mode(ckpool_t *ckp, proc_instance_t *pi) return ret; } -static void add_proxy(ckpool_t *ckp, const int num) +static proxy_instance_t *__add_proxy(ckpool_t *ckp, gdata_t *gdata, const int id) { proxy_instance_t *proxy; - server_instance_t *si; - ckp->servers = realloc(ckp->servers, sizeof(server_instance_t *) * (num + 1)); - ckp->servers[num] = ckzalloc(sizeof(server_instance_t)); - si = ckp->servers[num]; - si->url = strdup(ckp->proxyurl[num]); - si->auth = strdup(ckp->proxyauth[num]); - si->pass = strdup(ckp->proxypass[num]); proxy = ckzalloc(sizeof(proxy_instance_t)); - si->data = proxy; - proxy->auth = si->auth; - proxy->pass = si->pass; - proxy->si = si; + proxy->id = proxy->low_id = id; + proxy->url = strdup(ckp->proxyurl[id]); + proxy->auth = strdup(ckp->proxyauth[id]); + proxy->pass = strdup(ckp->proxypass[id]); proxy->ckp = ckp; - proxy->cs = &si->cs; - proxy->cs->ckp = ckp; + proxy->cs.ckp = ckp; mutex_init(&proxy->notify_lock); mutex_init(&proxy->share_lock); + HASH_ADD_I64(gdata->proxies, id, proxy); + return proxy; } static int proxy_mode(ckpool_t *ckp, proc_instance_t *pi) { gdata_t *gdata = ckp->data; proxy_instance_t *proxy; - server_instance_t *si; int i, ret; mutex_init(&gdata->lock); /* Create all our proxy structures and pointers */ - for (i = 0; i < ckp->proxies; i++) - add_proxy(ckp, i); + for (i = 0; i < ckp->proxies; i++) { + proxy = __add_proxy(ckp, gdata, i); + if (ckp->passthrough) { + create_pthread(&proxy->pth_precv, passthrough_recv, proxy); + proxy->passsends = create_ckmsgq(ckp, "passsend", &passthrough_send); + } else { + prepare_proxy(proxy); + } + } LOGWARNING("%s generator ready", ckp->name); @@ -2369,24 +2333,20 @@ static int proxy_mode(ckpool_t *ckp, proc_instance_t *pi) mutex_lock(&gdata->lock); for (i = 0; i < ckp->proxies; i++) { - si = ckp->servers[i]; - Close(si->cs.fd); - proxy = si->data; + continue; // FIXME: Find proxies + Close(proxy->cs.fd); free(proxy->enonce1); free(proxy->enonce1bin); pthread_cancel(proxy->pth_psend); pthread_cancel(proxy->pth_precv); join_pthread(proxy->pth_psend); join_pthread(proxy->pth_precv); - dealloc(si->data); - dealloc(si->url); - dealloc(si->auth); - dealloc(si->pass); - dealloc(si); + dealloc(proxy->url); + dealloc(proxy->auth); + dealloc(proxy->pass); } mutex_unlock(&gdata->lock); - dealloc(ckp->servers); return ret; } From bcc5ca1a463b5aa3f62c11a2e518e2afc1d5c877 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Mon, 16 Mar 2015 22:47:23 +1100 Subject: [PATCH 334/544] Properly encode error messages in API responses --- src/generator.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/generator.c b/src/generator.c index fd90be00..5d36f22c 100644 --- a/src/generator.c +++ b/src/generator.c @@ -2094,7 +2094,7 @@ static void send_sublist(gdata_t *gdata, const int sockd, const char *buf) array_val = json_array(); - val = json_loads(buf, 0, NULL); + val = json_loads(buf, 0, &err_val); if (unlikely(!val)) { val = json_encode_errormsg(&err_val); goto out; @@ -2140,7 +2140,7 @@ static void parse_addproxy(ckpool_t *ckp, gdata_t *gdata, const int sockd, const json_t *val = NULL; int id; - val = json_loads(buf, 0, NULL); + val = json_loads(buf, 0, &err_val); if (unlikely(!val)) { val = json_encode_errormsg(&err_val); goto out; @@ -2150,7 +2150,7 @@ static void parse_addproxy(ckpool_t *ckp, gdata_t *gdata, const int sockd, const json_get_string(&pass, val, "pass"); json_decref(val); if (unlikely(!url || !auth || !pass)) { - LOGWARNING("Failed to decode url/auth/pass in addproxy %s", buf); + val = json_errormsg("Failed to decode url/auth/pass in addproxy %s", buf); goto out; } From 2cee823d6c15c4138efc91bdeb5fdd6b594648d0 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Tue, 17 Mar 2015 10:06:52 +1100 Subject: [PATCH 335/544] Allow url to be unique per proxy and subproxy allowing us to not have to recruit a full new parent proxy with a complicated id --- src/generator.c | 285 ++++++++++++++++++++---------------------------- 1 file changed, 121 insertions(+), 164 deletions(-) diff --git a/src/generator.c b/src/generator.c index 5d36f22c..e5c3cf4a 100644 --- a/src/generator.c +++ b/src/generator.c @@ -84,8 +84,7 @@ struct proxy_instance { ckpool_t *ckp; connsock_t cs; bool passthrough; - int64_t id; /* Proxy server id*/ - int low_id; /* Low bits of id */ + int id; /* Proxy server id*/ int subid; /* Subproxy id */ char *url; @@ -107,8 +106,7 @@ struct proxy_instance { bool notified; /* Has this proxy received any notifies yet */ bool disabled; /* Subproxy no longer to be used */ bool reconnect; /* We need to drop and reconnect */ - bool reconnecting; /* Testing in progress */ - bool redirecting; /* Children have received a reconnect */ + bool reconnecting; /* Testing of parent in progress */ int64_t recruit; /* No of recruiting requests in progress */ bool alive; @@ -580,7 +578,7 @@ static bool parse_subscribe(connsock_t *cs, proxy_instance_t *proxi) retry: parsed = true; if (!(buf = new_proxy_line(cs))) { - LOGNOTICE("Proxy %ld:%d %s failed to receive line in parse_subscribe", + LOGNOTICE("Proxy %d:%d %s failed to receive line in parse_subscribe", proxi->id, proxi->subid, proxi->url); goto out; } @@ -613,7 +611,7 @@ retry: buf = NULL; goto retry; } - LOGNOTICE("Proxy %ld:%d %s failed to parse subscribe response in parse_subscribe", + LOGNOTICE("Proxy %d:%d %s failed to parse subscribe response in parse_subscribe", proxi->id, proxi->subid, proxi->url); goto out; } @@ -649,9 +647,9 @@ retry: if (size < 3) { if (!proxi->subid) { LOGWARNING("Proxy %d %s Nonce2 length %d too small for fast miners", - proxi->low_id, proxi->url, size); + proxi->id, proxi->url, size); } else { - LOGNOTICE("Proxy %ld:%d Nonce2 length %d too small for fast miners", + LOGNOTICE("Proxy %d:%d Nonce2 length %d too small for fast miners", proxi->id, proxi->subid, size); } } @@ -665,7 +663,7 @@ retry: parent->recruit = 0; mutex_unlock(&parent->proxy_lock); - LOGNOTICE("Found notify for new proxy %ld:%d with enonce %s nonce2len %d", proxi->id, + LOGNOTICE("Found notify for new proxy %d:%d with enonce %s nonce2len %d", proxi->id, proxi->subid, proxi->enonce1, proxi->nonce2len); ret = true; @@ -698,7 +696,7 @@ retry: ret = send_json_msg(cs, req); json_decref(req); if (!ret) { - LOGNOTICE("Proxy %ld:%d %s failed to send message in subscribe_stratum", + LOGNOTICE("Proxy %d:%d %s failed to send message in subscribe_stratum", proxi->id, proxi->subid, proxi->url); goto out; } @@ -707,16 +705,16 @@ retry: goto out; if (proxi->no_params) { - LOGNOTICE("Proxy %ld:%d %s failed all subscription options in subscribe_stratum", + LOGNOTICE("Proxy %d:%d %s failed all subscription options in subscribe_stratum", proxi->id, proxi->subid, proxi->url); goto out; } - LOGINFO("Proxy %ld:%d %s failed connecting with parameters in subscribe_stratum, retrying without", + LOGINFO("Proxy %d:%d %s failed connecting with parameters in subscribe_stratum, retrying without", proxi->id, proxi->subid, proxi->url); proxi->no_params = true; ret = connect_proxy(cs); if (!ret) { - LOGNOTICE("Proxy %ld:%d %s failed to reconnect in subscribe_stratum", + LOGNOTICE("Proxy %d:%d %s failed to reconnect in subscribe_stratum", proxi->id, proxi->subid, proxi->url); goto out; } @@ -812,7 +810,7 @@ static bool parse_notify(ckpool_t *ckp, proxy_instance_t *proxi, json_t *val) goto out; } - LOGDEBUG("Received new notify from proxy %ld:%d", proxi->id, proxi->subid); + LOGDEBUG("Received new notify from proxy %d:%d", proxi->id, proxi->subid); ni = ckzalloc(sizeof(notify_instance_t)); ni->jobid = job_id; jobidbuf = json_string_value(job_id); @@ -908,7 +906,39 @@ static bool send_pong(proxy_instance_t *proxi, json_t *val) } static void prepare_proxy(proxy_instance_t *proxi); -static proxy_instance_t *create_subproxy(gdata_t *gdata, proxy_instance_t *proxi); + +/* Creates a duplicate instance or proxi to be used as a subproxy, ignoring + * fields we don't use in the subproxy. */ +static proxy_instance_t *create_subproxy(gdata_t *gdata, proxy_instance_t *proxi, const char *url) +{ + proxy_instance_t *subproxy; + + mutex_lock(&gdata->lock); + if (gdata->dead_proxies) { + /* Recycle an old proxy instance if one exists */ + subproxy = gdata->dead_proxies; + DL_DELETE(gdata->dead_proxies, subproxy); + subproxy->disabled = false; + } else { + subproxy = ckzalloc(sizeof(proxy_instance_t)); + mutex_init(&subproxy->share_lock); + } + mutex_unlock(&gdata->lock); + + subproxy->cs.ckp = subproxy->ckp = proxi->ckp; + + mutex_lock(&proxi->proxy_lock); + subproxy->subid = ++proxi->subproxy_count; + mutex_unlock(&proxi->proxy_lock); + + subproxy->id = proxi->id; + subproxy->url = strdup(url); + subproxy->auth = strdup(proxi->auth); + subproxy->pass = strdup(proxi->pass); + subproxy->parent = proxi; + subproxy->epfd = proxi->epfd; + return subproxy; +} static void add_subproxy(proxy_instance_t *proxi, proxy_instance_t *subproxy) { @@ -928,7 +958,7 @@ static proxy_instance_t *__subproxy_by_id(proxy_instance_t *proxy, const int sub /* Add to the dead list to be recycled if possible */ static void store_proxy(gdata_t *gdata, proxy_instance_t *proxy) { - LOGINFO("Recycling data from proxy %ld:%d", proxy->id, proxy->subid); + LOGINFO("Recycling data from proxy %d:%d", proxy->id, proxy->subid); mutex_lock(&gdata->lock); dealloc(proxy->url); @@ -938,11 +968,11 @@ static void store_proxy(gdata_t *gdata, proxy_instance_t *proxy) mutex_unlock(&gdata->lock); } -static void send_stratifier_deadproxy(ckpool_t *ckp, const int64_t id, const int subid) +static void send_stratifier_deadproxy(ckpool_t *ckp, const int id, const int subid) { char buf[256]; - sprintf(buf, "deadproxy=%ld:%d", id, subid); + sprintf(buf, "deadproxy=%d:%d", id, subid); send_proc(ckp->stratifier, buf); } @@ -972,13 +1002,12 @@ static void disable_subproxy(gdata_t *gdata, proxy_instance_t *proxi, proxy_inst store_proxy(gdata, subproxy); } -static bool parse_reconnect(proxy_instance_t *proxi, json_t *val) +static bool parse_reconnect(proxy_instance_t *proxy, json_t *val) { - proxy_instance_t *parent, *newproxi; bool sameurl = false, ret = false; - int64_t high_id, low_id, new_id; - ckpool_t *ckp = proxi->ckp; + ckpool_t *ckp = proxy->ckp; gdata_t *gdata = ckp->data; + proxy_instance_t *parent; const char *new_url; int new_port; char *url; @@ -997,10 +1026,10 @@ static bool parse_reconnect(proxy_instance_t *proxi, json_t *val) char *dot_pool, *dot_reconnect; int len; - dot_pool = strchr(proxi->url, '.'); + dot_pool = strchr(proxy->url, '.'); if (!dot_pool) { LOGWARNING("Denied stratum reconnect request from server without domain %s", - proxi->url); + proxy->url); goto out; } dot_reconnect = strchr(new_url, '.'); @@ -1012,61 +1041,37 @@ static bool parse_reconnect(proxy_instance_t *proxi, json_t *val) len = strlen(dot_reconnect); if (strncmp(dot_pool, dot_reconnect, len)) { LOGWARNING("Denied stratum reconnect request from %s to non-matching domain %s", - proxi->url, new_url); + proxy->url, new_url); goto out; } ASPRINTF(&url, "%s:%d", new_url, new_port); } else { - url = strdup(proxi->url); + url = strdup(proxy->url); sameurl = true; } LOGINFO("Processing reconnect request to %s", url); ret = true; - parent = proxi->parent; - /* If this is the same url we don't need to replace any existing - * parents, just drop the connection and allow a new one to be - * recruited. */ - if (sameurl) { - disable_subproxy(gdata, parent, proxi); + parent = proxy->parent; + disable_subproxy(gdata, parent, proxy); + if (parent != proxy) { + /* If this is a subproxy we only need to create a new one if + * the url has changed. Otherwise automated recruiting will + * take care of creating one if needed. */ + if (!sameurl) + create_subproxy(gdata, parent, url); goto out; } - /* If this isn't a parent proxy, recruit a new parent! */ - if (parent != proxi) { - proxi->reconnect = true; - /* Do we already know this proxy is redirecting? */ - if (parent->redirecting) - goto out; - proxi = parent; - proxi->redirecting = true; - } else - proxi->redirecting = false; + proxy->reconnect = true; + LOGWARNING("Proxy %d:%s reconnect issue to %s, dropping existing connection", + proxy->id, proxy->url, url); + if (!sameurl) { + char *oldurl = proxy->url; - mutex_lock(&gdata->lock); - high_id = proxi->id >> 32; /* Use the high bits for the reconnect id */ - high_id++; - high_id <<= 32; - low_id = proxi->id & 0x00000000FFFFFFFFll; /* Use the low bits for the master id */ - new_id = high_id | low_id; - newproxi = ckzalloc(sizeof(proxy_instance_t)); - newproxi->url = url; - newproxi->auth = strdup(proxi->auth); - newproxi->pass = strdup(proxi->pass); - newproxi->ckp = ckp; - newproxi->cs.ckp = ckp; - newproxi->low_id = low_id; - newproxi->id = new_id; - newproxi->subproxy_count = ++proxi->subproxy_count; - HASH_ADD_I64(gdata->proxies, id, newproxi); - if (!proxi->redirecting) { - proxi->disabled = true; - HASH_DEL(gdata->proxies, proxi); - proxi->reconnect = true; + proxy->url = url; + free(oldurl); } - mutex_unlock(&gdata->lock); - - prepare_proxy(newproxi); out: return ret; } @@ -1170,7 +1175,7 @@ static bool parse_method(ckpool_t *ckp, proxy_instance_t *proxi, const char *msg goto out; } - LOGDEBUG("Proxy %ld:%d received method %s", proxi->id, proxi->subid, buf); + LOGDEBUG("Proxy %d:%d received method %s", proxi->id, proxi->subid, buf); if (cmdmatch(buf, "mining.notify")) { ret = parse_notify(ckp, proxi, params); goto out; @@ -1221,7 +1226,7 @@ static bool auth_stratum(ckpool_t *ckp, connsock_t *cs, proxy_instance_t *proxi) ret = send_json_msg(cs, req); json_decref(req); if (!ret) { - LOGNOTICE("Proxy %ld:%d %s failed to send message in auth_stratum", + LOGNOTICE("Proxy %d:%d %s failed to send message in auth_stratum", proxi->id, proxi->subid, proxi->url); if (cs->fd > 0) { epoll_ctl(proxi->epfd, EPOLL_CTL_DEL, cs->fd, NULL); @@ -1236,7 +1241,7 @@ static bool auth_stratum(ckpool_t *ckp, connsock_t *cs, proxy_instance_t *proxi) free(buf); buf = next_proxy_line(cs, proxi); if (!buf) { - LOGNOTICE("Proxy %ld:%d %s failed to receive line in auth_stratum", + LOGNOTICE("Proxy %d:%d %s failed to receive line in auth_stratum", proxi->id, proxi->subid, proxi->url); ret = false; goto out; @@ -1247,27 +1252,27 @@ static bool auth_stratum(ckpool_t *ckp, connsock_t *cs, proxy_instance_t *proxi) val = json_msg_result(buf, &res_val, &err_val); if (!val) { LOGWARNING("Proxy %d:%d %s failed to get a json result in auth_stratum, got: %s", - proxi->low_id, proxi->subid, proxi->url, buf); + proxi->id, proxi->subid, proxi->url, buf); goto out; } if (err_val && !json_is_null(err_val)) { LOGWARNING("Proxy %d:%d %s failed to authorise in auth_stratum due to err_val, got: %s", - proxi->low_id, proxi->subid, proxi->url, buf); + proxi->id, proxi->subid, proxi->url, buf); goto out; } if (res_val) { ret = json_is_true(res_val); if (!ret) { LOGWARNING("Proxy %d:%d %s failed to authorise in auth_stratum, got: %s", - proxi->low_id, proxi->subid, proxi->url, buf); + proxi->id, proxi->subid, proxi->url, buf); goto out; } } else { /* No result and no error but successful val means auth success */ ret = true; } - LOGINFO("Proxy %ld:%d %s auth success in auth_stratum", proxi->id, proxi->subid, proxi->url); + LOGINFO("Proxy %d:%d %s auth success in auth_stratum", proxi->id, proxi->subid, proxi->url); out: if (val) json_decref(val); @@ -1330,25 +1335,24 @@ static proxy_instance_t *subproxy_by_id(proxy_instance_t *proxy, const int subid static void drop_proxy(gdata_t *gdata, const char *buf) { proxy_instance_t *proxy, *subproxy; - int64_t id = 0; - int subid = 0; + int id = -1, subid = -1; - sscanf(buf, "dropproxy=%ld:%d", &id, &subid); + sscanf(buf, "dropproxy=%d:%d", &id, &subid); if (unlikely(!subid)) { - LOGWARNING("Generator asked to drop parent proxy %ld", id); + LOGWARNING("Generator asked to drop parent proxy %d", id); return; } proxy = proxy_by_id(gdata, id); if (unlikely(!proxy)) { - LOGINFO("Generator asked to drop subproxy from non-existent parent %ld", id); + LOGINFO("Generator asked to drop subproxy from non-existent parent %d", id); return; } subproxy = subproxy_by_id(proxy, subid); if (!subproxy) { - LOGINFO("Generator asked to drop non-existent subproxy %ld:%d", id, subid); + LOGINFO("Generator asked to drop non-existent subproxy %d:%d", id, subid); return; } - LOGNOTICE("Generator asked to drop proxy %ld:%d", id, subid); + LOGNOTICE("Generator asked to drop proxy %d:%d", id, subid); disable_subproxy(gdata, proxy, subproxy); } @@ -1364,11 +1368,11 @@ static void submit_share(gdata_t *gdata, json_t *val) { proxy_instance_t *proxy, *proxi; ckpool_t *ckp = gdata->ckp; - int64_t client_id, id; bool success = false; stratum_msg_t *msg; share_msg_t *share; - int subid; + int64_t client_id; + int id, subid; /* Get the client id so we can tell the stratifier to drop it if the * proxy it's bound to is not functional */ @@ -1376,7 +1380,7 @@ static void submit_share(gdata_t *gdata, json_t *val) LOGWARNING("Got no client_id in share"); goto out; } - if (unlikely(!json_get_int64(&id, val, "proxy"))) { + if (unlikely(!json_get_int(&id, val, "proxy"))) { LOGWARNING("Got no proxy in share"); goto out; } @@ -1386,20 +1390,20 @@ static void submit_share(gdata_t *gdata, json_t *val) } proxy = proxy_by_id(gdata, id); if (unlikely(!proxy)) { - LOGNOTICE("Client %"PRId64" sending shares to non existent proxy %ld, dropping", + LOGNOTICE("Client %"PRId64" sending shares to non existent proxy %d, dropping", client_id, id); stratifier_reconnect_client(ckp, client_id); goto out; } proxi = subproxy_by_id(proxy, subid); if (unlikely(!proxi)) { - LOGNOTICE("Client %"PRId64" sending shares to non existent subproxy %ld:%d, dropping", + LOGNOTICE("Client %"PRId64" sending shares to non existent subproxy %d:%d, dropping", client_id, id, subid); stratifier_reconnect_client(ckp, client_id); goto out; } if (!proxi->alive) { - LOGNOTICE("Client %"PRId64" sending shares to dead subproxy %ld:%d, dropping", + LOGNOTICE("Client %"PRId64" sending shares to dead subproxy %d:%d, dropping", client_id, id, subid); stratifier_reconnect_client(ckp, client_id); goto out; @@ -1470,11 +1474,11 @@ static bool parse_share(proxy_instance_t *proxi, const char *buf) * so long as we recognised it as a share response */ ret = true; if (!share) { - LOGINFO("Proxy %ld:%d failed to find matching share to result: %s", + LOGINFO("Proxy %d:%d failed to find matching share to result: %s", proxi->id, proxi->subid, buf); goto out; } - LOGINFO("Proxy %ld:%d share result %s from client %d", proxi->id, proxi->subid, + LOGINFO("Proxy %d:%d share result %s from client %d", proxi->id, proxi->subid, buf, share->client_id); free(share); out: @@ -1495,18 +1499,18 @@ static void *proxy_send(void *arg) rename_proc("proxysend"); while (42) { - int64_t proxyid = 0, client_id = 0, id; proxy_instance_t *subproxy; + int proxyid = 0, subid = 0; + int64_t client_id = 0, id; notify_instance_t *ni; json_t *jobid = NULL; bool ret = true; - int subid = 0; json_t *val; tv_t now; ts_t abs; if (unlikely(proxy->reconnect)) { - LOGINFO("Shutting down proxy_send thread for proxy %ld to reconnect", + LOGINFO("Shutting down proxy_send thread for proxy %d to reconnect", proxy->id); break; } @@ -1539,7 +1543,7 @@ static void *proxy_send(void *arg) LOGWARNING("Failed to find jobid in proxy_send msg"); continue; } - if (unlikely(!json_get_int64(&proxyid, msg->json_msg, "proxy"))) { + if (unlikely(!json_get_int(&proxyid, msg->json_msg, "proxy"))) { LOGWARNING("Failed to find proxy in proxy_send msg"); continue; } @@ -1548,7 +1552,7 @@ static void *proxy_send(void *arg) continue; } if (unlikely(proxyid != proxy->id)) { - LOGWARNING("Proxysend for proxy %ld got message for proxy %ld!", + LOGWARNING("Proxysend for proxy %d got message for proxy %d!", proxy->id, proxyid); } @@ -1572,16 +1576,16 @@ static void *proxy_send(void *arg) json_decref(val); } else if (!jobid) { stratifier_reconnect_client(ckp, client_id); - LOGNOTICE("Proxy %ld:%s failed to find matching jobid for %sknown subproxy in proxysend", + LOGNOTICE("Proxy %d:%s failed to find matching jobid for %sknown subproxy in proxysend", proxy->id, proxy->url, subproxy ? "" : "un"); } else { stratifier_reconnect_client(ckp, client_id); - LOGNOTICE("Failed to find subproxy %ld:%d to send message to", + LOGNOTICE("Failed to find subproxy %d:%d to send message to", proxy->id, subid); } if (!ret && subproxy) { - LOGNOTICE("Proxy %ld:%d %s failed to send msg in proxy_send, dropping to reconnect", - id, subid, proxy->url); + LOGNOTICE("Proxy %d:%d %s failed to send msg in proxy_send, dropping to reconnect", + proxy->id, subid, proxy->url); disable_subproxy(gdata, proxy, subproxy); } } @@ -1684,39 +1688,6 @@ out: return ret; } -/* Creates a duplicate instance or proxi to be used as a subproxy, ignoring - * fields we don't use in the subproxy. */ -static proxy_instance_t *create_subproxy(gdata_t *gdata, proxy_instance_t *proxi) -{ - proxy_instance_t *subproxy; - - mutex_lock(&gdata->lock); - if (gdata->dead_proxies) { - /* Recycle an old proxy instance if one exists */ - subproxy = gdata->dead_proxies; - DL_DELETE(gdata->dead_proxies, subproxy); - subproxy->disabled = false; - } else { - subproxy = ckzalloc(sizeof(proxy_instance_t)); - mutex_init(&subproxy->share_lock); - } - mutex_unlock(&gdata->lock); - - subproxy->cs.ckp = subproxy->ckp = proxi->ckp; - - mutex_lock(&proxi->proxy_lock); - subproxy->subid = ++proxi->subproxy_count; - mutex_unlock(&proxi->proxy_lock); - - subproxy->id = proxi->id; - subproxy->url = strdup(proxi->url); - subproxy->auth = strdup(proxi->auth); - subproxy->pass = strdup(proxi->pass); - subproxy->parent = proxi; - subproxy->epfd = proxi->epfd; - return subproxy; -} - static void *proxy_recruit(void *arg) { proxy_instance_t *proxy, *parent = (proxy_instance_t *)arg; @@ -1728,7 +1699,7 @@ static void *proxy_recruit(void *arg) retry: recruit = false; - proxy = create_subproxy(gdata, parent); + proxy = create_subproxy(gdata, parent, parent->url); alive = proxy_alive(ckp, proxy, &proxy->cs, false, parent->epfd); if (!alive) { LOGNOTICE("Subproxy failed proxy_alive testing"); @@ -1818,8 +1789,7 @@ static void *passthrough_recv(void *arg) if (proxy_alive(ckp, proxi, cs, false, epfd)) { reconnect_generator(ckp); - LOGWARNING("Proxy %d:%s connection established", - proxi->low_id, proxi->url); + LOGWARNING("Proxy %d:%s connection established", proxi->id, proxi->url); } alive = proxi->alive; @@ -1841,7 +1811,7 @@ static void *passthrough_recv(void *arg) if (likely(ret > 0)) ret = read_socket_line(cs, 60); if (ret < 1) { - LOGWARNING("Proxy %ld:%s failed to read_socket_line in proxy_recv, attempting reconnect", + LOGWARNING("Proxy %d:%s failed to read_socket_line in proxy_recv, attempting reconnect", proxi->id, proxi->url); alive = proxi->alive = false; reconnect_generator(ckp); @@ -1905,8 +1875,7 @@ static void *proxy_recv(void *arg) } if (proxy_alive(ckp, proxi, cs, false, epfd)) { - LOGWARNING("Proxy %d:%s connection established", - proxi->low_id, proxi->url); + LOGWARNING("Proxy %d:%s connection established", proxi->id, proxi->url); } alive = proxi->alive; @@ -1923,7 +1892,7 @@ static void *proxy_recv(void *arg) reconnect_proxy(proxi); if (alive) { LOGWARNING("Proxy %d:%s failed, attempting reconnect", - proxi->low_id, proxi->url); + proxi->id, proxi->url); alive = false; } sleep(5); @@ -1934,7 +1903,7 @@ static void *proxy_recv(void *arg) * to prevent switching to unstable pools. */ if (!alive && (!current_proxy(gdata) || time(NULL) - proxi->reconnect_time > 30)) { reconnect_generator(ckp); - LOGWARNING("Proxy %d:%s recovered", proxi->low_id, proxi->url); + LOGWARNING("Proxy %d:%s recovered", proxi->id, proxi->url); proxi->reconnect_time = 0; alive = true; } @@ -1974,33 +1943,23 @@ static void *proxy_recv(void *arg) ret = read_socket_line(cs, 5); } if (ret < 1) { - LOGNOTICE("Proxy %ld:%d %s failed to epoll/read_socket_line in proxy_recv", + LOGNOTICE("Proxy %d:%d %s failed to epoll/read_socket_line in proxy_recv", proxi->id, subproxy->subid, subproxy->url); disable_subproxy(gdata, proxi, subproxy); continue; } do { - if (parse_method(ckp, subproxy, cs->buf)) { - if (subproxy->reconnect) { - /* Call this proxy dead to allow us to fail - * over to a backup pool until the reconnect - * pool is up */ - disable_subproxy(gdata, proxi, subproxy); - if (parent_proxy(subproxy)) { - LOGWARNING("Proxy %d:%s reconnect issue, dropping existing connection", - subproxy->low_id, subproxy->url); - goto out; - } - } + /* subproxy may have been recycled here if it is not a + * parent and reconnect was issued */ + if (parse_method(ckp, subproxy, cs->buf)) continue; - } /* If it's not a method it should be a share result */ if (!parse_share(subproxy, cs->buf)) - LOGWARNING("Unhandled stratum message: %s", cs->buf); + LOGNOTICE("Proxy %d:%d unhandled stratum message: %s", + subproxy->id, subproxy->subid, cs->buf); } while ((ret = read_socket_line(cs, 0)) > 0); } -out: - mutex_lock(&proxi->proxy_lock); + HASH_ITER(sh, proxi->subproxies, subproxy, tmp) { subproxy->disabled = true; send_stratifier_deadproxy(ckp, subproxy->id, subproxy->subid); @@ -2039,9 +1998,7 @@ static proxy_instance_t *wait_best_proxy(ckpool_t *ckp, gdata_t *gdata) if (proxi->disabled) continue; if (proxi->alive || subproxies_alive(proxi)) { - if ((!ret) || - (proxi->low_id < ret->low_id) || - (proxi->low_id == ret->low_id && proxi->id > ret->id)) + if (!ret || proxi->id < ret->id) ret = proxi; } } @@ -2066,8 +2023,8 @@ static void send_list(gdata_t *gdata, const int sockd) mutex_lock(&gdata->lock); HASH_ITER(hh, gdata->proxies, proxy, tmp) { - JSON_CPACK(val, "{sI,si,ss,ss,sf,sb,sb,sb,si}", - "id", proxy->id, "low_id", proxy->low_id, + JSON_CPACK(val, "{si,ss,ss,sf,sb,sb,sb,si}", + "id", proxy->id, "auth", proxy->auth, "pass", proxy->pass, "diff", proxy->diff, "notified", proxy->notified, "disabled", proxy->disabled, "alive", proxy->alive, @@ -2191,10 +2148,10 @@ reconnect: if (proxi != cproxy) { proxi = cproxy; if (!ckp->passthrough) { - LOGWARNING("Successfully connected to proxy %d %s:%s as proxy", - proxi->low_id, proxi->url, proxi->cs.port); + LOGWARNING("Successfully connected to proxy %d %s as proxy", + proxi->id, proxi->url); dealloc(buf); - ASPRINTF(&buf, "proxy=%ld", proxi->id); + ASPRINTF(&buf, "proxy=%d", proxi->id); send_proc(ckp->stratifier, buf); } } @@ -2296,7 +2253,7 @@ static proxy_instance_t *__add_proxy(ckpool_t *ckp, gdata_t *gdata, const int id proxy_instance_t *proxy; proxy = ckzalloc(sizeof(proxy_instance_t)); - proxy->id = proxy->low_id = id; + proxy->id = id; proxy->url = strdup(ckp->proxyurl[id]); proxy->auth = strdup(ckp->proxyauth[id]); proxy->pass = strdup(ckp->proxypass[id]); @@ -2304,7 +2261,7 @@ static proxy_instance_t *__add_proxy(ckpool_t *ckp, gdata_t *gdata, const int id proxy->cs.ckp = ckp; mutex_init(&proxy->notify_lock); mutex_init(&proxy->share_lock); - HASH_ADD_I64(gdata->proxies, id, proxy); + HASH_ADD_INT(gdata->proxies, id, proxy); return proxy; } From d590869963fa7a30b57aa7b55a7a40e495215976 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Tue, 17 Mar 2015 10:33:39 +1100 Subject: [PATCH 336/544] Update stratifier to use integers for proxy IDs again --- src/generator.c | 4 +- src/stratifier.c | 104 +++++++++++++++++++++-------------------------- 2 files changed, 49 insertions(+), 59 deletions(-) diff --git a/src/generator.c b/src/generator.c index e5c3cf4a..c6a64703 100644 --- a/src/generator.c +++ b/src/generator.c @@ -1290,12 +1290,12 @@ out: return ret; } -static proxy_instance_t *proxy_by_id(gdata_t *gdata, const int64_t id) +static proxy_instance_t *proxy_by_id(gdata_t *gdata, const int id) { proxy_instance_t *proxi; mutex_lock(&gdata->lock); - HASH_FIND_I64(gdata->proxies, &id, proxi); + HASH_FIND_INT(gdata->proxies, &id, proxi); mutex_unlock(&gdata->lock); return proxi; diff --git a/src/stratifier.c b/src/stratifier.c index 95c6604c..9f219b63 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -278,7 +278,7 @@ struct stratum_instance { sdata_t *sdata; /* Which sdata this client is bound to */ proxy_t *proxy; /* Proxy this is bound to in proxy mode */ - int64_t proxyid; /* Which proxy id */ + int proxyid; /* Which proxy id */ int subproxyid; /* Which subproxy */ }; @@ -295,8 +295,7 @@ struct proxy_base { UT_hash_handle sh; /* For subproxy hashlist */ proxy_t *next; /* For retired subproxies */ proxy_t *prev; - int64_t id; - int low_id; + int id; int subid; double diff; @@ -1079,20 +1078,19 @@ static sdata_t *duplicate_sdata(const sdata_t *sdata) return dsdata; } -static proxy_t *__generate_proxy(sdata_t *sdata, const int64_t id) +static proxy_t *__generate_proxy(sdata_t *sdata, const int id) { proxy_t *proxy = ckzalloc(sizeof(proxy_t)); proxy->parent = proxy; proxy->id = id; - proxy->low_id = id & 0xFFFFFFFF; proxy->sdata = duplicate_sdata(sdata); proxy->sdata->subproxy = proxy; proxy->sdata->verbose = true; /* subid == 0 on parent proxy */ HASH_ADD(sh, proxy->subproxies, subid, sizeof(int), proxy); proxy->subproxy_count++; - HASH_ADD_I64(sdata->proxies, id, proxy); + HASH_ADD_INT(sdata->proxies, id, proxy); sdata->proxy_count++; return proxy; } @@ -1103,7 +1101,6 @@ static proxy_t *__generate_subproxy(sdata_t *sdata, proxy_t *proxy, const int su subproxy->parent = proxy; subproxy->id = proxy->id; - subproxy->low_id = proxy->low_id; subproxy->subid = subid; HASH_ADD(sh, proxy->subproxies, subid, sizeof(int), subproxy); proxy->subproxy_count++; @@ -1112,22 +1109,22 @@ static proxy_t *__generate_subproxy(sdata_t *sdata, proxy_t *proxy, const int su return subproxy; } -static proxy_t *__existing_proxy(const sdata_t *sdata, const int64_t id) +static proxy_t *__existing_proxy(const sdata_t *sdata, const int id) { proxy_t *proxy; - HASH_FIND_I64(sdata->proxies, &id, proxy); + HASH_FIND_INT(sdata->proxies, &id, proxy); return proxy; } /* Find proxy by id number, generate one if none exist yet by that id */ -static proxy_t *__proxy_by_id(sdata_t *sdata, const int64_t id) +static proxy_t *__proxy_by_id(sdata_t *sdata, const int id) { proxy_t *proxy = __existing_proxy(sdata, id); if (unlikely(!proxy)) { proxy = __generate_proxy(sdata, id); - LOGNOTICE("Stratifier added new proxy %ld", id); + LOGNOTICE("Stratifier added new proxy %d", id); } return proxy; @@ -1147,12 +1144,12 @@ static proxy_t *__subproxy_by_id(sdata_t *sdata, proxy_t *proxy, const int subid if (!subproxy) { subproxy = __generate_subproxy(sdata, proxy, subid); - LOGINFO("Stratifier added new subproxy %ld:%d", proxy->id, subid); + LOGINFO("Stratifier added new subproxy %d:%d", proxy->id, subid); } return subproxy; } -static proxy_t *subproxy_by_id(sdata_t *sdata, const int64_t id, const int subid) +static proxy_t *subproxy_by_id(sdata_t *sdata, const int id, const int subid) { proxy_t *proxy, *subproxy; @@ -1164,7 +1161,7 @@ static proxy_t *subproxy_by_id(sdata_t *sdata, const int64_t id, const int subid return subproxy; } -static proxy_t *existing_subproxy(sdata_t *sdata, const int64_t id, const int subid) +static proxy_t *existing_subproxy(sdata_t *sdata, const int id, const int subid) { proxy_t *proxy, *subproxy = NULL; @@ -1241,7 +1238,7 @@ static void reconnect_clients(sdata_t *sdata) ck_runlock(&sdata->instance_lock); if (reconnects) { - LOGNOTICE("%d clients flagged for reconnect to proxy %ld", reconnects, + LOGNOTICE("%d clients flagged for reconnect to proxy %d", reconnects, proxy->id); } if (headroom < 0) @@ -1261,7 +1258,7 @@ static proxy_t *current_proxy(sdata_t *sdata) } #endif -static void dead_proxyid(sdata_t *sdata, const int64_t id, const int subid) +static void dead_proxyid(sdata_t *sdata, const int id, const int subid) { stratum_instance_t *client, *tmp; int reconnects = 0, hard = 0; @@ -1271,7 +1268,7 @@ static void dead_proxyid(sdata_t *sdata, const int64_t id, const int subid) proxy = existing_subproxy(sdata, id, subid); if (proxy) proxy->dead = true; - LOGINFO("Stratifier dropping clients from proxy %ld:%d", id, subid); + LOGINFO("Stratifier dropping clients from proxy %d:%d", id, subid); headroom = current_headroom(sdata, &proxy); ck_rlock(&sdata->instance_lock); @@ -1288,7 +1285,7 @@ static void dead_proxyid(sdata_t *sdata, const int64_t id, const int subid) ck_runlock(&sdata->instance_lock); if (reconnects) { - LOGNOTICE("%d clients flagged to reconnect from dead proxy %ld:%d", reconnects, + LOGNOTICE("%d clients flagged to reconnect from dead proxy %d:%d", reconnects, id, subid); } if (headroom < 0) @@ -1299,9 +1296,8 @@ static void update_subscribe(ckpool_t *ckp, const char *cmd) { sdata_t *sdata = ckp->data, *dsdata; proxy_t *proxy, *old = NULL; + int id = 0, subid = 0; const char *buf; - int64_t id = 0; - int subid = 0; json_t *val; if (unlikely(strlen(cmd) < 11)) { @@ -1315,7 +1311,7 @@ static void update_subscribe(ckpool_t *ckp, const char *cmd) LOGWARNING("Failed to json decode subscribe response in update_subscribe %s", buf); return; } - if (unlikely(!json_get_int64(&id, val, "proxy"))) { + if (unlikely(!json_get_int(&id, val, "proxy"))) { LOGWARNING("Failed to json decode proxy value in update_subscribe %s", buf); return; } @@ -1325,9 +1321,9 @@ static void update_subscribe(ckpool_t *ckp, const char *cmd) } if (!subid) - LOGNOTICE("Got updated subscribe for proxy %ld", id); + LOGNOTICE("Got updated subscribe for proxy %d", id); else - LOGINFO("Got updated subscribe for proxy %ld:%d", id, subid); + LOGINFO("Got updated subscribe for proxy %d:%d", id, subid); /* Is this a replacement for an existing proxy id? */ old = existing_subproxy(sdata, id, subid); @@ -1363,15 +1359,15 @@ static void update_subscribe(ckpool_t *ckp, const char *cmd) /* Is this a replacement proxy for the current one */ mutex_lock(&sdata->proxy_lock); - if (sdata->proxy && sdata->proxy->low_id == proxy->low_id && !proxy->subid) + if (sdata->proxy && sdata->proxy->id == proxy->id && !proxy->subid) sdata->proxy = proxy; mutex_unlock(&sdata->proxy_lock); if (subid) { - LOGINFO("Upstream pool %ld:%d extranonce2 length %d, max proxy clients %"PRId64, + LOGINFO("Upstream pool %d:%d extranonce2 length %d, max proxy clients %"PRId64, id, subid, proxy->nonce2len, proxy->max_clients); } else { - LOGNOTICE("Upstream pool %ld extranonce2 length %d, max proxy clients %"PRId64, + LOGNOTICE("Upstream pool %d extranonce2 length %d, max proxy clients %"PRId64, id, proxy->nonce2len, proxy->max_clients); } json_decref(val); @@ -1382,11 +1378,10 @@ static void update_notify(ckpool_t *ckp, const char *cmd) sdata_t *sdata = ckp->data, *dsdata; bool new_block = false, clean; proxy_t *proxy, *current; + int i, id = 0, subid = 0; int64_t current_id; - int i, subid = 0; char header[228]; const char *buf; - int64_t id = 0; workbase_t *wb; json_t *val; @@ -1402,17 +1397,17 @@ static void update_notify(ckpool_t *ckp, const char *cmd) LOGWARNING("Failed to json decode in update_notify"); return; } - json_get_int64(&id, val, "proxy"); + json_get_int(&id, val, "proxy"); json_get_int(&subid, val, "subproxy"); proxy = existing_subproxy(sdata, id, subid); if (unlikely(!proxy || !proxy->subscribed)) { - LOGINFO("No valid proxy %ld:%d subscription to update notify yet", id, subid); + LOGINFO("No valid proxy %d:%d subscription to update notify yet", id, subid); goto out; } if (!subid) - LOGNOTICE("Got updated notify for proxy %ld", id); + LOGNOTICE("Got updated notify for proxy %d", id); else - LOGINFO("Got updated notify for proxy %ld:%d", id, subid); + LOGINFO("Got updated notify for proxy %d:%d", id, subid); wb = ckzalloc(sizeof(workbase_t)); wb->ckp = ckp; @@ -1466,9 +1461,9 @@ static void update_notify(ckpool_t *ckp, const char *cmd) add_base(ckp, dsdata, wb, &new_block); if (new_block) { if (subid) - LOGINFO("Block hash on proxy %ld:%d changed to %s", id, subid, dsdata->lastswaphash); + LOGINFO("Block hash on proxy %d:%d changed to %s", id, subid, dsdata->lastswaphash); else - LOGNOTICE("Block hash on proxy %ld changed to %s", id, dsdata->lastswaphash); + LOGNOTICE("Block hash on proxy %d changed to %s", id, dsdata->lastswaphash); } mutex_lock(&sdata->proxy_lock); @@ -1481,7 +1476,7 @@ static void update_notify(ckpool_t *ckp, const char *cmd) if (!proxy->subid && proxy->id == current_id) reconnect_clients(sdata); clean |= new_block; - LOGINFO("Proxy %ld:%d broadcast updated stratum notify with%s clean", id, + LOGINFO("Proxy %d:%d broadcast updated stratum notify with%s clean", id, subid, clean ? "" : "out"); stratum_broadcast_update(dsdata, wb, clean); out: @@ -1495,10 +1490,9 @@ static void update_diff(ckpool_t *ckp, const char *cmd) sdata_t *sdata = ckp->data, *dsdata; stratum_instance_t *client, *tmp; double old_diff, diff; + int id = 0, subid = 0; const char *buf; - int64_t id = 0; proxy_t *proxy; - int subid = 0; json_t *val; if (unlikely(strlen(cmd) < 6)) { @@ -1513,15 +1507,15 @@ static void update_diff(ckpool_t *ckp, const char *cmd) LOGWARNING("Failed to json decode in update_diff"); return; } - json_get_int64(&id, val, "proxy"); + json_get_int(&id, val, "proxy"); json_get_int(&subid, val, "subproxy"); json_dblcpy(&diff, val, "diff"); json_decref(val); - LOGINFO("Got updated diff for proxy %ld:%d", id, subid); + LOGINFO("Got updated diff for proxy %d:%d", id, subid); proxy = existing_subproxy(sdata, id, subid); if (!proxy) { - LOGINFO("No existing subproxy %ld:%d to update diff", id, subid); + LOGINFO("No existing subproxy %d:%d to update diff", id, subid); return; } @@ -1603,12 +1597,12 @@ static void reap_proxies(ckpool_t *ckp, sdata_t *sdata) if (!subproxy->dead) continue; if (unlikely(!subproxy->subid)) { - LOGWARNING("Unexepectedly found proxy %ld:%d as subproxy of %ld:%d", + LOGWARNING("Unexepectedly found proxy %d:%d as subproxy of %d:%d", subproxy->id, subproxy->subid, proxy->id, proxy->subid); continue; } if (unlikely(subproxy == sdata->proxy)) { - LOGWARNING("Unexepectedly found proxy %ld:%d as current", + LOGWARNING("Unexepectedly found proxy %d:%d as current", subproxy->id, subproxy->subid); continue; } @@ -2239,25 +2233,24 @@ static void reconnect_client(sdata_t *sdata, stratum_instance_t *client) * this proxy we are only given this message if all clients must move. */ static void set_proxy(sdata_t *sdata, const char *buf) { - int64_t id = 0; + int id = 0; proxy_t *proxy; - sscanf(buf, "proxy=%ld", &id); + sscanf(buf, "proxy=%d", &id); mutex_lock(&sdata->proxy_lock); proxy = __proxy_by_id(sdata, id); sdata->proxy = proxy; mutex_unlock(&sdata->proxy_lock); - LOGNOTICE("Stratifier setting active proxy to %ld", id); + LOGNOTICE("Stratifier setting active proxy to %d", id); } static void dead_proxy(sdata_t *sdata, const char *buf) { - int64_t id = 0; - int subid = 0; + int id = 0, subid = 0; - sscanf(buf, "deadproxy=%ld:%d", &id, &subid); + sscanf(buf, "deadproxy=%d:%d", &id, &subid); dead_proxyid(sdata, id, subid); } @@ -2522,8 +2515,7 @@ static void stratum_send_message(sdata_t *sdata, const stratum_instance_t *clien static sdata_t *select_sdata(const ckpool_t *ckp, sdata_t *ckp_sdata) { proxy_t *current, *proxy, *subproxy, *best = NULL, *tmp, *tmpsub; - int best_subid = 0, best_lowid; - int64_t best_id = 0; + int best_subid = 0, best_id; if (!ckp->proxy || ckp->passthrough) return ckp_sdata; @@ -2532,7 +2524,7 @@ static sdata_t *select_sdata(const ckpool_t *ckp, sdata_t *ckp_sdata) LOGWARNING("No proxy available yet to generate subscribes"); return NULL; } - best_lowid = ckp_sdata->proxy_count; + best_id = ckp_sdata->proxy_count; mutex_lock(&ckp_sdata->proxy_lock); HASH_ITER(hh, ckp_sdata->proxies, proxy, tmp) { @@ -2555,9 +2547,8 @@ static sdata_t *select_sdata(const ckpool_t *ckp, sdata_t *ckp_sdata) max_headroom = subproxy_headroom; } } - if (best && (best->low_id < best_lowid || (best->low_id == best_lowid && best->id > best_id))) { + if (best && best->id < best_id) { best_id = best->id; - best_lowid = best->low_id; best_subid = best->subid; if (proxy == current) break; @@ -2598,7 +2589,7 @@ static json_t *parse_subscribe(stratum_instance_t *client, const int64_t client_ return json_string("Initialising"); } if (ckp->proxy) { - LOGINFO("Current %ld, selecting proxy %ld:%d for client %"PRId64, ckp_sdata->proxy->id, + LOGINFO("Current %d, selecting proxy %d:%d for client %"PRId64, ckp_sdata->proxy->id, sdata->subproxy->id, sdata->subproxy->subid, client->id); } client->sdata = sdata; @@ -3187,7 +3178,7 @@ static json_t *parse_authorise(stratum_instance_t *client, const json_t *params_ client->authorised = ret; user->authorised = ret; if (ckp->proxy) { - LOGNOTICE("Authorised client %"PRId64" to proxy %ld:%d, worker %s as user %s", + LOGNOTICE("Authorised client %"PRId64" to proxy %d:%d, worker %s as user %s", client->id, client->proxyid, client->subproxyid, buf, user->username); } else { LOGNOTICE("Authorised client %"PRId64" worker %s as user %s", @@ -4902,9 +4893,8 @@ static void *statsupdate(void *arg) mutex_lock(&sdata->proxy_lock); HASH_ITER(hh, sdata->proxies, proxy, proxytmp) { - JSON_CPACK(val, "{sI,si,si,sI,sb}", + JSON_CPACK(val, "{sI,si,sI,sb}", "id", proxy->id, - "priority", proxy->low_id, "subproxies", proxy->subproxy_count, "clients", proxy->combined_clients, "alive", !proxy->dead); From 1d87d5a493056edc6d1638c7cdb0033d1303d745 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Tue, 17 Mar 2015 10:55:18 +1100 Subject: [PATCH 337/544] Send connector reject message only if we can't find an active pool after 5 seconds --- src/generator.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/generator.c b/src/generator.c index c6a64703..539a642e 100644 --- a/src/generator.c +++ b/src/generator.c @@ -1988,6 +1988,7 @@ static void prepare_proxy(proxy_instance_t *proxi) static proxy_instance_t *wait_best_proxy(ckpool_t *ckp, gdata_t *gdata) { proxy_instance_t *ret = NULL, *proxi, *tmp; + int retries = 0; while (42) { if (!ping_main(ckp)) @@ -2007,7 +2008,10 @@ static proxy_instance_t *wait_best_proxy(ckpool_t *ckp, gdata_t *gdata) if (ret) break; - send_proc(ckp->connector, "reject"); + /* Send reject message if we are unable to find an active + * proxy for more than 5 seconds */ + if (!((++retries) % 5)) + send_proc(ckp->connector, "reject"); sleep(1); } send_proc(ckp->connector, ret ? "accept" : "reject"); From e083b2ef3eb658207ffc3606dd841ebd50ad3490 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Tue, 17 Mar 2015 11:35:23 +1100 Subject: [PATCH 338/544] Provide an API based mechanism for deleting proxies on the fly --- src/generator.c | 66 ++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 63 insertions(+), 3 deletions(-) diff --git a/src/generator.c b/src/generator.c index 539a642e..9ed5b9f8 100644 --- a/src/generator.c +++ b/src/generator.c @@ -961,6 +961,7 @@ static void store_proxy(gdata_t *gdata, proxy_instance_t *proxy) LOGINFO("Recycling data from proxy %d:%d", proxy->id, proxy->subid); mutex_lock(&gdata->lock); + dealloc(proxy->enonce1); dealloc(proxy->url); dealloc(proxy->auth); dealloc(proxy->pass); @@ -1498,6 +1499,8 @@ static void *proxy_send(void *arg) rename_proc("proxysend"); + pthread_detach(pthread_self()); + while (42) { proxy_instance_t *subproxy; int proxyid = 0, subid = 0; @@ -2127,12 +2130,69 @@ static void parse_addproxy(ckpool_t *ckp, gdata_t *gdata, const int sockd, const mutex_unlock(&gdata->lock); prepare_proxy(proxy); - JSON_CPACK(val, "{sI,ss,ss,ss}", + JSON_CPACK(val, "{si,ss,ss,ss}", "id", proxy->id, "url", url, "auth", auth, "pass", pass); out: send_api_response(val, sockd); } +static void delete_proxy(gdata_t *gdata, proxy_instance_t *proxy) +{ + proxy_instance_t *subproxy; + + /* Remove the proxy from the master list first */ + mutex_lock(&gdata->lock); + HASH_DEL(gdata->proxies, proxy); + /* Disable all its threads */ + pthread_cancel(proxy->pth_psend); + pthread_cancel(proxy->pth_precv); + Close(proxy->cs.fd); + mutex_unlock(&gdata->lock); + + /* Recycle all its subproxies */ + do { + mutex_lock(&proxy->proxy_lock); + subproxy = proxy->subproxies; + if (subproxy) + HASH_DELETE(sh, proxy->subproxies, subproxy); + mutex_unlock(&proxy->proxy_lock); + + if (subproxy && proxy != subproxy) + store_proxy(gdata, subproxy); + } while (subproxy); + + /* Recycle the proxy itself */ + store_proxy(gdata, proxy); +} + +static void parse_delproxy(ckpool_t *ckp, gdata_t *gdata, const int sockd, const char *buf) +{ + proxy_instance_t *proxy; + + json_error_t err_val; + json_t *val = NULL; + int id = -1; + + val = json_loads(buf, 0, &err_val); + if (unlikely(!val)) { + val = json_encode_errormsg(&err_val); + goto out; + } + json_get_int(&id, val, "id"); + proxy = proxy_by_id(gdata, id); + if (!proxy) { + val = json_errormsg("Proxy id %d not found", id); + goto out; + } + JSON_CPACK(val, "{si,ss,ss,ss}", "id", proxy->id, "url", proxy->url, + "auth", proxy->auth, "pass", proxy->pass); + + delete_proxy(gdata, proxy); + reconnect_generator(ckp); +out: + send_api_response(val, sockd); +} + static int proxy_loop(proc_instance_t *pi) { proxy_instance_t *proxi = NULL, *cproxy; @@ -2201,6 +2261,8 @@ retry: send_sublist(gdata, sockd, buf + 8); } else if (cmdmatch(buf, "addproxy")) { parse_addproxy(ckp, gdata, sockd, buf + 9); + } else if (cmdmatch(buf, "delproxy")) { + parse_delproxy(ckp, gdata, sockd, buf + 9); } else if (cmdmatch(buf, "shutdown")) { ret = 0; goto out; @@ -2300,8 +2362,6 @@ static int proxy_mode(ckpool_t *ckp, proc_instance_t *pi) free(proxy->enonce1bin); pthread_cancel(proxy->pth_psend); pthread_cancel(proxy->pth_precv); - join_pthread(proxy->pth_psend); - join_pthread(proxy->pth_precv); dealloc(proxy->url); dealloc(proxy->auth); dealloc(proxy->pass); From 41202d22eb3191a170f3094579f44928b3f0f4db Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Tue, 17 Mar 2015 12:06:09 +1100 Subject: [PATCH 339/544] Add API commands to allow us to enable/disable proxies on the fly --- src/generator.c | 54 ++++++++++++++++++++++++++++++++++++++----------- 1 file changed, 42 insertions(+), 12 deletions(-) diff --git a/src/generator.c b/src/generator.c index 9ed5b9f8..8cb5a5d4 100644 --- a/src/generator.c +++ b/src/generator.c @@ -1628,6 +1628,8 @@ static bool proxy_alive(ckpool_t *ckp, proxy_instance_t *proxi, connsock_t *cs, /* Has this proxy already been reconnected? */ if (cs->fd > 0) return true; + if (proxi->disabled) + return false; if (!extract_sockaddr(proxi->url, &cs->url, &cs->port)) { LOGWARNING("Failed to extract address from %s", proxi->url); goto out; @@ -1861,8 +1863,8 @@ static bool subproxies_alive(proxy_instance_t *proxy) static void *proxy_recv(void *arg) { proxy_instance_t *proxi = (proxy_instance_t *)arg; - proxy_instance_t *subproxy, *tmp; connsock_t *cs = &proxi->cs; + proxy_instance_t *subproxy; ckpool_t *ckp = proxi->ckp; gdata_t *gdata = ckp->data; struct epoll_event event; @@ -1963,17 +1965,6 @@ static void *proxy_recv(void *arg) } while ((ret = read_socket_line(cs, 0)) > 0); } - HASH_ITER(sh, proxi->subproxies, subproxy, tmp) { - subproxy->disabled = true; - send_stratifier_deadproxy(ckp, subproxy->id, subproxy->subid); - if (subproxy->cs.fd > 0) { - epoll_ctl(epfd, EPOLL_CTL_DEL, subproxy->cs.fd, NULL); - Close(subproxy->cs.fd); - } - HASH_DELETE(sh, proxi->subproxies, subproxy); - } - mutex_unlock(&proxi->proxy_lock); - return NULL; } @@ -2193,6 +2184,41 @@ out: send_api_response(val, sockd); } +static void parse_ableproxy(ckpool_t *ckp, gdata_t *gdata, const int sockd, + const char *buf, bool disable) +{ + proxy_instance_t *proxy; + + json_error_t err_val; + json_t *val = NULL; + int id = -1; + + val = json_loads(buf, 0, &err_val); + if (unlikely(!val)) { + val = json_encode_errormsg(&err_val); + goto out; + } + json_get_int(&id, val, "id"); + proxy = proxy_by_id(gdata, id); + if (!proxy) { + val = json_errormsg("Proxy id %d not found", id); + goto out; + } + JSON_CPACK(val, "{si,ss,ss,ss}", "id", proxy->id, "url", proxy->url, + "auth", proxy->auth, "pass", proxy->pass); + if (proxy->disabled != disable) { + proxy->disabled = disable; + LOGNOTICE("%sabling proxy %d", disable ? "Dis" : "En", id); + } + if (disable) { + disable_subproxy(gdata, proxy, proxy); + reconnect_generator(ckp); + } else + reconnect_proxy(proxy); +out: + send_api_response(val, sockd); +} + static int proxy_loop(proc_instance_t *pi) { proxy_instance_t *proxi = NULL, *cproxy; @@ -2263,6 +2289,10 @@ retry: parse_addproxy(ckp, gdata, sockd, buf + 9); } else if (cmdmatch(buf, "delproxy")) { parse_delproxy(ckp, gdata, sockd, buf + 9); + } else if (cmdmatch(buf, "enableproxy")) { + parse_ableproxy(ckp, gdata, sockd, buf + 12, false); + } else if (cmdmatch(buf, "disableproxy")) { + parse_ableproxy(ckp, gdata, sockd, buf + 13, true); } else if (cmdmatch(buf, "shutdown")) { ret = 0; goto out; From d61471ee2ce54a8ae957b2cdbcfd4e82aef294a2 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Tue, 17 Mar 2015 12:08:53 +1100 Subject: [PATCH 340/544] Add more notice information during API proxy manipulation --- src/generator.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/generator.c b/src/generator.c index 8cb5a5d4..b2acc16a 100644 --- a/src/generator.c +++ b/src/generator.c @@ -2120,6 +2120,7 @@ static void parse_addproxy(ckpool_t *ckp, gdata_t *gdata, const int sockd, const proxy = __add_proxy(ckp, gdata, id); mutex_unlock(&gdata->lock); + LOGNOTICE("Adding proxy %d:%s", id, proxy->url); prepare_proxy(proxy); JSON_CPACK(val, "{si,ss,ss,ss}", "id", proxy->id, "url", url, "auth", auth, "pass", pass); @@ -2178,6 +2179,7 @@ static void parse_delproxy(ckpool_t *ckp, gdata_t *gdata, const int sockd, const JSON_CPACK(val, "{si,ss,ss,ss}", "id", proxy->id, "url", proxy->url, "auth", proxy->auth, "pass", proxy->pass); + LOGNOTICE("Deleting proxy %d:%s", proxy->id, proxy->url); delete_proxy(gdata, proxy); reconnect_generator(ckp); out: @@ -2208,7 +2210,7 @@ static void parse_ableproxy(ckpool_t *ckp, gdata_t *gdata, const int sockd, "auth", proxy->auth, "pass", proxy->pass); if (proxy->disabled != disable) { proxy->disabled = disable; - LOGNOTICE("%sabling proxy %d", disable ? "Dis" : "En", id); + LOGNOTICE("%sabling proxy %d:%s", disable ? "Dis" : "En", id, proxy->url); } if (disable) { disable_subproxy(gdata, proxy, proxy); From 73268c15c5ceb4242abf11882a1e36b2b8631295 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Tue, 17 Mar 2015 13:18:04 +1100 Subject: [PATCH 341/544] Create a thread for all userproxy receives --- src/generator.c | 98 +++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 83 insertions(+), 15 deletions(-) diff --git a/src/generator.c b/src/generator.c index b2acc16a..5feb1f35 100644 --- a/src/generator.c +++ b/src/generator.c @@ -103,6 +103,7 @@ struct proxy_instance { bool no_params; /* Doesn't want any parameters on subscribe */ + bool global; /* Part of the global list of proxies */ bool notified; /* Has this proxy received any notifies yet */ bool disabled; /* Subproxy no longer to be used */ bool reconnect; /* We need to drop and reconnect */ @@ -148,6 +149,8 @@ struct generator_data { proxy_instance_t *dead_proxies; /* Disabled proxies */ int proxy_notify_id; // Globally increasing notify id ckmsgq_t *srvchk; // Server check message queue + pthread_t pth_uprecv; // User proxy receive thread + pthread_t pth_upsend; // User proxy send thread }; typedef struct generator_data gdata_t; @@ -1816,7 +1819,7 @@ static void *passthrough_recv(void *arg) if (likely(ret > 0)) ret = read_socket_line(cs, 60); if (ret < 1) { - LOGWARNING("Proxy %d:%s failed to read_socket_line in proxy_recv, attempting reconnect", + LOGWARNING("Proxy %d:%s failed to read_socket_line in passthrough_recv, attempting reconnect", proxi->id, proxi->url); alive = proxi->alive = false; reconnect_generator(ckp); @@ -1872,6 +1875,7 @@ static void *proxy_recv(void *arg) int epfd; rename_proc("proxyrecv"); + pthread_detach(pthread_self()); proxi->epfd = epfd = epoll_create1(EPOLL_CLOEXEC); if (epfd < 0){ @@ -1968,6 +1972,83 @@ static void *proxy_recv(void *arg) return NULL; } +/* Thread that handles all received messages from user proxies */ +static void *userproxy_recv(void *arg) +{ + ckpool_t *ckp = (ckpool_t *)arg; + gdata_t *gdata = ckp->data; + struct epoll_event event; + int epfd; + + rename_proc("uproxyrecv"); + pthread_detach(pthread_self()); + + epfd = epoll_create1(EPOLL_CLOEXEC); + if (epfd < 0){ + LOGEMERG("FATAL: Failed to create epoll in userproxy_recv"); + return NULL; + } + + while (42) { + share_msg_t *share, *tmpshare; + notify_instance_t *ni, *tmp; + proxy_instance_t *proxy; + connsock_t *cs; + time_t now; + int ret; + + ret = epoll_wait(epfd, &event, 1, 1000); + if (ret < 1) { + if (likely(!ret)) + continue; + LOGEMERG("Failed to epoll_wait in userproxy_recv"); + break; + } + proxy = event.data.ptr; + cs = &proxy->cs; + if (event.events & EPOLLHUP) { + LOGNOTICE("Proxy %d:%d %s hung up in epoll_wait", proxy->id, + proxy->subid, proxy->url); + disable_subproxy(gdata, proxy->parent, proxy); + continue; + } + now = time(NULL); + + /* Age old notifications older than 10 mins old */ + mutex_lock(&proxy->notify_lock); + HASH_ITER(hh, proxy->notify_instances, ni, tmp) { + if (HASH_COUNT(proxy->notify_instances) < 3) + break; + if (ni->notify_time < now - 600) { + HASH_DEL(proxy->notify_instances, ni); + clear_notify(ni); + } + } + mutex_unlock(&proxy->notify_lock); + + /* Similary with shares older than 2 mins without response */ + mutex_lock(&proxy->share_lock); + HASH_ITER(hh, proxy->shares, share, tmpshare) { + if (share->submit_time < now - 120) { + HASH_DEL(proxy->shares, share); + } + } + mutex_unlock(&proxy->share_lock); + + do { + /* proxy may have been recycled here if it is not a + * parent and reconnect was issued */ + if (parse_method(ckp, proxy, cs->buf)) + continue; + /* If it's not a method it should be a share result */ + if (!parse_share(proxy, cs->buf)) + LOGNOTICE("Proxy %d:%d unhandled stratum message: %s", + proxy->id, proxy->subid, cs->buf); + } while ((ret = read_socket_line(cs, 0)) > 0); + } + return NULL; +} + static void prepare_proxy(proxy_instance_t *proxi) { proxi->parent = proxi; @@ -2379,6 +2460,7 @@ static int proxy_mode(ckpool_t *ckp, proc_instance_t *pi) proxy->passsends = create_ckmsgq(ckp, "passsend", &passthrough_send); } else { prepare_proxy(proxy); + create_pthread(&gdata->pth_uprecv, userproxy_recv, ckp); } } @@ -2386,20 +2468,6 @@ static int proxy_mode(ckpool_t *ckp, proc_instance_t *pi) ret = proxy_loop(pi); - mutex_lock(&gdata->lock); - for (i = 0; i < ckp->proxies; i++) { - continue; // FIXME: Find proxies - Close(proxy->cs.fd); - free(proxy->enonce1); - free(proxy->enonce1bin); - pthread_cancel(proxy->pth_psend); - pthread_cancel(proxy->pth_precv); - dealloc(proxy->url); - dealloc(proxy->auth); - dealloc(proxy->pass); - } - mutex_unlock(&gdata->lock); - return ret; } From af3773901886b87a32d0a14a5c3b35f7b8e7f182 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Tue, 17 Mar 2015 13:20:54 +1100 Subject: [PATCH 342/544] Notifies and shares should all be with the parent structure --- src/generator.c | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/src/generator.c b/src/generator.c index 5feb1f35..c3b1b746 100644 --- a/src/generator.c +++ b/src/generator.c @@ -1990,9 +1990,9 @@ static void *userproxy_recv(void *arg) } while (42) { + proxy_instance_t *proxy, *parent; share_msg_t *share, *tmpshare; notify_instance_t *ni, *tmp; - proxy_instance_t *proxy; connsock_t *cs; time_t now; int ret; @@ -2013,27 +2013,27 @@ static void *userproxy_recv(void *arg) continue; } now = time(NULL); + parent = proxy->parent; - /* Age old notifications older than 10 mins old */ - mutex_lock(&proxy->notify_lock); - HASH_ITER(hh, proxy->notify_instances, ni, tmp) { - if (HASH_COUNT(proxy->notify_instances) < 3) + mutex_lock(&parent->notify_lock); + HASH_ITER(hh, parent->notify_instances, ni, tmp) { + if (HASH_COUNT(parent->notify_instances) < 3) break; if (ni->notify_time < now - 600) { - HASH_DEL(proxy->notify_instances, ni); + HASH_DEL(parent->notify_instances, ni); clear_notify(ni); } } - mutex_unlock(&proxy->notify_lock); + mutex_unlock(&parent->notify_lock); /* Similary with shares older than 2 mins without response */ - mutex_lock(&proxy->share_lock); - HASH_ITER(hh, proxy->shares, share, tmpshare) { + mutex_lock(&parent->share_lock); + HASH_ITER(hh, parent->shares, share, tmpshare) { if (share->submit_time < now - 120) { - HASH_DEL(proxy->shares, share); + HASH_DEL(parent->shares, share); } } - mutex_unlock(&proxy->share_lock); + mutex_unlock(&parent->share_lock); do { /* proxy may have been recycled here if it is not a From 964b9fb095aec087c43f79b8048ec51d52b518a8 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Tue, 17 Mar 2015 14:25:55 +1100 Subject: [PATCH 343/544] Move all proxy sends to the same thread --- src/generator.c | 80 +++++++++++++++++++++++-------------------------- 1 file changed, 37 insertions(+), 43 deletions(-) diff --git a/src/generator.c b/src/generator.c index c3b1b746..02ff825b 100644 --- a/src/generator.c +++ b/src/generator.c @@ -115,11 +115,6 @@ struct proxy_instance { notify_instance_t *notify_instances; pthread_t pth_precv; - pthread_t pth_psend; - mutex_t psend_lock; - pthread_cond_t psend_cond; - - stratum_msg_t *psends; mutex_t share_lock; share_msg_t *shares; @@ -150,7 +145,12 @@ struct generator_data { int proxy_notify_id; // Globally increasing notify id ckmsgq_t *srvchk; // Server check message queue pthread_t pth_uprecv; // User proxy receive thread - pthread_t pth_upsend; // User proxy send thread + pthread_t pth_psend; // Combined proxy send thread + + mutex_t psend_lock; // Lock associated with conditional below + pthread_cond_t psend_cond; + + stratum_msg_t *psends; }; typedef struct generator_data gdata_t; @@ -1429,10 +1429,10 @@ static void submit_share(gdata_t *gdata, json_t *val) json_object_set_nocheck(val, "id", json_integer(share->id)); /* Add the new message to the psend list */ - mutex_lock(&proxy->psend_lock); - DL_APPEND(proxy->psends, msg); - pthread_cond_signal(&proxy->psend_cond); - mutex_unlock(&proxy->psend_lock); + mutex_lock(&gdata->psend_lock); + DL_APPEND(gdata->psends, msg); + pthread_cond_signal(&gdata->psend_cond); + mutex_unlock(&gdata->psend_lock); out: if (!success) @@ -1494,9 +1494,7 @@ out: /* For processing and sending shares. proxy refers to parent proxy here */ static void *proxy_send(void *arg) { - proxy_instance_t *proxy = (proxy_instance_t *)arg; - connsock_t *cs = &proxy->cs; - ckpool_t *ckp = cs->ckp; + ckpool_t *ckp = (ckpool_t *)arg; gdata_t *gdata = ckp->data; stratum_msg_t *msg = NULL; @@ -1505,22 +1503,17 @@ static void *proxy_send(void *arg) pthread_detach(pthread_self()); while (42) { - proxy_instance_t *subproxy; + proxy_instance_t *proxy, *subproxy; int proxyid = 0, subid = 0; int64_t client_id = 0, id; notify_instance_t *ni; json_t *jobid = NULL; bool ret = true; + connsock_t *cs; json_t *val; tv_t now; ts_t abs; - if (unlikely(proxy->reconnect)) { - LOGINFO("Shutting down proxy_send thread for proxy %d to reconnect", - proxy->id); - break; - } - tv_time(&now); tv_to_ts(&abs, &now); abs.tv_sec++; @@ -1530,13 +1523,13 @@ static void *proxy_send(void *arg) free(msg); } - mutex_lock(&proxy->psend_lock); - if (!proxy->psends) - cond_timedwait(&proxy->psend_cond, &proxy->psend_lock, &abs); - msg = proxy->psends; + mutex_lock(&gdata->psend_lock); + if (!gdata->psends) + cond_timedwait(&gdata->psend_cond, &gdata->psend_lock, &abs); + msg = gdata->psends; if (likely(msg)) - DL_DELETE(proxy->psends, msg); - mutex_unlock(&proxy->psend_lock); + DL_DELETE(gdata->psends, msg); + mutex_unlock(&gdata->psend_lock); if (!msg) continue; @@ -1557,9 +1550,11 @@ static void *proxy_send(void *arg) LOGWARNING("Failed to find client_id in proxy_send msg"); continue; } - if (unlikely(proxyid != proxy->id)) { - LOGWARNING("Proxysend for proxy %d got message for proxy %d!", - proxy->id, proxyid); + proxy = proxy_by_id(gdata, proxyid); + if (unlikely(!proxy)) { + LOGWARNING("Proxysend for got message for non-existent proxy %d", + proxyid); + continue; } mutex_lock(&proxy->notify_lock); @@ -1569,9 +1564,13 @@ static void *proxy_send(void *arg) mutex_unlock(&proxy->notify_lock); subproxy = subproxy_by_id(proxy, subid); - if (subproxy) - cs = &subproxy->cs; - if (jobid && subproxy) { + if (unlikely(!subproxy)) { + LOGWARNING("Proxysend for got message for non-existent subproxy %d:%d", + proxyid, subid); + continue; + } + cs = &subproxy->cs; + if (jobid) { JSON_CPACK(val, "{s[soooo]soss}", "params", proxy->auth, jobid, json_object_dup(msg->json_msg, "nonce2"), json_object_dup(msg->json_msg, "ntime"), @@ -1580,16 +1579,12 @@ static void *proxy_send(void *arg) "method", "mining.submit"); ret = send_json_msg(cs, val); json_decref(val); - } else if (!jobid) { - stratifier_reconnect_client(ckp, client_id); - LOGNOTICE("Proxy %d:%s failed to find matching jobid for %sknown subproxy in proxysend", - proxy->id, proxy->url, subproxy ? "" : "un"); } else { stratifier_reconnect_client(ckp, client_id); - LOGNOTICE("Failed to find subproxy %d:%d to send message to", - proxy->id, subid); + LOGNOTICE("Proxy %d:%s failed to find matching jobid in proxysend", + proxy->id, proxy->url); } - if (!ret && subproxy) { + if (!ret) { LOGNOTICE("Proxy %d:%d %s failed to send msg in proxy_send, dropping to reconnect", proxy->id, subid, proxy->url); disable_subproxy(gdata, proxy, subproxy); @@ -2054,9 +2049,6 @@ static void prepare_proxy(proxy_instance_t *proxi) proxi->parent = proxi; mutex_init(&proxi->proxy_lock); add_subproxy(proxi, proxi); - mutex_init(&proxi->psend_lock); - cond_init(&proxi->psend_cond); - create_pthread(&proxi->pth_psend, proxy_send, proxi); create_pthread(&proxi->pth_precv, proxy_recv, proxi); } @@ -2217,7 +2209,6 @@ static void delete_proxy(gdata_t *gdata, proxy_instance_t *proxy) mutex_lock(&gdata->lock); HASH_DEL(gdata->proxies, proxy); /* Disable all its threads */ - pthread_cancel(proxy->pth_psend); pthread_cancel(proxy->pth_precv); Close(proxy->cs.fd); mutex_unlock(&gdata->lock); @@ -2461,6 +2452,9 @@ static int proxy_mode(ckpool_t *ckp, proc_instance_t *pi) } else { prepare_proxy(proxy); create_pthread(&gdata->pth_uprecv, userproxy_recv, ckp); + mutex_init(&gdata->psend_lock); + cond_init(&gdata->psend_cond); + create_pthread(&gdata->pth_psend, proxy_send, ckp); } } From 7827c5f96b61f90a8d08bdcfb8b4a96ba2ab8dd3 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Tue, 17 Mar 2015 15:30:15 +1100 Subject: [PATCH 344/544] Small cleanup of ordering --- src/generator.c | 35 ++++++++++++++++++----------------- 1 file changed, 18 insertions(+), 17 deletions(-) diff --git a/src/generator.c b/src/generator.c index 02ff825b..a2eacfe5 100644 --- a/src/generator.c +++ b/src/generator.c @@ -1556,6 +1556,12 @@ static void *proxy_send(void *arg) proxyid); continue; } + subproxy = subproxy_by_id(proxy, subid); + if (unlikely(!subproxy)) { + LOGWARNING("Proxysend for got message for non-existent subproxy %d:%d", + proxyid, subid); + continue; + } mutex_lock(&proxy->notify_lock); HASH_FIND_INT(proxy->notify_instances, &id, ni); @@ -1563,27 +1569,22 @@ static void *proxy_send(void *arg) jobid = json_copy(ni->jobid); mutex_unlock(&proxy->notify_lock); - subproxy = subproxy_by_id(proxy, subid); - if (unlikely(!subproxy)) { - LOGWARNING("Proxysend for got message for non-existent subproxy %d:%d", - proxyid, subid); - continue; - } - cs = &subproxy->cs; - if (jobid) { - JSON_CPACK(val, "{s[soooo]soss}", "params", proxy->auth, jobid, - json_object_dup(msg->json_msg, "nonce2"), - json_object_dup(msg->json_msg, "ntime"), - json_object_dup(msg->json_msg, "nonce"), - "id", json_object_dup(msg->json_msg, "id"), - "method", "mining.submit"); - ret = send_json_msg(cs, val); - json_decref(val); - } else { + if (unlikely(!jobid)) { stratifier_reconnect_client(ckp, client_id); LOGNOTICE("Proxy %d:%s failed to find matching jobid in proxysend", proxy->id, proxy->url); + continue; } + + cs = &subproxy->cs; + JSON_CPACK(val, "{s[soooo]soss}", "params", proxy->auth, jobid, + json_object_dup(msg->json_msg, "nonce2"), + json_object_dup(msg->json_msg, "ntime"), + json_object_dup(msg->json_msg, "nonce"), + "id", json_object_dup(msg->json_msg, "id"), + "method", "mining.submit"); + ret = send_json_msg(cs, val); + json_decref(val); if (!ret) { LOGNOTICE("Proxy %d:%d %s failed to send msg in proxy_send, dropping to reconnect", proxy->id, subid, proxy->url); From ed6febe6dbaf59f1223f3de083da785d5f4a0b6b Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Tue, 17 Mar 2015 16:06:19 +1100 Subject: [PATCH 345/544] Handle proxy sends according to which socket is ready to receive data --- src/generator.c | 84 ++++++++++++++++++++++++++++++++++++------------- 1 file changed, 63 insertions(+), 21 deletions(-) diff --git a/src/generator.c b/src/generator.c index a2eacfe5..cd6bf604 100644 --- a/src/generator.c +++ b/src/generator.c @@ -1491,12 +1491,59 @@ out: return ret; } +typedef struct cs_msg cs_msg_t; + +struct cs_msg { + cs_msg_t *next; + cs_msg_t *prev; + proxy_instance_t *proxy; + json_t *val; +}; + +/* Sends all messages in the queue ready to be dispatched, leaving those that + * fail write select to be handled next pass */ +static void send_json_msgq(gdata_t *gdata, cs_msg_t **csmsgq) +{ + cs_msg_t *csmsg, *tmp; + int ret; + + DL_FOREACH_SAFE(*csmsgq, csmsg, tmp) { + connsock_t *cs = &(csmsg->proxy->cs); + + if ((ret = wait_write_select(cs->fd, 0))) { + DL_DELETE(*csmsgq, csmsg); + if (ret > 0) + ret = send_json_msg(cs, csmsg->val); + json_decref(csmsg->val); + if (ret < 1) { + proxy_instance_t *proxy = csmsg->proxy; + + LOGNOTICE("Proxy %d:%d %s failed to send msg in send_json_msgq, dropping", + proxy->id, proxy->subid, proxy->url); + disable_subproxy(gdata, proxy->parent, proxy); + } + free(csmsg); + } + } +} + +static void add_json_msgq(cs_msg_t **csmsgq, proxy_instance_t *proxy, json_t **val) +{ + cs_msg_t *csmsg = ckalloc(sizeof(cs_msg_t)); + + csmsg->val = *val; + *val = NULL; + csmsg->proxy = proxy; + DL_APPEND(*csmsgq, csmsg); +} + /* For processing and sending shares. proxy refers to parent proxy here */ static void *proxy_send(void *arg) { ckpool_t *ckp = (ckpool_t *)arg; gdata_t *gdata = ckp->data; stratum_msg_t *msg = NULL; + cs_msg_t *csmsgq = NULL; rename_proc("proxysend"); @@ -1508,15 +1555,7 @@ static void *proxy_send(void *arg) int64_t client_id = 0, id; notify_instance_t *ni; json_t *jobid = NULL; - bool ret = true; - connsock_t *cs; json_t *val; - tv_t now; - ts_t abs; - - tv_time(&now); - tv_to_ts(&abs, &now); - abs.tv_sec++; if (unlikely(msg)) { json_decref(msg->json_msg); @@ -1524,15 +1563,24 @@ static void *proxy_send(void *arg) } mutex_lock(&gdata->psend_lock); - if (!gdata->psends) - cond_timedwait(&gdata->psend_cond, &gdata->psend_lock, &abs); + if (!gdata->psends) { + /* Poll every 10ms */ + const ts_t polltime = {0, 10000000}; + ts_t timeout_ts; + + ts_realtime(&timeout_ts); + timeraddspec(&timeout_ts, &polltime); + cond_timedwait(&gdata->psend_cond, &gdata->psend_lock, &timeout_ts); + } msg = gdata->psends; if (likely(msg)) DL_DELETE(gdata->psends, msg); mutex_unlock(&gdata->psend_lock); - if (!msg) + if (!msg) { + send_json_msgq(gdata, &csmsgq); continue; + } if (unlikely(!json_get_int(&subid, msg->json_msg, "subproxy"))) { LOGWARNING("Failed to find subproxy in proxy_send msg"); @@ -1572,24 +1620,18 @@ static void *proxy_send(void *arg) if (unlikely(!jobid)) { stratifier_reconnect_client(ckp, client_id); LOGNOTICE("Proxy %d:%s failed to find matching jobid in proxysend", - proxy->id, proxy->url); + subproxy->id, subproxy->url); continue; } - cs = &subproxy->cs; - JSON_CPACK(val, "{s[soooo]soss}", "params", proxy->auth, jobid, + JSON_CPACK(val, "{s[soooo]soss}", "params", subproxy->auth, jobid, json_object_dup(msg->json_msg, "nonce2"), json_object_dup(msg->json_msg, "ntime"), json_object_dup(msg->json_msg, "nonce"), "id", json_object_dup(msg->json_msg, "id"), "method", "mining.submit"); - ret = send_json_msg(cs, val); - json_decref(val); - if (!ret) { - LOGNOTICE("Proxy %d:%d %s failed to send msg in proxy_send, dropping to reconnect", - proxy->id, subid, proxy->url); - disable_subproxy(gdata, proxy, subproxy); - } + add_json_msgq(&csmsgq, subproxy, &val); + send_json_msgq(gdata, &csmsgq); } return NULL; } From a43c94c4d5401ff9e1c60bb9d71828c7330f9c1d Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Tue, 17 Mar 2015 18:22:26 +1100 Subject: [PATCH 346/544] Add a getuser command for generating users and getting their current statistics --- src/stratifier.c | 40 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/src/stratifier.c b/src/stratifier.c index 9f219b63..76a1d5eb 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -27,6 +27,7 @@ #include "stratifier.h" #include "uthash.h" #include "utlist.h" +#include "api.h" /* Consistent across all pool instances */ static const char *workpadding = "000000800000000000000000000000000000000000000000000000000000000000000000000000000000000080020000"; @@ -2276,6 +2277,40 @@ static void lazy_reconnect_client(sdata_t *sdata, stratum_instance_t *client) } } +static user_instance_t *get_user(sdata_t *sdata, const char *username); + +static void getuser(sdata_t *sdata, const char *buf, int *sockd) +{ + char *username = NULL; + user_instance_t *user; + json_error_t err_val; + json_t *val = NULL; + + val = json_loads(buf, 0, &err_val); + if (unlikely(!val)) { + val = json_encode_errormsg(&err_val); + goto out; + } + if (!json_get_string(&username, val, "user")) { + val = json_errormsg("Failed to find user key"); + goto out; + } + if (!strlen(username)) { + val = json_errormsg("Zero length user key"); + goto out; + } + user = get_user(sdata, username); + JSON_CPACK(val, "{ss,sI,si,sf,sf,sf,sf,sf,sf,si}", + "user", user->username, "id", user->id, "workers", user->workers, + "bestdiff", user->best_diff, "dsps1", user->dsps1, "dsps5", user->dsps5, + "dsps60", user->dsps60, "dsps1440", user->dsps1440, "dsps10080", user->dsps10080, + "lastshare", user->last_update.tv_sec); +out: + free(username); + send_api_response(val, *sockd); + _Close(sockd); +} + static void reconnect_client_id(sdata_t *sdata, const int64_t client_id) { stratum_instance_t *client; @@ -2362,6 +2397,11 @@ retry: Close(sockd); goto retry; } + /* Parse API commands here to return a message to sockd */ + if (cmdmatch(buf, "getuser")) { + getuser(sdata, buf + 8, &sockd); + goto retry; + } Close(sockd); LOGDEBUG("Stratifier received request: %s", buf); From e92e423412ec94f93d9deabe34727df39d77bab1 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Thu, 19 Mar 2015 16:48:26 +1100 Subject: [PATCH 347/544] Send a random sessionid rather than a blank one to prevent miner assuming they've reconnected --- src/stratifier.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/stratifier.c b/src/stratifier.c index c449a3fa..d7ffc9d5 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -2681,10 +2681,14 @@ static json_t *parse_subscribe(stratum_instance_t *client, const int64_t client_ n2len = sdata->workbases->enonce2varlen; ck_runlock(&sdata->workbase_lock); - /* Send a blank sessionid in proxy mode so clients don't think we have + /* Send a random sessionid in proxy mode so clients don't think we have * resumed if enonce1 ends up matching on reconnect. */ if (ckp->proxy) { - JSON_CPACK(ret, "[[[s,s]],s,i]", "mining.notify", "", client->enonce1, + unsigned int now = time(NULL); + char nowx[12]; + + sprintf(nowx, "%x", now); + JSON_CPACK(ret, "[[[s,s]],s,i]", "mining.notify", nowx, client->enonce1, n2len); } else { JSON_CPACK(ret, "[[[s,s]],s,i]", "mining.notify", client->enonce1, client->enonce1, From d8b0f9a9a9737e4013cc41494f6f34d5734ae416 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Thu, 19 Mar 2015 17:24:55 +1100 Subject: [PATCH 348/544] Make stratifier determine best proxy to connect to, allowing us to remove management from the generator --- src/stratifier.c | 73 ++++++++++++++++++++++++++---------------------- 1 file changed, 40 insertions(+), 33 deletions(-) diff --git a/src/stratifier.c b/src/stratifier.c index d7ffc9d5..f61d67cc 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -1246,18 +1246,45 @@ static void reconnect_clients(sdata_t *sdata) generator_recruit(sdata->ckp, -headroom); } -#if 0 -static proxy_t *current_proxy(sdata_t *sdata) +static bool __subproxies_alive(proxy_t *proxy) { - proxy_t *proxy; + proxy_t *subproxy, *tmp; + bool alive = false; + + HASH_ITER(sh, proxy->subproxies, subproxy, tmp) { + if (!subproxy->dead) { + alive = true; + break; + } + } + return alive; +} + +/* Iterate over the current global proxy list and see if the current one is + * the highest priority alive one. Uses ckp sdata */ +static void check_bestproxy(sdata_t *sdata) +{ + proxy_t *proxy, *tmp; + int changed_id = -1; mutex_lock(&sdata->proxy_lock); - proxy = sdata->proxy; + if (sdata->proxy && !__subproxies_alive(sdata->proxy)) + sdata->proxy = NULL; + HASH_ITER(hh, sdata->proxies, proxy, tmp) { + if (!__subproxies_alive(proxy)) + continue; + if (!sdata->proxy || sdata->proxy->id > proxy->id) { + sdata->proxy = proxy; + changed_id = proxy->id; + } + } mutex_unlock(&sdata->proxy_lock); - return proxy; + if (changed_id != -1) { + LOGNOTICE("Stratifier setting active proxy to %d", changed_id); + reconnect_clients(sdata); + } } -#endif static void dead_proxyid(sdata_t *sdata, const int id, const int subid) { @@ -1269,6 +1296,7 @@ static void dead_proxyid(sdata_t *sdata, const int id, const int subid) proxy = existing_subproxy(sdata, id, subid); if (proxy) proxy->dead = true; + check_bestproxy(sdata); LOGINFO("Stratifier dropping clients from proxy %d:%d", id, subid); headroom = current_headroom(sdata, &proxy); @@ -1380,7 +1408,6 @@ static void update_notify(ckpool_t *ckp, const char *cmd) bool new_block = false, clean; proxy_t *proxy, *current; int i, id = 0, subid = 0; - int64_t current_id; char header[228]; const char *buf; workbase_t *wb; @@ -1471,11 +1498,9 @@ static void update_notify(ckpool_t *ckp, const char *cmd) current = sdata->proxy; if (unlikely(!current)) current = sdata->proxy = proxy; - current_id = current->id; mutex_unlock(&sdata->proxy_lock); - if (!proxy->subid && proxy->id == current_id) - reconnect_clients(sdata); + check_bestproxy(sdata); clean |= new_block; LOGINFO("Proxy %d:%d broadcast updated stratum notify with%s clean", id, subid, clean ? "" : "out"); @@ -2229,24 +2254,6 @@ static void reconnect_client(sdata_t *sdata, stratum_instance_t *client) stratum_add_send(sdata, json_msg, client->id); } -/* Sets the currently active proxy. Clients will be told to reconnect once the - * first notify data comes from this proxy. Even if we are already bound to - * this proxy we are only given this message if all clients must move. */ -static void set_proxy(sdata_t *sdata, const char *buf) -{ - int id = 0; - proxy_t *proxy; - - sscanf(buf, "proxy=%d", &id); - - mutex_lock(&sdata->proxy_lock); - proxy = __proxy_by_id(sdata, id); - sdata->proxy = proxy; - mutex_unlock(&sdata->proxy_lock); - - LOGNOTICE("Stratifier setting active proxy to %d", id); -} - static void dead_proxy(sdata_t *sdata, const char *buf) { int id = 0, subid = 0; @@ -2442,8 +2449,6 @@ retry: block_reject(sdata, buf + 8); } else if (cmdmatch(buf, "reconnect")) { request_reconnect(sdata, buf); - } else if (cmdmatch(buf, "proxy")) { - set_proxy(sdata, buf); } else if (cmdmatch(buf, "deadproxy")) { dead_proxy(sdata, buf); } else if (cmdmatch(buf, "loglevel")) { @@ -2548,6 +2553,8 @@ static bool new_enonce1(ckpool_t *ckp, sdata_t *ckp_sdata, sdata_t *sdata, strat static void stratum_send_message(sdata_t *sdata, const stratum_instance_t *client, const char *msg); +#define maxint 2147483647 + /* Choose the stratifier data for a new client. Use the main ckp_sdata except * in proxy mode where we find a subproxy based on the current proxy with room * for more clients. Signal the generator to recruit more subproxies if we are @@ -2564,7 +2571,7 @@ static sdata_t *select_sdata(const ckpool_t *ckp, sdata_t *ckp_sdata) LOGWARNING("No proxy available yet to generate subscribes"); return NULL; } - best_id = ckp_sdata->proxy_count; + best_id = maxint; mutex_lock(&ckp_sdata->proxy_lock); HASH_ITER(hh, ckp_sdata->proxies, proxy, tmp) { @@ -2598,8 +2605,8 @@ static sdata_t *select_sdata(const ckpool_t *ckp, sdata_t *ckp_sdata) if (best_id != current->id || current->headroom < 2) generator_recruit(ckp, 1); - if (best_id == ckp_sdata->proxy_count) { - LOGNOTICE("Temporarily insufficient subproxies to accept more clients"); + if (best_id == maxint) { + LOGWARNING("Temporarily insufficient subproxies to accept more clients"); return NULL; } best = subproxy_by_id(ckp_sdata, best_id, best_subid); From 172e60b567974afc5c6a7b5eed82d471afc7006a Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Thu, 19 Mar 2015 17:55:49 +1100 Subject: [PATCH 349/544] Remove concept of current proxy from generator, allowing stratifier to do all the management --- src/generator.c | 55 +++++++++++-------------------------------------- 1 file changed, 12 insertions(+), 43 deletions(-) diff --git a/src/generator.c b/src/generator.c index cd6bf604..d0f52643 100644 --- a/src/generator.c +++ b/src/generator.c @@ -124,8 +124,6 @@ struct proxy_instance { char_entry_t *recvd_lines; /* Linked list of unprocessed messages */ - time_t reconnect_time; - int epfd; /* Epoll fd used by the parent proxy */ mutex_t proxy_lock; /* Lock protecting hashlist of proxies */ @@ -140,7 +138,6 @@ struct generator_data { ckpool_t *ckp; mutex_t lock; /* Lock protecting linked lists */ proxy_instance_t *proxies; /* Hash list of all proxies */ - proxy_instance_t *proxy; /* Current proxy */ proxy_instance_t *dead_proxies; /* Disabled proxies */ int proxy_notify_id; // Globally increasing notify id ckmsgq_t *srvchk; // Server check message queue @@ -851,12 +848,6 @@ static bool parse_notify(ckpool_t *ckp, proxy_instance_t *proxi, json_t *val) mutex_unlock(&proxy->notify_lock); send_notify(ckp, proxi, ni); - /* We have all the ingredients necessary to switch to this proxy if - * it's the best one so reassess now. */ - if (unlikely(parent_proxy(proxi) && !proxi->notified)) { - proxi->notified = true; - reconnect_generator(ckp); - } out: return ret; } @@ -1002,8 +993,10 @@ static void disable_subproxy(gdata_t *gdata, proxy_instance_t *proxi, proxy_inst HASH_DELETE(sh, proxi->subproxies, subproxy); mutex_unlock(&proxi->proxy_lock); - if (subproxy) + if (subproxy) { + send_stratifier_deadproxy(gdata->ckp, subproxy->id, subproxy->subid); store_proxy(gdata, subproxy); + } } static bool parse_reconnect(proxy_instance_t *proxy, json_t *val) @@ -1717,8 +1710,6 @@ out: epoll_ctl(proxi->epfd, EPOLL_CTL_DEL, cs->fd, NULL); Close(cs->fd); } - if (parent_proxy(proxi)) - reconnect_generator(ckp); } else { keep_sockalive(cs->fd); event.events = EPOLLIN; @@ -1871,17 +1862,6 @@ static void *passthrough_recv(void *arg) return NULL; } -static proxy_instance_t *current_proxy(gdata_t *gdata) -{ - proxy_instance_t *ret; - - mutex_lock(&gdata->lock); - ret = gdata->proxy; - mutex_unlock(&gdata->lock); - - return ret; -} - static bool subproxies_alive(proxy_instance_t *proxy) { proxy_instance_t *subproxy, *tmp; @@ -1943,15 +1923,10 @@ static void *proxy_recv(void *arg) alive = false; } sleep(5); - proxi->reconnect_time = time(NULL); } } - /* Wait 30 seconds before declaring this upstream pool alive - * to prevent switching to unstable pools. */ - if (!alive && (!current_proxy(gdata) || time(NULL) - proxi->reconnect_time > 30)) { - reconnect_generator(ckp); + if (!alive) { LOGWARNING("Proxy %d:%s recovered", proxi->id, proxi->url); - proxi->reconnect_time = 0; alive = true; } @@ -2113,7 +2088,6 @@ static proxy_instance_t *wait_best_proxy(ckpool_t *ckp, gdata_t *gdata) ret = proxi; } } - gdata->proxy = ret; mutex_unlock(&gdata->lock); if (ret) @@ -2244,7 +2218,7 @@ out: send_api_response(val, sockd); } -static void delete_proxy(gdata_t *gdata, proxy_instance_t *proxy) +static void delete_proxy(ckpool_t *ckp, gdata_t *gdata, proxy_instance_t *proxy) { proxy_instance_t *subproxy; @@ -2264,6 +2238,7 @@ static void delete_proxy(gdata_t *gdata, proxy_instance_t *proxy) HASH_DELETE(sh, proxy->subproxies, subproxy); mutex_unlock(&proxy->proxy_lock); + send_stratifier_deadproxy(ckp, subproxy->id, subproxy->subid); if (subproxy && proxy != subproxy) store_proxy(gdata, subproxy); } while (subproxy); @@ -2295,14 +2270,12 @@ static void parse_delproxy(ckpool_t *ckp, gdata_t *gdata, const int sockd, const "auth", proxy->auth, "pass", proxy->pass); LOGNOTICE("Deleting proxy %d:%s", proxy->id, proxy->url); - delete_proxy(gdata, proxy); - reconnect_generator(ckp); + delete_proxy(ckp, gdata, proxy); out: send_api_response(val, sockd); } -static void parse_ableproxy(ckpool_t *ckp, gdata_t *gdata, const int sockd, - const char *buf, bool disable) +static void parse_ableproxy(gdata_t *gdata, const int sockd, const char *buf, bool disable) { proxy_instance_t *proxy; @@ -2327,10 +2300,9 @@ static void parse_ableproxy(ckpool_t *ckp, gdata_t *gdata, const int sockd, proxy->disabled = disable; LOGNOTICE("%sabling proxy %d:%s", disable ? "Dis" : "En", id, proxy->url); } - if (disable) { + if (disable) disable_subproxy(gdata, proxy, proxy); - reconnect_generator(ckp); - } else + else reconnect_proxy(proxy); out: send_api_response(val, sockd); @@ -2357,9 +2329,6 @@ reconnect: if (!ckp->passthrough) { LOGWARNING("Successfully connected to proxy %d %s as proxy", proxi->id, proxi->url); - dealloc(buf); - ASPRINTF(&buf, "proxy=%d", proxi->id); - send_proc(ckp->stratifier, buf); } } retry: @@ -2407,9 +2376,9 @@ retry: } else if (cmdmatch(buf, "delproxy")) { parse_delproxy(ckp, gdata, sockd, buf + 9); } else if (cmdmatch(buf, "enableproxy")) { - parse_ableproxy(ckp, gdata, sockd, buf + 12, false); + parse_ableproxy(gdata, sockd, buf + 12, false); } else if (cmdmatch(buf, "disableproxy")) { - parse_ableproxy(ckp, gdata, sockd, buf + 13, true); + parse_ableproxy(gdata, sockd, buf + 13, true); } else if (cmdmatch(buf, "shutdown")) { ret = 0; goto out; From 9a1e59637f7c2f98be1af55b7019238626b0c40a Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Fri, 20 Mar 2015 09:27:38 +1100 Subject: [PATCH 350/544] Add uthash helpers for iterating over hash entries without the need for safe against deletion --- src/uthash.h | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/uthash.h b/src/uthash.h index 31db3fa8..ce61e339 100644 --- a/src/uthash.h +++ b/src/uthash.h @@ -869,14 +869,21 @@ do { (sizeof(UT_hash_table)) + \ (HASH_BLOOM_BYTELEN))) +/* HASH_ITERATE and HASH_FOREACH are not safe for deletion */ +#define HASH_ITERATE(head,el) HASH_ITERATE2(hh,head,el) +#define HASH_FOREACH(hh,head,el) HASH_ITERATE2(hh,head,el) #ifdef NO_DECLTYPE #define HASH_ITER(hh,head,el,tmp) \ for((el)=(head), (*(char**)(&(tmp)))=(char*)((head)?(head)->hh.next:NULL); \ el; (el)=(tmp),(*(char**)(&(tmp)))=(char*)((tmp)?(tmp)->hh.next:NULL)) +#define HASH_ITERATE2(hh,head,el) \ +for ((el)=(head); ((head)!= NULL); (head)=(head)->hh.next) #else #define HASH_ITER(hh,head,el,tmp) \ for((el)=(head),(tmp)=DECLTYPE(el)((head)?(head)->hh.next:NULL); \ el; (el)=(tmp),(tmp)=DECLTYPE(el)((tmp)?(tmp)->hh.next:NULL)) +#define HASH_ITERATE2(hh,head,el) \ +for ((el)=(head); ((head)!= NULL); (head)=DECLTYPE(el)(head)->hh.next) #endif /* obtain a count of items in the hash */ From 950a20e76fc4aaa085ea9c3f1ecbe44d10f7e0fe Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Fri, 20 Mar 2015 09:37:38 +1100 Subject: [PATCH 351/544] Use the terse hash helpers in the stratifier --- src/stratifier.c | 76 ++++++++++++++++++++++++------------------------ 1 file changed, 38 insertions(+), 38 deletions(-) diff --git a/src/stratifier.c b/src/stratifier.c index f61d67cc..8520b3c7 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -1178,14 +1178,14 @@ static proxy_t *existing_subproxy(sdata_t *sdata, const int id, const int subid) /* Set proxy to the current proxy and calculate how much headroom it has */ static int64_t current_headroom(sdata_t *sdata, proxy_t **proxy) { - proxy_t *subproxy, *tmp; int64_t headroom = 0; + proxy_t *subproxy; mutex_lock(&sdata->proxy_lock); *proxy = sdata->proxy; if (!*proxy) goto out_unlock; - HASH_ITER(sh, (*proxy)->subproxies, subproxy, tmp) { + HASH_FOREACH(sh, (*proxy)->subproxies, subproxy) { if (subproxy->dead) continue; headroom += subproxy->max_clients - subproxy->clients; @@ -1212,8 +1212,8 @@ static void generator_recruit(const ckpool_t *ckp, const int recruits) * switched lazily. */ static void reconnect_clients(sdata_t *sdata) { - stratum_instance_t *client, *tmpclient; int reconnects = 0, hard = 0; + stratum_instance_t *client; int64_t headroom; proxy_t *proxy; @@ -1222,7 +1222,7 @@ static void reconnect_clients(sdata_t *sdata) return; ck_rlock(&sdata->instance_lock); - HASH_ITER(hh, sdata->stratum_instances, client, tmpclient) { + HASH_ITERATE(sdata->stratum_instances, client) { if (client->proxyid == proxy->id) continue; if (headroom-- < 1) @@ -1248,10 +1248,10 @@ static void reconnect_clients(sdata_t *sdata) static bool __subproxies_alive(proxy_t *proxy) { - proxy_t *subproxy, *tmp; bool alive = false; + proxy_t *subproxy; - HASH_ITER(sh, proxy->subproxies, subproxy, tmp) { + HASH_FOREACH(sh, proxy->subproxies, subproxy) { if (!subproxy->dead) { alive = true; break; @@ -1264,13 +1264,13 @@ static bool __subproxies_alive(proxy_t *proxy) * the highest priority alive one. Uses ckp sdata */ static void check_bestproxy(sdata_t *sdata) { - proxy_t *proxy, *tmp; int changed_id = -1; + proxy_t *proxy; mutex_lock(&sdata->proxy_lock); if (sdata->proxy && !__subproxies_alive(sdata->proxy)) sdata->proxy = NULL; - HASH_ITER(hh, sdata->proxies, proxy, tmp) { + HASH_ITERATE(sdata->proxies, proxy) { if (!__subproxies_alive(proxy)) continue; if (!sdata->proxy || sdata->proxy->id > proxy->id) { @@ -1288,8 +1288,8 @@ static void check_bestproxy(sdata_t *sdata) static void dead_proxyid(sdata_t *sdata, const int id, const int subid) { - stratum_instance_t *client, *tmp; int reconnects = 0, hard = 0; + stratum_instance_t *client; int64_t headroom; proxy_t *proxy; @@ -1301,7 +1301,7 @@ static void dead_proxyid(sdata_t *sdata, const int id, const int subid) headroom = current_headroom(sdata, &proxy); ck_rlock(&sdata->instance_lock); - HASH_ITER(hh, sdata->stratum_instances, client, tmp) { + HASH_ITERATE(sdata->stratum_instances, client) { if (client->proxyid != id || client->subproxyid != subid) continue; reconnects++; @@ -1514,7 +1514,7 @@ static void stratum_send_diff(sdata_t *sdata, const stratum_instance_t *client); static void update_diff(ckpool_t *ckp, const char *cmd) { sdata_t *sdata = ckp->data, *dsdata; - stratum_instance_t *client, *tmp; + stratum_instance_t *client; double old_diff, diff; int id = 0, subid = 0; const char *buf; @@ -1568,7 +1568,7 @@ static void update_diff(ckpool_t *ckp, const char *cmd) /* If the diff has dropped, iterate over all the clients and check * they're at or below the new diff, and update it if not. */ ck_rlock(&sdata->instance_lock); - HASH_ITER(hh, sdata->stratum_instances, client, tmp) { + HASH_ITERATE(sdata->stratum_instances, client) { if (client->proxyid != id) continue; if (client->subproxyid != subid) @@ -1602,14 +1602,14 @@ static void free_proxy(proxy_t *proxy) * those. */ static void reap_proxies(ckpool_t *ckp, sdata_t *sdata) { - proxy_t *proxy, *proxytmp, *subproxy, *subtmp; + proxy_t *proxy, *subproxy, *subtmp; int dead = 0; if (!ckp->proxy) return; mutex_lock(&sdata->proxy_lock); - HASH_ITER(hh, sdata->proxies, proxy, proxytmp) { + HASH_ITERATE(sdata->proxies, proxy) { HASH_ITER(sh, proxy->subproxies, subproxy, subtmp) { if (!subproxy->bound_clients && !subproxy->dead) { /* Reset the counter to reuse this proxy */ @@ -1682,10 +1682,10 @@ static stratum_instance_t *ref_instance_by_id(sdata_t *sdata, const int64_t id) /* Has this client_id already been used and is now in one of the dropped lists */ static bool __dropped_instance(sdata_t *sdata, const int64_t id) { - stratum_instance_t *client, *tmp; + stratum_instance_t *client; bool ret = false; - HASH_ITER(hh, sdata->disconnected_instances, client, tmp) { + HASH_ITERATE(sdata->disconnected_instances, client) { if (unlikely(client->id == id)) { ret = true; goto out; @@ -1802,8 +1802,8 @@ static stratum_instance_t *__stratum_add_instance(ckpool_t *ckp, const int64_t i static uint64_t disconnected_sessionid_exists(sdata_t *sdata, const char *sessionid, const int64_t id) { - stratum_instance_t *client, *tmp; uint64_t enonce1_64 = 0, ret = 0; + stratum_instance_t *client; int64_t old_id = 0; int slen; @@ -1820,7 +1820,7 @@ static uint64_t disconnected_sessionid_exists(sdata_t *sdata, const char *sessio hex2bin(&enonce1_64, sessionid, slen); ck_wlock(&sdata->instance_lock); - HASH_ITER(hh, sdata->stratum_instances, client, tmp) { + HASH_ITERATE(sdata->stratum_instances, client) { if (client->id == id) continue; if (client->enonce1_64 == enonce1_64) { @@ -1869,7 +1869,7 @@ static void stratum_broadcast(sdata_t *sdata, json_t *val) { ckpool_t *ckp = sdata->ckp; sdata_t *ckp_sdata = ckp->data; - stratum_instance_t *client, *tmp; + stratum_instance_t *client; ckmsg_t *bulk_send = NULL; time_t now_t = time(NULL); int hard_reconnect = 0; @@ -1882,7 +1882,7 @@ static void stratum_broadcast(sdata_t *sdata, json_t *val) /* Use this locking as an opportunity to test other clients. */ ck_rlock(&ckp_sdata->instance_lock); - HASH_ITER(hh, ckp_sdata->stratum_instances, client, tmp) { + HASH_ITERATE(ckp_sdata->stratum_instances, client) { ckmsg_t *client_msg; smsg_t *msg; @@ -2003,7 +2003,7 @@ static void stratum_broadcast_message(sdata_t *sdata, const char *msg) static void request_reconnect(sdata_t *sdata, const char *cmd) { char *port = strdupa(cmd), *url = NULL; - stratum_instance_t *client, *tmp; + stratum_instance_t *client; json_t *json_msg; strsep(&port, ":"); @@ -2023,7 +2023,7 @@ static void request_reconnect(sdata_t *sdata, const char *cmd) /* Tag all existing clients as dropped now so they can be removed * lazily */ ck_wlock(&sdata->instance_lock); - HASH_ITER(hh, sdata->stratum_instances, client, tmp) { + HASH_ITERATE(sdata->stratum_instances, client) { client->dropped = true; } ck_wunlock(&sdata->instance_lock); @@ -2031,14 +2031,14 @@ static void request_reconnect(sdata_t *sdata, const char *cmd) static void reset_bestshares(sdata_t *sdata) { - user_instance_t *user, *tmpuser; - stratum_instance_t *client, *tmp; + stratum_instance_t *client; + user_instance_t *user; ck_rlock(&sdata->instance_lock); - HASH_ITER(hh, sdata->stratum_instances, client, tmp) { + HASH_ITERATE(sdata->stratum_instances, client) { client->best_diff = 0; } - HASH_ITER(hh, sdata->user_instances, user, tmpuser) { + HASH_ITERATE(sdata->user_instances, user) { worker_instance_t *worker; user->best_diff = 0; @@ -2561,7 +2561,7 @@ static void stratum_send_message(sdata_t *sdata, const stratum_instance_t *clien * running out of room. */ static sdata_t *select_sdata(const ckpool_t *ckp, sdata_t *ckp_sdata) { - proxy_t *current, *proxy, *subproxy, *best = NULL, *tmp, *tmpsub; + proxy_t *current, *proxy, *subproxy, *best = NULL; int best_subid = 0, best_id; if (!ckp->proxy || ckp->passthrough) @@ -2574,12 +2574,12 @@ static sdata_t *select_sdata(const ckpool_t *ckp, sdata_t *ckp_sdata) best_id = maxint; mutex_lock(&ckp_sdata->proxy_lock); - HASH_ITER(hh, ckp_sdata->proxies, proxy, tmp) { + HASH_ITERATE(ckp_sdata->proxies, proxy) { int64_t max_headroom; best = NULL; proxy->headroom = max_headroom = 0; - HASH_ITER(sh, proxy->subproxies, subproxy, tmpsub) { + HASH_FOREACH(sh, proxy->subproxies, subproxy) { int64_t subproxy_headroom; if (subproxy->dead) @@ -4610,7 +4610,7 @@ out: static void update_workerstats(ckpool_t *ckp, sdata_t *sdata) { json_entry_t *json_list = NULL, *entry, *tmpentry; - user_instance_t *user, *tmp; + user_instance_t *user; char cdfield[64]; time_t now_t; ts_t ts_now; @@ -4628,7 +4628,7 @@ static void update_workerstats(ckpool_t *ckp, sdata_t *sdata) now_t = ts_now.tv_sec; ck_rlock(&sdata->instance_lock); - HASH_ITER(hh, sdata->user_instances, user, tmp) { + HASH_ITERATE(sdata->user_instances, user) { worker_instance_t *worker; uint8_t cycle_mask; @@ -4732,10 +4732,10 @@ static void *statsupdate(void *arg) double ghs, ghs1, ghs5, ghs15, ghs60, ghs360, ghs1440, ghs10080, per_tdiff; char suffix1[16], suffix5[16], suffix15[16], suffix60[16], cdfield[64]; char suffix360[16], suffix1440[16], suffix10080[16]; - stratum_instance_t *client, *tmp; log_entry_t *log_entries = NULL; - user_instance_t *user, *tmpuser; char_entry_t *char_list = NULL; + stratum_instance_t *client; + user_instance_t *user; int idle_workers = 0; char *fname, *s, *sp; tv_t now, diff; @@ -4748,7 +4748,7 @@ static void *statsupdate(void *arg) timersub(&now, &stats->start_time, &diff); ck_rlock(&sdata->instance_lock); - HASH_ITER(hh, sdata->stratum_instances, client, tmp) { + HASH_ITERATE(sdata->stratum_instances, client) { if (!client_active(client)) continue; @@ -4768,7 +4768,7 @@ static void *statsupdate(void *arg) } } - HASH_ITER(hh, sdata->user_instances, user, tmpuser) { + HASH_ITERATE(sdata->user_instances, user) { worker_instance_t *worker; bool idle = false; @@ -4934,7 +4934,7 @@ static void *statsupdate(void *arg) fclose(fp); if (ckp->proxy && sdata->proxy) { - proxy_t *proxy, *proxytmp, *subproxy, *subtmp; + proxy_t *proxy, *subproxy; mutex_lock(&sdata->proxy_lock); JSON_CPACK(val, "{sI,si,si}", @@ -4949,7 +4949,7 @@ static void *statsupdate(void *arg) dealloc(s); mutex_lock(&sdata->proxy_lock); - HASH_ITER(hh, sdata->proxies, proxy, proxytmp) { + HASH_ITERATE(sdata->proxies, proxy) { JSON_CPACK(val, "{sI,si,sI,sb}", "id", proxy->id, "subproxies", proxy->subproxy_count, @@ -4960,7 +4960,7 @@ static void *statsupdate(void *arg) ASPRINTF(&sp, "Proxies:%s", s); dealloc(s); add_msg_entry(&char_list, &sp); - HASH_ITER(sh, proxy->subproxies, subproxy, subtmp) { + HASH_FOREACH(sh, proxy->subproxies, subproxy) { JSON_CPACK(val, "{sI,si,si,sI,sI,sf,sb}", "id", subproxy->id, "subid", subproxy->subid, From 4b099e8445e67e6f642056c2991fc8ed12bcedeb Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Fri, 20 Mar 2015 09:49:12 +1100 Subject: [PATCH 352/544] Add a priority field to proxies, currently set to their id and sort them by priority --- src/stratifier.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/stratifier.c b/src/stratifier.c index 8520b3c7..2bbfa5c1 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -298,6 +298,7 @@ struct proxy_base { proxy_t *prev; int id; int subid; + int priority; double diff; @@ -1079,12 +1080,18 @@ static sdata_t *duplicate_sdata(const sdata_t *sdata) return dsdata; } +static int prio_sort(proxy_t *a, proxy_t *b) +{ + return (a->priority - b->priority); +} + static proxy_t *__generate_proxy(sdata_t *sdata, const int id) { proxy_t *proxy = ckzalloc(sizeof(proxy_t)); proxy->parent = proxy; proxy->id = id; + proxy->priority = id; proxy->sdata = duplicate_sdata(sdata); proxy->sdata->subproxy = proxy; proxy->sdata->verbose = true; @@ -1092,6 +1099,7 @@ static proxy_t *__generate_proxy(sdata_t *sdata, const int id) HASH_ADD(sh, proxy->subproxies, subid, sizeof(int), proxy); proxy->subproxy_count++; HASH_ADD_INT(sdata->proxies, id, proxy); + HASH_SORT(sdata->proxies, prio_sort); sdata->proxy_count++; return proxy; } From 93d820ab55caa31ccaa5893c22510f87218277ec Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Fri, 20 Mar 2015 10:20:37 +1100 Subject: [PATCH 353/544] Revert "Use the terse hash helpers in the stratifier" This reverts commit 950a20e76fc4aaa085ea9c3f1ecbe44d10f7e0fe. --- src/stratifier.c | 76 ++++++++++++++++++++++++------------------------ 1 file changed, 38 insertions(+), 38 deletions(-) diff --git a/src/stratifier.c b/src/stratifier.c index 2bbfa5c1..93303e69 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -1186,14 +1186,14 @@ static proxy_t *existing_subproxy(sdata_t *sdata, const int id, const int subid) /* Set proxy to the current proxy and calculate how much headroom it has */ static int64_t current_headroom(sdata_t *sdata, proxy_t **proxy) { + proxy_t *subproxy, *tmp; int64_t headroom = 0; - proxy_t *subproxy; mutex_lock(&sdata->proxy_lock); *proxy = sdata->proxy; if (!*proxy) goto out_unlock; - HASH_FOREACH(sh, (*proxy)->subproxies, subproxy) { + HASH_ITER(sh, (*proxy)->subproxies, subproxy, tmp) { if (subproxy->dead) continue; headroom += subproxy->max_clients - subproxy->clients; @@ -1220,8 +1220,8 @@ static void generator_recruit(const ckpool_t *ckp, const int recruits) * switched lazily. */ static void reconnect_clients(sdata_t *sdata) { + stratum_instance_t *client, *tmpclient; int reconnects = 0, hard = 0; - stratum_instance_t *client; int64_t headroom; proxy_t *proxy; @@ -1230,7 +1230,7 @@ static void reconnect_clients(sdata_t *sdata) return; ck_rlock(&sdata->instance_lock); - HASH_ITERATE(sdata->stratum_instances, client) { + HASH_ITER(hh, sdata->stratum_instances, client, tmpclient) { if (client->proxyid == proxy->id) continue; if (headroom-- < 1) @@ -1256,10 +1256,10 @@ static void reconnect_clients(sdata_t *sdata) static bool __subproxies_alive(proxy_t *proxy) { + proxy_t *subproxy, *tmp; bool alive = false; - proxy_t *subproxy; - HASH_FOREACH(sh, proxy->subproxies, subproxy) { + HASH_ITER(sh, proxy->subproxies, subproxy, tmp) { if (!subproxy->dead) { alive = true; break; @@ -1272,13 +1272,13 @@ static bool __subproxies_alive(proxy_t *proxy) * the highest priority alive one. Uses ckp sdata */ static void check_bestproxy(sdata_t *sdata) { + proxy_t *proxy, *tmp; int changed_id = -1; - proxy_t *proxy; mutex_lock(&sdata->proxy_lock); if (sdata->proxy && !__subproxies_alive(sdata->proxy)) sdata->proxy = NULL; - HASH_ITERATE(sdata->proxies, proxy) { + HASH_ITER(hh, sdata->proxies, proxy, tmp) { if (!__subproxies_alive(proxy)) continue; if (!sdata->proxy || sdata->proxy->id > proxy->id) { @@ -1296,8 +1296,8 @@ static void check_bestproxy(sdata_t *sdata) static void dead_proxyid(sdata_t *sdata, const int id, const int subid) { + stratum_instance_t *client, *tmp; int reconnects = 0, hard = 0; - stratum_instance_t *client; int64_t headroom; proxy_t *proxy; @@ -1309,7 +1309,7 @@ static void dead_proxyid(sdata_t *sdata, const int id, const int subid) headroom = current_headroom(sdata, &proxy); ck_rlock(&sdata->instance_lock); - HASH_ITERATE(sdata->stratum_instances, client) { + HASH_ITER(hh, sdata->stratum_instances, client, tmp) { if (client->proxyid != id || client->subproxyid != subid) continue; reconnects++; @@ -1522,7 +1522,7 @@ static void stratum_send_diff(sdata_t *sdata, const stratum_instance_t *client); static void update_diff(ckpool_t *ckp, const char *cmd) { sdata_t *sdata = ckp->data, *dsdata; - stratum_instance_t *client; + stratum_instance_t *client, *tmp; double old_diff, diff; int id = 0, subid = 0; const char *buf; @@ -1576,7 +1576,7 @@ static void update_diff(ckpool_t *ckp, const char *cmd) /* If the diff has dropped, iterate over all the clients and check * they're at or below the new diff, and update it if not. */ ck_rlock(&sdata->instance_lock); - HASH_ITERATE(sdata->stratum_instances, client) { + HASH_ITER(hh, sdata->stratum_instances, client, tmp) { if (client->proxyid != id) continue; if (client->subproxyid != subid) @@ -1610,14 +1610,14 @@ static void free_proxy(proxy_t *proxy) * those. */ static void reap_proxies(ckpool_t *ckp, sdata_t *sdata) { - proxy_t *proxy, *subproxy, *subtmp; + proxy_t *proxy, *proxytmp, *subproxy, *subtmp; int dead = 0; if (!ckp->proxy) return; mutex_lock(&sdata->proxy_lock); - HASH_ITERATE(sdata->proxies, proxy) { + HASH_ITER(hh, sdata->proxies, proxy, proxytmp) { HASH_ITER(sh, proxy->subproxies, subproxy, subtmp) { if (!subproxy->bound_clients && !subproxy->dead) { /* Reset the counter to reuse this proxy */ @@ -1690,10 +1690,10 @@ static stratum_instance_t *ref_instance_by_id(sdata_t *sdata, const int64_t id) /* Has this client_id already been used and is now in one of the dropped lists */ static bool __dropped_instance(sdata_t *sdata, const int64_t id) { - stratum_instance_t *client; + stratum_instance_t *client, *tmp; bool ret = false; - HASH_ITERATE(sdata->disconnected_instances, client) { + HASH_ITER(hh, sdata->disconnected_instances, client, tmp) { if (unlikely(client->id == id)) { ret = true; goto out; @@ -1810,8 +1810,8 @@ static stratum_instance_t *__stratum_add_instance(ckpool_t *ckp, const int64_t i static uint64_t disconnected_sessionid_exists(sdata_t *sdata, const char *sessionid, const int64_t id) { + stratum_instance_t *client, *tmp; uint64_t enonce1_64 = 0, ret = 0; - stratum_instance_t *client; int64_t old_id = 0; int slen; @@ -1828,7 +1828,7 @@ static uint64_t disconnected_sessionid_exists(sdata_t *sdata, const char *sessio hex2bin(&enonce1_64, sessionid, slen); ck_wlock(&sdata->instance_lock); - HASH_ITERATE(sdata->stratum_instances, client) { + HASH_ITER(hh, sdata->stratum_instances, client, tmp) { if (client->id == id) continue; if (client->enonce1_64 == enonce1_64) { @@ -1877,7 +1877,7 @@ static void stratum_broadcast(sdata_t *sdata, json_t *val) { ckpool_t *ckp = sdata->ckp; sdata_t *ckp_sdata = ckp->data; - stratum_instance_t *client; + stratum_instance_t *client, *tmp; ckmsg_t *bulk_send = NULL; time_t now_t = time(NULL); int hard_reconnect = 0; @@ -1890,7 +1890,7 @@ static void stratum_broadcast(sdata_t *sdata, json_t *val) /* Use this locking as an opportunity to test other clients. */ ck_rlock(&ckp_sdata->instance_lock); - HASH_ITERATE(ckp_sdata->stratum_instances, client) { + HASH_ITER(hh, ckp_sdata->stratum_instances, client, tmp) { ckmsg_t *client_msg; smsg_t *msg; @@ -2011,7 +2011,7 @@ static void stratum_broadcast_message(sdata_t *sdata, const char *msg) static void request_reconnect(sdata_t *sdata, const char *cmd) { char *port = strdupa(cmd), *url = NULL; - stratum_instance_t *client; + stratum_instance_t *client, *tmp; json_t *json_msg; strsep(&port, ":"); @@ -2031,7 +2031,7 @@ static void request_reconnect(sdata_t *sdata, const char *cmd) /* Tag all existing clients as dropped now so they can be removed * lazily */ ck_wlock(&sdata->instance_lock); - HASH_ITERATE(sdata->stratum_instances, client) { + HASH_ITER(hh, sdata->stratum_instances, client, tmp) { client->dropped = true; } ck_wunlock(&sdata->instance_lock); @@ -2039,14 +2039,14 @@ static void request_reconnect(sdata_t *sdata, const char *cmd) static void reset_bestshares(sdata_t *sdata) { - stratum_instance_t *client; - user_instance_t *user; + user_instance_t *user, *tmpuser; + stratum_instance_t *client, *tmp; ck_rlock(&sdata->instance_lock); - HASH_ITERATE(sdata->stratum_instances, client) { + HASH_ITER(hh, sdata->stratum_instances, client, tmp) { client->best_diff = 0; } - HASH_ITERATE(sdata->user_instances, user) { + HASH_ITER(hh, sdata->user_instances, user, tmpuser) { worker_instance_t *worker; user->best_diff = 0; @@ -2569,7 +2569,7 @@ static void stratum_send_message(sdata_t *sdata, const stratum_instance_t *clien * running out of room. */ static sdata_t *select_sdata(const ckpool_t *ckp, sdata_t *ckp_sdata) { - proxy_t *current, *proxy, *subproxy, *best = NULL; + proxy_t *current, *proxy, *subproxy, *best = NULL, *tmp, *tmpsub; int best_subid = 0, best_id; if (!ckp->proxy || ckp->passthrough) @@ -2582,12 +2582,12 @@ static sdata_t *select_sdata(const ckpool_t *ckp, sdata_t *ckp_sdata) best_id = maxint; mutex_lock(&ckp_sdata->proxy_lock); - HASH_ITERATE(ckp_sdata->proxies, proxy) { + HASH_ITER(hh, ckp_sdata->proxies, proxy, tmp) { int64_t max_headroom; best = NULL; proxy->headroom = max_headroom = 0; - HASH_FOREACH(sh, proxy->subproxies, subproxy) { + HASH_ITER(sh, proxy->subproxies, subproxy, tmpsub) { int64_t subproxy_headroom; if (subproxy->dead) @@ -4618,7 +4618,7 @@ out: static void update_workerstats(ckpool_t *ckp, sdata_t *sdata) { json_entry_t *json_list = NULL, *entry, *tmpentry; - user_instance_t *user; + user_instance_t *user, *tmp; char cdfield[64]; time_t now_t; ts_t ts_now; @@ -4636,7 +4636,7 @@ static void update_workerstats(ckpool_t *ckp, sdata_t *sdata) now_t = ts_now.tv_sec; ck_rlock(&sdata->instance_lock); - HASH_ITERATE(sdata->user_instances, user) { + HASH_ITER(hh, sdata->user_instances, user, tmp) { worker_instance_t *worker; uint8_t cycle_mask; @@ -4740,10 +4740,10 @@ static void *statsupdate(void *arg) double ghs, ghs1, ghs5, ghs15, ghs60, ghs360, ghs1440, ghs10080, per_tdiff; char suffix1[16], suffix5[16], suffix15[16], suffix60[16], cdfield[64]; char suffix360[16], suffix1440[16], suffix10080[16]; + stratum_instance_t *client, *tmp; log_entry_t *log_entries = NULL; + user_instance_t *user, *tmpuser; char_entry_t *char_list = NULL; - stratum_instance_t *client; - user_instance_t *user; int idle_workers = 0; char *fname, *s, *sp; tv_t now, diff; @@ -4756,7 +4756,7 @@ static void *statsupdate(void *arg) timersub(&now, &stats->start_time, &diff); ck_rlock(&sdata->instance_lock); - HASH_ITERATE(sdata->stratum_instances, client) { + HASH_ITER(hh, sdata->stratum_instances, client, tmp) { if (!client_active(client)) continue; @@ -4776,7 +4776,7 @@ static void *statsupdate(void *arg) } } - HASH_ITERATE(sdata->user_instances, user) { + HASH_ITER(hh, sdata->user_instances, user, tmpuser) { worker_instance_t *worker; bool idle = false; @@ -4942,7 +4942,7 @@ static void *statsupdate(void *arg) fclose(fp); if (ckp->proxy && sdata->proxy) { - proxy_t *proxy, *subproxy; + proxy_t *proxy, *proxytmp, *subproxy, *subtmp; mutex_lock(&sdata->proxy_lock); JSON_CPACK(val, "{sI,si,si}", @@ -4957,7 +4957,7 @@ static void *statsupdate(void *arg) dealloc(s); mutex_lock(&sdata->proxy_lock); - HASH_ITERATE(sdata->proxies, proxy) { + HASH_ITER(hh, sdata->proxies, proxy, proxytmp) { JSON_CPACK(val, "{sI,si,sI,sb}", "id", proxy->id, "subproxies", proxy->subproxy_count, @@ -4968,7 +4968,7 @@ static void *statsupdate(void *arg) ASPRINTF(&sp, "Proxies:%s", s); dealloc(s); add_msg_entry(&char_list, &sp); - HASH_FOREACH(sh, proxy->subproxies, subproxy) { + HASH_ITER(sh, proxy->subproxies, subproxy, subtmp) { JSON_CPACK(val, "{sI,si,si,sI,sI,sf,sb}", "id", subproxy->id, "subid", subproxy->subid, From 5517a211714ab938b28a443ad97c7be82a74843f Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Fri, 20 Mar 2015 10:27:34 +1100 Subject: [PATCH 354/544] Use the priority sorting to speed up best proxy selection --- src/stratifier.c | 41 ++++++++++++----------------------------- 1 file changed, 12 insertions(+), 29 deletions(-) diff --git a/src/stratifier.c b/src/stratifier.c index 93303e69..ec2f97e5 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -1269,7 +1269,8 @@ static bool __subproxies_alive(proxy_t *proxy) } /* Iterate over the current global proxy list and see if the current one is - * the highest priority alive one. Uses ckp sdata */ + * the highest priority alive one. Proxies are sorted by priority so the first + * available will be highest priority. Uses ckp sdata */ static void check_bestproxy(sdata_t *sdata) { proxy_t *proxy, *tmp; @@ -1281,10 +1282,11 @@ static void check_bestproxy(sdata_t *sdata) HASH_ITER(hh, sdata->proxies, proxy, tmp) { if (!__subproxies_alive(proxy)) continue; - if (!sdata->proxy || sdata->proxy->id > proxy->id) { + if (proxy != sdata->proxy) { sdata->proxy = proxy; changed_id = proxy->id; } + break; } mutex_unlock(&sdata->proxy_lock); @@ -1394,12 +1396,6 @@ static void update_subscribe(ckpool_t *ckp, const char *cmd) proxy->clients = 0; ck_wunlock(&dsdata->workbase_lock); - /* Is this a replacement proxy for the current one */ - mutex_lock(&sdata->proxy_lock); - if (sdata->proxy && sdata->proxy->id == proxy->id && !proxy->subid) - sdata->proxy = proxy; - mutex_unlock(&sdata->proxy_lock); - if (subid) { LOGINFO("Upstream pool %d:%d extranonce2 length %d, max proxy clients %"PRId64, id, subid, proxy->nonce2len, proxy->max_clients); @@ -1414,10 +1410,10 @@ static void update_notify(ckpool_t *ckp, const char *cmd) { sdata_t *sdata = ckp->data, *dsdata; bool new_block = false, clean; - proxy_t *proxy, *current; int i, id = 0, subid = 0; char header[228]; const char *buf; + proxy_t *proxy; workbase_t *wb; json_t *val; @@ -1502,12 +1498,6 @@ static void update_notify(ckpool_t *ckp, const char *cmd) LOGNOTICE("Block hash on proxy %d changed to %s", id, dsdata->lastswaphash); } - mutex_lock(&sdata->proxy_lock); - current = sdata->proxy; - if (unlikely(!current)) - current = sdata->proxy = proxy; - mutex_unlock(&sdata->proxy_lock); - check_bestproxy(sdata); clean |= new_block; LOGINFO("Proxy %d:%d broadcast updated stratum notify with%s clean", id, @@ -2561,8 +2551,6 @@ static bool new_enonce1(ckpool_t *ckp, sdata_t *ckp_sdata, sdata_t *sdata, strat static void stratum_send_message(sdata_t *sdata, const stratum_instance_t *client, const char *msg); -#define maxint 2147483647 - /* Choose the stratifier data for a new client. Use the main ckp_sdata except * in proxy mode where we find a subproxy based on the current proxy with room * for more clients. Signal the generator to recruit more subproxies if we are @@ -2570,7 +2558,6 @@ static void stratum_send_message(sdata_t *sdata, const stratum_instance_t *clien static sdata_t *select_sdata(const ckpool_t *ckp, sdata_t *ckp_sdata) { proxy_t *current, *proxy, *subproxy, *best = NULL, *tmp, *tmpsub; - int best_subid = 0, best_id; if (!ckp->proxy || ckp->passthrough) return ckp_sdata; @@ -2579,8 +2566,9 @@ static sdata_t *select_sdata(const ckpool_t *ckp, sdata_t *ckp_sdata) LOGWARNING("No proxy available yet to generate subscribes"); return NULL; } - best_id = maxint; + /* Proxies are ordered by priority so first available will be the best + * priority */ mutex_lock(&ckp_sdata->proxy_lock); HASH_ITER(hh, ckp_sdata->proxies, proxy, tmp) { int64_t max_headroom; @@ -2602,22 +2590,17 @@ static sdata_t *select_sdata(const ckpool_t *ckp, sdata_t *ckp_sdata) max_headroom = subproxy_headroom; } } - if (best && best->id < best_id) { - best_id = best->id; - best_subid = best->subid; - if (proxy == current) - break; - } + if (best) + break; } mutex_unlock(&ckp_sdata->proxy_lock); - if (best_id != current->id || current->headroom < 2) - generator_recruit(ckp, 1); - if (best_id == maxint) { + if (!best) { LOGWARNING("Temporarily insufficient subproxies to accept more clients"); return NULL; } - best = subproxy_by_id(ckp_sdata, best_id, best_subid); + if (best->id != current->id || current->headroom < 2) + generator_recruit(ckp, 1); return best->sdata; } From a5a5487023fb1d987c6f19e8e6327f4ea472d4d8 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Fri, 20 Mar 2015 10:27:40 +1100 Subject: [PATCH 355/544] Revert "Add uthash helpers for iterating over hash entries without the need for safe against deletion" This reverts commit 9a1e59637f7c2f98be1af55b7019238626b0c40a. Broken. --- src/uthash.h | 7 ------- 1 file changed, 7 deletions(-) diff --git a/src/uthash.h b/src/uthash.h index ce61e339..31db3fa8 100644 --- a/src/uthash.h +++ b/src/uthash.h @@ -869,21 +869,14 @@ do { (sizeof(UT_hash_table)) + \ (HASH_BLOOM_BYTELEN))) -/* HASH_ITERATE and HASH_FOREACH are not safe for deletion */ -#define HASH_ITERATE(head,el) HASH_ITERATE2(hh,head,el) -#define HASH_FOREACH(hh,head,el) HASH_ITERATE2(hh,head,el) #ifdef NO_DECLTYPE #define HASH_ITER(hh,head,el,tmp) \ for((el)=(head), (*(char**)(&(tmp)))=(char*)((head)?(head)->hh.next:NULL); \ el; (el)=(tmp),(*(char**)(&(tmp)))=(char*)((tmp)?(tmp)->hh.next:NULL)) -#define HASH_ITERATE2(hh,head,el) \ -for ((el)=(head); ((head)!= NULL); (head)=(head)->hh.next) #else #define HASH_ITER(hh,head,el,tmp) \ for((el)=(head),(tmp)=DECLTYPE(el)((head)?(head)->hh.next:NULL); \ el; (el)=(tmp),(tmp)=DECLTYPE(el)((tmp)?(tmp)->hh.next:NULL)) -#define HASH_ITERATE2(hh,head,el) \ -for ((el)=(head); ((head)!= NULL); (head)=DECLTYPE(el)(head)->hh.next) #endif /* obtain a count of items in the hash */ From fd7d62df15aa07e7f89849432102886b2bff50b9 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Fri, 20 Mar 2015 10:36:29 +1100 Subject: [PATCH 356/544] Send the proxy details from the generator to the stratifier --- src/generator.c | 9 ++++----- src/stratifier.c | 17 +++++++++++++---- 2 files changed, 17 insertions(+), 9 deletions(-) diff --git a/src/generator.c b/src/generator.c index d0f52643..1d1e4cc9 100644 --- a/src/generator.c +++ b/src/generator.c @@ -1303,11 +1303,10 @@ static void send_subscribe(ckpool_t *ckp, proxy_instance_t *proxi) json_t *json_msg; char *msg, *buf; - JSON_CPACK(json_msg, "{sIsisssi}", - "proxy", proxi->id, - "subproxy", proxi->subid, - "enonce1", proxi->enonce1, - "nonce2len", proxi->nonce2len); + JSON_CPACK(json_msg, "{ss,ss,ss,sI,si,ss,si}", + "url", proxi->url, "auth", proxi->auth, "pass", proxi->pass, + "proxy", proxi->id, "subproxy", proxi->subid, + "enonce1", proxi->enonce1, "nonce2len", proxi->nonce2len); msg = json_dumps(json_msg, JSON_NO_UTF8); json_decref(json_msg); ASPRINTF(&buf, "subscribe=%s", msg); diff --git a/src/stratifier.c b/src/stratifier.c index ec2f97e5..422e3203 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -302,6 +302,9 @@ struct proxy_base { double diff; + char url[128]; + char auth[128]; + char pass[128]; char enonce1[32]; uchar enonce1bin[16]; int enonce1constlen; @@ -1378,6 +1381,12 @@ static void update_subscribe(ckpool_t *ckp, const char *cmd) ck_wlock(&dsdata->workbase_lock); proxy->subscribed = true; proxy->diff = ckp->startdiff; + memset(proxy->url, 0, 128); + memset(proxy->auth, 0, 128); + memset(proxy->pass, 0, 128); + strncpy(proxy->url, json_string_value(json_object_get(val, "url")), 127); + strncpy(proxy->auth, json_string_value(json_object_get(val, "auth")), 127); + strncpy(proxy->pass, json_string_value(json_object_get(val, "pass")), 127); /* Length is checked by generator */ strcpy(proxy->enonce1, json_string_value(json_object_get(val, "enonce1"))); proxy->enonce1constlen = strlen(proxy->enonce1) / 2; @@ -1397,11 +1406,11 @@ static void update_subscribe(ckpool_t *ckp, const char *cmd) ck_wunlock(&dsdata->workbase_lock); if (subid) { - LOGINFO("Upstream pool %d:%d extranonce2 length %d, max proxy clients %"PRId64, - id, subid, proxy->nonce2len, proxy->max_clients); + LOGINFO("Upstream pool %s %d:%d extranonce2 length %d, max proxy clients %"PRId64, + proxy->url, id, subid, proxy->nonce2len, proxy->max_clients); } else { - LOGNOTICE("Upstream pool %d extranonce2 length %d, max proxy clients %"PRId64, - id, proxy->nonce2len, proxy->max_clients); + LOGNOTICE("Upstream pool %s %d extranonce2 length %d, max proxy clients %"PRId64, + proxy->url, id, proxy->nonce2len, proxy->max_clients); } json_decref(val); } From d7f09124a610657b4cbad922992ac0fcfbb09c3f Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Fri, 20 Mar 2015 11:02:02 +1100 Subject: [PATCH 357/544] Add an API command to get proxy details --- src/stratifier.c | 55 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) diff --git a/src/stratifier.c b/src/stratifier.c index 422e3203..6f707e7e 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -1129,6 +1129,17 @@ static proxy_t *__existing_proxy(const sdata_t *sdata, const int id) return proxy; } +static proxy_t *existing_proxy(sdata_t *sdata, const int id) +{ + proxy_t *proxy; + + mutex_lock(&sdata->proxy_lock); + proxy = __existing_proxy(sdata, id); + mutex_unlock(&sdata->proxy_lock); + + return proxy; +} + /* Find proxy by id number, generate one if none exist yet by that id */ static proxy_t *__proxy_by_id(sdata_t *sdata, const int id) { @@ -2325,6 +2336,46 @@ out: _Close(sockd); } +static void getproxy(sdata_t *sdata, const char *buf, int *sockd) +{ + proxy_t *proxy; + json_error_t err_val; + json_t *val = NULL; + int id, subid = 0; + + val = json_loads(buf, 0, &err_val); + if (unlikely(!val)) { + val = json_encode_errormsg(&err_val); + goto out; + } + if (!json_get_int(&id, val, "id")) { + val = json_errormsg("Failed to find id key"); + goto out; + } + json_get_int(&subid, val, "subid"); + if (!subid) + proxy = existing_proxy(sdata, id); + else + proxy = existing_subproxy(sdata, id, subid); + if (!proxy) { + val = json_errormsg("Failed to find proxy %d:%d", id, subid); + goto out; + } + JSON_CPACK(val, "{si,si,si,sf,ss,ss,ss,ss,si,si,si,si,sb,sb,sI,sI,sI,sI,si,sb}", + "id", proxy->id, "subid", proxy->subid, "priority", proxy->priority, + "diff", proxy->diff, "url", proxy->url, "auth", proxy->auth, "pass", proxy->pass, + "enonce1", proxy->enonce1, "enonce1constlen", proxy->enonce1constlen, + "enonce1varlen", proxy->enonce1varlen, "nonce2len", proxy->nonce2len, + "enonce2varlen", proxy->enonce2varlen, "subscribed", proxy->subscribed, + "notified", proxy->notified, "clients", proxy->clients, "max_clients", proxy->max_clients, + "bound_clients", proxy->bound_clients, "combined_clients", proxy->combined_clients, + "headroom", proxy->headroom, "subproxy_count", proxy->subproxy_count, + "dead", proxy->dead); +out: + send_api_response(val, *sockd); + _Close(sockd); +} + static void reconnect_client_id(sdata_t *sdata, const int64_t client_id) { stratum_instance_t *client; @@ -2416,6 +2467,10 @@ retry: getuser(sdata, buf + 8, &sockd); goto retry; } + if (cmdmatch(buf, "getproxy")) { + getproxy(sdata, buf + 9, &sockd); + goto retry; + } Close(sockd); LOGDEBUG("Stratifier received request: %s", buf); From b54c650d6a3c5a1c31a299175fda60d19b01b142 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Fri, 20 Mar 2015 11:13:30 +1100 Subject: [PATCH 358/544] Give parent proxy details where appropriate in getproxy --- src/stratifier.c | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/stratifier.c b/src/stratifier.c index 6f707e7e..0b9068d8 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -2338,7 +2338,7 @@ out: static void getproxy(sdata_t *sdata, const char *buf, int *sockd) { - proxy_t *proxy; + proxy_t *proxy, *parent; json_error_t err_val; json_t *val = NULL; int id, subid = 0; @@ -2361,15 +2361,16 @@ static void getproxy(sdata_t *sdata, const char *buf, int *sockd) val = json_errormsg("Failed to find proxy %d:%d", id, subid); goto out; } + parent = proxy->parent; JSON_CPACK(val, "{si,si,si,sf,ss,ss,ss,ss,si,si,si,si,sb,sb,sI,sI,sI,sI,si,sb}", - "id", proxy->id, "subid", proxy->subid, "priority", proxy->priority, + "id", proxy->id, "subid", proxy->subid, "priority", parent->priority, "diff", proxy->diff, "url", proxy->url, "auth", proxy->auth, "pass", proxy->pass, "enonce1", proxy->enonce1, "enonce1constlen", proxy->enonce1constlen, "enonce1varlen", proxy->enonce1varlen, "nonce2len", proxy->nonce2len, "enonce2varlen", proxy->enonce2varlen, "subscribed", proxy->subscribed, "notified", proxy->notified, "clients", proxy->clients, "max_clients", proxy->max_clients, - "bound_clients", proxy->bound_clients, "combined_clients", proxy->combined_clients, - "headroom", proxy->headroom, "subproxy_count", proxy->subproxy_count, + "bound_clients", proxy->bound_clients, "combined_clients", parent->combined_clients, + "headroom", proxy->headroom, "subproxy_count", parent->subproxy_count, "dead", proxy->dead); out: send_api_response(val, *sockd); From 6a7fc728a4c100b1b8873cba93ad3e77173290ed Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Fri, 20 Mar 2015 12:12:49 +1100 Subject: [PATCH 359/544] Add an API command to set global proxy priority and do the switch --- src/stratifier.c | 106 +++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 93 insertions(+), 13 deletions(-) diff --git a/src/stratifier.c b/src/stratifier.c index 0b9068d8..7936599b 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -1088,6 +1088,33 @@ static int prio_sort(proxy_t *a, proxy_t *b) return (a->priority - b->priority); } +/* Priority values can be sparse, they do not need to be sequential */ +static void __set_proxy_prio(sdata_t *sdata, proxy_t *proxy, const int priority) +{ + proxy_t *tmpa, *tmpb, *exists = NULL; + int next_prio = 0; + + /* See if the priority is already in use */ + HASH_ITER(hh, sdata->proxies, tmpa, tmpb) { + if (tmpa->priority > priority) + break; + if (tmpa->priority == priority) { + exists = tmpa; + next_prio = exists->priority + 1; + break; + } + } + /* See if we need to push the priority of everything after exists up */ + HASH_ITER(hh, exists, tmpa, tmpb) { + if (tmpa->priority > next_prio) + break; + tmpa->priority++; + next_prio++; + } + proxy->priority = priority; + HASH_SORT(sdata->proxies, prio_sort); +} + static proxy_t *__generate_proxy(sdata_t *sdata, const int id) { proxy_t *proxy = ckzalloc(sizeof(proxy_t)); @@ -1102,7 +1129,8 @@ static proxy_t *__generate_proxy(sdata_t *sdata, const int id) HASH_ADD(sh, proxy->subproxies, subid, sizeof(int), proxy); proxy->subproxy_count++; HASH_ADD_INT(sdata->proxies, id, proxy); - HASH_SORT(sdata->proxies, prio_sort); + /* Set the new proxy priority to its id */ + __set_proxy_prio(sdata, proxy, id); sdata->proxy_count++; return proxy; } @@ -1197,6 +1225,13 @@ static proxy_t *existing_subproxy(sdata_t *sdata, const int id, const int subid) return subproxy; } +static void set_proxy_prio(sdata_t *sdata, proxy_t *proxy, const int priority) +{ + mutex_lock(&sdata->proxy_lock); + __set_proxy_prio(sdata, proxy, priority); + mutex_unlock(&sdata->proxy_lock); +} + /* Set proxy to the current proxy and calculate how much headroom it has */ static int64_t current_headroom(sdata_t *sdata, proxy_t **proxy) { @@ -2336,12 +2371,30 @@ out: _Close(sockd); } +static json_t *json_proxyinfo(const proxy_t *proxy) +{ + const proxy_t *parent = proxy->parent; + json_t *val; + + JSON_CPACK(val, "{si,si,si,sf,ss,ss,ss,ss,si,si,si,si,sb,sb,sI,sI,sI,sI,si,sb}", + "id", proxy->id, "subid", proxy->subid, "priority", parent->priority, + "diff", proxy->diff, "url", proxy->url, "auth", proxy->auth, "pass", proxy->pass, + "enonce1", proxy->enonce1, "enonce1constlen", proxy->enonce1constlen, + "enonce1varlen", proxy->enonce1varlen, "nonce2len", proxy->nonce2len, + "enonce2varlen", proxy->enonce2varlen, "subscribed", proxy->subscribed, + "notified", proxy->notified, "clients", proxy->clients, "max_clients", proxy->max_clients, + "bound_clients", proxy->bound_clients, "combined_clients", parent->combined_clients, + "headroom", proxy->headroom, "subproxy_count", parent->subproxy_count, + "dead", proxy->dead); + return val; +} + static void getproxy(sdata_t *sdata, const char *buf, int *sockd) { - proxy_t *proxy, *parent; json_error_t err_val; json_t *val = NULL; int id, subid = 0; + proxy_t *proxy; val = json_loads(buf, 0, &err_val); if (unlikely(!val)) { @@ -2361,17 +2414,40 @@ static void getproxy(sdata_t *sdata, const char *buf, int *sockd) val = json_errormsg("Failed to find proxy %d:%d", id, subid); goto out; } - parent = proxy->parent; - JSON_CPACK(val, "{si,si,si,sf,ss,ss,ss,ss,si,si,si,si,sb,sb,sI,sI,sI,sI,si,sb}", - "id", proxy->id, "subid", proxy->subid, "priority", parent->priority, - "diff", proxy->diff, "url", proxy->url, "auth", proxy->auth, "pass", proxy->pass, - "enonce1", proxy->enonce1, "enonce1constlen", proxy->enonce1constlen, - "enonce1varlen", proxy->enonce1varlen, "nonce2len", proxy->nonce2len, - "enonce2varlen", proxy->enonce2varlen, "subscribed", proxy->subscribed, - "notified", proxy->notified, "clients", proxy->clients, "max_clients", proxy->max_clients, - "bound_clients", proxy->bound_clients, "combined_clients", parent->combined_clients, - "headroom", proxy->headroom, "subproxy_count", parent->subproxy_count, - "dead", proxy->dead); + val = json_proxyinfo(proxy); +out: + send_api_response(val, *sockd); + _Close(sockd); +} + +static void setproxy(sdata_t *sdata, const char *buf, int *sockd) +{ + json_error_t err_val; + json_t *val = NULL; + int id, priority; + proxy_t *proxy; + + val = json_loads(buf, 0, &err_val); + if (unlikely(!val)) { + val = json_encode_errormsg(&err_val); + goto out; + } + if (!json_get_int(&id, val, "id")) { + val = json_errormsg("Failed to find id key"); + goto out; + } + if (!json_get_int(&priority, val, "priority")) { + val = json_errormsg("Failed to find priority key"); + goto out; + } + proxy = existing_proxy(sdata, id); + if (!proxy) { + val = json_errormsg("Failed to find proxy %d", id); + goto out; + } + if (priority != proxy->priority) + set_proxy_prio(sdata, proxy, priority); + val = json_proxyinfo(proxy); out: send_api_response(val, *sockd); _Close(sockd); @@ -2472,6 +2548,10 @@ retry: getproxy(sdata, buf + 9, &sockd); goto retry; } + if (cmdmatch(buf, "setproxy")) { + setproxy(sdata, buf + 9, &sockd); + goto retry; + } Close(sockd); LOGDEBUG("Stratifier received request: %s", buf); From 9f3a1bfb4d41f24c84227b23a5738c2ebcec97da Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Fri, 20 Mar 2015 12:40:20 +1100 Subject: [PATCH 360/544] Add userid parsing to proxy.add command --- src/generator.c | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/src/generator.c b/src/generator.c index 1d1e4cc9..fbb67e4d 100644 --- a/src/generator.c +++ b/src/generator.c @@ -2182,7 +2182,8 @@ static void parse_addproxy(ckpool_t *ckp, gdata_t *gdata, const int sockd, const proxy_instance_t *proxy; json_error_t err_val; json_t *val = NULL; - int id; + int id, userid; + bool global; val = json_loads(buf, 0, &err_val); if (unlikely(!val)) { @@ -2197,6 +2198,10 @@ static void parse_addproxy(ckpool_t *ckp, gdata_t *gdata, const int sockd, const val = json_errormsg("Failed to decode url/auth/pass in addproxy %s", buf); goto out; } + if (json_get_int(&userid, val, "userid")) + global = false; + else + global = true; mutex_lock(&gdata->lock); id = ckp->proxies++; @@ -2209,7 +2214,10 @@ static void parse_addproxy(ckpool_t *ckp, gdata_t *gdata, const int sockd, const proxy = __add_proxy(ckp, gdata, id); mutex_unlock(&gdata->lock); - LOGNOTICE("Adding proxy %d:%s", id, proxy->url); + if (global) + LOGNOTICE("Adding global proxy %d:%s", id, proxy->url); + else + LOGNOTICE("Adding user %d proxy %d:%s", userid, id, proxy->url); prepare_proxy(proxy); JSON_CPACK(val, "{si,ss,ss,ss}", "id", proxy->id, "url", url, "auth", auth, "pass", pass); From 9c60f55b6163c1147de255dd4fcc3c9819343d40 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Fri, 20 Mar 2015 13:54:33 +1100 Subject: [PATCH 361/544] Add preliminary structures required to add user proxies to the generator --- src/generator.c | 83 +++++++++++++++++++++++++++++++++++++------------ 1 file changed, 63 insertions(+), 20 deletions(-) diff --git a/src/generator.c b/src/generator.c index fbb67e4d..8613f8c1 100644 --- a/src/generator.c +++ b/src/generator.c @@ -86,6 +86,7 @@ struct proxy_instance { bool passthrough; int id; /* Proxy server id*/ int subid; /* Subproxy id */ + int userid; /* User id if this proxy is bound to a user */ char *url; char *auth; @@ -137,8 +138,10 @@ struct proxy_instance { struct generator_data { ckpool_t *ckp; mutex_t lock; /* Lock protecting linked lists */ - proxy_instance_t *proxies; /* Hash list of all proxies */ + proxy_instance_t *proxies; /* Hash list of all global proxies */ + proxy_instance_t *user_proxies; /* Hash list of all user proxies */ proxy_instance_t *dead_proxies; /* Disabled proxies */ + int proxy_notify_id; // Globally increasing notify id ckmsgq_t *srvchk; // Server check message queue pthread_t pth_uprecv; // User proxy receive thread @@ -919,13 +922,15 @@ static proxy_instance_t *create_subproxy(gdata_t *gdata, proxy_instance_t *proxi } mutex_unlock(&gdata->lock); - subproxy->cs.ckp = subproxy->ckp = proxi->ckp; + subproxy->ckp = proxi->ckp; mutex_lock(&proxi->proxy_lock); subproxy->subid = ++proxi->subproxy_count; mutex_unlock(&proxi->proxy_lock); subproxy->id = proxi->id; + subproxy->userid = proxi->userid; + subproxy->global = proxi->global; subproxy->url = strdup(url); subproxy->auth = strdup(proxi->auth); subproxy->pass = strdup(proxi->pass); @@ -1303,10 +1308,11 @@ static void send_subscribe(ckpool_t *ckp, proxy_instance_t *proxi) json_t *json_msg; char *msg, *buf; - JSON_CPACK(json_msg, "{ss,ss,ss,sI,si,ss,si}", + JSON_CPACK(json_msg, "{ss,ss,ss,sI,si,ss,si,sb,si}", "url", proxi->url, "auth", proxi->auth, "pass", proxi->pass, "proxy", proxi->id, "subproxy", proxi->subid, - "enonce1", proxi->enonce1, "nonce2len", proxi->nonce2len); + "enonce1", proxi->enonce1, "nonce2len", proxi->nonce2len, + "global", proxi->global, "userid", proxi->userid); msg = json_dumps(json_msg, JSON_NO_UTF8); json_decref(json_msg); ASPRINTF(&buf, "subscribe=%s", msg); @@ -2002,13 +2008,22 @@ static void *userproxy_recv(void *arg) } while (42) { - proxy_instance_t *proxy, *parent; + proxy_instance_t *proxy, *tmpproxy, *parent; share_msg_t *share, *tmpshare; notify_instance_t *ni, *tmp; connsock_t *cs; time_t now; int ret; + mutex_lock(&gdata->lock); + HASH_ITER(hh, gdata->user_proxies, proxy, tmpproxy) { + if (!proxy->alive) { + proxy->epfd = epfd; + reconnect_proxy(proxy); + } + } + mutex_unlock(&gdata->lock); + ret = epoll_wait(epfd, &event, 1, 1000); if (ret < 1) { if (likely(!ret)) @@ -2066,7 +2081,8 @@ static void prepare_proxy(proxy_instance_t *proxi) proxi->parent = proxi; mutex_init(&proxi->proxy_lock); add_subproxy(proxi, proxi); - create_pthread(&proxi->pth_precv, proxy_recv, proxi); + if (proxi->global) + create_pthread(&proxi->pth_precv, proxy_recv, proxi); } static proxy_instance_t *wait_best_proxy(ckpool_t *ckp, gdata_t *gdata) @@ -2176,6 +2192,24 @@ out: static proxy_instance_t *__add_proxy(ckpool_t *ckp, gdata_t *gdata, const int num); +static proxy_instance_t *__add_userproxy(ckpool_t *ckp, gdata_t *gdata, const int id, + const int userid, char *url, char *auth, char *pass) +{ + proxy_instance_t *proxy; + + proxy = ckzalloc(sizeof(proxy_instance_t)); + proxy->id = id; + proxy->userid = userid; + proxy->url = url; + proxy->auth = auth; + proxy->pass = pass; + proxy->ckp = ckp; + mutex_init(&proxy->notify_lock); + mutex_init(&proxy->share_lock); + HASH_ADD_INT(gdata->proxies, id, proxy); + return proxy; +} + static void parse_addproxy(ckpool_t *ckp, gdata_t *gdata, const int sockd, const char *buf) { char *url = NULL, *auth = NULL, *pass = NULL; @@ -2193,25 +2227,28 @@ static void parse_addproxy(ckpool_t *ckp, gdata_t *gdata, const int sockd, const json_get_string(&url, val, "url"); json_get_string(&auth, val, "auth"); json_get_string(&pass, val, "pass"); + if (json_get_int(&userid, val, "userid")) + global = false; + else + global = true; json_decref(val); if (unlikely(!url || !auth || !pass)) { val = json_errormsg("Failed to decode url/auth/pass in addproxy %s", buf); goto out; } - if (json_get_int(&userid, val, "userid")) - global = false; - else - global = true; mutex_lock(&gdata->lock); id = ckp->proxies++; - ckp->proxyurl = realloc(ckp->proxyurl, sizeof(char **) * ckp->proxies); - ckp->proxyauth = realloc(ckp->proxyauth, sizeof(char **) * ckp->proxies); - ckp->proxypass = realloc(ckp->proxypass, sizeof(char **) * ckp->proxies); - ckp->proxyurl[id] = strdup(url); - ckp->proxyauth[id] = strdup(auth); - ckp->proxypass[id] = strdup(pass); - proxy = __add_proxy(ckp, gdata, id); + if (global) { + ckp->proxyurl = realloc(ckp->proxyurl, sizeof(char **) * ckp->proxies); + ckp->proxyauth = realloc(ckp->proxyauth, sizeof(char **) * ckp->proxies); + ckp->proxypass = realloc(ckp->proxypass, sizeof(char **) * ckp->proxies); + ckp->proxyurl[id] = url; + ckp->proxyauth[id] = auth; + ckp->proxypass[id] = pass; + proxy = __add_proxy(ckp, gdata, id); + } else + proxy = __add_userproxy(ckp, gdata, id, userid, url, auth, pass); mutex_unlock(&gdata->lock); if (global) @@ -2219,8 +2256,14 @@ static void parse_addproxy(ckpool_t *ckp, gdata_t *gdata, const int sockd, const else LOGNOTICE("Adding user %d proxy %d:%s", userid, id, proxy->url); prepare_proxy(proxy); - JSON_CPACK(val, "{si,ss,ss,ss}", - "id", proxy->id, "url", url, "auth", auth, "pass", pass); + if (global) { + JSON_CPACK(val, "{si,ss,ss,ss}", + "id", proxy->id, "url", url, "auth", auth, "pass", pass); + } else { + JSON_CPACK(val, "{si,ss,ss,ss,si}", + "id", proxy->id, "url", url, "auth", auth, "pass", pass, + "userid", proxy->userid); + } out: send_api_response(val, sockd); } @@ -2447,10 +2490,10 @@ static proxy_instance_t *__add_proxy(ckpool_t *ckp, gdata_t *gdata, const int id proxy->auth = strdup(ckp->proxyauth[id]); proxy->pass = strdup(ckp->proxypass[id]); proxy->ckp = ckp; - proxy->cs.ckp = ckp; mutex_init(&proxy->notify_lock); mutex_init(&proxy->share_lock); HASH_ADD_INT(gdata->proxies, id, proxy); + proxy->global = true; return proxy; } From 06489267293a150428f91aa381d9cdc8645337fa Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Fri, 20 Mar 2015 14:02:43 +1100 Subject: [PATCH 362/544] Add user proxies to their own hashlist --- src/generator.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/generator.c b/src/generator.c index 8613f8c1..473f265e 100644 --- a/src/generator.c +++ b/src/generator.c @@ -2206,7 +2206,7 @@ static proxy_instance_t *__add_userproxy(ckpool_t *ckp, gdata_t *gdata, const in proxy->ckp = ckp; mutex_init(&proxy->notify_lock); mutex_init(&proxy->share_lock); - HASH_ADD_INT(gdata->proxies, id, proxy); + HASH_ADD_INT(gdata->user_proxies, id, proxy); return proxy; } From 92d1572e0a27af20f8c59899d04e6c35343cf698 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Fri, 20 Mar 2015 14:06:38 +1100 Subject: [PATCH 363/544] Search the user proxy list when looking up proxy_by_id --- src/generator.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/generator.c b/src/generator.c index 473f265e..bf2b5cc7 100644 --- a/src/generator.c +++ b/src/generator.c @@ -1298,6 +1298,8 @@ static proxy_instance_t *proxy_by_id(gdata_t *gdata, const int id) mutex_lock(&gdata->lock); HASH_FIND_INT(gdata->proxies, &id, proxi); + if (!proxi) + HASH_FIND_INT(gdata->user_proxies, &id, proxi); mutex_unlock(&gdata->lock); return proxi; From 1818d867a0a2fd7038da2aa8aef6f79c115f3c1d Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Fri, 20 Mar 2015 14:20:00 +1100 Subject: [PATCH 364/544] Use the one list for proxies in the generator, separating only when needed --- src/generator.c | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/src/generator.c b/src/generator.c index bf2b5cc7..4bb4f973 100644 --- a/src/generator.c +++ b/src/generator.c @@ -138,8 +138,7 @@ struct proxy_instance { struct generator_data { ckpool_t *ckp; mutex_t lock; /* Lock protecting linked lists */ - proxy_instance_t *proxies; /* Hash list of all global proxies */ - proxy_instance_t *user_proxies; /* Hash list of all user proxies */ + proxy_instance_t *proxies; /* Hash list of all proxies */ proxy_instance_t *dead_proxies; /* Disabled proxies */ int proxy_notify_id; // Globally increasing notify id @@ -1298,8 +1297,6 @@ static proxy_instance_t *proxy_by_id(gdata_t *gdata, const int id) mutex_lock(&gdata->lock); HASH_FIND_INT(gdata->proxies, &id, proxi); - if (!proxi) - HASH_FIND_INT(gdata->user_proxies, &id, proxi); mutex_unlock(&gdata->lock); return proxi; @@ -2018,8 +2015,8 @@ static void *userproxy_recv(void *arg) int ret; mutex_lock(&gdata->lock); - HASH_ITER(hh, gdata->user_proxies, proxy, tmpproxy) { - if (!proxy->alive) { + HASH_ITER(hh, gdata->proxies, proxy, tmpproxy) { + if (!proxy->global && !proxy->alive) { proxy->epfd = epfd; reconnect_proxy(proxy); } @@ -2098,7 +2095,7 @@ static proxy_instance_t *wait_best_proxy(ckpool_t *ckp, gdata_t *gdata) mutex_lock(&gdata->lock); HASH_ITER(hh, gdata->proxies, proxi, tmp) { - if (proxi->disabled) + if (proxi->disabled || !proxi->global) continue; if (proxi->alive || subproxies_alive(proxi)) { if (!ret || proxi->id < ret->id) @@ -2128,8 +2125,8 @@ static void send_list(gdata_t *gdata, const int sockd) mutex_lock(&gdata->lock); HASH_ITER(hh, gdata->proxies, proxy, tmp) { - JSON_CPACK(val, "{si,ss,ss,sf,sb,sb,sb,si}", - "id", proxy->id, + JSON_CPACK(val, "{si,sb,si,ss,ss,sf,sb,sb,sb,si}", + "id", proxy->id, "global", proxy->global, "userid", proxy->userid, "auth", proxy->auth, "pass", proxy->pass, "diff", proxy->diff, "notified", proxy->notified, "disabled", proxy->disabled, "alive", proxy->alive, @@ -2208,7 +2205,7 @@ static proxy_instance_t *__add_userproxy(ckpool_t *ckp, gdata_t *gdata, const in proxy->ckp = ckp; mutex_init(&proxy->notify_lock); mutex_init(&proxy->share_lock); - HASH_ADD_INT(gdata->user_proxies, id, proxy); + HASH_ADD_INT(gdata->proxies, id, proxy); return proxy; } From 84a6bf7dfcdc5787fc24371bda9565e752b77ffd Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Fri, 20 Mar 2015 15:25:51 +1100 Subject: [PATCH 365/544] Encode priority into user proxies by using the userid as its high bits --- src/stratifier.c | 64 +++++++++++++++++++++++++++++++++++++----------- 1 file changed, 50 insertions(+), 14 deletions(-) diff --git a/src/stratifier.c b/src/stratifier.c index 7936599b..65f44259 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -298,7 +298,13 @@ struct proxy_base { proxy_t *prev; int id; int subid; - int priority; + + /* Priority has the user id encoded in the high bits if it's not a + * global proxy. */ + int64_t priority; + + bool global; /* Is this a global proxy */ + int userid; /* Userid for non global proxies */ double diff; @@ -1083,16 +1089,24 @@ static sdata_t *duplicate_sdata(const sdata_t *sdata) return dsdata; } -static int prio_sort(proxy_t *a, proxy_t *b) +static int64_t prio_sort(proxy_t *a, proxy_t *b) { return (a->priority - b->priority); } /* Priority values can be sparse, they do not need to be sequential */ -static void __set_proxy_prio(sdata_t *sdata, proxy_t *proxy, const int priority) +static void __set_proxy_prio(sdata_t *sdata, proxy_t *proxy, int64_t priority) { proxy_t *tmpa, *tmpb, *exists = NULL; - int next_prio = 0; + int64_t next_prio = 0; + + /* Encode the userid as the high bits in priority */ + if (!proxy->global) { + int64_t high_bits = proxy->userid; + + high_bits <<= 32; + priority |= high_bits; + } /* See if the priority is already in use */ HASH_ITER(hh, sdata->proxies, tmpa, tmpb) { @@ -1121,7 +1135,6 @@ static proxy_t *__generate_proxy(sdata_t *sdata, const int id) proxy->parent = proxy; proxy->id = id; - proxy->priority = id; proxy->sdata = duplicate_sdata(sdata); proxy->sdata->subproxy = proxy; proxy->sdata->verbose = true; @@ -1383,9 +1396,10 @@ static void dead_proxyid(sdata_t *sdata, const int id, const int subid) static void update_subscribe(ckpool_t *ckp, const char *cmd) { sdata_t *sdata = ckp->data, *dsdata; + int id = 0, subid = 0, userid = 0; proxy_t *proxy, *old = NULL; - int id = 0, subid = 0; const char *buf; + bool global; json_t *val; if (unlikely(strlen(cmd) < 11)) { @@ -1407,6 +1421,16 @@ static void update_subscribe(ckpool_t *ckp, const char *cmd) LOGWARNING("Failed to json decode subproxy value in update_subscribe %s", buf); return; } + if (unlikely(!json_get_bool(&global, val, "global"))) { + LOGWARNING("Failed to json decode global value in update_subscribe %s", buf); + return; + } + if (!global) { + if (unlikely(!json_get_int(&userid, val, "userid"))) { + LOGWARNING("Failed to json decode userid value in update_subscribe %s", buf); + return; + } + } if (!subid) LOGNOTICE("Got updated subscribe for proxy %d", id); @@ -1422,9 +1446,8 @@ static void update_subscribe(ckpool_t *ckp, const char *cmd) proxy->dead = false; } else proxy = subproxy_by_id(sdata, id, subid); - dsdata = proxy->sdata; - - ck_wlock(&dsdata->workbase_lock); + proxy->global = global; + proxy->userid = userid; proxy->subscribed = true; proxy->diff = ckp->startdiff; memset(proxy->url, 0, 128); @@ -1433,6 +1456,10 @@ static void update_subscribe(ckpool_t *ckp, const char *cmd) strncpy(proxy->url, json_string_value(json_object_get(val, "url")), 127); strncpy(proxy->auth, json_string_value(json_object_get(val, "auth")), 127); strncpy(proxy->pass, json_string_value(json_object_get(val, "pass")), 127); + + dsdata = proxy->sdata; + + ck_wlock(&dsdata->workbase_lock); /* Length is checked by generator */ strcpy(proxy->enonce1, json_string_value(json_object_get(val, "enonce1"))); proxy->enonce1constlen = strlen(proxy->enonce1) / 2; @@ -2371,13 +2398,21 @@ out: _Close(sockd); } +/* Return the user masked priority value of the proxy */ +static int proxy_prio(const proxy_t *proxy) +{ + int prio = proxy->priority & 0x00000000ffffffff; + + return prio; +} + static json_t *json_proxyinfo(const proxy_t *proxy) { const proxy_t *parent = proxy->parent; json_t *val; - JSON_CPACK(val, "{si,si,si,sf,ss,ss,ss,ss,si,si,si,si,sb,sb,sI,sI,sI,sI,si,sb}", - "id", proxy->id, "subid", proxy->subid, "priority", parent->priority, + JSON_CPACK(val, "{si,si,si,sf,ss,ss,ss,ss,si,si,si,si,sb,sb,sI,sI,sI,sI,si,sb,sb,si}", + "id", proxy->id, "subid", proxy->subid, "priority", proxy_prio(parent), "diff", proxy->diff, "url", proxy->url, "auth", proxy->auth, "pass", proxy->pass, "enonce1", proxy->enonce1, "enonce1constlen", proxy->enonce1constlen, "enonce1varlen", proxy->enonce1varlen, "nonce2len", proxy->nonce2len, @@ -2385,7 +2420,7 @@ static json_t *json_proxyinfo(const proxy_t *proxy) "notified", proxy->notified, "clients", proxy->clients, "max_clients", proxy->max_clients, "bound_clients", proxy->bound_clients, "combined_clients", parent->combined_clients, "headroom", proxy->headroom, "subproxy_count", parent->subproxy_count, - "dead", proxy->dead); + "dead", proxy->dead, "global", proxy->global, "userid", proxy->userid); return val; } @@ -2445,7 +2480,7 @@ static void setproxy(sdata_t *sdata, const char *buf, int *sockd) val = json_errormsg("Failed to find proxy %d", id); goto out; } - if (priority != proxy->priority) + if (priority != proxy_prio(proxy)) set_proxy_prio(sdata, proxy, priority); val = json_proxyinfo(proxy); out: @@ -2718,7 +2753,8 @@ static sdata_t *select_sdata(const ckpool_t *ckp, sdata_t *ckp_sdata) HASH_ITER(hh, ckp_sdata->proxies, proxy, tmp) { int64_t max_headroom; - best = NULL; + if (!proxy->global) + break; proxy->headroom = max_headroom = 0; HASH_ITER(sh, proxy->subproxies, subproxy, tmpsub) { int64_t subproxy_headroom; From 3628128d640ec1a11a8487bfcd10c5630c440729 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Fri, 20 Mar 2015 15:28:39 +1100 Subject: [PATCH 366/544] Set ckp variable in connsocks used by proxies --- src/generator.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/generator.c b/src/generator.c index 4bb4f973..2c72e9b6 100644 --- a/src/generator.c +++ b/src/generator.c @@ -2202,7 +2202,7 @@ static proxy_instance_t *__add_userproxy(ckpool_t *ckp, gdata_t *gdata, const in proxy->url = url; proxy->auth = auth; proxy->pass = pass; - proxy->ckp = ckp; + proxy->ckp = proxy->cs.ckp = ckp; mutex_init(&proxy->notify_lock); mutex_init(&proxy->share_lock); HASH_ADD_INT(gdata->proxies, id, proxy); @@ -2488,7 +2488,7 @@ static proxy_instance_t *__add_proxy(ckpool_t *ckp, gdata_t *gdata, const int id proxy->url = strdup(ckp->proxyurl[id]); proxy->auth = strdup(ckp->proxyauth[id]); proxy->pass = strdup(ckp->proxypass[id]); - proxy->ckp = ckp; + proxy->ckp = proxy->cs.ckp = ckp; mutex_init(&proxy->notify_lock); mutex_init(&proxy->share_lock); HASH_ADD_INT(gdata->proxies, id, proxy); From fae7c70dc2858f6dfee38aa54be0e2a22ab985aa Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Fri, 20 Mar 2015 16:06:23 +1100 Subject: [PATCH 367/544] Abstract out the test for the best subproxy for future use with user proxies --- src/stratifier.c | 51 ++++++++++++++++++++++++++++++------------------ 1 file changed, 32 insertions(+), 19 deletions(-) diff --git a/src/stratifier.c b/src/stratifier.c index 65f44259..8dd61b55 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -2731,13 +2731,40 @@ static bool new_enonce1(ckpool_t *ckp, sdata_t *ckp_sdata, sdata_t *sdata, strat static void stratum_send_message(sdata_t *sdata, const stratum_instance_t *client, const char *msg); +/* Need to hold sdata->proxy_lock */ +static proxy_t *__best_subproxy(proxy_t *proxy) +{ + proxy_t *subproxy, *best = NULL, *tmp; + int64_t max_headroom; + + proxy->headroom = max_headroom = 0; + HASH_ITER(sh, proxy->subproxies, subproxy, tmp) { + int64_t subproxy_headroom; + + if (subproxy->dead) + continue; + if (!subproxy->sdata->current_workbase) + continue; + subproxy_headroom = subproxy->max_clients - subproxy->clients; + + proxy->headroom += subproxy_headroom; + if (subproxy_headroom > max_headroom) { + best = subproxy; + max_headroom = subproxy_headroom; + } + if (best) + break; + } + return best; +} + /* Choose the stratifier data for a new client. Use the main ckp_sdata except * in proxy mode where we find a subproxy based on the current proxy with room * for more clients. Signal the generator to recruit more subproxies if we are * running out of room. */ static sdata_t *select_sdata(const ckpool_t *ckp, sdata_t *ckp_sdata) { - proxy_t *current, *proxy, *subproxy, *best = NULL, *tmp, *tmpsub; + proxy_t *current, *proxy, *tmp, *best = NULL; if (!ckp->proxy || ckp->passthrough) return ckp_sdata; @@ -2751,26 +2778,12 @@ static sdata_t *select_sdata(const ckpool_t *ckp, sdata_t *ckp_sdata) * priority */ mutex_lock(&ckp_sdata->proxy_lock); HASH_ITER(hh, ckp_sdata->proxies, proxy, tmp) { - int64_t max_headroom; - + /* FIXME: We need to check the user bound proxies though we + * currently only know users after they've authorised which is + * too late for this. */ if (!proxy->global) break; - proxy->headroom = max_headroom = 0; - HASH_ITER(sh, proxy->subproxies, subproxy, tmpsub) { - int64_t subproxy_headroom; - - if (subproxy->dead) - continue; - if (!subproxy->sdata->current_workbase) - continue; - subproxy_headroom = subproxy->max_clients - subproxy->clients; - - proxy->headroom += subproxy_headroom; - if (subproxy_headroom > max_headroom) { - best = subproxy; - max_headroom = subproxy_headroom; - } - } + best = __best_subproxy(proxy); if (best) break; } From 33e903cb4fea19bbe5c03c8ad04d43d5aab259e2 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Fri, 20 Mar 2015 17:20:36 +1100 Subject: [PATCH 368/544] Logic inversion --- src/stratifier.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/stratifier.c b/src/stratifier.c index 8dd61b55..95c0b57c 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -2711,9 +2711,9 @@ static bool new_enonce1(ckpool_t *ckp, sdata_t *ckp_sdata, sdata_t *sdata, strat * left depending on nonce2 length, we'll always get a changing value * for every next client.*/ ck_wlock(&ckp_sdata->instance_lock); - enonce1 = htole64(ckp_sdata->enonce1_64); + enonce1 = le64toh(ckp_sdata->enonce1_64); enonce1++; - client->enonce1_64 = ckp_sdata->enonce1_64 = le64toh(enonce1); + client->enonce1_64 = ckp_sdata->enonce1_64 = htole64(enonce1); if (proxy) { client->proxy = proxy; proxy->clients++; From eb3c78f52b635793f9fb157e45cb81e371f9c0a2 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Fri, 20 Mar 2015 17:36:22 +1100 Subject: [PATCH 369/544] Give a unique sessionid in proxy mode --- src/stratifier.c | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/src/stratifier.c b/src/stratifier.c index 95c0b57c..6fc1e311 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -281,6 +281,11 @@ struct stratum_instance { proxy_t *proxy; /* Proxy this is bound to in proxy mode */ int proxyid; /* Which proxy id */ int subproxyid; /* Which subproxy */ + + /* What session id we gave this instance to recognise users as they + * reconnect before they've sent their auth information if possible in + * proxy mode instead of supporting stratum resume with it. */ + int64_t session_id; }; struct share { @@ -331,7 +336,7 @@ struct proxy_base { int subproxy_count; /* Number of subproxies */ proxy_t *parent; /* Parent proxy of each subproxy */ proxy_t *subproxies; /* Hashlist of subproxies sorted by subid */ - sdata_t *sdata; /* Unique stratifer data for each subproxy */ + sdata_t *sdata; /* Unique stratifier data for each subproxy */ bool dead; }; @@ -374,6 +379,7 @@ struct stratifier_data { ckmsgq_t *stxnq; // Transaction requests int64_t user_instance_id; + int64_t session_id; /* Stratum_instances hashlist is stored by id, whereas disconnected_instances * is sorted by enonce1_64. */ @@ -2874,13 +2880,17 @@ static json_t *parse_subscribe(stratum_instance_t *client, const int64_t client_ ck_runlock(&sdata->workbase_lock); /* Send a random sessionid in proxy mode so clients don't think we have - * resumed if enonce1 ends up matching on reconnect. */ + * resumed if enonce1 ends up matching on reconnect but we can use it + * to recognise users reconnecting before they authorise. */ if (ckp->proxy) { - unsigned int now = time(NULL); - char nowx[12]; + char sessionid[20]; + + ck_wlock(&ckp_sdata->instance_lock); + client->session_id = ckp_sdata->session_id++; + ck_wunlock(&ckp_sdata->instance_lock); - sprintf(nowx, "%x", now); - JSON_CPACK(ret, "[[[s,s]],s,i]", "mining.notify", nowx, client->enonce1, + sprintf(sessionid, "%lx", client->session_id); + JSON_CPACK(ret, "[[[s,s]],s,i]", "mining.notify", sessionid, client->enonce1, n2len); } else { JSON_CPACK(ret, "[[[s,s]],s,i]", "mining.notify", client->enonce1, client->enonce1, @@ -5389,6 +5399,8 @@ int stratifier(proc_instance_t *pi) randomiser <<= 32; if (!ckp->proxy) sdata->blockchange_id = sdata->workbase_id = randomiser; + else + sdata->session_id = randomiser; if (!ckp->serverurls) { ckp->serverurl[0] = "127.0.0.1"; From 6cb3078025277c738ce2382a8fe9268312985cad Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Fri, 20 Mar 2015 17:46:35 +1100 Subject: [PATCH 370/544] Add a list of sessionids created with their associated user once we generate the user --- src/stratifier.c | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/src/stratifier.c b/src/stratifier.c index 6fc1e311..b847239f 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -340,6 +340,15 @@ struct proxy_base { bool dead; }; +typedef struct user_sessionid user_sessionid_t; + +struct user_sessionid { + UT_hash_handle hh; + int64_t session_id; + user_instance_t *user; + time_t added; +}; + struct stratifier_data { ckpool_t *ckp; @@ -414,6 +423,7 @@ struct stratifier_data { proxy_t *proxies; /* Hashlist of all proxies */ mutex_t proxy_lock; /* Protects all proxy data */ proxy_t *subproxy; /* Which subproxy this sdata belongs to in proxy mode */ + user_sessionid_t *user_sessionids; }; typedef struct json_entry json_entry_t; @@ -3338,6 +3348,20 @@ static void queue_delayed_auth(stratum_instance_t *client) ckdbq_add(ckp, ID_AUTH, val); } +static void add_user_sessionid(ckpool_t *ckp, int64_t session_id, user_instance_t *user) +{ + user_sessionid_t *user_sessionid = ckalloc(sizeof(user_sessionid_t)); + sdata_t *sdata = ckp->data; + + user_sessionid->session_id = session_id; + user_sessionid->user = user; + user_sessionid->added = time(NULL); + + ck_wlock(&sdata->instance_lock); + HASH_ADD_I64(sdata->user_sessionids, session_id, user_sessionid); + ck_wunlock(&sdata->instance_lock); +} + /* Needs to be entered with client holding a ref count. */ static json_t *parse_authorise(stratum_instance_t *client, const json_t *params_val, json_t **err_val, int *errnum) @@ -3380,6 +3404,8 @@ static json_t *parse_authorise(stratum_instance_t *client, const json_t *params_ goto out; } user = generate_user(ckp, client, buf); + if (ckp->proxy) + add_user_sessionid(ckp, client->session_id, user); client->user_id = user->id; ts_realtime(&now); client->start_time = now.tv_sec; From 74ea210ad2776aff97d1a695e18ca8129cae2052 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Fri, 20 Mar 2015 18:05:26 +1100 Subject: [PATCH 371/544] Match user from sessionid if possible in proxy mode --- src/stratifier.c | 49 +++++++++++++++++++++++++++++++++++++----------- 1 file changed, 38 insertions(+), 11 deletions(-) diff --git a/src/stratifier.c b/src/stratifier.c index b847239f..8f401506 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -2814,6 +2814,26 @@ static sdata_t *select_sdata(const ckpool_t *ckp, sdata_t *ckp_sdata) return best->sdata; } +static user_instance_t *user_from_sessionid(sdata_t *sdata, const int64_t session_id) +{ + user_sessionid_t *user_sessionid; + user_instance_t *user = NULL; + + ck_wlock(&sdata->instance_lock); + HASH_FIND_I64(sdata->user_sessionids, &session_id, user_sessionid); + if (user_sessionid) + HASH_DEL(sdata->user_sessionids, user_sessionid); + ck_wunlock(&sdata->instance_lock); + + if (user_sessionid) { + user = user_sessionid->user; + LOGINFO("Found matching user %s from sessionid %lx", user->username, + session_id); + dealloc(user_sessionid); + } + return user; +} + /* Extranonce1 must be set here. Needs to be entered with client holding a ref * count. */ static json_t *parse_subscribe(stratum_instance_t *client, const int64_t client_id, const json_t *params_val) @@ -2853,19 +2873,26 @@ static json_t *parse_subscribe(stratum_instance_t *client, const int64_t client_ client->useragent = strdup(buf); else client->useragent = ckzalloc(1); // Set to "" - if (arr_size > 1 && !ckp->proxy) { - /* This would be the session id for reconnect, it will - * not work for clients on a proxied connection. */ + buf = NULL; + if (arr_size > 1) { buf = json_string_value(json_array_get(params_val, 1)); LOGDEBUG("Found old session id %s", buf); - /* Add matching here */ - if ((client->enonce1_64 = disconnected_sessionid_exists(sdata, buf, client_id))) { - sprintf(client->enonce1, "%016lx", client->enonce1_64); - old_match = true; - - ck_rlock(&sdata->workbase_lock); - __fill_enonce1data(sdata->current_workbase, client); - ck_runlock(&sdata->workbase_lock); + if (!ckp->proxy) { + /* This would be the session id for reconnect, it will + * not work for clients on a proxied connection. */ + if ((client->enonce1_64 = disconnected_sessionid_exists(sdata, buf, client_id))) { + sprintf(client->enonce1, "%016lx", client->enonce1_64); + old_match = true; + + ck_rlock(&sdata->workbase_lock); + __fill_enonce1data(sdata->current_workbase, client); + ck_runlock(&sdata->workbase_lock); + } + } else { + int64_t session_id = 0; + + sscanf(buf, "%lx", &session_id); + client->user_instance = user_from_sessionid(ckp_sdata, session_id); } } } else From 49fe273718f1ac00ccb9d1c7518f6cfe0b6519fa Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Fri, 20 Mar 2015 18:17:43 +1100 Subject: [PATCH 372/544] Look up users based on session id and find a proxy from their list if possible --- src/stratifier.c | 34 ++++++++++++++++++++-------------- 1 file changed, 20 insertions(+), 14 deletions(-) diff --git a/src/stratifier.c b/src/stratifier.c index 8f401506..75dc5d37 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -2778,7 +2778,7 @@ static proxy_t *__best_subproxy(proxy_t *proxy) * in proxy mode where we find a subproxy based on the current proxy with room * for more clients. Signal the generator to recruit more subproxies if we are * running out of room. */ -static sdata_t *select_sdata(const ckpool_t *ckp, sdata_t *ckp_sdata) +static sdata_t *select_sdata(const ckpool_t *ckp, sdata_t *ckp_sdata, int userid) { proxy_t *current, *proxy, *tmp, *best = NULL; @@ -2794,10 +2794,9 @@ static sdata_t *select_sdata(const ckpool_t *ckp, sdata_t *ckp_sdata) * priority */ mutex_lock(&ckp_sdata->proxy_lock); HASH_ITER(hh, ckp_sdata->proxies, proxy, tmp) { - /* FIXME: We need to check the user bound proxies though we - * currently only know users after they've authorised which is - * too late for this. */ - if (!proxy->global) + if (proxy->userid < userid) + continue; + if (proxy->userid > userid) break; best = __best_subproxy(proxy); if (best) @@ -2806,10 +2805,11 @@ static sdata_t *select_sdata(const ckpool_t *ckp, sdata_t *ckp_sdata) mutex_unlock(&ckp_sdata->proxy_lock); if (!best) { - LOGWARNING("Temporarily insufficient subproxies to accept more clients"); + if (!userid) + LOGWARNING("Temporarily insufficient subproxies to accept more clients"); return NULL; } - if (best->id != current->id || current->headroom < 2) + if (!userid && (best->id != current->id || current->headroom < 2)) generator_recruit(ckp, 1); return best->sdata; } @@ -2850,17 +2850,12 @@ static json_t *parse_subscribe(stratum_instance_t *client, const int64_t client_ return json_string("params not an array"); } - sdata = select_sdata(ckp, ckp_sdata); + sdata = select_sdata(ckp, ckp_sdata, 0); if (unlikely(!sdata || !sdata->current_workbase)) { LOGWARNING("Failed to provide subscription due to no %s", sdata ? "current workbase" : "sdata"); stratum_send_message(ckp_sdata, client, "Pool Initialising"); return json_string("Initialising"); } - if (ckp->proxy) { - LOGINFO("Current %d, selecting proxy %d:%d for client %"PRId64, ckp_sdata->proxy->id, - sdata->subproxy->id, sdata->subproxy->subid, client->id); - } - client->sdata = sdata; arr_size = json_array_size(params_val); /* NOTE useragent is NULL prior to this so should not be used in code @@ -2893,10 +2888,21 @@ static json_t *parse_subscribe(stratum_instance_t *client, const int64_t client_ sscanf(buf, "%lx", &session_id); client->user_instance = user_from_sessionid(ckp_sdata, session_id); + if (client->user_instance) { + sdata_t *usdata = select_sdata(ckp, ckp_sdata, client->user_instance->id); + + if (usdata) + sdata = usdata; + } } } } else client->useragent = ckzalloc(1); + client->sdata = sdata; + if (ckp->proxy) { + LOGINFO("Current %d, selecting proxy %d:%d for client %"PRId64, ckp_sdata->proxy->id, + sdata->subproxy->id, sdata->subproxy->subid, client->id); + } if (!old_match) { /* Create a new extranonce1 based on a uint64_t pointer */ if (!new_enonce1(ckp, ckp_sdata, sdata, client)) { @@ -3101,7 +3107,7 @@ static user_instance_t *__create_user(sdata_t *sdata, const char *username) user->auth_backoff = DEFAULT_AUTH_BACKOFF; strcpy(user->username, username); - user->id = sdata->user_instance_id++; + user->id = ++sdata->user_instance_id; HASH_ADD_STR(sdata->user_instances, username, user); return user; } From 65df7bc72243b7f7bda7235b92b5729b4ffecef8 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Fri, 20 Mar 2015 23:56:44 +1100 Subject: [PATCH 373/544] Set cs.ckp on creating a new subproxy --- src/generator.c | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/generator.c b/src/generator.c index 2c72e9b6..5a1fac58 100644 --- a/src/generator.c +++ b/src/generator.c @@ -905,7 +905,8 @@ static void prepare_proxy(proxy_instance_t *proxi); /* Creates a duplicate instance or proxi to be used as a subproxy, ignoring * fields we don't use in the subproxy. */ -static proxy_instance_t *create_subproxy(gdata_t *gdata, proxy_instance_t *proxi, const char *url) +static proxy_instance_t *create_subproxy(ckpool_t *ckp, gdata_t *gdata, proxy_instance_t *proxi, + const char *url) { proxy_instance_t *subproxy; @@ -921,7 +922,7 @@ static proxy_instance_t *create_subproxy(gdata_t *gdata, proxy_instance_t *proxi } mutex_unlock(&gdata->lock); - subproxy->ckp = proxi->ckp; + subproxy->cs.ckp = subproxy->ckp = ckp; mutex_lock(&proxi->proxy_lock); subproxy->subid = ++proxi->subproxy_count; @@ -1060,7 +1061,7 @@ static bool parse_reconnect(proxy_instance_t *proxy, json_t *val) * the url has changed. Otherwise automated recruiting will * take care of creating one if needed. */ if (!sameurl) - create_subproxy(gdata, parent, url); + create_subproxy(ckp, gdata, parent, url); goto out; } @@ -1740,7 +1741,7 @@ static void *proxy_recruit(void *arg) retry: recruit = false; - proxy = create_subproxy(gdata, parent, parent->url); + proxy = create_subproxy(ckp, gdata, parent, parent->url); alive = proxy_alive(ckp, proxy, &proxy->cs, false, parent->epfd); if (!alive) { LOGNOTICE("Subproxy failed proxy_alive testing"); From 505abca664ac923bb3e3e66d89453d3980ef4077 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Sat, 21 Mar 2015 13:12:15 +1100 Subject: [PATCH 374/544] Revert sessionid code to merge master sessionid code --- src/stratifier.c | 133 +++++++++++------------------------------------ 1 file changed, 31 insertions(+), 102 deletions(-) diff --git a/src/stratifier.c b/src/stratifier.c index 75dc5d37..95c0b57c 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -281,11 +281,6 @@ struct stratum_instance { proxy_t *proxy; /* Proxy this is bound to in proxy mode */ int proxyid; /* Which proxy id */ int subproxyid; /* Which subproxy */ - - /* What session id we gave this instance to recognise users as they - * reconnect before they've sent their auth information if possible in - * proxy mode instead of supporting stratum resume with it. */ - int64_t session_id; }; struct share { @@ -336,19 +331,10 @@ struct proxy_base { int subproxy_count; /* Number of subproxies */ proxy_t *parent; /* Parent proxy of each subproxy */ proxy_t *subproxies; /* Hashlist of subproxies sorted by subid */ - sdata_t *sdata; /* Unique stratifier data for each subproxy */ + sdata_t *sdata; /* Unique stratifer data for each subproxy */ bool dead; }; -typedef struct user_sessionid user_sessionid_t; - -struct user_sessionid { - UT_hash_handle hh; - int64_t session_id; - user_instance_t *user; - time_t added; -}; - struct stratifier_data { ckpool_t *ckp; @@ -388,7 +374,6 @@ struct stratifier_data { ckmsgq_t *stxnq; // Transaction requests int64_t user_instance_id; - int64_t session_id; /* Stratum_instances hashlist is stored by id, whereas disconnected_instances * is sorted by enonce1_64. */ @@ -423,7 +408,6 @@ struct stratifier_data { proxy_t *proxies; /* Hashlist of all proxies */ mutex_t proxy_lock; /* Protects all proxy data */ proxy_t *subproxy; /* Which subproxy this sdata belongs to in proxy mode */ - user_sessionid_t *user_sessionids; }; typedef struct json_entry json_entry_t; @@ -2778,7 +2762,7 @@ static proxy_t *__best_subproxy(proxy_t *proxy) * in proxy mode where we find a subproxy based on the current proxy with room * for more clients. Signal the generator to recruit more subproxies if we are * running out of room. */ -static sdata_t *select_sdata(const ckpool_t *ckp, sdata_t *ckp_sdata, int userid) +static sdata_t *select_sdata(const ckpool_t *ckp, sdata_t *ckp_sdata) { proxy_t *current, *proxy, *tmp, *best = NULL; @@ -2794,9 +2778,10 @@ static sdata_t *select_sdata(const ckpool_t *ckp, sdata_t *ckp_sdata, int userid * priority */ mutex_lock(&ckp_sdata->proxy_lock); HASH_ITER(hh, ckp_sdata->proxies, proxy, tmp) { - if (proxy->userid < userid) - continue; - if (proxy->userid > userid) + /* FIXME: We need to check the user bound proxies though we + * currently only know users after they've authorised which is + * too late for this. */ + if (!proxy->global) break; best = __best_subproxy(proxy); if (best) @@ -2805,35 +2790,14 @@ static sdata_t *select_sdata(const ckpool_t *ckp, sdata_t *ckp_sdata, int userid mutex_unlock(&ckp_sdata->proxy_lock); if (!best) { - if (!userid) - LOGWARNING("Temporarily insufficient subproxies to accept more clients"); + LOGWARNING("Temporarily insufficient subproxies to accept more clients"); return NULL; } - if (!userid && (best->id != current->id || current->headroom < 2)) + if (best->id != current->id || current->headroom < 2) generator_recruit(ckp, 1); return best->sdata; } -static user_instance_t *user_from_sessionid(sdata_t *sdata, const int64_t session_id) -{ - user_sessionid_t *user_sessionid; - user_instance_t *user = NULL; - - ck_wlock(&sdata->instance_lock); - HASH_FIND_I64(sdata->user_sessionids, &session_id, user_sessionid); - if (user_sessionid) - HASH_DEL(sdata->user_sessionids, user_sessionid); - ck_wunlock(&sdata->instance_lock); - - if (user_sessionid) { - user = user_sessionid->user; - LOGINFO("Found matching user %s from sessionid %lx", user->username, - session_id); - dealloc(user_sessionid); - } - return user; -} - /* Extranonce1 must be set here. Needs to be entered with client holding a ref * count. */ static json_t *parse_subscribe(stratum_instance_t *client, const int64_t client_id, const json_t *params_val) @@ -2850,12 +2814,17 @@ static json_t *parse_subscribe(stratum_instance_t *client, const int64_t client_ return json_string("params not an array"); } - sdata = select_sdata(ckp, ckp_sdata, 0); + sdata = select_sdata(ckp, ckp_sdata); if (unlikely(!sdata || !sdata->current_workbase)) { LOGWARNING("Failed to provide subscription due to no %s", sdata ? "current workbase" : "sdata"); stratum_send_message(ckp_sdata, client, "Pool Initialising"); return json_string("Initialising"); } + if (ckp->proxy) { + LOGINFO("Current %d, selecting proxy %d:%d for client %"PRId64, ckp_sdata->proxy->id, + sdata->subproxy->id, sdata->subproxy->subid, client->id); + } + client->sdata = sdata; arr_size = json_array_size(params_val); /* NOTE useragent is NULL prior to this so should not be used in code @@ -2868,41 +2837,23 @@ static json_t *parse_subscribe(stratum_instance_t *client, const int64_t client_ client->useragent = strdup(buf); else client->useragent = ckzalloc(1); // Set to "" - buf = NULL; - if (arr_size > 1) { + if (arr_size > 1 && !ckp->proxy) { + /* This would be the session id for reconnect, it will + * not work for clients on a proxied connection. */ buf = json_string_value(json_array_get(params_val, 1)); LOGDEBUG("Found old session id %s", buf); - if (!ckp->proxy) { - /* This would be the session id for reconnect, it will - * not work for clients on a proxied connection. */ - if ((client->enonce1_64 = disconnected_sessionid_exists(sdata, buf, client_id))) { - sprintf(client->enonce1, "%016lx", client->enonce1_64); - old_match = true; - - ck_rlock(&sdata->workbase_lock); - __fill_enonce1data(sdata->current_workbase, client); - ck_runlock(&sdata->workbase_lock); - } - } else { - int64_t session_id = 0; - - sscanf(buf, "%lx", &session_id); - client->user_instance = user_from_sessionid(ckp_sdata, session_id); - if (client->user_instance) { - sdata_t *usdata = select_sdata(ckp, ckp_sdata, client->user_instance->id); - - if (usdata) - sdata = usdata; - } + /* Add matching here */ + if ((client->enonce1_64 = disconnected_sessionid_exists(sdata, buf, client_id))) { + sprintf(client->enonce1, "%016lx", client->enonce1_64); + old_match = true; + + ck_rlock(&sdata->workbase_lock); + __fill_enonce1data(sdata->current_workbase, client); + ck_runlock(&sdata->workbase_lock); } } } else client->useragent = ckzalloc(1); - client->sdata = sdata; - if (ckp->proxy) { - LOGINFO("Current %d, selecting proxy %d:%d for client %"PRId64, ckp_sdata->proxy->id, - sdata->subproxy->id, sdata->subproxy->subid, client->id); - } if (!old_match) { /* Create a new extranonce1 based on a uint64_t pointer */ if (!new_enonce1(ckp, ckp_sdata, sdata, client)) { @@ -2923,17 +2874,13 @@ static json_t *parse_subscribe(stratum_instance_t *client, const int64_t client_ ck_runlock(&sdata->workbase_lock); /* Send a random sessionid in proxy mode so clients don't think we have - * resumed if enonce1 ends up matching on reconnect but we can use it - * to recognise users reconnecting before they authorise. */ + * resumed if enonce1 ends up matching on reconnect. */ if (ckp->proxy) { - char sessionid[20]; + unsigned int now = time(NULL); + char nowx[12]; - ck_wlock(&ckp_sdata->instance_lock); - client->session_id = ckp_sdata->session_id++; - ck_wunlock(&ckp_sdata->instance_lock); - - sprintf(sessionid, "%lx", client->session_id); - JSON_CPACK(ret, "[[[s,s]],s,i]", "mining.notify", sessionid, client->enonce1, + sprintf(nowx, "%x", now); + JSON_CPACK(ret, "[[[s,s]],s,i]", "mining.notify", nowx, client->enonce1, n2len); } else { JSON_CPACK(ret, "[[[s,s]],s,i]", "mining.notify", client->enonce1, client->enonce1, @@ -3107,7 +3054,7 @@ static user_instance_t *__create_user(sdata_t *sdata, const char *username) user->auth_backoff = DEFAULT_AUTH_BACKOFF; strcpy(user->username, username); - user->id = ++sdata->user_instance_id; + user->id = sdata->user_instance_id++; HASH_ADD_STR(sdata->user_instances, username, user); return user; } @@ -3381,20 +3328,6 @@ static void queue_delayed_auth(stratum_instance_t *client) ckdbq_add(ckp, ID_AUTH, val); } -static void add_user_sessionid(ckpool_t *ckp, int64_t session_id, user_instance_t *user) -{ - user_sessionid_t *user_sessionid = ckalloc(sizeof(user_sessionid_t)); - sdata_t *sdata = ckp->data; - - user_sessionid->session_id = session_id; - user_sessionid->user = user; - user_sessionid->added = time(NULL); - - ck_wlock(&sdata->instance_lock); - HASH_ADD_I64(sdata->user_sessionids, session_id, user_sessionid); - ck_wunlock(&sdata->instance_lock); -} - /* Needs to be entered with client holding a ref count. */ static json_t *parse_authorise(stratum_instance_t *client, const json_t *params_val, json_t **err_val, int *errnum) @@ -3437,8 +3370,6 @@ static json_t *parse_authorise(stratum_instance_t *client, const json_t *params_ goto out; } user = generate_user(ckp, client, buf); - if (ckp->proxy) - add_user_sessionid(ckp, client->session_id, user); client->user_id = user->id; ts_realtime(&now); client->start_time = now.tv_sec; @@ -5458,8 +5389,6 @@ int stratifier(proc_instance_t *pi) randomiser <<= 32; if (!ckp->proxy) sdata->blockchange_id = sdata->workbase_id = randomiser; - else - sdata->session_id = randomiser; if (!ckp->serverurls) { ckp->serverurl[0] = "127.0.0.1"; From 7961aa1a29a525cfa4da48784fd07ac944fc18f9 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Sat, 21 Mar 2015 13:26:18 +1100 Subject: [PATCH 375/544] There is no need for user instance id to be an int64 --- src/stratifier.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/stratifier.c b/src/stratifier.c index 236db742..0471ce08 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -160,7 +160,7 @@ typedef struct stratum_instance stratum_instance_t; struct user_instance { UT_hash_handle hh; char username[128]; - int64_t id; + int id; char *secondaryuserid; bool btcaddress; @@ -385,7 +385,7 @@ struct stratifier_data { ckmsgq_t *sauthq; // Stratum authorisations ckmsgq_t *stxnq; // Transaction requests - int64_t user_instance_id; + int user_instance_id; stratum_instance_t *stratum_instances; stratum_instance_t *recycled_instances; From b60bebe7b0722f36c847feede27bb1f24033cedc Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Sat, 21 Mar 2015 15:27:01 +1100 Subject: [PATCH 376/544] Rework code to always grab the sessionid but use it for reconnecting only in pool mode --- src/stratifier.c | 52 +++++++++++++++++++++++++++++------------------- 1 file changed, 31 insertions(+), 21 deletions(-) diff --git a/src/stratifier.c b/src/stratifier.c index 0471ce08..79023158 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -343,6 +343,7 @@ struct session { int session_id; uint64_t enonce1_64; int64_t client_id; + int userid; time_t added; }; @@ -1040,6 +1041,7 @@ static void __disconnect_session(sdata_t *sdata, const stratum_instance_t *clien session->enonce1_64 = client->enonce1_64; session->session_id = client->session_id; session->client_id = client->id; + session->userid = client->user_instance->id; session->added = now_t; HASH_ADD_INT(sdata->disconnected_sessions, session_id, session); sdata->stats.disconnected++; @@ -1878,28 +1880,15 @@ static stratum_instance_t *__stratum_add_instance(ckpool_t *ckp, const int64_t i return client; } -static uint64_t disconnected_sessionid_exists(sdata_t *sdata, const char *sessionid, - int *session_id, const int64_t id) +static uint64_t disconnected_sessionid_exists(sdata_t *sdata, const int session_id, + const int64_t id) { session_t *session; int64_t old_id = 0; uint64_t ret = 0; - int slen; - - if (!sessionid) - goto out; - slen = strlen(sessionid) / 2; - if (slen < 1 || slen > 4) - goto out; - - if (!validhex(sessionid)) - goto out; - - sscanf(sessionid, "%x", session_id); - LOGDEBUG("Testing for sessionid %s %x", sessionid, *session_id); ck_wlock(&sdata->instance_lock); - HASH_FIND_INT(sdata->disconnected_sessions, session_id, session); + HASH_FIND_INT(sdata->disconnected_sessions, &session_id, session); if (!session) goto out_unlock; HASH_DEL(sdata->disconnected_sessions, session); @@ -1909,7 +1898,7 @@ static uint64_t disconnected_sessionid_exists(sdata_t *sdata, const char *sessio dealloc(session); out_unlock: ck_wunlock(&sdata->instance_lock); -out: + if (ret) LOGNOTICE("Reconnecting old instance %"PRId64" to instance %"PRId64, old_id, id); return ret; @@ -2775,6 +2764,24 @@ static sdata_t *select_sdata(const ckpool_t *ckp, sdata_t *ckp_sdata) return best->sdata; } +static int int_from_sessionid(const char *sessionid) +{ + int ret = 0, slen; + + if (!sessionid) + goto out; + slen = strlen(sessionid) / 2; + if (slen < 1 || slen > 4) + goto out; + + if (!validhex(sessionid)) + goto out; + + sscanf(sessionid, "%x", &ret); +out: + return ret; +} + /* Extranonce1 must be set here. Needs to be entered with client holding a ref * count. */ static json_t *parse_subscribe(stratum_instance_t *client, const int64_t client_id, const json_t *params_val) @@ -2808,6 +2815,7 @@ static json_t *parse_subscribe(stratum_instance_t *client, const int64_t client_ /* NOTE useragent is NULL prior to this so should not be used in code * till after this point */ if (arr_size > 0) { + int session_id = 0; const char *buf; buf = json_string_value(json_array_get(params_val, 0)); @@ -2815,13 +2823,15 @@ static json_t *parse_subscribe(stratum_instance_t *client, const int64_t client_ client->useragent = strdup(buf); else client->useragent = ckzalloc(1); // Set to "" - if (arr_size > 1 && !ckp->proxy) { + if (arr_size > 1) { /* This would be the session id for reconnect, it will * not work for clients on a proxied connection. */ buf = json_string_value(json_array_get(params_val, 1)); - LOGDEBUG("Found old session id %s", buf); - /* Add matching here */ - if ((client->enonce1_64 = disconnected_sessionid_exists(sdata, buf, &client->session_id, client_id))) { + session_id = int_from_sessionid(buf); + } + if (!ckp->proxy && session_id) { + LOGDEBUG("Found old session id %d", session_id); + if ((client->enonce1_64 = disconnected_sessionid_exists(sdata, session_id, client_id))) { sprintf(client->enonce1, "%016lx", client->enonce1_64); old_match = true; From 4322c7d7c583a077416bc1c91abb3f9b605b5b02 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Sat, 21 Mar 2015 17:50:39 +1100 Subject: [PATCH 377/544] Inherit user id from old session id in proxy mode --- src/stratifier.c | 30 ++++++++++++++++++++++++++---- 1 file changed, 26 insertions(+), 4 deletions(-) diff --git a/src/stratifier.c b/src/stratifier.c index 79023158..45cded1a 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -267,7 +267,7 @@ struct stratum_instance { char *useragent; char *workername; - int64_t user_id; + int user_id; int server; /* Which server is this instance bound to */ ckpool_t *ckp; @@ -1041,7 +1041,7 @@ static void __disconnect_session(sdata_t *sdata, const stratum_instance_t *clien session->enonce1_64 = client->enonce1_64; session->session_id = client->session_id; session->client_id = client->id; - session->userid = client->user_instance->id; + session->userid = client->user_id; session->added = now_t; HASH_ADD_INT(sdata->disconnected_sessions, session_id, session); sdata->stats.disconnected++; @@ -2782,6 +2782,25 @@ out: return ret; } +static int userid_from_sessionid(sdata_t *sdata, const int session_id) +{ + session_t *session; + int ret = 0; + + ck_wlock(&sdata->instance_lock); + HASH_FIND_INT(sdata->disconnected_sessions, &session_id, session); + if (!session) + goto out_unlock; + HASH_DEL(sdata->disconnected_sessions, session); + sdata->stats.disconnected--; + ret = session->userid; + dealloc(session); +out_unlock: + ck_wunlock(&sdata->instance_lock); + + return ret; +} + /* Extranonce1 must be set here. Needs to be entered with client holding a ref * count. */ static json_t *parse_subscribe(stratum_instance_t *client, const int64_t client_id, const json_t *params_val) @@ -2828,9 +2847,9 @@ static json_t *parse_subscribe(stratum_instance_t *client, const int64_t client_ * not work for clients on a proxied connection. */ buf = json_string_value(json_array_get(params_val, 1)); session_id = int_from_sessionid(buf); + LOGDEBUG("Found old session id %d", session_id); } if (!ckp->proxy && session_id) { - LOGDEBUG("Found old session id %d", session_id); if ((client->enonce1_64 = disconnected_sessionid_exists(sdata, session_id, client_id))) { sprintf(client->enonce1, "%016lx", client->enonce1_64); old_match = true; @@ -2839,6 +2858,9 @@ static json_t *parse_subscribe(stratum_instance_t *client, const int64_t client_ __fill_enonce1data(sdata->current_workbase, client); ck_runlock(&sdata->workbase_lock); } + } else if (ckp->proxy && session_id) { + /* Use the session_id to tell us which user this was */ + client->user_id = userid_from_sessionid(ckp_sdata, session_id); } } else client->useragent = ckzalloc(1); @@ -3031,7 +3053,7 @@ static user_instance_t *__create_user(sdata_t *sdata, const char *username) user->auth_backoff = DEFAULT_AUTH_BACKOFF; strcpy(user->username, username); - user->id = sdata->user_instance_id++; + user->id = ++sdata->user_instance_id; HASH_ADD_STR(sdata->user_instances, username, user); return user; } From ac3726e56f3d50970145f096c6be126aa8514a12 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Sat, 21 Mar 2015 18:09:01 +1100 Subject: [PATCH 378/544] Try to find a suitable pool by userid on clients with a known userid --- src/stratifier.c | 37 +++++++++++++++++++++++-------------- 1 file changed, 23 insertions(+), 14 deletions(-) diff --git a/src/stratifier.c b/src/stratifier.c index 45cded1a..982820ca 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -2728,7 +2728,7 @@ static proxy_t *__best_subproxy(proxy_t *proxy) * in proxy mode where we find a subproxy based on the current proxy with room * for more clients. Signal the generator to recruit more subproxies if we are * running out of room. */ -static sdata_t *select_sdata(const ckpool_t *ckp, sdata_t *ckp_sdata) +static sdata_t *select_sdata(const ckpool_t *ckp, sdata_t *ckp_sdata, const int userid) { proxy_t *current, *proxy, *tmp, *best = NULL; @@ -2744,10 +2744,9 @@ static sdata_t *select_sdata(const ckpool_t *ckp, sdata_t *ckp_sdata) * priority */ mutex_lock(&ckp_sdata->proxy_lock); HASH_ITER(hh, ckp_sdata->proxies, proxy, tmp) { - /* FIXME: We need to check the user bound proxies though we - * currently only know users after they've authorised which is - * too late for this. */ - if (!proxy->global) + if (proxy->userid < userid) + continue; + if (proxy->userid > userid) break; best = __best_subproxy(proxy); if (best) @@ -2818,17 +2817,12 @@ static json_t *parse_subscribe(stratum_instance_t *client, const int64_t client_ return json_string("params not an array"); } - sdata = select_sdata(ckp, ckp_sdata); + sdata = select_sdata(ckp, ckp_sdata, 0); if (unlikely(!sdata || !sdata->current_workbase)) { LOGWARNING("Failed to provide subscription due to no %s", sdata ? "current workbase" : "sdata"); stratum_send_message(ckp_sdata, client, "Pool Initialising"); return json_string("Initialising"); } - if (ckp->proxy) { - LOGINFO("Current %d, selecting proxy %d:%d for client %"PRId64, ckp_sdata->proxy->id, - sdata->subproxy->id, sdata->subproxy->subid, client->id); - } - client->sdata = sdata; arr_size = json_array_size(params_val); /* NOTE useragent is NULL prior to this so should not be used in code @@ -2854,16 +2848,31 @@ static json_t *parse_subscribe(stratum_instance_t *client, const int64_t client_ sprintf(client->enonce1, "%016lx", client->enonce1_64); old_match = true; - ck_rlock(&sdata->workbase_lock); + ck_rlock(&ckp_sdata->workbase_lock); __fill_enonce1data(sdata->current_workbase, client); - ck_runlock(&sdata->workbase_lock); + ck_runlock(&ckp_sdata->workbase_lock); } } else if (ckp->proxy && session_id) { + int userid; + /* Use the session_id to tell us which user this was */ - client->user_id = userid_from_sessionid(ckp_sdata, session_id); + userid = userid_from_sessionid(ckp_sdata, session_id); + if (userid) { + sdata_t *user_sdata = select_sdata(ckp, ckp_sdata, userid); + + if (user_sdata) + sdata = user_sdata; + } } } else client->useragent = ckzalloc(1); + + client->sdata = sdata; + if (ckp->proxy) { + LOGINFO("Current %d, selecting proxy %d:%d for client %"PRId64, ckp_sdata->proxy->id, + sdata->subproxy->id, sdata->subproxy->subid, client->id); + } + if (!old_match) { /* Create a new extranonce1 based on a uint64_t pointer */ if (!new_enonce1(ckp, ckp_sdata, sdata, client)) { From 68bfb5bab4f944488e62bc1db90fb450f1b4aecb Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Sat, 21 Mar 2015 18:26:27 +1100 Subject: [PATCH 379/544] Adjust the best global proxy only when there's a change in global proxies --- src/stratifier.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/stratifier.c b/src/stratifier.c index 982820ca..f1005db4 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -1373,6 +1373,8 @@ static void check_bestproxy(sdata_t *sdata) HASH_ITER(hh, sdata->proxies, proxy, tmp) { if (!__subproxies_alive(proxy)) continue; + if (!proxy->global) + break; if (proxy != sdata->proxy) { sdata->proxy = proxy; changed_id = proxy->id; @@ -1397,7 +1399,8 @@ static void dead_proxyid(sdata_t *sdata, const int id, const int subid) proxy = existing_subproxy(sdata, id, subid); if (proxy) proxy->dead = true; - check_bestproxy(sdata); + if (proxy->global) + check_bestproxy(sdata); LOGINFO("Stratifier dropping clients from proxy %d:%d", id, subid); headroom = current_headroom(sdata, &proxy); @@ -1609,7 +1612,8 @@ static void update_notify(ckpool_t *ckp, const char *cmd) LOGNOTICE("Block hash on proxy %d changed to %s", id, dsdata->lastswaphash); } - check_bestproxy(sdata); + if (proxy->global) + check_bestproxy(sdata); clean |= new_block; LOGINFO("Proxy %d:%d broadcast updated stratum notify with%s clean", id, subid, clean ? "" : "out"); From 0e1d4dd4f1024019a917ebf611c7489c87fb4b4c Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Sat, 21 Mar 2015 18:45:11 +1100 Subject: [PATCH 380/544] Don't reconnect clients bound to user proxies on global changes --- src/stratifier.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/stratifier.c b/src/stratifier.c index f1005db4..88fde89b 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -1308,7 +1308,7 @@ static void generator_recruit(const ckpool_t *ckp, const int recruits) /* Find how much headroom we have and connect up to that many clients that are * not currently on this pool, setting the reconnect for the remainder to be - * switched lazily. */ + * switched lazily. Only reconnect clients bound to global proxies. */ static void reconnect_clients(sdata_t *sdata) { stratum_instance_t *client, *tmpclient; @@ -1322,6 +1322,9 @@ static void reconnect_clients(sdata_t *sdata) ck_rlock(&sdata->instance_lock); HASH_ITER(hh, sdata->stratum_instances, client, tmpclient) { + /* This client is bound to a user proxy */ + if (client->sdata && client->sdata->proxy && client->sdata->proxy->userid) + continue; if (client->proxyid == proxy->id) continue; if (headroom-- < 1) From a2052cdf37bddc9ffadcaaff84d6e84f758488a0 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Sat, 21 Mar 2015 19:25:33 +1100 Subject: [PATCH 381/544] Reconnect clients when they are bound to a global proxy and have userproxies with headroom --- src/stratifier.c | 39 ++++++++++++++++++++++++++++++++++----- 1 file changed, 34 insertions(+), 5 deletions(-) diff --git a/src/stratifier.c b/src/stratifier.c index 88fde89b..0da948e9 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -1165,7 +1165,7 @@ static proxy_t *__generate_proxy(sdata_t *sdata, const int id) proxy->parent = proxy; proxy->id = id; proxy->sdata = duplicate_sdata(sdata); - proxy->sdata->subproxy = proxy; + proxy->sdata->proxy = proxy->sdata->subproxy = proxy; proxy->sdata->verbose = true; /* subid == 0 on parent proxy */ HASH_ADD(sh, proxy->subproxies, subid, sizeof(int), proxy); @@ -1187,7 +1187,7 @@ static proxy_t *__generate_subproxy(sdata_t *sdata, proxy_t *proxy, const int su HASH_ADD(sh, proxy->subproxies, subid, sizeof(int), subproxy); proxy->subproxy_count++; subproxy->sdata = duplicate_sdata(sdata); - subproxy->sdata->subproxy = subproxy; + proxy->sdata->proxy = subproxy->sdata->subproxy = subproxy; return subproxy; } @@ -1295,6 +1295,26 @@ out_unlock: return headroom; } +static int64_t userproxy_headroom(sdata_t *sdata, const int userid) +{ + proxy_t *proxy, *subproxy, *tmp, *subtmp; + int64_t headroom = 0; + + mutex_lock(&sdata->proxy_lock); + HASH_ITER(hh, sdata->proxies, proxy, tmp) { + if (proxy->userid != userid) + continue; + HASH_ITER(sh, proxy->subproxies, subproxy, subtmp) { + if (subproxy->dead) + continue; + headroom += subproxy->max_clients - subproxy->clients; + } + } + mutex_unlock(&sdata->proxy_lock); + + return headroom; +} + static void reconnect_client(sdata_t *sdata, stratum_instance_t *client); static void generator_recruit(const ckpool_t *ckp, const int recruits) @@ -1400,10 +1420,11 @@ static void dead_proxyid(sdata_t *sdata, const int id, const int subid) proxy_t *proxy; proxy = existing_subproxy(sdata, id, subid); - if (proxy) + if (proxy) { proxy->dead = true; - if (proxy->global) - check_bestproxy(sdata); + if (proxy->global) + check_bestproxy(sdata); + } LOGINFO("Stratifier dropping clients from proxy %d:%d", id, subid); headroom = current_headroom(sdata, &proxy); @@ -3433,6 +3454,14 @@ static json_t *parse_authorise(stratum_instance_t *client, const json_t *params_ if (ckp->proxy) { LOGNOTICE("Authorised client %"PRId64" to proxy %d:%d, worker %s as user %s", client->id, client->proxyid, client->subproxyid, buf, user->username); + if (client->sdata && client->sdata->proxy && client->sdata->proxy->global) { + sdata_t *ckp_sdata = ckp->data; + + if (userproxy_headroom(ckp_sdata, client->user_id)) { + LOGWARNING("reconnecting to user!"); + reconnect_client(ckp_sdata, client); + } + } } else { LOGNOTICE("Authorised client %"PRId64" worker %s as user %s", client->id, buf, user->username); From d897abff8f26818e71e36990c09c8c848d3925e0 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Sat, 21 Mar 2015 19:26:17 +1100 Subject: [PATCH 382/544] Remove debugging --- src/stratifier.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/stratifier.c b/src/stratifier.c index 0da948e9..ddeb174a 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -3457,10 +3457,8 @@ static json_t *parse_authorise(stratum_instance_t *client, const json_t *params_ if (client->sdata && client->sdata->proxy && client->sdata->proxy->global) { sdata_t *ckp_sdata = ckp->data; - if (userproxy_headroom(ckp_sdata, client->user_id)) { - LOGWARNING("reconnecting to user!"); + if (userproxy_headroom(ckp_sdata, client->user_id)) reconnect_client(ckp_sdata, client); - } } } else { LOGNOTICE("Authorised client %"PRId64" worker %s as user %s", From c7852c34bb6d6b8d0b8dceaa732ae5927e698d49 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Sat, 21 Mar 2015 19:30:57 +1100 Subject: [PATCH 383/544] Make the reconnect after auth a soft reconnect --- src/stratifier.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/stratifier.c b/src/stratifier.c index ddeb174a..c2c77942 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -3458,7 +3458,7 @@ static json_t *parse_authorise(stratum_instance_t *client, const json_t *params_ sdata_t *ckp_sdata = ckp->data; if (userproxy_headroom(ckp_sdata, client->user_id)) - reconnect_client(ckp_sdata, client); + client->reconnect = true; } } else { LOGNOTICE("Authorised client %"PRId64" worker %s as user %s", From 046f8affb0d4209d52be70671463475e6ebd4250 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Sat, 21 Mar 2015 19:47:52 +1100 Subject: [PATCH 384/544] Reconnect clients not bound to a user proxy when a new notify creates more headroom --- src/stratifier.c | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/src/stratifier.c b/src/stratifier.c index c2c77942..a7850c7c 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -1544,6 +1544,41 @@ static void update_subscribe(ckpool_t *ckp, const char *cmd) json_decref(val); } +/* Check how much headroom the userid proxies have and reconnect any clients + * that are not bound to it */ +static void check_userproxies(sdata_t *sdata, const int userid) +{ + int64_t headroom = userproxy_headroom(sdata, userid); + stratum_instance_t *client, *tmpclient; + int reconnects = 0, hard = 0; + + if (!headroom) + return; + + ck_rlock(&sdata->instance_lock); + HASH_ITER(hh, sdata->stratum_instances, client, tmpclient) { + if (client->user_id != userid) + continue; + if (client->sdata && client->sdata->proxy && client->sdata->proxy->userid == userid) + continue; + if (headroom-- < 1) + continue; + reconnects++; + if (client->reconnect && hard <= SOMAXCONN / 2) { + hard++; + reconnect_client(sdata, client); + } else + client->reconnect = true; + } + ck_runlock(&sdata->instance_lock); + + if (reconnects) { + LOGNOTICE("%d clients flagged for reconnect to user %d proxies", + reconnects, userid); + } + /* FIXME: Recruit extra user proxies when headroom < 0 */ +} + static void update_notify(ckpool_t *ckp, const char *cmd) { sdata_t *sdata = ckp->data, *dsdata; @@ -1638,6 +1673,8 @@ static void update_notify(ckpool_t *ckp, const char *cmd) if (proxy->global) check_bestproxy(sdata); + else + check_userproxies(sdata, proxy->userid); clean |= new_block; LOGINFO("Proxy %d:%d broadcast updated stratum notify with%s clean", id, subid, clean ? "" : "out"); From bf79e0793d7b72dc35e7aa1e4c9a100c9121e8b7 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Sat, 21 Mar 2015 21:00:50 +1100 Subject: [PATCH 385/544] Recruit extra user subproxies when there aren't enough --- src/generator.c | 17 +++++++++---- src/stratifier.c | 64 ++++++++++++++++++++++++++++++++++++------------ 2 files changed, 60 insertions(+), 21 deletions(-) diff --git a/src/generator.c b/src/generator.c index 5a1fac58..79a02dce 100644 --- a/src/generator.c +++ b/src/generator.c @@ -1779,12 +1779,19 @@ static void recruit_subproxies(proxy_instance_t *proxi, const int recruits) } /* Queue up to the requested amount */ -static void recruit_subproxy(proxy_instance_t *proxi, const char *buf) +static void recruit_subproxy(gdata_t *gdata, const char *buf) { - int recruits = 1; + int recruits = 1, id = 0; + proxy_instance_t *proxy; - sscanf(buf, "recruit=%d", &recruits); - recruit_subproxies(proxi, recruits); + sscanf(buf, "recruit=%d:%d", &id, &recruits); + proxy = proxy_by_id(gdata, id); + if (unlikely(!proxy)) { + LOGNOTICE("Generator failed to find proxy id %d to recruit subproxies", + id); + return; + } + recruit_subproxies(proxy, recruits); } static void *proxy_reconnect(void *arg) @@ -2442,7 +2449,7 @@ retry: LOGDEBUG("Proxy received ping request"); send_unix_msg(sockd, "pong"); } else if (cmdmatch(buf, "recruit")) { - recruit_subproxy(proxi, buf); + recruit_subproxy(gdata, buf); } else if (cmdmatch(buf, "dropproxy")) { drop_proxy(gdata, buf); } else { diff --git a/src/stratifier.c b/src/stratifier.c index a7850c7c..2058ebc1 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -1295,15 +1295,17 @@ out_unlock: return headroom; } -static int64_t userproxy_headroom(sdata_t *sdata, const int userid) +static int64_t proxy_headroom(sdata_t *sdata, const int userid) { proxy_t *proxy, *subproxy, *tmp, *subtmp; int64_t headroom = 0; mutex_lock(&sdata->proxy_lock); HASH_ITER(hh, sdata->proxies, proxy, tmp) { - if (proxy->userid != userid) + if (proxy->userid < userid) continue; + if (proxy->userid > userid) + break; HASH_ITER(sh, proxy->subproxies, subproxy, subtmp) { if (subproxy->dead) continue; @@ -1317,11 +1319,11 @@ static int64_t userproxy_headroom(sdata_t *sdata, const int userid) static void reconnect_client(sdata_t *sdata, stratum_instance_t *client); -static void generator_recruit(const ckpool_t *ckp, const int recruits) +static void generator_recruit(const ckpool_t *ckp, const int proxyid, const int recruits) { char buf[256]; - sprintf(buf, "recruit=%d", recruits); + sprintf(buf, "recruit=%d:%d", proxyid, recruits); LOGINFO("Stratifer requesting %d more proxies from generator", recruits); send_generator(ckp, buf, GEN_PRIORITY); } @@ -1365,7 +1367,7 @@ static void reconnect_clients(sdata_t *sdata) proxy->id); } if (headroom < 0) - generator_recruit(sdata->ckp, -headroom); + generator_recruit(sdata->ckp, proxy->id, -headroom); } static bool __subproxies_alive(proxy_t *proxy) @@ -1414,8 +1416,8 @@ static void check_bestproxy(sdata_t *sdata) static void dead_proxyid(sdata_t *sdata, const int id, const int subid) { + int reconnects = 0, hard = 0, proxyid = 0; stratum_instance_t *client, *tmp; - int reconnects = 0, hard = 0; int64_t headroom; proxy_t *proxy; @@ -1427,6 +1429,8 @@ static void dead_proxyid(sdata_t *sdata, const int id, const int subid) } LOGINFO("Stratifier dropping clients from proxy %d:%d", id, subid); headroom = current_headroom(sdata, &proxy); + if (proxy) + proxyid = proxy->id; ck_rlock(&sdata->instance_lock); HASH_ITER(hh, sdata->stratum_instances, client, tmp) { @@ -1445,8 +1449,10 @@ static void dead_proxyid(sdata_t *sdata, const int id, const int subid) LOGNOTICE("%d clients flagged to reconnect from dead proxy %d:%d", reconnects, id, subid); } + /* When a proxy dies, recruit more of the global proxies for them to + * fail over to in case user proxies are unavailable. */ if (headroom < 0) - generator_recruit(sdata->ckp, -headroom); + generator_recruit(sdata->ckp, proxyid, -headroom); } static void update_subscribe(ckpool_t *ckp, const char *cmd) @@ -1544,17 +1550,38 @@ static void update_subscribe(ckpool_t *ckp, const char *cmd) json_decref(val); } +/* Find the highest priority alive proxy belonging to userid and recruit extra + * subproxies. */ +static void recruit_best_userproxy(sdata_t *sdata, const int userid, const int recruits) +{ + proxy_t *proxy, *subproxy, *tmp, *subtmp, *best = NULL; + + mutex_lock(&sdata->proxy_lock); + HASH_ITER(hh, sdata->proxies, proxy, tmp) { + if (proxy->userid < userid) + continue; + if (proxy->userid > userid) + break; + HASH_ITER(sh, proxy->subproxies, subproxy, subtmp) { + if (subproxy->dead) + continue; + best = proxy; + } + } + mutex_unlock(&sdata->proxy_lock); + + if (best) + generator_recruit(sdata->ckp, proxy->id, recruits); +} + /* Check how much headroom the userid proxies have and reconnect any clients * that are not bound to it */ static void check_userproxies(sdata_t *sdata, const int userid) { - int64_t headroom = userproxy_headroom(sdata, userid); + int64_t headroom = proxy_headroom(sdata, userid); stratum_instance_t *client, *tmpclient; int reconnects = 0, hard = 0; - if (!headroom) - return; - ck_rlock(&sdata->instance_lock); HASH_ITER(hh, sdata->stratum_instances, client, tmpclient) { if (client->user_id != userid) @@ -1576,7 +1603,8 @@ static void check_userproxies(sdata_t *sdata, const int userid) LOGNOTICE("%d clients flagged for reconnect to user %d proxies", reconnects, userid); } - /* FIXME: Recruit extra user proxies when headroom < 0 */ + if (headroom < 0) + recruit_best_userproxy(sdata, userid, -headroom); } static void update_notify(ckpool_t *ckp, const char *cmd) @@ -2386,7 +2414,7 @@ static void lazy_reconnect_client(sdata_t *sdata, stratum_instance_t *client) LOGNOTICE("Reconnecting client %"PRId64, client->id); reconnect_client(sdata, client); } else { - generator_recruit(sdata->ckp, -headroom); + generator_recruit(sdata->ckp, proxy->id, -headroom); if (!client->reconnect) { LOGNOTICE("Flagging client %"PRId64, client->id); client->reconnect = true; @@ -2823,8 +2851,12 @@ static sdata_t *select_sdata(const ckpool_t *ckp, sdata_t *ckp_sdata, const int LOGWARNING("Temporarily insufficient subproxies to accept more clients"); return NULL; } - if (best->id != current->id || current->headroom < 2) - generator_recruit(ckp, 1); + if (best->global && (best->id != current->id || current->headroom < 2)) + generator_recruit(ckp, current->id, 1); + else if (userid) { + if (proxy_headroom(ckp_sdata, userid) < 2) + generator_recruit(ckp, best->id, 1); + } return best->sdata; } @@ -3494,7 +3526,7 @@ static json_t *parse_authorise(stratum_instance_t *client, const json_t *params_ if (client->sdata && client->sdata->proxy && client->sdata->proxy->global) { sdata_t *ckp_sdata = ckp->data; - if (userproxy_headroom(ckp_sdata, client->user_id)) + if (proxy_headroom(ckp_sdata, client->user_id)) client->reconnect = true; } } else { From 508bb71e9b86cad297f295cdf019dacebb95d637 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Sat, 21 Mar 2015 21:04:01 +1100 Subject: [PATCH 386/544] Recruit from best not proxy --- src/stratifier.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/stratifier.c b/src/stratifier.c index 2058ebc1..d80b1aac 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -1571,7 +1571,7 @@ static void recruit_best_userproxy(sdata_t *sdata, const int userid, const int r mutex_unlock(&sdata->proxy_lock); if (best) - generator_recruit(sdata->ckp, proxy->id, recruits); + generator_recruit(sdata->ckp, best->id, recruits); } /* Check how much headroom the userid proxies have and reconnect any clients From f31980b9d0fc529e5e28669fdc8c9442f3156ede Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Sat, 21 Mar 2015 21:41:05 +1100 Subject: [PATCH 387/544] Only reconnect authorised clients --- src/stratifier.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/stratifier.c b/src/stratifier.c index d80b1aac..b27f10d2 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -1344,6 +1344,8 @@ static void reconnect_clients(sdata_t *sdata) ck_rlock(&sdata->instance_lock); HASH_ITER(hh, sdata->stratum_instances, client, tmpclient) { + if (!client->authorised) + continue; /* This client is bound to a user proxy */ if (client->sdata && client->sdata->proxy && client->sdata->proxy->userid) continue; @@ -1584,6 +1586,8 @@ static void check_userproxies(sdata_t *sdata, const int userid) ck_rlock(&sdata->instance_lock); HASH_ITER(hh, sdata->stratum_instances, client, tmpclient) { + if (!client->authorised) + continue; if (client->user_id != userid) continue; if (client->sdata && client->sdata->proxy && client->sdata->proxy->userid == userid) From 4e4c8aa79a1a8a52921ee6a80c1589aeb561347e Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Sat, 21 Mar 2015 22:04:56 +1100 Subject: [PATCH 388/544] Add more reliable tests to know which clients should be redirected --- src/stratifier.c | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/stratifier.c b/src/stratifier.c index b27f10d2..43dbb254 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -1165,7 +1165,7 @@ static proxy_t *__generate_proxy(sdata_t *sdata, const int id) proxy->parent = proxy; proxy->id = id; proxy->sdata = duplicate_sdata(sdata); - proxy->sdata->proxy = proxy->sdata->subproxy = proxy; + proxy->sdata->subproxy = proxy; proxy->sdata->verbose = true; /* subid == 0 on parent proxy */ HASH_ADD(sh, proxy->subproxies, subid, sizeof(int), proxy); @@ -1187,7 +1187,7 @@ static proxy_t *__generate_subproxy(sdata_t *sdata, proxy_t *proxy, const int su HASH_ADD(sh, proxy->subproxies, subid, sizeof(int), subproxy); proxy->subproxy_count++; subproxy->sdata = duplicate_sdata(sdata); - proxy->sdata->proxy = subproxy->sdata->subproxy = subproxy; + subproxy->sdata->subproxy = subproxy; return subproxy; } @@ -1344,10 +1344,12 @@ static void reconnect_clients(sdata_t *sdata) ck_rlock(&sdata->instance_lock); HASH_ITER(hh, sdata->stratum_instances, client, tmpclient) { + if (client->dropped) + continue; if (!client->authorised) continue; /* This client is bound to a user proxy */ - if (client->sdata && client->sdata->proxy && client->sdata->proxy->userid) + if (client->proxy->userid) continue; if (client->proxyid == proxy->id) continue; @@ -1586,11 +1588,13 @@ static void check_userproxies(sdata_t *sdata, const int userid) ck_rlock(&sdata->instance_lock); HASH_ITER(hh, sdata->stratum_instances, client, tmpclient) { + if (client->dropped) + continue; if (!client->authorised) continue; if (client->user_id != userid) continue; - if (client->sdata && client->sdata->proxy && client->sdata->proxy->userid == userid) + if (client->proxy->userid == userid) continue; if (headroom-- < 1) continue; From 4f3e19325b88219308688c071c97e0a812cf3b8e Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Sat, 21 Mar 2015 22:11:26 +1100 Subject: [PATCH 389/544] Minor notice change --- src/stratifier.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/stratifier.c b/src/stratifier.c index 43dbb254..bffa5080 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -1324,7 +1324,8 @@ static void generator_recruit(const ckpool_t *ckp, const int proxyid, const int char buf[256]; sprintf(buf, "recruit=%d:%d", proxyid, recruits); - LOGINFO("Stratifer requesting %d more proxies from generator", recruits); + LOGINFO("Stratifer requesting %d more subproxies of proxy %d from generator", + recruits, proxyid); send_generator(ckp, buf, GEN_PRIORITY); } From 804a98e63c3777e0e5251e32c53a47fcf3c21131 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Sat, 21 Mar 2015 22:17:47 +1100 Subject: [PATCH 390/544] Do all reconnects on stratum notify lazily only to prevent multiple subproxies flagging reconnects --- src/stratifier.c | 22 ++++++++-------------- 1 file changed, 8 insertions(+), 14 deletions(-) diff --git a/src/stratifier.c b/src/stratifier.c index bffa5080..b3ba8815 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -1335,7 +1335,7 @@ static void generator_recruit(const ckpool_t *ckp, const int proxyid, const int static void reconnect_clients(sdata_t *sdata) { stratum_instance_t *client, *tmpclient; - int reconnects = 0, hard = 0; + int reconnects = 0; int64_t headroom; proxy_t *proxy; @@ -1354,16 +1354,12 @@ static void reconnect_clients(sdata_t *sdata) continue; if (client->proxyid == proxy->id) continue; + if (client->reconnect) + continue; if (headroom-- < 1) continue; reconnects++; - /* Limit reconnects sent concurrently to prevent a flood of new - * connections */ - if (client->reconnect && hard <= SOMAXCONN / 2) { - hard++; - reconnect_client(sdata, client); - } else - client->reconnect = true; + client->reconnect = true; } ck_runlock(&sdata->instance_lock); @@ -1585,7 +1581,7 @@ static void check_userproxies(sdata_t *sdata, const int userid) { int64_t headroom = proxy_headroom(sdata, userid); stratum_instance_t *client, *tmpclient; - int reconnects = 0, hard = 0; + int reconnects = 0; ck_rlock(&sdata->instance_lock); HASH_ITER(hh, sdata->stratum_instances, client, tmpclient) { @@ -1597,14 +1593,12 @@ static void check_userproxies(sdata_t *sdata, const int userid) continue; if (client->proxy->userid == userid) continue; + if (client->reconnect) + continue; if (headroom-- < 1) continue; reconnects++; - if (client->reconnect && hard <= SOMAXCONN / 2) { - hard++; - reconnect_client(sdata, client); - } else - client->reconnect = true; + client->reconnect = true; } ck_runlock(&sdata->instance_lock); From baef99118637012b0d66a15084760c43d95b8ce9 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Sat, 21 Mar 2015 22:26:27 +1100 Subject: [PATCH 391/544] Use correct test for recruiting more global proxies --- src/stratifier.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/stratifier.c b/src/stratifier.c index b3ba8815..3efc8043 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -2854,9 +2854,10 @@ static sdata_t *select_sdata(const ckpool_t *ckp, sdata_t *ckp_sdata, const int LOGWARNING("Temporarily insufficient subproxies to accept more clients"); return NULL; } - if (best->global && (best->id != current->id || current->headroom < 2)) - generator_recruit(ckp, current->id, 1); - else if (userid) { + if (!userid) { + if (best->id != current->id || current_headroom(ckp_sdata, &proxy) < 2) + generator_recruit(ckp, current->id, 1); + } else { if (proxy_headroom(ckp_sdata, userid) < 2) generator_recruit(ckp, best->id, 1); } From 104ab03f2b65f889bfb04d62ece2f92146a15d1c Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Sun, 22 Mar 2015 09:36:34 +1100 Subject: [PATCH 393/544] Use only one share hashlist in the generator --- src/generator.c | 54 ++++++++++++++++++++++++------------------------- 1 file changed, 26 insertions(+), 28 deletions(-) diff --git a/src/generator.c b/src/generator.c index 79a02dce..bad61581 100644 --- a/src/generator.c +++ b/src/generator.c @@ -47,7 +47,7 @@ typedef struct notify_instance notify_instance_t; struct share_msg { UT_hash_handle hh; - int64_t id; // Our own id for submitting upstream + int id; // Our own id for submitting upstream int client_id; time_t submit_time; @@ -117,10 +117,6 @@ struct proxy_instance { pthread_t pth_precv; - mutex_t share_lock; - share_msg_t *shares; - int64_t share_id; - ckmsgq_t *passsends; // passthrough sends char_entry_t *recvd_lines; /* Linked list of unprocessed messages */ @@ -150,6 +146,10 @@ struct generator_data { pthread_cond_t psend_cond; stratum_msg_t *psends; + + mutex_t share_lock; + share_msg_t *shares; + int64_t share_id; }; typedef struct generator_data gdata_t; @@ -918,7 +918,6 @@ static proxy_instance_t *create_subproxy(ckpool_t *ckp, gdata_t *gdata, proxy_in subproxy->disabled = false; } else { subproxy = ckzalloc(sizeof(proxy_instance_t)); - mutex_init(&subproxy->share_lock); } mutex_unlock(&gdata->lock); @@ -1419,10 +1418,10 @@ static void submit_share(gdata_t *gdata, json_t *val) msg->json_msg = val; /* Add new share entry to the share hashtable */ - mutex_lock(&proxi->share_lock); - share->id = proxi->share_id++; - HASH_ADD_I64(proxi->shares, id, share); - mutex_unlock(&proxi->share_lock); + mutex_lock(&gdata->share_lock); + share->id = gdata->share_id++; + HASH_ADD_I64(gdata->shares, id, share); + mutex_unlock(&gdata->share_lock); json_object_set_nocheck(val, "id", json_integer(share->id)); @@ -1446,8 +1445,8 @@ static void clear_notify(notify_instance_t *ni) free(ni); } -/* FIXME: Return something useful to the stratifier based on this result */ -static bool parse_share(proxy_instance_t *proxi, const char *buf) +/* FIXME: Return something useful to the stratifier based on this result? */ +static bool parse_share(gdata_t *gdata, proxy_instance_t *proxi, const char *buf) { json_t *val = NULL, *idval; share_msg_t *share; @@ -1466,11 +1465,11 @@ static bool parse_share(proxy_instance_t *proxi, const char *buf) } id = json_integer_value(idval); - mutex_lock(&proxi->share_lock); - HASH_FIND_I64(proxi->shares, &id, share); + mutex_lock(&gdata->share_lock); + HASH_FIND_I64(gdata->shares, &id, share); if (share) - HASH_DEL(proxi->shares, share); - mutex_unlock(&proxi->share_lock); + HASH_DEL(gdata->shares, share); + mutex_unlock(&gdata->share_lock); /* We set response to true even if we don't find the matching share, * so long as we recognised it as a share response */ @@ -1957,13 +1956,13 @@ static void *proxy_recv(void *arg) mutex_unlock(&proxi->notify_lock); /* Similary with shares older than 2 mins without response */ - mutex_lock(&proxi->share_lock); - HASH_ITER(hh, proxi->shares, share, tmpshare) { + mutex_lock(&gdata->share_lock); + HASH_ITER(hh, gdata->shares, share, tmpshare) { if (share->submit_time < now - 120) { - HASH_DEL(proxi->shares, share); + HASH_DEL(gdata->shares, share); } } - mutex_unlock(&proxi->share_lock); + mutex_unlock(&gdata->share_lock); /* If we don't get an update within 10 minutes the upstream pool * has likely stopped responding. */ @@ -1988,7 +1987,7 @@ static void *proxy_recv(void *arg) if (parse_method(ckp, subproxy, cs->buf)) continue; /* If it's not a method it should be a share result */ - if (!parse_share(subproxy, cs->buf)) + if (!parse_share(gdata, subproxy, cs->buf)) LOGNOTICE("Proxy %d:%d unhandled stratum message: %s", subproxy->id, subproxy->subid, cs->buf); } while ((ret = read_socket_line(cs, 0)) > 0); @@ -2061,13 +2060,13 @@ static void *userproxy_recv(void *arg) mutex_unlock(&parent->notify_lock); /* Similary with shares older than 2 mins without response */ - mutex_lock(&parent->share_lock); - HASH_ITER(hh, parent->shares, share, tmpshare) { + mutex_lock(&gdata->share_lock); + HASH_ITER(hh, gdata->shares, share, tmpshare) { if (share->submit_time < now - 120) { - HASH_DEL(parent->shares, share); + HASH_DEL(gdata->shares, share); } } - mutex_unlock(&parent->share_lock); + mutex_unlock(&gdata->share_lock); do { /* proxy may have been recycled here if it is not a @@ -2075,7 +2074,7 @@ static void *userproxy_recv(void *arg) if (parse_method(ckp, proxy, cs->buf)) continue; /* If it's not a method it should be a share result */ - if (!parse_share(proxy, cs->buf)) + if (!parse_share(gdata, proxy, cs->buf)) LOGNOTICE("Proxy %d:%d unhandled stratum message: %s", proxy->id, proxy->subid, cs->buf); } while ((ret = read_socket_line(cs, 0)) > 0); @@ -2212,7 +2211,6 @@ static proxy_instance_t *__add_userproxy(ckpool_t *ckp, gdata_t *gdata, const in proxy->pass = pass; proxy->ckp = proxy->cs.ckp = ckp; mutex_init(&proxy->notify_lock); - mutex_init(&proxy->share_lock); HASH_ADD_INT(gdata->proxies, id, proxy); return proxy; } @@ -2498,7 +2496,6 @@ static proxy_instance_t *__add_proxy(ckpool_t *ckp, gdata_t *gdata, const int id proxy->pass = strdup(ckp->proxypass[id]); proxy->ckp = proxy->cs.ckp = ckp; mutex_init(&proxy->notify_lock); - mutex_init(&proxy->share_lock); HASH_ADD_INT(gdata->proxies, id, proxy); proxy->global = true; return proxy; @@ -2511,6 +2508,7 @@ static int proxy_mode(ckpool_t *ckp, proc_instance_t *pi) int i, ret; mutex_init(&gdata->lock); + mutex_init(&gdata->share_lock); /* Create all our proxy structures and pointers */ for (i = 0; i < ckp->proxies; i++) { From 19674013cfdc1623376d581eeb592808c0affa63 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Sun, 22 Mar 2015 09:44:46 +1100 Subject: [PATCH 394/544] Use only the one notify hashlist in the generator --- src/generator.c | 45 +++++++++++++++++++++------------------------ 1 file changed, 21 insertions(+), 24 deletions(-) diff --git a/src/generator.c b/src/generator.c index bad61581..1f3f9da8 100644 --- a/src/generator.c +++ b/src/generator.c @@ -112,9 +112,6 @@ struct proxy_instance { int64_t recruit; /* No of recruiting requests in progress */ bool alive; - mutex_t notify_lock; - notify_instance_t *notify_instances; - pthread_t pth_precv; ckmsgq_t *passsends; // passthrough sends @@ -147,6 +144,9 @@ struct generator_data { stratum_msg_t *psends; + mutex_t notify_lock; + notify_instance_t *notify_instances; + mutex_t share_lock; share_msg_t *shares; int64_t share_id; @@ -780,7 +780,6 @@ static void reconnect_generator(const ckpool_t *ckp) static bool parse_notify(ckpool_t *ckp, proxy_instance_t *proxi, json_t *val) { const char *prev_hash, *bbversion, *nbit, *ntime; - proxy_instance_t *proxy = proxi->parent; gdata_t *gdata = proxi->ckp->data; char *coinbase1, *coinbase2; const char *jobidbuf; @@ -844,10 +843,10 @@ static bool parse_notify(ckpool_t *ckp, proxy_instance_t *proxi, json_t *val) ni->notify_time = time(NULL); /* Add the notify instance to the parent proxy list, not the subproxy */ - mutex_lock(&proxy->notify_lock); + mutex_lock(&gdata->notify_lock); ni->id = gdata->proxy_notify_id++; - HASH_ADD_INT(proxy->notify_instances, id, ni); - mutex_unlock(&proxy->notify_lock); + HASH_ADD_INT(gdata->notify_instances, id, ni); + mutex_unlock(&gdata->notify_lock); send_notify(ckp, proxi, ni); out: @@ -1608,11 +1607,11 @@ static void *proxy_send(void *arg) continue; } - mutex_lock(&proxy->notify_lock); - HASH_FIND_INT(proxy->notify_instances, &id, ni); + mutex_lock(&gdata->notify_lock); + HASH_FIND_INT(gdata->notify_instances, &id, ni); if (ni) jobid = json_copy(ni->jobid); - mutex_unlock(&proxy->notify_lock); + mutex_unlock(&gdata->notify_lock); if (unlikely(!jobid)) { stratifier_reconnect_client(ckp, client_id); @@ -1944,16 +1943,16 @@ static void *proxy_recv(void *arg) now = time(NULL); /* Age old notifications older than 10 mins old */ - mutex_lock(&proxi->notify_lock); - HASH_ITER(hh, proxi->notify_instances, ni, tmp) { - if (HASH_COUNT(proxi->notify_instances) < 3) + mutex_lock(&gdata->notify_lock); + HASH_ITER(hh, gdata->notify_instances, ni, tmp) { + if (HASH_COUNT(gdata->notify_instances) < 3) break; if (ni->notify_time < now - 600) { - HASH_DEL(proxi->notify_instances, ni); + HASH_DEL(gdata->notify_instances, ni); clear_notify(ni); } } - mutex_unlock(&proxi->notify_lock); + mutex_unlock(&gdata->notify_lock); /* Similary with shares older than 2 mins without response */ mutex_lock(&gdata->share_lock); @@ -2014,7 +2013,7 @@ static void *userproxy_recv(void *arg) } while (42) { - proxy_instance_t *proxy, *tmpproxy, *parent; + proxy_instance_t *proxy, *tmpproxy; share_msg_t *share, *tmpshare; notify_instance_t *ni, *tmp; connsock_t *cs; @@ -2046,18 +2045,17 @@ static void *userproxy_recv(void *arg) continue; } now = time(NULL); - parent = proxy->parent; - mutex_lock(&parent->notify_lock); - HASH_ITER(hh, parent->notify_instances, ni, tmp) { - if (HASH_COUNT(parent->notify_instances) < 3) + mutex_lock(&gdata->notify_lock); + HASH_ITER(hh, gdata->notify_instances, ni, tmp) { + if (HASH_COUNT(gdata->notify_instances) < 3) break; if (ni->notify_time < now - 600) { - HASH_DEL(parent->notify_instances, ni); + HASH_DEL(gdata->notify_instances, ni); clear_notify(ni); } } - mutex_unlock(&parent->notify_lock); + mutex_unlock(&gdata->notify_lock); /* Similary with shares older than 2 mins without response */ mutex_lock(&gdata->share_lock); @@ -2210,7 +2208,6 @@ static proxy_instance_t *__add_userproxy(ckpool_t *ckp, gdata_t *gdata, const in proxy->auth = auth; proxy->pass = pass; proxy->ckp = proxy->cs.ckp = ckp; - mutex_init(&proxy->notify_lock); HASH_ADD_INT(gdata->proxies, id, proxy); return proxy; } @@ -2495,7 +2492,6 @@ static proxy_instance_t *__add_proxy(ckpool_t *ckp, gdata_t *gdata, const int id proxy->auth = strdup(ckp->proxyauth[id]); proxy->pass = strdup(ckp->proxypass[id]); proxy->ckp = proxy->cs.ckp = ckp; - mutex_init(&proxy->notify_lock); HASH_ADD_INT(gdata->proxies, id, proxy); proxy->global = true; return proxy; @@ -2508,6 +2504,7 @@ static int proxy_mode(ckpool_t *ckp, proc_instance_t *pi) int i, ret; mutex_init(&gdata->lock); + mutex_init(&gdata->notify_lock); mutex_init(&gdata->share_lock); /* Create all our proxy structures and pointers */ From 99878feb2db1c8f4828c4589d0c63ca8453681b1 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Sun, 22 Mar 2015 20:27:35 +1100 Subject: [PATCH 395/544] Add basic generator stats --- src/generator.c | 60 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 60 insertions(+) diff --git a/src/generator.c b/src/generator.c index 1f3f9da8..5513825b 100644 --- a/src/generator.c +++ b/src/generator.c @@ -133,6 +133,8 @@ struct generator_data { mutex_t lock; /* Lock protecting linked lists */ proxy_instance_t *proxies; /* Hash list of all proxies */ proxy_instance_t *dead_proxies; /* Disabled proxies */ + int proxies_generated; + int subproxies_generated; int proxy_notify_id; // Globally increasing notify id ckmsgq_t *srvchk; // Server check message queue @@ -916,6 +918,7 @@ static proxy_instance_t *create_subproxy(ckpool_t *ckp, gdata_t *gdata, proxy_in DL_DELETE(gdata->dead_proxies, subproxy); subproxy->disabled = false; } else { + gdata->subproxies_generated++; subproxy = ckzalloc(sizeof(proxy_instance_t)); } mutex_unlock(&gdata->lock); @@ -2201,6 +2204,7 @@ static proxy_instance_t *__add_userproxy(ckpool_t *ckp, gdata_t *gdata, const in { proxy_instance_t *proxy; + gdata->proxies_generated++; proxy = ckzalloc(sizeof(proxy_instance_t)); proxy->id = id; proxy->userid = userid; @@ -2360,6 +2364,59 @@ out: send_api_response(val, sockd); } +static void send_stats(gdata_t *gdata, const int sockd) +{ + json_t *val = json_object(), *subval; + int total_objects, objects, generated; + proxy_instance_t *proxy; + int64_t memsize; + + mutex_lock(&gdata->lock); + objects = HASH_COUNT(gdata->proxies); + memsize = SAFE_HASH_OVERHEAD(gdata->proxies) + sizeof(proxy_instance_t) * objects; + generated = gdata->proxies_generated; + JSON_CPACK(subval, "{si,si,si}", "count", objects, "memory", memsize, "generated", generated); + json_set_object(val, "proxies", subval); + + DL_COUNT(gdata->dead_proxies, proxy, objects); + memsize = sizeof(proxy_instance_t) * objects; + JSON_CPACK(subval, "{si,si}", "count", objects, "memory", memsize); + json_set_object(val, "dead_proxies", subval); + + total_objects = memsize = 0; + for (proxy = gdata->proxies; proxy; proxy=proxy->hh.next) { + mutex_lock(&proxy->proxy_lock); + total_objects += objects = HASH_COUNT(proxy->subproxies); + memsize += SAFE_HASH_OVERHEAD(proxy->subproxies) + sizeof(proxy_instance_t) * objects; + mutex_unlock(&proxy->proxy_lock); + } + generated = gdata->subproxies_generated; + mutex_unlock(&gdata->lock); + + JSON_CPACK(subval, "{si,si,si}", "count", total_objects, "memory", memsize, "generated", generated); + json_set_object(val, "subproxies", subval); + + mutex_lock(&gdata->notify_lock); + objects = HASH_COUNT(gdata->notify_instances); + memsize = SAFE_HASH_OVERHEAD(gdata->notify_instances) + sizeof(notify_instance_t) * objects; + generated = gdata->proxy_notify_id; + mutex_unlock(&gdata->notify_lock); + + JSON_CPACK(subval, "{si,si,si}", "count", objects, "memory", memsize, "generated", generated); + json_set_object(val, "notifies", subval); + + mutex_lock(&gdata->share_lock); + objects = HASH_COUNT(gdata->shares); + memsize = SAFE_HASH_OVERHEAD(gdata->shares) + sizeof(share_msg_t) * objects; + generated = gdata->share_id; + mutex_unlock(&gdata->share_lock); + + JSON_CPACK(subval, "{si,si,si}", "count", objects, "memory", memsize, "generated", generated); + json_set_object(val, "shares", subval); + + send_api_response(val, sockd); +} + static int proxy_loop(proc_instance_t *pi) { proxy_instance_t *proxi = NULL, *cproxy; @@ -2419,6 +2476,8 @@ retry: else submit_share(gdata, val); } + } else if (cmdmatch(buf, "stats")) { + send_stats(gdata, sockd); } else if (cmdmatch(buf, "list")) { send_list(gdata, sockd); } else if (cmdmatch(buf, "sublist")) { @@ -2486,6 +2545,7 @@ static proxy_instance_t *__add_proxy(ckpool_t *ckp, gdata_t *gdata, const int id { proxy_instance_t *proxy; + gdata->proxies_generated++; proxy = ckzalloc(sizeof(proxy_instance_t)); proxy->id = id; proxy->url = strdup(ckp->proxyurl[id]); From b37ee220e9f529d70c331c44983bc38812a5b3ac Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Sun, 22 Mar 2015 20:36:33 +1100 Subject: [PATCH 396/544] Add psends to generator stats --- src/generator.c | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/generator.c b/src/generator.c index 5513825b..75e51849 100644 --- a/src/generator.c +++ b/src/generator.c @@ -145,6 +145,7 @@ struct generator_data { pthread_cond_t psend_cond; stratum_msg_t *psends; + int psends_generated; mutex_t notify_lock; notify_instance_t *notify_instances; @@ -1429,6 +1430,7 @@ static void submit_share(gdata_t *gdata, json_t *val) /* Add the new message to the psend list */ mutex_lock(&gdata->psend_lock); + gdata->psends_generated++; DL_APPEND(gdata->psends, msg); pthread_cond_signal(&gdata->psend_cond); mutex_unlock(&gdata->psend_lock); @@ -2369,6 +2371,7 @@ static void send_stats(gdata_t *gdata, const int sockd) json_t *val = json_object(), *subval; int total_objects, objects, generated; proxy_instance_t *proxy; + stratum_msg_t *msg; int64_t memsize; mutex_lock(&gdata->lock); @@ -2414,6 +2417,15 @@ static void send_stats(gdata_t *gdata, const int sockd) JSON_CPACK(subval, "{si,si,si}", "count", objects, "memory", memsize, "generated", generated); json_set_object(val, "shares", subval); + mutex_lock(&gdata->psend_lock); + DL_COUNT(gdata->psends, msg, objects); + generated = gdata->psends_generated; + mutex_unlock(&gdata->psend_lock); + + memsize = sizeof(stratum_msg_t) * objects; + JSON_CPACK(subval, "{si,si,si}", "count", objects, "memory", memsize, "generated", generated); + json_set_object(val, "psends", subval); + send_api_response(val, sockd); } From fe93a914e111888c5eda51b1b176b5209aacb1c1 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Sun, 22 Mar 2015 21:04:59 +1100 Subject: [PATCH 397/544] Add API function to get worker stats --- src/stratifier.c | 44 +++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 43 insertions(+), 1 deletion(-) diff --git a/src/stratifier.c b/src/stratifier.c index a0099030..5939a982 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -2451,13 +2451,51 @@ static void getuser(sdata_t *sdata, const char *buf, int *sockd) "user", user->username, "id", user->id, "workers", user->workers, "bestdiff", user->best_diff, "dsps1", user->dsps1, "dsps5", user->dsps5, "dsps60", user->dsps60, "dsps1440", user->dsps1440, "dsps10080", user->dsps10080, - "lastshare", user->last_update.tv_sec); + "lastshare", user->last_share.tv_sec); out: free(username); send_api_response(val, *sockd); _Close(sockd); } +static worker_instance_t *get_worker(sdata_t *sdata, user_instance_t *user, const char *workername); + +static void getworker(sdata_t *sdata, const char *buf, int *sockd) +{ + char *tmp, *username, *workername = NULL; + worker_instance_t *worker; + user_instance_t *user; + json_error_t err_val; + json_t *val = NULL; + + val = json_loads(buf, 0, &err_val); + if (unlikely(!val)) { + val = json_encode_errormsg(&err_val); + goto out; + } + if (!json_get_string(&workername, val, "worker")) { + val = json_errormsg("Failed to find worker key"); + goto out; + } + if (!strlen(workername)) { + val = json_errormsg("Zero length worker key"); + goto out; + } + tmp = strdupa(workername); + username = strsep(&tmp, "._"); + user = get_user(sdata, username); + worker = get_worker(sdata, user, workername); + JSON_CPACK(val, "{ss,ss,sI,sf,sf,sf,sf,si,sf,si,sb}", + "user", username, "worker", workername, "id", user->id, + "dsps1", worker->dsps1, "dsps5", worker->dsps5, "dsps60", worker->dsps60, + "dsps1440", worker->dsps1440, "lastshare", worker->last_share.tv_sec, + "bestdiff", worker->best_diff, "mindiff", worker->mindiff, "idle", worker->idle); +out: + free(workername); + send_api_response(val, *sockd); + _Close(sockd); +} + /* Return the user masked priority value of the proxy */ static int proxy_prio(const proxy_t *proxy) { @@ -2639,6 +2677,10 @@ retry: getuser(sdata, buf + 8, &sockd); goto retry; } + if (cmdmatch(buf, "getworker")) { + getworker(sdata, buf + 10, &sockd); + goto retry; + } if (cmdmatch(buf, "getproxy")) { getproxy(sdata, buf + 9, &sockd); goto retry; From 1a723cf04d58ecad619037e23bc7d9816fb84805 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Sun, 22 Mar 2015 21:26:32 +1100 Subject: [PATCH 398/544] Reconnect clients to global proxies when we receive an update from the current proxy --- src/stratifier.c | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/src/stratifier.c b/src/stratifier.c index 5939a982..16bc564f 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -1410,10 +1410,8 @@ static void check_bestproxy(sdata_t *sdata) } mutex_unlock(&sdata->proxy_lock); - if (changed_id != -1) { + if (changed_id != -1) LOGNOTICE("Stratifier setting active proxy to %d", changed_id); - reconnect_clients(sdata); - } } static void dead_proxyid(sdata_t *sdata, const int id, const int subid) @@ -1611,6 +1609,17 @@ static void check_userproxies(sdata_t *sdata, const int userid) recruit_best_userproxy(sdata, userid, -headroom); } +static proxy_t *best_proxy(sdata_t *sdata) +{ + proxy_t *proxy; + + mutex_lock(&sdata->proxy_lock); + proxy = sdata->proxy; + mutex_unlock(&sdata->proxy_lock); + + return proxy; +} + static void update_notify(ckpool_t *ckp, const char *cmd) { sdata_t *sdata = ckp->data, *dsdata; @@ -1703,9 +1712,11 @@ static void update_notify(ckpool_t *ckp, const char *cmd) LOGNOTICE("Block hash on proxy %d changed to %s", id, dsdata->lastswaphash); } - if (proxy->global) + if (proxy->global) { check_bestproxy(sdata); - else + if (proxy->parent == best_proxy(sdata)->parent) + reconnect_clients(sdata); + } else check_userproxies(sdata, proxy->userid); clean |= new_block; LOGINFO("Proxy %d:%d broadcast updated stratum notify with%s clean", id, From 888ef2e2d813347ec879d4081696130a8826cd15 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Sun, 22 Mar 2015 21:27:32 +1100 Subject: [PATCH 399/544] Only give the insufficient subproxy message when looking for a global proxy --- src/stratifier.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/stratifier.c b/src/stratifier.c index 16bc564f..817bfef2 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -2902,7 +2902,8 @@ static sdata_t *select_sdata(const ckpool_t *ckp, sdata_t *ckp_sdata, const int mutex_unlock(&ckp_sdata->proxy_lock); if (!best) { - LOGWARNING("Temporarily insufficient subproxies to accept more clients"); + if (!userid) + LOGWARNING("Temporarily insufficient subproxies to accept more clients"); return NULL; } if (!userid) { From 6db9dee72275e650c70c28fc80d4311791eb1d72 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Mon, 23 Mar 2015 11:09:57 +1100 Subject: [PATCH 400/544] Store share statistics fo the upstream proxied pools --- src/generator.c | 45 +++++++++++++++++++++++++++++++++++++-------- 1 file changed, 37 insertions(+), 8 deletions(-) diff --git a/src/generator.c b/src/generator.c index 75e51849..ba0ef4a4 100644 --- a/src/generator.c +++ b/src/generator.c @@ -45,12 +45,16 @@ struct notify_instance { typedef struct notify_instance notify_instance_t; +typedef struct proxy_instance proxy_instance_t; + struct share_msg { UT_hash_handle hh; int id; // Our own id for submitting upstream int client_id; time_t submit_time; + proxy_instance_t *proxy; + double diff; }; typedef struct share_msg share_msg_t; @@ -72,8 +76,6 @@ struct pass_msg { typedef struct pass_msg pass_msg_t; -typedef struct proxy_instance proxy_instance_t; - /* Per proxied pool instance data */ struct proxy_instance { UT_hash_handle hh; /* Proxy list */ @@ -100,6 +102,10 @@ struct proxy_instance { tv_t last_message; double diff; + double diff_accepted; + double diff_rejected; + double total_accepted; /* Used only by parent proxy structures */ + double total_rejected; /* "" */ tv_t last_share; bool no_params; /* Doesn't want any parameters on subscribe */ @@ -1418,6 +1424,8 @@ static void submit_share(gdata_t *gdata, json_t *val) share = ckzalloc(sizeof(share_msg_t)); share->submit_time = time(NULL); share->client_id = client_id; + share->proxy = proxi; + share->diff = proxi->diff; msg->json_msg = val; /* Add new share entry to the share hashtable */ @@ -1449,25 +1457,42 @@ static void clear_notify(notify_instance_t *ni) free(ni); } -/* FIXME: Return something useful to the stratifier based on this result? */ +static void account_shares(proxy_instance_t *proxy, const double diff, const bool result) +{ + proxy_instance_t *parent = proxy->parent; + + mutex_lock(&parent->proxy_lock); + if (result) { + proxy->diff_accepted += diff; + parent->total_accepted += diff; + } else { + proxy->diff_rejected += diff; + parent->total_rejected += diff; + } + mutex_unlock(&parent->proxy_lock); +} + static bool parse_share(gdata_t *gdata, proxy_instance_t *proxi, const char *buf) { + bool ret = false, result = false; json_t *val = NULL, *idval; share_msg_t *share; - bool ret = false; int64_t id; val = json_loads(buf, 0, NULL); - if (!val) { - LOGINFO("Failed to parse json msg: %s", buf); + if (unlikely(!val)) { + LOGINFO("Failed to parse upstream json msg: %s", buf); goto out; } idval = json_object_get(val, "id"); - if (!idval) { - LOGINFO("Failed to find id in json msg: %s", buf); + if (unlikely(!idval)) { + LOGINFO("Failed to find id in upstream json msg: %s", buf); goto out; } id = json_integer_value(idval); + if (unlikely(!json_get_bool(&result, val, "result"))) { + LOGINFO("Failed to find result in upstream json msg: %s", buf); + } mutex_lock(&gdata->share_lock); HASH_FIND_I64(gdata->shares, &id, share); @@ -1481,8 +1506,12 @@ static bool parse_share(gdata_t *gdata, proxy_instance_t *proxi, const char *buf if (!share) { LOGINFO("Proxy %d:%d failed to find matching share to result: %s", proxi->id, proxi->subid, buf); + /* We don't know what diff these shares are so assume the + * current proxy diff. */ + account_shares(share->proxy, share->proxy->diff, result); goto out; } + account_shares(share->proxy, share->diff, result); LOGINFO("Proxy %d:%d share result %s from client %d", proxi->id, proxi->subid, buf, share->client_id); free(share); From 406c07a5ea6378d2715ceea583bf35e605bd512e Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Mon, 23 Mar 2015 11:31:01 +1100 Subject: [PATCH 401/544] Add support for upstream proxy stats via the API --- src/generator.c | 66 +++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 58 insertions(+), 8 deletions(-) diff --git a/src/generator.c b/src/generator.c index ba0ef4a4..09ac75af 100644 --- a/src/generator.c +++ b/src/generator.c @@ -53,7 +53,6 @@ struct share_msg { int client_id; time_t submit_time; - proxy_instance_t *proxy; double diff; }; @@ -1335,8 +1334,6 @@ static proxy_instance_t *subproxy_by_id(proxy_instance_t *proxy, const int subid mutex_lock(&proxy->proxy_lock); subproxy = __subproxy_by_id(proxy, subid); - if (subproxy && subproxy->disabled) - subproxy = NULL; mutex_unlock(&proxy->proxy_lock); return subproxy; @@ -1424,7 +1421,6 @@ static void submit_share(gdata_t *gdata, json_t *val) share = ckzalloc(sizeof(share_msg_t)); share->submit_time = time(NULL); share->client_id = client_id; - share->proxy = proxi; share->diff = proxi->diff; msg->json_msg = val; @@ -1508,10 +1504,10 @@ static bool parse_share(gdata_t *gdata, proxy_instance_t *proxi, const char *buf proxi->id, proxi->subid, buf); /* We don't know what diff these shares are so assume the * current proxy diff. */ - account_shares(share->proxy, share->proxy->diff, result); + account_shares(proxi, proxi->diff, result); goto out; } - account_shares(share->proxy, share->diff, result); + account_shares(proxi, share->diff, result); LOGINFO("Proxy %d:%d share result %s from client %d", proxi->id, proxi->subid, buf, share->client_id); free(share); @@ -2337,7 +2333,6 @@ static void delete_proxy(ckpool_t *ckp, gdata_t *gdata, proxy_instance_t *proxy) static void parse_delproxy(ckpool_t *ckp, gdata_t *gdata, const int sockd, const char *buf) { proxy_instance_t *proxy; - json_error_t err_val; json_t *val = NULL; int id = -1; @@ -2365,7 +2360,6 @@ out: static void parse_ableproxy(gdata_t *gdata, const int sockd, const char *buf, bool disable) { proxy_instance_t *proxy; - json_error_t err_val; json_t *val = NULL; int id = -1; @@ -2458,6 +2452,60 @@ static void send_stats(gdata_t *gdata, const int sockd) send_api_response(val, sockd); } +static void parse_proxystats(gdata_t *gdata, const int sockd, const char *buf) +{ + proxy_instance_t *proxy; + json_error_t err_val; + bool totals = false; + json_t *val = NULL; + int id, subid = 0; + + val = json_loads(buf, 0, &err_val); + if (unlikely(!val)) { + val = json_encode_errormsg(&err_val); + goto out; + } + if (!json_get_int(&id, val, "id")) { + val = json_errormsg("Failed to find id key"); + goto out; + } + if (!json_get_int(&subid, val, "subid")) + totals = true; + proxy = proxy_by_id(gdata, id); + if (!proxy) { + val = json_errormsg("Proxy id %d not found", id); + goto out; + } + if (!totals) + proxy = subproxy_by_id(proxy, subid); + if (!proxy) { + val = json_errormsg("Proxy id %d:%d not found", id, subid); + goto out; + } + if (totals) { + JSON_CPACK(val, "{si,si,ss,ss,ss,ss,si,si,sf,sf,sf,si,sb,sb,sb,sb,sI,si}", + "id", proxy->id, "userid", proxy->userid, "url", proxy->url, + "auth", proxy->auth, "pass", proxy->pass, "enonce1", proxy->enonce1 ? proxy->enonce1 : "", + "nonce1len", proxy->nonce1len, "nonce2len", proxy->nonce2len, "diff", proxy->diff, + "accepted", proxy->total_accepted, "rejected", proxy->total_rejected, + "lastshare", proxy->last_share.tv_sec, "global", proxy->global, "notified", proxy->notified, + "disabled", proxy->disabled, "alive", proxy->alive, "maxclients", proxy->clients_per_proxy, + "subproxies", proxy->subproxy_count); + } else { + JSON_CPACK(val, "{si,si,si,ss,ss,ss,ss,si,si,sf,sf,sf,si,sb,sb,sb,sb,sI,si}", + "id", proxy->id, "subid", proxy->subid, "userid", proxy->userid, "url", proxy->url, + "auth", proxy->auth, "pass", proxy->pass, "enonce1", proxy->enonce1 ? proxy->enonce1 : "", + "nonce1len", proxy->nonce1len, "nonce2len", proxy->nonce2len, "diff", proxy->diff, + "accepted", proxy->diff_accepted, "rejected", proxy->diff_rejected, + "lastshare", proxy->last_share.tv_sec, "global", proxy->global, "notified", proxy->notified, + "disabled", proxy->disabled, "alive", proxy->alive, "maxclients", proxy->clients_per_proxy, + "subproxies", proxy->subproxy_count); + + } +out: + send_api_response(val, sockd); +} + static int proxy_loop(proc_instance_t *pi) { proxy_instance_t *proxi = NULL, *cproxy; @@ -2531,6 +2579,8 @@ retry: parse_ableproxy(gdata, sockd, buf + 12, false); } else if (cmdmatch(buf, "disableproxy")) { parse_ableproxy(gdata, sockd, buf + 13, true); + } else if (cmdmatch(buf, "proxystats")) { + parse_proxystats(gdata, sockd, buf + 11); } else if (cmdmatch(buf, "shutdown")) { ret = 0; goto out; From a13df377d4f1c38ac65e6dc837b2973951c8101c Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Mon, 23 Mar 2015 14:04:22 +1100 Subject: [PATCH 402/544] Add an API command to get a list of clients by username --- src/stratifier.c | 41 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/src/stratifier.c b/src/stratifier.c index 817bfef2..c42538a9 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -2469,6 +2469,43 @@ out: _Close(sockd); } +static void getclients(sdata_t *sdata, const char *buf, int *sockd) +{ + json_t *val = NULL, *client_arr; + stratum_instance_t *client; + char *username = NULL; + user_instance_t *user; + json_error_t err_val; + + val = json_loads(buf, 0, &err_val); + if (unlikely(!val)) { + val = json_encode_errormsg(&err_val); + goto out; + } + if (!json_get_string(&username, val, "user")) { + val = json_errormsg("Failed to find user key"); + goto out; + } + if (!strlen(username)) { + val = json_errormsg("Zero length user key"); + goto out; + } + user = get_user(sdata, username); + client_arr = json_array(); + + ck_rlock(&sdata->instance_lock); + DL_FOREACH(user->clients, client) { + json_array_append_new(client_arr, json_integer(client->id)); + } + ck_runlock(&sdata->instance_lock); + + JSON_CPACK(val, "{so}", "clients", client_arr); +out: + free(username); + send_api_response(val, *sockd); + _Close(sockd); +} + static worker_instance_t *get_worker(sdata_t *sdata, user_instance_t *user, const char *workername); static void getworker(sdata_t *sdata, const char *buf, int *sockd) @@ -2692,6 +2729,10 @@ retry: getworker(sdata, buf + 10, &sockd); goto retry; } + if (cmdmatch(buf, "userclients")) { + getclients(sdata, buf + 12, &sockd); + goto retry; + } if (cmdmatch(buf, "getproxy")) { getproxy(sdata, buf + 9, &sockd); goto retry; From eb529da3b261bd8a168f94a51b391edc08ef08a6 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Mon, 23 Mar 2015 14:14:30 +1100 Subject: [PATCH 403/544] Add an API command to get a list of clients by workername --- src/stratifier.c | 51 +++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 48 insertions(+), 3 deletions(-) diff --git a/src/stratifier.c b/src/stratifier.c index c42538a9..571db59b 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -2469,7 +2469,7 @@ out: _Close(sockd); } -static void getclients(sdata_t *sdata, const char *buf, int *sockd) +static void userclients(sdata_t *sdata, const char *buf, int *sockd) { json_t *val = NULL, *client_arr; stratum_instance_t *client; @@ -2499,13 +2499,54 @@ static void getclients(sdata_t *sdata, const char *buf, int *sockd) } ck_runlock(&sdata->instance_lock); - JSON_CPACK(val, "{so}", "clients", client_arr); + JSON_CPACK(val, "{ss,so}", "user", username, "clients", client_arr); out: free(username); send_api_response(val, *sockd); _Close(sockd); } +static void workerclients(sdata_t *sdata, const char *buf, int *sockd) +{ + char *tmp, *username, *workername = NULL; + json_t *val = NULL, *client_arr; + stratum_instance_t *client; + user_instance_t *user; + json_error_t err_val; + + val = json_loads(buf, 0, &err_val); + if (unlikely(!val)) { + val = json_encode_errormsg(&err_val); + goto out; + } + if (!json_get_string(&workername, val, "worker")) { + val = json_errormsg("Failed to find worker key"); + goto out; + } + if (!strlen(workername)) { + val = json_errormsg("Zero length worker key"); + goto out; + } + tmp = strdupa(workername); + username = strsep(&tmp, "._"); + user = get_user(sdata, username); + client_arr = json_array(); + + ck_rlock(&sdata->instance_lock); + DL_FOREACH(user->clients, client) { + if (strcmp(client->workername, workername)) + continue; + json_array_append_new(client_arr, json_integer(client->id)); + } + ck_runlock(&sdata->instance_lock); + + JSON_CPACK(val, "{ss,so}", "worker", workername, "clients", client_arr); +out: + free(workername); + send_api_response(val, *sockd); + _Close(sockd); +} + static worker_instance_t *get_worker(sdata_t *sdata, user_instance_t *user, const char *workername); static void getworker(sdata_t *sdata, const char *buf, int *sockd) @@ -2730,7 +2771,11 @@ retry: goto retry; } if (cmdmatch(buf, "userclients")) { - getclients(sdata, buf + 12, &sockd); + userclients(sdata, buf + 12, &sockd); + goto retry; + } + if (cmdmatch(buf, "workerclients")) { + workerclients(sdata, buf + 14, &sockd); goto retry; } if (cmdmatch(buf, "getproxy")) { From ae5b119900d4e3cb4a59c54f36d080cf00a73810 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Mon, 23 Mar 2015 14:47:34 +1100 Subject: [PATCH 404/544] Add an API command to get statistics per client --- src/stratifier.c | 57 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 57 insertions(+) diff --git a/src/stratifier.c b/src/stratifier.c index 571db59b..1c3d8a9a 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -2585,6 +2585,59 @@ out: _Close(sockd); } +static void getclient(sdata_t *sdata, const char *buf, int *sockd) +{ + stratum_instance_t *client; + json_error_t err_val; + json_t *val = NULL; + int64_t client_id; + + val = json_loads(buf, 0, &err_val); + if (unlikely(!val)) { + val = json_encode_errormsg(&err_val); + goto out; + } + if (!json_get_int64(&client_id, val, "id")) { + val = json_errormsg("Failed to find id key"); + goto out; + } + client = ref_instance_by_id(sdata, client_id); + if (!client) { + val = json_errormsg("Failed to find client %"PRId64, client_id); + goto out; + } + /* Too many fields for a pack object, do each discretely to keep track */ + val = json_object(); + json_set_int(val, "id", client->id); + json_set_string(val, "enonce1", client->enonce1); + json_set_string(val, "enonce1var", client->enonce1var); + json_set_int(val, "enonce1_64", client->enonce1_64); + json_set_double(val, "diff", client->diff); + json_set_double(val, "dsps1", client->dsps1); + json_set_double(val, "dsps5", client->dsps5); + json_set_double(val, "dsps60", client->dsps60); + json_set_double(val, "dsps1440", client->dsps1440); + json_set_double(val, "dsps10080", client->dsps10080); + json_set_int(val, "lastshare", client->last_share.tv_sec); + json_set_int(val, "starttime", client->start_time); + json_set_string(val, "address", client->address); + json_set_bool(val, "subscribed", client->subscribed); + json_set_bool(val, "authorised", client->authorised); + json_set_bool(val, "idle", client->idle); + json_set_string(val, "useragent", client->useragent ? client->useragent : ""); + json_set_string(val, "workername", client->workername ? client->workername : ""); + json_set_int(val, "userid", client->user_id); + json_set_int(val, "server", client->server); + json_set_double(val, "bestdiff", client->best_diff); + json_set_int(val, "proxyid", client->proxyid); + json_set_int(val, "subproxyid", client->subproxyid); + + dec_instance_ref(sdata, client); +out: + send_api_response(val, *sockd); + _Close(sockd); +} + /* Return the user masked priority value of the proxy */ static int proxy_prio(const proxy_t *proxy) { @@ -2762,6 +2815,10 @@ retry: goto retry; } /* Parse API commands here to return a message to sockd */ + if (cmdmatch(buf, "getclient")) { + getclient(sdata, buf + 10, &sockd); + goto retry; + } if (cmdmatch(buf, "getuser")) { getuser(sdata, buf + 8, &sockd); goto retry; From af1841001a27d6770b1064dadb63fca654a9f447 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Mon, 23 Mar 2015 15:26:44 +1100 Subject: [PATCH 406/544] Fix missing field --- src/stratifier.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/stratifier.c b/src/stratifier.c index 1c3d8a9a..1a4a0141 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -2651,7 +2651,7 @@ static json_t *json_proxyinfo(const proxy_t *proxy) const proxy_t *parent = proxy->parent; json_t *val; - JSON_CPACK(val, "{si,si,si,sf,ss,ss,ss,ss,si,si,si,si,sb,sb,sI,sI,sI,sI,si,sb,sb,si}", + JSON_CPACK(val, "{si,si,si,sf,ss,ss,ss,ss,si,si,si,si,sb,sb,sI,sI,sI,sI,si,si,sb,sb,si}", "id", proxy->id, "subid", proxy->subid, "priority", proxy_prio(parent), "diff", proxy->diff, "url", proxy->url, "auth", proxy->auth, "pass", proxy->pass, "enonce1", proxy->enonce1, "enonce1constlen", proxy->enonce1constlen, From e1fe8aaebff2ba4050114df829843e01f6200eb8 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Mon, 23 Mar 2015 15:53:29 +1100 Subject: [PATCH 407/544] Add an API command to return a full list or optionally a list by userid of all proxies and their stats --- src/stratifier.c | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/src/stratifier.c b/src/stratifier.c index 1a4a0141..910d88f7 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -2695,6 +2695,34 @@ out: _Close(sockd); } +static void proxyinfo(sdata_t *sdata, const char *buf, int *sockd) +{ + json_t *val = NULL, *arr_val = json_array(); + proxy_t *proxy, *subproxy; + bool all = true; + int userid = 0; + + if (buf) { + /* See if there's a userid specified */ + val = json_loads(buf, 0, NULL); + if (json_get_int(&userid, val, "userid")) + all = false; + } + + mutex_lock(&sdata->proxy_lock); + for (proxy = sdata->proxies; proxy; proxy = proxy->hh.next) { + if (!all && proxy->userid != userid) + continue; + for (subproxy = proxy->subproxies; subproxy; subproxy = subproxy->sh.next) + json_array_append_new(arr_val, json_proxyinfo(subproxy)); + } + mutex_unlock(&sdata->proxy_lock); + + JSON_CPACK(val, "{so}", "proxies", arr_val); + send_api_response(val, *sockd); + _Close(sockd); +} + static void setproxy(sdata_t *sdata, const char *buf, int *sockd) { json_error_t err_val; @@ -2843,6 +2871,10 @@ retry: setproxy(sdata, buf + 9, &sockd); goto retry; } + if (cmdmatch(buf, "proxyinfo")) { + proxyinfo(sdata, buf + 10, &sockd); + goto retry; + } Close(sockd); LOGDEBUG("Stratifier received request: %s", buf); From cda20cf00c478404c13840cb66f15d7e27b31f9b Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Mon, 23 Mar 2015 16:30:22 +1100 Subject: [PATCH 408/544] Add API commands to get all the client info by user or worker --- src/stratifier.c | 133 ++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 113 insertions(+), 20 deletions(-) diff --git a/src/stratifier.c b/src/stratifier.c index 910d88f7..2fcec69c 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -2585,29 +2585,11 @@ out: _Close(sockd); } -static void getclient(sdata_t *sdata, const char *buf, int *sockd) +static json_t *clientinfo(const stratum_instance_t *client) { - stratum_instance_t *client; - json_error_t err_val; - json_t *val = NULL; - int64_t client_id; + json_t *val = json_object(); - val = json_loads(buf, 0, &err_val); - if (unlikely(!val)) { - val = json_encode_errormsg(&err_val); - goto out; - } - if (!json_get_int64(&client_id, val, "id")) { - val = json_errormsg("Failed to find id key"); - goto out; - } - client = ref_instance_by_id(sdata, client_id); - if (!client) { - val = json_errormsg("Failed to find client %"PRId64, client_id); - goto out; - } /* Too many fields for a pack object, do each discretely to keep track */ - val = json_object(); json_set_int(val, "id", client->id); json_set_string(val, "enonce1", client->enonce1); json_set_string(val, "enonce1var", client->enonce1var); @@ -2631,6 +2613,31 @@ static void getclient(sdata_t *sdata, const char *buf, int *sockd) json_set_double(val, "bestdiff", client->best_diff); json_set_int(val, "proxyid", client->proxyid); json_set_int(val, "subproxyid", client->subproxyid); + return val; +} + +static void getclient(sdata_t *sdata, const char *buf, int *sockd) +{ + stratum_instance_t *client; + json_error_t err_val; + json_t *val = NULL; + int64_t client_id; + + val = json_loads(buf, 0, &err_val); + if (unlikely(!val)) { + val = json_encode_errormsg(&err_val); + goto out; + } + if (!json_get_int64(&client_id, val, "id")) { + val = json_errormsg("Failed to find id key"); + goto out; + } + client = ref_instance_by_id(sdata, client_id); + if (!client) { + val = json_errormsg("Failed to find client %"PRId64, client_id); + goto out; + } + val = clientinfo(client); dec_instance_ref(sdata, client); out: @@ -2638,6 +2645,84 @@ out: _Close(sockd); } +static void user_clientinfo(sdata_t *sdata, const char *buf, int *sockd) +{ + json_t *val = NULL, *client_arr; + stratum_instance_t *client; + char *username = NULL; + user_instance_t *user; + json_error_t err_val; + + val = json_loads(buf, 0, &err_val); + if (unlikely(!val)) { + val = json_encode_errormsg(&err_val); + goto out; + } + if (!json_get_string(&username, val, "user")) { + val = json_errormsg("Failed to find user key"); + goto out; + } + if (!strlen(username)) { + val = json_errormsg("Zero length user key"); + goto out; + } + user = get_user(sdata, username); + client_arr = json_array(); + + ck_rlock(&sdata->instance_lock); + DL_FOREACH(user->clients, client) { + json_array_append_new(client_arr, clientinfo(client)); + } + ck_runlock(&sdata->instance_lock); + + JSON_CPACK(val, "{ss,so}", "user", username, "clients", client_arr); +out: + free(username); + send_api_response(val, *sockd); + _Close(sockd); +} + +static void worker_clientinfo(sdata_t *sdata, const char *buf, int *sockd) +{ + char *tmp, *username, *workername = NULL; + json_t *val = NULL, *client_arr; + stratum_instance_t *client; + user_instance_t *user; + json_error_t err_val; + + val = json_loads(buf, 0, &err_val); + if (unlikely(!val)) { + val = json_encode_errormsg(&err_val); + goto out; + } + if (!json_get_string(&workername, val, "worker")) { + val = json_errormsg("Failed to find worker key"); + goto out; + } + if (!strlen(workername)) { + val = json_errormsg("Zero length worker key"); + goto out; + } + tmp = strdupa(workername); + username = strsep(&tmp, "._"); + user = get_user(sdata, username); + client_arr = json_array(); + + ck_rlock(&sdata->instance_lock); + DL_FOREACH(user->clients, client) { + if (strcmp(client->workername, workername)) + continue; + json_array_append_new(client_arr, clientinfo(client)); + } + ck_runlock(&sdata->instance_lock); + + JSON_CPACK(val, "{ss,so}", "worker", workername, "clients", client_arr); +out: + free(workername); + send_api_response(val, *sockd); + _Close(sockd); +} + /* Return the user masked priority value of the proxy */ static int proxy_prio(const proxy_t *proxy) { @@ -2875,6 +2960,14 @@ retry: proxyinfo(sdata, buf + 10, &sockd); goto retry; } + if (cmdmatch(buf, "ucinfo")) { + user_clientinfo(sdata, buf + 7, &sockd); + goto retry; + } + if (cmdmatch(buf, "wcinfo")) { + worker_clientinfo(sdata, buf + 7, &sockd); + goto retry; + } Close(sockd); LOGDEBUG("Stratifier received request: %s", buf); From c1b4f6e490c3372e85ae68003eac990156ed2ecf Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Mon, 23 Mar 2015 16:41:26 +1100 Subject: [PATCH 409/544] Add an API command to get the info of every single client --- src/stratifier.c | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/src/stratifier.c b/src/stratifier.c index 2fcec69c..acce72af 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -2645,6 +2645,24 @@ out: _Close(sockd); } +static void getclients(sdata_t *sdata, int *sockd) +{ + json_t *val = NULL, *client_arr; + stratum_instance_t *client; + + client_arr = json_array(); + + ck_rlock(&sdata->instance_lock); + for (client = sdata->stratum_instances; client; client = client->hh.next) { + json_array_append_new(client_arr, clientinfo(client)); + } + ck_runlock(&sdata->instance_lock); + + JSON_CPACK(val, "{so}", "clients", client_arr); + send_api_response(val, *sockd); + _Close(sockd); +} + static void user_clientinfo(sdata_t *sdata, const char *buf, int *sockd) { json_t *val = NULL, *client_arr; @@ -2928,6 +2946,10 @@ retry: goto retry; } /* Parse API commands here to return a message to sockd */ + if (cmdmatch(buf, "clients")) { + getclients(sdata, &sockd); + goto retry; + } if (cmdmatch(buf, "getclient")) { getclient(sdata, buf + 10, &sockd); goto retry; From 23d6cca7eeeaffff75d8c933ef02787fb948fe15 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Mon, 23 Mar 2015 16:53:22 +1100 Subject: [PATCH 410/544] Add an API command to list all workers --- src/stratifier.c | 43 ++++++++++++++++++++++++++++++++++++++----- 1 file changed, 38 insertions(+), 5 deletions(-) diff --git a/src/stratifier.c b/src/stratifier.c index acce72af..b542b71f 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -2549,6 +2549,18 @@ out: static worker_instance_t *get_worker(sdata_t *sdata, user_instance_t *user, const char *workername); +static json_t *workerinfo(const user_instance_t *user, const worker_instance_t *worker) +{ + json_t *val; + + JSON_CPACK(val, "{ss,ss,sI,sf,sf,sf,sf,si,sf,si,sb}", + "user", user->username, "worker", worker->workername, "id", user->id, + "dsps1", worker->dsps1, "dsps5", worker->dsps5, "dsps60", worker->dsps60, + "dsps1440", worker->dsps1440, "lastshare", worker->last_share.tv_sec, + "bestdiff", worker->best_diff, "mindiff", worker->mindiff, "idle", worker->idle); + return val; +} + static void getworker(sdata_t *sdata, const char *buf, int *sockd) { char *tmp, *username, *workername = NULL; @@ -2574,17 +2586,34 @@ static void getworker(sdata_t *sdata, const char *buf, int *sockd) username = strsep(&tmp, "._"); user = get_user(sdata, username); worker = get_worker(sdata, user, workername); - JSON_CPACK(val, "{ss,ss,sI,sf,sf,sf,sf,si,sf,si,sb}", - "user", username, "worker", workername, "id", user->id, - "dsps1", worker->dsps1, "dsps5", worker->dsps5, "dsps60", worker->dsps60, - "dsps1440", worker->dsps1440, "lastshare", worker->last_share.tv_sec, - "bestdiff", worker->best_diff, "mindiff", worker->mindiff, "idle", worker->idle); + val = workerinfo(user, worker); out: free(workername); send_api_response(val, *sockd); _Close(sockd); } +static void getworkers(sdata_t *sdata, int *sockd) +{ + json_t *val = NULL, *worker_arr; + worker_instance_t *worker; + user_instance_t *user; + + worker_arr = json_array(); + + ck_rlock(&sdata->instance_lock); + for (user = sdata->user_instances; user; user = user->hh.next) { + DL_FOREACH(user->worker_instances, worker) { + json_array_append_new(worker_arr, workerinfo(user, worker)); + } + } + ck_runlock(&sdata->instance_lock); + + JSON_CPACK(val, "{so}", "workers", worker_arr); + send_api_response(val, *sockd); + _Close(sockd); +} + static json_t *clientinfo(const stratum_instance_t *client) { json_t *val = json_object(); @@ -2950,6 +2979,10 @@ retry: getclients(sdata, &sockd); goto retry; } + if (cmdmatch(buf, "workers")) { + getworkers(sdata, &sockd); + goto retry; + } if (cmdmatch(buf, "getclient")) { getclient(sdata, buf + 10, &sockd); goto retry; From a3819b873938712fce7f9bcc93fe52b57b8e076b Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Mon, 23 Mar 2015 16:57:20 +1100 Subject: [PATCH 411/544] Add an API command to list all workers --- src/stratifier.c | 40 +++++++++++++++++++++++++++++++++++----- 1 file changed, 35 insertions(+), 5 deletions(-) diff --git a/src/stratifier.c b/src/stratifier.c index b542b71f..25548652 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -2437,6 +2437,18 @@ static void lazy_reconnect_client(sdata_t *sdata, stratum_instance_t *client) static user_instance_t *get_user(sdata_t *sdata, const char *username); +static json_t *userinfo(const user_instance_t *user) +{ + json_t *val; + + JSON_CPACK(val, "{ss,sI,si,sf,sf,sf,sf,sf,sf,si}", + "user", user->username, "id", user->id, "workers", user->workers, + "bestdiff", user->best_diff, "dsps1", user->dsps1, "dsps5", user->dsps5, + "dsps60", user->dsps60, "dsps1440", user->dsps1440, "dsps10080", user->dsps10080, + "lastshare", user->last_share.tv_sec); + return val; +} + static void getuser(sdata_t *sdata, const char *buf, int *sockd) { char *username = NULL; @@ -2458,11 +2470,7 @@ static void getuser(sdata_t *sdata, const char *buf, int *sockd) goto out; } user = get_user(sdata, username); - JSON_CPACK(val, "{ss,sI,si,sf,sf,sf,sf,sf,sf,si}", - "user", user->username, "id", user->id, "workers", user->workers, - "bestdiff", user->best_diff, "dsps1", user->dsps1, "dsps5", user->dsps5, - "dsps60", user->dsps60, "dsps1440", user->dsps1440, "dsps10080", user->dsps10080, - "lastshare", user->last_share.tv_sec); + val = userinfo(user); out: free(username); send_api_response(val, *sockd); @@ -2614,6 +2622,24 @@ static void getworkers(sdata_t *sdata, int *sockd) _Close(sockd); } +static void getusers(sdata_t *sdata, int *sockd) +{ + json_t *val = NULL, *user_array; + user_instance_t *user; + + user_array = json_array(); + + ck_rlock(&sdata->instance_lock); + for (user = sdata->user_instances; user; user = user->hh.next) { + json_array_append_new(user_array, userinfo(user)); + } + ck_runlock(&sdata->instance_lock); + + JSON_CPACK(val, "{so}", "users", user_array); + send_api_response(val, *sockd); + _Close(sockd); +} + static json_t *clientinfo(const stratum_instance_t *client) { json_t *val = json_object(); @@ -2983,6 +3009,10 @@ retry: getworkers(sdata, &sockd); goto retry; } + if (cmdmatch(buf, "users")) { + getusers(sdata, &sockd); + goto retry; + } if (cmdmatch(buf, "getclient")) { getclient(sdata, buf + 10, &sockd); goto retry; From 47a6877fef151b2befdbf95b3c3127b8f7626564 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Tue, 24 Mar 2015 11:48:55 +1100 Subject: [PATCH 412/544] Add an API command for global pool stats --- src/stratifier.c | 44 +++++++++++++++++++++++++++++++++++--------- 1 file changed, 35 insertions(+), 9 deletions(-) diff --git a/src/stratifier.c b/src/stratifier.c index f9f6c98e..6831e644 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -2445,6 +2445,21 @@ static void lazy_reconnect_client(sdata_t *sdata, stratum_instance_t *client) } } +static void reconnect_client_id(sdata_t *sdata, const int64_t client_id) +{ + stratum_instance_t *client; + + client = ref_instance_by_id(sdata, client_id); + if (!client) { + LOGINFO("reconnect_client_id failed to find client %"PRId64, client_id); + return; + } + lazy_reconnect_client(sdata, client); + dec_instance_ref(sdata, client); +} + +/* API commands */ + static user_instance_t *get_user(sdata_t *sdata, const char *username); static json_t *userinfo(const user_instance_t *user) @@ -2924,17 +2939,24 @@ out: _Close(sockd); } -static void reconnect_client_id(sdata_t *sdata, const int64_t client_id) +static void get_poolstats(sdata_t *sdata, int *sockd) { - stratum_instance_t *client; + pool_stats_t *stats = &sdata->stats; + json_t *val; - client = ref_instance_by_id(sdata, client_id); - if (!client) { - LOGINFO("reconnect_client_id failed to find client %"PRId64, client_id); - return; - } - lazy_reconnect_client(sdata, client); - dec_instance_ref(sdata, client); + mutex_lock(&sdata->stats_lock); + JSON_CPACK(val, "{si,si,si,si,si,sI,sf,sf,sf,sf,sI,sI,sf,sf,sf,sf,sf,sf,sf}", + "start", stats->start_time.tv_sec, "update", stats->last_update.tv_sec, + "workers", stats->workers, "users", stats->users, "disconnected", stats->disconnected, + "shares", stats->accounted_shares, "sps1", stats->sps1, "sps5", stats->sps5, + "sps15", stats->sps15, "sps60", stats->sps60, "accepted", stats->accounted_diff_shares, + "rejected", stats->accounted_rejects, "dsps1", stats->dsps1, "dsps5", stats->dsps5, + "dsps15", stats->dsps15, "dsps60", stats->dsps60, "dsps360", stats->dsps360, + "dsps1440", stats->dsps1440, "dsps10080", stats->dsps10080); + mutex_unlock(&sdata->stats_lock); + + send_api_response(val, *sockd); + _Close(sockd); } static int stratum_loop(ckpool_t *ckp, proc_instance_t *pi) @@ -3051,6 +3073,10 @@ retry: setproxy(sdata, buf + 9, &sockd); goto retry; } + if (cmdmatch(buf, "poolstats")) { + get_poolstats(sdata, &sockd); + goto retry; + } if (cmdmatch(buf, "proxyinfo")) { proxyinfo(sdata, buf + 10, &sockd); goto retry; From e042ec330f548084bed47cc534ae9d3043525863 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Tue, 24 Mar 2015 12:00:58 +1100 Subject: [PATCH 413/544] Clean up parsing of proxy.stats API command, removing unused notified bool --- src/generator.c | 49 ++++++++++++++++++++++++++----------------------- 1 file changed, 26 insertions(+), 23 deletions(-) diff --git a/src/generator.c b/src/generator.c index 09ac75af..0208f630 100644 --- a/src/generator.c +++ b/src/generator.c @@ -110,7 +110,6 @@ struct proxy_instance { bool no_params; /* Doesn't want any parameters on subscribe */ bool global; /* Part of the global list of proxies */ - bool notified; /* Has this proxy received any notifies yet */ bool disabled; /* Subproxy no longer to be used */ bool reconnect; /* We need to drop and reconnect */ bool reconnecting; /* Testing of parent in progress */ @@ -2160,10 +2159,10 @@ static void send_list(gdata_t *gdata, const int sockd) mutex_lock(&gdata->lock); HASH_ITER(hh, gdata->proxies, proxy, tmp) { - JSON_CPACK(val, "{si,sb,si,ss,ss,sf,sb,sb,sb,si}", + JSON_CPACK(val, "{si,sb,si,ss,ss,sf,sb,sb,si}", "id", proxy->id, "global", proxy->global, "userid", proxy->userid, "auth", proxy->auth, "pass", proxy->pass, - "diff", proxy->diff, "notified", proxy->notified, + "diff", proxy->diff, "disabled", proxy->disabled, "alive", proxy->alive, "subproxies", proxy->subproxy_count); if (proxy->enonce1) { @@ -2205,10 +2204,10 @@ static void send_sublist(gdata_t *gdata, const int sockd, const char *buf) mutex_lock(&gdata->lock); HASH_ITER(sh, proxy->subproxies, subproxy, tmp) { - JSON_CPACK(val, "{si,ss,ss,sf,sb,sb,sb}", + JSON_CPACK(val, "{si,ss,ss,sf,sb,sb}", "subid", subproxy->id, "auth", subproxy->auth, "pass", subproxy->pass, - "diff", subproxy->diff, "notified", subproxy->notified, + "diff", subproxy->diff, "disabled", subproxy->disabled, "alive", subproxy->alive); if (subproxy->enonce1) { json_set_string(val, "enonce1", subproxy->enonce1); @@ -2482,26 +2481,30 @@ static void parse_proxystats(gdata_t *gdata, const int sockd, const char *buf) val = json_errormsg("Proxy id %d:%d not found", id, subid); goto out; } + val = json_object(); + json_set_int(val, "id", proxy->id); + json_set_int(val, "userid", proxy->userid); + json_set_string(val, "url", proxy->url); + json_set_string(val, "auth", proxy->auth); + json_set_string(val, "pass", proxy->pass); + json_set_string(val, "enonce1", proxy->enonce1 ? proxy->enonce1 : ""); + json_set_int(val, "nonce1len", proxy->nonce1len); + json_set_int(val, "nonce2len", proxy->nonce2len); + json_set_double(val, "diff", proxy->diff); if (totals) { - JSON_CPACK(val, "{si,si,ss,ss,ss,ss,si,si,sf,sf,sf,si,sb,sb,sb,sb,sI,si}", - "id", proxy->id, "userid", proxy->userid, "url", proxy->url, - "auth", proxy->auth, "pass", proxy->pass, "enonce1", proxy->enonce1 ? proxy->enonce1 : "", - "nonce1len", proxy->nonce1len, "nonce2len", proxy->nonce2len, "diff", proxy->diff, - "accepted", proxy->total_accepted, "rejected", proxy->total_rejected, - "lastshare", proxy->last_share.tv_sec, "global", proxy->global, "notified", proxy->notified, - "disabled", proxy->disabled, "alive", proxy->alive, "maxclients", proxy->clients_per_proxy, - "subproxies", proxy->subproxy_count); + json_set_double(val, "accepted", proxy->total_accepted); + json_set_double(val, "rejected", proxy->total_rejected); } else { - JSON_CPACK(val, "{si,si,si,ss,ss,ss,ss,si,si,sf,sf,sf,si,sb,sb,sb,sb,sI,si}", - "id", proxy->id, "subid", proxy->subid, "userid", proxy->userid, "url", proxy->url, - "auth", proxy->auth, "pass", proxy->pass, "enonce1", proxy->enonce1 ? proxy->enonce1 : "", - "nonce1len", proxy->nonce1len, "nonce2len", proxy->nonce2len, "diff", proxy->diff, - "accepted", proxy->diff_accepted, "rejected", proxy->diff_rejected, - "lastshare", proxy->last_share.tv_sec, "global", proxy->global, "notified", proxy->notified, - "disabled", proxy->disabled, "alive", proxy->alive, "maxclients", proxy->clients_per_proxy, - "subproxies", proxy->subproxy_count); - - } + json_set_double(val, "accepted", proxy->diff_accepted); + json_set_double(val, "rejected", proxy->diff_rejected); + } + json_set_int(val, "lastshare", proxy->last_share.tv_sec); + json_set_bool(val, "global", proxy->global); + json_set_bool(val, "disabled", proxy->disabled); + json_set_bool(val, "alive", proxy->alive); + json_set_int(val, "maxclients", proxy->clients_per_proxy); + if (parent_proxy(proxy)) + json_set_int(val, "subproxies", proxy->subproxy_count); out: send_api_response(val, sockd); } From 154c216b46a009ac236a45ccd5ffd9387c745d17 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Tue, 24 Mar 2015 12:07:15 +1100 Subject: [PATCH 414/544] More cleanup of proxy stats API --- src/generator.c | 56 +++++++++++++++++++++++++++---------------------- 1 file changed, 31 insertions(+), 25 deletions(-) diff --git a/src/generator.c b/src/generator.c index 0208f630..74e3e784 100644 --- a/src/generator.c +++ b/src/generator.c @@ -568,7 +568,7 @@ out: return buf; } -static inline bool parent_proxy(proxy_instance_t *proxy) +static inline bool parent_proxy(const proxy_instance_t *proxy) { return (proxy->parent == proxy); } @@ -2451,6 +2451,35 @@ static void send_stats(gdata_t *gdata, const int sockd) send_api_response(val, sockd); } +static json_t *proxystats(const proxy_instance_t *proxy) +{ + json_t *val; + + val = json_object(); + json_set_int(val, "id", proxy->id); + json_set_int(val, "userid", proxy->userid); + json_set_string(val, "url", proxy->url); + json_set_string(val, "auth", proxy->auth); + json_set_string(val, "pass", proxy->pass); + json_set_string(val, "enonce1", proxy->enonce1 ? proxy->enonce1 : ""); + json_set_int(val, "nonce1len", proxy->nonce1len); + json_set_int(val, "nonce2len", proxy->nonce2len); + json_set_double(val, "diff", proxy->diff); + if (parent_proxy(proxy)) { + json_set_double(val, "total_accepted", proxy->total_accepted); + json_set_double(val, "total_rejected", proxy->total_rejected); + json_set_int(val, "subproxies", proxy->subproxy_count); + } + json_set_double(val, "accepted", proxy->diff_accepted); + json_set_double(val, "rejected", proxy->diff_rejected); + json_set_int(val, "lastshare", proxy->last_share.tv_sec); + json_set_bool(val, "global", proxy->global); + json_set_bool(val, "disabled", proxy->disabled); + json_set_bool(val, "alive", proxy->alive); + json_set_int(val, "maxclients", proxy->clients_per_proxy); + return val; +} + static void parse_proxystats(gdata_t *gdata, const int sockd, const char *buf) { proxy_instance_t *proxy; @@ -2481,30 +2510,7 @@ static void parse_proxystats(gdata_t *gdata, const int sockd, const char *buf) val = json_errormsg("Proxy id %d:%d not found", id, subid); goto out; } - val = json_object(); - json_set_int(val, "id", proxy->id); - json_set_int(val, "userid", proxy->userid); - json_set_string(val, "url", proxy->url); - json_set_string(val, "auth", proxy->auth); - json_set_string(val, "pass", proxy->pass); - json_set_string(val, "enonce1", proxy->enonce1 ? proxy->enonce1 : ""); - json_set_int(val, "nonce1len", proxy->nonce1len); - json_set_int(val, "nonce2len", proxy->nonce2len); - json_set_double(val, "diff", proxy->diff); - if (totals) { - json_set_double(val, "accepted", proxy->total_accepted); - json_set_double(val, "rejected", proxy->total_rejected); - } else { - json_set_double(val, "accepted", proxy->diff_accepted); - json_set_double(val, "rejected", proxy->diff_rejected); - } - json_set_int(val, "lastshare", proxy->last_share.tv_sec); - json_set_bool(val, "global", proxy->global); - json_set_bool(val, "disabled", proxy->disabled); - json_set_bool(val, "alive", proxy->alive); - json_set_int(val, "maxclients", proxy->clients_per_proxy); - if (parent_proxy(proxy)) - json_set_int(val, "subproxies", proxy->subproxy_count); + val = proxystats(proxy); out: send_api_response(val, sockd); } From dc31479af7fa8980a6f64adeab7aaed9274b0e7e Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Tue, 24 Mar 2015 12:08:38 +1100 Subject: [PATCH 415/544] Consistent maxclients key use --- src/stratifier.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/stratifier.c b/src/stratifier.c index 6831e644..30bf70e5 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -2840,7 +2840,7 @@ static json_t *json_proxyinfo(const proxy_t *proxy) "enonce1", proxy->enonce1, "enonce1constlen", proxy->enonce1constlen, "enonce1varlen", proxy->enonce1varlen, "nonce2len", proxy->nonce2len, "enonce2varlen", proxy->enonce2varlen, "subscribed", proxy->subscribed, - "notified", proxy->notified, "clients", proxy->clients, "max_clients", proxy->max_clients, + "notified", proxy->notified, "clients", proxy->clients, "maxclients", proxy->max_clients, "bound_clients", proxy->bound_clients, "combined_clients", parent->combined_clients, "headroom", proxy->headroom, "subproxy_count", parent->subproxy_count, "dead", proxy->dead, "global", proxy->global, "userid", proxy->userid); From 24c2a4a7bf098873b297903e547051d1ff316ad5 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Mon, 20 Apr 2015 16:03:54 +1000 Subject: [PATCH 416/544] Convert the generator loop to use unix receive queues --- src/generator.c | 63 ++++++++++++++++++++++++------------------------- 1 file changed, 31 insertions(+), 32 deletions(-) diff --git a/src/generator.c b/src/generator.c index 74e3e784..1a06206c 100644 --- a/src/generator.c +++ b/src/generator.c @@ -267,12 +267,20 @@ static void kill_server(server_instance_t *si) dealloc(si->data); } +static void clear_unix_msg(unix_msg_t **umsg) +{ + if (*umsg) { + free((*umsg)->buf); + free(*umsg); + *umsg = NULL; + } +} + static int gen_loop(proc_instance_t *pi) { - int sockd = -1, ret = 0, selret; server_instance_t *si = NULL; bool reconnecting = false; - unixsock_t *us = &pi->us; + unix_msg_t *umsg = NULL; ckpool_t *ckp = pi->ckp; gdata_t *gdata = ckp->data; bool started = false; @@ -280,9 +288,10 @@ static int gen_loop(proc_instance_t *pi) connsock_t *cs; gbtbase_t *gbt; char hash[68]; + int ret = 0; reconnect: - Close(sockd); + clear_unix_msg(&umsg); if (si) { kill_server(si); reconnecting = true; @@ -299,36 +308,24 @@ reconnect: } retry: - Close(sockd); + clear_unix_msg(&umsg); ckmsgq_add(gdata->srvchk, si); do { - selret = wait_read_select(us->sockd, 5); - if (!selret && !ping_main(ckp)) { + umsg = get_unix_msg(pi); + if (unlikely(!umsg &&!ping_main(ckp))) { LOGEMERG("Generator failed to ping main process, exiting"); ret = 1; goto out; } - } while (selret < 1); + } while (!umsg); if (unlikely(cs->fd < 0)) { LOGWARNING("Bitcoind socket invalidated, will attempt failover"); goto reconnect; } - sockd = accept(us->sockd, NULL, NULL); - if (sockd < 0) { - LOGEMERG("Failed to accept on generator socket"); - ret = 1; - goto out; - } - - dealloc(buf); - buf = recv_unix_msg(sockd); - if (!buf) { - LOGWARNING("Failed to get message in gen_loop"); - goto retry; - } + buf = umsg->buf; LOGDEBUG("Generator received request: %s", buf); if (cmdmatch(buf, "shutdown")) { ret = 0; @@ -338,41 +335,41 @@ retry: if (!gen_gbtbase(cs, gbt)) { LOGWARNING("Failed to get block template from %s:%s", cs->url, cs->port); - send_unix_msg(sockd, "Failed"); + send_unix_msg(umsg->sockd, "Failed"); goto reconnect; } else { char *s = json_dumps(gbt->json, JSON_NO_UTF8); - send_unix_msg(sockd, s); + send_unix_msg(umsg->sockd, s); free(s); clear_gbtbase(gbt); } } else if (cmdmatch(buf, "getbest")) { if (si->notify) - send_unix_msg(sockd, "notify"); + send_unix_msg(umsg->sockd, "notify"); else if (!get_bestblockhash(cs, hash)) { LOGINFO("No best block hash support from %s:%s", cs->url, cs->port); - send_unix_msg(sockd, "failed"); + send_unix_msg(umsg->sockd, "failed"); } else { if (unlikely(!started)) { started = true; LOGWARNING("%s generator ready", ckp->name); } - send_unix_msg(sockd, hash); + send_unix_msg(umsg->sockd, hash); } } else if (cmdmatch(buf, "getlast")) { int height; if (si->notify) - send_unix_msg(sockd, "notify"); + send_unix_msg(umsg->sockd, "notify"); else if ((height = get_blockcount(cs)) == -1) { - send_unix_msg(sockd, "failed"); + send_unix_msg(umsg->sockd, "failed"); goto reconnect; } else { LOGDEBUG("Height: %d", height); if (!get_blockhash(cs, height, hash)) { - send_unix_msg(sockd, "failed"); + send_unix_msg(umsg->sockd, "failed"); goto reconnect; } else { if (unlikely(!started)) { @@ -380,7 +377,7 @@ retry: LOGWARNING("%s generator ready", ckp->name); } - send_unix_msg(sockd, hash); + send_unix_msg(umsg->sockd, hash); LOGDEBUG("Hash: %s", hash); } } @@ -395,16 +392,16 @@ retry: send_proc(ckp->stratifier, blockmsg); } else if (cmdmatch(buf, "checkaddr:")) { if (validate_address(cs, buf + 10)) - send_unix_msg(sockd, "true"); + send_unix_msg(umsg->sockd, "true"); else - send_unix_msg(sockd, "false"); + send_unix_msg(umsg->sockd, "false"); } else if (cmdmatch(buf, "reconnect")) { goto reconnect; } else if (cmdmatch(buf, "loglevel")) { sscanf(buf, "loglevel=%d", &ckp->loglevel); } else if (cmdmatch(buf, "ping")) { LOGDEBUG("Generator received ping request"); - send_unix_msg(sockd, "pong"); + send_unix_msg(umsg->sockd, "pong"); } goto retry; @@ -2734,6 +2731,8 @@ int generator(proc_instance_t *pi) gdata = ckzalloc(sizeof(gdata_t)); ckp->data = gdata; gdata->ckp = ckp; + create_unix_receiver(pi); + if (ckp->proxy) { char *buf = NULL; From 690870bc92afd7341afc168170ca6b5280397bad Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Mon, 20 Apr 2015 16:08:43 +1000 Subject: [PATCH 417/544] Convert the proxy_loop to use unix receive queues --- src/generator.c | 47 +++++++++++++++++------------------------------ 1 file changed, 17 insertions(+), 30 deletions(-) diff --git a/src/generator.c b/src/generator.c index 1a06206c..3a3989fe 100644 --- a/src/generator.c +++ b/src/generator.c @@ -407,7 +407,6 @@ retry: out: kill_server(si); - dealloc(buf); return ret; } @@ -2515,14 +2514,14 @@ out: static int proxy_loop(proc_instance_t *pi) { proxy_instance_t *proxi = NULL, *cproxy; - int sockd = -1, ret = 0, selret; - unixsock_t *us = &pi->us; ckpool_t *ckp = pi->ckp; gdata_t *gdata = ckp->data; + unix_msg_t *umsg = NULL; char *buf = NULL; + int ret = 0; reconnect: - Close(sockd); + clear_unix_msg(&umsg); /* This does not necessarily mean we reconnect, but a change has * occurred and we need to reexamine the proxies. */ cproxy = wait_best_proxy(ckp, gdata); @@ -2536,28 +2535,17 @@ reconnect: } } retry: - Close(sockd); + clear_unix_msg(&umsg); do { - selret = wait_read_select(us->sockd, 5); - if (!selret && !ping_main(ckp)) { + umsg = get_unix_msg(pi); + if (unlikely(!umsg &&!ping_main(ckp))) { LOGEMERG("Generator failed to ping main process, exiting"); ret = 1; goto out; } - } while (selret < 1); + } while (!umsg); - sockd = accept(us->sockd, NULL, NULL); - if (sockd < 0) { - LOGEMERG("Failed to accept on proxy socket"); - ret = 1; - goto out; - } - dealloc(buf); - buf = recv_unix_msg(sockd); - if (!buf) { - LOGWARNING("Failed to get message in proxy_loop"); - goto retry; - } + buf = umsg->buf; LOGDEBUG("Proxy received request: %s", buf); if (likely(buf[0] == '{')) { if (ckp->passthrough) @@ -2572,21 +2560,21 @@ retry: submit_share(gdata, val); } } else if (cmdmatch(buf, "stats")) { - send_stats(gdata, sockd); + send_stats(gdata, umsg->sockd); } else if (cmdmatch(buf, "list")) { - send_list(gdata, sockd); + send_list(gdata, umsg->sockd); } else if (cmdmatch(buf, "sublist")) { - send_sublist(gdata, sockd, buf + 8); + send_sublist(gdata, umsg->sockd, buf + 8); } else if (cmdmatch(buf, "addproxy")) { - parse_addproxy(ckp, gdata, sockd, buf + 9); + parse_addproxy(ckp, gdata, umsg->sockd, buf + 9); } else if (cmdmatch(buf, "delproxy")) { - parse_delproxy(ckp, gdata, sockd, buf + 9); + parse_delproxy(ckp, gdata, umsg->sockd, buf + 9); } else if (cmdmatch(buf, "enableproxy")) { - parse_ableproxy(gdata, sockd, buf + 12, false); + parse_ableproxy(gdata, umsg->sockd, buf + 12, false); } else if (cmdmatch(buf, "disableproxy")) { - parse_ableproxy(gdata, sockd, buf + 13, true); + parse_ableproxy(gdata, umsg->sockd, buf + 13, true); } else if (cmdmatch(buf, "proxystats")) { - parse_proxystats(gdata, sockd, buf + 11); + parse_proxystats(gdata, umsg->sockd, buf + 11); } else if (cmdmatch(buf, "shutdown")) { ret = 0; goto out; @@ -2598,7 +2586,7 @@ retry: sscanf(buf, "loglevel=%d", &ckp->loglevel); } else if (cmdmatch(buf, "ping")) { LOGDEBUG("Proxy received ping request"); - send_unix_msg(sockd, "pong"); + send_unix_msg(umsg->sockd, "pong"); } else if (cmdmatch(buf, "recruit")) { recruit_subproxy(gdata, buf); } else if (cmdmatch(buf, "dropproxy")) { @@ -2608,7 +2596,6 @@ retry: } goto retry; out: - Close(sockd); return ret; } From 182e1b2d8da6310ea3506ebc969ba463e59534b4 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Tue, 28 Apr 2015 20:09:07 +1000 Subject: [PATCH 418/544] Move to a completely pull based mechanism for reconnecting clients once there are enough upstream connections instead of forcing reconnects when there may not be enough --- src/stratifier.c | 82 +++++++++++------------------------------------- 1 file changed, 19 insertions(+), 63 deletions(-) diff --git a/src/stratifier.c b/src/stratifier.c index d9182035..925a711b 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -259,7 +259,6 @@ struct stratum_instance { * or other problem and should be dropped lazily if * this is set to 2 */ - bool reconnect; /* Do we need to send this client a reconnect message */ time_t reconnect_request; /* The time we sent a reconnect message */ user_instance_t *user_instance; @@ -1363,8 +1362,8 @@ static void generator_recruit(const ckpool_t *ckp, const int proxyid, const int } /* Find how much headroom we have and connect up to that many clients that are - * not currently on this pool, setting the reconnect for the remainder to be - * switched lazily. Only reconnect clients bound to global proxies. */ + * not currently on this pool, recruiting more slots to switch more clients + * later on lazily. Only reconnect clients bound to global proxies. */ static void reconnect_clients(sdata_t *sdata) { stratum_instance_t *client, *tmpclient; @@ -1387,18 +1386,16 @@ static void reconnect_clients(sdata_t *sdata) continue; if (client->proxyid == proxy->id) continue; - if (client->reconnect) - continue; if (headroom-- < 1) continue; reconnects++; - client->reconnect = true; + reconnect_client(sdata, client); } ck_runlock(&sdata->instance_lock); if (reconnects) { - LOGNOTICE("%d clients flagged for reconnect to proxy %d", reconnects, - proxy->id); + LOGNOTICE("%d clients flagged for reconnect to global proxy %d", + reconnects, proxy->id); } if (headroom < 0) generator_recruit(sdata->ckp, proxy->id, -headroom); @@ -1448,8 +1445,8 @@ static void check_bestproxy(sdata_t *sdata) static void dead_proxyid(sdata_t *sdata, const int id, const int subid) { - int reconnects = 0, hard = 0, proxyid = 0; stratum_instance_t *client, *tmp; + int reconnects = 0, proxyid = 0; int64_t headroom; proxy_t *proxy; @@ -1468,12 +1465,13 @@ static void dead_proxyid(sdata_t *sdata, const int id, const int subid) HASH_ITER(hh, sdata->stratum_instances, client, tmp) { if (client->proxyid != id || client->subproxyid != subid) continue; + /* Clients could remain connected to a dead connection here + * but should be picked up when we recruit enough slots after + * another notify. */ + if (headroom-- < 1) + continue; reconnects++; - if (headroom-- > 0 && client->reconnect && hard <= SOMAXCONN / 2) { - hard++; - reconnect_client(sdata, client); - } else - client->reconnect = true; + reconnect_client(sdata, client); } ck_runlock(&sdata->instance_lock); @@ -1607,7 +1605,7 @@ static void recruit_best_userproxy(sdata_t *sdata, const int userid, const int r } /* Check how much headroom the userid proxies have and reconnect any clients - * that are not bound to it */ + * that are not bound to it that should be */ static void check_userproxies(sdata_t *sdata, const int userid) { int64_t headroom = proxy_headroom(sdata, userid); @@ -1624,12 +1622,10 @@ static void check_userproxies(sdata_t *sdata, const int userid) continue; if (client->proxy->userid == userid) continue; - if (client->reconnect) - continue; if (headroom-- < 1) continue; reconnects++; - client->reconnect = true; + reconnect_client(sdata, client); } ck_runlock(&sdata->instance_lock); @@ -2072,7 +2068,6 @@ static void stratum_broadcast(sdata_t *sdata, json_t *val) stratum_instance_t *client, *tmp; ckmsg_t *bulk_send = NULL; time_t now_t = time(NULL); - int hard_reconnect = 0; ckmsgq_t *ssends; if (unlikely(!val)) { @@ -2103,13 +2098,6 @@ static void stratum_broadcast(sdata_t *sdata, json_t *val) } continue; } - if (client->reconnect) { - if (hard_reconnect <= SOMAXCONN / 2) { - hard_reconnect++; - reconnect_client(ckp_sdata, client); - } - continue; - } if (!client_active(client)) continue; @@ -2420,15 +2408,15 @@ static char *stratifier_stats(ckpool_t *ckp, sdata_t *sdata) /* Send a single client a reconnect request, setting the time we sent the * request so we can drop the client lazily if it hasn't reconnected on its - * own one minute later */ + * own more than one minute later if we call reconnect again */ static void reconnect_client(sdata_t *sdata, stratum_instance_t *client) { json_t *json_msg; /* Already requested? */ if (client->reconnect_request) { - connector_drop_client(sdata->ckp, client->id); - client->dropped = true; + if (time(NULL) - client->reconnect_request >= 60) + connector_drop_client(sdata->ckp, client->id); return; } client->reconnect_request = time(NULL); @@ -2445,28 +2433,6 @@ static void dead_proxy(sdata_t *sdata, const char *buf) dead_proxyid(sdata, id, subid); } -/* Must hold a reference */ -static void lazy_reconnect_client(sdata_t *sdata, stratum_instance_t *client) -{ - int64_t headroom; - proxy_t *proxy; - - headroom = current_headroom(sdata, &proxy); - if (!proxy) - return; - if (headroom-- > 0) { - LOGNOTICE("Reconnecting client %"PRId64, client->id); - reconnect_client(sdata, client); - } else { - generator_recruit(sdata->ckp, proxy->id, -headroom); - if (!client->reconnect) { - LOGNOTICE("Flagging client %"PRId64, client->id); - client->reconnect = true; - } else /* Already been flagged, force the send */ - reconnect_client(sdata, client); - } -} - static void reconnect_client_id(sdata_t *sdata, const int64_t client_id) { stratum_instance_t *client; @@ -2476,7 +2442,7 @@ static void reconnect_client_id(sdata_t *sdata, const int64_t client_id) LOGINFO("reconnect_client_id failed to find client %"PRId64, client_id); return; } - lazy_reconnect_client(sdata, client); + reconnect_client(sdata, client); dec_instance_ref(sdata, client); } @@ -3987,12 +3953,6 @@ static json_t *parse_authorise(stratum_instance_t *client, const json_t *params_ if (ckp->proxy) { LOGNOTICE("Authorised client %"PRId64" to proxy %d:%d, worker %s as user %s", client->id, client->proxyid, client->subproxyid, buf, user->username); - if (client->sdata && client->sdata->proxy && client->sdata->proxy->global) { - sdata_t *ckp_sdata = ckp->data; - - if (proxy_headroom(ckp_sdata, client->user_id)) - client->reconnect = true; - } } else { LOGNOTICE("Authorised client %"PRId64" worker %s as user %s", client->id, buf, user->username); @@ -4912,7 +4872,7 @@ static void parse_instance_msg(ckpool_t *ckp, sdata_t *sdata, smsg_t *msg, strat json_t *val = msg->json_msg, *id_val, *method, *params; int64_t client_id = msg->client_id; - if (client->reject == 2 || (client->reconnect_request && time(NULL) - client->reconnect_request > 60)) { + if (client->reject == 2) { LOGINFO("Dropping client %"PRId64" %s tagged for lazy invalidation", client_id, client->address); connector_drop_client(ckp, client_id); @@ -5027,10 +4987,6 @@ static void srecv_process(ckpool_t *ckp, char *buf) LOGINFO("Stratifier added instance %"PRId64" server %d", client->id, server); parse_instance_msg(ckp, sdata, msg, client); - /* The client is still active but has been issued a reconnect request - * so use this opportunity to send it a reconnect message */ - if (unlikely(client->reconnect)) - lazy_reconnect_client(sdata, client); dec_instance_ref(sdata, client); out: free(buf); From 4e5ccb6e13170b640bdc4a490cd4e277a0b456bc Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Wed, 29 Apr 2015 18:30:19 +1000 Subject: [PATCH 419/544] check for clients that really need to reconnect and do them asap --- src/stratifier.c | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/src/stratifier.c b/src/stratifier.c index 925a711b..c6a50ca7 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -259,6 +259,7 @@ struct stratum_instance { * or other problem and should be dropped lazily if * this is set to 2 */ + bool reconnect; /* This client really needs to reconnect */ time_t reconnect_request; /* The time we sent a reconnect message */ user_instance_t *user_instance; @@ -1381,11 +1382,14 @@ static void reconnect_clients(sdata_t *sdata) continue; if (!client->authorised) continue; - /* This client is bound to a user proxy */ - if (client->proxy->userid) - continue; - if (client->proxyid == proxy->id) - continue; + /* Is this client boudn to a dead proxy? */ + if (!client->reconnect) { + /* This client is bound to a user proxy */ + if (client->proxy->userid) + continue; + if (client->proxyid == proxy->id) + continue; + } if (headroom-- < 1) continue; reconnects++; @@ -1468,8 +1472,10 @@ static void dead_proxyid(sdata_t *sdata, const int id, const int subid) /* Clients could remain connected to a dead connection here * but should be picked up when we recruit enough slots after * another notify. */ - if (headroom-- < 1) + if (headroom-- < 1) { + client->reconnect = true; continue; + } reconnects++; reconnect_client(sdata, client); } @@ -1620,7 +1626,8 @@ static void check_userproxies(sdata_t *sdata, const int userid) continue; if (client->user_id != userid) continue; - if (client->proxy->userid == userid) + /* Is this client bound to a dead proxy? */ + if (!client->reconnect && client->proxy->userid == userid) continue; if (headroom-- < 1) continue; @@ -2442,6 +2449,7 @@ static void reconnect_client_id(sdata_t *sdata, const int64_t client_id) LOGINFO("reconnect_client_id failed to find client %"PRId64, client_id); return; } + client->reconnect = true; reconnect_client(sdata, client); dec_instance_ref(sdata, client); } From 11023ecfec71833681d2407e9034aadd952aac2e Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Wed, 29 Apr 2015 19:47:14 +1000 Subject: [PATCH 420/544] Use non blocking reads in read_socket_line --- src/ckpool.c | 35 +++++++++++++++++------------------ 1 file changed, 17 insertions(+), 18 deletions(-) diff --git a/src/ckpool.c b/src/ckpool.c index 4688c59d..eb3885c6 100644 --- a/src/ckpool.c +++ b/src/ckpool.c @@ -530,35 +530,34 @@ int read_socket_line(connsock_t *cs, const int timeout) eom = strchr(cs->buf, '\n'); } + ret = wait_read_select(fd, timeout); + if (ret < 1) { + if (!ret) + LOGDEBUG("Select timed out in read_socket_line"); + else { + if (cs->ckp->proxy) + LOGINFO("Select failed in read_socket_line"); + else + LOGERR("Select failed in read_socket_line"); + } + goto out; + } while (42) { char readbuf[PAGESIZE] = {}; int backoff = 1; char *newbuf; - ret = wait_read_select(fd, eom ? 0 : timeout); - if (ret < 1) { - if (eom) - break; - if (!ret) - LOGDEBUG("Select timed out in read_socket_line"); - else { - if (cs->ckp->proxy) - LOGNOTICE("Select failed in read_socket_line"); - else - LOGERR("Select failed in read_socket_line"); - } - goto out; - } - ret = recv(fd, readbuf, PAGESIZE - 4, 0); + ret = recv(fd, readbuf, PAGESIZE - 4, MSG_DONTWAIT); if (ret < 1) { /* Closed socket after valid message */ - if (eom) + if (eom || !ret || errno == EAGAIN || errno == EWOULDBLOCK) { + ret = 0; break; + } if (cs->ckp->proxy) - LOGNOTICE("Failed to recv in read_socket_line"); + LOGINFO("Failed to recv in read_socket_line"); else LOGERR("Failed to recv in read_socket_line"); - ret = -1; goto out; } buflen = cs->bufofs + ret + 1; From d59c8c1b41384e6dc78e42f25377c016492a1c7a Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Wed, 29 Apr 2015 19:48:58 +1000 Subject: [PATCH 421/544] Demote proxy logging to info level --- src/stratifier.c | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/src/stratifier.c b/src/stratifier.c index c6a50ca7..b0a30410 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -502,6 +502,18 @@ static void notice_msg_entries(char_entry_t **entries) } } +static void info_msg_entries(char_entry_t **entries) +{ + char_entry_t *entry, *tmpentry; + + DL_FOREACH_SAFE(*entries, entry, tmpentry) { + DL_DELETE(*entries, entry); + LOGINFO("%s", entry->buf); + free(entry->buf); + free(entry); + } +} + static void generate_coinbase(const ckpool_t *ckp, workbase_t *wb) { uint64_t *u64, g64, d64 = 0; @@ -5715,7 +5727,7 @@ static void *statsupdate(void *arg) } } mutex_unlock(&sdata->proxy_lock); - notice_msg_entries(&char_list); + info_msg_entries(&char_list); } ts_realtime(&ts_now); From 25840cfec9adab8dfc89973f99d18d493841b6ed Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Wed, 29 Apr 2015 19:50:05 +1000 Subject: [PATCH 422/544] Demote client flag message to info level --- src/stratifier.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/stratifier.c b/src/stratifier.c index b0a30410..53e1df26 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -1410,8 +1410,8 @@ static void reconnect_clients(sdata_t *sdata) ck_runlock(&sdata->instance_lock); if (reconnects) { - LOGNOTICE("%d clients flagged for reconnect to global proxy %d", - reconnects, proxy->id); + LOGINFO("%d clients flagged for reconnect to global proxy %d", + reconnects, proxy->id); } if (headroom < 0) generator_recruit(sdata->ckp, proxy->id, -headroom); @@ -1649,8 +1649,8 @@ static void check_userproxies(sdata_t *sdata, const int userid) ck_runlock(&sdata->instance_lock); if (reconnects) { - LOGNOTICE("%d clients flagged for reconnect to user %d proxies", - reconnects, userid); + LOGINFO("%d clients flagged for reconnect to user %d proxies", + reconnects, userid); } if (headroom < 0) recruit_best_userproxy(sdata, userid, -headroom); From cb640b6ab2926eba7cd165ba42650de823868c20 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Wed, 29 Apr 2015 19:53:38 +1000 Subject: [PATCH 423/544] Check for epoll hangups fully --- src/connector.c | 2 +- src/generator.c | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/connector.c b/src/connector.c index ffaac679..5393ae1f 100644 --- a/src/connector.c +++ b/src/connector.c @@ -470,7 +470,7 @@ void *receiver(void *arg) for (i = 0; i < serverfds; i++) { /* The small values will be less than the first client ids */ event.data.u64 = i; - event.events = EPOLLIN; + event.events = EPOLLIN | EPOLLRDHUP; ret = epoll_ctl(epfd, EPOLL_CTL_ADD, cdata->serverfd[i], &event); if (ret < 0) { LOGEMERG("FATAL: Failed to add epfd %d to epoll_ctl", epfd); diff --git a/src/generator.c b/src/generator.c index 3a3989fe..a62a273e 100644 --- a/src/generator.c +++ b/src/generator.c @@ -1740,7 +1740,7 @@ out: } } else { keep_sockalive(cs->fd); - event.events = EPOLLIN; + event.events = EPOLLIN | EPOLLRDHUP; event.data.ptr = proxi; /* Add this connsock_t to the epoll list */ if (unlikely(epoll_ctl(epfd, EPOLL_CTL_ADD, cs->fd, &event) == -1)) { @@ -1994,7 +1994,7 @@ static void *proxy_recv(void *arg) if (likely(ret > 0)) { subproxy = event.data.ptr; cs = &subproxy->cs; - if (event.events & EPOLLHUP) + if (event.events & (EPOLLHUP | EPOLLERR | EPOLLRDHUP)) ret = -1; else ret = read_socket_line(cs, 5); @@ -2063,7 +2063,7 @@ static void *userproxy_recv(void *arg) } proxy = event.data.ptr; cs = &proxy->cs; - if (event.events & EPOLLHUP) { + if (event.events & (EPOLLHUP | EPOLLERR | EPOLLRDHUP)) { LOGNOTICE("Proxy %d:%d %s hung up in epoll_wait", proxy->id, proxy->subid, proxy->url); disable_subproxy(gdata, proxy->parent, proxy); From 7e2280ff2d7573ef3499ac7d1cc0516b3ddd1b51 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Wed, 29 Apr 2015 20:06:12 +1000 Subject: [PATCH 424/544] Check for eom more carefully in read_socket_line --- src/ckpool.c | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/src/ckpool.c b/src/ckpool.c index eb3885c6..360c4abb 100644 --- a/src/ckpool.c +++ b/src/ckpool.c @@ -530,11 +530,13 @@ int read_socket_line(connsock_t *cs, const int timeout) eom = strchr(cs->buf, '\n'); } - ret = wait_read_select(fd, timeout); + ret = wait_read_select(fd, eom ? 0 : timeout); if (ret < 1) { - if (!ret) + if (!ret) { + if (eom) + goto parse; LOGDEBUG("Select timed out in read_socket_line"); - else { + } else { if (cs->ckp->proxy) LOGINFO("Select failed in read_socket_line"); else @@ -549,11 +551,9 @@ int read_socket_line(connsock_t *cs, const int timeout) ret = recv(fd, readbuf, PAGESIZE - 4, MSG_DONTWAIT); if (ret < 1) { - /* Closed socket after valid message */ - if (eom || !ret || errno == EAGAIN || errno == EWOULDBLOCK) { - ret = 0; + /* No more to read or closed socket after valid message */ + if (eom) break; - } if (cs->ckp->proxy) LOGINFO("Failed to recv in read_socket_line"); else @@ -578,6 +578,7 @@ int read_socket_line(connsock_t *cs, const int timeout) cs->buf[cs->bufofs] = '\0'; eom = strchr(cs->buf, '\n'); } +parse: ret = eom - cs->buf; cs->buflen = cs->buf + cs->bufofs - eom - 1; From 0383888d5ec582830e76ca764718db09eeba0ce5 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Wed, 29 Apr 2015 20:14:31 +1000 Subject: [PATCH 425/544] Demote non existent proxy message to info --- src/generator.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/generator.c b/src/generator.c index a62a273e..ee9626a7 100644 --- a/src/generator.c +++ b/src/generator.c @@ -1392,21 +1392,21 @@ static void submit_share(gdata_t *gdata, json_t *val) } proxy = proxy_by_id(gdata, id); if (unlikely(!proxy)) { - LOGNOTICE("Client %"PRId64" sending shares to non existent proxy %d, dropping", - client_id, id); + LOGINFO("Client %"PRId64" sending shares to non existent proxy %d, dropping", + client_id, id); stratifier_reconnect_client(ckp, client_id); goto out; } proxi = subproxy_by_id(proxy, subid); if (unlikely(!proxi)) { - LOGNOTICE("Client %"PRId64" sending shares to non existent subproxy %d:%d, dropping", - client_id, id, subid); + LOGINFO("Client %"PRId64" sending shares to non existent subproxy %d:%d, dropping", + client_id, id, subid); stratifier_reconnect_client(ckp, client_id); goto out; } if (!proxi->alive) { - LOGNOTICE("Client %"PRId64" sending shares to dead subproxy %d:%d, dropping", - client_id, id, subid); + LOGINFO("Client %"PRId64" sending shares to dead subproxy %d:%d, dropping", + client_id, id, subid); stratifier_reconnect_client(ckp, client_id); goto out; } From d8a52285130644da08b64a69b7bf1cab910784bf Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Wed, 29 Apr 2015 20:16:33 +1000 Subject: [PATCH 426/544] Demote flagged message --- src/stratifier.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/stratifier.c b/src/stratifier.c index 53e1df26..21681d96 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -1494,8 +1494,8 @@ static void dead_proxyid(sdata_t *sdata, const int id, const int subid) ck_runlock(&sdata->instance_lock); if (reconnects) { - LOGNOTICE("%d clients flagged to reconnect from dead proxy %d:%d", reconnects, - id, subid); + LOGINFO("%d clients flagged to reconnect from dead proxy %d:%d", reconnects, + id, subid); } /* When a proxy dies, recruit more of the global proxies for them to * fail over to in case user proxies are unavailable. */ From 9bcacd0131007e355bb4d6187a6155a758b70ea4 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Thu, 30 Apr 2015 00:49:24 +1000 Subject: [PATCH 427/544] Increase precision of wait read/write select to accept floats --- src/libckpool.c | 4 ++-- src/libckpool.h | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/libckpool.c b/src/libckpool.c index 813ca7c5..8b73aee3 100644 --- a/src/libckpool.c +++ b/src/libckpool.c @@ -903,7 +903,7 @@ int wait_close(int sockd, int timeout) } /* Emulate a select read wait for high fds that select doesn't support */ -int wait_read_select(int sockd, int timeout) +int wait_read_select(int sockd, float timeout) { struct pollfd sfd; int ret = -1; @@ -985,7 +985,7 @@ out: } /* Emulate a select write wait for high fds that select doesn't support */ -int wait_write_select(int sockd, int timeout) +int wait_write_select(int sockd, float timeout) { struct pollfd sfd; int ret = -1; diff --git a/src/libckpool.h b/src/libckpool.h index 8901e3be..0180606c 100644 --- a/src/libckpool.h +++ b/src/libckpool.h @@ -490,7 +490,7 @@ int _open_unix_server(const char *server_path, const char *file, const char *fun int _open_unix_client(const char *server_path, const char *file, const char *func, const int line); #define open_unix_client(server_path) _open_unix_client(server_path, __FILE__, __func__, __LINE__) int wait_close(int sockd, int timeout); -int wait_read_select(int sockd, int timeout); +int wait_read_select(int sockd, float timeout); int read_length(int sockd, void *buf, int len); char *_recv_unix_msg(int sockd, int timeout1, int timeout2, const char *file, const char *func, const int line); #define RECV_UNIX_TIMEOUT1 30 @@ -498,7 +498,7 @@ char *_recv_unix_msg(int sockd, int timeout1, int timeout2, const char *file, co #define recv_unix_msg(sockd) _recv_unix_msg(sockd, RECV_UNIX_TIMEOUT1, RECV_UNIX_TIMEOUT2, __FILE__, __func__, __LINE__) #define recv_unix_msg_tmo(sockd, tmo) _recv_unix_msg(sockd, tmo, RECV_UNIX_TIMEOUT2, __FILE__, __func__, __LINE__) #define recv_unix_msg_tmo2(sockd, tmo1, tmo2) _recv_unix_msg(sockd, tmo1, tmo2, __FILE__, __func__, __LINE__) -int wait_write_select(int sockd, int timeout); +int wait_write_select(int sockd, float timeout); int write_length(int sockd, const void *buf, int len); bool _send_unix_msg(int sockd, const char *buf, const char *file, const char *func, const int line); #define send_unix_msg(sockd, buf) _send_unix_msg(sockd, buf, __FILE__, __func__, __LINE__) From 3fb8ff91d693b00389154a135f260d244840d5ca Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Thu, 30 Apr 2015 01:03:40 +1000 Subject: [PATCH 428/544] Continue waiting in read_socket_line if we have no more to read but have not used up the full timeout --- src/ckpool.c | 12 +++++++++++- src/ckpool.h | 2 +- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/src/ckpool.c b/src/ckpool.c index 360c4abb..5113fbfe 100644 --- a/src/ckpool.c +++ b/src/ckpool.c @@ -510,11 +510,13 @@ void empty_buffer(connsock_t *cs) /* Read from a socket into cs->buf till we get an '\n', converting it to '\0' * and storing how much extra data we've received, to be moved to the beginning * of the buffer for use on the next receive. */ -int read_socket_line(connsock_t *cs, const int timeout) +int read_socket_line(connsock_t *cs, float timeout) { int fd = cs->fd, ret = -1; char *eom = NULL; + tv_t start, now; size_t buflen; + float diff; if (unlikely(fd < 0)) goto out; @@ -530,6 +532,8 @@ int read_socket_line(connsock_t *cs, const int timeout) eom = strchr(cs->buf, '\n'); } + tv_time(&start); +rewait: ret = wait_read_select(fd, eom ? 0 : timeout); if (ret < 1) { if (!ret) { @@ -544,6 +548,9 @@ int read_socket_line(connsock_t *cs, const int timeout) } goto out; } + tv_time(&now); + diff = tvdiff(&now, &start); + timeout -= diff; while (42) { char readbuf[PAGESIZE] = {}; int backoff = 1; @@ -554,6 +561,9 @@ int read_socket_line(connsock_t *cs, const int timeout) /* No more to read or closed socket after valid message */ if (eom) break; + /* Have we used up all the timeout yet? */ + if (timeout > 0 && (errno == EAGAIN || errno == EWOULDBLOCK)) + goto rewait; if (cs->ckp->proxy) LOGINFO("Failed to recv in read_socket_line"); else diff --git a/src/ckpool.h b/src/ckpool.h index 9b09625e..5fbaed5b 100644 --- a/src/ckpool.h +++ b/src/ckpool.h @@ -242,7 +242,7 @@ ckpool_t *global_ckp; bool ping_main(ckpool_t *ckp); void empty_buffer(connsock_t *cs); -int read_socket_line(connsock_t *cs, const int timeout); +int read_socket_line(connsock_t *cs, float timeout); void _send_proc(proc_instance_t *pi, const char *msg, const char *file, const char *func, const int line); #define send_proc(pi, msg) _send_proc(pi, msg, __FILE__, __func__, __LINE__) char *_send_recv_proc(proc_instance_t *pi, const char *msg, const char *file, const char *func, const int line); From f359b571195d523c910d9a63750ccd3b1e7318c2 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Thu, 30 Apr 2015 01:31:36 +1000 Subject: [PATCH 429/544] Close umsg sockd on clearing unix message in generator --- src/generator.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/generator.c b/src/generator.c index ee9626a7..ffae0180 100644 --- a/src/generator.c +++ b/src/generator.c @@ -270,6 +270,7 @@ static void kill_server(server_instance_t *si) static void clear_unix_msg(unix_msg_t **umsg) { if (*umsg) { + Close((*umsg)->sockd); free((*umsg)->buf); free(*umsg); *umsg = NULL; From 6194af292fbc35ae9cdd1e776e70c1270f55c3cf Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Thu, 30 Apr 2015 08:45:06 +1000 Subject: [PATCH 430/544] Send queued json messages in proxy mode non-blocking --- src/generator.c | 38 ++++++++++++++++++++++++++------------ 1 file changed, 26 insertions(+), 12 deletions(-) diff --git a/src/generator.c b/src/generator.c index ffae0180..78b34d4d 100644 --- a/src/generator.c +++ b/src/generator.c @@ -411,6 +411,7 @@ out: return ret; } +/* This is for blocking sends of json messages */ static bool send_json_msg(connsock_t *cs, const json_t *json_msg) { int len, sent; @@ -1519,31 +1520,38 @@ struct cs_msg { cs_msg_t *next; cs_msg_t *prev; proxy_instance_t *proxy; - json_t *val; + char *buf; + int len; + int ofs; }; /* Sends all messages in the queue ready to be dispatched, leaving those that - * fail write select to be handled next pass */ + * would block to be handled next pass */ static void send_json_msgq(gdata_t *gdata, cs_msg_t **csmsgq) { cs_msg_t *csmsg, *tmp; int ret; DL_FOREACH_SAFE(*csmsgq, csmsg, tmp) { - connsock_t *cs = &(csmsg->proxy->cs); + while (csmsg->len) { + proxy_instance_t *proxy = csmsg->proxy; + int fd = proxy->cs.fd; - if ((ret = wait_write_select(cs->fd, 0))) { - DL_DELETE(*csmsgq, csmsg); - if (ret > 0) - ret = send_json_msg(cs, csmsg->val); - json_decref(csmsg->val); + ret = send(fd, csmsg->buf + csmsg->ofs, csmsg->len, MSG_DONTWAIT); if (ret < 1) { - proxy_instance_t *proxy = csmsg->proxy; - + if (errno == EAGAIN || errno == EWOULDBLOCK) + break; + csmsg->len = 0; LOGNOTICE("Proxy %d:%d %s failed to send msg in send_json_msgq, dropping", proxy->id, proxy->subid, proxy->url); disable_subproxy(gdata, proxy->parent, proxy); } + csmsg->ofs += ret; + csmsg->len -= ret; + } + if (!csmsg->len) { + DL_DELETE(*csmsgq, csmsg); + free(csmsg->buf); free(csmsg); } } @@ -1551,10 +1559,16 @@ static void send_json_msgq(gdata_t *gdata, cs_msg_t **csmsgq) static void add_json_msgq(cs_msg_t **csmsgq, proxy_instance_t *proxy, json_t **val) { - cs_msg_t *csmsg = ckalloc(sizeof(cs_msg_t)); + cs_msg_t *csmsg = ckzalloc(sizeof(cs_msg_t)); - csmsg->val = *val; + csmsg->buf = json_dumps(*val, JSON_ESCAPE_SLASH | JSON_EOL); + json_decref(*val); *val = NULL; + if (unlikely(!csmsg->buf)) { + LOGWARNING("Failed to create json dump in add_json_msgq"); + return; + } + csmsg->len = strlen(csmsg->buf); csmsg->proxy = proxy; DL_APPEND(*csmsgq, csmsg); } From 2242d44b3054fd404be6aaec08655876e2ce9cc3 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Thu, 30 Apr 2015 09:04:19 +1000 Subject: [PATCH 431/544] Check for no ret after send in generator --- src/generator.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/generator.c b/src/generator.c index 78b34d4d..e7cf6f47 100644 --- a/src/generator.c +++ b/src/generator.c @@ -1539,7 +1539,7 @@ static void send_json_msgq(gdata_t *gdata, cs_msg_t **csmsgq) ret = send(fd, csmsg->buf + csmsg->ofs, csmsg->len, MSG_DONTWAIT); if (ret < 1) { - if (errno == EAGAIN || errno == EWOULDBLOCK) + if (errno == EAGAIN || errno == EWOULDBLOCK || !ret) break; csmsg->len = 0; LOGNOTICE("Proxy %d:%d %s failed to send msg in send_json_msgq, dropping", From e2f08fe36b9ecc46a2c1dd2d0c5ab07ffe16adfa Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Thu, 30 Apr 2015 09:14:12 +1000 Subject: [PATCH 432/544] Prevent sending parts of different messages to upstream proxies --- src/generator.c | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/src/generator.c b/src/generator.c index e7cf6f47..5b475e3d 100644 --- a/src/generator.c +++ b/src/generator.c @@ -115,6 +115,7 @@ struct proxy_instance { bool reconnecting; /* Testing of parent in progress */ int64_t recruit; /* No of recruiting requests in progress */ bool alive; + bool sending; /* Are we in the middle of a blocked write? */ pthread_t pth_precv; @@ -1533,10 +1534,17 @@ static void send_json_msgq(gdata_t *gdata, cs_msg_t **csmsgq) int ret; DL_FOREACH_SAFE(*csmsgq, csmsg, tmp) { + proxy_instance_t *proxy = csmsg->proxy; + + /* Only try to send one message at a time to each proxy + * to avoid sending parts of different messages */ + if (proxy->sending) + continue; while (csmsg->len) { - proxy_instance_t *proxy = csmsg->proxy; - int fd = proxy->cs.fd; + int fd; + proxy->sending = true; + fd = proxy->cs.fd; ret = send(fd, csmsg->buf + csmsg->ofs, csmsg->len, MSG_DONTWAIT); if (ret < 1) { if (errno == EAGAIN || errno == EWOULDBLOCK || !ret) @@ -1550,6 +1558,7 @@ static void send_json_msgq(gdata_t *gdata, cs_msg_t **csmsgq) csmsg->len -= ret; } if (!csmsg->len) { + proxy->sending = false; DL_DELETE(*csmsgq, csmsg); free(csmsg->buf); free(csmsg); From ef67643d5e08baf3bda9152db3bc7337af5fa552 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Thu, 30 Apr 2015 09:19:32 +1000 Subject: [PATCH 433/544] Reference the actual message we're blocking on in the generator to a proxy and continue only that one --- src/generator.c | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/src/generator.c b/src/generator.c index 5b475e3d..bd49dadb 100644 --- a/src/generator.c +++ b/src/generator.c @@ -74,6 +74,7 @@ struct pass_msg { }; typedef struct pass_msg pass_msg_t; +typedef struct cs_msg cs_msg_t; /* Per proxied pool instance data */ struct proxy_instance { @@ -115,7 +116,9 @@ struct proxy_instance { bool reconnecting; /* Testing of parent in progress */ int64_t recruit; /* No of recruiting requests in progress */ bool alive; - bool sending; /* Are we in the middle of a blocked write? */ + + /* Are we in the middle of a blocked write of this message? */ + cs_msg_t *sending; pthread_t pth_precv; @@ -1515,8 +1518,6 @@ out: return ret; } -typedef struct cs_msg cs_msg_t; - struct cs_msg { cs_msg_t *next; cs_msg_t *prev; @@ -1538,12 +1539,12 @@ static void send_json_msgq(gdata_t *gdata, cs_msg_t **csmsgq) /* Only try to send one message at a time to each proxy * to avoid sending parts of different messages */ - if (proxy->sending) + if (proxy->sending != csmsg) continue; while (csmsg->len) { int fd; - proxy->sending = true; + proxy->sending = csmsg; fd = proxy->cs.fd; ret = send(fd, csmsg->buf + csmsg->ofs, csmsg->len, MSG_DONTWAIT); if (ret < 1) { @@ -1558,7 +1559,7 @@ static void send_json_msgq(gdata_t *gdata, cs_msg_t **csmsgq) csmsg->len -= ret; } if (!csmsg->len) { - proxy->sending = false; + proxy->sending = NULL; DL_DELETE(*csmsgq, csmsg); free(csmsg->buf); free(csmsg); From b21cc7176d440a17cf66839d136ea78df8c392fa Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Thu, 30 Apr 2015 09:20:41 +1000 Subject: [PATCH 434/544] Check for proxy sending being set as well --- src/generator.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/generator.c b/src/generator.c index bd49dadb..4fdd09da 100644 --- a/src/generator.c +++ b/src/generator.c @@ -1539,7 +1539,7 @@ static void send_json_msgq(gdata_t *gdata, cs_msg_t **csmsgq) /* Only try to send one message at a time to each proxy * to avoid sending parts of different messages */ - if (proxy->sending != csmsg) + if (proxy->sending && proxy->sending != csmsg) continue; while (csmsg->len) { int fd; From 2c1944534be2c523630aa2a64a9e71a2723484ec Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Thu, 30 Apr 2015 09:44:52 +1000 Subject: [PATCH 435/544] Unconditionally drop all clients bound to a proxy should we update its subscribe --- src/stratifier.c | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/stratifier.c b/src/stratifier.c index 21681d96..789716a4 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -1459,7 +1459,7 @@ static void check_bestproxy(sdata_t *sdata) LOGNOTICE("Stratifier setting active proxy to %d", changed_id); } -static void dead_proxyid(sdata_t *sdata, const int id, const int subid) +static void dead_proxyid(sdata_t *sdata, const int id, const int subid, const bool replaced) { stratum_instance_t *client, *tmp; int reconnects = 0, proxyid = 0; @@ -1469,7 +1469,7 @@ static void dead_proxyid(sdata_t *sdata, const int id, const int subid) proxy = existing_subproxy(sdata, id, subid); if (proxy) { proxy->dead = true; - if (proxy->global) + if (!replaced && proxy->global) check_bestproxy(sdata); } LOGINFO("Stratifier dropping clients from proxy %d:%d", id, subid); @@ -1550,8 +1550,7 @@ static void update_subscribe(ckpool_t *ckp, const char *cmd) /* Is this a replacement for an existing proxy id? */ old = existing_subproxy(sdata, id, subid); if (old) { - if (old->dead) - dead_proxyid(sdata, id, subid); + dead_proxyid(sdata, id, subid, true); proxy = old; proxy->dead = false; } else @@ -2449,7 +2448,7 @@ static void dead_proxy(sdata_t *sdata, const char *buf) int id = 0, subid = 0; sscanf(buf, "deadproxy=%d:%d", &id, &subid); - dead_proxyid(sdata, id, subid); + dead_proxyid(sdata, id, subid, false); } static void reconnect_client_id(sdata_t *sdata, const int64_t client_id) From dcf55ff1fb896840abd48e960e109802b6ae4985 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Thu, 30 Apr 2015 09:56:09 +1000 Subject: [PATCH 436/544] Maintain a reference to which message we're currently sending to a client to prevent sending interleaved messages should they block --- src/connector.c | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/src/connector.c b/src/connector.c index 5393ae1f..be006391 100644 --- a/src/connector.c +++ b/src/connector.c @@ -24,6 +24,7 @@ #define MAX_MSGSIZE 1024 typedef struct client_instance client_instance_t; +typedef struct sender_send sender_send_t; struct client_instance { /* For clients hashtable */ @@ -53,6 +54,8 @@ struct client_instance { char buf[PAGESIZE]; int bufofs; + /* Are we currently sending a blocked message from this client */ + sender_send_t *sending; bool passthrough; }; @@ -66,8 +69,6 @@ struct sender_send { int ofs; }; -typedef struct sender_send sender_send_t; - /* Private data for the connector */ struct connector_data { ckpool_t *ckp; @@ -556,7 +557,12 @@ static bool send_sender_send(ckpool_t *ckp, cdata_t *cdata, sender_send_t *sende client_instance_t *client = sender_send->client; if (unlikely(client->invalid)) - return true; + goto out_true; + + /* Make sure we only send one message at a time to each client */ + if (unlikely(client->sending && client->sending != sender_send)) + return false; + client->sending = sender_send; while (sender_send->len) { int ret = write(client->fd, sender_send->buf + sender_send->ofs, sender_send->len); @@ -567,11 +573,13 @@ static bool send_sender_send(ckpool_t *ckp, cdata_t *cdata, sender_send_t *sende LOGINFO("Client id %"PRId64" fd %d disconnected with write errno %d:%s", client->id, client->fd, errno, strerror(errno)); invalidate_client(ckp, cdata, client); - return true; + goto out_true; } sender_send->ofs += ret; sender_send->len -= ret; } +out_true: + client->sending = NULL; return true; } From 75f8a93d49eab96bd2509713bb6da99b256a76e1 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Thu, 30 Apr 2015 11:12:08 +1000 Subject: [PATCH 437/544] Check for zero ret in read_socket_line --- src/ckpool.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ckpool.c b/src/ckpool.c index 5113fbfe..c4537d50 100644 --- a/src/ckpool.c +++ b/src/ckpool.c @@ -562,7 +562,7 @@ rewait: if (eom) break; /* Have we used up all the timeout yet? */ - if (timeout > 0 && (errno == EAGAIN || errno == EWOULDBLOCK)) + if (timeout > 0 && (errno == EAGAIN || errno == EWOULDBLOCK || !ret)) goto rewait; if (cs->ckp->proxy) LOGINFO("Failed to recv in read_socket_line"); From c627c12cab8949a132f5d1ecf2d7b0193e7ded44 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Thu, 28 May 2015 15:19:07 +1000 Subject: [PATCH 438/544] User id is an integer --- src/stratifier.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/stratifier.c b/src/stratifier.c index 789716a4..b788266b 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -2473,7 +2473,7 @@ static json_t *userinfo(const user_instance_t *user) { json_t *val; - JSON_CPACK(val, "{ss,sI,si,sf,sf,sf,sf,sf,sf,si}", + JSON_CPACK(val, "{ss,si,si,sf,sf,sf,sf,sf,sf,si}", "user", user->username, "id", user->id, "workers", user->workers, "bestdiff", user->best_diff, "dsps1", user->dsps1, "dsps5", user->dsps5, "dsps60", user->dsps60, "dsps1440", user->dsps1440, "dsps10080", user->dsps10080, From f3a826d30f2ec78fa9405e263c36891a18427fa2 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Sun, 7 Jun 2015 08:16:33 +1000 Subject: [PATCH 439/544] User ID is an integer in workerinfo --- src/stratifier.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/stratifier.c b/src/stratifier.c index b788266b..1b414411 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -2593,7 +2593,7 @@ static json_t *workerinfo(const user_instance_t *user, const worker_instance_t * { json_t *val; - JSON_CPACK(val, "{ss,ss,sI,sf,sf,sf,sf,si,sf,si,sb}", + JSON_CPACK(val, "{ss,ss,si,sf,sf,sf,sf,si,sf,si,sb}", "user", user->username, "worker", worker->workername, "id", user->id, "dsps1", worker->dsps1, "dsps5", worker->dsps5, "dsps60", worker->dsps60, "dsps1440", worker->dsps1440, "lastshare", worker->last_share.tv_sec, From 31c205c138cd3dd42a75f2b148606e180e8ee6c2 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Thu, 11 Jun 2015 12:05:01 +1000 Subject: [PATCH 440/544] Add basic configuration options for redirector mode --- ckpassthrough.conf | 19 ++++++++++++++++ ckproxy.conf | 1 - ckredirector.conf | 22 +++++++++++++++++++ src/ckpool.c | 55 +++++++++++++++++++++++++++++++++++++++++----- src/ckpool.h | 7 ++++++ 5 files changed, 98 insertions(+), 6 deletions(-) create mode 100644 ckpassthrough.conf create mode 100644 ckredirector.conf diff --git a/ckpassthrough.conf b/ckpassthrough.conf new file mode 100644 index 00000000..a0c9a092 --- /dev/null +++ b/ckpassthrough.conf @@ -0,0 +1,19 @@ +{ +"proxy" : [ + { + "url" : "ckpool.org:3333", + "auth" : "user", + "pass" : "pass" + } +], +"update_interval" : 30, +"serverurl" : [ + "192.168.1.100:3334", + "127.0.0.1:3334" + ], +"mindiff" : 1, +"startdiff" : 42, +"maxdiff" : 0, +"logdir" : "logs" +} +Comments from here on are ignored. diff --git a/ckproxy.conf b/ckproxy.conf index 5daddb5b..fec5783a 100644 --- a/ckproxy.conf +++ b/ckproxy.conf @@ -19,7 +19,6 @@ "mindiff" : 1, "startdiff" : 42, "maxdiff" : 0, -"clientsvspeed" : false, "logdir" : "logs" } Comments from here on are ignored. diff --git a/ckredirector.conf b/ckredirector.conf new file mode 100644 index 00000000..1a0f8733 --- /dev/null +++ b/ckredirector.conf @@ -0,0 +1,22 @@ +{ +"proxy" : [ + { + "url" : "ckpool.org:3333", + "auth" : "user", + "pass" : "pass" + } +], +"update_interval" : 30, +"serverurl" : [ + "192.168.1.100:3334", + "127.0.0.1:3334" + ], +"redirecturl" : [ + "node.ckpool.org:3333" + ], +"mindiff" : 1, +"startdiff" : 42, +"maxdiff" : 0, +"logdir" : "logs" +} +Comments from here on are ignored. diff --git a/src/ckpool.c b/src/ckpool.c index c4537d50..044e6cb9 100644 --- a/src/ckpool.c +++ b/src/ckpool.c @@ -1258,6 +1258,36 @@ out: return ret; } +static bool parse_redirecturls(ckpool_t *ckp, const json_t *arr_val) +{ + bool ret = false; + int arr_size, i; + + if (!arr_val) + goto out; + if (!json_is_array(arr_val)) { + LOGNOTICE("Unable to parse redirecturl entries as an array"); + goto out; + } + arr_size = json_array_size(arr_val); + if (!arr_size) { + LOGWARNING("redirecturl array empty"); + goto out; + } + ckp->redirecturls = arr_size; + ckp->redirecturl = ckalloc(sizeof(char *) * arr_size); + for (i = 0; i < arr_size; i++) { + json_t *val = json_array_get(arr_val, i); + + if (!_json_get_string(&ckp->redirecturl[i], val, "redirecturl")) + LOGWARNING("Invalid redirecturl entry number %d", i); + } + ret = true; +out: + return ret; +} + + static void parse_config(ckpool_t *ckp) { json_t *json_conf, *arr_val; @@ -1307,6 +1337,10 @@ static void parse_config(ckpool_t *ckp) if (arr_size) parse_proxies(ckp, arr_val, arr_size); } + arr_val = json_object_get(json_conf, "redirecturl"); + if (arr_val) + parse_redirecturls(ckp, arr_val); + json_decref(json_conf); } @@ -1434,6 +1468,7 @@ static struct option long_options[] = { {"name", required_argument, 0, 'n'}, {"passthrough", no_argument, 0, 'P'}, {"proxy", no_argument, 0, 'p'}, + {"redirector", no_argument, 0, 'R'}, {"ckdb-sockdir",required_argument, 0, 'S'}, {"sockdir", required_argument, 0, 's'}, {0, 0, 0, 0} @@ -1451,6 +1486,7 @@ static struct option long_options[] = { {"name", required_argument, 0, 'n'}, {"passthrough", no_argument, 0, 'P'}, {"proxy", no_argument, 0, 'p'}, + {"redirector", no_argument, 0, 'R'}, {"sockdir", required_argument, 0, 's'}, {0, 0, 0, 0} }; @@ -1494,7 +1530,7 @@ int main(int argc, char **argv) ckp.initial_args[ckp.args] = strdup(argv[ckp.args]); ckp.initial_args[ckp.args] = NULL; - while ((c = getopt_long(argc, argv, "Ac:Dd:g:HhkLl:n:PpS:s:", long_options, &i)) != -1) { + while ((c = getopt_long(argc, argv, "Ac:Dd:g:HhkLl:n:PpRS:s:", long_options, &i)) != -1) { switch (c) { case 'A': ckp.standalone = true; @@ -1549,15 +1585,20 @@ int main(int argc, char **argv) ckp.name = optarg; break; case 'P': - if (ckp.proxy) - quit(1, "Cannot set both proxy and passthrough mode"); + if (ckp.proxy || ckp.redirector) + quit(1, "Cannot set both proxy or redirector and passthrough mode"); ckp.standalone = ckp.proxy = ckp.passthrough = true; break; case 'p': - if (ckp.passthrough) - quit(1, "Cannot set both passthrough and proxy mode"); + if (ckp.passthrough || ckp.redirector) + quit(1, "Cannot set both passthrough or redirector and proxy mode"); ckp.proxy = true; break; + case 'R': + if (ckp.proxy || ckp.passthrough) + quit(1, "Cannot set both proxy or passthrough and redirector modes"); + ckp.standalone = ckp.proxy = ckp.passthrough = ckp.redirector = true; + break; case 'S': ckp.ckdb_sockdir = strdup(optarg); break; @@ -1570,6 +1611,10 @@ int main(int argc, char **argv) if (!ckp.name) { if (ckp.proxy) ckp.name = "ckproxy"; + else if (ckp.redirector) + ckp.name = "ckredirector"; + else if (ckp.passthrough) + ckp.name = "ckpassthrough"; else ckp.name = "ckpool"; } diff --git a/src/ckpool.h b/src/ckpool.h index 5fbaed5b..010a4db5 100644 --- a/src/ckpool.h +++ b/src/ckpool.h @@ -177,6 +177,9 @@ struct ckpool_instance { /* Are we running in passthrough mode */ bool passthrough; + /* Are we a redirecting passthrough */ + bool redirector; + /* Are we running as a proxy */ bool proxy; @@ -219,6 +222,10 @@ struct ckpool_instance { char **proxyauth; char **proxypass; + /* Passthrough redirect options */ + int redirecturls; + char **redirecturl; + /* Private data for each process */ void *data; }; From a8e09a444d561aeaa4d809d74c465726a4626fb5 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Thu, 11 Jun 2015 12:22:21 +1000 Subject: [PATCH 441/544] Abstract out add_share in proxy to be used by redirector code --- src/generator.c | 36 ++++++++++++++++++++++-------------- 1 file changed, 22 insertions(+), 14 deletions(-) diff --git a/src/generator.c b/src/generator.c index 4fdd09da..bea4fffa 100644 --- a/src/generator.c +++ b/src/generator.c @@ -1372,15 +1372,33 @@ static void stratifier_reconnect_client(ckpool_t *ckp, const int64_t id) send_proc(ckp->stratifier, buf); } +/* Add a share to the gdata share hashlist. Returns the share id */ +static int add_share(gdata_t *gdata, const int64_t client_id, const double diff) +{ + share_msg_t *share = ckzalloc(sizeof(share_msg_t)); + int ret; + + share->submit_time = time(NULL); + share->client_id = client_id; + share->diff = diff; + + /* Add new share entry to the share hashtable */ + mutex_lock(&gdata->share_lock); + ret = share->id = gdata->share_id++; + HASH_ADD_I64(gdata->shares, id, share); + mutex_unlock(&gdata->share_lock); + + return ret; +} + static void submit_share(gdata_t *gdata, json_t *val) { proxy_instance_t *proxy, *proxi; ckpool_t *ckp = gdata->ckp; + int id, subid, share_id; bool success = false; stratum_msg_t *msg; - share_msg_t *share; int64_t client_id; - int id, subid; /* Get the client id so we can tell the stratifier to drop it if the * proxy it's bound to is not functional */ @@ -1419,19 +1437,9 @@ static void submit_share(gdata_t *gdata, json_t *val) success = true; msg = ckzalloc(sizeof(stratum_msg_t)); - share = ckzalloc(sizeof(share_msg_t)); - share->submit_time = time(NULL); - share->client_id = client_id; - share->diff = proxi->diff; msg->json_msg = val; - - /* Add new share entry to the share hashtable */ - mutex_lock(&gdata->share_lock); - share->id = gdata->share_id++; - HASH_ADD_I64(gdata->shares, id, share); - mutex_unlock(&gdata->share_lock); - - json_object_set_nocheck(val, "id", json_integer(share->id)); + share_id = add_share(gdata, client_id, proxi->diff); + json_object_set_nocheck(val, "id", json_integer(share_id)); /* Add the new message to the psend list */ mutex_lock(&gdata->psend_lock); From 75501901adc1bcfee41984eb6c46dd5d05004425 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Thu, 11 Jun 2015 12:26:03 +1000 Subject: [PATCH 442/544] Add more info to parse_share return code --- src/generator.c | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/src/generator.c b/src/generator.c index bea4fffa..4e4ec622 100644 --- a/src/generator.c +++ b/src/generator.c @@ -1477,11 +1477,14 @@ static void account_shares(proxy_instance_t *proxy, const double diff, const boo mutex_unlock(&parent->proxy_lock); } -static bool parse_share(gdata_t *gdata, proxy_instance_t *proxi, const char *buf) +/* Returns zero if it is not recognised as a share, 1 if it is a valid share + * and -1 if it is recognised as a share but invalid. */ +static int parse_share(gdata_t *gdata, proxy_instance_t *proxi, const char *buf) { - bool ret = false, result = false; json_t *val = NULL, *idval; + bool result = false; share_msg_t *share; + int ret = 0; int64_t id; val = json_loads(buf, 0, NULL); @@ -1505,17 +1508,16 @@ static bool parse_share(gdata_t *gdata, proxy_instance_t *proxi, const char *buf HASH_DEL(gdata->shares, share); mutex_unlock(&gdata->share_lock); - /* We set response to true even if we don't find the matching share, - * so long as we recognised it as a share response */ - ret = true; if (!share) { LOGINFO("Proxy %d:%d failed to find matching share to result: %s", proxi->id, proxi->subid, buf); /* We don't know what diff these shares are so assume the * current proxy diff. */ account_shares(proxi, proxi->diff, result); + ret = -1; goto out; } + ret = 1; account_shares(proxi, share->diff, result); LOGINFO("Proxy %d:%d share result %s from client %d", proxi->id, proxi->subid, buf, share->client_id); @@ -2044,9 +2046,10 @@ static void *proxy_recv(void *arg) if (parse_method(ckp, subproxy, cs->buf)) continue; /* If it's not a method it should be a share result */ - if (!parse_share(gdata, subproxy, cs->buf)) + if (!parse_share(gdata, subproxy, cs->buf)) { LOGNOTICE("Proxy %d:%d unhandled stratum message: %s", subproxy->id, subproxy->subid, cs->buf); + } } while ((ret = read_socket_line(cs, 0)) > 0); } @@ -2130,9 +2133,10 @@ static void *userproxy_recv(void *arg) if (parse_method(ckp, proxy, cs->buf)) continue; /* If it's not a method it should be a share result */ - if (!parse_share(gdata, proxy, cs->buf)) + if (!parse_share(gdata, proxy, cs->buf)) { LOGNOTICE("Proxy %d:%d unhandled stratum message: %s", proxy->id, proxy->subid, cs->buf); + } } while ((ret = read_socket_line(cs, 0)) > 0); } return NULL; From 03edcc03cdf58f04e1bfac5bedb294ced93ea6fb Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Thu, 11 Jun 2015 14:49:47 +1000 Subject: [PATCH 443/544] Check for valid shares in redirector and redirect for first url for now --- src/ckpool.c | 17 ++++++---- src/ckpool.h | 1 + src/connector.c | 2 +- src/generator.c | 85 +++++++++++++++++++++++++++++++++++++++++++------ 4 files changed, 89 insertions(+), 16 deletions(-) diff --git a/src/ckpool.c b/src/ckpool.c index 044e6cb9..63cedf66 100644 --- a/src/ckpool.c +++ b/src/ckpool.c @@ -1163,7 +1163,7 @@ bool json_get_bool(bool *store, const json_t *val, const char *res) goto out; } if (!json_is_boolean(entry)) { - LOGWARNING("Json entry %s is not a boolean", res); + LOGINFO("Json entry %s is not a boolean", res); goto out; } *store = json_is_true(entry); @@ -1262,6 +1262,7 @@ static bool parse_redirecturls(ckpool_t *ckp, const json_t *arr_val) { bool ret = false; int arr_size, i; + char redirecturl[INET6_ADDRSTRLEN], url[INET6_ADDRSTRLEN], port[8]; if (!arr_val) goto out; @@ -1276,11 +1277,15 @@ static bool parse_redirecturls(ckpool_t *ckp, const json_t *arr_val) } ckp->redirecturls = arr_size; ckp->redirecturl = ckalloc(sizeof(char *) * arr_size); + ckp->redirectport = ckalloc(sizeof(char *) * arr_size); for (i = 0; i < arr_size; i++) { json_t *val = json_array_get(arr_val, i); - if (!_json_get_string(&ckp->redirecturl[i], val, "redirecturl")) - LOGWARNING("Invalid redirecturl entry number %d", i); + strncpy(redirecturl, json_string_value(val), INET6_ADDRSTRLEN - 1); + if (!url_from_serverurl(redirecturl, url, port)) + quit(1, "Invalid redirecturl entry %d %s", i, redirecturl); + ckp->redirecturl[i] = strdup(url); + ckp->redirectport[i] = strdup(port); } ret = true; out: @@ -1609,10 +1614,10 @@ int main(int argc, char **argv) } if (!ckp.name) { - if (ckp.proxy) - ckp.name = "ckproxy"; - else if (ckp.redirector) + if (ckp.redirector) ckp.name = "ckredirector"; + else if (ckp.proxy) + ckp.name = "ckproxy"; else if (ckp.passthrough) ckp.name = "ckpassthrough"; else diff --git a/src/ckpool.h b/src/ckpool.h index 010a4db5..cd62bcd6 100644 --- a/src/ckpool.h +++ b/src/ckpool.h @@ -225,6 +225,7 @@ struct ckpool_instance { /* Passthrough redirect options */ int redirecturls; char **redirecturl; + char **redirectport; /* Private data for each process */ void *data; diff --git a/src/connector.c b/src/connector.c index be006391..8be76549 100644 --- a/src/connector.c +++ b/src/connector.c @@ -733,7 +733,7 @@ static void process_client_msg(cdata_t *cdata, const char *buf) json_msg = json_loads(buf, 0, NULL); if (unlikely(!json_msg)) { - LOGWARNING("Invalid json message: %s", buf); + LOGWARNING("Invalid json message in process_client_msg: %s", buf); return; } diff --git a/src/generator.c b/src/generator.c index 4e4ec622..9e9452c4 100644 --- a/src/generator.c +++ b/src/generator.c @@ -1375,17 +1375,22 @@ static void stratifier_reconnect_client(ckpool_t *ckp, const int64_t id) /* Add a share to the gdata share hashlist. Returns the share id */ static int add_share(gdata_t *gdata, const int64_t client_id, const double diff) { - share_msg_t *share = ckzalloc(sizeof(share_msg_t)); + share_msg_t *share = ckzalloc(sizeof(share_msg_t)), *tmpshare; + time_t now; int ret; - share->submit_time = time(NULL); + share->submit_time = now = time(NULL); share->client_id = client_id; share->diff = diff; - /* Add new share entry to the share hashtable */ + /* Add new share entry to the share hashtable. Age old shares */ mutex_lock(&gdata->share_lock); ret = share->id = gdata->share_id++; HASH_ADD_I64(gdata->shares, id, share); + HASH_ITER(hh, gdata->shares, share, tmpshare) { + if (share->submit_time < now - 120) + HASH_DEL(gdata->shares, share); + } mutex_unlock(&gdata->share_lock); return ret; @@ -1479,7 +1484,7 @@ static void account_shares(proxy_instance_t *proxy, const double diff, const boo /* Returns zero if it is not recognised as a share, 1 if it is a valid share * and -1 if it is recognised as a share but invalid. */ -static int parse_share(gdata_t *gdata, proxy_instance_t *proxi, const char *buf) +static int parse_share(ckpool_t *ckp, gdata_t *gdata, proxy_instance_t *proxi, const char *buf) { json_t *val = NULL, *idval; bool result = false; @@ -1500,6 +1505,7 @@ static int parse_share(gdata_t *gdata, proxy_instance_t *proxi, const char *buf) id = json_integer_value(idval); if (unlikely(!json_get_bool(&result, val, "result"))) { LOGINFO("Failed to find result in upstream json msg: %s", buf); + goto out; } mutex_lock(&gdata->share_lock); @@ -1513,12 +1519,14 @@ static int parse_share(gdata_t *gdata, proxy_instance_t *proxi, const char *buf) proxi->id, proxi->subid, buf); /* We don't know what diff these shares are so assume the * current proxy diff. */ - account_shares(proxi, proxi->diff, result); + if (!ckp->redirector) + account_shares(proxi, proxi->diff, result); ret = -1; goto out; } ret = 1; - account_shares(proxi, share->diff, result); + if (!ckp->redirector) + account_shares(proxi, share->diff, result); LOGINFO("Proxy %d:%d share result %s from client %d", proxi->id, proxi->subid, buf, share->client_id); free(share); @@ -1692,11 +1700,36 @@ static void *proxy_send(void *arg) return NULL; } -static void passthrough_send(ckpool_t __maybe_unused *ckp, pass_msg_t *pm) +static void parse_redirector_share(ckpool_t *ckp, const char *msg) +{ + int64_t client_id; + json_t *val; + + val = json_loads(msg, 0, NULL); + if (unlikely(!val)) { + LOGWARNING("Invalid json message in parse_redirector_share: %s", msg); + return; + } + /* Extract the client id from the json message */ + client_id = json_integer_value(json_object_get(val, "client_id")); + /* Make sure this is a passthrough client value! */ + if (unlikely(client_id < 0xffffffffll)) { + LOGERR("redirect_client got invalid client id %"PRId64, client_id); + goto out; + } + /* Diff is irrelevant here as we don't keep track of it so use 0 */ + add_share(ckp->data, client_id, 0); +out: + json_decref(val); +} + +static void passthrough_send(ckpool_t *ckp, pass_msg_t *pm) { int len, sent; LOGDEBUG("Sending upstream json msg: %s", pm->msg); + if (ckp->redirector && strstr(pm->msg, "mining.submit")) + parse_redirector_share(ckp, pm->msg); len = strlen(pm->msg); sent = write_socket(pm->cs->fd, pm->msg, len); if (sent != len) { @@ -1875,6 +1908,34 @@ static void reconnect_proxy(proxy_instance_t *proxi) create_pthread(&pth, proxy_reconnect, proxi); } +static void redirect_client(ckpool_t *ckp, const char *buf) +{ + json_t *json_msg, *val; + int64_t client_id; + char *msg; + + json_msg = json_loads(buf, 0, NULL); + if (unlikely(!json_msg)) { + LOGWARNING("Invalid json message in redirect_client: %s", buf); + return; + } + /* Extract the client id from the json message */ + client_id = json_integer_value(json_object_get(json_msg, "client_id")); + /* Make sure this is a passthrough client value! */ + if (unlikely(client_id < 0xffffffffll)) { + LOGERR("redirect_client got invalid client id %"PRId64, client_id); + goto out; + } + JSON_CPACK(val, "{sIsosss[ssi]}", "id", "client_id", client_id, json_null(), + "method", "client.reconnect", "params", ckp->redirecturl[0], ckp->redirectport[0], 0); + msg = json_dumps(val, JSON_NO_UTF8 | JSON_PRESERVE_ORDER); + json_decref(val); + send_proc(ckp->connector, msg); + free(msg); +out: + json_decref(json_msg); +} + /* For receiving messages from an upstream pool to pass downstream. Responsible * for setting up the connection and testing pool is live. */ static void *passthrough_recv(void *arg) @@ -1928,6 +1989,12 @@ static void *passthrough_recv(void *arg) * process. Possibly parse parameters sent by upstream pool * here */ send_proc(ckp->connector, cs->buf); + + /* If we're a redirecting passthrough, look for a share + * responses here and redirect on a valid share. */ + if (ckp->redirector && parse_share(ckp, ckp->data, proxi, cs->buf) > 0) + redirect_client(ckp, cs->buf); + } return NULL; } @@ -2046,7 +2113,7 @@ static void *proxy_recv(void *arg) if (parse_method(ckp, subproxy, cs->buf)) continue; /* If it's not a method it should be a share result */ - if (!parse_share(gdata, subproxy, cs->buf)) { + if (!parse_share(ckp, gdata, subproxy, cs->buf)) { LOGNOTICE("Proxy %d:%d unhandled stratum message: %s", subproxy->id, subproxy->subid, cs->buf); } @@ -2133,7 +2200,7 @@ static void *userproxy_recv(void *arg) if (parse_method(ckp, proxy, cs->buf)) continue; /* If it's not a method it should be a share result */ - if (!parse_share(gdata, proxy, cs->buf)) { + if (!parse_share(ckp, gdata, proxy, cs->buf)) { LOGNOTICE("Proxy %d:%d unhandled stratum message: %s", proxy->id, proxy->subid, cs->buf); } From 98a2197568f31fc2fe95f110a584eec665cf6385 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Thu, 11 Jun 2015 16:12:20 +1000 Subject: [PATCH 444/544] Repair functioning of passthrough and add basic functions for share management in redirector --- src/ckpool.c | 4 +-- src/generator.c | 70 ++++++++++++++++++++++--------------------------- 2 files changed, 33 insertions(+), 41 deletions(-) diff --git a/src/ckpool.c b/src/ckpool.c index 63cedf66..7587ee7c 100644 --- a/src/ckpool.c +++ b/src/ckpool.c @@ -1616,10 +1616,10 @@ int main(int argc, char **argv) if (!ckp.name) { if (ckp.redirector) ckp.name = "ckredirector"; - else if (ckp.proxy) - ckp.name = "ckproxy"; else if (ckp.passthrough) ckp.name = "ckpassthrough"; + else if (ckp.proxy) + ckp.name = "ckproxy"; else ckp.name = "ckpool"; } diff --git a/src/generator.c b/src/generator.c index 9e9452c4..461dc01c 100644 --- a/src/generator.c +++ b/src/generator.c @@ -433,8 +433,10 @@ static bool send_json_msg(connsock_t *cs, const json_t *json_msg) return true; } -static bool connect_proxy(connsock_t *cs) +static bool connect_proxy(ckpool_t *ckp, connsock_t *cs, proxy_instance_t *proxy) { + if (cs->fd > 0) + Close(cs->fd); cs->fd = connect_socket(cs->url, cs->port); if (cs->fd < 0) { LOGINFO("Failed to connect socket to %s:%s in connect_proxy", @@ -442,6 +444,18 @@ static bool connect_proxy(connsock_t *cs) return false; } keep_sockalive(cs->fd); + if (!ckp->passthrough) { + struct epoll_event event; + + event.events = EPOLLIN | EPOLLRDHUP; + event.data.ptr = proxy; + /* Add this connsock_t to the epoll list */ + if (unlikely(epoll_ctl(proxy->epfd, EPOLL_CTL_ADD, cs->fd, &event) == -1)) { + LOGERR("Failed to add fd %d to epfd %d to epoll_ctl in proxy_alive", + cs->fd, proxy->epfd); + return false; + } + } return true; } @@ -685,13 +699,13 @@ out: return ret; } -static bool subscribe_stratum(connsock_t *cs, proxy_instance_t *proxi) +static bool subscribe_stratum(ckpool_t *ckp, connsock_t *cs, proxy_instance_t *proxi) { bool ret = false; json_t *req; retry: - /* Attempt to connect with the client description */ + /* Attempt to connect with the client description g*/ if (!proxi->no_params) { JSON_CPACK(req, "{s:i,s:s,s:[s]}", "id", 0, @@ -723,7 +737,7 @@ retry: LOGINFO("Proxy %d:%d %s failed connecting with parameters in subscribe_stratum, retrying without", proxi->id, proxi->subid, proxi->url); proxi->no_params = true; - ret = connect_proxy(cs); + ret = connect_proxy(ckp, cs, proxi); if (!ret) { LOGNOTICE("Proxy %d:%d %s failed to reconnect in subscribe_stratum", proxi->id, proxi->subid, proxi->url); @@ -1714,7 +1728,7 @@ static void parse_redirector_share(ckpool_t *ckp, const char *msg) client_id = json_integer_value(json_object_get(val, "client_id")); /* Make sure this is a passthrough client value! */ if (unlikely(client_id < 0xffffffffll)) { - LOGERR("redirect_client got invalid client id %"PRId64, client_id); + LOGERR("parse_redirector_share got invalid client id %"PRId64, client_id); goto out; } /* Diff is irrelevant here as we don't keep track of it so use 0 */ @@ -1750,9 +1764,8 @@ static void passthrough_add_send(proxy_instance_t *proxi, const char *msg) } static bool proxy_alive(ckpool_t *ckp, proxy_instance_t *proxi, connsock_t *cs, - bool pinging, int epfd) + bool pinging) { - struct epoll_event event; bool ret = false; /* Has this proxy already been reconnected? */ @@ -1764,7 +1777,7 @@ static bool proxy_alive(ckpool_t *ckp, proxy_instance_t *proxi, connsock_t *cs, LOGWARNING("Failed to extract address from %s", proxi->url); goto out; } - if (!connect_proxy(cs)) { + if (!connect_proxy(ckp, cs, proxi)) { if (!pinging) { LOGINFO("Failed to connect to %s:%s in proxy_mode!", cs->url, cs->port); @@ -1781,7 +1794,7 @@ static bool proxy_alive(ckpool_t *ckp, proxy_instance_t *proxi, connsock_t *cs, goto out; } /* Test we can connect, authorise and get stratum information */ - if (!subscribe_stratum(cs, proxi)) { + if (!subscribe_stratum(ckp, cs, proxi)) { if (!pinging) { LOGWARNING("Failed initial subscribe to %s:%s !", cs->url, cs->port); @@ -1802,20 +1815,8 @@ out: if (!ret) { send_stratifier_deadproxy(ckp, proxi->id, proxi->subid); /* Close and invalidate the file handle */ - if (cs->fd > 0) { - epoll_ctl(proxi->epfd, EPOLL_CTL_DEL, cs->fd, NULL); + if (cs->fd > 0) Close(cs->fd); - } - } else { - keep_sockalive(cs->fd); - event.events = EPOLLIN | EPOLLRDHUP; - event.data.ptr = proxi; - /* Add this connsock_t to the epoll list */ - if (unlikely(epoll_ctl(epfd, EPOLL_CTL_ADD, cs->fd, &event) == -1)) { - LOGERR("Failed to add fd %d to epfd %d to epoll_ctl in proxy_alive", - cs->fd, epfd); - return false; - } } proxi->alive = ret; return ret; @@ -1833,7 +1834,7 @@ static void *proxy_recruit(void *arg) retry: recruit = false; proxy = create_subproxy(ckp, gdata, parent, parent->url); - alive = proxy_alive(ckp, proxy, &proxy->cs, false, parent->epfd); + alive = proxy_alive(ckp, proxy, &proxy->cs, false); if (!alive) { LOGNOTICE("Subproxy failed proxy_alive testing"); store_proxy(gdata, proxy); @@ -1892,7 +1893,7 @@ static void *proxy_reconnect(void *arg) ckpool_t *ckp = proxy->ckp; pthread_detach(pthread_self()); - proxy_alive(ckp, proxy, cs, true, proxy->epfd); + proxy_alive(ckp, proxy, cs, true); proxy->reconnecting = false; return NULL; } @@ -1943,19 +1944,11 @@ static void *passthrough_recv(void *arg) proxy_instance_t *proxi = (proxy_instance_t *)arg; connsock_t *cs = &proxi->cs; ckpool_t *ckp = proxi->ckp; - struct epoll_event event; bool alive; - int epfd; rename_proc("passrecv"); - proxi->epfd = epfd = epoll_create1(EPOLL_CLOEXEC); - if (epfd < 0){ - LOGEMERG("FATAL: Failed to create epoll in passrecv"); - return NULL; - } - - if (proxy_alive(ckp, proxi, cs, false, epfd)) { + if (proxy_alive(ckp, proxi, cs, false)) { reconnect_generator(ckp); LOGWARNING("Proxy %d:%s connection established", proxi->id, proxi->url); } @@ -1964,7 +1957,7 @@ static void *passthrough_recv(void *arg) while (42) { int ret; - while (!proxy_alive(ckp, proxi, cs, true, epfd)) { + while (!proxy_alive(ckp, proxi, cs, true)) { if (alive) { alive = false; reconnect_generator(ckp); @@ -1975,13 +1968,12 @@ static void *passthrough_recv(void *arg) reconnect_generator(ckp); /* Make sure we receive a line within 90 seconds */ - ret = epoll_wait(epfd, &event, 1, 90000); - if (likely(ret > 0)) - ret = read_socket_line(cs, 60); + ret = read_socket_line(cs, 90); if (ret < 1) { LOGWARNING("Proxy %d:%s failed to read_socket_line in passthrough_recv, attempting reconnect", proxi->id, proxi->url); alive = proxi->alive = false; + Close(cs->fd); reconnect_generator(ckp); continue; } @@ -2038,9 +2030,9 @@ static void *proxy_recv(void *arg) return NULL; } - if (proxy_alive(ckp, proxi, cs, false, epfd)) { + if (proxy_alive(ckp, proxi, cs, false)) LOGWARNING("Proxy %d:%s connection established", proxi->id, proxi->url); - } + alive = proxi->alive; while (42) { From 35b81bad1735bae50672e20c8786bd8ccbe01469 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Thu, 11 Jun 2015 16:29:23 +1000 Subject: [PATCH 445/544] Remove attempted share identification from generator in redirector mode as it hasn't enough information --- src/generator.c | 73 ++++--------------------------------------------- 1 file changed, 6 insertions(+), 67 deletions(-) diff --git a/src/generator.c b/src/generator.c index 461dc01c..a52e7e6b 100644 --- a/src/generator.c +++ b/src/generator.c @@ -1498,7 +1498,7 @@ static void account_shares(proxy_instance_t *proxy, const double diff, const boo /* Returns zero if it is not recognised as a share, 1 if it is a valid share * and -1 if it is recognised as a share but invalid. */ -static int parse_share(ckpool_t *ckp, gdata_t *gdata, proxy_instance_t *proxi, const char *buf) +static int parse_share(gdata_t *gdata, proxy_instance_t *proxi, const char *buf) { json_t *val = NULL, *idval; bool result = false; @@ -1533,14 +1533,12 @@ static int parse_share(ckpool_t *ckp, gdata_t *gdata, proxy_instance_t *proxi, c proxi->id, proxi->subid, buf); /* We don't know what diff these shares are so assume the * current proxy diff. */ - if (!ckp->redirector) - account_shares(proxi, proxi->diff, result); + account_shares(proxi, proxi->diff, result); ret = -1; goto out; } ret = 1; - if (!ckp->redirector) - account_shares(proxi, share->diff, result); + account_shares(proxi, share->diff, result); LOGINFO("Proxy %d:%d share result %s from client %d", proxi->id, proxi->subid, buf, share->client_id); free(share); @@ -1714,36 +1712,11 @@ static void *proxy_send(void *arg) return NULL; } -static void parse_redirector_share(ckpool_t *ckp, const char *msg) -{ - int64_t client_id; - json_t *val; - - val = json_loads(msg, 0, NULL); - if (unlikely(!val)) { - LOGWARNING("Invalid json message in parse_redirector_share: %s", msg); - return; - } - /* Extract the client id from the json message */ - client_id = json_integer_value(json_object_get(val, "client_id")); - /* Make sure this is a passthrough client value! */ - if (unlikely(client_id < 0xffffffffll)) { - LOGERR("parse_redirector_share got invalid client id %"PRId64, client_id); - goto out; - } - /* Diff is irrelevant here as we don't keep track of it so use 0 */ - add_share(ckp->data, client_id, 0); -out: - json_decref(val); -} - -static void passthrough_send(ckpool_t *ckp, pass_msg_t *pm) +static void passthrough_send(ckpool_t __maybe_unused *ckp, pass_msg_t *pm) { int len, sent; LOGDEBUG("Sending upstream json msg: %s", pm->msg); - if (ckp->redirector && strstr(pm->msg, "mining.submit")) - parse_redirector_share(ckp, pm->msg); len = strlen(pm->msg); sent = write_socket(pm->cs->fd, pm->msg, len); if (sent != len) { @@ -1909,34 +1882,6 @@ static void reconnect_proxy(proxy_instance_t *proxi) create_pthread(&pth, proxy_reconnect, proxi); } -static void redirect_client(ckpool_t *ckp, const char *buf) -{ - json_t *json_msg, *val; - int64_t client_id; - char *msg; - - json_msg = json_loads(buf, 0, NULL); - if (unlikely(!json_msg)) { - LOGWARNING("Invalid json message in redirect_client: %s", buf); - return; - } - /* Extract the client id from the json message */ - client_id = json_integer_value(json_object_get(json_msg, "client_id")); - /* Make sure this is a passthrough client value! */ - if (unlikely(client_id < 0xffffffffll)) { - LOGERR("redirect_client got invalid client id %"PRId64, client_id); - goto out; - } - JSON_CPACK(val, "{sIsosss[ssi]}", "id", "client_id", client_id, json_null(), - "method", "client.reconnect", "params", ckp->redirecturl[0], ckp->redirectport[0], 0); - msg = json_dumps(val, JSON_NO_UTF8 | JSON_PRESERVE_ORDER); - json_decref(val); - send_proc(ckp->connector, msg); - free(msg); -out: - json_decref(json_msg); -} - /* For receiving messages from an upstream pool to pass downstream. Responsible * for setting up the connection and testing pool is live. */ static void *passthrough_recv(void *arg) @@ -1981,12 +1926,6 @@ static void *passthrough_recv(void *arg) * process. Possibly parse parameters sent by upstream pool * here */ send_proc(ckp->connector, cs->buf); - - /* If we're a redirecting passthrough, look for a share - * responses here and redirect on a valid share. */ - if (ckp->redirector && parse_share(ckp, ckp->data, proxi, cs->buf) > 0) - redirect_client(ckp, cs->buf); - } return NULL; } @@ -2105,7 +2044,7 @@ static void *proxy_recv(void *arg) if (parse_method(ckp, subproxy, cs->buf)) continue; /* If it's not a method it should be a share result */ - if (!parse_share(ckp, gdata, subproxy, cs->buf)) { + if (!parse_share(gdata, subproxy, cs->buf)) { LOGNOTICE("Proxy %d:%d unhandled stratum message: %s", subproxy->id, subproxy->subid, cs->buf); } @@ -2192,7 +2131,7 @@ static void *userproxy_recv(void *arg) if (parse_method(ckp, proxy, cs->buf)) continue; /* If it's not a method it should be a share result */ - if (!parse_share(ckp, gdata, proxy, cs->buf)) { + if (!parse_share(gdata, proxy, cs->buf)) { LOGNOTICE("Proxy %d:%d unhandled stratum message: %s", proxy->id, proxy->subid, cs->buf); } From 61ade171740d4a5a8a92b8bc37de998efa8c3816 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Thu, 11 Jun 2015 17:00:12 +1000 Subject: [PATCH 446/544] Use int64 for client_id in the generator --- src/generator.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/generator.c b/src/generator.c index a52e7e6b..6ddfa64a 100644 --- a/src/generator.c +++ b/src/generator.c @@ -51,7 +51,7 @@ struct share_msg { UT_hash_handle hh; int id; // Our own id for submitting upstream - int client_id; + int64_t client_id; time_t submit_time; double diff; }; @@ -63,7 +63,7 @@ struct stratum_msg { struct stratum_msg *prev; json_t *json_msg; - int client_id; + int64_t client_id; }; typedef struct stratum_msg stratum_msg_t; @@ -1539,7 +1539,7 @@ static int parse_share(gdata_t *gdata, proxy_instance_t *proxi, const char *buf) } ret = 1; account_shares(proxi, share->diff, result); - LOGINFO("Proxy %d:%d share result %s from client %d", proxi->id, proxi->subid, + LOGINFO("Proxy %d:%d share result %s from client %"PRId64, proxi->id, proxi->subid, buf, share->client_id); free(share); out: From ec9e730b1fe0c699555c1fe718011ed2dae9a2d9 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Thu, 11 Jun 2015 19:25:09 +1000 Subject: [PATCH 447/544] Look for unique ID accepted shares in the connector in redirector mode --- src/ckpool.c | 2 +- src/connector.c | 106 +++++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 106 insertions(+), 2 deletions(-) diff --git a/src/ckpool.c b/src/ckpool.c index 7587ee7c..554580ef 100644 --- a/src/ckpool.c +++ b/src/ckpool.c @@ -1103,7 +1103,7 @@ bool json_get_int64(int64_t *store, const json_t *val, const char *res) goto out; } if (!json_is_integer(entry)) { - LOGWARNING("Json entry %s is not an integer", res); + LOGINFO("Json entry %s is not an integer", res); goto out; } *store = json_integer_value(entry); diff --git a/src/connector.c b/src/connector.c index 8be76549..1821832f 100644 --- a/src/connector.c +++ b/src/connector.c @@ -25,6 +25,7 @@ typedef struct client_instance client_instance_t; typedef struct sender_send sender_send_t; +typedef struct share share_t; struct client_instance { /* For clients hashtable */ @@ -56,7 +57,11 @@ struct client_instance { /* Are we currently sending a blocked message from this client */ sender_send_t *sending; + + /* Is this the parent passthrough client */ bool passthrough; + + share_t *shares; }; struct sender_send { @@ -69,6 +74,14 @@ struct sender_send { int ofs; }; +struct share { + share_t *next; + share_t *prev; + + time_t submitted; + int64_t id; +}; + /* Private data for the connector */ struct connector_data { ckpool_t *ckp; @@ -348,6 +361,47 @@ static int invalidate_client(ckpool_t *ckp, cdata_t *cdata, client_instance_t *c static void send_client(cdata_t *cdata, int64_t id, char *buf); +/* Look for shares being submitted via a redirector and add them to a linked + * list for looking up the responses */ +static void parse_redirector_share(client_instance_t *client, const char *msg, const json_t *val) +{ + share_t *share, *tmp; + time_t now; + int64_t id; + + if (!json_get_int64(&id, val, "id")) { + LOGNOTICE("Failed to find redirector share id"); + return; + } + /* If this is not a share, delete any matching ID messages so we + * don't falsely assume the client has had an accepted share based on + * a true result to a different message. */ + if (!strstr(msg, "mining.submit")) { + LOGDEBUG("Redirector client %"PRId64" non share message: %s", client->id, msg); + DL_FOREACH_SAFE(client->shares, share, tmp) { + if (share->id == id) { + DL_DELETE(client->shares, share); + dealloc(share); + } + } + return; + } + share = ckzalloc(sizeof(share_t)); + now = time(NULL); + share->submitted = now; + share->id = id; + DL_APPEND(client->shares, share); + LOGINFO("Redirector adding client %"PRId64" share id: %"PRId64, client->id, id); + + /* Age old shares. */ + DL_FOREACH_SAFE(client->shares, share, tmp) { + if (now > share->submitted + 120) { + DL_DELETE(client->shares, share); + dealloc(share); + } + } +} + /* Client is holding a reference count from being on the epoll list */ static void parse_client_msg(cdata_t *cdata, client_instance_t *client) { @@ -400,14 +454,17 @@ reparse: invalidate_client(ckp, cdata, client); return; } else { - int64_t passthrough_id; char *s; if (client->passthrough) { + int64_t passthrough_id; + json_getdel_int64(&passthrough_id, val, "client_id"); passthrough_id = (client->id << 32) | passthrough_id; json_object_set_new_nocheck(val, "client_id", json_integer(passthrough_id)); } else { + if (ckp->redirector && strstr(msg, "mining.submit")) + parse_redirector_share(client, msg, val); json_object_set_new_nocheck(val, "client_id", json_integer(client->id)); json_object_set_new_nocheck(val, "address", json_string(client->address_name)); } @@ -640,6 +697,51 @@ static void *sender(void *arg) return NULL; } +static void test_redirector_shares(ckpool_t *ckp, client_instance_t *client, const char *buf) +{ + json_t *val = json_loads(buf, 0, NULL); + share_t *share, *found = NULL; + int64_t id; + + if (!val) { + LOGNOTICE("Invalid json response to client %"PRId64, client->id); + return; + } + if (!json_get_int64(&id, val, "id")) { + LOGINFO("Failed to find response id"); + goto out; + } + DL_FOREACH(client->shares, share) { + if (share->id == id) { + LOGDEBUG("Found matching share %"PRId64" in trs for client %"PRId64, + id, client->id); + DL_DELETE(client->shares, share); + found = share; + break; + } + } + if (found) { + bool result = false; + + dealloc(found); + if (!json_get_bool(&result, val, "result")) { + LOGINFO("Failed to find result in trs share"); + goto out; + } + if (!json_is_null(json_object_get(val, "error"))) { + LOGINFO("Got error for trs share"); + goto out; + } + if (!result) { + LOGDEBUG("Rejected trs share"); + goto out; + } + LOGWARNING("Found accepted share for client %"PRId64, client->id); + } +out: + json_decref(val); +} + /* Send a client by id a heap allocated buffer, allowing this function to * free the ram. */ static void send_client(cdata_t *cdata, const int64_t id, char *buf) @@ -690,6 +792,8 @@ static void send_client(cdata_t *cdata, const int64_t id, char *buf) free(buf); return; } + if (ckp->redirector) + test_redirector_shares(ckp, client, buf); } sender_send = ckzalloc(sizeof(sender_send_t)); From 65957df813755ecbdc1759bcfa091f30a593dc34 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Thu, 11 Jun 2015 19:37:25 +1000 Subject: [PATCH 448/544] Issue reconnect to miner submitting valid share in redirector mode --- src/connector.c | 30 +++++++++++++++++++++++++++++- 1 file changed, 29 insertions(+), 1 deletion(-) diff --git a/src/connector.c b/src/connector.c index 1821832f..9747d8e7 100644 --- a/src/connector.c +++ b/src/connector.c @@ -697,6 +697,32 @@ static void *sender(void *arg) return NULL; } +static void redirect_client(ckpool_t *ckp, client_instance_t *client) +{ + sender_send_t *sender_send; + cdata_t *cdata = ckp->data; + json_t *val; + char *buf; + + JSON_CPACK(val, "{sosss[ssi]}", "id", json_null(), "method", "client.reconnect", + "params", ckp->redirecturl[0], ckp->redirectport[0], 0); + buf = json_dumps(val, JSON_EOL); + json_decref(val); + + sender_send = ckzalloc(sizeof(sender_send_t)); + sender_send->client = client; + sender_send->buf = buf; + sender_send->len = strlen(buf); + + mutex_lock(&cdata->sender_lock); + cdata->sends_generated++; + DL_APPEND(cdata->sender_sends, sender_send); + pthread_cond_signal(&cdata->sender_cond); + mutex_unlock(&cdata->sender_lock); +} + +/* Look for accepted shares in redirector mode to know we can redirect this + * client to a protected server. */ static void test_redirector_shares(ckpool_t *ckp, client_instance_t *client, const char *buf) { json_t *val = json_loads(buf, 0, NULL); @@ -736,7 +762,9 @@ static void test_redirector_shares(ckpool_t *ckp, client_instance_t *client, con LOGDEBUG("Rejected trs share"); goto out; } - LOGWARNING("Found accepted share for client %"PRId64, client->id); + LOGNOTICE("Found accepted share for client %"PRId64" - redirecting", + client->id); + redirect_client(ckp, client); } out: json_decref(val); From 1b890257da34d8204f148445194b1a35a34a5e40 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Thu, 11 Jun 2015 19:42:48 +1000 Subject: [PATCH 449/544] Preserve the name used in redirecturl config for the redirector --- src/ckpool.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/ckpool.c b/src/ckpool.c index 554580ef..2d5b54a3 100644 --- a/src/ckpool.c +++ b/src/ckpool.c @@ -1262,7 +1262,8 @@ static bool parse_redirecturls(ckpool_t *ckp, const json_t *arr_val) { bool ret = false; int arr_size, i; - char redirecturl[INET6_ADDRSTRLEN], url[INET6_ADDRSTRLEN], port[8]; + char *redirecturl, url[INET6_ADDRSTRLEN], port[8]; + redirecturl = alloca(INET6_ADDRSTRLEN); if (!arr_val) goto out; @@ -1282,9 +1283,10 @@ static bool parse_redirecturls(ckpool_t *ckp, const json_t *arr_val) json_t *val = json_array_get(arr_val, i); strncpy(redirecturl, json_string_value(val), INET6_ADDRSTRLEN - 1); + /* See that the url properly resolves */ if (!url_from_serverurl(redirecturl, url, port)) quit(1, "Invalid redirecturl entry %d %s", i, redirecturl); - ckp->redirecturl[i] = strdup(url); + ckp->redirecturl[i] = strdup(strsep(&redirecturl, ":")); ckp->redirectport[i] = strdup(port); } ret = true; From 3ba8e1d6c152e48a922373f1f3a21ba9a85a1a14 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Thu, 11 Jun 2015 20:06:24 +1000 Subject: [PATCH 450/544] Hold an extra reference for the redirect_client message in the redirector --- src/connector.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/connector.c b/src/connector.c index 9747d8e7..39e26cdb 100644 --- a/src/connector.c +++ b/src/connector.c @@ -134,6 +134,13 @@ static void __inc_instance_ref(client_instance_t *client) client->ref++; } +static void inc_instance_ref(cdata_t *cdata, client_instance_t *client) +{ + ck_wlock(&cdata->lock); + __inc_instance_ref(client); + ck_wunlock(&cdata->lock); +} + /* Increase the reference count of instance */ static void __dec_instance_ref(client_instance_t *client) { @@ -713,6 +720,7 @@ static void redirect_client(ckpool_t *ckp, client_instance_t *client) sender_send->client = client; sender_send->buf = buf; sender_send->len = strlen(buf); + inc_instance_ref(cdata, client); mutex_lock(&cdata->sender_lock); cdata->sends_generated++; From 58b296da388adfc90de617dd15375a4876af75ab Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Thu, 11 Jun 2015 20:12:27 +1000 Subject: [PATCH 451/544] Document redirector option and configuration --- README | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/README b/README index c701deb0..a9a162fc 100644 --- a/README +++ b/README @@ -131,6 +131,7 @@ ckpool supports the following options: -n NAME | --name NAME -P | --passthrough -p | --proxy +-R | --redirector -S CKDB-SOCKDIR | --ckdb-sockdir CKDB-SOCKDIR -s SOCKDIR | --sockdir SOCKDIR @@ -179,6 +180,13 @@ clients as separate entities while presenting shares as a single user to the upstream pool specified. Note that the upstream pool needs to be a ckpool for it to scale to large hashrates. Standalone mode is Optional. +-R will start ckpool in a variant of passthrough mode. It is designed to be a +front end to filter out users that never contribute any shares. Once an +accepted share from the upstream pool is detected, it will issue a redirect to +one of the redirecturl entries in the configuration file. It will cycle over +entries if multiple exist, but try to keep all clients from the same IP +redirecting to the same pool. + -S tells ckpool which directory to look for the ckdb socket to talk to. This option does not exist when built without ckdb support. @@ -256,6 +264,9 @@ and 3334 in proxy mode. Multiple entries can be specified as an array by either IP or resolvable domain name but the executable must be able to bind to all of them and ports up to 1024 usually require privileged access. +"redirecturl" : This is an array of URLs that ckpool will redirect active +miners to in redirector mode. They must be valid resolvable URLs+ports. + "mindiff" : Minimum diff that vardiff will allow miners to drop to. Default 1 "startdiff" : Starting diff that new clients are given. Default 42 From 55c43a75e44e164fa11c02ab50dbaceb57fab328 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Thu, 11 Jun 2015 20:14:34 +1000 Subject: [PATCH 452/544] Document names of config files and running processes --- README | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/README b/README index a9a162fc..0cf3c908 100644 --- a/README +++ b/README @@ -142,8 +142,9 @@ are automatically accepted without any attempt to authorise users in any way. This option is explicitly enabled when built without ckdb support. -c tells ckpool to override its default configuration filename and -load the specified one. If -c is not specified, ckpool looks for ckpool.conf -whereas in proxy or passthrough modes it will look for ckproxy.conf +load the specified one. If -c is not specified, ckpool looks for ckpool.conf, +in proxy mode it looks for ckproxy.conf, in passthrough mode for +ckpassthrough.conf and in redirector mode for ckredirector.conf -d tells ckpool what the name of the ckdb process is that it should speak to, otherwise it will look for ckdb. @@ -168,7 +169,8 @@ and then workbase. maximum debug is level 7. -n will change the ckpool process name to that specified, allowing -multiple different named instances to be running. +multiple different named instances to be running. By default the variant +names are used: ckpool, ckproxy, ckpassthrough, ckredirector. -P will start ckpool in passthrough proxy mode where it collates all incoming connections and streams all information on a single connection to an upstream From 0dafbe12b43004fbd49cb19ee6b2b7a91a3e5281 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Thu, 11 Jun 2015 20:21:44 +1000 Subject: [PATCH 453/544] Comments --- src/connector.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/connector.c b/src/connector.c index 39e26cdb..d73d9d0f 100644 --- a/src/connector.c +++ b/src/connector.c @@ -61,6 +61,7 @@ struct client_instance { /* Is this the parent passthrough client */ bool passthrough; + /* Linked list of shares in redirector mode.*/ share_t *shares; }; @@ -369,7 +370,9 @@ static int invalidate_client(ckpool_t *ckp, cdata_t *cdata, client_instance_t *c static void send_client(cdata_t *cdata, int64_t id, char *buf); /* Look for shares being submitted via a redirector and add them to a linked - * list for looking up the responses */ + * list for looking up the responses. Theoretically this could leak shares but + * we should only ever store one share before redirecting active clients unless + * they don't support redirection. */ static void parse_redirector_share(client_instance_t *client, const char *msg, const json_t *val) { share_t *share, *tmp; From 1ba52b3daa6bf7f581baf82e1d2793cf8d5444fa Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Thu, 11 Jun 2015 20:49:59 +1000 Subject: [PATCH 454/544] Round robin over the redirect URLs in redirector mode, keeping clients from the same IP on the same URL --- src/connector.c | 41 ++++++++++++++++++++++++++++++++++++++++- 1 file changed, 40 insertions(+), 1 deletion(-) diff --git a/src/connector.c b/src/connector.c index d73d9d0f..46142cc4 100644 --- a/src/connector.c +++ b/src/connector.c @@ -26,6 +26,7 @@ typedef struct client_instance client_instance_t; typedef struct sender_send sender_send_t; typedef struct share share_t; +typedef struct redirect redirect_t; struct client_instance { /* For clients hashtable */ @@ -83,6 +84,13 @@ struct share { int64_t id; }; +struct redirect { + UT_hash_handle hh; + char address_name[INET6_ADDRSTRLEN]; + int id; + int redirect_no; +}; + /* Private data for the connector */ struct connector_data { ckpool_t *ckp; @@ -125,6 +133,11 @@ struct connector_data { /* For protecting the pending sends list */ mutex_t sender_lock; pthread_cond_t sender_cond; + + /* Hash list of all redirected IP address in redirector mode */ + redirect_t *redirects; + /* What redirect we're currently up to */ + int redirect; }; typedef struct connector_data cdata_t; @@ -707,15 +720,41 @@ static void *sender(void *arg) return NULL; } +static int add_redirect(ckpool_t *ckp, cdata_t *cdata, client_instance_t *client) +{ + redirect_t *redirect; + bool found; + + ck_wlock(&cdata->lock); + HASH_FIND_STR(cdata->redirects, client->address_name, redirect); + if (!redirect) { + redirect = ckzalloc(sizeof(redirect_t)); + strcpy(redirect->address_name, client->address_name); + redirect->redirect_no = cdata->redirect++; + if (cdata->redirect >= ckp->redirecturls) + cdata->redirect = 0; + HASH_ADD_STR(cdata->redirects, address_name, redirect); + found = false; + } else + found = true; + ck_wunlock(&cdata->lock); + + LOGNOTICE("Redirecting client %"PRId64" from %s IP %s to redirecturl %d", + client->id, found ? "matching" : "new", client->address_name, redirect->redirect_no); + return redirect->redirect_no; +} + static void redirect_client(ckpool_t *ckp, client_instance_t *client) { sender_send_t *sender_send; cdata_t *cdata = ckp->data; json_t *val; char *buf; + int num; + num = add_redirect(ckp, cdata, client); JSON_CPACK(val, "{sosss[ssi]}", "id", json_null(), "method", "client.reconnect", - "params", ckp->redirecturl[0], ckp->redirectport[0], 0); + "params", ckp->redirecturl[num], ckp->redirectport[num], 0); buf = json_dumps(val, JSON_EOL); json_decref(val); From b2a95205a8bc3a2b38ae2f5ad63a44537f70886d Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Thu, 11 Jun 2015 20:51:13 +1000 Subject: [PATCH 455/544] Add extra URL to sample redirecturl config --- ckredirector.conf | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ckredirector.conf b/ckredirector.conf index 1a0f8733..2c499a57 100644 --- a/ckredirector.conf +++ b/ckredirector.conf @@ -12,7 +12,8 @@ "127.0.0.1:3334" ], "redirecturl" : [ - "node.ckpool.org:3333" + "node1.ckpool.org:3333", + "node2.ckpool.org:3333" ], "mindiff" : 1, "startdiff" : 42, From 7b0611119f606824a7b9c409a40e46be2833ebb0 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Tue, 16 Jun 2015 12:52:10 +1000 Subject: [PATCH 456/544] Support testnet diffs --- src/libckpool.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/libckpool.c b/src/libckpool.c index cb91331a..f1b5c323 100644 --- a/src/libckpool.c +++ b/src/libckpool.c @@ -1967,7 +1967,10 @@ double diff_from_nbits(char *nbits) pow = nbits[0]; powdiff = (8 * (0x1d - 3)) - (8 * (pow - 3)); diff32 = be32toh(*((uint32_t *)nbits)) & 0x00FFFFFF; - numerator = 0xFFFFULL << powdiff; + if (likely(powdiff > 0)) + numerator = 0xFFFFULL << powdiff; + else + numerator = 0xFFFFULL >> -powdiff; return numerator / (double)diff32; } From 69124695cc68a16fd0d2598a745007c821aff884 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Tue, 16 Jun 2015 13:19:58 +1000 Subject: [PATCH 457/544] Add options for supporting a passthrough node --- src/ckpool.c | 25 +++++++++++++++++-------- src/ckpool.h | 3 +++ 2 files changed, 20 insertions(+), 8 deletions(-) diff --git a/src/ckpool.c b/src/ckpool.c index 2d5b54a3..8a6c5895 100644 --- a/src/ckpool.c +++ b/src/ckpool.c @@ -1472,6 +1472,7 @@ static struct option long_options[] = { {"killold", no_argument, 0, 'k'}, {"log-shares", no_argument, 0, 'L'}, {"loglevel", required_argument, 0, 'l'}, + {"node", no_argument, 0, 'N'}, {"name", required_argument, 0, 'n'}, {"passthrough", no_argument, 0, 'P'}, {"proxy", no_argument, 0, 'p'}, @@ -1490,6 +1491,7 @@ static struct option long_options[] = { {"killold", no_argument, 0, 'k'}, {"log-shares", no_argument, 0, 'L'}, {"loglevel", required_argument, 0, 'l'}, + {"node", no_argument, 0, 'N'}, {"name", required_argument, 0, 'n'}, {"passthrough", no_argument, 0, 'P'}, {"proxy", no_argument, 0, 'p'}, @@ -1537,7 +1539,7 @@ int main(int argc, char **argv) ckp.initial_args[ckp.args] = strdup(argv[ckp.args]); ckp.initial_args[ckp.args] = NULL; - while ((c = getopt_long(argc, argv, "Ac:Dd:g:HhkLl:n:PpRS:s:", long_options, &i)) != -1) { + while ((c = getopt_long(argc, argv, "Ac:Dd:g:HhkLl:Nn:PpRS:s:", long_options, &i)) != -1) { switch (c) { case 'A': ckp.standalone = true; @@ -1588,22 +1590,27 @@ int main(int argc, char **argv) LOG_EMERG, LOG_DEBUG, ckp.loglevel); } break; + case 'N': + if (ckp.proxy || ckp.redirector || ckp.passthrough) + quit(1, "Cannot set combinations of proxy, node, redirector and passthrough"); + ckp.standalone = ckp.node = ckp.proxy = ckp.passthrough = true; + break; case 'n': ckp.name = optarg; break; case 'P': - if (ckp.proxy || ckp.redirector) - quit(1, "Cannot set both proxy or redirector and passthrough mode"); + if (ckp.node || ckp.proxy || ckp.redirector) + quit(1, "Cannot set combinations of proxy, node, redirector and passthrough"); ckp.standalone = ckp.proxy = ckp.passthrough = true; break; case 'p': - if (ckp.passthrough || ckp.redirector) - quit(1, "Cannot set both passthrough or redirector and proxy mode"); + if (ckp.node || ckp.passthrough || ckp.redirector) + quit(1, "Cannot set combinations of proxy, node, redirector and passthrough"); ckp.proxy = true; break; case 'R': - if (ckp.proxy || ckp.passthrough) - quit(1, "Cannot set both proxy or passthrough and redirector modes"); + if (ckp.node || ckp.proxy || ckp.passthrough) + quit(1, "Cannot set combinations of proxy, node, redirector and passthrough"); ckp.standalone = ckp.proxy = ckp.passthrough = ckp.redirector = true; break; case 'S': @@ -1616,7 +1623,9 @@ int main(int argc, char **argv) } if (!ckp.name) { - if (ckp.redirector) + if (ckp.node) + ckp.name = "cknode"; + else if (ckp.redirector) ckp.name = "ckredirector"; else if (ckp.passthrough) ckp.name = "ckpassthrough"; diff --git a/src/ckpool.h b/src/ckpool.h index cd62bcd6..f23dedf4 100644 --- a/src/ckpool.h +++ b/src/ckpool.h @@ -180,6 +180,9 @@ struct ckpool_instance { /* Are we a redirecting passthrough */ bool redirector; + /* Are we a relay node passthrough */ + bool node; + /* Are we running as a proxy */ bool proxy; From b7e35effa591362fd96783c474ee9bb0186ef2c0 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Tue, 16 Jun 2015 14:57:57 +1000 Subject: [PATCH 458/544] Add backup btcd parsing in node mode --- src/ckpool.h | 1 + src/connector.c | 26 ++++++++++++++++++---- src/generator.c | 57 +++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 80 insertions(+), 4 deletions(-) diff --git a/src/ckpool.h b/src/ckpool.h index f23dedf4..6bb7784a 100644 --- a/src/ckpool.h +++ b/src/ckpool.h @@ -224,6 +224,7 @@ struct ckpool_instance { char **proxyurl; char **proxyauth; char **proxypass; + server_instance_t *btcdbackup; /* Passthrough redirect options */ int redirecturls; diff --git a/src/connector.c b/src/connector.c index 46142cc4..1c8ff948 100644 --- a/src/connector.c +++ b/src/connector.c @@ -61,6 +61,8 @@ struct client_instance { /* Is this the parent passthrough client */ bool passthrough; + /* Is this a parent relay passthrough node client */ + bool node; /* Linked list of shares in redirector mode.*/ share_t *shares; @@ -897,13 +899,14 @@ static bool client_exists(cdata_t *cdata, const int64_t id) return !!client; } -static void passthrough_client(cdata_t *cdata, client_instance_t *client) +static void passthrough_client(cdata_t *cdata, client_instance_t *client, const bool node) { char *buf; - LOGINFO("Connector adding passthrough client %"PRId64, client->id); + LOGINFO("Connector adding passthrough %sclient %"PRId64, node ? "node " : "", client->id); client->passthrough = true; - ASPRINTF(&buf, "{\"result\": true}\n"); + client->node = node; + ASPRINTF(&buf, "{\"result\": true, \"node\": %s}\n", node ? "true" : "false"); send_client(cdata, client->id, buf); } @@ -1090,7 +1093,22 @@ retry: LOGINFO("Connector failed to find client id %"PRId64" to pass through", client_id); goto retry; } - passthrough_client(cdata, client); + passthrough_client(cdata, client, false); + dec_instance_ref(cdata, client); + } else if (cmdmatch(buf, "node")) { + client_instance_t *client; + + ret = sscanf(buf, "node=%"PRId64, &client_id); + if (ret < 0) { + LOGDEBUG("Connector failed to parse node command: %s", buf); + goto retry; + } + client = ref_client_by_id(cdata, client_id); + if (unlikely(!client)) { + LOGINFO("Connector failed to find client id %"PRId64" to node pass through", client_id); + goto retry; + } + passthrough_client(cdata, client, true); dec_instance_ref(cdata, client); } else if (cmdmatch(buf, "getxfd")) { int fdno = -1; diff --git a/src/generator.c b/src/generator.c index 6ddfa64a..3dae8a1f 100644 --- a/src/generator.c +++ b/src/generator.c @@ -2676,6 +2676,41 @@ static proxy_instance_t *__add_proxy(ckpool_t *ckp, gdata_t *gdata, const int id return proxy; } +static bool alive_btcd(server_instance_t *si) +{ + connsock_t *cs = &si->cs; + char *userpass = NULL; + gbtbase_t gbt; + + if (!extract_sockaddr(si->url, &cs->url, &cs->port)) { + LOGWARNING("Failed to extract address from btcd %s", si->url); + return false; + } + userpass = strdup(si->auth); + realloc_strcat(&userpass, ":"); + realloc_strcat(&userpass, si->pass); + cs->auth = http_base64(userpass); + dealloc(userpass); + if (!cs->auth) { + LOGWARNING("Failed to create base64 auth from btcd %s", userpass); + return false; + } + if (cs->fd < 0) { + LOGWARNING("Failed to connect socket to btcd %s:%s !", cs->url, cs->port); + return false; + } + keep_sockalive(cs->fd); + /* Test we can authorise by getting a gbt, but we won't be using it. */ + memset(&gbt, 0, sizeof(gbtbase_t)); + if (!gen_gbtbase(cs, &gbt)) { + LOGINFO("Failed to get test block template from btcd %s:%s!", + cs->url, cs->port); + return false; + } + clear_gbtbase(&gbt); + return true; +} + static int proxy_mode(ckpool_t *ckp, proc_instance_t *pi) { gdata_t *gdata = ckp->data; @@ -2701,6 +2736,28 @@ static int proxy_mode(ckpool_t *ckp, proc_instance_t *pi) } } + if (ckp->node) { + if (ckp->btcds) { + /* If we also have btcds set up in node mode, try to talk to + * one of them as a way to submit blocks if we find them when + * submitting them upstream. */ + server_instance_t *si = ckp->btcdbackup = ckzalloc(sizeof(server_instance_t)); + + si->url = ckp->btcdurl[0]; + si->auth = ckp->btcdauth[0]; + si->pass = ckp->btcdpass[0]; + if (alive_btcd(si)) { + LOGNOTICE("Node backup btcd %s:%s alive", si->cs.url, si->cs.port); + } else { + LOGWARNING("Node backup btcd %s:%s failed! Will run as ordinary passthrough", + si->cs.url, si->cs.port); + ckp->btcdbackup = NULL; + free(si); + } + } else + LOGWARNING("No backup btcd specified in node mode! Will run as ordinary passthrough"); + } + LOGWARNING("%s generator ready", ckp->name); ret = proxy_loop(pi); From fc38e4d672e2e90da61f42b2e94698a88106d333 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Tue, 16 Jun 2015 15:24:11 +1000 Subject: [PATCH 459/544] Use server_alive to determine if node backup btcd is alive --- src/generator.c | 41 +++-------------------------------------- 1 file changed, 3 insertions(+), 38 deletions(-) diff --git a/src/generator.c b/src/generator.c index 3dae8a1f..15d8971b 100644 --- a/src/generator.c +++ b/src/generator.c @@ -2676,41 +2676,6 @@ static proxy_instance_t *__add_proxy(ckpool_t *ckp, gdata_t *gdata, const int id return proxy; } -static bool alive_btcd(server_instance_t *si) -{ - connsock_t *cs = &si->cs; - char *userpass = NULL; - gbtbase_t gbt; - - if (!extract_sockaddr(si->url, &cs->url, &cs->port)) { - LOGWARNING("Failed to extract address from btcd %s", si->url); - return false; - } - userpass = strdup(si->auth); - realloc_strcat(&userpass, ":"); - realloc_strcat(&userpass, si->pass); - cs->auth = http_base64(userpass); - dealloc(userpass); - if (!cs->auth) { - LOGWARNING("Failed to create base64 auth from btcd %s", userpass); - return false; - } - if (cs->fd < 0) { - LOGWARNING("Failed to connect socket to btcd %s:%s !", cs->url, cs->port); - return false; - } - keep_sockalive(cs->fd); - /* Test we can authorise by getting a gbt, but we won't be using it. */ - memset(&gbt, 0, sizeof(gbtbase_t)); - if (!gen_gbtbase(cs, &gbt)) { - LOGINFO("Failed to get test block template from btcd %s:%s!", - cs->url, cs->port); - return false; - } - clear_gbtbase(&gbt); - return true; -} - static int proxy_mode(ckpool_t *ckp, proc_instance_t *pi) { gdata_t *gdata = ckp->data; @@ -2739,14 +2704,14 @@ static int proxy_mode(ckpool_t *ckp, proc_instance_t *pi) if (ckp->node) { if (ckp->btcds) { /* If we also have btcds set up in node mode, try to talk to - * one of them as a way to submit blocks if we find them when - * submitting them upstream. */ + * one of them as a way to submit blocks if we find them when + * submitting them upstream. */ server_instance_t *si = ckp->btcdbackup = ckzalloc(sizeof(server_instance_t)); si->url = ckp->btcdurl[0]; si->auth = ckp->btcdauth[0]; si->pass = ckp->btcdpass[0]; - if (alive_btcd(si)) { + if (server_alive(ckp, si, false)) { LOGNOTICE("Node backup btcd %s:%s alive", si->cs.url, si->cs.port); } else { LOGWARNING("Node backup btcd %s:%s failed! Will run as ordinary passthrough", From e96f1d60e23dec450253c288e70749fcdd7b6735 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Tue, 16 Jun 2015 17:55:53 +1000 Subject: [PATCH 460/544] Send and parse node request with passthrough message --- src/generator.c | 13 ++++++++----- src/stratifier.c | 11 +++++++++-- 2 files changed, 17 insertions(+), 7 deletions(-) diff --git a/src/generator.c b/src/generator.c index 15d8971b..65ee3492 100644 --- a/src/generator.c +++ b/src/generator.c @@ -753,14 +753,14 @@ out: return ret; } -static bool passthrough_stratum(connsock_t *cs, proxy_instance_t *proxi) +static bool passthrough_stratum(connsock_t *cs, proxy_instance_t *proxi, const bool node) { json_t *req, *val = NULL, *res_val, *err_val; bool ret = false; - JSON_CPACK(req, "{s:s,s:[s]}", + JSON_CPACK(req, "{s:s,s:[sb]}", "method", "mining.passthrough", - "params", PACKAGE"/"VERSION); + "params", PACKAGE"/"VERSION, node); ret = send_json_msg(cs, req); json_decref(req); if (!ret) { @@ -1758,7 +1758,7 @@ static bool proxy_alive(ckpool_t *ckp, proxy_instance_t *proxi, connsock_t *cs, goto out; } if (ckp->passthrough) { - if (!passthrough_stratum(cs, proxi)) { + if (!passthrough_stratum(cs, proxi, ckp->node)) { LOGWARNING("Failed initial passthrough to %s:%s !", cs->url, cs->port); goto out; @@ -2718,9 +2718,12 @@ static int proxy_mode(ckpool_t *ckp, proc_instance_t *pi) si->cs.url, si->cs.port); ckp->btcdbackup = NULL; free(si); + ckp->node = false; } - } else + } else { LOGWARNING("No backup btcd specified in node mode! Will run as ordinary passthrough"); + ckp->node = false; + } } LOGWARNING("%s generator ready", ckp->name); diff --git a/src/stratifier.c b/src/stratifier.c index 2d8f006a..65fe62e3 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -4817,14 +4817,21 @@ static void parse_method(ckpool_t *ckp, sdata_t *sdata, stratum_instance_t *clie } if (unlikely(cmdmatch(method, "mining.passthrough"))) { + json_t *nodeval = json_array_get(params_val, 1); char buf[256]; + bool node; + if (nodeval) + node = json_is_true(nodeval); + else + node = false; /* We need to inform the connector process that this client * is a passthrough and to manage its messages accordingly. No * data from this client id should ever come back to this * stratifier after this so drop the client in the stratifier. */ - LOGNOTICE("Adding passthrough client %"PRId64" %s", client_id, client->address); - snprintf(buf, 255, "passthrough=%"PRId64, client_id); + LOGNOTICE("Adding %spassthrough client %"PRId64" %s", node ? "node " : "", + client_id, client->address); + snprintf(buf, 255, "%s=%"PRId64, node ? "node" : "passthrough", client_id); send_proc(ckp->connector, buf); drop_client(ckp, sdata, client_id); return; From bce4cab1763342efe8ec26852a5bb3ec6a0207c9 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Sat, 1 Aug 2015 15:57:53 +1000 Subject: [PATCH 461/544] Roll back code for relay node. --- src/ckpool.c | 25 ++++++++----------------- src/ckpool.h | 5 ----- src/connector.c | 26 ++++---------------------- src/generator.c | 35 +++++------------------------------ src/stratifier.c | 11 ++--------- 5 files changed, 19 insertions(+), 83 deletions(-) diff --git a/src/ckpool.c b/src/ckpool.c index 8bda9dd0..11c12f7d 100644 --- a/src/ckpool.c +++ b/src/ckpool.c @@ -1481,7 +1481,6 @@ static struct option long_options[] = { {"killold", no_argument, 0, 'k'}, {"log-shares", no_argument, 0, 'L'}, {"loglevel", required_argument, 0, 'l'}, - {"node", no_argument, 0, 'N'}, {"name", required_argument, 0, 'n'}, {"passthrough", no_argument, 0, 'P'}, {"proxy", no_argument, 0, 'p'}, @@ -1500,7 +1499,6 @@ static struct option long_options[] = { {"killold", no_argument, 0, 'k'}, {"log-shares", no_argument, 0, 'L'}, {"loglevel", required_argument, 0, 'l'}, - {"node", no_argument, 0, 'N'}, {"name", required_argument, 0, 'n'}, {"passthrough", no_argument, 0, 'P'}, {"proxy", no_argument, 0, 'p'}, @@ -1548,7 +1546,7 @@ int main(int argc, char **argv) ckp.initial_args[ckp.args] = strdup(argv[ckp.args]); ckp.initial_args[ckp.args] = NULL; - while ((c = getopt_long(argc, argv, "Ac:Dd:g:HhkLl:Nn:PpRS:s:", long_options, &i)) != -1) { + while ((c = getopt_long(argc, argv, "Ac:Dd:g:HhkLl:n:PpRS:s:", long_options, &i)) != -1) { switch (c) { case 'A': ckp.standalone = true; @@ -1599,27 +1597,22 @@ int main(int argc, char **argv) LOG_EMERG, LOG_DEBUG, ckp.loglevel); } break; - case 'N': - if (ckp.proxy || ckp.redirector || ckp.passthrough) - quit(1, "Cannot set combinations of proxy, node, redirector and passthrough"); - ckp.standalone = ckp.node = ckp.proxy = ckp.passthrough = true; - break; case 'n': ckp.name = optarg; break; case 'P': - if (ckp.node || ckp.proxy || ckp.redirector) - quit(1, "Cannot set combinations of proxy, node, redirector and passthrough"); + if (ckp.proxy || ckp.redirector) + quit(1, "Cannot set both proxy or redirector and passthrough mode"); ckp.standalone = ckp.proxy = ckp.passthrough = true; break; case 'p': - if (ckp.node || ckp.passthrough || ckp.redirector) - quit(1, "Cannot set combinations of proxy, node, redirector and passthrough"); + if (ckp.passthrough || ckp.redirector) + quit(1, "Cannot set both passthrough or redirector and proxy mode"); ckp.proxy = true; break; case 'R': - if (ckp.node || ckp.proxy || ckp.passthrough) - quit(1, "Cannot set combinations of proxy, node, redirector and passthrough"); + if (ckp.proxy || ckp.passthrough) + quit(1, "Cannot set both proxy or passthrough and redirector modes"); ckp.standalone = ckp.proxy = ckp.passthrough = ckp.redirector = true; break; case 'S': @@ -1632,9 +1625,7 @@ int main(int argc, char **argv) } if (!ckp.name) { - if (ckp.node) - ckp.name = "cknode"; - else if (ckp.redirector) + if (ckp.redirector) ckp.name = "ckredirector"; else if (ckp.passthrough) ckp.name = "ckpassthrough"; diff --git a/src/ckpool.h b/src/ckpool.h index 0fcd4b5b..39d24c01 100644 --- a/src/ckpool.h +++ b/src/ckpool.h @@ -22,7 +22,6 @@ #define RPC_TIMEOUT 60 struct ckpool_instance; - typedef struct ckpool_instance ckpool_t; struct ckmsg { @@ -184,9 +183,6 @@ struct ckpool_instance { /* Are we a redirecting passthrough */ bool redirector; - /* Are we a relay node passthrough */ - bool node; - /* Are we running as a proxy */ bool proxy; @@ -228,7 +224,6 @@ struct ckpool_instance { char **proxyurl; char **proxyauth; char **proxypass; - server_instance_t *btcdbackup; /* Passthrough redirect options */ int redirecturls; diff --git a/src/connector.c b/src/connector.c index 1c8ff948..46142cc4 100644 --- a/src/connector.c +++ b/src/connector.c @@ -61,8 +61,6 @@ struct client_instance { /* Is this the parent passthrough client */ bool passthrough; - /* Is this a parent relay passthrough node client */ - bool node; /* Linked list of shares in redirector mode.*/ share_t *shares; @@ -899,14 +897,13 @@ static bool client_exists(cdata_t *cdata, const int64_t id) return !!client; } -static void passthrough_client(cdata_t *cdata, client_instance_t *client, const bool node) +static void passthrough_client(cdata_t *cdata, client_instance_t *client) { char *buf; - LOGINFO("Connector adding passthrough %sclient %"PRId64, node ? "node " : "", client->id); + LOGINFO("Connector adding passthrough client %"PRId64, client->id); client->passthrough = true; - client->node = node; - ASPRINTF(&buf, "{\"result\": true, \"node\": %s}\n", node ? "true" : "false"); + ASPRINTF(&buf, "{\"result\": true}\n"); send_client(cdata, client->id, buf); } @@ -1093,22 +1090,7 @@ retry: LOGINFO("Connector failed to find client id %"PRId64" to pass through", client_id); goto retry; } - passthrough_client(cdata, client, false); - dec_instance_ref(cdata, client); - } else if (cmdmatch(buf, "node")) { - client_instance_t *client; - - ret = sscanf(buf, "node=%"PRId64, &client_id); - if (ret < 0) { - LOGDEBUG("Connector failed to parse node command: %s", buf); - goto retry; - } - client = ref_client_by_id(cdata, client_id); - if (unlikely(!client)) { - LOGINFO("Connector failed to find client id %"PRId64" to node pass through", client_id); - goto retry; - } - passthrough_client(cdata, client, true); + passthrough_client(cdata, client); dec_instance_ref(cdata, client); } else if (cmdmatch(buf, "getxfd")) { int fdno = -1; diff --git a/src/generator.c b/src/generator.c index aaff49e1..dd135b80 100644 --- a/src/generator.c +++ b/src/generator.c @@ -755,15 +755,15 @@ out: return ret; } -static bool passthrough_stratum(connsock_t *cs, proxy_instance_t *proxi, const bool node) +static bool passthrough_stratum(connsock_t *cs, proxy_instance_t *proxi) { json_t *req, *val = NULL, *res_val, *err_val; float timeout = 10; bool ret = false; - JSON_CPACK(req, "{s:s,s:[sb]}", + JSON_CPACK(req, "{s:s,s:[s]}", "method", "mining.passthrough", - "params", PACKAGE"/"VERSION, node); + "params", PACKAGE"/"VERSION); ret = send_json_msg(cs, req); json_decref(req); if (!ret) { @@ -1761,7 +1761,7 @@ static bool proxy_alive(ckpool_t *ckp, proxy_instance_t *proxi, connsock_t *cs, goto out; } if (ckp->passthrough) { - if (!passthrough_stratum(cs, proxi, ckp->node)) { + if (!passthrough_stratum(cs, proxi)) { LOGWARNING("Failed initial passthrough to %s:%s !", cs->url, cs->port); goto out; @@ -2029,7 +2029,7 @@ static void *proxy_recv(void *arg) * has likely stopped responding. */ ret = epoll_wait(epfd, &event, 1, 600000); if (likely(ret > 0)) { - float timeout = 10; + float timeout = 30; subproxy = event.data.ptr; cs = &subproxy->cs; @@ -2707,31 +2707,6 @@ static int proxy_mode(ckpool_t *ckp, proc_instance_t *pi) } } - if (ckp->node) { - if (ckp->btcds) { - /* If we also have btcds set up in node mode, try to talk to - * one of them as a way to submit blocks if we find them when - * submitting them upstream. */ - server_instance_t *si = ckp->btcdbackup = ckzalloc(sizeof(server_instance_t)); - - si->url = ckp->btcdurl[0]; - si->auth = ckp->btcdauth[0]; - si->pass = ckp->btcdpass[0]; - if (server_alive(ckp, si, false)) { - LOGNOTICE("Node backup btcd %s:%s alive", si->cs.url, si->cs.port); - } else { - LOGWARNING("Node backup btcd %s:%s failed! Will run as ordinary passthrough", - si->cs.url, si->cs.port); - ckp->btcdbackup = NULL; - free(si); - ckp->node = false; - } - } else { - LOGWARNING("No backup btcd specified in node mode! Will run as ordinary passthrough"); - ckp->node = false; - } - } - LOGWARNING("%s generator ready", ckp->name); ret = proxy_loop(pi); diff --git a/src/stratifier.c b/src/stratifier.c index 8c6bddab..55cbc3dc 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -4817,21 +4817,14 @@ static void parse_method(ckpool_t *ckp, sdata_t *sdata, stratum_instance_t *clie } if (unlikely(cmdmatch(method, "mining.passthrough"))) { - json_t *nodeval = json_array_get(params_val, 1); char buf[256]; - bool node; - if (nodeval) - node = json_is_true(nodeval); - else - node = false; /* We need to inform the connector process that this client * is a passthrough and to manage its messages accordingly. No * data from this client id should ever come back to this * stratifier after this so drop the client in the stratifier. */ - LOGNOTICE("Adding %spassthrough client %"PRId64" %s", node ? "node " : "", - client_id, client->address); - snprintf(buf, 255, "%s=%"PRId64, node ? "node" : "passthrough", client_id); + LOGNOTICE("Adding passthrough client %"PRId64" %s", client_id, client->address); + snprintf(buf, 255, "passthrough=%"PRId64, client_id); send_proc(ckp->connector, buf); drop_client(ckp, sdata, client_id); return; From effcf78586f2453fc9d3938a7c0421d1f07caa88 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Tue, 18 Aug 2015 13:50:54 +1000 Subject: [PATCH 462/544] Pass variable to read_socket_line instead of constant --- src/generator.c | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/src/generator.c b/src/generator.c index dd135b80..be8c5db5 100644 --- a/src/generator.c +++ b/src/generator.c @@ -1981,6 +1981,7 @@ static void *proxy_recv(void *arg) while (42) { share_msg_t *share, *tmpshare; notify_instance_t *ni, *tmp; + float timeout; time_t now; int ret; @@ -2029,14 +2030,14 @@ static void *proxy_recv(void *arg) * has likely stopped responding. */ ret = epoll_wait(epfd, &event, 1, 600000); if (likely(ret > 0)) { - float timeout = 30; - subproxy = event.data.ptr; cs = &subproxy->cs; if (event.events & (EPOLLHUP | EPOLLERR | EPOLLRDHUP)) ret = -1; - else + else { + timeout = 30; ret = read_socket_line(cs, &timeout); + } } if (ret < 1) { LOGNOTICE("Proxy %d:%d %s failed to epoll/read_socket_line in proxy_recv", @@ -2054,7 +2055,8 @@ static void *proxy_recv(void *arg) LOGNOTICE("Proxy %d:%d unhandled stratum message: %s", subproxy->id, subproxy->subid, cs->buf); } - } while ((ret = read_socket_line(cs, 0)) > 0); + timeout = 0; + } while ((ret = read_socket_line(cs, &timeout)) > 0); } return NULL; @@ -2082,6 +2084,7 @@ static void *userproxy_recv(void *arg) share_msg_t *share, *tmpshare; notify_instance_t *ni, *tmp; connsock_t *cs; + float timeout; time_t now; int ret; @@ -2141,7 +2144,8 @@ static void *userproxy_recv(void *arg) LOGNOTICE("Proxy %d:%d unhandled stratum message: %s", proxy->id, proxy->subid, cs->buf); } - } while ((ret = read_socket_line(cs, 0)) > 0); + timeout = 0; + } while ((ret = read_socket_line(cs, &timeout)) > 0); } return NULL; } From ead0f1cae43396b1ab5152b63dfaa8ffcad13cb1 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Tue, 18 Aug 2015 13:51:39 +1000 Subject: [PATCH 463/544] Allow timeout to reach zero in read_socket_line for one non-blocking read --- src/ckpool.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/ckpool.c b/src/ckpool.c index 1748c522..faf9fd7d 100644 --- a/src/ckpool.c +++ b/src/ckpool.c @@ -536,7 +536,7 @@ int read_socket_line(connsock_t *cs, float *timeout) tv_time(&start); rewait: - if (*timeout <= 0) { + if (*timeout < 0) { LOGDEBUG("Timed out in read_socket_line"); ret = 0; goto out; @@ -569,7 +569,7 @@ rewait: if (eom) break; /* Have we used up all the timeout yet? */ - if (*timeout > 0 && (errno == EAGAIN || errno == EWOULDBLOCK || !ret)) + if (*timeout >= 0 && (errno == EAGAIN || errno == EWOULDBLOCK || !ret)) goto rewait; if (cs->ckp->proxy) LOGINFO("Failed to recv in read_socket_line"); From d510d1af534b2b165a3715d6e8b9f33f5284a5f3 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Mon, 21 Sep 2015 13:07:26 +1000 Subject: [PATCH 464/544] Increase default log buf size to fit mining notify messages --- src/libckpool.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libckpool.h b/src/libckpool.h index aed291dc..d4b48747 100644 --- a/src/libckpool.h +++ b/src/libckpool.h @@ -192,7 +192,7 @@ static inline void flip_80(void *dest_p, const void *src_p) void logmsg(int loglevel, const char *fmt, ...); -#define DEFLOGBUFSIZ 512 +#define DEFLOGBUFSIZ 1024 #define LOGMSGBUF(__lvl, __buf) do { \ logmsg(__lvl, "%s", __buf); \ From 99ce264a8e72309549904bcad35ceb7c1b7b0e84 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Mon, 21 Sep 2015 13:17:57 +1000 Subject: [PATCH 465/544] Send a dropall to the stratifier when we're in reject mode --- src/connector.c | 1 + src/stratifier.c | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/connector.c b/src/connector.c index 46142cc4..608fb556 100644 --- a/src/connector.c +++ b/src/connector.c @@ -1067,6 +1067,7 @@ retry: } else if (cmdmatch(buf, "reject")) { LOGDEBUG("Connector received reject signal"); cdata->accept = false; + send_proc(ckp->stratifier, "dropall"); } else if (cmdmatch(buf, "stats")) { char *msg; diff --git a/src/stratifier.c b/src/stratifier.c index 55cbc3dc..0b7890cd 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -1138,7 +1138,7 @@ static void drop_allclients(ckpool_t *ckp) ck_wunlock(&sdata->instance_lock); if (kills) - LOGNOTICE("Dropped %d instances", kills); + LOGNOTICE("Dropped %d instances for dropall request", kills); } /* Copy only the relevant parts of the master sdata for each subproxy */ From 21f80c3ecf17a163a4da95adcf77ba6ad8528725 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Mon, 21 Sep 2015 13:34:31 +1000 Subject: [PATCH 466/544] Don't allow passthrough subclients to reconnect --- src/stratifier.c | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/stratifier.c b/src/stratifier.c index 0b7890cd..36d161bf 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -3343,6 +3343,12 @@ out_unlock: return ret; } +/* passthrough subclients have client_ids in the high bits */ +static inline bool passthrough_subclient(const int64_t client_id) +{ + return (client_id > 0xffffffffll); +} + /* Extranonce1 must be set here. Needs to be entered with client holding a ref * count. */ static json_t *parse_subscribe(stratum_instance_t *client, const int64_t client_id, const json_t *params_val) @@ -3386,7 +3392,7 @@ static json_t *parse_subscribe(stratum_instance_t *client, const int64_t client_ session_id = int_from_sessionid(buf); LOGDEBUG("Found old session id %d", session_id); } - if (!ckp->proxy && session_id) { + if (!ckp->proxy && session_id && !passthrough_subclient(client_id)) { if ((client->enonce1_64 = disconnected_sessionid_exists(sdata, session_id, client_id))) { sprintf(client->enonce1, "%016lx", client->enonce1_64); old_match = true; From 250bf42cc085493ae21415d9d401126e73d757d7 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Mon, 21 Sep 2015 16:43:21 +1000 Subject: [PATCH 467/544] Show successful connection message in passthrough mode as well --- src/generator.c | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/generator.c b/src/generator.c index b5fd9e48..c9de7143 100644 --- a/src/generator.c +++ b/src/generator.c @@ -2575,10 +2575,8 @@ reconnect: goto out; if (proxi != cproxy) { proxi = cproxy; - if (!ckp->passthrough) { - LOGWARNING("Successfully connected to proxy %d %s as proxy", - proxi->id, proxi->url); - } + LOGWARNING("Successfully connected to proxy %d %s as proxy%s", + proxi->id, proxi->url, ckp->passthrough ? " in passthrough mode" : ""); } retry: clear_unix_msg(&umsg); From 629e86555d00b0daa2e36fa4f6e8400645daa7e9 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Mon, 21 Sep 2015 17:02:08 +1000 Subject: [PATCH 468/544] Clean up passthrough console messages --- src/generator.c | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/src/generator.c b/src/generator.c index c9de7143..2695cabc 100644 --- a/src/generator.c +++ b/src/generator.c @@ -1725,7 +1725,7 @@ static void passthrough_send(ckpool_t *ckp, pass_msg_t *pm) if (sent != len) { LOGWARNING("Failed to passthrough %d bytes of message %s, attempting reconnect", len, pm->msg); - send_proc(ckp->generator, "reconnect"); + reconnect_generator(ckp); } free(pm->msg); free(pm); @@ -1897,10 +1897,8 @@ static void *passthrough_recv(void *arg) rename_proc("passrecv"); - if (proxy_alive(ckp, proxi, cs, false)) { - reconnect_generator(ckp); - LOGWARNING("Proxy %d:%s connection established", proxi->id, proxi->url); - } + if (proxy_alive(ckp, proxi, cs, false)) + LOGWARNING("Passthrough proxy %d:%s connection established", proxi->id, proxi->url); alive = proxi->alive; while (42) { @@ -1924,7 +1922,6 @@ static void *passthrough_recv(void *arg) proxi->id, proxi->url); alive = proxi->alive = false; Close(cs->fd); - reconnect_generator(ckp); continue; } /* Simply forward the message on, as is, to the connector to @@ -2573,7 +2570,7 @@ reconnect: cproxy = wait_best_proxy(ckp, gdata); if (!cproxy) goto out; - if (proxi != cproxy) { + if (proxi != cproxy || ckp->passthrough) { proxi = cproxy; LOGWARNING("Successfully connected to proxy %d %s as proxy%s", proxi->id, proxi->url, ckp->passthrough ? " in passthrough mode" : ""); From e53a6e028fb2705a100ba3f0590674c6c0e588ca Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Mon, 21 Sep 2015 17:05:42 +1000 Subject: [PATCH 469/544] Close the passthrough socket if it's not already on a failed write in passthrough_send --- src/generator.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/generator.c b/src/generator.c index 2695cabc..d560199e 100644 --- a/src/generator.c +++ b/src/generator.c @@ -1717,14 +1717,16 @@ static void *proxy_send(void *arg) static void passthrough_send(ckpool_t *ckp, pass_msg_t *pm) { + connsock_t *cs = pm->cs; int len, sent; LOGDEBUG("Sending upstream json msg: %s", pm->msg); len = strlen(pm->msg); - sent = write_socket(pm->cs->fd, pm->msg, len); - if (sent != len) { + sent = write_socket(cs->fd, pm->msg, len); + if (unlikely(sent != len)) { LOGWARNING("Failed to passthrough %d bytes of message %s, attempting reconnect", len, pm->msg); + Close(cs->fd); reconnect_generator(ckp); } free(pm->msg); @@ -1912,8 +1914,6 @@ static void *passthrough_recv(void *arg) } sleep(5); } - if (!alive) - reconnect_generator(ckp); /* Make sure we receive a line within 90 seconds */ ret = read_socket_line(cs, &timeout); From 47d30b88432189a8f844d8f7c6ecc420a8bace9c Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Mon, 21 Sep 2015 17:13:50 +1000 Subject: [PATCH 470/544] Check for redirecturls in redirector mode --- src/ckpool.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/ckpool.c b/src/ckpool.c index faf9fd7d..2c2f39f5 100644 --- a/src/ckpool.c +++ b/src/ckpool.c @@ -1757,6 +1757,8 @@ int main(int argc, char **argv) ckp.serverurl = ckzalloc(sizeof(char *)); if (ckp.proxy && !ckp.proxies) quit(0, "No proxy entries found in config file %s", ckp.config); + if (ckp.redirector && !ckp.redirecturls) + quit(0, "No redirect entries found in config file %s", ckp.config); /* Create the log directory */ trail_slash(&ckp.logdir); From a129d4c57906b41ef35967092ce1aff3d8b8fcc2 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Mon, 21 Sep 2015 17:36:46 +1000 Subject: [PATCH 471/544] Sort out alive message and reconnect in passthrough_recv --- src/generator.c | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/generator.c b/src/generator.c index d560199e..5304d247 100644 --- a/src/generator.c +++ b/src/generator.c @@ -1908,12 +1908,13 @@ static void *passthrough_recv(void *arg) int ret; while (!proxy_alive(ckp, proxi, cs, true)) { - if (alive) { - alive = false; - reconnect_generator(ckp); - } + alive = false; sleep(5); } + if (!alive) { + reconnect_generator(ckp); + alive = true; + } /* Make sure we receive a line within 90 seconds */ ret = read_socket_line(cs, &timeout); From a1df332ec8f33fc5b49df7d0f65b1d30b3a07627 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Fri, 20 Nov 2015 14:17:34 +1100 Subject: [PATCH 472/544] MergeFix --- src/generator.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/generator.c b/src/generator.c index 5304d247..24d5493d 100644 --- a/src/generator.c +++ b/src/generator.c @@ -1978,8 +1978,8 @@ static void *proxy_recv(void *arg) alive = proxi->alive; while (42) { - share_msg_t *share, *tmpshare; notify_instance_t *ni, *tmp; + share_msg_t *share, *tmpshare; float timeout; time_t now; int ret; From 88e20dd1110d828962480848ff3553e864f7f8db Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Sat, 12 Dec 2015 14:45:51 +1100 Subject: [PATCH 473/544] Add a userproxy mode that connects to the same upstream pool according to the username supplied by clients connecting to the proxy --- src/ckpool.c | 21 ++++++++---- src/ckpool.h | 3 ++ src/generator.c | 86 ++++++++++++++++++++++++++++++++++++++++++++---- src/stratifier.c | 23 ++++++++++++- 4 files changed, 119 insertions(+), 14 deletions(-) diff --git a/src/ckpool.c b/src/ckpool.c index 5ab30979..a76db57b 100644 --- a/src/ckpool.c +++ b/src/ckpool.c @@ -1517,6 +1517,7 @@ static struct option long_options[] = { {"redirector", no_argument, 0, 'R'}, {"ckdb-sockdir",required_argument, 0, 'S'}, {"sockdir", required_argument, 0, 's'}, + {"userproxy", no_argument, 0, 'u'}, {0, 0, 0, 0} }; #else @@ -1534,6 +1535,7 @@ static struct option long_options[] = { {"proxy", no_argument, 0, 'p'}, {"redirector", no_argument, 0, 'R'}, {"sockdir", required_argument, 0, 's'}, + {"userproxy", no_argument, 0, 'u'}, {0, 0, 0, 0} }; #endif @@ -1577,7 +1579,7 @@ int main(int argc, char **argv) ckp.initial_args[ckp.args] = strdup(argv[ckp.args]); ckp.initial_args[ckp.args] = NULL; - while ((c = getopt_long(argc, argv, "Ac:Dd:g:HhkLl:n:PpRS:s:", long_options, &i)) != -1) { + while ((c = getopt_long(argc, argv, "Ac:Dd:g:HhkLl:n:PpRS:s:u", long_options, &i)) != -1) { switch (c) { case 'A': ckp.standalone = true; @@ -1632,18 +1634,18 @@ int main(int argc, char **argv) ckp.name = optarg; break; case 'P': - if (ckp.proxy || ckp.redirector) - quit(1, "Cannot set both proxy or redirector and passthrough mode"); + if (ckp.proxy || ckp.redirector || ckp.userproxy) + quit(1, "Cannot set another proxy type or redirector and passthrough mode"); ckp.standalone = ckp.proxy = ckp.passthrough = true; break; case 'p': - if (ckp.passthrough || ckp.redirector) - quit(1, "Cannot set both passthrough or redirector and proxy mode"); + if (ckp.passthrough || ckp.redirector || ckp.userproxy) + quit(1, "Cannot set another proxy type or redirector and proxy mode"); ckp.proxy = true; break; case 'R': - if (ckp.proxy || ckp.passthrough) - quit(1, "Cannot set both proxy or passthrough and redirector modes"); + if (ckp.proxy || ckp.passthrough || ckp.userproxy) + quit(1, "Cannot set a proxy type or passthrough and redirector modes"); ckp.standalone = ckp.proxy = ckp.passthrough = ckp.redirector = true; break; case 'S': @@ -1652,6 +1654,11 @@ int main(int argc, char **argv) case 's': ckp.socket_dir = strdup(optarg); break; + case 'u': + if (ckp.proxy || ckp.redirector || ckp.passthrough) + quit(1, "Cannot set both userproxy and another proxy type or redirector"); + ckp.userproxy = ckp.proxy = true; + break; } } diff --git a/src/ckpool.h b/src/ckpool.h index 39d24c01..2593c969 100644 --- a/src/ckpool.h +++ b/src/ckpool.h @@ -189,6 +189,9 @@ struct ckpool_instance { /* Are we running without ckdb */ bool standalone; + /* Are we running in userproxy mode */ + bool userproxy; + /* Should we daemonise the ckpool process */ bool daemon; diff --git a/src/generator.c b/src/generator.c index 24d5493d..f0053df0 100644 --- a/src/generator.c +++ b/src/generator.c @@ -1012,11 +1012,11 @@ static void disable_subproxy(gdata_t *gdata, proxy_instance_t *proxi, proxy_inst epoll_ctl(proxi->epfd, EPOLL_CTL_DEL, subproxy->cs.fd, NULL); Close(subproxy->cs.fd); } + subproxy->disabled = true; if (parent_proxy(subproxy)) return; mutex_lock(&proxi->proxy_lock); - subproxy->disabled = true; /* Make sure subproxy is still in the list */ subproxy = __subproxy_by_id(proxi, subproxy->subid); if (likely(subproxy)) @@ -1167,7 +1167,7 @@ static bool parse_method(ckpool_t *ckp, proxy_instance_t *proxi, const char *msg memset(&err, 0, sizeof(err)); val = json_loads(msg, 0, &err); if (!val) { - LOGWARNING("JSON decode failed(%d): %s", err.line, err.text); + LOGWARNING("JSON decode of msg %s failed(%d): %s", msg, err.line, err.text); goto out; } @@ -1278,8 +1278,13 @@ static bool auth_stratum(ckpool_t *ckp, connsock_t *cs, proxy_instance_t *proxi) val = json_msg_result(buf, &res_val, &err_val); if (!val) { - LOGWARNING("Proxy %d:%d %s failed to get a json result in auth_stratum, got: %s", - proxi->id, proxi->subid, proxi->url, buf); + if (proxi->global) { + LOGWARNING("Proxy %d:%d %s failed to get a json result in auth_stratum, got: %s", + proxi->id, proxi->subid, proxi->url, buf); + } else { + LOGNOTICE("Proxy %d:%d %s failed to get a json result in auth_stratum, got: %s", + proxi->id, proxi->subid, proxi->url, buf); + } goto out; } @@ -1314,6 +1319,12 @@ out: parse_method(ckp, proxi, buf); }; } + if (!proxi->global) { + LOGNOTICE("Disabling userproxy %d:%d %s that failed authorisation as %s", + proxi->id, proxi->subid, proxi->url, proxi->auth); + proxi->disabled = true; + disable_subproxy(ckp->data, proxi->parent, proxi); + } return ret; } @@ -2133,7 +2144,8 @@ static void *userproxy_recv(void *arg) } mutex_unlock(&gdata->share_lock); - do { + timeout = 0; + while ((ret = read_socket_line(cs, &timeout)) > 0) { /* proxy may have been recycled here if it is not a * parent and reconnect was issued */ if (parse_method(ckp, proxy, cs->buf)) @@ -2144,7 +2156,7 @@ static void *userproxy_recv(void *arg) proxy->id, proxy->subid, cs->buf); } timeout = 0; - } while ((ret = read_socket_line(cs, &timeout)) > 0); + } } return NULL; } @@ -2282,6 +2294,24 @@ static proxy_instance_t *__add_userproxy(ckpool_t *ckp, gdata_t *gdata, const in return proxy; } +static void add_userproxy(ckpool_t *ckp, gdata_t *gdata, const int userid, + const char *url, const char *auth, const char *pass) +{ + proxy_instance_t *proxy; + char *newurl = strdup(url); + char *newauth = strdup(auth); + char *newpass = strdup(pass); + int id; + + mutex_lock(&gdata->lock); + id = ckp->proxies++; + proxy = __add_userproxy(ckp, gdata, id, userid, newurl, newauth, newpass); + mutex_unlock(&gdata->lock); + + LOGWARNING("Adding non global user %s, %d proxy %d:%s", auth, userid, id, url); + prepare_proxy(proxy); +} + static void parse_addproxy(ckpool_t *ckp, gdata_t *gdata, const int sockd, const char *buf) { char *url = NULL, *auth = NULL, *pass = NULL; @@ -2555,6 +2585,48 @@ out: send_api_response(val, sockd); } +static void parse_globaluser(ckpool_t *ckp, gdata_t *gdata, const char *buf) +{ + char *url, *username, *pass = strdupa(buf); + int userid = -1, proxyid = -1; + proxy_instance_t *proxy, *tmp; + int64_t clientid = -1; + bool found = false; + + sscanf(buf, "%d:%d:%"PRId64":%s", &proxyid, &userid, &clientid, pass); + if (unlikely(clientid < 0 || userid < 0 || proxyid < 0)) { + LOGWARNING("Failed to parse_globaluser ids from command %s", buf); + return; + } + username = strsep(&pass, ","); + if (unlikely(!username)) { + LOGWARNING("Failed to parse_globaluser username from command %s", buf); + return; + } + + LOGDEBUG("Checking userproxy proxy %d user %d:%"PRId64" worker %s pass %s", + proxyid, userid, clientid, username, pass); + + if (unlikely(proxyid >= ckp->proxies)) { + LOGWARNING("Trying to find non-existent proxy id %d in parse_globaluser", proxyid); + return; + } + + mutex_lock(&gdata->lock); + url = ckp->proxyurl[proxyid]; + HASH_ITER(hh, gdata->proxies, proxy, tmp) { + if (!strcmp(proxy->auth, username)) { + found = true; + break; + } + } + mutex_unlock(&gdata->lock); + + if (found) + return; + add_userproxy(ckp, gdata, userid, url, username, pass); +} + static int proxy_loop(proc_instance_t *pi) { proxy_instance_t *proxi = NULL, *cproxy; @@ -2617,6 +2689,8 @@ retry: parse_ableproxy(gdata, umsg->sockd, buf + 13, true); } else if (cmdmatch(buf, "proxystats")) { parse_proxystats(gdata, umsg->sockd, buf + 11); + } else if (cmdmatch(buf, "globaluser")) { + parse_globaluser(ckp, gdata, buf + 11); } else if (cmdmatch(buf, "shutdown")) { ret = 0; goto out; diff --git a/src/stratifier.c b/src/stratifier.c index f6b8de85..6979d3b8 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -282,6 +282,7 @@ struct stratum_instance { char *useragent; char *workername; + char *password; int user_id; int server; /* Which server is this instance bound to */ @@ -1078,6 +1079,7 @@ static void __kill_instance(sdata_t *sdata, stratum_instance_t *client) client->proxy->parent->combined_clients--; } free(client->workername); + free(client->password); free(client->useragent); memset(client, 0, sizeof(stratum_instance_t)); DL_APPEND(sdata->recycled_instances, client); @@ -4071,14 +4073,26 @@ static void queue_delayed_auth(stratum_instance_t *client) ckdbq_add(ckp, ID_AUTH, val); } +static void check_global_user(ckpool_t *ckp, user_instance_t *user, stratum_instance_t *client) +{ + sdata_t *sdata = ckp->data; + proxy_t *proxy = best_proxy(sdata); + int proxyid = proxy->id; + char buf[256]; + + sprintf(buf, "globaluser=%d:%d:%"PRId64":%s,%s", proxyid, user->id, client->id, + user->username, client->password); + send_generator(ckp, buf, GEN_LAX); +} + /* Needs to be entered with client holding a ref count. */ static json_t *parse_authorise(stratum_instance_t *client, const json_t *params_val, json_t **err_val, int *errnum) { user_instance_t *user; ckpool_t *ckp = client->ckp; + const char *buf, *pass; bool ret = false; - const char *buf; int arr_size; ts_t now; @@ -4112,6 +4126,7 @@ static json_t *parse_authorise(stratum_instance_t *client, const json_t *params_ *err_val = json_string("Invalid character in username"); goto out; } + pass = json_string_value(json_array_get(params_val, 1)); user = generate_user(ckp, client, buf); client->user_id = user->id; ts_realtime(&now); @@ -4119,6 +4134,10 @@ static json_t *parse_authorise(stratum_instance_t *client, const json_t *params_ /* NOTE workername is NULL prior to this so should not be used in code * till after this point */ client->workername = strdup(buf); + if (pass) + client->password = strndup(pass, 64); + else + client->password = strdup(""); if (user->failed_authtime) { time_t now_t = time(NULL); @@ -4161,6 +4180,8 @@ static json_t *parse_authorise(stratum_instance_t *client, const json_t *params_ if (ckp->proxy) { LOGNOTICE("Authorised client %"PRId64" to proxy %d:%d, worker %s as user %s", client->id, client->proxyid, client->subproxyid, buf, user->username); + if (ckp->userproxy) + check_global_user(ckp, user, client); } else { LOGNOTICE("Authorised client %"PRId64" worker %s as user %s", client->id, buf, user->username); From 1c5b8ae516bb1b452a8c2b28ec107b509f55a369 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Sat, 12 Dec 2015 23:21:54 +1100 Subject: [PATCH 474/544] Reconnect all clients of a certain user when adding a userproxy in userproxy mode --- src/generator.c | 10 ++++++++++ src/stratifier.c | 26 +++++++++++++++++++++++++- 2 files changed, 35 insertions(+), 1 deletion(-) diff --git a/src/generator.c b/src/generator.c index f0053df0..36a1e7f1 100644 --- a/src/generator.c +++ b/src/generator.c @@ -1400,6 +1400,14 @@ static void stratifier_reconnect_client(ckpool_t *ckp, const int64_t id) send_proc(ckp->stratifier, buf); } +static void stratifier_reconnect_user(ckpool_t *ckp, const int userid) +{ + char buf[256]; + + sprintf(buf, "reconnuser=%d", userid); + send_proc(ckp->stratifier, buf); +} + /* Add a share to the gdata share hashlist. Returns the share id */ static int add_share(gdata_t *gdata, const int64_t client_id, const double diff) { @@ -1809,6 +1817,8 @@ out: Close(cs->fd); } proxi->alive = ret; + if (ckp->userproxy && ret && !proxi->global) + stratifier_reconnect_user(ckp, proxi->userid); return ret; } diff --git a/src/stratifier.c b/src/stratifier.c index 6979d3b8..e49d5b60 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -2611,6 +2611,22 @@ static void reconnect_client_id(sdata_t *sdata, const int64_t client_id) dec_instance_ref(sdata, client); } +/* Reconnect all clients with a particular userid */ +static void reconnect_user_id(sdata_t *sdata, int userid) +{ + stratum_instance_t *client, *tmp; + + LOGWARNING("Dropping user id %d", userid); + + ck_rlock(&sdata->instance_lock); + HASH_ITER(hh, sdata->stratum_instances, client, tmp) { + if (client->user_id != userid) + continue; + reconnect_client(sdata, client); + } + ck_runlock(&sdata->instance_lock); +} + /* API commands */ static user_instance_t *get_user(sdata_t *sdata, const char *username); @@ -3261,9 +3277,17 @@ retry: ret = sscanf(buf, "reconnclient=%"PRId64, &client_id); if (ret < 0) - LOGDEBUG("Stratifier failed to parse reconnclient command: %s", buf); + LOGWARNING("Stratifier failed to parse reconnclient command: %s", buf); else reconnect_client_id(sdata, client_id); + } else if (cmdmatch(buf, "reconnuser")) { + int userid; + + ret = sscanf(buf, "reconnuser=%d", &userid); + if (ret < 0) + LOGWARNING("Stratifier failed to parse reconnuser command: %s", buf); + else + reconnect_user_id(sdata, userid); } else if (cmdmatch(buf, "dropall")) { drop_allclients(ckp); } else if (cmdmatch(buf, "block")) { From 0395403f9b9da62e45bb90702d18b1d7ea68dcf2 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Sat, 12 Dec 2015 23:53:23 +1100 Subject: [PATCH 475/544] Allow null message to be passed to parse_method --- src/generator.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/generator.c b/src/generator.c index 36a1e7f1..6b058719 100644 --- a/src/generator.c +++ b/src/generator.c @@ -1164,6 +1164,8 @@ static bool parse_method(ckpool_t *ckp, proxy_instance_t *proxi, const char *msg bool ret = false; const char *buf; + if (!msg) + goto out; memset(&err, 0, sizeof(err)); val = json_loads(msg, 0, &err); if (!val) { From 542c4f457bd823c82dfa328b03acd0cae23bdf51 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Sun, 13 Dec 2015 01:03:11 +1100 Subject: [PATCH 476/544] Remove epoll entry when closing socket to reconnect in connect_proxy --- src/generator.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/generator.c b/src/generator.c index 6b058719..06054258 100644 --- a/src/generator.c +++ b/src/generator.c @@ -435,8 +435,10 @@ static bool send_json_msg(connsock_t *cs, const json_t *json_msg) static bool connect_proxy(ckpool_t *ckp, connsock_t *cs, proxy_instance_t *proxy) { - if (cs->fd > 0) + if (cs->fd > 0) { + epoll_ctl(proxy->epfd, EPOLL_CTL_DEL, cs->fd, NULL); Close(cs->fd); + } cs->fd = connect_socket(cs->url, cs->port); if (cs->fd < 0) { LOGINFO("Failed to connect socket to %s:%s in connect_proxy", From c90e46a4f754018b7e625a416e32aebd1b042d73 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Sun, 13 Dec 2015 01:07:32 +1100 Subject: [PATCH 477/544] Don't read from unauthorised proxies in uproxyrecv --- src/generator.c | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/generator.c b/src/generator.c index 06054258..0c7e936d 100644 --- a/src/generator.c +++ b/src/generator.c @@ -116,6 +116,7 @@ struct proxy_instance { bool reconnecting; /* Testing of parent in progress */ int64_t recruit; /* No of recruiting requests in progress */ bool alive; + bool authorised; /* Are we in the middle of a blocked write of this message? */ cs_msg_t *sending; @@ -1812,7 +1813,7 @@ static bool proxy_alive(ckpool_t *ckp, proxy_instance_t *proxi, connsock_t *cs, } goto out; } - ret = true; + proxi->authorised = ret = true; out: if (!ret) { send_stratifier_deadproxy(ckp, proxi->id, proxi->subid); @@ -2129,6 +2130,11 @@ static void *userproxy_recv(void *arg) break; } proxy = event.data.ptr; + /* Make sure we haven't popped this off before we've finished + * subscribe/auth */ + if (unlikely(!proxy->authorised)) + continue; + cs = &proxy->cs; if (event.events & (EPOLLHUP | EPOLLERR | EPOLLRDHUP)) { LOGNOTICE("Proxy %d:%d %s hung up in epoll_wait", proxy->id, From 428182816791013fbaaaff900f55ed679b27690f Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Sun, 13 Dec 2015 01:51:58 +1100 Subject: [PATCH 478/544] Fix typo dropping userproxies and simplify reconnects to just on notify --- src/generator.c | 13 +------------ src/stratifier.c | 24 ------------------------ 2 files changed, 1 insertion(+), 36 deletions(-) diff --git a/src/generator.c b/src/generator.c index 0c7e936d..cddd0fe9 100644 --- a/src/generator.c +++ b/src/generator.c @@ -1323,8 +1323,7 @@ out: break; parse_method(ckp, proxi, buf); }; - } - if (!proxi->global) { + } else if (!proxi->global) { LOGNOTICE("Disabling userproxy %d:%d %s that failed authorisation as %s", proxi->id, proxi->subid, proxi->url, proxi->auth); proxi->disabled = true; @@ -1405,14 +1404,6 @@ static void stratifier_reconnect_client(ckpool_t *ckp, const int64_t id) send_proc(ckp->stratifier, buf); } -static void stratifier_reconnect_user(ckpool_t *ckp, const int userid) -{ - char buf[256]; - - sprintf(buf, "reconnuser=%d", userid); - send_proc(ckp->stratifier, buf); -} - /* Add a share to the gdata share hashlist. Returns the share id */ static int add_share(gdata_t *gdata, const int64_t client_id, const double diff) { @@ -1822,8 +1813,6 @@ out: Close(cs->fd); } proxi->alive = ret; - if (ckp->userproxy && ret && !proxi->global) - stratifier_reconnect_user(ckp, proxi->userid); return ret; } diff --git a/src/stratifier.c b/src/stratifier.c index e49d5b60..e75d69ea 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -2611,22 +2611,6 @@ static void reconnect_client_id(sdata_t *sdata, const int64_t client_id) dec_instance_ref(sdata, client); } -/* Reconnect all clients with a particular userid */ -static void reconnect_user_id(sdata_t *sdata, int userid) -{ - stratum_instance_t *client, *tmp; - - LOGWARNING("Dropping user id %d", userid); - - ck_rlock(&sdata->instance_lock); - HASH_ITER(hh, sdata->stratum_instances, client, tmp) { - if (client->user_id != userid) - continue; - reconnect_client(sdata, client); - } - ck_runlock(&sdata->instance_lock); -} - /* API commands */ static user_instance_t *get_user(sdata_t *sdata, const char *username); @@ -3280,14 +3264,6 @@ retry: LOGWARNING("Stratifier failed to parse reconnclient command: %s", buf); else reconnect_client_id(sdata, client_id); - } else if (cmdmatch(buf, "reconnuser")) { - int userid; - - ret = sscanf(buf, "reconnuser=%d", &userid); - if (ret < 0) - LOGWARNING("Stratifier failed to parse reconnuser command: %s", buf); - else - reconnect_user_id(sdata, userid); } else if (cmdmatch(buf, "dropall")) { drop_allclients(ckp); } else if (cmdmatch(buf, "block")) { From 35d795bb1577300c28637735f8ee66695e1e4dd5 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Sun, 13 Dec 2015 22:55:41 +1100 Subject: [PATCH 479/544] Try to match by IP address if clients don't support sessionid on reconnect --- src/stratifier.c | 44 +++++++++++++++++++++++++++++++++++++++----- 1 file changed, 39 insertions(+), 5 deletions(-) diff --git a/src/stratifier.c b/src/stratifier.c index e75d69ea..4b13d9bf 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -361,6 +361,7 @@ struct session { int64_t client_id; int userid; time_t added; + char address[INET6_ADDRSTRLEN]; }; #define ID_AUTH 0 @@ -1126,6 +1127,7 @@ static void __disconnect_session(sdata_t *sdata, const stratum_instance_t *clien session->client_id = client->id; session->userid = client->user_id; session->added = now_t; + strcpy(session->address, client->address); HASH_ADD_INT(sdata->disconnected_sessions, session_id, session); sdata->stats.disconnected++; sdata->disconnected_generated++; @@ -3468,7 +3470,7 @@ out: static int userid_from_sessionid(sdata_t *sdata, const int session_id) { session_t *session; - int ret = 0; + int ret = -1; ck_wlock(&sdata->instance_lock); HASH_FIND_INT(sdata->disconnected_sessions, &session_id, session); @@ -3481,6 +3483,33 @@ static int userid_from_sessionid(sdata_t *sdata, const int session_id) out_unlock: ck_wunlock(&sdata->instance_lock); + if (ret != -1) + LOGINFO("Found old session id %d for userid %d", session_id, ret); + return ret; +} + +static int userid_from_sessionip(sdata_t *sdata, const char *address) +{ + session_t *session, *tmp; + int ret = -1; + + ck_wlock(&sdata->instance_lock); + HASH_ITER(hh, sdata->disconnected_sessions, session, tmp) { + if (!strcmp(session->address, address)) { + ret = session->userid; + break; + } + } + if (ret == -1) + goto out_unlock; + HASH_DEL(sdata->disconnected_sessions, session); + sdata->stats.disconnected--; + dealloc(session); +out_unlock: + ck_wunlock(&sdata->instance_lock); + + if (ret != -1) + LOGINFO("Found old session address %s for userid %d", address, ret); return ret; } @@ -3542,12 +3571,17 @@ static json_t *parse_subscribe(stratum_instance_t *client, const int64_t client_ __fill_enonce1data(sdata->current_workbase, client); ck_runlock(&ckp_sdata->workbase_lock); } - } else if (ckp->proxy && session_id) { + } else if (ckp->proxy) { int userid; - /* Use the session_id to tell us which user this was */ - userid = userid_from_sessionid(ckp_sdata, session_id); - if (userid) { + /* Use the session_id to tell us which user this was. + * If it's not available, see if there's an IP address + * which matches a recently disconnected session. */ + if (session_id) + userid = userid_from_sessionid(ckp_sdata, session_id); + else + userid = userid_from_sessionip(ckp_sdata, client->address); + if (userid != -1) { sdata_t *user_sdata = select_sdata(ckp, ckp_sdata, userid); if (user_sdata) From f54891fae839945dc1f4cc1b5b5c93a815ac4c24 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Sun, 13 Dec 2015 23:06:12 +1100 Subject: [PATCH 480/544] Try both userid and userip to detect disconnected clients --- src/stratifier.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/stratifier.c b/src/stratifier.c index 4b13d9bf..dcc0b412 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -3579,7 +3579,7 @@ static json_t *parse_subscribe(stratum_instance_t *client, const int64_t client_ * which matches a recently disconnected session. */ if (session_id) userid = userid_from_sessionid(ckp_sdata, session_id); - else + if (userid == -1) userid = userid_from_sessionip(ckp_sdata, client->address); if (userid != -1) { sdata_t *user_sdata = select_sdata(ckp, ckp_sdata, userid); From 914be773e620ef86cef8d5e56f8597045ba24f7f Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Sun, 13 Dec 2015 23:23:31 +1100 Subject: [PATCH 481/544] Detect old session clients with an empty array in proxy mode as well --- src/stratifier.c | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/src/stratifier.c b/src/stratifier.c index dcc0b412..d98e37fd 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -3525,6 +3525,7 @@ static json_t *parse_subscribe(stratum_instance_t *client, const int64_t client_ { ckpool_t *ckp = client->ckp; sdata_t *sdata, *ckp_sdata = ckp->data; + int session_id = 0, userid = -1; bool old_match = false; char sessionid[12]; int arr_size; @@ -3547,7 +3548,6 @@ static json_t *parse_subscribe(stratum_instance_t *client, const int64_t client_ /* NOTE useragent is NULL prior to this so should not be used in code * till after this point */ if (arr_size > 0) { - int session_id = 0; const char *buf; buf = json_string_value(json_array_get(params_val, 0)); @@ -3571,26 +3571,26 @@ static json_t *parse_subscribe(stratum_instance_t *client, const int64_t client_ __fill_enonce1data(sdata->current_workbase, client); ck_runlock(&ckp_sdata->workbase_lock); } - } else if (ckp->proxy) { - int userid; - - /* Use the session_id to tell us which user this was. - * If it's not available, see if there's an IP address - * which matches a recently disconnected session. */ - if (session_id) - userid = userid_from_sessionid(ckp_sdata, session_id); - if (userid == -1) - userid = userid_from_sessionip(ckp_sdata, client->address); - if (userid != -1) { - sdata_t *user_sdata = select_sdata(ckp, ckp_sdata, userid); - - if (user_sdata) - sdata = user_sdata; - } } } else client->useragent = ckzalloc(1); + if (ckp->proxy) { + /* Use the session_id to tell us which user this was. + * If it's not available, see if there's an IP address + * which matches a recently disconnected session. */ + if (session_id) + userid = userid_from_sessionid(ckp_sdata, session_id); + if (userid == -1) + userid = userid_from_sessionip(ckp_sdata, client->address); + if (userid != -1) { + sdata_t *user_sdata = select_sdata(ckp, ckp_sdata, userid); + + if (user_sdata) + sdata = user_sdata; + } + } + client->sdata = sdata; if (ckp->proxy) { LOGINFO("Current %d, selecting proxy %d:%d for client %"PRId64, ckp_sdata->proxy->id, From 67f6e0f55880837e92ecb095f84dd221628988bb Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Mon, 14 Dec 2015 09:45:06 +1100 Subject: [PATCH 482/544] Document userproxy mode --- README | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/README b/README index 630b6e29..5e48dac7 100644 --- a/README +++ b/README @@ -137,6 +137,7 @@ ckpool supports the following options: -R | --redirector -S CKDB-SOCKDIR | --ckdb-sockdir CKDB-SOCKDIR -s SOCKDIR | --sockdir SOCKDIR +-u | --userproxy -A Standalone mode tells ckpool not to try to communicate with ckdb or log any @@ -199,6 +200,12 @@ This option does not exist when built without ckdb support. -s tells ckpool which directory to place its own communication sockets (/tmp by default) +-u Userproxy mode will start ckpool in proxy mode as per the -p option above, +but in addition it will accept username/passwords from the stratum connects +and try to open additional connections with those credentials to the upstream +pool specified in the configuration file and then reconnect miners to mine with +their chosen username/password to the upstream pool. + ckdb takes the following options: From f6e9f49e1adb63f2518d7220856bdb7dae0a8d13 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Mon, 14 Dec 2015 09:52:23 +1100 Subject: [PATCH 483/544] Showing each notify is too verbose for regular logging --- src/stratifier.c | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/stratifier.c b/src/stratifier.c index d98e37fd..0a9e19b2 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -1737,10 +1737,7 @@ static void update_notify(ckpool_t *ckp, const char *cmd) LOGINFO("No valid proxy %d:%d subscription to update notify yet", id, subid); goto out; } - if (!subid) - LOGNOTICE("Got updated notify for proxy %d", id); - else - LOGINFO("Got updated notify for proxy %d:%d", id, subid); + LOGINFO("Got updated notify for proxy %d:%d", id, subid); wb = ckzalloc(sizeof(workbase_t)); wb->ckp = ckp; From 06bdd7fa6a79dd27589c096e9232ea2f94162ac1 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Tue, 15 Dec 2015 15:13:50 +1100 Subject: [PATCH 484/544] Silence lack of current workbase warning in proxy mode since it happens till there is a proxy workbase to work with --- src/stratifier.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/stratifier.c b/src/stratifier.c index 163f39f9..958511eb 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -4877,10 +4877,14 @@ static void stratum_broadcast_update(sdata_t *sdata, const workbase_t *wb, const /* For sending a single stratum template update */ static void stratum_send_update(sdata_t *sdata, const int64_t client_id, const bool clean) { + ckpool_t *ckp = sdata->ckp; json_t *json_msg; if (unlikely(!sdata->current_workbase)) { - LOGWARNING("No current workbase to send stratum update"); + if (!ckp->proxy) + LOGWARNING("No current workbase to send stratum update"); + else + LOGDEBUG("No current workbase to send stratum update for client %"PRId64, client_id); return; } From 6f842fc2d8fec25f5f5c1c6a05451b943414eb15 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Fri, 18 Dec 2015 17:52:17 +1100 Subject: [PATCH 485/544] Cope with proxy auths that have no password --- src/generator.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/generator.c b/src/generator.c index 20b0e81b..0d89c5cb 100644 --- a/src/generator.c +++ b/src/generator.c @@ -2800,7 +2800,10 @@ static proxy_instance_t *__add_proxy(ckpool_t *ckp, gdata_t *gdata, const int id proxy->id = id; proxy->url = strdup(ckp->proxyurl[id]); proxy->auth = strdup(ckp->proxyauth[id]); - proxy->pass = strdup(ckp->proxypass[id]); + if (proxy->pass) + proxy->pass = strdup(ckp->proxypass[id]); + else + proxy->pass = strdup(""); proxy->ckp = proxy->cs.ckp = ckp; HASH_ADD_INT(gdata->proxies, id, proxy); proxy->global = true; From 012d94dd35b6e16a2c77c812a0c4d8f18975392e Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Fri, 18 Dec 2015 18:12:10 +1100 Subject: [PATCH 486/544] Add reconnect sends to alive bool changing on proxy conditions to try and fail over/back --- src/generator.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/generator.c b/src/generator.c index 0d89c5cb..4dc48d36 100644 --- a/src/generator.c +++ b/src/generator.c @@ -1932,12 +1932,14 @@ static void *passthrough_recv(void *arg) } if (!alive) { reconnect_generator(ckp); + LOGWARNING("Passthrough %d:%s recovered", proxi->id, proxi->url); alive = true; } /* Make sure we receive a line within 90 seconds */ ret = read_socket_line(cs, &timeout); if (ret < 1) { + reconnect_generator(ckp); LOGWARNING("Proxy %d:%s failed to read_socket_line in passthrough_recv, attempting reconnect", proxi->id, proxi->url); alive = proxi->alive = false; @@ -2009,6 +2011,7 @@ static void *proxy_recv(void *arg) while (!subproxies_alive(proxi)) { reconnect_proxy(proxi); if (alive) { + reconnect_generator(ckp); LOGWARNING("Proxy %d:%s failed, attempting reconnect", proxi->id, proxi->url); alive = false; @@ -2017,6 +2020,7 @@ static void *proxy_recv(void *arg) } } if (!alive) { + reconnect_generator(ckp); LOGWARNING("Proxy %d:%s recovered", proxi->id, proxi->url); alive = true; } From 8ef853b8fcb787bd6acfe4d6871aec3386b5b66a Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Fri, 18 Dec 2015 18:26:34 +1100 Subject: [PATCH 487/544] Use the proxy connsock semaphore to serialise uses of the cs->buf to prevent races --- src/generator.c | 31 +++++++++++++++++++++++++++---- 1 file changed, 27 insertions(+), 4 deletions(-) diff --git a/src/generator.c b/src/generator.c index 4dc48d36..e40b920d 100644 --- a/src/generator.c +++ b/src/generator.c @@ -771,6 +771,9 @@ static bool passthrough_stratum(connsock_t *cs, proxy_instance_t *proxi) JSON_CPACK(req, "{s:s,s:[s]}", "method", "mining.passthrough", "params", PACKAGE"/"VERSION); + + /* Serialise all send/recvs */ + cksem_wait(&cs->sem); ret = send_json_msg(cs, req); json_decref(req); if (!ret) { @@ -796,6 +799,8 @@ static bool passthrough_stratum(connsock_t *cs, proxy_instance_t *proxi) } proxi->passthrough = true; out: + cksem_post(&cs->sem); + if (val) json_decref(val); if (!ret) @@ -1771,6 +1776,9 @@ static bool proxy_alive(ckpool_t *ckp, proxy_instance_t *proxi, connsock_t *cs, return true; if (proxi->disabled) return false; + + /* Serialise all send/recvs here with the cs semaphore */ + cksem_wait(&cs->sem); if (!extract_sockaddr(proxi->url, &cs->url, &cs->port)) { LOGWARNING("Failed to extract address from %s", proxi->url); goto out; @@ -1810,6 +1818,8 @@ static bool proxy_alive(ckpool_t *ckp, proxy_instance_t *proxi, connsock_t *cs, } proxi->authorised = ret = true; out: + cksem_post(&cs->sem); + if (!ret) { send_stratifier_deadproxy(ckp, proxi->id, proxi->subid); /* Close and invalidate the file handle */ @@ -1937,7 +1947,10 @@ static void *passthrough_recv(void *arg) } /* Make sure we receive a line within 90 seconds */ + cksem_wait(&cs->sem); ret = read_socket_line(cs, &timeout); + cksem_post(&cs->sem); + if (ret < 1) { reconnect_generator(ckp); LOGWARNING("Proxy %d:%s failed to read_socket_line in passthrough_recv, attempting reconnect", @@ -2048,12 +2061,17 @@ static void *proxy_recv(void *arg) } mutex_unlock(&gdata->share_lock); + cs = NULL; /* If we don't get an update within 10 minutes the upstream pool * has likely stopped responding. */ ret = epoll_wait(epfd, &event, 1, 600000); if (likely(ret > 0)) { subproxy = event.data.ptr; cs = &subproxy->cs; + + /* Serialise messages from here once we have a cs by + * holding the semaphore. */ + cksem_wait(&cs->sem); if (event.events & (EPOLLHUP | EPOLLERR | EPOLLRDHUP)) ret = -1; else { @@ -2065,9 +2083,7 @@ static void *proxy_recv(void *arg) LOGNOTICE("Proxy %d:%d %s failed to epoll/read_socket_line in proxy_recv", proxi->id, subproxy->subid, subproxy->url); disable_subproxy(gdata, proxi, subproxy); - continue; - } - do { + } else do { /* subproxy may have been recycled here if it is not a * parent and reconnect was issued */ if (parse_method(ckp, subproxy, cs->buf)) @@ -2079,6 +2095,8 @@ static void *proxy_recv(void *arg) } timeout = 0; } while ((ret = read_socket_line(cs, &timeout)) > 0); + if (cs) + cksem_post(&cs->sem); } return NULL; @@ -2132,7 +2150,6 @@ static void *userproxy_recv(void *arg) if (unlikely(!proxy->authorised)) continue; - cs = &proxy->cs; if (event.events & (EPOLLHUP | EPOLLERR | EPOLLRDHUP)) { LOGNOTICE("Proxy %d:%d %s hung up in epoll_wait", proxy->id, proxy->subid, proxy->url); @@ -2162,6 +2179,9 @@ static void *userproxy_recv(void *arg) mutex_unlock(&gdata->share_lock); timeout = 0; + cs = &proxy->cs; + + cksem_wait(&cs->sem); while ((ret = read_socket_line(cs, &timeout)) > 0) { /* proxy may have been recycled here if it is not a * parent and reconnect was issued */ @@ -2174,6 +2194,7 @@ static void *userproxy_recv(void *arg) } timeout = 0; } + cksem_post(&cs->sem); } return NULL; } @@ -2811,6 +2832,8 @@ static proxy_instance_t *__add_proxy(ckpool_t *ckp, gdata_t *gdata, const int id proxy->ckp = proxy->cs.ckp = ckp; HASH_ADD_INT(gdata->proxies, id, proxy); proxy->global = true; + cksem_init(&proxy->cs.sem); + cksem_post(&proxy->cs.sem); return proxy; } From 0574a545543f776eed99fcc15131ef1f9ee5eb3d Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Fri, 18 Dec 2015 19:10:39 +1100 Subject: [PATCH 488/544] Don't disable parent proxies in disable_subproxy for failover to work --- src/generator.c | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/generator.c b/src/generator.c index e40b920d..666484c8 100644 --- a/src/generator.c +++ b/src/generator.c @@ -1015,7 +1015,8 @@ static void send_stratifier_deadproxy(ckpool_t *ckp, const int id, const int sub /* Remove the subproxy from the proxi list and put it on the dead list. * Further use of the subproxy pointer may point to a new proxy but will not - * dereference */ + * dereference. This will only disable subproxies so parent proxies need to + * have their disabled bool set manually. */ static void disable_subproxy(gdata_t *gdata, proxy_instance_t *proxi, proxy_instance_t *subproxy) { subproxy->alive = false; @@ -1024,10 +1025,11 @@ static void disable_subproxy(gdata_t *gdata, proxy_instance_t *proxi, proxy_inst epoll_ctl(proxi->epfd, EPOLL_CTL_DEL, subproxy->cs.fd, NULL); Close(subproxy->cs.fd); } - subproxy->disabled = true; if (parent_proxy(subproxy)) return; + subproxy->disabled = true; + mutex_lock(&proxi->proxy_lock); /* Make sure subproxy is still in the list */ subproxy = __subproxy_by_id(proxi, subproxy->subid); @@ -2488,9 +2490,11 @@ static void parse_ableproxy(gdata_t *gdata, const int sockd, const char *buf, bo proxy->disabled = disable; LOGNOTICE("%sabling proxy %d:%s", disable ? "Dis" : "En", id, proxy->url); } - if (disable) + if (disable) { + /* Set disabled bool here in case this is a parent proxy */ + proxy->disabled = true; disable_subproxy(gdata, proxy, proxy); - else + } else reconnect_proxy(proxy); out: send_api_response(val, sockd); From 1e2e5bab7d0008b83dca2f1f737df5e32a93962e Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Fri, 18 Dec 2015 21:44:55 +1100 Subject: [PATCH 489/544] Add semaphores for subproxies --- src/generator.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/generator.c b/src/generator.c index 666484c8..54e030e1 100644 --- a/src/generator.c +++ b/src/generator.c @@ -973,6 +973,8 @@ static proxy_instance_t *create_subproxy(ckpool_t *ckp, gdata_t *gdata, proxy_in subproxy->pass = strdup(proxi->pass); subproxy->parent = proxi; subproxy->epfd = proxi->epfd; + cksem_init(&subproxy->cs.sem); + cksem_post(&subproxy->cs.sem); return subproxy; } From b6ceb9b48470e07228d70afd3c83e89c964ae0cc Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Sat, 19 Dec 2015 11:48:22 +1100 Subject: [PATCH 490/544] Fix userproxies not having their semaphores initialised and avoid trying to process epoll responses from proxies not marked alive --- src/generator.c | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/src/generator.c b/src/generator.c index 54e030e1..4fb72e30 100644 --- a/src/generator.c +++ b/src/generator.c @@ -1776,13 +1776,18 @@ static bool proxy_alive(ckpool_t *ckp, proxy_instance_t *proxi, connsock_t *cs, bool ret = false; /* Has this proxy already been reconnected? */ - if (cs->fd > 0) + if (proxi->alive) return true; if (proxi->disabled) return false; /* Serialise all send/recvs here with the cs semaphore */ cksem_wait(&cs->sem); + /* Check again after grabbing semaphore */ + if (unlikely(proxi->alive)) { + ret = true; + goto out; + } if (!extract_sockaddr(proxi->url, &cs->url, &cs->port)) { LOGWARNING("Failed to extract address from %s", proxi->url); goto out; @@ -1822,8 +1827,6 @@ static bool proxy_alive(ckpool_t *ckp, proxy_instance_t *proxi, connsock_t *cs, } proxi->authorised = ret = true; out: - cksem_post(&cs->sem); - if (!ret) { send_stratifier_deadproxy(ckp, proxi->id, proxi->subid); /* Close and invalidate the file handle */ @@ -1831,6 +1834,8 @@ out: Close(cs->fd); } proxi->alive = ret; + cksem_post(&cs->sem); + return ret; } @@ -2072,6 +2077,8 @@ static void *proxy_recv(void *arg) if (likely(ret > 0)) { subproxy = event.data.ptr; cs = &subproxy->cs; + if (!subproxy->alive) + continue; /* Serialise messages from here once we have a cs by * holding the semaphore. */ @@ -2185,6 +2192,9 @@ static void *userproxy_recv(void *arg) timeout = 0; cs = &proxy->cs; + if (!proxy->alive) + continue; + cksem_wait(&cs->sem); while ((ret = read_socket_line(cs, &timeout)) > 0) { /* proxy may have been recycled here if it is not a @@ -2332,6 +2342,8 @@ static proxy_instance_t *__add_userproxy(ckpool_t *ckp, gdata_t *gdata, const in proxy->auth = auth; proxy->pass = pass; proxy->ckp = proxy->cs.ckp = ckp; + cksem_init(&proxy->cs.sem); + cksem_post(&proxy->cs.sem); HASH_ADD_INT(gdata->proxies, id, proxy); return proxy; } From 45a44ace7e815e43f9adf6a1df93a2966f666cae Mon Sep 17 00:00:00 2001 From: ckolivas Date: Sun, 27 Dec 2015 08:29:44 +1100 Subject: [PATCH 491/544] Fix passthrough/redirector trying to grab semaphore twice --- src/generator.c | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/generator.c b/src/generator.c index 4fb72e30..75378090 100644 --- a/src/generator.c +++ b/src/generator.c @@ -708,6 +708,7 @@ out: return ret; } +/* cs semaphore must be held */ static bool subscribe_stratum(ckpool_t *ckp, connsock_t *cs, proxy_instance_t *proxi) { bool ret = false; @@ -762,6 +763,7 @@ out: return ret; } +/* cs semaphore must be held */ static bool passthrough_stratum(connsock_t *cs, proxy_instance_t *proxi) { json_t *req, *val = NULL, *res_val, *err_val; @@ -772,8 +774,6 @@ static bool passthrough_stratum(connsock_t *cs, proxy_instance_t *proxi) "method", "mining.passthrough", "params", PACKAGE"/"VERSION); - /* Serialise all send/recvs */ - cksem_wait(&cs->sem); ret = send_json_msg(cs, req); json_decref(req); if (!ret) { @@ -799,8 +799,6 @@ static bool passthrough_stratum(connsock_t *cs, proxy_instance_t *proxi) } proxi->passthrough = true; out: - cksem_post(&cs->sem); - if (val) json_decref(val); if (!ret) @@ -1258,6 +1256,7 @@ out: return ret; } +/* cs semaphore must be held */ static bool auth_stratum(ckpool_t *ckp, connsock_t *cs, proxy_instance_t *proxi) { json_t *val = NULL, *res_val, *req, *err_val; From c437d3283bb7ebb57b31655d6c7f7a936b8aa58a Mon Sep 17 00:00:00 2001 From: ckolivas Date: Sun, 27 Dec 2015 08:38:52 +1100 Subject: [PATCH 492/544] Only try to redirect clients once in redirector mode, acting as a regular passthrough thereafter --- src/connector.c | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/connector.c b/src/connector.c index 916de95d..87803eb0 100644 --- a/src/connector.c +++ b/src/connector.c @@ -66,6 +66,9 @@ struct client_instance { /* Linked list of shares in redirector mode.*/ share_t *shares; + /* Has this client already been told to redirect */ + bool redirected; + /* Time this client started blocking, 0 when not blocked */ time_t blocked_time; }; @@ -771,6 +774,9 @@ static void redirect_client(ckpool_t *ckp, client_instance_t *client) char *buf; int num; + /* Set the redirected boool to only try redirecting them once */ + client->redirected = true; + num = add_redirect(ckp, cdata, client); JSON_CPACK(val, "{sosss[ssi]}", "id", json_null(), "method", "client.reconnect", "params", ckp->redirecturl[num], ckp->redirectport[num], 0); @@ -889,7 +895,7 @@ static void send_client(cdata_t *cdata, const int64_t id, char *buf) free(buf); return; } - if (ckp->redirector) + if (ckp->redirector && !client->redirected) test_redirector_shares(ckp, client, buf); } From 2eaeb2a96c5997cc7bc0d468d1ba9c2ab1dcbba1 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Sun, 27 Dec 2015 14:36:56 +1100 Subject: [PATCH 493/544] Don't keep storing shares in redirector mode after we've attempted to redirect the client --- src/connector.c | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/connector.c b/src/connector.c index 87803eb0..d932855d 100644 --- a/src/connector.c +++ b/src/connector.c @@ -391,9 +391,7 @@ static int invalidate_client(ckpool_t *ckp, cdata_t *cdata, client_instance_t *c static void send_client(cdata_t *cdata, int64_t id, char *buf); /* Look for shares being submitted via a redirector and add them to a linked - * list for looking up the responses. Theoretically this could leak shares but - * we should only ever store one share before redirecting active clients unless - * they don't support redirection. */ + * list for looking up the responses. */ static void parse_redirector_share(client_instance_t *client, const char *msg, const json_t *val) { share_t *share, *tmp; @@ -494,7 +492,7 @@ reparse: passthrough_id = (client->id << 32) | passthrough_id; json_object_set_new_nocheck(val, "client_id", json_integer(passthrough_id)); } else { - if (ckp->redirector && strstr(msg, "mining.submit")) + if (ckp->redirector && !client->redirected && strstr(msg, "mining.submit")) parse_redirector_share(client, msg, val); json_object_set_new_nocheck(val, "client_id", json_integer(client->id)); json_object_set_new_nocheck(val, "address", json_string(client->address_name)); @@ -840,6 +838,12 @@ static void test_redirector_shares(ckpool_t *ckp, client_instance_t *client, con LOGNOTICE("Found accepted share for client %"PRId64" - redirecting", client->id); redirect_client(ckp, client); + + /* Clear the list now since we don't need it any more */ + DL_FOREACH_SAFE(client->shares, share, found) { + DL_DELETE(client->shares, share); + dealloc(share); + } } out: json_decref(val); From 85112ebce985f5ccae03333d412a80cb1f0cbbc1 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Sun, 27 Dec 2015 23:21:57 +1100 Subject: [PATCH 494/544] Flag a proxy as not alive when we are unable to send to it in passthrough mode --- src/generator.c | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/generator.c b/src/generator.c index 75378090..a270229c 100644 --- a/src/generator.c +++ b/src/generator.c @@ -69,6 +69,7 @@ struct stratum_msg { typedef struct stratum_msg stratum_msg_t; struct pass_msg { + proxy_instance_t *proxy; connsock_t *cs; char *msg; }; @@ -1754,19 +1755,21 @@ static void passthrough_send(ckpool_t *ckp, pass_msg_t *pm) LOGWARNING("Failed to passthrough %d bytes of message %s, attempting reconnect", len, pm->msg); Close(cs->fd); + pm->proxy->alive = false; reconnect_generator(ckp); } free(pm->msg); free(pm); } -static void passthrough_add_send(proxy_instance_t *proxi, const char *msg) +static void passthrough_add_send(proxy_instance_t *proxy, const char *msg) { pass_msg_t *pm = ckzalloc(sizeof(pass_msg_t)); - pm->cs = &proxi->cs; + pm->proxy = proxy; + pm->cs = &proxy->cs; ASPRINTF(&pm->msg, "%s\n", msg); - ckmsgq_add(proxi->passsends, pm); + ckmsgq_add(proxy->passsends, pm); } static bool proxy_alive(ckpool_t *ckp, proxy_instance_t *proxi, connsock_t *cs, From 6a39cdb299dc007e516afcb2aa7cc8f7aca22f7c Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Tue, 29 Dec 2015 10:34:50 +1100 Subject: [PATCH 495/544] Add configuration option for node mode --- src/ckpool.c | 21 +++++++++++++++------ src/ckpool.h | 3 +++ 2 files changed, 18 insertions(+), 6 deletions(-) diff --git a/src/ckpool.c b/src/ckpool.c index ba81394a..9d63db74 100644 --- a/src/ckpool.c +++ b/src/ckpool.c @@ -1518,6 +1518,7 @@ static struct option long_options[] = { {"log-shares", no_argument, 0, 'L'}, {"loglevel", required_argument, 0, 'l'}, {"name", required_argument, 0, 'n'}, + {"node", no_argument, 0, 'N'}, {"passthrough", no_argument, 0, 'P'}, {"proxy", no_argument, 0, 'p'}, {"redirector", no_argument, 0, 'R'}, @@ -1537,6 +1538,7 @@ static struct option long_options[] = { {"log-shares", no_argument, 0, 'L'}, {"loglevel", required_argument, 0, 'l'}, {"name", required_argument, 0, 'n'}, + {"node", no_argument, 0, 'N'}, {"passthrough", no_argument, 0, 'P'}, {"proxy", no_argument, 0, 'p'}, {"redirector", no_argument, 0, 'R'}, @@ -1585,7 +1587,7 @@ int main(int argc, char **argv) ckp.initial_args[ckp.args] = strdup(argv[ckp.args]); ckp.initial_args[ckp.args] = NULL; - while ((c = getopt_long(argc, argv, "Ac:Dd:g:HhkLl:n:PpRS:s:u", long_options, &i)) != -1) { + while ((c = getopt_long(argc, argv, "Ac:Dd:g:HhkLl:Nn:PpRS:s:u", long_options, &i)) != -1) { switch (c) { case 'A': ckp.standalone = true; @@ -1636,21 +1638,26 @@ int main(int argc, char **argv) LOG_EMERG, LOG_DEBUG, ckp.loglevel); } break; + case 'N': + if (ckp.proxy || ckp.redirector || ckp.userproxy || ckp.passthrough) + quit(1, "Cannot set another proxy type or redirector and node mode"); + ckp.standalone = ckp.proxy = ckp.node = true; + break; case 'n': ckp.name = optarg; break; case 'P': - if (ckp.proxy || ckp.redirector || ckp.userproxy) + if (ckp.proxy || ckp.redirector || ckp.userproxy || ckp.node) quit(1, "Cannot set another proxy type or redirector and passthrough mode"); ckp.standalone = ckp.proxy = ckp.passthrough = true; break; case 'p': - if (ckp.passthrough || ckp.redirector || ckp.userproxy) + if (ckp.passthrough || ckp.redirector || ckp.userproxy || ckp.node) quit(1, "Cannot set another proxy type or redirector and proxy mode"); ckp.proxy = true; break; case 'R': - if (ckp.proxy || ckp.passthrough || ckp.userproxy) + if (ckp.proxy || ckp.passthrough || ckp.userproxy || ckp.node) quit(1, "Cannot set a proxy type or passthrough and redirector modes"); ckp.standalone = ckp.proxy = ckp.passthrough = ckp.redirector = true; break; @@ -1661,7 +1668,7 @@ int main(int argc, char **argv) ckp.socket_dir = strdup(optarg); break; case 'u': - if (ckp.proxy || ckp.redirector || ckp.passthrough) + if (ckp.proxy || ckp.redirector || ckp.passthrough || ckp.node) quit(1, "Cannot set both userproxy and another proxy type or redirector"); ckp.userproxy = ckp.proxy = true; break; @@ -1669,7 +1676,9 @@ int main(int argc, char **argv) } if (!ckp.name) { - if (ckp.redirector) + if (ckp.node) + ckp.name = "cknode"; + else if (ckp.redirector) ckp.name = "ckredirector"; else if (ckp.passthrough) ckp.name = "ckpassthrough"; diff --git a/src/ckpool.h b/src/ckpool.h index f0e48c26..cde4f0aa 100644 --- a/src/ckpool.h +++ b/src/ckpool.h @@ -180,6 +180,9 @@ struct ckpool_instance { pthread_t pth_listener; pthread_t pth_watchdog; + /* Are we running in node proxy mode */ + bool node; + /* Are we running in passthrough mode */ bool passthrough; From 0c605e3dee289164640a23b7622e20ed2c5093cc Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Tue, 29 Dec 2015 10:50:58 +1100 Subject: [PATCH 496/544] Add basic node subscription functionality --- src/generator.c | 64 +++++++++++++++++++++++++++++++++++++++++++++--- src/stratifier.c | 12 +++++++++ 2 files changed, 73 insertions(+), 3 deletions(-) diff --git a/src/generator.c b/src/generator.c index a270229c..ac5656c6 100644 --- a/src/generator.c +++ b/src/generator.c @@ -87,6 +87,7 @@ struct proxy_instance { ckpool_t *ckp; connsock_t cs; bool passthrough; + bool node; int id; /* Proxy server id*/ int subid; /* Subproxy id */ int userid; /* User id if this proxy is bound to a user */ @@ -807,6 +808,49 @@ out: return ret; } +/* cs semaphore must be held */ +static bool node_stratum(connsock_t *cs, proxy_instance_t *proxi) +{ + json_t *req, *val = NULL, *res_val, *err_val; + bool res, ret = false; + float timeout = 10; + + JSON_CPACK(req, "{s:s,s:[s]}", + "method", "mining.node", + "params", PACKAGE"/"VERSION); + + res = send_json_msg(cs, req); + json_decref(req); + if (!res) { + LOGWARNING("Failed to send message in node_stratum"); + goto out; + } + if (read_socket_line(cs, &timeout) < 1) { + LOGWARNING("Failed to receive line in node_stratum"); + goto out; + } + /* Ignore err_val here since we should always get a result from an + * upstream server */ + val = json_msg_result(cs->buf, &res_val, &err_val); + if (!val || !res_val) { + LOGWARNING("Failed to get a json result in node_stratum, got: %s", + cs->buf); + goto out; + } + ret = json_is_true(res_val); + if (!ret) { + LOGWARNING("Denied node setup for stratum"); + goto out; + } + proxi->node = true; +out: + if (val) + json_decref(val); + if (!ret) + Close(cs->fd); + return ret; +} + static void send_notify(ckpool_t *ckp, proxy_instance_t *proxi, notify_instance_t *ni); static void reconnect_generator(const ckpool_t *ckp) @@ -1801,6 +1845,15 @@ static bool proxy_alive(ckpool_t *ckp, proxy_instance_t *proxi, connsock_t *cs, } goto out; } + if (ckp->node) { + if (!node_stratum(cs, proxi)) { + LOGWARNING("Failed initial node setup to %s:%s !", + cs->url, cs->port); + goto out; + } + ret = true; + goto out; + } if (ckp->passthrough) { if (!passthrough_stratum(cs, proxi)) { LOGWARNING("Failed initial passthrough to %s:%s !", @@ -2701,10 +2754,15 @@ reconnect: cproxy = wait_best_proxy(ckp, gdata); if (!cproxy) goto out; - if (proxi != cproxy || ckp->passthrough) { + if (proxi != cproxy) { proxi = cproxy; - LOGWARNING("Successfully connected to proxy %d %s as proxy%s", - proxi->id, proxi->url, ckp->passthrough ? " in passthrough mode" : ""); + if (ckp->node) { + LOGWARNING("Successfully connected to pool %d %s as proxy node", + proxi->id, proxi->url); + } else { + LOGWARNING("Successfully connected to pool %d %s as proxy%s", + proxi->id, proxi->url, ckp->passthrough ? " in passthrough mode" : ""); + } } retry: clear_unix_msg(&umsg); diff --git a/src/stratifier.c b/src/stratifier.c index 6fdf72c2..172f5d57 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -5052,6 +5052,18 @@ static void parse_method(ckpool_t *ckp, sdata_t *sdata, stratum_instance_t *clie return; } + if (unlikely(cmdmatch(method, "mining.node"))) { + char buf[256]; + + /* Add this client as a passthrough in the connector and + * add it to the list of mining nodes in the stratifier */ + LOGNOTICE("Adding mining client %"PRId64" %s", client_id, client->address); + snprintf(buf, 255, "passthrough=%"PRId64, client_id); + send_proc(ckp->connector, buf); + drop_client(ckp, sdata, client_id); + return; + } + if (unlikely(cmdmatch(method, "mining.passthrough"))) { char buf[256]; From eab56b032a83e1dfb797fde96af6264fbb2dcca3 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Tue, 29 Dec 2015 11:10:38 +1100 Subject: [PATCH 497/544] Set up local bitcoind connections in node mode --- src/ckpool.c | 2 +- src/generator.c | 53 +++++++++++++++++++++++++++++++++++++------------ 2 files changed, 41 insertions(+), 14 deletions(-) diff --git a/src/ckpool.c b/src/ckpool.c index 9d63db74..384b8a00 100644 --- a/src/ckpool.c +++ b/src/ckpool.c @@ -1736,7 +1736,7 @@ int main(int argc, char **argv) parse_config(&ckp); /* Set defaults if not found in config file */ - if (!ckp.btcds && !ckp.proxy) { + if (!ckp.btcds) { ckp.btcds = 1; ckp.btcdurl = ckzalloc(sizeof(char *)); ckp.btcdauth = ckzalloc(sizeof(char *)); diff --git a/src/generator.c b/src/generator.c index 16225b2e..36cd2647 100644 --- a/src/generator.c +++ b/src/generator.c @@ -2740,14 +2740,31 @@ static void parse_globaluser(ckpool_t *ckp, gdata_t *gdata, const char *buf) static int proxy_loop(proc_instance_t *pi) { proxy_instance_t *proxi = NULL, *cproxy; + server_instance_t *si = NULL, *old_si; ckpool_t *ckp = pi->ckp; gdata_t *gdata = ckp->data; unix_msg_t *umsg = NULL; + bool started = false; char *buf = NULL; int ret = 0; reconnect: clear_unix_msg(&umsg); + + if (ckp->node) { + connsock_t *cs; + + old_si = si; + si = live_server(ckp); + if (!si) + goto out; + cs = &si->cs; + if (!old_si) + LOGWARNING("Connected to bitcoind: %s:%s", cs->url, cs->port); + else if (si != old_si) + LOGWARNING("Failed over to bitcoind: %s:%s", cs->url, cs->port); + } + /* This does not necessarily mean we reconnect, but a change has * occurred and we need to reexamine the proxies. */ cproxy = wait_best_proxy(ckp, gdata); @@ -2755,13 +2772,13 @@ reconnect: goto out; if (proxi != cproxy) { proxi = cproxy; - if (ckp->node) { - LOGWARNING("Successfully connected to pool %d %s as proxy node", - proxi->id, proxi->url); - } else { - LOGWARNING("Successfully connected to pool %d %s as proxy%s", - proxi->id, proxi->url, ckp->passthrough ? " in passthrough mode" : ""); - } + LOGWARNING("Successfully connected to pool %d %s as proxy%s", + proxi->id, proxi->url, ckp->passthrough ? " in passthrough mode" : ""); + } + + if (unlikely(!started)) { + started = true; + LOGWARNING("%s generator ready", ckp->name); } retry: clear_unix_msg(&umsg); @@ -2859,14 +2876,14 @@ static void *server_watchdog(void *arg) return NULL; } -static int server_mode(ckpool_t *ckp, proc_instance_t *pi) +static void setup_servers(ckpool_t *ckp) { pthread_t pth_watchdog; - server_instance_t *si; - int i, ret; + int i; ckp->servers = ckalloc(sizeof(server_instance_t *) * ckp->btcds); for (i = 0; i < ckp->btcds; i++) { + server_instance_t *si; connsock_t *cs; ckp->servers[i] = ckzalloc(sizeof(server_instance_t)); @@ -2882,10 +2899,19 @@ static int server_mode(ckpool_t *ckp, proc_instance_t *pi) } create_pthread(&pth_watchdog, server_watchdog, ckp); +} + +static int server_mode(ckpool_t *ckp, proc_instance_t *pi) +{ + int i, ret; + + setup_servers(ckp); + ret = gen_loop(pi); for (i = 0; i < ckp->btcds; i++) { - si = ckp->servers[i]; + server_instance_t *si = ckp->servers[i]; + kill_server(si); dealloc(si); } @@ -2924,6 +2950,9 @@ static int proxy_mode(ckpool_t *ckp, proc_instance_t *pi) mutex_init(&gdata->notify_lock); mutex_init(&gdata->share_lock); + if (ckp->node) + setup_servers(ckp); + /* Create all our proxy structures and pointers */ for (i = 0; i < ckp->proxies; i++) { proxy = __add_proxy(ckp, gdata, i); @@ -2939,8 +2968,6 @@ static int proxy_mode(ckpool_t *ckp, proc_instance_t *pi) } } - LOGWARNING("%s generator ready", ckp->name); - ret = proxy_loop(pi); return ret; From b09bb253fc582e75147a81c9dde8b08ad502632f Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Tue, 29 Dec 2015 15:27:44 +1100 Subject: [PATCH 498/544] Act as a passthrough in node mode, passing what type of stratum message we're passing through --- src/ckpool.c | 2 +- src/ckpool.h | 32 ++++++++++++++++++++++++++++ src/connector.c | 16 ++++++++------ src/stratifier.c | 55 +++++++++++++++++++++++++++--------------------- 4 files changed, 74 insertions(+), 31 deletions(-) diff --git a/src/ckpool.c b/src/ckpool.c index 384b8a00..15a9a5bf 100644 --- a/src/ckpool.c +++ b/src/ckpool.c @@ -1641,7 +1641,7 @@ int main(int argc, char **argv) case 'N': if (ckp.proxy || ckp.redirector || ckp.userproxy || ckp.passthrough) quit(1, "Cannot set another proxy type or redirector and node mode"); - ckp.standalone = ckp.proxy = ckp.node = true; + ckp.standalone = ckp.proxy = ckp.passthrough = ckp.node = true; break; case 'n': ckp.name = optarg; diff --git a/src/ckpool.h b/src/ckpool.h index cde4f0aa..0a72eb36 100644 --- a/src/ckpool.h +++ b/src/ckpool.h @@ -243,6 +243,38 @@ struct ckpool_instance { void *data; }; +enum stratum_msgtype { + SM_RECONNECT = 0, + SM_DIFF, + SM_MSG, + SM_UPDATE, + SM_ERROR, + SM_SUBSCRIBE, + SM_SUBSCRIBERESULT, + SM_SHARE, + SM_SHARERESULT, + SM_AUTH, + SM_AUTHRESULT, + SM_TXNS, + SM_TXNSRESULT +}; + +static const char __maybe_unused *stratum_msgs[] = { + "reconnect", + "diff", + "message", + "update", + "error", + "subscribe", + "subscribe.result", + "share", + "share.result", + "auth", + "auth.result", + "txns", + "txns.result" +}; + #ifdef USE_CKDB #define CKP_STANDALONE(CKP) ((CKP)->standalone == true) #else diff --git a/src/connector.c b/src/connector.c index d932855d..189bb6da 100644 --- a/src/connector.c +++ b/src/connector.c @@ -504,10 +504,10 @@ reparse: * do this unlocked as the occasional false negative can be * filtered by the stratifier. */ if (likely(!client->invalid)) { + if (ckp->node || !ckp->passthrough) + send_proc(ckp->stratifier, s); if (ckp->passthrough) send_proc(ckp->generator, s); - else - send_proc(ckp->stratifier, s); } free(s); @@ -936,11 +936,11 @@ static void passthrough_client(cdata_t *cdata, client_instance_t *client) send_client(cdata, client->id, buf); } -static void process_client_msg(cdata_t *cdata, const char *buf) +static void process_client_msg(ckpool_t *ckp, cdata_t *cdata, const char *buf) { + char *msg, *node_msg; int64_t client_id; json_t *json_msg; - char *msg; json_msg = json_loads(buf, 0, NULL); if (unlikely(!json_msg)) { @@ -953,8 +953,12 @@ static void process_client_msg(cdata_t *cdata, const char *buf) json_object_del(json_msg, "client_id"); /* Put client_id back in for a passthrough subclient, passing its * upstream client_id instead of the passthrough's. */ - if (client_id > 0xffffffffll) + if (ckp->node || client_id > 0xffffffffll) { json_object_set_new_nocheck(json_msg, "client_id", json_integer(client_id & 0xffffffffll)); + if (json_get_string(&node_msg, json_msg, "node.method")) + LOGDEBUG("Got node method %s", node_msg); + } + msg = json_dumps(json_msg, JSON_EOL); send_client(cdata, client_id, msg); json_decref(json_msg); @@ -1055,7 +1059,7 @@ retry: /* The bulk of the messages will be json messages to send to clients * so look for them first. */ if (likely(buf[0] == '{')) { - process_client_msg(cdata, buf); + process_client_msg(ckp, cdata, buf); } else if (cmdmatch(buf, "dropclient")) { client_instance_t *client; diff --git a/src/stratifier.c b/src/stratifier.c index 172f5d57..2180a743 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -2192,11 +2192,21 @@ static void stratum_broadcast(sdata_t *sdata, json_t *val) mutex_unlock(ssends->lock); } -static void stratum_add_send(sdata_t *sdata, json_t *val, const int64_t client_id) +/* passthrough subclients have client_ids in the high bits */ +static inline bool passthrough_subclient(const int64_t client_id) { - smsg_t *msg; + return (client_id > 0xffffffffll); +} - msg = ckzalloc(sizeof(smsg_t)); +static void stratum_add_send(sdata_t *sdata, json_t *val, const int64_t client_id, + const int msg_type) +{ + smsg_t *msg = ckzalloc(sizeof(smsg_t)); + ckpool_t *ckp = sdata->ckp; + + if (ckp->node || passthrough_subclient(client_id)) + json_set_string(val, "node.method", stratum_msgs[msg_type]); + LOGDEBUG("Sending stratum message %s", stratum_msgs[msg_type]); msg->json_msg = val; msg->client_id = client_id; ckmsgq_add(sdata->ssends, msg); @@ -2595,7 +2605,7 @@ static void reconnect_client(sdata_t *sdata, stratum_instance_t *client) client->reconnect_request = time(NULL); JSON_CPACK(json_msg, "{sosss[]}", "id", json_null(), "method", "client.reconnect", "params"); - stratum_add_send(sdata, json_msg, client->id); + stratum_add_send(sdata, json_msg, client->id, SM_RECONNECT); } static void dead_proxy(sdata_t *sdata, const char *buf) @@ -3520,12 +3530,6 @@ out_unlock: return ret; } -/* passthrough subclients have client_ids in the high bits */ -static inline bool passthrough_subclient(const int64_t client_id) -{ - return (client_id > 0xffffffffll); -} - /* Extranonce1 must be set here. Needs to be entered with client holding a ref * count. */ static json_t *parse_subscribe(stratum_instance_t *client, const int64_t client_id, const json_t *params_val) @@ -4251,7 +4255,7 @@ static void stratum_send_diff(sdata_t *sdata, const stratum_instance_t *client) JSON_CPACK(json_msg, "{s[I]soss}", "params", client->diff, "id", json_null(), "method", "mining.set_difficulty"); - stratum_add_send(sdata, json_msg, client->id); + stratum_add_send(sdata, json_msg, client->id, SM_DIFF); } /* Needs to be entered with client holding a ref count. */ @@ -4261,7 +4265,7 @@ static void stratum_send_message(sdata_t *sdata, const stratum_instance_t *clien JSON_CPACK(json_msg, "{sosss[s]}", "id", json_null(), "method", "client.show_message", "params", msg); - stratum_add_send(sdata, json_msg, client->id); + stratum_add_send(sdata, json_msg, client->id, SM_MSG); } static double time_bias(const double tdiff, const double period) @@ -4897,7 +4901,7 @@ static void stratum_send_update(sdata_t *sdata, const int64_t client_id, const b json_msg = __stratum_notify(sdata->current_workbase, clean); ck_runlock(&sdata->workbase_lock); - stratum_add_send(sdata, json_msg, client_id); + stratum_add_send(sdata, json_msg, client_id, SM_UPDATE); } static void send_json_err(sdata_t *sdata, const int64_t client_id, json_t *id_val, const char *err_msg) @@ -4905,7 +4909,7 @@ static void send_json_err(sdata_t *sdata, const int64_t client_id, json_t *id_va json_t *val; JSON_CPACK(val, "{soss}", "id", json_deep_copy(id_val), "error", err_msg); - stratum_add_send(sdata, val, client_id); + stratum_add_send(sdata, val, client_id, SM_ERROR); } /* Needs to be entered with client holding a ref count. */ @@ -5028,6 +5032,15 @@ static void parse_method(ckpool_t *ckp, sdata_t *sdata, stratum_instance_t *clie return; } + if (cmdmatch(method, "mining.term")) { + LOGDEBUG("Mining terminate requested from %"PRId64" %s", client_id, client->address); + drop_client(ckp, sdata, client_id); + return; + } + + if (ckp->node) + return; + if (cmdmatch(method, "mining.subscribe")) { json_t *val, *result_val; @@ -5046,7 +5059,7 @@ static void parse_method(ckpool_t *ckp, sdata_t *sdata, stratum_instance_t *clie json_object_set_new_nocheck(val, "result", result_val); json_object_set_nocheck(val, "id", id_val); json_object_set_new_nocheck(val, "error", json_null()); - stratum_add_send(sdata, val, client_id); + stratum_add_send(sdata, val, client_id, SM_SUBSCRIBERESULT); if (likely(client->subscribed)) init_client(sdata, client, client_id); return; @@ -5118,12 +5131,6 @@ static void parse_method(ckpool_t *ckp, sdata_t *sdata, stratum_instance_t *clie return; } - if (cmdmatch(method, "mining.term")) { - LOGDEBUG("Mining terminate requested from %"PRId64" %s", client_id, client->address); - drop_client(ckp, sdata, client_id); - return; - } - /* Unhandled message here */ LOGINFO("Unhandled client %"PRId64" %s method %s", client_id, client->address, method); return; @@ -5319,7 +5326,7 @@ static void sshare_process(ckpool_t *ckp, json_params_t *jp) json_object_set_new_nocheck(json_msg, "result", result_val); json_object_set_new_nocheck(json_msg, "error", err_val ? err_val : json_null()); steal_json_id(json_msg, jp); - stratum_add_send(sdata, json_msg, client_id); + stratum_add_send(sdata, json_msg, client_id, SM_SHARERESULT); out_decref: dec_instance_ref(sdata, client); out: @@ -5381,7 +5388,7 @@ static void sauth_process(ckpool_t *ckp, json_params_t *jp) json_object_set_new_nocheck(json_msg, "result", result_val); json_object_set_new_nocheck(json_msg, "error", err_val ? err_val : json_null()); steal_json_id(json_msg, jp); - stratum_add_send(sdata, json_msg, client_id); + stratum_add_send(sdata, json_msg, client_id, SM_AUTHRESULT); if (!json_is_true(result_val) || !client->suggest_diff) goto out; @@ -5596,7 +5603,7 @@ static void send_transactions(ckpool_t *ckp, json_params_t *jp) } else json_set_string(val, "error", "Invalid job_id"); out_send: - stratum_add_send(sdata, val, jp->client_id); + stratum_add_send(sdata, val, jp->client_id, SM_TXNSRESULT); out: if (client) dec_instance_ref(sdata, client); From c5718d510e26b6e9cb629175f6774d26b72c4736 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Tue, 29 Dec 2015 17:07:22 +1100 Subject: [PATCH 499/544] Detect node message type in the generator --- src/ckpool.h | 6 ++++-- src/connector.c | 11 ++++------- src/generator.c | 25 +++++++++++++++++++++++++ src/stratifier.c | 10 +++++++--- 4 files changed, 40 insertions(+), 12 deletions(-) diff --git a/src/ckpool.h b/src/ckpool.h index 0a72eb36..680b65ea 100644 --- a/src/ckpool.h +++ b/src/ckpool.h @@ -256,7 +256,8 @@ enum stratum_msgtype { SM_AUTH, SM_AUTHRESULT, SM_TXNS, - SM_TXNSRESULT + SM_TXNSRESULT, + SM_NONE }; static const char __maybe_unused *stratum_msgs[] = { @@ -272,7 +273,8 @@ static const char __maybe_unused *stratum_msgs[] = { "auth", "auth.result", "txns", - "txns.result" + "txns.result", + "" }; #ifdef USE_CKDB diff --git a/src/connector.c b/src/connector.c index 189bb6da..ea3d84d4 100644 --- a/src/connector.c +++ b/src/connector.c @@ -504,10 +504,10 @@ reparse: * do this unlocked as the occasional false negative can be * filtered by the stratifier. */ if (likely(!client->invalid)) { - if (ckp->node || !ckp->passthrough) - send_proc(ckp->stratifier, s); if (ckp->passthrough) send_proc(ckp->generator, s); + else + send_proc(ckp->stratifier, s); } free(s); @@ -938,9 +938,9 @@ static void passthrough_client(cdata_t *cdata, client_instance_t *client) static void process_client_msg(ckpool_t *ckp, cdata_t *cdata, const char *buf) { - char *msg, *node_msg; int64_t client_id; json_t *json_msg; + char *msg; json_msg = json_loads(buf, 0, NULL); if (unlikely(!json_msg)) { @@ -953,11 +953,8 @@ static void process_client_msg(ckpool_t *ckp, cdata_t *cdata, const char *buf) json_object_del(json_msg, "client_id"); /* Put client_id back in for a passthrough subclient, passing its * upstream client_id instead of the passthrough's. */ - if (ckp->node || client_id > 0xffffffffll) { + if (client_id > 0xffffffffll) json_object_set_new_nocheck(json_msg, "client_id", json_integer(client_id & 0xffffffffll)); - if (json_get_string(&node_msg, json_msg, "node.method")) - LOGDEBUG("Got node method %s", node_msg); - } msg = json_dumps(json_msg, JSON_EOL); send_client(cdata, client_id, msg); diff --git a/src/generator.c b/src/generator.c index 36cd2647..e46a4746 100644 --- a/src/generator.c +++ b/src/generator.c @@ -1980,6 +1980,27 @@ static void reconnect_proxy(proxy_instance_t *proxi) create_pthread(&pth, proxy_reconnect, proxi); } +static int node_msg_type(json_t *val) +{ + int i, ret = -1; + char *node_msg; + + if (!val) + goto out; + if (!json_get_string(&node_msg, val, "node.method")) + goto out; + for (i = 0; i < SM_NONE; i++) { + if (!strcmp(node_msg, stratum_msgs[i])) { + ret = i; + LOGWARNING("Got node method %d:%s", i, node_msg); + break; + } + } + json_object_del(val, "node.method"); +out: + return ret; +} + /* For receiving messages from an upstream pool to pass downstream. Responsible * for setting up the connection and testing pool is live. */ static void *passthrough_recv(void *arg) @@ -2022,6 +2043,10 @@ static void *passthrough_recv(void *arg) Close(cs->fd); continue; } + if (ckp->node) { + json_t *val = json_loads(cs->buf, 0, NULL); + int msg_type = node_msg_type(val); + } /* Simply forward the message on, as is, to the connector to * process. Possibly parse parameters sent by upstream pool * here */ diff --git a/src/stratifier.c b/src/stratifier.c index 2180a743..8a2abab4 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -6240,10 +6240,14 @@ int stratifier(proc_instance_t *pi) threads = sysconf(_SC_NPROCESSORS_ONLN) / 2 ? : 1; sdata->sshareq = create_ckmsgqs(ckp, "sprocessor", &sshare_process, threads); /* Create 1/4 as many stratum processing threads as there are CPUs */ - threads = threads / 2 ? : 1; + if (ckp->node) + threads = 1; + else { + threads = threads / 2 ? : 1; + sdata->sauthq = create_ckmsgq(ckp, "authoriser", &sauth_process); + sdata->stxnq = create_ckmsgq(ckp, "stxnq", &send_transactions); + } sdata->srecvs = create_ckmsgqs(ckp, "sreceiver", &srecv_process, threads); - sdata->sauthq = create_ckmsgq(ckp, "authoriser", &sauth_process); - sdata->stxnq = create_ckmsgq(ckp, "stxnq", &send_transactions); if (!CKP_STANDALONE(ckp)) { sdata->ckdbq = create_ckmsgq(ckp, "ckdbqueue", &ckdbq_process); create_pthread(&pth_heartbeat, ckdb_heartbeat, ckp); From 50c9b5ecd26b9dd8be7bfed6c013937175f48540 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Tue, 29 Dec 2015 17:24:28 +1100 Subject: [PATCH 500/544] Move message type detection to stratifier --- src/connector.c | 16 ++++++++++++++-- src/generator.c | 25 ------------------------- src/stratifier.c | 34 ++++++++++++++++++++++++++++++---- 3 files changed, 44 insertions(+), 31 deletions(-) diff --git a/src/connector.c b/src/connector.c index ea3d84d4..e9935748 100644 --- a/src/connector.c +++ b/src/connector.c @@ -899,6 +899,18 @@ static void send_client(cdata_t *cdata, const int64_t id, char *buf) free(buf); return; } + if (ckp->node) { + json_t *val = json_loads(buf, 0, NULL); + char *msg; + + json_object_set_new_nocheck(val, "client_id", json_integer(client->id)); + json_object_set_new_nocheck(val, "address", json_string(client->address_name)); + json_object_set_new_nocheck(val, "server", json_integer(client->server)); + msg = json_dumps(val, 0); + json_decref(val); + send_proc(ckp->stratifier, msg); + free(msg); + } if (ckp->redirector && !client->redirected) test_redirector_shares(ckp, client, buf); } @@ -936,7 +948,7 @@ static void passthrough_client(cdata_t *cdata, client_instance_t *client) send_client(cdata, client->id, buf); } -static void process_client_msg(ckpool_t *ckp, cdata_t *cdata, const char *buf) +static void process_client_msg(cdata_t *cdata, const char *buf) { int64_t client_id; json_t *json_msg; @@ -1056,7 +1068,7 @@ retry: /* The bulk of the messages will be json messages to send to clients * so look for them first. */ if (likely(buf[0] == '{')) { - process_client_msg(ckp, cdata, buf); + process_client_msg(cdata, buf); } else if (cmdmatch(buf, "dropclient")) { client_instance_t *client; diff --git a/src/generator.c b/src/generator.c index e46a4746..36cd2647 100644 --- a/src/generator.c +++ b/src/generator.c @@ -1980,27 +1980,6 @@ static void reconnect_proxy(proxy_instance_t *proxi) create_pthread(&pth, proxy_reconnect, proxi); } -static int node_msg_type(json_t *val) -{ - int i, ret = -1; - char *node_msg; - - if (!val) - goto out; - if (!json_get_string(&node_msg, val, "node.method")) - goto out; - for (i = 0; i < SM_NONE; i++) { - if (!strcmp(node_msg, stratum_msgs[i])) { - ret = i; - LOGWARNING("Got node method %d:%s", i, node_msg); - break; - } - } - json_object_del(val, "node.method"); -out: - return ret; -} - /* For receiving messages from an upstream pool to pass downstream. Responsible * for setting up the connection and testing pool is live. */ static void *passthrough_recv(void *arg) @@ -2043,10 +2022,6 @@ static void *passthrough_recv(void *arg) Close(cs->fd); continue; } - if (ckp->node) { - json_t *val = json_loads(cs->buf, 0, NULL); - int msg_type = node_msg_type(val); - } /* Simply forward the message on, as is, to the connector to * process. Possibly parse parameters sent by upstream pool * here */ diff --git a/src/stratifier.c b/src/stratifier.c index 8a2abab4..d30877ee 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -5038,9 +5038,6 @@ static void parse_method(ckpool_t *ckp, sdata_t *sdata, stratum_instance_t *clie return; } - if (ckp->node) - return; - if (cmdmatch(method, "mining.subscribe")) { json_t *val, *result_val; @@ -5142,6 +5139,27 @@ static void free_smsg(smsg_t *msg) free(msg); } +static int node_msg_type(json_t *val) +{ + int i, ret = -1; + char *node_msg; + + if (!val) + goto out; + if (!json_get_string(&node_msg, val, "node.method")) + goto out; + for (i = 0; i < SM_NONE; i++) { + if (!strcmp(node_msg, stratum_msgs[i])) { + ret = i; + LOGWARNING("Got node method %d:%s", i, node_msg); + break; + } + } + json_object_del(val, "node.method"); +out: + return ret; +} + /* Entered with client holding ref count */ static void parse_instance_msg(ckpool_t *ckp, sdata_t *sdata, smsg_t *msg, stratum_instance_t *client) { @@ -5185,7 +5203,15 @@ static void parse_instance_msg(ckpool_t *ckp, sdata_t *sdata, smsg_t *msg, strat send_json_err(sdata, client_id, id_val, "-1:params not found"); goto out; } - parse_method(ckp, sdata, client, client_id, id_val, method, params); + if (ckp->node) { + int msg_type = node_msg_type(val); + + if (msg_type > -1) + LOGWARNING("Got node method %d:%s", msg_type, stratum_msgs[msg_type]); + else + LOGWARNING("Missing node method"); + } else + parse_method(ckp, sdata, client, client_id, id_val, method, params); out: free_smsg(msg); } From 5ddfcdb14e4f548cfafe609c62b5e23da70b4972 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Tue, 29 Dec 2015 18:44:48 +1100 Subject: [PATCH 501/544] Identify all node message types in the stratifier --- src/connector.c | 4 ++-- src/stratifier.c | 44 ++++++++++++++++++++++++++++++-------------- 2 files changed, 32 insertions(+), 16 deletions(-) diff --git a/src/connector.c b/src/connector.c index e9935748..89eee877 100644 --- a/src/connector.c +++ b/src/connector.c @@ -504,10 +504,10 @@ reparse: * do this unlocked as the occasional false negative can be * filtered by the stratifier. */ if (likely(!client->invalid)) { + if (!ckp->passthrough || ckp->node) + send_proc(ckp->stratifier, s); if (ckp->passthrough) send_proc(ckp->generator, s); - else - send_proc(ckp->stratifier, s); } free(s); diff --git a/src/stratifier.c b/src/stratifier.c index d30877ee..12a22720 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -5142,24 +5142,45 @@ static void free_smsg(smsg_t *msg) static int node_msg_type(json_t *val) { int i, ret = -1; - char *node_msg; + char *method; if (!val) goto out; - if (!json_get_string(&node_msg, val, "node.method")) + if (!json_get_string(&method, val, "node.method")) goto out; for (i = 0; i < SM_NONE; i++) { - if (!strcmp(node_msg, stratum_msgs[i])) { + if (!strcmp(method, stratum_msgs[i])) { ret = i; - LOGWARNING("Got node method %d:%s", i, node_msg); + LOGWARNING("Got node method %d:%s", i, method); break; } } json_object_del(val, "node.method"); out: + if (ret < 0 && json_get_string(&method, val, "method")) { + if (!safecmp(method, "mining.subscribe")) + ret = SM_SUBSCRIBE; + else if (!safecmp(method, "mining.submit")) + ret = SM_SHARE; + else if (cmdmatch(method, "mining.auth")) + ret = SM_AUTH; + else if (cmdmatch(method, "mining.get")) + ret = SM_TXNS; + } return ret; } +/* Entered with client holding ref count */ +static void parse_node_msg(json_t *val, const char *buf, stratum_instance_t *client) +{ + int msg_type = node_msg_type(val); + + if (msg_type > -1) + LOGWARNING("Got node method %d:%s", msg_type, stratum_msgs[msg_type]); + else + LOGWARNING("Missing node method from %s", buf); +} + /* Entered with client holding ref count */ static void parse_instance_msg(ckpool_t *ckp, sdata_t *sdata, smsg_t *msg, stratum_instance_t *client) { @@ -5203,15 +5224,7 @@ static void parse_instance_msg(ckpool_t *ckp, sdata_t *sdata, smsg_t *msg, strat send_json_err(sdata, client_id, id_val, "-1:params not found"); goto out; } - if (ckp->node) { - int msg_type = node_msg_type(val); - - if (msg_type > -1) - LOGWARNING("Got node method %d:%s", msg_type, stratum_msgs[msg_type]); - else - LOGWARNING("Missing node method"); - } else - parse_method(ckp, sdata, client, client_id, id_val, method, params); + parse_method(ckp, sdata, client, client_id, id_val, method, params); out: free_smsg(msg); } @@ -5288,7 +5301,10 @@ static void srecv_process(ckpool_t *ckp, char *buf) if (unlikely(noid)) LOGINFO("Stratifier added instance %"PRId64" server %d", client->id, server); - parse_instance_msg(ckp, sdata, msg, client); + if (ckp->node) + parse_node_msg(msg->json_msg, buf, client); + else + parse_instance_msg(ckp, sdata, msg, client); dec_instance_ref(sdata, client); out: free(buf); From 6f7dc2d2f0f873bcad6db27addd3641e3c1b1b3a Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Tue, 29 Dec 2015 22:15:23 +1100 Subject: [PATCH 502/544] Add update detection in parse_node_msg and remove double warning --- src/stratifier.c | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/src/stratifier.c b/src/stratifier.c index 12a22720..4ce51b4b 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -5151,17 +5151,18 @@ static int node_msg_type(json_t *val) for (i = 0; i < SM_NONE; i++) { if (!strcmp(method, stratum_msgs[i])) { ret = i; - LOGWARNING("Got node method %d:%s", i, method); break; } } json_object_del(val, "node.method"); out: if (ret < 0 && json_get_string(&method, val, "method")) { - if (!safecmp(method, "mining.subscribe")) - ret = SM_SUBSCRIBE; - else if (!safecmp(method, "mining.submit")) + if (!safecmp(method, "mining.submit")) ret = SM_SHARE; + else if (!safecmp(method, "mining.notify")) + ret = SM_UPDATE; + else if (!safecmp(method, "mining.subscribe")) + ret = SM_SUBSCRIBE; else if (cmdmatch(method, "mining.auth")) ret = SM_AUTH; else if (cmdmatch(method, "mining.get")) @@ -5176,9 +5177,9 @@ static void parse_node_msg(json_t *val, const char *buf, stratum_instance_t *cli int msg_type = node_msg_type(val); if (msg_type > -1) - LOGWARNING("Got node method %d:%s", msg_type, stratum_msgs[msg_type]); + LOGWARNING("Got client %"PRId64" node method %d:%s", client->id, msg_type, stratum_msgs[msg_type]); else - LOGWARNING("Missing node method from %s", buf); + LOGWARNING("Missing client %"PRId64" node method from %s", client->id, buf); } /* Entered with client holding ref count */ From 04eaabfaa2091e159792265c680f8e3295e40601 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Tue, 29 Dec 2015 23:11:49 +1100 Subject: [PATCH 503/544] Add mining nodes to a linked list and send them the workinfo for now --- src/ckpool.h | 2 ++ src/connector.c | 6 ++++++ src/stratifier.c | 54 ++++++++++++++++++++++++++++++++++++++++++++---- 3 files changed, 58 insertions(+), 4 deletions(-) diff --git a/src/ckpool.h b/src/ckpool.h index 680b65ea..5c4529c0 100644 --- a/src/ckpool.h +++ b/src/ckpool.h @@ -257,6 +257,7 @@ enum stratum_msgtype { SM_AUTHRESULT, SM_TXNS, SM_TXNSRESULT, + SM_WORKINFO, SM_NONE }; @@ -274,6 +275,7 @@ static const char __maybe_unused *stratum_msgs[] = { "auth.result", "txns", "txns.result", + "workinfo", "" }; diff --git a/src/connector.c b/src/connector.c index 89eee877..0c353ecf 100644 --- a/src/connector.c +++ b/src/connector.c @@ -869,6 +869,12 @@ static void send_client(cdata_t *cdata, const int64_t id, char *buf) return; } + if (unlikely(ckp->node && !id)) { + LOGWARNING("Message for node: %s", buf); + free(buf); + return; + } + /* Grab a reference to this client until the sender_send has * completed processing. Is this a passthrough subclient ? */ if (id > 0xffffffffll) { diff --git a/src/stratifier.c b/src/stratifier.c index 4ce51b4b..666c292e 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -265,6 +265,7 @@ struct stratum_instance { time_t start_time; char address[INET6_ADDRSTRLEN]; + bool node; /* Is this a mining node */ bool subscribed; bool authorising; /* In progress, protected by instance_lock */ bool authorised; @@ -459,6 +460,7 @@ struct stratifier_data { stratum_instance_t *stratum_instances; stratum_instance_t *recycled_instances; + stratum_instance_t *node_instances; int stratum_generated; int disconnected_generated; @@ -784,8 +786,13 @@ static void _ckdbq_add(ckpool_t *ckp, const int idtype, json_t *val, const char #define ckdbq_add(ckp, idtype, val) _ckdbq_add(ckp, idtype, val, __FILE__, __func__, __LINE__) -static void send_workinfo(ckpool_t *ckp, const workbase_t *wb) +static void stratum_add_send(sdata_t *sdata, json_t *val, const int64_t client_id, + const int msg_type); + +static void send_workinfo(ckpool_t *ckp, sdata_t *sdata, const workbase_t *wb) { + stratum_instance_t *client; + ckmsg_t *bulk_send = NULL; char cdfield[64]; json_t *val; @@ -807,6 +814,33 @@ static void send_workinfo(ckpool_t *ckp, const workbase_t *wb) "createby", "code", "createcode", __func__, "createinet", ckp->serverurl[0]); + + ck_rlock(&sdata->instance_lock); + DL_FOREACH(sdata->node_instances, client) { + ckmsg_t *client_msg; + smsg_t *msg; + json_t *json_msg = json_deep_copy(val); + + json_set_string(json_msg, "node.method", stratum_msgs[SM_WORKINFO]); + client_msg = ckalloc(sizeof(ckmsg_t)); + msg = ckzalloc(sizeof(smsg_t)); + msg->json_msg = json_msg; + msg->client_id = client->id; + client_msg->data = msg; + DL_APPEND(bulk_send, client_msg); + } + ck_runlock(&sdata->instance_lock); + + if (bulk_send) { + ckmsgq_t *ssends = sdata->ssends; + + LOGINFO("Sending workinfo to mining nodes"); + + mutex_lock(ssends->lock); + DL_CONCAT(ssends->msgs, bulk_send); + pthread_cond_signal(ssends->cond); + mutex_unlock(ssends->lock); + } ckdbq_add(ckp, ID_WORKINFO, val); } @@ -894,7 +928,7 @@ static void add_base(ckpool_t *ckp, sdata_t *sdata, workbase_t *wb, bool *new_bl if (*new_block) purge_share_hashtable(sdata, wb->id); - send_workinfo(ckp, wb); + send_workinfo(ckp, sdata, wb); /* Send the aged work message to ckdb once we have dropped the workbase lock * to prevent taking recursive locks */ @@ -1994,6 +2028,8 @@ static void __drop_client(sdata_t *sdata, stratum_instance_t *client, bool lazil { user_instance_t *user = client->user_instance; + if (unlikely(client->node)) + DL_DELETE(sdata->node_instances, client); if (client->workername) { if (user) { ASPRINTF(msg, "Dropped client %"PRId64" %s %suser %s worker %s %s", @@ -5014,6 +5050,17 @@ static void init_client(sdata_t *sdata, const stratum_instance_t *client, const stratum_send_update(sdata, client_id, true); } +static void add_mining_node(ckpool_t *ckp, sdata_t *sdata, stratum_instance_t *client) +{ + client->node = true; + + ck_wlock(&sdata->instance_lock); + DL_APPEND(sdata->node_instances, client); + ck_wunlock(&sdata->instance_lock); + + LOGWARNING("Added client %"PRId64" %s as mining node!", client->id, client->address); +} + /* Enter with client holding ref count */ static void parse_method(ckpool_t *ckp, sdata_t *sdata, stratum_instance_t *client, const int64_t client_id, json_t *id_val, json_t *method_val, @@ -5067,10 +5114,9 @@ static void parse_method(ckpool_t *ckp, sdata_t *sdata, stratum_instance_t *clie /* Add this client as a passthrough in the connector and * add it to the list of mining nodes in the stratifier */ - LOGNOTICE("Adding mining client %"PRId64" %s", client_id, client->address); + add_mining_node(ckp, sdata, client); snprintf(buf, 255, "passthrough=%"PRId64, client_id); send_proc(ckp->connector, buf); - drop_client(ckp, sdata, client_id); return; } From 1feea438554d758acf16151e4f2100f2605ce7da Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Tue, 29 Dec 2015 23:51:59 +1100 Subject: [PATCH 504/544] Don't send proxy information in pasthrough mode --- src/generator.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/generator.c b/src/generator.c index 36cd2647..370f9b28 100644 --- a/src/generator.c +++ b/src/generator.c @@ -1053,6 +1053,8 @@ static void send_stratifier_deadproxy(ckpool_t *ckp, const int id, const int sub { char buf[256]; + if (ckp->passthrough) + return; sprintf(buf, "deadproxy=%d:%d", id, subid); send_proc(ckp->stratifier, buf); } From a647842602ed43f0f0c079da14f31546a87ebab9 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Tue, 29 Dec 2015 23:59:34 +1100 Subject: [PATCH 505/544] Don't drop node clients for not being authorised --- src/stratifier.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/stratifier.c b/src/stratifier.c index 666c292e..03593652 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -1479,7 +1479,7 @@ static void reconnect_clients(sdata_t *sdata) continue; if (!client->authorised) continue; - /* Is this client boudn to a dead proxy? */ + /* Is this client bound to a dead proxy? */ if (!client->reconnect) { /* This client is bound to a user proxy */ if (client->proxy->userid) @@ -2192,6 +2192,10 @@ static void stratum_broadcast(sdata_t *sdata, json_t *val) connector_test_client(ckp, client->id); continue; } + + if (client->node) + continue; + /* Test for clients that haven't authed in over a minute and drop them */ if (!client->authorised) { if (now_t > client->start_time + 60) { From 71611d221b1d6ff15b588cb4c2c6f19f0413a2b4 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Wed, 30 Dec 2015 00:25:21 +1100 Subject: [PATCH 506/544] Send a more comprehensive workbase in node mode --- src/stratifier.c | 45 +++++++++++++++++++++++++++++++++------------ 1 file changed, 33 insertions(+), 12 deletions(-) diff --git a/src/stratifier.c b/src/stratifier.c index 03593652..39f3160a 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -816,18 +816,39 @@ static void send_workinfo(ckpool_t *ckp, sdata_t *sdata, const workbase_t *wb) "createinet", ckp->serverurl[0]); ck_rlock(&sdata->instance_lock); - DL_FOREACH(sdata->node_instances, client) { - ckmsg_t *client_msg; - smsg_t *msg; - json_t *json_msg = json_deep_copy(val); - - json_set_string(json_msg, "node.method", stratum_msgs[SM_WORKINFO]); - client_msg = ckalloc(sizeof(ckmsg_t)); - msg = ckzalloc(sizeof(smsg_t)); - msg->json_msg = json_msg; - msg->client_id = client->id; - client_msg->data = msg; - DL_APPEND(bulk_send, client_msg); + if (sdata->node_instances) { + json_t *wb_val = json_object(); + + json_set_string(wb_val, "target", wb->target); + json_set_double(wb_val, "diff", wb->diff); + json_set_int(wb_val, "version", wb->version); + json_set_int(wb_val, "curtime", wb->curtime); + json_set_string(wb_val, "prevhash", wb->prevhash); + json_set_string(wb_val, "ntime", wb->ntime); + json_set_string(wb_val, "bbversion", wb->bbversion); + json_set_string(wb_val, "nbit", wb->nbit); + json_set_int(wb_val, "coinbasevalue", wb->coinbasevalue); + json_set_int(wb_val, "height", wb->height); + json_set_int(wb_val, "transactions", wb->transactions); + json_set_string(wb_val, "txn_data", wb->txn_data); + /* We don't need txn_hashes */ + json_set_int(wb_val, "merkles", wb->merkles); + json_object_set_new_nocheck(wb_val, "merklehash", json_deep_copy(wb->merkle_array)); + + DL_FOREACH(sdata->node_instances, client) { + ckmsg_t *client_msg; + smsg_t *msg; + json_t *json_msg = json_deep_copy(wb_val); + + json_set_string(json_msg, "node.method", stratum_msgs[SM_WORKINFO]); + client_msg = ckalloc(sizeof(ckmsg_t)); + msg = ckzalloc(sizeof(smsg_t)); + msg->json_msg = json_msg; + msg->client_id = client->id; + client_msg->data = msg; + DL_APPEND(bulk_send, client_msg); + } + json_decref(wb_val); } ck_runlock(&sdata->instance_lock); From 3dc070fcc2f9c9c6285fefd503e21437a6f313f3 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Wed, 30 Dec 2015 02:28:04 +1100 Subject: [PATCH 507/544] Do initial parsing of subscribe/auth in node mode --- src/connector.c | 3 +- src/stratifier.c | 173 +++++++++++++++++++++++++++++++++++++++++++---- 2 files changed, 163 insertions(+), 13 deletions(-) diff --git a/src/connector.c b/src/connector.c index 0c353ecf..3ce2bb2d 100644 --- a/src/connector.c +++ b/src/connector.c @@ -870,7 +870,8 @@ static void send_client(cdata_t *cdata, const int64_t id, char *buf) } if (unlikely(ckp->node && !id)) { - LOGWARNING("Message for node: %s", buf); + LOGDEBUG("Message for node: %s", buf); + send_proc(ckp->stratifier, buf); free(buf); return; } diff --git a/src/stratifier.c b/src/stratifier.c index 39f3160a..0378b2cf 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -829,11 +829,19 @@ static void send_workinfo(ckpool_t *ckp, sdata_t *sdata, const workbase_t *wb) json_set_string(wb_val, "nbit", wb->nbit); json_set_int(wb_val, "coinbasevalue", wb->coinbasevalue); json_set_int(wb_val, "height", wb->height); + json_set_string(wb_val, "flags", wb->flags); json_set_int(wb_val, "transactions", wb->transactions); - json_set_string(wb_val, "txn_data", wb->txn_data); + if (likely(wb->transactions)) + json_set_string(wb_val, "txn_data", wb->txn_data); /* We don't need txn_hashes */ json_set_int(wb_val, "merkles", wb->merkles); json_object_set_new_nocheck(wb_val, "merklehash", json_deep_copy(wb->merkle_array)); + json_set_string(wb_val, "coinb1", wb->coinb1); + json_set_int(wb_val, "enonce1varlen", wb->enonce1varlen); + json_set_int(wb_val, "enonce2varlen", wb->enonce2varlen); + json_set_int(wb_val, "coinb1len", wb->coinb1len); + json_set_int(wb_val, "coinb2len", wb->coinb2len); + json_set_string(wb_val, "coinb2", wb->coinb2); DL_FOREACH(sdata->node_instances, client) { ckmsg_t *client_msg; @@ -949,7 +957,8 @@ static void add_base(ckpool_t *ckp, sdata_t *sdata, workbase_t *wb, bool *new_bl if (*new_block) purge_share_hashtable(sdata, wb->id); - send_workinfo(ckp, sdata, wb); + if (!ckp->passthrough) + send_workinfo(ckp, sdata, wb); /* Send the aged work message to ckdb once we have dropped the workbase lock * to prevent taking recursive locks */ @@ -1123,6 +1132,67 @@ out: return NULL; } +static void add_node_base(ckpool_t *ckp, json_t *val) +{ + workbase_t *wb = ckzalloc(sizeof(workbase_t)); + sdata_t *sdata = ckp->data; + bool new_block = false; + char header[228]; + + wb->ckp = ckp; + json_strcpy(wb->target, val, "target"); + json_dblcpy(&wb->diff, val, "diff"); + json_uintcpy(&wb->version, val, "version"); + json_uintcpy(&wb->curtime, val, "curtime"); + json_strcpy(wb->prevhash, val, "prevhash"); + json_strcpy(wb->ntime, val, "ntime"); + sscanf(wb->ntime, "%x", &wb->ntime32); + json_strcpy(wb->bbversion, val, "bbversion"); + json_strcpy(wb->nbit, val, "nbit"); + json_uint64cpy(&wb->coinbasevalue, val, "coinbasevalue"); + json_intcpy(&wb->height, val, "height"); + json_strdup(&wb->flags, val, "flags"); + json_intcpy(&wb->transactions, val, "transactions"); + if (wb->transactions) + json_strdup(&wb->txn_data, val, "txn_data"); + json_intcpy(&wb->merkles, val, "merkles"); + wb->merkle_array = json_array(); + if (wb->merkles) { + json_t *arr; + int i; + + arr = json_object_get(val, "merklehash"); + for (i = 0; i < wb->merkles; i++) { + strcpy(&wb->merklehash[i][0], json_string_value(json_array_get(arr, i))); + hex2bin(&wb->merklebin[i][0], &wb->merklehash[i][0], 32); + json_array_append_new(wb->merkle_array, json_string(&wb->merklehash[i][0])); + } + } + json_strdup(&wb->coinb1, val, "coinb1"); + json_intcpy(&wb->coinb1len, val, "coinb1len"); + wb->coinb1bin = ckzalloc(wb->coinb1len); + hex2bin(wb->coinb1bin, wb->coinb1, wb->coinb1len); + json_strdup(&wb->coinb2, val, "coinb2"); + json_intcpy(&wb->coinb2len, val, "coinb2len"); + wb->coinb2bin = ckzalloc(wb->coinb2len); + hex2bin(wb->coinb2bin, wb->coinb2, wb->coinb2len); + json_intcpy(&wb->enonce1varlen, val, "enonce1varlen"); + json_intcpy(&wb->enonce2varlen, val, "enonce2varlen"); + + snprintf(header, 225, "%s%s%s%s%s%s%s", + wb->bbversion, wb->prevhash, + "0000000000000000000000000000000000000000000000000000000000000000", + wb->ntime, wb->nbit, + "00000000", /* nonce */ + workpadding); + LOGDEBUG("Header: %s", header); + hex2bin(wb->headerbin, header, 112); + + add_base(ckp, sdata, wb, &new_block); + if (new_block) + LOGWARNING("Block hash changed to %s", sdata->lastswaphash); +} + static void update_base(ckpool_t *ckp, const int prio) { struct update_req *ur = ckalloc(sizeof(struct update_req)); @@ -3610,7 +3680,7 @@ static json_t *parse_subscribe(stratum_instance_t *client, const int64_t client_ } sdata = select_sdata(ckp, ckp_sdata, 0); - if (unlikely(!sdata || !sdata->current_workbase)) { + if (unlikely(!ckp->node && (!sdata || !sdata->current_workbase))) { LOGWARNING("Failed to provide subscription due to no %s", sdata ? "current workbase" : "sdata"); stratum_send_message(ckp_sdata, client, "Pool Initialising"); return json_string("Initialising"); @@ -3647,6 +3717,10 @@ static json_t *parse_subscribe(stratum_instance_t *client, const int64_t client_ } else client->useragent = ckzalloc(1); + /* We got what we needed */ + if (ckp->node) + return NULL; + if (ckp->proxy) { /* Use the session_id to tell us which user this was. * If it's not available, see if there's an IP address @@ -4306,6 +4380,7 @@ static json_t *parse_authorise(stratum_instance_t *client, const json_t *params_ /* We can set this outside of lock safely */ client->authorising = false; out: + LOGWARNING("Parsed %d", ret); return json_boolean(ret); } @@ -5075,7 +5150,7 @@ static void init_client(sdata_t *sdata, const stratum_instance_t *client, const stratum_send_update(sdata, client_id, true); } -static void add_mining_node(ckpool_t *ckp, sdata_t *sdata, stratum_instance_t *client) +static void add_mining_node(sdata_t *sdata, stratum_instance_t *client) { client->node = true; @@ -5139,7 +5214,7 @@ static void parse_method(ckpool_t *ckp, sdata_t *sdata, stratum_instance_t *clie /* Add this client as a passthrough in the connector and * add it to the list of mining nodes in the stratifier */ - add_mining_node(ckp, sdata, client); + add_mining_node(sdata, client); snprintf(buf, 255, "passthrough=%"PRId64, client_id); send_proc(ckp->connector, buf); return; @@ -5210,6 +5285,27 @@ static void free_smsg(smsg_t *msg) free(msg); } +static void parse_diff(stratum_instance_t *client, json_t *val) +{ + double diff = json_number_value(json_array_get(val, 0)); + + LOGINFO("Set client %"PRId64" to diff %lf", client->id, diff); + client->diff = diff; +} + +static void parse_subscribe_result(stratum_instance_t *client, json_t *val) +{ + strncpy(client->enonce1, json_string_value(json_array_get(val, 1)), 16); + LOGDEBUG("Client %"PRId64" got enonce1 %s", client->id, client->enonce1); + sprintf(client->enonce1, "%016lx", client->enonce1_64); +} + +static void parse_authorise_result(stratum_instance_t *client, json_t *val) +{ + client->authorised = json_is_true(val); + LOGDEBUG("Client %"PRId64" is %sauthorised", client->id, client->authorised ? "" : "not "); +} + static int node_msg_type(json_t *val) { int i, ret = -1; @@ -5243,14 +5339,64 @@ out: } /* Entered with client holding ref count */ -static void parse_node_msg(json_t *val, const char *buf, stratum_instance_t *client) +static void node_client_msg(ckpool_t *ckp, json_t *val, const char *buf, stratum_instance_t *client) { + json_t *params, *method, *res_val, *id_val, *err_val = NULL; int msg_type = node_msg_type(val); + sdata_t *sdata = ckp->data; + json_params_t *jp; + int errnum; - if (msg_type > -1) - LOGWARNING("Got client %"PRId64" node method %d:%s", client->id, msg_type, stratum_msgs[msg_type]); - else - LOGWARNING("Missing client %"PRId64" node method from %s", client->id, buf); + if (msg_type < 0) { + LOGERR("Missing client %"PRId64" node method from %s", client->id, buf); + return; + } + LOGWARNING("Got client %"PRId64" node method %d:%s", client->id, msg_type, stratum_msgs[msg_type]); + id_val = json_object_get(val, "id"); + method = json_object_get(val, "method"); + params = json_object_get(val, "params"); + res_val = json_object_get(val, "result"); + switch (msg_type) { + case SM_SHARE: + jp = create_json_params(client->id, method, params, id_val); + ckmsgq_add(sdata->sshareq, jp); + break; + case SM_DIFF: + parse_diff(client, params); + break; + case SM_SUBSCRIBE: + parse_subscribe(client, client->id, params); + break; + case SM_SUBSCRIBERESULT: + parse_subscribe_result(client, res_val); + break; + case SM_AUTH: + parse_authorise(client, params, &err_val, &errnum); + break; + case SM_AUTHRESULT: + parse_authorise_result(client, res_val); + break; + default: + break; + } +} + +static void parse_node_msg(ckpool_t *ckp, json_t *val, const char *buf) +{ + int msg_type = node_msg_type(val); + + if (msg_type < 0) { + LOGERR("Missing node method from %s", buf); + return; + } + LOGDEBUG("Got node method %d:%s", msg_type, stratum_msgs[msg_type]); + switch (msg_type) { + case SM_WORKINFO: + add_node_base(ckp, val); + break; + default: + break; + } } /* Entered with client holding ref count */ @@ -5320,7 +5466,10 @@ static void srecv_process(ckpool_t *ckp, char *buf) msg->json_msg = val; val = json_object_get(msg->json_msg, "client_id"); if (unlikely(!val)) { - LOGWARNING("Failed to extract client_id from connector json smsg %s", buf); + if (ckp->node) + parse_node_msg(ckp, msg->json_msg, buf); + else + LOGWARNING("Failed to extract client_id from connector json smsg %s", buf); json_decref(msg->json_msg); free(msg); goto out; @@ -5374,7 +5523,7 @@ static void srecv_process(ckpool_t *ckp, char *buf) LOGINFO("Stratifier added instance %"PRId64" server %d", client->id, server); if (ckp->node) - parse_node_msg(msg->json_msg, buf, client); + node_client_msg(ckp, msg->json_msg, buf, client); else parse_instance_msg(ckp, sdata, msg, client); dec_instance_ref(sdata, client); From b1fea9750170d5f2013dcc390b2dbba26df7a5ea Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Wed, 30 Dec 2015 03:17:28 +1100 Subject: [PATCH 508/544] Basic requirements for node share processing --- src/stratifier.c | 41 +++++++++++++++++++++++------------------ 1 file changed, 23 insertions(+), 18 deletions(-) diff --git a/src/stratifier.c b/src/stratifier.c index 0378b2cf..138c484a 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -819,6 +819,7 @@ static void send_workinfo(ckpool_t *ckp, sdata_t *sdata, const workbase_t *wb) if (sdata->node_instances) { json_t *wb_val = json_object(); + json_set_int(wb_val, "jobid", wb->id); json_set_string(wb_val, "target", wb->target); json_set_double(wb_val, "diff", wb->diff); json_set_int(wb_val, "version", wb->version); @@ -1138,8 +1139,10 @@ static void add_node_base(ckpool_t *ckp, json_t *val) sdata_t *sdata = ckp->data; bool new_block = false; char header[228]; + int i; wb->ckp = ckp; + json_int64cpy(&wb->id, val, "jobid"); json_strcpy(wb->target, val, "target"); json_dblcpy(&wb->diff, val, "diff"); json_uintcpy(&wb->version, val, "version"); @@ -1156,17 +1159,10 @@ static void add_node_base(ckpool_t *ckp, json_t *val) if (wb->transactions) json_strdup(&wb->txn_data, val, "txn_data"); json_intcpy(&wb->merkles, val, "merkles"); - wb->merkle_array = json_array(); - if (wb->merkles) { - json_t *arr; - int i; - - arr = json_object_get(val, "merklehash"); - for (i = 0; i < wb->merkles; i++) { - strcpy(&wb->merklehash[i][0], json_string_value(json_array_get(arr, i))); - hex2bin(&wb->merklebin[i][0], &wb->merklehash[i][0], 32); - json_array_append_new(wb->merkle_array, json_string(&wb->merklehash[i][0])); - } + wb->merkle_array = json_object_dup(val, "merklehash"); + for (i = 0; i < wb->merkles; i++) { + strcpy(&wb->merklehash[i][0], json_string_value(json_array_get(wb->merkle_array, i))); + hex2bin(&wb->merklebin[i][0], &wb->merklehash[i][0], 32); } json_strdup(&wb->coinb1, val, "coinb1"); json_intcpy(&wb->coinb1len, val, "coinb1len"); @@ -1178,6 +1174,7 @@ static void add_node_base(ckpool_t *ckp, json_t *val) hex2bin(wb->coinb2bin, wb->coinb2, wb->coinb2len); json_intcpy(&wb->enonce1varlen, val, "enonce1varlen"); json_intcpy(&wb->enonce2varlen, val, "enonce2varlen"); + ts_realtime(&wb->gentime); snprintf(header, 225, "%s%s%s%s%s%s%s", wb->bbversion, wb->prevhash, @@ -1190,7 +1187,7 @@ static void add_node_base(ckpool_t *ckp, json_t *val) add_base(ckp, sdata, wb, &new_block); if (new_block) - LOGWARNING("Block hash changed to %s", sdata->lastswaphash); + LOGNOTICE("Block hash changed to %s", sdata->lastswaphash); } static void update_base(ckpool_t *ckp, const int prio) @@ -4380,7 +4377,6 @@ static json_t *parse_authorise(stratum_instance_t *client, const json_t *params_ /* We can set this outside of lock safely */ client->authorising = false; out: - LOGWARNING("Parsed %d", ret); return json_boolean(ret); } @@ -4440,6 +4436,9 @@ static void add_submit(ckpool_t *ckp, stratum_instance_t *client, const int diff } else if (!submit) return; + if (ckp->node) + return; + tv_time(&now_t); ck_rlock(&sdata->workbase_lock); @@ -4809,8 +4808,11 @@ static json_t *parse_submit(stratum_instance_t *client, json_t *json_msg, ck_rlock(&sdata->workbase_lock); HASH_FIND_I64(sdata->workbases, &id, wb); if (unlikely(!wb)) { - if (!sdata->current_workbase) + if (!sdata->current_workbase) { + ck_runlock(&sdata->workbase_lock); + return json_boolean(false); + } id = sdata->current_workbase->id; err = SE_INVALID_JOBID; json_set_string(json_msg, "reject-reason", SHARE_ERR(err)); @@ -4895,7 +4897,7 @@ out_unlock: submit = false; } } else - LOGINFO("Rejected client %"PRId64" invalid share", client->id); + LOGINFO("Rejected client %"PRId64" invalid share %s", client->id, SHARE_ERR(err)); /* Submit share to upstream pool in proxy mode. We submit valid and * stale shares and filter out the rest. */ @@ -5296,7 +5298,7 @@ static void parse_diff(stratum_instance_t *client, json_t *val) static void parse_subscribe_result(stratum_instance_t *client, json_t *val) { strncpy(client->enonce1, json_string_value(json_array_get(val, 1)), 16); - LOGDEBUG("Client %"PRId64" got enonce1 %s", client->id, client->enonce1); + LOGINFO("Client %"PRId64" got enonce1 %s", client->id, client->enonce1); sprintf(client->enonce1, "%016lx", client->enonce1_64); } @@ -5351,7 +5353,7 @@ static void node_client_msg(ckpool_t *ckp, json_t *val, const char *buf, stratum LOGERR("Missing client %"PRId64" node method from %s", client->id, buf); return; } - LOGWARNING("Got client %"PRId64" node method %d:%s", client->id, msg_type, stratum_msgs[msg_type]); + LOGDEBUG("Got client %"PRId64" node method %d:%s", client->id, msg_type, stratum_msgs[msg_type]); id_val = json_object_get(val, "id"); method = json_object_get(val, "method"); params = json_object_get(val, "params"); @@ -5589,7 +5591,10 @@ static void sshare_process(ckpool_t *ckp, json_params_t *jp) json_object_set_new_nocheck(json_msg, "result", result_val); json_object_set_new_nocheck(json_msg, "error", err_val ? err_val : json_null()); steal_json_id(json_msg, jp); - stratum_add_send(sdata, json_msg, client_id, SM_SHARERESULT); + if (ckp->node) + json_decref(json_msg); + else + stratum_add_send(sdata, json_msg, client_id, SM_SHARERESULT); out_decref: dec_instance_ref(sdata, client); out: From 340d469a646e6b4d5eb5b664bef019c3d8364fb0 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Wed, 30 Dec 2015 09:14:24 +1100 Subject: [PATCH 509/544] Don't allow the stratifier to send any messages in node mode --- src/stratifier.c | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/src/stratifier.c b/src/stratifier.c index 138c484a..ee9373a9 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -2329,12 +2329,18 @@ static inline bool passthrough_subclient(const int64_t client_id) static void stratum_add_send(sdata_t *sdata, json_t *val, const int64_t client_id, const int msg_type) { - smsg_t *msg = ckzalloc(sizeof(smsg_t)); + smsg_t *msg; ckpool_t *ckp = sdata->ckp; - if (ckp->node || passthrough_subclient(client_id)) + if (ckp->node) { + json_decref(val); + return; + } + + if (passthrough_subclient(client_id)) json_set_string(val, "node.method", stratum_msgs[msg_type]); LOGDEBUG("Sending stratum message %s", stratum_msgs[msg_type]); + msg = ckzalloc(sizeof(smsg_t)); msg->json_msg = val; msg->client_id = client_id; ckmsgq_add(sdata->ssends, msg); @@ -5591,10 +5597,7 @@ static void sshare_process(ckpool_t *ckp, json_params_t *jp) json_object_set_new_nocheck(json_msg, "result", result_val); json_object_set_new_nocheck(json_msg, "error", err_val ? err_val : json_null()); steal_json_id(json_msg, jp); - if (ckp->node) - json_decref(json_msg); - else - stratum_add_send(sdata, json_msg, client_id, SM_SHARERESULT); + stratum_add_send(sdata, json_msg, client_id, SM_SHARERESULT); out_decref: dec_instance_ref(sdata, client); out: @@ -6503,7 +6506,6 @@ int stratifier(proc_instance_t *pi) mutex_init(&sdata->ckdb_lock); mutex_init(&sdata->ckdb_msg_lock); - sdata->ssends = create_ckmsgq(ckp, "ssender", &ssend_process); /* Create half as many share processing threads as there are CPUs */ threads = sysconf(_SC_NPROCESSORS_ONLN) / 2 ? : 1; sdata->sshareq = create_ckmsgqs(ckp, "sprocessor", &sshare_process, threads); @@ -6511,6 +6513,7 @@ int stratifier(proc_instance_t *pi) if (ckp->node) threads = 1; else { + sdata->ssends = create_ckmsgq(ckp, "ssender", &ssend_process); threads = threads / 2 ? : 1; sdata->sauthq = create_ckmsgq(ckp, "authoriser", &sauth_process); sdata->stxnq = create_ckmsgq(ckp, "stxnq", &send_transactions); From 520f9ed2f4076c7ec93c3dd4ce67a28dd746cd68 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Wed, 30 Dec 2015 09:16:28 +1100 Subject: [PATCH 510/544] Comment --- src/stratifier.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/stratifier.c b/src/stratifier.c index ee9373a9..257e9c71 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -2333,6 +2333,8 @@ static void stratum_add_send(sdata_t *sdata, json_t *val, const int64_t client_i ckpool_t *ckp = sdata->ckp; if (ckp->node) { + /* Node shouldn't be sending any messages as it only uses the + * stratifier for monitoring activity. */ json_decref(val); return; } From 231069b2ba775af9829dc0511a2e6049605aca71 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Wed, 30 Dec 2015 09:21:23 +1100 Subject: [PATCH 511/544] Tidy send_node_workinfo --- src/stratifier.c | 49 ++++++++++++++++++++++++++---------------------- 1 file changed, 27 insertions(+), 22 deletions(-) diff --git a/src/stratifier.c b/src/stratifier.c index 257e9c71..9aa523d9 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -789,31 +789,10 @@ static void _ckdbq_add(ckpool_t *ckp, const int idtype, json_t *val, const char static void stratum_add_send(sdata_t *sdata, json_t *val, const int64_t client_id, const int msg_type); -static void send_workinfo(ckpool_t *ckp, sdata_t *sdata, const workbase_t *wb) +static void send_node_workinfo(sdata_t *sdata, const workbase_t *wb) { stratum_instance_t *client; ckmsg_t *bulk_send = NULL; - char cdfield[64]; - json_t *val; - - sprintf(cdfield, "%lu,%lu", wb->gentime.tv_sec, wb->gentime.tv_nsec); - - JSON_CPACK(val, "{sI,ss,ss,ss,ss,ss,ss,ss,ss,sI,so,ss,ss,ss,ss}", - "workinfoid", wb->id, - "poolinstance", ckp->name, - "transactiontree", wb->txn_hashes, - "prevhash", wb->prevhash, - "coinbase1", wb->coinb1, - "coinbase2", wb->coinb2, - "version", wb->bbversion, - "ntime", wb->ntime, - "bits", wb->nbit, - "reward", wb->coinbasevalue, - "merklehash", json_deep_copy(wb->merkle_array), - "createdate", cdfield, - "createby", "code", - "createcode", __func__, - "createinet", ckp->serverurl[0]); ck_rlock(&sdata->instance_lock); if (sdata->node_instances) { @@ -871,7 +850,33 @@ static void send_workinfo(ckpool_t *ckp, sdata_t *sdata, const workbase_t *wb) pthread_cond_signal(ssends->cond); mutex_unlock(ssends->lock); } +} + +static void send_workinfo(ckpool_t *ckp, sdata_t *sdata, const workbase_t *wb) +{ + char cdfield[64]; + json_t *val; + + sprintf(cdfield, "%lu,%lu", wb->gentime.tv_sec, wb->gentime.tv_nsec); + + JSON_CPACK(val, "{sI,ss,ss,ss,ss,ss,ss,ss,ss,sI,so,ss,ss,ss,ss}", + "workinfoid", wb->id, + "poolinstance", ckp->name, + "transactiontree", wb->txn_hashes, + "prevhash", wb->prevhash, + "coinbase1", wb->coinb1, + "coinbase2", wb->coinb2, + "version", wb->bbversion, + "ntime", wb->ntime, + "bits", wb->nbit, + "reward", wb->coinbasevalue, + "merklehash", json_deep_copy(wb->merkle_array), + "createdate", cdfield, + "createby", "code", + "createcode", __func__, + "createinet", ckp->serverurl[0]); ckdbq_add(ckp, ID_WORKINFO, val); + send_node_workinfo(sdata, wb); } static void send_ageworkinfo(ckpool_t *ckp, const int64_t id) From 516b851a468e3cb6801375d27f93bcd62925c7e9 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Wed, 30 Dec 2015 10:47:31 +1100 Subject: [PATCH 512/544] Fix enonce1 parsing for correct share identification --- src/stratifier.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/stratifier.c b/src/stratifier.c index 9aa523d9..b0e32b63 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -5310,9 +5310,13 @@ static void parse_diff(stratum_instance_t *client, json_t *val) static void parse_subscribe_result(stratum_instance_t *client, json_t *val) { + int len; + strncpy(client->enonce1, json_string_value(json_array_get(val, 1)), 16); - LOGINFO("Client %"PRId64" got enonce1 %s", client->id, client->enonce1); - sprintf(client->enonce1, "%016lx", client->enonce1_64); + len = strlen(client->enonce1) / 2; + hex2bin(client->enonce1bin, client->enonce1, len); + memcpy(&client->enonce1_64, client->enonce1bin, 8); + LOGINFO("Client %"PRId64" got enonce1 %lx string %s", client->id, client->enonce1_64, client->enonce1); } static void parse_authorise_result(stratum_instance_t *client, json_t *val) From b479043ff327979b123717480fbc51c3f89fa69e Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Wed, 30 Dec 2015 10:53:01 +1100 Subject: [PATCH 513/544] Node doesn't need a valid btcaddress --- src/generator.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/generator.c b/src/generator.c index 370f9b28..a7e6071b 100644 --- a/src/generator.c +++ b/src/generator.c @@ -212,7 +212,7 @@ static bool server_alive(ckpool_t *ckp, server_instance_t *si, bool pinging) goto out; } clear_gbtbase(gbt); - if (!validate_address(cs, ckp->btcaddress)) { + if (!ckp->node && !validate_address(cs, ckp->btcaddress)) { LOGWARNING("Invalid btcaddress: %s !", ckp->btcaddress); goto out; } From 135d2b84e741c0f9133496749886b53f68517ff7 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Wed, 30 Dec 2015 11:09:07 +1100 Subject: [PATCH 514/544] Submit blocks locally in node mode --- src/generator.c | 10 ++++++++-- src/stratifier.c | 10 ++++++++-- 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/src/generator.c b/src/generator.c index a7e6071b..1f832ab8 100644 --- a/src/generator.c +++ b/src/generator.c @@ -2746,6 +2746,7 @@ static int proxy_loop(proc_instance_t *pi) ckpool_t *ckp = pi->ckp; gdata_t *gdata = ckp->data; unix_msg_t *umsg = NULL; + connsock_t *cs = NULL; bool started = false; char *buf = NULL; int ret = 0; @@ -2754,8 +2755,6 @@ reconnect: clear_unix_msg(&umsg); if (ckp->node) { - connsock_t *cs; - old_si = si; si = live_server(ckp); if (!si) @@ -2831,7 +2830,14 @@ retry: } else if (cmdmatch(buf, "reconnect")) { goto reconnect; } else if (cmdmatch(buf, "submitblock:")) { + char blockmsg[80]; + bool ret; + LOGNOTICE("Submitting likely block solve share to upstream pool"); + ret = submit_block(cs, buf + 12 + 64 + 1); + memset(buf + 12 + 64, 0, 1); + sprintf(blockmsg, "%sblock:%s", ret ? "" : "no", buf + 12); + send_proc(ckp->stratifier, blockmsg); } else if (cmdmatch(buf, "loglevel")) { sscanf(buf, "loglevel=%d", &ckp->loglevel); } else if (cmdmatch(buf, "ping")) { diff --git a/src/stratifier.c b/src/stratifier.c index b0e32b63..66b9bd33 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -2270,6 +2270,11 @@ static void stratum_broadcast(sdata_t *sdata, json_t *val) return; } + if (ckp->node) { + json_decref(val); + return; + } + /* Use this locking as an opportunity to test other clients. */ ck_rlock(&ckp_sdata->instance_lock); HASH_ITER(hh, ckp_sdata->stratum_instances, client, tmp) { @@ -2523,7 +2528,8 @@ static void block_solve(ckpool_t *ckp, const char *blockhash) ts_t ts_now; json_t *val; - update_base(ckp, GEN_PRIORITY); + if (!ckp->node) + update_base(ckp, GEN_PRIORITY); ts_realtime(&ts_now); sprintf(cdfield, "%lu,%lu", ts_now.tv_sec, ts_now.tv_nsec); @@ -4569,7 +4575,7 @@ test_blocksolve(const stratum_instance_t *client, const workbase_t *wb, const uc LOGWARNING("Possible block solve diff %f !", diff); /* Can't submit a block in proxy mode without the transactions */ - if (wb->proxy && wb->merkles) + if (!ckp->node && wb->proxy) return; ts_realtime(&ts_now); From d3ca56e4972e54e8b4c8eb0b5513d5650d9251df Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Wed, 30 Dec 2015 11:19:50 +1100 Subject: [PATCH 515/544] Handle missing stratum message types on broadcasts --- src/ckpool.h | 2 ++ src/stratifier.c | 24 +++++++++++++----------- 2 files changed, 15 insertions(+), 11 deletions(-) diff --git a/src/ckpool.h b/src/ckpool.h index 5c4529c0..0829d36c 100644 --- a/src/ckpool.h +++ b/src/ckpool.h @@ -257,6 +257,7 @@ enum stratum_msgtype { SM_AUTHRESULT, SM_TXNS, SM_TXNSRESULT, + SM_PING, SM_WORKINFO, SM_NONE }; @@ -275,6 +276,7 @@ static const char __maybe_unused *stratum_msgs[] = { "auth.result", "txns", "txns.result", + "ping", "workinfo", "" }; diff --git a/src/stratifier.c b/src/stratifier.c index 66b9bd33..e87fc7e4 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -2253,10 +2253,16 @@ static void connector_test_client(ckpool_t *ckp, const int64_t id) send_proc(ckp->connector, buf); } +/* passthrough subclients have client_ids in the high bits */ +static inline bool passthrough_subclient(const int64_t client_id) +{ + return (client_id > 0xffffffffll); +} + /* For creating a list of sends without locking that can then be concatenated * to the stratum_sends list. Minimises locking and avoids taking recursive * locks. Sends only to sdata bound clients (everyone in ckpool) */ -static void stratum_broadcast(sdata_t *sdata, json_t *val) +static void stratum_broadcast(sdata_t *sdata, json_t *val, const int msg_type) { ckpool_t *ckp = sdata->ckp; sdata_t *ckp_sdata = ckp->data; @@ -2307,6 +2313,8 @@ static void stratum_broadcast(sdata_t *sdata, json_t *val) continue; client_msg = ckalloc(sizeof(ckmsg_t)); msg = ckzalloc(sizeof(smsg_t)); + if (passthrough_subclient(client->id)) + json_set_string(val, "node.method", stratum_msgs[msg_type]); msg->json_msg = json_deep_copy(val); msg->client_id = client->id; client_msg->data = msg; @@ -2330,12 +2338,6 @@ static void stratum_broadcast(sdata_t *sdata, json_t *val) mutex_unlock(ssends->lock); } -/* passthrough subclients have client_ids in the high bits */ -static inline bool passthrough_subclient(const int64_t client_id) -{ - return (client_id > 0xffffffffll); -} - static void stratum_add_send(sdata_t *sdata, json_t *val, const int64_t client_id, const int msg_type) { @@ -2390,7 +2392,7 @@ static void stratum_broadcast_message(sdata_t *sdata, const char *msg) JSON_CPACK(json_msg, "{sosss[s]}", "id", json_null(), "method", "client.show_message", "params", msg); - stratum_broadcast(sdata, json_msg); + stratum_broadcast(sdata, json_msg, SM_MSG); } /* Send a generic reconnect to all clients without parameters to make them @@ -2410,7 +2412,7 @@ static void request_reconnect(sdata_t *sdata, const char *cmd) } else JSON_CPACK(json_msg, "{sosss[]}", "id", json_null(), "method", "client.reconnect", "params"); - stratum_broadcast(sdata, json_msg); + stratum_broadcast(sdata, json_msg, SM_RECONNECT); /* Tag all existing clients as dropped now so they can be removed * lazily */ @@ -2655,7 +2657,7 @@ static void broadcast_ping(sdata_t *sdata) "id", 42, "method", "mining.ping"); - stratum_broadcast(sdata, json_msg); + stratum_broadcast(sdata, json_msg, SM_PING); } static void ckmsgq_stats(ckmsgq_t *ckmsgq, const int size, json_t **val) @@ -5037,7 +5039,7 @@ static void stratum_broadcast_update(sdata_t *sdata, const workbase_t *wb, const json_msg = __stratum_notify(wb, clean); ck_runlock(&sdata->workbase_lock); - stratum_broadcast(sdata, json_msg); + stratum_broadcast(sdata, json_msg, SM_UPDATE); } /* For sending a single stratum template update */ From 46aef425f2c1175c8123ce4395f8c9129e19bdd3 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Wed, 30 Dec 2015 11:45:00 +1100 Subject: [PATCH 516/544] Drop clients when we don't have an upstream pool in passthrough/node mode --- src/connector.c | 49 +++++++++++++++++++++++++++++++++++-------------- src/generator.c | 2 +- 2 files changed, 36 insertions(+), 15 deletions(-) diff --git a/src/connector.c b/src/connector.c index 3ce2bb2d..f78cb9c8 100644 --- a/src/connector.c +++ b/src/connector.c @@ -298,25 +298,34 @@ static int accept_client(cdata_t *cdata, const int epfd, const uint64_t server) return 1; } +static int __drop_client(cdata_t *cdata, client_instance_t *client) +{ + int ret = -1; + + if (client->invalid) + goto out; + client->invalid = true; + ret = client->fd; + Close(client->fd); + epoll_ctl(cdata->epfd, EPOLL_CTL_DEL, ret, NULL); + HASH_DEL(cdata->clients, client); + DL_APPEND(cdata->dead_clients, client); + /* This is the reference to this client's presence in the + * epoll list. */ + __dec_instance_ref(client); + cdata->dead_generated++; +out: + return ret; +} + /* Client must hold a reference count */ static int drop_client(cdata_t *cdata, client_instance_t *client) { - int64_t client_id = 0; + int64_t client_id = client->id; int fd = -1; ck_wlock(&cdata->lock); - if (!client->invalid) { - client->invalid = true; - client_id = client->id; - fd = client->fd; - epoll_ctl(cdata->epfd, EPOLL_CTL_DEL, fd, NULL); - HASH_DEL(cdata->clients, client); - DL_APPEND(cdata->dead_clients, client); - /* This is the reference to this client's presence in the - * epoll list. */ - __dec_instance_ref(client); - cdata->dead_generated++; - } + fd = __drop_client(cdata, client); ck_wunlock(&cdata->lock); if (fd > -1) @@ -388,6 +397,17 @@ static int invalidate_client(ckpool_t *ckp, cdata_t *cdata, client_instance_t *c return ret; } +static void drop_all_clients(cdata_t *cdata) +{ + client_instance_t *client, *tmp; + + ck_wlock(&cdata->lock); + HASH_ITER(hh, cdata->clients, client, tmp) { + __drop_client(cdata, client); + } + ck_wunlock(&cdata->lock); +} + static void send_client(cdata_t *cdata, int64_t id, char *buf); /* Look for shares being submitted via a redirector and add them to a linked @@ -1116,7 +1136,8 @@ retry: } else if (cmdmatch(buf, "reject")) { LOGDEBUG("Connector received reject signal"); cdata->accept = false; - send_proc(ckp->stratifier, "dropall"); + if (ckp->passthrough) + drop_all_clients(cdata); } else if (cmdmatch(buf, "stats")) { char *msg; diff --git a/src/generator.c b/src/generator.c index 1f832ab8..3680535a 100644 --- a/src/generator.c +++ b/src/generator.c @@ -2302,7 +2302,7 @@ static proxy_instance_t *wait_best_proxy(ckpool_t *ckp, gdata_t *gdata) break; /* Send reject message if we are unable to find an active * proxy for more than 5 seconds */ - if (!((++retries) % 5)) + if (!((retries++) % 5)) send_proc(ckp->connector, "reject"); sleep(1); } From 9c41763a71ac3fefe4a2c4f652b8f21236c15150 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Wed, 30 Dec 2015 12:11:59 +1100 Subject: [PATCH 517/544] Document node mode --- README | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/README b/README index 5e48dac7..8fd75b81 100644 --- a/README +++ b/README @@ -172,9 +172,14 @@ and then workbase. -l will change the ckpool process name to that specified, allowing multiple different named instances to be running. By default the variant -names are used: ckpool, ckproxy, ckpassthrough, ckredirector. +names are used: ckpool, ckproxy, ckpassthrough, ckredirector, cknode. -P will start ckpool in passthrough proxy mode where it collates all incoming connections and streams all information on a single connection to an upstream From 6b3792c090e90ba2a1353e8161a862877c758233 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Wed, 30 Dec 2015 14:07:32 +1100 Subject: [PATCH 518/544] Bump version to 0.9.0 --- configure.ac | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configure.ac b/configure.ac index 468326e1..6097d1a4 100644 --- a/configure.ac +++ b/configure.ac @@ -1,4 +1,4 @@ -AC_INIT(ckpool, 0.8.8, kernel@kolivas.org) +AC_INIT(ckpool, 0.9.0, kernel@kolivas.org) AC_CANONICAL_SYSTEM AC_CONFIG_MACRO_DIR([m4]) From 950b855b2d799a06688344adc0f90948db8f9cce Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Fri, 1 Jan 2016 16:32:58 +1100 Subject: [PATCH 519/544] Implement backwardly compatible lz4 compression support to be used by passthroughs and nodes --- configure.ac | 19 ++--- src/Makefile.am | 4 +- src/ckpool.c | 202 +++++++++++++++++++++++++++++++++++++++++------- src/ckpool.h | 4 + src/connector.c | 60 +++++++++++++- src/generator.c | 14 +++- 6 files changed, 259 insertions(+), 44 deletions(-) diff --git a/configure.ac b/configure.ac index 6097d1a4..a805be4d 100644 --- a/configure.ac +++ b/configure.ac @@ -40,17 +40,11 @@ AC_CHECK_HEADERS(stdint.h netinet/in.h netinet/tcp.h sys/ioctl.h getopt.h) AC_CHECK_HEADERS(sys/epoll.h libpq-fe.h postgresql/libpq-fe.h grp.h) AC_CHECK_HEADERS(gsl/gsl_math.h gsl/gsl_cdf.h) AC_CHECK_HEADERS(openssl/x509.h openssl/hmac.h) - -PTHREAD_LIBS="-lpthread" -MATH_LIBS="-lm" -RT_LIBS="-lrt" +AC_CHECK_HEADERS(lz4.h) AC_CONFIG_SUBDIRS([src/jansson-2.6]) JANSSON_LIBS="jansson-2.6/src/.libs/libjansson.a" -AC_SUBST(PTHREAD_LIBS) -AC_SUBST(MATH_LIBS) -AC_SUBST(RT_LIBS) AC_SUBST(JANSSON_LIBS) AC_ARG_WITH([ckdb], @@ -58,6 +52,11 @@ AC_ARG_WITH([ckdb], [ckdb=$withval] ) +#AC_SEARCH_LIBS(whatgoeshere?, rt, , echo "Error: Required library realtime not found." && exit 1) +AC_SEARCH_LIBS(exp, m, , echo "Error: Required library math not found." && exit 1) +AC_SEARCH_LIBS(LZ4_compress, lz4, , echo "Error: Required library liblz4-dev not found." && exit 1) +AC_SEARCH_LIBS(pthread_create, pthread, , "Error: Required library pthreads not found." && exit 1) + if test "x$ckdb" != "xno"; then AC_CHECK_LIB([pq], [main],[PQ=-lpq],echo "Error: Required library libpq-dev not found. Install it or disable postgresql support with --without-ckdb" && exit 1) @@ -84,8 +83,10 @@ echo "Compilation............: make (or gmake)" echo " CPPFLAGS.............: $CPPFLAGS" echo " CFLAGS...............: $CFLAGS" echo " LDFLAGS..............: $LDFLAGS" -echo " LDADD................: $PTHREAD_LIBS $MATH_LIBS $RT_LIBS $JANSSON_LIBS" -echo " db LDADD.............: $DB_LIBS" +echo " LDADD................: $LIBS $JANSSON_LIBS" +if test "x$ckdb" != "xno"; then + echo " db LDADD.............: $LIBS $DB_LIBS $JANSSON_LIBS" +fi echo echo "Installation...........: make install (as root if needed, with 'su' or 'sudo')" echo " prefix...............: $prefix" diff --git a/src/Makefile.am b/src/Makefile.am index e95666ac..53561714 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -5,7 +5,7 @@ AM_CPPFLAGS = -I$(top_srcdir)/src/jansson-2.6/src lib_LTLIBRARIES = libckpool.la libckpool_la_SOURCES = libckpool.c libckpool.h sha2.c sha2.h -libckpool_la_LIBADD = @PTHREAD_LIBS@ @MATH_LIBS@ @RT_LIBS@ +libckpool_la_LIBADD = @LIBS@ bin_PROGRAMS = ckpool ckpmsg notifier ckpool_SOURCES = ckpool.c ckpool.h generator.c generator.h bitcoin.c bitcoin.h \ @@ -23,5 +23,5 @@ if WANT_CKDB bin_PROGRAMS += ckdb ckdb_SOURCES = ckdb.c ckdb_cmd.c ckdb_data.c ckdb_dbio.c ckdb_btc.c \ ckdb_crypt.c ckdb.h klist.c ktree.c klist.h ktree.h -ckdb_LDADD = libckpool.la @JANSSON_LIBS@ @DB_LIBS@ @MATH_LIBS@ +ckdb_LDADD = libckpool.la @JANSSON_LIBS@ @DB_LIBS@ @LIBS@ endif diff --git a/src/ckpool.c b/src/ckpool.c index 15a9a5bf..a5a41108 100644 --- a/src/ckpool.c +++ b/src/ckpool.c @@ -20,6 +20,7 @@ #include #include #include +#include #include #include #include @@ -510,6 +511,129 @@ void empty_buffer(connsock_t *cs) cs->buflen = cs->bufofs = 0; } +static void clear_bufline(connsock_t *cs) +{ + if (unlikely(!cs->buf)) + cs->buf = ckzalloc(PAGESIZE); + else if (cs->buflen) { + memmove(cs->buf, cs->buf + cs->bufofs, cs->buflen); + memset(cs->buf + cs->buflen, 0, cs->bufofs); + cs->bufofs = cs->buflen; + cs->buflen = 0; + cs->buf[cs->bufofs] = '\0'; + } +} + +static void add_bufline(connsock_t *cs, const char *readbuf, const int len) +{ + int backoff = 1; + size_t buflen; + char *newbuf; + + buflen = cs->bufofs + len + 1; + while (42) { + newbuf = realloc(cs->buf, buflen); + if (likely(newbuf)) + break; + if (backoff == 1) + fprintf(stderr, "Failed to realloc %d in read_socket_line, retrying\n", (int)buflen); + cksleep_ms(backoff); + backoff <<= 1; + } + cs->buf = newbuf; + if (unlikely(!cs->buf)) + quit(1, "Failed to alloc buf of %d bytes in read_socket_line", (int)buflen); + memcpy(cs->buf + cs->bufofs, readbuf, len); + cs->bufofs += len; + cs->buf[cs->bufofs] = '\0'; +} + +static int read_cs_length(connsock_t *cs, float *timeout, int len) +{ + int ret = len; + + while (cs->buflen < len) { + char readbuf[PAGESIZE] = {}; + + ret = wait_read_select(cs->fd, *timeout); + if (ret < 1) + goto out; + ret = recv(cs->fd, readbuf, len - cs->buflen, MSG_DONTWAIT); + if (ret < 1) + goto out; + add_bufline(cs, readbuf, ret); + } +out: + return ret; +} + +static int read_lz4_line(connsock_t *cs, float *timeout) +{ + int compsize, decompsize, ret, buflen; + char *buf, *dest = NULL, *eom; + uint32_t msglen; + + /* Remove lz4 header */ + clear_bufline(cs); + + /* Get data sizes */ + ret = read_cs_length(cs, timeout, 8); + if (ret != 8) { + ret = -1; + goto out; + } + memcpy(&msglen, cs->buf, 4); + compsize = le32toh(msglen); + memcpy(&msglen, cs->buf + 4, 4); + decompsize = le32toh(msglen); + cs->bufofs = 8; + clear_bufline(cs); + LOGWARNING("Trying to read lz4 %d/%d", compsize, decompsize); + + if (unlikely(compsize < 1 || compsize > (int)0x80000000 || + decompsize < 1 || decompsize > (int)0x80000000)) { + LOGWARNING("Invalid message length comp %d decomp %d sent to read_lz4_line", compsize, decompsize); + ret = -1; + goto out; + } + + /* Get compressed data */ + ret = read_cs_length(cs, timeout, compsize); + if (ret != compsize) { + LOGWARNING("Failed to read %d compressed bytes in read_lz4_line, got %d", compsize, ret); + ret = -1; + goto out; + } + /* Do decompresion and buffer reconstruction here */ + dest = ckalloc(decompsize); + ret = LZ4_decompress_safe(cs->buf, dest, compsize, decompsize); + /* Clear out all the compressed data */ + clear_bufline(cs); + if (ret != decompsize) { + LOGWARNING("Failed to decompress %d bytes in read_lz4_line, got %d", decompsize, ret); + ret = -1; + goto out; + } + eom = dest + decompsize - 1; + if (memcmp(eom, "\n", 1)) { + LOGWARNING("Failed to find EOM in decompressed data in read_lz4_line"); + ret = -1; + goto out; + } + *eom = '\0'; + /* Wedge the decompressed buffer back to the start of cs->buf */ + buf = cs->buf; + buflen = cs->buflen; + cs->buf = dest; + dest = NULL; + ret = cs->buflen = decompsize - 1; + if (buflen) + add_bufline(cs, buf, buflen); +out: + free(dest); + return ret; +} + /* Read from a socket into cs->buf till we get an '\n', converting it to '\0' * and storing how much extra data we've received, to be moved to the beginning * of the buffer for use on the next receive. */ @@ -517,7 +641,6 @@ int read_socket_line(connsock_t *cs, float *timeout) { char *eom = NULL; tv_t start, now; - size_t buflen; int ret = -1; bool polled; float diff; @@ -525,16 +648,8 @@ int read_socket_line(connsock_t *cs, float *timeout) if (unlikely(cs->fd < 0)) goto out; - if (unlikely(!cs->buf)) - cs->buf = ckzalloc(PAGESIZE); - else if (cs->buflen) { - memmove(cs->buf, cs->buf + cs->bufofs, cs->buflen); - memset(cs->buf + cs->buflen, 0, cs->bufofs); - cs->bufofs = cs->buflen; - cs->buflen = 0; - cs->buf[cs->bufofs] = '\0'; - eom = strchr(cs->buf, '\n'); - } + clear_bufline(cs); + eom = strchr(cs->buf, '\n'); tv_time(&start); rewait: @@ -564,8 +679,6 @@ rewait: *timeout -= diff; while (42) { char readbuf[PAGESIZE] = {}; - int backoff = 1; - char *newbuf; ret = recv(cs->fd, readbuf, PAGESIZE - 4, MSG_DONTWAIT); if (ret < 1) { @@ -585,22 +698,7 @@ rewait: goto out; } polled = false; - buflen = cs->bufofs + ret + 1; - while (42) { - newbuf = realloc(cs->buf, buflen); - if (likely(newbuf)) - break; - if (backoff == 1) - fprintf(stderr, "Failed to realloc %d in read_socket_line, retrying\n", (int)buflen); - cksleep_ms(backoff); - backoff <<= 1; - } - cs->buf = newbuf; - if (unlikely(!cs->buf)) - quit(1, "Failed to alloc buf of %d bytes in read_socket_line", (int)buflen); - memcpy(cs->buf + cs->bufofs, readbuf, ret); - cs->bufofs += ret; - cs->buf[cs->bufofs] = '\0'; + add_bufline(cs, readbuf, ret); eom = strchr(cs->buf, '\n'); } parse: @@ -616,7 +714,53 @@ out: if (ret < 0) { empty_buffer(cs); dealloc(cs->buf); + } else if (cs->buflen && !strncmp(cs->buf, "lz4", 3)) + ret = read_lz4_line(cs, timeout); + return ret; +} + +/* Lz4 compressed block structure: + * - 4 byte magic header LZ4\n + * - 4 byte LE encoded compressed size + * - 4 byte LE encoded decompressed size + */ +int write_cs(connsock_t *cs, const char *buf, int len) +{ + int compsize, decompsize = len; + char *dest = NULL; + uint32_t msglen; + int ret = -1; + + /* Connsock doesn't expect lz4 compressed messages */ + if (!cs->lz4) { + ret = write_socket(cs->fd, buf, len); + goto out; } + dest = ckalloc(len + 12); + /* Do compression here */ + compsize = LZ4_compress(buf, dest + 12, len); + if (!compsize) { + LOGWARNING("Failed to LZ4 compress in write_cs, writing uncompressed"); + ret = write_socket(cs->fd, buf, len); + goto out; + } + LOGDEBUG("Writing connsock message compressed %d from %d", compsize, decompsize); + /* Copy lz4 magic header */ + sprintf(dest, "lz4\n"); + /* Copy compressed message length */ + msglen = htole32(compsize); + memcpy(dest + 4, &msglen, 4); + /* Copy decompressed message length */ + msglen = htole32(decompsize); + memcpy(dest + 8, &msglen, 4); + len = compsize + 12; + ret = write_socket(cs->fd, dest, len); + if (ret == len) + ret = decompsize; + else + ret = -1; +out: + free(dest); return ret; } diff --git a/src/ckpool.h b/src/ckpool.h index 0829d36c..9f2f599a 100644 --- a/src/ckpool.h +++ b/src/ckpool.h @@ -82,6 +82,9 @@ struct connsock { ckpool_t *ckp; /* Semaphore used to serialise request/responses */ sem_t sem; + + /* Has the other end acknowledged it can receive lz4 compressed data */ + bool lz4; }; typedef struct connsock connsock_t; @@ -301,6 +304,7 @@ ckpool_t *global_ckp; bool ping_main(ckpool_t *ckp); void empty_buffer(connsock_t *cs); int read_socket_line(connsock_t *cs, float *timeout); +int write_cs(connsock_t *cs, const char *buf, int len); void _send_proc(proc_instance_t *pi, const char *msg, const char *file, const char *func, const int line); #define send_proc(pi, msg) _send_proc(pi, msg, __FILE__, __func__, __LINE__) char *_send_recv_proc(proc_instance_t *pi, const char *msg, int writetimeout, int readtimedout, diff --git a/src/connector.c b/src/connector.c index f78cb9c8..f2858a59 100644 --- a/src/connector.c +++ b/src/connector.c @@ -10,6 +10,7 @@ #include "config.h" #include +#include #include #include #include @@ -63,6 +64,12 @@ struct client_instance { /* Is this the parent passthrough client */ bool passthrough; + /* Does this client expect lz4 compression? */ + bool lz4; + bool compressed; /* Currently receiving a compressed message */ + int compsize; /* Expected compressed data size */ + int decompsize; /* Expected decompressed data size */ + /* Linked list of shares in redirector mode.*/ share_t *shares; @@ -478,6 +485,28 @@ retry: return; } client->bufofs += ret; +compressed: + if (client->compressed) { + if (client->bufofs < client->compsize) + goto retry; + ret = LZ4_decompress_safe(client->buf, msg, client->compsize, client->decompsize); + if (ret != client->decompsize) { + LOGNOTICE("Failed to decompress %d from %d bytes in parse_client_msg, got %d", + client->decompsize, client->compsize, ret); + invalidate_client(ckp, cdata, client); + return; + } + LOGDEBUG("Received client message compressed %d from %d", + client->compsize, client->decompsize); + msg[ret] = '\0'; + /* Flag this client as able to receive lz4 compressed data now */ + client->lz4 = true; + client->bufofs -= client->compsize; + if (client->bufofs) + memmove(client->buf, client->buf + buflen, client->bufofs); + client->compressed = false; + goto parse; + } reparse: eol = memchr(client->buf, '\n', client->bufofs); if (!eol) @@ -490,11 +519,40 @@ reparse: invalidate_client(ckp, cdata, client); return; } + + /* Look for a compression header */ + if (!strncmp(client->buf, "lz4", 3)) { + uint32_t msglen; + + /* Do we have the whole header? If not, keep reading */ + if (client->bufofs < 12) + goto retry; + memcpy(&msglen, client->buf + 4, 4); + client->compsize = le32toh(msglen); + memcpy(&msglen, client->buf + 8, 4); + client->decompsize = le32toh(msglen); + if (unlikely(!client->compsize || !client->decompsize || + client->compsize > MAX_MSGSIZE || client->decompsize > MAX_MSGSIZE)) { + LOGNOTICE("Client id %"PRId64" invalid compressed message size %d/%d, disconnecting", + client->id, client->compsize, client->decompsize); + invalidate_client(ckp, cdata, client); + return; + } + client->bufofs -= 12; + if (client->bufofs > 0) + memmove(client->buf, client->buf + 12, client->bufofs); + client->compressed = true; + if (client->bufofs >= client->compsize) + goto compressed; + goto retry; + } + memcpy(msg, client->buf, buflen); msg[buflen] = '\0'; client->bufofs -= buflen; memmove(client->buf, client->buf + buflen, client->bufofs); client->buf[client->bufofs] = '\0'; +parse: if (!(val = json_loads(msg, 0, NULL))) { char *buf = strdup("Invalid JSON, disconnecting\n"); @@ -971,7 +1029,7 @@ static void passthrough_client(cdata_t *cdata, client_instance_t *client) LOGINFO("Connector adding passthrough client %"PRId64, client->id); client->passthrough = true; - ASPRINTF(&buf, "{\"result\": true}\n"); + ASPRINTF(&buf, "{\"result\": true, \"lz4\": true}\n"); send_client(cdata, client->id, buf); } diff --git a/src/generator.c b/src/generator.c index 3680535a..e9d69ef4 100644 --- a/src/generator.c +++ b/src/generator.c @@ -772,8 +772,9 @@ static bool passthrough_stratum(connsock_t *cs, proxy_instance_t *proxi) bool res, ret = false; float timeout = 10; - JSON_CPACK(req, "{s:s,s:[s]}", + JSON_CPACK(req, "{ss,sb,s[s]}", "method", "mining.passthrough", + "lz4", json_true(), "params", PACKAGE"/"VERSION); res = send_json_msg(cs, req); json_decref(req); @@ -798,6 +799,9 @@ static bool passthrough_stratum(connsock_t *cs, proxy_instance_t *proxi) LOGWARNING("Denied passthrough for stratum"); goto out; } + json_get_bool(&cs->lz4, val, "lz4"); + if (cs->lz4) + LOGNOTICE("Negotiated lz4 compression with pool"); proxi->passthrough = true; out: if (val) @@ -814,8 +818,9 @@ static bool node_stratum(connsock_t *cs, proxy_instance_t *proxi) bool res, ret = false; float timeout = 10; - JSON_CPACK(req, "{s:s,s:[s]}", + JSON_CPACK(req, "{ss,sb,s[s]}", "method", "mining.node", + "lz4", json_true(), "params", PACKAGE"/"VERSION); res = send_json_msg(cs, req); @@ -841,6 +846,9 @@ static bool node_stratum(connsock_t *cs, proxy_instance_t *proxi) LOGWARNING("Denied node setup for stratum"); goto out; } + json_get_bool(&cs->lz4, val, "lz4"); + if (cs->lz4) + LOGNOTICE("Negotiated lz4 compression with pool"); proxi->node = true; out: if (val) @@ -1795,7 +1803,7 @@ static void passthrough_send(ckpool_t *ckp, pass_msg_t *pm) LOGDEBUG("Sending upstream json msg: %s", pm->msg); len = strlen(pm->msg); - sent = write_socket(cs->fd, pm->msg, len); + sent = write_cs(cs, pm->msg, len); if (unlikely(sent != len)) { LOGWARNING("Failed to passthrough %d bytes of message %s, attempting reconnect", len, pm->msg); From 4a52065a55b119dc20b106165aeb141dd1fcfa43 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Fri, 1 Jan 2016 16:35:28 +1100 Subject: [PATCH 520/544] Only send packets compressed if they're smaller --- src/ckpool.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/ckpool.c b/src/ckpool.c index a5a41108..35839db8 100644 --- a/src/ckpool.c +++ b/src/ckpool.c @@ -744,6 +744,12 @@ int write_cs(connsock_t *cs, const char *buf, int len) ret = write_socket(cs->fd, buf, len); goto out; } + if (compsize + 12 >= len) { + /* Selectively send compressed packets only when they're + * smaller. */ + ret = write_socket(cs->fd, buf, len); + goto out; + } LOGDEBUG("Writing connsock message compressed %d from %d", compsize, decompsize); /* Copy lz4 magic header */ sprintf(dest, "lz4\n"); From 17cc4411afa7b2b2c3a163bf4725b56e4e04f117 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Fri, 1 Jan 2016 17:07:19 +1100 Subject: [PATCH 521/544] Fix compilation and send lz4 compatible clients compressed data --- configure.ac | 2 +- src/connector.c | 31 +++++++++++++++++++++++++++++++ 2 files changed, 32 insertions(+), 1 deletion(-) diff --git a/configure.ac b/configure.ac index a805be4d..b99bb405 100644 --- a/configure.ac +++ b/configure.ac @@ -55,7 +55,7 @@ AC_ARG_WITH([ckdb], #AC_SEARCH_LIBS(whatgoeshere?, rt, , echo "Error: Required library realtime not found." && exit 1) AC_SEARCH_LIBS(exp, m, , echo "Error: Required library math not found." && exit 1) AC_SEARCH_LIBS(LZ4_compress, lz4, , echo "Error: Required library liblz4-dev not found." && exit 1) -AC_SEARCH_LIBS(pthread_create, pthread, , "Error: Required library pthreads not found." && exit 1) +AC_SEARCH_LIBS(pthread_mutex_trylock, pthread, , "Error: Required library pthreads not found." && exit 1) if test "x$ckdb" != "xno"; then AC_CHECK_LIB([pq], [main],[PQ=-lpq],echo "Error: Required library libpq-dev diff --git a/src/connector.c b/src/connector.c index f2858a59..e550c9f8 100644 --- a/src/connector.c +++ b/src/connector.c @@ -1000,6 +1000,37 @@ static void send_client(cdata_t *cdata, const int64_t id, char *buf) test_redirector_shares(ckp, client, buf); } + /* Does this client accept compressed data? */ + if (client->lz4) { + char *dest = ckalloc(len + 12); + uint32_t msglen; + int compsize; + + compsize = LZ4_compress(buf, dest + 12, len); + if (unlikely(!compsize)) { + LOGWARNING("Failed to LZ4 compress in send_client, sending uncompressed"); + free(dest); + goto out; + } + if (compsize + 12 >= len) { + /* Only end it compressed if it's smaller */ + free(dest); + goto out; + } + LOGDEBUG("Sending client message compressed %d from %d", compsize, len); + /* Copy lz4 magic header */ + sprintf(dest, "lz4\n"); + /* Copy compressed message length */ + msglen = htole32(compsize); + memcpy(dest + 4, &msglen, 4); + /* Copy decompressed message length */ + msglen = htole32(len); + memcpy(dest + 8, &msglen, 4); + len = compsize + 12; + free(buf); + buf = dest; + } +out: sender_send = ckzalloc(sizeof(sender_send_t)); sender_send->client = client; sender_send->buf = buf; From e3fc6a1e56c84bd7dfade6802c9bb10a660695ed Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Fri, 1 Jan 2016 17:48:21 +1100 Subject: [PATCH 522/544] Selectively compress only large packets greater than one MTU and identify lz4 compatible clients immediately --- src/ckpool.c | 8 +------- src/connector.c | 22 +++++++++++----------- src/stratifier.c | 17 +++++++++++++---- 3 files changed, 25 insertions(+), 22 deletions(-) diff --git a/src/ckpool.c b/src/ckpool.c index 35839db8..25ec3331 100644 --- a/src/ckpool.c +++ b/src/ckpool.c @@ -732,7 +732,7 @@ int write_cs(connsock_t *cs, const char *buf, int len) int ret = -1; /* Connsock doesn't expect lz4 compressed messages */ - if (!cs->lz4) { + if (!cs->lz4 || len <= 1492) { ret = write_socket(cs->fd, buf, len); goto out; } @@ -744,12 +744,6 @@ int write_cs(connsock_t *cs, const char *buf, int len) ret = write_socket(cs->fd, buf, len); goto out; } - if (compsize + 12 >= len) { - /* Selectively send compressed packets only when they're - * smaller. */ - ret = write_socket(cs->fd, buf, len); - goto out; - } LOGDEBUG("Writing connsock message compressed %d from %d", compsize, decompsize); /* Copy lz4 magic header */ sprintf(dest, "lz4\n"); diff --git a/src/connector.c b/src/connector.c index e550c9f8..1faa82ba 100644 --- a/src/connector.c +++ b/src/connector.c @@ -499,8 +499,6 @@ compressed: LOGDEBUG("Received client message compressed %d from %d", client->compsize, client->decompsize); msg[ret] = '\0'; - /* Flag this client as able to receive lz4 compressed data now */ - client->lz4 = true; client->bufofs -= client->compsize; if (client->bufofs) memmove(client->buf, client->buf + buflen, client->bufofs); @@ -1000,8 +998,9 @@ static void send_client(cdata_t *cdata, const int64_t id, char *buf) test_redirector_shares(ckp, client, buf); } - /* Does this client accept compressed data? */ - if (client->lz4) { + /* Does this client accept compressed data? Only compress if it's + * larger than one MTU. */ + if (client->lz4 && len > 1492) { char *dest = ckalloc(len + 12); uint32_t msglen; int compsize; @@ -1012,11 +1011,6 @@ static void send_client(cdata_t *cdata, const int64_t id, char *buf) free(dest); goto out; } - if (compsize + 12 >= len) { - /* Only end it compressed if it's smaller */ - free(dest); - goto out; - } LOGDEBUG("Sending client message compressed %d from %d", compsize, len); /* Copy lz4 magic header */ sprintf(dest, "lz4\n"); @@ -1237,10 +1231,15 @@ retry: sscanf(buf, "loglevel=%d", &ckp->loglevel); } else if (cmdmatch(buf, "shutdown")) { goto out; - } else if (cmdmatch(buf, "passthrough")) { + } else if (cmdmatch(buf, "pass")) { client_instance_t *client; + bool lz4 = false; - ret = sscanf(buf, "passthrough=%"PRId64, &client_id); + if (strstr(buf, "lz4")) { + lz4 = true; + ret = sscanf(buf, "passlz4=%"PRId64, &client_id); + } else + ret = sscanf(buf, "passthrough=%"PRId64, &client_id); if (ret < 0) { LOGDEBUG("Connector failed to parse passthrough command: %s", buf); goto retry; @@ -1250,6 +1249,7 @@ retry: LOGINFO("Connector failed to find client id %"PRId64" to pass through", client_id); goto retry; } + client->lz4 = lz4; passthrough_client(cdata, client); dec_instance_ref(cdata, client); } else if (cmdmatch(buf, "getxfd")) { diff --git a/src/stratifier.c b/src/stratifier.c index e87fc7e4..bc618dfb 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -5186,10 +5186,11 @@ static void add_mining_node(sdata_t *sdata, stratum_instance_t *client) /* Enter with client holding ref count */ static void parse_method(ckpool_t *ckp, sdata_t *sdata, stratum_instance_t *client, - const int64_t client_id, json_t *id_val, json_t *method_val, + const int64_t client_id, json_t *val, json_t *id_val, json_t *method_val, json_t *params_val) { const char *method; + bool var; /* Random broken clients send something not an integer as the id so we * copy the json item for id_val as is for the response. By far the @@ -5237,8 +5238,12 @@ static void parse_method(ckpool_t *ckp, sdata_t *sdata, stratum_instance_t *clie /* Add this client as a passthrough in the connector and * add it to the list of mining nodes in the stratifier */ + json_get_bool(&var, val, "lz4"); add_mining_node(sdata, client); - snprintf(buf, 255, "passthrough=%"PRId64, client_id); + if (var) + snprintf(buf, 255, "passlz4=%"PRId64, client_id); + else + snprintf(buf, 255, "passthrough=%"PRId64, client_id); send_proc(ckp->connector, buf); return; } @@ -5250,8 +5255,12 @@ static void parse_method(ckpool_t *ckp, sdata_t *sdata, stratum_instance_t *clie * is a passthrough and to manage its messages accordingly. No * data from this client id should ever come back to this * stratifier after this so drop the client in the stratifier. */ + json_get_bool(&var, val, "lz4"); LOGNOTICE("Adding passthrough client %"PRId64" %s", client_id, client->address); - snprintf(buf, 255, "passthrough=%"PRId64, client_id); + if (var) + snprintf(buf, 255, "passlz4=%"PRId64, client_id); + else + snprintf(buf, 255, "passthrough=%"PRId64, client_id); send_proc(ckp->connector, buf); drop_client(ckp, sdata, client_id); return; @@ -5469,7 +5478,7 @@ static void parse_instance_msg(ckpool_t *ckp, sdata_t *sdata, smsg_t *msg, strat send_json_err(sdata, client_id, id_val, "-1:params not found"); goto out; } - parse_method(ckp, sdata, client, client_id, id_val, method, params); + parse_method(ckp, sdata, client, client_id, val, id_val, method, params); out: free_smsg(msg); } From 11d14620e6818cbb78d6408ee9f8f4336a942e69 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Sat, 2 Jan 2016 00:16:10 +1100 Subject: [PATCH 523/544] Convert to gzip --- configure.ac | 4 +-- src/ckpool.c | 61 +++++++++++++++++++++++-------------------- src/ckpool.h | 6 +++-- src/connector.c | 67 ++++++++++++++++++++++++++---------------------- src/generator.c | 16 ++++++------ src/libckpool.c | 12 +++++++++ src/libckpool.h | 2 ++ src/stratifier.c | 8 +++--- 8 files changed, 102 insertions(+), 74 deletions(-) diff --git a/configure.ac b/configure.ac index b99bb405..ff33c36e 100644 --- a/configure.ac +++ b/configure.ac @@ -40,7 +40,7 @@ AC_CHECK_HEADERS(stdint.h netinet/in.h netinet/tcp.h sys/ioctl.h getopt.h) AC_CHECK_HEADERS(sys/epoll.h libpq-fe.h postgresql/libpq-fe.h grp.h) AC_CHECK_HEADERS(gsl/gsl_math.h gsl/gsl_cdf.h) AC_CHECK_HEADERS(openssl/x509.h openssl/hmac.h) -AC_CHECK_HEADERS(lz4.h) +AC_CHECK_HEADERS(zlib.h) AC_CONFIG_SUBDIRS([src/jansson-2.6]) JANSSON_LIBS="jansson-2.6/src/.libs/libjansson.a" @@ -54,7 +54,7 @@ AC_ARG_WITH([ckdb], #AC_SEARCH_LIBS(whatgoeshere?, rt, , echo "Error: Required library realtime not found." && exit 1) AC_SEARCH_LIBS(exp, m, , echo "Error: Required library math not found." && exit 1) -AC_SEARCH_LIBS(LZ4_compress, lz4, , echo "Error: Required library liblz4-dev not found." && exit 1) +AC_SEARCH_LIBS(compress, z , , echo "Error: Required library zlib1g-dev not found." && exit 1) AC_SEARCH_LIBS(pthread_mutex_trylock, pthread, , "Error: Required library pthreads not found." && exit 1) if test "x$ckdb" != "xno"; then diff --git a/src/ckpool.c b/src/ckpool.c index 25ec3331..e7b698ea 100644 --- a/src/ckpool.c +++ b/src/ckpool.c @@ -20,7 +20,7 @@ #include #include #include -#include +#include #include #include #include @@ -36,6 +36,8 @@ ckpool_t *global_ckp; +const char gzip_magic[] = "\x1f\xd5\x01\n"; + static void proclog(ckpool_t *ckp, char *msg) { FILE *LOGFP; @@ -567,13 +569,14 @@ out: return ret; } -static int read_lz4_line(connsock_t *cs, float *timeout) +static int read_gz_line(connsock_t *cs, float *timeout) { - int compsize, decompsize, ret, buflen; + unsigned long compsize, res, decompsize; char *buf, *dest = NULL, *eom; + int ret, buflen; uint32_t msglen; - /* Remove lz4 header */ + /* Remove gz header */ clear_bufline(cs); /* Get data sizes */ @@ -588,35 +591,35 @@ static int read_lz4_line(connsock_t *cs, float *timeout) decompsize = le32toh(msglen); cs->bufofs = 8; clear_bufline(cs); - LOGWARNING("Trying to read lz4 %d/%d", compsize, decompsize); - if (unlikely(compsize < 1 || compsize > (int)0x80000000 || - decompsize < 1 || decompsize > (int)0x80000000)) { - LOGWARNING("Invalid message length comp %d decomp %d sent to read_lz4_line", compsize, decompsize); + if (unlikely(compsize < 1 || compsize > 0x80000000 || + decompsize < 1 || decompsize > 0x80000000)) { + LOGWARNING("Invalid message length comp %lu decomp %lu sent to read_gz_line", compsize, decompsize); ret = -1; goto out; } /* Get compressed data */ ret = read_cs_length(cs, timeout, compsize); - if (ret != compsize) { - LOGWARNING("Failed to read %d compressed bytes in read_lz4_line, got %d", compsize, ret); + if (ret != (int)compsize) { + LOGWARNING("Failed to read %lu compressed bytes in read_gz_line, got %d", compsize, ret); ret = -1; goto out; } /* Do decompresion and buffer reconstruction here */ dest = ckalloc(decompsize); - ret = LZ4_decompress_safe(cs->buf, dest, compsize, decompsize); + res = decompsize; + ret = uncompress((Bytef *)dest, &res, (Bytef *)cs->buf, compsize); /* Clear out all the compressed data */ clear_bufline(cs); - if (ret != decompsize) { - LOGWARNING("Failed to decompress %d bytes in read_lz4_line, got %d", decompsize, ret); + if (ret != Z_OK || res != decompsize) { + LOGWARNING("Failed to decompress %lu bytes in read_gz_line, got %d", decompsize, ret); ret = -1; goto out; } eom = dest + decompsize - 1; if (memcmp(eom, "\n", 1)) { - LOGWARNING("Failed to find EOM in decompressed data in read_lz4_line"); + LOGWARNING("Failed to find EOM in decompressed data in read_gz_line"); ret = -1; goto out; } @@ -714,39 +717,41 @@ out: if (ret < 0) { empty_buffer(cs); dealloc(cs->buf); - } else if (cs->buflen && !strncmp(cs->buf, "lz4", 3)) - ret = read_lz4_line(cs, timeout); + } else if (cs->buflen && !strncmp(cs->buf, gzip_magic, 3)) + ret = read_gz_line(cs, timeout); return ret; } -/* Lz4 compressed block structure: - * - 4 byte magic header LZ4\n +/* gzip compressed block structure: + * - 4 byte magic header gzip_magic "\x1f\xd5\x01\n" * - 4 byte LE encoded compressed size * - 4 byte LE encoded decompressed size */ int write_cs(connsock_t *cs, const char *buf, int len) { - int compsize, decompsize = len; + unsigned long compsize, decompsize = len; char *dest = NULL; uint32_t msglen; int ret = -1; - /* Connsock doesn't expect lz4 compressed messages */ - if (!cs->lz4 || len <= 1492) { + /* Connsock doesn't expect gz compressed messages */ + if (!cs->gz || len <= 1492) { ret = write_socket(cs->fd, buf, len); goto out; } - dest = ckalloc(len + 12); + compsize = round_up_page(len + 12); + dest = ckalloc(compsize); + compsize -= 12; /* Do compression here */ - compsize = LZ4_compress(buf, dest + 12, len); - if (!compsize) { - LOGWARNING("Failed to LZ4 compress in write_cs, writing uncompressed"); + ret = compress((Bytef *)dest + 12, &compsize, (Bytef *)buf, len); + if (ret != Z_OK) { + LOGWARNING("Failed to gz compress in write_cs, writing uncompressed"); ret = write_socket(cs->fd, buf, len); goto out; } - LOGDEBUG("Writing connsock message compressed %d from %d", compsize, decompsize); - /* Copy lz4 magic header */ - sprintf(dest, "lz4\n"); + LOGDEBUG("Writing connsock message compressed %lu from %lu", compsize, decompsize); + /* Copy gz magic header */ + sprintf(dest, gzip_magic); /* Copy compressed message length */ msglen = htole32(compsize); memcpy(dest + 4, &msglen, 4); diff --git a/src/ckpool.h b/src/ckpool.h index 9f2f599a..52017027 100644 --- a/src/ckpool.h +++ b/src/ckpool.h @@ -83,8 +83,8 @@ struct connsock { /* Semaphore used to serialise request/responses */ sem_t sem; - /* Has the other end acknowledged it can receive lz4 compressed data */ - bool lz4; + /* Has the other end acknowledged it can receive gz compressed data */ + bool gz; }; typedef struct connsock connsock_t; @@ -284,6 +284,8 @@ static const char __maybe_unused *stratum_msgs[] = { "" }; +extern const char gzip_magic[]; + #ifdef USE_CKDB #define CKP_STANDALONE(CKP) ((CKP)->standalone == true) #else diff --git a/src/connector.c b/src/connector.c index 1faa82ba..cb1d240d 100644 --- a/src/connector.c +++ b/src/connector.c @@ -10,12 +10,12 @@ #include "config.h" #include -#include #include #include #include #include #include +#include #include "ckpool.h" #include "libckpool.h" @@ -56,7 +56,7 @@ struct client_instance { int server; char buf[PAGESIZE]; - int bufofs; + unsigned long bufofs; /* Are we currently sending a blocked message from this client */ sender_send_t *sending; @@ -64,11 +64,11 @@ struct client_instance { /* Is this the parent passthrough client */ bool passthrough; - /* Does this client expect lz4 compression? */ - bool lz4; + /* Does this client expect gz compression? */ + bool gz; bool compressed; /* Currently receiving a compressed message */ - int compsize; /* Expected compressed data size */ - int decompsize; /* Expected decompressed data size */ + unsigned long compsize; /* Expected compressed data size */ + unsigned long decompsize; /* Expected decompressed data size */ /* Linked list of shares in redirector mode.*/ share_t *shares; @@ -479,7 +479,7 @@ retry: if (ret < 1) { if (likely(errno == EAGAIN || errno == EWOULDBLOCK || !ret)) return; - LOGINFO("Client id %"PRId64" fd %d disconnected - recv fail with bufofs %d ret %d errno %d %s", + LOGINFO("Client id %"PRId64" fd %d disconnected - recv fail with bufofs %lu ret %d errno %d %s", client->id, client->fd, client->bufofs, ret, errno, ret && errno ? strerror(errno) : ""); invalidate_client(ckp, cdata, client); return; @@ -487,18 +487,21 @@ retry: client->bufofs += ret; compressed: if (client->compressed) { + unsigned long res; + if (client->bufofs < client->compsize) goto retry; - ret = LZ4_decompress_safe(client->buf, msg, client->compsize, client->decompsize); - if (ret != client->decompsize) { - LOGNOTICE("Failed to decompress %d from %d bytes in parse_client_msg, got %d", + res = client->decompsize; + ret = uncompress((Bytef *)msg, &res, (Bytef *)client->buf, client->compsize); + if (ret != Z_OK || res != client->decompsize) { + LOGNOTICE("Failed to decompress %lu from %lu bytes in parse_client_msg, got %d", client->decompsize, client->compsize, ret); invalidate_client(ckp, cdata, client); return; } - LOGDEBUG("Received client message compressed %d from %d", + LOGDEBUG("Received client message compressed %lu from %lu", client->compsize, client->decompsize); - msg[ret] = '\0'; + msg[res] = '\0'; client->bufofs -= client->compsize; if (client->bufofs) memmove(client->buf, client->buf + buflen, client->bufofs); @@ -519,7 +522,7 @@ reparse: } /* Look for a compression header */ - if (!strncmp(client->buf, "lz4", 3)) { + if (!strncmp(client->buf, gzip_magic, 3)) { uint32_t msglen; /* Do we have the whole header? If not, keep reading */ @@ -531,7 +534,7 @@ reparse: client->decompsize = le32toh(msglen); if (unlikely(!client->compsize || !client->decompsize || client->compsize > MAX_MSGSIZE || client->decompsize > MAX_MSGSIZE)) { - LOGNOTICE("Client id %"PRId64" invalid compressed message size %d/%d, disconnecting", + LOGNOTICE("Client id %"PRId64" invalid compressed message size %lu/%lu, disconnecting", client->id, client->compsize, client->decompsize); invalidate_client(ckp, cdata, client); return; @@ -1000,20 +1003,24 @@ static void send_client(cdata_t *cdata, const int64_t id, char *buf) /* Does this client accept compressed data? Only compress if it's * larger than one MTU. */ - if (client->lz4 && len > 1492) { - char *dest = ckalloc(len + 12); + if (client->gz && len > 1492) { + unsigned long compsize; uint32_t msglen; - int compsize; - - compsize = LZ4_compress(buf, dest + 12, len); - if (unlikely(!compsize)) { - LOGWARNING("Failed to LZ4 compress in send_client, sending uncompressed"); + char *dest; + int ret; + + compsize = round_up_page(len + 12); + dest = ckalloc(compsize); + compsize -= 12; + ret = compress((Bytef *)dest + 12, &compsize, (Bytef *)buf, len); + if (unlikely(ret != Z_OK)) { + LOGWARNING("Failed to gz compress in send_client, got %d sending uncompressed", ret); free(dest); goto out; } - LOGDEBUG("Sending client message compressed %d from %d", compsize, len); - /* Copy lz4 magic header */ - sprintf(dest, "lz4\n"); + LOGDEBUG("Sending client message compressed %lu from %d", compsize, len); + /* Copy gz magic header */ + sprintf(dest, gzip_magic); /* Copy compressed message length */ msglen = htole32(compsize); memcpy(dest + 4, &msglen, 4); @@ -1054,7 +1061,7 @@ static void passthrough_client(cdata_t *cdata, client_instance_t *client) LOGINFO("Connector adding passthrough client %"PRId64, client->id); client->passthrough = true; - ASPRINTF(&buf, "{\"result\": true, \"lz4\": true}\n"); + ASPRINTF(&buf, "{\"result\": true, \"gz\": true}\n"); send_client(cdata, client->id, buf); } @@ -1233,11 +1240,11 @@ retry: goto out; } else if (cmdmatch(buf, "pass")) { client_instance_t *client; - bool lz4 = false; + bool gz = false; - if (strstr(buf, "lz4")) { - lz4 = true; - ret = sscanf(buf, "passlz4=%"PRId64, &client_id); + if (strstr(buf, "gz")) { + gz = true; + ret = sscanf(buf, "passgz=%"PRId64, &client_id); } else ret = sscanf(buf, "passthrough=%"PRId64, &client_id); if (ret < 0) { @@ -1249,7 +1256,7 @@ retry: LOGINFO("Connector failed to find client id %"PRId64" to pass through", client_id); goto retry; } - client->lz4 = lz4; + client->gz = gz; passthrough_client(cdata, client); dec_instance_ref(cdata, client); } else if (cmdmatch(buf, "getxfd")) { diff --git a/src/generator.c b/src/generator.c index e9d69ef4..f40bbfb2 100644 --- a/src/generator.c +++ b/src/generator.c @@ -774,7 +774,7 @@ static bool passthrough_stratum(connsock_t *cs, proxy_instance_t *proxi) JSON_CPACK(req, "{ss,sb,s[s]}", "method", "mining.passthrough", - "lz4", json_true(), + "gz", json_true(), "params", PACKAGE"/"VERSION); res = send_json_msg(cs, req); json_decref(req); @@ -799,9 +799,9 @@ static bool passthrough_stratum(connsock_t *cs, proxy_instance_t *proxi) LOGWARNING("Denied passthrough for stratum"); goto out; } - json_get_bool(&cs->lz4, val, "lz4"); - if (cs->lz4) - LOGNOTICE("Negotiated lz4 compression with pool"); + json_get_bool(&cs->gz, val, "gz"); + if (cs->gz) + LOGNOTICE("Negotiated gz compression with pool"); proxi->passthrough = true; out: if (val) @@ -820,7 +820,7 @@ static bool node_stratum(connsock_t *cs, proxy_instance_t *proxi) JSON_CPACK(req, "{ss,sb,s[s]}", "method", "mining.node", - "lz4", json_true(), + "gz", json_true(), "params", PACKAGE"/"VERSION); res = send_json_msg(cs, req); @@ -846,9 +846,9 @@ static bool node_stratum(connsock_t *cs, proxy_instance_t *proxi) LOGWARNING("Denied node setup for stratum"); goto out; } - json_get_bool(&cs->lz4, val, "lz4"); - if (cs->lz4) - LOGNOTICE("Negotiated lz4 compression with pool"); + json_get_bool(&cs->gz, val, "gz"); + if (cs->gz) + LOGNOTICE("Negotiated gz compression with pool"); proxi->node = true; out: if (val) diff --git a/src/libckpool.c b/src/libckpool.c index a0557a71..2e78d330 100644 --- a/src/libckpool.c +++ b/src/libckpool.c @@ -1389,6 +1389,18 @@ void *_ckzalloc(size_t len, const char *file, const char *func, const int line) return ptr; } +/* Round up to the nearest page size for efficient malloc */ +size_t round_up_page(size_t len) +{ + int rem = len % PAGESIZE; + + if (rem) + len += PAGESIZE - rem; + return len; +} + + + /* Adequate size s==len*2 + 1 must be alloced to use this variant */ void __bin2hex(void *vs, const void *vp, size_t len) { diff --git a/src/libckpool.h b/src/libckpool.h index 9d8ba341..10c18248 100644 --- a/src/libckpool.h +++ b/src/libckpool.h @@ -531,6 +531,8 @@ void trail_slash(char **buf); void *_ckalloc(size_t len, const char *file, const char *func, const int line); void *json_ckalloc(size_t size); void *_ckzalloc(size_t len, const char *file, const char *func, const int line); +size_t round_up_page(size_t len); + extern const int hex2bin_tbl[]; void __bin2hex(void *vs, const void *vp, size_t len); void *bin2hex(const void *vp, size_t len); diff --git a/src/stratifier.c b/src/stratifier.c index bc618dfb..55bdf65c 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -5238,10 +5238,10 @@ static void parse_method(ckpool_t *ckp, sdata_t *sdata, stratum_instance_t *clie /* Add this client as a passthrough in the connector and * add it to the list of mining nodes in the stratifier */ - json_get_bool(&var, val, "lz4"); + json_get_bool(&var, val, "gz"); add_mining_node(sdata, client); if (var) - snprintf(buf, 255, "passlz4=%"PRId64, client_id); + snprintf(buf, 255, "passgz=%"PRId64, client_id); else snprintf(buf, 255, "passthrough=%"PRId64, client_id); send_proc(ckp->connector, buf); @@ -5255,10 +5255,10 @@ static void parse_method(ckpool_t *ckp, sdata_t *sdata, stratum_instance_t *clie * is a passthrough and to manage its messages accordingly. No * data from this client id should ever come back to this * stratifier after this so drop the client in the stratifier. */ - json_get_bool(&var, val, "lz4"); + json_get_bool(&var, val, "gz"); LOGNOTICE("Adding passthrough client %"PRId64" %s", client_id, client->address); if (var) - snprintf(buf, 255, "passlz4=%"PRId64, client_id); + snprintf(buf, 255, "passgz=%"PRId64, client_id); else snprintf(buf, 255, "passthrough=%"PRId64, client_id); send_proc(ckp->connector, buf); From c9f0858c560889fc5fb6a61deffcd963a7732343 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Sat, 2 Jan 2016 00:45:19 +1100 Subject: [PATCH 524/544] Fixes? --- src/ckpool.c | 7 +++---- src/connector.c | 16 +++++++--------- src/libckpool.c | 12 ------------ src/libckpool.h | 2 -- 4 files changed, 10 insertions(+), 27 deletions(-) diff --git a/src/ckpool.c b/src/ckpool.c index e7b698ea..2f49936a 100644 --- a/src/ckpool.c +++ b/src/ckpool.c @@ -739,13 +739,12 @@ int write_cs(connsock_t *cs, const char *buf, int len) ret = write_socket(cs->fd, buf, len); goto out; } - compsize = round_up_page(len + 12); - dest = ckalloc(compsize); - compsize -= 12; + dest = ckalloc(len + 12); /* Do compression here */ + compsize = len; ret = compress((Bytef *)dest + 12, &compsize, (Bytef *)buf, len); if (ret != Z_OK) { - LOGWARNING("Failed to gz compress in write_cs, writing uncompressed"); + LOGINFO("Failed to gz compress in write_cs, writing uncompressed"); ret = write_socket(cs->fd, buf, len); goto out; } diff --git a/src/connector.c b/src/connector.c index cb1d240d..dd173478 100644 --- a/src/connector.c +++ b/src/connector.c @@ -1003,29 +1003,27 @@ static void send_client(cdata_t *cdata, const int64_t id, char *buf) /* Does this client accept compressed data? Only compress if it's * larger than one MTU. */ - if (client->gz && len > 1492) { - unsigned long compsize; + if (client->gz) { + unsigned long compsize, decompsize = len; + char *dest = ckalloc(len + 12); uint32_t msglen; - char *dest; int ret; - compsize = round_up_page(len + 12); - dest = ckalloc(compsize); - compsize -= 12; + compsize = len; ret = compress((Bytef *)dest + 12, &compsize, (Bytef *)buf, len); if (unlikely(ret != Z_OK)) { - LOGWARNING("Failed to gz compress in send_client, got %d sending uncompressed", ret); + LOGINFO("Failed to gz compress in send_client, got %d sending uncompressed", ret); free(dest); goto out; } - LOGDEBUG("Sending client message compressed %lu from %d", compsize, len); + LOGDEBUG("Sending client message compressed %lu from %lu", compsize, decompsize); /* Copy gz magic header */ sprintf(dest, gzip_magic); /* Copy compressed message length */ msglen = htole32(compsize); memcpy(dest + 4, &msglen, 4); /* Copy decompressed message length */ - msglen = htole32(len); + msglen = htole32(decompsize); memcpy(dest + 8, &msglen, 4); len = compsize + 12; free(buf); diff --git a/src/libckpool.c b/src/libckpool.c index 2e78d330..a0557a71 100644 --- a/src/libckpool.c +++ b/src/libckpool.c @@ -1389,18 +1389,6 @@ void *_ckzalloc(size_t len, const char *file, const char *func, const int line) return ptr; } -/* Round up to the nearest page size for efficient malloc */ -size_t round_up_page(size_t len) -{ - int rem = len % PAGESIZE; - - if (rem) - len += PAGESIZE - rem; - return len; -} - - - /* Adequate size s==len*2 + 1 must be alloced to use this variant */ void __bin2hex(void *vs, const void *vp, size_t len) { diff --git a/src/libckpool.h b/src/libckpool.h index 10c18248..9d8ba341 100644 --- a/src/libckpool.h +++ b/src/libckpool.h @@ -531,8 +531,6 @@ void trail_slash(char **buf); void *_ckalloc(size_t len, const char *file, const char *func, const int line); void *json_ckalloc(size_t size); void *_ckzalloc(size_t len, const char *file, const char *func, const int line); -size_t round_up_page(size_t len); - extern const int hex2bin_tbl[]; void __bin2hex(void *vs, const void *vp, size_t len); void *bin2hex(const void *vp, size_t len); From d69ed7ffcf5aca6921aaba10734aeff8839c3260 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Sat, 2 Jan 2016 08:18:30 +1100 Subject: [PATCH 525/544] More fixes --- src/ckpool.c | 24 ++++++++++-------------- src/connector.c | 8 +++++--- src/libckpool.c | 12 ++++++++++++ src/libckpool.h | 2 ++ 4 files changed, 29 insertions(+), 17 deletions(-) diff --git a/src/ckpool.c b/src/ckpool.c index 2f49936a..6cfbd9f9 100644 --- a/src/ckpool.c +++ b/src/ckpool.c @@ -555,7 +555,7 @@ static int read_cs_length(connsock_t *cs, float *timeout, int len) int ret = len; while (cs->buflen < len) { - char readbuf[PAGESIZE] = {}; + char readbuf[PAGESIZE]; ret = wait_read_select(cs->fd, *timeout); if (ret < 1) @@ -629,7 +629,7 @@ static int read_gz_line(connsock_t *cs, float *timeout) buflen = cs->buflen; cs->buf = dest; dest = NULL; - ret = cs->buflen = decompsize - 1; + ret = cs->buflen = decompsize; if (buflen) add_bufline(cs, buf, buflen); out: @@ -732,23 +732,20 @@ int write_cs(connsock_t *cs, const char *buf, int len) unsigned long compsize, decompsize = len; char *dest = NULL; uint32_t msglen; - int ret = -1; + int ret; /* Connsock doesn't expect gz compressed messages */ - if (!cs->gz || len <= 1492) { - ret = write_socket(cs->fd, buf, len); - goto out; - } - dest = ckalloc(len + 12); + if (!cs->gz || len <= 1492) + return write_socket(cs->fd, buf, len); + compsize = round_up_page(len + 12); + dest = alloca(compsize); /* Do compression here */ - compsize = len; + compsize -= 12; ret = compress((Bytef *)dest + 12, &compsize, (Bytef *)buf, len); if (ret != Z_OK) { LOGINFO("Failed to gz compress in write_cs, writing uncompressed"); - ret = write_socket(cs->fd, buf, len); - goto out; + return write_socket(cs->fd, buf, len); } - LOGDEBUG("Writing connsock message compressed %lu from %lu", compsize, decompsize); /* Copy gz magic header */ sprintf(dest, gzip_magic); /* Copy compressed message length */ @@ -758,13 +755,12 @@ int write_cs(connsock_t *cs, const char *buf, int len) msglen = htole32(decompsize); memcpy(dest + 8, &msglen, 4); len = compsize + 12; + LOGDEBUG("Writing connsock message compressed %d from %lu", len, decompsize); ret = write_socket(cs->fd, dest, len); if (ret == len) ret = decompsize; else ret = -1; -out: - free(dest); return ret; } diff --git a/src/connector.c b/src/connector.c index dd173478..0f7633fb 100644 --- a/src/connector.c +++ b/src/connector.c @@ -1005,18 +1005,19 @@ static void send_client(cdata_t *cdata, const int64_t id, char *buf) * larger than one MTU. */ if (client->gz) { unsigned long compsize, decompsize = len; - char *dest = ckalloc(len + 12); uint32_t msglen; + char *dest; int ret; - compsize = len; + compsize = round_up_page(len + 12); + dest = ckalloc(compsize); + compsize -= 12; ret = compress((Bytef *)dest + 12, &compsize, (Bytef *)buf, len); if (unlikely(ret != Z_OK)) { LOGINFO("Failed to gz compress in send_client, got %d sending uncompressed", ret); free(dest); goto out; } - LOGDEBUG("Sending client message compressed %lu from %lu", compsize, decompsize); /* Copy gz magic header */ sprintf(dest, gzip_magic); /* Copy compressed message length */ @@ -1026,6 +1027,7 @@ static void send_client(cdata_t *cdata, const int64_t id, char *buf) msglen = htole32(decompsize); memcpy(dest + 8, &msglen, 4); len = compsize + 12; + LOGDEBUG("Sending client message compressed %d from %lu", len, decompsize); free(buf); buf = dest; } diff --git a/src/libckpool.c b/src/libckpool.c index a0557a71..2e78d330 100644 --- a/src/libckpool.c +++ b/src/libckpool.c @@ -1389,6 +1389,18 @@ void *_ckzalloc(size_t len, const char *file, const char *func, const int line) return ptr; } +/* Round up to the nearest page size for efficient malloc */ +size_t round_up_page(size_t len) +{ + int rem = len % PAGESIZE; + + if (rem) + len += PAGESIZE - rem; + return len; +} + + + /* Adequate size s==len*2 + 1 must be alloced to use this variant */ void __bin2hex(void *vs, const void *vp, size_t len) { diff --git a/src/libckpool.h b/src/libckpool.h index 9d8ba341..10c18248 100644 --- a/src/libckpool.h +++ b/src/libckpool.h @@ -531,6 +531,8 @@ void trail_slash(char **buf); void *_ckalloc(size_t len, const char *file, const char *func, const int line); void *json_ckalloc(size_t size); void *_ckzalloc(size_t len, const char *file, const char *func, const int line); +size_t round_up_page(size_t len); + extern const int hex2bin_tbl[]; void __bin2hex(void *vs, const void *vp, size_t len); void *bin2hex(const void *vp, size_t len); From 4ef8ab49e334d86b1795d3ae4f1168d75de86b5e Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Sat, 2 Jan 2016 11:02:17 +1100 Subject: [PATCH 526/544] Fix reinsertion logic --- src/ckpool.c | 31 ++++++++++++++++++++++++------- 1 file changed, 24 insertions(+), 7 deletions(-) diff --git a/src/ckpool.c b/src/ckpool.c index 6cfbd9f9..c54b75e7 100644 --- a/src/ckpool.c +++ b/src/ckpool.c @@ -523,7 +523,8 @@ static void clear_bufline(connsock_t *cs) cs->bufofs = cs->buflen; cs->buflen = 0; cs->buf[cs->bufofs] = '\0'; - } + } else + cs->bufofs = 0; } static void add_bufline(connsock_t *cs, const char *readbuf, const int len) @@ -554,7 +555,7 @@ static int read_cs_length(connsock_t *cs, float *timeout, int len) { int ret = len; - while (cs->buflen < len) { + while (cs->bufofs < len) { char readbuf[PAGESIZE]; ret = wait_read_select(cs->fd, *timeout); @@ -585,10 +586,14 @@ static int read_gz_line(connsock_t *cs, float *timeout) ret = -1; goto out; } + memcpy(&msglen, cs->buf, 4); compsize = le32toh(msglen); memcpy(&msglen, cs->buf + 4, 4); decompsize = le32toh(msglen); + + /* Remove the gz variables */ + cs->buflen = cs->bufofs - 8; cs->bufofs = 8; clear_bufline(cs); @@ -606,32 +611,44 @@ static int read_gz_line(connsock_t *cs, float *timeout) ret = -1; goto out; } + /* Clear out all the compressed data */ + cs->buflen = cs->bufofs - compsize; + cs->bufofs = compsize; + clear_bufline(cs); + /* Do decompresion and buffer reconstruction here */ dest = ckalloc(decompsize); res = decompsize; ret = uncompress((Bytef *)dest, &res, (Bytef *)cs->buf, compsize); - /* Clear out all the compressed data */ - clear_bufline(cs); + if (ret != Z_OK || res != decompsize) { LOGWARNING("Failed to decompress %lu bytes in read_gz_line, got %d", decompsize, ret); ret = -1; goto out; } + eom = dest + decompsize - 1; if (memcmp(eom, "\n", 1)) { LOGWARNING("Failed to find EOM in decompressed data in read_gz_line"); ret = -1; goto out; } + *eom = '\0'; + ret = decompsize - 1; /* Wedge the decompressed buffer back to the start of cs->buf */ buf = cs->buf; - buflen = cs->buflen; + buflen = cs->bufofs; cs->buf = dest; dest = NULL; - ret = cs->buflen = decompsize; - if (buflen) + cs->bufofs = decompsize; + if (buflen) { + LOGWARNING("Remainder %s", buf); add_bufline(cs, buf, buflen); + cs->buflen = buflen; + cs->bufofs = decompsize; + } else + cs->buflen = cs->bufofs = 0; out: free(dest); return ret; From ea945e863c0e538ed3bc66f6fde522e6327b0274 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Sat, 2 Jan 2016 11:03:04 +1100 Subject: [PATCH 527/544] Reinstate mtu size for compression to clients --- src/connector.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/connector.c b/src/connector.c index 0f7633fb..2a9500cf 100644 --- a/src/connector.c +++ b/src/connector.c @@ -1003,7 +1003,7 @@ static void send_client(cdata_t *cdata, const int64_t id, char *buf) /* Does this client accept compressed data? Only compress if it's * larger than one MTU. */ - if (client->gz) { + if (client->gz && len > 1492) { unsigned long compsize, decompsize = len; uint32_t msglen; char *dest; From 5146b715fcbbd445326742b27fe00b9677822837 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Sat, 2 Jan 2016 11:05:01 +1100 Subject: [PATCH 528/544] Empty buffer on failure --- src/ckpool.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/ckpool.c b/src/ckpool.c index c54b75e7..2dd7cd3e 100644 --- a/src/ckpool.c +++ b/src/ckpool.c @@ -651,6 +651,8 @@ static int read_gz_line(connsock_t *cs, float *timeout) cs->buflen = cs->bufofs = 0; out: free(dest); + if (ret < 1) + empty_buffer(cs); return ret; } From 5fb2ea342e8c966f360000c06e791f2c6aeea9c3 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Sat, 2 Jan 2016 11:48:06 +1100 Subject: [PATCH 529/544] Various fixes --- src/ckpool.c | 14 ++++++++++---- src/connector.c | 9 +++++++-- 2 files changed, 17 insertions(+), 6 deletions(-) diff --git a/src/ckpool.c b/src/ckpool.c index 2dd7cd3e..5fcc7c9b 100644 --- a/src/ckpool.c +++ b/src/ckpool.c @@ -510,6 +510,8 @@ bool ping_main(ckpool_t *ckp) void empty_buffer(connsock_t *cs) { + if (cs->buf) + cs->buf[0] = '\0'; cs->buflen = cs->bufofs = 0; } @@ -557,11 +559,15 @@ static int read_cs_length(connsock_t *cs, float *timeout, int len) while (cs->bufofs < len) { char readbuf[PAGESIZE]; + int readlen; ret = wait_read_select(cs->fd, *timeout); if (ret < 1) goto out; - ret = recv(cs->fd, readbuf, len - cs->buflen, MSG_DONTWAIT); + readlen = len - cs->buflen; + if (readlen >= PAGESIZE) + readlen = PAGESIZE - 4; + ret = recv(cs->fd, readbuf, readlen, MSG_DONTWAIT); if (ret < 1) goto out; add_bufline(cs, readbuf, ret); @@ -617,8 +623,8 @@ static int read_gz_line(connsock_t *cs, float *timeout) clear_bufline(cs); /* Do decompresion and buffer reconstruction here */ - dest = ckalloc(decompsize); - res = decompsize; + res = round_up_page(decompsize); + dest = ckalloc(res); ret = uncompress((Bytef *)dest, &res, (Bytef *)cs->buf, compsize); if (ret != Z_OK || res != decompsize) { @@ -736,7 +742,7 @@ out: if (ret < 0) { empty_buffer(cs); dealloc(cs->buf); - } else if (cs->buflen && !strncmp(cs->buf, gzip_magic, 3)) + } else if (ret == 3 && !strncmp(cs->buf, gzip_magic, 3)) ret = read_gz_line(cs, timeout); return ret; } diff --git a/src/connector.c b/src/connector.c index 2a9500cf..5593476b 100644 --- a/src/connector.c +++ b/src/connector.c @@ -491,7 +491,12 @@ compressed: if (client->bufofs < client->compsize) goto retry; - res = client->decompsize; + res = PAGESIZE - 4; + if (unlikely(client->decompsize > res)) { + LOGNOTICE("Client attempting to send oversize compressed message, disconnecting"); + invalidate_client(ckp, cdata, client); + return; + } ret = uncompress((Bytef *)msg, &res, (Bytef *)client->buf, client->compsize); if (ret != Z_OK || res != client->decompsize) { LOGNOTICE("Failed to decompress %lu from %lu bytes in parse_client_msg, got %d", @@ -1003,7 +1008,7 @@ static void send_client(cdata_t *cdata, const int64_t id, char *buf) /* Does this client accept compressed data? Only compress if it's * larger than one MTU. */ - if (client->gz && len > 1492) { + if (client->gz) { unsigned long compsize, decompsize = len; uint32_t msglen; char *dest; From 7e4fee659e1020e3a6fe8a62c1b7cde838a67a72 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Sat, 2 Jan 2016 14:46:03 +1100 Subject: [PATCH 530/544] Minor fixes --- src/ckpool.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/ckpool.c b/src/ckpool.c index 5fcc7c9b..442ac412 100644 --- a/src/ckpool.c +++ b/src/ckpool.c @@ -535,7 +535,7 @@ static void add_bufline(connsock_t *cs, const char *readbuf, const int len) size_t buflen; char *newbuf; - buflen = cs->bufofs + len + 1; + buflen = round_up_page(cs->bufofs + len + 1); while (42) { newbuf = realloc(cs->buf, buflen); if (likely(newbuf)) @@ -564,7 +564,7 @@ static int read_cs_length(connsock_t *cs, float *timeout, int len) ret = wait_read_select(cs->fd, *timeout); if (ret < 1) goto out; - readlen = len - cs->buflen; + readlen = len - cs->bufofs; if (readlen >= PAGESIZE) readlen = PAGESIZE - 4; ret = recv(cs->fd, readbuf, readlen, MSG_DONTWAIT); @@ -617,6 +617,7 @@ static int read_gz_line(connsock_t *cs, float *timeout) ret = -1; goto out; } + /* Clear out all the compressed data */ cs->buflen = cs->bufofs - compsize; cs->bufofs = compsize; @@ -626,9 +627,9 @@ static int read_gz_line(connsock_t *cs, float *timeout) res = round_up_page(decompsize); dest = ckalloc(res); ret = uncompress((Bytef *)dest, &res, (Bytef *)cs->buf, compsize); - if (ret != Z_OK || res != decompsize) { - LOGWARNING("Failed to decompress %lu bytes in read_gz_line, got %d", decompsize, ret); + LOGWARNING("Failed to decompress %lu bytes in read_gz_line, result %d got %lu", + decompsize, ret, res); ret = -1; goto out; } @@ -649,7 +650,6 @@ static int read_gz_line(connsock_t *cs, float *timeout) dest = NULL; cs->bufofs = decompsize; if (buflen) { - LOGWARNING("Remainder %s", buf); add_bufline(cs, buf, buflen); cs->buflen = buflen; cs->bufofs = decompsize; From 1912613e913aba4f4102072ddb0935eac3fc36b6 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Sat, 2 Jan 2016 15:00:57 +1100 Subject: [PATCH 531/544] Hold semaphore till we've sent the buffer --- src/generator.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/generator.c b/src/generator.c index f40bbfb2..353dc641 100644 --- a/src/generator.c +++ b/src/generator.c @@ -2022,20 +2022,20 @@ static void *passthrough_recv(void *arg) /* Make sure we receive a line within 90 seconds */ cksem_wait(&cs->sem); ret = read_socket_line(cs, &timeout); - cksem_post(&cs->sem); - if (ret < 1) { - reconnect_generator(ckp); LOGWARNING("Proxy %d:%s failed to read_socket_line in passthrough_recv, attempting reconnect", proxi->id, proxi->url); alive = proxi->alive = false; Close(cs->fd); + reconnect_generator(ckp); + cksem_post(&cs->sem); continue; } /* Simply forward the message on, as is, to the connector to * process. Possibly parse parameters sent by upstream pool * here */ send_proc(ckp->connector, cs->buf); + cksem_post(&cs->sem); } return NULL; } From daa3fde1ab9f195238061c6eb858ae5344b6527c Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Sat, 2 Jan 2016 15:00:57 +1100 Subject: [PATCH 532/544] Hold semaphore till we've sent the buffer --- src/generator.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/generator.c b/src/generator.c index 3680535a..c44bf8f3 100644 --- a/src/generator.c +++ b/src/generator.c @@ -2014,20 +2014,20 @@ static void *passthrough_recv(void *arg) /* Make sure we receive a line within 90 seconds */ cksem_wait(&cs->sem); ret = read_socket_line(cs, &timeout); - cksem_post(&cs->sem); - if (ret < 1) { - reconnect_generator(ckp); LOGWARNING("Proxy %d:%s failed to read_socket_line in passthrough_recv, attempting reconnect", proxi->id, proxi->url); alive = proxi->alive = false; Close(cs->fd); + reconnect_generator(ckp); + cksem_post(&cs->sem); continue; } /* Simply forward the message on, as is, to the connector to * process. Possibly parse parameters sent by upstream pool * here */ send_proc(ckp->connector, cs->buf); + cksem_post(&cs->sem); } return NULL; } From ff5e74fbd1fde42f7e80fc6782f5804f2367b7a4 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Sun, 3 Jan 2016 09:54:59 +1100 Subject: [PATCH 533/544] Rework read_socket_line loop to not bother reading more once it has a message --- src/ckpool.c | 53 ++++++++++++++++++++-------------------------------- 1 file changed, 20 insertions(+), 33 deletions(-) diff --git a/src/ckpool.c b/src/ckpool.c index 442ac412..58cb831a 100644 --- a/src/ckpool.c +++ b/src/ckpool.c @@ -670,7 +670,6 @@ int read_socket_line(connsock_t *cs, float *timeout) char *eom = NULL; tv_t start, now; int ret = -1; - bool polled; float diff; if (unlikely(cs->fd < 0)) @@ -680,56 +679,44 @@ int read_socket_line(connsock_t *cs, float *timeout) eom = strchr(cs->buf, '\n'); tv_time(&start); -rewait: - if (*timeout < 0) { - LOGDEBUG("Timed out in read_socket_line"); - ret = 0; - goto out; - } - ret = wait_read_select(cs->fd, eom ? 0 : *timeout); - polled = true; - if (ret < 1) { - if (!ret) { - if (eom) - goto parse; - LOGDEBUG("Select timed out in read_socket_line"); - } else { + + while (!eom) { + char readbuf[PAGESIZE]; + + if (*timeout < 0) { + if (cs->ckp->proxy) + LOGINFO("Timed out in read_socket_line"); + else + LOGERR("Timed out in read_socket_line"); + ret = 0; + goto out; + } + ret = wait_read_select(cs->fd, *timeout); + if (ret < 0) { if (cs->ckp->proxy) LOGINFO("Select failed in read_socket_line"); else LOGERR("Select failed in read_socket_line"); + goto out; } - goto out; - } - tv_time(&now); - diff = tvdiff(&now, &start); - copy_tv(&start, &now); - *timeout -= diff; - while (42) { - char readbuf[PAGESIZE] = {}; - ret = recv(cs->fd, readbuf, PAGESIZE - 4, MSG_DONTWAIT); if (ret < 1) { - /* No more to read or closed socket after valid message */ - if (eom) - break; - /* Have we used up all the timeout yet? If polled is - * set that means poll has said there should be + /* If we have done wait_read_select there should be * something to read and if we get nothing it means the * socket is closed. */ - if (!polled && *timeout >= 0 && (errno == EAGAIN || errno == EWOULDBLOCK || !ret)) - goto rewait; if (cs->ckp->proxy) LOGINFO("Failed to recv in read_socket_line"); else LOGERR("Failed to recv in read_socket_line"); goto out; } - polled = false; add_bufline(cs, readbuf, ret); eom = strchr(cs->buf, '\n'); + tv_time(&now); + diff = tvdiff(&now, &start); + copy_tv(&start, &now); + *timeout -= diff; } -parse: ret = eom - cs->buf; cs->buflen = cs->buf + cs->bufofs - eom - 1; From 85302af03f6dd6ee3174f56497f1c5fea7194a7c Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Sun, 3 Jan 2016 10:49:05 +1100 Subject: [PATCH 534/544] Decrement timeout in read_cs_length --- src/ckpool.c | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/src/ckpool.c b/src/ckpool.c index 58cb831a..f2c6054a 100644 --- a/src/ckpool.c +++ b/src/ckpool.c @@ -555,24 +555,36 @@ static void add_bufline(connsock_t *cs, const char *readbuf, const int len) static int read_cs_length(connsock_t *cs, float *timeout, int len) { + tv_t start, now; int ret = len; + float diff; + + tv_time(&start); while (cs->bufofs < len) { char readbuf[PAGESIZE]; int readlen; + if (*timeout < 0) { + LOGDEBUG("Timed out in read_cs_length"); + ret = 0; + break; + } ret = wait_read_select(cs->fd, *timeout); if (ret < 1) - goto out; + break; readlen = len - cs->bufofs; if (readlen >= PAGESIZE) readlen = PAGESIZE - 4; ret = recv(cs->fd, readbuf, readlen, MSG_DONTWAIT); if (ret < 1) - goto out; + break; add_bufline(cs, readbuf, ret); + tv_time(&now); + diff = tvdiff(&now, &start); + copy_tv(&start, &now); + *timeout -= diff; } -out: return ret; } From effeba49991c89cbcfa72f1f1c26a16bd214f12f Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Sun, 3 Jan 2016 18:19:19 +1100 Subject: [PATCH 535/544] Fix logic fails --- src/ckpool.c | 19 ++++++++++++------- src/connector.c | 25 ++++++++++++------------- 2 files changed, 24 insertions(+), 20 deletions(-) diff --git a/src/ckpool.c b/src/ckpool.c index f2c6054a..00f9a908 100644 --- a/src/ckpool.c +++ b/src/ckpool.c @@ -556,8 +556,8 @@ static void add_bufline(connsock_t *cs, const char *readbuf, const int len) static int read_cs_length(connsock_t *cs, float *timeout, int len) { tv_t start, now; - int ret = len; float diff; + int ret; tv_time(&start); @@ -568,23 +568,25 @@ static int read_cs_length(connsock_t *cs, float *timeout, int len) if (*timeout < 0) { LOGDEBUG("Timed out in read_cs_length"); ret = 0; - break; + goto out; } ret = wait_read_select(cs->fd, *timeout); if (ret < 1) - break; + goto out; readlen = len - cs->bufofs; if (readlen >= PAGESIZE) readlen = PAGESIZE - 4; ret = recv(cs->fd, readbuf, readlen, MSG_DONTWAIT); if (ret < 1) - break; + goto out; add_bufline(cs, readbuf, ret); tv_time(&now); diff = tvdiff(&now, &start); copy_tv(&start, &now); *timeout -= diff; } + ret = len; +out: return ret; } @@ -741,7 +743,7 @@ out: if (ret < 0) { empty_buffer(cs); dealloc(cs->buf); - } else if (ret == 3 && !strncmp(cs->buf, gzip_magic, 3)) + } else if (ret == 3 && !memcmp(cs->buf, gzip_magic, 3)) ret = read_gz_line(cs, timeout); return ret; } @@ -758,7 +760,8 @@ int write_cs(connsock_t *cs, const char *buf, int len) uint32_t msglen; int ret; - /* Connsock doesn't expect gz compressed messages */ + /* Connsock doesn't expect gz compressed messages. Only compress if it's + * larger than one MTU. */ if (!cs->gz || len <= 1492) return write_socket(cs->fd, buf, len); compsize = round_up_page(len + 12); @@ -770,8 +773,10 @@ int write_cs(connsock_t *cs, const char *buf, int len) LOGINFO("Failed to gz compress in write_cs, writing uncompressed"); return write_socket(cs->fd, buf, len); } + if (unlikely(compsize + 12 >= decompsize)) + return write_socket(cs->fd, buf, len); /* Copy gz magic header */ - sprintf(dest, gzip_magic); + memcpy(dest, gzip_magic, 4); /* Copy compressed message length */ msglen = htole32(compsize); memcpy(dest + 4, &msglen, 4); diff --git a/src/connector.c b/src/connector.c index 5593476b..2f373b18 100644 --- a/src/connector.c +++ b/src/connector.c @@ -1008,33 +1008,32 @@ static void send_client(cdata_t *cdata, const int64_t id, char *buf) /* Does this client accept compressed data? Only compress if it's * larger than one MTU. */ - if (client->gz) { + if (client->gz && len > 1492) { unsigned long compsize, decompsize = len; uint32_t msglen; - char *dest; + Bytef *dest; int ret; - compsize = round_up_page(len + 12); - dest = ckalloc(compsize); - compsize -= 12; - ret = compress((Bytef *)dest + 12, &compsize, (Bytef *)buf, len); + compsize = round_up_page(len); + dest = alloca(compsize); + ret = compress(dest, &compsize, (Bytef *)buf, len); if (unlikely(ret != Z_OK)) { - LOGINFO("Failed to gz compress in send_client, got %d sending uncompressed", ret); - free(dest); + LOGWARNING("Failed to gz compress in send_client, got %d sending uncompressed", ret); goto out; } + if (unlikely(compsize + 12 >= decompsize)) + goto out; /* Copy gz magic header */ - sprintf(dest, gzip_magic); + memcpy(buf, gzip_magic, 4); /* Copy compressed message length */ msglen = htole32(compsize); - memcpy(dest + 4, &msglen, 4); + memcpy(buf + 4, &msglen, 4); /* Copy decompressed message length */ msglen = htole32(decompsize); - memcpy(dest + 8, &msglen, 4); + memcpy(buf + 8, &msglen, 4); + memcpy(buf + 12, dest, compsize); len = compsize + 12; LOGDEBUG("Sending client message compressed %d from %lu", len, decompsize); - free(buf); - buf = dest; } out: sender_send = ckzalloc(sizeof(sender_send_t)); From 44e973734251f40eac81d612124f655ba8f870ce Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Sun, 3 Jan 2016 22:07:27 +1100 Subject: [PATCH 536/544] Update stats in cknode mode --- src/stratifier.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/stratifier.c b/src/stratifier.c index 55bdf65c..35c922ea 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -6561,7 +6561,7 @@ int stratifier(proc_instance_t *pi) } mutex_init(&sdata->stats_lock); - if (!ckp->passthrough) + if (!ckp->passthrough || ckp->node) create_pthread(&pth_statsupdate, statsupdate, ckp); mutex_init(&sdata->share_lock); From 3a9f5a491e494b744855a79e1d1ebee723d6fd71 Mon Sep 17 00:00:00 2001 From: ckolivas Date: Mon, 4 Jan 2016 09:14:08 +1100 Subject: [PATCH 537/544] Add an option to selectively enable/disable compression in passthrough modes --- README | 4 ++++ ckpassthrough.conf | 5 +---- src/ckpool.c | 4 ++++ src/ckpool.h | 3 +++ src/generator.c | 12 ++++++------ 5 files changed, 18 insertions(+), 10 deletions(-) diff --git a/README b/README index 8fd75b81..ee80ac62 100644 --- a/README +++ b/README @@ -265,6 +265,10 @@ new network blocks and is 100 by default. It is intended to be a backup only for when the notifier is not set up and only polls if the "notify" field is not set on a btcd. +"compress" : When running in a passthrough mode (redirector, passthrough, node), +should we gzip compress large packets. For passthroughs on a local network it +is recommended to disable this. Default is enabled. + "nonce1length" : This is optional allowing the extranonce1 length to be chosen from 2 to 8. Default 4 diff --git a/ckpassthrough.conf b/ckpassthrough.conf index a0c9a092..dc7b2ec1 100644 --- a/ckpassthrough.conf +++ b/ckpassthrough.conf @@ -6,14 +6,11 @@ "pass" : "pass" } ], -"update_interval" : 30, "serverurl" : [ "192.168.1.100:3334", "127.0.0.1:3334" ], -"mindiff" : 1, -"startdiff" : 42, -"maxdiff" : 0, +"compress" : true, "logdir" : "logs" } Comments from here on are ignored. diff --git a/src/ckpool.c b/src/ckpool.c index 00f9a908..3f8e4aab 100644 --- a/src/ckpool.c +++ b/src/ckpool.c @@ -1539,6 +1539,7 @@ static void parse_config(ckpool_t *ckp) ckp->btcsig[38] = '\0'; } json_get_int(&ckp->blockpoll, json_conf, "blockpoll"); + json_get_bool(&ckp->compress, json_conf, "compress"); json_get_int(&ckp->nonce1length, json_conf, "nonce1length"); json_get_int(&ckp->nonce2length, json_conf, "nonce2length"); json_get_int(&ckp->update_interval, json_conf, "update_interval"); @@ -1907,6 +1908,9 @@ int main(int argc, char **argv) if (ret && errno != EEXIST) quit(1, "Failed to make directory %s", ckp.socket_dir); + /* Set default on */ + ckp.compress = true; + parse_config(&ckp); /* Set defaults if not found in config file */ if (!ckp.btcds) { diff --git a/src/ckpool.h b/src/ckpool.h index 52017027..0ab30f4c 100644 --- a/src/ckpool.h +++ b/src/ckpool.h @@ -192,6 +192,9 @@ struct ckpool_instance { /* Are we a redirecting passthrough */ bool redirector; + /* Should we compress large packets in passthrough modes */ + bool compress; + /* Are we running as a proxy */ bool proxy; diff --git a/src/generator.c b/src/generator.c index 353dc641..49b0f215 100644 --- a/src/generator.c +++ b/src/generator.c @@ -766,7 +766,7 @@ out: } /* cs semaphore must be held */ -static bool passthrough_stratum(connsock_t *cs, proxy_instance_t *proxi) +static bool passthrough_stratum(ckpool_t *ckp, connsock_t *cs, proxy_instance_t *proxi) { json_t *req, *val = NULL, *res_val, *err_val; bool res, ret = false; @@ -774,7 +774,7 @@ static bool passthrough_stratum(connsock_t *cs, proxy_instance_t *proxi) JSON_CPACK(req, "{ss,sb,s[s]}", "method", "mining.passthrough", - "gz", json_true(), + "gz", ckp->compress, "params", PACKAGE"/"VERSION); res = send_json_msg(cs, req); json_decref(req); @@ -812,7 +812,7 @@ out: } /* cs semaphore must be held */ -static bool node_stratum(connsock_t *cs, proxy_instance_t *proxi) +static bool node_stratum(ckpool_t *ckp, connsock_t *cs, proxy_instance_t *proxi) { json_t *req, *val = NULL, *res_val, *err_val; bool res, ret = false; @@ -820,7 +820,7 @@ static bool node_stratum(connsock_t *cs, proxy_instance_t *proxi) JSON_CPACK(req, "{ss,sb,s[s]}", "method", "mining.node", - "gz", json_true(), + "gz", ckp->compress, "params", PACKAGE"/"VERSION); res = send_json_msg(cs, req); @@ -1855,7 +1855,7 @@ static bool proxy_alive(ckpool_t *ckp, proxy_instance_t *proxi, connsock_t *cs, goto out; } if (ckp->node) { - if (!node_stratum(cs, proxi)) { + if (!node_stratum(ckp, cs, proxi)) { LOGWARNING("Failed initial node setup to %s:%s !", cs->url, cs->port); goto out; @@ -1864,7 +1864,7 @@ static bool proxy_alive(ckpool_t *ckp, proxy_instance_t *proxi, connsock_t *cs, goto out; } if (ckp->passthrough) { - if (!passthrough_stratum(cs, proxi)) { + if (!passthrough_stratum(ckp, cs, proxi)) { LOGWARNING("Failed initial passthrough to %s:%s !", cs->url, cs->port); goto out; From 5221f2dde735b0cfa98b799a7233d2e03eefff77 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Mon, 4 Jan 2016 09:19:33 +1100 Subject: [PATCH 538/544] Add sample cknode.conf file. --- cknode.conf | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100644 cknode.conf diff --git a/cknode.conf b/cknode.conf new file mode 100644 index 00000000..118d57b6 --- /dev/null +++ b/cknode.conf @@ -0,0 +1,35 @@ +{ +"btcd" : [ + { + "url" : "localhost:8332", + "auth" : "user", + "pass" : "pass", + "notify" : true + }, + { + "url" : "backup:8332", + "auth" : "user", + "pass" : "pass", + "notify" : false + } +], +"proxy" : [ + { + "url" : "ckpool.org:3333", + "auth" : "user", + "pass" : "pass" + }, + { + "url" : "backup.ckpool.org:3333", + "auth" : "user", + "pass" : "pass" + } +], +"serverurl" : [ + "192.168.1.100:3334", + "127.0.0.1:3334" + ], +"compress" : true, +"logdir" : "logs" +} +Comments from here on are ignored. From 6c128328b7d0fc51a384d6626f6d7f2a157f8c8a Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Mon, 4 Jan 2016 09:57:10 +1100 Subject: [PATCH 539/544] Don't reconnect passthroughs when there are no clients --- src/generator.c | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/src/generator.c b/src/generator.c index 49b0f215..06fe79ce 100644 --- a/src/generator.c +++ b/src/generator.c @@ -1804,7 +1804,7 @@ static void passthrough_send(ckpool_t *ckp, pass_msg_t *pm) LOGDEBUG("Sending upstream json msg: %s", pm->msg); len = strlen(pm->msg); sent = write_cs(cs, pm->msg, len); - if (unlikely(sent != len)) { + if (unlikely(sent != len && cs->fd)) { LOGWARNING("Failed to passthrough %d bytes of message %s, attempting reconnect", len, pm->msg); Close(cs->fd); @@ -1894,8 +1894,7 @@ out: if (!ret) { send_stratifier_deadproxy(ckp, proxi->id, proxi->subid); /* Close and invalidate the file handle */ - if (cs->fd > 0) - Close(cs->fd); + Close(cs->fd); } proxi->alive = ret; cksem_post(&cs->sem); @@ -2022,19 +2021,21 @@ static void *passthrough_recv(void *arg) /* Make sure we receive a line within 90 seconds */ cksem_wait(&cs->sem); ret = read_socket_line(cs, &timeout); - if (ret < 1) { - LOGWARNING("Proxy %d:%s failed to read_socket_line in passthrough_recv, attempting reconnect", + /* Simply forward the message on, as is, to the connector to + * process. Possibly parse parameters sent by upstream pool + * here */ + if (likely(ret > 0)) { + LOGDEBUG("Received upstream msg: %s", cs->buf); + send_proc(ckp->connector, cs->buf); + } else if (ret < 0) { + /* Read failure */ + LOGWARNING("Passthrough %d:%s failed to read_socket_line in passthrough_recv, attempting reconnect", proxi->id, proxi->url); alive = proxi->alive = false; Close(cs->fd); reconnect_generator(ckp); - cksem_post(&cs->sem); - continue; - } - /* Simply forward the message on, as is, to the connector to - * process. Possibly parse parameters sent by upstream pool - * here */ - send_proc(ckp->connector, cs->buf); + } else /* Idle, likely no clients */ + LOGDEBUG("Passthrough %d:%s no messages received", proxi->id, proxi->url); cksem_post(&cs->sem); } return NULL; From 47ecddf88a4c0b5d10710e502c64795a77937364 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Mon, 4 Jan 2016 10:48:46 +1100 Subject: [PATCH 540/544] Pass select timed out return of zero in read_socket_line --- src/ckpool.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/ckpool.c b/src/ckpool.c index 3f8e4aab..358054b2 100644 --- a/src/ckpool.c +++ b/src/ckpool.c @@ -706,11 +706,11 @@ int read_socket_line(connsock_t *cs, float *timeout) goto out; } ret = wait_read_select(cs->fd, *timeout); - if (ret < 0) { + if (ret < 1) { if (cs->ckp->proxy) - LOGINFO("Select failed in read_socket_line"); + LOGINFO("Select %s in read_socket_line", !ret ? "timed out" : "failed"); else - LOGERR("Select failed in read_socket_line"); + LOGERR("Select %s in read_socket_line", !ret ? "timed out" : "failed"); goto out; } ret = recv(cs->fd, readbuf, PAGESIZE - 4, MSG_DONTWAIT); From 5b816982e5e1c9a715dbcba398b7014d34ffc1e1 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Mon, 4 Jan 2016 12:10:19 +1100 Subject: [PATCH 541/544] Compression of any sort is slower than ordinary network transfers so remove it --- README | 4 -- configure.ac | 1 - src/ckpool.c | 177 +---------------------------------------------- src/ckpool.h | 9 --- src/connector.c | 104 +--------------------------- src/generator.c | 22 ++---- src/stratifier.c | 17 ++--- 7 files changed, 15 insertions(+), 319 deletions(-) diff --git a/README b/README index ee80ac62..8fd75b81 100644 --- a/README +++ b/README @@ -265,10 +265,6 @@ new network blocks and is 100 by default. It is intended to be a backup only for when the notifier is not set up and only polls if the "notify" field is not set on a btcd. -"compress" : When running in a passthrough mode (redirector, passthrough, node), -should we gzip compress large packets. For passthroughs on a local network it -is recommended to disable this. Default is enabled. - "nonce1length" : This is optional allowing the extranonce1 length to be chosen from 2 to 8. Default 4 diff --git a/configure.ac b/configure.ac index ff33c36e..12c688e7 100644 --- a/configure.ac +++ b/configure.ac @@ -54,7 +54,6 @@ AC_ARG_WITH([ckdb], #AC_SEARCH_LIBS(whatgoeshere?, rt, , echo "Error: Required library realtime not found." && exit 1) AC_SEARCH_LIBS(exp, m, , echo "Error: Required library math not found." && exit 1) -AC_SEARCH_LIBS(compress, z , , echo "Error: Required library zlib1g-dev not found." && exit 1) AC_SEARCH_LIBS(pthread_mutex_trylock, pthread, , "Error: Required library pthreads not found." && exit 1) if test "x$ckdb" != "xno"; then diff --git a/src/ckpool.c b/src/ckpool.c index 358054b2..153126a2 100644 --- a/src/ckpool.c +++ b/src/ckpool.c @@ -36,8 +36,6 @@ ckpool_t *global_ckp; -const char gzip_magic[] = "\x1f\xd5\x01\n"; - static void proclog(ckpool_t *ckp, char *msg) { FILE *LOGFP; @@ -553,129 +551,6 @@ static void add_bufline(connsock_t *cs, const char *readbuf, const int len) cs->buf[cs->bufofs] = '\0'; } -static int read_cs_length(connsock_t *cs, float *timeout, int len) -{ - tv_t start, now; - float diff; - int ret; - - tv_time(&start); - - while (cs->bufofs < len) { - char readbuf[PAGESIZE]; - int readlen; - - if (*timeout < 0) { - LOGDEBUG("Timed out in read_cs_length"); - ret = 0; - goto out; - } - ret = wait_read_select(cs->fd, *timeout); - if (ret < 1) - goto out; - readlen = len - cs->bufofs; - if (readlen >= PAGESIZE) - readlen = PAGESIZE - 4; - ret = recv(cs->fd, readbuf, readlen, MSG_DONTWAIT); - if (ret < 1) - goto out; - add_bufline(cs, readbuf, ret); - tv_time(&now); - diff = tvdiff(&now, &start); - copy_tv(&start, &now); - *timeout -= diff; - } - ret = len; -out: - return ret; -} - -static int read_gz_line(connsock_t *cs, float *timeout) -{ - unsigned long compsize, res, decompsize; - char *buf, *dest = NULL, *eom; - int ret, buflen; - uint32_t msglen; - - /* Remove gz header */ - clear_bufline(cs); - - /* Get data sizes */ - ret = read_cs_length(cs, timeout, 8); - if (ret != 8) { - ret = -1; - goto out; - } - - memcpy(&msglen, cs->buf, 4); - compsize = le32toh(msglen); - memcpy(&msglen, cs->buf + 4, 4); - decompsize = le32toh(msglen); - - /* Remove the gz variables */ - cs->buflen = cs->bufofs - 8; - cs->bufofs = 8; - clear_bufline(cs); - - if (unlikely(compsize < 1 || compsize > 0x80000000 || - decompsize < 1 || decompsize > 0x80000000)) { - LOGWARNING("Invalid message length comp %lu decomp %lu sent to read_gz_line", compsize, decompsize); - ret = -1; - goto out; - } - - /* Get compressed data */ - ret = read_cs_length(cs, timeout, compsize); - if (ret != (int)compsize) { - LOGWARNING("Failed to read %lu compressed bytes in read_gz_line, got %d", compsize, ret); - ret = -1; - goto out; - } - - /* Clear out all the compressed data */ - cs->buflen = cs->bufofs - compsize; - cs->bufofs = compsize; - clear_bufline(cs); - - /* Do decompresion and buffer reconstruction here */ - res = round_up_page(decompsize); - dest = ckalloc(res); - ret = uncompress((Bytef *)dest, &res, (Bytef *)cs->buf, compsize); - if (ret != Z_OK || res != decompsize) { - LOGWARNING("Failed to decompress %lu bytes in read_gz_line, result %d got %lu", - decompsize, ret, res); - ret = -1; - goto out; - } - - eom = dest + decompsize - 1; - if (memcmp(eom, "\n", 1)) { - LOGWARNING("Failed to find EOM in decompressed data in read_gz_line"); - ret = -1; - goto out; - } - - *eom = '\0'; - ret = decompsize - 1; - /* Wedge the decompressed buffer back to the start of cs->buf */ - buf = cs->buf; - buflen = cs->bufofs; - cs->buf = dest; - dest = NULL; - cs->bufofs = decompsize; - if (buflen) { - add_bufline(cs, buf, buflen); - cs->buflen = buflen; - cs->bufofs = decompsize; - } else - cs->buflen = cs->bufofs = 0; -out: - free(dest); - if (ret < 1) - empty_buffer(cs); - return ret; -} - /* Read from a socket into cs->buf till we get an '\n', converting it to '\0' * and storing how much extra data we've received, to be moved to the beginning * of the buffer for use on the next receive. */ @@ -743,53 +618,7 @@ out: if (ret < 0) { empty_buffer(cs); dealloc(cs->buf); - } else if (ret == 3 && !memcmp(cs->buf, gzip_magic, 3)) - ret = read_gz_line(cs, timeout); - return ret; -} - -/* gzip compressed block structure: - * - 4 byte magic header gzip_magic "\x1f\xd5\x01\n" - * - 4 byte LE encoded compressed size - * - 4 byte LE encoded decompressed size - */ -int write_cs(connsock_t *cs, const char *buf, int len) -{ - unsigned long compsize, decompsize = len; - char *dest = NULL; - uint32_t msglen; - int ret; - - /* Connsock doesn't expect gz compressed messages. Only compress if it's - * larger than one MTU. */ - if (!cs->gz || len <= 1492) - return write_socket(cs->fd, buf, len); - compsize = round_up_page(len + 12); - dest = alloca(compsize); - /* Do compression here */ - compsize -= 12; - ret = compress((Bytef *)dest + 12, &compsize, (Bytef *)buf, len); - if (ret != Z_OK) { - LOGINFO("Failed to gz compress in write_cs, writing uncompressed"); - return write_socket(cs->fd, buf, len); - } - if (unlikely(compsize + 12 >= decompsize)) - return write_socket(cs->fd, buf, len); - /* Copy gz magic header */ - memcpy(dest, gzip_magic, 4); - /* Copy compressed message length */ - msglen = htole32(compsize); - memcpy(dest + 4, &msglen, 4); - /* Copy decompressed message length */ - msglen = htole32(decompsize); - memcpy(dest + 8, &msglen, 4); - len = compsize + 12; - LOGDEBUG("Writing connsock message compressed %d from %lu", len, decompsize); - ret = write_socket(cs->fd, dest, len); - if (ret == len) - ret = decompsize; - else - ret = -1; + } return ret; } @@ -1539,7 +1368,6 @@ static void parse_config(ckpool_t *ckp) ckp->btcsig[38] = '\0'; } json_get_int(&ckp->blockpoll, json_conf, "blockpoll"); - json_get_bool(&ckp->compress, json_conf, "compress"); json_get_int(&ckp->nonce1length, json_conf, "nonce1length"); json_get_int(&ckp->nonce2length, json_conf, "nonce2length"); json_get_int(&ckp->update_interval, json_conf, "update_interval"); @@ -1908,9 +1736,6 @@ int main(int argc, char **argv) if (ret && errno != EEXIST) quit(1, "Failed to make directory %s", ckp.socket_dir); - /* Set default on */ - ckp.compress = true; - parse_config(&ckp); /* Set defaults if not found in config file */ if (!ckp.btcds) { diff --git a/src/ckpool.h b/src/ckpool.h index 0ab30f4c..0829d36c 100644 --- a/src/ckpool.h +++ b/src/ckpool.h @@ -82,9 +82,6 @@ struct connsock { ckpool_t *ckp; /* Semaphore used to serialise request/responses */ sem_t sem; - - /* Has the other end acknowledged it can receive gz compressed data */ - bool gz; }; typedef struct connsock connsock_t; @@ -192,9 +189,6 @@ struct ckpool_instance { /* Are we a redirecting passthrough */ bool redirector; - /* Should we compress large packets in passthrough modes */ - bool compress; - /* Are we running as a proxy */ bool proxy; @@ -287,8 +281,6 @@ static const char __maybe_unused *stratum_msgs[] = { "" }; -extern const char gzip_magic[]; - #ifdef USE_CKDB #define CKP_STANDALONE(CKP) ((CKP)->standalone == true) #else @@ -309,7 +301,6 @@ ckpool_t *global_ckp; bool ping_main(ckpool_t *ckp); void empty_buffer(connsock_t *cs); int read_socket_line(connsock_t *cs, float *timeout); -int write_cs(connsock_t *cs, const char *buf, int len); void _send_proc(proc_instance_t *pi, const char *msg, const char *file, const char *func, const int line); #define send_proc(pi, msg) _send_proc(pi, msg, __FILE__, __func__, __LINE__) char *_send_recv_proc(proc_instance_t *pi, const char *msg, int writetimeout, int readtimedout, diff --git a/src/connector.c b/src/connector.c index 2f373b18..0c284fa1 100644 --- a/src/connector.c +++ b/src/connector.c @@ -64,12 +64,6 @@ struct client_instance { /* Is this the parent passthrough client */ bool passthrough; - /* Does this client expect gz compression? */ - bool gz; - bool compressed; /* Currently receiving a compressed message */ - unsigned long compsize; /* Expected compressed data size */ - unsigned long decompsize; /* Expected decompressed data size */ - /* Linked list of shares in redirector mode.*/ share_t *shares; @@ -485,34 +479,6 @@ retry: return; } client->bufofs += ret; -compressed: - if (client->compressed) { - unsigned long res; - - if (client->bufofs < client->compsize) - goto retry; - res = PAGESIZE - 4; - if (unlikely(client->decompsize > res)) { - LOGNOTICE("Client attempting to send oversize compressed message, disconnecting"); - invalidate_client(ckp, cdata, client); - return; - } - ret = uncompress((Bytef *)msg, &res, (Bytef *)client->buf, client->compsize); - if (ret != Z_OK || res != client->decompsize) { - LOGNOTICE("Failed to decompress %lu from %lu bytes in parse_client_msg, got %d", - client->decompsize, client->compsize, ret); - invalidate_client(ckp, cdata, client); - return; - } - LOGDEBUG("Received client message compressed %lu from %lu", - client->compsize, client->decompsize); - msg[res] = '\0'; - client->bufofs -= client->compsize; - if (client->bufofs) - memmove(client->buf, client->buf + buflen, client->bufofs); - client->compressed = false; - goto parse; - } reparse: eol = memchr(client->buf, '\n', client->bufofs); if (!eol) @@ -526,39 +492,11 @@ reparse: return; } - /* Look for a compression header */ - if (!strncmp(client->buf, gzip_magic, 3)) { - uint32_t msglen; - - /* Do we have the whole header? If not, keep reading */ - if (client->bufofs < 12) - goto retry; - memcpy(&msglen, client->buf + 4, 4); - client->compsize = le32toh(msglen); - memcpy(&msglen, client->buf + 8, 4); - client->decompsize = le32toh(msglen); - if (unlikely(!client->compsize || !client->decompsize || - client->compsize > MAX_MSGSIZE || client->decompsize > MAX_MSGSIZE)) { - LOGNOTICE("Client id %"PRId64" invalid compressed message size %lu/%lu, disconnecting", - client->id, client->compsize, client->decompsize); - invalidate_client(ckp, cdata, client); - return; - } - client->bufofs -= 12; - if (client->bufofs > 0) - memmove(client->buf, client->buf + 12, client->bufofs); - client->compressed = true; - if (client->bufofs >= client->compsize) - goto compressed; - goto retry; - } - memcpy(msg, client->buf, buflen); msg[buflen] = '\0'; client->bufofs -= buflen; memmove(client->buf, client->buf + buflen, client->bufofs); client->buf[client->bufofs] = '\0'; -parse: if (!(val = json_loads(msg, 0, NULL))) { char *buf = strdup("Invalid JSON, disconnecting\n"); @@ -1006,36 +944,6 @@ static void send_client(cdata_t *cdata, const int64_t id, char *buf) test_redirector_shares(ckp, client, buf); } - /* Does this client accept compressed data? Only compress if it's - * larger than one MTU. */ - if (client->gz && len > 1492) { - unsigned long compsize, decompsize = len; - uint32_t msglen; - Bytef *dest; - int ret; - - compsize = round_up_page(len); - dest = alloca(compsize); - ret = compress(dest, &compsize, (Bytef *)buf, len); - if (unlikely(ret != Z_OK)) { - LOGWARNING("Failed to gz compress in send_client, got %d sending uncompressed", ret); - goto out; - } - if (unlikely(compsize + 12 >= decompsize)) - goto out; - /* Copy gz magic header */ - memcpy(buf, gzip_magic, 4); - /* Copy compressed message length */ - msglen = htole32(compsize); - memcpy(buf + 4, &msglen, 4); - /* Copy decompressed message length */ - msglen = htole32(decompsize); - memcpy(buf + 8, &msglen, 4); - memcpy(buf + 12, dest, compsize); - len = compsize + 12; - LOGDEBUG("Sending client message compressed %d from %lu", len, decompsize); - } -out: sender_send = ckzalloc(sizeof(sender_send_t)); sender_send->client = client; sender_send->buf = buf; @@ -1065,7 +973,7 @@ static void passthrough_client(cdata_t *cdata, client_instance_t *client) LOGINFO("Connector adding passthrough client %"PRId64, client->id); client->passthrough = true; - ASPRINTF(&buf, "{\"result\": true, \"gz\": true}\n"); + ASPRINTF(&buf, "{\"result\": true}\n"); send_client(cdata, client->id, buf); } @@ -1242,15 +1150,10 @@ retry: sscanf(buf, "loglevel=%d", &ckp->loglevel); } else if (cmdmatch(buf, "shutdown")) { goto out; - } else if (cmdmatch(buf, "pass")) { + } else if (cmdmatch(buf, "passthrough")) { client_instance_t *client; - bool gz = false; - if (strstr(buf, "gz")) { - gz = true; - ret = sscanf(buf, "passgz=%"PRId64, &client_id); - } else - ret = sscanf(buf, "passthrough=%"PRId64, &client_id); + ret = sscanf(buf, "passthrough=%"PRId64, &client_id); if (ret < 0) { LOGDEBUG("Connector failed to parse passthrough command: %s", buf); goto retry; @@ -1260,7 +1163,6 @@ retry: LOGINFO("Connector failed to find client id %"PRId64" to pass through", client_id); goto retry; } - client->gz = gz; passthrough_client(cdata, client); dec_instance_ref(cdata, client); } else if (cmdmatch(buf, "getxfd")) { diff --git a/src/generator.c b/src/generator.c index 06fe79ce..b9d42a15 100644 --- a/src/generator.c +++ b/src/generator.c @@ -766,15 +766,14 @@ out: } /* cs semaphore must be held */ -static bool passthrough_stratum(ckpool_t *ckp, connsock_t *cs, proxy_instance_t *proxi) +static bool passthrough_stratum(connsock_t *cs, proxy_instance_t *proxi) { json_t *req, *val = NULL, *res_val, *err_val; bool res, ret = false; float timeout = 10; - JSON_CPACK(req, "{ss,sb,s[s]}", + JSON_CPACK(req, "{ss,s[s]}", "method", "mining.passthrough", - "gz", ckp->compress, "params", PACKAGE"/"VERSION); res = send_json_msg(cs, req); json_decref(req); @@ -799,9 +798,6 @@ static bool passthrough_stratum(ckpool_t *ckp, connsock_t *cs, proxy_instance_t LOGWARNING("Denied passthrough for stratum"); goto out; } - json_get_bool(&cs->gz, val, "gz"); - if (cs->gz) - LOGNOTICE("Negotiated gz compression with pool"); proxi->passthrough = true; out: if (val) @@ -812,15 +808,14 @@ out: } /* cs semaphore must be held */ -static bool node_stratum(ckpool_t *ckp, connsock_t *cs, proxy_instance_t *proxi) +static bool node_stratum(connsock_t *cs, proxy_instance_t *proxi) { json_t *req, *val = NULL, *res_val, *err_val; bool res, ret = false; float timeout = 10; - JSON_CPACK(req, "{ss,sb,s[s]}", + JSON_CPACK(req, "{ss,s[s]}", "method", "mining.node", - "gz", ckp->compress, "params", PACKAGE"/"VERSION); res = send_json_msg(cs, req); @@ -846,9 +841,6 @@ static bool node_stratum(ckpool_t *ckp, connsock_t *cs, proxy_instance_t *proxi) LOGWARNING("Denied node setup for stratum"); goto out; } - json_get_bool(&cs->gz, val, "gz"); - if (cs->gz) - LOGNOTICE("Negotiated gz compression with pool"); proxi->node = true; out: if (val) @@ -1803,7 +1795,7 @@ static void passthrough_send(ckpool_t *ckp, pass_msg_t *pm) LOGDEBUG("Sending upstream json msg: %s", pm->msg); len = strlen(pm->msg); - sent = write_cs(cs, pm->msg, len); + sent = write_socket(cs->fd, pm->msg, len); if (unlikely(sent != len && cs->fd)) { LOGWARNING("Failed to passthrough %d bytes of message %s, attempting reconnect", len, pm->msg); @@ -1855,7 +1847,7 @@ static bool proxy_alive(ckpool_t *ckp, proxy_instance_t *proxi, connsock_t *cs, goto out; } if (ckp->node) { - if (!node_stratum(ckp, cs, proxi)) { + if (!node_stratum(cs, proxi)) { LOGWARNING("Failed initial node setup to %s:%s !", cs->url, cs->port); goto out; @@ -1864,7 +1856,7 @@ static bool proxy_alive(ckpool_t *ckp, proxy_instance_t *proxi, connsock_t *cs, goto out; } if (ckp->passthrough) { - if (!passthrough_stratum(ckp, cs, proxi)) { + if (!passthrough_stratum(cs, proxi)) { LOGWARNING("Failed initial passthrough to %s:%s !", cs->url, cs->port); goto out; diff --git a/src/stratifier.c b/src/stratifier.c index 35c922ea..13bd666a 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -5186,11 +5186,10 @@ static void add_mining_node(sdata_t *sdata, stratum_instance_t *client) /* Enter with client holding ref count */ static void parse_method(ckpool_t *ckp, sdata_t *sdata, stratum_instance_t *client, - const int64_t client_id, json_t *val, json_t *id_val, json_t *method_val, + const int64_t client_id, json_t *id_val, json_t *method_val, json_t *params_val) { const char *method; - bool var; /* Random broken clients send something not an integer as the id so we * copy the json item for id_val as is for the response. By far the @@ -5238,12 +5237,8 @@ static void parse_method(ckpool_t *ckp, sdata_t *sdata, stratum_instance_t *clie /* Add this client as a passthrough in the connector and * add it to the list of mining nodes in the stratifier */ - json_get_bool(&var, val, "gz"); add_mining_node(sdata, client); - if (var) - snprintf(buf, 255, "passgz=%"PRId64, client_id); - else - snprintf(buf, 255, "passthrough=%"PRId64, client_id); + snprintf(buf, 255, "passthrough=%"PRId64, client_id); send_proc(ckp->connector, buf); return; } @@ -5255,12 +5250,8 @@ static void parse_method(ckpool_t *ckp, sdata_t *sdata, stratum_instance_t *clie * is a passthrough and to manage its messages accordingly. No * data from this client id should ever come back to this * stratifier after this so drop the client in the stratifier. */ - json_get_bool(&var, val, "gz"); LOGNOTICE("Adding passthrough client %"PRId64" %s", client_id, client->address); - if (var) - snprintf(buf, 255, "passgz=%"PRId64, client_id); - else - snprintf(buf, 255, "passthrough=%"PRId64, client_id); + snprintf(buf, 255, "passthrough=%"PRId64, client_id); send_proc(ckp->connector, buf); drop_client(ckp, sdata, client_id); return; @@ -5478,7 +5469,7 @@ static void parse_instance_msg(ckpool_t *ckp, sdata_t *sdata, smsg_t *msg, strat send_json_err(sdata, client_id, id_val, "-1:params not found"); goto out; } - parse_method(ckp, sdata, client, client_id, val, id_val, method, params); + parse_method(ckp, sdata, client, client_id, id_val, method, params); out: free_smsg(msg); } From 25aecd33c82dd9320c2f33116e9f9f596c3a9bcb Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Mon, 4 Jan 2016 12:19:52 +1100 Subject: [PATCH 542/544] Selectively compact json that does not need to be visualised --- src/connector.c | 10 +++++----- src/stratifier.c | 4 ++-- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/connector.c b/src/connector.c index 0c284fa1..8dc31059 100644 --- a/src/connector.c +++ b/src/connector.c @@ -344,7 +344,7 @@ static void generator_drop_client(ckpool_t *ckp, const client_instance_t *client JSON_CPACK(val, "{si,sI:ss:si:ss:s[]}", "id", 42, "client_id", client->id, "address", client->address_name, "server", client->server, "method", "mining.term", "params"); - s = json_dumps(val, 0); + s = json_dumps(val, JSON_COMPACT); json_decref(val); send_proc(ckp->generator, s); free(s); @@ -520,7 +520,7 @@ reparse: json_object_set_new_nocheck(val, "address", json_string(client->address_name)); } json_object_set_new_nocheck(val, "server", json_integer(client->server)); - s = json_dumps(val, 0); + s = json_dumps(val, JSON_COMPACT); /* Do not send messages of clients we've already dropped. We * do this unlocked as the occasional false negative can be @@ -800,7 +800,7 @@ static void redirect_client(ckpool_t *ckp, client_instance_t *client) num = add_redirect(ckp, cdata, client); JSON_CPACK(val, "{sosss[ssi]}", "id", json_null(), "method", "client.reconnect", "params", ckp->redirecturl[num], ckp->redirectport[num], 0); - buf = json_dumps(val, JSON_EOL); + buf = json_dumps(val, JSON_EOL | JSON_COMPACT); json_decref(val); sender_send = ckzalloc(sizeof(sender_send_t)); @@ -935,7 +935,7 @@ static void send_client(cdata_t *cdata, const int64_t id, char *buf) json_object_set_new_nocheck(val, "client_id", json_integer(client->id)); json_object_set_new_nocheck(val, "address", json_string(client->address_name)); json_object_set_new_nocheck(val, "server", json_integer(client->server)); - msg = json_dumps(val, 0); + msg = json_dumps(val, JSON_COMPACT); json_decref(val); send_proc(ckp->stratifier, msg); free(msg); @@ -997,7 +997,7 @@ static void process_client_msg(cdata_t *cdata, const char *buf) if (client_id > 0xffffffffll) json_object_set_new_nocheck(json_msg, "client_id", json_integer(client_id & 0xffffffffll)); - msg = json_dumps(json_msg, JSON_EOL); + msg = json_dumps(json_msg, JSON_EOL | JSON_COMPACT); send_client(cdata, client_id, msg); json_decref(json_msg); } diff --git a/src/stratifier.c b/src/stratifier.c index 13bd666a..4f6fa553 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -4740,7 +4740,7 @@ static void submit_share(stratum_instance_t *client, const int64_t jobid, const JSON_CPACK(json_msg, "{sIsssssssIsIsi}", "jobid", jobid, "nonce2", enonce2, "ntime", ntime, "nonce", nonce, "client_id", client->id, "proxy", client->proxyid, "subproxy", client->subproxyid); - msg = json_dumps(json_msg, 0); + msg = json_dumps(json_msg, JSON_COMPACT); json_decref(json_msg); send_generator(ckp, msg, GEN_LAX); free(msg); @@ -5571,7 +5571,7 @@ static void ssend_process(ckpool_t *ckp, smsg_t *msg) /* Add client_id to the json message and send it to the * connector process to be delivered */ json_object_set_new_nocheck(msg->json_msg, "client_id", json_integer(msg->client_id)); - s = json_dumps(msg->json_msg, 0); + s = json_dumps(msg->json_msg, JSON_COMPACT); send_proc(ckp->connector, s); free(s); free_smsg(msg); From 407d8a8734f1162e5901b74c881e95ea982310f9 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Mon, 4 Jan 2016 12:26:05 +1100 Subject: [PATCH 543/544] Remove defunct options from sample conf files --- cknode.conf | 1 - ckpassthrough.conf | 1 - 2 files changed, 2 deletions(-) diff --git a/cknode.conf b/cknode.conf index 118d57b6..3a025572 100644 --- a/cknode.conf +++ b/cknode.conf @@ -29,7 +29,6 @@ "192.168.1.100:3334", "127.0.0.1:3334" ], -"compress" : true, "logdir" : "logs" } Comments from here on are ignored. diff --git a/ckpassthrough.conf b/ckpassthrough.conf index dc7b2ec1..c5b85bcd 100644 --- a/ckpassthrough.conf +++ b/ckpassthrough.conf @@ -10,7 +10,6 @@ "192.168.1.100:3334", "127.0.0.1:3334" ], -"compress" : true, "logdir" : "logs" } Comments from here on are ignored. From cb2c0577cb33128e3f2cc167a1dc02bbc01ae6d3 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Mon, 4 Jan 2016 13:40:27 +1100 Subject: [PATCH 544/544] Placeholders for api files --- src/Makefile.am | 2 +- src/ckpool.c | 1 - src/ckpool.h | 14 ++++++++++++++ src/generator.c | 1 - src/stratifier.c | 1 - 5 files changed, 15 insertions(+), 4 deletions(-) diff --git a/src/Makefile.am b/src/Makefile.am index 53561714..c4832dab 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -10,7 +10,7 @@ libckpool_la_LIBADD = @LIBS@ bin_PROGRAMS = ckpool ckpmsg notifier ckpool_SOURCES = ckpool.c ckpool.h generator.c generator.h bitcoin.c bitcoin.h \ stratifier.c stratifier.h connector.c connector.h uthash.h \ - utlist.h api.c api.h + utlist.h ckpool_LDADD = libckpool.la @JANSSON_LIBS@ ckpmsg_SOURCES = ckpmsg.c diff --git a/src/ckpool.c b/src/ckpool.c index 153126a2..6178b225 100644 --- a/src/ckpool.c +++ b/src/ckpool.c @@ -32,7 +32,6 @@ #include "generator.h" #include "stratifier.h" #include "connector.h" -#include "api.h" ckpool_t *global_ckp; diff --git a/src/ckpool.h b/src/ckpool.h index 0829d36c..f2afd133 100644 --- a/src/ckpool.h +++ b/src/ckpool.h @@ -324,4 +324,18 @@ bool json_get_bool(bool *store, const json_t *val, const char *res); bool json_getdel_int(int *store, json_t *val, const char *res); bool json_getdel_int64(int64_t *store, json_t *val, const char *res); + +/* API Placeholders for future API implementation */ +typedef struct apimsg apimsg_t; + +struct apimsg { + char *buf; + int sockd; +}; + +static inline void ckpool_api(ckpool_t __maybe_unused *ckp, apimsg_t __maybe_unused *apimsg) {}; +static inline json_t *json_encode_errormsg(json_error_t __maybe_unused *err_val) { return NULL; }; +static inline json_t *json_errormsg(const char __maybe_unused *fmt, ...) { return NULL; }; +static inline void send_api_response(json_t __maybe_unused *val, const int __maybe_unused sockd) {}; + #endif /* CKPOOL_H */ diff --git a/src/generator.c b/src/generator.c index b9d42a15..65b82da6 100644 --- a/src/generator.c +++ b/src/generator.c @@ -21,7 +21,6 @@ #include "bitcoin.h" #include "uthash.h" #include "utlist.h" -#include "api.h" struct notify_instance { /* Hash table data */ diff --git a/src/stratifier.c b/src/stratifier.c index 24d96fa0..011cefc9 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -27,7 +27,6 @@ #include "stratifier.h" #include "uthash.h" #include "utlist.h" -#include "api.h" #define MIN1 60 #define MIN5 300