From 638ce65c5b3c5c6444c302aa8369d8cb2e973526 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Tue, 13 Jan 2015 11:40:45 +1100 Subject: [PATCH 01/10] Add best share support per client/worker/user and display it per worker/user --- src/stratifier.c | 27 ++++++++++++++++++++++----- 1 file changed, 22 insertions(+), 5 deletions(-) diff --git a/src/stratifier.c b/src/stratifier.c index 4d34a3a1..42cafc2a 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -1,5 +1,5 @@ /* - * Copyright 2014 Con Kolivas + * Copyright 2014-2015 Con Kolivas * * This program is free software; you can redistribute it and/or modify it * under the terms of the GNU General Public License as published by the Free @@ -172,6 +172,8 @@ struct user_instance { int workers; + double best_diff; /* Best share found by this user */ + double dsps1; /* Diff shares per second, 1 minute rolling average */ double dsps5; /* ... 5 minute ... */ double dsps60;/* etc */ @@ -195,6 +197,7 @@ struct worker_instance { tv_t last_share; time_t start_time; + double best_diff; /* Best share found by this worker */ int mindiff; /* User chosen mindiff */ bool idle; @@ -254,6 +257,7 @@ struct stratum_instance { time_t last_txns; /* Last time this worker requested txn hashes */ int64_t suggest_diff; /* Stratum client suggested diff */ + double best_diff; /* Best share found by this instance */ }; struct share { @@ -2533,6 +2537,17 @@ static json_t *parse_submit(stratum_instance_t *client, json_t *json_msg, nonce2[len] = '\0'; } sdiff = submission_diff(client, wb, nonce2, ntime32, nonce, hash); + if (sdiff > client->best_diff) { + worker_instance_t *worker = client->worker_instance; + + client->best_diff = sdiff; + LOGINFO("User %s worker %s client %ld new best diff %lf", user_instance->username, + worker->workername, client->id, sdiff); + if (sdiff > worker->best_diff) + worker->best_diff = sdiff; + if (sdiff > user_instance->best_diff) + user_instance->best_diff = sdiff; + } bswap_256(sharehash, hash); __bin2hex(hexhash, sharehash, 32); @@ -3543,11 +3558,12 @@ static void *statsupdate(void *arg) ghs = worker->dsps1440 * nonces; suffix_string(ghs, suffix1440, 16, 0); - JSON_CPACK(val, "{ss,ss,ss,ss}", + JSON_CPACK(val, "{ss,ss,ss,ss,sf}", "hashrate1m", suffix1, "hashrate5m", suffix5, "hashrate1hr", suffix60, - "hashrate1d", suffix1440); + "hashrate1d", suffix1440, + "bestshare", worker->best_diff); snprintf(fname, 511, "%s/workers/%s", ckp->logdir, worker->workername); fp = fopen(fname, "we"); @@ -3587,13 +3603,14 @@ static void *statsupdate(void *arg) ghs = instance->dsps10080 * nonces; suffix_string(ghs, suffix10080, 16, 0); - JSON_CPACK(val, "{ss,ss,ss,ss,ss,si}", + JSON_CPACK(val, "{ss,ss,ss,ss,ss,si,sf}", "hashrate1m", suffix1, "hashrate5m", suffix5, "hashrate1hr", suffix60, "hashrate1d", suffix1440, "hashrate7d", suffix10080, - "workers", instance->workers); + "workers", instance->workers, + "bestshare", instance->best_diff); snprintf(fname, 511, "%s/users/%s", ckp->logdir, instance->username); fp = fopen(fname, "we"); From 906d24a554afc9c68624250cb03c61682e3a8c69 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Tue, 13 Jan 2015 12:45:12 +1100 Subject: [PATCH 02/10] Add an option to add an EOL marker to json_dumps --- src/jansson-2.6/src/dump.c | 9 ++++++++- src/jansson-2.6/src/jansson.h | 1 + src/jansson-2.6/src/jansson_private.h | 2 ++ src/jansson-2.6/src/memory.c | 20 ++++++++++++++++++++ 4 files changed, 31 insertions(+), 1 deletion(-) diff --git a/src/jansson-2.6/src/dump.c b/src/jansson-2.6/src/dump.c index a8e99c12..776f294d 100644 --- a/src/jansson-2.6/src/dump.c +++ b/src/jansson-2.6/src/dump.c @@ -427,6 +427,13 @@ static int do_dump(const json_t *json, size_t flags, int depth, } } +char *json_dump_dup(const char *str, size_t flags) +{ + if (flags & JSON_EOL) + return jsonp_eolstrdup(str); + return jsonp_strdup(str); +} + char *json_dumps(const json_t *json, size_t flags) { strbuffer_t strbuff; @@ -438,7 +445,7 @@ char *json_dumps(const json_t *json, size_t flags) if(json_dump_callback(json, dump_to_strbuffer, (void *)&strbuff, flags)) result = NULL; else - result = jsonp_strdup(strbuffer_value(&strbuff)); + result = json_dump_dup(strbuffer_value(&strbuff), flags); strbuffer_close(&strbuff); return result; diff --git a/src/jansson-2.6/src/jansson.h b/src/jansson-2.6/src/jansson.h index 8cc2760e..2c86c1f3 100644 --- a/src/jansson-2.6/src/jansson.h +++ b/src/jansson-2.6/src/jansson.h @@ -261,6 +261,7 @@ json_t *json_load_callback(json_load_callback_t callback, void *data, size_t fla #define JSON_ENCODE_ANY 0x200 #define JSON_ESCAPE_SLASH 0x400 #define JSON_NO_UTF8 0x800 +#define JSON_EOL 0x1000 typedef int (*json_dump_callback_t)(const char *buffer, size_t size, void *data); diff --git a/src/jansson-2.6/src/jansson_private.h b/src/jansson-2.6/src/jansson_private.h index 403b53a4..3ca4dffa 100644 --- a/src/jansson-2.6/src/jansson_private.h +++ b/src/jansson-2.6/src/jansson_private.h @@ -1,5 +1,6 @@ /* * Copyright (c) 2009-2013 Petri Lehtinen + * Copyright (c) 2015 Con Kolivas * * Jansson is free software; you can redistribute it and/or modify * it under the terms of the MIT license. See LICENSE for details. @@ -83,6 +84,7 @@ void* jsonp_malloc(size_t size); void jsonp_free(void *ptr); char *jsonp_strndup(const char *str, size_t length); char *jsonp_strdup(const char *str); +char *jsonp_eolstrdup(const char *str); /* Windows compatibility */ #ifdef _WIN32 diff --git a/src/jansson-2.6/src/memory.c b/src/jansson-2.6/src/memory.c index eb6cec54..0be24380 100644 --- a/src/jansson-2.6/src/memory.c +++ b/src/jansson-2.6/src/memory.c @@ -1,6 +1,7 @@ /* * Copyright (c) 2009-2013 Petri Lehtinen * Copyright (c) 2011-2012 Basile Starynkevitch + * Copyright (c) 2015 Con Kolivas * * Jansson is free software; you can redistribute it and/or modify it * under the terms of the MIT license. See LICENSE for details. @@ -49,6 +50,25 @@ char *jsonp_strdup(const char *str) return new_str; } +char *jsonp_eolstrdup(const char *str) +{ + char *new_str; + size_t len; + + len = strlen(str); + if(len == (size_t)-1) + return NULL; + + new_str = jsonp_malloc(len + 2); + if(!new_str) + return NULL; + + memcpy(new_str, str, len); + new_str[len] = '\n'; + new_str[len + 1] = '\0'; + return new_str; +} + void json_set_alloc_funcs(json_malloc_t malloc_fn, json_free_t free_fn) { do_malloc = malloc_fn; From 5c18e7e0f88e414a8383865e62c0dd23aa18cd36 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Tue, 13 Jan 2015 12:45:44 +1100 Subject: [PATCH 03/10] Use JSON_EOL in generator --- src/generator.c | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/generator.c b/src/generator.c index c1804029..dca9a86e 100644 --- a/src/generator.c +++ b/src/generator.c @@ -1,5 +1,5 @@ /* - * Copyright 2014 Con Kolivas + * Copyright 2014-2015 Con Kolivas * * This program is free software; you can redistribute it and/or modify it * under the terms of the GNU General Public License as published by the Free @@ -388,13 +388,11 @@ static bool send_json_msg(connsock_t *cs, json_t *json_msg) int len, sent; char *s; - s = json_dumps(json_msg, JSON_ESCAPE_SLASH); + s = json_dumps(json_msg, JSON_ESCAPE_SLASH | JSON_EOL); LOGDEBUG("Sending json msg: %s", s); len = strlen(s); sent = write_socket(cs->fd, s, len); - len += 1; dealloc(s); - sent += write_socket(cs->fd, "\n", 1); if (sent != len) { LOGWARNING("Failed to send %d bytes sent %d in send_json_msg", len, sent); return false; From e1e2d06a4f0cff91a3a192b46a2bece3e174c584 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Tue, 13 Jan 2015 12:52:33 +1100 Subject: [PATCH 04/10] Use JSON_EOL where suitable in stratifier --- src/stratifier.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/stratifier.c b/src/stratifier.c index 42cafc2a..a0cabb63 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -2634,9 +2634,9 @@ out_unlock: if (ckp->logshares) { fp = fopen(fname, "ae"); if (likely(fp)) { - s = json_dumps(val, 0); + s = json_dumps(val, JSON_EOL); len = strlen(s); - len = fprintf(fp, "%s\n", s); + len = fprintf(fp, "%s", s); free(s); fclose(fp); if (unlikely(len < 0)) @@ -3571,8 +3571,8 @@ static void *statsupdate(void *arg) LOGERR("Failed to fopen %s", fname); continue; } - s = json_dumps(val, JSON_NO_UTF8 | JSON_PRESERVE_ORDER); - fprintf(fp, "%s\n", s); + s = json_dumps(val, JSON_NO_UTF8 | JSON_PRESERVE_ORDER | JSON_EOL); + fprintf(fp, "%s", s); dealloc(s); json_decref(val); fclose(fp); @@ -3618,8 +3618,8 @@ static void *statsupdate(void *arg) LOGERR("Failed to fopen %s", fname); continue; } - s = json_dumps(val, JSON_NO_UTF8 | JSON_PRESERVE_ORDER); - fprintf(fp, "%s\n", s); + s = json_dumps(val, JSON_NO_UTF8 | JSON_PRESERVE_ORDER | JSON_EOL); + fprintf(fp, "%s", s); if (!idle) LOGNOTICE("User %s:%s", instance->username, s); dealloc(s); From 88c7ef3c204ac7e640e3ae35b0b29921fcdc83ce Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Tue, 13 Jan 2015 12:54:22 +1100 Subject: [PATCH 05/10] Use JSON_EOL in connector --- src/connector.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/connector.c b/src/connector.c index 7355f9aa..c23173e1 100644 --- a/src/connector.c +++ b/src/connector.c @@ -1,5 +1,5 @@ /* - * Copyright 2014 Con Kolivas + * Copyright 2014-2015 Con Kolivas * * This program is free software; you can redistribute it and/or modify it * under the terms of the GNU General Public License as published by the Free @@ -618,8 +618,7 @@ static void process_client_msg(cdata_t *cdata, const char *buf) json_object_set_new_nocheck(json_msg, "client_id", json_integer(passthrough_id)); } else client_id = client_id64; - msg = json_dumps(json_msg, 0); - realloc_strcat(&msg, "\n"); + msg = json_dumps(json_msg, JSON_EOL); send_client(cdata, client_id, msg); json_decref(json_msg); } From de0f3438e9d3e18a78c639af02a529f4e215fddf Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Tue, 13 Jan 2015 12:57:33 +1100 Subject: [PATCH 06/10] Don't use JSON_EOL for logged text --- src/stratifier.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/stratifier.c b/src/stratifier.c index a0cabb63..0b974d5f 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -3618,8 +3618,8 @@ static void *statsupdate(void *arg) LOGERR("Failed to fopen %s", fname); continue; } - s = json_dumps(val, JSON_NO_UTF8 | JSON_PRESERVE_ORDER | JSON_EOL); - fprintf(fp, "%s", s); + s = json_dumps(val, JSON_NO_UTF8 | JSON_PRESERVE_ORDER); + fprintf(fp, "%s\n", s); if (!idle) LOGNOTICE("User %s:%s", instance->username, s); dealloc(s); From 43dc33a54caefc130570d403d8f95e89efa88c04 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Tue, 13 Jan 2015 16:07:59 +1100 Subject: [PATCH 07/10] Age and discard old disconnected instance data after it's been used or no longer relevant --- src/stratifier.c | 46 +++++++++++++++++++++++++++++++++++++--------- 1 file changed, 37 insertions(+), 9 deletions(-) diff --git a/src/stratifier.c b/src/stratifier.c index 0b974d5f..20c4f4a5 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -255,6 +255,7 @@ struct stratum_instance { ckpool_t *ckp; time_t last_txns; /* Last time this worker requested txn hashes */ + time_t disconnected_time; /* Time this instance disconnected */ int64_t suggest_diff; /* Stratum client suggested diff */ double best_diff; /* Best share found by this instance */ @@ -1102,6 +1103,14 @@ static stratum_instance_t *__stratum_add_instance(ckpool_t *ckp, int64_t id, int return instance; } +/* Add a stratum instance to the dead instances list */ +static void kill_instance(sdata_t *sdata, stratum_instance_t *client) +{ + if (client->user_instance) + DL_DELETE(client->user_instance->instances, client); + LL_PREPEND(sdata->dead_instances, client); +} + /* Only supports a full ckpool instance sessionid with an 8 byte sessionid */ static bool disconnected_sessionid_exists(sdata_t *sdata, const char *sessionid, int64_t id) { @@ -1116,7 +1125,7 @@ static bool disconnected_sessionid_exists(sdata_t *sdata, const char *sessionid, /* Number is in BE but we don't swap either of them */ hex2bin(&session64, sessionid, 8); - ck_rlock(&sdata->instance_lock); + ck_ilock(&sdata->instance_lock); HASH_ITER(hh, sdata->stratum_instances, instance, tmp) { if (instance->id == id) continue; @@ -1127,10 +1136,18 @@ static bool disconnected_sessionid_exists(sdata_t *sdata, const char *sessionid, } instance = NULL; HASH_FIND(hh, sdata->disconnected_instances, &session64, sizeof(uint64_t), instance); - if (instance) + if (instance) { + /* If we've found a matching disconnected instance, use it only + * once and discard it */ + ck_ulock(&sdata->instance_lock); + HASH_DEL(sdata->disconnected_instances, instance); + kill_instance(sdata, instance); + ck_dwilock(&sdata->instance_lock); + ret = true; + } out_unlock: - ck_runlock(&sdata->instance_lock); + ck_uilock(&sdata->instance_lock); out: return ret; } @@ -1213,6 +1230,7 @@ static void dec_worker(ckpool_t *ckp, user_instance_t *instance) static void drop_client(sdata_t *sdata, int64_t id) { stratum_instance_t *client, *tmp; + time_t now_t = time(NULL); bool dec = false; LOGINFO("Stratifier dropping client %ld", id); @@ -1231,13 +1249,11 @@ static void drop_client(sdata_t *sdata, int64_t id) HASH_DEL(sdata->stratum_instances, client); HASH_FIND(hh, sdata->disconnected_instances, &client->enonce1_64, sizeof(uint64_t), old_client); /* Only keep around one copy of the old client in server mode */ - if (!client->ckp->proxy && !old_client && client->enonce1_64) + if (!client->ckp->proxy && !old_client && client->enonce1_64) { HASH_ADD(hh, sdata->disconnected_instances, enonce1_64, sizeof(uint64_t), client); - else { - if (client->user_instance) - DL_DELETE(client->user_instance->instances, client); - LL_PREPEND(sdata->dead_instances, client); - } + client->disconnected_time = time(NULL); + } else + kill_instance(sdata, client); } ck_wunlock(&sdata->instance_lock); @@ -1251,6 +1267,18 @@ static void drop_client(sdata_t *sdata, int64_t id) ck_wlock(&sdata->instance_lock); if (client) __dec_instance_ref(client); + /* Old disconnected instances will not have any valid shares so remove + * them from the disconnected instances list if they've been dead for + * more than 10 minutes */ + HASH_ITER(hh, sdata->disconnected_instances, client, tmp) { + if (now_t - client->disconnected_time < 600) + continue; + LOGINFO("Discarding aged disconnected instance %ld", client->id); + HASH_DEL(sdata->disconnected_instances, client); + kill_instance(sdata, client); + } + /* Discard any dead instances that no longer hold any reference counts, + * freeing up their memory safely */ DL_FOREACH_SAFE(sdata->dead_instances, client, tmp) { if (!client->ref) { LOGINFO("Stratifier discarding instance %ld", client->id); From 4103a3bae8862d505c6eba817c071480f579aab7 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Tue, 13 Jan 2015 16:30:52 +1100 Subject: [PATCH 08/10] Tell the stratifier to drop a client if the connector is told to drop it and cannot find it any more --- src/connector.c | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/src/connector.c b/src/connector.c index c23173e1..235ca6de 100644 --- a/src/connector.c +++ b/src/connector.c @@ -674,19 +674,6 @@ retry: * so look for them first. */ if (likely(buf[0] == '{')) { process_client_msg(cdata, buf); - } else if (cmdmatch(buf, "ping")) { - LOGDEBUG("Connector received ping request"); - send_unix_msg(sockd, "pong"); - } else if (cmdmatch(buf, "accept")) { - LOGDEBUG("Connector received accept signal"); - cdata->accept = true; - } else if (cmdmatch(buf, "reject")) { - LOGDEBUG("Connector received reject signal"); - cdata->accept = false; - } else if (cmdmatch(buf, "loglevel")) { - sscanf(buf, "loglevel=%d", &ckp->loglevel); - } else if (cmdmatch(buf, "shutdown")) { - goto out; } else if (cmdmatch(buf, "dropclient")) { client_instance_t *client; @@ -699,12 +686,26 @@ retry: client = ref_client_by_id(cdata, client_id); if (unlikely(!client)) { LOGINFO("Connector failed to find client id %ld to drop", client_id); + stratifier_drop_client(ckp, client_id); goto retry; } ret = drop_client(cdata, client); dec_instance_ref(cdata, client); if (ret >= 0) LOGINFO("Connector dropped client id: %ld", client_id); + } else if (cmdmatch(buf, "ping")) { + LOGDEBUG("Connector received ping request"); + send_unix_msg(sockd, "pong"); + } else if (cmdmatch(buf, "accept")) { + LOGDEBUG("Connector received accept signal"); + cdata->accept = true; + } else if (cmdmatch(buf, "reject")) { + LOGDEBUG("Connector received reject signal"); + cdata->accept = false; + } else if (cmdmatch(buf, "loglevel")) { + sscanf(buf, "loglevel=%d", &ckp->loglevel); + } else if (cmdmatch(buf, "shutdown")) { + goto out; } else if (cmdmatch(buf, "passthrough")) { client_instance_t *client; From 2545b081288209441a005d1b0344fcec9dfbe0cf Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Tue, 13 Jan 2015 16:49:16 +1100 Subject: [PATCH 09/10] Use invalidate in the connector when dropping a client to ensure the stratifier drops it as well --- src/connector.c | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/connector.c b/src/connector.c index 235ca6de..0bfae189 100644 --- a/src/connector.c +++ b/src/connector.c @@ -236,13 +236,14 @@ static void stratifier_drop_client(ckpool_t *ckp, int64_t id) * regularly but keep the instances in a linked list until their ref count * drops to zero when we can remove them lazily. Client must hold a reference * count. */ -static void invalidate_client(ckpool_t *ckp, cdata_t *cdata, client_instance_t *client) +static int invalidate_client(ckpool_t *ckp, cdata_t *cdata, client_instance_t *client) { client_instance_t *tmp; + int ret; - drop_client(cdata, client); + ret = drop_client(cdata, client); if (ckp->passthrough) - return; + goto out; stratifier_drop_client(ckp, client->id); /* Cull old unused clients lazily when there are no more reference @@ -256,6 +257,9 @@ static void invalidate_client(ckpool_t *ckp, cdata_t *cdata, client_instance_t * } } ck_wunlock(&cdata->lock); + +out: + return ret; } static void send_client(cdata_t *cdata, int64_t id, char *buf); @@ -689,7 +693,7 @@ retry: stratifier_drop_client(ckp, client_id); goto retry; } - ret = drop_client(cdata, client); + ret = invalidate_client(ckp, cdata, client); dec_instance_ref(cdata, client); if (ret >= 0) LOGINFO("Connector dropped client id: %ld", client_id); From 546768bfb78e03e5aead95e57c23d161c07e4aca Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Wed, 14 Jan 2015 20:10:51 +1100 Subject: [PATCH 10/10] Push version number to 0.8.1 --- configure.ac | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configure.ac b/configure.ac index 65d311c0..3983bd0d 100644 --- a/configure.ac +++ b/configure.ac @@ -1,4 +1,4 @@ -AC_INIT(ckpool, 0.8, kernel@kolivas.org) +AC_INIT(ckpool, 0.8.1, kernel@kolivas.org) AC_CANONICAL_SYSTEM AC_CONFIG_MACRO_DIR([m4])