From d8f5ceb0d23ae7115e735138b7169e228a9d1d6b Mon Sep 17 00:00:00 2001 From: kanoi Date: Wed, 25 Mar 2015 03:18:35 +1100 Subject: [PATCH 01/32] ckdb - retain shares with an unknown workinfoid and recheck for up to 60s --- src/ckdb.c | 53 ++++ src/ckdb.h | 44 +++- src/ckdb_cmd.c | 6 +- src/ckdb_data.c | 14 ++ src/ckdb_dbio.c | 629 ++++++++++++++++++++++++++++++++++++++---------- 5 files changed, 604 insertions(+), 142 deletions(-) diff --git a/src/ckdb.c b/src/ckdb.c index 6d12e221..fcc41e71 100644 --- a/src/ckdb.c +++ b/src/ckdb.c @@ -379,11 +379,15 @@ double current_ndiff; K_TREE *shares_root; K_LIST *shares_free; K_STORE *shares_store; +K_TREE *shares_early_root; +K_STORE *shares_early_store; // SHAREERRORS shareerrors.id.json={...} K_TREE *shareerrors_root; K_LIST *shareerrors_free; K_STORE *shareerrors_store; +K_TREE *shareerrors_early_root; +K_STORE *shareerrors_early_store; // SHARESUMMARY K_TREE *sharesummary_root; @@ -995,12 +999,16 @@ static void alloc_storage() shares_free = k_new_list("Shares", sizeof(SHARES), ALLOC_SHARES, LIMIT_SHARES, true); shares_store = k_new_store(shares_free); + shares_early_store = k_new_store(shares_free); shares_root = new_ktree(); + shares_early_root = new_ktree(); shareerrors_free = k_new_list("ShareErrors", sizeof(SHAREERRORS), ALLOC_SHAREERRORS, LIMIT_SHAREERRORS, true); shareerrors_store = k_new_store(shareerrors_free); + shareerrors_early_store = k_new_store(shareerrors_free); shareerrors_root = new_ktree(); + shareerrors_early_root = new_ktree(); sharesummary_free = k_new_list("ShareSummary", sizeof(SHARESUMMARY), ALLOC_SHARESUMMARY, LIMIT_SHARESUMMARY, true); @@ -1106,6 +1114,11 @@ static void alloc_storage() static void dealloc_storage() { + SHAREERRORS *shareerrors; + SHARES *shares; + K_ITEM *s_item; + char *st = NULL; + LOGWARNING("%s() logqueue ...", __func__); FREE_LISTS(logqueue); @@ -1151,7 +1164,47 @@ static void dealloc_storage() FREE_STORE_DATA(sharesummary); FREE_LIST_DATA(sharesummary); + if (shareerrors_early_store->count > 0) { + LOGERR("%s() *** shareerrors_early count %d ***", + __func__, shareerrors_early_store->count); + s_item = shareerrors_early_store->head; + while (s_item) { + DATA_SHAREERRORS(shareerrors, s_item); + LOGERR("%s(): %"PRId64"/%s/%"PRId32"/%s/%ld,%ld", + __func__, + shareerrors->workinfoid, + st = safe_text(shareerrors->workername), + shareerrors->errn, + shareerrors->error, + shareerrors->createdate.tv_sec, + shareerrors->createdate.tv_usec); + FREENULL(st); + s_item = s_item->next; + } + } + FREE_TREE(shareerrors_early); + FREE_STORE(shareerrors_early); FREE_ALL(shareerrors); + if (shares_early_store->count > 0) { + LOGERR("%s() *** shares_early count %d ***", + __func__, shares_early_store->count); + s_item = shares_early_store->head; + while (s_item) { + DATA_SHARES(shares, s_item); + LOGERR("%s(): %"PRId64"/%s/%s/%"PRId32"/%ld,%ld", + __func__, + shares->workinfoid, + st = safe_text(shares->workername), + shares->nonce, + shares->errn, + shares->createdate.tv_sec, + shares->createdate.tv_usec); + FREENULL(st); + s_item = s_item->next; + } + } + FREE_TREE(shares_early); + FREE_STORE(shares_early); FREE_ALL(shares); LOGWARNING("%s() workinfo ...", __func__); diff --git a/src/ckdb.h b/src/ckdb.h index ed9d615d..a0fdb2a1 100644 --- a/src/ckdb.h +++ b/src/ckdb.h @@ -55,7 +55,7 @@ #define DB_VLOCK "1" #define DB_VERSION "1.0.0" -#define CKDB_VERSION DB_VERSION"-1.035" +#define CKDB_VERSION DB_VERSION"-1.040" #define WHERE_FFL " - from %s %s() line %d" #define WHERE_FFL_HERE __FILE__, __func__, __LINE__ @@ -226,6 +226,7 @@ enum data_type { TYPE_BIGINT, TYPE_INT, TYPE_TV, + TYPE_BTV, TYPE_TVS, TYPE_CTV, TYPE_FTV, @@ -721,7 +722,7 @@ typedef struct transfer { // Suggest malloc use MMAP - 1913 = largest under 2MB #define ALLOC_TRANSFER 1913 #define LIMIT_TRANSFER 0 -#define CULL_TRANSFER 16 +#define CULL_TRANSFER 64 #define INIT_TRANSFER(_item) INIT_GENERIC(_item, transfer) #define DATA_TRANSFER(_var, _item) DATA_GENERIC(_var, _item, transfer, true) @@ -747,7 +748,7 @@ extern tv_t missing_secuser_max; typedef struct users { int64_t userid; char username[TXT_BIG+1]; - char usertrim[TXT_BIG+1]; // Non DB field + char usertrim[TXT_BIG+1]; // non-DB field // Anything in 'status' fails mining authentication char status[TXT_BIG+1]; char emailaddress[TXT_BIG+1]; @@ -843,7 +844,7 @@ typedef struct paymentaddresses { char payaddress[TXT_BIG+1]; int32_t payratio; HISTORYDATECONTROLFIELDS; - bool match; // Non-db field + bool match; // non-DB field } PAYMENTADDRESSES; #define ALLOC_PAYMENTADDRESSES 1024 @@ -873,7 +874,7 @@ typedef struct payments { char committxn[TXT_BIG+1]; char commitblockhash[TXT_BIG+1]; HISTORYDATECONTROLFIELDS; - K_ITEM *old_item; // Non-db field + K_ITEM *old_item; // non-DB field } PAYMENTS; #define ALLOC_PAYMENTS 1024 @@ -1024,6 +1025,8 @@ typedef struct shares { char error[TXT_SML+1]; char secondaryuserid[TXT_SML+1]; HISTORYDATECONTROLFIELDS; + int32_t redo; // non-DB field + int32_t oldcount; // non-DB field } SHARES; #define ALLOC_SHARES 10000 @@ -1034,6 +1037,13 @@ typedef struct shares { extern K_TREE *shares_root; extern K_LIST *shares_free; extern K_STORE *shares_store; +// shares unexpectedly before the workinfo +extern K_TREE *shares_early_root; +extern K_STORE *shares_early_store; + +/* Once a share is this old, it can only once more be + check for it's workinfoid and then be discarded */ +#define EARLYSHARESLIMIT 60.0 // SHAREERRORS shareerrors.id.json={...} typedef struct shareerrors { @@ -1045,9 +1055,11 @@ typedef struct shareerrors { char error[TXT_SML+1]; char secondaryuserid[TXT_SML+1]; HISTORYDATECONTROLFIELDS; + int32_t redo; // non-DB field + int32_t oldcount; // non-DB field } SHAREERRORS; -#define ALLOC_SHAREERRORS 10000 +#define ALLOC_SHAREERRORS 1000 #define LIMIT_SHAREERRORS 0 #define INIT_SHAREERRORS(_item) INIT_GENERIC(_item, shareerrors) #define DATA_SHAREERRORS(_var, _item) DATA_GENERIC(_var, _item, shareerrors, true) @@ -1055,6 +1067,9 @@ typedef struct shareerrors { extern K_TREE *shareerrors_root; extern K_LIST *shareerrors_free; extern K_STORE *shareerrors_store; +// shareerrors unexpectedly before the workinfo +extern K_TREE *shareerrors_early_root; +extern K_STORE *shareerrors_early_store; // SHARESUMMARY typedef struct sharesummary { @@ -1123,12 +1138,12 @@ typedef struct blocks { int64_t elapsed; char statsconfirmed[TXT_FLAG+1]; HISTORYDATECONTROLFIELDS; - bool ignore; // Non DB field + bool ignore; // non-DB field // Calculated only when = 0 double netdiff; - /* Non DB fields for the web page + /* non-DB fields for the web page * Calculate them once off/recalc them when required */ double blockdiffratio; double blockcdf; @@ -1193,7 +1208,7 @@ typedef struct miningpayouts { double diffacc; int64_t amount; HISTORYDATECONTROLFIELDS; - K_ITEM *old_item; // Non-db field + K_ITEM *old_item; // non-DB field } MININGPAYOUTS; #define ALLOC_MININGPAYOUTS 1000 @@ -1305,11 +1320,11 @@ typedef struct poolstats { double hashrate5m; double hashrate1hr; double hashrate24hr; - bool stored; // Non-db field + bool stored; // non-DB field SIMPLEDATECONTROLFIELDS; } POOLSTATS; -#define ALLOC_POOLSTATS 10000 +#define ALLOC_POOLSTATS 1000 #define LIMIT_POOLSTATS 0 #define INIT_POOLSTATS(_item) INIT_GENERIC(_item, poolstats) #define DATA_POOLSTATS(_var, _item) DATA_GENERIC(_var, _item, poolstats, true) @@ -1336,7 +1351,7 @@ typedef struct userstats { double hashrate5m; double hashrate1hr; double hashrate24hr; - bool idle; // Non-db field + bool idle; // non-DB field char summarylevel[TXT_FLAG+1]; // SUMMARY_NONE in RAM int32_t summarycount; tv_t statsdate; @@ -1348,7 +1363,7 @@ typedef struct userstats { * createdate batch, and thus could move all (complete) records * matching the createdate from userstats_eos_store into the tree */ -#define ALLOC_USERSTATS 10000 +#define ALLOC_USERSTATS 1000 #define LIMIT_USERSTATS 0 #define INIT_USERSTATS(_item) INIT_GENERIC(_item, userstats) #define DATA_USERSTATS(_var, _item) DATA_GENERIC(_var, _item, userstats, true) @@ -1689,6 +1704,7 @@ extern char *_data_to_buf(enum data_type typ, void *data, char *buf, size_t siz, #define ctv_to_buf(_data, _buf, _siz) _ctv_to_buf(_data, _buf, _siz, WHERE_FFL_HERE) #define ftv_to_buf(_data, _buf, _siz) _ftv_to_buf(_data, _buf, _siz, WHERE_FFL_HERE) #define tvs_to_buf(_data, _buf, _siz) _tvs_to_buf(_data, _buf, _siz, WHERE_FFL_HERE) +#define btv_to_buf(_data, _buf, _siz) _btv_to_buf(_data, _buf, _siz, WHERE_FFL_HERE) //#define blob_to_buf(_data, _buf, _siz) _blob_to_buf(_data, _buf, _siz, WHERE_FFL_HERE) #define double_to_buf(_data, _buf, _siz) _double_to_buf(_data, _buf, _siz, WHERE_FFL_HERE) @@ -1702,6 +1718,8 @@ extern char *_ctv_to_buf(tv_t *data, char *buf, size_t siz, WHERE_FFL_ARGS); extern char *_ftv_to_buf(tv_t *data, char *buf, size_t siz, WHERE_FFL_ARGS); // Convert tv to seconds (ignore uS) extern char *_tvs_to_buf(tv_t *data, char *buf, size_t siz, WHERE_FFL_ARGS); +// Convert tv to (brief) DD HH:MM:SS +extern char *_btv_to_buf(tv_t *data, char *buf, size_t siz, WHERE_FFL_ARGS); /* unused yet extern char *_blob_to_buf(char *data, char *buf, size_t siz, WHERE_FFL_ARGS); */ diff --git a/src/ckdb_cmd.c b/src/ckdb_cmd.c index 6a0e1cc3..1796bc4c 100644 --- a/src/ckdb_cmd.c +++ b/src/ckdb_cmd.c @@ -4902,8 +4902,10 @@ static char *cmd_stats(__maybe_unused PGconn *conn, char *cmd, char *id, USEINFO(idcontrol, 1, 0); USEINFO(optioncontrol, 1, 1); USEINFO(workinfo, 1, 1); - USEINFO(shares, 1, 1); - USEINFO(shareerrors, 1, 1); + // Trees don't share items so count as 1 tree + USEINFO(shares, 2, 1); + // Trees don't share items so count as 1 tree + USEINFO(shareerrors, 2, 1); USEINFO(sharesummary, 1, 2); USEINFO(workmarkers, 1, 2); USEINFO(markersummary, 1, 2); diff --git a/src/ckdb_data.c b/src/ckdb_data.c index f1303265..13641b8d 100644 --- a/src/ckdb_data.c +++ b/src/ckdb_data.c @@ -417,6 +417,14 @@ char *_data_to_buf(enum data_type typ, void *data, char *buf, size_t siz, WHERE_ tm.tm_sec, (((tv_t *)data)->tv_usec)); break; + case TYPE_BTV: + gmtime_r(&(((tv_t *)data)->tv_sec), &tm); + snprintf(buf, siz, "%02d %02d:%02d:%02d", + tm.tm_mday, + tm.tm_hour, + tm.tm_min, + tm.tm_sec); + break; case TYPE_CTV: snprintf(buf, siz, "%ld,%ld", (((tv_t *)data)->tv_sec), @@ -476,6 +484,12 @@ char *_tvs_to_buf(tv_t *data, char *buf, size_t siz, WHERE_FFL_ARGS) return _data_to_buf(TYPE_TVS, (void *)data, buf, siz, WHERE_FFL_PASS); } +// Convert tv to (brief) DD HH:MM:SS +char *_btv_to_buf(tv_t *data, char *buf, size_t siz, WHERE_FFL_ARGS) +{ + return _data_to_buf(TYPE_BTV, (void *)data, buf, siz, WHERE_FFL_PASS); +} + /* unused yet char *_blob_to_buf(char *data, char *buf, size_t siz, WHERE_FFL_ARGS) { diff --git a/src/ckdb_dbio.c b/src/ckdb_dbio.c index 033112af..02db38a1 100644 --- a/src/ckdb_dbio.c +++ b/src/ckdb_dbio.c @@ -883,6 +883,7 @@ K_ITEM *useratts_add(PGconn *conn, char *username, char *attname, USERATTS *row; USERS *users; bool ok = false; + char *st = NULL; LOGDEBUG("%s(): add", __func__); @@ -895,11 +896,10 @@ K_ITEM *useratts_add(PGconn *conn, char *username, char *attname, u_item = find_users(username); K_RUNLOCK(users_free); if (!u_item) { - char *txt; LOGERR("%s(): unknown user '%s'", __func__, - txt = safe_text(username)); - free(txt); + st = safe_text(username)); + FREENULL(st); goto unitem; } DATA_USERS(users, u_item); @@ -2757,20 +2757,208 @@ bool workinfo_fill(PGconn *conn) return ok; } +static bool shares_process(PGconn *conn, SHARES *shares, K_TREE *trf_root) +{ + K_ITEM *w_item, *wm_item, *ss_item; + SHARESUMMARY *sharesummary; + char *st = NULL; + + LOGDEBUG("%s() add", __func__); + + w_item = new_default_worker(conn, false, shares->userid, + shares->workername, shares->createby, + shares->createcode, shares->createinet, + &(shares->createdate), trf_root); + if (!w_item) { + LOGDEBUG("%s(): new_default_worker failed %"PRId64"/%s/%ld,%ld", + __func__, shares->userid, + st = safe_text(shares->workername), + shares->createdate.tv_sec, shares->createdate.tv_usec); + FREENULL(st); + return false; + } + + if (reloading && !confirm_sharesummary) { + // We only need to know if the workmarker is processed + wm_item = find_workmarkers(shares->workinfoid, false, + MARKER_PROCESSED); + if (wm_item) { + LOGDEBUG("%s(): workmarker exists for wid %"PRId64 + " %"PRId64"/%s/%ld,%ld", + __func__, shares->workinfoid, shares->userid, + st = safe_text(shares->workername), + shares->createdate.tv_sec, + shares->createdate.tv_usec); + FREENULL(st); + return false; + } + + ss_item = find_sharesummary(shares->userid, shares->workername, + shares->workinfoid); + if (ss_item) { + DATA_SHARESUMMARY(sharesummary, ss_item); + if (sharesummary->complete[0] != SUMMARY_NEW) { + LOGDEBUG("%s(): '%s' sharesummary exists " + "%"PRId64" %"PRId64"/%s/%ld,%ld", + __func__, sharesummary->complete, + shares->workinfoid, shares->userid, + st = safe_text(shares->workername), + shares->createdate.tv_sec, + shares->createdate.tv_usec); + FREENULL(st); + return false; + } + + if (!sharesummary->reset) { + zero_sharesummary(sharesummary, + &(shares->createdate), + shares->diff); + sharesummary->reset = true; + } + } + } + + if (!confirm_sharesummary) + workerstatus_update(NULL, shares, NULL); + + sharesummary_update(conn, shares, NULL, NULL, shares->createby, + shares->createcode, shares->createinet, + &(shares->createdate)); + + return true; +} + +// If it exists and it can be processed, process the oldest early share +static void shares_process_early(PGconn *conn, int64_t good_wid, tv_t *good_cd, + K_TREE *trf_root) +{ + K_TREE_CTX ctx[1]; + K_ITEM *es_item, *wi_item; + SHARES *early_shares; + char cd_buf[DATE_BUFSIZ]; + char *why = EMPTY; + char *st = NULL; + char tmp[1024]; + double delta; + bool ok; + + LOGDEBUG("%s() add", __func__); + + K_WLOCK(shares_free); + if (shares_early_store->count == 0) { + K_WUNLOCK(shares_free); + // None + return; + } + es_item = last_in_ktree(shares_early_root, ctx); + if (es_item) { + shares_early_root = remove_from_ktree(shares_early_root, + es_item, + cmp_shares); + k_unlink_item(shares_early_store, es_item); + } + K_WUNLOCK(shares_free); + if (es_item) { + DATA_SHARES(early_shares, es_item); + /* If the last (oldest) is newer than the + * current workinfo, leave it til later */ + if (early_shares->workinfoid > good_wid) + goto redo; + + /* If it matches the 'ok' share we just processed, + * we don't need to check the workinfoid */ + if (early_shares->workinfoid == good_wid) { + ok = shares_process(conn, early_shares, trf_root); + if (ok) + goto keep; + else + goto discard; + } else { + wi_item = find_workinfo(early_shares->workinfoid, NULL); + if (!wi_item) { + // good_cd is 'now' + delta = tvdiff(good_cd, + &(early_shares->createdate)); + if (early_shares->oldcount > 0) { + snprintf(tmp, sizeof(tmp), + " too old (%.1fs/%"PRId32")", + delta, + early_shares->oldcount); + why = tmp; + goto discard; + } + if (delta > EARLYSHARESLIMIT) + early_shares->oldcount++; + early_shares->redo++; + goto redo; + } else { + ok = shares_process(conn, early_shares, trf_root); + if (ok) + goto keep; + else + goto discard; + } + } + } + return; +redo: + K_WLOCK(shares_free); + shares_early_root = add_to_ktree(shares_early_root, es_item, cmp_shares); + k_add_tail(shares_early_store, es_item); + K_WUNLOCK(shares_free); + return; +keep: + btv_to_buf(&(early_shares->createdate), cd_buf, sizeof(cd_buf)); + LOGERR("%s() %"PRId64"/%s/%ld,%ld %s/%"PRId32"/%"PRId32 + " Early share procured", + __func__, early_shares->workinfoid, + st = safe_text(early_shares->workername), + early_shares->createdate.tv_sec, + early_shares->createdate.tv_usec, cd_buf, + early_shares->oldcount, early_shares->redo); + FREENULL(st); + K_WLOCK(shares_free); + shares_root = add_to_ktree(shares_root, es_item, cmp_shares); + k_add_head(shares_store, es_item); + K_WUNLOCK(shares_free); + return; +discard: + btv_to_buf(&(early_shares->createdate), cd_buf, sizeof(cd_buf)); + LOGERR("%s() %"PRId64"/%s/%ld,%ld %s/%"PRId32"/%"PRId32 + " Early share discarded!%s", + __func__, early_shares->workinfoid, + st = safe_text(early_shares->workername), + early_shares->createdate.tv_sec, + early_shares->createdate.tv_usec, cd_buf, + early_shares->oldcount, early_shares->redo, why); + FREENULL(st); + K_WLOCK(shares_free); + k_add_head(shares_free, es_item); + K_WUNLOCK(shares_free); + return; +} + +static void shareerrors_process_early(PGconn *conn, int64_t good_wid, + tv_t *good_cd, K_TREE *trf_root); + // Memory (and log file) only bool shares_add(PGconn *conn, char *workinfoid, char *username, char *workername, char *clientid, char *errn, char *enonce1, char *nonce2, char *nonce, char *diff, char *sdiff, char *secondaryuserid, char *by, char *code, char *inet, tv_t *cd, K_TREE *trf_root) { - K_ITEM *s_item, *u_item, *wi_item, *w_item, *wm_item, *ss_item; + K_ITEM *s_item = NULL, *u_item, *wi_item; char cd_buf[DATE_BUFSIZ]; - SHARESUMMARY *sharesummary; - SHARES *shares; + SHARES *shares = NULL; USERS *users; bool ok = false; + char *st = NULL; - LOGDEBUG("%s(): add", __func__); + LOGDEBUG("%s(): %s/%s/%s/%s/%ld,%ld", + __func__, + workinfoid, st = safe_text(workername), nonce, + errn, cd->tv_sec, cd->tv_usec); + FREENULL(st); K_WLOCK(shares_free); s_item = k_unlink_head(shares_free); @@ -2782,13 +2970,15 @@ bool shares_add(PGconn *conn, char *workinfoid, char *username, char *workername u_item = find_users(username); K_RUNLOCK(users_free); if (!u_item) { - char *txt; - tv_to_buf(cd, cd_buf, sizeof(cd_buf)); - LOGERR("%s() %s/%ld,%ld %.19s no user! Share discarded!", - __func__, txt = safe_text(username), + btv_to_buf(cd, cd_buf, sizeof(cd_buf)); + /* This should never happen unless there's a bug in ckpool + or the authentication information got to ckdb after + the shares ... which shouldn't ever happen */ + LOGERR("%s() %s/%ld,%ld %s no user! Share discarded!", + __func__, st = safe_text(username), cd->tv_sec, cd->tv_usec, cd_buf); - free(txt); - goto unitem; + FREENULL(st); + goto tisbad; } DATA_USERS(users, u_item); @@ -2809,11 +2999,12 @@ bool shares_add(PGconn *conn, char *workinfoid, char *username, char *workername STRNCPY(shares->secondaryuserid, users->secondaryuserid); if (!tv_newer(&missing_secuser_min, cd) || !tv_newer(cd, &missing_secuser_max)) { - tv_to_buf(cd, cd_buf, sizeof(cd_buf)); - LOGERR("%s() %s/%ld,%ld %.19s missing secondaryuserid! " + btv_to_buf(cd, cd_buf, sizeof(cd_buf)); + LOGERR("%s() %s/%ld,%ld %s missing secondaryuserid! " "Share corrected", - __func__, username, + __func__, st = safe_text(username), cd->tv_sec, cd->tv_usec, cd_buf); + FREENULL(st); } } @@ -2822,80 +3013,259 @@ bool shares_add(PGconn *conn, char *workinfoid, char *username, char *workername wi_item = find_workinfo(shares->workinfoid, NULL); if (!wi_item) { - tv_to_buf(cd, cd_buf, sizeof(cd_buf)); - // TODO: store it for a few workinfoid changes - LOGERR("%s() %"PRId64"/%s/%ld,%ld %.19s no workinfo! Share discarded!", - __func__, shares->workinfoid, workername, + btv_to_buf(cd, cd_buf, sizeof(cd_buf)); + LOGERR("%s() %"PRId64"/%s/%ld,%ld %s no workinfo! " + "Early share queued!", + __func__, shares->workinfoid, + st = safe_text(workername), cd->tv_sec, cd->tv_usec, cd_buf); - goto unitem; + FREENULL(st); + shares->redo = 0; + shares->oldcount = 0; + K_WLOCK(shares_free); + // They need to be sorted by workinfoid + shares_early_root = add_to_ktree(shares_early_root, s_item, + cmp_shares); + k_add_head(shares_early_store, s_item); + K_WUNLOCK(shares_free); + /* It was all OK except the missing workinfoid + * and it was queued, so most likely OK */ + return true; } - w_item = new_default_worker(conn, false, shares->userid, shares->workername, - by, code, inet, cd, trf_root); - if (!w_item) - goto unitem; + ok = shares_process(conn, shares, trf_root); + if (ok) { + K_WLOCK(shares_free); + shares_root = add_to_ktree(shares_root, s_item, cmp_shares); + k_add_head(shares_store, s_item); + K_WUNLOCK(shares_free); + + shares_process_early(conn, shares->workinfoid, + &(shares->createdate), trf_root); + // Call both since shareerrors may be rare + shareerrors_process_early(conn, shares->workinfoid, + &(shares->createdate), trf_root); + + // The original share was ok + return true; + } + +tisbad: + K_WLOCK(shares_free); + k_add_head(shares_free, s_item); + K_WUNLOCK(shares_free); + return false; +} + +static bool shareerrors_process(PGconn *conn, SHAREERRORS *shareerrors, + K_TREE *trf_root) +{ + K_ITEM *w_item, *wm_item, *ss_item; + SHARESUMMARY *sharesummary; + char *st = NULL; + + LOGDEBUG("%s() add", __func__); + + w_item = new_default_worker(conn, false, shareerrors->userid, + shareerrors->workername, + shareerrors->createby, + shareerrors->createcode, + shareerrors->createinet, + &(shareerrors->createdate), trf_root); + if (!w_item) { + LOGDEBUG("%s(): new_default_worker failed %"PRId64"/%s/%ld,%ld", + __func__, shareerrors->userid, + st = safe_text(shareerrors->workername), + shareerrors->createdate.tv_sec, + shareerrors->createdate.tv_usec); + FREENULL(st); + return false; + } if (reloading && !confirm_sharesummary) { // We only need to know if the workmarker is processed - wm_item = find_workmarkers(shares->workinfoid, false, + wm_item = find_workmarkers(shareerrors->workinfoid, false, MARKER_PROCESSED); if (wm_item) { - K_WLOCK(shares_free); - k_add_head(shares_free, s_item); - K_WUNLOCK(shares_free); - return true; + LOGDEBUG("%s(): workmarker exists for wid %"PRId64 + " %"PRId64"/%s/%ld,%ld", + __func__, shareerrors->workinfoid, + shareerrors->userid, + st = safe_text(shareerrors->workername), + shareerrors->createdate.tv_sec, + shareerrors->createdate.tv_usec); + FREENULL(st); + return false; } - ss_item = find_sharesummary(shares->userid, shares->workername, shares->workinfoid); + + ss_item = find_sharesummary(shareerrors->userid, + shareerrors->workername, + shareerrors->workinfoid); if (ss_item) { DATA_SHARESUMMARY(sharesummary, ss_item); if (sharesummary->complete[0] != SUMMARY_NEW) { - K_WLOCK(shares_free); - k_add_head(shares_free, s_item); - K_WUNLOCK(shares_free); - return true; + LOGDEBUG("%s(): '%s' sharesummary exists " + "%"PRId64" %"PRId64"/%s/%ld,%ld", + __func__, sharesummary->complete, + shareerrors->workinfoid, + shareerrors->userid, + st = safe_text(shareerrors->workername), + shareerrors->createdate.tv_sec, + shareerrors->createdate.tv_usec); + FREENULL(st); + return false; } if (!sharesummary->reset) { - zero_sharesummary(sharesummary, cd, shares->diff); + zero_sharesummary(sharesummary, + &(shareerrors->createdate), + 0.0); sharesummary->reset = true; } } } - if (!confirm_sharesummary) - workerstatus_update(NULL, shares, NULL); + sharesummary_update(conn, NULL, shareerrors, NULL, + shareerrors->createby, + shareerrors->createcode, + shareerrors->createinet, + &(shareerrors->createdate)); - sharesummary_update(conn, shares, NULL, NULL, by, code, inet, cd); + return true; +} - ok = true; -unitem: - K_WLOCK(shares_free); - if (!ok) - k_add_head(shares_free, s_item); - else { - shares_root = add_to_ktree(shares_root, s_item, cmp_shares); - k_add_head(shares_store, s_item); - } - K_WUNLOCK(shares_free); +// If it exists and it can be processed, process the oldest early shareerror +static void shareerrors_process_early(PGconn *conn, int64_t good_wid, + tv_t *good_cd, K_TREE *trf_root) +{ + K_TREE_CTX ctx[1]; + K_ITEM *es_item, *wi_item; + SHAREERRORS *early_shareerrors; + char cd_buf[DATE_BUFSIZ]; + char *why = EMPTY; + char *st = NULL; + char tmp[1024]; + double delta; + bool ok; - return ok; + LOGDEBUG("%s() add", __func__); + + K_WLOCK(shareerrors_free); + if (shareerrors_early_store->count == 0) { + K_WUNLOCK(shareerrors_free); + // None + return; + } + es_item = last_in_ktree(shareerrors_early_root, ctx); + if (es_item) { + shareerrors_early_root = remove_from_ktree(shareerrors_early_root, + es_item, + cmp_shareerrors); + k_unlink_item(shareerrors_early_store, es_item); + } + K_WUNLOCK(shareerrors_free); + if (es_item) { + DATA_SHAREERRORS(early_shareerrors, es_item); + /* If the last (oldest) is newer than the + * current workinfo, leave it til later */ + if (early_shareerrors->workinfoid > good_wid) + goto redo; + + /* If it matches the 'ok' share/shareerror we just processed, + * we don't need to check the workinfoid */ + if (early_shareerrors->workinfoid == good_wid) { + ok = shareerrors_process(conn, early_shareerrors, + trf_root); + if (ok) + goto keep; + else + goto discard; + } else { + wi_item = find_workinfo(early_shareerrors->workinfoid, NULL); + if (!wi_item) { + // good_cd is 'now' + delta = tvdiff(good_cd, + &(early_shareerrors->createdate)); + if (early_shareerrors->oldcount > 0) { + snprintf(tmp, sizeof(tmp), + " too old (%.1fs/%"PRId32")", + delta, + early_shareerrors->oldcount); + why = tmp; + goto discard; + } + if (delta > EARLYSHARESLIMIT) + early_shareerrors->oldcount++; + early_shareerrors->redo++; + goto redo; + } else { + ok = shareerrors_process(conn, + early_shareerrors, + trf_root); + if (ok) + goto keep; + else + goto discard; + } + } + } + return; +redo: + K_WLOCK(shareerrors_free); + shareerrors_early_root = add_to_ktree(shareerrors_early_root, es_item, + cmp_shareerrors); + k_add_tail(shareerrors_early_store, es_item); + K_WUNLOCK(shareerrors_free); + return; +keep: + btv_to_buf(&(early_shareerrors->createdate), cd_buf, sizeof(cd_buf)); + LOGERR("%s() %"PRId64"/%s/%ld,%ld %s/%"PRId32"/%"PRId32 + " Early share procured", + __func__, early_shareerrors->workinfoid, + st = safe_text(early_shareerrors->workername), + early_shareerrors->createdate.tv_sec, + early_shareerrors->createdate.tv_usec, cd_buf, + early_shareerrors->oldcount, early_shareerrors->redo); + FREENULL(st); + K_WLOCK(shareerrors_free); + shareerrors_root = add_to_ktree(shareerrors_root, es_item, cmp_shareerrors); + k_add_head(shareerrors_store, es_item); + K_WUNLOCK(shareerrors_free); + return; +discard: + btv_to_buf(&(early_shareerrors->createdate), cd_buf, sizeof(cd_buf)); + LOGERR("%s() %"PRId64"/%s/%ld,%ld %s/%"PRId32"/%"PRId32 + " Early share discarded!%s", + __func__, early_shareerrors->workinfoid, + st = safe_text(early_shareerrors->workername), + early_shareerrors->createdate.tv_sec, + early_shareerrors->createdate.tv_usec, cd_buf, + early_shareerrors->oldcount, early_shareerrors->redo, why); + FREENULL(st); + K_WLOCK(shareerrors_free); + k_add_head(shareerrors_free, es_item); + K_WUNLOCK(shareerrors_free); + return; } // Memory (and log file) only -// TODO: handle shareerrors that appear after a workinfoid is aged or doesn't exist? bool shareerrors_add(PGconn *conn, char *workinfoid, char *username, char *workername, char *clientid, char *errn, char *error, char *secondaryuserid, char *by, char *code, char *inet, tv_t *cd, K_TREE *trf_root) { - K_ITEM *s_item, *u_item, *wi_item, *w_item, *wm_item, *ss_item; + K_ITEM *s_item = NULL, *u_item, *wi_item; char cd_buf[DATE_BUFSIZ]; - SHARESUMMARY *sharesummary; - SHAREERRORS *shareerrors; + SHAREERRORS *shareerrors = NULL; USERS *users; bool ok = false; + char *st = NULL; - LOGDEBUG("%s(): add", __func__); + LOGDEBUG("%s(): %s/%s/%s/%s/%ld,%ld", + __func__, + workinfoid, st = safe_text(workername), errn, + error, cd->tv_sec, cd->tv_usec); + FREENULL(st); K_WLOCK(shareerrors_free); s_item = k_unlink_head(shareerrors_free); @@ -2907,13 +3277,12 @@ bool shareerrors_add(PGconn *conn, char *workinfoid, char *username, u_item = find_users(username); K_RUNLOCK(users_free); if (!u_item) { - char *txt; - tv_to_buf(cd, cd_buf, sizeof(cd_buf)); - LOGERR("%s() %s/%ld,%ld %.19s no user! Shareerror discarded!", - __func__, txt = safe_text(username), + btv_to_buf(cd, cd_buf, sizeof(cd_buf)); + LOGERR("%s() %s/%ld,%ld %s no user! Shareerror discarded!", + __func__, st = safe_text(username), cd->tv_sec, cd->tv_usec, cd_buf); - free(txt); - goto unitem; + FREENULL(st); + goto tisbad; } DATA_USERS(users, u_item); @@ -2930,11 +3299,12 @@ bool shareerrors_add(PGconn *conn, char *workinfoid, char *username, STRNCPY(shareerrors->secondaryuserid, users->secondaryuserid); if (!tv_newer(&missing_secuser_min, cd) || !tv_newer(cd, &missing_secuser_max)) { - tv_to_buf(cd, cd_buf, sizeof(cd_buf)); - LOGERR("%s() %s/%ld,%ld %.19s missing secondaryuserid! " + btv_to_buf(cd, cd_buf, sizeof(cd_buf)); + LOGERR("%s() %s/%ld,%ld %s missing secondaryuserid! " "Sharerror corrected", - __func__, username, + __func__, st = safe_text(username), cd->tv_sec, cd->tv_usec, cd_buf); + FREENULL(st); } } @@ -2943,59 +3313,51 @@ bool shareerrors_add(PGconn *conn, char *workinfoid, char *username, wi_item = find_workinfo(shareerrors->workinfoid, NULL); if (!wi_item) { - tv_to_buf(cd, cd_buf, sizeof(cd_buf)); - LOGERR("%s() %"PRId64"/%s/%ld,%ld %.19s no workinfo! Shareerror discarded!", - __func__, shareerrors->workinfoid, workername, + btv_to_buf(cd, cd_buf, sizeof(cd_buf)); + LOGERR("%s() %"PRId64"/%s/%ld,%ld %s no workinfo! " + "Early shareerror queued!", + __func__, shareerrors->workinfoid, + st = safe_text(workername), cd->tv_sec, cd->tv_usec, cd_buf); - goto unitem; + FREENULL(st); + shareerrors->redo = 0; + shareerrors->oldcount = 0; + K_WLOCK(shareerrors_free); + // They need to be sorted by workinfoid + shareerrors_early_root = add_to_ktree(shareerrors_early_root, + s_item, + cmp_shareerrors); + k_add_head(shareerrors_early_store, s_item); + K_WUNLOCK(shareerrors_free); + /* It was all OK except the missing workinfoid + * and it was queued, so most likely OK */ + return true; } - w_item = new_default_worker(NULL, false, shareerrors->userid, shareerrors->workername, - by, code, inet, cd, trf_root); - if (!w_item) - goto unitem; + ok = shareerrors_process(conn, shareerrors, trf_root); + if (ok) { + K_WLOCK(shareerrors_free); + shareerrors_root = add_to_ktree(shareerrors_root, s_item, + cmp_shareerrors); + k_add_head(shareerrors_store, s_item); + K_WUNLOCK(shareerrors_free); - if (reloading && !confirm_sharesummary) { - // We only need to know if the workmarker is processed - wm_item = find_workmarkers(shareerrors->workinfoid, false, - MARKER_PROCESSED); - if (wm_item) { - K_WLOCK(shareerrors_free); - k_add_head(shareerrors_free, s_item); - K_WUNLOCK(shareerrors_free); - return true; - } - ss_item = find_sharesummary(shareerrors->userid, shareerrors->workername, shareerrors->workinfoid); - if (ss_item) { - DATA_SHARESUMMARY(sharesummary, ss_item); - if (sharesummary->complete[0] != SUMMARY_NEW) { - K_WLOCK(shareerrors_free); - k_add_head(shareerrors_free, s_item); - K_WUNLOCK(shareerrors_free); - return true; - } + shareerrors_process_early(conn, shareerrors->workinfoid, + &(shareerrors->createdate), + trf_root); + // Call both in case we are only getting errors on bad work + shares_process_early(conn, shareerrors->workinfoid, + &(shareerrors->createdate), trf_root); - if (!sharesummary->reset) { - zero_sharesummary(sharesummary, cd, 0.0); - sharesummary->reset = true; - } - } + // The original share was ok + return true; } - sharesummary_update(conn, NULL, shareerrors, NULL, by, code, inet, cd); - - ok = true; -unitem: +tisbad: K_WLOCK(shareerrors_free); - if (!ok) - k_add_head(shareerrors_free, s_item); - else { - shareerrors_root = add_to_ktree(shareerrors_root, s_item, cmp_shareerrors); - k_add_head(shareerrors_store, s_item); - } + k_add_head(shareerrors_free, s_item); K_WUNLOCK(shareerrors_free); - - return ok; + return false; } bool shareerrors_fill() @@ -3030,6 +3392,7 @@ bool sharesummaries_to_markersummaries(PGconn *conn, WORKMARKERS *workmarkers, char *params[2]; int n, par = 0, deleted = -7; int ss_count, ms_count; + char *st = NULL; char *del; LOGWARNING("%s() Processing: workmarkers %"PRId64"/%s/" @@ -3120,7 +3483,8 @@ bool sharesummaries_to_markersummaries(PGconn *conn, WORKMARKERS *workmarkers, LOGDEBUG("%s() new ms %"PRId64"/%"PRId64"/%s", shortname, markersummary->markerid, markersummary->userid, - markersummary->workername); + st = safe_text(markersummary->workername)); + FREENULL(st); } else { DATA_MARKERSUMMARY(markersummary, ms_item); } @@ -3335,6 +3699,7 @@ bool _sharesummary_update(PGconn *conn, SHARES *s_row, SHAREERRORS *e_row, K_ITE tv_t *sharecreatedate; bool must_update = false, conned = false; double diff = 0; + char *st = NULL, *db = NULL; LOGDEBUG("%s(): update", __func__); @@ -3378,16 +3743,16 @@ bool _sharesummary_update(PGconn *conn, SHARES *s_row, SHAREERRORS *e_row, K_ITE wm_item = find_workmarkers(workinfoid, false, MARKER_PROCESSED); K_RUNLOCK(workmarkers_free); if (wm_item) { - char *tmp; DATA_WORKMARKERS(wm, wm_item); LOGERR("%s(): attempt to update sharesummary " "with %s %"PRId64"/%"PRId64"/%s createdate %s" " but processed workmarkers %"PRId64" exists", __func__, s_row ? "shares" : "shareerrors", - workinfoid, userid, workername, - (tmp = ctv_to_buf(sharecreatedate, NULL, 0)), + workinfoid, userid, st = safe_text(workername), + db = ctv_to_buf(sharecreatedate, NULL, 0), wm->markerid); - free(tmp); + FREENULL(st); + FREENULL(db); return false; } @@ -3496,9 +3861,12 @@ bool _sharesummary_update(PGconn *conn, SHARES *s_row, SHAREERRORS *e_row, K_ITE free(tmp1); } if (row->complete[0] != SUMMARY_NEW) { - LOGDEBUG("%s(): updating sharesummary not '%c' %"PRId64"/%s/%"PRId64"/%s", - __func__, SUMMARY_NEW, row->userid, row->workername, + LOGDEBUG("%s(): updating sharesummary not '%c'" + " %"PRId64"/%s/%"PRId64"/%s", + __func__, SUMMARY_NEW, row->userid, + st = safe_text(row->workername), row->workinfoid, row->complete); + FREENULL(st); } } } @@ -4081,6 +4449,7 @@ bool blocks_add(PGconn *conn, char *height, char *blockhash, bool ok = false, update_old = false; int n, par = 0; char want = '?'; + char *st = NULL; LOGDEBUG("%s(): add", __func__); @@ -4404,8 +4773,9 @@ flail: snprintf(tmp, sizeof(tmp), " Reward: %f, Worker: %s, ShareEst: %.1f %s%s%% UTC:%s", BTC_TO_D(row->reward), - row->workername, + st = safe_text(row->workername), pool.diffacc, est, pct, cd_buf); + FREENULL(st); if (pool.workinfoid < row->workinfoid) { pool.workinfoid = row->workinfoid; pool.height = row->height; @@ -5104,6 +5474,7 @@ bool auths_add(PGconn *conn, char *poolinstance, char *username, char cd_buf[DATE_BUFSIZ]; AUTHS *row; bool ok = false; + char *st = NULL; LOGDEBUG("%s(): add", __func__); @@ -5121,11 +5492,10 @@ bool auths_add(PGconn *conn, char *poolinstance, char *username, u_item = users_add(conn, username, EMPTY, EMPTY, by, code, inet, cd, trf_root); } else { - char *txt; LOGDEBUG("%s(): unknown user '%s'", __func__, - txt = safe_text(username)); - free(txt); + st = safe_text(username)); + FREENULL(st); } if (!u_item) goto unitem; @@ -5165,7 +5535,9 @@ bool auths_add(PGconn *conn, char *poolinstance, char *username, // Shouldn't actually be possible unless twice in the logs tv_to_buf(cd, cd_buf, sizeof(cd_buf)); LOGERR("%s(): Duplicate auths ignored %s/%s/%s", - __func__, poolinstance, workername, cd_buf); + __func__, poolinstance, st = safe_text(workername), + cd_buf); + FREENULL(st); /* Let them mine, that's what matters :) * though this would normally only be during a reload */ @@ -5471,6 +5843,7 @@ bool userstats_add(char *poolinstance, char *elapsed, char *username, USERSTATS *row, *match, *next; USERS *users; K_TREE_CTX ctx[1]; + char *st = NULL; LOGDEBUG("%s(): add", __func__); @@ -5486,11 +5859,10 @@ bool userstats_add(char *poolinstance, char *elapsed, char *username, u_item = find_users(username); K_RUNLOCK(users_free); if (!u_item) { - char *txt; LOGERR("%s(): unknown user '%s'", __func__, - txt = safe_text(username)); - free(txt); + st = safe_text(username)); + FREENULL(st); return false; } DATA_USERS(users, u_item); @@ -5595,13 +5967,13 @@ bool workerstats_add(char *poolinstance, char *elapsed, char *username, u_item = find_users(username); K_RUNLOCK(users_free); if (!u_item) { - char *usr, *wrk; + char *usr = NULL, *wrk = NULL; LOGERR("%s(): unknown user '%s' (worker=%s)", __func__, usr = safe_text(username), wrk = safe_text(workername)); - free(usr); - free(wrk); + FREENULL(usr); + FREENULL(wrk); return false; } DATA_USERS(users, u_item); @@ -5654,6 +6026,7 @@ bool markersummary_add(PGconn *conn, K_ITEM *ms_item, char *by, char *code, int n, par = 0; char *ins; bool ok = false; + char *st = NULL; LOGDEBUG("%s(): add", __func__); @@ -5691,8 +6064,10 @@ bool markersummary_add(PGconn *conn, K_ITEM *ms_item, char *by, char *code, MODIFYDATECONTROL ") values (" PQPARAM26 ")"; LOGDEBUG("%s() adding ms %"PRId64"/%"PRId64"/%s/%.0f", - __func__, row->markerid, row->userid, row->workername, + __func__, row->markerid, row->userid, + st = safe_text(row->workername), row->diffacc); + FREENULL(st); if (!conn) { conn = dbconnect(); From a9e0df04d2317fff16b3d2ef4230ea76ab7b9c2e Mon Sep 17 00:00:00 2001 From: kanoi Date: Wed, 25 Mar 2015 03:23:12 +1100 Subject: [PATCH 02/32] php - add the next payout address link to the payments page --- pool/page_payments.php | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pool/page_payments.php b/pool/page_payments.php index 5bd97ff0..bdf66522 100644 --- a/pool/page_payments.php +++ b/pool/page_payments.php @@ -9,10 +9,12 @@ function dopayments($data, $user) { $bc = 'https://blockchain.info/address/'; $addr1 = '1KzFJddTvK9TQWsmWFKYJ9fRx9QeSATyrT'; + $addr2 = '16dRhawxuR3BmdmkzdzUdgEfGAQszgmtbc'; $pg = '

Payments

'; $pg .= 'The payment transactions on blockchain are here:'; - $pg .= " BTC
"; + $pg .= " BTCa and"; + $pg .= " BTCb
"; $pg .= "The payments below don't yet show when they have been sent.

"; $ans = getPayments($user); From a3e3363edada291e04c3184b3331e541199ce384 Mon Sep 17 00:00:00 2001 From: kanoi Date: Wed, 25 Mar 2015 03:28:53 +1100 Subject: [PATCH 03/32] Revert "Add atomically increasing sequence number to all ckdb messages" This reverts commit 970c4056cf4ddd8423db509a435a8056f255babb. --- src/stratifier.c | 21 +++++---------------- 1 file changed, 5 insertions(+), 16 deletions(-) diff --git a/src/stratifier.c b/src/stratifier.c index a90a167f..2b23c5e7 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -299,10 +299,6 @@ struct stratifier_data { /* Serialises sends/receives to ckdb if possible */ mutex_t ckdb_lock; - /* Protects sequence numbers */ - mutex_t ckdb_msg_lock; - /* Incrementing sequence number */ - int ckdb_seq; bool ckdb_offline; @@ -615,18 +611,12 @@ static char *status_chars = "|/-\\"; /* Absorbs the json and generates a ckdb json message, logs it to the ckdb * log and returns the malloced message. */ -static char *ckdb_msg(ckpool_t *ckp, sdata_t *sdata, json_t *val, const int idtype) +static char *ckdb_msg(ckpool_t *ckp, json_t *val, const int idtype) { - char *json_msg; + char *json_msg = json_dumps(val, JSON_COMPACT); char logname[512]; char *ret = NULL; - /* Set the atomically incrementing sequence number */ - mutex_lock(&sdata->ckdb_msg_lock); - json_set_int(val, "seq", sdata->ckdb_seq++); - mutex_unlock(&sdata->ckdb_msg_lock); - - json_msg = json_dumps(val, JSON_COMPACT); if (unlikely(!json_msg)) goto out; ASPRINTF(&ret, "%s.id.json=%s", ckdb_ids[idtype], json_msg); @@ -665,7 +655,7 @@ static void _ckdbq_add(ckpool_t *ckp, const int idtype, json_t *val, const char if (CKP_STANDALONE(ckp)) return json_decref(val); - json_msg = ckdb_msg(ckp, sdata, val, idtype); + json_msg = ckdb_msg(ckp, val, idtype); if (unlikely(!json_msg)) { LOGWARNING("Failed to dump json from %s %s:%d", file, func, line); return; @@ -2359,9 +2349,9 @@ static int send_recv_auth(stratum_instance_t *client) json_set_string(val, "createcode", __func__); json_set_string(val, "createinet", client->address); if (user->btcaddress) - json_msg = ckdb_msg(ckp, sdata, val, ID_ADDRAUTH); + json_msg = ckdb_msg(ckp, val, ID_ADDRAUTH); else - json_msg = ckdb_msg(ckp, sdata, val, ID_AUTH); + json_msg = ckdb_msg(ckp, val, ID_AUTH); if (unlikely(!json_msg)) { LOGWARNING("Failed to dump json in send_recv_auth"); goto out; @@ -4477,7 +4467,6 @@ int stratifier(proc_instance_t *pi) cklock_init(&sdata->instance_lock); mutex_init(&sdata->ckdb_lock); - mutex_init(&sdata->ckdb_msg_lock); sdata->ssends = create_ckmsgq(ckp, "ssender", &ssend_process); /* Create half as many share processing threads as there are CPUs */ threads = sysconf(_SC_NPROCESSORS_ONLN) / 2 ? : 1; From 7072135a72de3f965e39e0144e7bb2315195209a Mon Sep 17 00:00:00 2001 From: kanoi Date: Thu, 26 Mar 2015 15:19:29 +1100 Subject: [PATCH 04/32] ckdb - fix old worker lastshare in workerstatus --- src/ckdb.h | 3 +- src/ckdb_data.c | 92 ++++++++++++++++++++++--------------------------- 2 files changed, 44 insertions(+), 51 deletions(-) diff --git a/src/ckdb.h b/src/ckdb.h index a0fdb2a1..862ddc07 100644 --- a/src/ckdb.h +++ b/src/ckdb.h @@ -55,7 +55,7 @@ #define DB_VLOCK "1" #define DB_VERSION "1.0.0" -#define CKDB_VERSION DB_VERSION"-1.040" +#define CKDB_VERSION DB_VERSION"-1.041" #define WHERE_FFL " - from %s %s() line %d" #define WHERE_FFL_HERE __FILE__, __func__, __LINE__ @@ -1808,6 +1808,7 @@ extern cmp_t cmp_sharesummary_workinfoid(K_ITEM *a, K_ITEM *b); extern void zero_sharesummary(SHARESUMMARY *row, tv_t *cd, double diff); extern K_ITEM *find_sharesummary(int64_t userid, char *workername, int64_t workinfoid); +extern K_ITEM *find_last_sharesummary(int64_t userid, char *workername); extern void auto_age_older(PGconn *conn, int64_t workinfoid, char *poolinstance, char *by, char *code, char *inet, tv_t *cd); #define dbhash2btchash(_hash, _buf, _siz) \ diff --git a/src/ckdb_data.c b/src/ckdb_data.c index 13641b8d..a6171049 100644 --- a/src/ckdb_data.c +++ b/src/ckdb_data.c @@ -750,58 +750,28 @@ K_ITEM *_find_create_workerstatus(int64_t userid, char *workername, TODO: combine set_block_share_counters() with this? */ void workerstatus_ready() { - K_TREE_CTX ws_ctx[1], ss_ctx[1], ms_ctx[1]; - K_ITEM *ws_item, us_look, ss_look, *us_item, *ss_item; - K_ITEM *ms_item, ms_look, *wm_item; - USERSTATS *userstats; - SHARESUMMARY looksharesummary, *sharesummary; - MARKERSUMMARY *markersummary; + K_TREE_CTX ws_ctx[1]; + K_ITEM *ws_item, *ms_item, *ss_item; WORKERSTATUS *workerstatus; + MARKERSUMMARY *markersummary; + SHARESUMMARY *sharesummary; LOGWARNING("%s(): Updating workerstatus...", __func__); - INIT_USERSTATS(&us_look); - INIT_MARKERSUMMARY(&ms_look); - INIT_SHARESUMMARY(&ss_look); ws_item = first_in_ktree(workerstatus_root, ws_ctx); while (ws_item) { DATA_WORKERSTATUS(workerstatus, ws_item); - // Zero or one - K_RLOCK(userstats_free); - us_item = find_userstats(workerstatus->userid, - workerstatus->workername); - K_RUNLOCK(userstats_free); - if (us_item) { - DATA_USERSTATS(userstats, us_item); - if (userstats->idle) { - if (tv_newer(&(workerstatus->last_idle), - &(userstats->statsdate))) { - copy_tv(&(workerstatus->last_idle), - &(userstats->statsdate)); - } - } else { - if (tv_newer(&(workerstatus->last_stats), - &(userstats->statsdate))) { - copy_tv(&(workerstatus->last_stats), - &(userstats->statsdate)); - } - } - } - K_RLOCK(markersummary_free); - // This is the last one + // This is the last share datestamp ms_item = find_markersummary_userid(workerstatus->userid, - workerstatus->workername, ms_ctx); + workerstatus->workername, + NULL); K_RUNLOCK(markersummary_free); if (ms_item) { DATA_MARKERSUMMARY(markersummary, ms_item); - K_RLOCK(workmarkers_free); - wm_item = find_workmarkerid(markersummary->markerid, - false, MARKER_PROCESSED); - K_RUNLOCK(workmarkers_free); - if (wm_item && - tv_newer(&(workerstatus->last_share), &(markersummary->lastshare))) { + if (tv_newer(&(workerstatus->last_share), + &(markersummary->lastshare))) { copy_tv(&(workerstatus->last_share), &(markersummary->lastshare)); workerstatus->last_diff = @@ -809,14 +779,9 @@ void workerstatus_ready() } } - // The last one - looksharesummary.userid = workerstatus->userid; - looksharesummary.workername = workerstatus->workername; - looksharesummary.workinfoid = MAXID; - ss_look.data = (void *)(&looksharesummary); K_RLOCK(sharesummary_free); - ss_item = find_before_in_ktree(sharesummary_root, &ss_look, - cmp_sharesummary, ss_ctx); + ss_item = find_last_sharesummary(workerstatus->userid, + workerstatus->workername); K_RUNLOCK(sharesummary_free); if (ss_item) { DATA_SHARESUMMARY(sharesummary, ss_item); @@ -1931,6 +1896,28 @@ K_ITEM *find_sharesummary(int64_t userid, char *workername, int64_t workinfoid) return find_in_ktree(sharesummary_root, &look, cmp_sharesummary, ctx); } +K_ITEM *find_last_sharesummary(int64_t userid, char *workername) +{ + SHARESUMMARY look_sharesummary, *sharesummary; + K_TREE_CTX ctx[1]; + K_ITEM look, *item; + + look_sharesummary.userid = userid; + look_sharesummary.workername = workername; + look_sharesummary.workinfoid = MAXID; + + INIT_SHARESUMMARY(&look); + look.data = (void *)(&look_sharesummary); + item = find_before_in_ktree(sharesummary_root, &look, cmp_sharesummary, ctx); + if (item) { + DATA_SHARESUMMARY(sharesummary, item); + if (sharesummary->userid != userid || + strcmp(sharesummary->workername, workername) != 0) + item = NULL; + } + return item; +} + /* TODO: markersummary checking? * However, there should be no issues since the sharesummaries are removed */ void auto_age_older(PGconn *conn, int64_t workinfoid, char *poolinstance, @@ -3577,7 +3564,7 @@ void dsp_markersummary(K_ITEM *item, FILE *stream) } } -// order by markerid asc,userid asc,workername asc +// order by markerid asc,userid asc,workername asc (has no expirydate) cmp_t cmp_markersummary(K_ITEM *a, K_ITEM *b) { MARKERSUMMARY *ma, *mb; @@ -3592,7 +3579,7 @@ cmp_t cmp_markersummary(K_ITEM *a, K_ITEM *b) return c; } -// order by userid asc,workername asc,lastshare asc +// order by userid asc,workername asc,lastshare asc (has no expirydate) cmp_t cmp_markersummary_userid(K_ITEM *a, K_ITEM *b) { MARKERSUMMARY *ma, *mb; @@ -3607,12 +3594,17 @@ cmp_t cmp_markersummary_userid(K_ITEM *a, K_ITEM *b) return c; } -// Finds the last markersummary for the worker but also returns the CTX -K_ITEM *find_markersummary_userid(int64_t userid, char *workername, K_TREE_CTX *ctx) +// Finds the last markersummary for the worker and optionally return the CTX +K_ITEM *find_markersummary_userid(int64_t userid, char *workername, + K_TREE_CTX *ctx) { + K_TREE_CTX ctx0[1]; K_ITEM look, *ms_item = NULL; MARKERSUMMARY markersummary, *ms; + if (ctx == NULL) + ctx = ctx0; + markersummary.userid = userid; markersummary.workername = workername; markersummary.lastshare.tv_sec = DATE_S_EOT; From 7b4e3dd86d479f2a1278d2477514c7f524ac980d Mon Sep 17 00:00:00 2001 From: kanoi Date: Thu, 26 Mar 2015 15:44:12 +1100 Subject: [PATCH 05/32] ckdb - reloading aged shares is ok - the full ageing message is reported later anyway --- src/ckdb.h | 2 +- src/ckdb_dbio.c | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/ckdb.h b/src/ckdb.h index 862ddc07..33cca86c 100644 --- a/src/ckdb.h +++ b/src/ckdb.h @@ -55,7 +55,7 @@ #define DB_VLOCK "1" #define DB_VERSION "1.0.0" -#define CKDB_VERSION DB_VERSION"-1.041" +#define CKDB_VERSION DB_VERSION"-1.042" #define WHERE_FFL " - from %s %s() line %d" #define WHERE_FFL_HERE __FILE__, __func__, __LINE__ diff --git a/src/ckdb_dbio.c b/src/ckdb_dbio.c index 02db38a1..a91d27c3 100644 --- a/src/ckdb_dbio.c +++ b/src/ckdb_dbio.c @@ -2806,7 +2806,8 @@ static bool shares_process(PGconn *conn, SHARES *shares, K_TREE *trf_root) shares->createdate.tv_sec, shares->createdate.tv_usec); FREENULL(st); - return false; + // Reloading a share already summarised + return true; } if (!sharesummary->reset) { From c42de09e5ca87ff33b3ff853d2177f1585d9557a Mon Sep 17 00:00:00 2001 From: kanoi Date: Thu, 26 Mar 2015 18:20:41 +1100 Subject: [PATCH 06/32] ckdb/php - show only recent workers with an option to disable it --- pool/page.php | 2 +- pool/page_workmgt.php | 61 +++++--- src/ckdb.h | 4 +- src/ckdb_cmd.c | 325 +++++++++++++++++++++++++----------------- 4 files changed, 243 insertions(+), 149 deletions(-) diff --git a/pool/page.php b/pool/page.php index 9d377438..adecb75f 100644 --- a/pool/page.php +++ b/pool/page.php @@ -165,7 +165,7 @@ function pghead($script_marker, $name) $head .= "window.onpaint=jst();\n\n"; $head .= "\n"; $head .= ''; @@ -592,9 +625,10 @@ function pgfoot($info) function gopage($info, $data, $pagefun, $page, $menu, $name, $user, $ispage = true, $dotop = true, $douser = true) { global $dbg, $stt; - global $page_scripts; + global $page_css, $page_scripts; $dbg_marker = '[@dbg@]'; + $css_marker = '[@css@]'; $script_marker = '[@scripts@]'; if ($dbg === true) @@ -616,13 +650,15 @@ function gopage($info, $data, $pagefun, $page, $menu, $name, $user, $ispage = tr // if (isset($_SESSION['logkey'])) // unset($_SESSION['logkey']); - $head = pghead($script_marker, $name); + $head = pghead($css_marker, $script_marker, $name); $body = pgbody($info, $page, $menu, $dotop, $user, $douser); $foot = pgfoot($info); if ($dbg === true) $pg = str_replace($dbg_marker, cvtdbg(), $pg); + $head = str_replace($css_marker, $page_css, $head); + if ($page_scripts != '') $page_scripts .= ""; diff --git a/pool/page_shifts.php b/pool/page_shifts.php index 7627c0a3..8902f004 100644 --- a/pool/page_shifts.php +++ b/pool/page_shifts.php @@ -20,7 +20,7 @@ function doshifts($data, $user) else { $pre = $ans['prefix_all']; - + $count = $ans['rows']; $pg = '

Last '.($count+1).' Shifts

'.$pg; for ($i = 0; $i < $count; $i++) diff --git a/pool/page_usperf.php b/pool/page_usperf.php index 5edd947c..325d6305 100644 --- a/pool/page_usperf.php +++ b/pool/page_usperf.php @@ -1,25 +1,52 @@ ths){ymin=ths}if(ths>ymax)ymax=ths;d['nx:'+i]=sn(i,d['shift:'+i]);if(xmin==-1||xmin>s){xmin=s}if(xmaxs){xmin=s}if(xmaxths){ymin=ths}if(ths>ymax)ymax=ths;document.getElementById('worker'+j).value=d[pre+'worker']} +} var tav=(tda/(xmax-xmin))*Math.pow(2,32)/Math.pow(10,12); var p5=(ymax-ymin)*0.05;ymax+=p5;ymin-=p5;if(ymin<0){ymin=0} if(c['zerob']){ymin=0} ghg(c,xmax-xmin); -ggr(c,0.9,0.9,'TH/s',rows,xmin,xmax,ymin,ymax,d,'nx:','vx:','ths:',tav)} +ggr(c,0.9,0.9,'TH/s',rows,xmin,xmax,ymin,ymax,d,'nx:','vx:','ths:',tav,w,cols)} c={}; function dodrw(data,cbx){if(hasCan()){gdrw(c,sep(data),cbx)}} -function gact(t){if(t.checked){scnv(t.id,1)}else{scnv(t.id,0)}godrw(0)}"; +function gact(t){if(t.checked){scnv(t.id,1)}else{scnv(t.id,0)}godrw(0)} +function wch(){var w='';for(var i=1;i<=$nc;i++){var e=document.getElementById('worker'+i);if(e&&e.value&&e.value.trim()){if(i>1){w+=','}w+=e.value.trim()}}if(w){scnv('workers',w)}}"; return $g; } # function dousperf($data, $user) { - $ans = getShiftData($user); + global $fld_sep, $val_sep; + + // This also defines how many worker fields there are + $cols = array('#0000c0', '#00dd00', '#e06020', '#b020e0'); + $nc = count($cols); + + $workers = 'all'; + if (isset($_COOKIE['workers'])) + { + $w = substr(trim($_COOKIE['workers']), 0, 1024); + if ($w !== false) + { + $wa = explode(',', $w, $nc+1); + if (count($wa) > $nc) + { + $w = ''; + for ($i = 0; $i < $nc; $i++) + $w .= (($i == 0) ? '' : ',').$wa[$i]; + } + $workers = $w; + } + } + + $ans = getShiftData($user, $workers); $iCrap = strpos($_SERVER['HTTP_USER_AGENT'],'iP'); if ($iCrap) @@ -28,9 +55,11 @@ function dousperf($data, $user) $vlines = true; $pg = '

User Shift Reward Performance


'; + if ($ans['STATUS'] == 'ok' and $ans['DATA'] != '') { addGBase(); + addTips(); $cbx = array('skey' => 'shift key', 'slines' => 'shift lines', 'tkey' => 'time key', 'tlines' => 'time lines', 'over' => 'key overlap', 'smooth' => 'smooth', @@ -39,7 +68,29 @@ function dousperf($data, $user) if ($vlines === true) $xon['slines'] = 1; - $pg .= '
'; + $pg .= '
'; + + $tt = "
  • all = all workers
  • noname = worker with no workername
    • "; + $tt .= "
  • or full workername without the username i.e. .worker or _worker
"; + $pg .= "?"; + $pg .= "$tt"; + + $i = 0; + $datacols = ''; + $onch = " onchange='wch()'"; + foreach ($cols as $col) + { + $i++; + $pg .= " Worker$i:"; + $pg .= " "; + + if ($i > 1) + $datacols .= ','; + $datacols .= $col; + } + + $oncl = "wch();location.href=\"".makeURL('usperf')."\""; + $pg .= "
"; foreach ($cbx as $nam => $txt) { $pg .= ' '; @@ -52,8 +103,9 @@ function dousperf($data, $user) $pg .= 'A graph will show here if your browser supports html5/canvas'; $pg .= "
\n"; $data = str_replace(array("\\","'"), array("\\\\","\\'"), $ans['DATA']); + $data .= $fld_sep . 'cols' . $val_sep . $datacols; $pg .= "\n"; - + $head .= "\n"; $head .= "