From c3cbef6eca51207ff08f461a1b5a25c2e6cfb157 Mon Sep 17 00:00:00 2001 From: kanoi Date: Mon, 14 Sep 2015 21:16:14 +1000 Subject: [PATCH 01/62] ckdb - make a separate object for a ktree vs the tree nodes --- src/ckdb.c | 120 +++++++------ src/ckdb.h | 2 +- src/ckdb_cmd.c | 35 ++-- src/ckdb_data.c | 140 +++++++-------- src/ckdb_dbio.c | 311 ++++++++++++++------------------- src/ktree.c | 453 ++++++++++++++++++++++++++---------------------- src/ktree.h | 65 ++++--- 7 files changed, 553 insertions(+), 573 deletions(-) diff --git a/src/ckdb.c b/src/ckdb.c index 935c8aa6..154916f5 100644 --- a/src/ckdb.c +++ b/src/ckdb.c @@ -999,43 +999,43 @@ static void alloc_storage() users_free = k_new_list("Users", sizeof(USERS), ALLOC_USERS, LIMIT_USERS, true); users_store = k_new_store(users_free); - users_root = new_ktree(); - userid_root = new_ktree(); + users_root = new_ktree(cmp_users); + userid_root = new_ktree(cmp_userid); useratts_free = k_new_list("Useratts", sizeof(USERATTS), ALLOC_USERATTS, LIMIT_USERATTS, true); useratts_store = k_new_store(useratts_free); - useratts_root = new_ktree(); + useratts_root = new_ktree(cmp_useratts); optioncontrol_free = k_new_list("OptionControl", sizeof(OPTIONCONTROL), ALLOC_OPTIONCONTROL, LIMIT_OPTIONCONTROL, true); optioncontrol_store = k_new_store(optioncontrol_free); - optioncontrol_root = new_ktree(); + optioncontrol_root = new_ktree(cmp_optioncontrol); workers_free = k_new_list("Workers", sizeof(WORKERS), ALLOC_WORKERS, LIMIT_WORKERS, true); workers_store = k_new_store(workers_free); - workers_root = new_ktree(); + workers_root = new_ktree(cmp_workers); paymentaddresses_free = k_new_list("PaymentAddresses", sizeof(PAYMENTADDRESSES), ALLOC_PAYMENTADDRESSES, LIMIT_PAYMENTADDRESSES, true); paymentaddresses_store = k_new_store(paymentaddresses_free); - paymentaddresses_root = new_ktree(); - paymentaddresses_create_root = new_ktree(); + paymentaddresses_root = new_ktree(cmp_paymentaddresses); + paymentaddresses_create_root = new_ktree(cmp_payaddr_create); paymentaddresses_free->dsp_func = dsp_paymentaddresses; payments_free = k_new_list("Payments", sizeof(PAYMENTS), ALLOC_PAYMENTS, LIMIT_PAYMENTS, true); payments_store = k_new_store(payments_free); - payments_root = new_ktree(); + payments_root = new_ktree(cmp_payments); accountbalance_free = k_new_list("AccountBalance", sizeof(ACCOUNTBALANCE), ALLOC_ACCOUNTBALANCE, LIMIT_ACCOUNTBALANCE, true); accountbalance_store = k_new_store(accountbalance_free); - accountbalance_root = new_ktree(); + accountbalance_root = new_ktree(cmp_accountbalance); idcontrol_free = k_new_list("IDControl", sizeof(IDCONTROL), ALLOC_IDCONTROL, LIMIT_IDCONTROL, true); @@ -1044,98 +1044,98 @@ static void alloc_storage() workinfo_free = k_new_list("WorkInfo", sizeof(WORKINFO), ALLOC_WORKINFO, LIMIT_WORKINFO, true); workinfo_store = k_new_store(workinfo_free); - workinfo_root = new_ktree(); + workinfo_root = new_ktree(cmp_workinfo); if (!confirm_sharesummary) - workinfo_height_root = new_ktree(); + workinfo_height_root = new_ktree(cmp_workinfo_height); shares_free = k_new_list("Shares", sizeof(SHARES), ALLOC_SHARES, LIMIT_SHARES, true); shares_store = k_new_store(shares_free); shares_early_store = k_new_store(shares_free); - shares_root = new_ktree(); - shares_early_root = new_ktree(); + shares_root = new_ktree(cmp_shares); + shares_early_root = new_ktree(cmp_shares); shareerrors_free = k_new_list("ShareErrors", sizeof(SHAREERRORS), ALLOC_SHAREERRORS, LIMIT_SHAREERRORS, true); shareerrors_store = k_new_store(shareerrors_free); shareerrors_early_store = k_new_store(shareerrors_free); - shareerrors_root = new_ktree(); - shareerrors_early_root = new_ktree(); + shareerrors_root = new_ktree(cmp_shareerrors); + shareerrors_early_root = new_ktree(cmp_shareerrors); sharesummary_free = k_new_list("ShareSummary", sizeof(SHARESUMMARY), ALLOC_SHARESUMMARY, LIMIT_SHARESUMMARY, true); sharesummary_store = k_new_store(sharesummary_free); - sharesummary_root = new_ktree(); - sharesummary_workinfoid_root = new_ktree(); + sharesummary_root = new_ktree(cmp_sharesummary); + sharesummary_workinfoid_root = new_ktree(cmp_sharesummary_workinfoid); sharesummary_free->dsp_func = dsp_sharesummary; sharesummary_pool_store = k_new_store(sharesummary_free); - sharesummary_pool_root = new_ktree(); + sharesummary_pool_root = new_ktree(cmp_sharesummary); blocks_free = k_new_list("Blocks", sizeof(BLOCKS), ALLOC_BLOCKS, LIMIT_BLOCKS, true); blocks_store = k_new_store(blocks_free); - blocks_root = new_ktree(); + blocks_root = new_ktree(cmp_blocks); blocks_free->dsp_func = dsp_blocks; miningpayouts_free = k_new_list("MiningPayouts", sizeof(MININGPAYOUTS), ALLOC_MININGPAYOUTS, LIMIT_MININGPAYOUTS, true); miningpayouts_store = k_new_store(miningpayouts_free); - miningpayouts_root = new_ktree(); + miningpayouts_root = new_ktree(cmp_miningpayouts); payouts_free = k_new_list("Payouts", sizeof(PAYOUTS), ALLOC_PAYOUTS, LIMIT_PAYOUTS, true); payouts_store = k_new_store(payouts_free); - payouts_root = new_ktree(); - payouts_id_root = new_ktree(); - payouts_wid_root = new_ktree(); + payouts_root = new_ktree(cmp_payouts); + payouts_id_root = new_ktree(cmp_payouts_id); + payouts_wid_root = new_ktree(cmp_payouts_wid); auths_free = k_new_list("Auths", sizeof(AUTHS), ALLOC_AUTHS, LIMIT_AUTHS, true); auths_store = k_new_store(auths_free); - auths_root = new_ktree(); + auths_root = new_ktree(cmp_auths); poolstats_free = k_new_list("PoolStats", sizeof(POOLSTATS), ALLOC_POOLSTATS, LIMIT_POOLSTATS, true); poolstats_store = k_new_store(poolstats_free); - poolstats_root = new_ktree(); + poolstats_root = new_ktree(cmp_poolstats); userstats_free = k_new_list("UserStats", sizeof(USERSTATS), ALLOC_USERSTATS, LIMIT_USERSTATS, true); userstats_store = k_new_store(userstats_free); userstats_eos_store = k_new_store(userstats_free); - userstats_root = new_ktree(); + userstats_root = new_ktree(cmp_userstats); userstats_free->dsp_func = dsp_userstats; workerstatus_free = k_new_list("WorkerStatus", sizeof(WORKERSTATUS), ALLOC_WORKERSTATUS, LIMIT_WORKERSTATUS, true); workerstatus_store = k_new_store(workerstatus_free); - workerstatus_root = new_ktree(); + workerstatus_root = new_ktree(cmp_workerstatus); markersummary_free = k_new_list("MarkerSummary", sizeof(MARKERSUMMARY), ALLOC_MARKERSUMMARY, LIMIT_MARKERSUMMARY, true); markersummary_store = k_new_store(markersummary_free); - markersummary_root = new_ktree(); - markersummary_userid_root = new_ktree(); + markersummary_root = new_ktree(cmp_markersummary); + markersummary_userid_root = new_ktree(cmp_markersummary_userid); markersummary_free->dsp_func = dsp_markersummary; markersummary_pool_store = k_new_store(markersummary_free); - markersummary_pool_root = new_ktree(); + markersummary_pool_root = new_ktree(cmp_markersummary); workmarkers_free = k_new_list("WorkMarkers", sizeof(WORKMARKERS), ALLOC_WORKMARKERS, LIMIT_WORKMARKERS, true); workmarkers_store = k_new_store(workmarkers_free); - workmarkers_root = new_ktree(); - workmarkers_workinfoid_root = new_ktree(); + workmarkers_root = new_ktree(cmp_workmarkers); + workmarkers_workinfoid_root = new_ktree(cmp_workmarkers_workinfoid); workmarkers_free->dsp_func = dsp_workmarkers; marks_free = k_new_list("Marks", sizeof(MARKS), ALLOC_MARKS, LIMIT_MARKS, true); marks_store = k_new_store(marks_free); - marks_root = new_ktree(); + marks_root = new_ktree(cmp_marks); userinfo_free = k_new_list("UserInfo", sizeof(USERINFO), ALLOC_USERINFO, LIMIT_USERINFO, true); userinfo_store = k_new_store(userinfo_free); - userinfo_root = new_ktree(); + userinfo_root = new_ktree(cmp_userinfo); } #define SEQSETMSG(_set, _seqset, _msgtxt, _endtxt) do { \ @@ -1179,7 +1179,7 @@ static void alloc_storage() #define FREE_TREE(_tree) \ if (_tree ## _root) \ - _tree ## _root = free_ktree(_tree ## _root, NULL) \ + free_ktree(_tree ## _root, NULL) \ #define FREE_STORE(_list) \ if (_list ## _store) \ @@ -1487,13 +1487,13 @@ static bool setup_data() INIT_WORKINFO(&look); look.data = (void *)(&wi); // Find the first workinfo for this height - found = find_after_in_ktree(workinfo_height_root, &look, cmp_workinfo_height, ctx); + found = find_after_in_ktree(workinfo_height_root, &look, ctx); if (found) { DATA_WORKINFO(wif, found); copy_tv(&last_bc, &(wif->createdate)); } // No longer needed - workinfo_height_root = free_ktree(workinfo_height_root, NULL); + free_ktree(workinfo_height_root, NULL); } return true; @@ -2536,7 +2536,7 @@ static enum cmd_values breakdown(K_ITEM **ml_item, char *buf, tv_t *now, goto nogood; } - msgline->trf_root = new_ktree(); + msgline->trf_root = new_ktree(cmp_transfer); msgline->trf_store = k_new_store(transfer_free); next = data; if (next && strncmp(next, JSON_TRANSFER, JSON_TRANSFER_LEN) == 0) { @@ -2688,8 +2688,7 @@ static enum cmd_values breakdown(K_ITEM **ml_item, char *buf, tv_t *now, STRNCPYSIZ(transfer->svalue, next, siz+1); transfer->mvalue = transfer->svalue; } - msgline->trf_root = add_to_ktree(msgline->trf_root, - t_item, cmp_transfer); + add_to_ktree(msgline->trf_root, t_item); k_add_head(msgline->trf_store, t_item); t_item = NULL; @@ -2732,15 +2731,12 @@ static enum cmd_values breakdown(K_ITEM **ml_item, char *buf, tv_t *now, transfer->mvalue = transfer->svalue; // Discard duplicates - if (find_in_ktree(msgline->trf_root, t_item, - cmp_transfer, ctx)) { + if (find_in_ktree(msgline->trf_root, t_item, ctx)) { if (transfer->mvalue != transfer->svalue) FREENULL(transfer->mvalue); k_add_head(transfer_free, t_item); } else { - msgline->trf_root = add_to_ktree(msgline->trf_root, - t_item, - cmp_transfer); + add_to_ktree(msgline->trf_root, t_item); k_add_head(msgline->trf_store, t_item); } } @@ -2935,7 +2931,7 @@ static void summarise_blocks() K_RLOCK(markersummary_free); ss_item = find_before_in_ktree(sharesummary_workinfoid_root, &ss_look, - cmp_sharesummary_workinfoid, ss_ctx); + ss_ctx); DATA_SHARESUMMARY_NULL(sharesummary, ss_item); while (ss_item && sharesummary->workinfoid > wi_start) { if (sharesummary->complete[0] == SUMMARY_NEW) { @@ -2971,7 +2967,7 @@ static void summarise_blocks() INIT_WORKMARKERS(&wm_look); wm_look.data = (void *)(&lookworkmarkers); wm_item = find_before_in_ktree(workmarkers_workinfoid_root, &wm_look, - cmp_workmarkers_workinfoid, ctx); + ctx); DATA_WORKMARKERS_NULL(workmarkers, wm_item); while (wm_item && CURRENT(&(workmarkers->expirydate)) && @@ -2995,7 +2991,7 @@ static void summarise_blocks() INIT_MARKERSUMMARY(&ms_look); ms_look.data = (void *)(&lookmarkersummary); ms_item = find_before_in_ktree(markersummary_root, &ms_look, - cmp_markersummary, ms_ctx); + ms_ctx); DATA_MARKERSUMMARY_NULL(markersummary, ms_item); while (ms_item && markersummary->markerid == workmarkers->markerid) { has_ms = true; @@ -3350,7 +3346,7 @@ static void make_a_shift_mark() lookworkinfo.expirydate.tv_usec = default_expiry.tv_usec; wi_look.data = (void *)(&lookworkinfo); K_RLOCK(workinfo_free); - wi_item = find_after_in_ktree(workinfo_root, &wi_look, cmp_workinfo, wi_ctx); + wi_item = find_after_in_ktree(workinfo_root, &wi_look, wi_ctx); K_RUNLOCK(workinfo_free); marks_wid = 0; used_wid = 0; @@ -3394,8 +3390,8 @@ static void make_a_shift_mark() looksharesummary.workername = EMPTY; ss_look.data = (void *)(&looksharesummary); K_RLOCK(sharesummary_free); - ss_item = find_before_in_ktree(sharesummary_workinfoid_root, &ss_look, - cmp_sharesummary_workinfoid, ss_ctx); + ss_item = find_before_in_ktree(sharesummary_workinfoid_root, + &ss_look, ss_ctx); K_RUNLOCK(sharesummary_free); DATA_SHARESUMMARY_NULL(sharesummary, ss_item); if (ss_item && @@ -4815,7 +4811,7 @@ static void compare_summaries(K_TREE *leftsum, char *leftname, look.data = (void *)(&looksharesummary); total = ok = missing = diff = 0; - lss = find_after_in_ktree(leftsum, &look, cmp_sharesummary_workinfoid, ctxl); + lss = find_after_in_ktree(leftsum, &look, ctxl); while (lss) { DATA_SHARESUMMARY(l_ss, lss); if (l_ss->workinfoid > confirm_last_workinfoid) @@ -4827,7 +4823,7 @@ static void compare_summaries(K_TREE *leftsum, char *leftname, first_used = l_ss->workinfoid; last_used = l_ss->workinfoid; - rss = find_in_ktree(rightsum, lss, cmp_sharesummary_workinfoid, ctxr); + rss = find_in_ktree(rightsum, lss, ctxr); DATA_SHARESUMMARY_NULL(r_ss, rss); if (!rss) { missing++; @@ -5036,7 +5032,7 @@ static void confirm_reload() lookblocks.height = confirm_block; lookblocks.blockhash[0] = '\0'; b_look.data = (void *)(&lookblocks); - b_end_item = find_after_in_ktree(blocks_root, &b_look, cmp_blocks, ctx); + b_end_item = find_after_in_ktree(blocks_root, &b_look, ctx); if (!b_end_item) { LOGWARNING("%s(): no DB block height found matching or after %d", __func__, confirm_block); @@ -5049,7 +5045,7 @@ static void confirm_reload() lookblocks.height = e_blocks->height; lookblocks.blockhash[0] = '\0'; b_look.data = (void *)(&lookblocks); - b_begin_item = find_before_in_ktree(blocks_root, &b_look, cmp_blocks, ctx); + b_begin_item = find_before_in_ktree(blocks_root, &b_look, ctx); if (!b_begin_item) confirm_first_workinfoid = 0; else { @@ -5058,7 +5054,7 @@ static void confirm_reload() lookblocks.height = b_blocks->height; lookblocks.blockhash[0] = '\0'; b_look.data = (void *)(&lookblocks); - b_begin_item = find_after_in_ktree(blocks_root, &b_look, cmp_blocks, ctx); + b_begin_item = find_after_in_ktree(blocks_root, &b_look, ctx); // Not possible if (!b_begin_item) confirm_first_workinfoid = 0; @@ -5110,7 +5106,7 @@ static void confirm_reload() lookworkinfo.expirydate.tv_sec = date_begin.tv_sec; lookworkinfo.expirydate.tv_usec = date_begin.tv_usec; wi_look.data = (void *)(&lookworkinfo); - wi_item = find_before_in_ktree(workinfo_root, &wi_look, cmp_workinfo, ctx); + wi_item = find_before_in_ktree(workinfo_root, &wi_look, ctx); if (wi_item) { DATA_WORKINFO(workinfo, wi_item); copy_tv(&start, &(workinfo->createdate)); @@ -5138,7 +5134,7 @@ static void confirm_reload() lookworkinfo.expirydate.tv_sec = date_eot.tv_sec; lookworkinfo.expirydate.tv_usec = date_eot.tv_usec; wi_look.data = (void *)(&lookworkinfo); - wi_item = find_after_in_ktree(workinfo_root, &wi_look, cmp_workinfo, ctx); + wi_item = find_after_in_ktree(workinfo_root, &wi_look, ctx); if (wi_item) { DATA_WORKINFO(workinfo, wi_item); /* Now find the one after the one we found to determine the @@ -5147,7 +5143,7 @@ static void confirm_reload() lookworkinfo.expirydate.tv_sec = date_eot.tv_sec; lookworkinfo.expirydate.tv_usec = date_eot.tv_usec; wi_look.data = (void *)(&lookworkinfo); - wi_item = find_after_in_ktree(workinfo_root, &wi_look, cmp_workinfo, ctx); + wi_item = find_after_in_ktree(workinfo_root, &wi_look, ctx); if (wi_item) { DATA_WORKINFO(workinfo, wi_item); copy_tv(&confirm_finish, &(workinfo->createdate)); @@ -5178,9 +5174,9 @@ static void confirm_reload() sharesummary_save = sharesummary_root; workinfo_save = workinfo_root; - sharesummary_workinfoid_root = new_ktree(); - sharesummary_root = new_ktree(); - workinfo_root = new_ktree(); + sharesummary_workinfoid_root = new_ktree(cmp_sharesummary_workinfoid); + sharesummary_root = new_ktree(cmp_sharesummary); + workinfo_root = new_ktree(cmp_workinfo); if (start.tv_sec < DATE_BEGIN) { start.tv_sec = DATE_BEGIN; diff --git a/src/ckdb.h b/src/ckdb.h index af0ff79e..cefe3dbe 100644 --- a/src/ckdb.h +++ b/src/ckdb.h @@ -55,7 +55,7 @@ #define DB_VLOCK "1" #define DB_VERSION "1.0.2" -#define CKDB_VERSION DB_VERSION"-1.241" +#define CKDB_VERSION DB_VERSION"-1.300" #define WHERE_FFL " - from %s %s() line %d" #define WHERE_FFL_HERE __FILE__, __func__, __LINE__ diff --git a/src/ckdb_cmd.c b/src/ckdb_cmd.c index f4911289..0e1c23e7 100644 --- a/src/ckdb_cmd.c +++ b/src/ckdb_cmd.c @@ -926,7 +926,7 @@ static char *cmd_poolstats_do(PGconn *conn, char *cmd, char *id, char *by, row.createdate.tv_usec = date_eot.tv_usec; INIT_POOLSTATS(&look); look.data = (void *)(&row); - ps = find_before_in_ktree(poolstats_root, &look, cmp_poolstats, ctx); + ps = find_before_in_ktree(poolstats_root, &look, ctx); if (!ps) store = true; else { @@ -1730,7 +1730,7 @@ static char *cmd_percent(char *cmd, char *id, tv_t *now, USERS *users) lookworkers.workername[0] = '\0'; DATE_ZERO(&(lookworkers.expirydate)); w_look.data = (void *)(&lookworkers); - w_item = find_after_in_ktree(workers_root, &w_look, cmp_workers, w_ctx); + w_item = find_after_in_ktree(workers_root, &w_look, w_ctx); DATA_WORKERS_NULL(workers, w_item); while (w_item && workers->userid == users->userid) { if (CURRENT(&(workers->expirydate))) { @@ -1991,7 +1991,7 @@ static char *cmd_workers(__maybe_unused PGconn *conn, char *cmd, char *id, lookworkers.workername[0] = '\0'; DATE_ZERO(&(lookworkers.expirydate)); w_look.data = (void *)(&lookworkers); - w_item = find_after_in_ktree(workers_root, &w_look, cmp_workers, w_ctx); + w_item = find_after_in_ktree(workers_root, &w_look, w_ctx); DATA_WORKERS_NULL(workers, w_item); rows = 0; while (w_item && workers->userid == users->userid) { @@ -3285,7 +3285,7 @@ static char *cmd_homepage(__maybe_unused PGconn *conn, char *cmd, char *id, INIT_USERSTATS(&look); look.data = (void *)(&lookuserstats); K_RLOCK(userstats_free); - us_item = find_before_in_ktree(userstats_root, &look, cmp_userstats, ctx); + us_item = find_before_in_ktree(userstats_root, &look, ctx); DATA_USERSTATS_NULL(userstats, us_item); while (us_item && userstats->userid == users->userid) { if (tvdiff(now, &(userstats->statsdate)) < USERSTATS_PER_S) { @@ -4045,7 +4045,7 @@ static char *cmd_pplns(__maybe_unused PGconn *conn, char *cmd, char *id, INIT_BLOCKS(&b_look); b_look.data = (void *)(&lookblocks); K_RLOCK(blocks_free); - b_item = find_before_in_ktree(blocks_root, &b_look, cmp_blocks, ctx); + b_item = find_before_in_ktree(blocks_root, &b_look, ctx); if (!b_item) { K_RUNLOCK(blocks_free); snprintf(reply, siz, "ERR.no block height %d", height); @@ -4120,7 +4120,7 @@ static char *cmd_pplns(__maybe_unused PGconn *conn, char *cmd, char *id, ss_count = wm_count = ms_count = 0; mu_store = k_new_store(miningpayouts_free); - mu_root = new_ktree(); + mu_root = new_ktree(cmp_mu); looksharesummary.workinfoid = block_workinfoid; looksharesummary.userid = MAXID; @@ -4130,8 +4130,8 @@ static char *cmd_pplns(__maybe_unused PGconn *conn, char *cmd, char *id, K_RLOCK(sharesummary_free); K_RLOCK(workmarkers_free); K_RLOCK(markersummary_free); - ss_item = find_before_in_ktree(sharesummary_workinfoid_root, &ss_look, - cmp_sharesummary_workinfoid, ctx); + ss_item = find_before_in_ktree(sharesummary_workinfoid_root, + &ss_look, ctx); DATA_SHARESUMMARY_NULL(sharesummary, ss_item); if (ss_item) end_workinfoid = sharesummary->workinfoid; @@ -4204,8 +4204,8 @@ static char *cmd_pplns(__maybe_unused PGconn *conn, char *cmd, char *id, lookworkmarkers.workinfoidend = block_workinfoid + 1; INIT_WORKMARKERS(&wm_look); wm_look.data = (void *)(&lookworkmarkers); - wm_item = find_before_in_ktree(workmarkers_workinfoid_root, &wm_look, - cmp_workmarkers_workinfoid, wm_ctx); + wm_item = find_before_in_ktree(workmarkers_workinfoid_root, + &wm_look, wm_ctx); DATA_WORKMARKERS_NULL(workmarkers, wm_item); LOGDEBUG("%s(): workmarkers < %"PRId64, __func__, lookworkmarkers.workinfoidend); while (total_diff < diff_want && wm_item && CURRENT(&(workmarkers->expirydate))) { @@ -4220,8 +4220,8 @@ static char *cmd_pplns(__maybe_unused PGconn *conn, char *cmd, char *id, lookmarkersummary.workername = EMPTY; INIT_MARKERSUMMARY(&ms_look); ms_look.data = (void *)(&lookmarkersummary); - ms_item = find_before_in_ktree(markersummary_root, &ms_look, - cmp_markersummary, ms_ctx); + ms_item = find_before_in_ktree(markersummary_root, + &ms_look, ms_ctx); DATA_MARKERSUMMARY_NULL(markersummary, ms_item); // add the whole markerid while (ms_item && markersummary->markerid == workmarkers->markerid) { @@ -4431,7 +4431,7 @@ static char *cmd_pplns(__maybe_unused PGconn *conn, char *cmd, char *id, // So web can always verify it received all data APPEND_REALLOC(buf, off, len, "pplns_last=1"); - mu_root = free_ktree(mu_root, NULL); + free_ktree(mu_root, NULL); K_WLOCK(mu_store); k_list_transfer_to_head(mu_store, miningpayouts_free); K_WUNLOCK(mu_store); @@ -4441,7 +4441,8 @@ static char *cmd_pplns(__maybe_unused PGconn *conn, char *cmd, char *id, return buf; shazbot: - mu_root = free_ktree(mu_root, NULL); + + free_ktree(mu_root, NULL); K_WLOCK(mu_store); k_list_transfer_to_head(mu_store, miningpayouts_free); K_WUNLOCK(mu_store); @@ -4494,7 +4495,7 @@ static char *cmd_pplns2(__maybe_unused PGconn *conn, char *cmd, char *id, INIT_BLOCKS(&b_look); b_look.data = (void *)(&lookblocks); K_RLOCK(blocks_free); - b_item = find_after_in_ktree(blocks_root, &b_look, cmp_blocks, b_ctx); + b_item = find_after_in_ktree(blocks_root, &b_look, b_ctx); K_RUNLOCK(blocks_free); if (!b_item) { K_RUNLOCK(blocks_free); @@ -5257,8 +5258,8 @@ static char *cmd_shifts(__maybe_unused PGconn *conn, char *cmd, char *id, markersummary.userid = users->userid; markersummary.workername = EMPTY; K_RLOCK(markersummary_free); - ms_item = find_after_in_ktree(markersummary_root, &ms_look, - cmp_markersummary, ms_ctx); + ms_item = find_after_in_ktree(markersummary_root, + &ms_look, ms_ctx); DATA_MARKERSUMMARY_NULL(ms, ms_item); while (ms_item && ms->markerid == wm->markerid && ms->userid == users->userid) { diff --git a/src/ckdb_data.c b/src/ckdb_data.c index 63a872c0..a7cc7eb3 100644 --- a/src/ckdb_data.c +++ b/src/ckdb_data.c @@ -20,7 +20,7 @@ void free_msgline_data(K_ITEM *item, bool t_lock, bool t_cull) DATA_MSGLINE(msgline, item); if (msgline->trf_root) - msgline->trf_root = free_ktree(msgline->trf_root, NULL); + free_ktree(msgline->trf_root, NULL); if (msgline->trf_store) { t_item = msgline->trf_store->head; while (t_item) { @@ -691,7 +691,7 @@ K_ITEM *find_transfer(K_TREE *trf_root, char *name) STRNCPY(transfer.name, name); INIT_TRANSFER(&look); look.data = (void *)(&transfer); - return find_in_ktree(trf_root, &look, cmp_transfer, ctx); + return find_in_ktree(trf_root, &look, ctx); } K_ITEM *_optional_name(K_TREE *trf_root, char *name, int len, char *patt, @@ -827,7 +827,7 @@ K_ITEM *get_workerstatus(int64_t userid, char *workername) INIT_WORKERSTATUS(&look); look.data = (void *)(&workerstatus); K_RLOCK(workerstatus_free); - find = find_in_ktree(workerstatus_root, &look, cmp_workerstatus, ctx); + find = find_in_ktree(workerstatus_root, &look, ctx); K_RUNLOCK(workerstatus_free); return find; } @@ -876,7 +876,7 @@ K_ITEM *_find_create_workerstatus(int64_t userid, char *workername, row->userid = userid; STRNCPY(row->workername, workername); - workerstatus_root = add_to_ktree(workerstatus_root, ws_item, cmp_workerstatus); + add_to_ktree(workerstatus_root, ws_item); k_add_head(workerstatus_store, ws_item); K_WUNLOCK(workerstatus_free); @@ -1115,7 +1115,7 @@ K_ITEM *find_users(char *username) INIT_USERS(&look); look.data = (void *)(&users); - return find_in_ktree(users_root, &look, cmp_users, ctx); + return find_in_ktree(users_root, &look, ctx); } // Must be R or W locked before call @@ -1131,7 +1131,7 @@ K_ITEM *find_userid(int64_t userid) INIT_USERS(&look); look.data = (void *)(&users); - return find_in_ktree(userid_root, &look, cmp_userid, ctx); + return find_in_ktree(userid_root, &look, ctx); } // TODO: endian? (to avoid being all zeros?) @@ -1428,7 +1428,7 @@ K_ITEM *find_useratts(int64_t userid, char *attname) INIT_USERATTS(&look); look.data = (void *)(&useratts); - return find_in_ktree(useratts_root, &look, cmp_useratts, ctx); + return find_in_ktree(useratts_root, &look, ctx); } // order by userid asc,workername asc,expirydate desc @@ -1459,7 +1459,7 @@ K_ITEM *find_workers(int64_t userid, char *workername) INIT_WORKERS(&look); look.data = (void *)(&workers); - return find_in_ktree(workers_root, &look, cmp_workers, ctx); + return find_in_ktree(workers_root, &look, ctx); } K_ITEM *first_workers(int64_t userid, K_TREE_CTX *ctx) @@ -1478,7 +1478,7 @@ K_ITEM *first_workers(int64_t userid, K_TREE_CTX *ctx) INIT_WORKERS(&look); look.data = (void *)(&workers); // Caller needs to check userid/expirydate if the result != NULL - return find_after_in_ktree(workers_root, &look, cmp_workers, ctx); + return find_after_in_ktree(workers_root, &look, ctx); } K_ITEM *new_worker(PGconn *conn, bool update, int64_t userid, char *workername, @@ -1607,7 +1607,7 @@ K_ITEM *find_paymentaddresses(int64_t userid, K_TREE_CTX *ctx) INIT_PAYMENTADDRESSES(&look); look.data = (void *)(&paymentaddresses); - item = find_before_in_ktree(paymentaddresses_root, &look, cmp_paymentaddresses, ctx); + item = find_before_in_ktree(paymentaddresses_root, &look, ctx); if (item) { DATA_PAYMENTADDRESSES(pa, item); if (pa->userid == userid && CURRENT(&(pa->expirydate))) @@ -1631,8 +1631,7 @@ K_ITEM *find_paymentaddresses_create(int64_t userid, K_TREE_CTX *ctx) INIT_PAYMENTADDRESSES(&look); look.data = (void *)(&paymentaddresses); - item = find_after_in_ktree(paymentaddresses_create_root, &look, - cmp_payaddr_create, ctx); + item = find_after_in_ktree(paymentaddresses_create_root, &look, ctx); if (item) { DATA_PAYMENTADDRESSES(pa, item); if (pa->userid == userid) @@ -1655,7 +1654,7 @@ K_ITEM *find_one_payaddress(int64_t userid, char *payaddress, K_TREE_CTX *ctx) INIT_PAYMENTADDRESSES(&look); look.data = (void *)(&paymentaddresses); - return find_in_ktree(paymentaddresses_root, &look, cmp_paymentaddresses, ctx); + return find_in_ktree(paymentaddresses_root, &look, ctx); } /* This will match any user that has the payaddress @@ -1715,7 +1714,7 @@ K_ITEM *find_payments(int64_t payoutid, int64_t userid, char *subname) INIT_PAYMENTS(&look); look.data = (void *)(&payments); - return find_in_ktree(payments_root, &look, cmp_payments, ctx); + return find_in_ktree(payments_root, &look, ctx); } K_ITEM *find_first_payments(int64_t userid, K_TREE_CTX *ctx) @@ -1733,7 +1732,7 @@ K_ITEM *find_first_payments(int64_t userid, K_TREE_CTX *ctx) INIT_PAYMENTS(&look); look.data = (void *)(&payments); // userid needs to be checked if item returned != NULL - item = find_after_in_ktree(payments_root, &look, cmp_payments, ctx); + item = find_after_in_ktree(payments_root, &look, ctx); return item; } @@ -1753,7 +1752,7 @@ K_ITEM *find_first_paypayid(int64_t userid, int64_t payoutid, K_TREE_CTX *ctx) INIT_PAYMENTS(&look); look.data = (void *)(&payments); // userid+payoutid needs to be checked if item returned != NULL - item = find_after_in_ktree(payments_root, &look, cmp_payments, ctx); + item = find_after_in_ktree(payments_root, &look, ctx); return item; } @@ -1777,7 +1776,7 @@ K_ITEM *find_accountbalance(int64_t userid) INIT_ACCOUNTBALANCE(&look); look.data = (void *)(&accountbalance); K_RLOCK(accountbalance_free); - item = find_in_ktree(accountbalance_root, &look, cmp_accountbalance, ctx); + item = find_in_ktree(accountbalance_root, &look, ctx); K_RUNLOCK(accountbalance_free); return item; } @@ -1853,7 +1852,7 @@ K_ITEM *find_optioncontrol(char *optionname, tv_t *now, int32_t height) INIT_OPTIONCONTROL(&look); look.data = (void *)(&optioncontrol); - item = find_after_in_ktree(optioncontrol_root, &look, cmp_optioncontrol, ctx); + item = find_after_in_ktree(optioncontrol_root, &look, ctx); ocbest = NULL; best = NULL; while (item) { @@ -1989,7 +1988,7 @@ K_ITEM *find_workinfo(int64_t workinfoid, K_TREE_CTX *ctx) INIT_WORKINFO(&look); look.data = (void *)(&workinfo); K_RLOCK(workinfo_free); - item = find_in_ktree(workinfo_root, &look, cmp_workinfo, ctx); + item = find_in_ktree(workinfo_root, &look, ctx); K_RUNLOCK(workinfo_free); return item; } @@ -2010,7 +2009,7 @@ K_ITEM *next_workinfo(int64_t workinfoid, K_TREE_CTX *ctx) INIT_WORKINFO(&look); look.data = (void *)(&workinfo); K_RLOCK(workinfo_free); - item = find_after_in_ktree(workinfo_root, &look, cmp_workinfo, ctx); + item = find_after_in_ktree(workinfo_root, &look, ctx); if (item) { DATA_WORKINFO(wi, item); while (item && !CURRENT(&(wi->expirydate))) { @@ -2091,7 +2090,7 @@ bool workinfo_age(int64_t workinfoid, char *poolinstance, char *by, char *code, diff_tot = 0; ss_look.data = (void *)(&looksharesummary); K_RLOCK(sharesummary_free); - ss_item = find_after_in_ktree(sharesummary_workinfoid_root, &ss_look, cmp_sharesummary_workinfoid, ss_ctx); + ss_item = find_after_in_ktree(sharesummary_workinfoid_root, &ss_look, ss_ctx); K_RUNLOCK(sharesummary_free); DATA_SHARESUMMARY_NULL(sharesummary, ss_item); while (ss_item && sharesummary->workinfoid == workinfoid) { @@ -2142,7 +2141,7 @@ bool workinfo_age(int64_t workinfoid, char *poolinstance, char *by, char *code, s_look.data = (void *)(&lookshares); K_WLOCK(shares_free); - s_item = find_after_in_ktree(shares_root, &s_look, cmp_shares, s_ctx); + s_item = find_after_in_ktree(shares_root, &s_look, s_ctx); while (s_item) { DATA_SHARES(shares, s_item); if (shares->workinfoid != workinfoid || @@ -2154,7 +2153,7 @@ bool workinfo_age(int64_t workinfoid, char *poolinstance, char *by, char *code, if (shares->errn == SE_NONE) diff_tot += shares->diff; tmp_item = next_in_ktree(s_ctx); - shares_root = remove_from_ktree(shares_root, s_item, cmp_shares); + remove_from_ktree(shares_root, s_item); k_unlink_item(shares_store, s_item); if (reloading && skipupdate) shares_dumped++; @@ -2317,13 +2316,10 @@ K_ITEM *_find_sharesummary(int64_t userid, char *workername, int64_t workinfoid, INIT_SHARESUMMARY(&look); look.data = (void *)(&sharesummary); - if (pool) { - return find_in_ktree(sharesummary_pool_root, &look, - cmp_sharesummary, ctx); - } else { - return find_in_ktree(sharesummary_root, &look, - cmp_sharesummary, ctx); - } + if (pool) + return find_in_ktree(sharesummary_pool_root, &look, ctx); + else + return find_in_ktree(sharesummary_root, &look, ctx); } K_ITEM *find_last_sharesummary(int64_t userid, char *workername) @@ -2338,7 +2334,7 @@ K_ITEM *find_last_sharesummary(int64_t userid, char *workername) INIT_SHARESUMMARY(&look); look.data = (void *)(&look_sharesummary); - item = find_before_in_ktree(sharesummary_root, &look, cmp_sharesummary, ctx); + item = find_before_in_ktree(sharesummary_root, &look, ctx); if (item) { DATA_SHARESUMMARY(sharesummary, item); if (sharesummary->userid != userid || @@ -2381,8 +2377,7 @@ void auto_age_older(int64_t workinfoid, char *poolinstance, char *by, look.data = (void *)(&looksharesummary); K_RLOCK(sharesummary_free); - ss_item = find_after_in_ktree(sharesummary_workinfoid_root, &look, - cmp_sharesummary_workinfoid, ctx); + ss_item = find_after_in_ktree(sharesummary_workinfoid_root, &look, ctx); DATA_SHARESUMMARY_NULL(sharesummary, ss_item); DATE_ZERO(&ss_first_min); @@ -2613,7 +2608,7 @@ K_ITEM *find_blocks(int32_t height, char *blockhash, K_TREE_CTX *ctx) INIT_BLOCKS(&look); look.data = (void *)(&blocks); - return find_in_ktree(blocks_root, &look, cmp_blocks, ctx); + return find_in_ktree(blocks_root, &look, ctx); } // Must be R or W locked before call @@ -2632,7 +2627,7 @@ K_ITEM *find_prev_blocks(int32_t height) INIT_BLOCKS(&look); look.data = (void *)(&lookblocks); - b_item = find_before_in_ktree(blocks_root, &look, cmp_blocks, ctx); + b_item = find_before_in_ktree(blocks_root, &look, ctx); while (b_item) { DATA_BLOCKS(blocks, b_item); if (blocks->confirmed[0] != BLOCKS_NEW && @@ -2716,8 +2711,8 @@ void set_block_share_counters() looksharesummary.workername = sharesummary->workername; looksharesummary.workinfoid = -1; ss_look.data = (void *)(&looksharesummary); - ss_item = find_before_in_ktree(sharesummary_root, &ss_look, - cmp_sharesummary, ctx); + ss_item = find_before_in_ktree(sharesummary_root, + &ss_look, ctx); continue; } @@ -2800,7 +2795,8 @@ void set_block_share_counters() lookmarkersummary.userid = MAXID; lookmarkersummary.workername = EMPTY; ms_look.data = (void *)(&lookmarkersummary); - ms_item = find_before_in_ktree(markersummary_root, &ms_look, cmp_markersummary, ctx_ms); + ms_item = find_before_in_ktree(markersummary_root, + &ms_look, ctx_ms); while (ms_item) { DATA_MARKERSUMMARY(markersummary, ms_item); if (markersummary->markerid != workmarkers->markerid) @@ -3067,7 +3063,7 @@ bool _set_prevcreatedate(int32_t oldest_height, WHERE_FFL_ARGS) INIT_BLOCKS(&look); look.data = (void *)(&lookblocks); - b_item = find_before_in_ktree(blocks_root, &look, cmp_blocks, b_ctx); + b_item = find_before_in_ktree(blocks_root, &look, b_ctx); while (b_item) { DATA_BLOCKS(blocks, b_item); if (CURRENT(&(blocks->expirydate)) && @@ -3194,7 +3190,7 @@ K_ITEM *find_miningpayouts(int64_t payoutid, int64_t userid) INIT_MININGPAYOUTS(&look); look.data = (void *)(&miningpayouts); - return find_in_ktree(miningpayouts_root, &look, cmp_miningpayouts, ctx); + return find_in_ktree(miningpayouts_root, &look, ctx); } K_ITEM *first_miningpayouts(int64_t payoutid, K_TREE_CTX *ctx) @@ -3212,7 +3208,7 @@ K_ITEM *first_miningpayouts(int64_t payoutid, K_TREE_CTX *ctx) INIT_MININGPAYOUTS(&look); look.data = (void *)(&miningpayouts); - return find_after_in_ktree(miningpayouts_root, &look, cmp_miningpayouts, ctx); + return find_after_in_ktree(miningpayouts_root, &look, ctx); } /* Processing payouts uses it's own tree of miningpayouts keyed only on userid @@ -3240,7 +3236,7 @@ K_TREE *upd_add_mu(K_TREE *mu_root, K_STORE *mu_store, int64_t userid, INIT_MININGPAYOUTS(&look); look.data = (void *)(&lookminingpayouts); // No locking required since it's not a shared tree or store - mu_item = find_in_ktree(mu_root, &look, cmp_mu, ctx); + mu_item = find_in_ktree(mu_root, &look, ctx); if (mu_item) { DATA_MININGPAYOUTS(miningpayouts, mu_item); miningpayouts->diffacc += diffacc; @@ -3250,7 +3246,7 @@ K_TREE *upd_add_mu(K_TREE *mu_root, K_STORE *mu_store, int64_t userid, DATA_MININGPAYOUTS(miningpayouts, mu_item); miningpayouts->userid = userid; miningpayouts->diffacc = diffacc; - mu_root = add_to_ktree(mu_root, mu_item, cmp_mu); + add_to_ktree(mu_root, mu_item); k_add_head(mu_store, mu_item); K_WUNLOCK(mu_store); } @@ -3314,7 +3310,7 @@ K_ITEM *find_payouts(int32_t height, char *blockhash) INIT_PAYOUTS(&look); look.data = (void *)(&payouts); - return find_in_ktree(payouts_root, &look, cmp_payouts, ctx); + return find_in_ktree(payouts_root, &look, ctx); } // Last block payout calculated @@ -3346,7 +3342,7 @@ K_ITEM *find_payoutid(int64_t payoutid) INIT_PAYOUTS(&look); look.data = (void *)(&payouts); - return find_in_ktree(payouts_id_root, &look, cmp_payouts_id, ctx); + return find_in_ktree(payouts_id_root, &look, ctx); } // First payouts workinfoidend equal or before workinfoidend @@ -3364,7 +3360,7 @@ K_ITEM *find_payouts_wid(int64_t workinfoidend, K_TREE_CTX *ctx) INIT_PAYOUTS(&look); look.data = (void *)(&payouts); - return find_before_in_ktree(payouts_wid_root, &look, cmp_payouts_wid, ctx); + return find_before_in_ktree(payouts_wid_root, &look, ctx); } /* Values from payout stats, returns -1 if statname isn't found @@ -3603,7 +3599,7 @@ bool process_pplns(int32_t height, char *blockhash, tv_t *addr_cd) ss_count = wm_count = ms_count = 0; mu_store = k_new_store(miningpayouts_free); - mu_root = new_ktree(); + mu_root = new_ktree(cmp_mu); looksharesummary.workinfoid = blocks->workinfoid; looksharesummary.userid = MAXID; @@ -3614,7 +3610,7 @@ bool process_pplns(int32_t height, char *blockhash, tv_t *addr_cd) K_RLOCK(workmarkers_free); K_RLOCK(markersummary_free); ss_item = find_before_in_ktree(sharesummary_workinfoid_root, &ss_look, - cmp_sharesummary_workinfoid, ss_ctx); + ss_ctx); DATA_SHARESUMMARY_NULL(sharesummary, ss_item); if (ss_item) end_workinfoid = sharesummary->workinfoid; @@ -3713,8 +3709,8 @@ bool process_pplns(int32_t height, char *blockhash, tv_t *addr_cd) lookworkmarkers.workinfoidend = blocks->workinfoid + 1; INIT_WORKMARKERS(&wm_look); wm_look.data = (void *)(&lookworkmarkers); - wm_item = find_before_in_ktree(workmarkers_workinfoid_root, &wm_look, - cmp_workmarkers_workinfoid, wm_ctx); + wm_item = find_before_in_ktree(workmarkers_workinfoid_root, + &wm_look, wm_ctx); DATA_WORKMARKERS_NULL(workmarkers, wm_item); LOGDEBUG("%s(): workmarkers < %"PRId64, __func__, lookworkmarkers.workinfoidend); while (total_diff < diff_want && wm_item && CURRENT(&(workmarkers->expirydate))) { @@ -3729,8 +3725,8 @@ bool process_pplns(int32_t height, char *blockhash, tv_t *addr_cd) lookmarkersummary.workername = EMPTY; INIT_MARKERSUMMARY(&ms_look); ms_look.data = (void *)(&lookmarkersummary); - ms_item = find_before_in_ktree(markersummary_root, &ms_look, - cmp_markersummary, ms_ctx); + ms_item = find_before_in_ktree(markersummary_root, + &ms_look, ms_ctx); DATA_MARKERSUMMARY_NULL(markersummary, ms_item); // add the whole markerid while (ms_item && markersummary->markerid == workmarkers->markerid) { @@ -4053,7 +4049,7 @@ bool process_pplns(int32_t height, char *blockhash, tv_t *addr_cd) payouts_add_ram(true, p_item, old_p_item, &now); - mu_root = free_ktree(mu_root, NULL); + free_ktree(mu_root, NULL); mu_item = k_unlink_head(mu_store); while (mu_item) { DATA_MININGPAYOUTS(miningpayouts, mu_item); @@ -4144,7 +4140,7 @@ oku: ; ck_wunlock(&process_pplns_lock); if (mu_root) - mu_root = free_ktree(mu_root, NULL); + free_ktree(mu_root, NULL); if (mu_store) { if (mu_store->count) { K_WLOCK(mu_store); @@ -4245,7 +4241,7 @@ K_ITEM *find_userstats(int64_t userid, char *workername) INIT_USERSTATS(&look); look.data = (void *)(&userstats); - return find_in_ktree(userstats_root, &look, cmp_userstats, ctx); + return find_in_ktree(userstats_root, &look, ctx); } void dsp_markersummary(K_ITEM *item, FILE *stream) @@ -4312,7 +4308,7 @@ K_ITEM *find_markersummary_userid(int64_t userid, char *workername, INIT_MARKERSUMMARY(&look); look.data = (void *)(&markersummary); - ms_item = find_before_in_ktree(markersummary_userid_root, &look, cmp_markersummary_userid, ctx); + ms_item = find_before_in_ktree(markersummary_userid_root, &look, ctx); if (ms_item) { DATA_MARKERSUMMARY(ms, ms_item); if (ms->userid != userid || strcmp(ms->workername, workername)) @@ -4349,11 +4345,11 @@ K_ITEM *_find_markersummary(int64_t markerid, int64_t workinfoid, INIT_MARKERSUMMARY(&look); look.data = (void *)(&markersummary); if (pool) { - ms_item = find_in_ktree(markersummary_pool_root, &look, - cmp_markersummary, ctx); + ms_item = find_in_ktree(markersummary_pool_root, + &look, ctx); } else { - ms_item = find_in_ktree(markersummary_root, &look, - cmp_markersummary, ctx); + ms_item = find_in_ktree(markersummary_root, + &look, ctx); } } @@ -4476,7 +4472,7 @@ K_ITEM *find_workmarkers(int64_t workinfoid, bool anystatus, char status, K_TREE INIT_WORKMARKERS(&look); look.data = (void *)(&workmarkers); - wm_item = find_after_in_ktree(workmarkers_workinfoid_root, &look, cmp_workmarkers_workinfoid, ctx); + wm_item = find_after_in_ktree(workmarkers_workinfoid_root, &look, ctx); if (wm_item) { DATA_WORKMARKERS(wm, wm_item); if (!CURRENT(&(wm->expirydate)) || @@ -4500,7 +4496,7 @@ K_ITEM *find_workmarkerid(int64_t markerid, bool anystatus, char status) INIT_WORKMARKERS(&look); look.data = (void *)(&workmarkers); - wm_item = find_in_ktree(workmarkers_root, &look, cmp_workmarkers, ctx); + wm_item = find_in_ktree(workmarkers_root, &look, ctx); if (wm_item) { DATA_WORKMARKERS(wm, wm_item); if (!CURRENT(&(wm->expirydate)) || @@ -4530,8 +4526,7 @@ static bool gen_workmarkers(PGconn *conn, MARKS *stt, bool after, MARKS *fin, look.data = (void *)(&workinfo); K_RLOCK(workinfo_free); if (after) { - wi_stt_item = find_after_in_ktree(workinfo_root, &look, - cmp_workinfo, ctx); + wi_stt_item = find_after_in_ktree(workinfo_root, &look, ctx); while (wi_stt_item) { DATA_WORKINFO(wi_stt, wi_stt_item); if (CURRENT(&(wi_stt->expirydate))) @@ -4539,8 +4534,7 @@ static bool gen_workmarkers(PGconn *conn, MARKS *stt, bool after, MARKS *fin, wi_stt_item = next_in_ktree(ctx); } } else { - wi_stt_item = find_in_ktree(workinfo_root, &look, - cmp_workinfo, ctx); + wi_stt_item = find_in_ktree(workinfo_root, &look, ctx); DATA_WORKINFO_NULL(wi_stt, wi_stt_item); } K_RUNLOCK(workinfo_free); @@ -4556,8 +4550,7 @@ static bool gen_workmarkers(PGconn *conn, MARKS *stt, bool after, MARKS *fin, K_RLOCK(workinfo_free); if (before) { DATE_ZERO(&(workinfo.expirydate)); - wi_fin_item = find_before_in_ktree(workinfo_root, &look, - cmp_workinfo, ctx); + wi_fin_item = find_before_in_ktree(workinfo_root, &look, ctx); while (wi_fin_item) { DATA_WORKINFO(wi_fin, wi_fin_item); if (CURRENT(&(wi_fin->expirydate))) @@ -4567,8 +4560,7 @@ static bool gen_workmarkers(PGconn *conn, MARKS *stt, bool after, MARKS *fin, } else { workinfo.expirydate.tv_sec = default_expiry.tv_sec; workinfo.expirydate.tv_usec = default_expiry.tv_usec; - wi_fin_item = find_in_ktree(workinfo_root, &look, - cmp_workinfo, ctx); + wi_fin_item = find_in_ktree(workinfo_root, &look, ctx); DATA_WORKINFO_NULL(wi_fin, wi_fin_item); } K_RUNLOCK(workinfo_free); @@ -4647,7 +4639,7 @@ bool workmarkers_generate(PGconn *conn, char *err, size_t siz, char *by, INIT_MARKS(&look); look.data = (void *)(&marks); K_RLOCK(marks_free); - m_item = find_before_in_ktree(marks_root, &look, cmp_marks, ctx); + m_item = find_before_in_ktree(marks_root, &look, ctx); while (m_item) { DATA_MARKS(mused, m_item); if (CURRENT(&(mused->expirydate)) && MUSED(mused->status)) @@ -4854,7 +4846,7 @@ K_ITEM *find_marks(int64_t workinfoid) INIT_MARKS(&look); look.data = (void *)(&marks); - return find_in_ktree(marks_root, &look, cmp_marks, ctx); + return find_in_ktree(marks_root, &look, ctx); } const char *marks_marktype(char *marktype) @@ -5026,7 +5018,7 @@ K_ITEM *_get_userinfo(int64_t userid, bool lock) look.data = (void *)(&userinfo); if (lock) K_RLOCK(userinfo_free); - find = find_in_ktree(userinfo_root, &look, cmp_userinfo, ctx); + find = find_in_ktree(userinfo_root, &look, ctx); if (lock) K_RUNLOCK(userinfo_free); return find; @@ -5059,7 +5051,7 @@ K_ITEM *_find_create_userinfo(int64_t userid, bool lock, WHERE_FFL_ARGS) else bigint_to_buf(userid, row->username, sizeof(row->username)); - userinfo_root = add_to_ktree(userinfo_root, ui_item, cmp_userinfo); + add_to_ktree(userinfo_root, ui_item); k_add_head(userinfo_store, ui_item); if (lock) K_WUNLOCK(userinfo_free); diff --git a/src/ckdb_dbio.c b/src/ckdb_dbio.c index 7c6e4a6b..881f9b3f 100644 --- a/src/ckdb_dbio.c +++ b/src/ckdb_dbio.c @@ -513,14 +513,14 @@ unparam: free_users_data(item); k_add_head(users_free, item); } else { - users_root = remove_from_ktree(users_root, u_item, cmp_users); - userid_root = remove_from_ktree(userid_root, u_item, cmp_userid); + remove_from_ktree(users_root, u_item); + remove_from_ktree(userid_root, u_item); copy_tv(&(users->expirydate), cd); - users_root = add_to_ktree(users_root, u_item, cmp_users); - userid_root = add_to_ktree(userid_root, u_item, cmp_userid); + add_to_ktree(users_root, u_item); + add_to_ktree(userid_root, u_item); - users_root = add_to_ktree(users_root, item, cmp_users); - userid_root = add_to_ktree(userid_root, item, cmp_userid); + add_to_ktree(users_root, item); + add_to_ktree(userid_root, item); k_add_head(users_store, item); } K_WUNLOCK(users_free); @@ -648,8 +648,8 @@ unitem: free_users_data(item); k_add_head(users_free, item); } else { - users_root = add_to_ktree(users_root, item, cmp_users); - userid_root = add_to_ktree(userid_root, item, cmp_userid); + add_to_ktree(users_root, item); + add_to_ktree(userid_root, item); k_add_head(users_store, item); } K_WUNLOCK(users_free); @@ -760,14 +760,14 @@ unparam: free_users_data(u_item); k_add_head(users_free, u_item); } else { - users_root = remove_from_ktree(users_root, old_u_item, cmp_users); - userid_root = remove_from_ktree(userid_root, old_u_item, cmp_userid); + remove_from_ktree(users_root, old_u_item); + remove_from_ktree(userid_root, old_u_item); copy_tv(&(old_users->expirydate), cd); - users_root = add_to_ktree(users_root, old_u_item, cmp_users); - userid_root = add_to_ktree(userid_root, old_u_item, cmp_userid); + add_to_ktree(users_root, old_u_item); + add_to_ktree(userid_root, old_u_item); - users_root = add_to_ktree(users_root, u_item, cmp_users); - userid_root = add_to_ktree(userid_root, u_item, cmp_userid); + add_to_ktree(users_root, u_item); + add_to_ktree(userid_root, u_item); k_add_head(users_store, u_item); } K_WUNLOCK(users_free); @@ -883,8 +883,8 @@ bool users_fill(PGconn *conn) username_trim(row); - users_root = add_to_ktree(users_root, item, cmp_users); - userid_root = add_to_ktree(userid_root, item, cmp_userid); + add_to_ktree(users_root, item); + add_to_ktree(userid_root, item); k_add_head(users_store, item); } if (!ok) { @@ -1011,11 +1011,11 @@ unparam: if (ok) { // Update it if (old_item) { - useratts_root = remove_from_ktree(useratts_root, old_item, cmp_useratts); + remove_from_ktree(useratts_root, old_item); copy_tv(&(old_useratts->expirydate), cd); - useratts_root = add_to_ktree(useratts_root, old_item, cmp_useratts); + add_to_ktree(useratts_root, old_item); } - useratts_root = add_to_ktree(useratts_root, ua_item, cmp_useratts); + add_to_ktree(useratts_root, ua_item); k_add_head(useratts_store, ua_item); } K_WUNLOCK(useratts_free); @@ -1162,9 +1162,9 @@ unparam: K_WLOCK(useratts_free); if (ok && item) { - useratts_root = remove_from_ktree(useratts_root, item, cmp_useratts); + remove_from_ktree(useratts_root, item); copy_tv(&(useratts->expirydate), cd); - useratts_root = add_to_ktree(useratts_root, item, cmp_useratts); + add_to_ktree(useratts_root, item); } K_WUNLOCK(useratts_free); @@ -1269,7 +1269,7 @@ bool useratts_fill(PGconn *conn) if (!ok) break; - useratts_root = add_to_ktree(useratts_root, item, cmp_useratts); + add_to_ktree(useratts_root, item); k_add_head(useratts_store, item); } if (!ok) @@ -1400,7 +1400,7 @@ unitem: if (!ret) k_add_head(workers_free, item); else { - workers_root = add_to_ktree(workers_root, item, cmp_workers); + add_to_ktree(workers_root, item); k_add_head(workers_store, item); // Ensure there is a matching workerstatus find_create_workerstatus(userid, workername, @@ -1639,7 +1639,7 @@ bool workers_fill(PGconn *conn) break; TXT_TO_BIGINT("workerid", field, row->workerid); - workers_root = add_to_ktree(workers_root, item, cmp_workers); + add_to_ktree(workers_root, item); k_add_head(workers_store, item); /* Make sure a workerstatus exists for each worker @@ -1869,15 +1869,11 @@ unparam: else { // It wasn't a match, thus it was expired n++; - paymentaddresses_root = remove_from_ktree(paymentaddresses_root, item, - cmp_paymentaddresses); - paymentaddresses_create_root = remove_from_ktree(paymentaddresses_create_root, - item, cmp_payaddr_create); + remove_from_ktree(paymentaddresses_root, item); + remove_from_ktree(paymentaddresses_create_root, item); copy_tv(&(row->expirydate), cd); - paymentaddresses_root = add_to_ktree(paymentaddresses_root, item, - cmp_paymentaddresses); - paymentaddresses_create_root = add_to_ktree(paymentaddresses_create_root, - item, cmp_payaddr_create); + add_to_ktree(paymentaddresses_root, item); + add_to_ktree(paymentaddresses_create_root, item); } item = prev; DATA_PAYMENTADDRESSES_NULL(row, item); @@ -1889,10 +1885,8 @@ unparam: next = match->next; DATA_PAYMENTADDRESSES(pa, match); if (!pa->match) { - paymentaddresses_root = add_to_ktree(paymentaddresses_root, match, - cmp_paymentaddresses); - paymentaddresses_create_root = add_to_ktree(paymentaddresses_create_root, - match, cmp_payaddr_create); + add_to_ktree(paymentaddresses_root, match); + add_to_ktree(paymentaddresses_create_root, match); k_unlink_item(pa_store, match); k_add_head(paymentaddresses_store, match); count++; @@ -1981,10 +1975,8 @@ bool paymentaddresses_fill(PGconn *conn) if (!ok) break; - paymentaddresses_root = add_to_ktree(paymentaddresses_root, item, - cmp_paymentaddresses); - paymentaddresses_create_root = add_to_ktree(paymentaddresses_create_root, - item, cmp_payaddr_create); + add_to_ktree(paymentaddresses_root, item); + add_to_ktree(paymentaddresses_create_root, item); k_add_head(paymentaddresses_store, item); } if (!ok) @@ -2015,11 +2007,11 @@ void payments_add_ram(bool ok, K_ITEM *p_item, K_ITEM *old_p_item, tv_t *cd) } else { if (old_p_item) { DATA_PAYMENTS(oldp, old_p_item); - payments_root = remove_from_ktree(payments_root, old_p_item, cmp_payments); + remove_from_ktree(payments_root, old_p_item); copy_tv(&(oldp->expirydate), cd); - payments_root = add_to_ktree(payments_root, old_p_item, cmp_payments); + add_to_ktree(payments_root, old_p_item); } - payments_root = add_to_ktree(payments_root, p_item, cmp_payments); + add_to_ktree(payments_root, p_item); k_add_head(payments_store, p_item); } K_WUNLOCK(payments_free); @@ -2252,7 +2244,7 @@ bool payments_fill(PGconn *conn) if (!ok) break; - payments_root = add_to_ktree(payments_root, item, cmp_payments); + add_to_ktree(payments_root, item); k_add_head(payments_store, item); } if (!ok) @@ -2358,7 +2350,7 @@ K_ITEM *optioncontrol_item_add(PGconn *conn, K_ITEM *oc_item, tv_t *cd, bool beg INIT_OPTIONCONTROL(&look); look.data = (void *)row; K_RLOCK(optioncontrol_free); - old_item = find_in_ktree(optioncontrol_root, &look, cmp_optioncontrol, ctx); + old_item = find_in_ktree(optioncontrol_root, &look, ctx); K_RUNLOCK(optioncontrol_free); if (!conn) { @@ -2446,13 +2438,12 @@ nostart: } else { // Discard old if (old_item) { - optioncontrol_root = remove_from_ktree(optioncontrol_root, old_item, - cmp_optioncontrol); + remove_from_ktree(optioncontrol_root, old_item); k_unlink_item(optioncontrol_store, old_item); free_optioncontrol_data(old_item); k_add_head(optioncontrol_free, old_item); } - optioncontrol_root = add_to_ktree(optioncontrol_root, oc_item, cmp_optioncontrol); + add_to_ktree(optioncontrol_root, oc_item); k_add_head(optioncontrol_store, oc_item); if (strcmp(row->optionname, SWITCH_STATE_NAME) == 0) { switch_state = atoi(row->optionvalue); @@ -2584,7 +2575,7 @@ bool optioncontrol_fill(PGconn *conn) if (!ok) break; - optioncontrol_root = add_to_ktree(optioncontrol_root, item, cmp_optioncontrol); + add_to_ktree(optioncontrol_root, item); k_add_head(optioncontrol_store, item); // There should only be one CURRENT version of switch_state @@ -2659,7 +2650,7 @@ int64_t workinfo_add(PGconn *conn, char *workinfoidstr, char *poolinstance, HISTORYDATETRANSFER(trf_root, row); K_WLOCK(workinfo_free); - if (find_in_ktree(workinfo_root, item, cmp_workinfo, ctx)) { + if (find_in_ktree(workinfo_root, item, ctx)) { workinfoid = row->workinfoid; free_workinfo_data(item); k_add_head(workinfo_free, item); @@ -2732,7 +2723,7 @@ unparam: hex2bin(ndiffbin, row->bits, 4); current_ndiff = diff_from_nbits(ndiffbin); - workinfo_root = add_to_ktree(workinfo_root, item, cmp_workinfo); + add_to_ktree(workinfo_root, item); k_add_head(workinfo_store, item); // Remember the bc = 'cd' when the height changes @@ -2900,9 +2891,9 @@ bool workinfo_fill(PGconn *conn) if (!ok) break; - workinfo_root = add_to_ktree(workinfo_root, item, cmp_workinfo); + add_to_ktree(workinfo_root, item); if (!confirm_sharesummary) - workinfo_height_root = add_to_ktree(workinfo_height_root, item, cmp_workinfo_height); + add_to_ktree(workinfo_height_root, item); k_add_head(workinfo_store, item); if (tv_newer(&(dbstatus.newest_createdate_workinfo), &(row->createdate))) { @@ -3027,9 +3018,7 @@ static void shares_process_early(PGconn *conn, int64_t good_wid, tv_t *good_cd, } es_item = last_in_ktree(shares_early_root, ctx); if (es_item) { - shares_early_root = remove_from_ktree(shares_early_root, - es_item, - cmp_shares); + remove_from_ktree(shares_early_root, es_item); k_unlink_item(shares_early_store, es_item); } K_WUNLOCK(shares_free); @@ -3078,7 +3067,7 @@ static void shares_process_early(PGconn *conn, int64_t good_wid, tv_t *good_cd, return; redo: K_WLOCK(shares_free); - shares_early_root = add_to_ktree(shares_early_root, es_item, cmp_shares); + add_to_ktree(shares_early_root, es_item); k_add_tail(shares_early_store, es_item); K_WUNLOCK(shares_free); return; @@ -3093,7 +3082,7 @@ keep: early_shares->oldcount, early_shares->redo); FREENULL(st); K_WLOCK(shares_free); - shares_root = add_to_ktree(shares_root, es_item, cmp_shares); + add_to_ktree(shares_root, es_item); k_add_head(shares_store, es_item); K_WUNLOCK(shares_free); return; @@ -3200,8 +3189,7 @@ bool shares_add(PGconn *conn, char *workinfoid, char *username, char *workername shares->oldcount = 0; K_WLOCK(shares_free); // They need to be sorted by workinfoid - shares_early_root = add_to_ktree(shares_early_root, s_item, - cmp_shares); + add_to_ktree(shares_early_root, s_item); k_add_head(shares_early_store, s_item); K_WUNLOCK(shares_free); /* It was all OK except the missing workinfoid @@ -3212,7 +3200,7 @@ bool shares_add(PGconn *conn, char *workinfoid, char *username, char *workername ok = shares_process(conn, shares, trf_root); if (ok) { K_WLOCK(shares_free); - shares_root = add_to_ktree(shares_root, s_item, cmp_shares); + add_to_ktree(shares_root, s_item); k_add_head(shares_store, s_item); K_WUNLOCK(shares_free); @@ -3327,9 +3315,7 @@ static void shareerrors_process_early(PGconn *conn, int64_t good_wid, } es_item = last_in_ktree(shareerrors_early_root, ctx); if (es_item) { - shareerrors_early_root = remove_from_ktree(shareerrors_early_root, - es_item, - cmp_shareerrors); + remove_from_ktree(shareerrors_early_root, es_item); k_unlink_item(shareerrors_early_store, es_item); } K_WUNLOCK(shareerrors_free); @@ -3381,8 +3367,7 @@ static void shareerrors_process_early(PGconn *conn, int64_t good_wid, return; redo: K_WLOCK(shareerrors_free); - shareerrors_early_root = add_to_ktree(shareerrors_early_root, es_item, - cmp_shareerrors); + add_to_ktree(shareerrors_early_root, es_item); k_add_tail(shareerrors_early_store, es_item); K_WUNLOCK(shareerrors_free); return; @@ -3397,7 +3382,7 @@ keep: early_shareerrors->oldcount, early_shareerrors->redo); FREENULL(st); K_WLOCK(shareerrors_free); - shareerrors_root = add_to_ktree(shareerrors_root, es_item, cmp_shareerrors); + add_to_ktree(shareerrors_root, es_item); k_add_head(shareerrors_store, es_item); K_WUNLOCK(shareerrors_free); return; @@ -3494,9 +3479,7 @@ bool shareerrors_add(PGconn *conn, char *workinfoid, char *username, shareerrors->oldcount = 0; K_WLOCK(shareerrors_free); // They need to be sorted by workinfoid - shareerrors_early_root = add_to_ktree(shareerrors_early_root, - s_item, - cmp_shareerrors); + add_to_ktree(shareerrors_early_root, s_item); k_add_head(shareerrors_early_store, s_item); K_WUNLOCK(shareerrors_free); /* It was all OK except the missing workinfoid @@ -3507,8 +3490,7 @@ bool shareerrors_add(PGconn *conn, char *workinfoid, char *username, ok = shareerrors_process(conn, shareerrors, trf_root); if (ok) { K_WLOCK(shareerrors_free); - shareerrors_root = add_to_ktree(shareerrors_root, s_item, - cmp_shareerrors); + add_to_ktree(shareerrors_root, s_item); k_add_head(shareerrors_store, s_item); K_WUNLOCK(shareerrors_free); @@ -3597,7 +3579,7 @@ bool sharesummaries_to_markersummaries(PGconn *conn, WORKMARKERS *workmarkers, K_STORE *old_sharesummary_store = k_new_store(sharesummary_free); K_STORE *new_markersummary_store = k_new_store(markersummary_free); - K_TREE *ms_root = new_ktree(); + K_TREE *ms_root = new_ktree(cmp_markersummary); if (!CURRENT(&(workmarkers->expirydate))) { reason = "unexpired"; @@ -3617,8 +3599,7 @@ bool sharesummaries_to_markersummaries(PGconn *conn, WORKMARKERS *workmarkers, INIT_MARKERSUMMARY(&ms_look); ms_look.data = (void *)(&lookmarkersummary); K_RLOCK(markersummary_free); - ms_item = find_after_in_ktree(markersummary_root, &ms_look, - cmp_markersummary, ms_ctx); + ms_item = find_after_in_ktree(markersummary_root, &ms_look, ms_ctx); K_RUNLOCK(markersummary_free); DATA_MARKERSUMMARY_NULL(markersummary, ms_item); if (ms_item && markersummary->markerid == workmarkers->markerid) { @@ -3640,8 +3621,8 @@ bool sharesummaries_to_markersummaries(PGconn *conn, WORKMARKERS *workmarkers, * Those incoming shares will not be touching the sharesummaries * we are processing here */ K_RLOCK(sharesummary_free); - ss_item = find_before_in_ktree(sharesummary_workinfoid_root, &ss_look, - cmp_sharesummary_workinfoid, ss_ctx); + ss_item = find_before_in_ktree(sharesummary_workinfoid_root, + &ss_look, ss_ctx); K_RUNLOCK(sharesummary_free); while (ss_item) { DATA_SHARESUMMARY(sharesummary, ss_item); @@ -3659,8 +3640,7 @@ bool sharesummaries_to_markersummaries(PGconn *conn, WORKMARKERS *workmarkers, lookmarkersummary.workername = sharesummary->workername; ms_look.data = (void *)(&lookmarkersummary); - ms_item = find_in_ktree(ms_root, &ms_look, - cmp_markersummary, ms_ctx); + ms_item = find_in_ktree(ms_root, &ms_look, ms_ctx); if (!ms_item) { K_WLOCK(markersummary_free); ms_item = k_unlink_head(markersummary_free); @@ -3673,8 +3653,7 @@ bool sharesummaries_to_markersummaries(PGconn *conn, WORKMARKERS *workmarkers, DUP_POINTER(markersummary_free, markersummary->workername, sharesummary->workername); - ms_root = add_to_ktree(ms_root, ms_item, - cmp_markersummary); + add_to_ktree(ms_root, ms_item); LOGDEBUG("%s() new ms %"PRId64"/%"PRId64"/%s", shortname, markersummary->markerid, @@ -3796,12 +3775,8 @@ flail: ms_item = new_markersummary_store->head; while (ms_item) { // move the new markersummaries into the trees/stores - markersummary_root = add_to_ktree(markersummary_root, - ms_item, - cmp_markersummary); - markersummary_userid_root = add_to_ktree(markersummary_userid_root, - ms_item, - cmp_markersummary_userid); + add_to_ktree(markersummary_root, ms_item); + add_to_ktree(markersummary_userid_root, ms_item); // create/update the pool markersummaries DATA_MARKERSUMMARY(markersummary, ms_item); @@ -3812,9 +3787,7 @@ flail: bzero(p_markersummary, sizeof(*p_markersummary)); p_markersummary->markerid = markersummary->markerid; POOL_MS(p_markersummary); - markersummary_pool_root = add_to_ktree(markersummary_pool_root, - p_ms_item, - cmp_markersummary); + add_to_ktree(markersummary_pool_root, p_ms_item); k_add_head(markersummary_pool_store, p_ms_item); } markersummary_to_pool(p_markersummary, markersummary); @@ -3828,20 +3801,14 @@ flail: ss_item = old_sharesummary_store->head; while (ss_item) { // remove the old sharesummaries from the trees - sharesummary_root = remove_from_ktree(sharesummary_root, - ss_item, - cmp_sharesummary); - sharesummary_workinfoid_root = remove_from_ktree(sharesummary_workinfoid_root, - ss_item, - cmp_sharesummary_workinfoid); + remove_from_ktree(sharesummary_root, ss_item); + remove_from_ktree(sharesummary_workinfoid_root, ss_item); // remove the pool sharesummaries DATA_SHARESUMMARY(sharesummary, ss_item); p_ss_item = find_sharesummary_p(sharesummary->workinfoid); if (p_ss_item) { - sharesummary_pool_root = remove_from_ktree(sharesummary_pool_root, - p_ss_item, - cmp_sharesummary); + remove_from_ktree(sharesummary_pool_root, p_ss_item); k_unlink_item(sharesummary_pool_store, p_ss_item); free_sharesummary_data(p_ss_item); k_add_head(sharesummary_free, p_ss_item); @@ -3864,7 +3831,7 @@ flail: workmarkers->description, workmarkers->status); } - ms_root = free_ktree(ms_root, NULL); + free_ktree(ms_root, NULL); new_markersummary_store = k_free_store(new_markersummary_store); old_sharesummary_store = k_free_store(old_sharesummary_store); @@ -4140,16 +4107,12 @@ bool _sharesummary_update(SHARES *s_row, SHAREERRORS *e_row, K_ITEM *ss_item, if (new || p_new) { K_WLOCK(sharesummary_free); if (new) { - sharesummary_root = add_to_ktree(sharesummary_root, item, cmp_sharesummary); - sharesummary_workinfoid_root = add_to_ktree(sharesummary_workinfoid_root, - item, - cmp_sharesummary_workinfoid); + add_to_ktree(sharesummary_root, item); + add_to_ktree(sharesummary_workinfoid_root, item); k_add_head(sharesummary_store, item); } if (p_new) { - sharesummary_pool_root = add_to_ktree(sharesummary_pool_root, - p_item, - cmp_sharesummary); + add_to_ktree(sharesummary_pool_root, p_item); k_add_head(sharesummary_pool_store, p_item); } K_WUNLOCK(sharesummary_free); @@ -4290,14 +4253,14 @@ unparam: k_add_head(blocks_free, b_item); else { if (update_old) { - blocks_root = remove_from_ktree(blocks_root, old_b_item, cmp_blocks); + remove_from_ktree(blocks_root, old_b_item); copy_tv(&(oldblocks->expirydate), cd); - blocks_root = add_to_ktree(blocks_root, old_b_item, cmp_blocks); + add_to_ktree(blocks_root, old_b_item); // Copy it over to avoid having to recalculate it row->netdiff = oldblocks->netdiff; } else row->netdiff = 0; - blocks_root = add_to_ktree(blocks_root, b_item, cmp_blocks); + add_to_ktree(blocks_root, b_item); k_add_head(blocks_store, b_item); blocks_stats_rebuild = true; // 'confirmed' is unchanged so no need to recalc *createdate @@ -4670,14 +4633,14 @@ flail: k_add_head(blocks_free, b_item); else { if (update_old) { - blocks_root = remove_from_ktree(blocks_root, old_b_item, cmp_blocks); + remove_from_ktree(blocks_root, old_b_item); copy_tv(&(oldblocks->expirydate), cd); - blocks_root = add_to_ktree(blocks_root, old_b_item, cmp_blocks); + add_to_ktree(blocks_root, old_b_item); // Copy it over to avoid having to recalculate it row->netdiff = oldblocks->netdiff; } else row->netdiff = 0; - blocks_root = add_to_ktree(blocks_root, b_item, cmp_blocks); + add_to_ktree(blocks_root, b_item); k_add_head(blocks_store, b_item); blocks_stats_rebuild = true; // recalc the *createdate fields for possibly affected blocks @@ -4897,7 +4860,7 @@ bool blocks_fill(PGconn *conn) if (!ok) break; - blocks_root = add_to_ktree(blocks_root, item, cmp_blocks); + add_to_ktree(blocks_root, item); k_add_head(blocks_store, item); if (tv_newer(&(dbstatus.newest_createdate_blocks), &(row->createdate))) @@ -4958,11 +4921,11 @@ void miningpayouts_add_ram(bool ok, K_ITEM *mp_item, K_ITEM *old_mp_item, tv_t * } else { if (old_mp_item) { DATA_MININGPAYOUTS(oldmp, old_mp_item); - miningpayouts_root = remove_from_ktree(miningpayouts_root, old_mp_item, cmp_miningpayouts); + remove_from_ktree(miningpayouts_root, old_mp_item); copy_tv(&(oldmp->expirydate), cd); - miningpayouts_root = add_to_ktree(miningpayouts_root, old_mp_item, cmp_miningpayouts); + add_to_ktree(miningpayouts_root, old_mp_item); } - miningpayouts_root = add_to_ktree(miningpayouts_root, mp_item, cmp_miningpayouts); + add_to_ktree(miningpayouts_root, mp_item); k_add_head(miningpayouts_store, mp_item); } K_WUNLOCK(miningpayouts_free); @@ -5142,7 +5105,7 @@ bool miningpayouts_fill(PGconn *conn) if (!ok) break; - miningpayouts_root = add_to_ktree(miningpayouts_root, item, cmp_miningpayouts); + add_to_ktree(miningpayouts_root, item); k_add_head(miningpayouts_store, item); tick(); @@ -5176,17 +5139,17 @@ void payouts_add_ram(bool ok, K_ITEM *p_item, K_ITEM *old_p_item, tv_t *cd) } else { if (old_p_item) { DATA_PAYOUTS(oldp, old_p_item); - payouts_root = remove_from_ktree(payouts_root, old_p_item, cmp_payouts); - payouts_id_root = remove_from_ktree(payouts_id_root, old_p_item, cmp_payouts_id); - payouts_wid_root = remove_from_ktree(payouts_wid_root, old_p_item, cmp_payouts_wid); + remove_from_ktree(payouts_root, old_p_item); + remove_from_ktree(payouts_id_root, old_p_item); + remove_from_ktree(payouts_wid_root, old_p_item); copy_tv(&(oldp->expirydate), cd); - payouts_root = add_to_ktree(payouts_root, old_p_item, cmp_payouts); - payouts_id_root = add_to_ktree(payouts_id_root, old_p_item, cmp_payouts_id); - payouts_wid_root = add_to_ktree(payouts_wid_root, old_p_item, cmp_payouts_wid); + add_to_ktree(payouts_root, old_p_item); + add_to_ktree(payouts_id_root, old_p_item); + add_to_ktree(payouts_wid_root, old_p_item); } - payouts_root = add_to_ktree(payouts_root, p_item, cmp_payouts); - payouts_id_root = add_to_ktree(payouts_id_root, p_item, cmp_payouts_id); - payouts_wid_root = add_to_ktree(payouts_wid_root, p_item, cmp_payouts_wid); + add_to_ktree(payouts_root, p_item); + add_to_ktree(payouts_id_root, p_item); + add_to_ktree(payouts_wid_root, p_item); k_add_head(payouts_store, p_item); } K_WUNLOCK(payouts_free); @@ -5491,22 +5454,22 @@ K_ITEM *payouts_full_expire(PGconn *conn, int64_t payoutid, tv_t *now, bool lock // No more possible errors, so update the ram tables DATA_PAYOUTS(payouts, po_item); - payouts_root = remove_from_ktree(payouts_root, po_item, cmp_payouts); - payouts_id_root = remove_from_ktree(payouts_id_root, po_item, cmp_payouts_id); - payouts_wid_root = remove_from_ktree(payouts_wid_root, po_item, cmp_payouts_wid); + remove_from_ktree(payouts_root, po_item); + remove_from_ktree(payouts_id_root, po_item); + remove_from_ktree(payouts_wid_root, po_item); copy_tv(&(payouts->expirydate), now); - payouts_root = add_to_ktree(payouts_root, po_item, cmp_payouts); - payouts_id_root = add_to_ktree(payouts_id_root, po_item, cmp_payouts_id); - payouts_wid_root = add_to_ktree(payouts_wid_root, po_item, cmp_payouts_wid); + add_to_ktree(payouts_root, po_item); + add_to_ktree(payouts_id_root, po_item); + add_to_ktree(payouts_wid_root, po_item); mp_item = first_miningpayouts(payoutid, mp_ctx); DATA_MININGPAYOUTS_NULL(mp, mp_item); while (mp_item && mp->payoutid == payoutid) { if (CURRENT(&(mp->expirydate))) { next_item = next_in_ktree(mp_ctx); - miningpayouts_root = remove_from_ktree(miningpayouts_root, mp_item, cmp_miningpayouts); + remove_from_ktree(miningpayouts_root, mp_item); copy_tv(&(mp->expirydate), now); - miningpayouts_root = add_to_ktree(miningpayouts_root, mp_item, cmp_miningpayouts); + add_to_ktree(miningpayouts_root, mp_item); mp_item = next_item; } else mp_item = next_in_ktree(mp_ctx); @@ -5520,9 +5483,9 @@ K_ITEM *payouts_full_expire(PGconn *conn, int64_t payoutid, tv_t *now, bool lock if (payments->payoutid == payoutid && CURRENT(&(payments->expirydate))) { next_item = next_in_ktree(pm_ctx); - payments_root = remove_from_ktree(payments_root, pm_item, cmp_payments); + remove_from_ktree(payments_root, pm_item); copy_tv(&(payments->expirydate), now); - payments_root = add_to_ktree(payments_root, pm_item, cmp_payments); + add_to_ktree(payments_root, pm_item); pm_item = next_item; } else pm_item = next_in_ktree(pm_ctx); @@ -5698,9 +5661,9 @@ bool payouts_fill(PGconn *conn) &(blocks->blockcreatedate)); } - payouts_root = add_to_ktree(payouts_root, item, cmp_payouts); - payouts_id_root = add_to_ktree(payouts_id_root, item, cmp_payouts_id); - payouts_wid_root = add_to_ktree(payouts_wid_root, item, cmp_payouts_wid); + add_to_ktree(payouts_root, item); + add_to_ktree(payouts_id_root, item); + add_to_ktree(payouts_wid_root, item); k_add_head(payouts_store, item); if (CURRENT(&(row->expirydate)) && PAYGENERATED(row->status)) @@ -5792,7 +5755,7 @@ bool auths_add(PGconn *conn, char *poolinstance, char *username, HISTORYDATETRANSFER(trf_root, row); K_WLOCK(auths_free); - if (find_in_ktree(auths_root, a_item, cmp_auths, ctx)) { + if (find_in_ktree(auths_root, a_item, ctx)) { k_add_head(auths_free, a_item); K_WUNLOCK(auths_free); @@ -5825,7 +5788,7 @@ unitem: if (!ok) k_add_head(auths_free, a_item); else { - auths_root = add_to_ktree(auths_root, a_item, cmp_auths); + add_to_ktree(auths_root, a_item); k_add_head(auths_store, a_item); } #endif @@ -5875,7 +5838,7 @@ bool poolstats_add(PGconn *conn, bool store, char *poolinstance, SIMPLEDATEINIT(row, cd, by, code, inet); SIMPLEDATETRANSFER(trf_root, row); - if (igndup && find_in_ktree(poolstats_root, p_item, cmp_poolstats, ctx)) { + if (igndup && find_in_ktree(poolstats_root, p_item, ctx)) { K_WLOCK(poolstats_free); k_add_head(poolstats_free, p_item); K_WUNLOCK(poolstats_free); @@ -5937,7 +5900,7 @@ unparam: if (!ok) k_add_head(poolstats_free, p_item); else { - poolstats_root = add_to_ktree(poolstats_root, p_item, cmp_poolstats); + add_to_ktree(poolstats_root, p_item); k_add_head(poolstats_store, p_item); } K_WUNLOCK(poolstats_free); @@ -6083,7 +6046,7 @@ bool poolstats_fill(PGconn *conn) if (!ok) break; - poolstats_root = add_to_ktree(poolstats_root, item, cmp_poolstats); + add_to_ktree(poolstats_root, item); k_add_head(poolstats_store, item); if (tv_newer(&(dbstatus.newest_createdate_poolstats), &(row->createdate))) @@ -6186,15 +6149,13 @@ bool userstats_add(char *poolinstance, char *elapsed, char *username, K_WLOCK(userstats_free); us_next = userstats_eos_store->head; while (us_next) { - us_item = find_in_ktree(userstats_root, us_next, - cmp_userstats, ctx); + us_item = find_in_ktree(userstats_root, us_next, ctx); if (!us_item) { // New user+worker - store it in RAM us_match = us_next; us_next = us_match->next; k_unlink_item(userstats_eos_store, us_match); - userstats_root = add_to_ktree(userstats_root, us_match, - cmp_userstats); + add_to_ktree(userstats_root, us_match); k_add_head(userstats_store, us_match); } else { DATA_USERSTATS(next, us_next); @@ -6270,12 +6231,10 @@ bool workerstats_add(char *poolinstance, char *elapsed, char *username, workerstatus_update(NULL, NULL, row); K_WLOCK(userstats_free); - us_match = find_in_ktree(userstats_root, us_item, - cmp_userstats, ctx); + us_match = find_in_ktree(userstats_root, us_item, ctx); if (!us_match) { // New user+worker - store it in RAM - userstats_root = add_to_ktree(userstats_root, us_item, - cmp_userstats); + add_to_ktree(userstats_root, us_item); k_add_head(userstats_store, us_item); } else { DATA_USERSTATS(match, us_match); @@ -6520,8 +6479,8 @@ bool markersummary_fill(PGconn *conn) if (!ok) break; - markersummary_root = add_to_ktree(markersummary_root, item, cmp_markersummary); - markersummary_userid_root = add_to_ktree(markersummary_userid_root, item, cmp_markersummary_userid); + add_to_ktree(markersummary_root, item); + add_to_ktree(markersummary_userid_root, item); k_add_head(markersummary_store, item); p_item = find_markersummary_p(row->markerid); @@ -6533,9 +6492,7 @@ bool markersummary_fill(PGconn *conn) bzero(p_row, sizeof(*p_row)); p_row->markerid = row->markerid; POOL_MS(p_row); - markersummary_pool_root = add_to_ktree(markersummary_pool_root, - p_item, - cmp_markersummary); + add_to_ktree(markersummary_pool_root, p_item); k_add_head(markersummary_pool_store, p_item); } else { DATA_MARKERSUMMARY(p_row, p_item); @@ -6769,29 +6726,18 @@ unparam: } else { if (old_wm_item) { - workmarkers_root = remove_from_ktree(workmarkers_root, - old_wm_item, - cmp_workmarkers); - workmarkers_workinfoid_root = remove_from_ktree(workmarkers_workinfoid_root, - old_wm_item, - cmp_workmarkers_workinfoid); + remove_from_ktree(workmarkers_root, old_wm_item); + remove_from_ktree(workmarkers_workinfoid_root, + old_wm_item); copy_tv(&(oldworkmarkers->expirydate), cd); - workmarkers_root = add_to_ktree(workmarkers_root, - old_wm_item, - cmp_workmarkers); - workmarkers_workinfoid_root = add_to_ktree(workmarkers_workinfoid_root, - old_wm_item, - cmp_workmarkers_workinfoid); + add_to_ktree(workmarkers_root, old_wm_item); + add_to_ktree(workmarkers_workinfoid_root, old_wm_item); } if (wm_item) { shift_rewards(wm_item); - workmarkers_root = add_to_ktree(workmarkers_root, - wm_item, - cmp_workmarkers); - workmarkers_workinfoid_root = add_to_ktree(workmarkers_workinfoid_root, - wm_item, - cmp_workmarkers_workinfoid); + add_to_ktree(workmarkers_root, wm_item); + add_to_ktree(workmarkers_workinfoid_root, wm_item); k_add_head(workmarkers_store, wm_item); } } @@ -6887,9 +6833,8 @@ bool workmarkers_fill(PGconn *conn) if (!ok) break; - workmarkers_root = add_to_ktree(workmarkers_root, item, cmp_workmarkers); - workmarkers_workinfoid_root = add_to_ktree(workmarkers_workinfoid_root, - item, cmp_workmarkers_workinfoid); + add_to_ktree(workmarkers_root, item); + add_to_ktree(workmarkers_workinfoid_root, item); k_add_head(workmarkers_store, item); if (dbstatus.newest_workmarker_workinfoid < row->workinfoidend) { @@ -7086,12 +7031,12 @@ unparam: } } else { if (old_m_item) { - marks_root = remove_from_ktree(marks_root, old_m_item, cmp_marks); + remove_from_ktree(marks_root, old_m_item); copy_tv(&(oldmarks->expirydate), cd); - marks_root = add_to_ktree(marks_root, old_m_item, cmp_marks); + add_to_ktree(marks_root, old_m_item); } if (m_item) { - marks_root = add_to_ktree(marks_root, m_item, cmp_marks); + add_to_ktree(marks_root, m_item); k_add_head(marks_store, m_item); } } @@ -7186,7 +7131,7 @@ bool marks_fill(PGconn *conn) if (!ok) break; - marks_root = add_to_ktree(marks_root, item, cmp_marks); + add_to_ktree(marks_root, item); k_add_head(marks_store, item); tick(); diff --git a/src/ktree.c b/src/ktree.c index 588683b0..1181d069 100644 --- a/src/ktree.c +++ b/src/ktree.c @@ -1,5 +1,5 @@ /* - * Copyright 1995-2014 Andrew Smith + * Copyright 1995-2015 Andrew Smith * * This program is free software; you can redistribute it and/or modify it * under the terms of the GNU General Public License as published by the Free @@ -23,42 +23,56 @@ static const int dbg = 0; #define Yo true #define No false -static K_TREE nil[1] = { { Yo, RED_BLACK, NULL, NULL, NULL, NULL, 0 } }; +static K_NODE nil[1] = { { Yo, RED_BLACK, NULL, NULL, NULL, NULL, 0 } }; -K_TREE *_new_ktree(KTREE_FFL_ARGS) +static K_NODE *_new_knode(KTREE_FFL_ARGS) { - K_TREE *ktree = (K_TREE *)malloc(sizeof(*ktree)); + K_NODE *node = (K_NODE *)malloc(sizeof(*node)); - if (ktree == NULL) - FAIL("%s", "OOM"); + if (node == NULL) + FAIL("%s", "node OOM"); + + node->isNil = Yo; + node->red = RED_BLACK; + node->parent = nil; + node->left = nil; + node->right = nil; + node->data = NULL; + node->test = 0; + + return node; +} + +K_TREE *_new_ktree(cmp_t (*cmp_funct)(K_ITEM *, K_ITEM *), KTREE_FFL_ARGS) +{ + K_TREE *tree = (K_TREE *)malloc(sizeof(*tree)); + + if (tree == NULL) + FAIL("%s", "tree OOM"); + + tree->root = _new_knode(KTREE_FFL_PASS); - ktree->isNil = Yo; - ktree->red = RED_BLACK; - ktree->parent = nil; - ktree->left = nil; - ktree->right = nil; - ktree->data = NULL; - ktree->test = 0; + tree->cmp_funct = cmp_funct; - return ktree; + return tree; } -static K_TREE *new_data(K_ITEM *data, KTREE_FFL_ARGS) +static K_NODE *new_data(K_ITEM *data, KTREE_FFL_ARGS) { - K_TREE *ktree = (K_TREE *)malloc(sizeof(*ktree)); + K_NODE *knode = (K_NODE *)malloc(sizeof(*knode)); - if (ktree == NULL) + if (knode == NULL) FAIL("%s", "OOM"); - ktree->isNil = No; - ktree->red = RED_RED; - ktree->parent = nil; - ktree->left = nil; - ktree->right = nil; - ktree->data = data; - ktree->test = 0; + knode->isNil = No; + knode->red = RED_RED; + knode->parent = nil; + knode->left = nil; + knode->right = nil; + knode->data = data; + knode->test = 0; - return ktree; + return knode; } static int bCount = 0; @@ -73,54 +87,54 @@ static long getTestValue() return ++testValue; } -static void show_ktree(K_TREE *root, char *path, int pos, char *(*dsp_funct)(K_ITEM *)) +static void show_ktree(K_NODE *node, char *path, int pos, char *(*dsp_funct)(K_ITEM *)) { char col; - if (root->isNil == Yo) + if (node->isNil == Yo) return; - if (root->left->isNil == No) + if (node->left->isNil == No) { path[pos] = 'L'; path[pos+1] = '\0'; - show_ktree(root->left, path, pos+1, dsp_funct); + show_ktree(node->left, path, pos+1, dsp_funct); } path[pos] = '\0'; - if (root->red == RED_RED) + if (node->red == RED_RED) col = 'R'; else - // if (root->red == RED_BLACK) + // if (node->red == RED_BLACK) col = 'B'; - printf(" %c %s=%s\n", col, path, dsp_funct(root->data)); + printf(" %c %s=%s\n", col, path, dsp_funct(node->data)); - if (root->right->isNil == No) + if (node->right->isNil == No) { path[pos] = 'R'; path[pos+1] = '\0'; - show_ktree(root->right, path, pos+1, dsp_funct); + show_ktree(node->right, path, pos+1, dsp_funct); } } -void _dump_ktree(K_TREE *root, char *(*dsp_funct)(K_ITEM *), KTREE_FFL_ARGS) +void _dump_ktree(K_TREE *tree, char *(*dsp_funct)(K_ITEM *), KTREE_FFL_ARGS) { char buf[42424]; printf("dump:\n"); - if (root->isNil == No) + if (tree->root->isNil == No) { buf[0] = 'T'; buf[1] = '\0'; - show_ktree(root, buf, 1, dsp_funct); + show_ktree(tree->root, buf, 1, dsp_funct); } else - printf(" Empty ktree\n"); + printf(" Empty tree\n"); } -void _dsp_ktree(K_LIST *list, K_TREE *root, char *filename, char *msg, KTREE_FFL_ARGS) +void _dsp_ktree(K_LIST *list, K_TREE *tree, char *filename, char *msg, KTREE_FFL_ARGS) { K_TREE_CTX ctx[1]; K_ITEM *item; @@ -154,11 +168,11 @@ void _dsp_ktree(K_LIST *list, K_TREE *root, char *filename, char *msg, KTREE_FFL if (msg) fprintf(stream, "%s %s\n", stamp, msg); else - fprintf(stream, "%s Dump of ktree '%s':\n", stamp, list->name); + fprintf(stream, "%s Dump of tree '%s':\n", stamp, list->name); - if (root->isNil == No) + if (tree->root->isNil == No) { - item = first_in_ktree(root, ctx); + item = first_in_ktree(tree, ctx); while (item) { list->dsp_func(item, stream); @@ -172,7 +186,7 @@ void _dsp_ktree(K_LIST *list, K_TREE *root, char *filename, char *msg, KTREE_FFL fclose(stream); } -static int nilTest(K_TREE *node, char *msg, int depth, int count, K_TREE *nil2, KTREE_FFL_ARGS) +static int nilTest(K_NODE *node, char *msg, int depth, int count, K_NODE *nil2, KTREE_FFL_ARGS) { if (node->isNil == Yo || node == nil2) { @@ -231,7 +245,7 @@ static int nilTest(K_TREE *node, char *msg, int depth, int count, K_TREE *nil2, return(count); } -static void bTest(K_TREE *root, K_TREE *cur, char *msg, int count, KTREE_FFL_ARGS) +static void bTest(K_NODE *cur, char *msg, int count, KTREE_FFL_ARGS) { if (cur->red != RED_RED) count++; @@ -259,108 +273,118 @@ static void bTest(K_TREE *root, K_TREE *cur, char *msg, int count, KTREE_FFL_ARG else FAIL("BTESTVALUE '%s' count=%d", msg, count); - bTest(root, cur->left, msg, count, KTREE_FFL_PASS); - bTest(root, cur->right, msg, count, KTREE_FFL_PASS); + bTest(cur->left, msg, count, KTREE_FFL_PASS); + bTest(cur->right, msg, count, KTREE_FFL_PASS); } } -static void bTestInit(K_TREE *root, char *msg, KTREE_FFL_ARGS) +static void bTestInit(K_TREE *tree, char *msg, KTREE_FFL_ARGS) { bCount = 0; bTestValue = getTestValue(); - bTest(root, root, msg, 0, KTREE_FFL_PASS); + bTest(tree->root, msg, 0, KTREE_FFL_PASS); } -static void lrpTest(K_TREE *top, char *msg, KTREE_FFL_ARGS) +static void lrpTest(K_NODE *node, char *msg, KTREE_FFL_ARGS) { - if (top->test != lrpTestValue) - top->test = lrpTestValue; + if (node->test != lrpTestValue) + node->test = lrpTestValue; else FAIL("LRPTESTVALUE '%s'", msg); - if (top->left->isNil == No) + if (node->left->isNil == No) { - if (top->left->parent != top) + if (node->left->parent != node) FAIL("LRPTESTL '%s'", msg); - lrpTest(top->left, msg, KTREE_FFL_PASS); + lrpTest(node->left, msg, KTREE_FFL_PASS); } - if (top->right->isNil == No) + if (node->right->isNil == No) { - if (top->right->parent != top) + if (node->right->parent != node) FAIL("LRPTESTR '%s'", msg); - lrpTest(top->right, msg, KTREE_FFL_PASS); + lrpTest(node->right, msg, KTREE_FFL_PASS); } } -static __maybe_unused void check_ktree(K_TREE *root, char *msg, K_TREE *nil2, int debugNil, int debugLRP, int debugColor, KTREE_FFL_ARGS) +static __maybe_unused void check_ktree(K_TREE *tree, char *msg, K_NODE *nil2, int debugNil, int debugLRP, int debugColor, KTREE_FFL_ARGS) { - if (root->isNil == Yo) + if (tree->root->isNil == Yo) return; if (debugNil) { nilTestValue = getTestValue(); - nilTest(root, msg, 1, 0, nil2, KTREE_FFL_PASS); + nilTest(tree->root, msg, 1, 0, nil2, KTREE_FFL_PASS); } - if (debugLRP && root->isNil == No) + if (debugLRP && tree->root->isNil == No) { lrpTestValue = getTestValue(); - lrpTest(root, msg, KTREE_FFL_PASS); + lrpTest(tree->root, msg, KTREE_FFL_PASS); } - if (debugColor && root->isNil == No) - bTestInit(root, msg, KTREE_FFL_PASS); + if (debugColor && tree->root->isNil == No) + bTestInit(tree, msg, KTREE_FFL_PASS); } -K_ITEM *_first_in_ktree(K_TREE *root, K_TREE_CTX *ctx, KTREE_FFL_ARGS) +static K_ITEM *_first_in_knode(K_NODE *node, K_TREE_CTX *ctx, KTREE_FFL_ARGS) { - if (root->isNil == No) + if (node->isNil == No) { - while (root->left->isNil == No) - root = root->left; + while (node->left->isNil == No) + node = node->left; - *ctx = root; - return(root->data); + *ctx = node; + return(node->data); } *ctx = NULL; return(NULL); } -K_ITEM *_last_in_ktree(K_TREE *root, K_TREE_CTX *ctx, KTREE_FFL_ARGS) +K_ITEM *_first_in_ktree(K_TREE *tree, K_TREE_CTX *ctx, KTREE_FFL_ARGS) { - if (root->isNil == No) + return _first_in_knode(tree->root, ctx, KTREE_FFL_PASS); +} + +static K_ITEM *_last_in_knode(K_NODE *node, K_TREE_CTX *ctx, KTREE_FFL_ARGS) +{ + if (node->isNil == No) { - while (root->right->isNil == No) - root = root->right; + while (node->right->isNil == No) + node = node->right; - *ctx = root; - return(root->data); + *ctx = node; + return(node->data); } *ctx = NULL; return(NULL); } +K_ITEM *_last_in_ktree(K_TREE *tree, K_TREE_CTX *ctx, KTREE_FFL_ARGS) +{ + return _last_in_knode(tree->root, ctx, KTREE_FFL_PASS); +} + K_ITEM *_next_in_ktree(K_TREE_CTX *ctx, KTREE_FFL_ARGS) { - K_TREE *parent; - K_TREE *ktree = (K_TREE *)(*ctx); + K_NODE *parent; + K_NODE *knode = (K_NODE *)(*ctx); - if (ktree->isNil == No) + if (knode->isNil == No) { - if (ktree->right->isNil == No) - return(first_in_ktree(ktree->right, ctx)); + if (knode->right->isNil == No) + return(_first_in_knode(knode->right, ctx, KTREE_FFL_PASS)); else { - parent = ktree->parent; - while (parent->isNil == No && ktree == parent->right) + parent = knode->parent; + while (parent->isNil == No && knode == parent->right) { - ktree = parent; + knode = parent; parent = parent->parent; } if (parent->isNil == No) @@ -377,19 +401,19 @@ K_ITEM *_next_in_ktree(K_TREE_CTX *ctx, KTREE_FFL_ARGS) K_ITEM *_prev_in_ktree(K_TREE_CTX *ctx, KTREE_FFL_ARGS) { - K_TREE *parent; - K_TREE *ktree = (K_TREE *)(*ctx); + K_NODE *parent; + K_NODE *knode = (K_NODE *)(*ctx); - if (ktree->isNil == No) + if (knode->isNil == No) { - if (ktree->left->isNil == No) - return(last_in_ktree(ktree->left, ctx)); + if (knode->left->isNil == No) + return(_last_in_knode(knode->left, ctx, KTREE_FFL_PASS)); else { - parent = ktree->parent; - while (parent->isNil == No && ktree == parent->left) + parent = knode->parent; + while (parent->isNil == No && knode == parent->left) { - ktree = parent; + knode = parent; parent = parent->parent; } if (parent->isNil == No) @@ -404,9 +428,9 @@ K_ITEM *_prev_in_ktree(K_TREE_CTX *ctx, KTREE_FFL_ARGS) return(NULL); } -static K_TREE *left_rotate(K_TREE *root, K_TREE *about) +static K_NODE *left_rotate(K_NODE *root, K_NODE *about) { - K_TREE *rotate; + K_NODE *rotate; rotate = about->right; about->right = rotate->left; @@ -432,9 +456,9 @@ static K_TREE *left_rotate(K_TREE *root, K_TREE *about) return(root); } -static K_TREE *right_rotate(K_TREE *root, K_TREE *about) +static K_NODE *right_rotate(K_NODE *root, K_NODE *about) { - K_TREE *rotate; + K_NODE *rotate; rotate = about->left; about->left = rotate->right; @@ -458,50 +482,50 @@ static K_TREE *right_rotate(K_TREE *root, K_TREE *about) return(root); } -K_TREE *_add_to_ktree(K_TREE *root, K_ITEM *data, cmp_t (*cmp_funct)(K_ITEM *, K_ITEM *), KTREE_FFL_ARGS) +void _add_to_ktree(K_TREE *tree, K_ITEM *data, KTREE_FFL_ARGS) { - K_TREE *ktree; - K_TREE *x, *y; - K_TREE *pp; + K_NODE *knode; + K_NODE *x, *y; + K_NODE *pp; cmp_t cmp; - if (root == NULL) - FAIL("%s", "ADDNULL add ktree is NULL"); + if (tree == NULL) + FAIL("%s", "ADDNULL add tree is NULL"); -//check_ktree(root, ">add", NULL, 1, 1, 1, KTREE_FFL_PASS); +//check_ktree(tree, ">add", NULL, 1, 1, 1, KTREE_FFL_PASS); - if (root->parent != nil && root->parent != NULL) - FAIL("%s", "ADDROOT add root isn't the root"); + if (tree->root->parent != nil && tree->root->parent != NULL) + FAIL("%s", "ADDROOT add tree->root isn't the root"); - ktree = new_data(data, KTREE_FFL_PASS); + knode = new_data(data, KTREE_FFL_PASS); - if (root->isNil == Yo) + if (tree->root->isNil == Yo) { - if (root != nil) - free(root); + if (tree->root != nil) + free(tree->root); - root = ktree; + tree->root = knode; } else { - x = root; + x = tree->root; y = nil; while (x->isNil == No) { y = x; - if ((cmp = (*cmp_funct)(ktree->data, x->data)) < 0) + if ((cmp = tree->cmp_funct(knode->data, x->data)) < 0) x = x->left; else x = x->right; } - ktree->parent = y; + knode->parent = y; if (cmp < 0) - y->left = ktree; + y->left = knode; else - y->right = ktree; + y->right = knode; - x = ktree; - while (x != root && x->parent->red == RED_RED) + x = knode; + while (x != tree->root && x->parent->red == RED_RED) { pp = x->parent->parent; if (x->parent == pp->left) @@ -519,12 +543,12 @@ K_TREE *_add_to_ktree(K_TREE *root, K_ITEM *data, cmp_t (*cmp_funct)(K_ITEM *, K if (x == x->parent->right) { x = x->parent; - root = left_rotate(root, x); + tree->root = left_rotate(tree->root, x); pp = x->parent->parent; } x->parent->red = RED_BLACK; pp->red = RED_RED; - root = right_rotate(root, pp); + tree->root = right_rotate(tree->root, pp); } } else @@ -542,45 +566,49 @@ K_TREE *_add_to_ktree(K_TREE *root, K_ITEM *data, cmp_t (*cmp_funct)(K_ITEM *, K if (x == x->parent->left) { x = x->parent; - root = right_rotate(root, x); + tree->root = right_rotate(tree->root, x); pp = x->parent->parent; } x->parent->red = RED_BLACK; pp->red = RED_RED; - root = left_rotate(root, pp); + tree->root = left_rotate(tree->root, pp); } } } } - root->red = RED_BLACK; - -//check_ktree(root, "root->red = RED_BLACK; - return(root); +//check_ktree(tree, "root == NULL) + FAIL("%s", "FINDNULL find tree->root is NULL"); - while (ktree->isNil == No && cmp != 0) + knode = tree->root; + + while (knode->isNil == No && cmp != 0) { - if ((cmp = (*cmp_funct)(ktree->data, data))) + if ((cmp = tree->cmp_funct(knode->data, data))) { if (cmp > 0) - ktree = ktree->left; + knode = knode->left; else - ktree = ktree->right; + knode = knode->right; } } - if (ktree->isNil == No) + if (knode->isNil == No) { - *ctx = ktree; - return(ktree->data); + *ctx = knode; + return(knode->data); } else { @@ -589,30 +617,35 @@ K_ITEM *_find_in_ktree(K_TREE *ktree, K_ITEM *data, cmp_t (*cmp_funct)(K_ITEM *, } } -K_ITEM *_find_after_in_ktree(K_TREE *ktree, K_ITEM *data, cmp_t (*cmp_funct)(K_ITEM *, K_ITEM *), K_TREE_CTX *ctx, KTREE_FFL_ARGS) +K_ITEM *_find_after_in_ktree(K_TREE *tree, K_ITEM *data, K_TREE_CTX *ctx, KTREE_FFL_ARGS) { - K_TREE *old = NULL; + K_NODE *knode, *old = NULL; cmp_t cmp = -1, oldcmp = -1; - if (ktree == NULL) - FAIL("%s", "FINDNULL find_after ktree is NULL"); + if (tree == NULL) + FAIL("%s", "FINDNULL find_after tree is NULL"); + + if (tree->root == NULL) + FAIL("%s", "FINDNULL find_after tree->root is NULL"); - while (ktree->isNil == No && cmp != 0) + knode = tree->root; + + while (knode->isNil == No && cmp != 0) { - if ((cmp = (*cmp_funct)(ktree->data, data))) + if ((cmp = tree->cmp_funct(knode->data, data))) { - old = ktree; + old = knode; oldcmp = cmp; if (cmp > 0) - ktree = ktree->left; + knode = knode->left; else - ktree = ktree->right; + knode = knode->right; } } - if (ktree->isNil == No) + if (knode->isNil == No) { - *ctx = ktree; + *ctx = knode; return next_in_ktree(ctx); } else @@ -634,30 +667,35 @@ K_ITEM *_find_after_in_ktree(K_TREE *ktree, K_ITEM *data, cmp_t (*cmp_funct)(K_I } } -K_ITEM *_find_before_in_ktree(K_TREE *ktree, K_ITEM *data, cmp_t (*cmp_funct)(K_ITEM *, K_ITEM *), K_TREE_CTX *ctx, KTREE_FFL_ARGS) +K_ITEM *_find_before_in_ktree(K_TREE *tree, K_ITEM *data, K_TREE_CTX *ctx, KTREE_FFL_ARGS) { - K_TREE *old = NULL; + K_NODE *knode, *old = NULL; cmp_t cmp = 1, oldcmp = 1; - if (ktree == NULL) - FAIL("%s", "FINDNULL find_before ktree is NULL"); + if (tree == NULL) + FAIL("%s", "FINDNULL find_before tree is NULL"); + + if (tree->root == NULL) + FAIL("%s", "FINDNULL find_before tree->root is NULL"); + + knode = tree->root; - while (ktree->isNil == No && cmp != 0) + while (knode->isNil == No && cmp != 0) { - if ((cmp = (*cmp_funct)(ktree->data, data))) + if ((cmp = tree->cmp_funct(knode->data, data))) { - old = ktree; + old = knode; oldcmp = cmp; if (cmp > 0) - ktree = ktree->left; + knode = knode->left; else - ktree = ktree->right; + knode = knode->right; } } - if (ktree->isNil == No) + if (knode->isNil == No) { - *ctx = ktree; + *ctx = knode; return prev_in_ktree(ctx); } else @@ -679,9 +717,9 @@ K_ITEM *_find_before_in_ktree(K_TREE *ktree, K_ITEM *data, cmp_t (*cmp_funct)(K_ } } -static K_TREE *removeFixup(K_TREE *root, K_TREE *fix) +static K_NODE *removeFixup(K_NODE *root, K_NODE *fix) { - K_TREE *w = NULL; + K_NODE *w = NULL; while (fix != root && fix->red != RED_RED) { @@ -758,37 +796,40 @@ static K_TREE *removeFixup(K_TREE *root, K_TREE *fix) return root; } -// Does this work OK when you remove the last element in the ktree? +// Does this work OK when you remove the last element in the tree? // It should return the root as 'nil' -K_TREE *_remove_from_ktree(K_TREE *root, K_ITEM *data, cmp_t (*cmp_funct)(K_ITEM *, K_ITEM *), K_TREE_CTX *ctx, KTREE_FFL_ARGS) +void _remove_from_ktree(K_TREE *tree, K_ITEM *data, K_TREE_CTX *ctx, KTREE_FFL_ARGS) { K_TREE_CTX tmpctx[1]; - K_TREE *found; + K_NODE *found; K_ITEM *fdata; - K_TREE *x, *y, *nil2; + K_NODE *x, *y, *nil2; // cmp_t cmp; int yred; -//check_ktree(root, ">remove", NULL, 1, 1, 1, KTREE_FFL_PASS); +//check_ktree(tree, ">remove", NULL, 1, 1, 1, KTREE_FFL_PASS); - if (root == NULL) - FAIL("%s", "REMNULL remove ktree is NULL"); + if (tree == NULL) + FAIL("%s", "REMNULL remove tree is NULL"); - if (root->isNil == Yo) + if (tree->root == NULL) + FAIL("%s", "REMNULL remove tree->root is NULL"); + + if (tree->root->isNil == Yo) { *ctx = NULL; - return(root); + return; } - if (root->parent->isNil == No) - FAIL("%s", "REMROOT remove root isn't the root"); + if (tree->root->parent->isNil == No) + FAIL("%s", "REMROOT remove tree->root isn't the root"); - fdata = find_in_ktree(root, data, cmp_funct, ctx); + fdata = find_in_ktree(tree, data, ctx); if (fdata == NULL) - return(root); + return; - if (cmp_funct(fdata, data) != 0) + if (tree->cmp_funct(fdata, data) != 0) FAIL("%s", "BADFIND cmp(found, remove) != 0"); found = *ctx; @@ -819,14 +860,14 @@ K_TREE *_remove_from_ktree(K_TREE *root, K_ITEM *data, cmp_t (*cmp_funct)(K_ITEM nil2 = NULL; else { - nil2 = new_ktree(); + nil2 = _new_knode(KTREE_FFL_PASS); x = nil2; } x->parent = y->parent; if (x->parent->isNil == Yo) - root = x; + tree->root = x; else { if (x->parent->left == y) @@ -837,8 +878,8 @@ K_TREE *_remove_from_ktree(K_TREE *root, K_ITEM *data, cmp_t (*cmp_funct)(K_ITEM if (y != found) { - if (root == found) - root = y; + if (tree->root == found) + tree->root = y; if (x == found) x = y; @@ -864,7 +905,7 @@ K_TREE *_remove_from_ktree(K_TREE *root, K_ITEM *data, cmp_t (*cmp_funct)(K_ITEM } if (yred != RED_RED) - root = removeFixup(root, x); + tree->root = removeFixup(tree->root, x); if (nil2 != NULL) { @@ -874,8 +915,8 @@ K_TREE *_remove_from_ktree(K_TREE *root, K_ITEM *data, cmp_t (*cmp_funct)(K_ITEM if (nil2->parent->isNil == No && nil2->parent->right == nil2) nil2->parent->right = nil; - if (root == nil2) - root = nil; + if (tree->root == nil2) + tree->root = nil; /* if (dbg != 0) @@ -889,24 +930,24 @@ DBG("@remove nil2->left wasn't nil!!!\n"); DBG("@remove nil2->right wasn't nil!!!\n"); } cmp = 0; - fdata = first_in_ktree(root, tmpctx);; + fdata = first_in_ktree(tree, tmpctx);; while (fdata != NULL) { cmp++; x = *tmpctx; if (x == nil2) { -DBG("@remove found nil2 in ktree %f!!!\n", cmp); +DBG("@remove found nil2 in ktree %d!!!\n", (int)cmp); } else if (x->left == nil2) { -DBG("@remove found nil2 in ktree(left) %f!!!\n", cmp); +DBG("@remove found nil2 in ktree(left) %d!!!\n", (int)cmp); } else if (x->right == nil2) { -DBG("@remove found nil2 in ktree(right) %f!!!\n", cmp); +DBG("@remove found nil2 in ktree(right) %d!!!\n", (int)cmp); } fdata = next_in_ktree(tmpctx);; @@ -920,10 +961,10 @@ DBG("@remove found nil2 in ktree(right) %f!!!\n", cmp); if (dbg != 0) { cmp = 0; - fdata = first_in_ktree(root, tmpctx);; + fdata = first_in_ktree(tree, tmpctx);; while (fdata != NULL) { - if (cmp_funct(fdata, root->data) < 0) + if (tree->cmp_funct(fdata, tree->root->data) < 0) cmp--; else cmp++; @@ -932,52 +973,48 @@ if (dbg != 0) } if (cmp < -10 || cmp > 10) { -DBG("@remove after balance=%f :(\n", cmp); +DBG("@remove after balance=%d :(\n", (int)cmp); } } */ -//check_ktree(root, "data != NULL && free_funct) - (*free_funct)(ktree->data); + if (knode->data != NULL && free_funct) + free_funct(knode->data); - free_ktree_sub(ktree->left, free_funct); - free_ktree_sub(ktree->right, free_funct); + free_ktree_sub(knode->left, free_funct); + free_ktree_sub(knode->right, free_funct); - free(ktree); + free(knode); } } -K_TREE *_free_ktree(K_TREE *ktree, void (*free_funct)(void *), KTREE_FFL_ARGS) +void _free_ktree(K_TREE *tree, void (*free_funct)(void *), KTREE_FFL_ARGS) { - if (ktree == NULL) - FAIL("%s", "FREENULL free NULL ktree"); - - if (ktree->parent != NULL && ktree->parent != nil) - FAIL("%s", "FREENOTROOT free ktree not root"); + if (tree == NULL) + FAIL("%s", "FREENULL free NULL tree"); - free_ktree_sub(ktree, free_funct); + if (tree->root->parent != NULL && tree->root->parent != nil) + FAIL("%s", "FREENOTROOT free tree->root not root"); - return(nil); + free_ktree_sub(tree->root, free_funct); } diff --git a/src/ktree.h b/src/ktree.h index 0721610a..bf388c92 100644 --- a/src/ktree.h +++ b/src/ktree.h @@ -1,5 +1,5 @@ /* - * Copyright 1995-2014 Andrew Smith + * Copyright 1995-2015 Andrew Smith * * This program is free software; you can redistribute it and/or modify it * under the terms of the GNU General Public License as published by the Free @@ -34,45 +34,54 @@ #define CMP_BIGINT CMP_BIG #define CMP_DOUBLE CMP_BIG -typedef struct ktree +typedef struct knode { bool isNil; bool red; - struct ktree *parent; - struct ktree *left; - struct ktree *right; + struct knode *parent; + struct knode *left; + struct knode *right; K_ITEM *data; long test; +} K_NODE; + +typedef struct ktree +{ + K_NODE *root; + cmp_t (*cmp_funct)(K_ITEM *, K_ITEM *); } K_TREE; typedef void *K_TREE_CTX; -extern K_TREE *_new_ktree(KTREE_FFL_ARGS); -#define new_ktree() _new_ktree(KLIST_FFL_HERE) -extern void _dump_ktree(K_TREE *root, char *(*dsp_funct)(K_ITEM *), KTREE_FFL_ARGS); -#define dump_ktree(_root, _dsp_funct) _dump_ktree(_root, _dsp_funct, KLIST_FFL_HERE) -extern void _dsp_ktree(K_LIST *list, K_TREE *root, char *filename, char *msg, KTREE_FFL_ARGS); -#define dsp_ktree(_list, _root, _filename, _msg) _dsp_ktree(_list, _root, _filename, _msg, KLIST_FFL_HERE) -extern K_ITEM *_first_in_ktree(K_TREE *root, K_TREE_CTX *ctx, KTREE_FFL_ARGS); -#define first_in_ktree(_root, _ctx) _first_in_ktree(_root, _ctx, KLIST_FFL_HERE) -extern K_ITEM *_last_in_ktree(K_TREE *root, K_TREE_CTX *ctx, KTREE_FFL_ARGS); -#define last_in_ktree(_root, _ctx) _last_in_ktree(_root, _ctx, KLIST_FFL_HERE) +extern K_TREE *_new_ktree(cmp_t (*cmp_funct)(K_ITEM *, K_ITEM *), KTREE_FFL_ARGS); +#define new_ktree(_cmp_funct) _new_ktree(_cmp_funct, KLIST_FFL_HERE) +extern void _dump_ktree(K_TREE *tree, char *(*dsp_funct)(K_ITEM *), KTREE_FFL_ARGS); +#define dump_ktree(_tree, _dsp_funct) _dump_ktree(_tree, _dsp_funct, KLIST_FFL_HERE) +extern void _dsp_ktree(K_LIST *list, K_TREE *tree, char *filename, char *msg, KTREE_FFL_ARGS); +#define dsp_ktree(_list, _tree, _filename, _msg) _dsp_ktree(_list, _tree, _filename, _msg, KLIST_FFL_HERE) +extern K_ITEM *_first_in_ktree(K_TREE *tree, K_TREE_CTX *ctx, KTREE_FFL_ARGS); +#define first_in_ktree(_tree, _ctx) _first_in_ktree(_tree, _ctx, KLIST_FFL_HERE) +extern K_ITEM *_last_in_ktree(K_TREE *tree, K_TREE_CTX *ctx, KTREE_FFL_ARGS); +#define last_in_ktree(_tree, _ctx) _last_in_ktree(_tree, _ctx, KLIST_FFL_HERE) extern K_ITEM *_next_in_ktree(K_TREE_CTX *ctx, KTREE_FFL_ARGS); #define next_in_ktree(_ctx) _next_in_ktree(_ctx, KLIST_FFL_HERE) extern K_ITEM *_prev_in_ktree(K_TREE_CTX *ctx, KTREE_FFL_ARGS); #define prev_in_ktree(_ctx) _prev_in_ktree(_ctx, KLIST_FFL_HERE) -extern K_TREE *_add_to_ktree(K_TREE *root, K_ITEM *data, cmp_t (*cmp_func)(K_ITEM *, K_ITEM *), KTREE_FFL_ARGS); -#define add_to_ktree(_root, _data, _cmp_func) _add_to_ktree(_root, _data, _cmp_func, KLIST_FFL_HERE) -extern K_ITEM *_find_in_ktree(K_TREE *root, K_ITEM *data, cmp_t (*cmp_funct)(K_ITEM *, K_ITEM *), K_TREE_CTX *ctx, KTREE_FFL_ARGS); -#define find_in_ktree(_root, _data, _cmp_funct, _ctx) _find_in_ktree(_root, _data, _cmp_funct, _ctx, KLIST_FFL_HERE) -extern K_ITEM *_find_after_in_ktree(K_TREE *ktree, K_ITEM *data, cmp_t (*cmp_funct)(K_ITEM *, K_ITEM *), K_TREE_CTX *ctx, KTREE_FFL_ARGS); -#define find_after_in_ktree(_ktree, _data, _cmp_funct, _ctx) _find_after_in_ktree(_ktree, _data, _cmp_funct, _ctx, KLIST_FFL_HERE) -extern K_ITEM *_find_before_in_ktree(K_TREE *ktree, K_ITEM *data, cmp_t (*cmp_funct)(K_ITEM *, K_ITEM *), K_TREE_CTX *ctx, KTREE_FFL_ARGS); -#define find_before_in_ktree(_ktree, _data, _cmp_funct, _ctx) _find_before_in_ktree(_ktree, _data, _cmp_funct, _ctx, KLIST_FFL_HERE) -extern K_TREE *_remove_from_ktree(K_TREE *root, K_ITEM *data, cmp_t (*cmp_funct)(K_ITEM *, K_ITEM *), K_TREE_CTX *ctx, KTREE_FFL_ARGS); -extern K_TREE *_remove_from_ktree_free(K_TREE *root, K_ITEM *data, cmp_t (*cmp_funct)(K_ITEM *, K_ITEM *), KTREE_FFL_ARGS); -#define remove_from_ktree(_root, _data, _cmp_funct) _remove_from_ktree_free(_root, _data, _cmp_funct, KLIST_FFL_HERE) -extern K_TREE *_free_ktree(K_TREE *root, void (*free_funct)(void *), KTREE_FFL_ARGS); -#define free_ktree(_root, _free_funct) _free_ktree(_root, _free_funct, KLIST_FFL_HERE) +extern void _add_to_ktree(K_TREE *tree, K_ITEM *data, KTREE_FFL_ARGS); +#define add_to_ktree(_tree, _data) _add_to_ktree(_tree, _data, KLIST_FFL_HERE) +extern K_ITEM *_find_in_ktree(K_TREE *tree, K_ITEM *data, K_TREE_CTX *ctx, KTREE_FFL_ARGS); +#define find_in_ktree(_tree, _data, _ctx) _find_in_ktree(_tree, _data, _ctx, KLIST_FFL_HERE) +extern K_ITEM *_find_after_in_ktree(K_TREE *ktree, K_ITEM *data, K_TREE_CTX *ctx, KTREE_FFL_ARGS); +#define find_after_in_ktree(_ktree, _data, _ctx) _find_after_in_ktree(_ktree, _data, _ctx, KLIST_FFL_HERE) +extern K_ITEM *_find_before_in_ktree(K_TREE *ktree, K_ITEM *data, K_TREE_CTX *ctx, KTREE_FFL_ARGS); +#define find_before_in_ktree(_ktree, _data, _ctx) _find_before_in_ktree(_ktree, _data, _ctx, KLIST_FFL_HERE) +extern void _remove_from_ktree(K_TREE *tree, K_ITEM *data, K_TREE_CTX *ctx, KTREE_FFL_ARGS); +extern void _remove_from_ktree_free(K_TREE *tree, K_ITEM *data, KTREE_FFL_ARGS); +#define remove_from_ktree(_tree, _data) _remove_from_ktree_free(_tree, _data, KLIST_FFL_HERE) +extern void _free_ktree(K_TREE *tree, void (*free_funct)(void *), KTREE_FFL_ARGS); +#define free_ktree(_tree, _free_funct) do { \ + _free_ktree(_tree, _free_funct, KLIST_FFL_HERE); \ + _tree = NULL; \ + } while (0) #endif From 885be5d715b30bda134fcacefbd721852be26a42 Mon Sep 17 00:00:00 2001 From: kanoi Date: Tue, 15 Sep 2015 12:42:30 +1000 Subject: [PATCH 02/62] ckdb/php - add PPS calculations for shifts --- pool/base.php | 7 +++-- pool/page_shifts.php | 28 ++++++++++++++--- src/ckdb.h | 15 +++++++-- src/ckdb_cmd.c | 13 +++++++- src/ckdb_data.c | 73 ++++++++++++++++++++++++++++++++++++++++++-- src/ckdb_dbio.c | 17 +++++++++-- 6 files changed, 139 insertions(+), 14 deletions(-) diff --git a/pool/base.php b/pool/base.php index 828454a3..3e4c7c02 100644 --- a/pool/base.php +++ b/pool/base.php @@ -126,9 +126,12 @@ function btcfmt($amt) return number_format($amt, 8); } # -function utcd($when) +function utcd($when, $brief = false) { - return gmdate('Y-m-d H:i:s+00', round($when)); + if ($brief) + return gmdate('M-d H:i:s', round($when)); + else + return gmdate('Y-m-d H:i:s+00', round($when)); } # global $sipre; diff --git a/pool/page_shifts.php b/pool/page_shifts.php index 1be0d587..63ae6087 100644 --- a/pool/page_shifts.php +++ b/pool/page_shifts.php @@ -4,10 +4,11 @@ function doshifts($data, $user) { $ans = getShifts($user); - $pg = "\n"; + $pg = "Click here to jump to the start of the last payout

"; + $pg .= "
\n"; $pg .= ""; $pg .= ""; - $pg .= ""; + $pg .= ""; $pg .= ""; $pg .= ""; $pg .= ""; @@ -15,6 +16,8 @@ function doshifts($data, $user) $pg .= ""; $pg .= ""; $pg .= ""; + $pg .= ""; + $pg .= ""; $pg .= "\n"; if (($ans['STATUS'] != 'ok') || !isset($ans['prefix_all'])) @@ -28,9 +31,13 @@ function doshifts($data, $user) for ($i = 0; $i < $count; $i++) { $u = ''; + $mark = ''; if (isset($ans['lastpayoutstart:'.$i]) && $ans['lastpayoutstart:'.$i] != '') + { $u = 'u'; + $mark = ''; + } if (($i % 2) == 0) $row = "even$u"; else @@ -52,9 +59,9 @@ function doshifts($data, $user) $btc = ' '; else $btc = ''; - $pg .= ""; + $pg .= ""; $start = $ans['start:'.$i]; - $pg .= ''; + $pg .= ''; $nd = $ans['end:'.$i]; $elapsed = $nd - $start; $pg .= ''; @@ -72,10 +79,23 @@ function doshifts($data, $user) $avgsh = 0; $pg .= ''; $pg .= ''; + $ppsr = (float)$ans['ppsrewarded:'.$i]; + if ($ppsr > 0) + $ppsd = sprintf('%.3e', $ppsr); + else + $ppsd = '0'; + $pg .= ""; + $ppsv = (float)$ans['ppsvalue:'.$i]; + if ($ppsv > 0) + $pgot = number_format(100.0 * $ppsr / $ppsv, 2).'%'; + else + $pgot = '?'; + $pg .= ""; $pg .= "\n"; } } $pg .= "
ShiftStartStart UTCLengthYour DiffInv DiffSharesAvg ShareRewardsPPS*PPS%
$shif$btc$shif$btc$mark'.utcd($start).''.utcd($start, true).''.howmanyhrs($elapsed).''.number_format($avgsh, 2).''.$ans['rewards:'.$i].'$ppsd$pgot
\n"; + $pg .= "* The PPS value unit is satoshis
"; return $pg; } diff --git a/src/ckdb.h b/src/ckdb.h index cefe3dbe..4bb19b5b 100644 --- a/src/ckdb.h +++ b/src/ckdb.h @@ -55,7 +55,7 @@ #define DB_VLOCK "1" #define DB_VERSION "1.0.2" -#define CKDB_VERSION DB_VERSION"-1.300" +#define CKDB_VERSION DB_VERSION"-1.301" #define WHERE_FFL " - from %s %s() line %d" #define WHERE_FFL_HERE __FILE__, __func__, __LINE__ @@ -1294,6 +1294,7 @@ typedef struct optioncontrol { // Value it must default to (to work properly) #define OPTIONCONTROL_HEIGHT 1 +#define MAX_HEIGHT 999999999 // Test it here rather than obscuring the #define elsewhere #if ((OPTIONCONTROL_HEIGHT+1) != START_POOL_HEIGHT) @@ -1346,6 +1347,13 @@ extern tv_t last_bc; // current network diff extern double current_ndiff; +// Offset in binary coinbase1 of the block number +#define BLOCKNUM_OFFSET 42 +// Initial block reward (satoshi) +#define REWARD_BASE 5000000000.0 +// How many blocks per halving +#define REWARD_HALVE 210000.0 + // SHARES shares.id.json={...} typedef struct shares { int64_t workinfoid; @@ -2090,6 +2098,8 @@ extern void sequence_report(bool lock); #define REWARDOVERRIDE "MinerReward" +#define PPSOVERRIDE "PPSValue" + // Data free functions (first) #define FREE_ITEM(item) do { } while(0) // TODO: make a macro for all other to use above macro @@ -2264,7 +2274,7 @@ extern K_ITEM *find_first_paypayid(int64_t userid, int64_t payoutid, K_TREE_CTX extern cmp_t cmp_accountbalance(K_ITEM *a, K_ITEM *b); extern K_ITEM *find_accountbalance(int64_t userid); extern cmp_t cmp_optioncontrol(K_ITEM *a, K_ITEM *b); -extern K_ITEM *find_optioncontrol(char *optionname, tv_t *now, int32_t height); +extern K_ITEM *find_optioncontrol(char *optionname, const tv_t *now, int32_t height); extern int64_t user_sys_setting(int64_t userid, char *setting_name, int64_t setting_default, tv_t *now); extern cmp_t cmp_workinfo(K_ITEM *a, K_ITEM *b); @@ -2279,6 +2289,7 @@ extern bool workinfo_age(int64_t workinfoid, char *poolinstance, char *by, char *code, char *inet, tv_t *cd, tv_t *ss_first, tv_t *ss_last, int64_t *ss_count, int64_t *s_count, int64_t *s_diff); +extern double workinfo_pps(K_ITEM *w_item, int64_t workinfoid, bool lock); extern cmp_t cmp_shares(K_ITEM *a, K_ITEM *b); extern cmp_t cmp_shareerrors(K_ITEM *a, K_ITEM *b); extern void dsp_sharesummary(K_ITEM *item, FILE *stream); diff --git a/src/ckdb_cmd.c b/src/ckdb_cmd.c index 0e1c23e7..935adca5 100644 --- a/src/ckdb_cmd.c +++ b/src/ckdb_cmd.c @@ -5383,6 +5383,16 @@ static char *cmd_shifts(__maybe_unused PGconn *conn, char *cmd, char *id, rows, wm->rewards, FLDSEP); APPEND_REALLOC(buf, off, len, tmp); + // Use %.15e -> 16 non-leading-zero decimal places + snprintf(tmp, sizeof(tmp), "ppsvalue:%d=%.15f%c", + rows, wm->pps_value, FLDSEP); + APPEND_REALLOC(buf, off, len, tmp); + + // Use %.15e -> 16 non-leading-zero decimal places + snprintf(tmp, sizeof(tmp), "ppsrewarded:%d=%.15e%c", + rows, wm->rewarded, FLDSEP); + APPEND_REALLOC(buf, off, len, tmp); + snprintf(tmp, sizeof(tmp), "lastpayoutstart:%d=%s%c", rows, (wm->workinfoidstart == @@ -5431,7 +5441,8 @@ static char *cmd_shifts(__maybe_unused PGconn *conn, char *cmd, char *id, snprintf(tmp, sizeof(tmp), "rows=%d%cflds=%s%c", rows, FLDSEP, "markerid,shift,start,end,rewards," - "lastpayoutstart", FLDSEP); + "ppsvalue,ppsrewarded,lastpayoutstart", + FLDSEP); APPEND_REALLOC(buf, off, len, tmp); snprintf(tmp, sizeof(tmp), "arn=%s", "Shifts"); diff --git a/src/ckdb_data.c b/src/ckdb_data.c index a7cc7eb3..720f3a8e 100644 --- a/src/ckdb_data.c +++ b/src/ckdb_data.c @@ -1822,7 +1822,7 @@ static bool _reward_override_name(int32_t height, char *buf, size_t siz, } // Must be R or W locked before call -K_ITEM *find_optioncontrol(char *optionname, tv_t *now, int32_t height) +K_ITEM *find_optioncontrol(char *optionname, const tv_t *now, int32_t height) { OPTIONCONTROL optioncontrol, *oc, *ocbest; K_TREE_CTX ctx[1]; @@ -1832,6 +1832,9 @@ K_ITEM *find_optioncontrol(char *optionname, tv_t *now, int32_t height) * 1) activationdate is <= now * and * 2) height <= specified height (pool.height = current) + * The logic being: if 'now' is after the record activation date + * and 'height' is after the record activation height then + * the record is active * Remember the active record with the newest activationdate * If two records have the same activation date, then * remember the active record with the highest height @@ -2197,6 +2200,61 @@ bye: return ok; } + +// The PPS value of a 1diff share for the given workinfoid +double workinfo_pps(K_ITEM *w_item, int64_t workinfoid, bool lock) +{ + OPTIONCONTROL *optioncontrol; + K_ITEM *oc_item; + char oc_name[TXT_SML+1]; + char coinbase1bin[TXT_SML+1]; + char ndiffbin[TXT_SML+1]; + WORKINFO *workinfo; + double w_diff; + int w_blocknum; + size_t len; + + // Allow optioncontrol override for a given workinfoid + snprintf(oc_name, sizeof(oc_name), PPSOVERRIDE"_%"PRId64, workinfoid); + if (lock) + K_RLOCK(optioncontrol_free); + // No time/height control is used, just find the latest record + oc_item = find_optioncontrol(oc_name, &date_eot, MAX_HEIGHT); + if (lock) + K_RUNLOCK(optioncontrol_free); + + // Value is a floating point double of satoshi + if (oc_item) { + DATA_OPTIONCONTROL(optioncontrol, oc_item); + return atof(optioncontrol->optionvalue); + } + + if (!w_item) { + LOGERR("%s(): missing workinfo %"PRId64, + __func__, workinfoid); + return 0.0; + } + + DATA_WORKINFO(workinfo, w_item); + len = strlen(workinfo->coinbase1); + if (len < (BLOCKNUM_OFFSET * 2 + 4) || (len & 1)) { + LOGERR("%s(): Invalid coinbase1 len %d - " + "should be >= %d and even - for wid %"PRId64, + __func__, (int)len, (BLOCKNUM_OFFSET * 2 + 4), + workinfoid); + return 0.0; + } + + hex2bin(ndiffbin, workinfo->bits, 4); + w_diff = diff_from_nbits(ndiffbin); + hex2bin(coinbase1bin, workinfo->coinbase1 + (BLOCKNUM_OFFSET * 2), + (len - (BLOCKNUM_OFFSET * 2)) >> 1); + w_blocknum = get_sernumber((uchar *)coinbase1bin); + + // BASE halving to determine coinbase reward then divided by difficulty + return(REWARD_BASE * pow(0.5, floor((double)w_blocknum / REWARD_HALVE)) / w_diff); +} + // order by workinfoid asc,userid asc,workername asc,createdate asc,nonce asc,expirydate desc cmp_t cmp_shares(K_ITEM *a, K_ITEM *b) { @@ -4767,6 +4825,10 @@ bool reward_shifts(PAYOUTS *payouts, bool lock, int delta) K_ITEM *wm_item; WORKMARKERS *wm; bool did_one = false; + double payout_pps; + + payout_pps = (double)delta * (double)(payouts->minerreward) / + payouts->diffused; if (lock) K_WLOCK(workmarkers_free); @@ -4781,6 +4843,7 @@ bool reward_shifts(PAYOUTS *payouts, bool lock, int delta) * onto the PROCESSED status if it isn't already processed */ if (CURRENT(&(wm->expirydate))) { wm->rewards += delta; + wm->rewarded += payout_pps; did_one = true; } wm_item = next_in_ktree(ctx); @@ -4792,7 +4855,13 @@ bool reward_shifts(PAYOUTS *payouts, bool lock, int delta) return did_one; } -// (re)calculate rewards for a shift +/* (re)calculate rewards for a shift + * N.B. we don't need to zero/undo a workmarkers rewards directly + * since this is just a counter of how many times it's been rewarded + * and thus if the shift is expired the counter is ignored + * We only need to (re)calculate it when the workmarker is created + * Payouts code processing will increment/decrement all current rewards as + * needed with reward_shifts() when payouts are added/changed/removed */ bool shift_rewards(K_ITEM *wm_item) { PAYOUTS *payouts = NULL; diff --git a/src/ckdb_dbio.c b/src/ckdb_dbio.c index 881f9b3f..db2ed64f 100644 --- a/src/ckdb_dbio.c +++ b/src/ckdb_dbio.c @@ -6628,12 +6628,13 @@ bool _workmarkers_process(PGconn *conn, bool already, bool add, WHERE_FFL_PASS); goto rollback; } - w_item = find_workinfo(workinfoidend, NULL); + w_item = find_workinfo(workinfoidstart, NULL); if (!w_item) goto rollback; - w_item = find_workinfo(workinfoidstart, NULL); + w_item = find_workinfo(workinfoidend, NULL); if (!w_item) goto rollback; + K_WLOCK(workmarkers_free); wm_item = k_unlink_head(workmarkers_free); K_WUNLOCK(workmarkers_free); @@ -6698,6 +6699,7 @@ bool _workmarkers_process(PGconn *conn, bool already, bool add, PGLOGERR("Insert", rescode, conn); goto rollback; } + row->pps_value = workinfo_pps(w_item, workinfoidend, true); } ok = true; @@ -6837,9 +6839,18 @@ bool workmarkers_fill(PGconn *conn) add_to_ktree(workmarkers_workinfoid_root, item); k_add_head(workmarkers_store, item); + wi_item = find_workinfo(row->workinfoidend, NULL); + if (!wi_item) { + LOGERR("%s(): ERROR workmarkerid %"PRId64 + " wid end %"PRId64" doesn't exist! " + "PPS value will be zero", + __func__, row->markerid, + row->workinfoidend); + } + row->pps_value = workinfo_pps(wi_item, row->workinfoidend, false); + if (dbstatus.newest_workmarker_workinfoid < row->workinfoidend) { dbstatus.newest_workmarker_workinfoid = row->workinfoidend; - wi_item = find_workinfo(row->workinfoidend, NULL); if (!wi_item) { LOGEMERG("%s(): FAILURE workmarkerid %"PRId64 " wid end %"PRId64" doesn't exist! " From ee6f208812ae93b8cbfac5be669b868aa6bc55d2 Mon Sep 17 00:00:00 2001 From: kanoi Date: Tue, 15 Sep 2015 14:07:00 +1000 Subject: [PATCH 03/62] php - shift PPS is 'Rewarded' --- pool/page_shifts.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pool/page_shifts.php b/pool/page_shifts.php index 63ae6087..1753cce9 100644 --- a/pool/page_shifts.php +++ b/pool/page_shifts.php @@ -16,7 +16,7 @@ function doshifts($data, $user) $pg .= "Shares"; $pg .= "Avg Share"; $pg .= "Rewards"; - $pg .= "PPS*"; + $pg .= "Rewarded*"; $pg .= "PPS%"; $pg .= "\n"; From 78f17117c0aa50b0edc95e4b6fa3d305e984edd6 Mon Sep 17 00:00:00 2001 From: kanoi Date: Tue, 15 Sep 2015 14:36:56 +1000 Subject: [PATCH 04/62] php - update shift Rewarded unit text --- pool/page_shifts.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pool/page_shifts.php b/pool/page_shifts.php index 1753cce9..598a71a4 100644 --- a/pool/page_shifts.php +++ b/pool/page_shifts.php @@ -95,7 +95,7 @@ function doshifts($data, $user) } } $pg .= "\n"; - $pg .= "* The PPS value unit is satoshis
"; + $pg .= "* The Rewarded value unit is satoshis per 1diff share
"; return $pg; } From 8b8eb5dc62dbc59220f45b9be5fe366e355f0772 Mon Sep 17 00:00:00 2001 From: kanoi Date: Tue, 15 Sep 2015 20:27:21 +1000 Subject: [PATCH 05/62] ckdb - shift wasn't flagging workers used if not in the last shift --- src/ckdb.h | 2 +- src/ckdb_cmd.c | 30 ++++++++++++++++++++++++++---- src/ckdb_crypt.c | 6 +++--- 3 files changed, 30 insertions(+), 8 deletions(-) diff --git a/src/ckdb.h b/src/ckdb.h index 4bb19b5b..c34666c5 100644 --- a/src/ckdb.h +++ b/src/ckdb.h @@ -55,7 +55,7 @@ #define DB_VLOCK "1" #define DB_VERSION "1.0.2" -#define CKDB_VERSION DB_VERSION"-1.301" +#define CKDB_VERSION DB_VERSION"-1.302" #define WHERE_FFL " - from %s %s() line %d" #define WHERE_FFL_HERE __FILE__, __func__, __LINE__ diff --git a/src/ckdb_cmd.c b/src/ckdb_cmd.c index 935adca5..1f501c2d 100644 --- a/src/ckdb_cmd.c +++ b/src/ckdb_cmd.c @@ -269,6 +269,7 @@ static char *cmd_2fa(__maybe_unused PGconn *conn, char *cmd, char *id, __func__, st = safe_text_nonull(users->username), users->databits); + FREENULL(st); goto dame; } @@ -5073,6 +5074,7 @@ typedef struct worker_match { bool match; size_t len; bool used; + bool everused; } WM; static char *worker_offset(char *workername) @@ -5158,7 +5160,7 @@ static char *cmd_shifts(__maybe_unused PGconn *conn, char *cmd, char *id, size_t siz = sizeof(reply); char *select = NULL; WM workm[SELECT_LIMIT+1]; - char *buf, *work; + char *buf = NULL, *work, *st = NULL; size_t len, off; tv_t marker_end = { 0L, 0L }; int rows, want, i, where_all; @@ -5198,12 +5200,19 @@ static char *cmd_shifts(__maybe_unused PGconn *conn, char *cmd, char *id, if (i_select) select = strdup(transfer_data(i_select)); + APPEND_REALLOC_INIT(buf, off, len); + snprintf(tmp, sizeof(tmp), " select='%s'", + select ? st = safe_text_nonull(select) : "null"); + FREENULL(st); + APPEND_REALLOC(buf, off, len, tmp); + bzero(workm, sizeof(workm)); where_all = select_list(&(workm[0]), select); // Nothing selected = all if (workm[0].worker == NULL) { where_all = 0; workm[0].worker = WORKERS_ALL; + APPEND_REALLOC(buf, off, len, " no workers"); } else { for (i = 0; workm[i].worker; i++) { // N.B. len is only used if match is true @@ -5214,12 +5223,24 @@ static char *cmd_shifts(__maybe_unused PGconn *conn, char *cmd, char *id, workm[i].match = true; workm[i].len--; } + snprintf(tmp, sizeof(tmp), " workm[%d]=%s,%s,%d", + i, st = safe_text_nonull(workm[i].worker), + workm[i].match ? "Y" : "N", + (int)(workm[i].len)); + FREENULL(st); + APPEND_REALLOC(buf, off, len, tmp); } } if (where_all >= 0) workm[where_all].used = true; + snprintf(tmp, sizeof(tmp), " where_all=%d", where_all); + APPEND_REALLOC(buf, off, len, tmp); + LOGDEBUG("%s() user=%"PRId64"/%s' %s", + __func__, users->userid, users->username, buf+1); + FREENULL(buf); + APPEND_REALLOC_INIT(buf, off, len); APPEND_REALLOC(buf, off, len, "ok."); INIT_MARKERSUMMARY(&ms_look); @@ -5269,6 +5290,7 @@ static char *cmd_shifts(__maybe_unused PGconn *conn, char *cmd, char *id, (workm[want].match && strncmp(work, workm[want].worker, workm[want].len) == 0) || (!(workm[want].match) && strcmp(workm[want].worker, work) == 0)) { workm[want].used = true; + workm[want].everused = true; ms_add[want].diffacc += ms->diffacc; ms_add[want].diffsta += ms->diffsta; ms_add[want].diffdup += ms->diffdup; @@ -5414,7 +5436,7 @@ static char *cmd_shifts(__maybe_unused PGconn *conn, char *cmd, char *id, K_RUNLOCK(workmarkers_free); for (i = 0; workm[i].worker; i++) { - if (workm[i].used) { + if (workm[i].everused) { snprintf(tmp, sizeof(tmp), "%d_worker=%s%s%c", i, workm[i].worker, @@ -5448,7 +5470,7 @@ static char *cmd_shifts(__maybe_unused PGconn *conn, char *cmd, char *id, snprintf(tmp, sizeof(tmp), "arn=%s", "Shifts"); APPEND_REALLOC(buf, off, len, tmp); for (i = 0; workm[i].worker; i++) { - if (workm[i].used) { + if (workm[i].everused) { snprintf(tmp, sizeof(tmp), ",Worker_%d", i); APPEND_REALLOC(buf, off, len, tmp); } @@ -5457,7 +5479,7 @@ static char *cmd_shifts(__maybe_unused PGconn *conn, char *cmd, char *id, snprintf(tmp, sizeof(tmp), "%carp=", FLDSEP); APPEND_REALLOC(buf, off, len, tmp); for (i = 0; workm[i].worker; i++) { - if (workm[i].used) { + if (workm[i].everused) { snprintf(tmp, sizeof(tmp), ",%d_", i); APPEND_REALLOC(buf, off, len, tmp); } diff --git a/src/ckdb_crypt.c b/src/ckdb_crypt.c index 7ad0109f..e21cddf4 100644 --- a/src/ckdb_crypt.c +++ b/src/ckdb_crypt.c @@ -36,7 +36,7 @@ char *_tob32(USERS *users, unsigned char *bin, size_t len, char *name, if (osiz != olen) { LOGEMERG("%s() of '%s' data for '%s' invalid olen=%d != osiz=%d" WHERE_FFL, - __func__, name, safe_text_nonull(users->username), + __func__, name, st = safe_text_nonull(users->username), (int)olen, (int)osiz, WHERE_FFL_PASS); FREENULL(st); olen = osiz; @@ -59,8 +59,8 @@ char *_tob32(USERS *users, unsigned char *bin, size_t len, char *name, "ch=%d, i=%d j=%d bits=%d bin=0x%s len=%d " "olen=%d" WHERE_FFL, __func__, name, - safe_text_nonull(users->username), ch, i, j, - bits, binstr, (int)len, (int)olen, + st = safe_text_nonull(users->username), + ch, i, j, bits, binstr, (int)len, (int)olen, WHERE_FFL_PASS); FREENULL(st); FREENULL(binstr); From ea1a2c6edfbb7d75c4a836411350ef8920ed2da5 Mon Sep 17 00:00:00 2001 From: kanoi Date: Tue, 15 Sep 2015 22:28:00 +1000 Subject: [PATCH 06/62] ckdb - shifts created after a payout they are in, need rewarded set --- src/ckdb.h | 2 +- src/ckdb_data.c | 12 ++++++++++-- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/src/ckdb.h b/src/ckdb.h index c34666c5..a0963ae9 100644 --- a/src/ckdb.h +++ b/src/ckdb.h @@ -55,7 +55,7 @@ #define DB_VLOCK "1" #define DB_VERSION "1.0.2" -#define CKDB_VERSION DB_VERSION"-1.302" +#define CKDB_VERSION DB_VERSION"-1.303" #define WHERE_FFL " - from %s %s() line %d" #define WHERE_FFL_HERE __FILE__, __func__, __LINE__ diff --git a/src/ckdb_data.c b/src/ckdb_data.c index 720f3a8e..34f29848 100644 --- a/src/ckdb_data.c +++ b/src/ckdb_data.c @@ -4861,7 +4861,9 @@ bool reward_shifts(PAYOUTS *payouts, bool lock, int delta) * and thus if the shift is expired the counter is ignored * We only need to (re)calculate it when the workmarker is created * Payouts code processing will increment/decrement all current rewards as - * needed with reward_shifts() when payouts are added/changed/removed */ + * needed with reward_shifts() when payouts are added/changed/removed, + * however, the last shift in a payout can be created after the payout + * is generated so we need to update all from the payouts */ bool shift_rewards(K_ITEM *wm_item) { PAYOUTS *payouts = NULL; @@ -4869,6 +4871,7 @@ bool shift_rewards(K_ITEM *wm_item) WORKMARKERS *wm; K_ITEM *p_item; int rewards = 0; + double pps = 0.0; DATA_WORKMARKERS(wm, wm_item); @@ -4879,14 +4882,19 @@ bool shift_rewards(K_ITEM *wm_item) // a workmarker should not cross a payout boundary while (p_item && payouts->workinfoidstart <= wm->workinfoidstart && wm->workinfoidend <= payouts->workinfoidend) { - if (CURRENT(&(payouts->expirydate))) + if (CURRENT(&(payouts->expirydate))) { rewards++; + pps += (double)(payouts->minerreward) / + payouts->diffused; + } p_item = prev_in_ktree(ctx); DATA_PAYOUTS_NULL(payouts, p_item); } K_RUNLOCK(payouts_free); wm->rewards = rewards; + wm->rewarded = pps; + return (rewards > 0); } From 48eb9a9ee909fbb6889d3fa585b64c5116e5c917 Mon Sep 17 00:00:00 2001 From: kanoi Date: Thu, 17 Sep 2015 02:20:45 +1000 Subject: [PATCH 07/62] php - set shift rewarded format to %.5f --- pool/page_shifts.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pool/page_shifts.php b/pool/page_shifts.php index 598a71a4..2b330647 100644 --- a/pool/page_shifts.php +++ b/pool/page_shifts.php @@ -81,7 +81,7 @@ function doshifts($data, $user) $pg .= ''.$ans['rewards:'.$i].''; $ppsr = (float)$ans['ppsrewarded:'.$i]; if ($ppsr > 0) - $ppsd = sprintf('%.3e', $ppsr); + $ppsd = sprintf('%.5f', $ppsr); else $ppsd = '0'; $pg .= "$ppsd"; From 07d4e9512a1b299ebad7eff66fd77d16daf058f8 Mon Sep 17 00:00:00 2001 From: kanoi Date: Sun, 20 Sep 2015 01:24:32 +1000 Subject: [PATCH 08/62] ckdb/php - separate logging of ckpool msgs vs other, add a web timestamp --- pool/db.php | 6 +++-- src/ckdb.c | 69 ++++++++++++++++++++++++++++---------------------- src/ckdb.h | 16 +++++++++--- src/ckdb_cmd.c | 25 +++++++----------- 4 files changed, 64 insertions(+), 52 deletions(-) diff --git a/pool/db.php b/pool/db.php index a72591be..2b9d4c5e 100644 --- a/pool/db.php +++ b/pool/db.php @@ -139,13 +139,15 @@ function msgEncode($cmd, $id, $fields, $user) { global $send_sep, $fld_sep, $val_sep; - $t = time() % 10000; + $now = time(); + $t = $now % 10000; $msg = $cmd . $send_sep . $id.$t . $send_sep; foreach ($fields as $name => $value) $msg .= $name . $val_sep . $value . $fld_sep; $msg .= 'createcode' . $val_sep . 'php' . $fld_sep; $msg .= 'createby' . $val_sep . $user . $fld_sep; - $msg .= 'createinet' . $val_sep . zeip(); + $msg .= 'createinet' . $val_sep . zeip(). $fld_sep; + $msg .= 'webtime' . $val_sep . $now; adm($user, $msg); return $msg; } diff --git a/src/ckdb.c b/src/ckdb.c index 154916f5..6e4e03a7 100644 --- a/src/ckdb.c +++ b/src/ckdb.c @@ -497,17 +497,18 @@ K_TREE *userinfo_root; K_LIST *userinfo_free; K_STORE *userinfo_store; -static char logname[512]; +static char logname_db[512]; +static char logname_io[512]; static char *dbcode; // low spec version of rotating_log() - no locking -static bool rotating_log_nolock(char *msg) +static bool rotating_log_nolock(char *msg, char *prefix) { char *filename; FILE *fp; bool ok = false; - filename = rotating_filename(logname, time(NULL)); + filename = rotating_filename(prefix, time(NULL)); fp = fopen(filename, "a+e"); if (unlikely(!fp)) { LOGERR("Failed to fopen %s in rotating_log!", filename); @@ -523,7 +524,7 @@ stageleft: return ok; } -static void log_queue_message(char *msg) +static void log_queue_message(char *msg, bool db) { K_ITEM *lq_item; LOGQUEUE *lq; @@ -534,6 +535,7 @@ static void log_queue_message(char *msg) lq->msg = strdup(msg); if (!(lq->msg)) quithere(1, "malloc (%d) OOM", (int)strlen(msg)); + lq->db = db; k_add_tail(logqueue_store, lq_item); K_WUNLOCK(logqueue_free); } @@ -2518,12 +2520,18 @@ static enum cmd_values breakdown(K_ITEM **ml_item, char *buf, tv_t *now, } if (ckdb_cmds[msgline->which_cmds].cmd_val == CMD_END) { + LOGQUE(buf, false); LOGERR("Listener received unknown command: '%.42s...", st2 = safe_text(buf)); FREENULL(st2); goto nogood; } + if (ckdb_cmds[msgline->which_cmds].access & ACCESS_POOL) + LOGQUE(buf, true); + else + LOGQUE(buf, false); + if (noid) { if (ckdb_cmds[msgline->which_cmds].noid) { free(cmdptr); @@ -3614,7 +3622,8 @@ static void *logger(__maybe_unused void *arg) setnow(&now); snprintf(buf, sizeof(buf), "logstart.%ld,%ld", now.tv_sec, now.tv_usec); - LOGFILE(buf); + LOGFILE(buf, logname_db); + LOGFILE(buf, logname_io); while (!everyone_die) { K_WLOCK(logqueue_free); @@ -3622,7 +3631,10 @@ static void *logger(__maybe_unused void *arg) K_WUNLOCK(logqueue_free); while (lq_item) { DATA_LOGQUEUE(lq, lq_item); - LOGFILE(lq->msg); + if (lq->db) + LOGFILE(lq->msg, logname_db); + else + LOGFILE(lq->msg, logname_io); FREENULL(lq->msg); K_WLOCK(logqueue_free); @@ -3641,14 +3653,18 @@ static void *logger(__maybe_unused void *arg) setnow(&now); snprintf(buf, sizeof(buf), "logstopping.%d.%ld,%ld", count, now.tv_sec, now.tv_usec); - LOGFILE(buf); + LOGFILE(buf, logname_db); + LOGFILE(buf, logname_io); if (count) LOGERR("%s", buf); lq_item = logqueue_store->head; copy_tv(&then, &now); while (lq_item) { DATA_LOGQUEUE(lq, lq_item); - LOGFILE(lq->msg); + if (lq->db) + LOGFILE(lq->msg, logname_db); + else + LOGFILE(lq->msg, logname_io); FREENULL(lq->msg); count--; setnow(&now); @@ -3666,7 +3682,8 @@ static void *logger(__maybe_unused void *arg) setnow(&now); snprintf(buf, sizeof(buf), "logstop.%ld,%ld", now.tv_sec, now.tv_usec); - LOGFILE(buf); + LOGFILE(buf, logname_db); + LOGFILE(buf, logname_io); LOGWARNING("%s", buf); return NULL; @@ -3687,8 +3704,7 @@ static void *socketer(__maybe_unused void *arg) proc_instance_t *pi = (proc_instance_t *)arg; unixsock_t *us = &pi->us; char *end, *ans = NULL, *rep = NULL, *buf = NULL, *dot; - char *last_auth = NULL, *reply_auth = NULL; - char *last_addrauth = NULL, *reply_addrauth = NULL; + // No dup check for pool stats, the SEQ code will handle that char *last_chkpass = NULL, *reply_chkpass = NULL; char *last_adduser = NULL, *reply_adduser = NULL; char *last_newpass = NULL, *reply_newpass = NULL; @@ -3768,10 +3784,7 @@ static void *socketer(__maybe_unused void *arg) dup = false; show_dup = true; // These are ordered approximately most likely first - if (last_auth && strcmp(last_auth, buf) == 0) { - reply_last = reply_auth; - dup = true; - } else if (last_chkpass && strcmp(last_chkpass, buf) == 0) { + if (last_chkpass && strcmp(last_chkpass, buf) == 0) { reply_last = reply_chkpass; dup = true; } else if (last_adduser && strcmp(last_adduser, buf) == 0) { @@ -3783,9 +3796,6 @@ static void *socketer(__maybe_unused void *arg) } else if (last_newid && strcmp(last_newid, buf) == 0) { reply_last = reply_newid; dup = true; - } else if (last_addrauth && strcmp(last_addrauth, buf) == 0) { - reply_last = reply_addrauth; - dup = true; } else if (last_userset && strcmp(last_userset, buf) == 0) { reply_last = reply_userset; dup = true; @@ -3814,7 +3824,8 @@ static void *socketer(__maybe_unused void *arg) *dot = '\0'; snprintf(reply, sizeof(reply), "%s%ld,%ld.%s", LOGDUP, now.tv_sec, now.tv_usec, duptype); - LOGQUE(reply); + // dup cant be pool + LOGQUE(reply, false); if (show_dup) LOGWARNING("Duplicate '%s' message received", duptype); else @@ -3823,7 +3834,6 @@ static void *socketer(__maybe_unused void *arg) int seqentryflags = SE_SOCKET; if (!reload_queue_complete) seqentryflags = SE_EARLYSOCK; - LOGQUE(buf); cmdnum = breakdown(&ml_item, buf, &now, seqentryflags); DATA_MSGLINE(msgline, ml_item); replied = false; @@ -3956,12 +3966,6 @@ static void *socketer(__maybe_unused void *arg) send_unix_msg(sockd, rep); FREENULL(ans); switch (cmdnum) { - case CMD_AUTH: - STORELASTREPLY(auth); - break; - case CMD_ADDRAUTH: - STORELASTREPLY(addrauth); - break; case CMD_CHKPASS: STORELASTREPLY(chkpass); break; @@ -4233,7 +4237,6 @@ static void reload_line(PGconn *conn, char *filename, uint64_t count, char *buf) __func__, count); } - LOGQUE(buf); // ml_item is set for all but CMD_REPLY cmdnum = breakdown(&ml_item, buf, &now, SE_RELOAD); DATA_MSGLINE(msgline, ml_item); @@ -4444,7 +4447,8 @@ static bool reload_from(tv_t *start) copy_tv(&begin, &now); tvs_to_buf(&now, run, sizeof(run)); snprintf(reload_buf, MAX_READ, "reload.%s.s0", run); - LOGQUE(reload_buf); + LOGQUE(reload_buf, true); + LOGQUE(reload_buf, false); conn = dbconnect(); @@ -4537,7 +4541,8 @@ static bool reload_from(tv_t *start) diff = 1; snprintf(reload_buf, MAX_READ, "reload.%s.%"PRIu64, run, total); - LOGQUE(reload_buf); + LOGQUE(reload_buf, true); + LOGQUE(reload_buf, false); LOGWARNING("%s(): read %d file%s, total %"PRIu64" line%s %.2f/s", __func__, processing, processing == 1 ? "" : "s", @@ -5582,7 +5587,11 @@ int main(int argc, char **argv) quit(1, "Failed to open log file %s", buf); ckp.logfd = fileno(ckp.logfp); - snprintf(logname, sizeof(logname), "%s%s-db%s-", + // -db is ckpool messages + snprintf(logname_db, sizeof(logname_db), "%s%s-db%s-", + ckp.logdir, ckp.name, dbcode); + // -io is everything else + snprintf(logname_io, sizeof(logname_io), "%s%s-io%s-", ckp.logdir, ckp.name, dbcode); setnow(&now); diff --git a/src/ckdb.h b/src/ckdb.h index a0963ae9..85930c7b 100644 --- a/src/ckdb.h +++ b/src/ckdb.h @@ -55,7 +55,7 @@ #define DB_VLOCK "1" #define DB_VERSION "1.0.2" -#define CKDB_VERSION DB_VERSION"-1.303" +#define CKDB_VERSION DB_VERSION"-1.310" #define WHERE_FFL " - from %s %s() line %d" #define WHERE_FFL_HERE __FILE__, __func__, __LINE__ @@ -438,8 +438,8 @@ enum cmd_values { // CCLs are every ... #define ROLL_S 3600 -#define LOGQUE(_msg) log_queue_message(_msg) -#define LOGFILE(_msg) rotating_log_nolock(_msg) +#define LOGQUE(_msg, _db) log_queue_message(_msg, _db) +#define LOGFILE(_msg, _prefix) rotating_log_nolock(_msg, _prefix) #define LOGDUP "dup." // *** @@ -711,6 +711,7 @@ enum cmd_values { // LOGQUEUE typedef struct logqueue { char *msg; + bool db; } LOGQUEUE; #define ALLOC_LOGQUEUE 1024 @@ -2608,6 +2609,13 @@ extern bool check_db_version(PGconn *conn); // *** ckdb_cmd.c // *** +// TODO: limit access by having seperate sockets for each +#define ACCESS_POOL (1 << 0) +#define ACCESS_SYSTEM (1 << 1) +#define ACCESS_WEB (1 << 2) +#define ACCESS_PROXY (1 << 3) +#define ACCESS_CKDB (1 << 4) + struct CMDS { enum cmd_values cmd_val; char *cmd_str; @@ -2616,7 +2624,7 @@ struct CMDS { char *(*func)(PGconn *, char *, char *, tv_t *, char *, char *, char *, tv_t *, K_TREE *); enum seq_num seq; - char *access; + int access; }; extern struct CMDS ckdb_cmds[]; diff --git a/src/ckdb_cmd.c b/src/ckdb_cmd.c index 1f501c2d..45ac8393 100644 --- a/src/ckdb_cmd.c +++ b/src/ckdb_cmd.c @@ -6413,13 +6413,6 @@ static char *cmd_btcset(__maybe_unused PGconn *conn, char *cmd, char *id, return strdup(buf); } -// TODO: limit access by having seperate sockets for each -#define ACCESS_POOL "p" -#define ACCESS_SYSTEM "s" -#define ACCESS_WEB "w" -#define ACCESS_PROXY "x" -#define ACCESS_CKDB "c" - /* The socket command format is as follows: * Basic structure: * cmd.ID.fld1=value1 FLDSEP fld2=value2 FLDSEP fld3=... @@ -6482,8 +6475,8 @@ static char *cmd_btcset(__maybe_unused PGconn *conn, char *cmd, char *id, // cmd_val cmd_str noid createdate func seq access struct CMDS ckdb_cmds[] = { { CMD_TERMINATE, "terminate", true, false, NULL, SEQ_NONE, ACCESS_SYSTEM }, - { CMD_PING, "ping", true, false, NULL, SEQ_NONE, ACCESS_SYSTEM ACCESS_POOL ACCESS_WEB }, - { CMD_VERSION, "version", true, false, NULL, SEQ_NONE, ACCESS_SYSTEM ACCESS_POOL ACCESS_WEB }, + { CMD_PING, "ping", true, false, NULL, SEQ_NONE, ACCESS_SYSTEM | ACCESS_WEB }, + { CMD_VERSION, "version", true, false, NULL, SEQ_NONE, ACCESS_SYSTEM | ACCESS_WEB }, { CMD_LOGLEVEL, "loglevel", true, false, NULL, SEQ_NONE, ACCESS_SYSTEM }, { CMD_FLUSH, "flush", true, false, NULL, SEQ_NONE, ACCESS_SYSTEM }, { CMD_SHARELOG, STR_WORKINFO, false, true, cmd_sharelog, SEQ_WORKINFO, ACCESS_POOL }, @@ -6516,15 +6509,15 @@ struct CMDS ckdb_cmds[] = { { CMD_GETOPTS, "getopts", false, false, cmd_getopts, SEQ_NONE, ACCESS_WEB }, { CMD_SETOPTS, "setopts", false, false, cmd_setopts, SEQ_NONE, ACCESS_WEB }, { CMD_DSP, "dsp", false, false, cmd_dsp, SEQ_NONE, ACCESS_SYSTEM }, - { CMD_STATS, "stats", true, false, cmd_stats, SEQ_NONE, ACCESS_SYSTEM ACCESS_WEB }, - { CMD_PPLNS, "pplns", false, false, cmd_pplns, SEQ_NONE, ACCESS_SYSTEM ACCESS_WEB }, - { CMD_PPLNS2, "pplns2", false, false, cmd_pplns2, SEQ_NONE, ACCESS_SYSTEM ACCESS_WEB }, + { CMD_STATS, "stats", true, false, cmd_stats, SEQ_NONE, ACCESS_SYSTEM | ACCESS_WEB }, + { CMD_PPLNS, "pplns", false, false, cmd_pplns, SEQ_NONE, ACCESS_SYSTEM | ACCESS_WEB }, + { CMD_PPLNS2, "pplns2", false, false, cmd_pplns2, SEQ_NONE, ACCESS_SYSTEM | ACCESS_WEB }, { CMD_PAYOUTS, "payouts", false, false, cmd_payouts, SEQ_NONE, ACCESS_SYSTEM }, - { CMD_MPAYOUTS, "mpayouts", false, false, cmd_mpayouts, SEQ_NONE, ACCESS_SYSTEM ACCESS_WEB }, - { CMD_SHIFTS, "shifts", false, false, cmd_shifts, SEQ_NONE, ACCESS_SYSTEM ACCESS_WEB }, - { CMD_USERSTATUS,"userstatus", false, false, cmd_userstatus, SEQ_NONE, ACCESS_SYSTEM ACCESS_WEB }, + { CMD_MPAYOUTS, "mpayouts", false, false, cmd_mpayouts, SEQ_NONE, ACCESS_SYSTEM | ACCESS_WEB }, + { CMD_SHIFTS, "shifts", false, false, cmd_shifts, SEQ_NONE, ACCESS_SYSTEM | ACCESS_WEB }, + { CMD_USERSTATUS,"userstatus", false, false, cmd_userstatus, SEQ_NONE, ACCESS_SYSTEM | ACCESS_WEB }, { CMD_MARKS, "marks", false, false, cmd_marks, SEQ_NONE, ACCESS_SYSTEM }, - { CMD_PSHIFT, "pshift", false, false, cmd_pshift, SEQ_NONE, ACCESS_SYSTEM ACCESS_WEB }, + { CMD_PSHIFT, "pshift", false, false, cmd_pshift, SEQ_NONE, ACCESS_SYSTEM | ACCESS_WEB }, { CMD_SHSTA, "shsta", true, false, cmd_shsta, SEQ_NONE, ACCESS_SYSTEM }, { CMD_USERINFO, "userinfo", false, false, cmd_userinfo, SEQ_NONE, ACCESS_WEB }, { CMD_BTCSET, "btcset", false, false, cmd_btcset, SEQ_NONE, ACCESS_SYSTEM }, From e4f244eee6b7d22ae4f6a9a230df5ade02bafe97 Mon Sep 17 00:00:00 2001 From: kanoi Date: Sun, 20 Sep 2015 14:37:45 +1000 Subject: [PATCH 09/62] php - display the elapsed time of the worker share rate --- pool/page_workers.php | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/pool/page_workers.php b/pool/page_workers.php index 07987311..adf5ebf0 100644 --- a/pool/page_workers.php +++ b/pool/page_workers.php @@ -11,9 +11,10 @@ function worktitle($data, $user) $pg .= 'Shares'; $pg .= "<$r id=srtdiff data-sf=r4>:Diff"; $pg .= "<$r id=srtshrate data-sf=r5>:Share Rate"; - $pg .= "<$r id=srtinv data-sf=r6>:Invalid"; + $pg .= '«Elapsed'; + $pg .= "<$r id=srtinv data-sf=r7>:Invalid"; $pg .= 'Block %'; - $pg .= "<$r id=srtrate data-sf=r8>:Hash Rate"; + $pg .= "<$r id=srtrate data-sf=r9>:Hash Rate"; $pg .= "\n"; return $pg; } @@ -46,9 +47,10 @@ function workuser($data, $user, &$offset, &$totshare, &$totdiff, } $all = array(); $count = $ans['rows']; + $now = $ans['STAMP']; for ($i = 0; $i < $count; $i++) { - $lst = $ans['STAMP'] - $ans['w_lastshare:'.$i]; + $lst = $now - $ans['w_lastshare:'.$i]; if ($old !== false && $lst > $old) continue; @@ -74,7 +76,7 @@ function workuser($data, $user, &$offset, &$totshare, &$totdiff, for ($i = 0; $i < $count; $i++) { - $lst = $ans['STAMP'] - $all[$i]['w_lastshare']; + $lst = $now - $all[$i]['w_lastshare']; if ($old !== false && $lst > $old) continue; @@ -105,9 +107,12 @@ function workuser($data, $user, &$offset, &$totshare, &$totdiff, $acthr = '0'; $acthrv = 0; $actstt = $all[$i]['w_active_start']; - if ($actstt > 0) + if ($actstt <= 0 || ($now - $actstt) < 0) + $actsin = ' '; + else { - $elapsed = $ans['STAMP'] - $actstt; + $actsin = howmanyhrs($now - $actstt); + $elapsed = $now - $actstt; if ($elapsed > 0) { $acthrv = $all[$i]['w_active_diffacc'] * @@ -117,6 +122,7 @@ function workuser($data, $user, &$offset, &$totshare, &$totdiff, } } $pg .= "$acthr"; + $pg .= "$actsin"; $dinv = $all[$i]['w_diffinv']; $dtot = $dacc + $dinv; @@ -178,7 +184,7 @@ function worktotal($offset, $totshare, $totdiff, $totshrate, $totinvalid, $pg .= "$shareacc"; $diffacc = number_format($totdiff, 0); $pg .= "$diffacc"; - $pg .= "$totshrate"; + $pg .= "$totshrate "; $dtot = $totdiff + $totinvalid; if ($dtot > 0) $rej = number_format(100.0 * $totinvalid / $dtot, 3); From be18ff41a9da8f30840aa4826cf48350ab6c20f7 Mon Sep 17 00:00:00 2001 From: kanoi Date: Sun, 20 Sep 2015 16:53:55 +1000 Subject: [PATCH 10/62] ckdb - reset worker share rate every network block change --- src/ckdb.h | 3 ++- src/ckdb_data.c | 17 +++++++++++++++++ src/ckdb_dbio.c | 8 +++++++- 3 files changed, 26 insertions(+), 2 deletions(-) diff --git a/src/ckdb.h b/src/ckdb.h index 85930c7b..6eadecff 100644 --- a/src/ckdb.h +++ b/src/ckdb.h @@ -55,7 +55,7 @@ #define DB_VLOCK "1" #define DB_VERSION "1.0.2" -#define CKDB_VERSION DB_VERSION"-1.310" +#define CKDB_VERSION DB_VERSION"-1.311" #define WHERE_FFL " - from %s %s() line %d" #define WHERE_FFL_HERE __FILE__, __func__, __LINE__ @@ -2211,6 +2211,7 @@ extern K_ITEM *_find_create_workerstatus(int64_t userid, char *workername, bool create, const char *file2, const char *func2, const int line2, WHERE_FFL_ARGS); +extern void zero_all_active(tv_t *when); extern void workerstatus_ready(); #define workerstatus_update(_auths, _shares, _userstats) \ _workerstatus_update(_auths, _shares, _userstats, WHERE_FFL_HERE) diff --git a/src/ckdb_data.c b/src/ckdb_data.c index 34f29848..3393b99f 100644 --- a/src/ckdb_data.c +++ b/src/ckdb_data.c @@ -913,6 +913,23 @@ static void zero_on_idle(tv_t *when, WORKERSTATUS *workerstatus) workerstatus->active_sharehi = workerstatus->active_sharerej = 0.0; } +void zero_all_active(tv_t *when) +{ + WORKERSTATUS *workerstatus; + K_TREE_CTX ws_ctx[1]; + K_ITEM *ws_item; + + K_WLOCK(workerstatus_free); + ws_item = first_in_ktree(workerstatus_root, ws_ctx); + while (ws_item) { + DATA_WORKERSTATUS(workerstatus, ws_item); + zero_on_idle(when, workerstatus); + ws_item = next_in_ktree(ws_ctx); + } + + K_WUNLOCK(workerstatus_free); +} + /* All data is loaded, now update workerstatus fields TODO: combine set_block_share_counters() with this? */ void workerstatus_ready() diff --git a/src/ckdb_dbio.c b/src/ckdb_dbio.c index db2ed64f..d4a91555 100644 --- a/src/ckdb_dbio.c +++ b/src/ckdb_dbio.c @@ -2623,6 +2623,7 @@ int64_t workinfo_add(PGconn *conn, char *workinfoidstr, char *poolinstance, char *ins; char *params[11 + HISTORYDATECOUNT]; int n, par = 0; + bool zero_active = false; LOGDEBUG("%s(): add", __func__); @@ -2730,14 +2731,19 @@ unparam: if (workinfo_current) { WORKINFO *wic; DATA_WORKINFO(wic, workinfo_current); - if (cmp_height(wic->coinbase1, row->coinbase1) != 0) + if (cmp_height(wic->coinbase1, row->coinbase1) != 0) { copy_tv(&last_bc, cd); + zero_active = true; + } } workinfo_current = item; } K_WUNLOCK(workinfo_free); + if (zero_active) + zero_all_active(cd); + return workinfoid; } From a5ffba6d52310fdbd49933c0473dd5b27f46e145 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Mon, 21 Sep 2015 13:07:26 +1000 Subject: [PATCH 11/62] Increase default log buf size to fit mining notify messages --- src/libckpool.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libckpool.h b/src/libckpool.h index aed291dc..d4b48747 100644 --- a/src/libckpool.h +++ b/src/libckpool.h @@ -192,7 +192,7 @@ static inline void flip_80(void *dest_p, const void *src_p) void logmsg(int loglevel, const char *fmt, ...); -#define DEFLOGBUFSIZ 512 +#define DEFLOGBUFSIZ 1024 #define LOGMSGBUF(__lvl, __buf) do { \ logmsg(__lvl, "%s", __buf); \ From c0e4310c2da247982ca86b24a8183be86ce5230f Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Mon, 21 Sep 2015 13:17:57 +1000 Subject: [PATCH 12/62] Send a dropall to the stratifier when we're in reject mode --- src/connector.c | 1 + src/stratifier.c | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/connector.c b/src/connector.c index 3107901c..a7b776ab 100644 --- a/src/connector.c +++ b/src/connector.c @@ -864,6 +864,7 @@ retry: } else if (cmdmatch(buf, "reject")) { LOGDEBUG("Connector received reject signal"); cdata->accept = false; + send_proc(ckp->stratifier, "dropall"); } else if (cmdmatch(buf, "stats")) { char *msg; diff --git a/src/stratifier.c b/src/stratifier.c index 243a9df9..5bdb9146 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -1064,7 +1064,7 @@ static void drop_allclients(ckpool_t *ckp) ck_wunlock(&sdata->instance_lock); if (kills) - LOGNOTICE("Dropped %d instances", kills); + LOGNOTICE("Dropped %d instances for dropall request", kills); } static void update_subscribe(ckpool_t *ckp) From 3b9a84634011be606064024fa6795f493450c457 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Mon, 21 Sep 2015 13:40:42 +1000 Subject: [PATCH 13/62] Don't allow passthrough subclients to resume --- src/stratifier.c | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/stratifier.c b/src/stratifier.c index 5bdb9146..dc479330 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -1370,6 +1370,12 @@ static stratum_instance_t *__stratum_add_instance(ckpool_t *ckp, const int64_t i return client; } +/* passthrough subclients have client_ids in the high bits */ +static inline bool passthrough_subclient(const int64_t client_id) +{ + return (client_id > 0xffffffffll); +} + static uint64_t disconnected_sessionid_exists(sdata_t *sdata, const char *sessionid, int *session_id, const int64_t id) { @@ -1378,6 +1384,10 @@ static uint64_t disconnected_sessionid_exists(sdata_t *sdata, const char *sessio uint64_t ret = 0; int slen; + /* Don't allow passthrough subclients to resume */ + if (passthrough_subclient(id)) + goto out; + if (!sessionid) goto out; slen = strlen(sessionid) / 2; From 581b3c8902cede037071f2a35ea352ad68c51b46 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Mon, 21 Sep 2015 15:34:11 +1000 Subject: [PATCH 14/62] 1024 is upper limit to be considered text by most editors so change down to 1000 --- src/libckpool.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libckpool.h b/src/libckpool.h index d4b48747..9d8ba341 100644 --- a/src/libckpool.h +++ b/src/libckpool.h @@ -192,7 +192,7 @@ static inline void flip_80(void *dest_p, const void *src_p) void logmsg(int loglevel, const char *fmt, ...); -#define DEFLOGBUFSIZ 1024 +#define DEFLOGBUFSIZ 1000 #define LOGMSGBUF(__lvl, __buf) do { \ logmsg(__lvl, "%s", __buf); \ From 7d97b8d323de76469610762821399925032b70bc Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Mon, 21 Sep 2015 15:58:43 +1000 Subject: [PATCH 15/62] The stratifier shouldn't be broadcasting anything in passthrough mode --- src/stratifier.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/stratifier.c b/src/stratifier.c index dc479330..7073f97a 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -1773,7 +1773,7 @@ retry: LOGDEBUG("%ds elapsed in strat_loop, updating gbt base", ckp->update_interval); update_base(ckp, GEN_NORMAL); - } else { + } else if (!ckp->passthrough) { LOGDEBUG("%ds elapsed in strat_loop, pinging miners", ckp->update_interval); broadcast_ping(sdata); From 0d6fb758d698c9f0873c0d6a7338760c3d19db9b Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Mon, 21 Sep 2015 16:18:24 +1000 Subject: [PATCH 16/62] Timeout should be updated on each loop in proxy_recv --- src/generator.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/generator.c b/src/generator.c index 760f11e0..0336d257 100644 --- a/src/generator.c +++ b/src/generator.c @@ -1290,8 +1290,8 @@ static void *proxy_recv(void *arg) /* If we don't get an update within 10 minutes the upstream pool * has likely stopped responding. */ - timeout = 10; do { + timeout = 10; if (cs->fd == -1) { ret = -1; break; From 88e78a02da9d261e297903904daff77b2045db86 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Mon, 21 Sep 2015 16:22:09 +1000 Subject: [PATCH 17/62] Cope with read_socket_line timeouts properly in generator --- src/generator.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/generator.c b/src/generator.c index 0336d257..c8fec756 100644 --- a/src/generator.c +++ b/src/generator.c @@ -1261,9 +1261,9 @@ static void *proxy_recv(void *arg) while (42) { notify_instance_t *ni, *tmp; share_msg_t *share, *tmpshare; - int retries = 0, ret; float timeout; time_t now; + int ret; now = time(NULL); @@ -1290,14 +1290,14 @@ static void *proxy_recv(void *arg) /* If we don't get an update within 10 minutes the upstream pool * has likely stopped responding. */ + timeout = 600; do { - timeout = 10; if (cs->fd == -1) { ret = -1; break; } ret = read_socket_line(cs, &timeout); - } while (ret == 0 && ++retries < 120); + } while (ret == 0 && timeout > 0); if (ret < 1) { /* Send ourselves a reconnect message */ @@ -1403,7 +1403,7 @@ static void *passthrough_recv(void *arg) do { ret = read_socket_line(cs, &timeout); - } while (ret == 0); + } while (ret == 0 && timeout > 0); if (ret < 1) { /* Send ourselves a reconnect message */ From 83f111ae94010e61853745333aaf52c4927e9d98 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Mon, 21 Sep 2015 16:40:30 +1000 Subject: [PATCH 18/62] Make passthrough connection log to console --- src/generator.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/generator.c b/src/generator.c index c8fec756..27df2509 100644 --- a/src/generator.c +++ b/src/generator.c @@ -1538,12 +1538,13 @@ retry: } cs = alive->cs; - LOGNOTICE("Connected to upstream server %s:%s as proxy%s", cs->url, cs->port, - ckp->passthrough ? " in passthrough mode" : ""); if (ckp->passthrough) { + LOGWARNING("Connected to upstream server %s:%s as proxy in passthrough mode", + cs->url, cs->port); create_pthread(&alive->pth_precv, passthrough_recv, alive); alive->passsends = create_ckmsgq(ckp, "passsend", &passthrough_send); } else { + LOGNOTICE("Connected to upstream server %s:%s as proxy", cs->url, cs->port); create_pthread(&alive->pth_precv, proxy_recv, alive); mutex_init(&alive->psend_lock); cond_init(&alive->psend_cond); From 019062d94c7b889c5b448812fb709f60958ac185 Mon Sep 17 00:00:00 2001 From: kanoi Date: Tue, 22 Sep 2015 01:10:03 +1000 Subject: [PATCH 19/62] ckdb/php add last accepted share to tables, code and scripts --- pool/base.php | 11 +- pool/page_pplns2.php | 5 + pool/page_workers.php | 5 +- sql/ckdb.sql | 6 +- sql/v1.0.2-v1.0.3.sql | 38 ++++ src/ckdb.c | 47 +++-- src/ckdb.h | 26 ++- src/ckdb_cmd.c | 41 +++-- src/ckdb_data.c | 60 +++--- src/ckdb_dbio.c | 411 ++++++++++++++++++++++-------------------- 10 files changed, 385 insertions(+), 265 deletions(-) create mode 100644 sql/v1.0.2-v1.0.3.sql diff --git a/pool/base.php b/pool/base.php index 3e4c7c02..583ebcf6 100644 --- a/pool/base.php +++ b/pool/base.php @@ -99,7 +99,7 @@ function howlongago($sec) return $des; } # -function howmanyhrs($tot) +function howmanyhrs($tot, $days = false) { $sec = round($tot); if ($sec < 60) @@ -114,7 +114,14 @@ function howmanyhrs($tot) { $hr = floor($min / 60); $min -= $hr * 60; - $des = $hr.'hr '.$min.'m '.$sec.'s'; + if ($days && $hr > 23) + { + $dy = floor($hr / 24); + $hr -= $dy * 24; + $des = $dy.'d '.$hr.'hr '.$min.'m '.$sec.'s'; + } + else + $des = $hr.'hr '.$min.'m '.$sec.'s'; } } return $des; diff --git a/pool/page_pplns2.php b/pool/page_pplns2.php index 1d87007f..9455b655 100644 --- a/pool/page_pplns2.php +++ b/pool/page_pplns2.php @@ -121,6 +121,9 @@ function fmtdata($code, $val) case '.': $ret = number_format($val, 1); break; + case '@': + $ret = howmanyhrs($val, true); + break; default: $ret = $val; } @@ -186,6 +189,7 @@ Block: 'PPLNS Wanted' => '.diff_want', 'PPLNS Used' => '.diffacc_total', 'Elapsed Seconds' => ',pplns_elapsed', + 'Elapsed Time' => '@pplns_elapsed', 'Users' => 'rows', 'Oldest Workinfoid' => 'begin_workinfoid', 'Oldest Time' => 'begin_stamp', @@ -241,6 +245,7 @@ Block: { case ',': case '.': + case '@': $nm = substr($name, 1); $fmt = fmtdata($name[0], $ans[$nm]); break; diff --git a/pool/page_workers.php b/pool/page_workers.php index adf5ebf0..78173010 100644 --- a/pool/page_workers.php +++ b/pool/page_workers.php @@ -61,6 +61,7 @@ function workuser($data, $user, &$offset, &$totshare, &$totdiff, $all[] = array('workername' => $ans['workername:'.$i], 'w_lastshare' => $ans['w_lastshare:'.$i], + 'w_lastshareacc' => $ans['w_lastshareacc:'.$i], 'w_lastdiff' => $ans['w_lastdiff:'.$i], 'w_shareacc' => $ans['w_shareacc:'.$i], 'w_diffacc' => $ans['w_diffacc:'.$i], @@ -80,6 +81,8 @@ function workuser($data, $user, &$offset, &$totshare, &$totdiff, if ($old !== false && $lst > $old) continue; + $lstacc = $now - $all[$i]['w_lastshareacc']; + if ((($offset) % 2) == 0) $row = 'even'; else @@ -93,7 +96,7 @@ function workuser($data, $user, &$offset, &$totshare, &$totdiff, $ld = ' '; $pg .= "$ld"; - $pg .= "".howlongago($lst).''; + $pg .= "".howlongago($lstacc).''; $shareacc = number_format($all[$i]['w_shareacc'], 0); $totshare += $all[$i]['w_shareacc']; diff --git a/sql/ckdb.sql b/sql/ckdb.sql index f7d9775b..4f39ba26 100644 --- a/sql/ckdb.sql +++ b/sql/ckdb.sql @@ -247,6 +247,8 @@ CREATE TABLE sharesummary ( -- per workinfo for each user+worker errorcount bigint NOT NULL, firstshare timestamp with time zone NOT NULL, lastshare timestamp with time zone NOT NULL, + firstshareacc timestamp with time zone NOT NULL, + lastshareacc timestamp with time zone NOT NULL, lastdiffacc float NOT NULL, complete char NOT NULL, createdate timestamp with time zone NOT NULL, @@ -311,6 +313,8 @@ CREATE TABLE markersummary ( -- sum of sharesummary for a workinfo range errorcount bigint NOT NULL, firstshare timestamp with time zone NOT NULL, lastshare timestamp with time zone NOT NULL, + firstshareacc timestamp with time zone NOT NULL, + lastshareacc timestamp with time zone NOT NULL, lastdiffacc float NOT NULL, createdate timestamp with time zone NOT NULL, createby character varying(64) NOT NULL, @@ -464,4 +468,4 @@ CREATE TABLE version ( PRIMARY KEY (vlock) ); -insert into version (vlock,version) values (1,'1.0.2'); +insert into version (vlock,version) values (1,'1.0.3'); diff --git a/sql/v1.0.2-v1.0.3.sql b/sql/v1.0.2-v1.0.3.sql new file mode 100644 index 00000000..86494661 --- /dev/null +++ b/sql/v1.0.2-v1.0.3.sql @@ -0,0 +1,38 @@ +SET SESSION AUTHORIZATION 'postgres'; + +BEGIN transaction; + +DO $$ +DECLARE ver TEXT; +BEGIN + + UPDATE version set version='1.0.3' where vlock=1 and version='1.0.2'; + + IF found THEN + RETURN; + END IF; + + SELECT version into ver from version + WHERE vlock=1; + + RAISE EXCEPTION 'Wrong DB version - expect "1.0.2" - found "%"', ver; + +END $$; + +ALTER TABLE ONLY markersummary + ADD COLUMN firstshareacc timestamp with time zone DEFAULT '1970-01-01 00:00:00+00' NOT NULL, + ADD COLUMN lastshareacc timestamp with time zone DEFAULT '1970-01-01 00:00:00+00' NOT NULL; + +ALTER TABLE ONLY markersummary + ALTER COLUMN firstshareacc DROP DEFAULT, + ALTER COLUMN lastshareacc DROP DEFAULT; + +ALTER TABLE ONLY sharesummary + ADD COLUMN firstshareacc timestamp with time zone DEFAULT '1970-01-01 00:00:00+00' NOT NULL, + ADD COLUMN lastshareacc timestamp with time zone DEFAULT '1970-01-01 00:00:00+00' NOT NULL; + +ALTER TABLE ONLY sharesummary + ALTER COLUMN firstshareacc DROP DEFAULT, + ALTER COLUMN lastshareacc DROP DEFAULT; + +END transaction; diff --git a/src/ckdb.c b/src/ckdb.c index 6e4e03a7..63a51dfe 100644 --- a/src/ckdb.c +++ b/src/ckdb.c @@ -267,6 +267,7 @@ static bool seqdata_reload_lost = false; tv_t last_heartbeat; tv_t last_workinfo; tv_t last_share; +tv_t last_share_acc; tv_t last_share_inv; tv_t last_auth; cklock_t last_lock; @@ -2950,12 +2951,14 @@ static void summarise_blocks() return; } has_ss = true; - if (elapsed_start.tv_sec == 0 || - !tv_newer(&elapsed_start, &(sharesummary->firstshare))) { - copy_tv(&elapsed_start, &(sharesummary->firstshare)); + if (sharesummary->diffacc > 0) { + if (elapsed_start.tv_sec == 0 || + !tv_newer(&elapsed_start, &(sharesummary->firstshareacc))) { + copy_tv(&elapsed_start, &(sharesummary->firstshareacc)); + } + if (tv_newer(&elapsed_finish, &(sharesummary->lastshareacc))) + copy_tv(&elapsed_finish, &(sharesummary->lastshareacc)); } - if (tv_newer(&elapsed_finish, &(sharesummary->lastshare))) - copy_tv(&elapsed_finish, &(sharesummary->lastshare)); diffacc += sharesummary->diffacc; diffinv += sharesummary->diffsta + sharesummary->diffdup + @@ -3003,12 +3006,14 @@ static void summarise_blocks() DATA_MARKERSUMMARY_NULL(markersummary, ms_item); while (ms_item && markersummary->markerid == workmarkers->markerid) { has_ms = true; - if (elapsed_start.tv_sec == 0 || - !tv_newer(&elapsed_start, &(markersummary->firstshare))) { - copy_tv(&elapsed_start, &(markersummary->firstshare)); + if (markersummary->diffacc > 0) { + if (elapsed_start.tv_sec == 0 || + !tv_newer(&elapsed_start, &(markersummary->firstshareacc))) { + copy_tv(&elapsed_start, &(markersummary->firstshareacc)); + } + if (tv_newer(&elapsed_finish, &(markersummary->lastshareacc))) + copy_tv(&elapsed_finish, &(markersummary->lastshareacc)); } - if (tv_newer(&elapsed_finish, &(markersummary->lastshare))) - copy_tv(&elapsed_finish, &(markersummary->lastshare)); diffacc += markersummary->diffacc; diffinv += markersummary->diffsta + markersummary->diffdup + @@ -3149,7 +3154,7 @@ static void make_a_shift_mark() MARKS *marks = NULL, *sh_marks = NULL; int64_t ss_age_wid, last_marks_wid, marks_wid, prev_wid; bool was_block = false, ok; - char cd_buf[DATE_BUFSIZ], cd_buf2[DATE_BUFSIZ]; + char cd_buf[DATE_BUFSIZ], cd_buf2[DATE_BUFSIZ], cd_buf3[DATE_BUFSIZ]; int used_wid; /* If there are no CURRENT marks, make the first one by @@ -3225,12 +3230,13 @@ static void make_a_shift_mark() } K_RUNLOCK(sharesummary_free); if (ss_item) { - tv_to_buf(&(sharesummary->lastshare), cd_buf, sizeof(cd_buf)); - tv_to_buf(&(sharesummary->createdate), cd_buf2, sizeof(cd_buf2)); - LOGDEBUG("%s() last sharesummary %s/%s/%"PRId64"/%s/%s", + tv_to_buf(&(sharesummary->lastshareacc), cd_buf, sizeof(cd_buf)); + tv_to_buf(&(sharesummary->lastshare), cd_buf2, sizeof(cd_buf2)); + tv_to_buf(&(sharesummary->createdate), cd_buf3, sizeof(cd_buf3)); + LOGDEBUG("%s() last sharesummary %s/%s/%"PRId64"/%s/%s/%s", __func__, sharesummary->complete, sharesummary->workername, - ss_age_wid, cd_buf, cd_buf2); + ss_age_wid, cd_buf, cd_buf2, cd_buf3); } LOGDEBUG("%s() age sharesummary limit wid %"PRId64, __func__, ss_age_wid); @@ -3407,16 +3413,18 @@ static void make_a_shift_mark() /* Not aged = shift not complete * Though, it shouldn't happen */ if (sharesummary->complete[0] == SUMMARY_NEW) { - tv_to_buf(&(sharesummary->lastshare), + tv_to_buf(&(sharesummary->lastshareacc), cd_buf, sizeof(cd_buf)); - tv_to_buf(&(sharesummary->createdate), + tv_to_buf(&(sharesummary->lastshare), cd_buf2, sizeof(cd_buf2)); + tv_to_buf(&(sharesummary->createdate), + cd_buf3, sizeof(cd_buf3)); LOGEMERG("%s() ERR unaged sharesummary " - "%s/%s/%"PRId64"/%s/%s", + "%s/%s/%"PRId64"/%s/%s/%s", __func__, sharesummary->complete, sharesummary->workername, sharesummary->workinfoid, - cd_buf, cd_buf2); + cd_buf, cd_buf2, cd_buf3); return; } } @@ -4690,6 +4698,7 @@ static void *listener(void *arg) setnow(&last_heartbeat); copy_tv(&last_workinfo, &last_heartbeat); copy_tv(&last_share, &last_heartbeat); + copy_tv(&last_share_acc, &last_heartbeat); copy_tv(&last_share_inv, &last_heartbeat); copy_tv(&last_auth, &last_heartbeat); ck_wunlock(&last_lock); diff --git a/src/ckdb.h b/src/ckdb.h index 6eadecff..8b4360be 100644 --- a/src/ckdb.h +++ b/src/ckdb.h @@ -54,8 +54,8 @@ */ #define DB_VLOCK "1" -#define DB_VERSION "1.0.2" -#define CKDB_VERSION DB_VERSION"-1.311" +#define DB_VERSION "1.0.3" +#define CKDB_VERSION DB_VERSION"-1.320" #define WHERE_FFL " - from %s %s() line %d" #define WHERE_FFL_HERE __FILE__, __func__, __LINE__ @@ -312,6 +312,7 @@ extern bool everyone_die; extern tv_t last_heartbeat; extern tv_t last_workinfo; extern tv_t last_share; +extern tv_t last_share_acc; extern tv_t last_share_inv; extern tv_t last_auth; extern cklock_t last_lock; @@ -1435,6 +1436,8 @@ typedef struct sharesummary { int64_t errorcount; tv_t firstshare; tv_t lastshare; + tv_t firstshareacc; + tv_t lastshareacc; double lastdiffacc; char complete[TXT_FLAG+1]; MODIFYDATECONTROLPOINTERS; @@ -1830,7 +1833,8 @@ typedef struct workerstatus { char workername[TXT_BIG+1]; tv_t last_auth; tv_t last_share; - double last_diff; + tv_t last_share_acc; + double last_diff_acc; tv_t last_stats; tv_t last_idle; // Below gets reset on each block @@ -1890,6 +1894,8 @@ typedef struct markersummary { int64_t errorcount; tv_t firstshare; tv_t lastshare; + tv_t firstshareacc; + tv_t lastshareacc; double lastdiffacc; MODIFYDATECONTROLPOINTERS; } MARKERSUMMARY; @@ -2297,7 +2303,7 @@ extern cmp_t cmp_shareerrors(K_ITEM *a, K_ITEM *b); extern void dsp_sharesummary(K_ITEM *item, FILE *stream); extern cmp_t cmp_sharesummary(K_ITEM *a, K_ITEM *b); extern cmp_t cmp_sharesummary_workinfoid(K_ITEM *a, K_ITEM *b); -extern void zero_sharesummary(SHARESUMMARY *row, tv_t *cd, double diff); +extern void zero_sharesummary(SHARESUMMARY *row); #define find_sharesummary(_userid, _workername, _workinfoid) \ _find_sharesummary(_userid, _workername, _workinfoid, false) #define find_sharesummary_p(_workinfoid) \ @@ -2518,12 +2524,16 @@ extern bool sharesummaries_to_markersummaries(PGconn *conn, WORKMARKERS *workmar char *by, char *code, char *inet, tv_t *cd, K_TREE *trf_root); extern char *ooo_status(char *buf, size_t siz); -#define sharesummary_update(_s_row, _e_row, _ss_item, _by, _code, _inet, _cd) \ - _sharesummary_update(_s_row, _e_row, _ss_item, _by, _code, _inet, _cd, \ +#define sharesummary_update(_s_row, _e_row, _by, _code, _inet, _cd) \ + _sharesummary_update(_s_row, _e_row, _by, _code, _inet, _cd, \ WHERE_FFL_HERE) -extern bool _sharesummary_update(SHARES *s_row, SHAREERRORS *e_row, K_ITEM *ss_item, - char *by, char *code, char *inet, tv_t *cd, +extern bool _sharesummary_update(SHARES *s_row, SHAREERRORS *e_row, char *by, + char *code, char *inet, tv_t *cd, WHERE_FFL_ARGS); +#define sharesummary_age(_ss_item, _by, _code, _inet, _cd) \ + _sharesummary_age(_ss_item, _by, _code, _inet, _cd, WHERE_FFL_HERE) +extern bool _sharesummary_age(K_ITEM *ss_item, char *by, char *code, char *inet, + tv_t *cd, WHERE_FFL_ARGS); extern bool sharesummary_fill(PGconn *conn); extern bool blocks_stats(PGconn *conn, int32_t height, char *blockhash, double diffacc, double diffinv, double shareacc, diff --git a/src/ckdb_cmd.c b/src/ckdb_cmd.c index 45ac8393..ff472945 100644 --- a/src/ckdb_cmd.c +++ b/src/ckdb_cmd.c @@ -2001,6 +2001,7 @@ static char *cmd_workers(__maybe_unused PGconn *conn, char *cmd, char *id, if (ws_item) { DATA_WORKERSTATUS(workerstatus, ws_item); K_RLOCK(workerstatus_free); + // good or bad - either means active copy_tv(&last_share, &(workerstatus->last_share)); K_RUNLOCK(workerstatus_free); } else @@ -2028,7 +2029,9 @@ static char *cmd_workers(__maybe_unused PGconn *conn, char *cmd, char *id, double w_hashrate24hr; int64_t w_elapsed; tv_t w_lastshare; - double w_lastdiff, w_diffacc, w_diffinv; + tv_t w_lastshareacc; + double w_lastdiffacc, w_diffacc; + double w_diffinv; double w_diffsta, w_diffdup; double w_diffhi, w_diffrej; double w_shareacc, w_shareinv; @@ -2042,8 +2045,9 @@ static char *cmd_workers(__maybe_unused PGconn *conn, char *cmd, char *id, w_elapsed = -1; if (!ws_item) { - w_lastshare.tv_sec = 0L; - w_lastdiff = w_diffacc = + w_lastshare.tv_sec = + w_lastshareacc.tv_sec = 0L; + w_lastdiffacc = w_diffacc = w_diffinv = w_diffsta = w_diffdup = w_diffhi = w_diffrej = w_shareacc = @@ -2056,7 +2060,8 @@ static char *cmd_workers(__maybe_unused PGconn *conn, char *cmd, char *id, // It's bad to read possibly changing data K_RLOCK(workerstatus_free); w_lastshare.tv_sec = workerstatus->last_share.tv_sec; - w_lastdiff = workerstatus->last_diff; + w_lastshareacc.tv_sec = workerstatus->last_share_acc.tv_sec; + w_lastdiffacc = workerstatus->last_diff_acc; w_diffacc = workerstatus->block_diffacc; w_diffinv = workerstatus->block_diffinv; w_diffsta = workerstatus->block_diffsta; @@ -2112,7 +2117,11 @@ static char *cmd_workers(__maybe_unused PGconn *conn, char *cmd, char *id, snprintf(tmp, sizeof(tmp), "w_lastshare:%d=%s%c", rows, reply, FLDSEP); APPEND_REALLOC(buf, off, len, tmp); - double_to_buf(w_lastdiff, reply, sizeof(reply)); + int_to_buf((int)(w_lastshareacc.tv_sec), reply, sizeof(reply)); + snprintf(tmp, sizeof(tmp), "w_lastshareacc:%d=%s%c", rows, reply, FLDSEP); + APPEND_REALLOC(buf, off, len, tmp); + + double_to_buf(w_lastdiffacc, reply, sizeof(reply)); snprintf(tmp, sizeof(tmp), "w_lastdiff:%d=%s%c", rows, reply, FLDSEP); APPEND_REALLOC(buf, off, len, tmp); @@ -2185,7 +2194,7 @@ static char *cmd_workers(__maybe_unused PGconn *conn, char *cmd, char *id, "workername,difficultydefault,idlenotificationenabled," "idlenotificationtime", stats ? ",w_hashrate5m,w_hashrate1hr,w_hashrate24hr," - "w_elapsed,w_lastshare," + "w_elapsed,w_lastshare,w_lastshareacc," "w_lastdiff,w_diffacc,w_diffinv," "w_diffsta,w_diffdup,w_diffhi,w_diffrej," "w_shareacc,w_shareinv," @@ -2504,10 +2513,11 @@ wiconf: int32_t errn; TXT_TO_INT("errn", transfer_data(i_errn), errn); ck_wlock(&last_lock); + setnow(&last_share); if (errn == SE_NONE) - setnow(&last_share); + copy_tv(&last_share_acc, &last_share); else - setnow(&last_share_inv); + copy_tv(&last_share_inv, &last_share); ck_wunlock(&last_lock); } LOGDEBUG("%s.ok.added %s", id, transfer_data(i_nonce)); @@ -3114,6 +3124,9 @@ static char *cmd_homepage(__maybe_unused PGconn *conn, char *cmd, char *id, ftv_to_buf(&last_share, reply, siz); snprintf(tmp, sizeof(tmp), "lastsh=%s%c", reply, FLDSEP); APPEND_REALLOC(buf, off, len, tmp); + ftv_to_buf(&last_share_acc, reply, siz); + snprintf(tmp, sizeof(tmp), "lastshacc=%s%c", reply, FLDSEP); + APPEND_REALLOC(buf, off, len, tmp); ftv_to_buf(&last_share_inv, reply, siz); snprintf(tmp, sizeof(tmp), "lastshinv=%s%c", reply, FLDSEP); APPEND_REALLOC(buf, off, len, tmp); @@ -4137,7 +4150,7 @@ static char *cmd_pplns(__maybe_unused PGconn *conn, char *cmd, char *id, if (ss_item) end_workinfoid = sharesummary->workinfoid; /* add up all sharesummaries until >= diff_want - * also record the latest lastshare - that will be the end pplns time + * also record the latest lastshareacc - that will be the end pplns time * which will be >= block_tv */ while (total_diff < diff_want && ss_item) { switch (sharesummary->complete[0]) { @@ -4159,8 +4172,8 @@ static char *cmd_pplns(__maybe_unused PGconn *conn, char *cmd, char *id, acc_share_count += sharesummary->shareacc; total_diff += (int64_t)(sharesummary->diffacc); begin_workinfoid = sharesummary->workinfoid; - if (tv_newer(&end_tv, &(sharesummary->lastshare))) - copy_tv(&end_tv, &(sharesummary->lastshare)); + if (tv_newer(&end_tv, &(sharesummary->lastshareacc))) + copy_tv(&end_tv, &(sharesummary->lastshareacc)); mu_root = upd_add_mu(mu_root, mu_store, sharesummary->userid, (int64_t)(sharesummary->diffacc)); @@ -4233,8 +4246,8 @@ static char *cmd_pplns(__maybe_unused PGconn *conn, char *cmd, char *id, acc_share_count += markersummary->shareacc; total_diff += (int64_t)(markersummary->diffacc); begin_workinfoid = workmarkers->workinfoidstart; - if (tv_newer(&end_tv, &(markersummary->lastshare))) - copy_tv(&end_tv, &(markersummary->lastshare)); + if (tv_newer(&end_tv, &(markersummary->lastshareacc))) + copy_tv(&end_tv, &(markersummary->lastshareacc)); mu_root = upd_add_mu(mu_root, mu_store, markersummary->userid, (int64_t)(markersummary->diffacc)); @@ -6521,5 +6534,5 @@ struct CMDS ckdb_cmds[] = { { CMD_SHSTA, "shsta", true, false, cmd_shsta, SEQ_NONE, ACCESS_SYSTEM }, { CMD_USERINFO, "userinfo", false, false, cmd_userinfo, SEQ_NONE, ACCESS_WEB }, { CMD_BTCSET, "btcset", false, false, cmd_btcset, SEQ_NONE, ACCESS_SYSTEM }, - { CMD_END, NULL, false, false, NULL, SEQ_NONE, NULL } + { CMD_END, NULL, false, false, NULL, SEQ_NONE, 0 } }; diff --git a/src/ckdb_data.c b/src/ckdb_data.c index 3393b99f..2aa4feb4 100644 --- a/src/ckdb_data.c +++ b/src/ckdb_data.c @@ -958,7 +958,12 @@ void workerstatus_ready() &(markersummary->lastshare))) { copy_tv(&(workerstatus->last_share), &(markersummary->lastshare)); - workerstatus->last_diff = + } + if (tv_newer(&(workerstatus->last_share_acc), + &(markersummary->lastshareacc))) { + copy_tv(&(workerstatus->last_share_acc), + &(markersummary->lastshareacc)); + workerstatus->last_diff_acc = markersummary->lastdiffacc; } } @@ -973,7 +978,12 @@ void workerstatus_ready() &(sharesummary->lastshare))) { copy_tv(&(workerstatus->last_share), &(sharesummary->lastshare)); - workerstatus->last_diff = + } + if (tv_newer(&(workerstatus->last_share_acc), + &(sharesummary->lastshareacc))) { + copy_tv(&(workerstatus->last_share_acc), + &(sharesummary->lastshareacc)); + workerstatus->last_diff_acc = sharesummary->lastdiffacc; } } @@ -1017,10 +1027,8 @@ void _workerstatus_update(AUTHS *auths, SHARES *shares, if (item) { DATA_WORKERSTATUS(row, item); K_WLOCK(workerstatus_free); - if (tv_newer(&(row->last_share), &(shares->createdate))) { + if (tv_newer(&(row->last_share), &(shares->createdate))) copy_tv(&(row->last_share), &(shares->createdate)); - row->last_diff = shares->diff; - } if (row->active_start.tv_sec == 0) copy_tv(&(row->active_start), &(shares->createdate)); switch (shares->errn) { @@ -1029,6 +1037,12 @@ void _workerstatus_update(AUTHS *auths, SHARES *shares, row->block_shareacc++; row->active_diffacc += shares->diff; row->active_shareacc++; + if (tv_newer(&(row->last_share_acc), + &(shares->createdate))) { + copy_tv(&(row->last_share_acc), + &(shares->createdate)); + row->last_diff_acc = shares->diff; + } break; case SE_STALE: row->block_diffinv += shares->diff; @@ -2118,7 +2132,8 @@ bool workinfo_age(int64_t workinfoid, char *poolinstance, char *by, char *code, error[0] = '\0'; skipupdate = false; /* Reloading during a confirm will not have any old data - * so finding an aged sharesummary here is an error */ + * so finding an aged sharesummary here is an error + * N.B. this can only happen with (very) old reload files */ if (reloading) { if (sharesummary->complete[0] == SUMMARY_COMPLETE) { ss_already++; @@ -2134,7 +2149,7 @@ bool workinfo_age(int64_t workinfoid, char *poolinstance, char *by, char *code, } if (!skipupdate) { - if (!sharesummary_update(NULL, NULL, ss_item, by, code, inet, cd)) { + if (!sharesummary_age(ss_item, by, code, inet, cd)) { ss_failed++; LOGERR("%s(): Failed to age sharesummary %"PRId64"/%s/%"PRId64, __func__, sharesummary->userid, @@ -2364,17 +2379,17 @@ cmp_t cmp_sharesummary_workinfoid(K_ITEM *a, K_ITEM *b) return c; } -void zero_sharesummary(SHARESUMMARY *row, tv_t *cd, double diff) +void zero_sharesummary(SHARESUMMARY *row) { row->diffacc = row->diffsta = row->diffdup = row->diffhi = row->diffrej = row->shareacc = row->sharesta = row->sharedup = row->sharehi = row->sharerej = 0.0; row->sharecount = row->errorcount = 0; - row->firstshare.tv_sec = cd->tv_sec; - row->firstshare.tv_usec = cd->tv_usec; - row->lastshare.tv_sec = row->firstshare.tv_sec; - row->lastshare.tv_usec = row->firstshare.tv_usec; - row->lastdiffacc = diff; + DATE_ZERO(&(row->firstshare)); + DATE_ZERO(&(row->lastshare)); + DATE_ZERO(&(row->firstshareacc)); + DATE_ZERO(&(row->lastshareacc)); + row->lastdiffacc = 0; row->complete[0] = SUMMARY_NEW; row->complete[1] = '\0'; } @@ -3690,7 +3705,7 @@ bool process_pplns(int32_t height, char *blockhash, tv_t *addr_cd) if (ss_item) end_workinfoid = sharesummary->workinfoid; /* Add up all sharesummaries until >= diff_want - * also record the latest lastshare - that will be the end pplns time + * also record the latest lastshareacc - that will be the end pplns time * which will be >= blocks->blockcreatedate */ while (total_diff < diff_want && ss_item) { switch (sharesummary->complete[0]) { @@ -3723,10 +3738,8 @@ bool process_pplns(int32_t height, char *blockhash, tv_t *addr_cd) acc_share_count += sharesummary->shareacc; total_diff += sharesummary->diffacc; begin_workinfoid = sharesummary->workinfoid; - // TODO: add lastshareacc to sharesummary and markersummary - if (sharesummary->shareacc > 0 && - tv_newer(&end_tv, &(sharesummary->lastshare))) - copy_tv(&end_tv, &(sharesummary->lastshare)); + if (tv_newer(&end_tv, &(sharesummary->lastshareacc))) + copy_tv(&end_tv, &(sharesummary->lastshareacc)); mu_root = upd_add_mu(mu_root, mu_store, sharesummary->userid, sharesummary->diffacc); @@ -3760,10 +3773,8 @@ bool process_pplns(int32_t height, char *blockhash, tv_t *addr_cd) total_share_count += sharesummary->sharecount; acc_share_count += sharesummary->shareacc; total_diff += sharesummary->diffacc; - // TODO: add lastshareacc to sharesummary and markersummary - if (sharesummary->shareacc > 0 && - tv_newer(&end_tv, &(sharesummary->lastshare))) - copy_tv(&end_tv, &(sharesummary->lastshare)); + if (tv_newer(&end_tv, &(sharesummary->lastshareacc))) + copy_tv(&end_tv, &(sharesummary->lastshareacc)); mu_root = upd_add_mu(mu_root, mu_store, sharesummary->userid, sharesummary->diffacc); @@ -3812,9 +3823,8 @@ bool process_pplns(int32_t height, char *blockhash, tv_t *addr_cd) acc_share_count += markersummary->shareacc; total_diff += markersummary->diffacc; begin_workinfoid = workmarkers->workinfoidstart; - if (markersummary->shareacc > 0 && - tv_newer(&end_tv, &(markersummary->lastshare))) - copy_tv(&end_tv, &(markersummary->lastshare)); + if (tv_newer(&end_tv, &(markersummary->lastshareacc))) + copy_tv(&end_tv, &(markersummary->lastshareacc)); mu_root = upd_add_mu(mu_root, mu_store, markersummary->userid, markersummary->diffacc); diff --git a/src/ckdb_dbio.c b/src/ckdb_dbio.c index d4a91555..23abdbed 100644 --- a/src/ckdb_dbio.c +++ b/src/ckdb_dbio.c @@ -2993,9 +2993,8 @@ static bool shares_process(PGconn *conn, SHARES *shares, K_TREE *trf_root) userinfo_update(shares, NULL, NULL); } - sharesummary_update(shares, NULL, NULL, shares->createby, - shares->createcode, shares->createinet, - &(shares->createdate)); + sharesummary_update(shares, NULL, shares->createby, shares->createcode, + shares->createinet, &(shares->createdate)); return true; } @@ -3288,10 +3287,8 @@ static bool shareerrors_process(PGconn *conn, SHAREERRORS *shareerrors, } } - sharesummary_update(NULL, shareerrors, NULL, - shareerrors->createby, - shareerrors->createcode, - shareerrors->createinet, + sharesummary_update(NULL, shareerrors, shareerrors->createby, + shareerrors->createcode, shareerrors->createinet, &(shareerrors->createdate)); return true; @@ -3541,9 +3538,17 @@ static void markersummary_to_pool(MARKERSUMMARY *p_row, MARKERSUMMARY *row) !tv_newer(&(p_row->firstshare), &(row->firstshare))) { copy_tv(&(p_row->firstshare), &(row->firstshare)); } - if (tv_newer(&(p_row->lastshare), &(row->lastshare))) { + if (tv_newer(&(p_row->lastshare), &(row->lastshare))) copy_tv(&(p_row->lastshare), &(row->lastshare)); - p_row->lastdiffacc = row->lastdiffacc; + if (row->diffacc > 0) { + if (!p_row->firstshareacc.tv_sec || + !tv_newer(&(p_row->firstshareacc), &(row->firstshareacc))) { + copy_tv(&(p_row->firstshareacc), &(row->firstshareacc)); + } + if (tv_newer(&(p_row->lastshareacc), &(row->lastshareacc))) { + copy_tv(&(p_row->lastshareacc), &(row->lastshareacc)); + p_row->lastdiffacc = row->lastdiffacc; + } } } @@ -3553,7 +3558,7 @@ static void markersummary_to_pool(MARKERSUMMARY *p_row, MARKERSUMMARY *row) * so that is probably the best solution since * we should be watching the pool all the time :) * The cause would most likely be either a code bug or a DB problem - * so there many be no obvious automated fix + * so there may be no obvious automated fix * and flagging the workmarkers to be skipped may or may not be the solution, * thus manual intervention will be the rule for now */ bool sharesummaries_to_markersummaries(PGconn *conn, WORKMARKERS *workmarkers, @@ -3686,9 +3691,17 @@ bool sharesummaries_to_markersummaries(PGconn *conn, WORKMARKERS *workmarkers, !tv_newer(&(markersummary->firstshare), &(sharesummary->firstshare))) { copy_tv(&(markersummary->firstshare), &(sharesummary->firstshare)); } - if (tv_newer(&(markersummary->lastshare), &(sharesummary->lastshare))) { + if (tv_newer(&(markersummary->lastshare), &(sharesummary->lastshare))) copy_tv(&(markersummary->lastshare), &(sharesummary->lastshare)); - markersummary->lastdiffacc = sharesummary->lastdiffacc; + if (sharesummary->diffacc > 0) { + if (!markersummary->firstshareacc.tv_sec || + !tv_newer(&(markersummary->firstshareacc), &(sharesummary->firstshareacc))) { + copy_tv(&(markersummary->firstshareacc), &(sharesummary->firstshareacc)); + } + if (tv_newer(&(markersummary->lastshareacc), &(sharesummary->lastshareacc))) { + copy_tv(&(markersummary->lastshareacc), &(sharesummary->lastshareacc)); + markersummary->lastdiffacc = sharesummary->lastdiffacc; + } } diffacc += sharesummary->diffacc; @@ -3844,50 +3857,29 @@ flail: return ok; } -// no longer used -#if 0 -static void sharesummary_to_pool(SHARESUMMARY *p_row, SHARESUMMARY *row) -{ - p_row->diffacc += row->diffacc; - p_row->diffsta += row->diffsta; - p_row->diffdup += row->diffdup; - p_row->diffhi += row->diffhi; - p_row->diffrej += row->diffrej; - p_row->shareacc += row->shareacc; - p_row->sharesta += row->sharesta; - p_row->sharedup += row->sharedup; - p_row->sharehi += row->sharehi; - p_row->sharerej += row->sharerej; - p_row->sharecount += row->sharecount; - p_row->errorcount += row->errorcount; - if (!p_row->firstshare.tv_sec || - !tv_newer(&(p_row->firstshare), &(row->firstshare))) { - copy_tv(&(p_row->firstshare), &(row->firstshare)); - } - if (tv_newer(&(p_row->lastshare), &(row->lastshare))) { - copy_tv(&(p_row->lastshare), &(row->lastshare)); - p_row->lastdiffacc = row->lastdiffacc; - } -} -#endif - static void set_sharesummary_stats(SHARESUMMARY *row, SHARES *s_row, SHAREERRORS *e_row, bool new, double *tdf, double *tdl) { tv_t *createdate; - double diff; - if (s_row) { + if (s_row) createdate = &(s_row->createdate); - diff = s_row->diff; - } else { + else createdate = &(e_row->createdate); - diff = 0; - } - if (new) - zero_sharesummary(row, createdate, diff); + if (new) { + zero_sharesummary(row); + copy_tv(&(row->firstshare), createdate); + copy_tv(&(row->lastshare), createdate); + } else { + if (!row->firstshare.tv_sec || + !tv_newer(&(row->firstshare), createdate)) { + copy_tv(&(row->firstshare), createdate); + } + if (tv_newer(&(row->lastshare), createdate)) + copy_tv(&(row->lastshare), createdate); + } if (s_row) { row->sharecount += 1; @@ -3895,6 +3887,17 @@ static void set_sharesummary_stats(SHARESUMMARY *row, SHARES *s_row, case SE_NONE: row->diffacc += s_row->diff; row->shareacc++; + // should always be true + if (s_row->diff > 0) { + if (!row->firstshareacc.tv_sec || + !tv_newer(&(row->firstshareacc), createdate)) { + copy_tv(&(row->firstshareacc), createdate); + } + if (tv_newer(&(row->lastshareacc), createdate)) { + copy_tv(&(row->lastshareacc), createdate); + row->lastdiffacc = s_row->diff; + } + } break; case SE_STALE: row->diffsta += s_row->diff; @@ -3916,15 +3919,10 @@ static void set_sharesummary_stats(SHARESUMMARY *row, SHARES *s_row, } else row->errorcount += 1; + // Only if required if (!new) { *tdf = tvdiff(createdate, &(row->firstshare)); - if (*tdf < 0.0) - copy_tv(&(row->firstshare), createdate); *tdl = tvdiff(createdate, &(row->lastshare)); - if (*tdl >= 0.0) { - copy_tv(&(row->lastshare), createdate); - row->lastdiffacc = diff; - } } } @@ -3946,176 +3944,159 @@ char *ooo_status(char *buf, size_t siz) } // No longer stored in the DB but fields are updated as before -bool _sharesummary_update(SHARES *s_row, SHAREERRORS *e_row, K_ITEM *ss_item, - char *by, char *code, char *inet, tv_t *cd, - WHERE_FFL_ARGS) +bool _sharesummary_update(SHARES *s_row, SHAREERRORS *e_row, char *by, + char *code, char *inet, tv_t *cd, WHERE_FFL_ARGS) { WORKMARKERS *wm; SHARESUMMARY *row, *p_row; - K_ITEM *item, *wm_item, *p_item = NULL; + K_ITEM *ss_item, *wm_item, *p_item = NULL; bool new = false, p_new = false; int64_t userid, workinfoid; char *workername; - tv_t *createdate; char *st = NULL, *db = NULL; char ooo_buf[256]; double tdf, tdl; LOGDEBUG("%s(): update", __func__); - // this will never be a pool_ summary - if (ss_item) { - if (s_row || e_row) { - quithere(1, "ERR: only one of s_row, e_row and " - "ss_item allowed" WHERE_FFL, + if (s_row) { + if (e_row) { + quithere(1, "ERR: only one of s_row and e_row allowed" + WHERE_FFL, WHERE_FFL_PASS); } - item = ss_item; - DATA_SHARESUMMARY(row, item); - row->complete[0] = SUMMARY_COMPLETE; - row->complete[1] = '\0'; + userid = s_row->userid; + workername = s_row->workername; + workinfoid = s_row->workinfoid; } else { - if (s_row) { - if (e_row) { - quithere(1, "ERR: only one of s_row, e_row " - "(and ss_item) allowed" WHERE_FFL, - WHERE_FFL_PASS); - } - userid = s_row->userid; - workername = s_row->workername; - workinfoid = s_row->workinfoid; - createdate = &(s_row->createdate); - } else { - if (!e_row) { - quithere(1, "ERR: all s_row, e_row and " - "ss_item are NULL" WHERE_FFL, - WHERE_FFL_PASS); - } - userid = e_row->userid; - workername = e_row->workername; - workinfoid = e_row->workinfoid; - createdate = &(e_row->createdate); - } - - K_RLOCK(workmarkers_free); - wm_item = find_workmarkers(workinfoid, false, MARKER_PROCESSED, - NULL); - K_RUNLOCK(workmarkers_free); - if (wm_item) { - DATA_WORKMARKERS(wm, wm_item); - LOGERR("%s(): attempt to update sharesummary " - "with %s %"PRId64"/%"PRId64"/%s "CDDB" %s" - " but processed workmarkers %"PRId64" exists", - __func__, s_row ? "shares" : "shareerrors", - workinfoid, userid, st = safe_text(workername), - db = ctv_to_buf(createdate, NULL, 0), - wm->markerid); - FREENULL(st); - FREENULL(db); - return false; + if (!e_row) { + quithere(1, "ERR: both s_row and e_row are NULL" + WHERE_FFL, + WHERE_FFL_PASS); } + userid = e_row->userid; + workername = e_row->workername; + workinfoid = e_row->workinfoid; + } + + K_RLOCK(workmarkers_free); + wm_item = find_workmarkers(workinfoid, false, MARKER_PROCESSED, + NULL); + K_RUNLOCK(workmarkers_free); + if (wm_item) { + DATA_WORKMARKERS(wm, wm_item); + LOGERR("%s(): attempt to update sharesummary " + "with %s %"PRId64"/%"PRId64"/%s "CDDB" %s" + " but processed workmarkers %"PRId64" exists", + __func__, s_row ? "shares" : "shareerrors", + workinfoid, userid, st = safe_text(workername), + db = ctv_to_buf(cd, NULL, 0), + wm->markerid); + FREENULL(st); + FREENULL(db); + return false; + } - K_RLOCK(sharesummary_free); - item = find_sharesummary(userid, workername, workinfoid); - p_item = find_sharesummary_p(workinfoid); - K_RUNLOCK(sharesummary_free); + K_RLOCK(sharesummary_free); + ss_item = find_sharesummary(userid, workername, workinfoid); + p_item = find_sharesummary_p(workinfoid); + K_RUNLOCK(sharesummary_free); - if (item) { - DATA_SHARESUMMARY(row, item); - } else { - new = true; - K_WLOCK(sharesummary_free); - item = k_unlink_head(sharesummary_free); - K_WUNLOCK(sharesummary_free); - DATA_SHARESUMMARY(row, item); - bzero(row, sizeof(*row)); - row->userid = userid; - DUP_POINTER(sharesummary_free, row->workername, - workername); - row->workinfoid = workinfoid; - } + if (ss_item) { + DATA_SHARESUMMARY(row, ss_item); + } else { + new = true; + K_WLOCK(sharesummary_free); + ss_item = k_unlink_head(sharesummary_free); + K_WUNLOCK(sharesummary_free); + DATA_SHARESUMMARY(row, ss_item); + bzero(row, sizeof(*row)); + row->userid = userid; + DUP_POINTER(sharesummary_free, row->workername, + workername); + row->workinfoid = workinfoid; + } - // N.B. this directly updates the non-key data - set_sharesummary_stats(row, s_row, e_row, new, &tdf, &tdl); - - if (!new) { - // don't LOG '=' in case shares come from ckpool with the same timestamp - if (tdf < 0.0) { - char *tmp1, *tmp2; - int level = LOG_DEBUG; - // WARNING for shares exceeding the OOOLIMIT but not during startup - if (tdf < OOOLIMIT) { - ooof++; - if (startup_complete) - level = LOG_WARNING; - } else - ooof0++; - LOGMSG(level, "%s(): OoO %s "CDDB" (%s) is < summary" - " firstshare (%s) (%s)", - __func__, s_row ? "shares" : "shareerrors", - (tmp1 = ctv_to_buf(createdate, NULL, 0)), - (tmp2 = ctv_to_buf(&(row->firstshare), NULL, 0)), - ooo_status(ooo_buf, sizeof(ooo_buf))); - free(tmp2); - free(tmp1); - } + // N.B. this directly updates the non-key data + set_sharesummary_stats(row, s_row, e_row, new, &tdf, &tdl); - // don't LOG '=' in case shares come from ckpool with the same timestamp - if (tdl < 0.0) { - char *tmp1, *tmp2; - int level = LOG_DEBUG; - // WARNING for shares exceeding the OOOLIMIT but not during startup - if (tdl < OOOLIMIT) { - oool++; - if (startup_complete) - level = LOG_WARNING; - } else - oool0++; - LOGMSG(level, "%s(): OoO %s "CDDB" (%s) is < summary" - " lastshare (%s) (%s)", - __func__, s_row ? "shares" : "shareerrors", - (tmp1 = ctv_to_buf(createdate, NULL, 0)), - (tmp2 = ctv_to_buf(&(row->lastshare), NULL, 0)), - ooo_status(ooo_buf, sizeof(ooo_buf))); - free(tmp2); - free(tmp1); - } + if (!new) { + // don't LOG '=' in case shares come from ckpool with the same timestamp + if (tdf < 0.0) { + char *tmp1, *tmp2; + int level = LOG_DEBUG; + // WARNING for shares exceeding the OOOLIMIT but not during startup + if (tdf < OOOLIMIT) { + ooof++; + if (startup_complete) + level = LOG_WARNING; + } else + ooof0++; + LOGMSG(level, "%s(): OoO %s "CDDB" (%s) is < summary" + " firstshare (%s) (%s)", + __func__, s_row ? "shares" : "shareerrors", + (tmp1 = ctv_to_buf(cd, NULL, 0)), + (tmp2 = ctv_to_buf(&(row->firstshare), NULL, 0)), + ooo_status(ooo_buf, sizeof(ooo_buf))); + free(tmp2); + free(tmp1); + } - if (row->complete[0] != SUMMARY_NEW) { - LOGDEBUG("%s(): updating sharesummary not '%c'" - " %"PRId64"/%s/%"PRId64"/%s", - __func__, SUMMARY_NEW, row->userid, - st = safe_text_nonull(row->workername), - row->workinfoid, row->complete); - FREENULL(st); - } + // don't LOG '=' in case shares come from ckpool with the same timestamp + if (tdl < 0.0) { + char *tmp1, *tmp2; + int level = LOG_DEBUG; + // WARNING for shares exceeding the OOOLIMIT but not during startup + if (tdl < OOOLIMIT) { + oool++; + if (startup_complete) + level = LOG_WARNING; + } else + oool0++; + LOGMSG(level, "%s(): OoO %s "CDDB" (%s) is < summary" + " lastshare (%s) (%s)", + __func__, s_row ? "shares" : "shareerrors", + (tmp1 = ctv_to_buf(cd, NULL, 0)), + (tmp2 = ctv_to_buf(&(row->lastshare), NULL, 0)), + ooo_status(ooo_buf, sizeof(ooo_buf))); + free(tmp2); + free(tmp1); } - if (p_item) { - DATA_SHARESUMMARY(p_row, p_item); - } else { - p_new = true; - K_WLOCK(sharesummary_free); - p_item = k_unlink_head(sharesummary_free); - K_WUNLOCK(sharesummary_free); - DATA_SHARESUMMARY(p_row, p_item); - bzero(p_row, sizeof(*p_row)); - POOL_SS(p_row); - p_row->workinfoid = workinfoid; + if (row->complete[0] != SUMMARY_NEW) { + LOGDEBUG("%s(): updating sharesummary not '%c'" + " %"PRId64"/%s/%"PRId64"/%s", + __func__, SUMMARY_NEW, row->userid, + st = safe_text_nonull(row->workername), + row->workinfoid, row->complete); + FREENULL(st); } + } - set_sharesummary_stats(p_row, s_row, e_row, p_new, &tdf, &tdl); + if (p_item) { + DATA_SHARESUMMARY(p_row, p_item); + } else { + p_new = true; + K_WLOCK(sharesummary_free); + p_item = k_unlink_head(sharesummary_free); + K_WUNLOCK(sharesummary_free); + DATA_SHARESUMMARY(p_row, p_item); + bzero(p_row, sizeof(*p_row)); + POOL_SS(p_row); + p_row->workinfoid = workinfoid; } + set_sharesummary_stats(p_row, s_row, e_row, p_new, &tdf, &tdl); + MODIFYDATEPOINTERS(sharesummary_free, row, cd, by, code, inet); // Store either new item if (new || p_new) { K_WLOCK(sharesummary_free); if (new) { - add_to_ktree(sharesummary_root, item); - add_to_ktree(sharesummary_workinfoid_root, item); - k_add_head(sharesummary_store, item); + add_to_ktree(sharesummary_root, ss_item); + add_to_ktree(sharesummary_workinfoid_root, ss_item); + k_add_head(sharesummary_store, ss_item); } if (p_new) { add_to_ktree(sharesummary_pool_root, p_item); @@ -4127,6 +4108,23 @@ bool _sharesummary_update(SHARES *s_row, SHAREERRORS *e_row, K_ITEM *ss_item, return true; } +// No key fields are modified +bool _sharesummary_age(K_ITEM *ss_item, char *by, char *code, char *inet, + tv_t *cd, WHERE_FFL_ARGS) +{ + SHARESUMMARY *row; + + LOGDEBUG("%s(): update", __func__); + + DATA_SHARESUMMARY(row, ss_item); + row->complete[0] = SUMMARY_COMPLETE; + row->complete[1] = '\0'; + + MODIFYDATEPOINTERS(sharesummary_free, row, cd, by, code, inet); + + return true; +} + bool blocks_stats(PGconn *conn, int32_t height, char *blockhash, double diffacc, double diffinv, double shareacc, double shareinv, int64_t elapsed, @@ -6263,7 +6261,7 @@ bool markersummary_add(PGconn *conn, K_ITEM *ms_item, char *by, char *code, bool conned = false; PGresult *res; MARKERSUMMARY *row; - char *params[18 + MODIFYDATECOUNT]; + char *params[20 + MODIFYDATECOUNT]; int n, par = 0; char *ins; bool ok = false; @@ -6294,6 +6292,8 @@ bool markersummary_add(PGconn *conn, K_ITEM *ms_item, char *by, char *code, params[par++] = bigint_to_buf(row->errorcount, NULL, 0); params[par++] = tv_to_buf(&(row->firstshare), NULL, 0); params[par++] = tv_to_buf(&(row->lastshare), NULL, 0); + params[par++] = tv_to_buf(&(row->firstshareacc), NULL, 0); + params[par++] = tv_to_buf(&(row->lastshareacc), NULL, 0); params[par++] = double_to_buf(row->lastdiffacc, NULL, 0); MODIFYDATEPARAMS(params, par, row); PARCHK(par, params); @@ -6301,7 +6301,8 @@ bool markersummary_add(PGconn *conn, K_ITEM *ms_item, char *by, char *code, ins = "insert into markersummary " "(markerid,userid,workername,diffacc,diffsta,diffdup,diffhi," "diffrej,shareacc,sharesta,sharedup,sharehi,sharerej," - "sharecount,errorcount,firstshare,lastshare,lastdiffacc" + "sharecount,errorcount,firstshare,lastshare,firstshareacc," + "lastshareacc,lastdiffacc" MODIFYDATECONTROL ") values (" PQPARAM26 ")"; LOGDEBUG("%s() adding ms %"PRId64"/%"PRId64"/%s/%.0f", @@ -6344,7 +6345,7 @@ bool markersummary_fill(PGconn *conn) MARKERSUMMARY *row, *p_row; char *field; char *sel; - int fields = 18; + int fields = 20; bool ok; LOGDEBUG("%s(): select", __func__); @@ -6356,8 +6357,8 @@ bool markersummary_fill(PGconn *conn) sel = "select " "markerid,userid,workername,diffacc,diffsta,diffdup,diffhi," "diffrej,shareacc,sharesta,sharedup,sharehi,sharerej," - "sharecount,errorcount,firstshare,lastshare," - "lastdiffacc" + "sharecount,errorcount,firstshare,lastshare,firstshareacc," + "lastshareacc,lastdiffacc" MODIFYDATECONTROL " from markersummary"; res = PQexec(conn, sel, CKPQ_READ); @@ -6476,6 +6477,16 @@ bool markersummary_fill(PGconn *conn) break; TXT_TO_TV("lastshare", field, row->lastshare); + PQ_GET_FLD(res, i, "firstshareacc", field, ok); + if (!ok) + break; + TXT_TO_TV("firstshareacc", field, row->firstshareacc); + + PQ_GET_FLD(res, i, "lastshareacc", field, ok); + if (!ok) + break; + TXT_TO_TV("lastshareacc", field, row->lastshareacc); + PQ_GET_FLD(res, i, "lastdiffacc", field, ok); if (!ok) break; @@ -6485,6 +6496,16 @@ bool markersummary_fill(PGconn *conn) if (!ok) break; + /* Save having to do this everywhere in the code for old data + * It's not always accurate, but soon after when it's not, + * and also what was used before the 2 fields were added */ + if (row->diffacc > 0) { + if (row->firstshareacc.tv_sec == 0L) + copy_tv(&(row->firstshareacc), &(row->firstshare)); + if (row->lastshareacc.tv_sec == 0L) + copy_tv(&(row->lastshareacc), &(row->lastshare)); + } + add_to_ktree(markersummary_root, item); add_to_ktree(markersummary_userid_root, item); k_add_head(markersummary_store, item); From 43c89c323accdce3d39f716edd0ad1320c4e02df Mon Sep 17 00:00:00 2001 From: kanoi Date: Tue, 22 Sep 2015 10:46:26 +1000 Subject: [PATCH 20/62] ckdb - correct markersummary param count --- src/ckdb.h | 2 +- src/ckdb_dbio.c | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/ckdb.h b/src/ckdb.h index 8b4360be..ded07a54 100644 --- a/src/ckdb.h +++ b/src/ckdb.h @@ -55,7 +55,7 @@ #define DB_VLOCK "1" #define DB_VERSION "1.0.3" -#define CKDB_VERSION DB_VERSION"-1.320" +#define CKDB_VERSION DB_VERSION"-1.321" #define WHERE_FFL " - from %s %s() line %d" #define WHERE_FFL_HERE __FILE__, __func__, __LINE__ diff --git a/src/ckdb_dbio.c b/src/ckdb_dbio.c index 23abdbed..a5de06e1 100644 --- a/src/ckdb_dbio.c +++ b/src/ckdb_dbio.c @@ -184,6 +184,7 @@ char *pqerrmsg(PGconn *conn) #define PQPARAM23 PQPARAM16 ",$17,$18,$19,$20,$21,$22,$23" #define PQPARAM26 PQPARAM22 ",$23,$24,$25,$26" #define PQPARAM27 PQPARAM26 ",$27" +#define PQPARAM28 PQPARAM26 ",$27,$28" #define PARCHK(_par, _params) do { \ if (_par != (int)(sizeof(_params)/sizeof(_params[0]))) { \ @@ -6303,7 +6304,7 @@ bool markersummary_add(PGconn *conn, K_ITEM *ms_item, char *by, char *code, "diffrej,shareacc,sharesta,sharedup,sharehi,sharerej," "sharecount,errorcount,firstshare,lastshare,firstshareacc," "lastshareacc,lastdiffacc" - MODIFYDATECONTROL ") values (" PQPARAM26 ")"; + MODIFYDATECONTROL ") values (" PQPARAM28 ")"; LOGDEBUG("%s() adding ms %"PRId64"/%"PRId64"/%s/%.0f", __func__, row->markerid, row->userid, From 92c841c980a21d0be730b7acb093302b5807412f Mon Sep 17 00:00:00 2001 From: kanoi Date: Tue, 22 Sep 2015 11:34:36 +1000 Subject: [PATCH 21/62] ckdb - remove unused variables in SS_to_MS --- src/ckdb_dbio.c | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/ckdb_dbio.c b/src/ckdb_dbio.c index a5de06e1..f77abd57 100644 --- a/src/ckdb_dbio.c +++ b/src/ckdb_dbio.c @@ -3578,8 +3578,6 @@ bool sharesummaries_to_markersummaries(PGconn *conn, WORKMARKERS *workmarkers, bool ok = false, conned = false; int64_t diffacc, shareacc; char *reason = NULL; - char *params[2]; - int n, par = 0; int ss_count, ms_count; char *st = NULL; @@ -3753,9 +3751,6 @@ rollback: PQclear(res); flail: - for (n = 0; n < par; n++) - free(params[n]); - if (conned) PQfinish(conn); From d383285b3db1c3f9b3da6438b296873118f59d7a Mon Sep 17 00:00:00 2001 From: kanoi Date: Fri, 25 Sep 2015 23:29:15 +1000 Subject: [PATCH 22/62] ckdb - ignore non-processed workmarkers when determining the reload point --- src/ckdb.h | 2 +- src/ckdb_dbio.c | 21 +++++++++++++++++++-- 2 files changed, 20 insertions(+), 3 deletions(-) diff --git a/src/ckdb.h b/src/ckdb.h index ded07a54..045d99f2 100644 --- a/src/ckdb.h +++ b/src/ckdb.h @@ -55,7 +55,7 @@ #define DB_VLOCK "1" #define DB_VERSION "1.0.3" -#define CKDB_VERSION DB_VERSION"-1.321" +#define CKDB_VERSION DB_VERSION"-1.322" #define WHERE_FFL " - from %s %s() line %d" #define WHERE_FFL_HERE __FILE__, __func__, __LINE__ diff --git a/src/ckdb_dbio.c b/src/ckdb_dbio.c index f77abd57..ffd858e9 100644 --- a/src/ckdb_dbio.c +++ b/src/ckdb_dbio.c @@ -6872,13 +6872,30 @@ bool workmarkers_fill(PGconn *conn) } row->pps_value = workinfo_pps(wi_item, row->workinfoidend, false); - if (dbstatus.newest_workmarker_workinfoid < row->workinfoidend) { + if (CURRENT(&(row->expirydate)) && + !WMPROCESSED(row->status)) { + LOGWARNING("%s(): WARNING workmarkerid %"PRId64" (%s)" + " wid end %"PRId64" isn't processed! (%s) " + "You should abort ckdb and mark it if it " + "actually has already been processed", + __func__, row->markerid, row->description, + row->workinfoidend, row->status); + } + + /* Ignore any workmarker that isn't processed, so that the + * necessary data to process it can be reloaded, if the + * workmarker is after the last processed shift + * Any CURRENT non-processed workmarkers will give a console + * warning (above) */ + if (CURRENT(&(row->expirydate)) && + WMPROCESSED(row->status) && + dbstatus.newest_workmarker_workinfoid < row->workinfoidend) { dbstatus.newest_workmarker_workinfoid = row->workinfoidend; if (!wi_item) { LOGEMERG("%s(): FAILURE workmarkerid %"PRId64 " wid end %"PRId64" doesn't exist! " "You should abort ckdb and fix it, " - " since the reload may skip some data", + "since the reload may skip some data", __func__, row->markerid, row->workinfoidend); } else { From 067d99d018794d05c0f4f4c963c02914a9e034dd Mon Sep 17 00:00:00 2001 From: kanoi Date: Sat, 26 Sep 2015 14:49:55 +1000 Subject: [PATCH 23/62] ckdb - end shifts on a diff change --- src/ckdb.c | 29 +++++++++++++++++++++++++++++ src/ckdb.h | 2 +- 2 files changed, 30 insertions(+), 1 deletion(-) diff --git a/src/ckdb.c b/src/ckdb.c index 63a51dfe..e0f81575 100644 --- a/src/ckdb.c +++ b/src/ckdb.c @@ -3153,6 +3153,7 @@ static void make_a_shift_mark() BLOCKS *blocks = NULL; MARKS *marks = NULL, *sh_marks = NULL; int64_t ss_age_wid, last_marks_wid, marks_wid, prev_wid; + char wi_bits[TXT_SML+1]; bool was_block = false, ok; char cd_buf[DATE_BUFSIZ], cd_buf2[DATE_BUFSIZ], cd_buf3[DATE_BUFSIZ]; int used_wid; @@ -3365,6 +3366,7 @@ static void make_a_shift_mark() marks_wid = 0; used_wid = 0; prev_wid = 0; + wi_bits[0] = '\0'; while (wi_item) { DATA_WORKINFO(workinfo, wi_item); if (CURRENT(&(workinfo->expirydate))) { @@ -3376,6 +3378,33 @@ static void make_a_shift_mark() __func__, used_wid); return; } + if (wi_bits[0] == '\0') + STRNCPY(wi_bits, workinfo->bits); + else { + /* Did difficulty change? + * Stop at the last workinfo, before the diff + * changed + * IMPORTANT: this change will create different + * shifts if you were to reload a database, + * that you rolled back from after a diff + * change to before a diff change, if the data + * at the diff change was created with a + * version of ckdb before V1.323 + * THUS if that was the case, then some payouts + * that were generated before the rollback will + * be a small bit different to those created + * after reloading the same data */ + if (strcmp(wi_bits, workinfo->bits) != 0) { + LOGDEBUG("%s() OK shift stops at diff" + " change '%s->%s' %"PRId64 + "->%"PRId64, + __func__, wi_bits, + workinfo->bits, prev_wid, + workinfo->workinfoid); + marks_wid = prev_wid; + break; + } + } /* Did we find a pool restart? i.e. a wid skip * These will usually be a much larger jump, * however the pool should never skip any */ diff --git a/src/ckdb.h b/src/ckdb.h index 045d99f2..88976a4d 100644 --- a/src/ckdb.h +++ b/src/ckdb.h @@ -55,7 +55,7 @@ #define DB_VLOCK "1" #define DB_VERSION "1.0.3" -#define CKDB_VERSION DB_VERSION"-1.322" +#define CKDB_VERSION DB_VERSION"-1.323" #define WHERE_FFL " - from %s %s() line %d" #define WHERE_FFL_HERE __FILE__, __func__, __LINE__ From 1ad54736a736069b5cfdcd82ce8e8b413c60633a Mon Sep 17 00:00:00 2001 From: kanoi Date: Sat, 26 Sep 2015 21:21:38 +1000 Subject: [PATCH 24/62] ckdb - add to marks command deleting the markersummaries and reversing a workmarker back to ready --- src/ckdb.h | 3 +- src/ckdb_cmd.c | 96 ++++++++++++++++++++++++++- src/ckdb_dbio.c | 171 +++++++++++++++++++++++++++++++++++++++++++++++- 3 files changed, 264 insertions(+), 6 deletions(-) diff --git a/src/ckdb.h b/src/ckdb.h index 88976a4d..4186db66 100644 --- a/src/ckdb.h +++ b/src/ckdb.h @@ -55,7 +55,7 @@ #define DB_VLOCK "1" #define DB_VERSION "1.0.3" -#define CKDB_VERSION DB_VERSION"-1.323" +#define CKDB_VERSION DB_VERSION"-1.330" #define WHERE_FFL " - from %s %s() line %d" #define WHERE_FFL_HERE __FILE__, __func__, __LINE__ @@ -2523,6 +2523,7 @@ extern bool shareerrors_add(PGconn *conn, char *workinfoid, char *username, extern bool sharesummaries_to_markersummaries(PGconn *conn, WORKMARKERS *workmarkers, char *by, char *code, char *inet, tv_t *cd, K_TREE *trf_root); +extern bool delete_markersummaries(PGconn *conn, WORKMARKERS *wm); extern char *ooo_status(char *buf, size_t siz); #define sharesummary_update(_s_row, _e_row, _by, _code, _inet, _cd) \ _sharesummary_update(_s_row, _e_row, _by, _code, _inet, _cd, \ diff --git a/src/ckdb_cmd.c b/src/ckdb_cmd.c index ff472945..7fab93af 100644 --- a/src/ckdb_cmd.c +++ b/src/ckdb_cmd.c @@ -5738,7 +5738,7 @@ static char *cmd_marks(PGconn *conn, char *cmd, char *id, action = transfer_data(i_action); if (strcasecmp(action, "add") == 0) { - /* Add a mark + /* Add a mark, -m will automatically do this * Require marktype * Require workinfoid for all but 'b' * If marktype is 'b' or 'p' then require height/block (number) @@ -5944,7 +5944,7 @@ static char *cmd_marks(PGconn *conn, char *cmd, char *id, trf_root); } } else if (strcasecmp(action, "generate") == 0) { - /* Generate workmarkers + /* Generate workmarkers, -m will automatically do this * No parameters */ tmp[0] = '\0'; ok = workmarkers_generate(conn, tmp, sizeof(tmp), @@ -6005,6 +6005,7 @@ static char *cmd_marks(PGconn *conn, char *cmd, char *id, } else if (strcasecmp(action, "sum") == 0) { /* For the last available workmarker, * summarise it's sharesummaries into markersummaries + * -m will automatically do this * No parameters */ ok = make_markersummaries(true, by, code, inet, cd, trf_root); if (!ok) { @@ -6012,6 +6013,40 @@ static char *cmd_marks(PGconn *conn, char *cmd, char *id, LOGERR("%s.%s", id, reply); return strdup(reply); } + } else if (strcasecmp(action, "ready") == 0) { + /* Mark a processed workmarker as ready + * for fixing problems with markersummaries + * Requires markerid */ + i_markerid = require_name(trf_root, "markerid", 1, (char *)intpatt, reply, siz); + if (!i_markerid) + return strdup(reply); + TXT_TO_BIGINT("markerid", transfer_data(i_markerid), markerid); + K_RLOCK(workmarkers_free); + wm_item = find_workmarkerid(markerid, true, '\0'); + K_RUNLOCK(workmarkers_free); + if (!wm_item) { + snprintf(reply, siz, + "unknown workmarkers with markerid %"PRId64, markerid); + return strdup(reply); + } + DATA_WORKMARKERS(workmarkers, wm_item); + if (!WMPROCESSED(workmarkers->status)) { + snprintf(reply, siz, + "markerid isn't processed %"PRId64, markerid); + return strdup(reply); + } + ok = workmarkers_process(NULL, false, true, markerid, + workmarkers->poolinstance, + workmarkers->workinfoidend, + workmarkers->workinfoidstart, + workmarkers->description, + MARKER_READY_STR, + by, code, inet, cd, trf_root); + if (!ok) { + snprintf(reply, siz, "%s failed", action); + LOGERR("%s.%s", id, reply); + return strdup(reply); + } } else if (strcasecmp(action, "processed") == 0) { /* Mark a workmarker as processed * Requires markerid */ @@ -6045,6 +6080,63 @@ static char *cmd_marks(PGconn *conn, char *cmd, char *id, LOGERR("%s.%s", id, reply); return strdup(reply); } + } else if (strcasecmp(action, "cancel") == 0) { + /* Cancel(delete) all the markersummaries in a workmarker + * This can only be done if the workmarker isn't processed + * It reports on the console, summary information of the + * markersummaries that were deleted + * + * WARNING ... if you do this after the workmarker has been + * processed, there will no longer be any matching shares, + * sharesummaries or workmarkers in ram, so you'd need to + * restart ckdb to regenerate the markersummaries + * HOWEVER, ckdb wont reload the shares if there is a later + * workmarker that is processed + * + * SS_to_MS will complain if any markersummaries already exist + * when processing a workmarker + * Normally you would use 'processed' if the markersummaries + * are OK, and just the workmarker failed to be updated to + * processed status + * However, if there is actually something wrong with the + * shift data (markersummaries) you can delete them and they + * will be regenerated + * This will usually only work as expected if the last + * workmarker isn't marked as processed, but somehow there + * are markersummaries for it in the DB, thus the reload + * will reload all the shares for the workmarker then it + * will print a warning every 13s on the console saying + * that it can't process the workmarker + * In this case you would cancel the workmarker then ckdb + * will regenerate it from the shares/sharesummaries in ram + * + * Requires markerid */ + i_markerid = require_name(trf_root, "markerid", 1, (char *)intpatt, reply, siz); + if (!i_markerid) + return strdup(reply); + TXT_TO_BIGINT("markerid", transfer_data(i_markerid), markerid); + K_RLOCK(workmarkers_free); + wm_item = find_workmarkerid(markerid, true, '\0'); + K_RUNLOCK(workmarkers_free); + if (!wm_item) { + snprintf(reply, siz, + "unknown workmarkers with markerid %"PRId64, markerid); + return strdup(reply); + } + DATA_WORKMARKERS(workmarkers, wm_item); + if (WMPROCESSED(workmarkers->status)) { + snprintf(reply, siz, + "can't cancel a processed markerid %"PRId64, + markerid); + return strdup(reply); + } + + ok = delete_markersummaries(NULL, workmarkers); + if (!ok) { + snprintf(reply, siz, "%s failed", action); + LOGERR("%s.%s", id, reply); + return strdup(reply); + } } else { snprintf(reply, siz, "unknown action '%s'", action); LOGERR("%s.%s", id, reply); diff --git a/src/ckdb_dbio.c b/src/ckdb_dbio.c index ffd858e9..fd790a13 100644 --- a/src/ckdb_dbio.c +++ b/src/ckdb_dbio.c @@ -3567,7 +3567,7 @@ bool sharesummaries_to_markersummaries(PGconn *conn, WORKMARKERS *workmarkers, K_TREE *trf_root) { // shorter name for log messages - const char *shortname = "SS_to_MS"; + static const char *shortname = "SS_to_MS"; ExecStatusType rescode; PGresult *res; K_TREE_CTX ss_ctx[1], ms_ctx[1]; @@ -3613,6 +3613,14 @@ bool sharesummaries_to_markersummaries(PGconn *conn, WORKMARKERS *workmarkers, K_RUNLOCK(markersummary_free); DATA_MARKERSUMMARY_NULL(markersummary, ms_item); if (ms_item && markersummary->markerid == workmarkers->markerid) { + /* The fix here is either to set the workmarker as processed + * with the marks action=processed + * if the markersummaries are OK but the workmarker failed to + * have it's status set to processed + * OR + * delete the markersummaries with the marks action=cancel + * so this will continue and regenerate the markersummaries + */ reason = "markersummaries already exist"; goto flail; } @@ -3853,6 +3861,162 @@ flail: return ok; } +bool delete_markersummaries(PGconn *conn, WORKMARKERS *wm) +{ + // shorter name for log messages + static const char *shortname = "DEL_MS"; + K_STORE *del_markersummary_store = NULL; + ExecStatusType rescode; + PGresult *res; + K_TREE_CTX ms_ctx[1]; + MARKERSUMMARY *markersummary = NULL, lookmarkersummary; + K_ITEM *ms_item, ms_look, *p_ms_item = NULL; + bool ok = false, conned = false; + int64_t diffacc, shareacc; + char *reason = "unknown"; + int ms_count; + char *params[1]; + int par = 0, ms_del = 0; + char *del, *tuples = NULL; + + if (WMPROCESSED(wm->status)) { + reason = "status processed"; + goto flail; + } + + LOGWARNING("%s() Deleting: markersummaries for workmarkers " + "%"PRId64"/%s/End %"PRId64"/Stt %"PRId64"/%s/%s", + shortname, wm->markerid, wm->poolinstance, + wm->workinfoidend, wm->workinfoidstart, wm->description, + wm->status); + + del_markersummary_store = k_new_store(markersummary_free); + + lookmarkersummary.markerid = wm->markerid; + lookmarkersummary.userid = 0; + lookmarkersummary.workername = EMPTY; + + ms_count = diffacc = shareacc = 0; + ms_item = NULL; + + INIT_MARKERSUMMARY(&ms_look); + ms_look.data = (void *)(&lookmarkersummary); + + K_WLOCK(workmarkers_free); + K_WLOCK(markersummary_free); + + ms_item = find_after_in_ktree(markersummary_root, &ms_look, ms_ctx); + DATA_MARKERSUMMARY_NULL(markersummary, ms_item); + if (!ms_item || markersummary->markerid != wm->markerid) { + reason = "no markersummaries"; + goto flail; + } + + // Build the delete list of markersummaries + while (ms_item && markersummary->markerid == wm->markerid) { + ms_count++; + diffacc += markersummary->diffacc; + shareacc += markersummary->shareacc; + + k_unlink_item(markersummary_store, ms_item); + k_add_tail(del_markersummary_store, ms_item); + + ms_item = next_in_ktree(ms_ctx); + DATA_MARKERSUMMARY_NULL(markersummary, ms_item); + } + + par = 0; + params[par++] = bigint_to_buf(wm->markerid, NULL, 0); + PARCHK(par, params); + + del = "delete from markersummary where markerid=$1"; + + if (conn == NULL) { + conn = dbconnect(); + conned = true; + } + + res = PQexecParams(conn, del, par, NULL, (const char **)params, NULL, NULL, 0, CKPQ_WRITE); + rescode = PQresultStatus(res); + if (!PGOK(rescode)) { + PGLOGERR("Delete", rescode, conn); + reason = "db error"; + goto unparam; + } + + if (PGOK(rescode)) { + tuples = PQcmdTuples(res); + if (tuples && *tuples) { + ms_del = atoi(tuples); + if (ms_del != ms_count) { + LOGERR("%s() deleted markersummaries should be" + " %d but deleted=%d", + shortname, ms_count, ms_del); + reason = "del mismatch"; + goto unparam; + } + } + } + + ok = true; +unparam: + PQclear(res); +flail: + if (conned) + PQfinish(conn); + + if (!ok) { + if (del_markersummary_store && del_markersummary_store->count) { + k_list_transfer_to_head(del_markersummary_store, + markersummary_store); + } + } else { + /* TODO: add a list garbage collection thread so as to not + * invalidate the data immediately (free_*), rather after + * some delay */ + ms_item = del_markersummary_store->head; + while (ms_item) { + remove_from_ktree(markersummary_root, ms_item); + remove_from_ktree(markersummary_userid_root, ms_item); + free_markersummary_data(ms_item); + ms_item = ms_item->next; + } + + k_list_transfer_to_head(del_markersummary_store, + markersummary_free); + + p_ms_item = find_markersummary_p(wm->markerid); + if (p_ms_item) { + remove_from_ktree(markersummary_pool_root, p_ms_item); + free_markersummary_data(p_ms_item); + k_unlink_item(markersummary_pool_store, p_ms_item); + k_add_head(markersummary_free, p_ms_item); + } + } + + K_WUNLOCK(markersummary_free); + K_WUNLOCK(workmarkers_free); + + if (!ok) { + // already displayed the full workmarkers detail at the top + LOGERR("%s() %s: workmarkers %"PRId64"/%s/%s", + shortname, reason, wm->markerid, wm->description, + wm->status); + } else { + LOGWARNING("%s() Deleted: %d ms %"PRId64" shares " + "%"PRId64" diff for workmarkers %"PRId64"/%s/" + "End %"PRId64"/Stt %"PRId64"/%s/%s", + shortname, ms_count, shareacc, diffacc, + wm->markerid, wm->poolinstance, wm->workinfoidend, + wm->workinfoidstart, wm->description, wm->status); + } + + if (del_markersummary_store) + del_markersummary_store = k_free_store(del_markersummary_store); + + return ok; +} + static void set_sharesummary_stats(SHARESUMMARY *row, SHARES *s_row, SHAREERRORS *e_row, bool new, double *tdf, double *tdl) @@ -6876,8 +7040,9 @@ bool workmarkers_fill(PGconn *conn) !WMPROCESSED(row->status)) { LOGWARNING("%s(): WARNING workmarkerid %"PRId64" (%s)" " wid end %"PRId64" isn't processed! (%s) " - "You should abort ckdb and mark it if it " - "actually has already been processed", + "You need to correct it after the startup " + "completes, with a marks action: processed" + " or cancel", __func__, row->markerid, row->description, row->workinfoidend, row->status); } From 7226734ab4601ec18609181c8db4794dd77c3a85 Mon Sep 17 00:00:00 2001 From: kanoi Date: Sat, 26 Sep 2015 23:41:48 +1000 Subject: [PATCH 25/62] ckdb - allow turning marks generation on/off using the marks command actions genon/genoff --- src/ckdb.c | 3 +-- src/ckdb.h | 4 +++- src/ckdb_cmd.c | 46 +++++++++++++++++++++++++++++++++++++++------- 3 files changed, 43 insertions(+), 10 deletions(-) diff --git a/src/ckdb.c b/src/ckdb.c index e0f81575..637f089b 100644 --- a/src/ckdb.c +++ b/src/ckdb.c @@ -125,8 +125,7 @@ static char *status_chars = "|/-\\"; static char *restorefrom; -// Only accessed in here -static bool markersummary_auto; +bool markersummary_auto; int switch_state = SWITCH_STATE_ALL; diff --git a/src/ckdb.h b/src/ckdb.h index 4186db66..61f38f69 100644 --- a/src/ckdb.h +++ b/src/ckdb.h @@ -55,7 +55,7 @@ #define DB_VLOCK "1" #define DB_VERSION "1.0.3" -#define CKDB_VERSION DB_VERSION"-1.330" +#define CKDB_VERSION DB_VERSION"-1.331" #define WHERE_FFL " - from %s %s() line %d" #define WHERE_FFL_HERE __FILE__, __func__, __LINE__ @@ -95,6 +95,8 @@ extern int switch_state; #define SWITCH_STATE_AUTHWORKERS 1 #define SWITCH_STATE_ALL 666666 +extern bool markersummary_auto; + #define BLANK " " extern char *EMPTY; diff --git a/src/ckdb_cmd.c b/src/ckdb_cmd.c index 7fab93af..519a02a1 100644 --- a/src/ckdb_cmd.c +++ b/src/ckdb_cmd.c @@ -5738,7 +5738,7 @@ static char *cmd_marks(PGconn *conn, char *cmd, char *id, action = transfer_data(i_action); if (strcasecmp(action, "add") == 0) { - /* Add a mark, -m will automatically do this + /* Add a mark, -m/genon will automatically do this * Require marktype * Require workinfoid for all but 'b' * If marktype is 'b' or 'p' then require height/block (number) @@ -5944,7 +5944,7 @@ static char *cmd_marks(PGconn *conn, char *cmd, char *id, trf_root); } } else if (strcasecmp(action, "generate") == 0) { - /* Generate workmarkers, -m will automatically do this + /* Generate workmarkers, -m/genon will automatically do this * No parameters */ tmp[0] = '\0'; ok = workmarkers_generate(conn, tmp, sizeof(tmp), @@ -6005,7 +6005,7 @@ static char *cmd_marks(PGconn *conn, char *cmd, char *id, } else if (strcasecmp(action, "sum") == 0) { /* For the last available workmarker, * summarise it's sharesummaries into markersummaries - * -m will automatically do this + * -m/genon will automatically do this * No parameters */ ok = make_markersummaries(true, by, code, inet, cd, trf_root); if (!ok) { @@ -6087,11 +6087,21 @@ static char *cmd_marks(PGconn *conn, char *cmd, char *id, * markersummaries that were deleted * * WARNING ... if you do this after the workmarker has been - * processed, there will no longer be any matching shares, - * sharesummaries or workmarkers in ram, so you'd need to - * restart ckdb to regenerate the markersummaries + * processed, after switching it to ready, there will no + * longer be any matching shares or sharesummaries in ram + * to regenerate the markersummaries, so you'd need to restart + * ckdb to reload the shares to regenerate the markersummaries * HOWEVER, ckdb wont reload the shares if there is a later - * workmarker that is processed + * workmarker that is already processed + * + * To reprocess an already processed workmarker, you'd have + * to firstly turn off auto processing with genoff, then + * change the required workmarker, and all after it, to + * ready, then cancel them all, then finally restart ckdb + * which will reload all the necessary shares and regenerate + * the markersummaries + * Of course if you don't have ALL the necessary shares in + * the CCLs then you'd lose data doing this * * SS_to_MS will complain if any markersummaries already exist * when processing a workmarker @@ -6137,6 +6147,28 @@ static char *cmd_marks(PGconn *conn, char *cmd, char *id, LOGERR("%s.%s", id, reply); return strdup(reply); } + } else if (strcasecmp(action, "genon") == 0) { + /* Turn on auto marker generation and processing + * and report the before/after status + * No parameters */ + bool old = markersummary_auto; + markersummary_auto = true; + snprintf(msg, sizeof(msg), "mark generation state was %s," + " now %s", + old ? "On" : "Off", + markersummary_auto ? "On" : "Off"); + ok = true; + } else if (strcasecmp(action, "genoff") == 0) { + /* Turn off auto marker generation and processing + * and report the before/after status + * No parameters */ + bool old = markersummary_auto; + markersummary_auto = false; + snprintf(msg, sizeof(msg), "mark generation state was %s," + " now %s", + old ? "On" : "Off", + markersummary_auto ? "On" : "Off"); + ok = true; } else { snprintf(reply, siz, "unknown action '%s'", action); LOGERR("%s.%s", id, reply); From 10a69a6db1dbb2e69df31b9fbf622b0dee959269 Mon Sep 17 00:00:00 2001 From: kanoi Date: Sat, 26 Sep 2015 23:47:00 +1000 Subject: [PATCH 26/62] ckdb - set the -m flag to handle all mark related generation, not just markersummaries --- src/ckdb.c | 12 ++++++++---- src/ckdb.h | 2 +- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/src/ckdb.c b/src/ckdb.c index 637f089b..b152b7aa 100644 --- a/src/ckdb.c +++ b/src/ckdb.c @@ -3611,8 +3611,10 @@ static void *marker(__maybe_unused void *arg) } if (everyone_die) break; - else - make_a_shift_mark(); + else { + if (markersummary_auto) + make_a_shift_mark(); + } for (i = 0; i < 4; i++) { if (!everyone_die) @@ -3620,8 +3622,10 @@ static void *marker(__maybe_unused void *arg) } if (everyone_die) break; - else - make_a_workmarker(); + else { + if (markersummary_auto) + make_a_workmarker(); + } for (i = 0; i < 4; i++) { if (!everyone_die) diff --git a/src/ckdb.h b/src/ckdb.h index 61f38f69..b9170fc2 100644 --- a/src/ckdb.h +++ b/src/ckdb.h @@ -55,7 +55,7 @@ #define DB_VLOCK "1" #define DB_VERSION "1.0.3" -#define CKDB_VERSION DB_VERSION"-1.331" +#define CKDB_VERSION DB_VERSION"-1.332" #define WHERE_FFL " - from %s %s() line %d" #define WHERE_FFL_HERE __FILE__, __func__, __LINE__ From ad52dc6771e4c1930fb877e7e8068ff9c90a8c18 Mon Sep 17 00:00:00 2001 From: kanoi Date: Sun, 27 Sep 2015 01:17:41 +1000 Subject: [PATCH 27/62] ckdb - add -g flag to enable payout generation - off by default - and payouts commands genon/genoff to enable/disable it --- src/ckdb.c | 24 +++++++++++++++++++++--- src/ckdb.h | 3 ++- src/ckdb_cmd.c | 22 ++++++++++++++++++++++ 3 files changed, 45 insertions(+), 4 deletions(-) diff --git a/src/ckdb.c b/src/ckdb.c index b152b7aa..c9525a63 100644 --- a/src/ckdb.c +++ b/src/ckdb.c @@ -125,6 +125,7 @@ static char *status_chars = "|/-\\"; static char *restorefrom; +bool genpayout_auto; bool markersummary_auto; int switch_state = SWITCH_STATE_ALL; @@ -1432,6 +1433,13 @@ static bool setup_data() mutex_init(&wq_waitlock); cond_init(&wq_waitcond); + LOGWARNING("%sStartup payout generation state is %s", + genpayout_auto ? "" : "WARNING: ", + genpayout_auto ? "On" : "Off"); + LOGWARNING("%sStartup mark generation state is %s", + markersummary_auto ? "" : "WARNING: ", + markersummary_auto ? "On" : "Off"); + alloc_storage(); setnow(&db_stt); @@ -3054,7 +3062,12 @@ static void summarise_blocks() diffacc, diffinv, shareacc, shareinv, elapsed); // Now the summarisation is confirmed, generate the payout data - pplns_block(blocks); + if (genpayout_auto) + pplns_block(blocks); + else { + LOGWARNING("%s() Auto payout generation disabled", + __func__); + } } else { LOGERR("%s() block %d, failed to confirm stats", __func__, blocks->height); @@ -5402,11 +5415,13 @@ static void check_restore_dir(char *name) static struct option long_options[] = { { "config", required_argument, 0, 'c' }, { "dbname", required_argument, 0, 'd' }, + // generate = enable payout pplns auto generation + { "generate", no_argument, 0, 'g' }, { "help", no_argument, 0, 'h' }, { "killold", no_argument, 0, 'k' }, { "loglevel", required_argument, 0, 'l' }, - // markersummary = enable markersummary auto generation - { "markersummary", no_argument, 0, 'm' }, + // marker = enable mark/workmarker/markersummary auto generation + { "marker", no_argument, 0, 'm' }, { "name", required_argument, 0, 'n' }, { "dbpass", required_argument, 0, 'p' }, { "btc-pass", required_argument, 0, 'P' }, @@ -5462,6 +5477,9 @@ int main(int argc, char **argv) while (*kill) *(kill++) = ' '; break; + case 'g': + genpayout_auto = true; + break; case 'h': for (j = 0; long_options[j].val; j++) { struct option *jopt = &long_options[j]; diff --git a/src/ckdb.h b/src/ckdb.h index b9170fc2..eb454c02 100644 --- a/src/ckdb.h +++ b/src/ckdb.h @@ -55,7 +55,7 @@ #define DB_VLOCK "1" #define DB_VERSION "1.0.3" -#define CKDB_VERSION DB_VERSION"-1.332" +#define CKDB_VERSION DB_VERSION"-1.333" #define WHERE_FFL " - from %s %s() line %d" #define WHERE_FFL_HERE __FILE__, __func__, __LINE__ @@ -95,6 +95,7 @@ extern int switch_state; #define SWITCH_STATE_AUTHWORKERS 1 #define SWITCH_STATE_ALL 666666 +extern bool genpayout_auto; extern bool markersummary_auto; #define BLANK " " diff --git a/src/ckdb_cmd.c b/src/ckdb_cmd.c index 519a02a1..4a757840 100644 --- a/src/ckdb_cmd.c +++ b/src/ckdb_cmd.c @@ -4947,6 +4947,28 @@ static char *cmd_payouts(PGconn *conn, char *cmd, char *id, tv_t *now, else ok = process_pplns(height, blockhash, &addrdate); + } else if (strcasecmp(action, "genon") == 0) { + /* Turn on auto payout generation + * and report the before/after status + * No parameters */ + bool old = genpayout_auto; + genpayout_auto = true; + snprintf(msg, sizeof(msg), "payout generation state was %s," + " now %s", + old ? "On" : "Off", + genpayout_auto ? "On" : "Off"); + ok = true; + } else if (strcasecmp(action, "genoff") == 0) { + /* Turn off auto payout generation + * and report the before/after status + * No parameters */ + bool old = genpayout_auto; + genpayout_auto = false; + snprintf(msg, sizeof(msg), "payout generation state was %s," + " now %s", + old ? "On" : "Off", + genpayout_auto ? "On" : "Off"); + ok = true; } else { snprintf(reply, siz, "unknown action '%s'", action); LOGERR("%s.%s", id, reply); From 17f9bdf44728340866fee235fd3f62c9687cafd6 Mon Sep 17 00:00:00 2001 From: kanoi Date: Mon, 28 Sep 2015 19:16:21 +1000 Subject: [PATCH 28/62] ckdb - add a lower block limit to when diff change shift ends occur --- pool/page_payout.php | 18 ++++++++----- src/ckdb.c | 63 +++++++++++++++++++++++++++++--------------- src/ckdb.h | 5 ++-- src/ckdb_cmd.c | 3 +-- src/ckdb_data.c | 2 +- 5 files changed, 58 insertions(+), 33 deletions(-) diff --git a/pool/page_payout.php b/pool/page_payout.php index 8f7ff36b..b691f1f4 100644 --- a/pool/page_payout.php +++ b/pool/page_payout.php @@ -2,9 +2,10 @@ # function dopayout($data, $user) { - $t = '5'; - $ot = '1/5'; - $n = '5Nd'; + $N = 5; + $t = "$N"; + $ot = "1/$N"; + $n = "${N}Nd"; $n1 = 'N'; $n1d = 'Nd'; $bc = '+101 Confirms'; @@ -13,7 +14,7 @@ function dopayout($data, $user) if (isset($data['info']['currndiff'])) $nd = $data['info']['currndiff']; $nv = number_format($nd, 1); - $nv5 = number_format(5.0*$nd, 1); + $nvx = number_format($N*$nd, 1); $pg = "

Payouts

"; $pg .= ""; $pg .= ""; + $pg .= ""; $pg .= "\n"; $since = $data['info']['lastblock']; @@ -121,6 +122,7 @@ function doblocks($data, $user) $bg = " bgcolor=$bg"; $luck = number_format(100 * $ans['s_luck:'.$i], 2); + $txm = number_format(100 * $ans['s_txmean:'.$i], 1); $pg .= ""; $pg .= ""; @@ -129,6 +131,7 @@ function doblocks($data, $user) $pg .= ""; $pg .= ""; $pg .= ""; + $pg .= ""; $pg .= "\n"; } $pg .= "
@@ -41,11 +42,14 @@ Shifts are ~50min or less in length.
Aproximately every 30s, the pool generates new work and sends that to all the miners.
The pool also sends new work every time a block is found on the Bitcoin network.
A shift summarises all the shares submitted to the pool for 100 work changes.
-However, when we find pool blocks, the current shift ends at the work in which the block was found.
-A ckpool restart will also end the current shift and start a new one.

+However, when we find pool blocks, the current shift ends at the work in which the block was found
+and a new shift starts.
+A ckpool restart will also end the current shift and start a new shift.
+Also, a network difficulty change will end the current shift and start a new shift.
+The network difficulty changes every 2016 block or about every 2 weeks.
So, what's the $n value?

-The current Bitcoin network value for $n1d is $nv and thus $n is $nv5
+The current Bitcoin network value for $n1d is $nv and thus $n is $nvx
Bitcoin adjusts the $n1d value every 2016 blocks, which is about every 2 weeks.

When a block is found, the reward process counts back shifts until the total share difficulty included is $n.
Since shares are summarised into shifts, it will include the full shift at the end of the range counting backwards,
diff --git a/src/ckdb.c b/src/ckdb.c index c9525a63..581f1967 100644 --- a/src/ckdb.c +++ b/src/ckdb.c @@ -3154,6 +3154,12 @@ ASSERT4((sizeof(shift_words) == (sizeof(char *) * SHIFT_WORDS))); // Number of workinfoids per shift #define WID_PER_SHIFT 100 +// A diff change will end the current shift if it occurs after this block +#define SHIFT_DIFF_BLOCK 376500 + +// optioncontrol name to override the SHIFT_DIFF_BLOCK value +#define SHIFT_DIFF_BLOCK_STR "ShiftDiffBlock" + static void make_a_shift_mark() { K_TREE_CTX ss_ctx[1], m_ctx[1], wi_ctx[1], b_ctx[1]; @@ -3165,8 +3171,10 @@ static void make_a_shift_mark() BLOCKS *blocks = NULL; MARKS *marks = NULL, *sh_marks = NULL; int64_t ss_age_wid, last_marks_wid, marks_wid, prev_wid; + int64_t shiftdiffblock = SHIFT_DIFF_BLOCK; + int32_t height; char wi_bits[TXT_SML+1]; - bool was_block = false, ok; + bool was_block = false, ok, oc_look = true; char cd_buf[DATE_BUFSIZ], cd_buf2[DATE_BUFSIZ], cd_buf3[DATE_BUFSIZ]; int used_wid; @@ -3393,28 +3401,41 @@ static void make_a_shift_mark() if (wi_bits[0] == '\0') STRNCPY(wi_bits, workinfo->bits); else { + /* Make sure you set the SHIFT_DIFF_BLOCK_STR + * optioncontrol if you changed ckdb to V1.323 + * before the diff change, before the block= + * SHIFT_DIFF_BLOCK default value + * This however, would only affect a reload + * after that if the reload crossed the + * previous diff change and the shifts had not + * already been stored in the DB (i.e. next to + * zero probability) + * However a DB rollback and reload across that + * diff change would be affected */ + if (oc_look) { + shiftdiffblock = sys_setting(SHIFT_DIFF_BLOCK_STR, + SHIFT_DIFF_BLOCK, + &date_eot); + oc_look = false; + } /* Did difficulty change? * Stop at the last workinfo, before the diff - * changed - * IMPORTANT: this change will create different - * shifts if you were to reload a database, - * that you rolled back from after a diff - * change to before a diff change, if the data - * at the diff change was created with a - * version of ckdb before V1.323 - * THUS if that was the case, then some payouts - * that were generated before the rollback will - * be a small bit different to those created - * after reloading the same data */ + * changed */ if (strcmp(wi_bits, workinfo->bits) != 0) { - LOGDEBUG("%s() OK shift stops at diff" - " change '%s->%s' %"PRId64 - "->%"PRId64, - __func__, wi_bits, - workinfo->bits, prev_wid, - workinfo->workinfoid); - marks_wid = prev_wid; - break; + height = coinbase1height(workinfo->coinbase1); + if (height > (int32_t)shiftdiffblock) { + LOGDEBUG("%s() OK shift stops at diff" + " change '%s->%s' %"PRId64 + "->%"PRId64" height %"PRId32 + " limit %"PRId64, + __func__, wi_bits, + workinfo->bits, prev_wid, + workinfo->workinfoid, + height, + shiftdiffblock); + marks_wid = prev_wid; + break; + } } } /* Did we find a pool restart? i.e. a wid skip @@ -5466,7 +5487,7 @@ int main(int argc, char **argv) memset(&ckp, 0, sizeof(ckp)); ckp.loglevel = LOG_NOTICE; - while ((c = getopt_long(argc, argv, "c:d:hkl:mn:p:P:r:R:s:S:t:u:U:vw:yY:", long_options, &i)) != -1) { + while ((c = getopt_long(argc, argv, "c:d:ghkl:mn:p:P:r:R:s:S:t:u:U:vw:yY:", long_options, &i)) != -1) { switch(c) { case 'c': ckp.config = strdup(optarg); diff --git a/src/ckdb.h b/src/ckdb.h index eb454c02..cda98b56 100644 --- a/src/ckdb.h +++ b/src/ckdb.h @@ -55,7 +55,7 @@ #define DB_VLOCK "1" #define DB_VERSION "1.0.3" -#define CKDB_VERSION DB_VERSION"-1.333" +#define CKDB_VERSION DB_VERSION"-1.334" #define WHERE_FFL " - from %s %s() line %d" #define WHERE_FFL_HERE __FILE__, __func__, __LINE__ @@ -2286,8 +2286,9 @@ extern cmp_t cmp_accountbalance(K_ITEM *a, K_ITEM *b); extern K_ITEM *find_accountbalance(int64_t userid); extern cmp_t cmp_optioncontrol(K_ITEM *a, K_ITEM *b); extern K_ITEM *find_optioncontrol(char *optionname, const tv_t *now, int32_t height); +#define sys_setting(_name, _def, _now) user_sys_setting(0, _name, _def, _now) extern int64_t user_sys_setting(int64_t userid, char *setting_name, - int64_t setting_default, tv_t *now); + int64_t setting_default, const tv_t *now); extern cmp_t cmp_workinfo(K_ITEM *a, K_ITEM *b); #define coinbase1height(_cb1) _coinbase1height(_cb1, WHERE_FFL_HERE) extern int32_t _coinbase1height(char *coinbase1, WHERE_FFL_ARGS); diff --git a/src/ckdb_cmd.c b/src/ckdb_cmd.c index 4a757840..d1ec179c 100644 --- a/src/ckdb_cmd.c +++ b/src/ckdb_cmd.c @@ -1156,8 +1156,7 @@ static char *cmd_blocklist(__maybe_unused PGconn *conn, char *cmd, char *id, LOGDEBUG("%s(): cmd '%s'", __func__, cmd); - // 0 means just the system setting - maxrows = user_sys_setting(0, BLOCKS_SETTING_NAME, BLOCKS_DEFAULT, now); + maxrows = sys_setting(BLOCKS_SETTING_NAME, BLOCKS_DEFAULT, now); APPEND_REALLOC_INIT(buf, off, len); APPEND_REALLOC(buf, off, len, "ok."); diff --git a/src/ckdb_data.c b/src/ckdb_data.c index 2aa4feb4..dd32fb69 100644 --- a/src/ckdb_data.c +++ b/src/ckdb_data.c @@ -1922,7 +1922,7 @@ K_ITEM *find_optioncontrol(char *optionname, const tv_t *now, int32_t height) * i.e. ensure now and pool.height are correct (e.g. during a reload) */ int64_t user_sys_setting(int64_t userid, char *setting_name, - int64_t setting_default, tv_t *now) + int64_t setting_default, const tv_t *now) { OPTIONCONTROL *optioncontrol; K_ITEM *ua_item, *oc_item; From 4f4e30eda8421c973204b45c64654781034e0490 Mon Sep 17 00:00:00 2001 From: kanoi Date: Mon, 28 Sep 2015 21:17:56 +1000 Subject: [PATCH 29/62] sql - a couple of useful script when copying DB data for testing --- sql/rollback.sh | 195 ++++++++++++++++++++++++++++++++++++++++++++++++ sql/tabdump.sh | 63 ++++++++++++++++ 2 files changed, 258 insertions(+) create mode 100755 sql/rollback.sh create mode 100755 sql/tabdump.sh diff --git a/sql/rollback.sh b/sql/rollback.sh new file mode 100755 index 00000000..c348d7ab --- /dev/null +++ b/sql/rollback.sh @@ -0,0 +1,195 @@ +#!/bin/bash +# +# WARNING!!! +# 1) Don't use this +# 2) If you do use this, only use it if your "ckpool's ckdb logfiles" +# (CCLs) are 100% certainly OK +# Data in the DB will be lost if the rollback requires reloading +# data that is missing from the CCLs +# 3) If you do use this, make sure ckdb is NOT running +# 4) The rollback is only for the tables that are generated from the CCLs +# If you need data from other tables, see tabdump.sql +# 5) If you need to reload missing data that may or may not be in the CCLs +# then you need to convince Kano to update the ckdb -y option to handle +# that, with the current db, and also add code to the -y option to +# update the DB +# i.e. the ckdb -y option needs updating +# +# The basic use of this script is for when something goes wrong with the +# database and the CCLs are OK - so you can roll back to before the +# database problem and reload all the data +# +# Deleting markersummaries would only be an option if you know for sure +# that all data is in the CCLs +# It would be ideal to do a markersummary table dump before a rollback +# However, -e will be OK if the CCLs aren't missing anything reqired for +# the reload +# +# In all cases, if data is missing between the start and end of the +# rollback, then the reload SEQ checking will report it, unless ... +# by coincidence the missing data is at the end of a ckpool sequence +# and no gaps exist in the data before the truncation point +# e.g. seqall goes from N..M then a new SEQ starts at 0.. +# however if there actually was data after M, but all missing exactly up +# to before the SEQ restart 0 point, then ckdb couldn't know this unless +# the data was inconsistent +# +# -e will of course allow you to compare all but markersummary, before and +# after rolling back, by manually comparing the expired record with the +# new replacement unexpired record - but it deletes the markersummary data +# -e is untidy simply because it leaves around unnecesary data, if the +# reload after the rollback is not missing anything and succeeds +# -e deletes all the records that were already expired so you will lose +# the original DB history of those records, however a reload should +# regenerate a similar history +# +# -m is the same as -e except it doesn't touch markersummary at all +# -m will probably cause duplicate markersummary issues for ckdb +# +# -r ... use at your own risk :) - but is the best option if nothing is +# missing from your CCLs +# it simply deletes all reload data back to the specified point and +# that data should all reappear with a successful reload +# +usAge() +{ + echo "usAge: `basename $0` [-n] -e|-r|-m workinfoid" + echo " Preferred option is just -e as long as you aren't missing reload data" + echo + echo " -n = use 'now()' instead of the date timestamp of now, for expiry date" + echo + echo " -e = rollback by deleteing expired data and expiring unexpired data," + echo " but deleting markersummary" + echo + echo " -r = rollback by deleting all relevant data (expired or unexpired)" + echo " -m = rollback by deleteing expired data and expiring unexpired data," + echo " but don't touch markersummary" + echo " N.B. -m is expected to cause problems with ckdb" + echo + echo " generates the sql on stdout" + echo + echo " Read `basename $0` for an explanation" + exit 1 +} +# +idctl() +{ + echo "\\echo 'update idcontrol $1' +update idcontrol +set lastid=(select max($1) from $2), +modifydate=$n,modifyby='$idn', +modifycode='$mc',modifyinet='$mi' +where idname='$1';" +} +# +process() +{ + # so all stamps are exactly the same + now="`date +%s`" + idn="rollback_${wi}_`date -d "@$now" '+%Y%m%d%H%M%S%:::z'`" + mc="`basename $0`" + mi="127.0.0.1" + if [ "$usenow" ] ; then + n="now()" + else + n="'`date -u -d "@$now" '+%Y-%m-%d %H:%M:%S+00'`'" + fi + ex="expirydate = $n" + unex="expirydate > '6666-06-01'" + oldex="expirydate < '6666-06-01'" + # + if [ "$opt" = "-r" -o "$opt" = "-e" ] ; then + echo "\\echo 'delete markersummary'" + echo "delete from markersummary where markerid in" + echo " (select distinct markerid from workmarkers where workinfoidend >= $wi);" + fi + if [ "$opt" = "-r" ] ; then + echo "\\echo 'delete marks'" + echo "delete from marks where workinfoid >= $wi;" + echo "\\echo 'delete workmarkers'" + echo "delete from workmarkers where workinfoidend >= $wi;" + else + echo "\\echo 'delete/update marks'" + echo "delete from marks where $oldex and workinfoid >= $wi;" + echo "update marks set $ex where $unex and workinfoid >= $wi;" + echo "\\echo 'delete/update workmarkers'" + echo "delete from workmarkers where $oldex and workinfoidend >= $wi;" + echo "update workmarkers set $ex where $unex and workinfoidend >= $wi;" + fi + # + if [ "$opt" = "-r" ] ; then + echo "\\echo 'delete blocks'" + echo "delete from blocks where workinfoid >= $wi;" + echo "\\echo 'delete workinfo'" + echo "delete from workinfo where workinfoid >= $wi;" + else + echo "\\echo 'delete/update blocks'" + echo "delete from blocks where $oldex and workinfoid >= $wi;" + echo "update blocks set $ex where $unex and workinfoid >= $wi;" + echo "\\echo 'delete/update workinfo'" + echo "delete from workinfo where $oldex and workinfoid >= $wi;" + echo "update workinfo set $ex where $unex and workinfoid >= $wi;" + fi + # + if [ "$opt" = "-r" ] ; then + echo "\\echo 'delete miningpayouts'" + echo "delete from miningpayouts where payoutid in" + echo " (select distinct payoutid from payouts where workinfoidend >= $wi);" + echo "\\echo 'delete payments'" + echo "delete from payments where payoutid in" + echo " (select distinct payoutid from payouts where workinfoidend >= $wi);" + echo "\\echo 'delete payouts'" + echo "delete from payouts where workinfoidend >= $wi;" + else + echo "\\echo 'delete/update miningpayouts'" + echo "delete from miningpayouts where $oldex and payoutid in" + echo " (select distinct payoutid from payouts where workinfoidend >= $wi);" + echo "update miningpayouts set $ex where $unex and payoutid in" + echo " (select distinct payoutid from payouts where workinfoidend >= $wi);" + echo "\\echo 'delete/update payments'" + echo "delete from payments where $oldex and payoutid in" + echo " (select distinct payoutid from payouts where workinfoidend >= $wi);" + echo "update payments set $ex where $unex and payoutid in" + echo " (select distinct payoutid from payouts where workinfoidend >= $wi);" + echo "\\echo 'delete/update payouts'" + echo "delete from payouts where $oldex and workinfoidend >= $wi;" + echo "update payouts set $ex where workinfoidend >= $wi and $unex;" + fi + # + idctl markerid workmarkers + idctl paymentid payments + idctl payoutid payouts + # this makes sure the worker data is consistent - don't remove this + idctl workerid workers +} +# +if [ -z "$1" ] ; then + usAge +fi +# +if [ "$1" = "-?" -o "$1" = "-h" -o "$1" = "-help" -o "$1" = "--help" ] ; then + usAge +fi +# +usenow="" +if [ "$1" = "-n" ] ; then + usenow="y" + shift +fi +# +if [ "$1" != "-r" -a "$1" != "-e" -a "$1" != "-m" ] ; then + echo "ERR: Unknown p1='$1'" + usAge +fi +# +opt="$1" +shift +# +if [ -z "$1" ] ; then + echo "ERR: missing workinfoid" + usAge +fi +# +wi="$1" +# +process diff --git a/sql/tabdump.sh b/sql/tabdump.sh new file mode 100755 index 00000000..2c684740 --- /dev/null +++ b/sql/tabdump.sh @@ -0,0 +1,63 @@ +#!/bin/bash +# +# idcontrol is updated after the rows are loaded +# for users, paymentaddresses and workers +# +# note that this doesn't save idcontrol since it +# would cause a consistency problem with the reload +# Instead it generates it based on the current DB +# +# Obviously ... don't run the output file when CKDB is running +# +# initid.sh would also be useful to create any missing idcontrol +# records, before using this to update them +# +t0="optioncontrol paymentaddresses useratts users workers version" +# +usAge() +{ + echo "usAge: `basename $0` -r > log.sql" + echo " -r = do it" + echo " dump tables not part of the reload" + exit 1 +} +# +idctl() +{ + echo " +update idcontrol +set lastid=(select max($1) from $2), +modifydate=now(),modifyby='$idn', +modifycode='$mc',modifyinet='$mi' +where idname='$1';" +} +# +process() +{ + t="" + for i in $t0 ; do + t="$t -t $i" + echo "delete from $i;" + done + pg_dump -a $t ckdb + # + idn="tabdump-`date "+%Y%m%d%H%M%S"`" + mc="`basename $0`" + mi="127.0.0.1" + # + idctl userid users + idctl workerid workers + idctl paymentaddressid paymentaddresses + + # these below are just to make sure the DB is consistent + idctl markerid workmarkers + idctl paymentid payments + idctl payoutid payouts +} +# +if [ "$1" != "-r" ] ; then + echo "Missing -r" + usAge +fi +# +process From 915ce9eb775f4c34856bfc4b42735a5facd2225e Mon Sep 17 00:00:00 2001 From: kanoi Date: Tue, 29 Sep 2015 22:29:26 +1000 Subject: [PATCH 30/62] ckdb - cmd_query to return various data in ckdb --- src/ckdb.c | 4 +- src/ckdb.h | 5 +- src/ckdb_cmd.c | 485 +++++++++++++++++++++++++++++++++++++++++++++++- src/ckdb_data.c | 7 +- 4 files changed, 495 insertions(+), 6 deletions(-) diff --git a/src/ckdb.c b/src/ckdb.c index 581f1967..2be37ab4 100644 --- a/src/ckdb.c +++ b/src/ckdb.c @@ -2909,7 +2909,7 @@ static void summarise_blocks() diffacc = diffinv = shareacc = shareinv = 0; elapsed = 0; K_RLOCK(blocks_free); - b_prev = find_prev_blocks(blocks->height); + b_prev = find_prev_blocks(blocks->height, NULL); K_RUNLOCK(blocks_free); if (!b_prev) { wi_start = 0; @@ -4128,6 +4128,7 @@ static void *socketer(__maybe_unused void *arg) /* Process, but reject (loading) until startup_complete * and don't test for duplicates */ case CMD_MARKS: + case CMD_QUERY: if (!startup_complete) { snprintf(reply, sizeof(reply), "%s.%ld.loading.%s", @@ -4356,6 +4357,7 @@ static void reload_line(PGconn *conn, char *filename, uint64_t count, char *buf) case CMD_SHSTA: case CMD_USERINFO: case CMD_BTCSET: + case CMD_QUERY: LOGERR("%s() INVALID message line %"PRIu64 " ignored '%.42s...", __func__, count, diff --git a/src/ckdb.h b/src/ckdb.h index cda98b56..17bd6b70 100644 --- a/src/ckdb.h +++ b/src/ckdb.h @@ -55,7 +55,7 @@ #define DB_VLOCK "1" #define DB_VERSION "1.0.3" -#define CKDB_VERSION DB_VERSION"-1.334" +#define CKDB_VERSION DB_VERSION"-1.340" #define WHERE_FFL " - from %s %s() line %d" #define WHERE_FFL_HERE __FILE__, __func__, __LINE__ @@ -410,6 +410,7 @@ enum cmd_values { CMD_SHSTA, CMD_USERINFO, CMD_BTCSET, + CMD_QUERY, CMD_END }; @@ -2332,7 +2333,7 @@ extern double _blockhash_diff(char *hash, WHERE_FFL_ARGS); extern void dsp_blocks(K_ITEM *item, FILE *stream); extern cmp_t cmp_blocks(K_ITEM *a, K_ITEM *b); extern K_ITEM *find_blocks(int32_t height, char *blockhash, K_TREE_CTX *ctx); -extern K_ITEM *find_prev_blocks(int32_t height); +extern K_ITEM *find_prev_blocks(int32_t height, K_TREE_CTX *ctx); extern const char *blocks_confirmed(char *confirmed); extern void zero_on_new_block(); extern void set_block_share_counters(); diff --git a/src/ckdb_cmd.c b/src/ckdb_cmd.c index d1ec179c..8dccf664 100644 --- a/src/ckdb_cmd.c +++ b/src/ckdb_cmd.c @@ -5795,7 +5795,7 @@ static char *cmd_marks(PGconn *conn, char *cmd, char *id, TXT_TO_INT("height", transfer_data(i_height), height); K_RLOCK(blocks_free); - b_item = find_prev_blocks(height+1); + b_item = find_prev_blocks(height+1, NULL); K_RUNLOCK(blocks_free); if (b_item) { DATA_BLOCKS(blocks, b_item); @@ -6571,6 +6571,488 @@ static char *cmd_btcset(__maybe_unused PGconn *conn, char *cmd, char *id, return strdup(buf); } +/* Query CKDB for certain information + * See each string compare below of 'request' for the list of queries + * For non-error conditions, rows=0 means there were no matching results + * for the request, and rows=n is placed last in the reply */ +static char *cmd_query(__maybe_unused PGconn *conn, char *cmd, char *id, + __maybe_unused tv_t *now, __maybe_unused char *by, + __maybe_unused char *code, __maybe_unused char *inet, + __maybe_unused tv_t *cd, K_TREE *trf_root) +{ + K_TREE_CTX ctx[1]; + char cd_buf[DATE_BUFSIZ]; + char reply[1024] = ""; + size_t siz = sizeof(reply); + char tmp[1024] = ""; + char msg[1024] = ""; + char *buf = NULL; + size_t len, off; + K_ITEM *i_request; + char *request; + bool ok = false; + int rows = 0; + + LOGDEBUG("%s(): cmd '%s'", __func__, cmd); + + i_request = require_name(trf_root, "request", 1, NULL, reply, siz); + if (!i_request) + return strdup(reply); + request = transfer_data(i_request); + + APPEND_REALLOC_INIT(buf, off, len); + APPEND_REALLOC(buf, off, len, "ok."); + + if (strcasecmp(request, "block") == 0) { + /* return DB information for the blocks with height=value + * if expired= is present, it will also return expired records */ + K_ITEM *i_height, *i_expired, *b_item; + bool expired = false; + BLOCKS *blocks; + int32_t height; + + i_height = require_name(trf_root, "height", + 1, (char *)intpatt, + reply, siz); + if (!i_height) + return strdup(reply); + TXT_TO_INT("height", transfer_data(i_height), height); + + i_expired = optional_name(trf_root, "expired", + 0, NULL, reply, siz); + if (i_expired) + expired = true; + + int_to_buf(height, reply, sizeof(reply)); + snprintf(msg, sizeof(msg), "height=%s", reply); + + K_RLOCK(blocks_free); + b_item = find_prev_blocks(height, ctx); + DATA_BLOCKS_NULL(blocks, b_item); + while (b_item && blocks->height <= height) { + if ((expired || CURRENT(&(blocks->expirydate))) && + blocks->height == height) { + int_to_buf(blocks->height, reply, sizeof(reply)); + snprintf(tmp, sizeof(tmp), + "height:%d=%s%c", + rows, reply, FLDSEP); + APPEND_REALLOC(buf, off, len, tmp); + snprintf(tmp, sizeof(tmp), + "blockhash:%d=%s%c", + rows, blocks->blockhash, FLDSEP); + APPEND_REALLOC(buf, off, len, tmp); + snprintf(tmp, sizeof(tmp), + "confirmed:%d=%s%c", + rows, blocks->confirmed, FLDSEP); + APPEND_REALLOC(buf, off, len, tmp); + tv_to_buf(&(blocks->expirydate), cd_buf, + sizeof(cd_buf)); + snprintf(tmp, sizeof(tmp), + EDDB"_str:%d=%s%c", + rows, cd_buf, FLDSEP); + APPEND_REALLOC(buf, off, len, tmp); + tv_to_buf(&(blocks->createdate), cd_buf, + sizeof(cd_buf)); + snprintf(tmp, sizeof(tmp), + CDDB"_str:%d=%s%c", + rows, cd_buf, FLDSEP); + APPEND_REALLOC(buf, off, len, tmp); + tv_to_buf(&(blocks->blockcreatedate), cd_buf, + sizeof(cd_buf)); + snprintf(tmp, sizeof(tmp), + "block"CDDB"_str:%d=%s%c", + rows, cd_buf, FLDSEP); + APPEND_REALLOC(buf, off, len, tmp); + bigint_to_buf(blocks->workinfoid, reply, sizeof(reply)); + snprintf(tmp, sizeof(tmp), + "workinfoid:%d=%s%c", + rows, reply, FLDSEP); + APPEND_REALLOC(buf, off, len, tmp); + + rows++; + } + b_item = next_in_ktree(ctx); + DATA_BLOCKS_NULL(blocks, b_item); + } + K_RUNLOCK(blocks_free); + + snprintf(tmp, sizeof(tmp), "flds=%s%c", + "height,blockhash,confirmed,"EDDB"_str," + CDDB"_str,block"CDDB"_str,workinfoid", FLDSEP); + APPEND_REALLOC(buf, off, len, tmp); + snprintf(tmp, sizeof(tmp), "arn=%s%carp=%s%c", + "Blocks", FLDSEP, "", FLDSEP); + APPEND_REALLOC(buf, off, len, tmp); + + ok = true; + } else if (strcasecmp(request, "workinfo") == 0) { + /* return DB information for the workinfo with wid=value + * if expired= is present, it will also return expired records + * though ckdb doesn't expire workinfo records - only external + * pgsql scripts would do that to the DB, then ckdb would + * load them the next time it (re)starts */ + K_ITEM *i_wid, *i_expired, *wi_item, *wm_item; + char ndiffbin[TXT_SML+1]; + bool expired = false; + WORKINFO *workinfo; + WORKMARKERS *wm; + int64_t wid; + + i_wid = require_name(trf_root, "wid", + 1, (char *)intpatt, + reply, siz); + if (!i_wid) + return strdup(reply); + TXT_TO_BIGINT("wid", transfer_data(i_wid), wid); + + i_expired = optional_name(trf_root, "expired", + 0, NULL, reply, siz); + if (i_expired) + expired = true; + + bigint_to_buf(wid, reply, sizeof(reply)); + snprintf(msg, sizeof(msg), "wid=%s", reply); + + /* We look for the 'next' (or last) workinfo then go backwards + * to ensure we find all expired records in case they + * were requested */ + K_RLOCK(workinfo_free); + wi_item = next_workinfo(wid, ctx); + if (!wi_item) + wi_item = last_in_ktree(workinfo_root, ctx); + DATA_WORKINFO_NULL(workinfo, wi_item); + while (wi_item && workinfo->workinfoid >= wid) { + if ((expired || CURRENT(&(workinfo->expirydate))) && + workinfo->workinfoid == wid) { + bigint_to_buf(workinfo->workinfoid, + reply, sizeof(reply)); + snprintf(tmp, sizeof(tmp), + "workinfoid:%d=%s%c", + rows, reply, FLDSEP); + APPEND_REALLOC(buf, off, len, tmp); + int_to_buf(coinbase1height(workinfo->coinbase1), + reply, sizeof(reply)); + snprintf(tmp, sizeof(tmp), + "height:%d=%s%c", + rows, reply, FLDSEP); + APPEND_REALLOC(buf, off, len, tmp); + snprintf(tmp, sizeof(tmp), + "prevhash:%d=%s%c", + rows, workinfo->prevhash, FLDSEP); + APPEND_REALLOC(buf, off, len, tmp); + tv_to_buf(&(workinfo->expirydate), cd_buf, + sizeof(cd_buf)); + snprintf(tmp, sizeof(tmp), + EDDB"_str:%d=%s%c", + rows, cd_buf, FLDSEP); + APPEND_REALLOC(buf, off, len, tmp); + tv_to_buf(&(workinfo->createdate), cd_buf, + sizeof(cd_buf)); + snprintf(tmp, sizeof(tmp), + CDDB"_str:%d=%s%c", + rows, cd_buf, FLDSEP); + APPEND_REALLOC(buf, off, len, tmp); + hex2bin(ndiffbin, workinfo->bits, 4); + snprintf(tmp, sizeof(tmp), + "ndiff:%d=%.1f%c", rows, + diff_from_nbits(ndiffbin), + FLDSEP); + APPEND_REALLOC(buf, off, len, tmp); + snprintf(tmp, sizeof(tmp), + "ppsvalue:%d=%.15f%c", rows, + workinfo_pps(wi_item, + workinfo->workinfoid, + true), + FLDSEP); + APPEND_REALLOC(buf, off, len, tmp); + K_RLOCK(workmarkers_free); + wm_item = find_workmarkers(wid, false, + MARKER_PROCESSED, + NULL); + K_RUNLOCK(workmarkers_free); + if (!wm_item) { + snprintf(tmp, sizeof(tmp), + "markerid:%d=%c", rows, FLDSEP); + APPEND_REALLOC(buf, off, len, tmp); + snprintf(tmp, sizeof(tmp), + "shift:%d=%c", rows, FLDSEP); + APPEND_REALLOC(buf, off, len, tmp); + snprintf(tmp, sizeof(tmp), + "shiftend:%d=%c", rows, FLDSEP); + APPEND_REALLOC(buf, off, len, tmp); + snprintf(tmp, sizeof(tmp), + "shiftstart:%d=%c", rows, FLDSEP); + APPEND_REALLOC(buf, off, len, tmp); + } else { + DATA_WORKMARKERS(wm, wm_item); + bigint_to_buf(wm->markerid, + reply, sizeof(reply)); + snprintf(tmp, sizeof(tmp), + "markerid:%d=%s%c", + rows, reply, FLDSEP); + APPEND_REALLOC(buf, off, len, tmp); + snprintf(tmp, sizeof(tmp), + "shift:%d=%s%c", + rows, wm->description, FLDSEP); + APPEND_REALLOC(buf, off, len, tmp); + bigint_to_buf(wm->workinfoidend, + reply, sizeof(reply)); + snprintf(tmp, sizeof(tmp), + "shiftend:%d=%s%c", + rows, reply, FLDSEP); + APPEND_REALLOC(buf, off, len, tmp); + bigint_to_buf(wm->workinfoidstart, + reply, sizeof(reply)); + snprintf(tmp, sizeof(tmp), + "shiftstart:%d=%s%c", + rows, reply, FLDSEP); + APPEND_REALLOC(buf, off, len, tmp); + } + rows++; + } + wi_item = prev_in_ktree(ctx); + DATA_WORKINFO_NULL(workinfo, wi_item); + } + K_RUNLOCK(workinfo_free); + + snprintf(tmp, sizeof(tmp), "flds=%s%c", + "workinfoid,height,prevhash,"EDDB"_str,"CDDB"_str," + "ndiff,ppsvalue,markerid,shift,shiftend,shiftstart", + FLDSEP); + APPEND_REALLOC(buf, off, len, tmp); + snprintf(tmp, sizeof(tmp), "arn=%s%carp=%s%c", + "Workinfo", FLDSEP, "", FLDSEP); + APPEND_REALLOC(buf, off, len, tmp); + + ok = true; + } else if (strcasecmp(request, "range") == 0) { + /* Return the workinfoid range that has block height=height + * WARNING! This will traverse workinfo from the end back to + * the given height, and thus since workinfo is the 2nd + * largest tree, it may access swapped data if you request + * older data */ + K_ITEM *i_height, *wi_item; + WORKINFO *workinfo; + int32_t height, this_height; + int64_t idend, idstt; + + i_height = require_name(trf_root, "height", + 1, (char *)intpatt, + reply, siz); + if (!i_height) + return strdup(reply); + TXT_TO_INT("height", transfer_data(i_height), height); + + int_to_buf(height, reply, sizeof(reply)); + snprintf(msg, sizeof(msg), "height=%s", reply); + + idend = idstt = 0L; + /* Start from the last workinfo and continue until we get + * below block 'height' */ + K_RLOCK(workinfo_free); + wi_item = last_in_ktree(workinfo_root, ctx); + DATA_WORKINFO_NULL(workinfo, wi_item); + while (wi_item) { + this_height = coinbase1height(workinfo->coinbase1); + if (this_height < height) + break; + if (CURRENT(&(workinfo->expirydate)) && + this_height == height) { + if (idend == 0L) + idend = workinfo->workinfoid; + idstt = workinfo->workinfoid; + } + wi_item = prev_in_ktree(ctx); + DATA_WORKINFO_NULL(workinfo, wi_item); + } + K_RUNLOCK(workinfo_free); + + int_to_buf(height, reply, sizeof(reply)); + snprintf(tmp, sizeof(tmp), "height:%d=%s%c", + rows, reply, FLDSEP); + APPEND_REALLOC(buf, off, len, tmp); + bigint_to_buf(idend, reply, sizeof(reply)); + snprintf(tmp, sizeof(tmp), "workinfoidend:%d=%s%c", + rows, reply, FLDSEP); + APPEND_REALLOC(buf, off, len, tmp); + bigint_to_buf(idstt, reply, sizeof(reply)); + snprintf(tmp, sizeof(tmp), "workinfoidstart:%d=%s%c", + rows, reply, FLDSEP); + APPEND_REALLOC(buf, off, len, tmp); + + rows++; + + snprintf(tmp, sizeof(tmp), "flds=%s%c", + "height,workinfoidend,workinfoidstart", FLDSEP); + APPEND_REALLOC(buf, off, len, tmp); + snprintf(tmp, sizeof(tmp), "arn=%s%carp=%s%c", + "WorkinfoRange", FLDSEP, "", FLDSEP); + APPEND_REALLOC(buf, off, len, tmp); + + ok = true; + } else if (strcasecmp(request, "diff") == 0) { + /* return the details of the next diff change after + * block height=height + * WARNING! This will traverse workinfo from the end back to + * the given height, and thus since workinfo is the 2nd + * largest tree, it may access swapped data if you request + * older data */ + K_ITEM *i_height, *wi_item; + WORKINFO *workinfo = NULL; + int32_t height, this_height; + char ndiffbin[TXT_SML+1]; + char bits[TXT_SML+1]; + bool got = false; + + i_height = require_name(trf_root, "height", + 1, (char *)intpatt, + reply, siz); + if (!i_height) + return strdup(reply); + TXT_TO_INT("height", transfer_data(i_height), height); + + int_to_buf(height, reply, sizeof(reply)); + snprintf(msg, sizeof(msg), "height=%s", reply); + + snprintf(tmp, sizeof(tmp), "height0:%d=%s%c", + rows, reply, FLDSEP); + APPEND_REALLOC(buf, off, len, tmp); + + /* Start from the last workinfo and continue until we get + * below block 'height' */ + K_RLOCK(workinfo_free); + wi_item = last_in_ktree(workinfo_root, ctx); + DATA_WORKINFO_NULL(workinfo, wi_item); + while (wi_item) { + if (CURRENT(&(workinfo->expirydate))) { + this_height = coinbase1height(workinfo->coinbase1); + if (this_height < height) + break; + } + wi_item = prev_in_ktree(ctx); + DATA_WORKINFO_NULL(workinfo, wi_item); + } + // If we fell off the front use the first one + if (!wi_item) + wi_item = first_in_ktree(workinfo_root, ctx); + DATA_WORKINFO_NULL(workinfo, wi_item); + while (wi_item) { + if (CURRENT(&(workinfo->expirydate))) { + this_height = coinbase1height(workinfo->coinbase1); + if (this_height >= height) + break; + } + wi_item = next_in_ktree(ctx); + DATA_WORKINFO_NULL(workinfo, wi_item); + } + if (wi_item) { + DATA_WORKINFO(workinfo, wi_item); + this_height = coinbase1height(workinfo->coinbase1); + if (this_height == height) { + // We have our starting point + STRNCPY(bits, workinfo->bits); + got = true; + + bigint_to_buf(workinfo->workinfoid, + reply, sizeof(reply)); + snprintf(tmp, sizeof(tmp), + "workinfoid0:%d=%s%c", + rows, reply, FLDSEP); + APPEND_REALLOC(buf, off, len, tmp); + hex2bin(ndiffbin, workinfo->bits, 4); + snprintf(tmp, sizeof(tmp), + "ndiff0:%d=%.1f%c", rows, + diff_from_nbits(ndiffbin), + FLDSEP); + APPEND_REALLOC(buf, off, len, tmp); + + while (wi_item) { + if (CURRENT(&(workinfo->expirydate))) { + if (strcmp(bits, workinfo->bits) != 0) + break; + } + wi_item = next_in_ktree(ctx); + DATA_WORKINFO_NULL(workinfo, wi_item); + } + } else + wi_item = NULL; + } + K_RUNLOCK(workinfo_free); + + if (!got) { + snprintf(tmp, sizeof(tmp), "workinfoid0:%d=%c", + rows, FLDSEP); + APPEND_REALLOC(buf, off, len, tmp); + snprintf(tmp, sizeof(tmp), "ndiff0:%d=%c", + rows, FLDSEP); + APPEND_REALLOC(buf, off, len, tmp); + } + + if (!wi_item) { + snprintf(tmp, sizeof(tmp), "height:%d=%c", + rows, FLDSEP); + APPEND_REALLOC(buf, off, len, tmp); + snprintf(tmp, sizeof(tmp), "workinfoid:%d=%c", + rows, FLDSEP); + APPEND_REALLOC(buf, off, len, tmp); + snprintf(tmp, sizeof(tmp), "ndiff:%d=%c", + rows, FLDSEP); + APPEND_REALLOC(buf, off, len, tmp); + } else { + this_height = coinbase1height(workinfo->coinbase1); + int_to_buf(this_height, reply, sizeof(reply)); + snprintf(tmp, sizeof(tmp), "height:%d=%s%c", + rows, reply, FLDSEP); + APPEND_REALLOC(buf, off, len, tmp); + bigint_to_buf(workinfo->workinfoid, + reply, sizeof(reply)); + snprintf(tmp, sizeof(tmp), "workinfoid:%d=%s%c", + rows, reply, FLDSEP); + APPEND_REALLOC(buf, off, len, tmp); + hex2bin(ndiffbin, workinfo->bits, 4); + snprintf(tmp, sizeof(tmp), "ndiff:%d=%.1f%c", + rows, + diff_from_nbits(ndiffbin), + FLDSEP); + APPEND_REALLOC(buf, off, len, tmp); + } + + rows++; + + snprintf(tmp, sizeof(tmp), "flds=%s%c", + "height0,workinfoid0,ndiff0,height,workinfo,ndiff", + FLDSEP); + APPEND_REALLOC(buf, off, len, tmp); + snprintf(tmp, sizeof(tmp), "arn=%s%carp=%s%c", + "WorkinfoRange", FLDSEP, "", FLDSEP); + APPEND_REALLOC(buf, off, len, tmp); + + ok = true; + } else { + free(buf); + snprintf(reply, siz, "unknown request '%s'", request); + LOGERR("%s() %s.%s", __func__, id, reply); + return strdup(reply); + } + + if (!ok) { + free(buf); + snprintf(reply, siz, "failed.%s%s%s", + request, + msg[0] ? " " : "", + msg[0] ? msg : ""); + LOGERR("%s() %s.%s", __func__, id, reply); + return strdup(reply); + } + + snprintf(tmp, sizeof(tmp), "rows=%d", rows); + APPEND_REALLOC(buf, off, len, tmp); + LOGWARNING("%s() %s.%s%s%s", __func__, id, request, + msg[0] ? " " : "", + msg[0] ? msg : ""); + return buf; +} + /* The socket command format is as follows: * Basic structure: * cmd.ID.fld1=value1 FLDSEP fld2=value2 FLDSEP fld3=... @@ -6679,5 +7161,6 @@ struct CMDS ckdb_cmds[] = { { CMD_SHSTA, "shsta", true, false, cmd_shsta, SEQ_NONE, ACCESS_SYSTEM }, { CMD_USERINFO, "userinfo", false, false, cmd_userinfo, SEQ_NONE, ACCESS_WEB }, { CMD_BTCSET, "btcset", false, false, cmd_btcset, SEQ_NONE, ACCESS_SYSTEM }, + { CMD_QUERY, "query", false, false, cmd_query, SEQ_NONE, ACCESS_SYSTEM }, { CMD_END, NULL, false, false, NULL, SEQ_NONE, 0 } }; diff --git a/src/ckdb_data.c b/src/ckdb_data.c index dd32fb69..eb2572c5 100644 --- a/src/ckdb_data.c +++ b/src/ckdb_data.c @@ -2702,12 +2702,15 @@ K_ITEM *find_blocks(int32_t height, char *blockhash, K_TREE_CTX *ctx) } // Must be R or W locked before call -K_ITEM *find_prev_blocks(int32_t height) +K_ITEM *find_prev_blocks(int32_t height, K_TREE_CTX *ctx) { BLOCKS lookblocks, *blocks; - K_TREE_CTX ctx[1]; + K_TREE_CTX ctx0[1]; K_ITEM look, *b_item; + if (ctx == NULL) + ctx = ctx0; + /* TODO: For self orphaned (if that ever happens) * this will find based on blockhash order if it has two, * not NEW, blocks, which might not find the right one */ From 8577384fed3979471d49278eca25cf2aeef35de1 Mon Sep 17 00:00:00 2001 From: kanoi Date: Wed, 30 Sep 2015 00:20:24 +1000 Subject: [PATCH 31/62] ckdb - during a reload, do all blocks and ignore errors based on the block number --- src/ckdb.c | 3 ++- src/ckdb.h | 5 +++-- src/ckdb_btc.c | 17 ++++++----------- src/ckdb_cmd.c | 45 ++++++++++++++++++++++++++------------------- src/ckdb_dbio.c | 17 ++++++++++------- 5 files changed, 47 insertions(+), 40 deletions(-) diff --git a/src/ckdb.c b/src/ckdb.c index 2be37ab4..78045855 100644 --- a/src/ckdb.c +++ b/src/ckdb.c @@ -826,7 +826,8 @@ static bool reload() tv_to_buf(&(dbstatus.newest_createdate_poolstats), buf, sizeof(buf)); LOGWARNING("%s(): %s newest DB poolstats (ignored)", __func__, buf); tv_to_buf(&(dbstatus.newest_createdate_blocks), buf, sizeof(buf)); - LOGWARNING("%s(): %s newest DB blocks (ignored)", __func__, buf); + LOGWARNING("%s(): %"PRId32"/%s newest DB blocks (ignored)", + __func__, dbstatus.newest_height_blocks, buf); copy_tv(&start, &(dbstatus.newest_createdate_workmarker_workinfo)); reason = "workmarkers"; diff --git a/src/ckdb.h b/src/ckdb.h index 17bd6b70..403acf0c 100644 --- a/src/ckdb.h +++ b/src/ckdb.h @@ -55,7 +55,7 @@ #define DB_VLOCK "1" #define DB_VERSION "1.0.3" -#define CKDB_VERSION DB_VERSION"-1.340" +#define CKDB_VERSION DB_VERSION"-1.350" #define WHERE_FFL " - from %s %s() line %d" #define WHERE_FFL_HERE __FILE__, __func__, __LINE__ @@ -138,6 +138,7 @@ typedef struct loadstatus { tv_t newest_createdate_workinfo; tv_t newest_createdate_poolstats; tv_t newest_createdate_blocks; + int32_t newest_height_blocks; } LOADSTATUS; extern LOADSTATUS dbstatus; @@ -2545,7 +2546,7 @@ extern bool blocks_stats(PGconn *conn, int32_t height, char *blockhash, double diffacc, double diffinv, double shareacc, double shareinv, int64_t elapsed, char *by, char *code, char *inet, tv_t *cd); -extern bool blocks_add(PGconn *conn, char *height, char *blockhash, +extern bool blocks_add(PGconn *conn, int32_t height, char *blockhash, char *confirmed, char *info, char *workinfoid, char *username, char *workername, char *clientid, char *enonce1, char *nonce2, char *nonce, char *reward, diff --git a/src/ckdb_btc.c b/src/ckdb_btc.c index 93759bc2..1e258800 100644 --- a/src/ckdb_btc.c +++ b/src/ckdb_btc.c @@ -308,7 +308,6 @@ bool btc_valid_address(char *addr) void btc_blockstatus(BLOCKS *blocks) { char hash[TXT_BIG+1]; - char height_str[32]; char *blockhash; int32_t confirms; size_t len; @@ -348,14 +347,12 @@ void btc_blockstatus(BLOCKS *blocks) return; if (strcmp(blockhash, hash) != 0) { - snprintf(height_str, sizeof(height_str), "%d", blocks->height); - LOGERR("%s() flagging block %d(%s) as %s pool=%s btc=%s", - __func__, - blocks->height, height_str, + LOGERR("%s() flagging block %d as %s pool=%s btc=%s", + __func__, blocks->height, blocks_confirmed(BLOCKS_ORPHAN_STR), hash, blockhash); - ok = blocks_add(NULL, height_str, + ok = blocks_add(NULL, blocks->height, blocks->blockhash, BLOCKS_ORPHAN_STR, EMPTY, EMPTY, EMPTY, EMPTY, EMPTY, @@ -371,14 +368,12 @@ void btc_blockstatus(BLOCKS *blocks) confirms = btc_confirms(hash); if (confirms >= BLOCKS_42_VALUE) { - snprintf(height_str, sizeof(height_str), "%d", blocks->height); - LOGERR("%s() flagging block %d(%s) as %s confirms=%d(%d)", - __func__, - blocks->height, height_str, + LOGERR("%s() flagging block %d as %s confirms=%d(%d)", + __func__, blocks->height, blocks_confirmed(BLOCKS_42_STR), confirms, BLOCKS_42_VALUE); - ok = blocks_add(NULL, height_str, + ok = blocks_add(NULL, blocks->height, blocks->blockhash, BLOCKS_42_STR, EMPTY, EMPTY, EMPTY, EMPTY, EMPTY, diff --git a/src/ckdb_cmd.c b/src/ckdb_cmd.c index 8dccf664..2f8b396f 100644 --- a/src/ckdb_cmd.c +++ b/src/ckdb_cmd.c @@ -1387,8 +1387,8 @@ redo: return buf; } -static char *cmd_blockstatus(__maybe_unused PGconn *conn, char *cmd, char *id, - tv_t *now, char *by, char *code, char *inet, +static char *cmd_blockstatus(PGconn *conn, char *cmd, char *id, tv_t *now, + char *by, char *code, char *inet, __maybe_unused tv_t *cd, K_TREE *trf_root) { K_ITEM *i_height, *i_blockhash, *i_action, *i_info; @@ -1437,7 +1437,7 @@ static char *cmd_blockstatus(__maybe_unused PGconn *conn, char *cmd, char *id, switch (blocks->confirmed[0]) { case BLOCKS_NEW: case BLOCKS_CONFIRM: - ok = blocks_add(conn, transfer_data(i_height), + ok = blocks_add(conn, height, blocks->blockhash, BLOCKS_ORPHAN_STR, info, EMPTY, EMPTY, EMPTY, EMPTY, @@ -1478,7 +1478,7 @@ static char *cmd_blockstatus(__maybe_unused PGconn *conn, char *cmd, char *id, case BLOCKS_CONFIRM: case BLOCKS_ORPHAN: case BLOCKS_REJECT: - ok = blocks_add(conn, transfer_data(i_height), + ok = blocks_add(conn, height, blocks->blockhash, BLOCKS_REJECT_STR, info, EMPTY, EMPTY, EMPTY, EMPTY, @@ -1507,7 +1507,7 @@ static char *cmd_blockstatus(__maybe_unused PGconn *conn, char *cmd, char *id, // Confirm a new block that wasn't confirmed due to some bug switch (blocks->confirmed[0]) { case BLOCKS_NEW: - ok = blocks_add(conn, transfer_data(i_height), + ok = blocks_add(conn, height, blocks->blockhash, BLOCKS_CONFIRM_STR, info, EMPTY, EMPTY, EMPTY, EMPTY, @@ -2644,13 +2644,13 @@ awconf: } // TODO: the confirm update: identify block changes from workinfo height? -static char *cmd_blocks_do(PGconn *conn, char *cmd, char *id, char *by, - char *code, char *inet, tv_t *cd, bool igndup, - K_TREE *trf_root) +static char *cmd_blocks_do(PGconn *conn, char *cmd, int32_t height, char *id, + char *by, char *code, char *inet, tv_t *cd, + bool igndup, K_TREE *trf_root) { char reply[1024] = ""; size_t siz = sizeof(reply); - K_ITEM *i_height, *i_blockhash, *i_confirmed, *i_workinfoid, *i_username; + K_ITEM *i_blockhash, *i_confirmed, *i_workinfoid, *i_username; K_ITEM *i_workername, *i_clientid, *i_enonce1, *i_nonce2, *i_nonce, *i_reward; TRANSFER *transfer; char *msg; @@ -2658,10 +2658,6 @@ static char *cmd_blocks_do(PGconn *conn, char *cmd, char *id, char *by, LOGDEBUG("%s(): cmd '%s'", __func__, cmd); - i_height = require_name(trf_root, "height", 1, NULL, reply, siz); - if (!i_height) - return strdup(reply); - i_blockhash = require_name(trf_root, "blockhash", 1, NULL, reply, siz); if (!i_blockhash) return strdup(reply); @@ -2707,7 +2703,7 @@ static char *cmd_blocks_do(PGconn *conn, char *cmd, char *id, char *by, return strdup(reply); msg = "added"; - ok = blocks_add(conn, transfer_data(i_height), + ok = blocks_add(conn, height, transfer_data(i_blockhash), transfer_data(i_confirmed), EMPTY, @@ -2724,7 +2720,7 @@ static char *cmd_blocks_do(PGconn *conn, char *cmd, char *id, char *by, break; case BLOCKS_CONFIRM: msg = "confirmed"; - ok = blocks_add(conn, transfer_data(i_height), + ok = blocks_add(conn, height, transfer_data(i_blockhash), transfer_data(i_confirmed), EMPTY, @@ -2757,17 +2753,28 @@ static char *cmd_blocks(PGconn *conn, char *cmd, char *id, char *code, char *inet, tv_t *cd, K_TREE *trf_root) { + char reply[1024] = ""; + size_t siz = sizeof(reply); bool igndup = false; + K_ITEM *i_height; + int32_t height; + + LOGDEBUG("%s(): cmd '%s'", __func__, cmd); + + i_height = require_name(trf_root, "height", 1, NULL, reply, siz); + if (!i_height) + return strdup(reply); + + TXT_TO_INT("height", transfer_data(i_height), height); // confirm_summaries() doesn't call this if (reloading) { - if (tv_equal(cd, &(dbstatus.newest_createdate_blocks))) + // Since they're blocks, just try them all + if (height <= dbstatus.newest_height_blocks) igndup = true; - else if (tv_newer(cd, &(dbstatus.newest_createdate_blocks))) - return NULL; } - return cmd_blocks_do(conn, cmd, id, by, code, inet, cd, igndup, trf_root); + return cmd_blocks_do(conn, cmd, height, id, by, code, inet, cd, igndup, trf_root); } static char *cmd_auth_do(PGconn *conn, char *cmd, char *id, char *by, diff --git a/src/ckdb_dbio.c b/src/ckdb_dbio.c index fd790a13..fd839ad7 100644 --- a/src/ckdb_dbio.c +++ b/src/ckdb_dbio.c @@ -4434,7 +4434,7 @@ unparam: return ok; } -bool blocks_add(PGconn *conn, char *height, char *blockhash, +bool blocks_add(PGconn *conn, int32_t height, char *blockhash, char *confirmed, char *info, char *workinfoid, char *username, char *workername, char *clientid, char *enonce1, char *nonce2, char *nonce, char *reward, @@ -4465,7 +4465,7 @@ bool blocks_add(PGconn *conn, char *height, char *blockhash, DATA_BLOCKS(row, b_item); bzero(row, sizeof(*row)); - TXT_TO_INT("height", height, row->height); + row->height = height; STRNCPY(row->blockhash, blockhash); dsp_hash(blockhash, hash_dsp, sizeof(hash_dsp)); @@ -4486,7 +4486,7 @@ bool blocks_add(PGconn *conn, char *height, char *blockhash, if (!igndup) { tv_to_buf(cd, cd_buf, sizeof(cd_buf)); LOGERR("%s(): Duplicate (%s) blocks ignored, Status: " - "%s, Block: %s/...%s/%s", + "%s, Block: %"PRId32"/...%s/%s", __func__, blocks_confirmed(oldblocks->confirmed), blocks_confirmed(confirmed), @@ -4578,7 +4578,7 @@ bool blocks_add(PGconn *conn, char *height, char *blockhash, if (!startup_complete) { tv_to_buf(cd, cd_buf, sizeof(cd_buf)); LOGERR("%s(): Status: %s invalid during startup. " - "Ignored: Block: %s/...%s/%s", + "Ignored: Block: %"PRId32"/...%s/%s", __func__, blocks_confirmed(confirmed), height, hash_dsp, cd_buf); @@ -4587,7 +4587,7 @@ bool blocks_add(PGconn *conn, char *height, char *blockhash, case BLOCKS_CONFIRM: if (!old_b_item) { tv_to_buf(cd, cd_buf, sizeof(cd_buf)); - LOGERR("%s(): Can't %s a non-existent Block: %s/...%s/%s", + LOGERR("%s(): Can't %s a non-existent Block: %"PRId32"/...%s/%s", __func__, blocks_confirmed(confirmed), height, hash_dsp, cd_buf); goto flail; @@ -4619,7 +4619,7 @@ bool blocks_add(PGconn *conn, char *height, char *blockhash, if (startup_complete) { tv_to_buf(cd, cd_buf, sizeof(cd_buf)); LOGERR("%s(): New Status: (%s)%s requires Status: %s. " - "Ignored: Status: (%s)%s, Block: %s/...%s/%s", + "Ignored: Status: (%s)%s, Block: %"PRId32"/...%s/%s", __func__, confirmed, blocks_confirmed(confirmed), want, oldblocks->confirmed, @@ -4870,7 +4870,7 @@ flail: break; } - LOGWARNING("%s(): %sStatus: %s, Block: %s/...%s Diff %s%s", + LOGWARNING("%s(): %sStatus: %s, Block: %"PRId32"/...%s Diff %s%s", __func__, blk ? "BLOCK! " : "", blocks_confirmed(confirmed), height, hash_dsp, diff, tmp); @@ -5030,6 +5030,9 @@ bool blocks_fill(PGconn *conn) if (tv_newer(&(dbstatus.newest_createdate_blocks), &(row->createdate))) copy_tv(&(dbstatus.newest_createdate_blocks), &(row->createdate)); + if (dbstatus.newest_height_blocks < row->height) + dbstatus.newest_height_blocks = row->height; + if (pool.workinfoid < row->workinfoid) { pool.workinfoid = row->workinfoid; pool.height = row->height; From e65ddb3b26457c0b4e467c60202702b05183b293 Mon Sep 17 00:00:00 2001 From: kanoi Date: Wed, 30 Sep 2015 10:48:01 +1000 Subject: [PATCH 32/62] sql - update rollback.sh to include a count report option and a blocks table option --- sql/rollback.sh | 115 ++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 97 insertions(+), 18 deletions(-) diff --git a/sql/rollback.sh b/sql/rollback.sh index c348d7ab..41df058d 100755 --- a/sql/rollback.sh +++ b/sql/rollback.sh @@ -13,12 +13,24 @@ # then you need to convince Kano to update the ckdb -y option to handle # that, with the current db, and also add code to the -y option to # update the DB -# i.e. the ckdb -y option needs updating +# i.e. the ckdb -y options/code need updating +# 6) Blocks commands manually entered using ckpmsg (confirm,orphan,reject,etc) +# will not be automatically restored if you use -b, you'll need to redo +# them again manually # # The basic use of this script is for when something goes wrong with the # database and the CCLs are OK - so you can roll back to before the # database problem and reload all the data # +# It may be unnecessary to delete/expire blocks records since the reload will +# attempt to load all old and new blocks records in the CCLs being +# reloaded, no matter what +# Also, deleting blocks history will lose all manual block changes +# done with confirms, orphans, rejects, etc +# However, if the block summarisations are wrong or the blocks table is +# corrupt, you will need to delete/expire them to correct them and then +# redo the manual blocks changes after reloading +# # Deleting markersummaries would only be an option if you know for sure # that all data is in the CCLs # It would be ideal to do a markersummary table dump before a rollback @@ -34,6 +46,8 @@ # to before the SEQ restart 0 point, then ckdb couldn't know this unless # the data was inconsistent # +# -c will show you record counts but not change anything +# # -e will of course allow you to compare all but markersummary, before and # after rolling back, by manually comparing the expired record with the # new replacement unexpired record - but it deletes the markersummary data @@ -53,11 +67,16 @@ # usAge() { - echo "usAge: `basename $0` [-n] -e|-r|-m workinfoid" + echo "usAge: `basename $0` [-b] [-n] -c|-e|-r|-m workinfoid" echo " Preferred option is just -e as long as you aren't missing reload data" + echo " The order of options must match the usAge order" + echo + echo " -b = include blocks when deleting/expiring (otherwise left untouched)" echo echo " -n = use 'now()' instead of the date timestamp of now, for expiry date" echo + echo " -c = report the counts of rows that would be affected, but don't do it" + echo echo " -e = rollback by deleteing expired data and expiring unexpired data," echo " but deleting markersummary" echo @@ -66,9 +85,9 @@ usAge() echo " but don't touch markersummary" echo " N.B. -m is expected to cause problems with ckdb" echo - echo " generates the sql on stdout" + echo " generates the sql on stdout, that you would feed into pgsql" echo - echo " Read `basename $0` for an explanation" + echo " Read `basename $0` for a more detailed explanation" exit 1 } # @@ -98,17 +117,37 @@ process() unex="expirydate > '6666-06-01'" oldex="expirydate < '6666-06-01'" # + if [ "$opt" = "-c" ] ; then + echo "\\echo 'count markersummary'" + echo "select count(*),min(markerid) as min_markerid,max(markerid) as max_markerid," + echo " sum(diffacc) as sum_diffacc,min(firstshareacc) as min_firstshareacc," + echo " max(lastshareacc) as max_lastshareacc" + echo " from markersummary where markerid in" + echo " (select distinct markerid from workmarkers where workinfoidend >= $wi);" + fi if [ "$opt" = "-r" -o "$opt" = "-e" ] ; then echo "\\echo 'delete markersummary'" echo "delete from markersummary where markerid in" echo " (select distinct markerid from workmarkers where workinfoidend >= $wi);" fi + # + if [ "$opt" = "-c" ] ; then + echo "\\echo 'count marks'" + echo "select count(*),min(workinfoid) as min_workinfoid," + echo " max(workinfoid) as max_workinfoid" + echo " from marks where workinfoid >= $wi;" + echo "\\echo 'count workmarkers'" + echo "select count(*),min(workinfoidstart) as min_workinfoidstart," + echo " max(workinfoidend) as max_workinfoidend" + echo " from workmarkers where workinfoidend >= $wi;" + fi if [ "$opt" = "-r" ] ; then echo "\\echo 'delete marks'" echo "delete from marks where workinfoid >= $wi;" echo "\\echo 'delete workmarkers'" echo "delete from workmarkers where workinfoidend >= $wi;" - else + fi + if [ "$opt" = "-e" -o "$opt" = "-m" ] ; then echo "\\echo 'delete/update marks'" echo "delete from marks where $oldex and workinfoid >= $wi;" echo "update marks set $ex where $unex and workinfoid >= $wi;" @@ -117,20 +156,51 @@ process() echo "update workmarkers set $ex where $unex and workinfoidend >= $wi;" fi # + if [ "$opt" = "-c" ] ; then + echo "\\echo 'count blocks'" + echo "select count(*),min(workinfoid) as min_workinfoid," + echo " max(workinfoid) as max_workinfoid" + echo " from blocks where workinfoid >= $wi;" + echo "\\echo 'count workinfo'" + echo "select count(*),min(workinfoid) as min_workinfoid," + echo " max(workinfoid) as max_workinfoid" + echo " from workinfo where workinfoid >= $wi;" + fi if [ "$opt" = "-r" ] ; then - echo "\\echo 'delete blocks'" - echo "delete from blocks where workinfoid >= $wi;" + if [ "$deblk" ] ; then + echo "\\echo 'delete blocks'" + echo "delete from blocks where workinfoid >= $wi;" + fi echo "\\echo 'delete workinfo'" echo "delete from workinfo where workinfoid >= $wi;" - else - echo "\\echo 'delete/update blocks'" - echo "delete from blocks where $oldex and workinfoid >= $wi;" - echo "update blocks set $ex where $unex and workinfoid >= $wi;" + fi + if [ "$opt" = "-e" -o "$opt" = "-m" ] ; then + if [ "$deblk" ] ; then + echo "\\echo 'delete/update blocks'" + echo "delete from blocks where $oldex and workinfoid >= $wi;" + echo "update blocks set $ex where $unex and workinfoid >= $wi;" + fi echo "\\echo 'delete/update workinfo'" echo "delete from workinfo where $oldex and workinfoid >= $wi;" echo "update workinfo set $ex where $unex and workinfoid >= $wi;" fi # + if [ "$opt" = "-c" ] ; then + echo "\\echo 'count miningpayouts'" + echo "select count(*),min(payoutid) as min_payoutid," + echo " max(payoutid) as max_payoutid" + echo " from miningpayouts where payoutid in" + echo " (select distinct payoutid from payouts where workinfoidend >= $wi);" + echo "\\echo 'count payments'" + echo "select count(*),min(payoutid) as min_payoutid," + echo " max(payoutid) as max_payoutid" + echo " from payments where payoutid in" + echo " (select distinct payoutid from payouts where workinfoidend >= $wi);" + echo "\\echo 'count payouts'" + echo "select count(*),min(workinfoidstart) as min_workinfoidstart," + echo " max(workinfoidend) as max_workinfoidend" + echo " from payouts where workinfoidend >= $wi;" + fi if [ "$opt" = "-r" ] ; then echo "\\echo 'delete miningpayouts'" echo "delete from miningpayouts where payoutid in" @@ -140,7 +210,8 @@ process() echo " (select distinct payoutid from payouts where workinfoidend >= $wi);" echo "\\echo 'delete payouts'" echo "delete from payouts where workinfoidend >= $wi;" - else + fi + if [ "$opt" = "-e" -o "$opt" = "-m" ] ; then echo "\\echo 'delete/update miningpayouts'" echo "delete from miningpayouts where $oldex and payoutid in" echo " (select distinct payoutid from payouts where workinfoidend >= $wi);" @@ -156,11 +227,13 @@ process() echo "update payouts set $ex where workinfoidend >= $wi and $unex;" fi # - idctl markerid workmarkers - idctl paymentid payments - idctl payoutid payouts - # this makes sure the worker data is consistent - don't remove this - idctl workerid workers + if [ "$opt" = "-e" -o "$opt" = "-m" -o "$opt" = "-r" ] ; then + idctl markerid workmarkers + idctl paymentid payments + idctl payoutid payouts + # this makes sure the worker data is consistent - don't remove this + idctl workerid workers + fi } # if [ -z "$1" ] ; then @@ -171,13 +244,19 @@ if [ "$1" = "-?" -o "$1" = "-h" -o "$1" = "-help" -o "$1" = "--help" ] ; then usAge fi # +deblk="" +if [ "$1" = "-b" ] ; then + deblk="y" + shift +fi +# usenow="" if [ "$1" = "-n" ] ; then usenow="y" shift fi # -if [ "$1" != "-r" -a "$1" != "-e" -a "$1" != "-m" ] ; then +if [ "$1" != "-c" -a "$1" != "-e" -a "$1" != "-r" -a "$1" != "-m" ] ; then echo "ERR: Unknown p1='$1'" usAge fi From 01ee4fbfd164bd15dcd0fd6caea209c8dacd4dd3 Mon Sep 17 00:00:00 2001 From: kanoi Date: Wed, 30 Sep 2015 19:16:16 +1000 Subject: [PATCH 33/62] ckdb - allow shift processing to start during the reload --- src/ckdb.c | 62 ++++++++++++++++++++++++++++++++++++++++++++++--- src/ckdb.h | 29 ++++++++++++----------- src/ckdb_cmd.c | 6 +++-- src/ckdb_data.c | 61 ++++++++++++++++++++++++------------------------ src/ckdb_dbio.c | 20 +++++++++------- 5 files changed, 120 insertions(+), 58 deletions(-) diff --git a/src/ckdb.c b/src/ckdb.c index 78045855..0143a0c3 100644 --- a/src/ckdb.c +++ b/src/ckdb.c @@ -253,6 +253,8 @@ bool db_users_complete = false; bool db_load_complete = false; // Different input data handling bool reloading = false; +// Start marks processing during a larger reload +static bool reloaded_N_files = false; // Data load is complete bool startup_complete = false; // Set to true the first time workqueue reaches 0 after startup @@ -1483,13 +1485,27 @@ static bool setup_data() sec -= min * 60.0; LOGWARNING("reload complete %.0fm %.3fs", min, sec); + // full lock access since mark processing can occur + ck_wlock(&process_pplns_lock); + K_WLOCK(workerstatus_free); + K_RLOCK(sharesummary_free); + K_RLOCK(workmarkers_free); + K_RLOCK(markersummary_free); + set_block_share_counters(); + if (!everyone_die) + workerstatus_ready(); + + K_RUNLOCK(markersummary_free); + K_RUNLOCK(workmarkers_free); + K_RUNLOCK(sharesummary_free); + K_WUNLOCK(workerstatus_free); + ck_wunlock(&process_pplns_lock); + if (everyone_die) return false; - workerstatus_ready(); - workinfo_current = last_in_ktree(workinfo_height_root, ctx); if (workinfo_current) { DATA_WORKINFO(wic, workinfo_current); @@ -3083,9 +3099,20 @@ static void *summariser(__maybe_unused void *arg) rename_proc("db_summariser"); + /* Don't do any summarisation until the reload queue completes coz: + * 1) It locks/accesses a lot of data - workinfo/markersummary that + * can slow down the reload + * 2) If you stop and restart ckdb this wont affect the restart point + * Thus it's OK to do it later + * 3) It does I/O to bitcoind which is slow ... + * 4) It triggers the payout generation which also accesses a lot of + * data - workinfo/markersummary - but it wont affect a later + * restart point if it hasn't been done. Thus it's OK to do it later + */ while (!everyone_die && !reload_queue_complete) cksleep_ms(42); + LOGWARNING("%s() Start processing...", __func__); summariser_using_data = true; while (!everyone_die) { @@ -3628,7 +3655,13 @@ static void *marker(__maybe_unused void *arg) rename_proc("db_marker"); - while (!everyone_die && !reload_queue_complete) + /* We want this to start during the CCL reload so that if we run a + * large reload and it fails at some point, the next reload will not + * always have to go back to the same reload point as before due to + * no new workmarkers being completed/processed + * However, don't start during the first N reload files so that a + * normal ckdb restart reload won't slow down */ + while (!everyone_die && !reloaded_N_files && !reload_queue_complete) cksleep_ms(42); if (sharesummary_marks_limit) { @@ -3637,6 +3670,7 @@ static void *marker(__maybe_unused void *arg) return NULL; } + LOGWARNING("%s() Start processing...", __func__); marker_using_data = true; while (!everyone_die) { @@ -3692,6 +3726,7 @@ static void *logger(__maybe_unused void *arg) snprintf(buf, sizeof(buf), "db%s_logger", dbcode); rename_proc(buf); + LOGWARNING("%s() Start processing...", __func__); logger_using_data = true; setnow(&now); @@ -3809,6 +3844,7 @@ static void *socketer(__maybe_unused void *arg) while (!everyone_die && !db_users_complete) cksem_mswait(&socketer_sem, 420); + LOGWARNING("%s() Start processing...", __func__); socketer_using_data = true; want_first = true; @@ -4484,6 +4520,14 @@ static bool logopen(char **filename, FILE **fp, bool *apipe) return false; } +// How many files need to be processed before flagging reloaded_N_files +#define RELOAD_N_FILES 2 +// optioncontrol name to override the above value +#define RELOAD_N_FILES_STR "ReloadNFiles" + +// How many lines in a reload file required to count it +#define RELOAD_N_COUNT 1000 + /* If the reload start file is missing and -r was specified correctly: * touch the filename reported in "Failed to open 'filename'", * if ckdb aborts at the beginning of the reload, then start again */ @@ -4501,11 +4545,15 @@ static bool reload_from(tv_t *start) tv_t now, begin; double diff; FILE *fp = NULL; + int file_N_limit; reload_buf = malloc(MAX_READ); if (!reload_buf) quithere(1, "(%d) OOM", MAX_READ); + file_N_limit = (int)sys_setting(RELOAD_N_FILES_STR, RELOAD_N_FILES, + &date_eot); + reloading = true; copy_tv(&reload_timestamp, start); @@ -4569,6 +4617,14 @@ static bool reload_from(tv_t *start) LOGWARNING("%s(): confirm range complete", __func__); break; } + + /* Used by marker() to start mark generation during a longer + * than normal reload */ + if (count > RELOAD_N_COUNT) { + if (--file_N_limit < 1) + reloaded_N_files = true; + } + filename = rotating_filename(restorefrom, reload_timestamp.tv_sec); ok = logopen(&filename, &fp, &apipe); if (!ok) { diff --git a/src/ckdb.h b/src/ckdb.h index 403acf0c..a45dc0be 100644 --- a/src/ckdb.h +++ b/src/ckdb.h @@ -55,7 +55,7 @@ #define DB_VLOCK "1" #define DB_VERSION "1.0.3" -#define CKDB_VERSION DB_VERSION"-1.350" +#define CKDB_VERSION DB_VERSION"-1.400" #define WHERE_FFL " - from %s %s() line %d" #define WHERE_FFL_HERE __FILE__, __func__, __LINE__ @@ -2212,16 +2212,16 @@ extern K_ITEM *_optional_name(K_TREE *trf_root, char *name, int len, char *patt, extern K_ITEM *_require_name(K_TREE *trf_root, char *name, int len, char *patt, char *reply, size_t siz, WHERE_FFL_ARGS); extern cmp_t cmp_workerstatus(K_ITEM *a, K_ITEM *b); -extern K_ITEM *get_workerstatus(int64_t userid, char *workername); -#define find_create_workerstatus(_u, _w, _file, _func, _line) \ - _find_create_workerstatus(_u, _w, true, _file, _func, _line, WHERE_FFL_HERE) -#define find_workerstatus(_u, _w, _file, _func, _line) \ - _find_create_workerstatus(_u, _w, false, _file, _func, _line, WHERE_FFL_HERE) - -extern K_ITEM *_find_create_workerstatus(int64_t userid, char *workername, - bool create, const char *file2, - const char *func2, const int line2, - WHERE_FFL_ARGS); +extern K_ITEM *get_workerstatus(bool lock, int64_t userid, char *workername); +#define find_create_workerstatus(_l, _u, _w, _file, _func, _line) \ + _find_create_workerstatus(_l, _u, _w, true, _file, _func, _line, WHERE_FFL_HERE) +#define find_workerstatus(_l, _u, _w, _file, _func, _line) \ + _find_create_workerstatus(_l, _u, _w, false, _file, _func, _line, WHERE_FFL_HERE) + +extern K_ITEM *_find_create_workerstatus(bool lock, int64_t userid, + char *workername, bool create, + const char *file2, const char *func2, + const int line2, WHERE_FFL_ARGS); extern void zero_all_active(tv_t *when); extern void workerstatus_ready(); #define workerstatus_update(_auths, _shares, _userstats) \ @@ -2336,7 +2336,7 @@ extern cmp_t cmp_blocks(K_ITEM *a, K_ITEM *b); extern K_ITEM *find_blocks(int32_t height, char *blockhash, K_TREE_CTX *ctx); extern K_ITEM *find_prev_blocks(int32_t height, K_TREE_CTX *ctx); extern const char *blocks_confirmed(char *confirmed); -extern void zero_on_new_block(); +extern void zero_on_new_block(bool lock); extern void set_block_share_counters(); extern bool check_update_blocks_stats(tv_t *stats); #define set_blockcreatedate(_h) _set_blockcreatedate(_h, WHERE_FFL_HERE) @@ -2484,8 +2484,9 @@ extern K_ITEM *useratts_add(PGconn *conn, char *username, char *attname, bool begun); extern bool useratts_item_expire(PGconn *conn, K_ITEM *ua_item, tv_t *cd); extern bool useratts_fill(PGconn *conn); -extern K_ITEM *workers_add(PGconn *conn, int64_t userid, char *workername, - char *difficultydefault, char *idlenotificationenabled, +extern K_ITEM *workers_add(PGconn *conn, bool lock, int64_t userid, + char *workername, char *difficultydefault, + char *idlenotificationenabled, char *idlenotificationtime, char *by, char *code, char *inet, tv_t *cd, K_TREE *trf_root); extern bool workers_update(PGconn *conn, K_ITEM *item, char *difficultydefault, diff --git a/src/ckdb_cmd.c b/src/ckdb_cmd.c index 2f8b396f..d7bed90e 100644 --- a/src/ckdb_cmd.c +++ b/src/ckdb_cmd.c @@ -1734,7 +1734,8 @@ static char *cmd_percent(char *cmd, char *id, tv_t *now, USERS *users) DATA_WORKERS_NULL(workers, w_item); while (w_item && workers->userid == users->userid) { if (CURRENT(&(workers->expirydate))) { - ws_item = get_workerstatus(users->userid, workers->workername); + ws_item = get_workerstatus(true, users->userid, + workers->workername); if (ws_item) { DATA_WORKERSTATUS(workerstatus, ws_item); t_diffacc += workerstatus->block_diffacc; @@ -1996,7 +1997,8 @@ static char *cmd_workers(__maybe_unused PGconn *conn, char *cmd, char *id, rows = 0; while (w_item && workers->userid == users->userid) { if (CURRENT(&(workers->expirydate))) { - ws_item = get_workerstatus(users->userid, workers->workername); + ws_item = get_workerstatus(true, users->userid, + workers->workername); if (ws_item) { DATA_WORKERSTATUS(workerstatus, ws_item); K_RLOCK(workerstatus_free); diff --git a/src/ckdb_data.c b/src/ckdb_data.c index eb2572c5..95d38469 100644 --- a/src/ckdb_data.c +++ b/src/ckdb_data.c @@ -815,7 +815,7 @@ cmp_t cmp_workerstatus(K_ITEM *a, K_ITEM *b) /* TODO: replace a lot of the code for all data types that codes finds, * each with specific functions for finding, to centralise the finds, * with passed ctx's */ -K_ITEM *get_workerstatus(int64_t userid, char *workername) +K_ITEM *get_workerstatus(bool lock, int64_t userid, char *workername) { WORKERSTATUS workerstatus; K_TREE_CTX ctx[1]; @@ -826,9 +826,11 @@ K_ITEM *get_workerstatus(int64_t userid, char *workername) INIT_WORKERSTATUS(&look); look.data = (void *)(&workerstatus); - K_RLOCK(workerstatus_free); + if (lock) + K_RLOCK(workerstatus_free); find = find_in_ktree(workerstatus_root, &look, ctx); - K_RUNLOCK(workerstatus_free); + if (lock) + K_RUNLOCK(workerstatus_free); return find; } @@ -839,7 +841,7 @@ K_ITEM *get_workerstatus(int64_t userid, char *workername) * This has 2 sets of file/func/line to allow 2 levels of traceback * to see why it happened */ -K_ITEM *_find_create_workerstatus(int64_t userid, char *workername, +K_ITEM *_find_create_workerstatus(bool lock, int64_t userid, char *workername, bool create, const char *file2, const char *func2, const int line2, WHERE_FFL_ARGS) @@ -849,7 +851,7 @@ K_ITEM *_find_create_workerstatus(int64_t userid, char *workername, bool ws_err = false, w_err = false; tv_t now; - ws_item = get_workerstatus(userid, workername); + ws_item = get_workerstatus(lock, userid, workername); if (!ws_item) { if (!create) { ws_err = true; @@ -858,7 +860,8 @@ K_ITEM *_find_create_workerstatus(int64_t userid, char *workername, if (!w_item) { w_err = true; setnow(&now); - w_item = workers_add(NULL, userid, workername, + w_item = workers_add(NULL, lock, userid, + workername, NULL, NULL, NULL, by_default, (char *)__func__, @@ -867,7 +870,8 @@ K_ITEM *_find_create_workerstatus(int64_t userid, char *workername, } } - K_WLOCK(workerstatus_free); + if (lock) + K_WLOCK(workerstatus_free); ws_item = k_unlink_head(workerstatus_free); DATA_WORKERSTATUS(row, ws_item); @@ -878,7 +882,8 @@ K_ITEM *_find_create_workerstatus(int64_t userid, char *workername, add_to_ktree(workerstatus_root, ws_item); k_add_head(workerstatus_store, ws_item); - K_WUNLOCK(workerstatus_free); + if (lock) + K_WUNLOCK(workerstatus_free); if (ws_err) { LOGNOTICE("%s(): CREATED Missing workerstatus" @@ -946,12 +951,10 @@ void workerstatus_ready() while (ws_item) { DATA_WORKERSTATUS(workerstatus, ws_item); - K_RLOCK(markersummary_free); // This is the last share datestamp ms_item = find_markersummary_userid(workerstatus->userid, workerstatus->workername, NULL); - K_RUNLOCK(markersummary_free); if (ms_item) { DATA_MARKERSUMMARY(markersummary, ms_item); if (tv_newer(&(workerstatus->last_share), @@ -968,10 +971,8 @@ void workerstatus_ready() } } - K_RLOCK(sharesummary_free); ss_item = find_last_sharesummary(workerstatus->userid, workerstatus->workername); - K_RUNLOCK(sharesummary_free); if (ss_item) { DATA_SHARESUMMARY(sharesummary, ss_item); if (tv_newer(&(workerstatus->last_share), @@ -1001,7 +1002,7 @@ void _workerstatus_update(AUTHS *auths, SHARES *shares, K_ITEM *item; if (auths) { - item = find_workerstatus(auths->userid, auths->workername, + item = find_workerstatus(true, auths->userid, auths->workername, file, func, line); if (item) { DATA_WORKERSTATUS(row, item); @@ -1022,7 +1023,8 @@ void _workerstatus_update(AUTHS *auths, SHARES *shares, pool.diffinv += shares->diff; pool.shareinv++; } - item = find_workerstatus(shares->userid, shares->workername, + item = find_workerstatus(true, shares->userid, + shares->workername, file, func, line); if (item) { DATA_WORKERSTATUS(row, item); @@ -1090,7 +1092,8 @@ void _workerstatus_update(AUTHS *auths, SHARES *shares, } if (startup_complete && userstats) { - item = find_workerstatus(userstats->userid, userstats->workername, + item = find_workerstatus(true, userstats->userid, + userstats->workername, file, func, line); if (item) { DATA_WORKERSTATUS(row, item); @@ -1535,7 +1538,7 @@ K_ITEM *new_worker(PGconn *conn, bool update, int64_t userid, char *workername, } // TODO: limit how many? - item = workers_add(conn, userid, workername, diffdef, + item = workers_add(conn, true, userid, workername, diffdef, idlenotificationenabled, idlenotificationtime, by, code, inet, cd, trf_root); } @@ -2748,13 +2751,14 @@ const char *blocks_confirmed(char *confirmed) return blocks_unknown; } -void zero_on_new_block() +void zero_on_new_block(bool lock) { WORKERSTATUS *workerstatus; K_TREE_CTX ctx[1]; K_ITEM *ws_item; - K_WLOCK(workerstatus_free); + if (lock) + K_WLOCK(workerstatus_free); pool.diffacc = pool.diffinv = pool.shareacc = pool.shareinv = pool.best_sdiff = 0; ws_item = first_in_ktree(workerstatus_root, ctx); @@ -2768,12 +2772,11 @@ void zero_on_new_block() workerstatus->block_sharehi = workerstatus->block_sharerej = 0.0; ws_item = next_in_ktree(ctx); } - K_WUNLOCK(workerstatus_free); - + if (lock) + K_WUNLOCK(workerstatus_free); } -/* Currently only used at the end of the startup - * Will need to add locking if it's used, later, after startup completes */ +// Currently only used at the end of the startup void set_block_share_counters() { K_TREE_CTX ctx[1], ctx_ms[1]; @@ -2788,13 +2791,12 @@ void set_block_share_counters() INIT_SHARESUMMARY(&ss_look); INIT_MARKERSUMMARY(&ms_look); - zero_on_new_block(); + zero_on_new_block(false); ws_item = NULL; /* From the end backwards so we can skip the workinfoid's we don't * want by jumping back to just before the current worker when the * workinfoid goes below the limit */ - K_RLOCK(sharesummary_free); ss_item = last_in_ktree(sharesummary_root, ctx); while (ss_item) { DATA_SHARESUMMARY(sharesummary, ss_item); @@ -2819,16 +2821,14 @@ void set_block_share_counters() * since it should always exist * However, it is simplest to simply create it * and keep going */ - K_RUNLOCK(sharesummary_free); - ws_item = find_workerstatus(sharesummary->userid, + ws_item = find_workerstatus(false, sharesummary->userid, sharesummary->workername, __FILE__, __func__, __LINE__); if (!ws_item) { - ws_item = find_create_workerstatus(sharesummary->userid, + ws_item = find_create_workerstatus(false, sharesummary->userid, sharesummary->workername, __FILE__, __func__, __LINE__); } - K_RLOCK(sharesummary_free); DATA_WORKERSTATUS(workerstatus, ws_item); } @@ -2857,7 +2857,6 @@ void set_block_share_counters() ss_item = prev_in_ktree(ctx); } - K_RUNLOCK(sharesummary_free); LOGWARNING("%s(): Updating block markersummary counters...", __func__); @@ -2905,11 +2904,11 @@ void set_block_share_counters() * since it should always exist * However, it is simplest to simply create it * and keep going */ - ws_item = find_workerstatus(markersummary->userid, + ws_item = find_workerstatus(false, markersummary->userid, markersummary->workername, __FILE__, __func__, __LINE__); if (!ws_item) { - ws_item = find_create_workerstatus(markersummary->userid, + ws_item = find_create_workerstatus(false, markersummary->userid, markersummary->workername, __FILE__, __func__, __LINE__); } diff --git a/src/ckdb_dbio.c b/src/ckdb_dbio.c index fd839ad7..60940a30 100644 --- a/src/ckdb_dbio.c +++ b/src/ckdb_dbio.c @@ -1287,7 +1287,7 @@ bool useratts_fill(PGconn *conn) return ok; } -K_ITEM *workers_add(PGconn *conn, int64_t userid, char *workername, +K_ITEM *workers_add(PGconn *conn, bool lock, int64_t userid, char *workername, char *difficultydefault, char *idlenotificationenabled, char *idlenotificationtime, char *by, char *code, char *inet, tv_t *cd, K_TREE *trf_root) @@ -1305,9 +1305,11 @@ K_ITEM *workers_add(PGconn *conn, int64_t userid, char *workername, LOGDEBUG("%s(): add", __func__); - K_WLOCK(workers_free); + if (lock) + K_WLOCK(workers_free); item = k_unlink_head(workers_free); - K_WUNLOCK(workers_free); + if (lock) + K_WUNLOCK(workers_free); DATA_WORKERS(row, item); @@ -1397,17 +1399,19 @@ unparam: unitem: if (conned) PQfinish(conn); - K_WLOCK(workers_free); + if (lock) + K_WLOCK(workers_free); if (!ret) k_add_head(workers_free, item); else { add_to_ktree(workers_root, item); k_add_head(workers_store, item); // Ensure there is a matching workerstatus - find_create_workerstatus(userid, workername, + find_create_workerstatus(lock, userid, workername, __FILE__, __func__, __LINE__); } - K_WUNLOCK(workers_free); + if (lock) + K_WUNLOCK(workers_free); return ret; } @@ -1647,7 +1651,7 @@ bool workers_fill(PGconn *conn) * This is to ensure that code can use the workerstatus tree * to reference other tables and not miss workers in the * other tables */ - find_create_workerstatus(row->userid, row->workername, + find_create_workerstatus(false, row->userid, row->workername, __FILE__, __func__, __LINE__); } if (!ok) @@ -4858,7 +4862,7 @@ flail: if (pool.workinfoid < row->workinfoid) { pool.workinfoid = row->workinfoid; pool.height = row->height; - zero_on_new_block(); + zero_on_new_block(true); } break; case BLOCKS_ORPHAN: From 3273ce9924289317b8fae0a574ef65bcf2d6db59 Mon Sep 17 00:00:00 2001 From: kanoi Date: Wed, 30 Sep 2015 20:41:24 +1000 Subject: [PATCH 34/62] ckdb - add a payout request to cmd_query --- src/ckdb.h | 3 +- src/ckdb_cmd.c | 121 ++++++++++++++++++++++++++++++++++++++++++++++++ src/ckdb_data.c | 19 ++++++++ 3 files changed, 142 insertions(+), 1 deletion(-) diff --git a/src/ckdb.h b/src/ckdb.h index a45dc0be..94ac6fbc 100644 --- a/src/ckdb.h +++ b/src/ckdb.h @@ -55,7 +55,7 @@ #define DB_VLOCK "1" #define DB_VERSION "1.0.3" -#define CKDB_VERSION DB_VERSION"-1.400" +#define CKDB_VERSION DB_VERSION"-1.401" #define WHERE_FFL " - from %s %s() line %d" #define WHERE_FFL_HERE __FILE__, __func__, __LINE__ @@ -2353,6 +2353,7 @@ extern cmp_t cmp_payouts(K_ITEM *a, K_ITEM *b); extern cmp_t cmp_payouts_id(K_ITEM *a, K_ITEM *b); extern cmp_t cmp_payouts_wid(K_ITEM *a, K_ITEM *b); extern K_ITEM *find_payouts(int32_t height, char *blockhash); +extern K_ITEM *first_payouts(int32_t height, K_TREE_CTX *ctx); extern K_ITEM *find_last_payouts(); extern K_ITEM *find_payoutid(int64_t payoutid); extern K_ITEM *find_payouts_wid(int64_t workinfoidend, K_TREE_CTX *ctx); diff --git a/src/ckdb_cmd.c b/src/ckdb_cmd.c index d7bed90e..2a6fd845 100644 --- a/src/ckdb_cmd.c +++ b/src/ckdb_cmd.c @@ -7036,6 +7036,127 @@ static char *cmd_query(__maybe_unused PGconn *conn, char *cmd, char *id, "WorkinfoRange", FLDSEP, "", FLDSEP); APPEND_REALLOC(buf, off, len, tmp); + ok = true; + } else if (strcasecmp(request, "payout") == 0) { + /* return the details of the payouts for block height=height + * if expired= is present, also return expired records */ + K_ITEM *i_height, *i_expired, *p_item; + PAYOUTS *payouts = NULL; + bool expired = false; + int32_t height; + char *stats = NULL, *ptr; + + i_height = require_name(trf_root, "height", + 1, (char *)intpatt, + reply, siz); + if (!i_height) + return strdup(reply); + TXT_TO_INT("height", transfer_data(i_height), height); + + int_to_buf(height, reply, sizeof(reply)); + snprintf(msg, sizeof(msg), "height=%s", reply); + + i_expired = optional_name(trf_root, "expired", + 0, NULL, reply, siz); + if (i_expired) + expired = true; + + K_RLOCK(payouts_free); + p_item = first_payouts(height, ctx); + DATA_PAYOUTS_NULL(payouts, p_item); + while (p_item && payouts->height == height) { + if (expired || CURRENT(&(payouts->expirydate))) { + int_to_buf(payouts->height, + reply, sizeof(reply)); + snprintf(tmp, sizeof(tmp), + "height:%d=%s%c", + rows, reply, FLDSEP); + APPEND_REALLOC(buf, off, len, tmp); + bigint_to_buf(payouts->payoutid, + reply, sizeof(reply)); + snprintf(tmp, sizeof(tmp), + "payoutid:%d=%s%c", + rows, reply, FLDSEP); + APPEND_REALLOC(buf, off, len, tmp); + bigint_to_buf(payouts->minerreward, + reply, sizeof(reply)); + snprintf(tmp, sizeof(tmp), + "minerreward:%d=%s%c", + rows, reply, FLDSEP); + APPEND_REALLOC(buf, off, len, tmp); + bigint_to_buf(payouts->workinfoidstart, + reply, sizeof(reply)); + snprintf(tmp, sizeof(tmp), + "workinfoidstart:%d=%s%c", + rows, reply, FLDSEP); + APPEND_REALLOC(buf, off, len, tmp); + bigint_to_buf(payouts->workinfoidend, + reply, sizeof(reply)); + snprintf(tmp, sizeof(tmp), + "workinfoidend:%d=%s%c", + rows, reply, FLDSEP); + APPEND_REALLOC(buf, off, len, tmp); + bigint_to_buf(payouts->elapsed, + reply, sizeof(reply)); + snprintf(tmp, sizeof(tmp), + "elapsed:%d=%s%c", + rows, reply, FLDSEP); + APPEND_REALLOC(buf, off, len, tmp); + snprintf(tmp, sizeof(tmp), + "status:%d=%s%c", + rows, payouts->status, FLDSEP); + APPEND_REALLOC(buf, off, len, tmp); + snprintf(tmp, sizeof(tmp), + "diffwanted:%d=%f%c", + rows, payouts->diffwanted, FLDSEP); + APPEND_REALLOC(buf, off, len, tmp); + snprintf(tmp, sizeof(tmp), + "diffused:%d=%f%c", + rows, payouts->diffused, FLDSEP); + APPEND_REALLOC(buf, off, len, tmp); + ptr = stats = strdup(payouts->stats); + if (!stats) { + quithere(1, "strdup (%"PRId64") OOM", + payouts->payoutid); + } + while (*ptr) { + if (*ptr == FLDSEP) + *ptr = ' '; + ptr++; + } + snprintf(tmp, sizeof(tmp), + "stats:%d=%s%c", + rows, stats, FLDSEP); + APPEND_REALLOC(buf, off, len, tmp); + FREENULL(stats); + tv_to_buf(&(payouts->expirydate), cd_buf, + sizeof(cd_buf)); + snprintf(tmp, sizeof(tmp), + EDDB"_str:%d=%s%c", + rows, cd_buf, FLDSEP); + APPEND_REALLOC(buf, off, len, tmp); + tv_to_buf(&(payouts->createdate), cd_buf, + sizeof(cd_buf)); + snprintf(tmp, sizeof(tmp), + CDDB"_str:%d=%s%c", + rows, cd_buf, FLDSEP); + APPEND_REALLOC(buf, off, len, tmp); + rows++; + } + p_item = next_in_ktree(ctx); + DATA_PAYOUTS_NULL(payouts, p_item); + } + + snprintf(tmp, sizeof(tmp), "flds=%s%c", + "height,payoutid,minerreward,workinfoidstart," + "workinfoidend,elapsed,status,diffwanted,diffused," + "stats,"EDDB"_str,"CDDB"_str", + FLDSEP); + APPEND_REALLOC(buf, off, len, tmp); + snprintf(tmp, sizeof(tmp), "arn=%s%carp=%s%c", + "Payouts", FLDSEP, "", FLDSEP); + APPEND_REALLOC(buf, off, len, tmp); + ok = true; } else { free(buf); diff --git a/src/ckdb_data.c b/src/ckdb_data.c index 95d38469..5a98946e 100644 --- a/src/ckdb_data.c +++ b/src/ckdb_data.c @@ -3405,6 +3405,25 @@ K_ITEM *find_payouts(int32_t height, char *blockhash) return find_in_ktree(payouts_root, &look, ctx); } +// The first (any state) payouts record with the given height +K_ITEM *first_payouts(int32_t height, K_TREE_CTX *ctx) +{ + PAYOUTS payouts; + K_TREE_CTX ctx0[1]; + K_ITEM look; + + if (ctx == NULL) + ctx = ctx0; + + payouts.height = height; + payouts.blockhash[0] = '\0'; + DATE_ZERO(&(payouts.expirydate)); + + INIT_PAYOUTS(&look); + look.data = (void *)(&payouts); + return find_after_in_ktree(payouts_root, &look, ctx); +} + // Last block payout calculated K_ITEM *find_last_payouts() { From 7b9233054a6a757e161f4b95d20d643e688d0e53 Mon Sep 17 00:00:00 2001 From: kanoi Date: Thu, 1 Oct 2015 17:00:02 +1000 Subject: [PATCH 35/62] ckdb/php - add block reward mean to block stats --- pool/page_blocks.php | 3 +++ src/ckdb.c | 2 +- src/ckdb.h | 6 +++++- src/ckdb_cmd.c | 8 +++++--- src/ckdb_data.c | 27 +++++++++++++++++++++------ 5 files changed, 35 insertions(+), 11 deletions(-) diff --git a/pool/page_blocks.php b/pool/page_blocks.php index e8d473bd..86e923b4 100644 --- a/pool/page_blocks.php +++ b/pool/page_blocks.php @@ -98,6 +98,7 @@ function doblocks($data, $user) $pg .= "		Mean%CDF[Erl]Luck%MeanTx%
$desc Blocks$mean%$cdferldsp$luck%$txm%
\n"; diff --git a/src/ckdb.c b/src/ckdb.c index 0143a0c3..909e5b41 100644 --- a/src/ckdb.c +++ b/src/ckdb.c @@ -4621,7 +4621,7 @@ static bool reload_from(tv_t *start) /* Used by marker() to start mark generation during a longer * than normal reload */ if (count > RELOAD_N_COUNT) { - if (--file_N_limit < 1) + if (file_N_limit-- < 1) reloaded_N_files = true; } diff --git a/src/ckdb.h b/src/ckdb.h index 94ac6fbc..2dd2d5a7 100644 --- a/src/ckdb.h +++ b/src/ckdb.h @@ -55,7 +55,7 @@ #define DB_VLOCK "1" #define DB_VERSION "1.0.3" -#define CKDB_VERSION DB_VERSION"-1.401" +#define CKDB_VERSION DB_VERSION"-1.402" #define WHERE_FFL " - from %s %s() line %d" #define WHERE_FFL_HERE __FILE__, __func__, __LINE__ @@ -1514,6 +1514,9 @@ typedef struct blocks { double cdferl; double luck; + // Mean reward ratio per block from last to this + double txmean; + // To save looking them up when needed tv_t prevcreatedate; // non-DB field tv_t blockcreatedate; // non-DB field @@ -2303,6 +2306,7 @@ extern bool workinfo_age(int64_t workinfoid, char *poolinstance, char *by, char *code, char *inet, tv_t *cd, tv_t *ss_first, tv_t *ss_last, int64_t *ss_count, int64_t *s_count, int64_t *s_diff); +extern double coinbase_reward(int32_t height); extern double workinfo_pps(K_ITEM *w_item, int64_t workinfoid, bool lock); extern cmp_t cmp_shares(K_ITEM *a, K_ITEM *b); extern cmp_t cmp_shareerrors(K_ITEM *a, K_ITEM *b); diff --git a/src/ckdb_cmd.c b/src/ckdb_cmd.c index 2a6fd845..f5669b95 100644 --- a/src/ckdb_cmd.c +++ b/src/ckdb_cmd.c @@ -1329,7 +1329,8 @@ redo: "s_diffratio:%d=%.8f%c" "s_diffmean:%d=%.8f%c" "s_cdferl:%d=%.8f%c" - "s_luck:%d=%.8f%c", + "s_luck:%d=%.8f%c" + "s_txmean:%d=%.8f%c", srows, seq, FLDSEP, srows, desc, FLDSEP, srows, (int)(blocks->height), FLDSEP, @@ -1338,7 +1339,8 @@ redo: srows, blocks->diffratio, FLDSEP, srows, blocks->diffmean, FLDSEP, srows, blocks->cdferl, FLDSEP, - srows, blocks->luck, FLDSEP); + srows, blocks->luck, FLDSEP, + srows, blocks->txmean, FLDSEP); APPEND_REALLOC(buf, off, len, tmp); srows++; } @@ -1367,7 +1369,7 @@ redo: "s_rows=%d%cs_flds=%s%c", srows, FLDSEP, "s_seq,s_desc,s_height,s_"CDTRF",s_prev"CDTRF",s_diffratio," - "s_diffmean,s_cdferl,s_luck", + "s_diffmean,s_cdferl,s_luck,s_txmean", FLDSEP); APPEND_REALLOC(buf, off, len, tmp); diff --git a/src/ckdb_data.c b/src/ckdb_data.c index 5a98946e..ff6f102c 100644 --- a/src/ckdb_data.c +++ b/src/ckdb_data.c @@ -2235,6 +2235,15 @@ bye: return ok; } +// Block height coinbase reward value +double coinbase_reward(int32_t height) +{ + double value; + + value = REWARD_BASE * pow(0.5, floor((double)height / REWARD_HALVE)); + + return(value); +} // The PPS value of a 1diff share for the given workinfoid double workinfo_pps(K_ITEM *w_item, int64_t workinfoid, bool lock) @@ -2246,7 +2255,7 @@ double workinfo_pps(K_ITEM *w_item, int64_t workinfoid, bool lock) char ndiffbin[TXT_SML+1]; WORKINFO *workinfo; double w_diff; - int w_blocknum; + int32_t w_blocknum; size_t len; // Allow optioncontrol override for a given workinfoid @@ -2284,10 +2293,10 @@ double workinfo_pps(K_ITEM *w_item, int64_t workinfoid, bool lock) w_diff = diff_from_nbits(ndiffbin); hex2bin(coinbase1bin, workinfo->coinbase1 + (BLOCKNUM_OFFSET * 2), (len - (BLOCKNUM_OFFSET * 2)) >> 1); - w_blocknum = get_sernumber((uchar *)coinbase1bin); + w_blocknum = (int32_t)get_sernumber((uchar *)coinbase1bin); - // BASE halving to determine coinbase reward then divided by difficulty - return(REWARD_BASE * pow(0.5, floor((double)w_blocknum / REWARD_HALVE)) / w_diff); + // PPS 1diff is worth coinbase reward divided by difficulty + return(coinbase_reward(w_blocknum) / w_diff); } // order by workinfoid asc,userid asc,workername asc,createdate asc,nonce asc,expirydate desc @@ -2965,7 +2974,7 @@ bool check_update_blocks_stats(tv_t *stats) WORKINFO *workinfo; BLOCKS *blocks; char ndiffbin[TXT_SML+1]; - double ok, diffacc, netsumm, diffmean, pending; + double ok, diffacc, netsumm, diffmean, pending, txmean, cr; tv_t now; /* Wait for startup_complete rather than db_load_complete @@ -2994,7 +3003,7 @@ bool check_update_blocks_stats(tv_t *stats) } b_item = next_in_ktree(ctx); } - ok = diffacc = netsumm = diffmean = 0.0; + ok = diffacc = netsumm = diffmean = 0.0, txmean = 0.0; b_item = last_in_ktree(blocks_root, ctx); while (b_item) { DATA_BLOCKS(blocks, b_item); @@ -3043,6 +3052,7 @@ bool check_update_blocks_stats(tv_t *stats) blocks->diffmean = 0.0; blocks->cdferl = 0.0; blocks->luck = 0.0; + blocks->txmean = 0.0; } else { ok++; diffacc += blocks->diffcalc; @@ -3064,6 +3074,11 @@ bool check_update_blocks_stats(tv_t *stats) blocks->cdferl = gsl_cdf_gamma_P(diffmean, ok, 1.0 / ok); blocks->luck = 1.0 / diffmean; } + + cr = coinbase_reward(blocks->height); + txmean = ((txmean * (ok - 1)) + + ((double)(blocks->reward) / cr)) / ok; + blocks->txmean = txmean; } } b_item = prev_in_ktree(ctx); From f4db3e14bd739f2780f2004ce366283a21efa841 Mon Sep 17 00:00:00 2001 From: kanoi Date: Thu, 1 Oct 2015 22:10:32 +1000 Subject: [PATCH 36/62] ckdb/sql/php - add an optional address idname for users with multiple payment addressess --- pool/db.php | 4 ++++ pool/page_addrmgt.php | 39 ++++++++++++++++++++++++++------------- pool/page_percent.php | 6 +++++- sql/ckdb.sql | 4 +++- sql/v1.0.3-v1.0.4.sql | 26 ++++++++++++++++++++++++++ src/ckdb.c | 18 ++++++++++++------ src/ckdb.h | 3 ++- src/ckdb_cmd.c | 24 ++++++++++++++++++++---- src/ckdb_dbio.c | 26 ++++++++++++++++++-------- 9 files changed, 116 insertions(+), 34 deletions(-) create mode 100644 sql/v1.0.3-v1.0.4.sql diff --git a/pool/db.php b/pool/db.php index 2b9d4c5e..c845d5e9 100644 --- a/pool/db.php +++ b/pool/db.php @@ -242,6 +242,7 @@ function userReg($user, $email, $pass) # function userSettings($user, $email = null, $addr = null, $pass = null, $twofa = null) { + global $fld_sep; $tmo = false; $flds = array('username' => $user); if ($email != null) @@ -253,6 +254,9 @@ function userSettings($user, $email = null, $addr = null, $pass = null, $twofa = foreach ($addr as $ar) { $flds['address:'.$i] = $ar['addr']; + // optional - missing = blank + if (isset($ar['payname'])) + $flds['payname:'.$i] = str_replace($fld_sep, ' ', trim($ar['payname'])); // optional - missing = use default if (isset($ar['ratio'])) $flds['ratio:'.$i] = $ar['ratio']; diff --git a/pool/page_addrmgt.php b/pool/page_addrmgt.php index d0300796..fe886b44 100644 --- a/pool/page_addrmgt.php +++ b/pool/page_addrmgt.php @@ -11,6 +11,7 @@ function addrmgtuser($data, $user, $err) $pg .= "\n"; $pg .= ''; $pg .= ''; + $pg .= ''; $pg .= ''; $pg .= ''; $pg .= ''; @@ -18,7 +19,10 @@ function addrmgtuser($data, $user, $err) # new row template for '+' $pg .= ''; $pg .= ''; + $pg .= ''; $pg .= '"; $addr = $ans['addr:'.$i]; $pg .= ''; + $nam = htmlspecialchars($ans['payname:'.$i]); + $pg .= ''; $ratio = intval($ans['ratio:'.$i]); $pg .= '"; - $pg .= ''; @@ -72,20 +80,21 @@ function addrmgtuser($data, $user, $err) else $row = 'odd'; $pg .= ""; - $pg .= ''; + $pg .= ''; + $pg .= ''; + $pg .= ''; if ((($offset++) % 2) == 0) $row = 'even'; else $row = 'odd'; $pg .= ""; - $pg .= ''; + $pg .= ''; + $pg .= ''; + $pg .= ''; - $pg .= "'; @@ -97,8 +106,9 @@ function addrmgtuser($data, $user, $err) $pg .= "var r=document.getElementById('rows');var c=parseInt(r.value);"; $pg .= "var bs=document.getElementById('bs');var n=bs.cloneNode(true);n.id='z';"; $pg .= "n.className='odd';var ia=n.childNodes[0].firstChild;ia.name='addr:'+c;ia.value='';"; - $pg .= "var ir=n.childNodes[1].firstChild;ir.id='rat'+c;ir.name='ratio:'+c;ir.value='0';"; - $pg .= "var ip=n.childNodes[2].firstChild;ip.id='per'+c;ip.innerHTML='0.00%';"; + $pg .= "var ipn=n.childNodes[1].firstChild;ipn.name='payname:'+c;ipn.value='';"; + $pg .= "var ir=n.childNodes[2].firstChild;ir.id='rat'+c;ir.name='ratio:'+c;ir.value='0';"; + $pg .= "var ip=n.childNodes[3].firstChild;ip.id='per'+c;ip.innerHTML='0.00%';"; $pg .= "p.parentNode.insertBefore(n, p);"; $pg .= "c++;r.value=c;return true}\n"; $pg .= "function repc(){var c=parseInt(document.getElementById('rows').value);"; @@ -127,9 +137,12 @@ function doaddrmgt($data, $user) for ($i = 0; $i < $count; $i++) { $addr = getparam('addr:'.$i, false); + $nam = getparam('payname:'.$i, false); + if (nuem($nam)) + $nam = ''; $ratio = getparam('ratio:'.$i, false); if (!nuem($addr) && !nuem($ratio)) - $addrarr[] = array('addr' => $addr, 'ratio' => $ratio); + $addrarr[] = array('addr' => $addr, 'payname' => $nam, 'ratio' => $ratio); } $ans = userSettings($user, null, $addrarr, $pass, $twofa); if ($ans['STATUS'] != 'ok') diff --git a/pool/page_percent.php b/pool/page_percent.php index 2cd5b887..1e303691 100644 --- a/pool/page_percent.php +++ b/pool/page_percent.php @@ -4,6 +4,7 @@ function pertitle($data, $user) { $pg = ''; $pg .= ''; + $pg .= ''; $pg .= ''; $pg .= ''; $pg .= ''; @@ -40,6 +41,7 @@ function peruser($data, $user, &$offset, &$totshare, &$totdiff, $all[] = array('payaddress' => $ans['payaddress:'.$i], 'payratio' => $ans['payratio:'.$i], 'paypercent' => $ans['paypercent:'.$i], + 'payname' => $ans['payname:'.$i], 'p_shareacc' => $ans['p_shareacc:'.$i], 'p_diffacc' => $ans['p_diffacc:'.$i], 'p_diffinv' => $ans['p_diffinv:'.$i], @@ -58,6 +60,7 @@ function peruser($data, $user, &$offset, &$totshare, &$totdiff, $pg .= ""; $pg .= ''; + $pg .= ''; $shareacc = number_format($all[$i]['p_shareacc'], 0); $totshare += $all[$i]['p_shareacc']; @@ -113,6 +116,7 @@ function pertotal($offset, $totshare, $totdiff, $totinvalid, $totrate, $blockacc else $row = 'odd'; $pg .= ""; + $pg .= ""; $shareacc = number_format($totshare, 0); $pg .= ""; $diffacc = number_format($totdiff, 0); @@ -156,7 +160,7 @@ function dopercent($data, $user) if ($blockacc > 0 && $blockreward > 0) { $btc = btcfmt($totdiff / $blockacc * $blockreward); - $pg .= ''; } diff --git a/sql/ckdb.sql b/sql/ckdb.sql index 4f39ba26..614990ea 100644 --- a/sql/ckdb.sql +++ b/sql/ckdb.sql @@ -75,6 +75,8 @@ CREATE TABLE paymentaddresses ( userid bigint NOT NULL, payaddress character varying(256) DEFAULT ''::character varying NOT NULL, payratio integer DEFAULT 1000000 NOT NULL, + payname character varying(64) DEFAULT ''::character varying NOT NULL, + status char DEFAULT ' ' NOT NULL, createdate timestamp with time zone NOT NULL, createby character varying(64) DEFAULT ''::character varying NOT NULL, createcode character varying(128) DEFAULT ''::character varying NOT NULL, @@ -468,4 +470,4 @@ CREATE TABLE version ( PRIMARY KEY (vlock) ); -insert into version (vlock,version) values (1,'1.0.3'); +insert into version (vlock,version) values (1,'1.0.4'); diff --git a/sql/v1.0.3-v1.0.4.sql b/sql/v1.0.3-v1.0.4.sql new file mode 100644 index 00000000..ba49dc7f --- /dev/null +++ b/sql/v1.0.3-v1.0.4.sql @@ -0,0 +1,26 @@ +SET SESSION AUTHORIZATION 'postgres'; + +BEGIN transaction; + +DO $$ +DECLARE ver TEXT; +BEGIN + + UPDATE version set version='1.0.4' where vlock=1 and version='1.0.3'; + + IF found THEN + RETURN; + END IF; + + SELECT version into ver from version + WHERE vlock=1; + + RAISE EXCEPTION 'Wrong DB version - expect "1.0.3" - found "%"', ver; + +END $$; + +ALTER TABLE ONLY paymentaddresses + ADD COLUMN payname character varying(64) DEFAULT ''::character varying NOT NULL, + ADD COLUMN status char DEFAULT ' ' NOT NULL; + +END transaction; diff --git a/src/ckdb.c b/src/ckdb.c index 909e5b41..041ec653 100644 --- a/src/ckdb.c +++ b/src/ckdb.c @@ -3112,8 +3112,10 @@ static void *summariser(__maybe_unused void *arg) while (!everyone_die && !reload_queue_complete) cksleep_ms(42); - LOGWARNING("%s() Start processing...", __func__); - summariser_using_data = true; + if (!everyone_die) { + LOGWARNING("%s() Start processing...", __func__); + summariser_using_data = true; + } while (!everyone_die) { for (i = 0; i < 5; i++) { @@ -3670,8 +3672,10 @@ static void *marker(__maybe_unused void *arg) return NULL; } - LOGWARNING("%s() Start processing...", __func__); - marker_using_data = true; + if (!everyone_die) { + LOGWARNING("%s() Start processing...", __func__); + marker_using_data = true; + } while (!everyone_die) { for (i = 0; i < 5; i++) { @@ -3844,8 +3848,10 @@ static void *socketer(__maybe_unused void *arg) while (!everyone_die && !db_users_complete) cksem_mswait(&socketer_sem, 420); - LOGWARNING("%s() Start processing...", __func__); - socketer_using_data = true; + if (!everyone_die) { + LOGWARNING("%s() Start processing...", __func__); + socketer_using_data = true; + } want_first = true; while (!everyone_die) { diff --git a/src/ckdb.h b/src/ckdb.h index 2dd2d5a7..3c0ef443 100644 --- a/src/ckdb.h +++ b/src/ckdb.h @@ -54,7 +54,7 @@ */ #define DB_VLOCK "1" -#define DB_VERSION "1.0.3" +#define DB_VERSION "1.0.4" #define CKDB_VERSION DB_VERSION"-1.402" #define WHERE_FFL " - from %s %s() line %d" @@ -1186,6 +1186,7 @@ typedef struct paymentaddresses { int64_t userid; char payaddress[TXT_BIG+1]; int32_t payratio; + char payname[TXT_SML+1]; HISTORYDATECONTROLFIELDS; bool match; // non-DB field } PAYMENTADDRESSES; diff --git a/src/ckdb_cmd.c b/src/ckdb_cmd.c index f5669b95..4f2bce33 100644 --- a/src/ckdb_cmd.c +++ b/src/ckdb_cmd.c @@ -445,8 +445,8 @@ static char *cmd_userset(PGconn *conn, char *cmd, char *id, __maybe_unused tv_t *notcd, K_TREE *trf_root) { K_ITEM *i_username, *i_passwordhash, *i_2fa, *i_rows, *i_address; - K_ITEM *i_ratio, *i_email, *u_item, *pa_item, *old_pa_item; - char *email, *address; + K_ITEM *i_ratio, *i_payname, *i_email, *u_item, *pa_item, *old_pa_item; + char *email, *address, *payname; char reply[1024] = ""; size_t siz = sizeof(reply); char tmp[1024]; @@ -507,6 +507,9 @@ static char *cmd_userset(PGconn *conn, char *cmd, char *id, snprintf(tmp, sizeof(tmp), "ratio:%d=%d%c", rows, row->payratio, FLDSEP); APPEND_REALLOC(answer, off, len, tmp); + snprintf(tmp, sizeof(tmp), "payname:%d=%s%c", + rows, row->payname, FLDSEP); + APPEND_REALLOC(answer, off, len, tmp); rows++; pa_item = prev_in_ktree(ctx); @@ -517,7 +520,7 @@ static char *cmd_userset(PGconn *conn, char *cmd, char *id, snprintf(tmp, sizeof(tmp), "rows=%d%cflds=%s%c", rows, FLDSEP, - "addr,ratio", FLDSEP); + "addr,ratio,payname", FLDSEP); APPEND_REALLOC(answer, off, len, tmp); snprintf(tmp, sizeof(tmp), "arn=%s%carp=%s", "PaymentAddresses", FLDSEP, ""); @@ -615,11 +618,20 @@ static char *cmd_userset(PGconn *conn, char *cmd, char *id, } pa_item = pa_item->next; } + snprintf(tmp, sizeof(tmp), "payname:%d", i); + i_payname = optional_name(trf_root, tmp, + 0, NULL, + reply, siz); + if (i_payname) + payname = transfer_data(i_payname); + else + payname = EMPTY; pa_item = k_unlink_head(paymentaddresses_free); DATA_PAYMENTADDRESSES(row, pa_item); bzero(row, sizeof(*row)); STRNCPY(row->payaddress, address); row->payratio = ratio; + STRNCPY(row->payname, payname); k_add_head(pa_store, pa_item); } K_WUNLOCK(paymentaddresses_free); @@ -1808,6 +1820,10 @@ static char *cmd_percent(char *cmd, char *id, tv_t *now, USERS *users) rows, ratio * 100.0, FLDSEP); APPEND_REALLOC(buf, off, len, tmp); + snprintf(tmp, sizeof(tmp), "payname:%d=%s%c", + rows, pa->payname, FLDSEP); + APPEND_REALLOC(buf, off, len, tmp); + snprintf(tmp, sizeof(tmp), "p_hashrate5m:%d=%.1f%c", rows, (double)t_hashrate5m * ratio, FLDSEP); @@ -1893,7 +1909,7 @@ static char *cmd_percent(char *cmd, char *id, tv_t *now, USERS *users) snprintf(tmp, sizeof(tmp), "rows=%d%cflds=%s%c", rows, FLDSEP, - "payaddress,payratio,paypercent," + "payaddress,payratio,paypercent,payname," "p_hashrate5m,p_hashrate1hr,p_hashrate24hr," "p_diffacc,p_diffinv," "p_diffsta,p_diffdup,p_diffhi,p_diffrej," diff --git a/src/ckdb_dbio.c b/src/ckdb_dbio.c index 60940a30..90f34033 100644 --- a/src/ckdb_dbio.c +++ b/src/ckdb_dbio.c @@ -1723,7 +1723,9 @@ bool paymentaddresses_set(PGconn *conn, int64_t userid, K_STORE *pa_store, /* Since we are merging the changes in rather than just * replacing the db contents, lock the data for the duration - * of the update to ensure nothing else changes it */ + * of the update to ensure nothing else changes it + * N.B. 'payname' isn't the start of the key + * thus 2 different addresses can have the same 'payname' */ K_WLOCK(paymentaddresses_free); locked = true; @@ -1741,7 +1743,8 @@ bool paymentaddresses_set(PGconn *conn, int64_t userid, K_STORE *pa_store, while (match) { DATA_PAYMENTADDRESSES(pa, match); if (strcmp(pa->payaddress, row->payaddress) == 0 && - pa->payratio == row->payratio) { + pa->payratio == row->payratio && + strcmp(pa->payname, row->payname) == 0) { pa->match = true; // Don't store it matches++; break; @@ -1791,8 +1794,8 @@ bool paymentaddresses_set(PGconn *conn, int64_t userid, K_STORE *pa_store, // Second step - add the non-matching records to the DB LOGDEBUG("%s(): Step 2", __func__); ins = "insert into paymentaddresses " - "(paymentaddressid,userid,payaddress,payratio" - HISTORYDATECONTROL ") values (" PQPARAM9 ")"; + "(paymentaddressid,userid,payaddress,payratio,payname" + HISTORYDATECONTROL ") values (" PQPARAM10 ")"; count = 0; match = pa_store->head; @@ -1814,8 +1817,9 @@ bool paymentaddresses_set(PGconn *conn, int64_t userid, K_STORE *pa_store, params[par++] = bigint_to_buf(row->userid, NULL, 0); params[par++] = str_to_buf(row->payaddress, NULL, 0); params[par++] = int_to_buf(row->payratio, NULL, 0); + params[par++] = str_to_buf(row->payname, NULL, 0); HISTORYDATEPARAMS(params, par, row); - PARCHKVAL(par, 9, params); // As per PQPARAM9 above + PARCHKVAL(par, 10, params); // As per PQPARAM10 above res = PQexecParams(conn, ins, par, NULL, (const char **)params, NULL, NULL, 0, CKPQ_WRITE); @@ -1864,7 +1868,8 @@ unparam: while (match) { DATA_PAYMENTADDRESSES(pa, match); if (strcmp(pa->payaddress, row->payaddress) == 0 && - pa->payratio == row->payratio) { + pa->payratio == row->payratio && + strcmp(pa->payname, row->payname) == 0) { break; } match = match->next; @@ -1917,13 +1922,13 @@ bool paymentaddresses_fill(PGconn *conn) int n, i; char *field; char *sel; - int fields = 4; + int fields = 5; bool ok; LOGDEBUG("%s(): select", __func__); sel = "select " - "paymentaddressid,userid,payaddress,payratio" + "paymentaddressid,userid,payaddress,payratio,payname" HISTORYDATECONTROL " from paymentaddresses"; res = PQexec(conn, sel, CKPQ_READ); @@ -1976,6 +1981,11 @@ bool paymentaddresses_fill(PGconn *conn) break; TXT_TO_INT("payratio", field, row->payratio); + PQ_GET_FLD(res, i, "payname", field, ok); + if (!ok) + break; + TXT_TO_STR("payname", field, row->payname); + HISTORYDATEFLDS(res, i, row, ok); if (!ok) break; From 9660f55c27239edc3dfc4d7c29f8665dbf5e96cc Mon Sep 17 00:00:00 2001 From: kanoi Date: Fri, 2 Oct 2015 16:01:32 +1000 Subject: [PATCH 37/62] ckdb/php - define a limit for multi-address accounts --- pool/page_addrmgt.php | 46 ++++++++++++++++++++++++++++--------------- src/ckdb.h | 5 ++++- src/ckdb_cmd.c | 27 +++++++++++++++++++++---- src/ckdb_dbio.c | 17 +++++++++------- 4 files changed, 67 insertions(+), 28 deletions(-) diff --git a/pool/page_addrmgt.php b/pool/page_addrmgt.php index fe886b44..7e4c639b 100644 --- a/pool/page_addrmgt.php +++ b/pool/page_addrmgt.php @@ -10,6 +10,7 @@ function addrmgtuser($data, $user, $err) $pg .= makeForm('addrmgt'); $pg .= "
AddressIDRatio%
'; - $pg .= ""; + $pg .= ""; + $pg .= ''; + $pg .= ""; $pg .= ''; $pg .= ""; @@ -45,7 +49,11 @@ function addrmgtuser($data, $user, $err) $pg .= "
'; - $pg .= ""; + $pg .= ""; + $pg .= ''; + $pg .= ""; $pg .= ''; @@ -63,7 +71,7 @@ function addrmgtuser($data, $user, $err) else $row = 'odd'; $pg .= "
'; + $pg .= ''; $pg .= ""; $pg .= '
'; - $pg .= 'Password: '; - $pg .= ' 
Password: 
'; - $pg .= '*2nd Authentication: '; - $pg .= '
*2nd Authentication:'; + $pg .= '  
"; + $pg .= "
"; $pg .= "*Leave blank if you haven't enabled it
"; $pg .= 'You must enter your password to save changes
'; $pg .= 'A ratio of 0, will remove the address from the payouts
AddressIDSharesDiffInvalid
'.$all[$i]['payaddress'].''.$all[$i]['payname'].'
Total: $shareacc
'; + $pg .= '
'; $pg .= "
Payout est if block found at 100%: ~$btc BTC"; $pg .= '
\n"; $pg .= ''; + $pg .= ''; $pg .= ''; $pg .= ''; $pg .= ''; @@ -18,6 +19,7 @@ function addrmgtuser($data, $user, $err) # new row template for '+' $pg .= ''; + $pg .= ''; $pg .= ''; @@ -38,7 +40,12 @@ function addrmgtuser($data, $user, $err) $count = 0; if ($ans['STATUS'] == 'ok') { + if (isset($ans['limit'])) + $limit = $ans['limit']; + else + $limit = 1; $count = $ans['rows']; + # this will output any DB rows > limit but DB update will ignore extras for ($i = 0; $i < $count; $i++) { if ((($offset) % 2) == 0) @@ -48,6 +55,7 @@ function addrmgtuser($data, $user, $err) $pg .= ""; $addr = $ans['addr:'.$i]; + $pg .= ''; $pg .= ''; @@ -66,20 +74,22 @@ function addrmgtuser($data, $user, $err) $offset++; } - if ((($offset++) % 2) == 0) - $row = 'even'; - else - $row = 'odd'; - $pg .= ""; - $pg .= ''; + if ($offset < $limit) { + if ((($offset++) % 2) == 0) + $row = 'even'; + else + $row = 'odd'; + $pg .= ""; + } if ((($offset++) % 2) == 0) $row = 'even'; else $row = 'odd'; $pg .= ""; + $pg .= ''; $pg .= ''; $pg .= ''; $pg .= ''; @@ -89,28 +99,32 @@ function addrmgtuser($data, $user, $err) else $row = 'odd'; $pg .= ""; + $pg .= ''; $pg .= ''; $pg .= ''; $pg .= ''; - $pg .= "'; + $pg .= 'A ratio of 0, will remove the address from the payouts'; } $pg .= "
#AddressIDRatio
'; $pg .= ""; $pg .= '
'.($i+1).''; $pg .= ""; $pg .= '
'; - $pg .= ""; - $pg .= '
"; + $pg .= ""; + $pg .= "limit $limit
 Password: 
 *2nd Authentication:'; $pg .= '  
"; + $pg .= '
'; $pg .= "*Leave blank if you haven't enabled it
"; $pg .= 'You must enter your password to save changes
'; - $pg .= 'A ratio of 0, will remove the address from the payouts
\n"; + # TODO - adrw() update the odd/even class for the new row and rows below it + # TODO - move the js functions into inc.php $pg .= "\n"; + } + + return $pg; +} +# +function show_luck($info, $page, $menu, $name, $user) +{ + gopage($info, NULL, 'doluck', $page, $menu, $name, $user); +} +# +?> diff --git a/pool/prime.php b/pool/prime.php index 5b774e41..86e142b1 100644 --- a/pool/prime.php +++ b/pool/prime.php @@ -86,7 +86,8 @@ function check() 'Stats' => 'stats', 'Blocks' => 'blocks', 'Graph' => 'psperf', - 'Acclaim' => 'userinfo' + 'Acclaim' => 'userinfo', + 'Luck' => 'luck' ), 'Admin' => NULL, 'gap' => array( # options not shown From f35c77fa59dbcd8170355abea9bc3af33bb1c900 Mon Sep 17 00:00:00 2001 From: kanoi Date: Sun, 25 Oct 2015 00:58:01 +1100 Subject: [PATCH 41/62] php - set better visual defaults --- pool/page_luck.php | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/pool/page_luck.php b/pool/page_luck.php index 411d6b85..9a806743 100644 --- a/pool/page_luck.php +++ b/pool/page_luck.php @@ -29,12 +29,6 @@ function doluck($data, $user) else $ans = getBlocks($user); - $iCrap = strpos($_SERVER['HTTP_USER_AGENT'],'iP'); - if ($iCrap) - $vlines = false; - else - $vlines = true; - $pg = '

Pool Avg Block Luck History


'; if ($ans['STATUS'] == 'ok' and isset($ans['rows']) and $ans['rows'] > 0) @@ -53,9 +47,7 @@ function doluck($data, $user) 'tkey' => 'time key', 'tlines' => 'time lines', 'over' => 'key overlap', 'smooth' => 'smooth', 'utc' => 'utc'); - $xon = array('skey' => 1, 'utc' => 1); - if ($vlines === true) - $xon['slines'] = 1; + $xon = array('skey' => 1, 'tkey' => 1, 'tlines' => 1, 'utc' => 1); $pg .= '
'; foreach ($cbx as $nam => $txt) From 594793645a7ef46ffebbdcddf821f5ae88dcfa55 Mon Sep 17 00:00:00 2001 From: kanoi Date: Mon, 26 Oct 2015 23:53:04 +1100 Subject: [PATCH 42/62] php - add 'h' for hours on the graph time key --- pool/inc.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pool/inc.php b/pool/inc.php index e75395c3..cf29fbef 100644 --- a/pool/inc.php +++ b/pool/inc.php @@ -29,7 +29,7 @@ function gfl(c){c['ctx'].fill()} function gst(c){c['ctx'].stroke()} function gfi(c){gle(c);gst(c)} function gbd(c){gbe(c,0,0);gln(c,1,0);gln(c,1,1);gln(c,0,1);gle(c);gfl(c);gst(c)} -function ggr(c,xs,ys,yt,xn,x0,x1,y0,y1,ar,nx,vx,vy,av,w,cols){gtso(c,xs,ys);gss(c,'black');glw(c,1.5);gbe(c,0,1);gln(c,0,0);gln(c,1,0);gst(c);glw(c,0.2);var hi=c['ctx'].measureText('M').width, wi=c['ctx'].measureText('0').width;for(var i=0;i<11;i++){var y=i/10.0;gbe(c,-0.01,y);gln(c,1,y);gst(c);var t=''+(((y1-y0)*i/10+y0).toFixed(2));gfz(c,0,y,-wi,0,t,'black','end')}gfz(c,gx0(c),0.55,wi,0,yt,'#0080ff','left');var m=Math.round(0.5+xn/20.0);for(var i=0;i=x0;i-=hrs){var xo=(i-x0)/(x1-x0);if(c['hrs'][c['hr']]<=48){n=dfmt(c,i)}else{n=dfmtm(c,i)}if(xo<=1 && c['tkey'] && ((l%hlv)==0)){gbe(c,xo,0);gln(c,xo,-0.02);gst(c);gfz(c,xo,0,0,-hi*tpos,n,'brown','center')}if(xo<=1 && c['tlines']){gbe(c,xo,0);gln(c,xo,1);gst(c)}l++}}glw(c,1);if(c['smooth']){for(var j=1;j0 && c['lin'+j]){gss(c,'red');var y=(av[j-1]-y0)/(y1-y0);gbe(c,0,y);gln(c,1,y);gst(c);var t=''+av[j-1].toFixed(2)+'av';gfz(c,1,y,1,0,t,cols[j-1],'left')}}if(c['tkey']){var i,dsp;i=c['hrs'][c['hr']];if(i < 24){dsp=''+i}else{dsp=''+(i/24)+'d'}gfz(c,1,0,c['xo']-c['pxe'],hi,dsp,'red','end');i=c['hln'][c['hl']];gfz(c,1,0,c['xo']-c['pxe'],hi*3,''+i,'red','end')}} +function ggr(c,xs,ys,yt,xn,x0,x1,y0,y1,ar,nx,vx,vy,av,w,cols){gtso(c,xs,ys);gss(c,'black');glw(c,1.5);gbe(c,0,1);gln(c,0,0);gln(c,1,0);gst(c);glw(c,0.2);var hi=c['ctx'].measureText('M').width, wi=c['ctx'].measureText('0').width;for(var i=0;i<11;i++){var y=i/10.0;gbe(c,-0.01,y);gln(c,1,y);gst(c);var t=''+(((y1-y0)*i/10+y0).toFixed(2));gfz(c,0,y,-wi,0,t,'black','end')}gfz(c,gx0(c),0.55,wi,0,yt,'#0080ff','left');var m=Math.round(0.5+xn/20.0);for(var i=0;i=x0;i-=hrs){var xo=(i-x0)/(x1-x0);if(c['hrs'][c['hr']]<=48){n=dfmt(c,i)}else{n=dfmtm(c,i)}if(xo<=1 && c['tkey'] && ((l%hlv)==0)){gbe(c,xo,0);gln(c,xo,-0.02);gst(c);gfz(c,xo,0,0,-hi*tpos,n,'brown','center')}if(xo<=1 && c['tlines']){gbe(c,xo,0);gln(c,xo,1);gst(c)}l++}}glw(c,1);if(c['smooth']){for(var j=1;j0 && c['lin'+j]){gss(c,'red');var y=(av[j-1]-y0)/(y1-y0);gbe(c,0,y);gln(c,1,y);gst(c);var t=''+av[j-1].toFixed(2)+'av';gfz(c,1,y,1,0,t,cols[j-1],'left')}}if(c['tkey']){var i,dsp;i=c['hrs'][c['hr']];if(i < 24){dsp=''+i+'h'}else{dsp=''+(i/24)+'d'}gfz(c,1,0,c['xo']-c['pxe'],hi,dsp,'red','end');i=c['hln'][c['hl']];gfz(c,1,0,c['xo']-c['pxe'],hi*3,''+i,'red','end')}} function sn(i,shi){if(shi.indexOf(' Shift ')<0){return ''+(i%10)}else{return shi.replace(/.* ([a-z])[a-z]*$/,'$1')}} function gc2(c){var div=document.getElementById('can0');while (div.firstChild){div.removeChild(div.firstChild)}c['can']=document.createElement('canvas');c['can'].id='can';c['xo']=0.0;c['yo']=0.0;c['ctx']=c['can'].getContext('2d');c['ctx'].canvas.width=c['xm']+1;c['ctx'].canvas.height=c['ym']+1;div.appendChild(c['can']);c['pxe']=Math.max(Math.round(c['xm']/250),1)} function gc(c){c['wx']=window.innerWidth;c['wy']=window.innerHeight;c['xm']=Math.max(Math.round(c['wx']*0.9+0.5),400);c['ym']=Math.max(Math.round(c['wy']*0.8+0.5),400);if(c['ym']>c['xm']){c['ym']=c['xm']}gc2(c)} From eedbd3e2adc1a5f165ce37831574f281936b0c9d Mon Sep 17 00:00:00 2001 From: kanoi Date: Tue, 27 Oct 2015 02:14:07 +1100 Subject: [PATCH 43/62] ckdb - use psql cursors for larger tables --- src/ckdb.h | 2 +- src/ckdb_dbio.c | 935 ++++++++++++++++++++++++++++-------------------- 2 files changed, 549 insertions(+), 388 deletions(-) diff --git a/src/ckdb.h b/src/ckdb.h index 204d9ee9..ab48fccb 100644 --- a/src/ckdb.h +++ b/src/ckdb.h @@ -55,7 +55,7 @@ #define DB_VLOCK "1" #define DB_VERSION "1.0.4" -#define CKDB_VERSION DB_VERSION"-1.403" +#define CKDB_VERSION DB_VERSION"-1.500" #define WHERE_FFL " - from %s %s() line %d" #define WHERE_FFL_HERE __FILE__, __func__, __LINE__ diff --git a/src/ckdb_dbio.c b/src/ckdb_dbio.c index 6d1277ca..6319aed4 100644 --- a/src/ckdb_dbio.c +++ b/src/ckdb_dbio.c @@ -41,8 +41,13 @@ char *pqerrmsg(PGconn *conn) if (__col == -1) { \ LOGERR("%s(): Unknown field '%s' row %d", __func__, __name, __row); \ __ok = false; \ - } else \ + } else { \ __fld = PQgetvalue(__res, __row, __col); \ + if (__fld == NULL) { \ + LOGERR("%s(): Invalid field '%s' or row %d", __func__, __name, __row); \ + __ok = false; \ + }\ + } \ } while (0) // HISTORY FIELDS @@ -1561,26 +1566,44 @@ bool workers_fill(PGconn *conn) ExecStatusType rescode; PGresult *res; K_ITEM *item; - int n, i; + int n, t, i; WORKERS *row; char *field; char *sel; int fields = 7; - bool ok; + bool ok = false; LOGDEBUG("%s(): select", __func__); - sel = "select " + sel = "declare wk cursor for select " "userid,workername,difficultydefault," "idlenotificationenabled,idlenotificationtime,workerbits" HISTORYDATECONTROL ",workerid from workers"; + res = PQexec(conn, "Begin", CKPQ_READ); + rescode = PQresultStatus(res); + PQclear(res); + if (!PGOK(rescode)) { + PGLOGERR("Begin", rescode, conn); + return false; + } + res = PQexec(conn, sel, CKPQ_READ); rescode = PQresultStatus(res); + PQclear(res); if (!PGOK(rescode)) { - PGLOGERR("Select", rescode, conn); + PGLOGERR("Declare", rescode, conn); + goto flail; + } + + LOGDEBUG("%s(): fetching ...", __func__); + + res = PQexec(conn, "fetch 1 in wk", CKPQ_READ); + rescode = PQresultStatus(res); + if (!PGOK(rescode)) { + PGLOGERR("Fetch first", rescode, conn); PQclear(res); - return false; + goto flail; } n = PQnfields(res); @@ -1588,81 +1611,95 @@ bool workers_fill(PGconn *conn) LOGERR("%s(): Invalid field count - should be %d, but is %d", __func__, fields + HISTORYDATECOUNT, n); PQclear(res); - return false; + goto flail; } - n = PQntuples(res); - LOGDEBUG("%s(): tree build count %d", __func__, n); + n = 0; ok = true; K_WLOCK(workers_free); - for (i = 0; i < n; i++) { - item = k_unlink_head(workers_free); - DATA_WORKERS(row, item); - bzero(row, sizeof(*row)); - - if (everyone_die) { - ok = false; - break; - } + while ((t = PQntuples(res)) > 0) { + for (i = 0; i < t; i++) { + item = k_unlink_head(workers_free); + DATA_WORKERS(row, item); + bzero(row, sizeof(*row)); + + if (everyone_die) { + ok = false; + break; + } - PQ_GET_FLD(res, i, "userid", field, ok); - if (!ok) - break; - TXT_TO_BIGINT("userid", field, row->userid); + PQ_GET_FLD(res, i, "userid", field, ok); + if (!ok) + break; + TXT_TO_BIGINT("userid", field, row->userid); - PQ_GET_FLD(res, i, "workername", field, ok); - if (!ok) - break; - TXT_TO_STR("workername", field, row->workername); + PQ_GET_FLD(res, i, "workername", field, ok); + if (!ok) + break; + TXT_TO_STR("workername", field, row->workername); - PQ_GET_FLD(res, i, "difficultydefault", field, ok); - if (!ok) - break; - TXT_TO_INT("difficultydefault", field, row->difficultydefault); + PQ_GET_FLD(res, i, "difficultydefault", field, ok); + if (!ok) + break; + TXT_TO_INT("difficultydefault", field, row->difficultydefault); - PQ_GET_FLD(res, i, "idlenotificationenabled", field, ok); - if (!ok) - break; - TXT_TO_STR("idlenotificationenabled", field, row->idlenotificationenabled); + PQ_GET_FLD(res, i, "idlenotificationenabled", field, ok); + if (!ok) + break; + TXT_TO_STR("idlenotificationenabled", field, row->idlenotificationenabled); - PQ_GET_FLD(res, i, "idlenotificationtime", field, ok); - if (!ok) - break; - TXT_TO_INT("idlenotificationtime", field, row->idlenotificationtime); + PQ_GET_FLD(res, i, "idlenotificationtime", field, ok); + if (!ok) + break; + TXT_TO_INT("idlenotificationtime", field, row->idlenotificationtime); - PQ_GET_FLD(res, i, "workerbits", field, ok); - if (!ok) - break; - TXT_TO_BIGINT("workerbits", field, row->workerbits); + PQ_GET_FLD(res, i, "workerbits", field, ok); + if (!ok) + break; + TXT_TO_BIGINT("workerbits", field, row->workerbits); - HISTORYDATEFLDS(res, i, row, ok); - if (!ok) - break; + HISTORYDATEFLDS(res, i, row, ok); + if (!ok) + break; - PQ_GET_FLD(res, i, "workerid", field, ok); - if (!ok) + PQ_GET_FLD(res, i, "workerid", field, ok); + if (!ok) + break; + TXT_TO_BIGINT("workerid", field, row->workerid); + + add_to_ktree(workers_root, item); + k_add_head(workers_store, item); + + /* Make sure a workerstatus exists for each worker + * This is to ensure that code can use the workerstatus tree + * to reference other tables and not miss workers in the + * other tables */ + find_create_workerstatus(false, row->userid, row->workername, + __FILE__, __func__, __LINE__); + tick(); + n++; + } + PQclear(res); + res = PQexec(conn, "fetch 9999 in wk", CKPQ_READ); + rescode = PQresultStatus(res); + if (!PGOK(rescode)) { + PGLOGERR("Fetch next", rescode, conn); + ok = false; break; - TXT_TO_BIGINT("workerid", field, row->workerid); - - add_to_ktree(workers_root, item); - k_add_head(workers_store, item); - - /* Make sure a workerstatus exists for each worker - * This is to ensure that code can use the workerstatus tree - * to reference other tables and not miss workers in the - * other tables */ - find_create_workerstatus(false, row->userid, row->workername, - __FILE__, __func__, __LINE__); + } } if (!ok) k_add_head(workers_free, item); K_WUNLOCK(workers_free); PQclear(res); +flail: + res = PQexec(conn, "Commit", CKPQ_READ); + PQclear(res); if (ok) { LOGDEBUG("%s(): built", __func__); - LOGWARNING("%s(): loaded %d workers records", __func__, n); + LOGWARNING("%s(): fetched %d workers records", __func__, n); } return ok; @@ -2160,25 +2197,43 @@ bool payments_fill(PGconn *conn) PGresult *res; K_ITEM *item; PAYMENTS *row; - int n, i; + int n, t, i; char *field; char *sel; int fields = 11; - bool ok; + bool ok = false; LOGDEBUG("%s(): select", __func__); - sel = "select " + sel = "declare ps cursor for select " "paymentid,payoutid,userid,subname,paydate,payaddress," "originaltxn,amount,diffacc,committxn,commitblockhash" HISTORYDATECONTROL " from payments"; + res = PQexec(conn, "Begin", CKPQ_READ); + rescode = PQresultStatus(res); + PQclear(res); + if (!PGOK(rescode)) { + PGLOGERR("Begin", rescode, conn); + return false; + } + res = PQexec(conn, sel, CKPQ_READ); rescode = PQresultStatus(res); + PQclear(res); if (!PGOK(rescode)) { - PGLOGERR("Select", rescode, conn); + PGLOGERR("Declare", rescode, conn); + goto flail; + } + + LOGDEBUG("%s(): fetching ...", __func__); + + res = PQexec(conn, "fetch 1 in ps", CKPQ_READ); + rescode = PQresultStatus(res); + if (!PGOK(rescode)) { + PGLOGERR("Fetch first", rescode, conn); PQclear(res); - return false; + goto flail; } n = PQnfields(res); @@ -2186,94 +2241,108 @@ bool payments_fill(PGconn *conn) LOGERR("%s(): Invalid field count - should be %d, but is %d", __func__, fields + HISTORYDATECOUNT, n); PQclear(res); - return false; + goto flail; } - n = PQntuples(res); - LOGDEBUG("%s(): tree build count %d", __func__, n); + n = 0; ok = true; K_WLOCK(payments_free); - for (i = 0; i < n; i++) { - item = k_unlink_head(payments_free); - DATA_PAYMENTS(row, item); - bzero(row, sizeof(*row)); + while ((t = PQntuples(res)) > 0) { + for (i = 0; i < t; i++) { + item = k_unlink_head(payments_free); + DATA_PAYMENTS(row, item); + bzero(row, sizeof(*row)); + + if (everyone_die) { + ok = false; + break; + } - if (everyone_die) { - ok = false; - break; - } + PQ_GET_FLD(res, i, "paymentid", field, ok); + if (!ok) + break; + TXT_TO_BIGINT("paymentid", field, row->paymentid); - PQ_GET_FLD(res, i, "paymentid", field, ok); - if (!ok) - break; - TXT_TO_BIGINT("paymentid", field, row->paymentid); + PQ_GET_FLD(res, i, "payoutid", field, ok); + if (!ok) + break; + TXT_TO_BIGINT("payoutid", field, row->payoutid); - PQ_GET_FLD(res, i, "payoutid", field, ok); - if (!ok) - break; - TXT_TO_BIGINT("payoutid", field, row->payoutid); + PQ_GET_FLD(res, i, "userid", field, ok); + if (!ok) + break; + TXT_TO_BIGINT("userid", field, row->userid); - PQ_GET_FLD(res, i, "userid", field, ok); - if (!ok) - break; - TXT_TO_BIGINT("userid", field, row->userid); + PQ_GET_FLD(res, i, "subname", field, ok); + if (!ok) + break; + TXT_TO_STR("subname", field, row->subname); - PQ_GET_FLD(res, i, "subname", field, ok); - if (!ok) - break; - TXT_TO_STR("subname", field, row->subname); + PQ_GET_FLD(res, i, "paydate", field, ok); + if (!ok) + break; + TXT_TO_TV("paydate", field, row->paydate); - PQ_GET_FLD(res, i, "paydate", field, ok); - if (!ok) - break; - TXT_TO_TV("paydate", field, row->paydate); + PQ_GET_FLD(res, i, "payaddress", field, ok); + if (!ok) + break; + TXT_TO_STR("payaddress", field, row->payaddress); - PQ_GET_FLD(res, i, "payaddress", field, ok); - if (!ok) - break; - TXT_TO_STR("payaddress", field, row->payaddress); + PQ_GET_FLD(res, i, "originaltxn", field, ok); + if (!ok) + break; + TXT_TO_STR("originaltxn", field, row->originaltxn); - PQ_GET_FLD(res, i, "originaltxn", field, ok); - if (!ok) - break; - TXT_TO_STR("originaltxn", field, row->originaltxn); + PQ_GET_FLD(res, i, "amount", field, ok); + if (!ok) + break; + TXT_TO_BIGINT("amount", field, row->amount); - PQ_GET_FLD(res, i, "amount", field, ok); - if (!ok) - break; - TXT_TO_BIGINT("amount", field, row->amount); + PQ_GET_FLD(res, i, "diffacc", field, ok); + if (!ok) + break; + TXT_TO_DOUBLE("diffacc", field, row->diffacc); - PQ_GET_FLD(res, i, "diffacc", field, ok); - if (!ok) - break; - TXT_TO_DOUBLE("diffacc", field, row->diffacc); + PQ_GET_FLD(res, i, "committxn", field, ok); + if (!ok) + break; + TXT_TO_STR("committxn", field, row->committxn); - PQ_GET_FLD(res, i, "committxn", field, ok); - if (!ok) - break; - TXT_TO_STR("committxn", field, row->committxn); + PQ_GET_FLD(res, i, "commitblockhash", field, ok); + if (!ok) + break; + TXT_TO_STR("commitblockhash", field, row->commitblockhash); - PQ_GET_FLD(res, i, "commitblockhash", field, ok); - if (!ok) - break; - TXT_TO_STR("commitblockhash", field, row->commitblockhash); + HISTORYDATEFLDS(res, i, row, ok); + if (!ok) + break; - HISTORYDATEFLDS(res, i, row, ok); - if (!ok) + add_to_ktree(payments_root, item); + k_add_head(payments_store, item); + tick(); + n++; + } + PQclear(res); + res = PQexec(conn, "fetch 9999 in ps", CKPQ_READ); + rescode = PQresultStatus(res); + if (!PGOK(rescode)) { + PGLOGERR("Fetch next", rescode, conn); + ok = false; break; - - add_to_ktree(payments_root, item); - k_add_head(payments_store, item); + } } if (!ok) k_add_head(payments_free, item); K_WUNLOCK(payments_free); PQclear(res); +flail: + res = PQexec(conn, "Commit", CKPQ_READ); + PQclear(res); if (ok) { LOGDEBUG("%s(): built", __func__); - LOGWARNING("%s(): loaded %d payments records", __func__, n); + LOGWARNING("%s(): fetched %d payments records", __func__, n); } return ok; @@ -2772,12 +2841,12 @@ bool workinfo_fill(PGconn *conn) K_ITEM *item; WORKINFO *row; char *params[3]; - int n, i, par = 0; + int n, t, i, par = 0; char *field; char *sel = NULL; size_t len, off; int fields = 10; - bool ok; + bool ok = false; LOGDEBUG("%s(): select", __func__); @@ -2790,7 +2859,7 @@ bool workinfo_fill(PGconn *conn) APPEND_REALLOC_INIT(sel, off, len); APPEND_REALLOC(sel, off, len, - "select " + "declare wi cursor for select " // "workinfoid,poolinstance,transactiontree,merklehash,prevhash," "workinfoid,poolinstance,merklehash,prevhash," "coinbase1,coinbase2,version,bits,ntime,reward" @@ -2820,12 +2889,31 @@ bool workinfo_fill(PGconn *conn) params[par++] = bigint_to_buf(dbload_workinfoid_start, NULL, 0); params[par++] = bigint_to_buf(dbload_workinfoid_finish, NULL, 0); PARCHK(par, params); + + res = PQexec(conn, "Begin", CKPQ_READ); + rescode = PQresultStatus(res); + PQclear(res); + if (!PGOK(rescode)) { + PGLOGERR("Begin", rescode, conn); + return false; + } + res = PQexecParams(conn, sel, par, NULL, (const char **)params, NULL, NULL, 0, CKPQ_READ); rescode = PQresultStatus(res); + PQclear(res); + if (!PGOK(rescode)) { + PGLOGERR("Declare", rescode, conn); + goto flail; + } + + LOGDEBUG("%s(): fetching ...", __func__); + + res = PQexec(conn, "fetch 1 in wi", CKPQ_READ); + rescode = PQresultStatus(res); if (!PGOK(rescode)) { - PGLOGERR("Select", rescode, conn); + PGLOGERR("Fetch first", rescode, conn); PQclear(res); - return false; + goto flail; } n = PQnfields(res); @@ -2833,106 +2921,115 @@ bool workinfo_fill(PGconn *conn) LOGERR("%s(): Invalid field count - should be %d, but is %d", __func__, fields + HISTORYDATECOUNT, n); PQclear(res); - return false; + goto flail; } - n = PQntuples(res); - LOGDEBUG("%s(): tree build count %d", __func__, n); + n = 0; ok = true; //K_WLOCK(workinfo_free); - for (i = 0; i < n; i++) { - item = k_unlink_head(workinfo_free); - DATA_WORKINFO(row, item); - bzero(row, sizeof(*row)); - - if (everyone_die) { - ok = false; - break; - } + while ((t = PQntuples(res)) > 0) { + for (i = 0; i < t; i++) { + item = k_unlink_head(workinfo_free); + DATA_WORKINFO(row, item); + bzero(row, sizeof(*row)); + + if (everyone_die) { + ok = false; + break; + } - PQ_GET_FLD(res, i, "workinfoid", field, ok); - if (!ok) - break; - TXT_TO_BIGINT("workinfoid", field, row->workinfoid); + PQ_GET_FLD(res, i, "workinfoid", field, ok); + if (!ok) + break; + TXT_TO_BIGINT("workinfoid", field, row->workinfoid); - PQ_GET_FLD(res, i, "poolinstance", field, ok); - if (!ok) - break; - TXT_TO_STR("poolinstance", field, row->poolinstance); + PQ_GET_FLD(res, i, "poolinstance", field, ok); + if (!ok) + break; + TXT_TO_STR("poolinstance", field, row->poolinstance); /* Not currently needed in RAM - PQ_GET_FLD(res, i, "transactiontree", field, ok); - if (!ok) - break; - TXT_TO_BLOB("transactiontree", field, row->transactiontree); - LIST_MEM_ADD(workinfo_free, row->transactiontree); + PQ_GET_FLD(res, i, "transactiontree", field, ok); + if (!ok) + break; + TXT_TO_BLOB("transactiontree", field, row->transactiontree); + LIST_MEM_ADD(workinfo_free, row->transactiontree); */ - row->transactiontree = EMPTY; + row->transactiontree = EMPTY; - PQ_GET_FLD(res, i, "merklehash", field, ok); - if (!ok) - break; - TXT_TO_BLOB("merklehash", field, row->merklehash); - LIST_MEM_ADD(workinfo_free, row->merklehash); + PQ_GET_FLD(res, i, "merklehash", field, ok); + if (!ok) + break; + TXT_TO_BLOB("merklehash", field, row->merklehash); + LIST_MEM_ADD(workinfo_free, row->merklehash); - PQ_GET_FLD(res, i, "prevhash", field, ok); - if (!ok) - break; - TXT_TO_STR("prevhash", field, row->prevhash); + PQ_GET_FLD(res, i, "prevhash", field, ok); + if (!ok) + break; + TXT_TO_STR("prevhash", field, row->prevhash); - PQ_GET_FLD(res, i, "coinbase1", field, ok); - if (!ok) - break; - TXT_TO_STR("coinbase1", field, row->coinbase1); + PQ_GET_FLD(res, i, "coinbase1", field, ok); + if (!ok) + break; + TXT_TO_STR("coinbase1", field, row->coinbase1); - PQ_GET_FLD(res, i, "coinbase2", field, ok); - if (!ok) - break; - TXT_TO_STR("coinbase2", field, row->coinbase2); + PQ_GET_FLD(res, i, "coinbase2", field, ok); + if (!ok) + break; + TXT_TO_STR("coinbase2", field, row->coinbase2); - PQ_GET_FLD(res, i, "version", field, ok); - if (!ok) - break; - TXT_TO_STR("version", field, row->version); + PQ_GET_FLD(res, i, "version", field, ok); + if (!ok) + break; + TXT_TO_STR("version", field, row->version); - PQ_GET_FLD(res, i, "bits", field, ok); - if (!ok) - break; - TXT_TO_STR("bits", field, row->bits); + PQ_GET_FLD(res, i, "bits", field, ok); + if (!ok) + break; + TXT_TO_STR("bits", field, row->bits); - PQ_GET_FLD(res, i, "ntime", field, ok); - if (!ok) - break; - TXT_TO_STR("ntime", field, row->ntime); + PQ_GET_FLD(res, i, "ntime", field, ok); + if (!ok) + break; + TXT_TO_STR("ntime", field, row->ntime); - PQ_GET_FLD(res, i, "reward", field, ok); - if (!ok) - break; - TXT_TO_BIGINT("reward", field, row->reward); - pool.reward = row->reward; + PQ_GET_FLD(res, i, "reward", field, ok); + if (!ok) + break; + TXT_TO_BIGINT("reward", field, row->reward); + pool.reward = row->reward; - HISTORYDATEFLDS(res, i, row, ok); - if (!ok) - break; + HISTORYDATEFLDS(res, i, row, ok); + if (!ok) + break; - add_to_ktree(workinfo_root, item); - if (!confirm_sharesummary) - add_to_ktree(workinfo_height_root, item); - k_add_head(workinfo_store, item); + add_to_ktree(workinfo_root, item); + if (!confirm_sharesummary) + add_to_ktree(workinfo_height_root, item); + k_add_head(workinfo_store, item); - if (tv_newer(&(dbstatus.newest_createdate_workinfo), &(row->createdate))) { - copy_tv(&(dbstatus.newest_createdate_workinfo), &(row->createdate)); - dbstatus.newest_workinfoid = row->workinfoid; - } + if (tv_newer(&(dbstatus.newest_createdate_workinfo), &(row->createdate))) { + copy_tv(&(dbstatus.newest_createdate_workinfo), &(row->createdate)); + dbstatus.newest_workinfoid = row->workinfoid; + } - if (i == 0 || ((i+1) % 100000) == 0) { - printf(TICK_PREFIX"wi "); - pcom(i+1); - putchar('\r'); - fflush(stdout); + if (n == 0 || ((n+1) % 100000) == 0) { + printf(TICK_PREFIX"wi "); + pcom(n+1); + putchar('\r'); + fflush(stdout); + } + tick(); + n++; + } + PQclear(res); + res = PQexec(conn, "fetch 9999 in wi", CKPQ_READ); + rescode = PQresultStatus(res); + if (!PGOK(rescode)) { + PGLOGERR("Fetch next", rescode, conn); + ok = false; + break; } - - tick(); } if (!ok) { free_workinfo_data(item); @@ -2942,10 +3039,14 @@ bool workinfo_fill(PGconn *conn) //K_WUNLOCK(workinfo_free); PQclear(res); +flail: + res = PQexec(conn, "Commit", CKPQ_READ); + PQclear(res); + if (ok) { LOGDEBUG("%s(): built", __func__); - LOGWARNING("%s(): loaded %d workinfo records", __func__, n); + LOGWARNING("%s(): fetched %d workinfo records", __func__, n); } return ok; @@ -5223,24 +5324,42 @@ bool miningpayouts_fill(PGconn *conn) PGresult *res; K_ITEM *item; MININGPAYOUTS *row; - int n, i; + int n, t, i; char *field; char *sel; int fields = 4; - bool ok; + bool ok = false; LOGDEBUG("%s(): select", __func__); - sel = "select " + sel = "declare mp cursor for select " "payoutid,userid,diffacc,amount" HISTORYDATECONTROL " from miningpayouts"; + res = PQexec(conn, "Begin", CKPQ_READ); + rescode = PQresultStatus(res); + PQclear(res); + if (!PGOK(rescode)) { + PGLOGERR("Begin", rescode, conn); + return false; + } + res = PQexec(conn, sel, CKPQ_READ); rescode = PQresultStatus(res); + PQclear(res); if (!PGOK(rescode)) { - PGLOGERR("Select", rescode, conn); + PGLOGERR("Declare", rescode, conn); + goto flail; + } + + LOGDEBUG("%s(): fetching ...", __func__); + + res = PQexec(conn, "fetch 1 in mp", CKPQ_READ); + rescode = PQresultStatus(res); + if (!PGOK(rescode)) { + PGLOGERR("Fetch first", rescode, conn); PQclear(res); - return false; + goto flail; } n = PQnfields(res); @@ -5248,61 +5367,73 @@ bool miningpayouts_fill(PGconn *conn) LOGERR("%s(): Invalid field count - should be %d, but is %d", __func__, fields + HISTORYDATECOUNT, n); PQclear(res); - return false; + goto flail; } - n = PQntuples(res); - LOGDEBUG("%s(): tree build count %d", __func__, n); + n = 0; ok = true; K_WLOCK(miningpayouts_free); - for (i = 0; i < n; i++) { - item = k_unlink_head(miningpayouts_free); - DATA_MININGPAYOUTS(row, item); - bzero(row, sizeof(*row)); + while ((t = PQntuples(res)) > 0) { + for (i = 0; i < t; i++) { + item = k_unlink_head(miningpayouts_free); + DATA_MININGPAYOUTS(row, item); + bzero(row, sizeof(*row)); + + if (everyone_die) { + ok = false; + break; + } - if (everyone_die) { - ok = false; - break; - } + PQ_GET_FLD(res, i, "payoutid", field, ok); + if (!ok) + break; + TXT_TO_BIGINT("payoutid", field, row->payoutid); - PQ_GET_FLD(res, i, "payoutid", field, ok); - if (!ok) - break; - TXT_TO_BIGINT("payoutid", field, row->payoutid); + PQ_GET_FLD(res, i, "userid", field, ok); + if (!ok) + break; + TXT_TO_BIGINT("userid", field, row->userid); - PQ_GET_FLD(res, i, "userid", field, ok); - if (!ok) - break; - TXT_TO_BIGINT("userid", field, row->userid); + PQ_GET_FLD(res, i, "diffacc", field, ok); + if (!ok) + break; + TXT_TO_DOUBLE("diffacc", field, row->diffacc); - PQ_GET_FLD(res, i, "diffacc", field, ok); - if (!ok) - break; - TXT_TO_DOUBLE("diffacc", field, row->diffacc); + PQ_GET_FLD(res, i, "amount", field, ok); + if (!ok) + break; + TXT_TO_BIGINT("amount", field, row->amount); - PQ_GET_FLD(res, i, "amount", field, ok); - if (!ok) - break; - TXT_TO_BIGINT("amount", field, row->amount); + HISTORYDATEFLDS(res, i, row, ok); + if (!ok) + break; - HISTORYDATEFLDS(res, i, row, ok); - if (!ok) + add_to_ktree(miningpayouts_root, item); + k_add_head(miningpayouts_store, item); + tick(); + n++; + } + PQclear(res); + res = PQexec(conn, "fetch 9999 in mp", CKPQ_READ); + rescode = PQresultStatus(res); + if (!PGOK(rescode)) { + PGLOGERR("Fetch next", rescode, conn); + ok = false; break; - - add_to_ktree(miningpayouts_root, item); - k_add_head(miningpayouts_store, item); - - tick(); + } } if (!ok) k_add_head(miningpayouts_free, item); K_WUNLOCK(miningpayouts_free); PQclear(res); +flail: + res = PQexec(conn, "Commit", CKPQ_READ); + PQclear(res); if (ok) { LOGDEBUG("%s(): built", __func__); - LOGWARNING("%s(): loaded %d miningpayout records", __func__, n); + LOGWARNING("%s(): fetched %d miningpayout records", __func__, n); } return ok; @@ -6521,12 +6652,12 @@ bool markersummary_fill(PGconn *conn) ExecStatusType rescode; PGresult *res; K_ITEM *item, *p_item; - int n, i, p_n; + int n, t, i, p_n; MARKERSUMMARY *row, *p_row; char *field; char *sel; int fields = 20; - bool ok; + bool ok = false; LOGDEBUG("%s(): select", __func__); @@ -6534,19 +6665,37 @@ bool markersummary_fill(PGconn *conn) fflush(stdout); // TODO: limit how far back - sel = "select " + sel = "declare ws cursor for select " "markerid,userid,workername,diffacc,diffsta,diffdup,diffhi," "diffrej,shareacc,sharesta,sharedup,sharehi,sharerej," "sharecount,errorcount,firstshare,lastshare,firstshareacc," "lastshareacc,lastdiffacc" MODIFYDATECONTROL " from markersummary"; + res = PQexec(conn, "Begin", CKPQ_READ); + rescode = PQresultStatus(res); + PQclear(res); + if (!PGOK(rescode)) { + PGLOGERR("Begin", rescode, conn); + return false; + } + res = PQexec(conn, sel, CKPQ_READ); rescode = PQresultStatus(res); + PQclear(res); if (!PGOK(rescode)) { - PGLOGERR("Select", rescode, conn); + PGLOGERR("Declare", rescode, conn); + goto flail; + } + + LOGDEBUG("%s(): fetching ...", __func__); + + res = PQexec(conn, "fetch 1 in ws", CKPQ_READ); + rescode = PQresultStatus(res); + if (!PGOK(rescode)) { + PGLOGERR("Fetch first", rescode, conn); PQclear(res); - return false; + goto flail; } n = PQnfields(res); @@ -6554,169 +6703,178 @@ bool markersummary_fill(PGconn *conn) LOGERR("%s(): Invalid field count - should be %d, but is %d", __func__, fields + MODIFYDATECOUNT, n); PQclear(res); - return false; + goto flail; } - n = PQntuples(res); - LOGDEBUG("%s(): tree build count %d", __func__, n); + n = 0; ok = true; //K_WLOCK(markersummary_free); - for (i = 0; i < n; i++) { - item = k_unlink_head(markersummary_free); - DATA_MARKERSUMMARY(row, item); - bzero(row, sizeof(*row)); - - if (everyone_die) { - ok = false; - break; - } + while ((t = PQntuples(res)) > 0) { + for (i = 0; i < t; i++) { + item = k_unlink_head(markersummary_free); + DATA_MARKERSUMMARY(row, item); + bzero(row, sizeof(*row)); + + if (everyone_die) { + ok = false; + break; + } - PQ_GET_FLD(res, i, "markerid", field, ok); - if (!ok) - break; - TXT_TO_BIGINT("markerid", field, row->markerid); + PQ_GET_FLD(res, i, "markerid", field, ok); + if (!ok) + break; + TXT_TO_BIGINT("markerid", field, row->markerid); - PQ_GET_FLD(res, i, "userid", field, ok); - if (!ok) - break; - TXT_TO_BIGINT("userid", field, row->userid); + PQ_GET_FLD(res, i, "userid", field, ok); + if (!ok) + break; + TXT_TO_BIGINT("userid", field, row->userid); - PQ_GET_FLD(res, i, "workername", field, ok); - if (!ok) - break; - TXT_TO_PTR("workername", field, row->workername); - LIST_MEM_ADD(markersummary_free, row->workername); + PQ_GET_FLD(res, i, "workername", field, ok); + if (!ok) + break; + TXT_TO_PTR("workername", field, row->workername); + LIST_MEM_ADD(markersummary_free, row->workername); - PQ_GET_FLD(res, i, "diffacc", field, ok); - if (!ok) - break; - TXT_TO_DOUBLE("diffacc", field, row->diffacc); + PQ_GET_FLD(res, i, "diffacc", field, ok); + if (!ok) + break; + TXT_TO_DOUBLE("diffacc", field, row->diffacc); - PQ_GET_FLD(res, i, "diffsta", field, ok); - if (!ok) - break; - TXT_TO_DOUBLE("diffsta", field, row->diffsta); + PQ_GET_FLD(res, i, "diffsta", field, ok); + if (!ok) + break; + TXT_TO_DOUBLE("diffsta", field, row->diffsta); - PQ_GET_FLD(res, i, "diffdup", field, ok); - if (!ok) - break; - TXT_TO_DOUBLE("diffdup", field, row->diffdup); + PQ_GET_FLD(res, i, "diffdup", field, ok); + if (!ok) + break; + TXT_TO_DOUBLE("diffdup", field, row->diffdup); - PQ_GET_FLD(res, i, "diffhi", field, ok); - if (!ok) - break; - TXT_TO_DOUBLE("diffhi", field, row->diffhi); + PQ_GET_FLD(res, i, "diffhi", field, ok); + if (!ok) + break; + TXT_TO_DOUBLE("diffhi", field, row->diffhi); - PQ_GET_FLD(res, i, "diffrej", field, ok); - if (!ok) - break; - TXT_TO_DOUBLE("diffrej", field, row->diffrej); + PQ_GET_FLD(res, i, "diffrej", field, ok); + if (!ok) + break; + TXT_TO_DOUBLE("diffrej", field, row->diffrej); - PQ_GET_FLD(res, i, "shareacc", field, ok); - if (!ok) - break; - TXT_TO_DOUBLE("shareacc", field, row->shareacc); + PQ_GET_FLD(res, i, "shareacc", field, ok); + if (!ok) + break; + TXT_TO_DOUBLE("shareacc", field, row->shareacc); - PQ_GET_FLD(res, i, "sharesta", field, ok); - if (!ok) - break; - TXT_TO_DOUBLE("sharesta", field, row->sharesta); + PQ_GET_FLD(res, i, "sharesta", field, ok); + if (!ok) + break; + TXT_TO_DOUBLE("sharesta", field, row->sharesta); - PQ_GET_FLD(res, i, "sharedup", field, ok); - if (!ok) - break; - TXT_TO_DOUBLE("sharedup", field, row->sharedup); + PQ_GET_FLD(res, i, "sharedup", field, ok); + if (!ok) + break; + TXT_TO_DOUBLE("sharedup", field, row->sharedup); - PQ_GET_FLD(res, i, "sharehi", field, ok); - if (!ok) - break; - TXT_TO_DOUBLE("sharehi", field, row->sharehi); + PQ_GET_FLD(res, i, "sharehi", field, ok); + if (!ok) + break; + TXT_TO_DOUBLE("sharehi", field, row->sharehi); - PQ_GET_FLD(res, i, "sharerej", field, ok); - if (!ok) - break; - TXT_TO_DOUBLE("sharerej", field, row->sharerej); + PQ_GET_FLD(res, i, "sharerej", field, ok); + if (!ok) + break; + TXT_TO_DOUBLE("sharerej", field, row->sharerej); - PQ_GET_FLD(res, i, "sharecount", field, ok); - if (!ok) - break; - TXT_TO_BIGINT("sharecount", field, row->sharecount); + PQ_GET_FLD(res, i, "sharecount", field, ok); + if (!ok) + break; + TXT_TO_BIGINT("sharecount", field, row->sharecount); - PQ_GET_FLD(res, i, "errorcount", field, ok); - if (!ok) - break; - TXT_TO_BIGINT("errorcount", field, row->errorcount); + PQ_GET_FLD(res, i, "errorcount", field, ok); + if (!ok) + break; + TXT_TO_BIGINT("errorcount", field, row->errorcount); - PQ_GET_FLD(res, i, "firstshare", field, ok); - if (!ok) - break; - TXT_TO_TV("firstshare", field, row->firstshare); + PQ_GET_FLD(res, i, "firstshare", field, ok); + if (!ok) + break; + TXT_TO_TV("firstshare", field, row->firstshare); - PQ_GET_FLD(res, i, "lastshare", field, ok); - if (!ok) - break; - TXT_TO_TV("lastshare", field, row->lastshare); + PQ_GET_FLD(res, i, "lastshare", field, ok); + if (!ok) + break; + TXT_TO_TV("lastshare", field, row->lastshare); - PQ_GET_FLD(res, i, "firstshareacc", field, ok); - if (!ok) - break; - TXT_TO_TV("firstshareacc", field, row->firstshareacc); + PQ_GET_FLD(res, i, "firstshareacc", field, ok); + if (!ok) + break; + TXT_TO_TV("firstshareacc", field, row->firstshareacc); - PQ_GET_FLD(res, i, "lastshareacc", field, ok); - if (!ok) - break; - TXT_TO_TV("lastshareacc", field, row->lastshareacc); + PQ_GET_FLD(res, i, "lastshareacc", field, ok); + if (!ok) + break; + TXT_TO_TV("lastshareacc", field, row->lastshareacc); - PQ_GET_FLD(res, i, "lastdiffacc", field, ok); - if (!ok) - break; - TXT_TO_DOUBLE("lastdiffacc", field, row->lastdiffacc); + PQ_GET_FLD(res, i, "lastdiffacc", field, ok); + if (!ok) + break; + TXT_TO_DOUBLE("lastdiffacc", field, row->lastdiffacc); - MODIFYDATEFLDPOINTERS(markersummary_free, res, i, row, ok); - if (!ok) - break; + MODIFYDATEFLDPOINTERS(markersummary_free, res, i, row, ok); + if (!ok) + break; - /* Save having to do this everywhere in the code for old data - * It's not always accurate, but soon after when it's not, - * and also what was used before the 2 fields were added */ - if (row->diffacc > 0) { - if (row->firstshareacc.tv_sec == 0L) - copy_tv(&(row->firstshareacc), &(row->firstshare)); - if (row->lastshareacc.tv_sec == 0L) - copy_tv(&(row->lastshareacc), &(row->lastshare)); - } + /* Save having to do this everywhere in the code for old data + * It's not always accurate, but soon after when it's not, + * and also what was used before the 2 fields were added */ + if (row->diffacc > 0) { + if (row->firstshareacc.tv_sec == 0L) + copy_tv(&(row->firstshareacc), &(row->firstshare)); + if (row->lastshareacc.tv_sec == 0L) + copy_tv(&(row->lastshareacc), &(row->lastshare)); + } - add_to_ktree(markersummary_root, item); - add_to_ktree(markersummary_userid_root, item); - k_add_head(markersummary_store, item); - - p_item = find_markersummary_p(row->markerid); - if (!p_item) { - /* N.B. this could be false due to the markerid - * having the wrong status TODO: deal with that? */ - p_item = k_unlink_head(markersummary_free); - DATA_MARKERSUMMARY(p_row, p_item); - bzero(p_row, sizeof(*p_row)); - p_row->markerid = row->markerid; - POOL_MS(p_row); - add_to_ktree(markersummary_pool_root, p_item); - k_add_head(markersummary_pool_store, p_item); - } else { - DATA_MARKERSUMMARY(p_row, p_item); - } + add_to_ktree(markersummary_root, item); + add_to_ktree(markersummary_userid_root, item); + k_add_head(markersummary_store, item); + + p_item = find_markersummary_p(row->markerid); + if (!p_item) { + /* N.B. this could be false due to the markerid + * having the wrong status TODO: deal with that? */ + p_item = k_unlink_head(markersummary_free); + DATA_MARKERSUMMARY(p_row, p_item); + bzero(p_row, sizeof(*p_row)); + p_row->markerid = row->markerid; + POOL_MS(p_row); + add_to_ktree(markersummary_pool_root, p_item); + k_add_head(markersummary_pool_store, p_item); + } else { + DATA_MARKERSUMMARY(p_row, p_item); + } - markersummary_to_pool(p_row, row); + markersummary_to_pool(p_row, row); - _userinfo_update(NULL, NULL, row, false, false); + _userinfo_update(NULL, NULL, row, false, false); - if (i == 0 || ((i+1) % 100000) == 0) { - printf(TICK_PREFIX"ms "); - pcom(i+1); - putchar('\r'); - fflush(stdout); + if (n == 0 || ((n+1) % 100000) == 0) { + printf(TICK_PREFIX"ms "); + pcom(n+1); + putchar('\r'); + fflush(stdout); + } + tick(); + n++; + } + PQclear(res); + res = PQexec(conn, "fetch 9999 in ws", CKPQ_READ); + rescode = PQresultStatus(res); + if (!PGOK(rescode)) { + PGLOGERR("Fetch next", rescode, conn); + ok = false; + break; } - - tick(); } if (!ok) { free_markersummary_data(item); @@ -6727,10 +6885,13 @@ bool markersummary_fill(PGconn *conn) //K_WUNLOCK(markersummary_free); PQclear(res); +flail: + res = PQexec(conn, "Commit", CKPQ_READ); + PQclear(res); if (ok) { LOGDEBUG("%s(): built", __func__); - LOGWARNING("%s(): loaded %d markersummary records", __func__, n); + LOGWARNING("%s(): fetched %d markersummary records", __func__, n); LOGWARNING("%s(): created %d markersummary pool records", __func__, p_n); } From ddaa0f7102701dd7d4e10dfc8ffb2deabe48d855 Mon Sep 17 00:00:00 2001 From: kanoi Date: Tue, 27 Oct 2015 13:16:18 +1100 Subject: [PATCH 44/62] php - adjust the payout help wording --- pool/page_payout.php | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pool/page_payout.php b/pool/page_payout.php index b691f1f4..3bb9a791 100644 --- a/pool/page_payout.php +++ b/pool/page_payout.php @@ -45,8 +45,7 @@ A shift summarises all the shares submitted to the pool for 100 work changes.
and a new shift starts.
A ckpool restart will also end the current shift and start a new shift.
-Also, a network difficulty change will end the current shift and start a new shift.
-The network difficulty changes every 2016 block or about every 2 weeks.
+A network difficulty change will also end the current shift and start a new shift.

So, what's the $n value?

The current Bitcoin network value for $n1d is $nv and thus $n is $nvx
From 48b1947778a82f54e4be1e76b414e89e8fb69847 Mon Sep 17 00:00:00 2001 From: kanoi Date: Sat, 31 Oct 2015 20:16:59 +1100 Subject: [PATCH 45/62] ckdb - store and use workinfo diff and height in the ram structrue --- src/ckdb.c | 8 +++---- src/ckdb.h | 10 ++++---- src/ckdb_cmd.c | 38 ++++++++++------------------- src/ckdb_data.c | 64 ++++++++++++++++++++----------------------------- src/ckdb_dbio.c | 10 ++++++-- 5 files changed, 55 insertions(+), 75 deletions(-) diff --git a/src/ckdb.c b/src/ckdb.c index 041ec653..d13997c7 100644 --- a/src/ckdb.c +++ b/src/ckdb.c @@ -2915,7 +2915,7 @@ static void summarise_blocks() if (workinfo_current) { WORKINFO *wic; DATA_WORKINFO(wic, workinfo_current); - hi = coinbase1height(wic->coinbase1); + hi = wic->height; } K_RUNLOCK(workinfo_free); @@ -3202,7 +3202,6 @@ static void make_a_shift_mark() MARKS *marks = NULL, *sh_marks = NULL; int64_t ss_age_wid, last_marks_wid, marks_wid, prev_wid; int64_t shiftdiffblock = SHIFT_DIFF_BLOCK; - int32_t height; char wi_bits[TXT_SML+1]; bool was_block = false, ok, oc_look = true; char cd_buf[DATE_BUFSIZ], cd_buf2[DATE_BUFSIZ], cd_buf3[DATE_BUFSIZ]; @@ -3452,8 +3451,7 @@ static void make_a_shift_mark() * Stop at the last workinfo, before the diff * changed */ if (strcmp(wi_bits, workinfo->bits) != 0) { - height = coinbase1height(workinfo->coinbase1); - if (height > (int32_t)shiftdiffblock) { + if (workinfo->height > (int32_t)shiftdiffblock) { LOGDEBUG("%s() OK shift stops at diff" " change '%s->%s' %"PRId64 "->%"PRId64" height %"PRId32 @@ -3461,7 +3459,7 @@ static void make_a_shift_mark() __func__, wi_bits, workinfo->bits, prev_wid, workinfo->workinfoid, - height, + workinfo->height, shiftdiffblock); marks_wid = prev_wid; break; diff --git a/src/ckdb.h b/src/ckdb.h index ab48fccb..779b70c2 100644 --- a/src/ckdb.h +++ b/src/ckdb.h @@ -55,7 +55,7 @@ #define DB_VLOCK "1" #define DB_VERSION "1.0.4" -#define CKDB_VERSION DB_VERSION"-1.500" +#define CKDB_VERSION DB_VERSION"-1.501" #define WHERE_FFL " - from %s %s() line %d" #define WHERE_FFL_HERE __FILE__, __func__, __LINE__ @@ -1336,6 +1336,8 @@ typedef struct workinfo { char bits[TXT_SML+1]; char ntime[TXT_SML+1]; int64_t reward; + int32_t height; // non-DB field + double diff_target; // non-DB field HISTORYDATECONTROLFIELDS; } WORKINFO; @@ -2299,10 +2301,8 @@ extern K_ITEM *find_optioncontrol(char *optionname, const tv_t *now, int32_t hei extern int64_t user_sys_setting(int64_t userid, char *setting_name, int64_t setting_default, const tv_t *now); extern cmp_t cmp_workinfo(K_ITEM *a, K_ITEM *b); -#define coinbase1height(_cb1) _coinbase1height(_cb1, WHERE_FFL_HERE) -extern int32_t _coinbase1height(char *coinbase1, WHERE_FFL_ARGS); -#define cmp_height(_cb1a, _cb1b) _cmp_height(_cb1a, _cb1b, WHERE_FFL_HERE) -extern cmp_t _cmp_height(char *coinbase1a, char *coinbase1b, WHERE_FFL_ARGS); +#define coinbase1height(_wi) _coinbase1height(_wi, WHERE_FFL_HERE) +extern int32_t _coinbase1height(WORKINFO *wi, WHERE_FFL_ARGS); extern cmp_t cmp_workinfo_height(K_ITEM *a, K_ITEM *b); extern K_ITEM *find_workinfo(int64_t workinfoid, K_TREE_CTX *ctx); extern K_ITEM *next_workinfo(int64_t workinfoid, K_TREE_CTX *ctx); diff --git a/src/ckdb_cmd.c b/src/ckdb_cmd.c index 5ad59a5a..9322d1ae 100644 --- a/src/ckdb_cmd.c +++ b/src/ckdb_cmd.c @@ -3187,11 +3187,9 @@ static char *cmd_homepage(__maybe_unused PGconn *conn, char *cmd, char *id, K_RLOCK(workinfo_free); if (workinfo_current) { WORKINFO *wic; - int32_t hi; DATA_WORKINFO(wic, workinfo_current); - hi = coinbase1height(wic->coinbase1); snprintf(tmp, sizeof(tmp), "lastheight=%d%c", - hi-1, FLDSEP); + wic->height-1, FLDSEP); APPEND_REALLOC(buf, off, len, tmp); } else { snprintf(tmp, sizeof(tmp), "lastheight=?%c", FLDSEP); @@ -4066,7 +4064,6 @@ static char *cmd_pplns(__maybe_unused PGconn *conn, char *cmd, char *id, double diff_add = 0.0; double diff_want; bool allow_aged = false, countbacklimit; - char ndiffbin[TXT_SML+1]; size_t len, off; int rows; @@ -4156,8 +4153,7 @@ static char *cmd_pplns(__maybe_unused PGconn *conn, char *cmd, char *id, } DATA_WORKINFO(workinfo, w_item); - hex2bin(ndiffbin, workinfo->bits, 4); - ndiff = diff_from_nbits(ndiffbin); + ndiff = workinfo->diff_target; diff_want = ndiff * diff_times + diff_add; if (diff_want < 1.0) { snprintf(reply, siz, @@ -4527,8 +4523,6 @@ static char *cmd_pplns2(__maybe_unused PGconn *conn, char *cmd, char *id, PAYOUTS *payouts; BLOCKS lookblocks, *blocks; WORKINFO *bworkinfo, *workinfo; - char ndiffbin[TXT_SML+1]; - double ndiff; USERS *users; int32_t height; K_TREE_CTX b_ctx[1], mp_ctx[1], pay_ctx[1]; @@ -4759,9 +4753,8 @@ static char *cmd_pplns2(__maybe_unused PGconn *conn, char *cmd, char *id, APPEND_REALLOC(buf, off, len, tmp); snprintf(tmp, sizeof(tmp), "%s%c", payouts->stats, FLDSEP); APPEND_REALLOC(buf, off, len, tmp); - hex2bin(ndiffbin, bworkinfo->bits, 4); - ndiff = diff_from_nbits(ndiffbin); - snprintf(tmp, sizeof(tmp), "block_ndiff=%f%c", ndiff, FLDSEP); + snprintf(tmp, sizeof(tmp), "block_ndiff=%f%c", + bworkinfo->diff_target, FLDSEP); APPEND_REALLOC(buf, off, len, tmp); snprintf(tmp, sizeof(tmp), "diff_want=%.1f%c", payouts->diffwanted, FLDSEP); @@ -6738,7 +6731,6 @@ static char *cmd_query(__maybe_unused PGconn *conn, char *cmd, char *id, * pgsql scripts would do that to the DB, then ckdb would * load them the next time it (re)starts */ K_ITEM *i_wid, *i_expired, *wi_item, *wm_item; - char ndiffbin[TXT_SML+1]; bool expired = false; WORKINFO *workinfo; WORKMARKERS *wm; @@ -6776,7 +6768,7 @@ static char *cmd_query(__maybe_unused PGconn *conn, char *cmd, char *id, "workinfoid:%d=%s%c", rows, reply, FLDSEP); APPEND_REALLOC(buf, off, len, tmp); - int_to_buf(coinbase1height(workinfo->coinbase1), + int_to_buf(workinfo->height, reply, sizeof(reply)); snprintf(tmp, sizeof(tmp), "height:%d=%s%c", @@ -6798,10 +6790,9 @@ static char *cmd_query(__maybe_unused PGconn *conn, char *cmd, char *id, CDDB"_str:%d=%s%c", rows, cd_buf, FLDSEP); APPEND_REALLOC(buf, off, len, tmp); - hex2bin(ndiffbin, workinfo->bits, 4); snprintf(tmp, sizeof(tmp), "ndiff:%d=%.1f%c", rows, - diff_from_nbits(ndiffbin), + workinfo->diff_target, FLDSEP); APPEND_REALLOC(buf, off, len, tmp); snprintf(tmp, sizeof(tmp), @@ -6899,7 +6890,7 @@ static char *cmd_query(__maybe_unused PGconn *conn, char *cmd, char *id, wi_item = last_in_ktree(workinfo_root, ctx); DATA_WORKINFO_NULL(workinfo, wi_item); while (wi_item) { - this_height = coinbase1height(workinfo->coinbase1); + this_height = workinfo->height; if (this_height < height) break; if (CURRENT(&(workinfo->expirydate)) && @@ -6946,7 +6937,6 @@ static char *cmd_query(__maybe_unused PGconn *conn, char *cmd, char *id, K_ITEM *i_height, *wi_item; WORKINFO *workinfo = NULL; int32_t height, this_height; - char ndiffbin[TXT_SML+1]; char bits[TXT_SML+1]; bool got = false; @@ -6971,7 +6961,7 @@ static char *cmd_query(__maybe_unused PGconn *conn, char *cmd, char *id, DATA_WORKINFO_NULL(workinfo, wi_item); while (wi_item) { if (CURRENT(&(workinfo->expirydate))) { - this_height = coinbase1height(workinfo->coinbase1); + this_height = workinfo->height; if (this_height < height) break; } @@ -6984,7 +6974,7 @@ static char *cmd_query(__maybe_unused PGconn *conn, char *cmd, char *id, DATA_WORKINFO_NULL(workinfo, wi_item); while (wi_item) { if (CURRENT(&(workinfo->expirydate))) { - this_height = coinbase1height(workinfo->coinbase1); + this_height = workinfo->height; if (this_height >= height) break; } @@ -6993,7 +6983,7 @@ static char *cmd_query(__maybe_unused PGconn *conn, char *cmd, char *id, } if (wi_item) { DATA_WORKINFO(workinfo, wi_item); - this_height = coinbase1height(workinfo->coinbase1); + this_height = workinfo->height; if (this_height == height) { // We have our starting point STRNCPY(bits, workinfo->bits); @@ -7005,10 +6995,9 @@ static char *cmd_query(__maybe_unused PGconn *conn, char *cmd, char *id, "workinfoid0:%d=%s%c", rows, reply, FLDSEP); APPEND_REALLOC(buf, off, len, tmp); - hex2bin(ndiffbin, workinfo->bits, 4); snprintf(tmp, sizeof(tmp), "ndiff0:%d=%.1f%c", rows, - diff_from_nbits(ndiffbin), + workinfo->diff_target, FLDSEP); APPEND_REALLOC(buf, off, len, tmp); @@ -7045,7 +7034,7 @@ static char *cmd_query(__maybe_unused PGconn *conn, char *cmd, char *id, rows, FLDSEP); APPEND_REALLOC(buf, off, len, tmp); } else { - this_height = coinbase1height(workinfo->coinbase1); + this_height = workinfo->height; int_to_buf(this_height, reply, sizeof(reply)); snprintf(tmp, sizeof(tmp), "height:%d=%s%c", rows, reply, FLDSEP); @@ -7055,10 +7044,9 @@ static char *cmd_query(__maybe_unused PGconn *conn, char *cmd, char *id, snprintf(tmp, sizeof(tmp), "workinfoid:%d=%s%c", rows, reply, FLDSEP); APPEND_REALLOC(buf, off, len, tmp); - hex2bin(ndiffbin, workinfo->bits, 4); snprintf(tmp, sizeof(tmp), "ndiff:%d=%.1f%c", rows, - diff_from_nbits(ndiffbin), + workinfo->diff_target, FLDSEP); APPEND_REALLOC(buf, off, len, tmp); } diff --git a/src/ckdb_data.c b/src/ckdb_data.c index ff6f102c..30347485 100644 --- a/src/ckdb_data.c +++ b/src/ckdb_data.c @@ -1965,20 +1965,36 @@ cmp_t cmp_workinfo(K_ITEM *a, K_ITEM *b) return c; } -int32_t _coinbase1height(char *coinbase1, WHERE_FFL_ARGS) +int32_t _coinbase1height(WORKINFO *wi, WHERE_FFL_ARGS) { int32_t height = 0; + char *st = NULL; uchar *cb1; + size_t len; int siz; - cb1 = ((uchar *)coinbase1) + 84; + len = strlen(wi->coinbase1); + if (len < (BLOCKNUM_OFFSET * 2 + 4) || (len & 1)) { + LOGERR("ERR %s(): Invalid coinbase1 len %d - " + "should be >= %d and even - wid %"PRId64 + " (cb1 %.10s%s)", + __func__, (int)len, (BLOCKNUM_OFFSET * 2 + 4), + wi->workinfoid, st = safe_text_nonull(wi->coinbase1), + len <= 10 ? "" : "..."); + FREENULL(st); + return height; + } + + cb1 = ((uchar *)(wi->coinbase1)) + (BLOCKNUM_OFFSET * 2); siz = ((hex2bin_tbl[*cb1]) << 4) + (hex2bin_tbl[*(cb1+1)]); // limit to 4 for int32_t and since ... that should last a while :) if (siz < 1 || siz > 4) { - LOGERR("%s(): Invalid coinbase1 block height size (%d)" - " require: 1..4 (cb1 %s)" WHERE_FFL, - __func__, siz, coinbase1, WHERE_FFL_PASS); + LOGERR("ERR %s(): Invalid coinbase1 block height size (%d)" + " require: 1..4 - wid %"PRId64" (cb1 %.10s...)" + WHERE_FFL, + __func__, siz, wi->workinfoid, wi->coinbase1, + WHERE_FFL_PASS); return height; } @@ -1991,19 +2007,13 @@ int32_t _coinbase1height(char *coinbase1, WHERE_FFL_ARGS) return height; } -cmp_t _cmp_height(char *coinbase1a, char *coinbase1b, WHERE_FFL_ARGS) -{ - return CMP_INT(_coinbase1height(coinbase1a, WHERE_FFL_PASS), - _coinbase1height(coinbase1b, WHERE_FFL_PASS)); -} - // order by height asc,createdate asc cmp_t cmp_workinfo_height(K_ITEM *a, K_ITEM *b) { WORKINFO *wa, *wb; DATA_WORKINFO(wa, a); DATA_WORKINFO(wb, b); - cmp_t c = cmp_height(wa->coinbase1, wb->coinbase1); + cmp_t c = CMP_INT(wa->height, wb->height); if (c == 0) c = CMP_TV(wa->createdate, wb->createdate); return c; @@ -2251,12 +2261,7 @@ double workinfo_pps(K_ITEM *w_item, int64_t workinfoid, bool lock) OPTIONCONTROL *optioncontrol; K_ITEM *oc_item; char oc_name[TXT_SML+1]; - char coinbase1bin[TXT_SML+1]; - char ndiffbin[TXT_SML+1]; WORKINFO *workinfo; - double w_diff; - int32_t w_blocknum; - size_t len; // Allow optioncontrol override for a given workinfoid snprintf(oc_name, sizeof(oc_name), PPSOVERRIDE"_%"PRId64, workinfoid); @@ -2280,23 +2285,9 @@ double workinfo_pps(K_ITEM *w_item, int64_t workinfoid, bool lock) } DATA_WORKINFO(workinfo, w_item); - len = strlen(workinfo->coinbase1); - if (len < (BLOCKNUM_OFFSET * 2 + 4) || (len & 1)) { - LOGERR("%s(): Invalid coinbase1 len %d - " - "should be >= %d and even - for wid %"PRId64, - __func__, (int)len, (BLOCKNUM_OFFSET * 2 + 4), - workinfoid); - return 0.0; - } - - hex2bin(ndiffbin, workinfo->bits, 4); - w_diff = diff_from_nbits(ndiffbin); - hex2bin(coinbase1bin, workinfo->coinbase1 + (BLOCKNUM_OFFSET * 2), - (len - (BLOCKNUM_OFFSET * 2)) >> 1); - w_blocknum = (int32_t)get_sernumber((uchar *)coinbase1bin); // PPS 1diff is worth coinbase reward divided by difficulty - return(coinbase_reward(w_blocknum) / w_diff); + return(coinbase_reward(workinfo->height) / workinfo->diff_target); } // order by workinfoid asc,userid asc,workername asc,createdate asc,nonce asc,expirydate desc @@ -2973,7 +2964,6 @@ bool check_update_blocks_stats(tv_t *stats) K_ITEM *b_item, *w_item; WORKINFO *workinfo; BLOCKS *blocks; - char ndiffbin[TXT_SML+1]; double ok, diffacc, netsumm, diffmean, pending, txmean, cr; tv_t now; @@ -3028,8 +3018,7 @@ bool check_update_blocks_stats(tv_t *stats) return false; } DATA_WORKINFO(workinfo, w_item); - hex2bin(ndiffbin, workinfo->bits, 4); - blocks->netdiff = diff_from_nbits(ndiffbin); + blocks->netdiff = workinfo->diff_target; } /* Stats for each blocks are independent of * if they are orphans or not */ @@ -3599,7 +3588,7 @@ bool process_pplns(int32_t height, char *blockhash, tv_t *addr_cd) K_TREE *mu_root = NULL; int usercount; double ndiff, total_diff, diff_want, elapsed; - char ndiffbin[TXT_SML+1], rewardbuf[32]; + char rewardbuf[32]; double diff_times, diff_add; char cd_buf[CDATE_BUFSIZ]; tv_t end_tv = { 0L, 0L }; @@ -3699,8 +3688,7 @@ bool process_pplns(int32_t height, char *blockhash, tv_t *addr_cd) DATA_OPTIONCONTROL(optioncontrol, oc_item); diff_add = atof(optioncontrol->optionvalue); - hex2bin(ndiffbin, workinfo->bits, 4); - ndiff = diff_from_nbits(ndiffbin); + ndiff = workinfo->diff_target; diff_want = ndiff * diff_times + diff_add; if (diff_want < 1.0) { LOGERR("%s(): invalid diff_want %.1f, block %"PRId32"/%" diff --git a/src/ckdb_dbio.c b/src/ckdb_dbio.c index 6319aed4..843d3e8d 100644 --- a/src/ckdb_dbio.c +++ b/src/ckdb_dbio.c @@ -2808,8 +2808,9 @@ unparam: LIST_MEM_SUB(workinfo_free, row->transactiontree); FREENULL(row->transactiontree); + row->height = coinbase1height(row); hex2bin(ndiffbin, row->bits, 4); - current_ndiff = diff_from_nbits(ndiffbin); + row->diff_target = current_ndiff = diff_from_nbits(ndiffbin); add_to_ktree(workinfo_root, item); k_add_head(workinfo_store, item); @@ -2818,7 +2819,7 @@ unparam: if (workinfo_current) { WORKINFO *wic; DATA_WORKINFO(wic, workinfo_current); - if (cmp_height(wic->coinbase1, row->coinbase1) != 0) { + if (wic->height != row->height) { copy_tv(&last_bc, cd); zero_active = true; } @@ -2836,6 +2837,7 @@ unparam: bool workinfo_fill(PGconn *conn) { + char ndiffbin[TXT_SML+1]; ExecStatusType rescode; PGresult *res; K_ITEM *item; @@ -3003,6 +3005,10 @@ bool workinfo_fill(PGconn *conn) if (!ok) break; + row->height = coinbase1height(row); + hex2bin(ndiffbin, row->bits, 4); + row->diff_target = diff_from_nbits(ndiffbin); + add_to_ktree(workinfo_root, item); if (!confirm_sharesummary) add_to_ktree(workinfo_height_root, item); From 58cd4551f2235aafed1f68fb5afb615335ee0a06 Mon Sep 17 00:00:00 2001 From: kanoi Date: Sat, 31 Oct 2015 22:29:40 +1100 Subject: [PATCH 46/62] ckdb - log valid/invalid shares default 5% or better of a being block --- src/ckdb.c | 2 ++ src/ckdb.h | 13 +++++++- src/ckdb_dbio.c | 88 +++++++++++++++++++++++++++++++++++++++++-------- 3 files changed, 89 insertions(+), 14 deletions(-) diff --git a/src/ckdb.c b/src/ckdb.c index d13997c7..c9faa92e 100644 --- a/src/ckdb.c +++ b/src/ckdb.c @@ -385,6 +385,8 @@ K_STORE *shares_store; K_TREE *shares_early_root; K_STORE *shares_early_store; +double diff_percent = DIFF_VAL(DIFF_PERCENT_DEFAULT); + // SHAREERRORS shareerrors.id.json={...} K_TREE *shareerrors_root; K_LIST *shareerrors_free; diff --git a/src/ckdb.h b/src/ckdb.h index 779b70c2..bc14329a 100644 --- a/src/ckdb.h +++ b/src/ckdb.h @@ -55,7 +55,7 @@ #define DB_VLOCK "1" #define DB_VERSION "1.0.4" -#define CKDB_VERSION DB_VERSION"-1.501" +#define CKDB_VERSION DB_VERSION"-1.502" #define WHERE_FFL " - from %s %s() line %d" #define WHERE_FFL_HERE __FILE__, __func__, __LINE__ @@ -1402,6 +1402,17 @@ extern K_STORE *shares_early_store; check for it's workinfoid and then be discarded */ #define EARLYSHARESLIMIT 60.0 +/* All shares this % less than beng a block, or higher, + will be reported on the console */ +#define DIFF_PERCENT_DEFAULT 5 +// OptionControl can override it, > 100 means don't do it +#define DIFF_PERCENT_NAME "ShareDiffPercent" + +// int diff % -> ratio +#define DIFF_VAL(_v) (1.0 - ((double)(_v) / 100.0)) + +extern double diff_percent; + // SHAREERRORS shareerrors.id.json={...} typedef struct shareerrors { int64_t workinfoid; diff --git a/src/ckdb_dbio.c b/src/ckdb_dbio.c index 843d3e8d..eed9f5cf 100644 --- a/src/ckdb_dbio.c +++ b/src/ckdb_dbio.c @@ -2434,6 +2434,13 @@ K_ITEM *optioncontrol_item_add(PGconn *conn, K_ITEM *oc_item, tv_t *cd, bool beg row->activationheight = OPTIONCONTROL_HEIGHT; } + // Update if it's changed + if (strcmp(row->optionname, DIFF_PERCENT_NAME) == 0) { + diff_percent = DIFF_VAL(strtod(row->optionvalue, NULL)); + if (errno == ERANGE) + diff_percent = DIFF_VAL(DIFF_PERCENT_DEFAULT); + } + INIT_OPTIONCONTROL(&look); look.data = (void *)row; K_RLOCK(optioncontrol_free); @@ -2672,6 +2679,14 @@ bool optioncontrol_fill(PGconn *conn) LOGWARNING("%s() set switch_state to %d", __func__, switch_state); } + + // The last loaded CURRENT value will be used + if (CURRENT(&(row->expirydate)) && + strcmp(row->optionname, DIFF_PERCENT_NAME) == 0) { + diff_percent = DIFF_VAL(strtod(row->optionvalue, NULL)); + if (errno == ERANGE) + diff_percent = DIFF_VAL(DIFF_PERCENT_DEFAULT); + } } if (!ok) { free_optioncontrol_data(item); @@ -3058,14 +3073,58 @@ flail: return ok; } -static bool shares_process(PGconn *conn, SHARES *shares, K_TREE *trf_root) +static bool shares_process(PGconn *conn, SHARES *shares, K_ITEM *wi_item, + K_TREE *trf_root) { K_ITEM *w_item, *wm_item, *ss_item; SHARESUMMARY *sharesummary; + WORKINFO *workinfo; char *st = NULL; LOGDEBUG("%s() add", __func__); + if (diff_percent >= 0.0) { + DATA_WORKINFO(workinfo, wi_item); + if (shares->sdiff >= (workinfo->diff_target * diff_percent)) { + bool block = (shares->sdiff >= workinfo->diff_target); + char *sta = NULL, pct[16] = "?", est[16] = ""; + switch (shares->errn) { + case SE_NONE: + break; + case SE_STALE: + sta = "STALE"; + break; + case SE_DUPE: + sta = "Dup"; + break; + case SE_HIGH_DIFF: + sta = "HI"; + break; + default: + sta = "UNKNOWN"; + break; + } + if (pool.diffacc >= 1000.0) { + est[0] = ' '; + suffix_string(pool.diffacc, est+1, sizeof(est)-2, 0); + } + if (workinfo->diff_target > 0.0) { + snprintf(pct, sizeof(pct), " %.2f%%", + 100.0 * pool.diffacc / + workinfo->diff_target); + } + LOGWARNING("%s %s Diff %.1f%% (%.0f/%.1f) %s " + "Pool %.1f%s%s", + block ? "BLOCK!" : "Share", + (sta == NULL) ? "ok" : sta, + 100.0 * shares->sdiff / workinfo->diff_target, + shares->sdiff, workinfo->diff_target, + st = safe_text_nonull(shares->workername), + pool.diffacc, est, pct); + FREENULL(st); + } + } + w_item = new_default_worker(conn, false, shares->userid, shares->workername, shares->createby, shares->createcode, shares->createinet, @@ -3125,11 +3184,12 @@ static bool shares_process(PGconn *conn, SHARES *shares, K_TREE *trf_root) } // If it exists and it can be processed, process the oldest early share -static void shares_process_early(PGconn *conn, int64_t good_wid, tv_t *good_cd, +static void shares_process_early(PGconn *conn, K_ITEM *wi, tv_t *good_cd, K_TREE *trf_root) { K_TREE_CTX ctx[1]; K_ITEM *es_item, *wi_item; + WORKINFO *workinfo; SHARES *early_shares; char cd_buf[DATE_BUFSIZ]; char *why = EMPTY; @@ -3140,6 +3200,7 @@ static void shares_process_early(PGconn *conn, int64_t good_wid, tv_t *good_cd, LOGDEBUG("%s() add", __func__); + DATA_WORKINFO(workinfo, wi); K_WLOCK(shares_free); if (shares_early_store->count == 0) { K_WUNLOCK(shares_free); @@ -3156,13 +3217,13 @@ static void shares_process_early(PGconn *conn, int64_t good_wid, tv_t *good_cd, DATA_SHARES(early_shares, es_item); /* If the last (oldest) is newer than the * current workinfo, leave it til later */ - if (early_shares->workinfoid > good_wid) + if (early_shares->workinfoid > workinfo->workinfoid) goto redo; /* If it matches the 'ok' share we just processed, * we don't need to check the workinfoid */ - if (early_shares->workinfoid == good_wid) { - ok = shares_process(conn, early_shares, trf_root); + if (early_shares->workinfoid == workinfo->workinfoid) { + ok = shares_process(conn, early_shares, wi, trf_root); if (ok) goto keep; else @@ -3186,7 +3247,8 @@ static void shares_process_early(PGconn *conn, int64_t good_wid, tv_t *good_cd, early_shares->redo++; goto redo; } else { - ok = shares_process(conn, early_shares, trf_root); + ok = shares_process(conn, early_shares, + wi_item, trf_root); if (ok) goto keep; else @@ -3327,15 +3389,15 @@ bool shares_add(PGconn *conn, char *workinfoid, char *username, char *workername return true; } - ok = shares_process(conn, shares, trf_root); + ok = shares_process(conn, shares, wi_item, trf_root); if (ok) { K_WLOCK(shares_free); add_to_ktree(shares_root, s_item); k_add_head(shares_store, s_item); K_WUNLOCK(shares_free); - shares_process_early(conn, shares->workinfoid, - &(shares->createdate), trf_root); + shares_process_early(conn, wi_item, &(shares->createdate), + trf_root); // Call both since shareerrors may be rare shareerrors_process_early(conn, shares->workinfoid, &(shares->createdate), trf_root); @@ -3626,8 +3688,8 @@ bool shareerrors_add(PGconn *conn, char *workinfoid, char *username, &(shareerrors->createdate), trf_root); // Call both in case we are only getting errors on bad work - shares_process_early(conn, shareerrors->workinfoid, - &(shareerrors->createdate), trf_root); + shares_process_early(conn, wi_item, &(shareerrors->createdate), + trf_root); // The original share was ok return true; @@ -4970,11 +5032,11 @@ flail: } if (pool.diffacc >= 1000.0) { suffix_string(pool.diffacc, est, sizeof(est)-1, 0); - strcat(est, " "); + strcat(est, "% "); } tv_to_buf(&(row->createdate), cd_buf, sizeof(cd_buf)); snprintf(tmp, sizeof(tmp), - " Reward: %f, Worker: %s, ShareEst: %.1f %s%s%% UTC:%s", + " Reward: %f, Worker: %s, ShareEst: %.1f %s%sUTC:%s", BTC_TO_D(row->reward), st = safe_text_nonull(row->workername), pool.diffacc, est, pct, cd_buf); From 42d009350381de69b79dd76f96803306214c8afc Mon Sep 17 00:00:00 2001 From: kanoi Date: Sat, 31 Oct 2015 23:01:03 +1100 Subject: [PATCH 47/62] ckdb - use new workinfo diff_target rather than recalc it --- src/ckdb.h | 2 +- src/ckdb_dbio.c | 9 +++------ 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/src/ckdb.h b/src/ckdb.h index bc14329a..313635d7 100644 --- a/src/ckdb.h +++ b/src/ckdb.h @@ -55,7 +55,7 @@ #define DB_VLOCK "1" #define DB_VERSION "1.0.4" -#define CKDB_VERSION DB_VERSION"-1.502" +#define CKDB_VERSION DB_VERSION"-1.503" #define WHERE_FFL " - from %s %s() line %d" #define WHERE_FFL_HERE __FILE__, __func__, __LINE__ diff --git a/src/ckdb_dbio.c b/src/ckdb_dbio.c index eed9f5cf..d60fa1f1 100644 --- a/src/ckdb_dbio.c +++ b/src/ckdb_dbio.c @@ -5019,15 +5019,12 @@ flail: blk = true; w_item = find_workinfo(row->workinfoid, NULL); if (w_item) { - char wdiffbin[TXT_SML+1]; - double wdiff; WORKINFO *workinfo; DATA_WORKINFO(workinfo, w_item); - hex2bin(wdiffbin, workinfo->bits, 4); - wdiff = diff_from_nbits(wdiffbin); - if (wdiff > 0.0) { + if (workinfo->diff_target > 0.0) { snprintf(pct, sizeof(pct), "%.2f", - 100.0 * pool.diffacc / wdiff); + 100.0 * pool.diffacc / + workinfo->diff_target); } } if (pool.diffacc >= 1000.0) { From 09bac4ece8f0703cc3ea36c3011e5b5e0c4395e7 Mon Sep 17 00:00:00 2001 From: kanoi Date: Sat, 31 Oct 2015 23:43:47 +1100 Subject: [PATCH 48/62] ckdb - dealloc_storage optional memory deallocation to speed up shutdown --- src/ckdb.c | 81 ++++++++++++++++++++++++++++++++++++++---------------- src/ckdb.h | 14 +++++++++- 2 files changed, 71 insertions(+), 24 deletions(-) diff --git a/src/ckdb.c b/src/ckdb.c index c9faa92e..44b6f2cc 100644 --- a/src/ckdb.c +++ b/src/ckdb.c @@ -128,6 +128,8 @@ static char *restorefrom; bool genpayout_auto; bool markersummary_auto; +enum free_modes free_mode = FREE_MODE_FAST; + int switch_state = SWITCH_STATE_ALL; // disallow: '/' WORKSEP1 WORKSEP2 and FLDSEP @@ -1277,6 +1279,11 @@ static void dealloc_storage() SHARES *shares; int seq; + if (free_mode == FREE_MODE_NONE) { + LOGWARNING("%s() skipped", __func__); + return; + } + LOGWARNING("%s() logqueue ...", __func__); FREE_LISTS(logqueue); @@ -1292,15 +1299,19 @@ static void dealloc_storage() FREE_STORE_DATA(workmarkers); FREE_LIST_DATA(workmarkers); - LOGWARNING("%s() markersummary ...", __func__); - - FREE_TREE(markersummary_pool); - k_list_transfer_to_tail(markersummary_pool_store, markersummary_store); - FREE_STORE(markersummary_pool); - FREE_TREE(markersummary_userid); - FREE_TREE(markersummary); - FREE_STORE_DATA(markersummary); - FREE_LIST_DATA(markersummary); + if (free_mode != FREE_MODE_ALL) + LOGWARNING("%s() markersummary skipped", __func__); + else { + LOGWARNING("%s() markersummary ...", __func__); + + FREE_TREE(markersummary_pool); + k_list_transfer_to_tail(markersummary_pool_store, markersummary_store); + FREE_STORE(markersummary_pool); + FREE_TREE(markersummary_userid); + FREE_TREE(markersummary); + FREE_STORE_DATA(markersummary); + FREE_LIST_DATA(markersummary); + } FREE_ALL(workerstatus); @@ -1376,12 +1387,16 @@ static void dealloc_storage() FREE_STORE(shares_early); FREE_ALL(shares); - LOGWARNING("%s() workinfo ...", __func__); + if (free_mode != FREE_MODE_ALL) + LOGWARNING("%s() workinfo skipped", __func__); + else { + LOGWARNING("%s() workinfo ...", __func__); - FREE_TREE(workinfo_height); - FREE_TREE(workinfo); - FREE_STORE_DATA(workinfo); - FREE_LIST_DATA(workinfo); + FREE_TREE(workinfo_height); + FREE_TREE(workinfo); + FREE_STORE_DATA(workinfo); + FREE_LIST_DATA(workinfo); + } FREE_LISTS(idcontrol); FREE_ALL(accountbalance); @@ -1409,18 +1424,22 @@ static void dealloc_storage() FREE_LISTS(workqueue); FREE_LISTS(msgline); - LOGWARNING("%s() seqset ...", __func__); - sequence_report(false); + if (free_mode != FREE_MODE_ALL) + LOGWARNING("%s() seqset skipped", __func__); + else { + LOGWARNING("%s() seqset ...", __func__); + sequence_report(false); - FREE_STORE_DATA(seqset); - FREE_LIST_DATA(seqset); - FREE_LISTS(seqset); + FREE_STORE_DATA(seqset); + FREE_LIST_DATA(seqset); + FREE_LISTS(seqset); - // Must be after seqset - FREE_LIST(seqtrans); + // Must be after seqset + FREE_LIST(seqtrans); - for (seq = 0; seq < SEQ_MAX; seq++) - FREENULL(seqnam[seq]); + for (seq = 0; seq < SEQ_MAX; seq++) + FREENULL(seqnam[seq]); + } LOGWARNING("%s() finished", __func__); } @@ -5501,6 +5520,7 @@ static void check_restore_dir(char *name) static struct option long_options[] = { { "config", required_argument, 0, 'c' }, { "dbname", required_argument, 0, 'd' }, + { "free", required_argument, 0, 'f' }, // generate = enable payout pplns auto generation { "generate", no_argument, 0, 'g' }, { "help", no_argument, 0, 'h' }, @@ -5563,6 +5583,21 @@ int main(int argc, char **argv) while (*kill) *(kill++) = ' '; break; + case 'f': + if (strcasecmp(optarg, FREE_MODE_ALL_STR) == 0) + free_mode = FREE_MODE_ALL; + else if (strcasecmp(optarg, FREE_MODE_NONE_STR) == 0) + free_mode = FREE_MODE_NONE; + else if (strcasecmp(optarg, FREE_MODE_FAST_STR) == 0) + free_mode = FREE_MODE_FAST; + else { + quit(1, "Invalid free '%s' must be: " + FREE_MODE_ALL_STR", " + FREE_MODE_NONE_STR" or " + FREE_MODE_FAST_STR, + optarg); + } + break; case 'g': genpayout_auto = true; break; diff --git a/src/ckdb.h b/src/ckdb.h index 313635d7..932e4777 100644 --- a/src/ckdb.h +++ b/src/ckdb.h @@ -55,7 +55,7 @@ #define DB_VLOCK "1" #define DB_VERSION "1.0.4" -#define CKDB_VERSION DB_VERSION"-1.503" +#define CKDB_VERSION DB_VERSION"-1.504" #define WHERE_FFL " - from %s %s() line %d" #define WHERE_FFL_HERE __FILE__, __func__, __LINE__ @@ -98,6 +98,18 @@ extern int switch_state; extern bool genpayout_auto; extern bool markersummary_auto; +enum free_modes { + FREE_MODE_ALL, + FREE_MODE_NONE, + FREE_MODE_FAST +}; + +#define FREE_MODE_ALL_STR "all" +#define FREE_MODE_NONE_STR "none" +#define FREE_MODE_FAST_STR "fast" + +extern enum free_modes free_mode; + #define BLANK " " extern char *EMPTY; From 7057da5dce4ed73b616b80cc78a634b7d2110093 Mon Sep 17 00:00:00 2001 From: kanoi Date: Mon, 2 Nov 2015 12:09:45 +1100 Subject: [PATCH 49/62] ckdb - fix the % character position in the block message --- src/ckdb.h | 2 +- src/ckdb_dbio.c | 16 +++++++--------- 2 files changed, 8 insertions(+), 10 deletions(-) diff --git a/src/ckdb.h b/src/ckdb.h index 932e4777..3b458a0d 100644 --- a/src/ckdb.h +++ b/src/ckdb.h @@ -55,7 +55,7 @@ #define DB_VLOCK "1" #define DB_VERSION "1.0.4" -#define CKDB_VERSION DB_VERSION"-1.504" +#define CKDB_VERSION DB_VERSION"-1.505" #define WHERE_FFL " - from %s %s() line %d" #define WHERE_FFL_HERE __FILE__, __func__, __LINE__ diff --git a/src/ckdb_dbio.c b/src/ckdb_dbio.c index d60fa1f1..90e16fb3 100644 --- a/src/ckdb_dbio.c +++ b/src/ckdb_dbio.c @@ -5004,8 +5004,8 @@ flail: char est[16] = ""; char diff[16] = ""; K_ITEM *w_item; - char tmp[256]; - bool blk; + char tmp[256] = ""; + bool blk = false; suffix_string(hash_diff, diff, sizeof(diff)-1, 0); @@ -5017,20 +5017,20 @@ flail: break; case BLOCKS_CONFIRM: blk = true; + if (pool.diffacc >= 1000.0) { + suffix_string(pool.diffacc, est, sizeof(est)-2, 0); + strcat(est, " "); + } w_item = find_workinfo(row->workinfoid, NULL); if (w_item) { WORKINFO *workinfo; DATA_WORKINFO(workinfo, w_item); if (workinfo->diff_target > 0.0) { - snprintf(pct, sizeof(pct), "%.2f", + snprintf(pct, sizeof(pct), "%.2f%% ", 100.0 * pool.diffacc / workinfo->diff_target); } } - if (pool.diffacc >= 1000.0) { - suffix_string(pool.diffacc, est, sizeof(est)-1, 0); - strcat(est, "% "); - } tv_to_buf(&(row->createdate), cd_buf, sizeof(cd_buf)); snprintf(tmp, sizeof(tmp), " Reward: %f, Worker: %s, ShareEst: %.1f %s%sUTC:%s", @@ -5048,8 +5048,6 @@ flail: case BLOCKS_REJECT: case BLOCKS_42: default: - blk = false; - tmp[0] = '\0'; break; } From 8f3b616d00b3ffc8d7f97d9538de1cf4af65069d Mon Sep 17 00:00:00 2001 From: kanoi Date: Sat, 7 Nov 2015 10:14:23 +1100 Subject: [PATCH 50/62] php - 1 block luck of course isn't an 'average' --- pool/page_luck.php | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/pool/page_luck.php b/pool/page_luck.php index 9a806743..434a3e4e 100644 --- a/pool/page_luck.php +++ b/pool/page_luck.php @@ -70,7 +70,11 @@ function doluck($data, $user) $chk = ''; $pg .= ""; $pg .= ": "; - $pg .= $nams[$i-1].' Block Luck Avg'; + if ($nams[$i-1] == 1) + $avs = ''; + else + $avs = ' Avg'; + $pg .= $nams[$i-1]." Block Luck$avs"; if ($i > 1) $datacols .= ','; From 5592fa2c2de48f08f288a695366b403034e8ecd3 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Mon, 5 Oct 2015 15:01:07 +1100 Subject: [PATCH 51/62] Add 7 day stats to workers --- src/stratifier.c | 138 ++++++++++++++++++++++++++--------------------- 1 file changed, 77 insertions(+), 61 deletions(-) diff --git a/src/stratifier.c b/src/stratifier.c index 7073f97a..8cebb54e 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -28,6 +28,14 @@ #include "uthash.h" #include "utlist.h" +#define MIN1 60 +#define MIN5 300 +#define MIN15 900 +#define HOUR 3600 +#define HOUR6 21600 +#define DAY 86400 +#define WEEK 604800 + /* Consistent across all pool instances */ static const char *workpadding = "000000800000000000000000000000000000000000000000000000000000000000000000000000000000000080020000"; static const char *scriptsig_header = "01000000010000000000000000000000000000000000000000000000000000000000000000ffffffff"; @@ -200,6 +208,7 @@ struct worker_instance { double dsps5; double dsps60; double dsps1440; + double dsps10080; tv_t last_share; tv_t last_update; time_t start_time; @@ -2155,11 +2164,11 @@ static void read_userstats(ckpool_t *ckp, user_instance_t *user) if (tvsec_diff > 60) { LOGINFO("Old user stats indicate not logged for %d seconds, decaying stats", tvsec_diff); - decay_time(&user->dsps1, 0, tvsec_diff, 60); - decay_time(&user->dsps5, 0, tvsec_diff, 300); - decay_time(&user->dsps60, 0, tvsec_diff, 3600); - decay_time(&user->dsps1440, 0, tvsec_diff, 86400); - decay_time(&user->dsps10080, 0, tvsec_diff, 604800); + decay_time(&user->dsps1, 0, tvsec_diff, MIN1); + decay_time(&user->dsps5, 0, tvsec_diff, MIN5); + decay_time(&user->dsps60, 0, tvsec_diff, HOUR); + decay_time(&user->dsps1440, 0, tvsec_diff, DAY); + decay_time(&user->dsps10080, 0, tvsec_diff, WEEK); } } @@ -2197,6 +2206,7 @@ static void read_workerstats(ckpool_t *ckp, worker_instance_t *worker) worker->dsps5 = dsps_from_key(val, "hashrate5m"); worker->dsps60 = dsps_from_key(val, "hashrate1d"); worker->dsps1440 = dsps_from_key(val, "hashrate1d"); + worker->dsps10080 = dsps_from_key(val, "hashrate7d"); json_get_double(&worker->best_diff, val, "bestshare"); json_get_int64(&worker->last_update.tv_sec, val, "lastupdate"); LOGINFO("Successfully read worker %s stats %f %f %f %f %f", worker->workername, @@ -2207,10 +2217,10 @@ static void read_workerstats(ckpool_t *ckp, worker_instance_t *worker) if (tvsec_diff > 60) { LOGINFO("Old worker stats indicate not logged for %d seconds, decaying stats", tvsec_diff); - decay_time(&worker->dsps1, 0, tvsec_diff, 60); - decay_time(&worker->dsps5, 0, tvsec_diff, 300); - decay_time(&worker->dsps60, 0, tvsec_diff, 3600); - decay_time(&worker->dsps1440, 0, tvsec_diff, 86400); + decay_time(&worker->dsps1, 0, tvsec_diff, MIN1); + decay_time(&worker->dsps5, 0, tvsec_diff, MIN5); + decay_time(&worker->dsps60, 0, tvsec_diff, HOUR); + decay_time(&worker->dsps1440, 0, tvsec_diff, DAY); } } @@ -2685,27 +2695,28 @@ static void add_submit(ckpool_t *ckp, stratum_instance_t *client, const int diff } tdiff = sane_tdiff(&now_t, &client->last_share); - decay_time(&client->dsps1, diff, tdiff, 60); - decay_time(&client->dsps5, diff, tdiff, 300); - decay_time(&client->dsps60, diff, tdiff, 3600); - decay_time(&client->dsps1440, diff, tdiff, 86400); - decay_time(&client->dsps10080, diff, tdiff, 604800); + decay_time(&client->dsps1, diff, tdiff, MIN1); + decay_time(&client->dsps5, diff, tdiff, MIN5); + decay_time(&client->dsps60, diff, tdiff, HOUR); + decay_time(&client->dsps1440, diff, tdiff, DAY); + decay_time(&client->dsps10080, diff, tdiff, WEEK); copy_tv(&client->last_share, &now_t); tdiff = sane_tdiff(&now_t, &worker->last_share); - decay_time(&worker->dsps1, diff, tdiff, 60); - decay_time(&worker->dsps5, diff, tdiff, 300); - decay_time(&worker->dsps60, diff, tdiff, 3600); - decay_time(&worker->dsps1440, diff, tdiff, 86400); + decay_time(&worker->dsps1, diff, tdiff, MIN1); + decay_time(&worker->dsps5, diff, tdiff, MIN5); + decay_time(&worker->dsps60, diff, tdiff, HOUR); + decay_time(&worker->dsps1440, diff, tdiff, DAY); + decay_time(&worker->dsps10080, diff, tdiff, WEEK); copy_tv(&worker->last_share, &now_t); worker->idle = false; tdiff = sane_tdiff(&now_t, &user->last_share); - decay_time(&user->dsps1, diff, tdiff, 60); - decay_time(&user->dsps5, diff, tdiff, 300); - decay_time(&user->dsps60, diff, tdiff, 3600); - decay_time(&user->dsps1440, diff, tdiff, 86400); - decay_time(&user->dsps10080, diff, tdiff, 604800); + decay_time(&user->dsps1, diff, tdiff, MIN1); + decay_time(&user->dsps5, diff, tdiff, MIN5); + decay_time(&user->dsps60, diff, tdiff, HOUR); + decay_time(&user->dsps1440, diff, tdiff, DAY); + decay_time(&user->dsps10080, diff, tdiff, WEEK); copy_tv(&user->last_share, &now_t); client->idle = false; @@ -4110,11 +4121,11 @@ static void *statsupdate(void *arg) /* Decay times per connected instance */ if (per_tdiff > 60) { /* No shares for over a minute, decay to 0 */ - decay_time(&client->dsps1, 0, per_tdiff, 60); - decay_time(&client->dsps5, 0, per_tdiff, 300); - decay_time(&client->dsps60, 0, per_tdiff, 3600); - decay_time(&client->dsps1440, 0, per_tdiff, 86400); - decay_time(&client->dsps10080, 0, per_tdiff, 604800); + decay_time(&client->dsps1, 0, per_tdiff, MIN1); + decay_time(&client->dsps5, 0, per_tdiff, MIN5); + decay_time(&client->dsps60, 0, per_tdiff, HOUR); + decay_time(&client->dsps1440, 0, per_tdiff, DAY); + decay_time(&client->dsps10080, 0, per_tdiff, WEEK); idle_workers++; if (per_tdiff > 600) client->idle = true; @@ -4133,10 +4144,11 @@ static void *statsupdate(void *arg) DL_FOREACH(user->worker_instances, worker) { per_tdiff = tvdiff(&now, &worker->last_share); if (per_tdiff > 60) { - decay_time(&worker->dsps1, 0, per_tdiff, 60); - decay_time(&worker->dsps5, 0, per_tdiff, 300); - decay_time(&worker->dsps60, 0, per_tdiff, 3600); - decay_time(&worker->dsps1440, 0, per_tdiff, 86400); + decay_time(&worker->dsps1, 0, per_tdiff, MIN1); + decay_time(&worker->dsps5, 0, per_tdiff, MIN5); + decay_time(&worker->dsps60, 0, per_tdiff, HOUR); + decay_time(&worker->dsps1440, 0, per_tdiff, DAY); + decay_time(&worker->dsps10080, 0, per_tdiff, WEEK); worker->idle = true; } ghs = worker->dsps1 * nonces; @@ -4151,13 +4163,17 @@ static void *statsupdate(void *arg) ghs = worker->dsps1440 * nonces; suffix_string(ghs, suffix1440, 16, 0); + ghs = worker->dsps10080 * nonces; + suffix_string(ghs, suffix10080, 16, 0); + copy_tv(&worker->last_update, &now); - JSON_CPACK(val, "{ss,ss,ss,ss,si,sf}", + JSON_CPACK(val, "{ss,ss,ss,ss,ss,si,sf}", "hashrate1m", suffix1, "hashrate5m", suffix5, "hashrate1hr", suffix60, "hashrate1d", suffix1440, + "hashrate7d", suffix10080, "lastupdate", now.tv_sec, "bestshare", worker->best_diff); @@ -4170,11 +4186,11 @@ static void *statsupdate(void *arg) /* Decay times per user */ per_tdiff = tvdiff(&now, &user->last_share); if (per_tdiff > 60) { - decay_time(&user->dsps1, 0, per_tdiff, 60); - decay_time(&user->dsps5, 0, per_tdiff, 300); - decay_time(&user->dsps60, 0, per_tdiff, 3600); - decay_time(&user->dsps1440, 0, per_tdiff, 86400); - decay_time(&user->dsps10080, 0, per_tdiff, 604800); + decay_time(&user->dsps1, 0, per_tdiff, MIN1); + decay_time(&user->dsps5, 0, per_tdiff, MIN5); + decay_time(&user->dsps60, 0, per_tdiff, HOUR); + decay_time(&user->dsps1440, 0, per_tdiff, DAY); + decay_time(&user->dsps10080, 0, per_tdiff, WEEK); idle = true; } ghs = user->dsps1 * nonces; @@ -4323,18 +4339,18 @@ static void *statsupdate(void *arg) stats->accounted_diff_shares += stats->unaccounted_diff_shares; stats->accounted_rejects += stats->unaccounted_rejects; - decay_time(&stats->sps1, stats->unaccounted_shares, 1.875, 60); - decay_time(&stats->sps5, stats->unaccounted_shares, 1.875, 300); - decay_time(&stats->sps15, stats->unaccounted_shares, 1.875, 900); - decay_time(&stats->sps60, stats->unaccounted_shares, 1.875, 3600); + decay_time(&stats->sps1, stats->unaccounted_shares, 1.875, MIN1); + decay_time(&stats->sps5, stats->unaccounted_shares, 1.875, MIN5); + decay_time(&stats->sps15, stats->unaccounted_shares, 1.875, MIN15); + decay_time(&stats->sps60, stats->unaccounted_shares, 1.875, HOUR); - decay_time(&stats->dsps1, stats->unaccounted_diff_shares, 1.875, 60); - decay_time(&stats->dsps5, stats->unaccounted_diff_shares, 1.875, 300); - decay_time(&stats->dsps15, stats->unaccounted_diff_shares, 1.875, 900); - decay_time(&stats->dsps60, stats->unaccounted_diff_shares, 1.875, 3600); - decay_time(&stats->dsps360, stats->unaccounted_diff_shares, 1.875, 21600); - decay_time(&stats->dsps1440, stats->unaccounted_diff_shares, 1.875, 86400); - decay_time(&stats->dsps10080, stats->unaccounted_diff_shares, 1.875, 604800); + decay_time(&stats->dsps1, stats->unaccounted_diff_shares, 1.875, MIN1); + decay_time(&stats->dsps5, stats->unaccounted_diff_shares, 1.875, MIN5); + decay_time(&stats->dsps15, stats->unaccounted_diff_shares, 1.875, MIN15); + decay_time(&stats->dsps60, stats->unaccounted_diff_shares, 1.875, HOUR); + decay_time(&stats->dsps360, stats->unaccounted_diff_shares, 1.875, HOUR6); + decay_time(&stats->dsps1440, stats->unaccounted_diff_shares, 1.875, DAY); + decay_time(&stats->dsps10080, stats->unaccounted_diff_shares, 1.875, WEEK); stats->unaccounted_shares = stats->unaccounted_diff_shares = @@ -4453,18 +4469,18 @@ static void read_poolstats(ckpool_t *ckp) if (tvsec_diff > 60) { LOGNOTICE("Old pool stats indicate pool down for %d seconds, decaying stats", tvsec_diff); - decay_time(&stats->sps1, 0, tvsec_diff, 60); - decay_time(&stats->sps5, 0, tvsec_diff, 300); - decay_time(&stats->sps15, 0, tvsec_diff, 900); - decay_time(&stats->sps60, 0, tvsec_diff, 3600); - - decay_time(&stats->dsps1, 0, tvsec_diff, 60); - decay_time(&stats->dsps5, 0, tvsec_diff, 300); - decay_time(&stats->dsps15, 0, tvsec_diff, 900); - decay_time(&stats->dsps60, 0, tvsec_diff, 3600); - decay_time(&stats->dsps360, 0, tvsec_diff, 21600); - decay_time(&stats->dsps1440, 0, tvsec_diff, 86400); - decay_time(&stats->dsps10080, 0, tvsec_diff, 604800); + decay_time(&stats->sps1, 0, tvsec_diff, MIN1); + decay_time(&stats->sps5, 0, tvsec_diff, MIN5); + decay_time(&stats->sps15, 0, tvsec_diff, MIN15); + decay_time(&stats->sps60, 0, tvsec_diff, HOUR); + + decay_time(&stats->dsps1, 0, tvsec_diff, MIN1); + decay_time(&stats->dsps5, 0, tvsec_diff, MIN5); + decay_time(&stats->dsps15, 0, tvsec_diff, MIN15); + decay_time(&stats->dsps60, 0, tvsec_diff, HOUR); + decay_time(&stats->dsps360, 0, tvsec_diff, HOUR6); + decay_time(&stats->dsps1440, 0, tvsec_diff, DAY); + decay_time(&stats->dsps10080, 0, tvsec_diff, WEEK); } } From 814cf36bf78d512bcccddf740f476f726552ed04 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Mon, 5 Oct 2015 15:21:08 +1100 Subject: [PATCH 52/62] Log valid shares per user and worker --- src/stratifier.c | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/src/stratifier.c b/src/stratifier.c index 8cebb54e..28a15011 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -181,6 +181,7 @@ struct user_instance { double best_diff; /* Best share found by this user */ + int64_t shares; double dsps1; /* Diff shares per second, 1 minute rolling average */ double dsps5; /* ... 5 minute ... */ double dsps60;/* etc */ @@ -204,6 +205,7 @@ struct worker_instance { worker_instance_t *next; worker_instance_t *prev; + int64_t shares; double dsps1; double dsps5; double dsps60; @@ -2154,6 +2156,7 @@ static void read_userstats(ckpool_t *ckp, user_instance_t *user) user->dsps1440 = dsps_from_key(val, "hashrate1d"); user->dsps10080 = dsps_from_key(val, "hashrate7d"); json_get_int64(&user->last_update.tv_sec, val, "lastupdate"); + json_get_int64(&user->shares, val, "shares"); json_get_double(&user->best_diff, val, "bestshare"); LOGINFO("Successfully read user %s stats %f %f %f %f %f %f", user->username, user->dsps1, user->dsps5, user->dsps60, user->dsps1440, @@ -2209,6 +2212,7 @@ static void read_workerstats(ckpool_t *ckp, worker_instance_t *worker) worker->dsps10080 = dsps_from_key(val, "hashrate7d"); json_get_double(&worker->best_diff, val, "bestshare"); json_get_int64(&worker->last_update.tv_sec, val, "lastupdate"); + json_get_int64(&worker->shares, val, "shares"); LOGINFO("Successfully read worker %s stats %f %f %f %f %f", worker->workername, worker->dsps1, worker->dsps5, worker->dsps60, worker->dsps1440, worker->best_diff); json_decref(val); @@ -2676,7 +2680,10 @@ static void add_submit(ckpool_t *ckp, stratum_instance_t *client, const int diff mutex_unlock(&sdata->stats_lock); /* Count only accepted and stale rejects in diff calculation. */ - if (!valid && !submit) + if (valid) { + worker->shares += diff; + user->shares += diff; + } else if (!submit) return; tv_time(&now_t); @@ -4168,13 +4175,14 @@ static void *statsupdate(void *arg) copy_tv(&worker->last_update, &now); - JSON_CPACK(val, "{ss,ss,ss,ss,ss,si,sf}", + JSON_CPACK(val, "{ss,ss,ss,ss,ss,si,sI,sf}", "hashrate1m", suffix1, "hashrate5m", suffix5, "hashrate1hr", suffix60, "hashrate1d", suffix1440, "hashrate7d", suffix10080, "lastupdate", now.tv_sec, + "shares", worker->shares, "bestshare", worker->best_diff); ASPRINTF(&fname, "%s/workers/%s", ckp->logdir, worker->workername); @@ -4210,7 +4218,7 @@ static void *statsupdate(void *arg) copy_tv(&user->last_update, &now); - JSON_CPACK(val, "{ss,ss,ss,ss,ss,si,si,sf}", + JSON_CPACK(val, "{ss,ss,ss,ss,ss,si,si,sI,sf}", "hashrate1m", suffix1, "hashrate5m", suffix5, "hashrate1hr", suffix60, @@ -4218,6 +4226,7 @@ static void *statsupdate(void *arg) "hashrate7d", suffix10080, "lastupdate", now.tv_sec, "workers", user->workers, + "shares", user->shares, "bestshare", user->best_diff); ASPRINTF(&fname, "%s/users/%s", ckp->logdir, user->username); From fa55ab48d8c1603ad7362d98b2730a2337e0cef7 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Mon, 5 Oct 2015 16:00:29 +1100 Subject: [PATCH 53/62] Display workername in block solves --- src/stratifier.c | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/src/stratifier.c b/src/stratifier.c index 28a15011..7927f339 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -1574,12 +1574,12 @@ static void reset_bestshares(sdata_t *sdata) static void block_solve(ckpool_t *ckp, const char *blockhash) { ckmsg_t *block, *tmp, *found = NULL; + char *msg, *workername = NULL; sdata_t *sdata = ckp->data; char cdfield[64]; int height = 0; ts_t ts_now; json_t *val; - char *msg; update_base(ckp, GEN_PRIORITY); @@ -1598,6 +1598,7 @@ static void block_solve(ckpool_t *ckp, const char *blockhash) } if (!strcmp(solvehash, blockhash)) { dealloc(solvehash); + json_get_string(&workername, val, "workername"); found = block; DL_DELETE(sdata->block_solves, block); break; @@ -1619,11 +1620,16 @@ static void block_solve(ckpool_t *ckp, const char *blockhash) ckdbq_add(ckp, ID_BLOCK, val); free(found); - ASPRINTF(&msg, "Block %d solved by %s!", height, ckp->name); + if (unlikely(!workername)) + workername = strdup(""); + + ASPRINTF(&msg, "Block %d solved by %s @ %s!", height, workername, ckp->name); stratum_broadcast_message(sdata, msg); free(msg); - LOGWARNING("Solved and confirmed block %d", height); + LOGWARNING("Solved and confirmed block %d by %s", height, workername); + free(workername); + reset_bestshares(sdata); } From 4ba1fd6156874ffd5702ece64added83540399bf Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Fri, 13 Nov 2015 14:52:01 +1100 Subject: [PATCH 54/62] Silence dropping of throttled clients --- src/stratifier.c | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/src/stratifier.c b/src/stratifier.c index 7927f339..9c83c1d9 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -1302,9 +1302,11 @@ static void __drop_client(sdata_t *sdata, stratum_instance_t *client, bool lazil if (client->workername) { if (user) { - ASPRINTF(msg, "Dropped client %"PRId64" %s %suser %s worker %s %s", - client->id, client->address, user->throttled ? "throttled " : "", - user->username, client->workername, lazily ? "lazily" : ""); + if (!user->throttled) { + ASPRINTF(msg, "Dropped client %"PRId64" %s user %s worker %s %s", + client->id, client->address, user->username, + client->workername, lazily ? "lazily" : ""); + } } else { ASPRINTF(msg, "Dropped client %"PRId64" %s no user worker %s %s", client->id, client->address, client->workername, @@ -1323,7 +1325,7 @@ static void _dec_instance_ref(sdata_t *sdata, stratum_instance_t *client, const const char *func, const int line) { char_entry_t *entries = NULL; - char *msg; + char *msg = NULL; int ref; ck_wlock(&sdata->instance_lock); @@ -1332,7 +1334,10 @@ static void _dec_instance_ref(sdata_t *sdata, stratum_instance_t *client, const * moved due to holding a reference and drop them now. */ if (unlikely(client->dropped && !ref)) { __drop_client(sdata, client, true, &msg); - add_msg_entry(&entries, &msg); + /* Make sure there is a message as throttled users don't + * generate a message */ + if (msg) + add_msg_entry(&entries, &msg); } ck_wunlock(&sdata->instance_lock); From 434ce77b3e6aba2c90fdd1ef03dcd73091851142 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Fri, 13 Nov 2015 15:16:15 +1100 Subject: [PATCH 55/62] Fix user/worker/client stats decaying too fast by generalising decay in shared functions and noting last time value was decayed --- src/stratifier.c | 111 +++++++++++++++++++++++++---------------------- 1 file changed, 59 insertions(+), 52 deletions(-) diff --git a/src/stratifier.c b/src/stratifier.c index 9c83c1d9..cfe8ec04 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -188,6 +188,7 @@ struct user_instance { double dsps1440; double dsps10080; tv_t last_share; + tv_t last_decay; tv_t last_update; bool authorised; /* Has this username ever been authorised? */ @@ -212,6 +213,7 @@ struct worker_instance { double dsps1440; double dsps10080; tv_t last_share; + tv_t last_decay; tv_t last_update; time_t start_time; @@ -252,6 +254,7 @@ struct stratum_instance { int ssdc; /* Shares since diff change */ tv_t first_share; tv_t last_share; + tv_t last_decay; time_t first_invalid; /* Time of first invalid in run of non stale rejects */ time_t start_time; @@ -2131,6 +2134,52 @@ static double dsps_from_key(json_t *val, const char *key) return ret; } +/* Sanity check to prevent clock adjustments backwards from screwing up stats */ +static double sane_tdiff(tv_t *end, tv_t *start) +{ + double tdiff = tvdiff(end, start); + + if (unlikely(tdiff < 0.001)) + tdiff = 0.001; + return tdiff; +} + +static void decay_client(stratum_instance_t *client, double diff, tv_t *now_t) +{ + double tdiff = sane_tdiff(now_t, &client->last_decay); + + decay_time(&client->dsps1, diff, tdiff, MIN1); + decay_time(&client->dsps5, diff, tdiff, MIN5); + decay_time(&client->dsps60, diff, tdiff, HOUR); + decay_time(&client->dsps1440, diff, tdiff, DAY); + decay_time(&client->dsps10080, diff, tdiff, WEEK); + copy_tv(&client->last_decay, now_t); +} + +static void decay_worker(worker_instance_t *worker, double diff, tv_t *now_t) +{ + double tdiff = sane_tdiff(now_t, &worker->last_decay); + + decay_time(&worker->dsps1, diff, tdiff, MIN1); + decay_time(&worker->dsps5, diff, tdiff, MIN5); + decay_time(&worker->dsps60, diff, tdiff, HOUR); + decay_time(&worker->dsps1440, diff, tdiff, DAY); + decay_time(&worker->dsps10080, diff, tdiff, WEEK); + copy_tv(&worker->last_decay, now_t); +} + +static void decay_user(user_instance_t *user, double diff, tv_t *now_t) +{ + double tdiff = sane_tdiff(now_t, &user->last_decay); + + decay_time(&user->dsps1, diff, tdiff, MIN1); + decay_time(&user->dsps5, diff, tdiff, MIN5); + decay_time(&user->dsps60, diff, tdiff, HOUR); + decay_time(&user->dsps1440, diff, tdiff, DAY); + decay_time(&user->dsps10080, diff, tdiff, WEEK); + copy_tv(&user->last_decay, now_t); +} + /* Enter holding a reference count */ static void read_userstats(ckpool_t *ckp, user_instance_t *user) { @@ -2161,6 +2210,7 @@ static void read_userstats(ckpool_t *ckp, user_instance_t *user) tv_time(&now); copy_tv(&user->last_share, &now); + copy_tv(&user->last_decay, &now); user->dsps1 = dsps_from_key(val, "hashrate1m"); user->dsps5 = dsps_from_key(val, "hashrate5m"); user->dsps60 = dsps_from_key(val, "hashrate1hr"); @@ -2178,11 +2228,7 @@ static void read_userstats(ckpool_t *ckp, user_instance_t *user) if (tvsec_diff > 60) { LOGINFO("Old user stats indicate not logged for %d seconds, decaying stats", tvsec_diff); - decay_time(&user->dsps1, 0, tvsec_diff, MIN1); - decay_time(&user->dsps5, 0, tvsec_diff, MIN5); - decay_time(&user->dsps60, 0, tvsec_diff, HOUR); - decay_time(&user->dsps1440, 0, tvsec_diff, DAY); - decay_time(&user->dsps10080, 0, tvsec_diff, WEEK); + decay_user(user, 0, &now); } } @@ -2216,6 +2262,7 @@ static void read_workerstats(ckpool_t *ckp, worker_instance_t *worker) tv_time(&now); copy_tv(&worker->last_share, &now); + copy_tv(&worker->last_decay, &now); worker->dsps1 = dsps_from_key(val, "hashrate1m"); worker->dsps5 = dsps_from_key(val, "hashrate5m"); worker->dsps60 = dsps_from_key(val, "hashrate1d"); @@ -2232,10 +2279,7 @@ static void read_workerstats(ckpool_t *ckp, worker_instance_t *worker) if (tvsec_diff > 60) { LOGINFO("Old worker stats indicate not logged for %d seconds, decaying stats", tvsec_diff); - decay_time(&worker->dsps1, 0, tvsec_diff, MIN1); - decay_time(&worker->dsps5, 0, tvsec_diff, MIN5); - decay_time(&worker->dsps60, 0, tvsec_diff, HOUR); - decay_time(&worker->dsps1440, 0, tvsec_diff, DAY); + decay_worker(worker, 0, &now); } } @@ -2661,16 +2705,6 @@ static double time_bias(const double tdiff, const double period) return 1.0 - 1.0 / exp(dexp); } -/* Sanity check to prevent clock adjustments backwards from screwing up stats */ -static double sane_tdiff(tv_t *end, tv_t *start) -{ - double tdiff = tvdiff(end, start); - - if (unlikely(tdiff < 0.001)) - tdiff = 0.001; - return tdiff; -} - /* Needs to be entered with client holding a ref count. */ static void add_submit(ckpool_t *ckp, stratum_instance_t *client, const int diff, const bool valid, const bool submit) @@ -2712,29 +2746,14 @@ static void add_submit(ckpool_t *ckp, stratum_instance_t *client, const int diff copy_tv(&client->ldc, &now_t); } - tdiff = sane_tdiff(&now_t, &client->last_share); - decay_time(&client->dsps1, diff, tdiff, MIN1); - decay_time(&client->dsps5, diff, tdiff, MIN5); - decay_time(&client->dsps60, diff, tdiff, HOUR); - decay_time(&client->dsps1440, diff, tdiff, DAY); - decay_time(&client->dsps10080, diff, tdiff, WEEK); + decay_client(client, diff, &now_t); copy_tv(&client->last_share, &now_t); - tdiff = sane_tdiff(&now_t, &worker->last_share); - decay_time(&worker->dsps1, diff, tdiff, MIN1); - decay_time(&worker->dsps5, diff, tdiff, MIN5); - decay_time(&worker->dsps60, diff, tdiff, HOUR); - decay_time(&worker->dsps1440, diff, tdiff, DAY); - decay_time(&worker->dsps10080, diff, tdiff, WEEK); + decay_worker(worker, diff, &now_t); copy_tv(&worker->last_share, &now_t); worker->idle = false; - tdiff = sane_tdiff(&now_t, &user->last_share); - decay_time(&user->dsps1, diff, tdiff, MIN1); - decay_time(&user->dsps5, diff, tdiff, MIN5); - decay_time(&user->dsps60, diff, tdiff, HOUR); - decay_time(&user->dsps1440, diff, tdiff, DAY); - decay_time(&user->dsps10080, diff, tdiff, WEEK); + decay_user(user, diff, &now_t); copy_tv(&user->last_share, &now_t); client->idle = false; @@ -4139,11 +4158,7 @@ static void *statsupdate(void *arg) /* Decay times per connected instance */ if (per_tdiff > 60) { /* No shares for over a minute, decay to 0 */ - decay_time(&client->dsps1, 0, per_tdiff, MIN1); - decay_time(&client->dsps5, 0, per_tdiff, MIN5); - decay_time(&client->dsps60, 0, per_tdiff, HOUR); - decay_time(&client->dsps1440, 0, per_tdiff, DAY); - decay_time(&client->dsps10080, 0, per_tdiff, WEEK); + decay_client(client, 0, &now); idle_workers++; if (per_tdiff > 600) client->idle = true; @@ -4162,11 +4177,7 @@ static void *statsupdate(void *arg) DL_FOREACH(user->worker_instances, worker) { per_tdiff = tvdiff(&now, &worker->last_share); if (per_tdiff > 60) { - decay_time(&worker->dsps1, 0, per_tdiff, MIN1); - decay_time(&worker->dsps5, 0, per_tdiff, MIN5); - decay_time(&worker->dsps60, 0, per_tdiff, HOUR); - decay_time(&worker->dsps1440, 0, per_tdiff, DAY); - decay_time(&worker->dsps10080, 0, per_tdiff, WEEK); + decay_worker(worker, 0, &now); worker->idle = true; } ghs = worker->dsps1 * nonces; @@ -4205,11 +4216,7 @@ static void *statsupdate(void *arg) /* Decay times per user */ per_tdiff = tvdiff(&now, &user->last_share); if (per_tdiff > 60) { - decay_time(&user->dsps1, 0, per_tdiff, MIN1); - decay_time(&user->dsps5, 0, per_tdiff, MIN5); - decay_time(&user->dsps60, 0, per_tdiff, HOUR); - decay_time(&user->dsps1440, 0, per_tdiff, DAY); - decay_time(&user->dsps10080, 0, per_tdiff, WEEK); + decay_user(user, 0, &now); idle = true; } ghs = user->dsps1 * nonces; From 5cc1812271c9ec35442c62cecacb8225f68ba08b Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Fri, 13 Nov 2015 15:59:25 +1100 Subject: [PATCH 56/62] Allow add_msg_entry to be called with a null buffer safely --- src/stratifier.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/stratifier.c b/src/stratifier.c index cfe8ec04..7902db36 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -454,8 +454,11 @@ struct json_entry { * to the log outside of lock */ static void add_msg_entry(char_entry_t **entries, char **buf) { - char_entry_t *entry = ckalloc(sizeof(char_entry_t)); + char_entry_t *entry; + if (!*buf) + return; + entry = ckalloc(sizeof(char_entry_t)); entry->buf = *buf; *buf = NULL; DL_APPEND(*entries, entry); @@ -1337,10 +1340,7 @@ static void _dec_instance_ref(sdata_t *sdata, stratum_instance_t *client, const * moved due to holding a reference and drop them now. */ if (unlikely(client->dropped && !ref)) { __drop_client(sdata, client, true, &msg); - /* Make sure there is a message as throttled users don't - * generate a message */ - if (msg) - add_msg_entry(&entries, &msg); + add_msg_entry(&entries, &msg); } ck_wunlock(&sdata->instance_lock); From f42f1875294e6cb9081183b77450e61a70206300 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Fri, 13 Nov 2015 16:12:25 +1100 Subject: [PATCH 57/62] Revert "Allow add_msg_entry to be called with a null buffer safely" This reverts commit 5cc1812271c9ec35442c62cecacb8225f68ba08b. --- src/stratifier.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/stratifier.c b/src/stratifier.c index 7902db36..cfe8ec04 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -454,11 +454,8 @@ struct json_entry { * to the log outside of lock */ static void add_msg_entry(char_entry_t **entries, char **buf) { - char_entry_t *entry; + char_entry_t *entry = ckalloc(sizeof(char_entry_t)); - if (!*buf) - return; - entry = ckalloc(sizeof(char_entry_t)); entry->buf = *buf; *buf = NULL; DL_APPEND(*entries, entry); @@ -1340,7 +1337,10 @@ static void _dec_instance_ref(sdata_t *sdata, stratum_instance_t *client, const * moved due to holding a reference and drop them now. */ if (unlikely(client->dropped && !ref)) { __drop_client(sdata, client, true, &msg); - add_msg_entry(&entries, &msg); + /* Make sure there is a message as throttled users don't + * generate a message */ + if (msg) + add_msg_entry(&entries, &msg); } ck_wunlock(&sdata->instance_lock); From b709dbea44d26e7163626e009fad0b5c68d4ba66 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Fri, 13 Nov 2015 16:12:35 +1100 Subject: [PATCH 58/62] Revert "Silence dropping of throttled clients" This reverts commit 4ba1fd6156874ffd5702ece64added83540399bf. --- src/stratifier.c | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/src/stratifier.c b/src/stratifier.c index cfe8ec04..358055b2 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -1305,11 +1305,9 @@ static void __drop_client(sdata_t *sdata, stratum_instance_t *client, bool lazil if (client->workername) { if (user) { - if (!user->throttled) { - ASPRINTF(msg, "Dropped client %"PRId64" %s user %s worker %s %s", - client->id, client->address, user->username, - client->workername, lazily ? "lazily" : ""); - } + ASPRINTF(msg, "Dropped client %"PRId64" %s %suser %s worker %s %s", + client->id, client->address, user->throttled ? "throttled " : "", + user->username, client->workername, lazily ? "lazily" : ""); } else { ASPRINTF(msg, "Dropped client %"PRId64" %s no user worker %s %s", client->id, client->address, client->workername, @@ -1328,7 +1326,7 @@ static void _dec_instance_ref(sdata_t *sdata, stratum_instance_t *client, const const char *func, const int line) { char_entry_t *entries = NULL; - char *msg = NULL; + char *msg; int ref; ck_wlock(&sdata->instance_lock); @@ -1337,10 +1335,7 @@ static void _dec_instance_ref(sdata_t *sdata, stratum_instance_t *client, const * moved due to holding a reference and drop them now. */ if (unlikely(client->dropped && !ref)) { __drop_client(sdata, client, true, &msg); - /* Make sure there is a message as throttled users don't - * generate a message */ - if (msg) - add_msg_entry(&entries, &msg); + add_msg_entry(&entries, &msg); } ck_wunlock(&sdata->instance_lock); From 3f83abab8d40f7ea3c30d7faec6bf3fb880f299b Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Tue, 17 Nov 2015 18:52:05 +1100 Subject: [PATCH 59/62] Fix url_from_socket to work with IPV6 sized names --- src/libckpool.c | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/libckpool.c b/src/libckpool.c index 91799eff..0c1b891e 100644 --- a/src/libckpool.c +++ b/src/libckpool.c @@ -570,14 +570,15 @@ out: * INET6_ADDRSTRLEN size, port at least a string of 6 bytes */ bool url_from_socket(const int sockd, char *url, char *port) { - socklen_t addrlen = sizeof(struct sockaddr); - struct sockaddr addr; + struct sockaddr_storage storage; + socklen_t addrlen = sizeof(struct sockaddr_storage); + struct sockaddr *addr = (struct sockaddr *)&storage; if (sockd < 1) return false; - if (getsockname(sockd, &addr, &addrlen)) + if (getsockname(sockd, addr, &addrlen)) return false; - if (!url_from_sockaddr(&addr, url, port)) + if (!url_from_sockaddr(addr, url, port)) return false; return true; } From 449e60511385a66428d79deaa1b9444f64729ba6 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Tue, 17 Nov 2015 19:05:36 +1100 Subject: [PATCH 60/62] Specify what message didn't receive a response in send_recv_path --- src/ckpool.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/ckpool.c b/src/ckpool.c index 6fbffb98..350876cc 100644 --- a/src/ckpool.c +++ b/src/ckpool.c @@ -1467,7 +1467,8 @@ static bool send_recv_path(const char *path, const char *msg) ret = true; LOGWARNING("Received: %s in response to %s request", response, msg); dealloc(response); - } + } else + LOGWARNING("Received not response to %s request", msg); Close(sockd); return ret; } From 08fcf808a70ec8ecfdc27266720591f7acbb2793 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Tue, 17 Nov 2015 19:12:37 +1100 Subject: [PATCH 61/62] Specify what url has been inherited upon restart if possible --- src/ckpool.c | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/src/ckpool.c b/src/ckpool.c index 350876cc..eca22e9c 100644 --- a/src/ckpool.c +++ b/src/ckpool.c @@ -1708,6 +1708,7 @@ int main(int argc, char **argv) if (send_recv_path(path, "ping")) { for (i = 0; i < ckp.serverurls; i++) { + char oldurl[INET6_ADDRSTRLEN], oldport[8]; char getfd[16]; int sockd; @@ -1719,10 +1720,16 @@ int main(int argc, char **argv) break; ckp.oldconnfd[i] = get_fd(sockd); Close(sockd); - if (!ckp.oldconnfd[i]) + sockd = ckp.oldconnfd[i]; + if (!sockd) break; - LOGWARNING("Inherited old server socket %d with new file descriptor %d!", - i, ckp.oldconnfd[i]); + if (url_from_socket(sockd, oldurl, oldport)) { + LOGWARNING("Inherited old server socket %d url %s:%s !", + i, oldurl, oldport); + } else { + LOGWARNING("Inherited old server socket %d with new file descriptor %d!", + i, ckp.oldconnfd[i]); + } } send_recv_path(path, "reject"); send_recv_path(path, "reconnect"); From b0c30a3f7deb558d71a5a43a2528b817d6f815d9 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Fri, 20 Nov 2015 10:09:19 +1100 Subject: [PATCH 62/62] Disconnect clients whose sends block for more than 60 seconds --- src/connector.c | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/src/connector.c b/src/connector.c index a7b776ab..981e8173 100644 --- a/src/connector.c +++ b/src/connector.c @@ -57,6 +57,9 @@ struct client_instance { /* Are we currently sending a blocked message from this client */ sender_send_t *sending; bool passthrough; + + /* Time this client started blocking, 0 when not blocked */ + time_t blocked_time; }; struct sender_send { @@ -556,6 +559,7 @@ out: static bool send_sender_send(ckpool_t *ckp, cdata_t *cdata, sender_send_t *sender_send) { client_instance_t *client = sender_send->client; + time_t now_t; if (unlikely(client->invalid)) goto out_true; @@ -563,14 +567,26 @@ static bool send_sender_send(ckpool_t *ckp, cdata_t *cdata, sender_send_t *sende /* Make sure we only send one message at a time to each client */ if (unlikely(client->sending && client->sending != sender_send)) return false; + client->sending = sender_send; + now_t = time(NULL); while (sender_send->len) { int ret = write(client->fd, sender_send->buf + sender_send->ofs, sender_send->len); if (unlikely(ret < 1)) { - if (errno == EAGAIN || errno == EWOULDBLOCK || !ret) + /* Invalidate clients that block for more than 60 seconds */ + if (unlikely(client->blocked_time && now_t - client->blocked_time >= 60)) { + LOGNOTICE("Client id %"PRId64" fd %d blocked for >60 seconds, disconnecting", + client->id, client->fd); + invalidate_client(ckp, cdata, client); + goto out_true; + } + if (errno == EAGAIN || errno == EWOULDBLOCK || !ret) { + if (!client->blocked_time) + client->blocked_time = now_t; return false; + } LOGINFO("Client id %"PRId64" fd %d disconnected with write errno %d:%s", client->id, client->fd, errno, strerror(errno)); invalidate_client(ckp, cdata, client); @@ -578,6 +594,7 @@ static bool send_sender_send(ckpool_t *ckp, cdata_t *cdata, sender_send_t *sende } sender_send->ofs += ret; sender_send->len -= ret; + client->blocked_time = 0; } out_true: client->sending = NULL;