From 951f9019a9192808dd60ab28f55274ce96bb8c7e Mon Sep 17 00:00:00 2001 From: kanoi Date: Wed, 11 Mar 2015 18:38:32 +1100 Subject: [PATCH] ckdb/php - more block statistics --- configure.ac | 7 ++- pool/page_blocks.php | 112 +++++++++++++++++++---------------- src/Makefile.am | 2 +- src/ckdb.c | 2 + src/ckdb.h | 29 ++++++++- src/ckdb_cmd.c | 136 +++++++++++++++++++++++++++++++++++-------- src/ckdb_data.c | 108 ++++++++++++++++++++++++++++++++++ src/ckdb_dbio.c | 12 +++- 8 files changed, 327 insertions(+), 81 deletions(-) diff --git a/configure.ac b/configure.ac index 49d28ee2..d55e77b6 100644 --- a/configure.ac +++ b/configure.ac @@ -38,6 +38,7 @@ AC_CHECK_HEADERS(alloca.h pthread.h stdio.h math.h signal.h sys/prctl.h) AC_CHECK_HEADERS(sys/types.h sys/socket.h sys/stat.h linux/un.h netdb.h) AC_CHECK_HEADERS(stdint.h netinet/in.h netinet/tcp.h sys/ioctl.h getopt.h) AC_CHECK_HEADERS(sys/epoll.h libpq-fe.h postgresql/libpq-fe.h grp.h) +AC_CHECK_HEADERS(gsl/gsl_math.h gsl/gsl_cdf.h) PTHREAD_LIBS="-lpthread" MATH_LIBS="-lm" @@ -59,8 +60,12 @@ AC_ARG_WITH([ckdb], if test "x$ckdb" != "xno"; then AC_CHECK_LIB([pq], [main],[PQ=-lpq],echo "Error: Required library libpq-dev not found. Install it or disable postgresql support with --without-ckdb" && exit 1) + AC_CHECK_LIB([gsl], [main],[GSL=-lgsl],echo "Error: Required library gsl-dev + not found. Install it or disable support with --without-ckdb" && exit 1) + AC_CHECK_LIB([gslcblas], [main],[GSLCBLAS=-lgslcblas],echo "Error: Required library gslcblas + not found. Install it or disable support with --without-ckdb" && exit 1) AC_DEFINE([USE_CKDB], [1], [Defined to 1 if ckdb support required]) - PQ_LIBS="-lpq" + PQ_LIBS="-lpq -lgsl -lgslcblas" else PQ_LIBS="" fi diff --git a/pool/page_blocks.php b/pool/page_blocks.php index d7b7d189..1556323c 100644 --- a/pool/page_blocks.php +++ b/pool/page_blocks.php @@ -56,6 +56,43 @@ function doblocks($data, $user) if ($wantcsv === false) { + if ($ans['STATUS'] == 'ok' and isset($ans['s_rows']) and $ans['s_rows'] > 0) + { + $pg .= '

Block Statistics

'; + $pg .= "\n"; + $pg .= ""; + $pg .= ""; + $pg .= ""; + $pg .= ""; + $pg .= ""; + $pg .= ""; + $pg .= "\n"; + + $count = $ans['s_rows']; + for ($i = 0; $i < $count; $i++) + { + if (($i % 2) == 0) + $row = 'even'; + else + $row = 'odd'; + + $desc = $ans['s_desc:'.$i]; + $diff = number_format(100 * $ans['s_diffratio:'.$i], 2); + $mean = number_format(100 * $ans['s_diffmean:'.$i], 2); + $cdferl = number_format($ans['s_cdferl:'.$i], 4); + $luck = number_format(100 * $ans['s_luck:'.$i], 2); + + $pg .= ""; + $pg .= ""; + $pg .= ""; + $pg .= ""; + $pg .= ""; + $pg .= ""; + $pg .= "\n"; + } + $pg .= "
DescriptionDiff%Mean%CDF[Erl]Luck%
$desc Blocks$diff%$mean%$cdferl$luck%
\n"; + } + if ($ans['STATUS'] == 'ok') { $count = $ans['rows']; @@ -70,15 +107,15 @@ function doblocks($data, $user) $s = 's'; } - $pg = "

Last$num Block$s

"; + $pg .= "

Last$num Block$s

"; } else - $pg = '

Blocks

'; + $pg .= '

Blocks

'; list($fg, $bg) = pctcolour(25.0); $pg .= ""; $pg .= " Green  "; - $pg .= 'is good luck. Lower Diff% and brighter green is best luck.
'; + $pg .= 'is good luck. Lower Diff% and brighter green is better luck.
'; list($fg, $bg) = pctcolour(100.0); $pg .= ""; $pg .= " 100%  "; @@ -94,7 +131,7 @@ function doblocks($data, $user) $pg .= "Height"; if ($user !== null) $pg .= "Who"; - $pg .= "Reward"; + $pg .= "Block Reward"; $pg .= "When"; $pg .= "Status"; $pg .= "Diff"; @@ -109,7 +146,6 @@ function doblocks($data, $user) $csv = "Sequence,Height,Status,Timestamp,DiffAcc,NetDiff,Hash\n"; if ($ans['STATUS'] == 'ok') { - $tot = $ans['tot']; $count = $ans['rows']; for ($i = 0; $i < $count; $i++) { @@ -130,7 +166,7 @@ function doblocks($data, $user) $seq = ''; } else - $seq = $tot--; + $seq = $ans['seq:'.$i]; if ($stat == '1-Confirm') { if (isset($data['info']['lastheight'])) @@ -143,37 +179,34 @@ function doblocks($data, $user) } $stara = ''; - $starp = ''; - if (isset($ans['status:'.($i+1)])) - if ($ans['status:'.($i+1)] == 'Orphan' - && $stat != 'Orphan') - { - $stara = '*'; - $starp = '*'; - } + if ($stat == 'Orphan') + $stara = '*'; $diffacc = $ans['diffacc:'.$i]; $acc = number_format($diffacc, 0); $netdiff = $ans['netdiff:'.$i]; - if ($netdiff > 0) + $diffratio = $ans['diffratio:'.$i]; + $cdf = $ans['cdf:'.$i]; + $luck = $ans['luck:'.$i]; + + if ($diffratio > 0) { - $pct = 100.0 * $diffacc / $netdiff; + $pct = 100.0 * $diffratio; list($fg, $bg) = pctcolour($pct); - $bpct = "$starp".number_format($pct, 2).'%'; + $bpct = "".number_format($pct, 2).'%'; $bg = " bgcolor=$bg"; $blktot += $diffacc; if ($stat != 'Orphan') $nettot += $netdiff; - $cdfv = 1 - exp(-1 * $diffacc / $netdiff); - $cdf = number_format($cdfv, 2); + $cdfdsp = number_format($cdf, 2); } else { $bg = ''; $bpct = '?'; - $cdf = '?'; + $cdfdsp = '?'; } if ($wantcsv === false) @@ -188,7 +221,7 @@ function doblocks($data, $user) $pg .= "".$stat.''; $pg .= "$stara$acc"; $pg .= "$bpct"; - $pg .= "$cdf"; + $pg .= "$cdfdsp"; $pg .= "\n"; } else @@ -208,39 +241,16 @@ function doblocks($data, $user) echo $csv; exit(0); } - if ($nettot > 0) + if ($orph === true) { - if (($i % 2) == 0) - $row = 'even'; - else - $row = 'odd'; - - $pct = 100.0 * $blktot / $nettot; - list($fg, $bg) = pctcolour($pct); - $bpct = "".number_format($pct, 2).'%'; - $bg = " bgcolor=$bg"; - - $pg .= ""; - $pg .= 'Total:'; - $pg .= ''; - $pg .= "\n"; - if ($orph === true) - { - $pg .= ' #endif +#include +#include + #include "ckpool.h" #include "libckpool.h" @@ -52,7 +55,7 @@ #define DB_VLOCK "1" #define DB_VERSION "1.0.0" -#define CKDB_VERSION DB_VERSION"-1.012" +#define CKDB_VERSION DB_VERSION"-1.020" #define WHERE_FFL " - from %s %s() line %d" #define WHERE_FFL_HERE __FILE__, __func__, __LINE__ @@ -192,6 +195,11 @@ extern POOLSTATUS pool; _dstoff += _srclen; \ } while(0) +#define APPEND_REALLOC_RESET(_buf, _off) do { \ + (_buf)[0] = '\0'; \ + _off = 0; \ + } while(0) + enum data_type { TYPE_STR, TYPE_BIGINT, @@ -1095,6 +1103,22 @@ typedef struct blocks { char statsconfirmed[TXT_FLAG+1]; HISTORYDATECONTROLFIELDS; bool ignore; // Non DB field + + // Calculated only when = 0 + double netdiff; + + /* Non DB fields for the web page + * Calculate them once off/recalc them when required */ + double blockdiffratio; + double blockcdf; + double blockluck; + + /* From the last found block to this one + * Orphans have these set to zero */ + double diffratio; + double diffmean; + double cdferl; + double luck; } BLOCKS; #define ALLOC_BLOCKS 100 @@ -1134,6 +1158,8 @@ extern const char *blocks_unknown; extern K_TREE *blocks_root; extern K_LIST *blocks_free; extern K_STORE *blocks_store; +extern tv_t blocks_stats_time; +extern bool blocks_stats_rebuild; // MININGPAYOUTS typedef struct miningpayouts { @@ -1750,6 +1776,7 @@ extern K_ITEM *find_prev_blocks(int32_t height); extern const char *blocks_confirmed(char *confirmed); extern void zero_on_new_block(); extern void set_block_share_counters(); +extern bool check_update_blocks_stats(tv_t *stats); extern cmp_t cmp_miningpayouts(K_ITEM *a, K_ITEM *b); extern K_ITEM *find_miningpayouts(int64_t payoutid, int64_t userid); extern K_ITEM *first_miningpayouts(int64_t payoutid, K_TREE_CTX *ctx); diff --git a/src/ckdb_cmd.c b/src/ckdb_cmd.c index 303be5cd..411507e6 100644 --- a/src/ckdb_cmd.c +++ b/src/ckdb_cmd.c @@ -811,23 +811,30 @@ static char *cmd_blocklist(__maybe_unused PGconn *conn, char *cmd, char *id, __maybe_unused K_TREE *trf_root) { K_TREE_CTX ctx[1]; - K_ITEM *b_item, *w_item; + K_ITEM *b_item; BLOCKS *blocks; char reply[1024] = ""; char tmp[1024]; - char *buf; + char *buf, *desc, desc_buf[64]; size_t len, off; int32_t height = -1; - tv_t first_cd = {0,0}; - int rows, tot; + tv_t first_cd = {0,0}, stats_tv = {0,0}, stats_tv2 = {0,0}; + int rows, srows, tot, seq; + bool has_stats; LOGDEBUG("%s(): cmd '%s'", __func__, cmd); APPEND_REALLOC_INIT(buf, off, len); APPEND_REALLOC(buf, off, len, "ok."); - rows = 0; + +redo: + K_WLOCK(blocks_free); + has_stats = check_update_blocks_stats(&stats_tv); + K_WUNLOCK(blocks_free); + + srows = rows = 0; K_RLOCK(blocks_free); - b_item = last_in_ktree(blocks_root, ctx); + b_item = first_in_ktree(blocks_root, ctx); tot = 0; while (b_item) { DATA_BLOCKS(blocks, b_item); @@ -835,16 +842,31 @@ static char *cmd_blocklist(__maybe_unused PGconn *conn, char *cmd, char *id, if (blocks->confirmed[0] != BLOCKS_ORPHAN) tot++; } - b_item = prev_in_ktree(ctx); + b_item = next_in_ktree(ctx); } + seq = tot; b_item = last_in_ktree(blocks_root, ctx); while (b_item && rows < 42) { DATA_BLOCKS(blocks, b_item); + /* For each block remember the initial createdate + * Reverse sort order the oldest expirydate is first + * which should be the 'n' record */ if (height != blocks->height) { height = blocks->height; copy_tv(&first_cd, &(blocks->createdate)); } if (CURRENT(&(blocks->expirydate))) { + if (blocks->confirmed[0] == BLOCKS_ORPHAN) { + snprintf(tmp, sizeof(tmp), + "seq:%d=o%c", + rows, FLDSEP); + APPEND_REALLOC(buf, off, len, tmp); + } else { + snprintf(tmp, sizeof(tmp), + "seq:%d=%d%c", + rows, seq--, FLDSEP); + APPEND_REALLOC(buf, off, len, tmp); + } int_to_buf(blocks->height, reply, sizeof(reply)); snprintf(tmp, sizeof(tmp), "height:%d=%s%c", rows, reply, FLDSEP); APPEND_REALLOC(buf, off, len, tmp); @@ -900,21 +922,21 @@ static char *cmd_blocklist(__maybe_unused PGconn *conn, char *cmd, char *id, snprintf(tmp, sizeof(tmp), "elapsed:%d=%s%c", rows, reply, FLDSEP); APPEND_REALLOC(buf, off, len, tmp); - w_item = find_workinfo(blocks->workinfoid, NULL); - if (w_item) { - char wdiffbin[TXT_SML+1]; - double wdiff; - WORKINFO *workinfo; - DATA_WORKINFO(workinfo, w_item); - hex2bin(wdiffbin, workinfo->bits, 4); - wdiff = diff_from_nbits(wdiffbin); - snprintf(tmp, sizeof(tmp), - "netdiff:%d=%.1f%c", - rows, wdiff, FLDSEP); - APPEND_REALLOC(buf, off, len, tmp); + if (has_stats) { + snprintf(tmp, sizeof(tmp), + "netdiff:%d=%.8f%cdiffratio:%d=%.8f%c" + "cdf:%d=%.8f%cluck:%d=%.8f%c", + rows, blocks->netdiff, FLDSEP, + rows, blocks->blockdiffratio, FLDSEP, + rows, blocks->blockcdf, FLDSEP, + rows, blocks->blockluck, FLDSEP); + APPEND_REALLOC(buf, off, len, tmp); } else { snprintf(tmp, sizeof(tmp), - "netdiff:%d=?%c", rows, FLDSEP); + "netdiff:%d=?%cdiffratio:%d=?%c" + "cdf:%d=?%cluck:%d=?%c", + rows, FLDSEP, rows, FLDSEP, + rows, FLDSEP, rows, FLDSEP); APPEND_REALLOC(buf, off, len, tmp); } @@ -922,17 +944,81 @@ static char *cmd_blocklist(__maybe_unused PGconn *conn, char *cmd, char *id, } b_item = prev_in_ktree(ctx); } + if (has_stats) { + seq = tot; + b_item = last_in_ktree(blocks_root, ctx); + while (b_item) { + DATA_BLOCKS(blocks, b_item); + if (CURRENT(&(blocks->expirydate)) && + blocks->confirmed[0] != BLOCKS_ORPHAN) { + desc = NULL; + if (seq == 1) { + snprintf(desc_buf, sizeof(desc_buf), + "All - Last %d", tot); + desc = desc_buf; + } else if (seq == tot - 4) { + desc = "Last 5"; + } else if (seq == tot - 9) { + desc = "Last 10"; + } else if (seq == tot - 19) { + desc = "Last 20"; + } else if (seq == tot - 41) { + desc = "Last 42"; + } + if (desc) { + snprintf(tmp, sizeof(tmp), + "s_seq:%d=%d%c" + "s_desc:%d=%s%c" + "s_diffratio:%d=%.8f%c" + "s_diffmean:%d=%.8f%c" + "s_cdferl:%d=%.8f%c" + "s_luck:%d=%.8f%c", + srows, seq, FLDSEP, + srows, desc, FLDSEP, + srows, blocks->diffratio, FLDSEP, + srows, blocks->diffmean, FLDSEP, + srows, blocks->cdferl, FLDSEP, + srows, blocks->luck, FLDSEP); + APPEND_REALLOC(buf, off, len, tmp); + srows++; + } + seq--; + } + b_item = prev_in_ktree(ctx); + } + copy_tv(&stats_tv2, &blocks_stats_time); + } K_RUNLOCK(blocks_free); + + // Only check for a redo if we used the stats values + if (has_stats) { + /* If the stats changed then redo with the new corrected values + * This isn't likely at all, but it guarantees the blocks + * page shows correct information since any code that wants + * to modify the blocks table must have it under write lock + * then flag the stats as needing to be recalculated */ + if (!tv_equal(&stats_tv, &stats_tv2)) { + APPEND_REALLOC_RESET(buf, off); + goto redo; + } + } + + snprintf(tmp, sizeof(tmp), + "s_rows=%d%cs_flds=%s%c", + srows, FLDSEP, + "s_seq,s_desc,s_diffratio,s_diffmean,s_cdferl,s_luck", + FLDSEP); + APPEND_REALLOC(buf, off, len, tmp); + snprintf(tmp, sizeof(tmp), - "tot=%d%crows=%d%cflds=%s%c", - tot, FLDSEP, + "rows=%d%cflds=%s%c", rows, FLDSEP, - "height,blockhash,nonce,reward,workername,firstcreatedate," + "seq,height,blockhash,nonce,reward,workername,firstcreatedate," "createdate,status,diffacc,diffinv,shareacc,shareinv,elapsed," - "netdiff", FLDSEP); + "netdiff,diffratio,cdf,luck", FLDSEP); APPEND_REALLOC(buf, off, len, tmp); - snprintf(tmp, sizeof(tmp), "arn=%s%carp=%s", "Blocks", FLDSEP, ""); + snprintf(tmp, sizeof(tmp), "arn=%s%carp=%s", "Blocks,BlockStats", FLDSEP, ",s"); APPEND_REALLOC(buf, off, len, tmp); LOGDEBUG("%s.ok.%d_blocks", id, rows); diff --git a/src/ckdb_data.c b/src/ckdb_data.c index 3878f342..5d9907f6 100644 --- a/src/ckdb_data.c +++ b/src/ckdb_data.c @@ -2399,6 +2399,114 @@ void set_block_share_counters() LOGWARNING("%s(): Update block counters complete", __func__); } +/* Must be under K_WLOCK(blocks_free) when called + * Call this before using the block stats and again check (under lock) + * the blocks_stats_time didn't change after you finish processing + * If it has changed, redo the processing from scratch + * If return is false, then stats aren't available + * TODO: consider storing the partial calculations in the BLOCKS structure + * and only recalc from the last block modified (remembered) + * Will be useful with a large block history */ +bool check_update_blocks_stats(tv_t *stats) +{ + static int64_t last_missing_workinfoid = 0; + static tv_t last_message = { 0L, 0L }; + K_TREE_CTX ctx[1]; + K_ITEM *b_item, *w_item; + WORKINFO *workinfo; + BLOCKS *blocks; + char ndiffbin[TXT_SML+1]; + double ok, diffacc, netsumm, diffmean, pending; + tv_t now; + + /* Wait for startup_complete rather than db_load_complete + * This avoids doing a 'long' lock stats update while reloading */ + if (!startup_complete) + return false; + + if (blocks_stats_rebuild) { + ok = diffacc = netsumm = diffmean = pending = 0.0; + b_item = last_in_ktree(blocks_root, ctx); + while (b_item) { + DATA_BLOCKS(blocks, b_item); + if (CURRENT(&(blocks->expirydate))) { + if (blocks->netdiff == 0) { + // Deadlock alert + K_RLOCK(workinfo_free); + w_item = find_workinfo(blocks->workinfoid, NULL); + K_RUNLOCK(workinfo_free); + if (!w_item) { + setnow(&now); + if (!blocks->workinfoid != last_missing_workinfoid || + tvdiff(&now, &last_message) >= 5.0) { + LOGEMERG("%s(): missing block workinfoid %" + PRId32"/%"PRId64"/%s", + __func__, blocks->height, + blocks->workinfoid, + blocks->confirmed); + } + last_missing_workinfoid = blocks->workinfoid; + copy_tv(&last_message, &now); + return false; + } + DATA_WORKINFO(workinfo, w_item); + hex2bin(ndiffbin, workinfo->bits, 4); + blocks->netdiff = diff_from_nbits(ndiffbin); + } + pending += blocks->diffacc; + + /* Stats for each blocks are independent of + * if they are orphans or not */ + if (blocks->netdiff == 0.0) + blocks->blockdiffratio = 0.0; + else + blocks->blockdiffratio = blocks->diffacc / blocks->netdiff; + blocks->blockcdf = 1.0 - exp(-1.0 * blocks->blockdiffratio); + if (blocks->blockdiffratio == 0.0) + blocks->blockluck = 0.0; + else + blocks->blockluck = 1.0 / blocks->blockdiffratio; + + /* Orphans are treated as +diffacc but no block + * i.e. they simply add shares to the later block + * and have running stats set to zero */ + if (blocks->confirmed[0] == BLOCKS_ORPHAN) { + blocks->diffratio = 0.0; + blocks->diffmean = 0.0; + blocks->cdferl = 0.0; + blocks->luck = 0.0; + } else { + ok++; + diffacc += pending; + pending = 0.0; + netsumm += blocks->netdiff; + + if (netsumm == 0.0) + blocks->diffratio = 0.0; + else + blocks->diffratio = diffacc / netsumm; + + diffmean = (diffmean * (ok - 1) + blocks->blockdiffratio) / ok; + blocks->diffmean = diffmean; + + if (diffmean == 0.0) { + blocks->cdferl = 0.0; + blocks->luck = 0.0; + } else { + blocks->cdferl = gsl_cdf_gamma_P(diffmean, ok, 1.0 / ok); + blocks->luck = 1.0 / diffmean; + } + } + } + b_item = prev_in_ktree(ctx); + } + setnow(&blocks_stats_time); + blocks_stats_rebuild = false; + } + copy_tv(stats, &blocks_stats_time); + return true; +} + /* order by payoutid asc,userid asc,expirydate asc * i.e. only one payout amount per block per user */ cmp_t cmp_miningpayouts(K_ITEM *a, K_ITEM *b) diff --git a/src/ckdb_dbio.c b/src/ckdb_dbio.c index 4db4f0ba..91e43fd8 100644 --- a/src/ckdb_dbio.c +++ b/src/ckdb_dbio.c @@ -3996,9 +3996,13 @@ unparam: blocks_root = remove_from_ktree(blocks_root, old_b_item, cmp_blocks); copy_tv(&(oldblocks->expirydate), cd); blocks_root = add_to_ktree(blocks_root, old_b_item, cmp_blocks); - } + // Copy it over to avoid having to recalculate it + row->netdiff = oldblocks->netdiff; + } else + row->netdiff = 0; blocks_root = add_to_ktree(blocks_root, b_item, cmp_blocks); k_add_head(blocks_store, b_item); + blocks_stats_rebuild = true; } K_WUNLOCK(blocks_free); @@ -4300,9 +4304,13 @@ flail: blocks_root = remove_from_ktree(blocks_root, old_b_item, cmp_blocks); copy_tv(&(oldblocks->expirydate), cd); blocks_root = add_to_ktree(blocks_root, old_b_item, cmp_blocks); - } + // Copy it over to avoid having to recalculate it + row->netdiff = oldblocks->netdiff; + } else + row->netdiff = 0; blocks_root = add_to_ktree(blocks_root, b_item, cmp_blocks); k_add_head(blocks_store, b_item); + blocks_stats_rebuild = true; } K_WUNLOCK(blocks_free);