From 3beb8cc0493e95230780b431b9a7c4e04c432640 Mon Sep 17 00:00:00 2001 From: kanoi Date: Fri, 23 Sep 2016 21:27:53 +1000 Subject: [PATCH 01/33] ckdb - add thread control for b/clistener and comment about ckdb thread setup --- src/ckdb.c | 351 ++++++++++++++++++++++++++++++++++++------------- src/ckdb.h | 7 +- src/ckdb_cmd.c | 16 +++ 3 files changed, 278 insertions(+), 96 deletions(-) diff --git a/src/ckdb.c b/src/ckdb.c index 1b38ff00..76591c6c 100644 --- a/src/ckdb.c +++ b/src/ckdb.c @@ -14,6 +14,48 @@ * Consider adding row level locking (a per kitem usage count) if needed */ +/* Thread layout + * ------------- + * Any thread that manages a thread count will have 00 in it's name + * and name each subsequent thread it creates, with the same name + * but with 01, 02 etc, and then wait on them all before exiting + * The 2 digit 00 relates to THREAD_LIMIT which is 99 + * there's a limit of THREAD_LIMIT active threads per thread manager + * WARNING - however the total number of threads created is limited by the + * LOCK_CHECK code which allows only MAX_THREADS total ckdb thread to + * be created irrelevant of any being deleted + * + * The threads that can be managed have a command option to set them when + * running ckdb and can also be changed via the cmd_threads socket command + * + * The main() 'ckdb' thread starts: + * iomsgs() for filelog '_fiomsgs' and console '_ciomsgs' + * listener() '_p00qproc' + * which manages it's thread count in pqproc() + * + * listener() starts: + * breakdown() for reload '_r00breaker' and cmd '_c00breaker' + * each of which manage their thead counts + * logger() '_logger' + * socksetup() '_socksetup' + * summariser() '_summarise' + * marker() '_marker' + * then calls setup_data() which calls reload() + * then calls pqproc() + * which manages the thread count + * + * socksetup() starts: + * replier() for pool '_preplier' cmd '_creplier' and btc '_breplier' + * listener_all() for cmd '_c00listen' and btc '_b00listen' + * each of which manage their thead counts + * process_socket() '_procsock' + * sockrun() for ckpool '_psockrun' web '_wsockrun' and cmd '_csockrun' + * + * reload() starts: + * process_reload() '_p00rload' + * which manages it's thread count + */ + /* Startup * ------- * During startup we load the DB and track where it is up to with @@ -157,6 +199,11 @@ static int cmd_breakdown_threads = -1; int reload_breakdown_threads_delta = 0; int cmd_breakdown_threads_delta = 0; +int cmd_listener_threads = 2; +int btc_listener_threads = 2; +int cmd_listener_threads_delta = 0; +int btc_listener_threads_delta = 0; + // Lock used to determine when the last breakdown thread exits static cklock_t breakdown_lock; @@ -381,10 +428,10 @@ static uint64_t sock_acc[MAXSOCK], sock_recv[MAXSOCK]; // breaker() summarised static tv_t break_reload_stt, break_cmd_stt, break_reload_fin; static uint64_t break_reload_processed, break_cmd_processed; -// clistener() +// listener_all() +static cklock_t listener_all_lock; static double clis_us; static uint64_t clis_processed; -// blistener() static double blis_us; static uint64_t blis_processed; @@ -958,7 +1005,7 @@ static void ioprocess(IOQUEUE *io) flock(logfd, LOCK_EX); if (io->errn) { fprintf(LOGFP, "%s%s with errno %d: %s\n", - stamp, io->msg, + stamp, io->msg, io->errn, strerror(io->errn)); } else fprintf(LOGFP, "%s%s\n", stamp, io->msg); @@ -3202,7 +3249,7 @@ static bool update_seq(enum seq_num seq, uint64_t n_seqcmd, goto gotseqset; } } - } + } // Need to setup a new seqset newseq = true; @@ -3464,7 +3511,7 @@ gotseqset: } } seqdata->seqbase++; - } + } seqdata->maxseq++; } // store n_seqcmd @@ -4525,7 +4572,7 @@ static void *breaker(void *arg) } K_WUNLOCK(breakqueue_free); while (!everyone_die) { - if (mythread && !breaker_running[typ][mythread]) + if (mythread && !breaker_running[typ][mythread]) break; K_WLOCK(breakqueue_free); @@ -6051,37 +6098,163 @@ static void process_sockd(PGconn *conn, K_ITEM *wq_item, enum reply_type reply_t tick(); } -static void *clistener(__maybe_unused void *arg) +struct listener_setup { + int bc; + int thread; +}; + +#define BC_B 0 +#define BC_C 1 + +static pthread_t listener_pt[2][THREAD_LIMIT]; +static struct listener_setup listener_setup[2][THREAD_LIMIT]; + +static void *listener_all(void *arg) { + static bool running[2][THREAD_LIMIT]; + + struct listener_setup *setup; PGconn *conn = NULL; K_ITEM *wq_item; tv_t now1, now2; char buf[128]; time_t now; ts_t when, when_add; - int ret; + int i, typ, mythread, done, tot, ret; + int listener_delta = 0; - pthread_detach(pthread_self()); + setup = (struct listener_setup *)(arg); + typ = setup->bc; + mythread = setup->thread; - snprintf(buf, sizeof(buf), "db%s_%s", dbcode, __func__); + snprintf(buf, sizeof(buf), "db%s_%c%02d%s", + dbcode, (typ == BC_B) ? 'b' : 'c', mythread, "listen"); LOCK_INIT(buf); rename_proc(buf); - LOGNOTICE("%s() processing", __func__); + if (mythread == 0) { + pthread_detach(pthread_self()); + + for (i = 1; i < THREAD_LIMIT; i++) { + listener_setup[typ][i].thread = i; + listener_setup[typ][i].bc = typ; + running[typ][i] = false; + } + running[typ][0] = true; + + if (typ == BC_B) + listener_delta = btc_listener_threads - 1; + else + listener_delta = cmd_listener_threads - 1; + + LOGNOTICE("%s() %s initialised - delta %d", + __func__, (typ == BC_B) ? "btc" : "cmd", listener_delta); + } + LOGNOTICE("%s() %s processing", __func__, buf); when_add.tv_sec = CMD_QUEUE_SLEEP_MS / 1000; when_add.tv_nsec = (CMD_QUEUE_SLEEP_MS % 1000) * 1000000; - clistener_using_data = true; + if (typ == BC_B) + blistener_using_data = true; + else + clistener_using_data = true; conn = dbconnect(); now = time(NULL); while (!everyone_die) { + if (mythread && !running[typ][mythread]) + break; + + wq_item = NULL; K_WLOCK(workqueue_free); - wq_item = k_unlink_head(cmd_workqueue_store); + if (mythread == 0 && typ == BC_B && + btc_listener_threads_delta != 0) { + listener_delta = btc_listener_threads_delta; + btc_listener_threads_delta = 0; + } else if (mythread == 0 && typ == BC_C && + cmd_listener_threads_delta != 0) { + listener_delta = cmd_listener_threads_delta; + cmd_listener_threads_delta = 0; + } else { + if (typ == BC_B) + wq_item = k_unlink_head(btc_workqueue_store); + else + wq_item = k_unlink_head(cmd_workqueue_store); + } K_WUNLOCK(workqueue_free); + // TODO: deal with thread creation/shutdown failure + if (listener_delta != 0) { + if (listener_delta > 0) { + // Add threads + tot = 1; + done = 0; + for (i = 1; i < THREAD_LIMIT; i++) { + if (!running[typ][i]) { + if (listener_delta > 0) { + listener_delta--; + running[typ][i] = true; + create_pthread(&(listener_pt[typ][i]), + listener_all, + &(listener_setup[typ][i])); + done++; + tot++; + } + } else + tot++; + } + LOGWARNING("%s() created %d %s thread%s total=%d" +#if LOCK_CHECK + " next_thread_id=%d" +#endif + , __func__, + done, + (typ == BC_B) ? "btc" : "cmd", + (done == 1) ? EMPTY : "s", + tot +#if LOCK_CHECK + , next_thread_id +#endif + ); + } else { + // Notify and wait for each to exit + tot = 1; + done = 0; + for (i = THREAD_LIMIT - 1; i > 0; i--) { + if (running[typ][i]) { + if (listener_delta < 0) { + listener_delta++; + LOGNOTICE("%s() %s stopping %d", + __func__, + (typ == BC_B) ? "btc" : "cmd", + i); + running[typ][i] = false; + join_pthread(listener_pt[typ][i]); + done++; + } else + tot++; + } + } + LOGWARNING("%s() stopped %d %s thread%s total=%d" +#if LOCK_CHECK + " next_thread_id=%d" +#endif + , __func__, + done, + (typ == BC_B) ? "btc" : "cmd", + (done == 1) ? EMPTY : "s", + tot +#if LOCK_CHECK + , next_thread_id +#endif + ); + } + listener_delta = 0; + continue; + } + // Don't keep a connection for more than ~10s if ((time(NULL) - now) > 10) { PQfinish(conn); @@ -6091,100 +6264,68 @@ static void *clistener(__maybe_unused void *arg) if (wq_item) { setnow(&now1); - process_sockd(conn, wq_item, REPLIER_CMD); + process_sockd(conn, wq_item, + (typ == BC_B) ? REPLIER_BTC : REPLIER_CMD); setnow(&now2); - clis_us += us_tvdiff(&now2, &now1); - clis_processed++; + ck_wlock(&listener_all_lock); + if (typ == BC_B) { + blis_us += us_tvdiff(&now2, &now1); + blis_processed++; + } else { + clis_us += us_tvdiff(&now2, &now1); + clis_processed++; + } + ck_wunlock(&listener_all_lock); } else { setnowts(&when); timeraddspec(&when, &when_add); - mutex_lock(&wq_cmd_waitlock); - ret = cond_timedwait(&wq_cmd_waitcond, - &wq_cmd_waitlock, &when); - if (ret == 0) - wq_cmd_wakes++; - else if (errno == ETIMEDOUT) - wq_cmd_timeouts++; - mutex_unlock(&wq_cmd_waitlock); + if (typ == BC_B) { + mutex_lock(&wq_btc_waitlock); + ret = cond_timedwait(&wq_btc_waitcond, + &wq_btc_waitlock, &when); + if (ret == 0) + wq_btc_wakes++; + else if (errno == ETIMEDOUT) + wq_btc_timeouts++; + mutex_unlock(&wq_btc_waitlock); + } else { + mutex_lock(&wq_cmd_waitlock); + ret = cond_timedwait(&wq_cmd_waitcond, + &wq_cmd_waitlock, &when); + if (ret == 0) + wq_cmd_wakes++; + else if (errno == ETIMEDOUT) + wq_cmd_timeouts++; + mutex_unlock(&wq_cmd_waitlock); + } } } - LOGNOTICE("%s() exiting, processed %"PRIu64, __func__, clis_processed); - - clistener_using_data = false; - if (conn) PQfinish(conn); - return NULL; -} - -static void *blistener(__maybe_unused void *arg) -{ - PGconn *conn = NULL; - K_ITEM *wq_item; - tv_t now1, now2; - char buf[128]; - time_t now; - ts_t when, when_add; - int ret; - - pthread_detach(pthread_self()); - - snprintf(buf, sizeof(buf), "db%s_%s", dbcode, __func__); - LOCK_INIT(buf); - rename_proc(buf); - - LOGNOTICE("%s() processing", __func__); - - when_add.tv_sec = CMD_QUEUE_SLEEP_MS / 1000; - when_add.tv_nsec = (CMD_QUEUE_SLEEP_MS % 1000) * 1000000; - - blistener_using_data = true; - - now = time(NULL); - - while (!everyone_die) { - K_WLOCK(workqueue_free); - wq_item = k_unlink_head(btc_workqueue_store); - K_WUNLOCK(workqueue_free); - - // Don't keep a connection for more than ~10s - if ((time(NULL) - now) > 10) { - PQfinish(conn); - conn = dbconnect(); - now = time(NULL); + if (mythread != 0) + LOGNOTICE("%s() %s exiting", __func__, buf); + else { + for (i = 1; i < THREAD_LIMIT; i++) { + if (running[typ][i]) { + running[typ][i] = false; + LOGNOTICE("%s() %s waiting for %d", + __func__, buf, i); + join_pthread(listener_pt[typ][i]); + } } - if (wq_item) { - setnow(&now1); - process_sockd(conn, wq_item, REPLIER_BTC); - setnow(&now2); - blis_us += us_tvdiff(&now2, &now1); - blis_processed++; + if (typ == BC_B) { + LOGNOTICE("%s() %s exiting, processed %"PRIu64, __func__, buf, blis_processed); + blistener_using_data = false; } else { - setnowts(&when); - timeraddspec(&when, &when_add); - - mutex_lock(&wq_btc_waitlock); - ret = cond_timedwait(&wq_btc_waitcond, - &wq_btc_waitlock, &when); - if (ret == 0) - wq_btc_wakes++; - else if (errno == ETIMEDOUT) - wq_btc_timeouts++; - mutex_unlock(&wq_btc_waitlock); + LOGNOTICE("%s() %s exiting, processed %"PRIu64, __func__, buf, clis_processed); + clistener_using_data = false; } } - LOGNOTICE("%s() exiting, processed %"PRIu64, __func__, blis_processed); - - blistener_using_data = false; - - if (conn) - PQfinish(conn); - return NULL; } @@ -6841,7 +6982,7 @@ static void *socksetup(__maybe_unused void *arg) { pthread_t prep_pt, crep_pt, brep_pt; enum reply_type p_typ, c_typ, b_typ; - pthread_t clis_pt, blis_pt, proc_pt; + pthread_t proc_pt; pthread_t psock_pt, wsock_pt, csock_pt; char nbuf[64]; @@ -6868,9 +7009,15 @@ static void *socksetup(__maybe_unused void *arg) LOGWARNING("%s() Start processing...", __func__); socksetup_using_data = true; - create_pthread(&clis_pt, clistener, NULL); + listener_setup[BC_B][0].bc = BC_B; + listener_setup[BC_B][0].thread = 0; + create_pthread(&listener_pt[BC_B][0], listener_all, + &(listener_setup[BC_B][0])); - create_pthread(&blis_pt, blistener, NULL); + listener_setup[BC_C][0].bc = BC_C; + listener_setup[BC_C][0].thread = 0; + create_pthread(&listener_pt[BC_C][0], listener_all, + &(listener_setup[BC_C][0])); create_pthread(&proc_pt, process_socket, NULL); @@ -7429,7 +7576,7 @@ static bool reload_from(tv_t *start, const tv_t *finish) * Also since ckpool messages are not in order, we could be * aborting early and not get the few slightly later out of * order messages in the log file */ - while (!everyone_die && + while (!everyone_die && logline(reload_buf, MAX_READ, fp, filename)) { reload_line(filename, reload_buf, ++count); @@ -8995,6 +9142,7 @@ static struct option long_options[] = { // override calculated value { "breakdown-threads", required_argument, 0, 'B' }, { "config", required_argument, 0, 'c' }, + { "cmd-listener-threads", required_argument, 0, 'C' }, { "dbname", required_argument, 0, 'd' }, { "minsdiff", required_argument, 0, 'D' }, { "free", required_argument, 0, 'f' }, @@ -9077,7 +9225,7 @@ int main(int argc, char **argv) memset(&ckpcmd, 0, sizeof(ckp)); ckp.loglevel = LOG_NOTICE; - while ((c = getopt_long(argc, argv, "a:Ab:B:c:d:D:f:ghi:IkK:l:L:mM:n:N:o:p:P:q:Q:r:R:s:S:t:Tu:U:vw:xXyY:", long_options, &i)) != -1) { + while ((c = getopt_long(argc, argv, "a:Ab:B:c:C:d:D:f:ghi:IkK:l:L:mM:n:N:o:p:P:q:Q:r:R:s:S:t:Tu:U:vw:xXyY:", long_options, &i)) != -1) { switch(c) { case '?': case ':': @@ -9120,6 +9268,18 @@ int main(int argc, char **argv) case 'c': ckp.config = strdup(optarg); break; + case 'C': + { + int cl = atoi(optarg); + if (cl < 1 || cl > THREAD_LIMIT) { + quit(1, "Invalid listener " + "thread count %d " + "- must be >0 and <=%d", + cl, THREAD_LIMIT); + } + cmd_listener_threads = cl; + } + break; case 'd': db_name = strdup(optarg); kill = optarg; @@ -9467,6 +9627,7 @@ int main(int argc, char **argv) cklock_init(&breakdown_lock); cklock_init(&replier_lock); + cklock_init(&listener_all_lock); cklock_init(&last_lock); cklock_init(&btc_lock); cklock_init(&poolinstance_lock); diff --git a/src/ckdb.h b/src/ckdb.h index 135494ce..73258212 100644 --- a/src/ckdb.h +++ b/src/ckdb.h @@ -58,7 +58,7 @@ #define DB_VLOCK "1" #define DB_VERSION "1.0.7" -#define CKDB_VERSION DB_VERSION"-2.508" +#define CKDB_VERSION DB_VERSION"-2.509" #define WHERE_FFL " - from %s %s() line %d" #define WHERE_FFL_HERE __FILE__, __func__, __LINE__ @@ -129,6 +129,11 @@ extern int proc_queue_threads_delta; extern int reload_breakdown_threads_delta; extern int cmd_breakdown_threads_delta; +extern int cmd_listener_threads; +extern int btc_listener_threads; +extern int cmd_listener_threads_delta; +extern int btc_listener_threads_delta; + #define BLANK " " extern char *EMPTY; extern const char *nullstr; diff --git a/src/ckdb_cmd.c b/src/ckdb_cmd.c index 8754dddf..15727195 100644 --- a/src/ckdb_cmd.c +++ b/src/ckdb_cmd.c @@ -8463,6 +8463,22 @@ static char *cmd_threads(__maybe_unused PGconn *conn, char *cmd, char *id, K_WUNLOCK(breakqueue_free); snprintf(reply, siz, "ok.delta %d request sent", delta_value); return strdup(reply); + } else if (strcasecmp(name, "cl") == 0 || + strcasecmp(name, "cmd_listener") == 0) { + K_WLOCK(workqueue_free); + // Just overwrite whatever's there + cmd_listener_threads_delta = delta_value; + K_WUNLOCK(workqueue_free); + snprintf(reply, siz, "ok.delta %d request sent", delta_value); + return strdup(reply); + } else if (strcasecmp(name, "bl") == 0 || + strcasecmp(name, "btc_listener") == 0) { + K_WLOCK(workqueue_free); + // Just overwrite whatever's there + btc_listener_threads_delta = delta_value; + K_WUNLOCK(workqueue_free); + snprintf(reply, siz, "ok.delta %d request sent", delta_value); + return strdup(reply); } else { snprintf(reply, siz, "unknown name '%s'", name); LOGERR("%s() %s.%s", __func__, id, reply); From 8ed2003afc7969af48a10e4a5e5d14b4f28bd6a4 Mon Sep 17 00:00:00 2001 From: kanoi Date: Sat, 24 Sep 2016 19:46:14 +1000 Subject: [PATCH 02/33] ckdb - completely ignore dup shares, since it can stop workmarker processing --- src/ckdb.h | 2 +- src/ckdb_dbio.c | 8 ++++++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/src/ckdb.h b/src/ckdb.h index 73258212..afe5a1e1 100644 --- a/src/ckdb.h +++ b/src/ckdb.h @@ -58,7 +58,7 @@ #define DB_VLOCK "1" #define DB_VERSION "1.0.7" -#define CKDB_VERSION DB_VERSION"-2.509" +#define CKDB_VERSION DB_VERSION"-2.510" #define WHERE_FFL " - from %s %s() line %d" #define WHERE_FFL_HERE __FILE__, __func__, __LINE__ diff --git a/src/ckdb_dbio.c b/src/ckdb_dbio.c index e422f628..ff914504 100644 --- a/src/ckdb_dbio.c +++ b/src/ckdb_dbio.c @@ -4074,8 +4074,12 @@ bool shares_db(PGconn *conn, K_ITEM *s_item) res = PQexecParams(conn, ins, par, NULL, (const char **)params, NULL, NULL, 0, CKPQ_WRITE); rescode = PQresultStatus(res); if (!PGOK(rescode)) { - PGLOGERR("Insert", rescode, conn); - goto unparam; + // If the share is already in the db ignore the error + char *code = PQresultErrorField(res, PG_DIAG_SQLSTATE); + if (!code || strcmp(code, SQL_UNIQUE_VIOLATION)) { + PGLOGERR("Insert", rescode, conn); + goto unparam; + } } ok = true; From c94a57f311d684ca2ceacbcc1bfcae8a6456f417 Mon Sep 17 00:00:00 2001 From: kanoi Date: Sun, 25 Sep 2016 14:07:29 +1000 Subject: [PATCH 03/33] php - convert TAB to tabs in ckdb.php - optionally on by default --- src/ckdb.php | 27 +++++++++++++++++++-------- 1 file changed, 19 insertions(+), 8 deletions(-) diff --git a/src/ckdb.php b/src/ckdb.php index 9ae4b1a9..8fa2b97e 100644 --- a/src/ckdb.php +++ b/src/ckdb.php @@ -19,10 +19,12 @@ function getsock2($fun, $tmo) return _getsock($fun, "$socket_dir$socket_name/$socket_file", $tmo); } # -function msg($line, $tmo = false) +function msg($line, $tabs, $tmo = false) { global $fld_sep, $val_sep; + if ($tabs) + $line = str_replace("TAB", "\t", $line); $fun = 'stdin'; $ret = false; $socket = getsock2($fun, $tmo); @@ -40,7 +42,8 @@ function msg($line, $tmo = false) function usAge($a0) { global $socket_name_def, $socket_dir_def, $socket_file_def; - echo "usAge: php $a0 [name [dir [socket]]]\n"; + echo "usAge: php $a0 [-t] [name [dir [socket]]]\n"; + echo " -t = don't convert 'TAB' to a tab character\n"; echo " default name = $socket_name_def\n"; echo " default dir = $socket_dir_def\n"; echo " default socket = $socket_file_def\n"; @@ -48,18 +51,26 @@ function usAge($a0) exit(1); } # +$tabs = true; +# if (count($argv) > 1) { if ($argv[1] == '-?' || $argv[1] == '-h' || $argv[1] == '-help' || $argv[1] == '--help') usAge($argv[0]); - $socket_name = $argv[1]; - if (count($argv) > 2) + $a = 1; + if ($argv[$a] == '-t') + { + $tabs = false; + $a++; + } + $socket_name = $argv[$a++]; + if (count($argv) > $a) { - $socket_dir = $argv[2]; - if (count($argv) > 3) - $socket_file = $argv[3]; + $socket_dir = $argv[$a++]; + if (count($argv) > $a) + $socket_file = $argv[$a]; } } # @@ -68,7 +79,7 @@ while ($line = fgets(STDIN)) $line = trim($line); if (strlen($line) > 0) { - $rep = msg($line); + $rep = msg($line, $tabs); if ($rep === false) echo "Failed\n"; else From fefce2810e219fadd1d9229f993ac9e6fadac7e1 Mon Sep 17 00:00:00 2001 From: kanoi Date: Wed, 28 Sep 2016 10:19:09 +1000 Subject: [PATCH 04/33] php - scale up the rewards unit --- pool/page_shifts.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pool/page_shifts.php b/pool/page_shifts.php index 5de1b39c..1c0ffbe6 100644 --- a/pool/page_shifts.php +++ b/pool/page_shifts.php @@ -81,7 +81,7 @@ function doshifts($data, $user) $pg .= ''.$ans['rewards:'.$i].''; $ppsr = (float)$ans['ppsrewarded:'.$i]; if ($ppsr > 0) - $ppsd = sprintf('%.5f', $ppsr); + $ppsd = sprintf('%.5f', $ppsr*1000.0); else $ppsd = '0'; $pg .= "$ppsd"; @@ -96,7 +96,7 @@ function doshifts($data, $user) $pg .= ''; } $pg .= "\n"; - $pg .= "* The Rewarded value unit is satoshis per 1diff share
"; + $pg .= "* The Rewarded value unit is satoshis per 1000diff share
"; return $pg; } From 762f8ffad388e301d91f27488e2b2f0d3193095a Mon Sep 17 00:00:00 2001 From: kanoi Date: Wed, 28 Sep 2016 17:09:21 +1000 Subject: [PATCH 05/33] ckdb - store shares with unknown usernames --- src/ckdb.h | 12 +++++-- src/ckdb_cmd.c | 7 ++-- src/ckdb_dbio.c | 91 +++++++++++++++++++++++++++++++++++-------------- 3 files changed, 79 insertions(+), 31 deletions(-) diff --git a/src/ckdb.h b/src/ckdb.h index afe5a1e1..646a5e68 100644 --- a/src/ckdb.h +++ b/src/ckdb.h @@ -58,7 +58,7 @@ #define DB_VLOCK "1" #define DB_VERSION "1.0.7" -#define CKDB_VERSION DB_VERSION"-2.510" +#define CKDB_VERSION DB_VERSION"-2.511" #define WHERE_FFL " - from %s %s() line %d" #define WHERE_FFL_HERE __FILE__, __func__, __LINE__ @@ -1844,6 +1844,9 @@ typedef struct users { // Address account, not a username account #define USER_ADDRESS 0x1 +// Username created due to a share that had an unknown username +#define USER_MISSING 0x2 + // 16 x base 32 (5 bits) = 10 bytes (8 bits) #define TOTPAUTH_KEYSIZE 10 #define TOTPAUTH_DSP_KEYSIZE 16 @@ -3652,11 +3655,14 @@ extern bool users_update(PGconn *conn, K_ITEM *u_item, char *oldhash, int *event); extern K_ITEM *users_add(PGconn *conn, INTRANSIENT *in_username, char *emailaddress, char *passwordhash, - int64_t userbits, char *by, char *code, char *inet, - tv_t *cd, K_TREE *trf_root); + char *secondaryuserid, int64_t userbits, char *by, + char *code, char *inet, tv_t *cd, K_TREE *trf_root); extern bool users_replace(PGconn *conn, K_ITEM *u_item, K_ITEM *old_u_item, char *by, char *code, char *inet, tv_t *cd, K_TREE *trf_root); +extern K_ITEM *create_missing_user(PGconn *conn, char *username, + char *secondaryuserid, char *by, char *code, + char *inet, tv_t *cd, K_TREE *trf_root); extern bool users_fill(PGconn *conn); extern bool useratts_item_add(PGconn *conn, K_ITEM *ua_item, tv_t *cd, bool begun); diff --git a/src/ckdb_cmd.c b/src/ckdb_cmd.c index 15727195..e4578fa5 100644 --- a/src/ckdb_cmd.c +++ b/src/ckdb_cmd.c @@ -72,8 +72,9 @@ static char *cmd_adduser(PGconn *conn, char *cmd, char *id, tv_t *now, char *by, if (event == EVENT_OK) { u_item = users_add(conn, in_username, transfer_data(i_emailaddress), - transfer_data(i_passwordhash), 0, - by, code, inet, now, trf_root); + transfer_data(i_passwordhash), + NULL, 0, by, code, inet, now, + trf_root); } } @@ -3116,7 +3117,7 @@ static char *cmd_auth_do(PGconn *conn, char *cmd, char *id, char *by, DATA_OPTIONCONTROL(optioncontrol, oc_item); u_item = users_add(conn, in_username, EMPTY, optioncontrol->optionvalue, - 0, by, code, inet, cd, + NULL, 0, by, code, inet, cd, trf_root); } else ok = false; diff --git a/src/ckdb_dbio.c b/src/ckdb_dbio.c index ff914504..e24a9943 100644 --- a/src/ckdb_dbio.c +++ b/src/ckdb_dbio.c @@ -706,8 +706,9 @@ unparam: } K_ITEM *users_add(PGconn *conn, INTRANSIENT *in_username, char *emailaddress, - char *passwordhash, int64_t userbits, char *by, - char *code, char *inet, tv_t *cd, K_TREE *trf_root) + char *passwordhash, char *secondaryuserid, + int64_t userbits, char *by, char *code, char *inet, + tv_t *cd, K_TREE *trf_root) { ExecStatusType rescode; bool conned = false; @@ -771,9 +772,14 @@ K_ITEM *users_add(PGconn *conn, INTRANSIENT *in_username, char *emailaddress, row->status[0] = '\0'; STRNCPY(row->emailaddress, emailaddress); - snprintf(tohash, sizeof(tohash), "%s&#%s", in_username->str, emailaddress); - HASH_BER(tohash, strlen(tohash), 1, hash, tmp); - __bin2hex(row->secondaryuserid, (void *)(&hash), sizeof(hash)); + if (secondaryuserid == NULL) { + snprintf(tohash, sizeof(tohash), "%s&#%s", + in_username->str, + emailaddress); + HASH_BER(tohash, strlen(tohash), 1, hash, tmp); + __bin2hex(row->secondaryuserid, (void *)(&hash), sizeof(hash)); + } else + STRNCPY(row->secondaryuserid, secondaryuserid); make_salt(row); if (passwordhash == EMPTY) { @@ -969,6 +975,27 @@ unparam: return ok; } +/* If a share contains an unknown username then it will most likely be from + * reloading a data file when the users table is incomplete or corrupt + * Since the share is valid, it should be counted against the pool stats and + * creating a user also means the payout rewards are calculated correctly + * This also gives 2 options to resolve it later: + * 1) Correct the database/redistribute the rewards to the correct user + * 2) Rollback the database and reload it with a corrected users table + * Option 1) gives a simpler solution vs option 2) if 2) is too far in + * the past to easily do a reload */ +K_ITEM *create_missing_user(PGconn *conn, char *username, char *secondaryuserid, + char *by, char *code, char *inet, tv_t *cd, + K_TREE *trf_root) +{ + INTRANSIENT *in_username; + + in_username = get_intransient("username", username); + + return users_add(conn, in_username, EMPTY, EMPTY, secondaryuserid, + USER_MISSING, by, code, inet, cd, trf_root); +} + bool users_fill(PGconn *conn) { ExecStatusType rescode; @@ -3656,10 +3683,11 @@ static bool shares_process(PGconn *conn, SHARES *shares, K_ITEM *wi_item, shares->createcode, shares->createinet, &(shares->createdate), trf_root); if (!w_item) { - LOGDEBUG("%s(): new_default_worker failed %"PRId64"/%s/%ld,%ld", - __func__, shares->userid, - st = safe_text_nonull(shares->in_workername), - shares->createdate.tv_sec, shares->createdate.tv_usec); + LOGERR("%s(): ERR new_default_worker failed" + " %"PRId64"/%s/%ld,%ld", + __func__, shares->userid, + st = safe_text_nonull(shares->in_workername), + shares->createdate.tv_sec, shares->createdate.tv_usec); FREENULL(st); return false; } @@ -3671,12 +3699,12 @@ static bool shares_process(PGconn *conn, SHARES *shares, K_ITEM *wi_item, MARKER_PROCESSED, NULL); K_RUNLOCK(workmarkers_free); if (wm_item) { - LOGDEBUG("%s(): workmarker exists for wid %"PRId64 - " %"PRId64"/%s/%ld,%ld", - __func__, shares->workinfoid, shares->userid, - st = safe_text_nonull(shares->in_workername), - shares->createdate.tv_sec, - shares->createdate.tv_usec); + LOGERR("%s(): ERR workmarker exists for wid %"PRId64 + " %"PRId64"/%s/%ld,%ld", + __func__, shares->workinfoid, shares->userid, + st = safe_text_nonull(shares->in_workername), + shares->createdate.tv_sec, + shares->createdate.tv_usec); FREENULL(st); return false; } @@ -3875,18 +3903,31 @@ bool shares_add(PGconn *conn, char *workinfoid, char *username, K_RLOCK(users_free); u_item = find_users(username); K_RUNLOCK(users_free); - /* Can't change outside lock since we don't delete users + /* Won't change outside lock since we don't delete users * or change their *userid */ if (!u_item) { - btv_to_buf(cd, cd_buf, sizeof(cd_buf)); /* This should never happen unless there's a bug in ckpool or the authentication information got to ckdb after - the shares ... which shouldn't ever happen */ - LOGERR("%s() %s/%ld,%ld %s no user! Share discarded!", - __func__, st = safe_text_nonull(username), - cd->tv_sec, cd->tv_usec, cd_buf); - FREENULL(st); - goto tisbad; + the shares or the users table is missing data ... + which shouldn't ever happen + However, since it's a valid share, store it */ + u_item = create_missing_user(conn, username, secondaryuserid, + by, code, inet, cd, trf_root); + btv_to_buf(cd, cd_buf, sizeof(cd_buf)); + if (!u_item) { + LOGERR("%s() ERR %s/%ld,%ld %s no/failed user! Share" + " discarded!", + __func__, st = safe_text_nonull(username), + cd->tv_sec, cd->tv_usec, cd_buf); + FREENULL(st); + goto tisbad; + } else { + DATA_USERS(users, u_item); + LOGERR("%s() MISSING %s/%ld,%ld %s created", + __func__, st = safe_text_nonull(username), + cd->tv_sec, cd->tv_usec, cd_buf); + FREENULL(st); + } } DATA_USERS(users, u_item); shares->userid = users->userid; @@ -7818,8 +7859,8 @@ bool auths_add(PGconn *conn, INTRANSIENT *in_poolinstance, if (!u_item) { if (addressuser) { u_item = users_add(conn, in_username, EMPTY, EMPTY, - USER_ADDRESS, by, code, inet, cd, - trf_root); + NULL, USER_ADDRESS, by, code, inet, + cd, trf_root); } else { LOGDEBUG("%s(): unknown user '%s'", __func__, From 4ab63834abdf68ec979d9afd39c9b4de5a6e4459 Mon Sep 17 00:00:00 2001 From: kanoi Date: Thu, 29 Sep 2016 17:26:03 +1000 Subject: [PATCH 06/33] ckdb - update cmd_dsp --- src/ckdb.h | 2 +- src/ckdb_cmd.c | 72 ++++++++++++++++++++++++++++++++++--------------- src/ckdb_data.c | 6 +++-- src/klist.c | 57 +++++++++++++++++++++++++++++++++++++++ src/klist.h | 2 ++ src/ktree.c | 6 +++-- 6 files changed, 118 insertions(+), 27 deletions(-) diff --git a/src/ckdb.h b/src/ckdb.h index 646a5e68..f8bed5a7 100644 --- a/src/ckdb.h +++ b/src/ckdb.h @@ -58,7 +58,7 @@ #define DB_VLOCK "1" #define DB_VERSION "1.0.7" -#define CKDB_VERSION DB_VERSION"-2.511" +#define CKDB_VERSION DB_VERSION"-2.512" #define WHERE_FFL " - from %s %s() line %d" #define WHERE_FFL_HERE __FILE__, __func__, __LINE__ diff --git a/src/ckdb_cmd.c b/src/ckdb_cmd.c index e4578fa5..82b2c401 100644 --- a/src/ckdb_cmd.c +++ b/src/ckdb_cmd.c @@ -5904,45 +5904,73 @@ static char *cmd_dsp(__maybe_unused PGconn *conn, __maybe_unused char *cmd, __maybe_unused K_TREE *trf_root, __maybe_unused bool reload_data) { - __maybe_unused K_ITEM *i_file; - __maybe_unused char reply[1024] = ""; - __maybe_unused size_t siz = sizeof(reply); - LOGDEBUG("%s(): cmd '%s'", __func__, cmd); - // WARNING: This is a gaping security hole - only use in development +#if 1 LOGDEBUG("%s.disabled.dsp", id); return strdup("disabled.dsp"); -/* +#else + // WARNING: This is a gaping security hole - only use in development + K_ITEM *i_file, *i_name, *i_type; + char reply[1024] = "", *fil, *name, *typ; + size_t siz = sizeof(reply); + i_file = require_name(trf_root, "file", 1, NULL, reply, siz); if (!i_file) return strdup(reply); - dsp_ktree(blocks_free, blocks_root, transfer_data(i_file), NULL); + i_name = require_name(trf_root, "name", 1, NULL, reply, siz); + if (!i_name) + return strdup(reply); + + i_type = optional_name(trf_root, "type", 1, NULL, reply, siz); + if (*reply) + return strdup(reply); + + fil = transfer_data(i_file); + name = transfer_data(i_name); + if (i_type) + typ = transfer_data(i_type); + else + typ = "tree"; + + if (strcasecmp(typ, "tree") == 0) { + if (strcasecmp(name, "blocks") == 0) + dsp_ktree(blocks_root, fil, NULL); - dsp_ktree(transfer_free, trf_root, transfer_data(i_file), NULL); + if (strcasecmp(name, "transfer") == 0) + dsp_ktree(trf_root, fil, NULL); - dsp_ktree(paymentaddresses_free, paymentaddresses_root, - transfer_data(i_file), NULL); + if (strcasecmp(name, "paymentaddresses") == 0) + dsp_ktree(paymentaddresses_root, fil, NULL); - dsp_ktree(paymentaddresses_create_free, paymentaddresses_root, - transfer_data(i_file), NULL); + if (strcasecmp(name, "paymentaddresses_create") == 0) + dsp_ktree(paymentaddresses_root, fil, NULL); - dsp_ktree(sharesummary_free, sharesummary_root, - transfer_data(i_file), NULL); + if (strcasecmp(name, "sharesummary") == 0) + dsp_ktree(sharesummary_root, fil, NULL); - dsp_ktree(userstats_free, userstats_root, - transfer_data(i_file), NULL); + if (strcasecmp(name, "userstats") == 0) + dsp_ktree(userstats_root, fil, NULL); - dsp_ktree(markersummary_free, markersummary_root, - transfer_data(i_file), NULL); + if (strcasecmp(name, "markersummary") == 0) + dsp_ktree(markersummary_root, fil, NULL); - dsp_ktree(workmarkers_free, workmarkers_root, - transfer_data(i_file), NULL); + if (strcasecmp(name, "workmarkers") == 0) + dsp_ktree(workmarkers_root, fil, NULL); + } + + if (strcasecmp(typ, "store") == 0) { + if (strcasecmp(name, "blocks") == 0) + dsp_kstore(blocks_store, fil, NULL); - LOGDEBUG("%s.ok.dsp.file='%s'", id, transfer_data(i_file)); + if (strcasecmp(name, "markersummary") == 0) + dsp_kstore(markersummary_store, fil, NULL); + } + + LOGDEBUG("%s.ok.dsp.file='%s'", id, fil); return strdup("ok.dsp"); -*/ +#endif } static char *cmd_stats(__maybe_unused PGconn *conn, char *cmd, char *id, diff --git a/src/ckdb_data.c b/src/ckdb_data.c index d4bc4461..c24f6a6e 100644 --- a/src/ckdb_data.c +++ b/src/ckdb_data.c @@ -856,8 +856,10 @@ void dsp_transfer(K_ITEM *item, FILE *stream) fprintf(stream, "%s() called with (null) item\n", __func__); else { DATA_TRANSFER(t, item); - fprintf(stream, " name='%s' mvalue='%s' malloc=%"PRIu64"\n", - t->name, t->mvalue, t->msiz); + fprintf(stream, " name='%s' mvalue='%s' malloc=%"PRIu64 + " intransient=%c\n", + t->name, t->mvalue, t->msiz, + t->intransient ? 'Y' : 'N'); } } diff --git a/src/klist.c b/src/klist.c index 1741489e..d1f65ae0 100644 --- a/src/klist.c +++ b/src/klist.c @@ -60,6 +60,63 @@ K_LISTS *all_klists; #define CHKITEM(__item, __list) _CHKITEM(__item, __list, "item") +void _dsp_kstore(K_STORE *store, char *filename, char *msg, KLIST_FFL_ARGS) +{ + K_ITEM *item; + FILE *stream; + struct tm tm; + time_t now_t; + char stamp[128]; + + if (!(store->master->dsp_func)) { + quithere(1, "List %s has no dsp_func" KLIST_FFL, + store->master->name, KLIST_FFL_PASS); + } + + now_t = time(NULL); + localtime_r(&now_t, &tm); + snprintf(stamp, sizeof(stamp), + "[%d-%02d-%02d %02d:%02d:%02d]", + tm.tm_year + 1900, + tm.tm_mon + 1, + tm.tm_mday, + tm.tm_hour, + tm.tm_min, + tm.tm_sec); + + stream = fopen(filename, "ae"); + if (!stream) + { + fprintf(stderr, "%s %s() failed to open '%s' (%d) %s", + stamp, __func__, filename, errno, strerror(errno)); + return; + } + + if (msg) + fprintf(stream, "%s %s\n", stamp, msg); + else + fprintf(stream, "%s Dump of store '%s':\n", stamp, store->master->name); + + if (store->count > 0) + { + K_RLOCK(store->master); + + item = store->head; + while (item) + { + store->master->dsp_func(item, stream); + item = item->next; + } + K_RUNLOCK(store->master); + + fprintf(stream, "End\n\n"); + } + else + fprintf(stream, "Empty kstore\n\n"); + + fclose(stream); +} + static void k_alloc_items(K_LIST *list, KLIST_FFL_ARGS) { K_ITEM *item; diff --git a/src/klist.h b/src/klist.h index d40b199b..eb9e6dc7 100644 --- a/src/klist.h +++ b/src/klist.h @@ -663,6 +663,8 @@ static inline K_ITEM *list_rtail(K_LIST *list) #define STORE_HEAD_NOLOCK(_s) LIST_HEAD_NOLOCK(_s) #define STORE_TAIL_NOLOCK(_s) LIST_TAIL_NOLOCK(_s) +extern void _dsp_kstore(K_STORE *store, char *filename, char *msg, KLIST_FFL_ARGS); +#define dsp_kstore(_store, _file, _msg) _dsp_kstore(_store, _file, _msg, KLIST_FFL_HERE) extern K_STORE *_k_new_store(K_LIST *list, KLIST_FFL_ARGS); #define k_new_store(_list) _k_new_store(_list, KLIST_FFL_HERE) extern K_LIST *_k_new_list(const char *name, size_t siz, int allocate, diff --git a/src/ktree.c b/src/ktree.c index 78b2ef98..5675a5ae 100644 --- a/src/ktree.c +++ b/src/ktree.c @@ -182,8 +182,6 @@ void _dsp_ktree(K_TREE *tree, char *filename, char *msg, KTREE_FFL_ARGS) if (!(tree->master->dsp_func)) FAIL("NULLDSP NULL dsp_func in %s", tree->master->name); - _TREE_READ(tree, true, file, func, line); - now_t = time(NULL); localtime_r(&now_t, &tm); snprintf(stamp, sizeof(stamp), @@ -210,12 +208,16 @@ void _dsp_ktree(K_TREE *tree, char *filename, char *msg, KTREE_FFL_ARGS) if (tree->root->isNil == No) { + K_RLOCK(tree->master); + item = first_in_ktree(tree, ctx); while (item) { tree->master->dsp_func(item, stream); item = next_in_ktree(ctx); } + K_RUNLOCK(tree->master); + fprintf(stream, "End\n\n"); } else From 62c7fce61ed85a59c232233c086c39c0fe134d23 Mon Sep 17 00:00:00 2001 From: kanoi Date: Thu, 29 Sep 2016 21:02:13 +1000 Subject: [PATCH 07/33] ckdb - add msgline to cmd_dsp() code --- src/ckdb.c | 1 + src/ckdb.h | 3 ++- src/ckdb_cmd.c | 3 +++ src/ckdb_data.c | 31 +++++++++++++++++++++++++++++++ 4 files changed, 37 insertions(+), 1 deletion(-) diff --git a/src/ckdb.c b/src/ckdb.c index 76591c6c..78934f9f 100644 --- a/src/ckdb.c +++ b/src/ckdb.c @@ -2023,6 +2023,7 @@ static void alloc_storage() msgline_free = k_new_list("MsgLine", sizeof(MSGLINE), ALLOC_MSGLINE, LIMIT_MSGLINE, true); msgline_store = k_new_store(msgline_free); + msgline_free->dsp_func = dsp_msgline; workqueue_free = k_new_list("WorkQueue", sizeof(WORKQUEUE), ALLOC_WORKQUEUE, LIMIT_WORKQUEUE, true); diff --git a/src/ckdb.h b/src/ckdb.h index f8bed5a7..eeb759f8 100644 --- a/src/ckdb.h +++ b/src/ckdb.h @@ -58,7 +58,7 @@ #define DB_VLOCK "1" #define DB_VERSION "1.0.7" -#define CKDB_VERSION DB_VERSION"-2.512" +#define CKDB_VERSION DB_VERSION"-2.513" #define WHERE_FFL " - from %s %s() line %d" #define WHERE_FFL_HERE __FILE__, __func__, __LINE__ @@ -3333,6 +3333,7 @@ extern INTRANSIENT *_get_intransient(const char *fldnam, char *value, #define intransient_str(_fld, _val) \ _intransient_str(_fld, _val, WHERE_FFL_HERE) extern char *_intransient_str(char *fldnam, char *value, WHERE_FFL_ARGS); +extern void dsp_msgline(K_ITEM *item, FILE *stream); extern char *_transfer_data(K_ITEM *item, WHERE_FFL_ARGS); extern void dsp_transfer(K_ITEM *item, FILE *stream); extern cmp_t cmp_transfer(K_ITEM *a, K_ITEM *b); diff --git a/src/ckdb_cmd.c b/src/ckdb_cmd.c index 82b2c401..ae6c436f 100644 --- a/src/ckdb_cmd.c +++ b/src/ckdb_cmd.c @@ -5966,6 +5966,9 @@ static char *cmd_dsp(__maybe_unused PGconn *conn, __maybe_unused char *cmd, if (strcasecmp(name, "markersummary") == 0) dsp_kstore(markersummary_store, fil, NULL); + + if (strcasecmp(name, "msgline") == 0) + dsp_kstore(msgline_store, fil, NULL); } LOGDEBUG("%s.ok.dsp.file='%s'", id, fil); diff --git a/src/ckdb_data.c b/src/ckdb_data.c index c24f6a6e..4b2d3280 100644 --- a/src/ckdb_data.c +++ b/src/ckdb_data.c @@ -816,6 +816,37 @@ char *_intransient_str(char *fldnam, char *value, WHERE_FFL_ARGS) return in->str; } +void dsp_msgline(K_ITEM *item, FILE *stream) +{ + K_ITEM *t_item; + MSGLINE *m; + int c; + + if (!item) + fprintf(stream, "%s() called with (null) item\n", __func__); + else { + DATA_MSGLINE(m, item); + if (m->trf_store) + c = m->trf_store->count; + else + c = 0; + + fprintf(stream, " which=%d id='%s' cmd='%s' msg='%.42s' " + "trf_store=%c count=%d\n", + m->which_cmds, m->id, m->cmd, m->msg, + m->trf_store ? 'Y' : 'N', c); + + if (m->trf_store) { + t_item = m->trf_store->head; + while (t_item) { + fputc(' ', stream); + dsp_transfer(t_item, stream); + t_item = t_item->next; + } + } + } +} + // For mutiple variable function calls that need the data char *_transfer_data(K_ITEM *item, WHERE_FFL_ARGS) { From ed9a9aae3db7c6512095fee2996083b0199b830e Mon Sep 17 00:00:00 2001 From: kanoi Date: Thu, 29 Sep 2016 22:31:04 +1000 Subject: [PATCH 08/33] ckdb - handle culling in the klist code --- pool/page_ckp.php | 12 +++-- src/ckdb.c | 54 +++++++++-------------- src/ckdb.h | 16 ++++--- src/ckdb_cmd.c | 7 +-- src/ckdb_data.c | 7 +-- src/klist.c | 109 +++++++++++++++++++++++++++++----------------- src/klist.h | 16 +++---- 7 files changed, 120 insertions(+), 101 deletions(-) diff --git a/pool/page_ckp.php b/pool/page_ckp.php index 07fb370c..57c0f34f 100644 --- a/pool/page_ckp.php +++ b/pool/page_ckp.php @@ -16,7 +16,7 @@ function stnum($num) # function dockp($data, $user) { - $pg = '

CKPool

'; + $pg = '

CKDB

'; $msg = msgEncode('stats', 'stats', array(), $user); $rep = sendsockreply('stats', $msg); @@ -33,10 +33,11 @@ function dockp($data, $user) $pg .= "Name:<$r id=srtname data-sf=s0>"; $pg .= 'Initial'; $pg .= 'Allocated'; - $pg .= "<$r id=srtname data-sf=r3>:In Store"; - $pg .= "<$r id=srtname data-sf=r4>:RAM"; - $pg .= "<$r id=srtname data-sf=r5>:RAM2"; + $pg .= "<$r id=srtstore data-sf=r3>:In Store"; + $pg .= "<$r id=srtram data-sf=r4>:RAM"; + $pg .= "<$r id=srtram2 data-sf=r5>:RAM2"; $pg .= 'Cull'; + $pg .= 'Limit'; $pg .= "\n"; if ($ans['STATUS'] == 'ok') { @@ -57,11 +58,14 @@ function dockp($data, $user) $pg .= "".stnum($ans['ram:'.$i]).''; $pg .= "".stnum($ans['ram2:'.$i]).''; $pg .= ''.stnum($ans['cull:'.$i]).''; + $pg .= ''.stnum($ans['cull_limit:'.$i]).''; $pg .= "\n"; } $pg .= ''; } $pg .= "\n"; + $pg .= "\n"; return $pg; } diff --git a/src/ckdb.c b/src/ckdb.c index 78934f9f..778cb140 100644 --- a/src/ckdb.c +++ b/src/ckdb.c @@ -1962,8 +1962,9 @@ static void alloc_storage() ALLOC_LOGQUEUE, LIMIT_LOGQUEUE, true); logqueue_store = k_new_store(logqueue_free); - breakqueue_free = k_new_list("BreakQueue", sizeof(BREAKQUEUE), - ALLOC_BREAKQUEUE, LIMIT_BREAKQUEUE, true); + breakqueue_free = k_new_list_cull("BreakQueue", sizeof(BREAKQUEUE), + ALLOC_BREAKQUEUE, LIMIT_BREAKQUEUE, + true, CULL_BREAKQUEUE); reload_breakqueue_store = k_new_store(breakqueue_free); reload_done_breakqueue_store = k_new_store(breakqueue_free); cmd_breakqueue_store = k_new_store(breakqueue_free); @@ -2017,16 +2018,19 @@ static void alloc_storage() } } - seqtrans_free = k_new_list("SeqTrans", sizeof(SEQTRANS), - ALLOC_SEQTRANS, LIMIT_SEQTRANS, true); + seqtrans_free = k_new_list_cull("SeqTrans", sizeof(SEQTRANS), + ALLOC_SEQTRANS, LIMIT_SEQTRANS, true, + CULL_SEQTRANS); - msgline_free = k_new_list("MsgLine", sizeof(MSGLINE), - ALLOC_MSGLINE, LIMIT_MSGLINE, true); + msgline_free = k_new_list_cull("MsgLine", sizeof(MSGLINE), + ALLOC_MSGLINE, LIMIT_MSGLINE, true, + CULL_MSGLINE); msgline_store = k_new_store(msgline_free); msgline_free->dsp_func = dsp_msgline; - workqueue_free = k_new_list("WorkQueue", sizeof(WORKQUEUE), - ALLOC_WORKQUEUE, LIMIT_WORKQUEUE, true); + workqueue_free = k_new_list_cull("WorkQueue", sizeof(WORKQUEUE), + ALLOC_WORKQUEUE, LIMIT_WORKQUEUE, + true, CULL_WORKQUEUE); pool0_workqueue_store = k_new_store(workqueue_free); pool_workqueue_store = k_new_store(workqueue_free); cmd_workqueue_store = k_new_store(workqueue_free); @@ -2045,8 +2049,9 @@ static void alloc_storage() LIMIT_HEARTBEATQUEUE, true); heartbeatqueue_store = k_new_store(heartbeatqueue_free); - transfer_free = k_new_list(Transfer, sizeof(TRANSFER), - ALLOC_TRANSFER, LIMIT_TRANSFER, true); + transfer_free = k_new_list_cull(Transfer, sizeof(TRANSFER), + ALLOC_TRANSFER, LIMIT_TRANSFER, true, + CULL_TRANSFER); transfer_free->dsp_func = dsp_transfer; users_free = k_new_list("Users", sizeof(USERS), @@ -3097,9 +3102,6 @@ static void trans_seq(tv_t *now) if (store->count) { K_WLOCK(seqtrans_free); k_list_transfer_to_head(store, seqtrans_free); - if (seqtrans_free->count == seqtrans_free->total && - seqtrans_free->total >= ALLOC_SEQTRANS * CULL_SEQTRANS) - k_cull_list(seqtrans_free); K_WUNLOCK(seqtrans_free); } } @@ -3878,9 +3880,6 @@ setitemdata: } K_WLOCK(seqtrans_free); k_list_transfer_to_head(lost, seqtrans_free); - if (seqtrans_free->count == seqtrans_free->total && - seqtrans_free->total >= ALLOC_SEQTRANS * CULL_SEQTRANS) - k_cull_list(seqtrans_free); K_WUNLOCK(seqtrans_free); } @@ -4772,10 +4771,6 @@ static void *breaker(void *arg) pthread_cond_signal(&process_socket_waitcond); mutex_unlock(&process_socket_waitlock); } - - if (breakqueue_free->count == breakqueue_free->total && - breakqueue_free->total >= ALLOC_BREAKQUEUE * CULL_BREAKQUEUE) - k_cull_list(breakqueue_free); K_WUNLOCK(breakqueue_free); } @@ -6083,7 +6078,7 @@ static void process_sockd(PGconn *conn, K_ITEM *wq_item, enum reply_type reply_t K_WUNLOCK(breakqueue_free); FREENULL(ans); - free_msgline_data(ml_item, true, true); + free_msgline_data(ml_item, true); K_WLOCK(msgline_free); msgline_free->ram -= msgline->msgsiz; k_add_head(msgline_free, ml_item); @@ -6091,9 +6086,6 @@ static void process_sockd(PGconn *conn, K_ITEM *wq_item, enum reply_type reply_t K_WLOCK(workqueue_free); k_add_head(workqueue_free, wq_item); - if (workqueue_free->count == workqueue_free->total && - workqueue_free->total >= ALLOC_WORKQUEUE * CULL_WORKQUEUE) - k_cull_list(workqueue_free); K_WUNLOCK(workqueue_free); tick(); @@ -6755,7 +6747,7 @@ static void *process_socket(__maybe_unused void *arg) K_ITEM *ml_item = wq->msgline_item; MSGLINE *ml; DATA_MSGLINE(ml, ml_item); - free_msgline_data(ml_item, true, false); + free_msgline_data(ml_item, true); K_WLOCK(msgline_free); msgline_free->ram -= ml->msgsiz; k_add_head(msgline_free, ml_item); @@ -6798,7 +6790,7 @@ skippy: if (bq->ml_item) { MSGLINE *ml; DATA_MSGLINE(ml, bq->ml_item); - free_msgline_data(bq->ml_item, true, true); + free_msgline_data(bq->ml_item, true); K_WLOCK(msgline_free); msgline_free->ram -= ml->msgsiz; k_add_head(msgline_free, bq->ml_item); @@ -7158,7 +7150,7 @@ static void process_reload_item(PGconn *conn, K_ITEM *bq_item) if (bq->ml_item) { DATA_MSGLINE(msgline, bq->ml_item); - free_msgline_data(bq->ml_item, true, true); + free_msgline_data(bq->ml_item, true); K_WLOCK(msgline_free); msgline_free->ram -= msgline->msgsiz; k_add_head(msgline_free, bq->ml_item); @@ -7797,7 +7789,7 @@ static void process_queued(PGconn *conn, K_ITEM *wq_item) break; } - free_msgline_data(ml_item, true, true); + free_msgline_data(ml_item, true); K_WLOCK(msgline_free); msgline_free->ram -= msgline->msgsiz; k_add_head(msgline_free, ml_item); @@ -7805,9 +7797,6 @@ static void process_queued(PGconn *conn, K_ITEM *wq_item) K_WLOCK(workqueue_free); k_add_head(workqueue_free, wq_item); - if (workqueue_free->count == workqueue_free->total && - workqueue_free->total >= ALLOC_WORKQUEUE * CULL_WORKQUEUE) - k_cull_list(workqueue_free); K_WUNLOCK(workqueue_free); } @@ -7816,9 +7805,6 @@ static void free_lost(SEQDATA *seqdata) if (seqdata->reload_lost) { K_WLOCK(seqtrans_free); k_list_transfer_to_head(seqdata->reload_lost, seqtrans_free); - if (seqtrans_free->count == seqtrans_free->total && - seqtrans_free->total >= ALLOC_SEQTRANS * CULL_SEQTRANS) - k_cull_list(seqtrans_free); K_WUNLOCK(seqtrans_free); seqdata->reload_lost = NULL; } diff --git a/src/ckdb.h b/src/ckdb.h index eeb759f8..c22a4334 100644 --- a/src/ckdb.h +++ b/src/ckdb.h @@ -58,7 +58,7 @@ #define DB_VLOCK "1" #define DB_VERSION "1.0.7" -#define CKDB_VERSION DB_VERSION"-2.513" +#define CKDB_VERSION DB_VERSION"-2.514" #define WHERE_FFL " - from %s %s() line %d" #define WHERE_FFL_HERE __FILE__, __func__, __LINE__ @@ -1370,7 +1370,7 @@ typedef struct msgline { #define ALLOC_MSGLINE 8192 #define LIMIT_MSGLINE 0 -#define CULL_MSGLINE 8 +#define CULL_MSGLINE (8 * ALLOC_MSGLINE) #define INIT_MSGLINE(_item) INIT_GENERIC(_item, msgline) #define DATA_MSGLINE(_var, _item) DATA_GENERIC(_var, _item, msgline, true) #define DATA_MSGLINE_NULL(_var, _item) DATA_GENERIC(_var, _item, msgline, false) @@ -1396,7 +1396,7 @@ typedef struct breakqueue { #define ALLOC_BREAKQUEUE 16384 #define LIMIT_BREAKQUEUE 0 -#define CULL_BREAKQUEUE 4 +#define CULL_BREAKQUEUE (4 * ALLOC_BREAKQUEUE) #define INIT_BREAKQUEUE(_item) INIT_GENERIC(_item, breakqueue) #define DATA_BREAKQUEUE(_var, _item) DATA_GENERIC(_var, _item, breakqueue, true) @@ -1463,7 +1463,7 @@ typedef struct workqueue { #define ALLOC_WORKQUEUE 1024 #define LIMIT_WORKQUEUE 0 -#define CULL_WORKQUEUE 32 +#define CULL_WORKQUEUE (32 * ALLOC_WORKQUEUE) #define INIT_WORKQUEUE(_item) INIT_GENERIC(_item, workqueue) #define DATA_WORKQUEUE(_var, _item) DATA_GENERIC(_var, _item, workqueue, true) @@ -1563,7 +1563,9 @@ typedef struct transfer { // Suggest malloc use MMAP = largest under 2MB #define ALLOC_TRANSFER ((int)(2*1024*1024/sizeof(TRANSFER))) #define LIMIT_TRANSFER 0 -#define CULL_TRANSFER 16 +/* ALLOC_TRANSFER is ~14k, but it should only ever get this big during a reload + * so set it a bit above that */ +#define CULL_TRANSFER 32768 #define INIT_TRANSFER(_item) INIT_GENERIC(_item, transfer) #define DATA_TRANSFER(_var, _item) DATA_GENERIC(_var, _item, transfer, true) @@ -1787,7 +1789,7 @@ extern K_LIST *seqtrans_free; #define ALLOC_SEQTRANS 1024 #define LIMIT_SEQTRANS 0 -#define CULL_SEQTRANS 64 +#define CULL_SEQTRANS (16 * ALLOC_SEQTRANS) #define INIT_SEQTRANS(_item) INIT_GENERIC(_item, seqtrans) #define DATA_SEQTRANS(_var, _item) DATA_GENERIC(_var, _item, seqtrans, true) #define DATA_SEQTRANS_NULL(_var, _item) DATA_GENERIC(_var, _item, seqtrans, false) @@ -3227,7 +3229,7 @@ extern void sequence_report(bool lock); #define FREE_ITEM(item) do { } while(0) // TODO: make a macro for all other to use above macro extern void free_transfer_data(TRANSFER *transfer); -extern void free_msgline_data(K_ITEM *item, bool t_lock, bool t_cull); +extern void free_msgline_data(K_ITEM *item, bool t_lock); extern void free_users_data(K_ITEM *item); extern void free_workinfo_data(K_ITEM *item); #define free_sharesummary_data(_i) FREE_ITEM(_i) diff --git a/src/ckdb_cmd.c b/src/ckdb_cmd.c index ae6c436f..e536802d 100644 --- a/src/ckdb_cmd.c +++ b/src/ckdb_cmd.c @@ -6032,7 +6032,7 @@ static char *cmd_stats(__maybe_unused PGconn *conn, char *cmd, char *id, snprintf(tmp, sizeof(tmp), "name:%d=%s%s%s%cinitial:%d=%d%callocated:%d=%d%c" "instore:%d=%d%cram:%d=%"PRIu64"%c" - "ram2:%d=%"PRIu64"%ccull:%d=%d%c", + "ram2:%d=%"PRIu64"%ccull:%d=%d%ccull_limit:%d=%d%c", rows, name, istree ? " (tree)" : "", klist->is_lock_only ? " (lock)" : "", FLDSEP, rows, klist->allocate, FLDSEP, @@ -6040,7 +6040,8 @@ static char *cmd_stats(__maybe_unused PGconn *conn, char *cmd, char *id, rows, klist->total - klist->count, FLDSEP, rows, ram, FLDSEP, rows, ram2, FLDSEP, - rows, klist->cull_count, FLDSEP); + rows, klist->cull_count, FLDSEP, + rows, klist->cull_limit, FLDSEP); APPEND_REALLOC(buf, off, len, tmp); tot += ram + ram2; @@ -6056,7 +6057,7 @@ static char *cmd_stats(__maybe_unused PGconn *conn, char *cmd, char *id, snprintf(tmp, sizeof(tmp), "rows=%d%cflds=%s%c", rows, FLDSEP, - "name,initial,allocated,instore,ram,cull", FLDSEP); + "name,initial,allocated,instore,ram,cull,cull_limit", FLDSEP); APPEND_REALLOC(buf, off, len, tmp); snprintf(tmp, sizeof(tmp), "arn=%s%carp=%s", "Stats", FLDSEP, ""); diff --git a/src/ckdb_data.c b/src/ckdb_data.c index 4b2d3280..905cf5bb 100644 --- a/src/ckdb_data.c +++ b/src/ckdb_data.c @@ -18,7 +18,7 @@ void free_transfer_data(TRANSFER *transfer) FREENULL(transfer->mvalue); } -void free_msgline_data(K_ITEM *item, bool t_lock, bool t_cull) +void free_msgline_data(K_ITEM *item, bool t_lock) { K_ITEM *t_item = NULL; TRANSFER *transfer; @@ -40,11 +40,6 @@ void free_msgline_data(K_ITEM *item, bool t_lock, bool t_cull) K_WLOCK(transfer_free); transfer_free->ram -= ram2; k_list_transfer_to_head(msgline->trf_store, transfer_free); - if (t_cull) { - if (transfer_free->count == transfer_free->total && - transfer_free->total >= ALLOC_TRANSFER * CULL_TRANSFER) - k_cull_list(transfer_free); - } if (t_lock) K_WUNLOCK(transfer_free); msgline->trf_store = k_free_store(msgline->trf_store); diff --git a/src/klist.c b/src/klist.c index d1f65ae0..a33a7e9c 100644 --- a/src/klist.c +++ b/src/klist.c @@ -213,7 +213,8 @@ K_STORE *_k_new_store(K_LIST *list, KLIST_FFL_ARGS) K_LIST *_k_new_list(const char *name, size_t siz, int allocate, int limit, bool do_tail, bool lock_only, bool without_lock, - bool local_list, const char *name2, KLIST_FFL_ARGS) + bool local_list, const char *name2, int cull_limit, + KLIST_FFL_ARGS) { K_LIST *list; @@ -223,6 +224,11 @@ K_LIST *_k_new_list(const char *name, size_t siz, int allocate, int limit, if (limit < 0) quithere(1, "Invalid new list %s with limit %d must be >= 0", name, limit); + /* after culling, the first block of items are again allocated, + * so there's no point culling a single block of items */ + if (cull_limit > 0 && cull_limit <= allocate) + quithere(1, "Invalid new list %s with cull_limit %d must be > allocate (%d)", name, cull_limit, allocate); + list = calloc(1, sizeof(*list)); if (!list) quithere(1, "Failed to calloc list %s", name); @@ -248,6 +254,7 @@ K_LIST *_k_new_list(const char *name, size_t siz, int allocate, int limit, list->allocate = allocate; list->limit = limit; list->do_tail = do_tail; + list->cull_limit = cull_limit; if (!(list->is_lock_only)) k_alloc_items(list, KLIST_FFL_PASS); @@ -360,6 +367,58 @@ K_ITEM *_k_unlink_tail(K_LIST *list, LOCK_MAYBE bool chklock, KLIST_FFL_ARGS) return item; } +#define CHKCULL(_list) \ + do { \ + if (!((_list)->is_store) && !((_list)->is_lock_only) && \ + (_list)->cull_limit > 0 && \ + (_list)->count == (_list)->total && \ + (_list)->total >= (_list)->cull_limit) { \ + k_cull_list(_list, file, func, line); \ + } \ + } while(0); + +static void k_cull_list(K_LIST *list, KLIST_FFL_ARGS) +{ + int i; + + CHKLIST(list); + _LIST_WRITE(list, true, file, func, line); + + if (list->is_store) { + quithere(1, "List %s can't %s() a store" KLIST_FFL, + list->name, __func__, KLIST_FFL_PASS); + } + + if (list->is_lock_only) { + quithere(1, "List %s can't %s() a lock_only" KLIST_FFL, + list->name, __func__, KLIST_FFL_PASS); + } + + if (list->count != list->total) { + quithere(1, "List %s can't %s() a list in use" KLIST_FFL, + list->name, __func__, KLIST_FFL_PASS); + } + + for (i = 0; i < list->item_mem_count; i++) + free(list->item_memory[i]); + free(list->item_memory); + list->item_memory = NULL; + list->item_mem_count = 0; + + for (i = 0; i < list->data_mem_count; i++) + free(list->data_memory[i]); + free(list->data_memory); + list->data_memory = NULL; + list->data_mem_count = 0; + + list->total = list->count = list->count_up = 0; + list->head = list->tail = NULL; + + list->cull_count++; + + k_alloc_items(list, KLIST_FFL_PASS); +} + void _k_add_head(K_LIST *list, K_ITEM *item, LOCK_MAYBE bool chklock, KLIST_FFL_ARGS) { CHKLS(list); @@ -390,6 +449,8 @@ void _k_add_head(K_LIST *list, K_ITEM *item, LOCK_MAYBE bool chklock, KLIST_FFL_ list->count++; list->count_up++; + + CHKCULL(list); } /* slows it down (of course) - only for debugging @@ -437,6 +498,8 @@ void _k_add_tail(K_LIST *list, K_ITEM *item, LOCK_MAYBE bool chklock, KLIST_FFL_ list->count++; list->count_up++; + + CHKCULL(list); } // Insert item into the list next after 'after' @@ -475,6 +538,8 @@ void _k_insert_after(K_LIST *list, K_ITEM *item, K_ITEM *after, LOCK_MAYBE bool list->count++; list->count_up++; + + // no point checking cull since this wouldn't be an _free list } void _k_unlink_item(K_LIST *list, K_ITEM *item, LOCK_MAYBE bool chklock, KLIST_FFL_ARGS) @@ -541,6 +606,8 @@ void _k_list_transfer_to_head(K_LIST *from, K_LIST *to, LOCK_MAYBE bool chklock, from->count = 0; to->count_up += from->count_up; from->count_up = 0; + + CHKCULL(to); } void _k_list_transfer_to_tail(K_LIST *from, K_LIST *to, LOCK_MAYBE bool chklock, KLIST_FFL_ARGS) @@ -577,6 +644,8 @@ void _k_list_transfer_to_tail(K_LIST *from, K_LIST *to, LOCK_MAYBE bool chklock, from->count = 0; to->count_up += from->count_up; from->count_up = 0; + + CHKCULL(to); } K_LIST *_k_free_list(K_LIST *list, KLIST_FFL_ARGS) @@ -655,41 +724,3 @@ K_STORE *_k_free_store(K_STORE *store, KLIST_FFL_ARGS) return NULL; } - -// Must be locked and none in use and/or unlinked -void _k_cull_list(K_LIST *list, LOCK_MAYBE bool chklock, KLIST_FFL_ARGS) -{ - int i; - - CHKLIST(list); - _LIST_WRITE(list, chklock, file, func, line); - - if (list->is_store) { - quithere(1, "List %s can't %s() a store" KLIST_FFL, - list->name, __func__, KLIST_FFL_PASS); - } - - if (list->count != list->total) { - quithere(1, "List %s can't %s() a list in use" KLIST_FFL, - list->name, __func__, KLIST_FFL_PASS); - } - - for (i = 0; i < list->item_mem_count; i++) - free(list->item_memory[i]); - free(list->item_memory); - list->item_memory = NULL; - list->item_mem_count = 0; - - for (i = 0; i < list->data_mem_count; i++) - free(list->data_memory[i]); - free(list->data_memory); - list->data_memory = NULL; - list->data_mem_count = 0; - - list->total = list->count = list->count_up = 0; - list->head = list->tail = NULL; - - list->cull_count++; - - k_alloc_items(list, KLIST_FFL_PASS); -} diff --git a/src/klist.h b/src/klist.h index eb9e6dc7..ee8a021f 100644 --- a/src/klist.h +++ b/src/klist.h @@ -151,7 +151,8 @@ typedef struct k_list { int data_mem_count; // how many item data memory buffers have been allocated void **data_memory; // allocated item data memory buffers void (*dsp_func)(K_ITEM *, FILE *); // optional data display to a file - int cull_count; + int cull_limit; // <1 means don't cull, otherwise total to cull at + int cull_count; // number of times culled uint64_t ram; // ram allocated for data pointers - code must manage it int stores; // how many stores it currently has #if LOCK_CHECK @@ -670,13 +671,15 @@ extern K_STORE *_k_new_store(K_LIST *list, KLIST_FFL_ARGS); extern K_LIST *_k_new_list(const char *name, size_t siz, int allocate, int limit, bool do_tail, bool lock_only, bool without_lock, bool local_list, - const char *name2, KLIST_FFL_ARGS); + const char *name2, int cull_limit, KLIST_FFL_ARGS); #define k_new_list(_name, _siz, _allocate, _limit, _do_tail) \ - _k_new_list(_name, _siz, _allocate, _limit, _do_tail, false, false, false, NULL, KLIST_FFL_HERE) + _k_new_list(_name, _siz, _allocate, _limit, _do_tail, false, false, false, NULL, 0, KLIST_FFL_HERE) #define k_lock_only_list(_name) \ - _k_new_list(_name, 1, 1, 1, true, true, false, false, NULL, KLIST_FFL_HERE) + _k_new_list(_name, 1, 1, 1, true, true, false, false, NULL, 0, KLIST_FFL_HERE) #define k_new_tree_list(_name, _siz, _allocate, _limit, _do_tail, _local_tree, _name2) \ - _k_new_list(_name, _siz, _allocate, _limit, _do_tail, false, true, _local_tree, _name2, KLIST_FFL_HERE) + _k_new_list(_name, _siz, _allocate, _limit, _do_tail, false, true, _local_tree, _name2, 0, KLIST_FFL_HERE) +#define k_new_list_cull(_name, _siz, _allocate, _limit, _do_tail, _cull) \ + _k_new_list(_name, _siz, _allocate, _limit, _do_tail, false, false, false, NULL, _cull, KLIST_FFL_HERE) extern K_ITEM *_k_unlink_head(K_LIST *list, LOCK_MAYBE bool chklock, KLIST_FFL_ARGS); #define k_unlink_head(_list) _k_unlink_head(_list, true, KLIST_FFL_HERE) #define k_unlink_head_nolock(_list) _k_unlink_head(_list, false, KLIST_FFL_HERE) @@ -711,8 +714,5 @@ extern K_LIST *_k_free_list(K_LIST *list, KLIST_FFL_ARGS); #define k_free_list(_list) _k_free_list(_list, KLIST_FFL_HERE) extern K_STORE *_k_free_store(K_STORE *store, KLIST_FFL_ARGS); #define k_free_store(_store) _k_free_store(_store, KLIST_FFL_HERE) -extern void _k_cull_list(K_LIST *list, LOCK_MAYBE bool chklock, KLIST_FFL_ARGS); -#define k_cull_list(_list) _k_cull_list(_list, true, KLIST_FFL_HERE) -//#define k_cull_list_nolock(_list) _k_cull_list(_list, false, KLIST_FFL_HERE) #endif From 9fd830bbaeb2ae20554e6c69a6f0bf28c5331656 Mon Sep 17 00:00:00 2001 From: kanoi Date: Fri, 30 Sep 2016 06:00:39 +1000 Subject: [PATCH 09/33] ckdb - ensure unused payments fields aren't null --- src/ckdb.h | 2 +- src/ckdb_data.c | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/src/ckdb.h b/src/ckdb.h index c22a4334..fcfa21d5 100644 --- a/src/ckdb.h +++ b/src/ckdb.h @@ -58,7 +58,7 @@ #define DB_VLOCK "1" #define DB_VERSION "1.0.7" -#define CKDB_VERSION DB_VERSION"-2.514" +#define CKDB_VERSION DB_VERSION"-2.515" #define WHERE_FFL " - from %s %s() line %d" #define WHERE_FFL_HERE __FILE__, __func__, __LINE__ diff --git a/src/ckdb_data.c b/src/ckdb_data.c index 905cf5bb..38bd969b 100644 --- a/src/ckdb_data.c +++ b/src/ckdb_data.c @@ -5012,6 +5012,9 @@ bool process_pplns(int32_t height, char *blockhash, tv_t *addr_cd) (double)(pa->payratio) / (double)paytotal; used += d64; + payments->in_originaltxn = + payments->in_committxn = + payments->in_commitblockhash = EMPTY; k_add_tail_nolock(pay_store, pay_item); ok = payments_add(conn, true, pay_item, &(payments->old_item), @@ -5041,6 +5044,9 @@ bool process_pplns(int32_t height, char *blockhash, tv_t *addr_cd) payments->amount = amount; payments->diffacc = miningpayouts->diffacc; used = amount; + payments->in_originaltxn = + payments->in_committxn = + payments->in_commitblockhash = EMPTY; k_add_tail_nolock(pay_store, pay_item); ok = payments_add(conn, true, pay_item, &(payments->old_item), From 6b59f1442af9bf3160bee6aab79b9d407f704a6f Mon Sep 17 00:00:00 2001 From: kanoi Date: Fri, 30 Sep 2016 06:20:08 +1000 Subject: [PATCH 10/33] ckdb - log errors for calls to _data_to_buf with null data --- src/ckdb.h | 2 +- src/ckdb_data.c | 18 ++++++++++++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/src/ckdb.h b/src/ckdb.h index fcfa21d5..ff6ca9d9 100644 --- a/src/ckdb.h +++ b/src/ckdb.h @@ -58,7 +58,7 @@ #define DB_VLOCK "1" #define DB_VERSION "1.0.7" -#define CKDB_VERSION DB_VERSION"-2.515" +#define CKDB_VERSION DB_VERSION"-2.516" #define WHERE_FFL " - from %s %s() line %d" #define WHERE_FFL_HERE __FILE__, __func__, __LINE__ diff --git a/src/ckdb_data.c b/src/ckdb_data.c index 38bd969b..74d30032 100644 --- a/src/ckdb_data.c +++ b/src/ckdb_data.c @@ -505,9 +505,27 @@ void _txt_to_double(char *nam, char *fld, double *data, size_t siz, WHERE_FFL_AR char *_data_to_buf(enum data_type typ, void *data, char *buf, size_t siz, WHERE_FFL_ARGS) { + static bool had_null = false; struct tm tm; double d; + // Return an empty string but only log a console message the first time + if (!data) { + // locking doesn't matter - if we get extra messages + if (!had_null) { + had_null = true; + LOGEMERG("%s() BUG - called with null data - check" + " log file" WHERE_FFL, + __func__, WHERE_FFL_PASS); + } + LOGNOTICE("%s() BUG - called with null data typ=%d" WHERE_FFL, + __func__, (int)typ, WHERE_FFL_PASS); + if (!buf) + buf = malloc(1); + *buf = '\0'; + return buf; + } + if (!buf) { switch (typ) { case TYPE_STR: From d15e332b2af7b8124d3f098d7ece8aaa67031486 Mon Sep 17 00:00:00 2001 From: kanoi Date: Fri, 30 Sep 2016 09:34:06 +1000 Subject: [PATCH 11/33] ckdb - use a signed number for ram to handle numerical errors --- src/ckdb.h | 2 +- src/ckdb_cmd.c | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/ckdb.h b/src/ckdb.h index ff6ca9d9..a2f0fd2d 100644 --- a/src/ckdb.h +++ b/src/ckdb.h @@ -58,7 +58,7 @@ #define DB_VLOCK "1" #define DB_VERSION "1.0.7" -#define CKDB_VERSION DB_VERSION"-2.516" +#define CKDB_VERSION DB_VERSION"-2.517" #define WHERE_FFL " - from %s %s() line %d" #define WHERE_FFL_HERE __FILE__, __func__, __LINE__ diff --git a/src/ckdb_cmd.c b/src/ckdb_cmd.c index e536802d..d0bdc3fa 100644 --- a/src/ckdb_cmd.c +++ b/src/ckdb_cmd.c @@ -5986,7 +5986,7 @@ static char *cmd_stats(__maybe_unused PGconn *conn, char *cmd, char *id, char tmp[1024], *buf; const char *name; size_t len, off; - uint64_t ram, ram2, tot = 0; + int64_t ram, ram2, tot = 0; K_LIST *klist; K_LISTS *klists; int rows = 0; @@ -6031,8 +6031,8 @@ static char *cmd_stats(__maybe_unused PGconn *conn, char *cmd, char *id, snprintf(tmp, sizeof(tmp), "name:%d=%s%s%s%cinitial:%d=%d%callocated:%d=%d%c" - "instore:%d=%d%cram:%d=%"PRIu64"%c" - "ram2:%d=%"PRIu64"%ccull:%d=%d%ccull_limit:%d=%d%c", + "instore:%d=%d%cram:%d=%"PRId64"%c" + "ram2:%d=%"PRId64"%ccull:%d=%d%ccull_limit:%d=%d%c", rows, name, istree ? " (tree)" : "", klist->is_lock_only ? " (lock)" : "", FLDSEP, rows, klist->allocate, FLDSEP, @@ -6051,7 +6051,7 @@ static char *cmd_stats(__maybe_unused PGconn *conn, char *cmd, char *id, } ck_wunlock(&lock_check_lock); - snprintf(tmp, sizeof(tmp), "totalram=%"PRIu64"%c", tot, FLDSEP); + snprintf(tmp, sizeof(tmp), "totalram=%"PRId64"%c", tot, FLDSEP); APPEND_REALLOC(buf, off, len, tmp); snprintf(tmp, sizeof(tmp), From 8496314df3e77a9bd03b2839b6ad4b06093cf034 Mon Sep 17 00:00:00 2001 From: kanoi Date: Sat, 1 Oct 2016 10:40:28 +1000 Subject: [PATCH 12/33] ckdb - log duplicate DB store of shares as NOTICE --- src/ckdb.h | 3 ++- src/ckdb_dbio.c | 4 +++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/src/ckdb.h b/src/ckdb.h index a2f0fd2d..10d48a5e 100644 --- a/src/ckdb.h +++ b/src/ckdb.h @@ -58,7 +58,7 @@ #define DB_VLOCK "1" #define DB_VERSION "1.0.7" -#define CKDB_VERSION DB_VERSION"-2.517" +#define CKDB_VERSION DB_VERSION"-2.518" #define WHERE_FFL " - from %s %s() line %d" #define WHERE_FFL_HERE __FILE__, __func__, __LINE__ @@ -3641,6 +3641,7 @@ extern PGresult *_CKPQexecParams(PGconn *conn, const char *qry, #define PGLOGERR(_str, _rescode, _conn) PGLOG(LOGERR, _str, _rescode, _conn) #define PGLOGEMERG(_str, _rescode, _conn) PGLOG(LOGEMERG, _str, _rescode, _conn) +#define PGLOGNOTICE(_str, _rescode, _conn) PGLOG(LOGNOTICE, _str, _rescode, _conn) extern char *pqerrmsg(PGconn *conn); extern bool CKPQConn(PGconn **conn); diff --git a/src/ckdb_dbio.c b/src/ckdb_dbio.c index e24a9943..ad6f9caa 100644 --- a/src/ckdb_dbio.c +++ b/src/ckdb_dbio.c @@ -4115,11 +4115,13 @@ bool shares_db(PGconn *conn, K_ITEM *s_item) res = PQexecParams(conn, ins, par, NULL, (const char **)params, NULL, NULL, 0, CKPQ_WRITE); rescode = PQresultStatus(res); if (!PGOK(rescode)) { - // If the share is already in the db ignore the error char *code = PQresultErrorField(res, PG_DIAG_SQLSTATE); if (!code || strcmp(code, SQL_UNIQUE_VIOLATION)) { PGLOGERR("Insert", rescode, conn); goto unparam; + } else { + // If the share is already in the db use NOTICE + PGLOGNOTICE("Insert", rescode, conn); } } From ac6aff79188fc44ed5db12e0dcdfcf7d56fa83df Mon Sep 17 00:00:00 2001 From: kanoi Date: Thu, 6 Oct 2016 17:48:57 +1100 Subject: [PATCH 13/33] ckdb - wrap most PQ functions and simplify --- src/ckdb.c | 68 +-- src/ckdb.h | 28 +- src/ckdb_cmd.c | 91 +--- src/ckdb_data.c | 6 +- src/ckdb_dbio.c | 1051 ++++++++++++++++++----------------------------- 5 files changed, 473 insertions(+), 771 deletions(-) diff --git a/src/ckdb.c b/src/ckdb.c index 778cb140..4532993e 100644 --- a/src/ckdb.c +++ b/src/ckdb.c @@ -1716,9 +1716,11 @@ PGconn *dbconnect() */ static bool getdata1() { - PGconn *conn = dbconnect(); + PGconn *conn = NULL; bool ok = true; + CKPQConn(&conn); + if (!(ok = check_db_version(conn))) goto matane; if (!(ok = optioncontrol_fill(conn))) @@ -1731,7 +1733,7 @@ static bool getdata1() matane: - PQfinish(conn); + CKPQFinish(&conn); return ok; } @@ -1740,19 +1742,25 @@ matane: */ static bool getdata2() { - PGconn *conn = dbconnect(); - bool ok = blocks_fill(conn); + PGconn *conn = NULL; + bool ok; + + CKPQConn(&conn); + + ok = blocks_fill(conn); - PQfinish(conn); + CKPQFinish(&conn); return ok; } static bool getdata3() { - PGconn *conn = dbconnect(); + PGconn *conn = NULL; bool ok = true; + CKPQConn(&conn); + if (!key_update && !confirm_sharesummary) { if (!(ok = paymentaddresses_fill(conn)) || everyone_die) goto sukamudai; @@ -1762,12 +1770,12 @@ static bool getdata3() if (!(ok = miningpayouts_fill(conn)) || everyone_die) goto sukamudai; } - PQfinish(conn); - conn = dbconnect(); + CKPQFinish(&conn); + CKPQConn(&conn); if (!(ok = workinfo_fill(conn)) || everyone_die) goto sukamudai; - PQfinish(conn); - conn = dbconnect(); + CKPQFinish(&conn); + CKPQConn(&conn); if (!(ok = marks_fill(conn)) || everyone_die) goto sukamudai; /* must be after workinfo */ @@ -1778,14 +1786,14 @@ static bool getdata3() if (!(ok = payouts_fill(conn)) || everyone_die) goto sukamudai; } - PQfinish(conn); - conn = dbconnect(); + CKPQFinish(&conn); + CKPQConn(&conn); if (!key_update) { if (!(ok = markersummary_fill(conn)) || everyone_die) goto sukamudai; } - PQfinish(conn); - conn = dbconnect(); + CKPQFinish(&conn); + CKPQConn(&conn); if (!key_update) { if (!(ok = shares_fill(conn)) || everyone_die) goto sukamudai; @@ -1795,7 +1803,7 @@ static bool getdata3() sukamudai: - PQfinish(conn); + CKPQFinish(&conn); return ok; } @@ -6153,7 +6161,7 @@ static void *listener_all(void *arg) else clistener_using_data = true; - conn = dbconnect(); + CKPQConn(&conn); now = time(NULL); while (!everyone_die) { @@ -6250,8 +6258,8 @@ static void *listener_all(void *arg) // Don't keep a connection for more than ~10s if ((time(NULL) - now) > 10) { - PQfinish(conn); - conn = dbconnect(); + CKPQFinish(&conn); + CKPQConn(&conn); now = time(NULL); } @@ -6294,9 +6302,7 @@ static void *listener_all(void *arg) } } } - - if (conn) - PQfinish(conn); + CKPQFinish(&conn); if (mythread != 0) LOGNOTICE("%s() %s exiting", __func__, buf); @@ -7202,7 +7208,7 @@ static void *process_reload(__maybe_unused void *arg) when_add.tv_sec = RELOAD_QUEUE_SLEEP_MS / 1000; when_add.tv_nsec = (RELOAD_QUEUE_SLEEP_MS % 1000) * 1000000; - conn = dbconnect(); + CKPQConn(&conn); now = time(NULL); while (!everyone_die) { @@ -7306,8 +7312,8 @@ static void *process_reload(__maybe_unused void *arg) // Don't keep a connection for more than ~10s ... of processing if ((time(NULL) - now) > 10) { - PQfinish(conn); - conn = dbconnect(); + CKPQFinish(&conn); + CKPQConn(&conn); now = time(NULL); } @@ -7322,9 +7328,7 @@ static void *process_reload(__maybe_unused void *arg) tick(); } - - if (conn) - PQfinish(conn); + CKPQFinish(&conn); if (mythread == 0) { for (i = 1; i < THREAD_LIMIT; i++) { @@ -7859,7 +7863,7 @@ static void *pqproc(void *arg) when_add.tv_nsec = (CMD_QUEUE_SLEEP_MS % 1000) * 1000000; now = time(NULL); - conn = dbconnect(); + CKPQConn(&conn); wqgot = 0; // Override checking until pool0 is complete @@ -7972,8 +7976,8 @@ static void *pqproc(void *arg) /* Don't keep a connection for more than ~10s or ~10000 items * but always have a connection open */ if ((time(NULL) - now) > 10 || wqgot > 10000) { - PQfinish(conn); - conn = dbconnect(); + CKPQFinish(&conn); + CKPQConn(&conn); now = time(NULL); wqgot = 0; } @@ -8043,9 +8047,7 @@ static void *pqproc(void *arg) mutex_unlock(&wq_pool_waitlock); } } - - if (conn) - PQfinish(conn); + CKPQFinish(&conn); if (mythread == 0) { for (i = 1; i < THREAD_LIMIT; i++) { diff --git a/src/ckdb.h b/src/ckdb.h index 10d48a5e..05a77bca 100644 --- a/src/ckdb.h +++ b/src/ckdb.h @@ -58,7 +58,7 @@ #define DB_VLOCK "1" #define DB_VERSION "1.0.7" -#define CKDB_VERSION DB_VERSION"-2.518" +#define CKDB_VERSION DB_VERSION"-2.600" #define WHERE_FFL " - from %s %s() line %d" #define WHERE_FFL_HERE __FILE__, __func__, __LINE__ @@ -3614,12 +3614,12 @@ extern void userinfo_block(BLOCKS *blocks, enum info_type isnew, int delta); #define CKPQ_READ true #define CKPQ_WRITE false -#define CKPQexec(_conn, _qry, _isread) _CKPQexec(_conn, _qry, _isread, WHERE_FFL_HERE) -extern PGresult *_CKPQexec(PGconn *conn, const char *qry, bool isread, WHERE_FFL_ARGS); -#define CKPQexecParams(_conn, _qry, _p1, _p2, _p3, _p4, _p5, _p6, _isread) \ - _CKPQexecParams(_conn, _qry, _p1, _p2, _p3, _p4, _p5, _p6, \ +#define CKPQExec(_conn, _qry, _isread) _CKPQExec(_conn, _qry, _isread, WHERE_FFL_HERE) +extern PGresult *_CKPQExec(PGconn *conn, const char *qry, bool isread, WHERE_FFL_ARGS); +#define CKPQExecParams(_conn, _qry, _p1, _p2, _p3, _p4, _p5, _p6, _isread) \ + _CKPQExecParams(_conn, _qry, _p1, _p2, _p3, _p4, _p5, _p6, \ _isread, WHERE_FFL_HERE) -extern PGresult *_CKPQexecParams(PGconn *conn, const char *qry, +extern PGresult *_CKPQExecParams(PGconn *conn, const char *qry, int nParams, const Oid *paramTypes, const char *const * paramValues, @@ -3627,10 +3627,10 @@ extern PGresult *_CKPQexecParams(PGconn *conn, const char *qry, const int *paramFormats, int resultFormat, bool isread, WHERE_FFL_ARGS); - -// Force use CKPQ... for PQ functions in use -#define PQexec CKPQexec -#define PQexecParams CKPQexecParams +extern ExecStatusType _CKPQResultStatus(PGresult *res, WHERE_FFL_ARGS); +#define CKPQResultStatus(_res) _CKPQResultStatus(_res, WHERE_FFL_HERE) +extern void _CKPQClear(PGresult *res, WHERE_FFL_ARGS); +#define CKPQClear(_res) _CKPQClear(_res, WHERE_FFL_HERE) #define PGLOG(__LOG, __str, __rescode, __conn) do { \ char *__buf = pqerrmsg(__conn); \ @@ -3644,12 +3644,16 @@ extern PGresult *_CKPQexecParams(PGconn *conn, const char *qry, #define PGLOGNOTICE(_str, _rescode, _conn) PGLOG(LOGNOTICE, _str, _rescode, _conn) extern char *pqerrmsg(PGconn *conn); -extern bool CKPQConn(PGconn **conn); -extern void CKPQDisco(PGconn **conn, bool conned); +extern bool _CKPQConn(PGconn **conn, WHERE_FFL_ARGS); +#define CKPQConn(_conn) _CKPQConn(_conn, WHERE_FFL_HERE) +extern bool _CKPQDisco(PGconn **conn, bool conned, WHERE_FFL_ARGS); +#define CKPQDisco(_conn, _conned) _CKPQDisco(_conn, _conned, WHERE_FFL_HERE) +#define CKPQFinish(_conn) CKPQDisco(_conn, true) extern bool _CKPQBegin(PGconn *conn, WHERE_FFL_ARGS); #define CKPQBegin(_conn) _CKPQBegin(conn, WHERE_FFL_HERE) extern void _CKPQEnd(PGconn *conn, bool commit, WHERE_FFL_ARGS); #define CKPQEnd(_conn, _commit) _CKPQEnd(_conn, _commit, WHERE_FFL_HERE) +#define CKPQCommit(_conn) _CKPQEnd(_conn, true, WHERE_FFL_HERE) extern int64_t nextid(PGconn *conn, char *idname, int64_t increment, tv_t *cd, char *by, char *code, char *inet); diff --git a/src/ckdb_cmd.c b/src/ckdb_cmd.c index d0bdc3fa..2dbd3eab 100644 --- a/src/ckdb_cmd.c +++ b/src/ckdb_cmd.c @@ -3872,8 +3872,6 @@ static char *cmd_setatts(PGconn *conn, char *cmd, char *id, __maybe_unused tv_t *notcd, K_TREE *trf_root, __maybe_unused bool reload_data) { - ExecStatusType rescode; - PGresult *res; bool conned = false; K_ITEM *t_item, *u_item, *ua_item = NULL; INTRANSIENT *in_username; @@ -3921,21 +3919,13 @@ static char *cmd_setatts(PGconn *conn, char *cmd, char *id, *(dot++) = '\0'; // If we already had a different one, save it to the DB if (ua_item && strcmp(useratts->attname, attname) != 0) { - if (conn == NULL) { - conn = dbconnect(); - conned = true; - } + conned = CKPQConn(&conn); if (!begun) { - // Beginning of a write txn - res = PQexec(conn, "Begin", CKPQ_WRITE); - rescode = PQresultStatus(res); - PQclear(res); - if (!PGOK(rescode)) { - PGLOGERR("Begin", rescode, conn); + begun = CKPQBegin(conn); + if (!begun) { reason = "DBERR"; goto bats; } - begun = true; } if (useratts_item_add(conn, ua_item, now, begun)) { ua_item = NULL; @@ -3983,21 +3973,13 @@ static char *cmd_setatts(PGconn *conn, char *cmd, char *id, t_item = next_in_ktree(ctx); } if (ua_item) { - if (conn == NULL) { - conn = dbconnect(); - conned = true; - } + conned = CKPQConn(&conn); if (!begun) { - // Beginning of a write txn - res = PQexec(conn, "Begin", CKPQ_WRITE); - rescode = PQresultStatus(res); - PQclear(res); - if (!PGOK(rescode)) { - PGLOGERR("Begin", rescode, conn); + begun = CKPQBegin(conn); + if (!begun) { reason = "DBERR"; goto bats; } - begun = true; } if (!useratts_item_add(conn, ua_item, now, begun)) { reason = "DBERR"; @@ -4007,15 +3989,11 @@ static char *cmd_setatts(PGconn *conn, char *cmd, char *id, } } rollback: - if (!reason) - res = PQexec(conn, "Commit", CKPQ_WRITE); - else - res = PQexec(conn, "Rollback", CKPQ_WRITE); - PQclear(res); + CKPQEnd(conn, (reason == NULL)); + bats: - if (conned) - PQfinish(conn); + conned = CKPQDisco(&conn, conned); if (reason) { if (ua_item) { K_WLOCK(useratts_free); @@ -4207,8 +4185,6 @@ static char *cmd_setopts(PGconn *conn, char *cmd, char *id, __maybe_unused tv_t *notcd, K_TREE *trf_root, __maybe_unused bool reload_data) { - ExecStatusType rescode; - PGresult *res; bool conned = false; K_ITEM *t_item, *oc_item = NULL, *ok = NULL; K_TREE_CTX ctx[1]; @@ -4242,21 +4218,13 @@ static char *cmd_setopts(PGconn *conn, char *cmd, char *id, reason = "Missing value"; goto rollback; } - if (conn == NULL) { - conn = dbconnect(); - conned = true; - } + conned = CKPQConn(&conn); if (!begun) { - // Beginning of a write txn - res = PQexec(conn, "Begin", CKPQ_WRITE); - rescode = PQresultStatus(res); - PQclear(res); - if (!PGOK(rescode)) { - PGLOGERR("Begin", rescode, conn); + begun = CKPQBegin(conn); + if (!begun) { reason = "DBERR"; goto rollback; } - begun = true; } ok = optioncontrol_item_add(conn, oc_item, now, begun); oc_item = NULL; @@ -4299,21 +4267,13 @@ static char *cmd_setopts(PGconn *conn, char *cmd, char *id, reason = "Missing value"; goto rollback; } - if (conn == NULL) { - conn = dbconnect(); - conned = true; - } + conned = CKPQConn(&conn); if (!begun) { - // Beginning of a write txn - res = PQexec(conn, "Begin", CKPQ_WRITE); - rescode = PQresultStatus(res); - PQclear(res); - if (!PGOK(rescode)) { - PGLOGERR("Begin", rescode, conn); + begun = CKPQBegin(conn); + if (!begun) { reason = "DBERR"; goto rollback; } - begun = true; } ok = optioncontrol_item_add(conn, oc_item, now, begun); oc_item = NULL; @@ -4325,17 +4285,10 @@ static char *cmd_setopts(PGconn *conn, char *cmd, char *id, } } rollback: - if (begun) { - if (reason) - res = PQexec(conn, "Rollback", CKPQ_WRITE); - else - res = PQexec(conn, "Commit", CKPQ_WRITE); + if (begun) + CKPQEnd(conn, (reason == NULL)); - PQclear(res); - } - - if (conned) - PQfinish(conn); + conned = CKPQDisco(&conn, conned); if (reason) { snprintf(reply, siz, "ERR.%s", reason); LOGERR("%s.%s.%s", cmd, id, reply); @@ -8386,10 +8339,7 @@ static char *cmd_high(PGconn *conn, char *cmd, char *id, if (strcasecmp(action, "store") == 0) { /* Store the shares_hi_root list in the db now, * rather than wait for a shift process to do it */ - if (!conn) { - conn = dbconnect(); - conned = true; - } + conned = CKPQConn(&conn); count = 0; do { did = false; @@ -8404,8 +8354,7 @@ static char *cmd_high(PGconn *conn, char *cmd, char *id, count++; } } while (did); - if (conned) - PQfinish(conn); + conned = CKPQDisco(&conn, conned); if (count) { LOGWARNING("%s() Stored: %d high shares", __func__, count); diff --git a/src/ckdb_data.c b/src/ckdb_data.c index 74d30032..8a1755a1 100644 --- a/src/ckdb_data.c +++ b/src/ckdb_data.c @@ -6397,7 +6397,7 @@ K_ITEM *_find_markersummary(int64_t markerid, int64_t workinfoid, bool make_markersummaries(bool msg, char *by, char *code, char *inet, tv_t *cd, K_TREE *trf_root) { - PGconn *conn; + PGconn *conn = NULL; K_TREE_CTX ctx[1]; WORKMARKERS *workmarkers; K_ITEM *wm_item, *wm_last = NULL, *s_item = NULL; @@ -6427,7 +6427,7 @@ bool make_markersummaries(bool msg, char *by, char *code, char *inet, return false; } - conn = dbconnect(); + CKPQConn(&conn); /* Store all shares in the DB before processing the workmarker * This way we know that the high shares in the DB will match the start @@ -6486,7 +6486,7 @@ bool make_markersummaries(bool msg, char *by, char *code, char *inet, tvdiff(&proc_lock_fin, &proc_lock_got)); flailed: - PQfinish(conn); + CKPQDisco(&conn, true); if (count > 0) { LOGWARNING("%s() Stored: %d high shares %.3fs", diff --git a/src/ckdb_dbio.c b/src/ckdb_dbio.c index ad6f9caa..d23ed730 100644 --- a/src/ckdb_dbio.c +++ b/src/ckdb_dbio.c @@ -291,9 +291,6 @@ char *pqerrmsg(PGconn *conn) } \ } while (0) -#undef PQexec -#undef PQexecParams - /* Debug level to display write transactions - 0 removes the code * Also enables checking the isread flag */ #define CKPQ_SHOW_WRITE 0 @@ -306,7 +303,7 @@ char *pqerrmsg(PGconn *conn) #define CKPQ_ISREAD3LEN (sizeof(CKPQ_ISREAD3)-1) // Bug check to ensure no unexpected write txns occur -PGresult *_CKPQexec(PGconn *conn, const char *qry, bool isread, WHERE_FFL_ARGS) +PGresult *_CKPQExec(PGconn *conn, const char *qry, bool isread, WHERE_FFL_ARGS) { // It would slow it down, but could check qry for insert/update/... if (!isread && confirm_sharesummary) @@ -349,7 +346,7 @@ PGresult *_CKPQexec(PGconn *conn, const char *qry, bool isread, WHERE_FFL_ARGS) return PQexec(conn, qry); } -PGresult *_CKPQexecParams(PGconn *conn, const char *qry, +PGresult *_CKPQExecParams(PGconn *conn, const char *qry, int nParams, const Oid *paramTypes, const char *const * paramValues, @@ -407,11 +404,17 @@ PGresult *_CKPQexecParams(PGconn *conn, const char *qry, paramFormats, resultFormat); } -#define PQexec CKPQexec -#define PQexecParams CKPQexecParams +ExecStatusType _CKPQResultStatus(PGresult *res, WHERE_FFL_ARGS) +{ + return PQresultStatus(res); +} + +void _CKPQClear(PGresult *res, WHERE_FFL_ARGS) +{ + PQclear(res); +} -// TODO: switch all to use this -bool CKPQConn(PGconn **conn) +bool _CKPQConn(PGconn **conn, WHERE_FFL_ARGS) { if (*conn == NULL) { LOGDEBUG("%s(): connecting", __func__); @@ -421,24 +424,24 @@ bool CKPQConn(PGconn **conn) return false; } -// TODO: switch all to use this -void CKPQDisco(PGconn **conn, bool conned) +bool _CKPQDisco(PGconn **conn, bool conned, WHERE_FFL_ARGS) { - if (conned) { + if (conned && *conn) { LOGDEBUG("%s(): disco", __func__); PQfinish(*conn); + *conn = NULL; } + return false; } -// TODO: switch all to use this bool _CKPQBegin(PGconn *conn, WHERE_FFL_ARGS) { ExecStatusType rescode; PGresult *res; - res = PQexec(conn, "Begin", CKPQ_WRITE); - rescode = PQresultStatus(res); - PQclear(res); + res = _CKPQExec(conn, "Begin", CKPQ_WRITE, WHERE_FFL_PASS); + rescode = _CKPQResultStatus(res, WHERE_FFL_PASS); + _CKPQClear(res, WHERE_FFL_PASS); if (!PGOK(rescode)) { char *buf = pqerrmsg(conn); LOGEMERG("%s(): Begin failed (%d) '%s'" WHERE_FFL, @@ -450,7 +453,6 @@ bool _CKPQBegin(PGconn *conn, WHERE_FFL_ARGS) return true; } -// TODO: switch all to use this void _CKPQEnd(PGconn *conn, bool commit, WHERE_FFL_ARGS) { ExecStatusType rescode; @@ -458,13 +460,13 @@ void _CKPQEnd(PGconn *conn, bool commit, WHERE_FFL_ARGS) if (commit) { LOGDEBUG("%s(): commit", __func__); - res = PQexec(conn, "Commit", CKPQ_WRITE); + res = _CKPQExec(conn, "Commit", CKPQ_WRITE, WHERE_FFL_PASS); } else { LOGDEBUG("%s(): rollback", __func__); - res = PQexec(conn, "Rollback", CKPQ_WRITE); + res = _CKPQExec(conn, "Rollback", CKPQ_WRITE, WHERE_FFL_PASS); } - rescode = PQresultStatus(res); - PQclear(res); + rescode = _CKPQResultStatus(res, WHERE_FFL_PASS); + _CKPQClear(res, WHERE_FFL_PASS); if (!PGOK(rescode)) { char *buf = pqerrmsg(conn); LOGEMERG("%s(): %s failed (%d) '%s'" WHERE_FFL, @@ -493,13 +495,9 @@ int64_t nextid(PGconn *conn, char *idname, int64_t increment, "where idname='%s' for update", idname); - if (conn == NULL) { - conn = dbconnect(); - conned = true; - } - - res = PQexec(conn, qry, CKPQ_WRITE); - rescode = PQresultStatus(res); + conned = CKPQConn(&conn); + res = CKPQExec(conn, qry, CKPQ_WRITE); + rescode = CKPQResultStatus(res); if (!PGOK(rescode)) { PGLOGERR("Select", rescode, conn); goto cleanup; @@ -524,7 +522,7 @@ int64_t nextid(PGconn *conn, char *idname, int64_t increment, goto cleanup; TXT_TO_BIGINT("lastid", field, lastid); - PQclear(res); + CKPQClear(res); lastid += increment; snprintf(qry, sizeof(qry), "update idcontrol set " @@ -541,8 +539,9 @@ int64_t nextid(PGconn *conn, char *idname, int64_t increment, params[par++] = str_to_buf(inet, NULL, 0); PARCHK(par, params); - res = PQexecParams(conn, qry, par, NULL, (const char **)params, NULL, NULL, 0, CKPQ_WRITE); - rescode = PQresultStatus(res); + res = CKPQExecParams(conn, qry, par, NULL, (const char **)params, NULL, NULL, 0, CKPQ_WRITE); + rescode = CKPQResultStatus(res); + CKPQClear(res); if (!PGOK(rescode)) { PGLOGERR("Update", rescode, conn); lastid = 0; @@ -551,9 +550,7 @@ int64_t nextid(PGconn *conn, char *idname, int64_t increment, for (n = 0; n < par; n++) free(params[n]); cleanup: - PQclear(res); - if (conned) - PQfinish(conn); + conned = CKPQDisco(&conn, conned); return lastid; } @@ -617,23 +614,19 @@ bool users_update(PGconn *conn, K_ITEM *u_item, char *oldhash, params[par++] = tv_to_buf((tv_t *)&default_expiry, NULL, 0); PARCHKVAL(par, 3, params); - if (conn == NULL) { - conn = dbconnect(); - conned = true; - } - + conned = CKPQConn(&conn); // Beginning of a write txn - res = PQexec(conn, "Begin", CKPQ_WRITE); - rescode = PQresultStatus(res); - PQclear(res); + res = CKPQExec(conn, "Begin", CKPQ_WRITE); + rescode = CKPQResultStatus(res); + CKPQClear(res); if (!PGOK(rescode)) { PGLOGERR("Begin", rescode, conn); goto unparam; } - res = PQexecParams(conn, upd, par, NULL, (const char **)params, NULL, NULL, 0, CKPQ_WRITE); - rescode = PQresultStatus(res); - PQclear(res); + res = CKPQExecParams(conn, upd, par, NULL, (const char **)params, NULL, NULL, 0, CKPQ_WRITE); + rescode = CKPQResultStatus(res); + CKPQClear(res); if (!PGOK(rescode)) { PGLOGERR("Update", rescode, conn); goto rollback; @@ -663,9 +656,9 @@ bool users_update(PGconn *conn, K_ITEM *u_item, char *oldhash, "$7,$8,$9,$10,$11 from users where " "userid=$1 and "EDDB"=$2"; - res = PQexecParams(conn, ins, par, NULL, (const char **)params, NULL, NULL, 0, CKPQ_WRITE); - rescode = PQresultStatus(res); - PQclear(res); + res = CKPQExecParams(conn, ins, par, NULL, (const char **)params, NULL, NULL, 0, CKPQ_WRITE); + rescode = CKPQResultStatus(res); + CKPQClear(res); if (!PGOK(rescode)) { PGLOGERR("Insert", rescode, conn); goto rollback; @@ -673,15 +666,11 @@ bool users_update(PGconn *conn, K_ITEM *u_item, char *oldhash, ok = true; rollback: - if (ok) - res = PQexec(conn, "Commit", CKPQ_WRITE); - else - res = PQexec(conn, "Rollback", CKPQ_WRITE); - PQclear(res); + CKPQEnd(conn, ok); + unparam: - if (conned) - PQfinish(conn); + conned = CKPQDisco(&conn, conned); for (n = 0; n < par; n++) free(params[n]); @@ -819,13 +808,10 @@ K_ITEM *users_add(PGconn *conn, INTRANSIENT *in_username, char *emailaddress, "secondaryuserid,salt,userdata,userbits" HISTORYDATECONTROL ") values (" PQPARAM15 ")"; - if (!conn) { - conn = dbconnect(); - conned = true; - } - - res = PQexecParams(conn, ins, par, NULL, (const char **)params, NULL, NULL, 0, CKPQ_WRITE); - rescode = PQresultStatus(res); + conned = CKPQConn(&conn); + res = CKPQExecParams(conn, ins, par, NULL, (const char **)params, NULL, NULL, 0, CKPQ_WRITE); + rescode = CKPQResultStatus(res); + CKPQClear(res); if (!PGOK(rescode)) { PGLOGERR("Insert", rescode, conn); goto unparam; @@ -833,9 +819,7 @@ K_ITEM *users_add(PGconn *conn, INTRANSIENT *in_username, char *emailaddress, ok = true; unparam: - PQclear(res); - if (conned) - PQfinish(conn); + conned = CKPQDisco(&conn, conned); for (n = 0; n < par; n++) free(params[n]); unitem: @@ -888,23 +872,19 @@ bool users_replace(PGconn *conn, K_ITEM *u_item, K_ITEM *old_u_item, char *by, params[par++] = tv_to_buf((tv_t *)&default_expiry, NULL, 0); PARCHKVAL(par, 3, params); - if (conn == NULL) { - conn = dbconnect(); - conned = true; - } - + conned = CKPQConn(&conn); // Beginning of a write txn - res = PQexec(conn, "Begin", CKPQ_WRITE); - rescode = PQresultStatus(res); - PQclear(res); + res = CKPQExec(conn, "Begin", CKPQ_WRITE); + rescode = CKPQResultStatus(res); + CKPQClear(res); if (!PGOK(rescode)) { PGLOGERR("Begin", rescode, conn); goto unparam; } - res = PQexecParams(conn, upd, par, NULL, (const char **)params, NULL, NULL, 0, CKPQ_WRITE); - rescode = PQresultStatus(res); - PQclear(res); + res = CKPQExecParams(conn, upd, par, NULL, (const char **)params, NULL, NULL, 0, CKPQ_WRITE); + rescode = CKPQResultStatus(res); + CKPQClear(res); if (!PGOK(rescode)) { PGLOGERR("Update", rescode, conn); goto rollback; @@ -932,9 +912,9 @@ bool users_replace(PGconn *conn, K_ITEM *u_item, K_ITEM *old_u_item, char *by, "passwordhash,secondaryuserid,salt,userdata,userbits" HISTORYDATECONTROL ") values (" PQPARAM15 ")"; - res = PQexecParams(conn, ins, par, NULL, (const char **)params, NULL, NULL, 0, CKPQ_WRITE); - rescode = PQresultStatus(res); - PQclear(res); + res = CKPQExecParams(conn, ins, par, NULL, (const char **)params, NULL, NULL, 0, CKPQ_WRITE); + rescode = CKPQResultStatus(res); + CKPQClear(res); if (!PGOK(rescode)) { PGLOGERR("Insert", rescode, conn); goto rollback; @@ -942,15 +922,11 @@ bool users_replace(PGconn *conn, K_ITEM *u_item, K_ITEM *old_u_item, char *by, ok = true; rollback: - if (ok) - res = PQexec(conn, "Commit", CKPQ_WRITE); - else - res = PQexec(conn, "Rollback", CKPQ_WRITE); - PQclear(res); + CKPQEnd(conn, ok); + unparam: - if (conned) - PQfinish(conn); + conned = CKPQDisco(&conn, conned); for (n = 0; n < par; n++) free(params[n]); @@ -1015,11 +991,11 @@ bool users_fill(PGconn *conn) "passwordhash,secondaryuserid,salt,userdata,userbits" HISTORYDATECONTROL " from users"; - res = PQexec(conn, sel, CKPQ_READ); - rescode = PQresultStatus(res); + res = CKPQExec(conn, sel, CKPQ_READ); + rescode = CKPQResultStatus(res); if (!PGOK(rescode)) { PGLOGERR("Select", rescode, conn); - PQclear(res); + CKPQClear(res); return false; } @@ -1027,7 +1003,7 @@ bool users_fill(PGconn *conn) if (n != (fields + HISTORYDATECOUNT)) { LOGERR("%s(): Invalid field count - should be %d, but is %d", __func__, fields + HISTORYDATECOUNT, n); - PQclear(res); + CKPQClear(res); return false; } @@ -1114,7 +1090,7 @@ bool users_fill(PGconn *conn) } K_WUNLOCK(users_free); - PQclear(res); + CKPQClear(res); if (ok) { LOGDEBUG("%s(): built", __func__); @@ -1148,20 +1124,10 @@ bool useratts_item_add(PGconn *conn, K_ITEM *ua_item, tv_t *cd, bool begun) /* N.B. the values of the old ua_item record, if it exists, * are completely ignored i.e. you must provide all values required */ - if (!conn) { - conn = dbconnect(); - conned = true; - } - + conned = CKPQConn(&conn); if (!begun) { - // Beginning of a write txn - res = PQexec(conn, "Begin", CKPQ_WRITE); - rescode = PQresultStatus(res); - PQclear(res); - if (!PGOK(rescode)) { - PGLOGERR("Begin", rescode, conn); + if (!CKPQBegin(conn)) goto unparam; - } } if (old_item) { @@ -1174,9 +1140,9 @@ bool useratts_item_add(PGconn *conn, K_ITEM *ua_item, tv_t *cd, bool begun) params[par++] = tv_to_buf((tv_t *)&default_expiry, NULL, 0); PARCHKVAL(par, 4, params); - res = PQexecParams(conn, upd, par, NULL, (const char **)params, NULL, NULL, 0, CKPQ_WRITE); - rescode = PQresultStatus(res); - PQclear(res); + res = CKPQExecParams(conn, upd, par, NULL, (const char **)params, NULL, NULL, 0, CKPQ_WRITE); + rescode = CKPQResultStatus(res); + CKPQClear(res); if (!PGOK(rescode)) { PGLOGERR("Update", rescode, conn); goto unparam; @@ -1204,9 +1170,9 @@ bool useratts_item_add(PGconn *conn, K_ITEM *ua_item, tv_t *cd, bool begun) "attdate,attdate2" HISTORYDATECONTROL ") values (" PQPARAM14 ")"; - res = PQexecParams(conn, ins, par, NULL, (const char **)params, NULL, NULL, 0, CKPQ_WRITE); - rescode = PQresultStatus(res); - PQclear(res); + res = CKPQExecParams(conn, ins, par, NULL, (const char **)params, NULL, NULL, 0, CKPQ_WRITE); + rescode = CKPQResultStatus(res); + CKPQClear(res); if (!PGOK(rescode)) { PGLOGERR("Insert", rescode, conn); goto rollback; @@ -1214,17 +1180,11 @@ bool useratts_item_add(PGconn *conn, K_ITEM *ua_item, tv_t *cd, bool begun) ok = true; rollback: - if (!begun) { - if (ok) - res = PQexec(conn, "Commit", CKPQ_WRITE); - else - res = PQexec(conn, "Rollback", CKPQ_WRITE); + if (!begun) + CKPQEnd(conn, ok); - PQclear(res); - } unparam: - if (conned) - PQfinish(conn); + conned = CKPQDisco(&conn, conned); for (n = 0; n < par; n++) free(params[n]); @@ -1350,11 +1310,7 @@ bool useratts_item_expire(PGconn *conn, K_ITEM *ua_item, tv_t *cd) if (item) { DATA_USERATTS(useratts, item); - if (!conn) { - conn = dbconnect(); - conned = true; - } - + conned = CKPQConn(&conn); upd = "update useratts set "EDDB"=$1 where userid=$2 and " "attname=$3 and "EDDB"=$4"; par = 0; @@ -1364,8 +1320,9 @@ bool useratts_item_expire(PGconn *conn, K_ITEM *ua_item, tv_t *cd) params[par++] = tv_to_buf((tv_t *)&default_expiry, NULL, 0); PARCHKVAL(par, 4, params); - res = PQexecParams(conn, upd, par, NULL, (const char **)params, NULL, NULL, 0, CKPQ_WRITE); - rescode = PQresultStatus(res); + res = CKPQExecParams(conn, upd, par, NULL, (const char **)params, NULL, NULL, 0, CKPQ_WRITE); + rescode = CKPQResultStatus(res); + CKPQClear(res); if (!PGOK(rescode)) { PGLOGERR("Update", rescode, conn); goto unparam; @@ -1374,9 +1331,7 @@ bool useratts_item_expire(PGconn *conn, K_ITEM *ua_item, tv_t *cd) ok = true; unparam: if (par) { - PQclear(res); - if (conned) - PQfinish(conn); + conned = CKPQDisco(&conn, conned); for (n = 0; n < par; n++) free(params[n]); } @@ -1411,11 +1366,11 @@ bool useratts_fill(PGconn *conn) ",attdate,attdate2" HISTORYDATECONTROL " from useratts"; - res = PQexec(conn, sel, CKPQ_READ); - rescode = PQresultStatus(res); + res = CKPQExec(conn, sel, CKPQ_READ); + rescode = CKPQResultStatus(res); if (!PGOK(rescode)) { PGLOGERR("Select", rescode, conn); - PQclear(res); + CKPQClear(res); return false; } @@ -1423,7 +1378,7 @@ bool useratts_fill(PGconn *conn) if (n != (fields + HISTORYDATECOUNT)) { LOGERR("%s(): Invalid field count - should be %d, but is %d", __func__, fields + HISTORYDATECOUNT, n); - PQclear(res); + CKPQClear(res); return false; } @@ -1497,7 +1452,7 @@ bool useratts_fill(PGconn *conn) k_add_head(useratts_free, item); K_WUNLOCK(useratts_free); - PQclear(res); + CKPQClear(res); if (ok) { LOGDEBUG("%s(): built", __func__); @@ -1545,11 +1500,7 @@ K_ITEM *workers_add(PGconn *conn, int64_t userid, char *workername, bool add_ws, DATA_WORKERS(row, item); - if (conn == NULL) { - conn = dbconnect(); - conned = true; - } - + conned = CKPQConn(&conn); bzero(row, sizeof(*row)); row->workerid = nextid(conn, "workerid", (int64_t)1, cd, by, code, inet); if (row->workerid == 0) @@ -1616,8 +1567,9 @@ K_ITEM *workers_add(PGconn *conn, int64_t userid, char *workername, bool add_ws, "idlenotificationenabled,idlenotificationtime,workerbits" HISTORYDATECONTROL ") values (" PQPARAM12 ")"; - res = PQexecParams(conn, ins, par, NULL, (const char **)params, NULL, NULL, 0, CKPQ_WRITE); - rescode = PQresultStatus(res); + res = CKPQExecParams(conn, ins, par, NULL, (const char **)params, NULL, NULL, 0, CKPQ_WRITE); + rescode = CKPQResultStatus(res); + CKPQClear(res); if (!PGOK(rescode)) { PGLOGERR("Insert", rescode, conn); goto unparam; @@ -1625,12 +1577,10 @@ K_ITEM *workers_add(PGconn *conn, int64_t userid, char *workername, bool add_ws, ret = item; unparam: - PQclear(res); for (n = 0; n < par; n++) free(params[n]); unitem: - if (conned) - PQfinish(conn); + conned = CKPQDisco(&conn, conned); K_WLOCK(workers_free); if (!ret) k_add_head(workers_free, item); @@ -1736,22 +1686,18 @@ bool workers_update(PGconn *conn, K_ITEM *item, char *difficultydefault, params[par++] = tv_to_buf((tv_t *)&default_expiry, NULL, 0); PARCHKVAL(par, 3, params); - if (conn == NULL) { - conn = dbconnect(); - conned = true; - } - - res = PQexec(conn, "Begin", CKPQ_WRITE); - rescode = PQresultStatus(res); - PQclear(res); + conned = CKPQConn(&conn); + res = CKPQExec(conn, "Begin", CKPQ_WRITE); + rescode = CKPQResultStatus(res); + CKPQClear(res); if (!PGOK(rescode)) { PGLOGERR("Begin", rescode, conn); goto unparam; } - res = PQexecParams(conn, upd, par, NULL, (const char **)params, NULL, NULL, 0, CKPQ_WRITE); - rescode = PQresultStatus(res); - PQclear(res); + res = CKPQExecParams(conn, upd, par, NULL, (const char **)params, NULL, NULL, 0, CKPQ_WRITE); + rescode = CKPQResultStatus(res); + CKPQClear(res); if (!PGOK(rescode)) { PGLOGERR("Update", rescode, conn); goto rollback; @@ -1776,9 +1722,9 @@ bool workers_update(PGconn *conn, K_ITEM *item, char *difficultydefault, HISTORYDATEPARAMS(params, par, row); PARCHK(par, params); - res = PQexecParams(conn, ins, par, NULL, (const char **)params, NULL, NULL, 0, CKPQ_WRITE); - rescode = PQresultStatus(res); - PQclear(res); + res = CKPQExecParams(conn, ins, par, NULL, (const char **)params, NULL, NULL, 0, CKPQ_WRITE); + rescode = CKPQResultStatus(res); + CKPQClear(res); if (!PGOK(rescode)) { PGLOGERR("Insert", rescode, conn); goto rollback; @@ -1786,15 +1732,11 @@ bool workers_update(PGconn *conn, K_ITEM *item, char *difficultydefault, ok = true; rollback: - if (ok) - res = PQexec(conn, "Commit", CKPQ_WRITE); - else - res = PQexec(conn, "Rollback", CKPQ_WRITE); - PQclear(res); + CKPQEnd(conn, ok); + unparam: - if (conned) - PQfinish(conn); + conned = CKPQDisco(&conn, conned); for (n = 0; n < par; n++) free(params[n]); early: @@ -1823,20 +1765,15 @@ bool workers_fill(PGconn *conn) "idlenotificationenabled,idlenotificationtime,workerbits" HISTORYDATECONTROL ",workerid from workers"; - res = PQexec(conn, "Begin", CKPQ_READ); - rescode = PQresultStatus(res); - PQclear(res); - if (!PGOK(rescode)) { - PGLOGERR("Begin", rescode, conn); + if (!CKPQBegin(conn)) return false; - } // See workers_add() about this lock K_WLOCK(workers_db_free); - res = PQexec(conn, sel, CKPQ_READ); - rescode = PQresultStatus(res); - PQclear(res); + res = CKPQExec(conn, sel, CKPQ_READ); + rescode = CKPQResultStatus(res); + CKPQClear(res); if (!PGOK(rescode)) { PGLOGERR("Declare", rescode, conn); goto flail; @@ -1844,11 +1781,11 @@ bool workers_fill(PGconn *conn) LOGDEBUG("%s(): fetching ...", __func__); - res = PQexec(conn, "fetch 1 in wk", CKPQ_READ); - rescode = PQresultStatus(res); + res = CKPQExec(conn, "fetch 1 in wk", CKPQ_READ); + rescode = CKPQResultStatus(res); if (!PGOK(rescode)) { PGLOGERR("Fetch first", rescode, conn); - PQclear(res); + CKPQClear(res); goto flail; } @@ -1856,7 +1793,7 @@ bool workers_fill(PGconn *conn) if (n != (fields + HISTORYDATECOUNT)) { LOGERR("%s(): Invalid field count - should be %d, but is %d", __func__, fields + HISTORYDATECOUNT, n); - PQclear(res); + CKPQClear(res); goto flail; } @@ -1926,9 +1863,9 @@ bool workers_fill(PGconn *conn) tick(); n++; } - PQclear(res); - res = PQexec(conn, "fetch 9999 in wk", CKPQ_READ); - rescode = PQresultStatus(res); + CKPQClear(res); + res = CKPQExec(conn, "fetch 9999 in wk", CKPQ_READ); + rescode = CKPQResultStatus(res); if (!PGOK(rescode)) { PGLOGERR("Fetch next", rescode, conn); ok = false; @@ -1938,11 +1875,10 @@ bool workers_fill(PGconn *conn) if (!ok) k_add_head(workers_free, item); - PQclear(res); + CKPQClear(res); flail: - res = PQexec(conn, "Commit", CKPQ_READ); - PQclear(res); + CKPQCommit(conn); K_WUNLOCK(workers_db_free); if (ok) { @@ -1982,20 +1918,11 @@ bool paymentaddresses_set(PGconn *conn, int64_t userid, K_STORE *pa_store, if (pa_store->count > ABS_ADDR_LIMIT) return false; - if (conn == NULL) { - conn = dbconnect(); - conned = true; - } - + conned = CKPQConn(&conn); /* This means the nextid updates will rollback on an error, but also * means that it will lock the nextid record for the whole update */ - res = PQexec(conn, "Begin", CKPQ_WRITE); - rescode = PQresultStatus(res); - if (!PGOK(rescode)) { - PGLOGERR("Begin", rescode, conn); + if (!CKPQBegin(conn)) goto unparam; - } - PQclear(res); // First step - DB expire all the old/changed records in RAM LOGDEBUG("%s(): Step 1 userid=%"PRId64, __func__, userid); @@ -2064,9 +1991,9 @@ bool paymentaddresses_set(PGconn *conn, int64_t userid, K_STORE *pa_store, } else { APPEND_REALLOC(upd, off, len, ")"); PARCHKVAL(par, par, params); - res = PQexecParams(conn, upd, par, NULL, (const char **)params, NULL, NULL, 0, CKPQ_WRITE); - rescode = PQresultStatus(res); - PQclear(res); + res = CKPQExecParams(conn, upd, par, NULL, (const char **)params, NULL, NULL, 0, CKPQ_WRITE); + rescode = CKPQResultStatus(res); + CKPQClear(res); if (!PGOK(rescode)) { PGLOGERR("Update", rescode, conn); goto rollback; @@ -2109,10 +2036,10 @@ bool paymentaddresses_set(PGconn *conn, int64_t userid, K_STORE *pa_store, HISTORYDATEPARAMSIN(params, par, row); PARCHKVAL(par, 10, params); // As per PQPARAM10 above - res = PQexecParams(conn, ins, par, NULL, (const char **)params, + res = CKPQExecParams(conn, ins, par, NULL, (const char **)params, NULL, NULL, 0, CKPQ_WRITE); - rescode = PQresultStatus(res); - PQclear(res); + rescode = CKPQResultStatus(res); + CKPQClear(res); if (!PGOK(rescode)) { PGLOGERR("Insert", rescode, conn); goto rollback; @@ -2130,15 +2057,11 @@ bool paymentaddresses_set(PGconn *conn, int64_t userid, K_STORE *pa_store, ok = true; rollback: - if (ok) - res = PQexec(conn, "Commit", CKPQ_WRITE); - else - res = PQexec(conn, "Rollback", CKPQ_WRITE); - PQclear(res); + CKPQEnd(conn, ok); + unparam: - if (conned) - PQfinish(conn); + conned = CKPQDisco(&conn, conned); for (n = 0; n < par; n++) free(params[n]); FREENULL(upd); @@ -2219,11 +2142,11 @@ bool paymentaddresses_fill(PGconn *conn) "paymentaddressid,userid,payaddress,payratio,payname" HISTORYDATECONTROL " from paymentaddresses"; - res = PQexec(conn, sel, CKPQ_READ); - rescode = PQresultStatus(res); + res = CKPQExec(conn, sel, CKPQ_READ); + rescode = CKPQResultStatus(res); if (!PGOK(rescode)) { PGLOGERR("Select", rescode, conn); - PQclear(res); + CKPQClear(res); return false; } @@ -2231,7 +2154,7 @@ bool paymentaddresses_fill(PGconn *conn) if (n != (fields + HISTORYDATECOUNT)) { LOGERR("%s(): Invalid field count - should be %d, but is %d", __func__, fields + HISTORYDATECOUNT, n); - PQclear(res); + CKPQClear(res); return false; } @@ -2286,7 +2209,7 @@ bool paymentaddresses_fill(PGconn *conn) k_add_head(paymentaddresses_free, item); K_WUNLOCK(paymentaddresses_free); - PQclear(res); + CKPQClear(res); if (ok) { LOGDEBUG("%s(): built", __func__); @@ -2366,9 +2289,9 @@ bool payments_add(PGconn *conn, bool add, K_ITEM *p_item, K_ITEM **old_p_item, params[par++] = tv_to_buf((tv_t *)&default_expiry, NULL, 0); PARCHKVAL(par, 3, params); - res = PQexecParams(conn, upd, par, NULL, (const char **)params, NULL, NULL, 0, CKPQ_WRITE); - rescode = PQresultStatus(res); - PQclear(res); + res = CKPQExecParams(conn, upd, par, NULL, (const char **)params, NULL, NULL, 0, CKPQ_WRITE); + rescode = CKPQResultStatus(res); + CKPQClear(res); if (!PGOK(rescode)) { PGLOGERR("Update", rescode, conn); goto rollback; @@ -2415,8 +2338,9 @@ bool payments_add(PGconn *conn, bool add, K_ITEM *p_item, K_ITEM **old_p_item, "originaltxn,amount,diffacc,committxn,commitblockhash" HISTORYDATECONTROL ") values (" PQPARAM16 ")"; - res = PQexecParams(conn, ins, par, NULL, (const char **)params, NULL, NULL, 0, CKPQ_WRITE); - rescode = PQresultStatus(res); + res = CKPQExecParams(conn, ins, par, NULL, (const char **)params, NULL, NULL, 0, CKPQ_WRITE); + rescode = CKPQResultStatus(res); + CKPQClear(res); if (!PGOK(rescode)) { PGLOGERR("Insert", rescode, conn); goto unparam; @@ -2431,7 +2355,7 @@ unparam: for (n = 0; n < par; n++) free(params[n]); - CKPQDisco(&conn, conned); + conned = CKPQDisco(&conn, conned); if (!already) payments_add_ram(ok, p_item, *old_p_item, cd); @@ -2462,17 +2386,12 @@ bool payments_fill(PGconn *conn) "originaltxn,amount,diffacc,committxn,commitblockhash" HISTORYDATECONTROL " from payments"; - res = PQexec(conn, "Begin", CKPQ_READ); - rescode = PQresultStatus(res); - PQclear(res); - if (!PGOK(rescode)) { - PGLOGERR("Begin", rescode, conn); + if (!CKPQBegin(conn)) return false; - } - res = PQexec(conn, sel, CKPQ_READ); - rescode = PQresultStatus(res); - PQclear(res); + res = CKPQExec(conn, sel, CKPQ_READ); + rescode = CKPQResultStatus(res); + CKPQClear(res); if (!PGOK(rescode)) { PGLOGERR("Declare", rescode, conn); goto flail; @@ -2480,11 +2399,11 @@ bool payments_fill(PGconn *conn) LOGDEBUG("%s(): fetching ...", __func__); - res = PQexec(conn, "fetch 1 in ps", CKPQ_READ); - rescode = PQresultStatus(res); + res = CKPQExec(conn, "fetch 1 in ps", CKPQ_READ); + rescode = CKPQResultStatus(res); if (!PGOK(rescode)) { PGLOGERR("Fetch first", rescode, conn); - PQclear(res); + CKPQClear(res); goto flail; } @@ -2492,7 +2411,7 @@ bool payments_fill(PGconn *conn) if (n != (fields + HISTORYDATECOUNT)) { LOGERR("%s(): Invalid field count - should be %d, but is %d", __func__, fields + HISTORYDATECOUNT, n); - PQclear(res); + CKPQClear(res); goto flail; } @@ -2581,9 +2500,9 @@ bool payments_fill(PGconn *conn) tick(); n++; } - PQclear(res); - res = PQexec(conn, "fetch 9999 in ps", CKPQ_READ); - rescode = PQresultStatus(res); + CKPQClear(res); + res = CKPQExec(conn, "fetch 9999 in ps", CKPQ_READ); + rescode = CKPQResultStatus(res); if (!PGOK(rescode)) { PGLOGERR("Fetch next", rescode, conn); ok = false; @@ -2594,11 +2513,10 @@ bool payments_fill(PGconn *conn) k_add_head(payments_free, item); K_WUNLOCK(payments_free); - PQclear(res); + CKPQClear(res); flail: - res = PQexec(conn, "Commit", CKPQ_READ); - PQclear(res); + CKPQCommit(conn); if (ok) { LOGDEBUG("%s(): built", __func__); LOGWARNING("%s(): fetched %d payments records", __func__, n); @@ -2642,14 +2560,10 @@ bool idcontrol_add(PGconn *conn, char *idname, char *idvalue, char *by, ins = "insert into idcontrol " "(idname,lastid" MODIFYDATECONTROL ") values (" PQPARAM10 ")"; - if (!conn) { - conn = dbconnect(); - conned = true; - } - - res = PQexecParams(conn, ins, par, NULL, (const char **)params, NULL, NULL, 0, CKPQ_WRITE); - rescode = PQresultStatus(res); - PQclear(res); + conned = CKPQConn(&conn); + res = CKPQExecParams(conn, ins, par, NULL, (const char **)params, NULL, NULL, 0, CKPQ_WRITE); + rescode = CKPQResultStatus(res); + CKPQClear(res); if (!PGOK(rescode)) { PGLOGERR("Insert", rescode, conn); goto foil; @@ -2657,8 +2571,7 @@ bool idcontrol_add(PGconn *conn, char *idname, char *idvalue, char *by, ok = true; foil: - if (conned) - PQfinish(conn); + conned = CKPQDisco(&conn, conned); for (n = 0; n < par; n++) free(params[n]); @@ -2977,19 +2890,10 @@ K_ITEM *optioncontrol_item_add(PGconn *conn, K_ITEM *oc_item, tv_t *cd, bool beg old_item = find_in_ktree(optioncontrol_root, &look, ctx); K_RUNLOCK(optioncontrol_free); - if (!conn) { - conn = dbconnect(); - conned = true; - } - + conned = CKPQConn(&conn); if (!begun) { - res = PQexec(conn, "Begin", CKPQ_WRITE); - rescode = PQresultStatus(res); - PQclear(res); - if (!PGOK(rescode)) { - PGLOGERR("Begin", rescode, conn); + if (!CKPQBegin(conn)) goto nostart; - } } if (old_item) { @@ -3006,9 +2910,9 @@ K_ITEM *optioncontrol_item_add(PGconn *conn, K_ITEM *oc_item, tv_t *cd, bool beg params[par++] = tv_to_buf((tv_t *)&default_expiry, NULL, 0); PARCHKVAL(par, 5, params); - res = PQexecParams(conn, upd, par, NULL, (const char **)params, NULL, NULL, 0, CKPQ_WRITE); - rescode = PQresultStatus(res); - PQclear(res); + res = CKPQExecParams(conn, upd, par, NULL, (const char **)params, NULL, NULL, 0, CKPQ_WRITE); + rescode = CKPQResultStatus(res); + CKPQClear(res); if (!PGOK(rescode)) { PGLOGERR("Update", rescode, conn); goto rollback; @@ -3030,9 +2934,9 @@ K_ITEM *optioncontrol_item_add(PGconn *conn, K_ITEM *oc_item, tv_t *cd, bool beg "(optionname,optionvalue,activationdate,activationheight" HISTORYDATECONTROL ") values (" PQPARAM9 ")"; - res = PQexecParams(conn, ins, par, NULL, (const char **)params, NULL, NULL, 0, CKPQ_WRITE); - rescode = PQresultStatus(res); - PQclear(res); + res = CKPQExecParams(conn, ins, par, NULL, (const char **)params, NULL, NULL, 0, CKPQ_WRITE); + rescode = CKPQResultStatus(res); + CKPQClear(res); if (!PGOK(rescode)) { PGLOGERR("Insert", rescode, conn); goto rollback; @@ -3040,17 +2944,11 @@ K_ITEM *optioncontrol_item_add(PGconn *conn, K_ITEM *oc_item, tv_t *cd, bool beg ok = true; rollback: - if (!begun) { - if (ok) - res = PQexec(conn, "Commit", CKPQ_WRITE); - else - res = PQexec(conn, "Rollback", CKPQ_WRITE); + if (!begun) + CKPQEnd(conn, ok); - PQclear(res); - } nostart: - if (conned) - PQfinish(conn); + conned = CKPQDisco(&conn, conned); for (n = 0; n < par; n++) free(params[n]); @@ -3145,11 +3043,11 @@ bool optioncontrol_fill(PGconn *conn) par = 0; params[par++] = tv_to_buf((tv_t *)(&default_expiry), NULL, 0); PARCHK(par, params); - res = PQexecParams(conn, sel, par, NULL, (const char **)params, NULL, NULL, 0, CKPQ_READ); - rescode = PQresultStatus(res); + res = CKPQExecParams(conn, sel, par, NULL, (const char **)params, NULL, NULL, 0, CKPQ_READ); + rescode = CKPQResultStatus(res); if (!PGOK(rescode)) { PGLOGERR("Select", rescode, conn); - PQclear(res); + CKPQClear(res); return false; } @@ -3157,7 +3055,7 @@ bool optioncontrol_fill(PGconn *conn) if (n != (fields + HISTORYDATECOUNT)) { LOGERR("%s(): Invalid field count - should be %d, but is %d", __func__, fields + HISTORYDATECOUNT, n); - PQclear(res); + CKPQClear(res); return false; } @@ -3211,7 +3109,7 @@ bool optioncontrol_fill(PGconn *conn) } K_WUNLOCK(optioncontrol_free); - PQclear(res); + CKPQClear(res); for (n = 0; n < par; n++) free(params[n]); @@ -3330,13 +3228,10 @@ int64_t workinfo_add(PGconn *conn, char *workinfoidstr, "prevhash,coinbase1,coinbase2,version,bits,ntime,reward" HISTORYDATECONTROL ") values (" PQPARAM16 ")"; - if (!conn) { - conn = dbconnect(); - conned = true; - } - - res = PQexecParams(conn, ins, par, NULL, (const char **)params, NULL, NULL, 0, CKPQ_WRITE); - rescode = PQresultStatus(res); + conned = CKPQConn(&conn); + res = CKPQExecParams(conn, ins, par, NULL, (const char **)params, NULL, NULL, 0, CKPQ_WRITE); + rescode = CKPQResultStatus(res); + CKPQClear(res); if (!PGOK(rescode)) { PGLOGERR("Insert", rescode, conn); goto unparam; @@ -3347,9 +3242,7 @@ int64_t workinfo_add(PGconn *conn, char *workinfoidstr, unparam: if (par) { - PQclear(res); - if (conned) - PQfinish(conn); + conned = CKPQDisco(&conn, conned); for (n = 0; n < par; n++) free(params[n]); } @@ -3446,27 +3339,22 @@ bool workinfo_fill(PGconn *conn) params[par++] = bigint_to_buf(dbload_workinfoid_finish, NULL, 0); PARCHK(par, params); - res = PQexec(conn, "Begin", CKPQ_READ); - rescode = PQresultStatus(res); - PQclear(res); - if (!PGOK(rescode)) { - PGLOGERR("Begin", rescode, conn); + if (!CKPQBegin(conn)) return false; - } if (exclusive_db) { - res = PQexec(conn, "Lock table workinfo in access exclusive mode", CKPQ_READ); - rescode = PQresultStatus(res); - PQclear(res); + res = CKPQExec(conn, "Lock table workinfo in access exclusive mode", CKPQ_READ); + rescode = CKPQResultStatus(res); + CKPQClear(res); if (!PGOK(rescode)) { PGLOGERR("Lock", rescode, conn); goto flail; } } - res = PQexecParams(conn, sel, par, NULL, (const char **)params, NULL, NULL, 0, CKPQ_READ); - rescode = PQresultStatus(res); - PQclear(res); + res = CKPQExecParams(conn, sel, par, NULL, (const char **)params, NULL, NULL, 0, CKPQ_READ); + rescode = CKPQResultStatus(res); + CKPQClear(res); if (!PGOK(rescode)) { PGLOGERR("Declare", rescode, conn); goto flail; @@ -3474,11 +3362,11 @@ bool workinfo_fill(PGconn *conn) LOGDEBUG("%s(): fetching ...", __func__); - res = PQexec(conn, "fetch 1 in wi", CKPQ_READ); - rescode = PQresultStatus(res); + res = CKPQExec(conn, "fetch 1 in wi", CKPQ_READ); + rescode = CKPQResultStatus(res); if (!PGOK(rescode)) { PGLOGERR("Fetch first", rescode, conn); - PQclear(res); + CKPQClear(res); goto flail; } @@ -3486,7 +3374,7 @@ bool workinfo_fill(PGconn *conn) if (n != (fields + HISTORYDATECOUNT)) { LOGERR("%s(): Invalid field count - should be %d, but is %d", __func__, fields + HISTORYDATECOUNT, n); - PQclear(res); + CKPQClear(res); goto flail; } @@ -3587,9 +3475,9 @@ bool workinfo_fill(PGconn *conn) tick(); n++; } - PQclear(res); - res = PQexec(conn, "fetch 9999 in wi", CKPQ_READ); - rescode = PQresultStatus(res); + CKPQClear(res); + res = CKPQExec(conn, "fetch 9999 in wi", CKPQ_READ); + rescode = CKPQResultStatus(res); if (!PGOK(rescode)) { PGLOGERR("Fetch next", rescode, conn); ok = false; @@ -3606,10 +3494,10 @@ bool workinfo_fill(PGconn *conn) } K_WUNLOCK(workinfo_free); - PQclear(res); + CKPQClear(res); flail: - res = PQexec(conn, "Commit", CKPQ_READ); - PQclear(res); + + CKPQCommit(conn); for (i = 0; i < par; i++) free(params[i]); par = 0; @@ -4107,13 +3995,9 @@ bool shares_db(PGconn *conn, K_ITEM *s_item) "diff,sdiff,errn,error,secondaryuserid,ntime,minsdiff,address," "agent" HISTORYDATECONTROL ") values (" PQPARAM21 ")"; - if (!conn) { - conn = dbconnect(); - conned = true; - } - - res = PQexecParams(conn, ins, par, NULL, (const char **)params, NULL, NULL, 0, CKPQ_WRITE); - rescode = PQresultStatus(res); + conned = CKPQConn(&conn); + res = CKPQExecParams(conn, ins, par, NULL, (const char **)params, NULL, NULL, 0, CKPQ_WRITE); + rescode = CKPQResultStatus(res); if (!PGOK(rescode)) { char *code = PQresultErrorField(res, PG_DIAG_SQLSTATE); if (!code || strcmp(code, SQL_UNIQUE_VIOLATION)) { @@ -4128,9 +4012,8 @@ bool shares_db(PGconn *conn, K_ITEM *s_item) ok = true; unparam: if (par) { - PQclear(res); - if (conned) - PQfinish(conn); + CKPQClear(res); + conned = CKPQDisco(&conn, conned); for (n = 0; n < par; n++) free(params[n]); } @@ -4210,27 +4093,22 @@ bool shares_fill(PGconn *conn) params[par++] = bigint_to_buf(workinfoid, NULL, 0); PARCHK(par, params); - res = PQexec(conn, "Begin", CKPQ_READ); - rescode = PQresultStatus(res); - PQclear(res); - if (!PGOK(rescode)) { - PGLOGERR("Begin", rescode, conn); + if (!CKPQBegin(conn)) return false; - } if (exclusive_db) { - res = PQexec(conn, "Lock table shares in access exclusive mode", CKPQ_READ); - rescode = PQresultStatus(res); - PQclear(res); + res = CKPQExec(conn, "Lock table shares in access exclusive mode", CKPQ_READ); + rescode = CKPQResultStatus(res); + CKPQClear(res); if (!PGOK(rescode)) { PGLOGERR("Lock", rescode, conn); goto flail; } } - res = PQexecParams(conn, sel, par, NULL, (const char **)params, NULL, NULL, 0, CKPQ_READ); - rescode = PQresultStatus(res); - PQclear(res); + res = CKPQExecParams(conn, sel, par, NULL, (const char **)params, NULL, NULL, 0, CKPQ_READ); + rescode = CKPQResultStatus(res); + CKPQClear(res); if (!PGOK(rescode)) { PGLOGERR("Declare", rescode, conn); goto flail; @@ -4238,11 +4116,11 @@ bool shares_fill(PGconn *conn) LOGDEBUG("%s(): fetching ...", __func__); - res = PQexec(conn, "fetch 1 in sh", CKPQ_READ); - rescode = PQresultStatus(res); + res = CKPQExec(conn, "fetch 1 in sh", CKPQ_READ); + rescode = CKPQResultStatus(res); if (!PGOK(rescode)) { PGLOGERR("Fetch first", rescode, conn); - PQclear(res); + CKPQClear(res); goto flail; } @@ -4250,7 +4128,7 @@ bool shares_fill(PGconn *conn) if (n != (fields + HISTORYDATECOUNT)) { LOGERR("%s(): Invalid field count - should be %d, but is %d", __func__, fields + HISTORYDATECOUNT, n); - PQclear(res); + CKPQClear(res); goto flail; } @@ -4368,9 +4246,9 @@ bool shares_fill(PGconn *conn) tick(); n++; } - PQclear(res); - res = PQexec(conn, "fetch 9999 in sh", CKPQ_READ); - rescode = PQresultStatus(res); + CKPQClear(res); + res = CKPQExec(conn, "fetch 9999 in sh", CKPQ_READ); + rescode = CKPQResultStatus(res); if (!PGOK(rescode)) { PGLOGERR("Fetch next", rescode, conn); ok = false; @@ -4381,11 +4259,10 @@ bool shares_fill(PGconn *conn) k_add_head(shares_free, item); K_WUNLOCK(shares_free); - PQclear(res); + CKPQClear(res); flail: - res = PQexec(conn, "Commit", CKPQ_READ); - PQclear(res); + CKPQCommit(conn); if (ok) { LOGDEBUG("%s(): built", __func__); LOGWARNING("%s(): fetched %d shares records", __func__, n); @@ -4761,8 +4638,7 @@ bool sharesummaries_to_markersummaries(PGconn *conn, WORKMARKERS *workmarkers, static const char *shortname = "K/SS_to_K/MS"; static const char *sshortname = "SS_to_MS"; static const char *kshortname = "KSS_to_KS"; - ExecStatusType rescode; - PGresult *res; + K_TREE_CTX ss_ctx[1], kss_ctx[1], ms_ctx[1], ks_ctx[1]; SHARESUMMARY *sharesummary, looksharesummary; KEYSHARESUMMARY *keysharesummary, lookkeysharesummary; @@ -5084,16 +4960,8 @@ dokey: setnow(&kadd_fin); setnow(&db_stt); - if (conn == NULL) { - conn = dbconnect(); - conned = true; - } - - res = PQexec(conn, "Begin", CKPQ_WRITE); - rescode = PQresultStatus(res); - PQclear(res); - if (!PGOK(rescode)) { - PGLOGERR("Begin", rescode, conn); + conned = CKPQConn(&conn); + if (!CKPQBegin(conn)) { setnow(&db_fin); goto flail; } @@ -5137,15 +5005,11 @@ dokey: } setnow(&kdb_fin); rollback: - if (ok) - res = PQexec(conn, "Commit", CKPQ_WRITE); - else - res = PQexec(conn, "Rollback", CKPQ_WRITE); - PQclear(res); + CKPQEnd(conn, ok); + flail: - if (conned) - PQfinish(conn); + conned = CKPQDisco(&conn, conned); if (reason) { // already displayed the full workmarkers detail at the top @@ -5403,13 +5267,9 @@ bool delete_markersummaries(PGconn *conn, WORKMARKERS *wm) del = "delete from markersummary where markerid=$1"; - if (conn == NULL) { - conn = dbconnect(); - conned = true; - } - - res = PQexecParams(conn, del, par, NULL, (const char **)params, NULL, NULL, 0, CKPQ_WRITE); - rescode = PQresultStatus(res); + conned = CKPQConn(&conn); + res = CKPQExecParams(conn, del, par, NULL, (const char **)params, NULL, NULL, 0, CKPQ_WRITE); + rescode = CKPQResultStatus(res); if (!PGOK(rescode)) { PGLOGERR("Delete", rescode, conn); reason = "db error"; @@ -5432,10 +5292,9 @@ bool delete_markersummaries(PGconn *conn, WORKMARKERS *wm) ok = true; unparam: - PQclear(res); + CKPQClear(res); flail: - if (conned) - PQfinish(conn); + conned = CKPQDisco(&conn, conned); if (!ok) { if (del_markersummary_store && del_markersummary_store->count) { @@ -5924,22 +5783,13 @@ bool blocks_stats(PGconn *conn, int32_t height, char *blockhash, params[par++] = tv_to_buf((tv_t *)&default_expiry, NULL, 0); PARCHKVAL(par, 3, params); - if (conn == NULL) { - conn = dbconnect(); - conned = true; - } - - res = PQexec(conn, "Begin", CKPQ_WRITE); - rescode = PQresultStatus(res); - PQclear(res); - if (!PGOK(rescode)) { - PGLOGERR("Begin", rescode, conn); + conned = CKPQConn(&conn); + if (!CKPQBegin(conn)) goto unparam; - } - res = PQexecParams(conn, upd, par, NULL, (const char **)params, NULL, NULL, 0, CKPQ_WRITE); - rescode = PQresultStatus(res); - PQclear(res); + res = CKPQExecParams(conn, upd, par, NULL, (const char **)params, NULL, NULL, 0, CKPQ_WRITE); + rescode = CKPQResultStatus(res); + CKPQClear(res); if (!PGOK(rescode)) { PGLOGERR("Update", rescode, conn); goto rollback; @@ -5974,9 +5824,9 @@ bool blocks_stats(PGconn *conn, int32_t height, char *blockhash, "$9,$10,$11,$12,$13 from blocks where " "blockhash=$1 and "EDDB"=$2"; - res = PQexecParams(conn, ins, par, NULL, (const char **)params, NULL, NULL, 0, CKPQ_WRITE); - rescode = PQresultStatus(res); - PQclear(res); + res = CKPQExecParams(conn, ins, par, NULL, (const char **)params, NULL, NULL, 0, CKPQ_WRITE); + rescode = CKPQResultStatus(res); + CKPQClear(res); if (!PGOK(rescode)) { PGLOGERR("Insert", rescode, conn); goto rollback; @@ -5984,19 +5834,14 @@ bool blocks_stats(PGconn *conn, int32_t height, char *blockhash, ok = true; rollback: - if (ok) - res = PQexec(conn, "Commit", CKPQ_WRITE); - else - res = PQexec(conn, "Rollback", CKPQ_WRITE); - PQclear(res); + CKPQEnd(conn, ok); + unparam: + conned = CKPQDisco(&conn, conned); for (n = 0; n < par; n++) free(params[n]); - if (conned) - PQfinish(conn); - K_WLOCK(blocks_free); if (!ok) k_add_head(blocks_free, b_item); @@ -6146,14 +5991,10 @@ bool blocks_add(PGconn *conn, int32_t height, char *blockhash, "statsconfirmed" HISTORYDATECONTROL ") values (" PQPARAM23 ")"; - if (conn == NULL) { - conn = dbconnect(); - conned = true; - } - - res = PQexecParams(conn, ins, par, NULL, (const char **)params, NULL, NULL, 0, CKPQ_WRITE); - rescode = PQresultStatus(res); - PQclear(res); + conned = CKPQConn(&conn); + res = CKPQExecParams(conn, ins, par, NULL, (const char **)params, NULL, NULL, 0, CKPQ_WRITE); + rescode = CKPQResultStatus(res); + CKPQClear(res); if (!PGOK(rescode)) { PGLOGERR("Insert", rescode, conn); goto unparam; @@ -6228,11 +6069,7 @@ bool blocks_add(PGconn *conn, int32_t height, char *blockhash, params[par++] = tv_to_buf((tv_t *)&default_expiry, NULL, 0); PARCHKVAL(par, 3, params); - if (conn == NULL) { - conn = dbconnect(); - conned = true; - } - + conned = CKPQConn(&conn); // New is mostly a copy of the old copy_blocks(row, oldblocks); STRNCPY(row->confirmed, confirmed); @@ -6248,17 +6085,11 @@ bool blocks_add(PGconn *conn, int32_t height, char *blockhash, HISTORYDATEINIT(row, cd, by, code, inet); HISTORYDATETRANSFER(trf_root, row); - res = PQexec(conn, "Begin", CKPQ_WRITE); - rescode = PQresultStatus(res); - PQclear(res); - if (!PGOK(rescode)) { - PGLOGERR("Begin", rescode, conn); + if (!CKPQBegin(conn)) goto unparam; - } - - res = PQexecParams(conn, upd, par, NULL, (const char **)params, NULL, NULL, 0, CKPQ_WRITE); - rescode = PQresultStatus(res); - PQclear(res); + res = CKPQExecParams(conn, upd, par, NULL, (const char **)params, NULL, NULL, 0, CKPQ_WRITE); + rescode = CKPQResultStatus(res); + CKPQClear(res); if (!PGOK(rescode)) { PGLOGERR("Update", rescode, conn); goto rollback; @@ -6310,9 +6141,9 @@ bool blocks_add(PGconn *conn, int32_t height, char *blockhash, "blockhash=$1 and "EDDB"=$2"; } - res = PQexecParams(conn, ins, par, NULL, (const char **)params, NULL, NULL, 0, CKPQ_WRITE); - rescode = PQresultStatus(res); - PQclear(res); + res = CKPQExecParams(conn, ins, par, NULL, (const char **)params, NULL, NULL, 0, CKPQ_WRITE); + rescode = CKPQResultStatus(res); + CKPQClear(res); if (!PGOK(rescode)) { PGLOGERR("Insert", rescode, conn); goto rollback; @@ -6371,18 +6202,14 @@ bool blocks_add(PGconn *conn, int32_t height, char *blockhash, ok = true; rollback: - if (ok) - res = PQexec(conn, "Commit", CKPQ_WRITE); - else - res = PQexec(conn, "Rollback", CKPQ_WRITE); - PQclear(res); + CKPQEnd(conn, ok); + unparam: for (n = 0; n < par; n++) free(params[n]); flail: - if (conned) - PQfinish(conn); + conned = CKPQDisco(&conn, conned); K_RLOCK(workinfo_free); K_WLOCK(blocks_free); @@ -6489,11 +6316,11 @@ bool blocks_fill(PGconn *conn) "diffacc,diffinv,shareacc,shareinv,elapsed,statsconfirmed" HISTORYDATECONTROL " from blocks"; - res = PQexec(conn, sel, CKPQ_READ); - rescode = PQresultStatus(res); + res = CKPQExec(conn, sel, CKPQ_READ); + rescode = CKPQResultStatus(res); if (!PGOK(rescode)) { PGLOGERR("Select", rescode, conn); - PQclear(res); + CKPQClear(res); return false; } @@ -6501,7 +6328,7 @@ bool blocks_fill(PGconn *conn) if (n != (fields + HISTORYDATECOUNT)) { LOGERR("%s(): Invalid field count - should be %d, but is %d", __func__, fields + HISTORYDATECOUNT, n); - PQclear(res); + CKPQClear(res); return false; } @@ -6653,7 +6480,7 @@ bool blocks_fill(PGconn *conn) } K_WUNLOCK(blocks_free); - PQclear(res); + CKPQClear(res); if (ok) { LOGDEBUG("%s(): built", __func__); @@ -6734,9 +6561,9 @@ bool miningpayouts_add(PGconn *conn, bool add, K_ITEM *mp_item, params[par++] = tv_to_buf((tv_t *)&default_expiry, NULL, 0); PARCHKVAL(par, 4, params); - res = PQexecParams(conn, upd, par, NULL, (const char **)params, NULL, NULL, 0, CKPQ_WRITE); - rescode = PQresultStatus(res); - PQclear(res); + res = CKPQExecParams(conn, upd, par, NULL, (const char **)params, NULL, NULL, 0, CKPQ_WRITE); + rescode = CKPQResultStatus(res); + CKPQClear(res); if (!PGOK(rescode)) { PGLOGERR("Update", rescode, conn); goto rollback; @@ -6765,8 +6592,9 @@ bool miningpayouts_add(PGconn *conn, bool add, K_ITEM *mp_item, "(payoutid,userid,diffacc,amount" HISTORYDATECONTROL ") values (" PQPARAM9 ")"; - res = PQexecParams(conn, ins, par, NULL, (const char **)params, NULL, NULL, 0, CKPQ_WRITE); - rescode = PQresultStatus(res); + res = CKPQExecParams(conn, ins, par, NULL, (const char **)params, NULL, NULL, 0, CKPQ_WRITE); + rescode = CKPQResultStatus(res); + CKPQClear(res); if (!PGOK(rescode)) { PGLOGERR("Insert", rescode, conn); goto unparam; @@ -6781,7 +6609,7 @@ unparam: for (n = 0; n < par; n++) free(params[n]); - CKPQDisco(&conn, conned); + conned = CKPQDisco(&conn, conned); if (!already) miningpayouts_add_ram(ok, mp_item, *old_mp_item, cd); @@ -6811,27 +6639,22 @@ bool miningpayouts_fill(PGconn *conn) "payoutid,userid,diffacc,amount" HISTORYDATECONTROL " from miningpayouts"; - res = PQexec(conn, "Begin", CKPQ_READ); - rescode = PQresultStatus(res); - PQclear(res); - if (!PGOK(rescode)) { - PGLOGERR("Begin", rescode, conn); + if (!CKPQBegin(conn)) return false; - } if (exclusive_db) { - res = PQexec(conn, "Lock table miningpayouts in access exclusive mode", CKPQ_READ); - rescode = PQresultStatus(res); - PQclear(res); + res = CKPQExec(conn, "Lock table miningpayouts in access exclusive mode", CKPQ_READ); + rescode = CKPQResultStatus(res); + CKPQClear(res); if (!PGOK(rescode)) { PGLOGERR("Lock", rescode, conn); goto flail; } } - res = PQexec(conn, sel, CKPQ_READ); - rescode = PQresultStatus(res); - PQclear(res); + res = CKPQExec(conn, sel, CKPQ_READ); + rescode = CKPQResultStatus(res); + CKPQClear(res); if (!PGOK(rescode)) { PGLOGERR("Declare", rescode, conn); goto flail; @@ -6839,11 +6662,11 @@ bool miningpayouts_fill(PGconn *conn) LOGDEBUG("%s(): fetching ...", __func__); - res = PQexec(conn, "fetch 1 in mp", CKPQ_READ); - rescode = PQresultStatus(res); + res = CKPQExec(conn, "fetch 1 in mp", CKPQ_READ); + rescode = CKPQResultStatus(res); if (!PGOK(rescode)) { PGLOGERR("Fetch first", rescode, conn); - PQclear(res); + CKPQClear(res); goto flail; } @@ -6851,7 +6674,7 @@ bool miningpayouts_fill(PGconn *conn) if (n != (fields + HISTORYDATECOUNT)) { LOGERR("%s(): Invalid field count - should be %d, but is %d", __func__, fields + HISTORYDATECOUNT, n); - PQclear(res); + CKPQClear(res); goto flail; } @@ -6905,9 +6728,9 @@ bool miningpayouts_fill(PGconn *conn) tick(); n++; } - PQclear(res); - res = PQexec(conn, "fetch 9999 in mp", CKPQ_READ); - rescode = PQresultStatus(res); + CKPQClear(res); + res = CKPQExec(conn, "fetch 9999 in mp", CKPQ_READ); + rescode = CKPQResultStatus(res); if (!PGOK(rescode)) { PGLOGERR("Fetch next", rescode, conn); ok = false; @@ -6918,11 +6741,10 @@ bool miningpayouts_fill(PGconn *conn) k_add_head(miningpayouts_free, item); K_WUNLOCK(miningpayouts_free); - PQclear(res); + CKPQClear(res); flail: - res = PQexec(conn, "Commit", CKPQ_READ); - PQclear(res); + CKPQCommit(conn); if (ok) { LOGDEBUG("%s(): built", __func__); LOGWARNING("%s(): fetched %d miningpayout records", __func__, n); @@ -7008,9 +6830,9 @@ bool payouts_add(PGconn *conn, bool add, K_ITEM *p_item, K_ITEM **old_p_item, params[par++] = tv_to_buf((tv_t *)&default_expiry, NULL, 0); PARCHKVAL(par, 3, params); - res = PQexecParams(conn, upd, par, NULL, (const char **)params, NULL, NULL, 0, CKPQ_WRITE); - rescode = PQresultStatus(res); - PQclear(res); + res = CKPQExecParams(conn, upd, par, NULL, (const char **)params, NULL, NULL, 0, CKPQ_WRITE); + rescode = CKPQResultStatus(res); + CKPQClear(res); if (!PGOK(rescode)) { PGLOGERR("Update", rescode, conn); goto rollback; @@ -7060,8 +6882,9 @@ bool payouts_add(PGconn *conn, bool add, K_ITEM *p_item, K_ITEM **old_p_item, "lastshareacc,stats" HISTORYDATECONTROL ") values (" PQPARAM18 ")"; - res = PQexecParams(conn, ins, par, NULL, (const char **)params, NULL, NULL, 0, CKPQ_WRITE); - rescode = PQresultStatus(res); + res = CKPQExecParams(conn, ins, par, NULL, (const char **)params, NULL, NULL, 0, CKPQ_WRITE); + rescode = CKPQResultStatus(res); + CKPQClear(res); if (!PGOK(rescode)) { PGLOGERR("Insert", rescode, conn); goto unparam; @@ -7076,7 +6899,7 @@ unparam: for (n = 0; n < par; n++) free(params[n]); - CKPQDisco(&conn, conned); + conned = CKPQDisco(&conn, conned); if (!already) payouts_add_ram(ok, p_item, *old_p_item, cd); @@ -7119,7 +6942,6 @@ K_ITEM *payouts_full_expire(PGconn *conn, int64_t payoutid, tv_t *now, bool lock } conned = CKPQConn(&conn); - begun = CKPQBegin(conn); if (!begun) goto matane; @@ -7131,8 +6953,8 @@ K_ITEM *payouts_full_expire(PGconn *conn, int64_t payoutid, tv_t *now, bool lock params[par++] = tv_to_buf((tv_t *)&default_expiry, NULL, 0); PARCHKVAL(par, 3, params); - res = PQexecParams(conn, upd, par, NULL, (const char **)params, NULL, NULL, 0, CKPQ_WRITE); - rescode = PQresultStatus(res); + res = CKPQExecParams(conn, upd, par, NULL, (const char **)params, NULL, NULL, 0, CKPQ_WRITE); + rescode = CKPQResultStatus(res); if (PGOK(rescode)) { tuples = PQcmdTuples(res); if (tuples && *tuples) { @@ -7141,11 +6963,12 @@ K_ITEM *payouts_full_expire(PGconn *conn, int64_t payoutid, tv_t *now, bool lock LOGERR("%s() updated payouts should be 1" " but updated=%d", __func__, po_upd); + CKPQClear(res); goto matane; } } } - PQclear(res); + CKPQClear(res); if (!PGOK(rescode)) { PGLOGERR("Update payouts", rescode, conn); goto matane; @@ -7161,14 +6984,14 @@ K_ITEM *payouts_full_expire(PGconn *conn, int64_t payoutid, tv_t *now, bool lock params[par++] = tv_to_buf((tv_t *)&default_expiry, NULL, 0); PARCHKVAL(par, 3, params); - res = PQexecParams(conn, upd, par, NULL, (const char **)params, NULL, NULL, 0, CKPQ_WRITE); - rescode = PQresultStatus(res); + res = CKPQExecParams(conn, upd, par, NULL, (const char **)params, NULL, NULL, 0, CKPQ_WRITE); + rescode = CKPQResultStatus(res); if (PGOK(rescode)) { tuples = PQcmdTuples(res); if (tuples && *tuples) mp_upd = atoi(tuples); } - PQclear(res); + CKPQClear(res); if (!PGOK(rescode)) { PGLOGERR("Update miningpayouts", rescode, conn); goto matane; @@ -7184,14 +7007,14 @@ K_ITEM *payouts_full_expire(PGconn *conn, int64_t payoutid, tv_t *now, bool lock params[par++] = tv_to_buf((tv_t *)&default_expiry, NULL, 0); PARCHKVAL(par, 3, params); - res = PQexecParams(conn, upd, par, NULL, (const char **)params, NULL, NULL, 0, CKPQ_WRITE); - rescode = PQresultStatus(res); + res = CKPQExecParams(conn, upd, par, NULL, (const char **)params, NULL, NULL, 0, CKPQ_WRITE); + rescode = CKPQResultStatus(res); if (PGOK(rescode)) { tuples = PQcmdTuples(res); if (tuples && *tuples) pm_upd = atoi(tuples); } - PQclear(res); + CKPQClear(res); if (!PGOK(rescode)) { PGLOGERR("Update payments", rescode, conn); goto matane; @@ -7317,7 +7140,7 @@ matane: K_WUNLOCK(payouts_free); } - CKPQDisco(&conn, conned); + conned = CKPQDisco(&conn, conned); if (lock) K_WUNLOCK(process_pplns_free); @@ -7352,11 +7175,11 @@ bool payouts_fill(PGconn *conn) "elapsed,status,diffwanted,diffused,shareacc,lastshareacc,stats" HISTORYDATECONTROL " from payouts"; - res = PQexec(conn, sel, CKPQ_READ); - rescode = PQresultStatus(res); + res = CKPQExec(conn, sel, CKPQ_READ); + rescode = CKPQResultStatus(res); if (!PGOK(rescode)) { PGLOGERR("Select", rescode, conn); - PQclear(res); + CKPQClear(res); return false; } @@ -7364,7 +7187,7 @@ bool payouts_fill(PGconn *conn) if (n != (fields + HISTORYDATECOUNT)) { LOGERR("%s(): Invalid field count - should be %d, but is %d", __func__, fields + HISTORYDATECOUNT, n); - PQclear(res); + CKPQClear(res); return false; } @@ -7486,7 +7309,7 @@ bool payouts_fill(PGconn *conn) } K_WUNLOCK(payouts_free); - PQclear(res); + CKPQClear(res); if (ok) { LOGDEBUG("%s(): built", __func__); @@ -8011,13 +7834,9 @@ bool poolstats_add(PGconn *conn, bool store, INTRANSIENT *in_poolinstance, "hashrate5m,hashrate1hr,hashrate24hr" SIMPLEDATECONTROL ") values (" PQPARAM12 ")"; - if (!conn) { - conn = dbconnect(); - conned = true; - } - - res = PQexecParams(conn, ins, par, NULL, (const char **)params, NULL, NULL, 0, CKPQ_WRITE); - rescode = PQresultStatus(res); + conned = CKPQConn(&conn); + res = CKPQExecParams(conn, ins, par, NULL, (const char **)params, NULL, NULL, 0, CKPQ_WRITE); + rescode = CKPQResultStatus(res); if (!PGOK(rescode)) { bool show_msg = true; char *code; @@ -8037,9 +7856,8 @@ bool poolstats_add(PGconn *conn, bool store, INTRANSIENT *in_poolinstance, ok = true; unparam: if (store) { - PQclear(res); - if (conned) - PQfinish(conn); + CKPQClear(res); + conned = CKPQDisco(&conn, conned); for (n = 0; n < par; n++) free(params[n]); } @@ -8116,11 +7934,11 @@ bool poolstats_fill(PGconn *conn) " from poolstats where "CDDB">"); APPEND_REALLOC(sel, off, len, stamp); - res = PQexec(conn, sel, CKPQ_READ); - rescode = PQresultStatus(res); + res = CKPQExec(conn, sel, CKPQ_READ); + rescode = CKPQResultStatus(res); if (!PGOK(rescode)) { PGLOGERR("Select", rescode, conn); - PQclear(res); + CKPQClear(res); ok = false; goto clean; } @@ -8129,7 +7947,7 @@ bool poolstats_fill(PGconn *conn) if (n != (fields + SIMPLEDATECOUNT)) { LOGERR("%s(): Invalid field count - should be %d, but is %d", __func__, fields + SIMPLEDATECOUNT, n); - PQclear(res); + CKPQClear(res); ok = false; goto clean; } @@ -8211,7 +8029,7 @@ bool poolstats_fill(PGconn *conn) k_add_head(poolstats_free, item); K_WUNLOCK(poolstats_free); - PQclear(res); + CKPQClear(res); if (ok) { LOGDEBUG("%s(): built", __func__); @@ -8463,13 +8281,10 @@ bool markersummary_add(PGconn *conn, K_ITEM *ms_item, char *by, char *code, row->diffacc); FREENULL(st); - if (!conn) { - conn = dbconnect(); - conned = true; - } - - res = PQexecParams(conn, ins, par, NULL, (const char **)params, NULL, NULL, 0, CKPQ_WRITE); - rescode = PQresultStatus(res); + conned = CKPQConn(&conn); + res = CKPQExecParams(conn, ins, par, NULL, (const char **)params, NULL, NULL, 0, CKPQ_WRITE); + rescode = CKPQResultStatus(res); + CKPQClear(res); if (!PGOK(rescode)) { PGLOGERR("Insert", rescode, conn); goto unparam; @@ -8477,9 +8292,7 @@ bool markersummary_add(PGconn *conn, K_ITEM *ms_item, char *by, char *code, ok = true; unparam: - PQclear(res); - if (conned) - PQfinish(conn); + conned = CKPQDisco(&conn, conned); for (n = 0; n < par; n++) free(params[n]); @@ -8595,27 +8408,22 @@ bool markersummary_fill(PGconn *conn) STRNCPY(tickbuf, TICK_PREFIX"ms 0"); cr_msg(false, tickbuf); - res = PQexec(conn, "Begin", CKPQ_READ); - rescode = PQresultStatus(res); - PQclear(res); - if (!PGOK(rescode)) { - PGLOGERR("Begin", rescode, conn); + if (!CKPQBegin(conn)) return false; - } if (exclusive_db) { - res = PQexec(conn, "Lock table markersummary in access exclusive mode", CKPQ_READ); - rescode = PQresultStatus(res); - PQclear(res); + res = CKPQExec(conn, "Lock table markersummary in access exclusive mode", CKPQ_READ); + rescode = CKPQResultStatus(res); + CKPQClear(res); if (!PGOK(rescode)) { PGLOGERR("Lock", rescode, conn); goto flail; } } - res = PQexecParams(conn, sel, par, NULL, (const char **)params, NULL, NULL, 0, CKPQ_READ); - rescode = PQresultStatus(res); - PQclear(res); + res = CKPQExecParams(conn, sel, par, NULL, (const char **)params, NULL, NULL, 0, CKPQ_READ); + rescode = CKPQResultStatus(res); + CKPQClear(res); if (!PGOK(rescode)) { PGLOGERR("Declare", rescode, conn); goto flail; @@ -8623,11 +8431,11 @@ bool markersummary_fill(PGconn *conn) LOGDEBUG("%s(): fetching ...", __func__); - res = PQexec(conn, "fetch 1 in ws", CKPQ_READ); - rescode = PQresultStatus(res); + res = CKPQExec(conn, "fetch 1 in ws", CKPQ_READ); + rescode = CKPQResultStatus(res); if (!PGOK(rescode)) { PGLOGERR("Fetch first", rescode, conn); - PQclear(res); + CKPQClear(res); goto flail; } @@ -8635,7 +8443,7 @@ bool markersummary_fill(PGconn *conn) if (n != (fields + MODIFYDATECOUNT)) { LOGERR("%s(): Invalid field count - should be %d, but is %d", __func__, fields + MODIFYDATECOUNT, n); - PQclear(res); + CKPQClear(res); goto flail; } @@ -8804,9 +8612,9 @@ bool markersummary_fill(PGconn *conn) } K_WUNLOCK(userinfo_free); K_RUNLOCK(workmarkers_free); - PQclear(res); - res = PQexec(conn, "fetch 9999 in ws", CKPQ_READ); - rescode = PQresultStatus(res); + CKPQClear(res); + res = CKPQExec(conn, "fetch 9999 in ws", CKPQ_READ); + rescode = CKPQResultStatus(res); if (!PGOK(rescode)) { PGLOGERR("Fetch next", rescode, conn); ok = false; @@ -8821,11 +8629,10 @@ bool markersummary_fill(PGconn *conn) p_n = markersummary_pool_store->count; K_WUNLOCK(markersummary_free); - PQclear(res); + CKPQClear(res); flail: - res = PQexec(conn, "Commit", CKPQ_READ); - PQclear(res); + CKPQCommit(conn); for (i = 0; i < par; i++) free(params[i]); par = 0; @@ -8895,13 +8702,10 @@ bool keysummary_add(PGconn *conn, K_ITEM *ks_item, char *by, char *code, row->diffacc); FREENULL(st); - if (!conn) { - conn = dbconnect(); - conned = true; - } - - res = PQexecParams(conn, ins, par, NULL, (const char **)params, NULL, NULL, 0, CKPQ_WRITE); - rescode = PQresultStatus(res); + conned = CKPQConn(&conn); + res = CKPQExecParams(conn, ins, par, NULL, (const char **)params, NULL, NULL, 0, CKPQ_WRITE); + rescode = CKPQResultStatus(res); + CKPQClear(res); if (!PGOK(rescode)) { PGLOGERR("Insert", rescode, conn); /* Don't fail on a duplicate during key_update @@ -8913,9 +8717,7 @@ bool keysummary_add(PGconn *conn, K_ITEM *ks_item, char *by, char *code, ok = true; unparam: - PQclear(res); - if (conned) - PQfinish(conn); + conned = CKPQDisco(&conn, conned); for (n = 0; n < par; n++) free(params[n]); @@ -8972,20 +8774,11 @@ bool _workmarkers_process(PGconn *conn, bool already, bool add, LOGDEBUG("%s(): updating old", __func__); DATA_WORKMARKERS(oldworkmarkers, old_wm_item); - if (!conn) { - conn = dbconnect(); - conned = true; - } + conned = CKPQConn(&conn); if (!already) { - res = PQexec(conn, "Begin", CKPQ_WRITE); - rescode = PQresultStatus(res); - PQclear(res); - if (!PGOK(rescode)) { - PGLOGERR("Begin", rescode, conn); + begun = CKPQBegin(conn); + if (!begun) goto unparam; - } - - begun = true; } upd = "update workmarkers set "EDDB"=$1 where markerid=$2" @@ -8996,9 +8789,9 @@ bool _workmarkers_process(PGconn *conn, bool already, bool add, params[par++] = tv_to_buf((tv_t *)&default_expiry, NULL, 0); PARCHKVAL(par, 3, params); - res = PQexecParams(conn, upd, par, NULL, (const char **)params, NULL, NULL, 0, CKPQ_WRITE); - rescode = PQresultStatus(res); - PQclear(res); + res = CKPQExecParams(conn, upd, par, NULL, (const char **)params, NULL, NULL, 0, CKPQ_WRITE); + rescode = CKPQResultStatus(res); + CKPQClear(res); if (!PGOK(rescode)) { PGLOGERR("Update", rescode, conn); goto rollback; @@ -9035,20 +8828,11 @@ bool _workmarkers_process(PGconn *conn, bool already, bool add, DATA_WORKMARKERS(row, wm_item); bzero(row, sizeof(*row)); - if (conn == NULL) { - conn = dbconnect(); - conned = true; - } - + conned = CKPQConn(&conn); if (!already && !begun) { - res = PQexec(conn, "Begin", CKPQ_WRITE); - rescode = PQresultStatus(res); - PQclear(res); - if (!PGOK(rescode)) { - PGLOGERR("Begin", rescode, conn); + begun = CKPQBegin(conn); + if (!begun) goto unparam; - } - begun = true; } if (old_wm_item) @@ -9086,9 +8870,9 @@ bool _workmarkers_process(PGconn *conn, bool already, bool add, HISTORYDATEPARAMS(params, par, row); PARCHK(par, params); - res = PQexecParams(conn, ins, par, NULL, (const char **)params, NULL, NULL, 0, CKPQ_WRITE); - rescode = PQresultStatus(res); - PQclear(res); + res = CKPQExecParams(conn, ins, par, NULL, (const char **)params, NULL, NULL, 0, CKPQ_WRITE); + rescode = CKPQResultStatus(res); + CKPQClear(res); if (!PGOK(rescode)) { PGLOGERR("Insert", rescode, conn); goto rollback; @@ -9098,21 +8882,14 @@ bool _workmarkers_process(PGconn *conn, bool already, bool add, ok = true; rollback: - if (begun) { - if (ok) - res = PQexec(conn, "Commit", CKPQ_WRITE); - else - res = PQexec(conn, "Rollback", CKPQ_WRITE); + if (begun) + CKPQEnd(conn, ok); - PQclear(res); - } unparam: + conned = CKPQDisco(&conn, conned); for (n = 0; n < par; n++) free(params[n]); - if (conned) - PQfinish(conn); - if (!ok) { if (wm_item) { K_WLOCK(workmarkers_free); @@ -9169,19 +8946,19 @@ bool workmarkers_fill(PGconn *conn) par = 0; params[par++] = bigint_to_buf(dbload_workinfoid_start, NULL, 0); PARCHK(par, params); - res = PQexecParams(conn, sel, par, NULL, (const char **)params, NULL, NULL, 0, CKPQ_READ); + res = CKPQExecParams(conn, sel, par, NULL, (const char **)params, NULL, NULL, 0, CKPQ_READ); } else { sel = "select " "markerid,poolinstance,workinfoidend,workinfoidstart," "description,status" HISTORYDATECONTROL " from workmarkers"; - res = PQexec(conn, sel, CKPQ_READ); + res = CKPQExec(conn, sel, CKPQ_READ); } - rescode = PQresultStatus(res); + rescode = CKPQResultStatus(res); if (!PGOK(rescode)) { PGLOGERR("Select", rescode, conn); - PQclear(res); + CKPQClear(res); return false; } @@ -9189,7 +8966,7 @@ bool workmarkers_fill(PGconn *conn) if (n != (fields + HISTORYDATECOUNT)) { LOGERR("%s(): Invalid field count - should be %d, but is %d", __func__, fields + HISTORYDATECOUNT, n); - PQclear(res); + CKPQClear(res); return false; } @@ -9303,7 +9080,7 @@ bool workmarkers_fill(PGconn *conn) } K_WUNLOCK(workmarkers_free); - PQclear(res); + CKPQClear(res); for (i = 0; i < par; i++) free(params[i]); par = 0; @@ -9345,19 +9122,10 @@ bool _marks_process(PGconn *conn, bool add, char *poolinstance, LOGDEBUG("%s(): updating old", __func__); DATA_MARKS(oldmarks, old_m_item); - if (!conn) { - conn = dbconnect(); - conned = true; - } - res = PQexec(conn, "Begin", CKPQ_WRITE); - rescode = PQresultStatus(res); - PQclear(res); - if (!PGOK(rescode)) { - PGLOGERR("Begin", rescode, conn); + conned = CKPQConn(&conn); + begun = CKPQBegin(conn); + if (!begun) goto unparam; - } - - begun = true; upd = "update marks set "EDDB"=$1 where workinfoid=$2" " and "EDDB"=$3"; @@ -9367,9 +9135,9 @@ bool _marks_process(PGconn *conn, bool add, char *poolinstance, params[par++] = tv_to_buf((tv_t *)&default_expiry, NULL, 0); PARCHKVAL(par, 3, params); - res = PQexecParams(conn, upd, par, NULL, (const char **)params, NULL, NULL, 0, CKPQ_WRITE); - rescode = PQresultStatus(res); - PQclear(res); + res = CKPQExecParams(conn, upd, par, NULL, (const char **)params, NULL, NULL, 0, CKPQ_WRITE); + rescode = CKPQResultStatus(res); + CKPQClear(res); if (!PGOK(rescode)) { PGLOGERR("Update", rescode, conn); goto rollback; @@ -9426,25 +9194,16 @@ bool _marks_process(PGconn *conn, bool add, char *poolinstance, HISTORYDATEPARAMS(params, par, row); PARCHK(par, params); - if (conn == NULL) { - conn = dbconnect(); - conned = true; - } - + conned = CKPQConn(&conn); if (!begun) { - res = PQexec(conn, "Begin", CKPQ_WRITE); - rescode = PQresultStatus(res); - PQclear(res); - if (!PGOK(rescode)) { - PGLOGERR("Begin", rescode, conn); + begun = CKPQBegin(conn); + if (!begun) goto unparam; - } - begun = true; } - res = PQexecParams(conn, ins, par, NULL, (const char **)params, NULL, NULL, 0, CKPQ_WRITE); - rescode = PQresultStatus(res); - PQclear(res); + res = CKPQExecParams(conn, ins, par, NULL, (const char **)params, NULL, NULL, 0, CKPQ_WRITE); + rescode = CKPQResultStatus(res); + CKPQClear(res); if (!PGOK(rescode)) { PGLOGERR("Insert", rescode, conn); goto rollback; @@ -9453,21 +9212,14 @@ bool _marks_process(PGconn *conn, bool add, char *poolinstance, ok = true; rollback: - if (begun) { - if (ok) - res = PQexec(conn, "Commit", CKPQ_WRITE); - else - res = PQexec(conn, "Rollback", CKPQ_WRITE); + if (begun) + CKPQEnd(conn, ok); - PQclear(res); - } unparam: + conned = CKPQDisco(&conn, conned); for (n = 0; n < par; n++) free(params[n]); - if (conned) - PQfinish(conn); - K_WLOCK(marks_free); if (!ok) { if (m_item) { @@ -9509,11 +9261,11 @@ bool marks_fill(PGconn *conn) "poolinstance,workinfoid,description,extra,marktype,status" HISTORYDATECONTROL " from marks"; - res = PQexec(conn, sel, CKPQ_READ); - rescode = PQresultStatus(res); + res = CKPQExec(conn, sel, CKPQ_READ); + rescode = CKPQResultStatus(res); if (!PGOK(rescode)) { PGLOGERR("Select", rescode, conn); - PQclear(res); + CKPQClear(res); return false; } @@ -9521,7 +9273,7 @@ bool marks_fill(PGconn *conn) if (n != (fields + HISTORYDATECOUNT)) { LOGERR("%s(): Invalid field count - should be %d, but is %d", __func__, fields + HISTORYDATECOUNT, n); - PQclear(res); + CKPQClear(res); return false; } @@ -9591,7 +9343,7 @@ bool marks_fill(PGconn *conn) } K_WUNLOCK(marks_free); - PQclear(res); + CKPQClear(res); if (ok) { LOGDEBUG("%s(): built", __func__); @@ -9616,11 +9368,11 @@ bool check_db_version(PGconn *conn) LOGDEBUG("%s(): select", __func__); sel = "select version() as pgv,* from version;"; - res = PQexec(conn, sel, CKPQ_READ); - rescode = PQresultStatus(res); + res = CKPQExec(conn, sel, CKPQ_READ); + rescode = CKPQResultStatus(res); if (!PGOK(rescode)) { PGLOGEMERG("Select", rescode, conn); - PQclear(res); + CKPQClear(res); return false; } @@ -9628,7 +9380,7 @@ bool check_db_version(PGconn *conn) if (n != fields) { LOGEMERG("%s(): Invalid field count - should be %d, but is %d", __func__, fields, n); - PQclear(res); + CKPQClear(res); return false; } @@ -9636,7 +9388,7 @@ bool check_db_version(PGconn *conn) if (n != 1) { LOGEMERG("%s(): Invalid record count - should be %d, but is %d", __func__, 1, n); - PQclear(res); + CKPQClear(res); return false; } @@ -9644,14 +9396,14 @@ bool check_db_version(PGconn *conn) PQ_GET_FLD(res, 0, "vlock", field, ok); if (!ok) { LOGEMERG("%s(): Missing field vlock", __func__); - PQclear(res); + CKPQClear(res); return false; } if (strcmp(field, DB_VLOCK)) { LOGEMERG("%s(): incorrect vlock '%s' - should be '%s'", __func__, field, DB_VLOCK); - PQclear(res); + CKPQClear(res); return false; } @@ -9659,14 +9411,14 @@ bool check_db_version(PGconn *conn) PQ_GET_FLD(res, 0, "version", field, ok); if (!ok) { LOGEMERG("%s(): Missing field version", __func__); - PQclear(res); + CKPQClear(res); return false; } if (strcmp(field, DB_VERSION)) { LOGEMERG("%s(): incorrect version '%s' - should be '%s'", __func__, field, DB_VERSION); - PQclear(res); + CKPQClear(res); return false; } @@ -9676,7 +9428,7 @@ bool check_db_version(PGconn *conn) else pgv = strdup("Failed to get postgresql version information"); - PQclear(res); + CKPQClear(res); LOGWARNING("%s(): DB version (%s) correct (CKDB V%s)", __func__, DB_VERSION, CKDB_VERSION); @@ -9732,13 +9484,10 @@ char *cmd_newid(PGconn *conn, char *cmd, char *id, tv_t *now, char *by, ins = "insert into idcontrol " "(idname,lastid" MODIFYDATECONTROL ") values (" PQPARAM10 ")"; - if (!conn) { - conn = dbconnect(); - conned = true; - } - - res = PQexecParams(conn, ins, par, NULL, (const char **)params, NULL, NULL, 0, CKPQ_WRITE); - rescode = PQresultStatus(res); + conned = CKPQConn(&conn); + res = CKPQExecParams(conn, ins, par, NULL, (const char **)params, NULL, NULL, 0, CKPQ_WRITE); + rescode = CKPQResultStatus(res); + CKPQClear(res); if (!PGOK(rescode)) { PGLOGERR("Insert", rescode, conn); goto foil; @@ -9746,9 +9495,7 @@ char *cmd_newid(PGconn *conn, char *cmd, char *id, tv_t *now, char *by, ok = true; foil: - PQclear(res); - if (conned) - PQfinish(conn); + conned = CKPQDisco(&conn, conned); for (n = 0; n < par; n++) free(params[n]); From 11bbc5970d2a2bcf97d274ff0a5e67002d4472c5 Mon Sep 17 00:00:00 2001 From: kanoi Date: Thu, 6 Oct 2016 18:50:57 +1100 Subject: [PATCH 14/33] ckdb - remaining begin statements --- src/ckdb.h | 2 +- src/ckdb_dbio.c | 23 +++-------------------- 2 files changed, 4 insertions(+), 21 deletions(-) diff --git a/src/ckdb.h b/src/ckdb.h index 05a77bca..9c414fca 100644 --- a/src/ckdb.h +++ b/src/ckdb.h @@ -58,7 +58,7 @@ #define DB_VLOCK "1" #define DB_VERSION "1.0.7" -#define CKDB_VERSION DB_VERSION"-2.600" +#define CKDB_VERSION DB_VERSION"-2.601" #define WHERE_FFL " - from %s %s() line %d" #define WHERE_FFL_HERE __FILE__, __func__, __LINE__ diff --git a/src/ckdb_dbio.c b/src/ckdb_dbio.c index d23ed730..de1d9996 100644 --- a/src/ckdb_dbio.c +++ b/src/ckdb_dbio.c @@ -615,14 +615,8 @@ bool users_update(PGconn *conn, K_ITEM *u_item, char *oldhash, PARCHKVAL(par, 3, params); conned = CKPQConn(&conn); - // Beginning of a write txn - res = CKPQExec(conn, "Begin", CKPQ_WRITE); - rescode = CKPQResultStatus(res); - CKPQClear(res); - if (!PGOK(rescode)) { - PGLOGERR("Begin", rescode, conn); + if (!CKPQBegin(conn)) goto unparam; - } res = CKPQExecParams(conn, upd, par, NULL, (const char **)params, NULL, NULL, 0, CKPQ_WRITE); rescode = CKPQResultStatus(res); @@ -873,14 +867,8 @@ bool users_replace(PGconn *conn, K_ITEM *u_item, K_ITEM *old_u_item, char *by, PARCHKVAL(par, 3, params); conned = CKPQConn(&conn); - // Beginning of a write txn - res = CKPQExec(conn, "Begin", CKPQ_WRITE); - rescode = CKPQResultStatus(res); - CKPQClear(res); - if (!PGOK(rescode)) { - PGLOGERR("Begin", rescode, conn); + if (!CKPQBegin(conn)) goto unparam; - } res = CKPQExecParams(conn, upd, par, NULL, (const char **)params, NULL, NULL, 0, CKPQ_WRITE); rescode = CKPQResultStatus(res); @@ -1687,13 +1675,8 @@ bool workers_update(PGconn *conn, K_ITEM *item, char *difficultydefault, PARCHKVAL(par, 3, params); conned = CKPQConn(&conn); - res = CKPQExec(conn, "Begin", CKPQ_WRITE); - rescode = CKPQResultStatus(res); - CKPQClear(res); - if (!PGOK(rescode)) { - PGLOGERR("Begin", rescode, conn); + if (!CKPQBegin(conn)) goto unparam; - } res = CKPQExecParams(conn, upd, par, NULL, (const char **)params, NULL, NULL, 0, CKPQ_WRITE); rescode = CKPQResultStatus(res); From 9b8aaaf038f958510f4c017ee29ead73c0e15876 Mon Sep 17 00:00:00 2001 From: kanoi Date: Fri, 7 Oct 2016 16:19:38 +1100 Subject: [PATCH 15/33] ckdb - allow changing CULL_TRANSFER --- src/ckdb.c | 9 +++++---- src/ckdb.h | 12 ++++++++---- src/ckdb_dbio.c | 21 +++++++++++++++++++++ 3 files changed, 34 insertions(+), 8 deletions(-) diff --git a/src/ckdb.c b/src/ckdb.c index 4532993e..9e78c097 100644 --- a/src/ckdb.c +++ b/src/ckdb.c @@ -624,6 +624,7 @@ K_STORE *heartbeatqueue_store; // TRANSFER K_LIST *transfer_free; +int cull_transfer = CULL_TRANSFER; // SEQSET K_LIST *seqset_free; @@ -2059,7 +2060,7 @@ static void alloc_storage() transfer_free = k_new_list_cull(Transfer, sizeof(TRANSFER), ALLOC_TRANSFER, LIMIT_TRANSFER, true, - CULL_TRANSFER); + cull_transfer); transfer_free->dsp_func = dsp_transfer; users_free = k_new_list("Users", sizeof(USERS), @@ -2277,6 +2278,9 @@ static void alloc_storage() userinfo_root = new_ktree(NULL, cmp_userinfo, userinfo_free); #if LOCK_CHECK + // Above all needed for optioncontrol_trigger() + DLPRIO(optioncontrol, 96); + DLPRIO(seqset, 91); DLPRIO(transfer, 90); @@ -2302,9 +2306,6 @@ static void alloc_storage() DLPRIO(userinfo, 50); - // Uses event_limits - DLPRIO(optioncontrol, 49); - // Needs to check users and ips and uses events_limits DLPRIO(events, 48); DLPRIO(ovents, 47); diff --git a/src/ckdb.h b/src/ckdb.h index 9c414fca..4aa172ff 100644 --- a/src/ckdb.h +++ b/src/ckdb.h @@ -58,7 +58,7 @@ #define DB_VLOCK "1" #define DB_VERSION "1.0.7" -#define CKDB_VERSION DB_VERSION"-2.601" +#define CKDB_VERSION DB_VERSION"-2.602" #define WHERE_FFL " - from %s %s() line %d" #define WHERE_FFL_HERE __FILE__, __func__, __LINE__ @@ -1563,14 +1563,18 @@ typedef struct transfer { // Suggest malloc use MMAP = largest under 2MB #define ALLOC_TRANSFER ((int)(2*1024*1024/sizeof(TRANSFER))) #define LIMIT_TRANSFER 0 -/* ALLOC_TRANSFER is ~14k, but it should only ever get this big during a reload - * so set it a bit above that */ -#define CULL_TRANSFER 32768 +// ALLOC_TRANSFER is ~14k, allocated often is 3 +#define CULL_TRANSFER (4 * ALLOC_TRANSFER) #define INIT_TRANSFER(_item) INIT_GENERIC(_item, transfer) #define DATA_TRANSFER(_var, _item) DATA_GENERIC(_var, _item, transfer, true) extern K_LIST *transfer_free; +/* Allow defining and adjusting it on a running system + * cull_limit is set to the optionvalue * ALLOC_TRANSFER */ +#define CULL_TRANSFER_NAME "CullTransfer" +extern int cull_transfer; + #define transfer_data(_item) _transfer_data(_item, WHERE_FFL_HERE) extern const char Transfer[]; diff --git a/src/ckdb_dbio.c b/src/ckdb_dbio.c index de1d9996..274f9d72 100644 --- a/src/ckdb_dbio.c +++ b/src/ckdb_dbio.c @@ -2786,12 +2786,33 @@ void oc_ips(OPTIONCONTROL *oc, const char *from) } } +void oc_trf(OPTIONCONTROL *oc, const char *from) +{ + int ct; + + ct = atoi(oc->optionvalue); + if (ct < 0 || ct > 64) { + LOGERR("%s(%s) ERR set cull_transfer ignored '%s' (%d)" + " must be 0..64", + from, __func__, oc->optionvalue, ct); + } else { + K_WLOCK(transfer_free); + // ct isn't the value, it's the multiplier + cull_transfer = ct * ALLOC_TRANSFER; + transfer_free->cull_limit = cull_transfer; + K_WUNLOCK(transfer_free); + LOGWARNING("%s(%s) set cull_transfer to %d->%d", + from, __func__, ct, cull_transfer); + } +} + OC_TRIGGER oc_trigger[] = { { SWITCH_STATE_NAME, true, oc_switch_state }, { DIFF_PERCENT_NAME, true, oc_diff_percent }, { OC_LIMITS, false, oc_event_limits }, { OC_OLIMITS, false, oc_ovent_limits }, { OC_IPS, false, oc_ips }, + { CULL_TRANSFER_NAME, true, oc_trf }, { NULL, 0, NULL } }; From 0fb49e96be77a5e3509323b4a7803aac64163255 Mon Sep 17 00:00:00 2001 From: kanoi Date: Fri, 7 Oct 2016 16:36:19 +1100 Subject: [PATCH 16/33] php - add more sorting options to page_ckp --- pool/page_ckp.php | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/pool/page_ckp.php b/pool/page_ckp.php index 57c0f34f..769d0d60 100644 --- a/pool/page_ckp.php +++ b/pool/page_ckp.php @@ -32,12 +32,12 @@ function dockp($data, $user) $pg .= ''; $pg .= "Name:<$r id=srtname data-sf=s0>"; $pg .= 'Initial'; - $pg .= 'Allocated'; + $pg .= "<$r id=srtalloc data-sf=r2>:Alloc"; $pg .= "<$r id=srtstore data-sf=r3>:In Store"; $pg .= "<$r id=srtram data-sf=r4>:RAM"; $pg .= "<$r id=srtram2 data-sf=r5>:RAM2"; - $pg .= 'Cull'; - $pg .= 'Limit'; + $pg .= "<$r id=srtcull data-sf=r6>:Cull"; + $pg .= "<$r id=srtlim data-sf=r7>:Limit"; $pg .= "\n"; if ($ans['STATUS'] == 'ok') { @@ -53,12 +53,12 @@ function dockp($data, $user) $pg .= ""; $pg .= "".$ans['name:'.$i].''; $pg .= ''.stnum($ans['initial:'.$i]).''; - $pg .= ''.stnum($ans['allocated:'.$i]).''; + $pg .= "".stnum($ans['allocated:'.$i]).''; $pg .= "".stnum($ans['instore:'.$i]).''; $pg .= "".stnum($ans['ram:'.$i]).''; $pg .= "".stnum($ans['ram2:'.$i]).''; - $pg .= ''.stnum($ans['cull:'.$i]).''; - $pg .= ''.stnum($ans['cull_limit:'.$i]).''; + $pg .= "".stnum($ans['cull:'.$i]).''; + $pg .= "".stnum($ans['cull_limit:'.$i]).''; $pg .= "\n"; } $pg .= ''; From eb25c7f94a1eb11db837d031b8e2783d04caae51 Mon Sep 17 00:00:00 2001 From: kanoi Date: Tue, 11 Oct 2016 20:45:59 +1100 Subject: [PATCH 17/33] ckdb - cache field numbers when loading tables --- src/ckdb.h | 46 ++-- src/ckdb_dbio.c | 642 +++++++++++++++++++++++++++++++----------------- 2 files changed, 442 insertions(+), 246 deletions(-) diff --git a/src/ckdb.h b/src/ckdb.h index 4aa172ff..a7ce6052 100644 --- a/src/ckdb.h +++ b/src/ckdb.h @@ -58,7 +58,7 @@ #define DB_VLOCK "1" #define DB_VERSION "1.0.7" -#define CKDB_VERSION DB_VERSION"-2.602" +#define CKDB_VERSION DB_VERSION"-2.700" #define WHERE_FFL " - from %s %s() line %d" #define WHERE_FFL_HERE __FILE__, __func__, __LINE__ @@ -70,6 +70,10 @@ #define STRINT(x) STRINT2(x) #define STRINT2(x) #x +// Same as above but name it what we are using it for +#define STRMACRO(x) STRMAC2(x) +#define STRMAC2(x) #x + // So they can fit into a 1 byte flag field #define TRUE_STR "Y" #define TRUE2_STR "T" @@ -407,19 +411,33 @@ extern int btc_timeout; // Lock access to the above variables so they can be changed extern cklock_t btc_lock; -#define EDDB "expirydate" -#define CDDB "createdate" -#define CDTRF CDDB -#define BYDB "createby" -#define BYTRF BYDB -#define CODEDB "createcode" -#define CODETRF CODEDB -#define INETDB "createinet" -#define INETTRF INETDB -#define MDDB "modifydate" -#define MBYDB "modifyby" -#define MCODEDB "modifycode" -#define MINETDB "modifyinet" +#define _EDDB expirydate +#define _CDDB createdate +#define _CDTRF _CDDB +#define _BYDB createby +#define _BYTRF _BYDB +#define _CODEDB createcode +#define _CODETRF _CODEDB +#define _INETDB createinet +#define _INETTRF _INETDB +#define _MDDB modifydate +#define _MBYDB modifyby +#define _MCODEDB modifycode +#define _MINETDB modifyinet + +#define EDDB STRMACRO(_EDDB) +#define CDDB STRMACRO(_CDDB) +#define CDTRF STRMACRO(_CDTRF) +#define BYDB STRMACRO(_BYDB) +#define BYTRF STRMACRO(_BYTRF) +#define CODEDB STRMACRO(_CODEDB) +#define CODETRF STRMACRO(_CODETRF) +#define INETDB STRMACRO(_INETDB) +#define INETTRF STRMACRO(_INETTRF) +#define MDDB STRMACRO(_MDDB) +#define MBYDB STRMACRO(_MBYDB) +#define MCODEDB STRMACRO(_MCODEDB) +#define MINETDB STRMACRO(_MINETDB) extern char *by_default; extern char *inet_default; diff --git a/src/ckdb_dbio.c b/src/ckdb_dbio.c index 274f9d72..296005a6 100644 --- a/src/ckdb_dbio.c +++ b/src/ckdb_dbio.c @@ -25,40 +25,74 @@ char *pqerrmsg(PGconn *conn) return buf; } -#define PQ_GET_FLD(__res, __row, __name, __fld, __ok) do { \ - int __col = PQfnumber(__res, __name); \ - if (__col == -1) { \ - LOGERR("%s(): Unknown field '%s' row %d", __func__, __name, __row); \ - __ok = false; \ - } else { \ - __fld = PQgetvalue(__res, __row, __col); \ - if (__fld == NULL) { \ - LOGERR("%s(): Invalid field '%s' or row %d", __func__, __name, __row); \ +/* *** WARNING: each field used in PQ_VAL_FLD() must (of course) have __num + * defined, but must also (re)initialise it to -1 with each call to PQexec + * N.B. it may? not be necessary to reinitialise them with each subsequent + * fetch, however I can't find that clearly documented anywhere, it may change + * in the future, and once per fetch reduces the number of times by a rather + * large factor of 9999 so it doesn't really matter repeating it each fetch */ +#define CKPQFETCHSIZ 9999 +#define CKPQFETCHSTR STRINT(CKPQFETCHSIZ) +#define FETCHTICK 100000 +#define CKPQFUNDEF -1 +#define CKPQ_VAL_FLD(__res, __row, __num, __name, __fld, __ok) do { \ + if (__num == CKPQFUNDEF) { \ + __num = PQfnumber(__res, __name); \ + if (__num == CKPQFUNDEF) { \ + LOGERR("%s():%d: Unknown field '%s' row %d", \ + __func__, __LINE__, __name, __row); \ __ok = false; \ - }\ + } \ } \ + __fld = PQgetvalue(__res, __row, __num); \ + if (__fld == NULL) { \ + LOGERR("%s():%d Invalid field '%s' or row %d", \ + __func__, __LINE__, __name, __row); \ + __ok = false; \ + }\ } while (0) +// Allow params to be macros + +#define CKPQ_VAL_FLD_tail2(__res, __row, __name, __fld, __ok) \ + CKPQ_VAL_FLD(__res, __row, __name ## _num__, #__name, __fld, __ok) +#define CKPQ_VAL_FLD_tail(__res, __row, __name, __fld, __ok) \ + CKPQ_VAL_FLD_tail2(__res, __row, __name, __fld, __ok) +#define CKPQ_VAL_FLD_num2(__res, __row, __name, __fld, __ok) \ + CKPQ_VAL_FLD(__res, __row, __name ## _num, #__name, __fld, __ok) +#define CKPQ_VAL_FLD_num(__res, __row, __name, __fld, __ok) \ + CKPQ_VAL_FLD_num2(__res, __row, __name, __fld, __ok) +#define CKPQADDNUM2(__name) __name ## _num__ +#define CKPQADDNUM(__name) CKPQADDNUM2(__name) + // HISTORY FIELDS +#define HISTORYDATE_num \ + int CKPQADDNUM(_CDDB), CKPQADDNUM(_BYDB), CKPQADDNUM(_CODEDB), \ + CKPQADDNUM(_INETDB), CKPQADDNUM(_EDDB) + +#define HISTORYDATE_init \ + CKPQADDNUM(_CDDB) = CKPQADDNUM(_BYDB) = CKPQADDNUM(_CODEDB) = \ + CKPQADDNUM(_INETDB) = CKPQADDNUM(_EDDB) = CKPQFUNDEF + #define HISTORYDATEFLDS(_res, _row, _data, _ok) do { \ char *_fld; \ - PQ_GET_FLD(_res, _row, CDDB, _fld, _ok); \ + CKPQ_VAL_FLD_tail(_res, _row, _CDDB, _fld, _ok); \ if (!_ok) \ break; \ TXT_TO_TVDB(CDDB, _fld, (_data)->createdate); \ - PQ_GET_FLD(_res, _row, BYDB, _fld, _ok); \ + CKPQ_VAL_FLD_tail(_res, _row, _BYDB, _fld, _ok); \ if (!_ok) \ break; \ TXT_TO_STR(BYDB, _fld, (_data)->createby); \ - PQ_GET_FLD(_res, _row, CODEDB, _fld, _ok); \ + CKPQ_VAL_FLD_tail(_res, _row, _CODEDB, _fld, _ok); \ if (!_ok) \ break; \ TXT_TO_STR(CODEDB, _fld, (_data)->createcode); \ - PQ_GET_FLD(_res, _row, INETDB, _fld, _ok); \ + CKPQ_VAL_FLD_tail(_res, _row, _INETDB, _fld, _ok); \ if (!_ok) \ break; \ TXT_TO_STR(INETDB, _fld, (_data)->createinet); \ - PQ_GET_FLD(_res, _row, EDDB, _fld, _ok); \ + CKPQ_VAL_FLD_tail(_res, _row, _EDDB, _fld, _ok); \ if (!_ok) \ break; \ TXT_TO_TVDB(EDDB, _fld, (_data)->expirydate); \ @@ -67,23 +101,23 @@ char *pqerrmsg(PGconn *conn) #define HISTORYDATEIN(_res, _row, _data, _ok) do { \ char *_fld; \ - PQ_GET_FLD(_res, _row, CDDB, _fld, _ok); \ + CKPQ_VAL_FLD_tail(_res, _row, _CDDB, _fld, _ok); \ if (!_ok) \ break; \ TXT_TO_TVDB(CDDB, _fld, (_data)->createdate); \ - PQ_GET_FLD(_res, _row, BYDB, _fld, _ok); \ + CKPQ_VAL_FLD_tail(_res, _row, _BYDB, _fld, _ok); \ if (!_ok) \ break; \ (_data)->in_createby = intransient_str(BYDB, _fld); \ - PQ_GET_FLD(_res, _row, CODEDB, _fld, _ok); \ + CKPQ_VAL_FLD_tail(_res, _row, _CODEDB, _fld, _ok); \ if (!_ok) \ break; \ (_data)->in_createcode = intransient_str(CODEDB, _fld); \ - PQ_GET_FLD(_res, _row, INETDB, _fld, _ok); \ + CKPQ_VAL_FLD_tail(_res, _row, _INETDB, _fld, _ok); \ if (!_ok) \ break; \ (_data)->in_createinet = intransient_str(INETDB, _fld); \ - PQ_GET_FLD(_res, _row, EDDB, _fld, _ok); \ + CKPQ_VAL_FLD_tail(_res, _row, _EDDB, _fld, _ok); \ if (!_ok) \ break; \ TXT_TO_TVDB(EDDB, _fld, (_data)->expirydate); \ @@ -107,74 +141,47 @@ char *pqerrmsg(PGconn *conn) } while (0) // MODIFY FIELDS -#define MODIFYDATEFLDPOINTERS(_list, _res, _row, _data, _ok) do { \ - char *_fld; \ - PQ_GET_FLD(_res, _row, CDDB, _fld, _ok); \ - if (!_ok) \ - break; \ - TXT_TO_TVDB(CDDB, _fld, (_data)->createdate); \ - PQ_GET_FLD(_res, _row, BYDB, _fld, _ok); \ - if (!_ok) \ - break; \ - SET_CREATEBY(_list, (_data)->createby, _fld); \ - PQ_GET_FLD(_res, _row, CODEDB, _fld, _ok); \ - if (!_ok) \ - break; \ - SET_CREATECODE(_list, (_data)->createcode, _fld); \ - PQ_GET_FLD(_res, _row, INETDB, _fld, _ok); \ - if (!_ok) \ - break; \ - SET_CREATEINET(_list, (_data)->createinet, _fld); \ - PQ_GET_FLD(_res, _row, MDDB, _fld, _ok); \ - if (!_ok) \ - break; \ - TXT_TO_TVDB(MDDB, _fld, (_data)->modifydate); \ - PQ_GET_FLD(_res, _row, MBYDB, _fld, _ok); \ - if (!_ok) \ - break; \ - SET_MODIFYBY(_list, (_data)->modifyby, _fld); \ - PQ_GET_FLD(_res, _row, MCODEDB, _fld, _ok); \ - if (!_ok) \ - break; \ - SET_MODIFYCODE(_list, (_data)->modifycode, _fld); \ - PQ_GET_FLD(_res, _row, MINETDB, _fld, _ok); \ - if (!_ok) \ - break; \ - SET_MODIFYINET(_list, (_data)->modifyinet, _fld); \ - (_data)->pointers = (_data)->pointers; \ - } while (0) +#define MODIFYDATE_num \ + int CKPQADDNUM(_CDDB), CKPQADDNUM(_BYDB), CKPQADDNUM(_CODEDB), \ + CKPQADDNUM(_INETDB), CKPQADDNUM(_MDDB), CKPQADDNUM(_MBYDB), \ + CKPQADDNUM(_MCODEDB), CKPQADDNUM(_MINETDB) + +#define MODIFYDATE_init \ + CKPQADDNUM(_CDDB) = CKPQADDNUM(_BYDB) = CKPQADDNUM(_CODEDB) = \ + CKPQADDNUM(_INETDB) = CKPQADDNUM(_MDDB) = CKPQADDNUM(_MBYDB) = \ + CKPQADDNUM(_MCODEDB) = CKPQADDNUM(_MINETDB) = CKPQFUNDEF #define MODIFYDATEIN(_res, _row, _data, _ok) do { \ char *_fld; \ - PQ_GET_FLD(_res, _row, CDDB, _fld, _ok); \ + CKPQ_VAL_FLD_tail(_res, _row, _CDDB, _fld, _ok); \ if (!_ok) \ break; \ TXT_TO_TVDB(CDDB, _fld, (_data)->createdate); \ - PQ_GET_FLD(_res, _row, BYDB, _fld, _ok); \ + CKPQ_VAL_FLD_tail(_res, _row, _BYDB, _fld, _ok); \ if (!_ok) \ break; \ (_data)->in_createby = intransient_str(BYDB, _fld); \ - PQ_GET_FLD(_res, _row, CODEDB, _fld, _ok); \ + CKPQ_VAL_FLD_tail(_res, _row, _CODEDB, _fld, _ok); \ if (!_ok) \ break; \ (_data)->in_createcode = intransient_str(CODEDB, _fld); \ - PQ_GET_FLD(_res, _row, INETDB, _fld, _ok); \ + CKPQ_VAL_FLD_tail(_res, _row, _INETDB, _fld, _ok); \ if (!_ok) \ break; \ (_data)->in_createinet = intransient_str(INETDB, _fld); \ - PQ_GET_FLD(_res, _row, MDDB, _fld, _ok); \ + CKPQ_VAL_FLD_tail(_res, _row, _MDDB, _fld, _ok); \ if (!_ok) \ break; \ TXT_TO_TVDB(MDDB, _fld, (_data)->modifydate); \ - PQ_GET_FLD(_res, _row, MBYDB, _fld, _ok); \ + CKPQ_VAL_FLD_tail(_res, _row, _MBYDB, _fld, _ok); \ if (!_ok) \ break; \ (_data)->in_modifyby = intransient_str(MBYDB, _fld); \ - PQ_GET_FLD(_res, _row, MCODEDB, _fld, _ok); \ + CKPQ_VAL_FLD_tail(_res, _row, _MCODEDB, _fld, _ok); \ if (!_ok) \ break; \ (_data)->in_modifycode = intransient_str(MCODEDB, _fld); \ - PQ_GET_FLD(_res, _row, MINETDB, _fld, _ok); \ + CKPQ_VAL_FLD_tail(_res, _row, _MINETDB, _fld, _ok); \ if (!_ok) \ break; \ (_data)->in_modifyinet = intransient_str(MINETDB, _fld); \ @@ -211,21 +218,29 @@ char *pqerrmsg(PGconn *conn) } while (0) // SIMPLE FIELDS +#define SIMPLEDATE_num \ + int CKPQADDNUM(_CDDB), CKPQADDNUM(_BYDB), CKPQADDNUM(_CODEDB), \ + CKPQADDNUM(_INETDB) + +#define SIMPLEDATE_init \ + CKPQADDNUM(_CDDB) = CKPQADDNUM(_BYDB) = CKPQADDNUM(_CODEDB) = \ + CKPQADDNUM(_INETDB) = CKPQFUNDEF + #define SIMPLEDATEFLDS(_res, _row, _data, _ok) do { \ char *_fld; \ - PQ_GET_FLD(_res, _row, CDDB, _fld, _ok); \ + CKPQ_VAL_FLD_tail(_res, _row, _CDDB, _fld, _ok); \ if (!_ok) \ break; \ TXT_TO_TVDB(CDDB, _fld, (_data)->createdate); \ - PQ_GET_FLD(_res, _row, BYDB, _fld, _ok); \ + CKPQ_VAL_FLD_tail(_res, _row, _BYDB, _fld, _ok); \ if (!_ok) \ break; \ TXT_TO_STR(BYDB, _fld, (_data)->createby); \ - PQ_GET_FLD(_res, _row, CODEDB, _fld, _ok); \ + CKPQ_VAL_FLD_tail(_res, _row, _CODEDB, _fld, _ok); \ if (!_ok) \ break; \ TXT_TO_STR(CODEDB, _fld, (_data)->createcode); \ - PQ_GET_FLD(_res, _row, INETDB, _fld, _ok); \ + CKPQ_VAL_FLD_tail(_res, _row, _INETDB, _fld, _ok); \ if (!_ok) \ break; \ TXT_TO_STR(INETDB, _fld, (_data)->createinet); \ @@ -484,7 +499,7 @@ int64_t nextid(PGconn *conn, char *idname, int64_t increment, PGresult *res; char qry[1024]; char *params[5]; - int n, par = 0; + int n, f, par = 0; int64_t lastid; char *field; bool ok; @@ -517,7 +532,8 @@ int64_t nextid(PGconn *conn, char *idname, int64_t increment, } ok = true; - PQ_GET_FLD(res, 0, "lastid", field, ok); + f = CKPQFUNDEF; + CKPQ_VAL_FLD(res, 0, f, "lastid", field, ok); if (!ok) goto cleanup; TXT_TO_BIGINT("lastid", field, lastid); @@ -974,6 +990,11 @@ bool users_fill(PGconn *conn) LOGDEBUG("%s(): select", __func__); + int userid_num, username_num, status_num, emailaddress_num; + int joineddate_num, passwordhash_num, secondaryuserid_num, salt_num; + int userdata_num, userbits_num; + HISTORYDATE_num; + sel = "select " "userid,username,status,emailaddress,joineddate," "passwordhash,secondaryuserid,salt,userdata,userbits" @@ -998,6 +1019,10 @@ bool users_fill(PGconn *conn) n = PQntuples(res); LOGDEBUG("%s(): tree build count %d", __func__, n); ok = true; + userid_num = username_num = status_num = emailaddress_num = + joineddate_num = passwordhash_num = secondaryuserid_num = salt_num = + userdata_num = userbits_num = CKPQFUNDEF; + HISTORYDATE_init; K_WLOCK(users_free); for (i = 0; i < n; i++) { item = k_unlink_head(users_free); @@ -1009,55 +1034,55 @@ bool users_fill(PGconn *conn) break; } - PQ_GET_FLD(res, i, "userid", field, ok); + CKPQ_VAL_FLD_num(res, i, userid, field, ok); if (!ok) break; TXT_TO_BIGINT("userid", field, row->userid); - PQ_GET_FLD(res, i, "username", field, ok); + CKPQ_VAL_FLD_num(res, i, username, field, ok); if (!ok) break; row->in_username = intransient_str("username", field); - PQ_GET_FLD(res, i, "status", field, ok); + CKPQ_VAL_FLD_num(res, i, status, field, ok); if (!ok) break; TXT_TO_STR("status", field, row->status); - PQ_GET_FLD(res, i, "emailaddress", field, ok); + CKPQ_VAL_FLD_num(res, i, emailaddress, field, ok); if (!ok) break; TXT_TO_STR("emailaddress", field, row->emailaddress); - PQ_GET_FLD(res, i, "joineddate", field, ok); + CKPQ_VAL_FLD_num(res, i, joineddate, field, ok); if (!ok) break; TXT_TO_TVDB("joineddate", field, row->joineddate); - PQ_GET_FLD(res, i, "passwordhash", field, ok); + CKPQ_VAL_FLD_num(res, i, passwordhash, field, ok); if (!ok) break; TXT_TO_STR("passwordhash", field, row->passwordhash); - PQ_GET_FLD(res, i, "secondaryuserid", field, ok); + CKPQ_VAL_FLD_num(res, i, secondaryuserid, field, ok); if (!ok) break; TXT_TO_STR("secondaryuserid", field, row->secondaryuserid); - PQ_GET_FLD(res, i, "salt", field, ok); + CKPQ_VAL_FLD_num(res, i, salt, field, ok); if (!ok) break; TXT_TO_STR("salt", field, row->salt); // TODO: good case for invariant - PQ_GET_FLD(res, i, "userdata", field, ok); + CKPQ_VAL_FLD_num(res, i, userdata, field, ok); if (!ok) break; TXT_TO_PTR("userdata", field, row->userdata); LIST_MEM_ADD(users_free, row->userdata); users_databits(row); - PQ_GET_FLD(res, i, "userbits", field, ok); + CKPQ_VAL_FLD_num(res, i, userbits, field, ok); if (!ok) break; TXT_TO_BIGINT("userbits", field, row->userbits); @@ -1194,7 +1219,7 @@ unparam: K_ITEM *useratts_add(PGconn *conn, char *username, char *attname, char *status, char *attstr, char *attstr2, - char *attnum, char *attnum2, char *attdate, + char *attnum, char *attnum2, char *attdate, char *attdate2, char *by, char *code, char *inet, tv_t *cd, K_TREE *trf_root, bool begun) @@ -1349,6 +1374,10 @@ bool useratts_fill(PGconn *conn) LOGDEBUG("%s(): select", __func__); + int userid_num, attname_num, status_num, attstr_num, attstr2_num; + int attnum_num, attnum2_num, attdate_num, attdate2_num; + HISTORYDATE_num; + sel = "select " "userid,attname,status,attstr,attstr2,attnum,attnum2" ",attdate,attdate2" @@ -1373,6 +1402,9 @@ bool useratts_fill(PGconn *conn) n = PQntuples(res); LOGDEBUG("%s(): tree build count %d", __func__, n); ok = true; + userid_num = attname_num = status_num = attstr_num = attstr2_num = + attnum_num = attnum2_num = attdate_num = attdate2_num = CKPQFUNDEF; + HISTORYDATE_init; K_WLOCK(useratts_free); for (i = 0; i < n; i++) { item = k_unlink_head(useratts_free); @@ -1384,47 +1416,47 @@ bool useratts_fill(PGconn *conn) break; } - PQ_GET_FLD(res, i, "userid", field, ok); + CKPQ_VAL_FLD_num(res, i, userid, field, ok); if (!ok) break; TXT_TO_BIGINT("userid", field, row->userid); - PQ_GET_FLD(res, i, "attname", field, ok); + CKPQ_VAL_FLD_num(res, i, attname, field, ok); if (!ok) break; TXT_TO_STR("attname", field, row->attname); - PQ_GET_FLD(res, i, "status", field, ok); + CKPQ_VAL_FLD_num(res, i, status, field, ok); if (!ok) break; TXT_TO_STR("status", field, row->status); - PQ_GET_FLD(res, i, "attstr", field, ok); + CKPQ_VAL_FLD_num(res, i, attstr, field, ok); if (!ok) break; TXT_TO_STR("attstr", field, row->attstr); - PQ_GET_FLD(res, i, "attstr2", field, ok); + CKPQ_VAL_FLD_num(res, i, attstr2, field, ok); if (!ok) break; TXT_TO_STR("attstr2", field, row->attstr2); - PQ_GET_FLD(res, i, "attnum", field, ok); + CKPQ_VAL_FLD_num(res, i, attnum, field, ok); if (!ok) break; TXT_TO_BIGINT("attnum", field, row->attnum); - PQ_GET_FLD(res, i, "attnum2", field, ok); + CKPQ_VAL_FLD_num(res, i, attnum2, field, ok); if (!ok) break; TXT_TO_BIGINT("attnum2", field, row->attnum2); - PQ_GET_FLD(res, i, "attdate", field, ok); + CKPQ_VAL_FLD_num(res, i, attdate, field, ok); if (!ok) break; TXT_TO_TVDB("attdate", field, row->attdate); - PQ_GET_FLD(res, i, "attdate2", field, ok); + CKPQ_VAL_FLD_num(res, i, attdate2, field, ok); if (!ok) break; TXT_TO_TVDB("attdate2", field, row->attdate2); @@ -1743,6 +1775,11 @@ bool workers_fill(PGconn *conn) LOGDEBUG("%s(): select", __func__); + int userid_num, workername_num, difficultydefault_num; + int idlenotificationenabled_num, idlenotificationtime_num; + int workerbits_num, workerid_num; + HISTORYDATE_num; + sel = "declare wk cursor for select " "userid,workername,difficultydefault," "idlenotificationenabled,idlenotificationtime,workerbits" @@ -1782,6 +1819,10 @@ bool workers_fill(PGconn *conn) n = 0; ok = true; + userid_num = workername_num = difficultydefault_num = + idlenotificationenabled_num = idlenotificationtime_num = + workerbits_num = workerid_num = CKPQFUNDEF; + HISTORYDATE_init; while ((t = PQntuples(res)) > 0) { for (i = 0; i < t; i++) { K_WLOCK(workers_free); @@ -1795,32 +1836,32 @@ bool workers_fill(PGconn *conn) break; } - PQ_GET_FLD(res, i, "userid", field, ok); + CKPQ_VAL_FLD_num(res, i, userid, field, ok); if (!ok) break; TXT_TO_BIGINT("userid", field, row->userid); - PQ_GET_FLD(res, i, "workername", field, ok); + CKPQ_VAL_FLD_num(res, i, workername, field, ok); if (!ok) break; row->in_workername = intransient_str("workername", field); - PQ_GET_FLD(res, i, "difficultydefault", field, ok); + CKPQ_VAL_FLD_num(res, i, difficultydefault, field, ok); if (!ok) break; TXT_TO_INT("difficultydefault", field, row->difficultydefault); - PQ_GET_FLD(res, i, "idlenotificationenabled", field, ok); + CKPQ_VAL_FLD_num(res, i, idlenotificationenabled, field, ok); if (!ok) break; TXT_TO_STR("idlenotificationenabled", field, row->idlenotificationenabled); - PQ_GET_FLD(res, i, "idlenotificationtime", field, ok); + CKPQ_VAL_FLD_num(res, i, idlenotificationtime, field, ok); if (!ok) break; TXT_TO_INT("idlenotificationtime", field, row->idlenotificationtime); - PQ_GET_FLD(res, i, "workerbits", field, ok); + CKPQ_VAL_FLD_num(res, i, workerbits, field, ok); if (!ok) break; TXT_TO_BIGINT("workerbits", field, row->workerbits); @@ -1829,7 +1870,7 @@ bool workers_fill(PGconn *conn) if (!ok) break; - PQ_GET_FLD(res, i, "workerid", field, ok); + CKPQ_VAL_FLD_num(res, i, workerid, field, ok); if (!ok) break; TXT_TO_BIGINT("workerid", field, row->workerid); @@ -1847,13 +1888,17 @@ bool workers_fill(PGconn *conn) n++; } CKPQClear(res); - res = CKPQExec(conn, "fetch 9999 in wk", CKPQ_READ); + res = CKPQExec(conn, "fetch "CKPQFETCHSTR" in wk", CKPQ_READ); rescode = CKPQResultStatus(res); if (!PGOK(rescode)) { PGLOGERR("Fetch next", rescode, conn); ok = false; break; } + userid_num = workername_num = difficultydefault_num = + idlenotificationenabled_num = idlenotificationtime_num = + workerbits_num = workerid_num = CKPQFUNDEF; + HISTORYDATE_init; } if (!ok) k_add_head(workers_free, item); @@ -2121,6 +2166,10 @@ bool paymentaddresses_fill(PGconn *conn) LOGDEBUG("%s(): select", __func__); + int paymentaddressid_num, userid_num, payaddress_num, payratio_num; + int payname_num; + HISTORYDATE_num; + sel = "select " "paymentaddressid,userid,payaddress,payratio,payname" HISTORYDATECONTROL @@ -2144,6 +2193,9 @@ bool paymentaddresses_fill(PGconn *conn) n = PQntuples(res); LOGDEBUG("%s(): tree build count %d", __func__, n); ok = true; + paymentaddressid_num = userid_num = payaddress_num = payratio_num = + payname_num = CKPQFUNDEF; + HISTORYDATE_init; K_WLOCK(paymentaddresses_free); for (i = 0; i < n; i++) { item = k_unlink_head(paymentaddresses_free); @@ -2155,27 +2207,27 @@ bool paymentaddresses_fill(PGconn *conn) break; } - PQ_GET_FLD(res, i, "paymentaddressid", field, ok); + CKPQ_VAL_FLD_num(res, i, paymentaddressid, field, ok); if (!ok) break; TXT_TO_BIGINT("paymentaddressid", field, row->paymentaddressid); - PQ_GET_FLD(res, i, "userid", field, ok); + CKPQ_VAL_FLD_num(res, i, userid, field, ok); if (!ok) break; TXT_TO_BIGINT("userid", field, row->userid); - PQ_GET_FLD(res, i, "payaddress", field, ok); + CKPQ_VAL_FLD_num(res, i, payaddress, field, ok); if (!ok) break; row->in_payaddress = intransient_str("payaddress", field); - PQ_GET_FLD(res, i, "payratio", field, ok); + CKPQ_VAL_FLD_num(res, i, payratio, field, ok); if (!ok) break; TXT_TO_INT("payratio", field, row->payratio); - PQ_GET_FLD(res, i, "payname", field, ok); + CKPQ_VAL_FLD_num(res, i, payname, field, ok); if (!ok) break; TXT_TO_STR("payname", field, row->payname); @@ -2364,6 +2416,11 @@ bool payments_fill(PGconn *conn) STRNCPY(tickbuf, TICK_PREFIX"pm 0"); cr_msg(false, tickbuf); + int paymentid_num, payoutid_num, userid_num, subname_num, paydate_num; + int payaddress_num, originaltxn_num, amount_num, diffacc_num; + int committxn_num, commitblockhash_num; + HISTORYDATE_num; + sel = "declare ps cursor for select " "paymentid,payoutid,userid,subname,paydate,payaddress," "originaltxn,amount,diffacc,committxn,commitblockhash" @@ -2400,6 +2457,10 @@ bool payments_fill(PGconn *conn) n = 0; ok = true; + paymentid_num = payoutid_num = userid_num = subname_num = paydate_num = + payaddress_num = originaltxn_num = amount_num = diffacc_num = + committxn_num = commitblockhash_num = CKPQFUNDEF; + HISTORYDATE_init; K_WLOCK(payments_free); while ((t = PQntuples(res)) > 0) { for (i = 0; i < t; i++) { @@ -2412,57 +2473,57 @@ bool payments_fill(PGconn *conn) break; } - PQ_GET_FLD(res, i, "paymentid", field, ok); + CKPQ_VAL_FLD_num(res, i, paymentid, field, ok); if (!ok) break; TXT_TO_BIGINT("paymentid", field, row->paymentid); - PQ_GET_FLD(res, i, "payoutid", field, ok); + CKPQ_VAL_FLD_num(res, i, payoutid, field, ok); if (!ok) break; TXT_TO_BIGINT("payoutid", field, row->payoutid); - PQ_GET_FLD(res, i, "userid", field, ok); + CKPQ_VAL_FLD_num(res, i, userid, field, ok); if (!ok) break; TXT_TO_BIGINT("userid", field, row->userid); - PQ_GET_FLD(res, i, "subname", field, ok); + CKPQ_VAL_FLD_num(res, i, subname, field, ok); if (!ok) break; row->in_subname = intransient_str("subname", field); - PQ_GET_FLD(res, i, "paydate", field, ok); + CKPQ_VAL_FLD_num(res, i, paydate, field, ok); if (!ok) break; TXT_TO_TVDB("paydate", field, row->paydate); - PQ_GET_FLD(res, i, "payaddress", field, ok); + CKPQ_VAL_FLD_num(res, i, payaddress, field, ok); if (!ok) break; row->in_payaddress = intransient_str("payaddress", field); - PQ_GET_FLD(res, i, "originaltxn", field, ok); + CKPQ_VAL_FLD_num(res, i, originaltxn, field, ok); if (!ok) break; row->in_originaltxn = intransient_str("originaltxn", field); - PQ_GET_FLD(res, i, "amount", field, ok); + CKPQ_VAL_FLD_num(res, i, amount, field, ok); if (!ok) break; TXT_TO_BIGINT("amount", field, row->amount); - PQ_GET_FLD(res, i, "diffacc", field, ok); + CKPQ_VAL_FLD_num(res, i, diffacc, field, ok); if (!ok) break; TXT_TO_DOUBLE("diffacc", field, row->diffacc); - PQ_GET_FLD(res, i, "committxn", field, ok); + CKPQ_VAL_FLD_num(res, i, committxn, field, ok); if (!ok) break; row->in_committxn = intransient_str("committxn", field); - PQ_GET_FLD(res, i, "commitblockhash", field, ok); + CKPQ_VAL_FLD_num(res, i, commitblockhash, field, ok); if (!ok) break; row->in_commitblockhash = intransient_str("commitblockhash", field); @@ -2474,7 +2535,7 @@ bool payments_fill(PGconn *conn) add_to_ktree(payments_root, item); k_add_head(payments_store, item); - if (n == 0 || ((n+1) % 100000) == 0) { + if (n == 0 || ((n+1) % FETCHTICK) == 0) { pcom(n+1, pcombuf, sizeof(pcombuf)); snprintf(tickbuf, sizeof(tickbuf), TICK_PREFIX"pm %s", pcombuf); @@ -2484,13 +2545,17 @@ bool payments_fill(PGconn *conn) n++; } CKPQClear(res); - res = CKPQExec(conn, "fetch 9999 in ps", CKPQ_READ); + res = CKPQExec(conn, "fetch "CKPQFETCHSTR" in ps", CKPQ_READ); rescode = CKPQResultStatus(res); if (!PGOK(rescode)) { PGLOGERR("Fetch next", rescode, conn); ok = false; break; } + paymentid_num = payoutid_num = userid_num = subname_num = + paydate_num = payaddress_num = originaltxn_num = amount_num = + diffacc_num = committxn_num = commitblockhash_num = CKPQFUNDEF; + HISTORYDATE_init; } if (!ok) k_add_head(payments_free, item); @@ -3039,6 +3104,10 @@ bool optioncontrol_fill(PGconn *conn) LOGDEBUG("%s(): select", __func__); + int optionname_num, optionvalue_num, activationdate_num; + int activationheight_num; + HISTORYDATE_num; + // No need to keep old versions in ram for now ... sel = "select " "optionname,optionvalue,activationdate,activationheight" @@ -3066,6 +3135,9 @@ bool optioncontrol_fill(PGconn *conn) n = PQntuples(res); LOGDEBUG("%s(): tree build count %d", __func__, n); ok = true; + optionname_num = optionvalue_num = activationdate_num = + activationheight_num = CKPQFUNDEF; + HISTORYDATE_init; K_WLOCK(optioncontrol_free); for (i = 0; i < n; i++) { item = k_unlink_head(optioncontrol_free); @@ -3077,23 +3149,23 @@ bool optioncontrol_fill(PGconn *conn) break; } - PQ_GET_FLD(res, i, "optionname", field, ok); + CKPQ_VAL_FLD_num(res, i, optionname, field, ok); if (!ok) break; TXT_TO_STR("optionname", field, row->optionname); - PQ_GET_FLD(res, i, "optionvalue", field, ok); + CKPQ_VAL_FLD_num(res, i, optionvalue, field, ok); if (!ok) break; TXT_TO_BLOB("optionvalue", field, row->optionvalue); LIST_MEM_ADD(optioncontrol_free, row->optionvalue); - PQ_GET_FLD(res, i, "activationdate", field, ok); + CKPQ_VAL_FLD_num(res, i, activationdate, field, ok); if (!ok) break; TXT_TO_TVDB("activationdate", field, row->activationdate); - PQ_GET_FLD(res, i, "activationheight", field, ok); + CKPQ_VAL_FLD_num(res, i, activationheight, field, ok); if (!ok) break; TXT_TO_INT("activationheight", field, row->activationheight); @@ -3310,6 +3382,11 @@ bool workinfo_fill(PGconn *conn) STRNCPY(tickbuf, TICK_PREFIX"wi 0"); cr_msg(false, tickbuf); + int workinfoid_num, poolinstance_num, merklehash_num, prevhash_num; + int coinbase1_num, coinbase2_num, version_num, bits_num, ntime_num; + int reward_num; + HISTORYDATE_num; + APPEND_REALLOC_INIT(sel, off, len); APPEND_REALLOC(sel, off, len, "declare wi cursor for select " @@ -3384,6 +3461,10 @@ bool workinfo_fill(PGconn *conn) n = 0; ok = true; + workinfoid_num = poolinstance_num = merklehash_num = prevhash_num = + coinbase1_num = coinbase2_num = version_num = bits_num = ntime_num = + reward_num = CKPQFUNDEF; + HISTORYDATE_init; K_WLOCK(workinfo_free); while ((t = PQntuples(res)) > 0) { for (i = 0; i < t; i++) { @@ -3396,7 +3477,7 @@ bool workinfo_fill(PGconn *conn) break; } - PQ_GET_FLD(res, i, "poolinstance", field, ok); + CKPQ_VAL_FLD_num(res, i, poolinstance, field, ok); if (!ok) break; if (sys_poolinstance && strcmp(field, sys_poolinstance)) { @@ -3406,7 +3487,7 @@ bool workinfo_fill(PGconn *conn) } row->in_poolinstance = intransient_str("poolinstance", field); - PQ_GET_FLD(res, i, "workinfoid", field, ok); + CKPQ_VAL_FLD_num(res, i, workinfoid, field, ok); if (!ok) break; TXT_TO_BIGINT("workinfoid", field, row->workinfoid); @@ -3414,39 +3495,39 @@ bool workinfo_fill(PGconn *conn) row->transactiontree = EMPTY; row->merklehash = EMPTY; - PQ_GET_FLD(res, i, "prevhash", field, ok); + CKPQ_VAL_FLD_num(res, i, prevhash, field, ok); if (!ok) break; row->in_prevhash = intransient_str("prevhash", field); - PQ_GET_FLD(res, i, "coinbase1", field, ok); + CKPQ_VAL_FLD_num(res, i, coinbase1, field, ok); if (!ok) break; TXT_TO_BLOB("coinbase1", field, row->coinbase1); LIST_MEM_ADD(workinfo_free, row->coinbase1); - PQ_GET_FLD(res, i, "coinbase2", field, ok); + CKPQ_VAL_FLD_num(res, i, coinbase2, field, ok); if (!ok) break; TXT_TO_BLOB("coinbase2", field, row->coinbase2); LIST_MEM_ADD(workinfo_free, row->coinbase2); - PQ_GET_FLD(res, i, "version", field, ok); + CKPQ_VAL_FLD_num(res, i, version, field, ok); if (!ok) break; row->in_version = intransient_str("version", field); - PQ_GET_FLD(res, i, "bits", field, ok); + CKPQ_VAL_FLD_num(res, i, bits, field, ok); if (!ok) break; row->in_bits = intransient_str("bits", field); - PQ_GET_FLD(res, i, "ntime", field, ok); + CKPQ_VAL_FLD_num(res, i, ntime, field, ok); if (!ok) break; TXT_TO_STR("ntime", field, row->ntime); - PQ_GET_FLD(res, i, "reward", field, ok); + CKPQ_VAL_FLD_num(res, i, reward, field, ok); if (!ok) break; TXT_TO_BIGINT("reward", field, row->reward); @@ -3470,7 +3551,7 @@ bool workinfo_fill(PGconn *conn) dbstatus.newest_workinfoid = row->workinfoid; } - if (n == 0 || ((n+1) % 100000) == 0) { + if (n == 0 || ((n+1) % FETCHTICK) == 0) { pcom(n+1, pcombuf, sizeof(pcombuf)); snprintf(tickbuf, sizeof(tickbuf), TICK_PREFIX"wi %s", pcombuf); @@ -3480,13 +3561,17 @@ bool workinfo_fill(PGconn *conn) n++; } CKPQClear(res); - res = CKPQExec(conn, "fetch 9999 in wi", CKPQ_READ); + res = CKPQExec(conn, "fetch "CKPQFETCHSTR" in wi", CKPQ_READ); rescode = CKPQResultStatus(res); if (!PGOK(rescode)) { PGLOGERR("Fetch next", rescode, conn); ok = false; break; } + workinfoid_num = poolinstance_num = merklehash_num = + prevhash_num = coinbase1_num = coinbase2_num = version_num = + bits_num = ntime_num = reward_num = CKPQFUNDEF; + HISTORYDATE_init; } if (!ok) { free_workinfo_data(item); @@ -4087,6 +4172,12 @@ bool shares_fill(PGconn *conn) STRNCPY(tickbuf, TICK_PREFIX"sh 0"); cr_msg(false, tickbuf); + int workinfoid_num, userid_num, workername_num, clientid_num; + int enonce1_num, nonce2_num, nonce_num, diff_num, sdiff_num, errn_num; + int error_num, secondaryuserid_num, ntime_num, minsdiff_num, agent_num; + int address_num; + HISTORYDATE_num; + sel = "declare sh cursor for select " "workinfoid,userid,workername,clientid,enonce1,nonce2,nonce," "diff,sdiff,errn,error,secondaryuserid,ntime,minsdiff,agent," @@ -4138,6 +4229,11 @@ bool shares_fill(PGconn *conn) n = 0; ok = true; + workinfoid_num = userid_num = workername_num = clientid_num = + enonce1_num = nonce2_num = nonce_num = diff_num = sdiff_num = errn_num = + error_num = secondaryuserid_num = ntime_num = minsdiff_num = agent_num = + address_num = CKPQFUNDEF; + HISTORYDATE_init; K_WLOCK(shares_free); while ((t = PQntuples(res)) > 0) { for (i = 0; i < t; i++) { @@ -4150,72 +4246,72 @@ bool shares_fill(PGconn *conn) break; } - PQ_GET_FLD(res, i, "workinfoid", field, ok); + CKPQ_VAL_FLD_num(res, i, workinfoid, field, ok); if (!ok) break; TXT_TO_BIGINT("workinfoid", field, row->workinfoid); - PQ_GET_FLD(res, i, "userid", field, ok); + CKPQ_VAL_FLD_num(res, i, userid, field, ok); if (!ok) break; TXT_TO_BIGINT("userid", field, row->userid); - PQ_GET_FLD(res, i, "workername", field, ok); + CKPQ_VAL_FLD_num(res, i, workername, field, ok); if (!ok) break; row->in_workername = intransient_str("workername", field); - PQ_GET_FLD(res, i, "clientid", field, ok); + CKPQ_VAL_FLD_num(res, i, clientid, field, ok); if (!ok) break; TXT_TO_INT("clientid", field, row->clientid); - PQ_GET_FLD(res, i, "enonce1", field, ok); + CKPQ_VAL_FLD_num(res, i, enonce1, field, ok); if (!ok) break; TXT_TO_STR("enonce1", field, row->enonce1); - PQ_GET_FLD(res, i, "nonce2", field, ok); + CKPQ_VAL_FLD_num(res, i, nonce2, field, ok); if (!ok) break; TXT_TO_STR("nonce2", field, row->nonce2); - PQ_GET_FLD(res, i, "nonce", field, ok); + CKPQ_VAL_FLD_num(res, i, nonce, field, ok); if (!ok) break; TXT_TO_STR("nonce", field, row->nonce); - PQ_GET_FLD(res, i, "diff", field, ok); + CKPQ_VAL_FLD_num(res, i, diff, field, ok); if (!ok) break; TXT_TO_DOUBLE("diff", field, row->diff); - PQ_GET_FLD(res, i, "sdiff", field, ok); + CKPQ_VAL_FLD_num(res, i, sdiff, field, ok); if (!ok) break; TXT_TO_DOUBLE("sdiff", field, row->sdiff); - PQ_GET_FLD(res, i, "errn", field, ok); + CKPQ_VAL_FLD_num(res, i, errn, field, ok); if (!ok) break; TXT_TO_INT("errn", field, row->errn); - PQ_GET_FLD(res, i, "error", field, ok); + CKPQ_VAL_FLD_num(res, i, error, field, ok); if (!ok) break; TXT_TO_STR("error", field, row->error); - PQ_GET_FLD(res, i, "secondaryuserid", field, ok); + CKPQ_VAL_FLD_num(res, i, secondaryuserid, field, ok); if (!ok) break; TXT_TO_STR("secondaryuserid", field, row->secondaryuserid); - PQ_GET_FLD(res, i, "ntime", field, ok); + CKPQ_VAL_FLD_num(res, i, ntime, field, ok); if (!ok) break; TXT_TO_STR("ntime", field, row->ntime); - PQ_GET_FLD(res, i, "minsdiff", field, ok); + CKPQ_VAL_FLD_num(res, i, minsdiff, field, ok); if (!ok) break; TXT_TO_DOUBLE("minsdiff", field, row->sdiff); @@ -4224,14 +4320,14 @@ bool shares_fill(PGconn *conn) if (!ok) break; - PQ_GET_FLD(res, i, "agent", field, ok); + CKPQ_VAL_FLD_num(res, i, agent, field, ok); if (!ok) break; if (!(*field)) no_agent++; TXT_TO_STR("agent", field, row->agent); - PQ_GET_FLD(res, i, "address", field, ok); + CKPQ_VAL_FLD_num(res, i, address, field, ok); if (!ok) break; if (!(*field)) @@ -4241,7 +4337,7 @@ bool shares_fill(PGconn *conn) add_to_ktree(shares_db_root, item); k_add_head(shares_hi_store, item); - if (n == 0 || ((n+1) % 100000) == 0) { + if (n == 0 || ((n+1) % FETCHTICK) == 0) { pcom(n+1, pcombuf, sizeof(pcombuf)); snprintf(tickbuf, sizeof(tickbuf), TICK_PREFIX"sh %s", pcombuf); @@ -4251,13 +4347,18 @@ bool shares_fill(PGconn *conn) n++; } CKPQClear(res); - res = CKPQExec(conn, "fetch 9999 in sh", CKPQ_READ); + res = CKPQExec(conn, "fetch "CKPQFETCHSTR" in sh", CKPQ_READ); rescode = CKPQResultStatus(res); if (!PGOK(rescode)) { PGLOGERR("Fetch next", rescode, conn); ok = false; break; } + workinfoid_num = userid_num = workername_num = clientid_num = + enonce1_num = nonce2_num = nonce_num = diff_num = sdiff_num = + errn_num = error_num = secondaryuserid_num = ntime_num = + minsdiff_num = agent_num = address_num = CKPQFUNDEF; + HISTORYDATE_init; } if (!ok) k_add_head(shares_free, item); @@ -6314,6 +6415,12 @@ bool blocks_fill(PGconn *conn) LOGDEBUG("%s(): select", __func__); + int height_num, blockhash_num, workinfoid_num, userid_num; + int workername_num, clientid_num, enonce1_num, nonce2_num, nonce_num; + int reward_num, confirmed_num, info_num, diffacc_num, diffinv_num; + int shareacc_num, shareinv_num, elapsed_num, statsconfirmed_num; + HISTORYDATE_num; + sel = "select " "height,blockhash,workinfoid,userid,workername," "clientid,enonce1,nonce2,nonce,reward,confirmed,info," @@ -6339,6 +6446,12 @@ bool blocks_fill(PGconn *conn) n = PQntuples(res); LOGDEBUG("%s(): tree build count %d", __func__, n); ok = true; + height_num = blockhash_num = workinfoid_num = userid_num = + workername_num = clientid_num = enonce1_num = nonce2_num = nonce_num = + reward_num = confirmed_num = info_num = diffacc_num = diffinv_num = + shareacc_num = shareinv_num = elapsed_num = + statsconfirmed_num = CKPQFUNDEF; + HISTORYDATE_init; K_WLOCK(blocks_free); for (i = 0; i < n; i++) { item = k_unlink_head(blocks_free); @@ -6350,92 +6463,92 @@ bool blocks_fill(PGconn *conn) break; } - PQ_GET_FLD(res, i, "height", field, ok); + CKPQ_VAL_FLD_num(res, i, height, field, ok); if (!ok) break; TXT_TO_INT("height", field, row->height); - PQ_GET_FLD(res, i, "blockhash", field, ok); + CKPQ_VAL_FLD_num(res, i, blockhash, field, ok); if (!ok) break; TXT_TO_STR("blockhash", field, row->blockhash); - PQ_GET_FLD(res, i, "workinfoid", field, ok); + CKPQ_VAL_FLD_num(res, i, workinfoid, field, ok); if (!ok) break; TXT_TO_BIGINT("workinfoid", field, row->workinfoid); - PQ_GET_FLD(res, i, "userid", field, ok); + CKPQ_VAL_FLD_num(res, i, userid, field, ok); if (!ok) break; TXT_TO_BIGINT("userid", field, row->userid); - PQ_GET_FLD(res, i, "workername", field, ok); + CKPQ_VAL_FLD_num(res, i, workername, field, ok); if (!ok) break; row->in_workername = intransient_str("workername", field); - PQ_GET_FLD(res, i, "clientid", field, ok); + CKPQ_VAL_FLD_num(res, i, clientid, field, ok); if (!ok) break; TXT_TO_INT("clientid", field, row->clientid); - PQ_GET_FLD(res, i, "enonce1", field, ok); + CKPQ_VAL_FLD_num(res, i, enonce1, field, ok); if (!ok) break; TXT_TO_STR("enonce1", field, row->enonce1); - PQ_GET_FLD(res, i, "nonce2", field, ok); + CKPQ_VAL_FLD_num(res, i, nonce2, field, ok); if (!ok) break; TXT_TO_STR("nonce2", field, row->nonce2); - PQ_GET_FLD(res, i, "nonce", field, ok); + CKPQ_VAL_FLD_num(res, i, nonce, field, ok); if (!ok) break; TXT_TO_STR("nonce", field, row->nonce); - PQ_GET_FLD(res, i, "reward", field, ok); + CKPQ_VAL_FLD_num(res, i, reward, field, ok); if (!ok) break; TXT_TO_BIGINT("reward", field, row->reward); - PQ_GET_FLD(res, i, "confirmed", field, ok); + CKPQ_VAL_FLD_num(res, i, confirmed, field, ok); if (!ok) break; TXT_TO_STR("confirmed", field, row->confirmed); - PQ_GET_FLD(res, i, "info", field, ok); + CKPQ_VAL_FLD_num(res, i, info, field, ok); if (!ok) break; TXT_TO_STR("info", field, row->info); - PQ_GET_FLD(res, i, "diffacc", field, ok); + CKPQ_VAL_FLD_num(res, i, diffacc, field, ok); if (!ok) break; TXT_TO_DOUBLE("diffacc", field, row->diffacc); - PQ_GET_FLD(res, i, "diffinv", field, ok); + CKPQ_VAL_FLD_num(res, i, diffinv, field, ok); if (!ok) break; TXT_TO_DOUBLE("diffinv", field, row->diffinv); - PQ_GET_FLD(res, i, "shareacc", field, ok); + CKPQ_VAL_FLD_num(res, i, shareacc, field, ok); if (!ok) break; TXT_TO_DOUBLE("shareacc", field, row->shareacc); - PQ_GET_FLD(res, i, "shareinv", field, ok); + CKPQ_VAL_FLD_num(res, i, shareinv, field, ok); if (!ok) break; TXT_TO_DOUBLE("shareinv", field, row->shareinv); - PQ_GET_FLD(res, i, "elapsed", field, ok); + CKPQ_VAL_FLD_num(res, i, elapsed, field, ok); if (!ok) break; TXT_TO_BIGINT("elapsed", field, row->elapsed); - PQ_GET_FLD(res, i, "statsconfirmed", field, ok); + CKPQ_VAL_FLD_num(res, i, statsconfirmed, field, ok); if (!ok) break; TXT_TO_STR("statsconfirmed", field, row->statsconfirmed); @@ -6639,6 +6752,10 @@ bool miningpayouts_fill(PGconn *conn) STRNCPY(tickbuf, TICK_PREFIX"mp 0"); cr_msg(false, tickbuf); + + int payoutid_num, userid_num, diffacc_num, amount_num; + HISTORYDATE_num; + sel = "declare mp cursor for select " "payoutid,userid,diffacc,amount" HISTORYDATECONTROL @@ -6684,6 +6801,8 @@ bool miningpayouts_fill(PGconn *conn) n = 0; ok = true; + payoutid_num = userid_num = diffacc_num = amount_num = CKPQFUNDEF; + HISTORYDATE_init; K_WLOCK(miningpayouts_free); while ((t = PQntuples(res)) > 0) { for (i = 0; i < t; i++) { @@ -6696,22 +6815,22 @@ bool miningpayouts_fill(PGconn *conn) break; } - PQ_GET_FLD(res, i, "payoutid", field, ok); + CKPQ_VAL_FLD_num(res, i, payoutid, field, ok); if (!ok) break; TXT_TO_BIGINT("payoutid", field, row->payoutid); - PQ_GET_FLD(res, i, "userid", field, ok); + CKPQ_VAL_FLD_num(res, i, userid, field, ok); if (!ok) break; TXT_TO_BIGINT("userid", field, row->userid); - PQ_GET_FLD(res, i, "diffacc", field, ok); + CKPQ_VAL_FLD_num(res, i, diffacc, field, ok); if (!ok) break; TXT_TO_DOUBLE("diffacc", field, row->diffacc); - PQ_GET_FLD(res, i, "amount", field, ok); + CKPQ_VAL_FLD_num(res, i, amount, field, ok); if (!ok) break; TXT_TO_BIGINT("amount", field, row->amount); @@ -6723,7 +6842,7 @@ bool miningpayouts_fill(PGconn *conn) add_to_ktree(miningpayouts_root, item); k_add_head(miningpayouts_store, item); - if (n == 0 || ((n+1) % 100000) == 0) { + if (n == 0 || ((n+1) % FETCHTICK) == 0) { pcom(n+1, pcombuf, sizeof(pcombuf)); snprintf(tickbuf, sizeof(tickbuf), TICK_PREFIX"mp %s", pcombuf); @@ -6733,13 +6852,15 @@ bool miningpayouts_fill(PGconn *conn) n++; } CKPQClear(res); - res = CKPQExec(conn, "fetch 9999 in mp", CKPQ_READ); + res = CKPQExec(conn, "fetch "CKPQFETCHSTR" in mp", CKPQ_READ); rescode = CKPQResultStatus(res); if (!PGOK(rescode)) { PGLOGERR("Fetch next", rescode, conn); ok = false; break; } + payoutid_num = userid_num = diffacc_num = amount_num = CKPQFUNDEF; + HISTORYDATE_init; } if (!ok) k_add_head(miningpayouts_free, item); @@ -7174,6 +7295,12 @@ bool payouts_fill(PGconn *conn) LOGDEBUG("%s(): select", __func__); + int payoutid_num, height_num, blockhash_num, minerreward_num; + int workinfoidstart_num, workinfoidend_num, elapsed_num, status_num; + int diffwanted_num, diffused_num, shareacc_num, lastshareacc_num; + int stats_num; + HISTORYDATE_num; + sel = "select " "payoutid,height,blockhash,minerreward,workinfoidstart,workinfoidend," "elapsed,status,diffwanted,diffused,shareacc,lastshareacc,stats" @@ -7198,6 +7325,11 @@ bool payouts_fill(PGconn *conn) n = PQntuples(res); LOGDEBUG("%s(): tree build count %d", __func__, n); ok = true; + payoutid_num = height_num = blockhash_num = minerreward_num = + workinfoidstart_num = workinfoidend_num = elapsed_num = status_num = + diffwanted_num = diffused_num = shareacc_num = lastshareacc_num = + stats_num = CKPQFUNDEF; + HISTORYDATE_init; K_WLOCK(payouts_free); for (i = 0; i < n; i++) { item = k_unlink_head(payouts_free); @@ -7209,67 +7341,67 @@ bool payouts_fill(PGconn *conn) break; } - PQ_GET_FLD(res, i, "payoutid", field, ok); + CKPQ_VAL_FLD_num(res, i, payoutid, field, ok); if (!ok) break; TXT_TO_BIGINT("payoutid", field, row->payoutid); - PQ_GET_FLD(res, i, "height", field, ok); + CKPQ_VAL_FLD_num(res, i, height, field, ok); if (!ok) break; TXT_TO_INT("height", field, row->height); - PQ_GET_FLD(res, i, "blockhash", field, ok); + CKPQ_VAL_FLD_num(res, i, blockhash, field, ok); if (!ok) break; TXT_TO_STR("blockhash", field, row->blockhash); - PQ_GET_FLD(res, i, "minerreward", field, ok); + CKPQ_VAL_FLD_num(res, i, minerreward, field, ok); if (!ok) break; TXT_TO_BIGINT("minerreward", field, row->minerreward); - PQ_GET_FLD(res, i, "workinfoidstart", field, ok); + CKPQ_VAL_FLD_num(res, i, workinfoidstart, field, ok); if (!ok) break; TXT_TO_BIGINT("workinfoidstart", field, row->workinfoidstart); - PQ_GET_FLD(res, i, "workinfoidend", field, ok); + CKPQ_VAL_FLD_num(res, i, workinfoidend, field, ok); if (!ok) break; TXT_TO_BIGINT("workinfoidend", field, row->workinfoidend); - PQ_GET_FLD(res, i, "elapsed", field, ok); + CKPQ_VAL_FLD_num(res, i, elapsed, field, ok); if (!ok) break; TXT_TO_BIGINT("elapsed", field, row->elapsed); - PQ_GET_FLD(res, i, "status", field, ok); + CKPQ_VAL_FLD_num(res, i, status, field, ok); if (!ok) break; TXT_TO_STR("status", field, row->status); - PQ_GET_FLD(res, i, "diffwanted", field, ok); + CKPQ_VAL_FLD_num(res, i, diffwanted, field, ok); if (!ok) break; TXT_TO_DOUBLE("diffwanted", field, row->diffwanted); - PQ_GET_FLD(res, i, "diffused", field, ok); + CKPQ_VAL_FLD_num(res, i, diffused, field, ok); if (!ok) break; TXT_TO_DOUBLE("diffused", field, row->diffused); - PQ_GET_FLD(res, i, "shareacc", field, ok); + CKPQ_VAL_FLD_num(res, i, shareacc, field, ok); if (!ok) break; TXT_TO_DOUBLE("shareacc", field, row->shareacc); - PQ_GET_FLD(res, i, "lastshareacc", field, ok); + CKPQ_VAL_FLD_num(res, i, lastshareacc, field, ok); if (!ok) break; TXT_TO_TVDB("lastshareacc", field, row->lastshareacc); - PQ_GET_FLD(res, i, "stats", field, ok); + CKPQ_VAL_FLD_num(res, i, stats, field, ok); if (!ok) break; TXT_TO_BLOB("stats", field, row->stats); @@ -7929,6 +8061,10 @@ bool poolstats_fill(PGconn *conn) tm.tm_sec, tzinfo); + int poolinstance_num, elapsed_num, users_num, workers_num, hashrate_num; + int hashrate5m_num, hashrate1hr_num, hashrate24hr_num; + SIMPLEDATE_num; + APPEND_REALLOC_INIT(sel, off, len); APPEND_REALLOC(sel, off, len, "select " @@ -7959,6 +8095,10 @@ bool poolstats_fill(PGconn *conn) n = PQntuples(res); LOGDEBUG("%s(): tree build count %d", __func__, n); ok = true; + poolinstance_num = elapsed_num = users_num = workers_num = + hashrate_num = hashrate5m_num = hashrate1hr_num = + hashrate24hr_num = CKPQFUNDEF; + SIMPLEDATE_init; K_WLOCK(poolstats_free); for (i = 0; i < n; i++) { item = k_unlink_head(poolstats_free); @@ -7972,7 +8112,7 @@ bool poolstats_fill(PGconn *conn) row->stored = true; - PQ_GET_FLD(res, i, "poolinstance", field, ok); + CKPQ_VAL_FLD_num(res, i, poolinstance, field, ok); if (!ok) break; if (sys_poolinstance && strcmp(field, sys_poolinstance)) { @@ -7982,37 +8122,37 @@ bool poolstats_fill(PGconn *conn) } row->in_poolinstance = intransient_str("poolinstance", field); - PQ_GET_FLD(res, i, "elapsed", field, ok); + CKPQ_VAL_FLD_num(res, i, elapsed, field, ok); if (!ok) break; TXT_TO_BIGINT("elapsed", field, row->elapsed); - PQ_GET_FLD(res, i, "users", field, ok); + CKPQ_VAL_FLD_num(res, i, users, field, ok); if (!ok) break; TXT_TO_INT("users", field, row->users); - PQ_GET_FLD(res, i, "workers", field, ok); + CKPQ_VAL_FLD_num(res, i, workers, field, ok); if (!ok) break; TXT_TO_INT("workers", field, row->workers); - PQ_GET_FLD(res, i, "hashrate", field, ok); + CKPQ_VAL_FLD_num(res, i, hashrate, field, ok); if (!ok) break; TXT_TO_DOUBLE("hashrate", field, row->hashrate); - PQ_GET_FLD(res, i, "hashrate5m", field, ok); + CKPQ_VAL_FLD_num(res, i, hashrate5m, field, ok); if (!ok) break; TXT_TO_DOUBLE("hashrate5m", field, row->hashrate5m); - PQ_GET_FLD(res, i, "hashrate1hr", field, ok); + CKPQ_VAL_FLD_num(res, i, hashrate1hr, field, ok); if (!ok) break; TXT_TO_DOUBLE("hashrate1hr", field, row->hashrate1hr); - PQ_GET_FLD(res, i, "hashrate24hr", field, ok); + CKPQ_VAL_FLD_num(res, i, hashrate24hr, field, ok); if (!ok) break; TXT_TO_DOUBLE("hashrate24hr", field, row->hashrate24hr); @@ -8390,6 +8530,13 @@ bool markersummary_fill(PGconn *conn) } } + int markerid_num, userid_num, workername_num, diffacc_num, diffsta_num; + int diffdup_num, diffhi_num, diffrej_num, shareacc_num, sharesta_num; + int sharedup_num, sharehi_num, sharerej_num, sharecount_num; + int errorcount_num, firstshare_num, lastshare_num, firstshareacc_num; + int lastshareacc_num, lastdiffacc_num; + MODIFYDATE_num; + // TODO: limit how far back sel = "declare ws cursor for select " "markerid,userid,workername,diffacc,diffsta,diffdup,diffhi," @@ -8453,6 +8600,12 @@ bool markersummary_fill(PGconn *conn) n = 0; ok = true; + markerid_num = userid_num = workername_num = diffacc_num = diffsta_num = + diffdup_num = diffhi_num = diffrej_num = shareacc_num = sharesta_num = + sharedup_num = sharehi_num = sharerej_num = sharecount_num = + errorcount_num = firstshare_num = lastshare_num = firstshareacc_num = + lastshareacc_num = lastdiffacc_num = CKPQFUNDEF; + MODIFYDATE_init; K_WLOCK(markersummary_free); while ((t = PQntuples(res)) > 0) { // Avoid locking them too many times @@ -8468,102 +8621,102 @@ bool markersummary_fill(PGconn *conn) break; } - PQ_GET_FLD(res, i, "markerid", field, ok); + CKPQ_VAL_FLD_num(res, i, markerid, field, ok); if (!ok) break; TXT_TO_BIGINT("markerid", field, row->markerid); - PQ_GET_FLD(res, i, "userid", field, ok); + CKPQ_VAL_FLD_num(res, i, userid, field, ok); if (!ok) break; TXT_TO_BIGINT("userid", field, row->userid); - PQ_GET_FLD(res, i, "workername", field, ok); + CKPQ_VAL_FLD_num(res, i, workername, field, ok); if (!ok) break; row->in_workername = intransient_str("workername", field); - PQ_GET_FLD(res, i, "diffacc", field, ok); + CKPQ_VAL_FLD_num(res, i, diffacc, field, ok); if (!ok) break; TXT_TO_DOUBLE("diffacc", field, row->diffacc); - PQ_GET_FLD(res, i, "diffsta", field, ok); + CKPQ_VAL_FLD_num(res, i, diffsta, field, ok); if (!ok) break; TXT_TO_DOUBLE("diffsta", field, row->diffsta); - PQ_GET_FLD(res, i, "diffdup", field, ok); + CKPQ_VAL_FLD_num(res, i, diffdup, field, ok); if (!ok) break; TXT_TO_DOUBLE("diffdup", field, row->diffdup); - PQ_GET_FLD(res, i, "diffhi", field, ok); + CKPQ_VAL_FLD_num(res, i, diffhi, field, ok); if (!ok) break; TXT_TO_DOUBLE("diffhi", field, row->diffhi); - PQ_GET_FLD(res, i, "diffrej", field, ok); + CKPQ_VAL_FLD_num(res, i, diffrej, field, ok); if (!ok) break; TXT_TO_DOUBLE("diffrej", field, row->diffrej); - PQ_GET_FLD(res, i, "shareacc", field, ok); + CKPQ_VAL_FLD_num(res, i, shareacc, field, ok); if (!ok) break; TXT_TO_DOUBLE("shareacc", field, row->shareacc); - PQ_GET_FLD(res, i, "sharesta", field, ok); + CKPQ_VAL_FLD_num(res, i, sharesta, field, ok); if (!ok) break; TXT_TO_DOUBLE("sharesta", field, row->sharesta); - PQ_GET_FLD(res, i, "sharedup", field, ok); + CKPQ_VAL_FLD_num(res, i, sharedup, field, ok); if (!ok) break; TXT_TO_DOUBLE("sharedup", field, row->sharedup); - PQ_GET_FLD(res, i, "sharehi", field, ok); + CKPQ_VAL_FLD_num(res, i, sharehi, field, ok); if (!ok) break; TXT_TO_DOUBLE("sharehi", field, row->sharehi); - PQ_GET_FLD(res, i, "sharerej", field, ok); + CKPQ_VAL_FLD_num(res, i, sharerej, field, ok); if (!ok) break; TXT_TO_DOUBLE("sharerej", field, row->sharerej); - PQ_GET_FLD(res, i, "sharecount", field, ok); + CKPQ_VAL_FLD_num(res, i, sharecount, field, ok); if (!ok) break; TXT_TO_BIGINT("sharecount", field, row->sharecount); - PQ_GET_FLD(res, i, "errorcount", field, ok); + CKPQ_VAL_FLD_num(res, i, errorcount, field, ok); if (!ok) break; TXT_TO_BIGINT("errorcount", field, row->errorcount); - PQ_GET_FLD(res, i, "firstshare", field, ok); + CKPQ_VAL_FLD_num(res, i, firstshare, field, ok); if (!ok) break; TXT_TO_TVDB("firstshare", field, row->firstshare); - PQ_GET_FLD(res, i, "lastshare", field, ok); + CKPQ_VAL_FLD_num(res, i, lastshare, field, ok); if (!ok) break; TXT_TO_TVDB("lastshare", field, row->lastshare); - PQ_GET_FLD(res, i, "firstshareacc", field, ok); + CKPQ_VAL_FLD_num(res, i, firstshareacc, field, ok); if (!ok) break; TXT_TO_TVDB("firstshareacc", field, row->firstshareacc); - PQ_GET_FLD(res, i, "lastshareacc", field, ok); + CKPQ_VAL_FLD_num(res, i, lastshareacc, field, ok); if (!ok) break; TXT_TO_TVDB("lastshareacc", field, row->lastshareacc); - PQ_GET_FLD(res, i, "lastdiffacc", field, ok); + CKPQ_VAL_FLD_num(res, i, lastdiffacc, field, ok); if (!ok) break; TXT_TO_DOUBLE("lastdiffacc", field, row->lastdiffacc); @@ -8605,7 +8758,7 @@ bool markersummary_fill(PGconn *conn) userinfo_update(NULL, NULL, row, false); - if (n == 0 || ((n+1) % 100000) == 0) { + if (n == 0 || ((n+1) % FETCHTICK) == 0) { pcom(n+1, pcombuf, sizeof(pcombuf)); snprintf(tickbuf, sizeof(tickbuf), TICK_PREFIX"ms %s", pcombuf); @@ -8617,13 +8770,20 @@ bool markersummary_fill(PGconn *conn) K_WUNLOCK(userinfo_free); K_RUNLOCK(workmarkers_free); CKPQClear(res); - res = CKPQExec(conn, "fetch 9999 in ws", CKPQ_READ); + res = CKPQExec(conn, "fetch "CKPQFETCHSTR" in ws", CKPQ_READ); rescode = CKPQResultStatus(res); if (!PGOK(rescode)) { PGLOGERR("Fetch next", rescode, conn); ok = false; break; } + markerid_num = userid_num = workername_num = diffacc_num = + diffsta_num = diffdup_num = diffhi_num = diffrej_num = + shareacc_num = sharesta_num = sharedup_num = sharehi_num = + sharerej_num = sharecount_num = errorcount_num = + firstshare_num = lastshare_num = firstshareacc_num = + lastshareacc_num = lastdiffacc_num = CKPQFUNDEF; + MODIFYDATE_init; } if (!ok) { free_markersummary_data(item); @@ -8940,6 +9100,10 @@ bool workmarkers_fill(PGconn *conn) LOGDEBUG("%s(): select", __func__); + int markerid_num, poolinstance_num, workinfoidend_num; + int workinfoidstart_num, description_num, status_num; + HISTORYDATE_num; + // Allow limiting the load for key_update if (key_update && dbload_workinfoid_start != -1) { sel = "select " @@ -8977,6 +9141,9 @@ bool workmarkers_fill(PGconn *conn) n = PQntuples(res); LOGDEBUG("%s(): tree build count %d", __func__, n); ok = true; + markerid_num = poolinstance_num = workinfoidend_num = + workinfoidstart_num = description_num = status_num = CKPQFUNDEF; + HISTORYDATE_init; K_WLOCK(workmarkers_free); for (i = 0; i < n; i++) { item = k_unlink_head(workmarkers_free); @@ -8988,7 +9155,7 @@ bool workmarkers_fill(PGconn *conn) break; } - PQ_GET_FLD(res, i, "poolinstance", field, ok); + CKPQ_VAL_FLD_num(res, i, poolinstance, field, ok); if (!ok) break; if (sys_poolinstance && strcmp(field, sys_poolinstance)) { @@ -8998,28 +9165,28 @@ bool workmarkers_fill(PGconn *conn) } row->in_poolinstance = intransient_str("poolinstance", field); - PQ_GET_FLD(res, i, "markerid", field, ok); + CKPQ_VAL_FLD_num(res, i, markerid, field, ok); if (!ok) break; TXT_TO_BIGINT("markerid", field, row->markerid); - PQ_GET_FLD(res, i, "workinfoidend", field, ok); + CKPQ_VAL_FLD_num(res, i, workinfoidend, field, ok); if (!ok) break; TXT_TO_BIGINT("workinfoidend", field, row->workinfoidend); - PQ_GET_FLD(res, i, "workinfoidstart", field, ok); + CKPQ_VAL_FLD_num(res, i, workinfoidstart, field, ok); if (!ok) break; TXT_TO_BIGINT("workinfoidstart", field, row->workinfoidstart); - PQ_GET_FLD(res, i, "description", field, ok); + CKPQ_VAL_FLD_num(res, i, description, field, ok); if (!ok) break; TXT_TO_PTR("description", field, row->description); LIST_MEM_ADD(workmarkers_free, row->description); - PQ_GET_FLD(res, i, "status", field, ok); + CKPQ_VAL_FLD_num(res, i, status, field, ok); if (!ok) break; TXT_TO_STR("status", field, row->status); @@ -9260,6 +9427,10 @@ bool marks_fill(PGconn *conn) LOGDEBUG("%s(): select", __func__); + int poolinstance_num, workinfoid_num, description_num, extra_num; + int marktype_num, status_num; + HISTORYDATE_num; + // TODO: limit how far back sel = "select " "poolinstance,workinfoid,description,extra,marktype,status" @@ -9284,6 +9455,9 @@ bool marks_fill(PGconn *conn) n = PQntuples(res); LOGDEBUG("%s(): tree build count %d", __func__, n); ok = true; + poolinstance_num = workinfoid_num = description_num = extra_num = + marktype_num = status_num = CKPQFUNDEF; + HISTORYDATE_init; K_WLOCK(marks_free); for (i = 0; i < n; i++) { item = k_unlink_head(marks_free); @@ -9295,7 +9469,7 @@ bool marks_fill(PGconn *conn) break; } - PQ_GET_FLD(res, i, "poolinstance", field, ok); + CKPQ_VAL_FLD_num(res, i, poolinstance, field, ok); if (!ok) break; if (sys_poolinstance && strcmp(field, sys_poolinstance)) { @@ -9305,29 +9479,29 @@ bool marks_fill(PGconn *conn) } row->in_poolinstance = intransient_str("poolinstance", field); - PQ_GET_FLD(res, i, "workinfoid", field, ok); + CKPQ_VAL_FLD_num(res, i, workinfoid, field, ok); if (!ok) break; TXT_TO_BIGINT("workinfoid", field, row->workinfoid); - PQ_GET_FLD(res, i, "description", field, ok); + CKPQ_VAL_FLD_num(res, i, description, field, ok); if (!ok) break; TXT_TO_PTR("description", field, row->description); LIST_MEM_ADD(marks_free, row->description); - PQ_GET_FLD(res, i, "extra", field, ok); + CKPQ_VAL_FLD_num(res, i, extra, field, ok); if (!ok) break; TXT_TO_PTR("extra", field, row->extra); LIST_MEM_ADD(marks_free, row->extra); - PQ_GET_FLD(res, i, "marktype", field, ok); + CKPQ_VAL_FLD_num(res, i, marktype, field, ok); if (!ok) break; TXT_TO_STR("marktype", field, row->marktype); - PQ_GET_FLD(res, i, "status", field, ok); + CKPQ_VAL_FLD_num(res, i, status, field, ok); if (!ok) break; TXT_TO_STR("status", field, row->status); @@ -9367,7 +9541,7 @@ bool check_db_version(PGconn *conn) char *pgv; int fields = 3; bool ok; - int n; + int n, f; LOGDEBUG("%s(): select", __func__); @@ -9397,7 +9571,8 @@ bool check_db_version(PGconn *conn) } ok = true; - PQ_GET_FLD(res, 0, "vlock", field, ok); + f = CKPQFUNDEF; + CKPQ_VAL_FLD(res, 0, f, "vlock", field, ok); if (!ok) { LOGEMERG("%s(): Missing field vlock", __func__); CKPQClear(res); @@ -9412,7 +9587,8 @@ bool check_db_version(PGconn *conn) } ok = true; - PQ_GET_FLD(res, 0, "version", field, ok); + f = CKPQFUNDEF; + CKPQ_VAL_FLD(res, 0, f, "version", field, ok); if (!ok) { LOGEMERG("%s(): Missing field version", __func__); CKPQClear(res); @@ -9426,7 +9602,9 @@ bool check_db_version(PGconn *conn) return false; } - PQ_GET_FLD(res, 0, "pgv", field, ok); + ok = true; + f = CKPQFUNDEF; + CKPQ_VAL_FLD(res, 0, f, "pgv", field, ok); if (ok) pgv = strdup(field); else From 67ba50fe8968d671e13d35cf6eb3bc38511a31a8 Mon Sep 17 00:00:00 2001 From: kanoi Date: Wed, 12 Oct 2016 16:18:49 +1100 Subject: [PATCH 18/33] ckdb - fix optioncontrol DLPRIO change --- src/ckdb.c | 6 +++--- src/ckdb.h | 2 +- src/ckdb_dbio.c | 22 ++++++++++++++++------ 3 files changed, 20 insertions(+), 10 deletions(-) diff --git a/src/ckdb.c b/src/ckdb.c index 9e78c097..b3c6cb0e 100644 --- a/src/ckdb.c +++ b/src/ckdb.c @@ -2278,9 +2278,6 @@ static void alloc_storage() userinfo_root = new_ktree(NULL, cmp_userinfo, userinfo_free); #if LOCK_CHECK - // Above all needed for optioncontrol_trigger() - DLPRIO(optioncontrol, 96); - DLPRIO(seqset, 91); DLPRIO(transfer, 90); @@ -2306,6 +2303,9 @@ static void alloc_storage() DLPRIO(userinfo, 50); + // Uses event_limits + DLPRIO(optioncontrol, 49); + // Needs to check users and ips and uses events_limits DLPRIO(events, 48); DLPRIO(ovents, 47); diff --git a/src/ckdb.h b/src/ckdb.h index a7ce6052..a6c25058 100644 --- a/src/ckdb.h +++ b/src/ckdb.h @@ -58,7 +58,7 @@ #define DB_VLOCK "1" #define DB_VERSION "1.0.7" -#define CKDB_VERSION DB_VERSION"-2.700" +#define CKDB_VERSION DB_VERSION"-2.701" #define WHERE_FFL " - from %s %s() line %d" #define WHERE_FFL_HERE __FILE__, __func__, __LINE__ diff --git a/src/ckdb_dbio.c b/src/ckdb_dbio.c index 296005a6..54671d4f 100644 --- a/src/ckdb_dbio.c +++ b/src/ckdb_dbio.c @@ -2871,6 +2871,8 @@ void oc_trf(OPTIONCONTROL *oc, const char *from) } } +/* Trigger functions shouldn't touch expirydate since the optioncontrol data + * isn't locked - that's the only field that could change */ OC_TRIGGER oc_trigger[] = { { SWITCH_STATE_NAME, true, oc_switch_state }, { DIFF_PERCENT_NAME, true, oc_diff_percent }, @@ -2883,7 +2885,11 @@ OC_TRIGGER oc_trigger[] = { /* For oc items that aren't date/height controlled, and use global variables * rather than having to look up the value every time it's needed - * Called from within the write lock that loaded/added the oc_item */ + * Called from within the write lock that loaded/added the oc_item + * The write lock is released before calling the trigger function, + * and regained after it returns, thus expirydate shouldn't be accessed + * since it's value could be changed by another thread, under lock + * None of the other oc field values will change */ static void optioncontrol_trigger(K_ITEM *oc_item, const char *from) { char cd_buf[DATE_BUFSIZ], cd2_buf[DATE_BUFSIZ]; @@ -2909,6 +2915,9 @@ static void optioncontrol_trigger(K_ITEM *oc_item, const char *from) } } if (got > -1) { + /* Don't hold the lock during debug messages or + * during the call to the trigger function */ + K_WUNLOCK(optioncontrol_free); // If it's date/height controlled, display an ERR if (oc->activationheight != OPTIONCONTROL_HEIGHT || tv_newer(&date_begin, &(oc->activationdate))) @@ -2925,6 +2934,7 @@ static void optioncontrol_trigger(K_ITEM *oc_item, const char *from) OPTIONCONTROL_HEIGHT, cd2_buf); } else oc_trigger[i].func(oc, from); + K_WLOCK(optioncontrol_free); } } } @@ -2936,7 +2946,7 @@ K_ITEM *optioncontrol_item_add(PGconn *conn, K_ITEM *oc_item, tv_t *cd, bool beg K_TREE_CTX ctx[1]; PGresult *res; K_ITEM *old_item, look; - OPTIONCONTROL *row; + OPTIONCONTROL *row, *oc; char *upd, *ins; bool ok = false; char *params[4 + HISTORYDATECOUNT]; @@ -3029,12 +3039,12 @@ nostart: free_optioncontrol_data(oc_item); k_add_head(optioncontrol_free, oc_item); } else { - // Discard old + // Keep old to ensure the new item can be read outside lock if (old_item) { remove_from_ktree(optioncontrol_root, old_item); - k_unlink_item(optioncontrol_store, old_item); - free_optioncontrol_data(old_item); - k_add_head(optioncontrol_free, old_item); + DATA_OPTIONCONTROL(oc, old_item); + copy_tv(&(oc->expirydate), cd); + add_to_ktree(optioncontrol_root, old_item); } add_to_ktree(optioncontrol_root, oc_item); k_add_head(optioncontrol_store, oc_item); From ff8f097cc9c0472f758bf5e0521b9629861b51ba Mon Sep 17 00:00:00 2001 From: kanoi Date: Wed, 12 Oct 2016 17:41:51 +1100 Subject: [PATCH 19/33] ckdb - add commas to db load record counts --- src/ckdb.h | 2 +- src/ckdb_dbio.c | 74 ++++++++++++++++++++++++++++++++++++++----------- 2 files changed, 59 insertions(+), 17 deletions(-) diff --git a/src/ckdb.h b/src/ckdb.h index a6c25058..c7a20e50 100644 --- a/src/ckdb.h +++ b/src/ckdb.h @@ -58,7 +58,7 @@ #define DB_VLOCK "1" #define DB_VERSION "1.0.7" -#define CKDB_VERSION DB_VERSION"-2.701" +#define CKDB_VERSION DB_VERSION"-2.702" #define WHERE_FFL " - from %s %s() line %d" #define WHERE_FFL_HERE __FILE__, __func__, __LINE__ diff --git a/src/ckdb_dbio.c b/src/ckdb_dbio.c index 54671d4f..aff62a08 100644 --- a/src/ckdb_dbio.c +++ b/src/ckdb_dbio.c @@ -978,6 +978,7 @@ K_ITEM *create_missing_user(PGconn *conn, char *username, char *secondaryuserid, bool users_fill(PGconn *conn) { + char pcombuf[64]; ExecStatusType rescode; PGresult *res; K_ITEM *item; @@ -1107,7 +1108,9 @@ bool users_fill(PGconn *conn) if (ok) { LOGDEBUG("%s(): built", __func__); - LOGWARNING("%s(): loaded %d users records", __func__, n); + pcom(n, pcombuf, sizeof(pcombuf)); + LOGWARNING("%s(): loaded %s users records", + __func__, pcombuf); } return ok; @@ -1362,6 +1365,7 @@ unparam: bool useratts_fill(PGconn *conn) { + char pcombuf[64]; ExecStatusType rescode; PGresult *res; K_ITEM *item; @@ -1476,7 +1480,9 @@ bool useratts_fill(PGconn *conn) if (ok) { LOGDEBUG("%s(): built", __func__); - LOGWARNING("%s(): loaded %d useratts records", __func__, n); + pcom(n, pcombuf, sizeof(pcombuf)); + LOGWARNING("%s(): loaded %s useratts records", + __func__, pcombuf); } return ok; @@ -1763,6 +1769,7 @@ early: bool workers_fill(PGconn *conn) { + char pcombuf[64]; ExecStatusType rescode; PGresult *res; K_ITEM *item = NULL; @@ -1911,7 +1918,9 @@ flail: if (ok) { LOGDEBUG("%s(): built", __func__); - LOGWARNING("%s(): fetched %d workers records", __func__, n); + pcom(n, pcombuf, sizeof(pcombuf)); + LOGWARNING("%s(): fetched %s workers records", + __func__, pcombuf); } return ok; @@ -2154,6 +2163,7 @@ unparam: bool paymentaddresses_fill(PGconn *conn) { + char pcombuf[64]; ExecStatusType rescode; PGresult *res; K_ITEM *item; @@ -2248,7 +2258,9 @@ bool paymentaddresses_fill(PGconn *conn) if (ok) { LOGDEBUG("%s(): built", __func__); - LOGWARNING("%s(): loaded %d paymentaddresses records", __func__, n); + pcom(n, pcombuf, sizeof(pcombuf)); + LOGWARNING("%s(): loaded %s paymentaddresses records", + __func__, pcombuf); } return ok; @@ -2567,7 +2579,9 @@ flail: CKPQCommit(conn); if (ok) { LOGDEBUG("%s(): built", __func__); - LOGWARNING("%s(): fetched %d payments records", __func__, n); + pcom(n, pcombuf, sizeof(pcombuf)); + LOGWARNING("%s(): fetched %s payments records", + __func__, pcombuf); } return ok; @@ -3099,6 +3113,7 @@ K_ITEM *optioncontrol_add(PGconn *conn, char *optionname, char *optionvalue, bool optioncontrol_fill(PGconn *conn) { + char pcombuf[64]; ExecStatusType rescode; PGresult *res; K_ITEM *item; @@ -3201,7 +3216,9 @@ bool optioncontrol_fill(PGconn *conn) if (ok) { LOGDEBUG("%s(): built", __func__); - LOGWARNING("%s(): loaded %d optioncontrol records", __func__, n); + pcom(n, pcombuf, sizeof(pcombuf)); + LOGWARNING("%s(): loaded %s optioncontrol records", + __func__, pcombuf); LOGWARNING("%s() switch_state initially %d", __func__, switch_state); @@ -3605,7 +3622,9 @@ flail: if (ok) { LOGDEBUG("%s(): built", __func__); - LOGWARNING("%s(): fetched %d workinfo records", __func__, n); + pcom(n, pcombuf, sizeof(pcombuf)); + LOGWARNING("%s(): fetched %s workinfo records", + __func__, pcombuf); } POOLINSTANCE_DBLOAD_MSG(workinfo); @@ -4380,7 +4399,9 @@ flail: CKPQCommit(conn); if (ok) { LOGDEBUG("%s(): built", __func__); - LOGWARNING("%s(): fetched %d shares records", __func__, n); + pcom(n, pcombuf, sizeof(pcombuf)); + LOGWARNING("%s(): fetched %s shares records", + __func__, pcombuf); if (no_addr || no_agent) { if (no_addr == no_agent) { LOGWARNING(" %d had no address and agent", @@ -6412,6 +6433,7 @@ flail: bool blocks_fill(PGconn *conn) { + char pcombuf[64]; ExecStatusType rescode; PGresult *res; K_TREE_CTX ctx[1]; @@ -6611,7 +6633,9 @@ bool blocks_fill(PGconn *conn) if (ok) { LOGDEBUG("%s(): built", __func__); - LOGWARNING("%s(): loaded %d blocks records", __func__, n); + pcom(n, pcombuf, sizeof(pcombuf)); + LOGWARNING("%s(): loaded %s blocks records", + __func__, pcombuf); } return ok; @@ -6882,7 +6906,9 @@ flail: CKPQCommit(conn); if (ok) { LOGDEBUG("%s(): built", __func__); - LOGWARNING("%s(): fetched %d miningpayout records", __func__, n); + pcom(n, pcombuf, sizeof(pcombuf)); + LOGWARNING("%s(): fetched %s miningpayout records", + __func__, pcombuf); } return ok; @@ -7291,6 +7317,7 @@ matane: bool payouts_fill(PGconn *conn) { + char pcombuf[64]; ExecStatusType rescode; PGresult *res; K_ITEM *item, *b_item; @@ -7459,7 +7486,9 @@ bool payouts_fill(PGconn *conn) if (ok) { LOGDEBUG("%s(): built", __func__); - LOGWARNING("%s(): loaded %d payout records", __func__, n); + pcom(n, pcombuf, sizeof(pcombuf)); + LOGWARNING("%s(): loaded %s payout records", + __func__, pcombuf); } return ok; @@ -8023,6 +8052,7 @@ unparam: // TODO: data selection - only require ? bool poolstats_fill(PGconn *conn) { + char pcombuf[64]; ExecStatusType rescode; PGresult *res; K_ITEM *item; @@ -8187,7 +8217,9 @@ bool poolstats_fill(PGconn *conn) if (ok) { LOGDEBUG("%s(): built", __func__); - LOGWARNING("%s(): loaded %d poolstats records", __func__, n); + pcom(n, pcombuf, sizeof(pcombuf)); + LOGWARNING("%s(): loaded %s poolstats records", + __func__, pcombuf); } clean: free(sel); @@ -8813,8 +8845,12 @@ flail: if (ok) { LOGDEBUG("%s(): built", __func__); - LOGWARNING("%s(): fetched %d markersummary records", __func__, n); - LOGWARNING("%s(): created %d markersummary pool records", __func__, p_n); + pcom(n, pcombuf, sizeof(pcombuf)); + LOGWARNING("%s(): fetched %s markersummary records", + __func__, pcombuf); + pcom(p_n, pcombuf, sizeof(pcombuf)); + LOGWARNING("%s(): created %s markersummary pool records", + __func__, pcombuf); } return ok; @@ -9096,6 +9132,7 @@ unparam: bool workmarkers_fill(PGconn *conn) { + char pcombuf[64]; ExecStatusType rescode; PGresult *res; K_ITEM *item, *wi_item; @@ -9268,7 +9305,9 @@ bool workmarkers_fill(PGconn *conn) if (ok) { LOGDEBUG("%s(): built", __func__); - LOGWARNING("%s(): loaded %d workmarkers records", __func__, n); + pcom(n, pcombuf, sizeof(pcombuf)); + LOGWARNING("%s(): loaded %s workmarkers records", + __func__, pcombuf); } POOLINSTANCE_DBLOAD_MSG(workmarkers); @@ -9425,6 +9464,7 @@ unparam: bool marks_fill(PGconn *conn) { + char pcombuf[64]; ExecStatusType rescode; PGresult *res; K_ITEM *item; @@ -9535,7 +9575,9 @@ bool marks_fill(PGconn *conn) if (ok) { LOGDEBUG("%s(): built", __func__); - LOGWARNING("%s(): loaded %d marks records", __func__, n); + pcom(n, pcombuf, sizeof(pcombuf)); + LOGWARNING("%s(): loaded %s marks records", + __func__, pcombuf); } POOLINSTANCE_DBLOAD_MSG(marks); From 00b5a6b1bef0626be85846636e08fe0ff424827f Mon Sep 17 00:00:00 2001 From: kanoi Date: Thu, 13 Oct 2016 09:44:30 +1100 Subject: [PATCH 20/33] ckdb - track idcontrol in ram --- src/ckdb.c | 14 ++-- src/ckdb.h | 11 ++- src/ckdb_cmd.c | 70 +++++++++++++---- src/ckdb_data.c | 46 +++++++++++ src/ckdb_dbio.c | 199 +++++++++++++++++++++++++++++------------------- 5 files changed, 236 insertions(+), 104 deletions(-) diff --git a/src/ckdb.c b/src/ckdb.c index b3c6cb0e..046df82f 100644 --- a/src/ckdb.c +++ b/src/ckdb.c @@ -688,8 +688,7 @@ K_LIST *accountadjustment_free; K_STORE *accountadjustment_store; // IDCONTROL -// These are only used for db access - not stored in memory -//K_TREE *idcontrol_root; +K_TREE *idcontrol_root; K_LIST *idcontrol_free; K_STORE *idcontrol_store; @@ -1711,7 +1710,6 @@ PGconn *dbconnect() } /* Load tables required to support auths,adduser,chkpass and newid - * N.B. idcontrol is DB internal so is always ready * OptionControl is loaded first in case it is needed by other loads * (though not yet) */ @@ -1726,6 +1724,8 @@ static bool getdata1() goto matane; if (!(ok = optioncontrol_fill(conn))) goto matane; + if (!(ok = idcontrol_fill(conn))) + goto matane; if (!(ok = users_fill(conn))) goto matane; if (!(ok = workers_fill(conn))) @@ -2113,6 +2113,8 @@ static void alloc_storage() idcontrol_free = k_new_list("IDControl", sizeof(IDCONTROL), ALLOC_IDCONTROL, LIMIT_IDCONTROL, true); idcontrol_store = k_new_store(idcontrol_free); + idcontrol_root = new_ktree(NULL, cmp_idcontrol, idcontrol_free); + idcontrol_free->dsp_func = dsp_idcontrol; esm_free = k_new_list("ESM", sizeof(ESM), ALLOC_ESM, LIMIT_ESM, true); esm_store = k_new_store(esm_free); @@ -2331,10 +2333,12 @@ static void alloc_storage() DLPRIO(paymentaddresses, 5); + // Must be above instransient + DLPRIO(idcontrol, 3); + // Don't currently nest any locks in these: DLPRIO(esm, PRIO_TERMINAL); DLPRIO(workers, PRIO_TERMINAL); - DLPRIO(idcontrol, PRIO_TERMINAL); DLPRIO(ips, PRIO_TERMINAL); DLPRIO(replies, PRIO_TERMINAL); @@ -2674,7 +2678,7 @@ static void dealloc_storage() esm_report(); FREE_ALL(esm); - FREE_LISTS(idcontrol); + FREE_ALL(idcontrol); FREE_ALL(accountbalance); FREE_ALL(payments); diff --git a/src/ckdb.h b/src/ckdb.h index c7a20e50..4a9b9560 100644 --- a/src/ckdb.h +++ b/src/ckdb.h @@ -58,7 +58,7 @@ #define DB_VLOCK "1" #define DB_VERSION "1.0.7" -#define CKDB_VERSION DB_VERSION"-2.702" +#define CKDB_VERSION DB_VERSION"-2.703" #define WHERE_FFL " - from %s %s() line %d" #define WHERE_FFL_HERE __FILE__, __func__, __LINE__ @@ -2062,7 +2062,7 @@ extern K_STORE *accountadjustment_store; typedef struct idcontrol { char idname[TXT_SML+1]; int64_t lastid; - MODIFYDATECONTROLFIELDS; + MODIFYDATECONTROLIN; } IDCONTROL; #define ALLOC_IDCONTROL 16 @@ -2070,8 +2070,7 @@ typedef struct idcontrol { #define INIT_IDCONTROL(_item) INIT_GENERIC(_item, idcontrol) #define DATA_IDCONTROL(_var, _item) DATA_GENERIC(_var, _item, idcontrol, true) -// These are only used for db access - not stored in memory -//extern K_TREE *idcontrol_root; +extern K_TREE *idcontrol_root; extern K_LIST *idcontrol_free; extern K_STORE *idcontrol_store; @@ -3460,6 +3459,9 @@ extern K_ITEM *find_first_payments(int64_t userid, K_TREE_CTX *ctx); extern K_ITEM *find_first_paypayid(int64_t userid, int64_t payoutid, K_TREE_CTX *ctx); extern cmp_t cmp_accountbalance(K_ITEM *a, K_ITEM *b); extern K_ITEM *find_accountbalance(int64_t userid); +extern void dsp_idcontrol(K_ITEM *item, FILE *stream); +extern cmp_t cmp_idcontrol(K_ITEM *a, K_ITEM *b); +extern K_ITEM *find_idcontrol(char *idname); extern cmp_t cmp_optioncontrol(K_ITEM *a, K_ITEM *b); extern K_ITEM *find_optioncontrol(char *optionname, const tv_t *now, int32_t height); #define sys_setting(_name, _def, _now) user_sys_setting(0, _name, _def, _now) @@ -3726,6 +3728,7 @@ extern bool payments_add(PGconn *conn, bool add, K_ITEM *p_item, extern bool payments_fill(PGconn *conn); extern bool idcontrol_add(PGconn *conn, char *idname, char *idvalue, char *by, char *code, char *inet, tv_t *cd, K_TREE *trf_root); +extern bool idcontrol_fill(PGconn *conn); extern K_ITEM *optioncontrol_item_add(PGconn *conn, K_ITEM *oc_item, tv_t *cd, bool begun); extern K_ITEM *optioncontrol_add(PGconn *conn, char *optionname, char *optionvalue, char *activationdate, char *activationheight, diff --git a/src/ckdb_cmd.c b/src/ckdb_cmd.c index 2dbd3eab..5eb52d35 100644 --- a/src/ckdb_cmd.c +++ b/src/ckdb_cmd.c @@ -5867,6 +5867,9 @@ static char *cmd_dsp(__maybe_unused PGconn *conn, __maybe_unused char *cmd, K_ITEM *i_file, *i_name, *i_type; char reply[1024] = "", *fil, *name, *typ; size_t siz = sizeof(reply); + K_STORE *store = NULL; + K_TREE *tree = NULL; + bool unknown_typ = true, unknown_name = true, msg = false; i_file = require_name(trf_root, "file", 1, NULL, reply, siz); if (!i_file) @@ -5888,44 +5891,81 @@ static char *cmd_dsp(__maybe_unused PGconn *conn, __maybe_unused char *cmd, typ = "tree"; if (strcasecmp(typ, "tree") == 0) { + unknown_typ = false; + if (strcasecmp(name, "blocks") == 0) - dsp_ktree(blocks_root, fil, NULL); + tree = blocks_root; if (strcasecmp(name, "transfer") == 0) - dsp_ktree(trf_root, fil, NULL); + tree = trf_root; if (strcasecmp(name, "paymentaddresses") == 0) - dsp_ktree(paymentaddresses_root, fil, NULL); + tree = paymentaddresses_root; if (strcasecmp(name, "paymentaddresses_create") == 0) - dsp_ktree(paymentaddresses_root, fil, NULL); + tree = paymentaddresses_create_root; if (strcasecmp(name, "sharesummary") == 0) - dsp_ktree(sharesummary_root, fil, NULL); + tree = sharesummary_root; if (strcasecmp(name, "userstats") == 0) - dsp_ktree(userstats_root, fil, NULL); + tree = userstats_root; if (strcasecmp(name, "markersummary") == 0) - dsp_ktree(markersummary_root, fil, NULL); + tree = markersummary_root; if (strcasecmp(name, "workmarkers") == 0) - dsp_ktree(workmarkers_root, fil, NULL); - } + tree = workmarkers_root; + + if (strcasecmp(name, "idcontrol") == 0) + tree = idcontrol_root; + + if (tree) { + unknown_name = false; + if (tree->master->dsp_func) + dsp_ktree(tree, fil, NULL); + else { + snprintf(reply, siz, + "%s %s has no dsp_func", + typ, name); + msg = true; + } + } + } else if (strcasecmp(typ, "store") == 0) { + unknown_typ = false; - if (strcasecmp(typ, "store") == 0) { if (strcasecmp(name, "blocks") == 0) - dsp_kstore(blocks_store, fil, NULL); + store = blocks_store; if (strcasecmp(name, "markersummary") == 0) - dsp_kstore(markersummary_store, fil, NULL); + store = markersummary_store; if (strcasecmp(name, "msgline") == 0) - dsp_kstore(msgline_store, fil, NULL); + store = msgline_store; + + if (store) { + unknown_name = false; + if (store->master->dsp_func) + dsp_kstore(store, fil, NULL); + else { + snprintf(reply, siz, + "%s %s has no dsp_func", + typ, name); + msg = true; + } + } } - LOGDEBUG("%s.ok.dsp.file='%s'", id, fil); - return strdup("ok.dsp"); + if (unknown_typ) { + snprintf(reply, siz, "unknown typ '%s'", typ); + } else if (unknown_name) { + snprintf(reply, siz, "unknown name '%s' for '%s'", name, typ); + } else { + if (!msg) + snprintf(reply, siz, "ok.dsp.file='%s'", fil); + } + LOGDEBUG("%s.%s'", id, reply); + return strdup(reply); #endif } diff --git a/src/ckdb_data.c b/src/ckdb_data.c index 8a1755a1..9a7323d5 100644 --- a/src/ckdb_data.c +++ b/src/ckdb_data.c @@ -2187,6 +2187,52 @@ K_ITEM *find_accountbalance(int64_t userid) return item; } +void dsp_idcontrol(K_ITEM *item, FILE *stream) +{ + char createdate_buf[DATE_BUFSIZ], modifydate_buf[DATE_BUFSIZ]; + IDCONTROL *i; + + if (!item) + fprintf(stream, "%s() called with (null) item\n", __func__); + else { + DATA_IDCONTROL(i, item); + tv_to_buf(&(i->createdate), createdate_buf, sizeof(createdate_buf)); + tv_to_buf(&(i->modifydate), modifydate_buf, sizeof(modifydate_buf)); + fprintf(stream, " idname='%s' lastid=%"PRId64" cdate='%s'" + " cby='%s' ccode='%s' cinet='%s' mdate='%s'" + " mby='%s' mcode='%s' minet='%s'\n", + i->idname, i->lastid, createdate_buf, + i->in_createby, i->in_createcode, + i->in_createinet, modifydate_buf, + i->in_modifyby, i->in_modifycode, + i->in_modifyinet); + } +} + +// order by idname asc +cmp_t cmp_idcontrol(K_ITEM *a, K_ITEM *b) +{ + IDCONTROL *ida, *idb; + DATA_IDCONTROL(ida, a); + DATA_IDCONTROL(idb, b); + return CMP_STR(ida->idname, idb->idname); +} + +// idcontrol must be R or W locked +K_ITEM *find_idcontrol(char *idname) +{ + IDCONTROL idcontrol; + K_TREE_CTX ctx[1]; + K_ITEM look, *item; + + STRNCPY(idcontrol.idname, idname); + + INIT_IDCONTROL(&look); + look.data = (void *)(&idcontrol); + item = find_in_ktree(idcontrol_root, &look, ctx); + return item; +} + // order by optionname asc,activationdate asc,activationheight asc,expirydate desc cmp_t cmp_optioncontrol(K_ITEM *a, K_ITEM *b) { diff --git a/src/ckdb_dbio.c b/src/ckdb_dbio.c index aff62a08..3d53c11b 100644 --- a/src/ckdb_dbio.c +++ b/src/ckdb_dbio.c @@ -496,6 +496,8 @@ int64_t nextid(PGconn *conn, char *idname, int64_t increment, { ExecStatusType rescode; bool conned = false; + IDCONTROL *idcontrol; + K_ITEM *item; PGresult *res; char qry[1024]; char *params[5]; @@ -506,6 +508,14 @@ int64_t nextid(PGconn *conn, char *idname, int64_t increment, lastid = 0; + K_WLOCK(idcontrol_free); + item = find_idcontrol(idname); + if (!item) + { + LOGERR("%s(): No matching idname='%s' in tree", __func__, idname); + goto cleanup; + } + snprintf(qry, sizeof(qry), "select lastid from idcontrol " "where idname='%s' for update", idname); @@ -561,11 +571,19 @@ int64_t nextid(PGconn *conn, char *idname, int64_t increment, if (!PGOK(rescode)) { PGLOGERR("Update", rescode, conn); lastid = 0; + } else { + DATA_IDCONTROL(idcontrol, item); + idcontrol->lastid = lastid; + copy_tv(&(idcontrol->modifydate), cd); + idcontrol->in_modifyby = intransient_str(MBYDB, by); + idcontrol->in_modifycode = intransient_str(MCODEDB, code); + idcontrol->in_modifyinet = intransient_str(MINETDB, inet); } for (n = 0; n < par; n++) free(params[n]); cleanup: + K_WUNLOCK(idcontrol_free); conned = CKPQDisco(&conn, conned); return lastid; } @@ -2591,7 +2609,7 @@ bool idcontrol_add(PGconn *conn, char *idname, char *idvalue, char *by, char *code, char *inet, tv_t *cd, __maybe_unused K_TREE *trf_root) { - K_ITEM *look; + K_ITEM *item; IDCONTROL *row; char *params[2 + MODIFYDATECOUNT]; int n, par = 0; @@ -2604,19 +2622,19 @@ bool idcontrol_add(PGconn *conn, char *idname, char *idvalue, char *by, LOGDEBUG("%s(): add", __func__); K_WLOCK(idcontrol_free); - look = k_unlink_head(idcontrol_free); + item = k_unlink_head(idcontrol_free); K_WUNLOCK(idcontrol_free); - DATA_IDCONTROL(row, look); + DATA_IDCONTROL(row, item); STRNCPY(row->idname, idname); TXT_TO_BIGINT("idvalue", idvalue, row->lastid); - MODIFYDATEINIT(row, cd, by, code, inet); + MODIFYDATEINTRANS(row, cd, by, code, inet); par = 0; params[par++] = str_to_buf(row->idname, NULL, 0); params[par++] = bigint_to_buf(row->lastid, NULL, 0); - MODIFYDATEPARAMS(params, par, row); + MODIFYDATEPARAMSIN(params, par, row); PARCHK(par, params); ins = "insert into idcontrol " @@ -2637,13 +2655,108 @@ foil: for (n = 0; n < par; n++) free(params[n]); + /* N.B. The DB key matches the tree key, + * the tree depends on this to be valid */ K_WLOCK(idcontrol_free); - k_add_head(idcontrol_free, look); + if (ok) { + add_to_ktree(idcontrol_root, item); + k_add_head(idcontrol_store, item); + } else + k_add_head(idcontrol_free, item); K_WUNLOCK(idcontrol_free); return ok; } +bool idcontrol_fill(PGconn *conn) +{ + char pcombuf[64]; + ExecStatusType rescode; + PGresult *res; + K_ITEM *item; + int n, i; + IDCONTROL *row; + char *field; + char *sel; + int fields = 2; + bool ok; + + LOGDEBUG("%s(): select", __func__); + + int idname_num, lastid_num; + MODIFYDATE_num; + + sel = "select " + "idname,lastid" + MODIFYDATECONTROL + " from idcontrol"; + res = CKPQExec(conn, sel, CKPQ_READ); + rescode = CKPQResultStatus(res); + if (!PGOK(rescode)) { + PGLOGERR("Select", rescode, conn); + CKPQClear(res); + return false; + } + + n = PQnfields(res); + if (n != (fields + MODIFYDATECOUNT)) { + LOGERR("%s(): Invalid field count - should be %d, but is %d", + __func__, fields + MODIFYDATECOUNT, n); + CKPQClear(res); + return false; + } + + n = PQntuples(res); + LOGDEBUG("%s(): tree build count %d", __func__, n); + ok = true; + idname_num = lastid_num = CKPQFUNDEF; + MODIFYDATE_init; + K_WLOCK(idcontrol_free); + for (i = 0; i < n; i++) { + item = k_unlink_head(idcontrol_free); + DATA_IDCONTROL(row, item); + bzero(row, sizeof(*row)); + + if (everyone_die) { + ok = false; + break; + } + + CKPQ_VAL_FLD_num(res, i, idname, field, ok); + if (!ok) + break; + TXT_TO_STR("idname", field, row->idname); + + CKPQ_VAL_FLD_num(res, i, lastid, field, ok); + if (!ok) + break; + TXT_TO_BIGINT("lastid", field, row->lastid); + + MODIFYDATEIN(res, i, row, ok); + if (!ok) + break; + + /* N.B. The DB key matches the tree key, + * the tree depends on this to be valid */ + add_to_ktree(idcontrol_root, item); + k_add_head(idcontrol_store, item); + } + if (!ok) + k_add_head(idcontrol_free, item); + + K_WUNLOCK(idcontrol_free); + CKPQClear(res); + + if (ok) { + LOGDEBUG("%s(): built", __func__); + pcom(n, pcombuf, sizeof(pcombuf)); + LOGWARNING("%s(): loaded %s idcontrol records", + __func__, pcombuf); + } + + return ok; +} + void oc_switch_state(OPTIONCONTROL *oc, const char *from) { switch_state = atoi(oc->optionvalue); @@ -9672,77 +9785,3 @@ bool check_db_version(PGconn *conn) return true; } - -char *cmd_newid(PGconn *conn, char *cmd, char *id, tv_t *now, char *by, - char *code, char *inet, __maybe_unused tv_t *cd, - K_TREE *trf_root) -{ - char reply[1024] = ""; - size_t siz = sizeof(reply); - K_ITEM *i_idname, *i_idvalue, *look; - IDCONTROL *row; - char *params[2 + MODIFYDATECOUNT]; - int n, par = 0; - bool ok = false; - ExecStatusType rescode; - bool conned = false; - PGresult *res; - char *ins; - - LOGDEBUG("%s(): cmd '%s'", __func__, cmd); - - i_idname = require_name(trf_root, "idname", 3, (char *)idpatt, reply, siz); - if (!i_idname) - return strdup(reply); - - i_idvalue = require_name(trf_root, "idvalue", 1, (char *)intpatt, reply, siz); - if (!i_idvalue) - return strdup(reply); - - K_WLOCK(idcontrol_free); - look = k_unlink_head(idcontrol_free); - K_WUNLOCK(idcontrol_free); - - DATA_IDCONTROL(row, look); - - STRNCPY(row->idname, transfer_data(i_idname)); - TXT_TO_BIGINT("idvalue", transfer_data(i_idvalue), row->lastid); - MODIFYDATEINIT(row, now, by, code, inet); - - par = 0; - params[par++] = str_to_buf(row->idname, NULL, 0); - params[par++] = bigint_to_buf(row->lastid, NULL, 0); - MODIFYDATEPARAMS(params, par, row); - PARCHK(par, params); - - ins = "insert into idcontrol " - "(idname,lastid" MODIFYDATECONTROL ") values (" PQPARAM10 ")"; - - conned = CKPQConn(&conn); - res = CKPQExecParams(conn, ins, par, NULL, (const char **)params, NULL, NULL, 0, CKPQ_WRITE); - rescode = CKPQResultStatus(res); - CKPQClear(res); - if (!PGOK(rescode)) { - PGLOGERR("Insert", rescode, conn); - goto foil; - } - - ok = true; -foil: - conned = CKPQDisco(&conn, conned); - for (n = 0; n < par; n++) - free(params[n]); - - K_WLOCK(idcontrol_free); - k_add_head(idcontrol_free, look); - K_WUNLOCK(idcontrol_free); - - if (!ok) { - LOGERR("%s() %s.failed.DBE", __func__, id); - return strdup("failed.DBE"); - } - LOGDEBUG("%s.ok.added %s %"PRId64, id, transfer_data(i_idname), row->lastid); - snprintf(reply, siz, "ok.added %s %"PRId64, - transfer_data(i_idname), row->lastid); - return strdup(reply); -} From c6e1a62a022276e6de6d78f4112dbe317f277655 Mon Sep 17 00:00:00 2001 From: kanoi Date: Fri, 14 Oct 2016 14:24:49 +1100 Subject: [PATCH 21/33] ckdb - pause command to help with culling --- src/ckdb.c | 36 ++++++++++++++++++++------- src/ckdb.h | 14 +++++++++-- src/ckdb_cmd.c | 66 ++++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 105 insertions(+), 11 deletions(-) diff --git a/src/ckdb.c b/src/ckdb.c index 046df82f..b191f347 100644 --- a/src/ckdb.c +++ b/src/ckdb.c @@ -26,7 +26,7 @@ * be created irrelevant of any being deleted * * The threads that can be managed have a command option to set them when - * running ckdb and can also be changed via the cmd_threads socket command + * starting ckdb and can also be changed via the cmd_threads socket command * * The main() 'ckdb' thread starts: * iomsgs() for filelog '_fiomsgs' and console '_ciomsgs' @@ -71,13 +71,13 @@ * completes and just process authorise messages immediately while the * reload runs * However, we start the ckpool message queue after loading - * the optioncontrol, users, workers and useratts DB tables, before loading - * the much larger DB tables, so that ckdb is effectively ready for messages - * almost immediately + * the optioncontrol, idcontrol, users, workers and useratts DB tables, + * before loading the much larger DB tables, so that ckdb is effectively + * ready for messages almost immediately * The first ckpool message allows us to know where ckpool is up to * in the CCLs - see reload_from() for how this is handled * The users table, required for the authorise messages, is always updated - * immediately + * in the disk DB immediately */ /* Reload data needed @@ -116,7 +116,7 @@ * RAM accountbalance: TODO: created as data is loaded * * idcontrol: only userid reuse is critical and the user is added - * immeditately to the DB before replying to the add message + * immeditately to the disk DB before replying to the add message * * Tables that are/will be written straight to the DB, so are OK: * users, useraccounts, paymentaddresses, payments, @@ -375,7 +375,7 @@ bool dbload_only_sharesummary = false; * markersummaries and pplns payouts may not be correct */ bool sharesummary_marks_limit = false; -// DB optioncontrol,users,workers,useratts load is complete +// DB optioncontrol,idcontrol,users,workers,useratts load is complete bool db_users_complete = false; // DB load is complete bool db_load_complete = false; @@ -388,7 +388,7 @@ bool reloaded_N_files = false; // Data load is complete bool startup_complete = false; // Set to true when pool0 completes, pool0 = socket data during reload -static bool reload_queue_complete = false; +bool reload_queue_complete = false; // Tell everyone to die bool everyone_die = false; // Set to true every time a store is created @@ -551,6 +551,8 @@ int reload_processing; int cmd_processing; int sockd_count; int max_sockd_count; +ts_t breaker_sleep_stt; +int breaker_sleep_ms; // Trigger breaker() processing mutex_t bq_reload_waitlock; @@ -4516,6 +4518,9 @@ static void *breaker(void *arg) ts_t when, when_add; int i, typ, mythread, done, tot, ret; int breaker_delta = 0; + ts_t last_sleep = { 0L, 0L }; + int last_sleep_ms = 0; + bool do_sleep = false; setup = (struct breaker_setup *)(arg); mythread = setup->thread; @@ -4591,7 +4596,11 @@ static void *breaker(void *arg) K_WLOCK(breakqueue_free); bq_item = NULL; was_null = false; - if (mythread == 0 && reload && reload_breakdown_threads_delta != 0) { + if (breaker_sleep_stt.tv_sec > last_sleep.tv_sec) { + copy_ts(&last_sleep, &breaker_sleep_stt); + last_sleep_ms = breaker_sleep_ms; + do_sleep = true; + } else if (mythread == 0 && reload && reload_breakdown_threads_delta != 0) { breaker_delta = reload_breakdown_threads_delta; reload_breakdown_threads_delta = 0; } else if (mythread == 0 && !reload && cmd_breakdown_threads_delta != 0) { @@ -4621,6 +4630,12 @@ static void *breaker(void *arg) } K_WUNLOCK(breakqueue_free); + if (do_sleep) { + do_sleep = false; + cksleep_ms_r(&last_sleep, last_sleep_ms); + continue; + } + // TODO: deal with thread creation/shutdown failure if (breaker_delta != 0) { if (breaker_delta > 0) { @@ -6396,6 +6411,7 @@ static void *process_socket(__maybe_unused void *arg) case CMD_CHKPASS: case CMD_GETATTS: case CMD_THREADS: + case CMD_PAUSE: case CMD_HOMEPAGE: case CMD_QUERY: break; @@ -6597,6 +6613,7 @@ static void *process_socket(__maybe_unused void *arg) case CMD_EVENTS: case CMD_HIGH: case CMD_THREADS: + case CMD_PAUSE: case CMD_QUERY: msgline->sockd = bq->sockd; bq->sockd = -1; @@ -7111,6 +7128,7 @@ static void process_reload_item(PGconn *conn, K_ITEM *bq_item) case CMD_EVENTS: case CMD_HIGH: case CMD_THREADS: + case CMD_PAUSE: LOGERR("%s() INVALID message line %"PRIu64 " ignored '%.42s...", __func__, bq->count, diff --git a/src/ckdb.h b/src/ckdb.h index 4a9b9560..86f407d4 100644 --- a/src/ckdb.h +++ b/src/ckdb.h @@ -58,7 +58,7 @@ #define DB_VLOCK "1" #define DB_VERSION "1.0.7" -#define CKDB_VERSION DB_VERSION"-2.703" +#define CKDB_VERSION DB_VERSION"-2.704" #define WHERE_FFL " - from %s %s() line %d" #define WHERE_FFL_HERE __FILE__, __func__, __LINE__ @@ -355,7 +355,7 @@ extern bool dbload_only_sharesummary; * markersummaries and pplns payouts may not be correct */ extern bool sharesummary_marks_limit; -// DB users,workers load is complete +// DB optioncontrol,idcontrol,users,workers,useratts load is complete extern bool db_users_complete; // DB load is complete extern bool db_load_complete; @@ -367,6 +367,8 @@ extern bool reloading; extern bool reloaded_N_files; // Data load is complete extern bool startup_complete; +// Set to true when pool0 completes, pool0 = socket data during reload +extern bool reload_queue_complete; // Tell everyone to die extern bool everyone_die; @@ -747,6 +749,7 @@ enum cmd_values { CMD_EVENTS, CMD_HIGH, CMD_THREADS, + CMD_PAUSE, CMD_END }; @@ -1450,6 +1453,8 @@ extern int reload_processing; extern int cmd_processing; extern int sockd_count; extern int max_sockd_count; +extern ts_t breaker_sleep_stt; +extern int breaker_sleep_ms; // Trigger breaker() processing extern mutex_t bq_reload_waitlock; @@ -2877,6 +2882,11 @@ extern K_STORE *userstats_eos_store; // newer OR equal #define tv_newer_eq(_old, _new) (!(tv_newer(_new, _old))) +#define copy_ts(_dest, _src) do { \ + (_dest)->tv_sec = (_src)->tv_sec; \ + (_dest)->tv_nsec = (_src)->tv_nsec; \ + } while(0) + // WORKERSTATUS from various incoming data typedef struct workerstatus { int64_t userid; diff --git a/src/ckdb_cmd.c b/src/ckdb_cmd.c index 5eb52d35..775221b0 100644 --- a/src/ckdb_cmd.c +++ b/src/ckdb_cmd.c @@ -8510,6 +8510,71 @@ static char *cmd_threads(__maybe_unused PGconn *conn, char *cmd, char *id, return buf; } +static char *cmd_pause(__maybe_unused PGconn *conn, char *cmd, char *id, + __maybe_unused tv_t *now, __maybe_unused char *by, + __maybe_unused char *code, __maybe_unused char *inet, + __maybe_unused tv_t *cd, K_TREE *trf_root, + __maybe_unused bool reload_data) +{ + K_ITEM *i_name; + char reply[1024] = ""; + size_t siz = sizeof(reply); + char *name; + + LOGDEBUG("%s(): cmd '%s'", __func__, cmd); + + i_name = require_name(trf_root, "name", 1, NULL, reply, siz); + if (!i_name) + return strdup(reply); + name = transfer_data(i_name); + + /* Pause the breaker threads to help culling to take place for some + * tables that can be culled but 'never' empty due to threads always + * creating new data before the old data has finished being processed + * N.B. this should only be needed on a sizeable pool, once after + * the reload completes ... and even 499ms would be a long time to + * pause in the case of a sizeable pool ... DANGER, WILL ROBINSON! */ + if (strcasecmp(name, "breaker") == 0) { + K_ITEM *i_ms; + int ms = 100; + + i_ms = optional_name(trf_root, "ms", 1, NULL, reply, siz); + if (*reply) + return strdup(reply); + if (i_ms) { + ms = atoi(transfer_data(i_ms)); + // 4999 is too long, don't do it! + if (ms < 10 || ms > 4999) { + snprintf(reply, siz, + "%s ms %d outside range 10-4999", + name, ms); + goto out; + } + } + + if (!reload_queue_complete && !key_update) { + snprintf(reply, siz, + "no point pausing %s before reload completes", + name); + goto out; + } + + /* Use an absolute start time to try to get all threads asleep + * at the same time */ + K_WLOCK(breakqueue_free); + cksleep_prepare_r(&breaker_sleep_stt); + breaker_sleep_ms = ms; + K_WUNLOCK(breakqueue_free); + snprintf(reply, siz, "ok.%s %s%dms pause sent", name, + ms > 499 ? "ALERT!!! " : EMPTY, ms); + } else + snprintf(reply, siz, "unknown name '%s'", name); + +out: + LOGWARNING("%s() %s.%s", __func__, id, reply); + return strdup(reply); +} + /* The socket command format is as follows: * Basic structure: * cmd.ID.fld1=value1 FLDSEP fld2=value2 FLDSEP fld3=... @@ -8623,5 +8688,6 @@ struct CMDS ckdb_cmds[] = { { CMD_EVENTS, "events", false, false, cmd_events, SEQ_NONE, ACCESS_SYSTEM | ACCESS_WEB }, { CMD_HIGH, "high", false, false, cmd_high, SEQ_NONE, ACCESS_SYSTEM }, { CMD_THREADS, "threads", false, false, cmd_threads, SEQ_NONE, ACCESS_SYSTEM }, + { CMD_PAUSE, "pause", false, false, cmd_pause, SEQ_NONE, ACCESS_SYSTEM }, { CMD_END, NULL, false, false, NULL, SEQ_NONE, 0 } }; From fa9fcee9228f91b08b96fdbae00df748827e3395 Mon Sep 17 00:00:00 2001 From: kanoi Date: Fri, 14 Oct 2016 18:49:06 +1100 Subject: [PATCH 22/33] ckdb - keep track of the list of kstores in a klist --- src/ckdb.h | 2 +- src/klist.c | 33 +++++++++++++++++++++++++++++++-- src/klist.h | 2 ++ 3 files changed, 34 insertions(+), 3 deletions(-) diff --git a/src/ckdb.h b/src/ckdb.h index 86f407d4..cfc78143 100644 --- a/src/ckdb.h +++ b/src/ckdb.h @@ -58,7 +58,7 @@ #define DB_VLOCK "1" #define DB_VERSION "1.0.7" -#define CKDB_VERSION DB_VERSION"-2.704" +#define CKDB_VERSION DB_VERSION"-2.710" #define WHERE_FFL " - from %s %s() line %d" #define WHERE_FFL_HERE __FILE__, __func__, __LINE__ diff --git a/src/klist.c b/src/klist.c index a33a7e9c..cdad46c5 100644 --- a/src/klist.c +++ b/src/klist.c @@ -206,7 +206,21 @@ K_STORE *_k_new_store(K_LIST *list, KLIST_FFL_ARGS) store->lock = NULL; store->name = list->name; store->do_tail = list->do_tail; - list->stores++; + store->prev_store = NULL; + // Only tracked for lists with a lock + if (store->master->lock == NULL) { + store->next_store = NULL; + store->master->stores++; + } else { + K_WLOCK(list); + // In the master list, next is the head + if (list->next_store) + list->next_store->prev_store = store; + store->next_store = list->next_store; + list->next_store = store; + list->stores++; + K_WUNLOCK(list); + } return store; } @@ -255,6 +269,7 @@ K_LIST *_k_new_list(const char *name, size_t siz, int allocate, int limit, list->limit = limit; list->do_tail = do_tail; list->cull_limit = cull_limit; + list->next_store = list->prev_store = NULL; if (!(list->is_lock_only)) k_alloc_items(list, KLIST_FFL_PASS); @@ -718,7 +733,21 @@ K_STORE *_k_free_store(K_STORE *store, KLIST_FFL_ARGS) store->name, __func__, KLIST_FFL_PASS); } - store->master->stores--; + if (store->master->lock == NULL) + store->master->stores--; + else { + K_WLOCK(store->master); + // unlink store from the list + if (store->prev_store) + store->prev_store->next_store = store->next_store; + if (store->next_store) + store->next_store->prev_store = store->prev_store; + // correct the head if we are the head + if (store->master->next_store == store) + store->master->next_store = store->next_store; + store->master->stores--; + K_WUNLOCK(store->master); + } free(store); diff --git a/src/klist.h b/src/klist.h index ee8a021f..e009d708 100644 --- a/src/klist.h +++ b/src/klist.h @@ -154,6 +154,8 @@ typedef struct k_list { int cull_limit; // <1 means don't cull, otherwise total to cull at int cull_count; // number of times culled uint64_t ram; // ram allocated for data pointers - code must manage it + struct k_list *next_store; // list of all stores - the head is next_store in the list master + struct k_list *prev_store; // the stores themselves have their prev and next int stores; // how many stores it currently has #if LOCK_CHECK // Since each thread has it's own k_lock no locking is required on this From 17d0e1afdcab3c682733703107b7ee980ae2680f Mon Sep 17 00:00:00 2001 From: kanoi Date: Sat, 15 Oct 2016 12:48:30 +1100 Subject: [PATCH 23/33] ckdb - fix inconsequential mem leaks in cmd_query and add defined out transfer debugging --- src/ckdb.h | 2 +- src/ckdb_cmd.c | 92 ++++++++++++++++++++++++++++++++++++++++++++------ 2 files changed, 83 insertions(+), 11 deletions(-) diff --git a/src/ckdb.h b/src/ckdb.h index cfc78143..724d580b 100644 --- a/src/ckdb.h +++ b/src/ckdb.h @@ -58,7 +58,7 @@ #define DB_VLOCK "1" #define DB_VERSION "1.0.7" -#define CKDB_VERSION DB_VERSION"-2.710" +#define CKDB_VERSION DB_VERSION"-2.711" #define WHERE_FFL " - from %s %s() line %d" #define WHERE_FFL_HERE __FILE__, __func__, __LINE__ diff --git a/src/ckdb_cmd.c b/src/ckdb_cmd.c index 775221b0..8e8216e2 100644 --- a/src/ckdb_cmd.c +++ b/src/ckdb_cmd.c @@ -7056,7 +7056,7 @@ static char *cmd_query(__maybe_unused PGconn *conn, char *cmd, char *id, 1, (char *)intpatt, reply, siz); if (!i_height) - return strdup(reply); + goto badreply; TXT_TO_INT("height", transfer_data(i_height), height); i_expired = optional_name(trf_root, "expired", @@ -7142,7 +7142,7 @@ static char *cmd_query(__maybe_unused PGconn *conn, char *cmd, char *id, 1, (char *)intpatt, reply, siz); if (!i_wid) - return strdup(reply); + goto badreply; TXT_TO_BIGINT("wid", transfer_data(i_wid), wid); i_expired = optional_name(trf_root, "expired", @@ -7278,7 +7278,7 @@ static char *cmd_query(__maybe_unused PGconn *conn, char *cmd, char *id, 1, (char *)intpatt, reply, siz); if (!i_height) - return strdup(reply); + goto badreply; TXT_TO_INT("height", transfer_data(i_height), height); int_to_buf(height, reply, sizeof(reply)); @@ -7345,7 +7345,7 @@ static char *cmd_query(__maybe_unused PGconn *conn, char *cmd, char *id, 1, (char *)intpatt, reply, siz); if (!i_height) - return strdup(reply); + goto badreply; TXT_TO_INT("height", transfer_data(i_height), height); int_to_buf(height, reply, sizeof(reply)); @@ -7476,7 +7476,7 @@ static char *cmd_query(__maybe_unused PGconn *conn, char *cmd, char *id, 1, (char *)intpatt, reply, siz); if (!i_height) - return strdup(reply); + goto badreply; TXT_TO_INT("height", transfer_data(i_height), height); int_to_buf(height, reply, sizeof(reply)); @@ -7597,7 +7597,7 @@ static char *cmd_query(__maybe_unused PGconn *conn, char *cmd, char *id, 1, (char *)intpatt, reply, siz); if (!i_wid) - return strdup(reply); + goto badreply; TXT_TO_BIGINT("wid", transfer_data(i_wid), selwid); INIT_SHARES(&s_look); @@ -7729,21 +7729,89 @@ static char *cmd_query(__maybe_unused PGconn *conn, char *cmd, char *id, APPEND_REALLOC(buf, off, len, tmp); ok = true; +#if 0 + } else if (strcasecmp(request, "transfer") == 0) { + /* Code for debugging the transfer stores + * limit is set to avoid a very large reply, + * since transfer can be millions of items during a reload */ + TRANSFER *trf = NULL; + K_STORE *trf_store; + K_ITEM *trf_item, *i_limit; + int store = 0, limit = 20; + bool exceeded = false; + + i_limit = optional_name(trf_root, "limit", + 1, (char *)intpatt, + reply, siz); + if (*reply) { + snprintf(reply, siz, "unknown request '%s'", request); + LOGERR("%s() %s.%s", __func__, id, reply); + goto badreply; + } + if (i_limit) + limit = atoi(transfer_data(i_limit)); + + snprintf(tmp, sizeof(tmp), "limit=%d%c", limit, FLDSEP); + APPEND_REALLOC(buf, off, len, tmp); + + K_RLOCK(transfer_free); + trf_store = transfer_free->next_store; + while (!exceeded && trf_store) { + trf_item = trf_store->head; + while (trf_item) { + if (rows >= limit) { + exceeded = true; + break; + } + DATA_TRANSFER(trf, trf_item); + snprintf(tmp, sizeof(tmp), "store:%d=%d%c", + rows, store, FLDSEP); + APPEND_REALLOC(buf, off, len, tmp); + snprintf(tmp, sizeof(tmp), "storename:%d=%s%c", + rows, trf_store->name, FLDSEP); + APPEND_REALLOC(buf, off, len, tmp); + snprintf(tmp, sizeof(tmp), "name:%d=%s%c", + rows, trf->name, FLDSEP); + APPEND_REALLOC(buf, off, len, tmp); + snprintf(tmp, sizeof(tmp), "mvalue:%d=%s%c", + rows, trf->mvalue, FLDSEP); + APPEND_REALLOC(buf, off, len, tmp); + snprintf(tmp, sizeof(tmp), + "malloc:%d=%"PRIu64"%c", + rows, trf->msiz, FLDSEP); + APPEND_REALLOC(buf, off, len, tmp); + snprintf(tmp, sizeof(tmp), "intrans:%d=%c%c", + rows, trf->intransient ? 'Y' : 'N', + FLDSEP); + APPEND_REALLOC(buf, off, len, tmp); + + trf_item = trf_item->next; + rows++; + } + trf_store = trf_store->next_store; + store++; + } + K_RUNLOCK(transfer_free); + + snprintf(tmp, sizeof(tmp), "limitexceeded=%c%c", + exceeded ? 'Y' : 'N', FLDSEP); + APPEND_REALLOC(buf, off, len, tmp); + + ok = true; +#endif } else { - free(buf); snprintf(reply, siz, "unknown request '%s'", request); LOGERR("%s() %s.%s", __func__, id, reply); - return strdup(reply); + goto badreply; } if (!ok) { - free(buf); snprintf(reply, siz, "failed.%s%s%s", request, msg[0] ? " " : "", msg[0] ? msg : ""); LOGERR("%s() %s.%s", __func__, id, reply); - return strdup(reply); + goto badreply; } snprintf(tmp, sizeof(tmp), "rows=%d", rows); @@ -7752,6 +7820,10 @@ static char *cmd_query(__maybe_unused PGconn *conn, char *cmd, char *id, msg[0] ? " " : "", msg[0] ? msg : ""); return buf; + +badreply: + free(buf); + return strdup(reply); } // Query and disable internal lock detection code From d2dc72706136331f408b72cf2978bef9e56d45ac Mon Sep 17 00:00:00 2001 From: kanoi Date: Sat, 15 Oct 2016 13:27:29 +1100 Subject: [PATCH 24/33] ckdb - add to cmd_query transfer stats --- src/ckdb.h | 2 +- src/ckdb_cmd.c | 27 +++++++++++++++++++++++---- 2 files changed, 24 insertions(+), 5 deletions(-) diff --git a/src/ckdb.h b/src/ckdb.h index 724d580b..f62cd442 100644 --- a/src/ckdb.h +++ b/src/ckdb.h @@ -58,7 +58,7 @@ #define DB_VLOCK "1" #define DB_VERSION "1.0.7" -#define CKDB_VERSION DB_VERSION"-2.711" +#define CKDB_VERSION DB_VERSION"-2.712" #define WHERE_FFL " - from %s %s() line %d" #define WHERE_FFL_HERE __FILE__, __func__, __LINE__ diff --git a/src/ckdb_cmd.c b/src/ckdb_cmd.c index 8e8216e2..fb77bbdc 100644 --- a/src/ckdb_cmd.c +++ b/src/ckdb_cmd.c @@ -7737,14 +7737,13 @@ static char *cmd_query(__maybe_unused PGconn *conn, char *cmd, char *id, TRANSFER *trf = NULL; K_STORE *trf_store; K_ITEM *trf_item, *i_limit; - int store = 0, limit = 20; + int stores = 0, limit = 20, tot_stores = 0; bool exceeded = false; i_limit = optional_name(trf_root, "limit", 1, (char *)intpatt, reply, siz); if (*reply) { - snprintf(reply, siz, "unknown request '%s'", request); LOGERR("%s() %s.%s", __func__, id, reply); goto badreply; } @@ -7765,7 +7764,7 @@ static char *cmd_query(__maybe_unused PGconn *conn, char *cmd, char *id, } DATA_TRANSFER(trf, trf_item); snprintf(tmp, sizeof(tmp), "store:%d=%d%c", - rows, store, FLDSEP); + rows, stores, FLDSEP); APPEND_REALLOC(buf, off, len, tmp); snprintf(tmp, sizeof(tmp), "storename:%d=%s%c", rows, trf_store->name, FLDSEP); @@ -7789,14 +7788,34 @@ static char *cmd_query(__maybe_unused PGconn *conn, char *cmd, char *id, rows++; } trf_store = trf_store->next_store; - store++; + stores++; + } + tot_stores = stores; + if (exceeded) { + while (trf_store) { + trf_store = trf_store->next_store; + tot_stores++; + } } K_RUNLOCK(transfer_free); + snprintf(tmp, sizeof(tmp), "rowstores=%d%c", + stores, FLDSEP); + APPEND_REALLOC(buf, off, len, tmp); + snprintf(tmp, sizeof(tmp), "totstores=%d%c", + tot_stores, FLDSEP); + APPEND_REALLOC(buf, off, len, tmp); snprintf(tmp, sizeof(tmp), "limitexceeded=%c%c", exceeded ? 'Y' : 'N', FLDSEP); APPEND_REALLOC(buf, off, len, tmp); + snprintf(tmp, sizeof(tmp), "flds=%s%c", + "store,storename,name,mvalue,malloc,intrans", FLDSEP); + APPEND_REALLOC(buf, off, len, tmp); + snprintf(tmp, sizeof(tmp), "arn=%s%carp=%s%c", + transfer_free->name, FLDSEP, "", FLDSEP); + APPEND_REALLOC(buf, off, len, tmp); + ok = true; #endif } else { From 373f9b01dfef1679405fbca809da74ced3a58e9f Mon Sep 17 00:00:00 2001 From: kanoi Date: Sun, 16 Oct 2016 11:48:27 +1100 Subject: [PATCH 25/33] ckdb - extend the limit on seq found messages --- src/ckdb.c | 11 +++++++---- src/ckdb.h | 2 +- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/src/ckdb.c b/src/ckdb.c index b191f347..0a1c79c5 100644 --- a/src/ckdb.c +++ b/src/ckdb.c @@ -2916,13 +2916,16 @@ static bool setup_data() #define DATASETTRANS(_seqdata, _u) \ ENTRYSETTRANS(&((_seqdata)->entry[(_u) & ((_seqdata)->size - 1)])) -// Check for transient missing every 2s +// Check for transient missing every X seconds #define TRANCHECKLIMIT 2.0 static tv_t last_trancheck; // Don't let these messages be slowed down by a trans_process() #define TRANCHKSEQOK(_seq) ((_seq) != SEQ_SHARES && (_seq) != SEQ_AUTH && \ (_seq) != SEQ_ADDRAUTH && (_seq) != SEQ_BLOCK) +// How many seconds to allow the build up of trans range messages +#define TRANSAGELIMIT 10.0 + /* time (now) is used, not cd, since cd is only relevant to reloading * and we don't run trans_process() during reloading * We also only know now, not cd, for a missing item @@ -3743,10 +3746,10 @@ setitemdata: found[seq].forced_msg = true; } } - // Check if there are any ranges >= 2s old (or forced) + // Check if there are any ranges >= the limit (or forced) for (i = 0; i < SEQ_MAX; i++) { if (found[i].forced_msg || (found[i].last.tv_sec != 0 && - tvdiff(&found_now, &(found[i].last)) >= 2.0)) { + tvdiff(&found_now, &(found[i].last)) >= TRANSAGELIMIT)) { memcpy(&(found_msgs[i]), &(found[i]), sizeof(SEQFOUND)); // will be displayed, so erase it @@ -3780,7 +3783,7 @@ setitemdata: setnow(&found_now); for (i = 0; i < SEQ_MAX; i++) { if (found[i].last.tv_sec != 0 && - tvdiff(&found_now, &(found[i].last)) >= 2.0) { + tvdiff(&found_now, &(found[i].last)) >= TRANSAGELIMIT) { memcpy(&(found_msgs[i]), &(found[i]), sizeof(SEQFOUND)); diff --git a/src/ckdb.h b/src/ckdb.h index f62cd442..51c5e1e8 100644 --- a/src/ckdb.h +++ b/src/ckdb.h @@ -58,7 +58,7 @@ #define DB_VLOCK "1" #define DB_VERSION "1.0.7" -#define CKDB_VERSION DB_VERSION"-2.712" +#define CKDB_VERSION DB_VERSION"-2.713" #define WHERE_FFL " - from %s %s() line %d" #define WHERE_FFL_HERE __FILE__, __func__, __LINE__ From 7cedce99d0600a3d300c438b91063b424b5cee70 Mon Sep 17 00:00:00 2001 From: kanoi Date: Sun, 16 Oct 2016 22:34:33 +1100 Subject: [PATCH 26/33] ckdb - allow creating a store inside a list lock --- src/ckdb.h | 2 +- src/ckdb_cmd.c | 2 +- src/klist.c | 8 +++++--- src/klist.h | 5 +++-- 4 files changed, 10 insertions(+), 7 deletions(-) diff --git a/src/ckdb.h b/src/ckdb.h index 51c5e1e8..2414a40a 100644 --- a/src/ckdb.h +++ b/src/ckdb.h @@ -58,7 +58,7 @@ #define DB_VLOCK "1" #define DB_VERSION "1.0.7" -#define CKDB_VERSION DB_VERSION"-2.713" +#define CKDB_VERSION DB_VERSION"-2.714" #define WHERE_FFL " - from %s %s() line %d" #define WHERE_FFL_HERE __FILE__, __func__, __LINE__ diff --git a/src/ckdb_cmd.c b/src/ckdb_cmd.c index fb77bbdc..18e61693 100644 --- a/src/ckdb_cmd.c +++ b/src/ckdb_cmd.c @@ -3348,7 +3348,7 @@ static char *cmd_heartbeat(__maybe_unused PGconn *conn, char *cmd, char *id, goto pulse; } - hq_store = k_new_store(heartbeatqueue_free); + hq_store = k_new_store_locked(heartbeatqueue_free); k_list_transfer_to_head(heartbeatqueue_store, hq_store); K_WUNLOCK(heartbeatqueue_free); diff --git a/src/klist.c b/src/klist.c index cdad46c5..105b7b4e 100644 --- a/src/klist.c +++ b/src/klist.c @@ -191,7 +191,7 @@ static void k_alloc_items(K_LIST *list, KLIST_FFL_ARGS) list->count_up = allocate; } -K_STORE *_k_new_store(K_LIST *list, KLIST_FFL_ARGS) +K_STORE *_k_new_store(K_LIST *list, bool gotlock, KLIST_FFL_ARGS) { K_STORE *store; @@ -212,14 +212,16 @@ K_STORE *_k_new_store(K_LIST *list, KLIST_FFL_ARGS) store->next_store = NULL; store->master->stores++; } else { - K_WLOCK(list); + if (!gotlock) + K_WLOCK(list); // In the master list, next is the head if (list->next_store) list->next_store->prev_store = store; store->next_store = list->next_store; list->next_store = store; list->stores++; - K_WUNLOCK(list); + if (!gotlock) + K_WUNLOCK(list); } return store; diff --git a/src/klist.h b/src/klist.h index e009d708..2d9ed190 100644 --- a/src/klist.h +++ b/src/klist.h @@ -668,8 +668,9 @@ static inline K_ITEM *list_rtail(K_LIST *list) extern void _dsp_kstore(K_STORE *store, char *filename, char *msg, KLIST_FFL_ARGS); #define dsp_kstore(_store, _file, _msg) _dsp_kstore(_store, _file, _msg, KLIST_FFL_HERE) -extern K_STORE *_k_new_store(K_LIST *list, KLIST_FFL_ARGS); -#define k_new_store(_list) _k_new_store(_list, KLIST_FFL_HERE) +extern K_STORE *_k_new_store(K_LIST *list, bool gotlock, KLIST_FFL_ARGS); +#define k_new_store(_list) _k_new_store(_list, false, KLIST_FFL_HERE) +#define k_new_store_locked(_list) _k_new_store(_list, true, KLIST_FFL_HERE) extern K_LIST *_k_new_list(const char *name, size_t siz, int allocate, int limit, bool do_tail, bool lock_only, bool without_lock, bool local_list, From 202ff78f11fec873dd4992b270917bde1819fbe2 Mon Sep 17 00:00:00 2001 From: kanoi Date: Mon, 17 Oct 2016 21:07:29 +1100 Subject: [PATCH 27/33] ckdb - track the number of db connections --- src/ckdb.c | 9 +++++++++ src/ckdb.h | 6 +++++- src/ckdb_cmd.c | 9 +++++++++ src/ckdb_dbio.c | 6 ++++++ 4 files changed, 29 insertions(+), 1 deletion(-) diff --git a/src/ckdb.c b/src/ckdb.c index 0a1c79c5..824db5f0 100644 --- a/src/ckdb.c +++ b/src/ckdb.c @@ -453,6 +453,11 @@ char *by_default = "code"; char *inet_default = "127.0.0.1"; char *id_default = "42"; +// Emulate a list for lock checking +K_LIST *pgdb_free; +// Count of db connections +int pgdb_count; + // NULL or poolinstance must match const char *sys_poolinstance = NULL; // lock for accessing all mismatch variables @@ -2281,6 +2286,9 @@ static void alloc_storage() userinfo_store = k_new_store(userinfo_free); userinfo_root = new_ktree(NULL, cmp_userinfo, userinfo_free); + // Emulate a list for lock checking + pgdb_free = k_lock_only_list("PGDB"); + #if LOCK_CHECK DLPRIO(seqset, 91); @@ -2343,6 +2351,7 @@ static void alloc_storage() DLPRIO(workers, PRIO_TERMINAL); DLPRIO(ips, PRIO_TERMINAL); DLPRIO(replies, PRIO_TERMINAL); + DLPRIO(pgdb, PRIO_TERMINAL); DLPCHECK(); diff --git a/src/ckdb.h b/src/ckdb.h index 2414a40a..5e163394 100644 --- a/src/ckdb.h +++ b/src/ckdb.h @@ -58,7 +58,7 @@ #define DB_VLOCK "1" #define DB_VERSION "1.0.7" -#define CKDB_VERSION DB_VERSION"-2.714" +#define CKDB_VERSION DB_VERSION"-2.715" #define WHERE_FFL " - from %s %s() line %d" #define WHERE_FFL_HERE __FILE__, __func__, __LINE__ @@ -445,6 +445,10 @@ extern char *by_default; extern char *inet_default; extern char *id_default; +// Emulate a list for lock checking +extern K_LIST *pgdb_free; +extern int pgdb_count; + // Number of seconds per poolinstance message for run #define POOLINSTANCE_MSG_EVERY 30 diff --git a/src/ckdb_cmd.c b/src/ckdb_cmd.c index 18e61693..b4ac3730 100644 --- a/src/ckdb_cmd.c +++ b/src/ckdb_cmd.c @@ -7728,6 +7728,15 @@ static char *cmd_query(__maybe_unused PGconn *conn, char *cmd, char *id, "ESM", FLDSEP, "", FLDSEP); APPEND_REALLOC(buf, off, len, tmp); + ok = true; + } else if (strcasecmp(request, "pg") == 0) { + K_RLOCK(pgdb_free); + snprintf(tmp, sizeof(tmp), "connections=%d%c", + pgdb_count, FLDSEP); + APPEND_REALLOC(buf, off, len, tmp); + K_RUNLOCK(pgdb_free); + rows++; + ok = true; #if 0 } else if (strcasecmp(request, "transfer") == 0) { diff --git a/src/ckdb_dbio.c b/src/ckdb_dbio.c index 3d53c11b..2acfabe1 100644 --- a/src/ckdb_dbio.c +++ b/src/ckdb_dbio.c @@ -434,6 +434,9 @@ bool _CKPQConn(PGconn **conn, WHERE_FFL_ARGS) if (*conn == NULL) { LOGDEBUG("%s(): connecting", __func__); *conn = dbconnect(); + K_WLOCK(pgdb_free); + pgdb_count++; + K_WUNLOCK(pgdb_free); return true; } return false; @@ -445,6 +448,9 @@ bool _CKPQDisco(PGconn **conn, bool conned, WHERE_FFL_ARGS) LOGDEBUG("%s(): disco", __func__); PQfinish(*conn); *conn = NULL; + K_WLOCK(pgdb_free); + pgdb_count--; + K_WUNLOCK(pgdb_free); } return false; } From d793da51c7b5b7dd30c3348e694b1127d27773a4 Mon Sep 17 00:00:00 2001 From: kanoi Date: Tue, 18 Oct 2016 11:20:33 +1100 Subject: [PATCH 28/33] php - some monthly block stats --- pool/page_blocks.php | 91 +++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 90 insertions(+), 1 deletion(-) diff --git a/pool/page_blocks.php b/pool/page_blocks.php index 1affd737..bc71d2b2 100644 --- a/pool/page_blocks.php +++ b/pool/page_blocks.php @@ -72,6 +72,7 @@ function pctcolour($pct) function doblocks($data, $user) { $blink = ' 0) + { + $pg .= '

Monthly Statistics

'; + $pg .= "\n"; + $pg .= ""; + $pg .= ""; + $pg .= ""; + $pg .= ""; + $pg .= ""; + $pg .= ""; + $pg .= ""; + $pg .= ""; + $pg .= "\n"; + + $pg .= ''; + $count = $ans['rows']; + $rout = $bcount = $bcd = $bmon = $byyyy = $bdiffratio = $btxn = 0; + $skipped = false; + for ($i = 0; $i < $count; $i++) + { + $conf = $ans['confirmed:'.$i]; + // Skip leading orphans + if (!$skipped && ($conf == 'O' || $conf == 'R')) + continue; + + $skipped = true; + + // If anything is missing, skip this table + $diffratio = $ans['diffratio:'.$i]; + if ($diffratio == '?') + break; + + $cd = $ans['firstcreatedate:'.$i]; + $mon = intval(gmdate('n', $cd)); + $yyyy = intval(gmdate('Y', $cd)); + // all orphans after a block must be included with that block + if (($conf != 'O' && $conf != 'R') + && ($mon != $bmon || $yyyy != $byyyy)) + { + if ($bcount != 0) + { + if (($rout % 2) == 0) + $row = 'even'; + else + $row = 'odd'; + + $name = gmdate('Y M', $bcd); + $exc = number_format($bdiffratio, 2); + $md = number_format(100 * $bdiffratio / $bcount, 2); + $mr = number_format(100 * $btxn / $bcount, 2); + $ml = number_format(100 * $bcount / $bdiffratio, 2); + $o = number_format((100 - $poolfee) * ($bcount / $bdiffratio) * ($btxn / $bcount), 2); + + $pg .= ""; + $pg .= ""; + $pg .= ""; + $pg .= ""; + $pg .= ""; + $pg .= ""; + $pg .= ""; + $pg .= ""; + $pg .= "\n"; + + $rout++; + } + if ($rout > 7) + break; + + $bcd = $cd; + $bmon = $mon; + $byyyy = $yyyy; + $bcount = 0; + $bdiffratio = 0; + $btxn = 0; + } + $bdiffratio += floatval($ans['diffratio:'.$i]); + + if ($conf != 'O' and $conf != 'R') + { + $height = $ans['height:'.$i]; + $reward = floatval($ans['reward:'.$i]); + $re = 5000000000.0 * pow(0.5, floor($height / 210000.0)); + $btxn += $reward / $re; + $bcount++; + } + } + $pg .= '
UTC MonthBlocksExpectedMean Diff%MeanTx%Luck%PPS%
$name$bcount$exc$md%$mr%$ml%$o%
'; + } + if ($ans['STATUS'] == 'ok') { $count = $ans['rows']; From 8b3f5d32254e4e6f690c62ddd2b15c16a2d126e6 Mon Sep 17 00:00:00 2001 From: kanoi Date: Tue, 18 Oct 2016 16:09:23 +1100 Subject: [PATCH 29/33] php - block information --- pool/page_blocks.php | 325 ++++++++++++++++++++++++------------------- 1 file changed, 178 insertions(+), 147 deletions(-) diff --git a/pool/page_blocks.php b/pool/page_blocks.php index bc71d2b2..cff64aa6 100644 --- a/pool/page_blocks.php +++ b/pool/page_blocks.php @@ -69,6 +69,182 @@ function pctcolour($pct) return array($fg, $bg); } # +function statstable($poolfee, $ans, $data) +{ + if ($ans['STATUS'] != 'ok' or !isset($ans['s_rows']) or $ans['s_rows'] < 1) + return ''; + + $pg = '

Block Statistics

'; + $pg .= "\n"; + $pg .= ""; + $pg .= ""; + $pg .= ""; + $pg .= ""; + $pg .= ""; + $pg .= ""; + $pg .= ""; + $pg .= ""; + + $tt = ""; + $tt .= '?'; + $tt .= ""; + $tt .= "Pool PPS%: MeanTx% * Luck% minus the pool fee"; + + $pg .= ""; + $pg .= "\n"; + + $since = $data['info']['lastblock']; + + $count = $ans['s_rows']; + for ($i = 0; $i < $count; $i++) + { + if (($i % 2) == 0) + $row = 'even'; + else + $row = 'odd'; + + $desc = $ans['s_desc:'.$i]; + $age = daysago($since - $ans['s_prevcreatedate:'.$i]); + $diff = number_format(100 * $ans['s_diffratio:'.$i], 2); + $mean = number_format(100 * $ans['s_diffmean:'.$i], 2); + + $cdferl = $ans['s_cdferl:'.$i]; + list($fg, $bg) = erlcolour($cdferl); + $cdferldsp = "".number_format($cdferl, 4).''; + $bg = " bgcolor=$bg"; + + $luck = number_format(100 * $ans['s_luck:'.$i], 2); + $txm = number_format(100 * $ans['s_txmean:'.$i], 1); + + $o = number_format((100 - $poolfee) * $ans['s_txmean:'.$i] / $ans['s_diffmean:'.$i], 2); + + $pg .= ""; + $pg .= ""; + $pg .= ""; + $pg .= ""; + $pg .= ""; + $pg .= ""; + $pg .= ""; + $pg .= ""; + $pg .= ""; + $pg .= "\n"; + } + $pg .= "
DescriptionTimeMeanTx%Diff%Mean%CDF[Erl]Luck%${tt}PPS%
$desc Blocks$age$txm%$diff%$mean%$cdferldsp$luck%$o%
\n"; + return $pg; +} +# +function monthtable($poolfee, $ans, $limit) +{ + if ($ans['STATUS'] != 'ok' or !isset($ans['rows']) or $ans['rows'] < 1) + return ''; + + $nowmon = intval(gmdate('n', $ans['STAMP'])); + $nowyyyy = intval(gmdate('Y', $ans['STAMP'])); + + $pg = '

Monthly Statistics

'; + $pg .= "\n"; + $pg .= ""; + $pg .= ""; + $pg .= ""; + $pg .= ""; + $pg .= ""; + $pg .= ""; + $pg .= ""; + $pg .= ""; + $pg .= ""; + $pg .= "\n"; + + $pg .= ''; + $count = $ans['rows']; + $rout = $bcount = $bcd = $bmon = $byyyy = $bdiffacc = $bdiffratio = $btxn = 0; + $skipped = false; + for ($i = 0; $i < $count; $i++) + { + $conf = $ans['confirmed:'.$i]; + // Skip leading orphans + if (!$skipped && ($conf == 'O' || $conf == 'R')) + continue; + + $skipped = true; + + // If anything is missing, skip this table + $diffratio = $ans['diffratio:'.$i]; + if ($diffratio == '?') + break; + + $cd = $ans['firstcreatedate:'.$i]; + $mon = intval(gmdate('n', $cd)); + $yyyy = intval(gmdate('Y', $cd)); + // all orphans after a block must be included with that block + if (($conf != 'O' && $conf != 'R') + && ($mon != $bmon || $yyyy != $byyyy)) + { + if ($bcount != 0) + { + if (($rout % 2) == 0) + $row = 'even'; + else + $row = 'odd'; + + if ($bmon == $nowmon && $byyyy == $nowyyyy) + $dots = '…'; + else + $dots = ''; + + $elap = $bcd - $cd; + $phr = ($bdiffacc / $elap) * pow(2, 32); + $phrdsp = siprefmt($phr); + + $name = gmdate('Y M', $bcd); + $exc = number_format($bdiffratio, 2); + if ($bdiffratio > $bcount) + $bcol = 'darkred'; + else + $bcol = 'darkgreen'; + $md = number_format(100 * $bdiffratio / $bcount, 2); + $mr = number_format(100 * $btxn / $bcount, 2); + $ml = number_format(100 * $bcount / $bdiffratio, 2); + $oa = (100 - $poolfee) * ($bcount / $bdiffratio) * ($btxn / $bcount); + $odsp = number_format($oa, 2); + + $pg .= ""; + $pg .= ""; + $pg .= ""; + $pg .= ""; + $pg .= ""; + $pg .= ""; + $pg .= ""; + $pg .= ""; + $pg .= ""; + $pg .= "\n"; + + $rout++; + } + if ($rout > $limit) + break; + + $bcd = $cd; + $bmon = $mon; + $byyyy = $yyyy; + $bcount = $bdiffacc = $bdiffratio = $btxn = 0; + } + $bdiffratio += floatval($ans['diffratio:'.$i]); + $bdiffacc += floatval($ans['diffacc:'.$i]); + + if ($conf != 'O' and $conf != 'R') + { + $height = $ans['height:'.$i]; + $reward = floatval($ans['reward:'.$i]); + $re = 5000000000.0 * pow(0.5, floor($height / 210000.0)); + $btxn += $reward / $re; + $bcount++; + } + } + $pg .= '
UTC MonthPool AvgBlocksExpectedMean Diff%MeanTx%Luck%PPS%
$name$dots${phrdsp}Hs$bcount$exc$md%$mr%$ml%$odsp%
'; + + return $pg; +} +# function doblocks($data, $user) { $blink = ' 0) - { - $pg .= '

Block Statistics

'; - $pg .= "\n"; - $pg .= ""; - $pg .= ""; - $pg .= ""; - $pg .= ""; - $pg .= ""; - $pg .= ""; - $pg .= ""; - $pg .= ""; - - $tt = ""; - $tt .= '?'; - $tt .= ""; - $tt .= "Pool PPS%: MeanTx% * Luck% minus the pool fee"; - - $pg .= ""; - $pg .= "\n"; - - $since = $data['info']['lastblock']; - - $count = $ans['s_rows']; - for ($i = 0; $i < $count; $i++) - { - if (($i % 2) == 0) - $row = 'even'; - else - $row = 'odd'; - - $desc = $ans['s_desc:'.$i]; - $age = daysago($since - $ans['s_prevcreatedate:'.$i]); - $diff = number_format(100 * $ans['s_diffratio:'.$i], 2); - $mean = number_format(100 * $ans['s_diffmean:'.$i], 2); - - $cdferl = $ans['s_cdferl:'.$i]; - list($fg, $bg) = erlcolour($cdferl); - $cdferldsp = "".number_format($cdferl, 4).''; - $bg = " bgcolor=$bg"; - - $luck = number_format(100 * $ans['s_luck:'.$i], 2); - $txm = number_format(100 * $ans['s_txmean:'.$i], 1); - - $o = number_format((100 - $poolfee) * $ans['s_txmean:'.$i] / $ans['s_diffmean:'.$i], 2); - - $pg .= ""; - $pg .= ""; - $pg .= ""; - $pg .= ""; - $pg .= ""; - $pg .= ""; - $pg .= ""; - $pg .= ""; - $pg .= ""; - $pg .= "\n"; - } - $pg .= "
DescriptionTimeMeanTx%Diff%Mean%CDF[Erl]Luck%${tt}PPS%
$desc Blocks$age$txm%$diff%$mean%$cdferldsp$luck%$o%
\n"; - } - - if ($ans['STATUS'] == 'ok' and isset($ans['rows']) and $ans['rows'] > 0) - { - $pg .= '

Monthly Statistics

'; - $pg .= "\n"; - $pg .= ""; - $pg .= ""; - $pg .= ""; - $pg .= ""; - $pg .= ""; - $pg .= ""; - $pg .= ""; - $pg .= ""; - $pg .= "\n"; - - $pg .= ''; - $count = $ans['rows']; - $rout = $bcount = $bcd = $bmon = $byyyy = $bdiffratio = $btxn = 0; - $skipped = false; - for ($i = 0; $i < $count; $i++) - { - $conf = $ans['confirmed:'.$i]; - // Skip leading orphans - if (!$skipped && ($conf == 'O' || $conf == 'R')) - continue; - - $skipped = true; - - // If anything is missing, skip this table - $diffratio = $ans['diffratio:'.$i]; - if ($diffratio == '?') - break; - - $cd = $ans['firstcreatedate:'.$i]; - $mon = intval(gmdate('n', $cd)); - $yyyy = intval(gmdate('Y', $cd)); - // all orphans after a block must be included with that block - if (($conf != 'O' && $conf != 'R') - && ($mon != $bmon || $yyyy != $byyyy)) - { - if ($bcount != 0) - { - if (($rout % 2) == 0) - $row = 'even'; - else - $row = 'odd'; - - $name = gmdate('Y M', $bcd); - $exc = number_format($bdiffratio, 2); - $md = number_format(100 * $bdiffratio / $bcount, 2); - $mr = number_format(100 * $btxn / $bcount, 2); - $ml = number_format(100 * $bcount / $bdiffratio, 2); - $o = number_format((100 - $poolfee) * ($bcount / $bdiffratio) * ($btxn / $bcount), 2); - - $pg .= ""; - $pg .= ""; - $pg .= ""; - $pg .= ""; - $pg .= ""; - $pg .= ""; - $pg .= ""; - $pg .= ""; - $pg .= "\n"; - - $rout++; - } - if ($rout > 7) - break; - - $bcd = $cd; - $bmon = $mon; - $byyyy = $yyyy; - $bcount = 0; - $bdiffratio = 0; - $btxn = 0; - } - $bdiffratio += floatval($ans['diffratio:'.$i]); + $pg .= statstable($poolfee, $ans, $data); - if ($conf != 'O' and $conf != 'R') - { - $height = $ans['height:'.$i]; - $reward = floatval($ans['reward:'.$i]); - $re = 5000000000.0 * pow(0.5, floor($height / 210000.0)); - $btxn += $reward / $re; - $bcount++; - } - } - $pg .= '
UTC MonthBlocksExpectedMean Diff%MeanTx%Luck%PPS%
$name$bcount$exc$md%$mr%$ml%$o%
'; - } + $pg .= monthtable($poolfee, $ans, 7); if ($ans['STATUS'] == 'ok') { From 9e86ce8805e29e460bf2c7cdde7f499f64213dfe Mon Sep 17 00:00:00 2001 From: kanoi Date: Thu, 20 Oct 2016 21:32:52 +1100 Subject: [PATCH 30/33] ckdb - fix lost db connections - and some incomplete pause code --- src/ckdb.c | 16 ++- src/ckdb.h | 44 ++++++++ src/ckdb_cmd.c | 21 ++-- src/ckdb_data.c | 3 +- src/ckdb_dbio.c | 281 +++++++++++++++++++++++++++++++++++++----------- 5 files changed, 293 insertions(+), 72 deletions(-) diff --git a/src/ckdb.c b/src/ckdb.c index 824db5f0..2ebebbb8 100644 --- a/src/ckdb.c +++ b/src/ckdb.c @@ -211,7 +211,7 @@ static int replier_count = 0; static cklock_t replier_lock; char *EMPTY = ""; -const char *nullstr = "(null)"; +const char *nullstr = NULLSTR; const char *true_str = "true"; const char *false_str = "false"; @@ -457,6 +457,19 @@ char *id_default = "42"; K_LIST *pgdb_free; // Count of db connections int pgdb_count; +__thread char *connect_file = NULLSTR; +__thread char *connect_func = NULLSTR; +__thread int connect_line = 0; +__thread bool connect_dis = true; +// Pause all DB IO (permanently) +cklock_t pgdb_pause_lock; +__thread int pause_read_count = 0; +__thread char *pause_read_file = NULLSTR; +__thread char *pause_read_func = NULLSTR; +__thread int pause_read_line = 0; +__thread bool pause_read_unlock = false; +bool pgdb_paused = false; +bool pgdb_pause_disabled = false; // NULL or poolinstance must match const char *sys_poolinstance = NULL; @@ -9654,6 +9667,7 @@ int main(int argc, char **argv) cklock_init(&listener_all_lock); cklock_init(&last_lock); cklock_init(&btc_lock); + cklock_init(&pgdb_pause_lock); cklock_init(&poolinstance_lock); cklock_init(&seq_found_lock); diff --git a/src/ckdb.h b/src/ckdb.h index 5e163394..1a9419fd 100644 --- a/src/ckdb.h +++ b/src/ckdb.h @@ -140,6 +140,7 @@ extern int btc_listener_threads_delta; #define BLANK " " extern char *EMPTY; +#define NULLSTR "(null)" extern const char *nullstr; extern const char *true_str; @@ -447,7 +448,46 @@ extern char *id_default; // Emulate a list for lock checking extern K_LIST *pgdb_free; +// Count of db connections extern int pgdb_count; +extern __thread char *connect_file; +extern __thread char *connect_func; +extern __thread int connect_line; +extern __thread bool connect_dis; +/* (WHEN FINISHED) Pause all DB IO (permanently) pause.1.name=pgTABconfirm=Y + * Without confirm=Y it will return the pause state + * All DB IO commands must take out a read of this lock before + * starting anything that shouldn't be done partially + * This also means that the whole of CKDB can lock up for a short + * time if e.g. shift processing has taken the read lock shortly before + * the write lock is taken for the pause request to set the flag + * Once pgdb_paused is true, all DB IO will not access the database + * and all web access will be in read only mode i.e. no user changes + * All connections to the DB will close and thus DB changes and outages + * can then occur without affecting CKDB at all + * check the connection count with query.1.request=pg + * Shift generation is unchanged, Payout generation is permanently disabled + * To restart CKDB you must terminate it then rerun it, then CKDB will + * reload/redo all ckpool data it didn't store in the database + * The aim is to look the same as a normal running CKDB except doing no + * DB I/O - that will be deferred until CKDB is later restarted + * You can't pause CKDB until the dbload has completed, but you can + * during the CCL reload + * The function to take out the pause lock increments the thread's + * pause_read_count so each function that expects the lock to be held can + * easily test that and incorrectly calling it multiple times is tracked + * If the read lock code is called incorrectly, i.e. a code bug, + * pgdb_pause_disabled will be set to true and the pause command can no + * longer be activated, however if it was already activated, this wont + * affect anything */ +extern cklock_t pgdb_pause_lock; +extern __thread int pause_read_count; +extern __thread char *pause_read_file; +extern __thread char *pause_read_func; +extern __thread int pause_read_line; +extern __thread bool pause_read_unlock; +extern bool pgdb_paused; +extern bool pgdb_pause_disabled; // Number of seconds per poolinstance message for run #define POOLINSTANCE_MSG_EVERY 30 @@ -3681,6 +3721,10 @@ extern void _CKPQClear(PGresult *res, WHERE_FFL_ARGS); #define PGLOGEMERG(_str, _rescode, _conn) PGLOG(LOGEMERG, _str, _rescode, _conn) #define PGLOGNOTICE(_str, _rescode, _conn) PGLOG(LOGNOTICE, _str, _rescode, _conn) +extern void _pause_read_lock(WHERE_FFL_ARGS); +#define pause_read_lock() _pause_read_lock(WHERE_FFL_HERE) +extern void _pause_read_unlock(WHERE_FFL_ARGS); +#define pause_read_unlock() _pause_read_unlock(WHERE_FFL_HERE) extern char *pqerrmsg(PGconn *conn); extern bool _CKPQConn(PGconn **conn, WHERE_FFL_ARGS); #define CKPQConn(_conn) _CKPQConn(_conn, WHERE_FFL_HERE) diff --git a/src/ckdb_cmd.c b/src/ckdb_cmd.c index b4ac3730..b64f2e43 100644 --- a/src/ckdb_cmd.c +++ b/src/ckdb_cmd.c @@ -3919,7 +3919,8 @@ static char *cmd_setatts(PGconn *conn, char *cmd, char *id, *(dot++) = '\0'; // If we already had a different one, save it to the DB if (ua_item && strcmp(useratts->attname, attname) != 0) { - conned = CKPQConn(&conn); + if (CKPQConn(&conn)) + conned = true; if (!begun) { begun = CKPQBegin(conn); if (!begun) { @@ -3973,7 +3974,8 @@ static char *cmd_setatts(PGconn *conn, char *cmd, char *id, t_item = next_in_ktree(ctx); } if (ua_item) { - conned = CKPQConn(&conn); + if (CKPQConn(&conn)) + conned = true; if (!begun) { begun = CKPQBegin(conn); if (!begun) { @@ -3993,7 +3995,7 @@ rollback: CKPQEnd(conn, (reason == NULL)); bats: - conned = CKPQDisco(&conn, conned); + CKPQDisco(&conn, conned); if (reason) { if (ua_item) { K_WLOCK(useratts_free); @@ -4218,7 +4220,8 @@ static char *cmd_setopts(PGconn *conn, char *cmd, char *id, reason = "Missing value"; goto rollback; } - conned = CKPQConn(&conn); + if (CKPQConn(&conn)) + conned = true; if (!begun) { begun = CKPQBegin(conn); if (!begun) { @@ -4267,7 +4270,8 @@ static char *cmd_setopts(PGconn *conn, char *cmd, char *id, reason = "Missing value"; goto rollback; } - conned = CKPQConn(&conn); + if (CKPQConn(&conn)) + conned = true; if (!begun) { begun = CKPQBegin(conn); if (!begun) { @@ -4288,7 +4292,7 @@ rollback: if (begun) CKPQEnd(conn, (reason == NULL)); - conned = CKPQDisco(&conn, conned); + CKPQDisco(&conn, conned); if (reason) { snprintf(reply, siz, "ERR.%s", reason); LOGERR("%s.%s.%s", cmd, id, reply); @@ -8479,7 +8483,8 @@ static char *cmd_high(PGconn *conn, char *cmd, char *id, if (strcasecmp(action, "store") == 0) { /* Store the shares_hi_root list in the db now, * rather than wait for a shift process to do it */ - conned = CKPQConn(&conn); + if (CKPQConn(&conn)) + conned = true; count = 0; do { did = false; @@ -8494,7 +8499,7 @@ static char *cmd_high(PGconn *conn, char *cmd, char *id, count++; } } while (did); - conned = CKPQDisco(&conn, conned); + CKPQDisco(&conn, conned); if (count) { LOGWARNING("%s() Stored: %d high shares", __func__, count); diff --git a/src/ckdb_data.c b/src/ckdb_data.c index 9a7323d5..d7eba801 100644 --- a/src/ckdb_data.c +++ b/src/ckdb_data.c @@ -4973,7 +4973,8 @@ bool process_pplns(int32_t height, char *blockhash, tv_t *addr_cd) FLDSEP, cd_buf); DUP_POINTER(payouts_free, payouts->stats, &buf[0]); - conned = CKPQConn(&conn); + if (CKPQConn(&conn)) + conned = true; begun = CKPQBegin(conn); if (!begun) goto shazbot; diff --git a/src/ckdb_dbio.c b/src/ckdb_dbio.c index 2acfabe1..8726892b 100644 --- a/src/ckdb_dbio.c +++ b/src/ckdb_dbio.c @@ -9,10 +9,60 @@ #include "ckdb.h" +void _pause_read_lock(WHERE_FFL_ARGS) +{ + if (pgdb_pause_disabled) + return; + + if (pause_read_count > 0) { + LOGEMERG("%s() ERR lock >0 (%d) (%s/%s/%d/%c) pause disabled" + WHERE_FFL, + __func__, pause_read_count, pause_read_file, + pause_read_func, pause_read_line, + pause_read_unlock ? 'U' : 'L', WHERE_FFL_PASS); + pgdb_pause_disabled = true; + return; + } + + ck_rlock(&pgdb_pause_lock); + pause_read_count++; + pause_read_file = (char *)file; + pause_read_func = (char *)func; + pause_read_line = line; + pause_read_unlock = false; +} + +void _pause_read_unlock(WHERE_FFL_ARGS) +{ + if (pgdb_pause_disabled) + return; + + if (pause_read_count != 1) { + LOGEMERG("%s() ERR lock !=1 (%d) (%s/%s/%d/%c) pause disabled" + WHERE_FFL, + __func__, pause_read_count, pause_read_file, + pause_read_func, pause_read_line, + pause_read_unlock ? 'U' : 'L', WHERE_FFL_PASS); + pgdb_pause_disabled = true; + return; + } + + ck_runlock(&pgdb_pause_lock); + pause_read_count--; + pause_read_file = (char *)file; + pause_read_func = (char *)func; + pause_read_line = line; + pause_read_unlock = true; +} + char *pqerrmsg(PGconn *conn) { - char *ptr, *buf = strdup(PQerrorMessage(conn)); + char *ptr, *buf; + + if (pgdb_paused) + return strdup("pgdb_paused"); + buf = strdup(PQerrorMessage(conn)); if (!buf) quithere(1, "malloc OOM"); ptr = buf + strlen(buf) - 1; @@ -36,6 +86,8 @@ char *pqerrmsg(PGconn *conn) #define FETCHTICK 100000 #define CKPQFUNDEF -1 #define CKPQ_VAL_FLD(__res, __row, __num, __name, __fld, __ok) do { \ + if (pgdb_paused) \ + break; \ if (__num == CKPQFUNDEF) { \ __num = PQfnumber(__res, __name); \ if (__num == CKPQFUNDEF) { \ @@ -76,6 +128,8 @@ char *pqerrmsg(PGconn *conn) #define HISTORYDATEFLDS(_res, _row, _data, _ok) do { \ char *_fld; \ + if (pgdb_paused) \ + break; \ CKPQ_VAL_FLD_tail(_res, _row, _CDDB, _fld, _ok); \ if (!_ok) \ break; \ @@ -101,6 +155,8 @@ char *pqerrmsg(PGconn *conn) #define HISTORYDATEIN(_res, _row, _data, _ok) do { \ char *_fld; \ + if (pgdb_paused) \ + break; \ CKPQ_VAL_FLD_tail(_res, _row, _CDDB, _fld, _ok); \ if (!_ok) \ break; \ @@ -153,6 +209,8 @@ char *pqerrmsg(PGconn *conn) #define MODIFYDATEIN(_res, _row, _data, _ok) do { \ char *_fld; \ + if (pgdb_paused) \ + break; \ CKPQ_VAL_FLD_tail(_res, _row, _CDDB, _fld, _ok); \ if (!_ok) \ break; \ @@ -228,6 +286,8 @@ char *pqerrmsg(PGconn *conn) #define SIMPLEDATEFLDS(_res, _row, _data, _ok) do { \ char *_fld; \ + if (pgdb_paused) \ + break; \ CKPQ_VAL_FLD_tail(_res, _row, _CDDB, _fld, _ok); \ if (!_ok) \ break; \ @@ -320,6 +380,9 @@ char *pqerrmsg(PGconn *conn) // Bug check to ensure no unexpected write txns occur PGresult *_CKPQExec(PGconn *conn, const char *qry, bool isread, WHERE_FFL_ARGS) { + if (pgdb_paused) + return NULL; + // It would slow it down, but could check qry for insert/update/... if (!isread && confirm_sharesummary) quitfrom(1, file, func, line, "BUG: write txn during confirm"); @@ -370,6 +433,9 @@ PGresult *_CKPQExecParams(PGconn *conn, const char *qry, int resultFormat, bool isread, WHERE_FFL_ARGS) { + if (pgdb_paused) + return NULL; + // It would slow it down, but could check qry for insert/update/... if (!isread && confirm_sharesummary) quitfrom(1, file, func, line, "BUG: write txn during confirm"); @@ -421,22 +487,44 @@ PGresult *_CKPQExecParams(PGconn *conn, const char *qry, ExecStatusType _CKPQResultStatus(PGresult *res, WHERE_FFL_ARGS) { + if (pgdb_paused) + return PGRES_COMMAND_OK; + return PQresultStatus(res); } void _CKPQClear(PGresult *res, WHERE_FFL_ARGS) { - PQclear(res); + if (!pgdb_paused) + PQclear(res); } bool _CKPQConn(PGconn **conn, WHERE_FFL_ARGS) { if (*conn == NULL) { - LOGDEBUG("%s(): connecting", __func__); - *conn = dbconnect(); - K_WLOCK(pgdb_free); - pgdb_count++; - K_WUNLOCK(pgdb_free); + if (connect_dis == false) { + LOGEMERG("%s() ERR already (%s/%s/%d)" WHERE_FFL +#if LOCK_CHECK + " @%s" +#endif + , __func__, connect_file, connect_func, + connect_line, WHERE_FFL_PASS +#if LOCK_CHECK + , my_thread_name +#endif + ); + } + if (!pgdb_paused) { + LOGDEBUG("%s(): connecting", __func__); + *conn = dbconnect(); + K_WLOCK(pgdb_free); + pgdb_count++; + K_WUNLOCK(pgdb_free); + connect_file = (char *)file; + connect_func = (char *)func; + connect_line = line; + connect_dis = false; + } return true; } return false; @@ -451,6 +539,10 @@ bool _CKPQDisco(PGconn **conn, bool conned, WHERE_FFL_ARGS) K_WLOCK(pgdb_free); pgdb_count--; K_WUNLOCK(pgdb_free); + connect_file = (char *)file; + connect_func = (char *)func; + connect_line = line; + connect_dis = true; } return false; } @@ -460,6 +552,9 @@ bool _CKPQBegin(PGconn *conn, WHERE_FFL_ARGS) ExecStatusType rescode; PGresult *res; + if (pgdb_paused) + return true; + res = _CKPQExec(conn, "Begin", CKPQ_WRITE, WHERE_FFL_PASS); rescode = _CKPQResultStatus(res, WHERE_FFL_PASS); _CKPQClear(res, WHERE_FFL_PASS); @@ -479,6 +574,9 @@ void _CKPQEnd(PGconn *conn, bool commit, WHERE_FFL_ARGS) ExecStatusType rescode; PGresult *res; + if (pgdb_paused) + return; + if (commit) { LOGDEBUG("%s(): commit", __func__); res = _CKPQExec(conn, "Commit", CKPQ_WRITE, WHERE_FFL_PASS); @@ -526,7 +624,8 @@ int64_t nextid(PGconn *conn, char *idname, int64_t increment, "where idname='%s' for update", idname); - conned = CKPQConn(&conn); + if (CKPQConn(&conn)) + conned = true; res = CKPQExec(conn, qry, CKPQ_WRITE); rescode = CKPQResultStatus(res); if (!PGOK(rescode)) { @@ -590,7 +689,7 @@ int64_t nextid(PGconn *conn, char *idname, int64_t increment, free(params[n]); cleanup: K_WUNLOCK(idcontrol_free); - conned = CKPQDisco(&conn, conned); + CKPQDisco(&conn, conned); return lastid; } @@ -612,6 +711,9 @@ bool users_update(PGconn *conn, K_ITEM *u_item, char *oldhash, LOGDEBUG("%s(): change", __func__); + if (pgdb_paused) + LOGEMERG("ERR: %s() called when paused - data lost", __func__); + if (oldhash != NULL) hash = true; else @@ -654,7 +756,8 @@ bool users_update(PGconn *conn, K_ITEM *u_item, char *oldhash, params[par++] = tv_to_buf((tv_t *)&default_expiry, NULL, 0); PARCHKVAL(par, 3, params); - conned = CKPQConn(&conn); + if (CKPQConn(&conn)) + conned = true; if (!CKPQBegin(conn)) goto unparam; @@ -704,7 +807,7 @@ rollback: CKPQEnd(conn, ok); unparam: - conned = CKPQDisco(&conn, conned); + CKPQDisco(&conn, conned); for (n = 0; n < par; n++) free(params[n]); @@ -748,6 +851,9 @@ K_ITEM *users_add(PGconn *conn, INTRANSIENT *in_username, char *emailaddress, LOGDEBUG("%s(): add", __func__); + if (pgdb_paused && userbits != USER_MISSING) + LOGEMERG("ERR: %s() called when paused - data lost", __func__); + /* 2 attempts to add the same user at the same time will only do it once * The 2nd attempt will get back the data provided by the 1st * and thus throw away any differences in the 2nd */ @@ -842,7 +948,8 @@ K_ITEM *users_add(PGconn *conn, INTRANSIENT *in_username, char *emailaddress, "secondaryuserid,salt,userdata,userbits" HISTORYDATECONTROL ") values (" PQPARAM15 ")"; - conned = CKPQConn(&conn); + if (CKPQConn(&conn)) + conned = true; res = CKPQExecParams(conn, ins, par, NULL, (const char **)params, NULL, NULL, 0, CKPQ_WRITE); rescode = CKPQResultStatus(res); CKPQClear(res); @@ -853,7 +960,7 @@ K_ITEM *users_add(PGconn *conn, INTRANSIENT *in_username, char *emailaddress, ok = true; unparam: - conned = CKPQDisco(&conn, conned); + CKPQDisco(&conn, conned); for (n = 0; n < par; n++) free(params[n]); unitem: @@ -893,6 +1000,9 @@ bool users_replace(PGconn *conn, K_ITEM *u_item, K_ITEM *old_u_item, char *by, LOGDEBUG("%s(): replace", __func__); + if (pgdb_paused) + LOGEMERG("ERR: %s() called when paused - data lost", __func__); + DATA_USERS(users, u_item); DATA_USERS(old_users, old_u_item); @@ -906,7 +1016,8 @@ bool users_replace(PGconn *conn, K_ITEM *u_item, K_ITEM *old_u_item, char *by, params[par++] = tv_to_buf((tv_t *)&default_expiry, NULL, 0); PARCHKVAL(par, 3, params); - conned = CKPQConn(&conn); + if (CKPQConn(&conn)) + conned = true; if (!CKPQBegin(conn)) goto unparam; @@ -954,7 +1065,7 @@ rollback: CKPQEnd(conn, ok); unparam: - conned = CKPQDisco(&conn, conned); + CKPQDisco(&conn, conned); for (n = 0; n < par; n++) free(params[n]); @@ -1154,6 +1265,9 @@ bool useratts_item_add(PGconn *conn, K_ITEM *ua_item, tv_t *cd, bool begun) LOGDEBUG("%s(): add", __func__); + if (pgdb_paused) + LOGEMERG("ERR: %s() called when paused - data lost", __func__); + DATA_USERATTS(useratts, ua_item); K_RLOCK(useratts_free); @@ -1164,7 +1278,8 @@ bool useratts_item_add(PGconn *conn, K_ITEM *ua_item, tv_t *cd, bool begun) /* N.B. the values of the old ua_item record, if it exists, * are completely ignored i.e. you must provide all values required */ - conned = CKPQConn(&conn); + if (CKPQConn(&conn)) + conned = true; if (!begun) { if (!CKPQBegin(conn)) goto unparam; @@ -1224,7 +1339,7 @@ rollback: CKPQEnd(conn, ok); unparam: - conned = CKPQDisco(&conn, conned); + CKPQDisco(&conn, conned); for (n = 0; n < par; n++) free(params[n]); @@ -1259,6 +1374,9 @@ K_ITEM *useratts_add(PGconn *conn, char *username, char *attname, LOGDEBUG("%s(): add", __func__); + if (pgdb_paused) + LOGEMERG("ERR: %s() called when paused - data lost", __func__); + K_WLOCK(useratts_free); item = k_unlink_head(useratts_free); K_WUNLOCK(useratts_free); @@ -1339,6 +1457,9 @@ bool useratts_item_expire(PGconn *conn, K_ITEM *ua_item, tv_t *cd) LOGDEBUG("%s(): add", __func__); + if (pgdb_paused) + LOGEMERG("ERR: %s() called when paused - data lost", __func__); + DATA_USERATTS(useratts, ua_item); /* This is pointless if ua_item is part of the tree, however, @@ -1350,7 +1471,8 @@ bool useratts_item_expire(PGconn *conn, K_ITEM *ua_item, tv_t *cd) if (item) { DATA_USERATTS(useratts, item); - conned = CKPQConn(&conn); + if (CKPQConn(&conn)) + conned = true; upd = "update useratts set "EDDB"=$1 where userid=$2 and " "attname=$3 and "EDDB"=$4"; par = 0; @@ -1371,7 +1493,7 @@ bool useratts_item_expire(PGconn *conn, K_ITEM *ua_item, tv_t *cd) ok = true; unparam: if (par) { - conned = CKPQDisco(&conn, conned); + CKPQDisco(&conn, conned); for (n = 0; n < par; n++) free(params[n]); } @@ -1550,7 +1672,8 @@ K_ITEM *workers_add(PGconn *conn, int64_t userid, char *workername, bool add_ws, DATA_WORKERS(row, item); - conned = CKPQConn(&conn); + if (CKPQConn(&conn)) + conned = true; bzero(row, sizeof(*row)); row->workerid = nextid(conn, "workerid", (int64_t)1, cd, by, code, inet); if (row->workerid == 0) @@ -1630,7 +1753,7 @@ unparam: for (n = 0; n < par; n++) free(params[n]); unitem: - conned = CKPQDisco(&conn, conned); + CKPQDisco(&conn, conned); K_WLOCK(workers_free); if (!ret) k_add_head(workers_free, item); @@ -1679,6 +1802,9 @@ bool workers_update(PGconn *conn, K_ITEM *item, char *difficultydefault, LOGDEBUG("%s(): update", __func__); + if (pgdb_paused) + LOGEMERG("ERR: %s() called when paused - data lost", __func__); + /* Two attempts to update the same worker at the same time * will determine the final state based on which gets the lock last, * i.e. randomly, but without overwriting at the same time */ @@ -1736,7 +1862,8 @@ bool workers_update(PGconn *conn, K_ITEM *item, char *difficultydefault, params[par++] = tv_to_buf((tv_t *)&default_expiry, NULL, 0); PARCHKVAL(par, 3, params); - conned = CKPQConn(&conn); + if (CKPQConn(&conn)) + conned = true; if (!CKPQBegin(conn)) goto unparam; @@ -1781,7 +1908,7 @@ rollback: CKPQEnd(conn, ok); unparam: - conned = CKPQDisco(&conn, conned); + CKPQDisco(&conn, conned); for (n = 0; n < par; n++) free(params[n]); early: @@ -1975,11 +2102,15 @@ bool paymentaddresses_set(PGconn *conn, int64_t userid, K_STORE *pa_store, LOGDEBUG("%s(): add", __func__); + if (pgdb_paused) + LOGEMERG("ERR: %s() called when paused - data lost", __func__); + // Quick early abort if (pa_store->count > ABS_ADDR_LIMIT) return false; - conned = CKPQConn(&conn); + if (CKPQConn(&conn)) + conned = true; /* This means the nextid updates will rollback on an error, but also * means that it will lock the nextid record for the whole update */ if (!CKPQBegin(conn)) @@ -2122,7 +2253,7 @@ rollback: CKPQEnd(conn, ok); unparam: - conned = CKPQDisco(&conn, conned); + CKPQDisco(&conn, conned); for (n = 0; n < par; n++) free(params[n]); FREENULL(upd); @@ -2340,7 +2471,8 @@ bool payments_add(PGconn *conn, bool add, K_ITEM *p_item, K_ITEM **old_p_item, *old_p_item = find_payments(row->payoutid, row->userid, row->in_subname); K_RUNLOCK(payments_free); - conned = CKPQConn(&conn); + if (CKPQConn(&conn)) + conned = true; if (!already) { begun = CKPQBegin(conn); if (!begun) @@ -2426,7 +2558,7 @@ unparam: for (n = 0; n < par; n++) free(params[n]); - conned = CKPQDisco(&conn, conned); + CKPQDisco(&conn, conned); if (!already) payments_add_ram(ok, p_item, *old_p_item, cd); @@ -2646,7 +2778,8 @@ bool idcontrol_add(PGconn *conn, char *idname, char *idvalue, char *by, ins = "insert into idcontrol " "(idname,lastid" MODIFYDATECONTROL ") values (" PQPARAM10 ")"; - conned = CKPQConn(&conn); + if (CKPQConn(&conn)) + conned = true; res = CKPQExecParams(conn, ins, par, NULL, (const char **)params, NULL, NULL, 0, CKPQ_WRITE); rescode = CKPQResultStatus(res); CKPQClear(res); @@ -2657,7 +2790,7 @@ bool idcontrol_add(PGconn *conn, char *idname, char *idvalue, char *by, ok = true; foil: - conned = CKPQDisco(&conn, conned); + CKPQDisco(&conn, conned); for (n = 0; n < par; n++) free(params[n]); @@ -3087,6 +3220,9 @@ K_ITEM *optioncontrol_item_add(PGconn *conn, K_ITEM *oc_item, tv_t *cd, bool beg LOGDEBUG("%s(): add", __func__); + if (pgdb_paused) + LOGEMERG("ERR: %s() called when paused - data lost", __func__); + DATA_OPTIONCONTROL(row, oc_item); // Enforce the rule that switch_state isn't date/height controlled @@ -3102,7 +3238,8 @@ K_ITEM *optioncontrol_item_add(PGconn *conn, K_ITEM *oc_item, tv_t *cd, bool beg old_item = find_in_ktree(optioncontrol_root, &look, ctx); K_RUNLOCK(optioncontrol_free); - conned = CKPQConn(&conn); + if (CKPQConn(&conn)) + conned = true; if (!begun) { if (!CKPQBegin(conn)) goto nostart; @@ -3160,7 +3297,7 @@ rollback: CKPQEnd(conn, ok); nostart: - conned = CKPQDisco(&conn, conned); + CKPQDisco(&conn, conned); for (n = 0; n < par; n++) free(params[n]); @@ -3202,6 +3339,9 @@ K_ITEM *optioncontrol_add(PGconn *conn, char *optionname, char *optionvalue, LOGDEBUG("%s(): add", __func__); + if (pgdb_paused) + LOGEMERG("ERR: %s() called when paused - data lost", __func__); + K_WLOCK(optioncontrol_free); item = k_unlink_head(optioncontrol_free); K_WUNLOCK(optioncontrol_free); @@ -3450,7 +3590,8 @@ int64_t workinfo_add(PGconn *conn, char *workinfoidstr, "prevhash,coinbase1,coinbase2,version,bits,ntime,reward" HISTORYDATECONTROL ") values (" PQPARAM16 ")"; - conned = CKPQConn(&conn); + if (CKPQConn(&conn)) + conned = true; res = CKPQExecParams(conn, ins, par, NULL, (const char **)params, NULL, NULL, 0, CKPQ_WRITE); rescode = CKPQResultStatus(res); CKPQClear(res); @@ -3464,7 +3605,7 @@ int64_t workinfo_add(PGconn *conn, char *workinfoidstr, unparam: if (par) { - conned = CKPQDisco(&conn, conned); + CKPQDisco(&conn, conned); for (n = 0; n < par; n++) free(params[n]); } @@ -4232,7 +4373,8 @@ bool shares_db(PGconn *conn, K_ITEM *s_item) "diff,sdiff,errn,error,secondaryuserid,ntime,minsdiff,address," "agent" HISTORYDATECONTROL ") values (" PQPARAM21 ")"; - conned = CKPQConn(&conn); + if (CKPQConn(&conn)) + conned = true; res = CKPQExecParams(conn, ins, par, NULL, (const char **)params, NULL, NULL, 0, CKPQ_WRITE); rescode = CKPQResultStatus(res); if (!PGOK(rescode)) { @@ -4250,7 +4392,7 @@ bool shares_db(PGconn *conn, K_ITEM *s_item) unparam: if (par) { CKPQClear(res); - conned = CKPQDisco(&conn, conned); + CKPQDisco(&conn, conned); for (n = 0; n < par; n++) free(params[n]); } @@ -5215,7 +5357,8 @@ dokey: setnow(&kadd_fin); setnow(&db_stt); - conned = CKPQConn(&conn); + if (CKPQConn(&conn)) + conned = true; if (!CKPQBegin(conn)) { setnow(&db_fin); goto flail; @@ -5264,7 +5407,7 @@ rollback: CKPQEnd(conn, ok); flail: - conned = CKPQDisco(&conn, conned); + CKPQDisco(&conn, conned); if (reason) { // already displayed the full workmarkers detail at the top @@ -5522,7 +5665,8 @@ bool delete_markersummaries(PGconn *conn, WORKMARKERS *wm) del = "delete from markersummary where markerid=$1"; - conned = CKPQConn(&conn); + if (CKPQConn(&conn)) + conned = true; res = CKPQExecParams(conn, del, par, NULL, (const char **)params, NULL, NULL, 0, CKPQ_WRITE); rescode = CKPQResultStatus(res); if (!PGOK(rescode)) { @@ -5549,7 +5693,7 @@ bool delete_markersummaries(PGconn *conn, WORKMARKERS *wm) unparam: CKPQClear(res); flail: - conned = CKPQDisco(&conn, conned); + CKPQDisco(&conn, conned); if (!ok) { if (del_markersummary_store && del_markersummary_store->count) { @@ -6038,7 +6182,8 @@ bool blocks_stats(PGconn *conn, int32_t height, char *blockhash, params[par++] = tv_to_buf((tv_t *)&default_expiry, NULL, 0); PARCHKVAL(par, 3, params); - conned = CKPQConn(&conn); + if (CKPQConn(&conn)) + conned = true; if (!CKPQBegin(conn)) goto unparam; @@ -6093,7 +6238,7 @@ rollback: CKPQEnd(conn, ok); unparam: - conned = CKPQDisco(&conn, conned); + CKPQDisco(&conn, conned); for (n = 0; n < par; n++) free(params[n]); @@ -6246,7 +6391,8 @@ bool blocks_add(PGconn *conn, int32_t height, char *blockhash, "statsconfirmed" HISTORYDATECONTROL ") values (" PQPARAM23 ")"; - conned = CKPQConn(&conn); + if (CKPQConn(&conn)) + conned = true; res = CKPQExecParams(conn, ins, par, NULL, (const char **)params, NULL, NULL, 0, CKPQ_WRITE); rescode = CKPQResultStatus(res); CKPQClear(res); @@ -6324,7 +6470,8 @@ bool blocks_add(PGconn *conn, int32_t height, char *blockhash, params[par++] = tv_to_buf((tv_t *)&default_expiry, NULL, 0); PARCHKVAL(par, 3, params); - conned = CKPQConn(&conn); + if (CKPQConn(&conn)) + conned = true; // New is mostly a copy of the old copy_blocks(row, oldblocks); STRNCPY(row->confirmed, confirmed); @@ -6464,7 +6611,7 @@ unparam: for (n = 0; n < par; n++) free(params[n]); flail: - conned = CKPQDisco(&conn, conned); + CKPQDisco(&conn, conned); K_RLOCK(workinfo_free); K_WLOCK(blocks_free); @@ -6810,7 +6957,8 @@ bool miningpayouts_add(PGconn *conn, bool add, K_ITEM *mp_item, *old_mp_item = find_miningpayouts(row->payoutid, row->userid); K_RUNLOCK(miningpayouts_free); - conned = CKPQConn(&conn); + if (CKPQConn(&conn)) + conned = true; if (!already) { begun = CKPQBegin(conn); if (!begun) @@ -6879,7 +7027,7 @@ unparam: for (n = 0; n < par; n++) free(params[n]); - conned = CKPQDisco(&conn, conned); + CKPQDisco(&conn, conned); if (!already) miningpayouts_add_ram(ok, mp_item, *old_mp_item, cd); @@ -7090,7 +7238,8 @@ bool payouts_add(PGconn *conn, bool add, K_ITEM *p_item, K_ITEM **old_p_item, *old_p_item = find_payouts(row->height, row->blockhash); K_RUNLOCK(payouts_free); - conned = CKPQConn(&conn); + if (CKPQConn(&conn)) + conned = true; if (!already) { begun = CKPQBegin(conn); if (!begun) @@ -7179,7 +7328,7 @@ unparam: for (n = 0; n < par; n++) free(params[n]); - conned = CKPQDisco(&conn, conned); + CKPQDisco(&conn, conned); if (!already) payouts_add_ram(ok, p_item, *old_p_item, cd); @@ -7221,7 +7370,8 @@ K_ITEM *payouts_full_expire(PGconn *conn, int64_t payoutid, tv_t *now, bool lock goto matane; } - conned = CKPQConn(&conn); + if (CKPQConn(&conn)) + conned = true; begun = CKPQBegin(conn); if (!begun) goto matane; @@ -7420,7 +7570,7 @@ matane: K_WUNLOCK(payouts_free); } - conned = CKPQDisco(&conn, conned); + CKPQDisco(&conn, conned); if (lock) K_WUNLOCK(process_pplns_free); @@ -8128,7 +8278,8 @@ bool poolstats_add(PGconn *conn, bool store, INTRANSIENT *in_poolinstance, "hashrate5m,hashrate1hr,hashrate24hr" SIMPLEDATECONTROL ") values (" PQPARAM12 ")"; - conned = CKPQConn(&conn); + if (CKPQConn(&conn)) + conned = true; res = CKPQExecParams(conn, ins, par, NULL, (const char **)params, NULL, NULL, 0, CKPQ_WRITE); rescode = CKPQResultStatus(res); if (!PGOK(rescode)) { @@ -8151,7 +8302,7 @@ bool poolstats_add(PGconn *conn, bool store, INTRANSIENT *in_poolinstance, unparam: if (store) { CKPQClear(res); - conned = CKPQDisco(&conn, conned); + CKPQDisco(&conn, conned); for (n = 0; n < par; n++) free(params[n]); } @@ -8586,7 +8737,8 @@ bool markersummary_add(PGconn *conn, K_ITEM *ms_item, char *by, char *code, row->diffacc); FREENULL(st); - conned = CKPQConn(&conn); + if (CKPQConn(&conn)) + conned = true; res = CKPQExecParams(conn, ins, par, NULL, (const char **)params, NULL, NULL, 0, CKPQ_WRITE); rescode = CKPQResultStatus(res); CKPQClear(res); @@ -8597,7 +8749,7 @@ bool markersummary_add(PGconn *conn, K_ITEM *ms_item, char *by, char *code, ok = true; unparam: - conned = CKPQDisco(&conn, conned); + CKPQDisco(&conn, conned); for (n = 0; n < par; n++) free(params[n]); @@ -9031,7 +9183,8 @@ bool keysummary_add(PGconn *conn, K_ITEM *ks_item, char *by, char *code, row->diffacc); FREENULL(st); - conned = CKPQConn(&conn); + if (CKPQConn(&conn)) + conned = true; res = CKPQExecParams(conn, ins, par, NULL, (const char **)params, NULL, NULL, 0, CKPQ_WRITE); rescode = CKPQResultStatus(res); CKPQClear(res); @@ -9046,7 +9199,7 @@ bool keysummary_add(PGconn *conn, K_ITEM *ks_item, char *by, char *code, ok = true; unparam: - conned = CKPQDisco(&conn, conned); + CKPQDisco(&conn, conned); for (n = 0; n < par; n++) free(params[n]); @@ -9103,7 +9256,8 @@ bool _workmarkers_process(PGconn *conn, bool already, bool add, LOGDEBUG("%s(): updating old", __func__); DATA_WORKMARKERS(oldworkmarkers, old_wm_item); - conned = CKPQConn(&conn); + if (CKPQConn(&conn)) + conned = true; if (!already) { begun = CKPQBegin(conn); if (!begun) @@ -9157,7 +9311,8 @@ bool _workmarkers_process(PGconn *conn, bool already, bool add, DATA_WORKMARKERS(row, wm_item); bzero(row, sizeof(*row)); - conned = CKPQConn(&conn); + if (CKPQConn(&conn)) + conned = true; if (!already && !begun) { begun = CKPQBegin(conn); if (!begun) @@ -9215,7 +9370,7 @@ rollback: CKPQEnd(conn, ok); unparam: - conned = CKPQDisco(&conn, conned); + CKPQDisco(&conn, conned); for (n = 0; n < par; n++) free(params[n]); @@ -9461,7 +9616,8 @@ bool _marks_process(PGconn *conn, bool add, char *poolinstance, LOGDEBUG("%s(): updating old", __func__); DATA_MARKS(oldmarks, old_m_item); - conned = CKPQConn(&conn); + if (CKPQConn(&conn)) + conned = true; begun = CKPQBegin(conn); if (!begun) goto unparam; @@ -9533,7 +9689,8 @@ bool _marks_process(PGconn *conn, bool add, char *poolinstance, HISTORYDATEPARAMS(params, par, row); PARCHK(par, params); - conned = CKPQConn(&conn); + if (CKPQConn(&conn)) + conned = true; if (!begun) { begun = CKPQBegin(conn); if (!begun) @@ -9555,7 +9712,7 @@ rollback: CKPQEnd(conn, ok); unparam: - conned = CKPQDisco(&conn, conned); + CKPQDisco(&conn, conned); for (n = 0; n < par; n++) free(params[n]); From 50b43a7cf7cb874ab16b31b3781296c512923767 Mon Sep 17 00:00:00 2001 From: kanoi Date: Mon, 24 Oct 2016 14:03:27 +1100 Subject: [PATCH 31/33] php - allow directly disabling web logins --- pool/base.php | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/pool/base.php b/pool/base.php index ec419c75..1c12e703 100644 --- a/pool/base.php +++ b/pool/base.php @@ -479,8 +479,17 @@ session_start(); # include_once('db.php'); # +global $disable_login; +$disable_login = false; +if (file_exists('../pool/disable_login.php')) + include_once('../pool/disable_login.php'); +# function validUserPass($user, $pass, $twofa) { + global $disable_login; + if ($disable_login == true) + exit(0); + # $rep = checkPass($user, $pass, $twofa); if ($rep != null) $ans = repDecode($rep); From 15901996b1ef1f9a10bf82e8d73b7575a807c4ea Mon Sep 17 00:00:00 2001 From: kanoi Date: Tue, 25 Oct 2016 17:15:38 +1100 Subject: [PATCH 32/33] php - allow an option login check function --- pool/base.php | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pool/base.php b/pool/base.php index 1c12e703..9afb01e4 100644 --- a/pool/base.php +++ b/pool/base.php @@ -487,6 +487,8 @@ if (file_exists('../pool/disable_login.php')) function validUserPass($user, $pass, $twofa) { global $disable_login; + if (function_exists('checklogin')) + checklogin($user); if ($disable_login == true) exit(0); # From 177b8165352d32dd18ae2f3620b49be1338fbddd Mon Sep 17 00:00:00 2001 From: kanoi Date: Fri, 28 Oct 2016 08:41:56 +1100 Subject: [PATCH 33/33] php - colour the monthly block count using luck limits: 0.9 to 1.1 --- pool/page_blocks.php | 43 ++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 40 insertions(+), 3 deletions(-) diff --git a/pool/page_blocks.php b/pool/page_blocks.php index cff64aa6..f6b59547 100644 --- a/pool/page_blocks.php +++ b/pool/page_blocks.php @@ -69,6 +69,41 @@ function pctcolour($pct) return array($fg, $bg); } # +function mthcolour($luck) +{ + if ($luck == 1.0) + { + $fg = 'white'; + $bg = 'black'; + } + else if ($luck > 1.0) + { + // 1.0 .. 1.1 (> 1.1 = max) + $grn = ($luck - 1.0) * 2550.0; + if ($grn > 255) + $grn = 255; + if ($grn < 0) + $grn = 0; + if ($grn > 190) + $fg = 'blue'; + else + $fg = 'white'; + $bg = sprintf("#00%02x00", $grn); + } + else + { + // 0.9 .. 1.0 (< 0.9 = max) + $red = (1.0 - $luck) * 2550.0; + if ($red > 255) + $red = 255; + if ($red < 0) + $red = 0; + $fg = 'white'; + $bg = sprintf("#%02x0000", $red); + } + return array($fg, $bg); +} +# function statstable($poolfee, $ans, $data) { if ($ans['STATUS'] != 'ok' or !isset($ans['s_rows']) or $ans['s_rows'] < 1) @@ -203,18 +238,20 @@ function monthtable($poolfee, $ans, $limit) $bcol = 'darkgreen'; $md = number_format(100 * $bdiffratio / $bcount, 2); $mr = number_format(100 * $btxn / $bcount, 2); - $ml = number_format(100 * $bcount / $bdiffratio, 2); + $ml = $bcount / $bdiffratio; + $mldsp = number_format(100 * $ml, 2); $oa = (100 - $poolfee) * ($bcount / $bdiffratio) * ($btxn / $bcount); $odsp = number_format($oa, 2); + list($fg, $bg) = mthcolour($ml); $pg .= ""; $pg .= "$name$dots"; $pg .= "${phrdsp}Hs"; - $pg .= "$bcount"; + $pg .= "$bcount"; $pg .= "$exc"; $pg .= "$md%"; $pg .= "$mr%"; - $pg .= "$ml%"; + $pg .= "$mldsp%"; $pg .= "$odsp%"; $pg .= "\n";