diff --git a/pool/base.php b/pool/base.php index ec419c75..9afb01e4 100644 --- a/pool/base.php +++ b/pool/base.php @@ -479,8 +479,19 @@ session_start(); # include_once('db.php'); # +global $disable_login; +$disable_login = false; +if (file_exists('../pool/disable_login.php')) + include_once('../pool/disable_login.php'); +# function validUserPass($user, $pass, $twofa) { + global $disable_login; + if (function_exists('checklogin')) + checklogin($user); + if ($disable_login == true) + exit(0); + # $rep = checkPass($user, $pass, $twofa); if ($rep != null) $ans = repDecode($rep); diff --git a/pool/page_blocks.php b/pool/page_blocks.php index 1affd737..f6b59547 100644 --- a/pool/page_blocks.php +++ b/pool/page_blocks.php @@ -69,9 +69,223 @@ function pctcolour($pct) return array($fg, $bg); } # +function mthcolour($luck) +{ + if ($luck == 1.0) + { + $fg = 'white'; + $bg = 'black'; + } + else if ($luck > 1.0) + { + // 1.0 .. 1.1 (> 1.1 = max) + $grn = ($luck - 1.0) * 2550.0; + if ($grn > 255) + $grn = 255; + if ($grn < 0) + $grn = 0; + if ($grn > 190) + $fg = 'blue'; + else + $fg = 'white'; + $bg = sprintf("#00%02x00", $grn); + } + else + { + // 0.9 .. 1.0 (< 0.9 = max) + $red = (1.0 - $luck) * 2550.0; + if ($red > 255) + $red = 255; + if ($red < 0) + $red = 0; + $fg = 'white'; + $bg = sprintf("#%02x0000", $red); + } + return array($fg, $bg); +} +# +function statstable($poolfee, $ans, $data) +{ + if ($ans['STATUS'] != 'ok' or !isset($ans['s_rows']) or $ans['s_rows'] < 1) + return ''; + + $pg = '

Block Statistics

'; + $pg .= "\n"; + $pg .= ""; + $pg .= ""; + $pg .= ""; + $pg .= ""; + $pg .= ""; + $pg .= ""; + $pg .= ""; + $pg .= ""; + + $tt = ""; + $tt .= '?'; + $tt .= ""; + $tt .= "Pool PPS%: MeanTx% * Luck% minus the pool fee"; + + $pg .= ""; + $pg .= "\n"; + + $since = $data['info']['lastblock']; + + $count = $ans['s_rows']; + for ($i = 0; $i < $count; $i++) + { + if (($i % 2) == 0) + $row = 'even'; + else + $row = 'odd'; + + $desc = $ans['s_desc:'.$i]; + $age = daysago($since - $ans['s_prevcreatedate:'.$i]); + $diff = number_format(100 * $ans['s_diffratio:'.$i], 2); + $mean = number_format(100 * $ans['s_diffmean:'.$i], 2); + + $cdferl = $ans['s_cdferl:'.$i]; + list($fg, $bg) = erlcolour($cdferl); + $cdferldsp = "".number_format($cdferl, 4).''; + $bg = " bgcolor=$bg"; + + $luck = number_format(100 * $ans['s_luck:'.$i], 2); + $txm = number_format(100 * $ans['s_txmean:'.$i], 1); + + $o = number_format((100 - $poolfee) * $ans['s_txmean:'.$i] / $ans['s_diffmean:'.$i], 2); + + $pg .= ""; + $pg .= ""; + $pg .= ""; + $pg .= ""; + $pg .= ""; + $pg .= ""; + $pg .= ""; + $pg .= ""; + $pg .= ""; + $pg .= "\n"; + } + $pg .= "
DescriptionTimeMeanTx%Diff%Mean%CDF[Erl]Luck%${tt}PPS%
$desc Blocks$age$txm%$diff%$mean%$cdferldsp$luck%$o%
\n"; + return $pg; +} +# +function monthtable($poolfee, $ans, $limit) +{ + if ($ans['STATUS'] != 'ok' or !isset($ans['rows']) or $ans['rows'] < 1) + return ''; + + $nowmon = intval(gmdate('n', $ans['STAMP'])); + $nowyyyy = intval(gmdate('Y', $ans['STAMP'])); + + $pg = '

Monthly Statistics

'; + $pg .= "\n"; + $pg .= ""; + $pg .= ""; + $pg .= ""; + $pg .= ""; + $pg .= ""; + $pg .= ""; + $pg .= ""; + $pg .= ""; + $pg .= ""; + $pg .= "\n"; + + $pg .= ''; + $count = $ans['rows']; + $rout = $bcount = $bcd = $bmon = $byyyy = $bdiffacc = $bdiffratio = $btxn = 0; + $skipped = false; + for ($i = 0; $i < $count; $i++) + { + $conf = $ans['confirmed:'.$i]; + // Skip leading orphans + if (!$skipped && ($conf == 'O' || $conf == 'R')) + continue; + + $skipped = true; + + // If anything is missing, skip this table + $diffratio = $ans['diffratio:'.$i]; + if ($diffratio == '?') + break; + + $cd = $ans['firstcreatedate:'.$i]; + $mon = intval(gmdate('n', $cd)); + $yyyy = intval(gmdate('Y', $cd)); + // all orphans after a block must be included with that block + if (($conf != 'O' && $conf != 'R') + && ($mon != $bmon || $yyyy != $byyyy)) + { + if ($bcount != 0) + { + if (($rout % 2) == 0) + $row = 'even'; + else + $row = 'odd'; + + if ($bmon == $nowmon && $byyyy == $nowyyyy) + $dots = '…'; + else + $dots = ''; + + $elap = $bcd - $cd; + $phr = ($bdiffacc / $elap) * pow(2, 32); + $phrdsp = siprefmt($phr); + + $name = gmdate('Y M', $bcd); + $exc = number_format($bdiffratio, 2); + if ($bdiffratio > $bcount) + $bcol = 'darkred'; + else + $bcol = 'darkgreen'; + $md = number_format(100 * $bdiffratio / $bcount, 2); + $mr = number_format(100 * $btxn / $bcount, 2); + $ml = $bcount / $bdiffratio; + $mldsp = number_format(100 * $ml, 2); + $oa = (100 - $poolfee) * ($bcount / $bdiffratio) * ($btxn / $bcount); + $odsp = number_format($oa, 2); + list($fg, $bg) = mthcolour($ml); + + $pg .= ""; + $pg .= ""; + $pg .= ""; + $pg .= ""; + $pg .= ""; + $pg .= ""; + $pg .= ""; + $pg .= ""; + $pg .= ""; + $pg .= "\n"; + + $rout++; + } + if ($rout > $limit) + break; + + $bcd = $cd; + $bmon = $mon; + $byyyy = $yyyy; + $bcount = $bdiffacc = $bdiffratio = $btxn = 0; + } + $bdiffratio += floatval($ans['diffratio:'.$i]); + $bdiffacc += floatval($ans['diffacc:'.$i]); + + if ($conf != 'O' and $conf != 'R') + { + $height = $ans['height:'.$i]; + $reward = floatval($ans['reward:'.$i]); + $re = 5000000000.0 * pow(0.5, floor($height / 210000.0)); + $btxn += $reward / $re; + $bcount++; + } + } + $pg .= '
UTC MonthPool AvgBlocksExpectedMean Diff%MeanTx%Luck%PPS%
$name$dots${phrdsp}Hs$bcount$exc$md%$mr%$mldsp%$odsp%
'; + + return $pg; +} +# function doblocks($data, $user) { $blink = ' 0) - { - $pg .= '

Block Statistics

'; - $pg .= "\n"; - $pg .= ""; - $pg .= ""; - $pg .= ""; - $pg .= ""; - $pg .= ""; - $pg .= ""; - $pg .= ""; - $pg .= ""; - - $tt = ""; - $tt .= '?'; - $tt .= ""; - $tt .= "Pool PPS%: MeanTx% * Luck% minus the pool fee"; - - $pg .= ""; - $pg .= "\n"; - - $since = $data['info']['lastblock']; - - $count = $ans['s_rows']; - for ($i = 0; $i < $count; $i++) - { - if (($i % 2) == 0) - $row = 'even'; - else - $row = 'odd'; - - $desc = $ans['s_desc:'.$i]; - $age = daysago($since - $ans['s_prevcreatedate:'.$i]); - $diff = number_format(100 * $ans['s_diffratio:'.$i], 2); - $mean = number_format(100 * $ans['s_diffmean:'.$i], 2); - - $cdferl = $ans['s_cdferl:'.$i]; - list($fg, $bg) = erlcolour($cdferl); - $cdferldsp = "".number_format($cdferl, 4).''; - $bg = " bgcolor=$bg"; - - $luck = number_format(100 * $ans['s_luck:'.$i], 2); - $txm = number_format(100 * $ans['s_txmean:'.$i], 1); - - $poolfee = 0.9; # pool fee as a % out of 100 - $o = number_format((100 - $poolfee) * $ans['s_txmean:'.$i] / $ans['s_diffmean:'.$i], 2); - - $pg .= ""; - $pg .= ""; - $pg .= ""; - $pg .= ""; - $pg .= ""; - $pg .= ""; - $pg .= ""; - $pg .= ""; - $pg .= ""; - $pg .= "\n"; - } - $pg .= "
DescriptionTimeMeanTx%Diff%Mean%CDF[Erl]Luck%${tt}PPS%
$desc Blocks$age$txm%$diff%$mean%$cdferldsp$luck%$o%
\n"; - } + $pg .= statstable($poolfee, $ans, $data); + + $pg .= monthtable($poolfee, $ans, 7); if ($ans['STATUS'] == 'ok') { diff --git a/pool/page_ckp.php b/pool/page_ckp.php index 07fb370c..769d0d60 100644 --- a/pool/page_ckp.php +++ b/pool/page_ckp.php @@ -16,7 +16,7 @@ function stnum($num) # function dockp($data, $user) { - $pg = '

CKPool

'; + $pg = '

CKDB

'; $msg = msgEncode('stats', 'stats', array(), $user); $rep = sendsockreply('stats', $msg); @@ -32,11 +32,12 @@ function dockp($data, $user) $pg .= ''; $pg .= "Name:<$r id=srtname data-sf=s0>"; $pg .= 'Initial'; - $pg .= 'Allocated'; - $pg .= "<$r id=srtname data-sf=r3>:In Store"; - $pg .= "<$r id=srtname data-sf=r4>:RAM"; - $pg .= "<$r id=srtname data-sf=r5>:RAM2"; - $pg .= 'Cull'; + $pg .= "<$r id=srtalloc data-sf=r2>:Alloc"; + $pg .= "<$r id=srtstore data-sf=r3>:In Store"; + $pg .= "<$r id=srtram data-sf=r4>:RAM"; + $pg .= "<$r id=srtram2 data-sf=r5>:RAM2"; + $pg .= "<$r id=srtcull data-sf=r6>:Cull"; + $pg .= "<$r id=srtlim data-sf=r7>:Limit"; $pg .= "\n"; if ($ans['STATUS'] == 'ok') { @@ -52,16 +53,19 @@ function dockp($data, $user) $pg .= ""; $pg .= "".$ans['name:'.$i].''; $pg .= ''.stnum($ans['initial:'.$i]).''; - $pg .= ''.stnum($ans['allocated:'.$i]).''; + $pg .= "".stnum($ans['allocated:'.$i]).''; $pg .= "".stnum($ans['instore:'.$i]).''; $pg .= "".stnum($ans['ram:'.$i]).''; $pg .= "".stnum($ans['ram2:'.$i]).''; - $pg .= ''.stnum($ans['cull:'.$i]).''; + $pg .= "".stnum($ans['cull:'.$i]).''; + $pg .= "".stnum($ans['cull_limit:'.$i]).''; $pg .= "\n"; } $pg .= ''; } $pg .= "\n"; + $pg .= "\n"; return $pg; } diff --git a/pool/page_shifts.php b/pool/page_shifts.php index 5de1b39c..1c0ffbe6 100644 --- a/pool/page_shifts.php +++ b/pool/page_shifts.php @@ -81,7 +81,7 @@ function doshifts($data, $user) $pg .= ''.$ans['rewards:'.$i].''; $ppsr = (float)$ans['ppsrewarded:'.$i]; if ($ppsr > 0) - $ppsd = sprintf('%.5f', $ppsr); + $ppsd = sprintf('%.5f', $ppsr*1000.0); else $ppsd = '0'; $pg .= "$ppsd"; @@ -96,7 +96,7 @@ function doshifts($data, $user) $pg .= ''; } $pg .= "\n"; - $pg .= "* The Rewarded value unit is satoshis per 1diff share
"; + $pg .= "* The Rewarded value unit is satoshis per 1000diff share
"; return $pg; } diff --git a/src/ckdb.c b/src/ckdb.c index 1b38ff00..2ebebbb8 100644 --- a/src/ckdb.c +++ b/src/ckdb.c @@ -14,6 +14,48 @@ * Consider adding row level locking (a per kitem usage count) if needed */ +/* Thread layout + * ------------- + * Any thread that manages a thread count will have 00 in it's name + * and name each subsequent thread it creates, with the same name + * but with 01, 02 etc, and then wait on them all before exiting + * The 2 digit 00 relates to THREAD_LIMIT which is 99 + * there's a limit of THREAD_LIMIT active threads per thread manager + * WARNING - however the total number of threads created is limited by the + * LOCK_CHECK code which allows only MAX_THREADS total ckdb thread to + * be created irrelevant of any being deleted + * + * The threads that can be managed have a command option to set them when + * starting ckdb and can also be changed via the cmd_threads socket command + * + * The main() 'ckdb' thread starts: + * iomsgs() for filelog '_fiomsgs' and console '_ciomsgs' + * listener() '_p00qproc' + * which manages it's thread count in pqproc() + * + * listener() starts: + * breakdown() for reload '_r00breaker' and cmd '_c00breaker' + * each of which manage their thead counts + * logger() '_logger' + * socksetup() '_socksetup' + * summariser() '_summarise' + * marker() '_marker' + * then calls setup_data() which calls reload() + * then calls pqproc() + * which manages the thread count + * + * socksetup() starts: + * replier() for pool '_preplier' cmd '_creplier' and btc '_breplier' + * listener_all() for cmd '_c00listen' and btc '_b00listen' + * each of which manage their thead counts + * process_socket() '_procsock' + * sockrun() for ckpool '_psockrun' web '_wsockrun' and cmd '_csockrun' + * + * reload() starts: + * process_reload() '_p00rload' + * which manages it's thread count + */ + /* Startup * ------- * During startup we load the DB and track where it is up to with @@ -29,13 +71,13 @@ * completes and just process authorise messages immediately while the * reload runs * However, we start the ckpool message queue after loading - * the optioncontrol, users, workers and useratts DB tables, before loading - * the much larger DB tables, so that ckdb is effectively ready for messages - * almost immediately + * the optioncontrol, idcontrol, users, workers and useratts DB tables, + * before loading the much larger DB tables, so that ckdb is effectively + * ready for messages almost immediately * The first ckpool message allows us to know where ckpool is up to * in the CCLs - see reload_from() for how this is handled * The users table, required for the authorise messages, is always updated - * immediately + * in the disk DB immediately */ /* Reload data needed @@ -74,7 +116,7 @@ * RAM accountbalance: TODO: created as data is loaded * * idcontrol: only userid reuse is critical and the user is added - * immeditately to the DB before replying to the add message + * immeditately to the disk DB before replying to the add message * * Tables that are/will be written straight to the DB, so are OK: * users, useraccounts, paymentaddresses, payments, @@ -157,6 +199,11 @@ static int cmd_breakdown_threads = -1; int reload_breakdown_threads_delta = 0; int cmd_breakdown_threads_delta = 0; +int cmd_listener_threads = 2; +int btc_listener_threads = 2; +int cmd_listener_threads_delta = 0; +int btc_listener_threads_delta = 0; + // Lock used to determine when the last breakdown thread exits static cklock_t breakdown_lock; @@ -164,7 +211,7 @@ static int replier_count = 0; static cklock_t replier_lock; char *EMPTY = ""; -const char *nullstr = "(null)"; +const char *nullstr = NULLSTR; const char *true_str = "true"; const char *false_str = "false"; @@ -328,7 +375,7 @@ bool dbload_only_sharesummary = false; * markersummaries and pplns payouts may not be correct */ bool sharesummary_marks_limit = false; -// DB optioncontrol,users,workers,useratts load is complete +// DB optioncontrol,idcontrol,users,workers,useratts load is complete bool db_users_complete = false; // DB load is complete bool db_load_complete = false; @@ -341,7 +388,7 @@ bool reloaded_N_files = false; // Data load is complete bool startup_complete = false; // Set to true when pool0 completes, pool0 = socket data during reload -static bool reload_queue_complete = false; +bool reload_queue_complete = false; // Tell everyone to die bool everyone_die = false; // Set to true every time a store is created @@ -381,10 +428,10 @@ static uint64_t sock_acc[MAXSOCK], sock_recv[MAXSOCK]; // breaker() summarised static tv_t break_reload_stt, break_cmd_stt, break_reload_fin; static uint64_t break_reload_processed, break_cmd_processed; -// clistener() +// listener_all() +static cklock_t listener_all_lock; static double clis_us; static uint64_t clis_processed; -// blistener() static double blis_us; static uint64_t blis_processed; @@ -406,6 +453,24 @@ char *by_default = "code"; char *inet_default = "127.0.0.1"; char *id_default = "42"; +// Emulate a list for lock checking +K_LIST *pgdb_free; +// Count of db connections +int pgdb_count; +__thread char *connect_file = NULLSTR; +__thread char *connect_func = NULLSTR; +__thread int connect_line = 0; +__thread bool connect_dis = true; +// Pause all DB IO (permanently) +cklock_t pgdb_pause_lock; +__thread int pause_read_count = 0; +__thread char *pause_read_file = NULLSTR; +__thread char *pause_read_func = NULLSTR; +__thread int pause_read_line = 0; +__thread bool pause_read_unlock = false; +bool pgdb_paused = false; +bool pgdb_pause_disabled = false; + // NULL or poolinstance must match const char *sys_poolinstance = NULL; // lock for accessing all mismatch variables @@ -504,6 +569,8 @@ int reload_processing; int cmd_processing; int sockd_count; int max_sockd_count; +ts_t breaker_sleep_stt; +int breaker_sleep_ms; // Trigger breaker() processing mutex_t bq_reload_waitlock; @@ -577,6 +644,7 @@ K_STORE *heartbeatqueue_store; // TRANSFER K_LIST *transfer_free; +int cull_transfer = CULL_TRANSFER; // SEQSET K_LIST *seqset_free; @@ -640,8 +708,7 @@ K_LIST *accountadjustment_free; K_STORE *accountadjustment_store; // IDCONTROL -// These are only used for db access - not stored in memory -//K_TREE *idcontrol_root; +K_TREE *idcontrol_root; K_LIST *idcontrol_free; K_STORE *idcontrol_store; @@ -958,7 +1025,7 @@ static void ioprocess(IOQUEUE *io) flock(logfd, LOCK_EX); if (io->errn) { fprintf(LOGFP, "%s%s with errno %d: %s\n", - stamp, io->msg, + stamp, io->msg, io->errn, strerror(io->errn)); } else fprintf(LOGFP, "%s%s\n", stamp, io->msg); @@ -1663,19 +1730,22 @@ PGconn *dbconnect() } /* Load tables required to support auths,adduser,chkpass and newid - * N.B. idcontrol is DB internal so is always ready * OptionControl is loaded first in case it is needed by other loads * (though not yet) */ static bool getdata1() { - PGconn *conn = dbconnect(); + PGconn *conn = NULL; bool ok = true; + CKPQConn(&conn); + if (!(ok = check_db_version(conn))) goto matane; if (!(ok = optioncontrol_fill(conn))) goto matane; + if (!(ok = idcontrol_fill(conn))) + goto matane; if (!(ok = users_fill(conn))) goto matane; if (!(ok = workers_fill(conn))) @@ -1684,7 +1754,7 @@ static bool getdata1() matane: - PQfinish(conn); + CKPQFinish(&conn); return ok; } @@ -1693,19 +1763,25 @@ matane: */ static bool getdata2() { - PGconn *conn = dbconnect(); - bool ok = blocks_fill(conn); + PGconn *conn = NULL; + bool ok; - PQfinish(conn); + CKPQConn(&conn); + + ok = blocks_fill(conn); + + CKPQFinish(&conn); return ok; } static bool getdata3() { - PGconn *conn = dbconnect(); + PGconn *conn = NULL; bool ok = true; + CKPQConn(&conn); + if (!key_update && !confirm_sharesummary) { if (!(ok = paymentaddresses_fill(conn)) || everyone_die) goto sukamudai; @@ -1715,12 +1791,12 @@ static bool getdata3() if (!(ok = miningpayouts_fill(conn)) || everyone_die) goto sukamudai; } - PQfinish(conn); - conn = dbconnect(); + CKPQFinish(&conn); + CKPQConn(&conn); if (!(ok = workinfo_fill(conn)) || everyone_die) goto sukamudai; - PQfinish(conn); - conn = dbconnect(); + CKPQFinish(&conn); + CKPQConn(&conn); if (!(ok = marks_fill(conn)) || everyone_die) goto sukamudai; /* must be after workinfo */ @@ -1731,14 +1807,14 @@ static bool getdata3() if (!(ok = payouts_fill(conn)) || everyone_die) goto sukamudai; } - PQfinish(conn); - conn = dbconnect(); + CKPQFinish(&conn); + CKPQConn(&conn); if (!key_update) { if (!(ok = markersummary_fill(conn)) || everyone_die) goto sukamudai; } - PQfinish(conn); - conn = dbconnect(); + CKPQFinish(&conn); + CKPQConn(&conn); if (!key_update) { if (!(ok = shares_fill(conn)) || everyone_die) goto sukamudai; @@ -1748,7 +1824,7 @@ static bool getdata3() sukamudai: - PQfinish(conn); + CKPQFinish(&conn); return ok; } @@ -1915,8 +1991,9 @@ static void alloc_storage() ALLOC_LOGQUEUE, LIMIT_LOGQUEUE, true); logqueue_store = k_new_store(logqueue_free); - breakqueue_free = k_new_list("BreakQueue", sizeof(BREAKQUEUE), - ALLOC_BREAKQUEUE, LIMIT_BREAKQUEUE, true); + breakqueue_free = k_new_list_cull("BreakQueue", sizeof(BREAKQUEUE), + ALLOC_BREAKQUEUE, LIMIT_BREAKQUEUE, + true, CULL_BREAKQUEUE); reload_breakqueue_store = k_new_store(breakqueue_free); reload_done_breakqueue_store = k_new_store(breakqueue_free); cmd_breakqueue_store = k_new_store(breakqueue_free); @@ -1970,15 +2047,19 @@ static void alloc_storage() } } - seqtrans_free = k_new_list("SeqTrans", sizeof(SEQTRANS), - ALLOC_SEQTRANS, LIMIT_SEQTRANS, true); + seqtrans_free = k_new_list_cull("SeqTrans", sizeof(SEQTRANS), + ALLOC_SEQTRANS, LIMIT_SEQTRANS, true, + CULL_SEQTRANS); - msgline_free = k_new_list("MsgLine", sizeof(MSGLINE), - ALLOC_MSGLINE, LIMIT_MSGLINE, true); + msgline_free = k_new_list_cull("MsgLine", sizeof(MSGLINE), + ALLOC_MSGLINE, LIMIT_MSGLINE, true, + CULL_MSGLINE); msgline_store = k_new_store(msgline_free); + msgline_free->dsp_func = dsp_msgline; - workqueue_free = k_new_list("WorkQueue", sizeof(WORKQUEUE), - ALLOC_WORKQUEUE, LIMIT_WORKQUEUE, true); + workqueue_free = k_new_list_cull("WorkQueue", sizeof(WORKQUEUE), + ALLOC_WORKQUEUE, LIMIT_WORKQUEUE, + true, CULL_WORKQUEUE); pool0_workqueue_store = k_new_store(workqueue_free); pool_workqueue_store = k_new_store(workqueue_free); cmd_workqueue_store = k_new_store(workqueue_free); @@ -1997,8 +2078,9 @@ static void alloc_storage() LIMIT_HEARTBEATQUEUE, true); heartbeatqueue_store = k_new_store(heartbeatqueue_free); - transfer_free = k_new_list(Transfer, sizeof(TRANSFER), - ALLOC_TRANSFER, LIMIT_TRANSFER, true); + transfer_free = k_new_list_cull(Transfer, sizeof(TRANSFER), + ALLOC_TRANSFER, LIMIT_TRANSFER, true, + cull_transfer); transfer_free->dsp_func = dsp_transfer; users_free = k_new_list("Users", sizeof(USERS), @@ -2051,6 +2133,8 @@ static void alloc_storage() idcontrol_free = k_new_list("IDControl", sizeof(IDCONTROL), ALLOC_IDCONTROL, LIMIT_IDCONTROL, true); idcontrol_store = k_new_store(idcontrol_free); + idcontrol_root = new_ktree(NULL, cmp_idcontrol, idcontrol_free); + idcontrol_free->dsp_func = dsp_idcontrol; esm_free = k_new_list("ESM", sizeof(ESM), ALLOC_ESM, LIMIT_ESM, true); esm_store = k_new_store(esm_free); @@ -2215,6 +2299,9 @@ static void alloc_storage() userinfo_store = k_new_store(userinfo_free); userinfo_root = new_ktree(NULL, cmp_userinfo, userinfo_free); + // Emulate a list for lock checking + pgdb_free = k_lock_only_list("PGDB"); + #if LOCK_CHECK DLPRIO(seqset, 91); @@ -2269,12 +2356,15 @@ static void alloc_storage() DLPRIO(paymentaddresses, 5); + // Must be above instransient + DLPRIO(idcontrol, 3); + // Don't currently nest any locks in these: DLPRIO(esm, PRIO_TERMINAL); DLPRIO(workers, PRIO_TERMINAL); - DLPRIO(idcontrol, PRIO_TERMINAL); DLPRIO(ips, PRIO_TERMINAL); DLPRIO(replies, PRIO_TERMINAL); + DLPRIO(pgdb, PRIO_TERMINAL); DLPCHECK(); @@ -2612,7 +2702,7 @@ static void dealloc_storage() esm_report(); FREE_ALL(esm); - FREE_LISTS(idcontrol); + FREE_ALL(idcontrol); FREE_ALL(accountbalance); FREE_ALL(payments); @@ -2848,13 +2938,16 @@ static bool setup_data() #define DATASETTRANS(_seqdata, _u) \ ENTRYSETTRANS(&((_seqdata)->entry[(_u) & ((_seqdata)->size - 1)])) -// Check for transient missing every 2s +// Check for transient missing every X seconds #define TRANCHECKLIMIT 2.0 static tv_t last_trancheck; // Don't let these messages be slowed down by a trans_process() #define TRANCHKSEQOK(_seq) ((_seq) != SEQ_SHARES && (_seq) != SEQ_AUTH && \ (_seq) != SEQ_ADDRAUTH && (_seq) != SEQ_BLOCK) +// How many seconds to allow the build up of trans range messages +#define TRANSAGELIMIT 10.0 + /* time (now) is used, not cd, since cd is only relevant to reloading * and we don't run trans_process() during reloading * We also only know now, not cd, for a missing item @@ -3049,9 +3142,6 @@ static void trans_seq(tv_t *now) if (store->count) { K_WLOCK(seqtrans_free); k_list_transfer_to_head(store, seqtrans_free); - if (seqtrans_free->count == seqtrans_free->total && - seqtrans_free->total >= ALLOC_SEQTRANS * CULL_SEQTRANS) - k_cull_list(seqtrans_free); K_WUNLOCK(seqtrans_free); } } @@ -3202,7 +3292,7 @@ static bool update_seq(enum seq_num seq, uint64_t n_seqcmd, goto gotseqset; } } - } + } // Need to setup a new seqset newseq = true; @@ -3464,7 +3554,7 @@ gotseqset: } } seqdata->seqbase++; - } + } seqdata->maxseq++; } // store n_seqcmd @@ -3678,10 +3768,10 @@ setitemdata: found[seq].forced_msg = true; } } - // Check if there are any ranges >= 2s old (or forced) + // Check if there are any ranges >= the limit (or forced) for (i = 0; i < SEQ_MAX; i++) { if (found[i].forced_msg || (found[i].last.tv_sec != 0 && - tvdiff(&found_now, &(found[i].last)) >= 2.0)) { + tvdiff(&found_now, &(found[i].last)) >= TRANSAGELIMIT)) { memcpy(&(found_msgs[i]), &(found[i]), sizeof(SEQFOUND)); // will be displayed, so erase it @@ -3715,7 +3805,7 @@ setitemdata: setnow(&found_now); for (i = 0; i < SEQ_MAX; i++) { if (found[i].last.tv_sec != 0 && - tvdiff(&found_now, &(found[i].last)) >= 2.0) { + tvdiff(&found_now, &(found[i].last)) >= TRANSAGELIMIT) { memcpy(&(found_msgs[i]), &(found[i]), sizeof(SEQFOUND)); @@ -3830,9 +3920,6 @@ setitemdata: } K_WLOCK(seqtrans_free); k_list_transfer_to_head(lost, seqtrans_free); - if (seqtrans_free->count == seqtrans_free->total && - seqtrans_free->total >= ALLOC_SEQTRANS * CULL_SEQTRANS) - k_cull_list(seqtrans_free); K_WUNLOCK(seqtrans_free); } @@ -4456,6 +4543,9 @@ static void *breaker(void *arg) ts_t when, when_add; int i, typ, mythread, done, tot, ret; int breaker_delta = 0; + ts_t last_sleep = { 0L, 0L }; + int last_sleep_ms = 0; + bool do_sleep = false; setup = (struct breaker_setup *)(arg); mythread = setup->thread; @@ -4525,13 +4615,17 @@ static void *breaker(void *arg) } K_WUNLOCK(breakqueue_free); while (!everyone_die) { - if (mythread && !breaker_running[typ][mythread]) + if (mythread && !breaker_running[typ][mythread]) break; K_WLOCK(breakqueue_free); bq_item = NULL; was_null = false; - if (mythread == 0 && reload && reload_breakdown_threads_delta != 0) { + if (breaker_sleep_stt.tv_sec > last_sleep.tv_sec) { + copy_ts(&last_sleep, &breaker_sleep_stt); + last_sleep_ms = breaker_sleep_ms; + do_sleep = true; + } else if (mythread == 0 && reload && reload_breakdown_threads_delta != 0) { breaker_delta = reload_breakdown_threads_delta; reload_breakdown_threads_delta = 0; } else if (mythread == 0 && !reload && cmd_breakdown_threads_delta != 0) { @@ -4561,6 +4655,12 @@ static void *breaker(void *arg) } K_WUNLOCK(breakqueue_free); + if (do_sleep) { + do_sleep = false; + cksleep_ms_r(&last_sleep, last_sleep_ms); + continue; + } + // TODO: deal with thread creation/shutdown failure if (breaker_delta != 0) { if (breaker_delta > 0) { @@ -4724,10 +4824,6 @@ static void *breaker(void *arg) pthread_cond_signal(&process_socket_waitcond); mutex_unlock(&process_socket_waitlock); } - - if (breakqueue_free->count == breakqueue_free->total && - breakqueue_free->total >= ALLOC_BREAKQUEUE * CULL_BREAKQUEUE) - k_cull_list(breakqueue_free); K_WUNLOCK(breakqueue_free); } @@ -6035,7 +6131,7 @@ static void process_sockd(PGconn *conn, K_ITEM *wq_item, enum reply_type reply_t K_WUNLOCK(breakqueue_free); FREENULL(ans); - free_msgline_data(ml_item, true, true); + free_msgline_data(ml_item, true); K_WLOCK(msgline_free); msgline_free->ram -= msgline->msgsiz; k_add_head(msgline_free, ml_item); @@ -6043,147 +6139,236 @@ static void process_sockd(PGconn *conn, K_ITEM *wq_item, enum reply_type reply_t K_WLOCK(workqueue_free); k_add_head(workqueue_free, wq_item); - if (workqueue_free->count == workqueue_free->total && - workqueue_free->total >= ALLOC_WORKQUEUE * CULL_WORKQUEUE) - k_cull_list(workqueue_free); K_WUNLOCK(workqueue_free); tick(); } -static void *clistener(__maybe_unused void *arg) +struct listener_setup { + int bc; + int thread; +}; + +#define BC_B 0 +#define BC_C 1 + +static pthread_t listener_pt[2][THREAD_LIMIT]; +static struct listener_setup listener_setup[2][THREAD_LIMIT]; + +static void *listener_all(void *arg) { + static bool running[2][THREAD_LIMIT]; + + struct listener_setup *setup; PGconn *conn = NULL; K_ITEM *wq_item; tv_t now1, now2; char buf[128]; time_t now; ts_t when, when_add; - int ret; + int i, typ, mythread, done, tot, ret; + int listener_delta = 0; - pthread_detach(pthread_self()); + setup = (struct listener_setup *)(arg); + typ = setup->bc; + mythread = setup->thread; - snprintf(buf, sizeof(buf), "db%s_%s", dbcode, __func__); + snprintf(buf, sizeof(buf), "db%s_%c%02d%s", + dbcode, (typ == BC_B) ? 'b' : 'c', mythread, "listen"); LOCK_INIT(buf); rename_proc(buf); - LOGNOTICE("%s() processing", __func__); - - when_add.tv_sec = CMD_QUEUE_SLEEP_MS / 1000; - when_add.tv_nsec = (CMD_QUEUE_SLEEP_MS % 1000) * 1000000; - - clistener_using_data = true; - - conn = dbconnect(); - now = time(NULL); - - while (!everyone_die) { - K_WLOCK(workqueue_free); - wq_item = k_unlink_head(cmd_workqueue_store); - K_WUNLOCK(workqueue_free); + if (mythread == 0) { + pthread_detach(pthread_self()); - // Don't keep a connection for more than ~10s - if ((time(NULL) - now) > 10) { - PQfinish(conn); - conn = dbconnect(); - now = time(NULL); + for (i = 1; i < THREAD_LIMIT; i++) { + listener_setup[typ][i].thread = i; + listener_setup[typ][i].bc = typ; + running[typ][i] = false; } + running[typ][0] = true; - if (wq_item) { - setnow(&now1); - process_sockd(conn, wq_item, REPLIER_CMD); - setnow(&now2); - clis_us += us_tvdiff(&now2, &now1); - clis_processed++; - } else { - setnowts(&when); - timeraddspec(&when, &when_add); + if (typ == BC_B) + listener_delta = btc_listener_threads - 1; + else + listener_delta = cmd_listener_threads - 1; - mutex_lock(&wq_cmd_waitlock); - ret = cond_timedwait(&wq_cmd_waitcond, - &wq_cmd_waitlock, &when); - if (ret == 0) - wq_cmd_wakes++; - else if (errno == ETIMEDOUT) - wq_cmd_timeouts++; - mutex_unlock(&wq_cmd_waitlock); - } + LOGNOTICE("%s() %s initialised - delta %d", + __func__, (typ == BC_B) ? "btc" : "cmd", listener_delta); } - - LOGNOTICE("%s() exiting, processed %"PRIu64, __func__, clis_processed); - - clistener_using_data = false; - - if (conn) - PQfinish(conn); - - return NULL; -} - -static void *blistener(__maybe_unused void *arg) -{ - PGconn *conn = NULL; - K_ITEM *wq_item; - tv_t now1, now2; - char buf[128]; - time_t now; - ts_t when, when_add; - int ret; - - pthread_detach(pthread_self()); - - snprintf(buf, sizeof(buf), "db%s_%s", dbcode, __func__); - LOCK_INIT(buf); - rename_proc(buf); - - LOGNOTICE("%s() processing", __func__); + LOGNOTICE("%s() %s processing", __func__, buf); when_add.tv_sec = CMD_QUEUE_SLEEP_MS / 1000; when_add.tv_nsec = (CMD_QUEUE_SLEEP_MS % 1000) * 1000000; - blistener_using_data = true; + if (typ == BC_B) + blistener_using_data = true; + else + clistener_using_data = true; + CKPQConn(&conn); now = time(NULL); while (!everyone_die) { + if (mythread && !running[typ][mythread]) + break; + + wq_item = NULL; K_WLOCK(workqueue_free); - wq_item = k_unlink_head(btc_workqueue_store); + if (mythread == 0 && typ == BC_B && + btc_listener_threads_delta != 0) { + listener_delta = btc_listener_threads_delta; + btc_listener_threads_delta = 0; + } else if (mythread == 0 && typ == BC_C && + cmd_listener_threads_delta != 0) { + listener_delta = cmd_listener_threads_delta; + cmd_listener_threads_delta = 0; + } else { + if (typ == BC_B) + wq_item = k_unlink_head(btc_workqueue_store); + else + wq_item = k_unlink_head(cmd_workqueue_store); + } K_WUNLOCK(workqueue_free); + // TODO: deal with thread creation/shutdown failure + if (listener_delta != 0) { + if (listener_delta > 0) { + // Add threads + tot = 1; + done = 0; + for (i = 1; i < THREAD_LIMIT; i++) { + if (!running[typ][i]) { + if (listener_delta > 0) { + listener_delta--; + running[typ][i] = true; + create_pthread(&(listener_pt[typ][i]), + listener_all, + &(listener_setup[typ][i])); + done++; + tot++; + } + } else + tot++; + } + LOGWARNING("%s() created %d %s thread%s total=%d" +#if LOCK_CHECK + " next_thread_id=%d" +#endif + , __func__, + done, + (typ == BC_B) ? "btc" : "cmd", + (done == 1) ? EMPTY : "s", + tot +#if LOCK_CHECK + , next_thread_id +#endif + ); + } else { + // Notify and wait for each to exit + tot = 1; + done = 0; + for (i = THREAD_LIMIT - 1; i > 0; i--) { + if (running[typ][i]) { + if (listener_delta < 0) { + listener_delta++; + LOGNOTICE("%s() %s stopping %d", + __func__, + (typ == BC_B) ? "btc" : "cmd", + i); + running[typ][i] = false; + join_pthread(listener_pt[typ][i]); + done++; + } else + tot++; + } + } + LOGWARNING("%s() stopped %d %s thread%s total=%d" +#if LOCK_CHECK + " next_thread_id=%d" +#endif + , __func__, + done, + (typ == BC_B) ? "btc" : "cmd", + (done == 1) ? EMPTY : "s", + tot +#if LOCK_CHECK + , next_thread_id +#endif + ); + } + listener_delta = 0; + continue; + } + // Don't keep a connection for more than ~10s if ((time(NULL) - now) > 10) { - PQfinish(conn); - conn = dbconnect(); + CKPQFinish(&conn); + CKPQConn(&conn); now = time(NULL); } if (wq_item) { setnow(&now1); - process_sockd(conn, wq_item, REPLIER_BTC); + process_sockd(conn, wq_item, + (typ == BC_B) ? REPLIER_BTC : REPLIER_CMD); setnow(&now2); - blis_us += us_tvdiff(&now2, &now1); - blis_processed++; + ck_wlock(&listener_all_lock); + if (typ == BC_B) { + blis_us += us_tvdiff(&now2, &now1); + blis_processed++; + } else { + clis_us += us_tvdiff(&now2, &now1); + clis_processed++; + } + ck_wunlock(&listener_all_lock); } else { setnowts(&when); timeraddspec(&when, &when_add); - mutex_lock(&wq_btc_waitlock); - ret = cond_timedwait(&wq_btc_waitcond, - &wq_btc_waitlock, &when); - if (ret == 0) - wq_btc_wakes++; - else if (errno == ETIMEDOUT) - wq_btc_timeouts++; - mutex_unlock(&wq_btc_waitlock); + if (typ == BC_B) { + mutex_lock(&wq_btc_waitlock); + ret = cond_timedwait(&wq_btc_waitcond, + &wq_btc_waitlock, &when); + if (ret == 0) + wq_btc_wakes++; + else if (errno == ETIMEDOUT) + wq_btc_timeouts++; + mutex_unlock(&wq_btc_waitlock); + } else { + mutex_lock(&wq_cmd_waitlock); + ret = cond_timedwait(&wq_cmd_waitcond, + &wq_cmd_waitlock, &when); + if (ret == 0) + wq_cmd_wakes++; + else if (errno == ETIMEDOUT) + wq_cmd_timeouts++; + mutex_unlock(&wq_cmd_waitlock); + } } } + CKPQFinish(&conn); - LOGNOTICE("%s() exiting, processed %"PRIu64, __func__, blis_processed); - - blistener_using_data = false; + if (mythread != 0) + LOGNOTICE("%s() %s exiting", __func__, buf); + else { + for (i = 1; i < THREAD_LIMIT; i++) { + if (running[typ][i]) { + running[typ][i] = false; + LOGNOTICE("%s() %s waiting for %d", + __func__, buf, i); + join_pthread(listener_pt[typ][i]); + } + } - if (conn) - PQfinish(conn); + if (typ == BC_B) { + LOGNOTICE("%s() %s exiting, processed %"PRIu64, __func__, buf, blis_processed); + blistener_using_data = false; + } else { + LOGNOTICE("%s() %s exiting, processed %"PRIu64, __func__, buf, clis_processed); + clistener_using_data = false; + } + } return NULL; } @@ -6251,6 +6436,7 @@ static void *process_socket(__maybe_unused void *arg) case CMD_CHKPASS: case CMD_GETATTS: case CMD_THREADS: + case CMD_PAUSE: case CMD_HOMEPAGE: case CMD_QUERY: break; @@ -6452,6 +6638,7 @@ static void *process_socket(__maybe_unused void *arg) case CMD_EVENTS: case CMD_HIGH: case CMD_THREADS: + case CMD_PAUSE: case CMD_QUERY: msgline->sockd = bq->sockd; bq->sockd = -1; @@ -6613,7 +6800,7 @@ static void *process_socket(__maybe_unused void *arg) K_ITEM *ml_item = wq->msgline_item; MSGLINE *ml; DATA_MSGLINE(ml, ml_item); - free_msgline_data(ml_item, true, false); + free_msgline_data(ml_item, true); K_WLOCK(msgline_free); msgline_free->ram -= ml->msgsiz; k_add_head(msgline_free, ml_item); @@ -6656,7 +6843,7 @@ skippy: if (bq->ml_item) { MSGLINE *ml; DATA_MSGLINE(ml, bq->ml_item); - free_msgline_data(bq->ml_item, true, true); + free_msgline_data(bq->ml_item, true); K_WLOCK(msgline_free); msgline_free->ram -= ml->msgsiz; k_add_head(msgline_free, bq->ml_item); @@ -6841,7 +7028,7 @@ static void *socksetup(__maybe_unused void *arg) { pthread_t prep_pt, crep_pt, brep_pt; enum reply_type p_typ, c_typ, b_typ; - pthread_t clis_pt, blis_pt, proc_pt; + pthread_t proc_pt; pthread_t psock_pt, wsock_pt, csock_pt; char nbuf[64]; @@ -6868,9 +7055,15 @@ static void *socksetup(__maybe_unused void *arg) LOGWARNING("%s() Start processing...", __func__); socksetup_using_data = true; - create_pthread(&clis_pt, clistener, NULL); + listener_setup[BC_B][0].bc = BC_B; + listener_setup[BC_B][0].thread = 0; + create_pthread(&listener_pt[BC_B][0], listener_all, + &(listener_setup[BC_B][0])); - create_pthread(&blis_pt, blistener, NULL); + listener_setup[BC_C][0].bc = BC_C; + listener_setup[BC_C][0].thread = 0; + create_pthread(&listener_pt[BC_C][0], listener_all, + &(listener_setup[BC_C][0])); create_pthread(&proc_pt, process_socket, NULL); @@ -6960,6 +7153,7 @@ static void process_reload_item(PGconn *conn, K_ITEM *bq_item) case CMD_EVENTS: case CMD_HIGH: case CMD_THREADS: + case CMD_PAUSE: LOGERR("%s() INVALID message line %"PRIu64 " ignored '%.42s...", __func__, bq->count, @@ -7010,7 +7204,7 @@ static void process_reload_item(PGconn *conn, K_ITEM *bq_item) if (bq->ml_item) { DATA_MSGLINE(msgline, bq->ml_item); - free_msgline_data(bq->ml_item, true, true); + free_msgline_data(bq->ml_item, true); K_WLOCK(msgline_free); msgline_free->ram -= msgline->msgsiz; k_add_head(msgline_free, bq->ml_item); @@ -7062,7 +7256,7 @@ static void *process_reload(__maybe_unused void *arg) when_add.tv_sec = RELOAD_QUEUE_SLEEP_MS / 1000; when_add.tv_nsec = (RELOAD_QUEUE_SLEEP_MS % 1000) * 1000000; - conn = dbconnect(); + CKPQConn(&conn); now = time(NULL); while (!everyone_die) { @@ -7166,8 +7360,8 @@ static void *process_reload(__maybe_unused void *arg) // Don't keep a connection for more than ~10s ... of processing if ((time(NULL) - now) > 10) { - PQfinish(conn); - conn = dbconnect(); + CKPQFinish(&conn); + CKPQConn(&conn); now = time(NULL); } @@ -7182,9 +7376,7 @@ static void *process_reload(__maybe_unused void *arg) tick(); } - - if (conn) - PQfinish(conn); + CKPQFinish(&conn); if (mythread == 0) { for (i = 1; i < THREAD_LIMIT; i++) { @@ -7429,7 +7621,7 @@ static bool reload_from(tv_t *start, const tv_t *finish) * Also since ckpool messages are not in order, we could be * aborting early and not get the few slightly later out of * order messages in the log file */ - while (!everyone_die && + while (!everyone_die && logline(reload_buf, MAX_READ, fp, filename)) { reload_line(filename, reload_buf, ++count); @@ -7649,7 +7841,7 @@ static void process_queued(PGconn *conn, K_ITEM *wq_item) break; } - free_msgline_data(ml_item, true, true); + free_msgline_data(ml_item, true); K_WLOCK(msgline_free); msgline_free->ram -= msgline->msgsiz; k_add_head(msgline_free, ml_item); @@ -7657,9 +7849,6 @@ static void process_queued(PGconn *conn, K_ITEM *wq_item) K_WLOCK(workqueue_free); k_add_head(workqueue_free, wq_item); - if (workqueue_free->count == workqueue_free->total && - workqueue_free->total >= ALLOC_WORKQUEUE * CULL_WORKQUEUE) - k_cull_list(workqueue_free); K_WUNLOCK(workqueue_free); } @@ -7668,9 +7857,6 @@ static void free_lost(SEQDATA *seqdata) if (seqdata->reload_lost) { K_WLOCK(seqtrans_free); k_list_transfer_to_head(seqdata->reload_lost, seqtrans_free); - if (seqtrans_free->count == seqtrans_free->total && - seqtrans_free->total >= ALLOC_SEQTRANS * CULL_SEQTRANS) - k_cull_list(seqtrans_free); K_WUNLOCK(seqtrans_free); seqdata->reload_lost = NULL; } @@ -7725,7 +7911,7 @@ static void *pqproc(void *arg) when_add.tv_nsec = (CMD_QUEUE_SLEEP_MS % 1000) * 1000000; now = time(NULL); - conn = dbconnect(); + CKPQConn(&conn); wqgot = 0; // Override checking until pool0 is complete @@ -7838,8 +8024,8 @@ static void *pqproc(void *arg) /* Don't keep a connection for more than ~10s or ~10000 items * but always have a connection open */ if ((time(NULL) - now) > 10 || wqgot > 10000) { - PQfinish(conn); - conn = dbconnect(); + CKPQFinish(&conn); + CKPQConn(&conn); now = time(NULL); wqgot = 0; } @@ -7909,9 +8095,7 @@ static void *pqproc(void *arg) mutex_unlock(&wq_pool_waitlock); } } - - if (conn) - PQfinish(conn); + CKPQFinish(&conn); if (mythread == 0) { for (i = 1; i < THREAD_LIMIT; i++) { @@ -8995,6 +9179,7 @@ static struct option long_options[] = { // override calculated value { "breakdown-threads", required_argument, 0, 'B' }, { "config", required_argument, 0, 'c' }, + { "cmd-listener-threads", required_argument, 0, 'C' }, { "dbname", required_argument, 0, 'd' }, { "minsdiff", required_argument, 0, 'D' }, { "free", required_argument, 0, 'f' }, @@ -9077,7 +9262,7 @@ int main(int argc, char **argv) memset(&ckpcmd, 0, sizeof(ckp)); ckp.loglevel = LOG_NOTICE; - while ((c = getopt_long(argc, argv, "a:Ab:B:c:d:D:f:ghi:IkK:l:L:mM:n:N:o:p:P:q:Q:r:R:s:S:t:Tu:U:vw:xXyY:", long_options, &i)) != -1) { + while ((c = getopt_long(argc, argv, "a:Ab:B:c:C:d:D:f:ghi:IkK:l:L:mM:n:N:o:p:P:q:Q:r:R:s:S:t:Tu:U:vw:xXyY:", long_options, &i)) != -1) { switch(c) { case '?': case ':': @@ -9120,6 +9305,18 @@ int main(int argc, char **argv) case 'c': ckp.config = strdup(optarg); break; + case 'C': + { + int cl = atoi(optarg); + if (cl < 1 || cl > THREAD_LIMIT) { + quit(1, "Invalid listener " + "thread count %d " + "- must be >0 and <=%d", + cl, THREAD_LIMIT); + } + cmd_listener_threads = cl; + } + break; case 'd': db_name = strdup(optarg); kill = optarg; @@ -9467,8 +9664,10 @@ int main(int argc, char **argv) cklock_init(&breakdown_lock); cklock_init(&replier_lock); + cklock_init(&listener_all_lock); cklock_init(&last_lock); cklock_init(&btc_lock); + cklock_init(&pgdb_pause_lock); cklock_init(&poolinstance_lock); cklock_init(&seq_found_lock); diff --git a/src/ckdb.h b/src/ckdb.h index 135494ce..1a9419fd 100644 --- a/src/ckdb.h +++ b/src/ckdb.h @@ -58,7 +58,7 @@ #define DB_VLOCK "1" #define DB_VERSION "1.0.7" -#define CKDB_VERSION DB_VERSION"-2.508" +#define CKDB_VERSION DB_VERSION"-2.715" #define WHERE_FFL " - from %s %s() line %d" #define WHERE_FFL_HERE __FILE__, __func__, __LINE__ @@ -70,6 +70,10 @@ #define STRINT(x) STRINT2(x) #define STRINT2(x) #x +// Same as above but name it what we are using it for +#define STRMACRO(x) STRMAC2(x) +#define STRMAC2(x) #x + // So they can fit into a 1 byte flag field #define TRUE_STR "Y" #define TRUE2_STR "T" @@ -129,8 +133,14 @@ extern int proc_queue_threads_delta; extern int reload_breakdown_threads_delta; extern int cmd_breakdown_threads_delta; +extern int cmd_listener_threads; +extern int btc_listener_threads; +extern int cmd_listener_threads_delta; +extern int btc_listener_threads_delta; + #define BLANK " " extern char *EMPTY; +#define NULLSTR "(null)" extern const char *nullstr; extern const char *true_str; @@ -346,7 +356,7 @@ extern bool dbload_only_sharesummary; * markersummaries and pplns payouts may not be correct */ extern bool sharesummary_marks_limit; -// DB users,workers load is complete +// DB optioncontrol,idcontrol,users,workers,useratts load is complete extern bool db_users_complete; // DB load is complete extern bool db_load_complete; @@ -358,6 +368,8 @@ extern bool reloading; extern bool reloaded_N_files; // Data load is complete extern bool startup_complete; +// Set to true when pool0 completes, pool0 = socket data during reload +extern bool reload_queue_complete; // Tell everyone to die extern bool everyone_die; @@ -402,24 +414,81 @@ extern int btc_timeout; // Lock access to the above variables so they can be changed extern cklock_t btc_lock; -#define EDDB "expirydate" -#define CDDB "createdate" -#define CDTRF CDDB -#define BYDB "createby" -#define BYTRF BYDB -#define CODEDB "createcode" -#define CODETRF CODEDB -#define INETDB "createinet" -#define INETTRF INETDB -#define MDDB "modifydate" -#define MBYDB "modifyby" -#define MCODEDB "modifycode" -#define MINETDB "modifyinet" +#define _EDDB expirydate +#define _CDDB createdate +#define _CDTRF _CDDB +#define _BYDB createby +#define _BYTRF _BYDB +#define _CODEDB createcode +#define _CODETRF _CODEDB +#define _INETDB createinet +#define _INETTRF _INETDB +#define _MDDB modifydate +#define _MBYDB modifyby +#define _MCODEDB modifycode +#define _MINETDB modifyinet + +#define EDDB STRMACRO(_EDDB) +#define CDDB STRMACRO(_CDDB) +#define CDTRF STRMACRO(_CDTRF) +#define BYDB STRMACRO(_BYDB) +#define BYTRF STRMACRO(_BYTRF) +#define CODEDB STRMACRO(_CODEDB) +#define CODETRF STRMACRO(_CODETRF) +#define INETDB STRMACRO(_INETDB) +#define INETTRF STRMACRO(_INETTRF) +#define MDDB STRMACRO(_MDDB) +#define MBYDB STRMACRO(_MBYDB) +#define MCODEDB STRMACRO(_MCODEDB) +#define MINETDB STRMACRO(_MINETDB) extern char *by_default; extern char *inet_default; extern char *id_default; +// Emulate a list for lock checking +extern K_LIST *pgdb_free; +// Count of db connections +extern int pgdb_count; +extern __thread char *connect_file; +extern __thread char *connect_func; +extern __thread int connect_line; +extern __thread bool connect_dis; +/* (WHEN FINISHED) Pause all DB IO (permanently) pause.1.name=pgTABconfirm=Y + * Without confirm=Y it will return the pause state + * All DB IO commands must take out a read of this lock before + * starting anything that shouldn't be done partially + * This also means that the whole of CKDB can lock up for a short + * time if e.g. shift processing has taken the read lock shortly before + * the write lock is taken for the pause request to set the flag + * Once pgdb_paused is true, all DB IO will not access the database + * and all web access will be in read only mode i.e. no user changes + * All connections to the DB will close and thus DB changes and outages + * can then occur without affecting CKDB at all + * check the connection count with query.1.request=pg + * Shift generation is unchanged, Payout generation is permanently disabled + * To restart CKDB you must terminate it then rerun it, then CKDB will + * reload/redo all ckpool data it didn't store in the database + * The aim is to look the same as a normal running CKDB except doing no + * DB I/O - that will be deferred until CKDB is later restarted + * You can't pause CKDB until the dbload has completed, but you can + * during the CCL reload + * The function to take out the pause lock increments the thread's + * pause_read_count so each function that expects the lock to be held can + * easily test that and incorrectly calling it multiple times is tracked + * If the read lock code is called incorrectly, i.e. a code bug, + * pgdb_pause_disabled will be set to true and the pause command can no + * longer be activated, however if it was already activated, this wont + * affect anything */ +extern cklock_t pgdb_pause_lock; +extern __thread int pause_read_count; +extern __thread char *pause_read_file; +extern __thread char *pause_read_func; +extern __thread int pause_read_line; +extern __thread bool pause_read_unlock; +extern bool pgdb_paused; +extern bool pgdb_pause_disabled; + // Number of seconds per poolinstance message for run #define POOLINSTANCE_MSG_EVERY 30 @@ -724,6 +793,7 @@ enum cmd_values { CMD_EVENTS, CMD_HIGH, CMD_THREADS, + CMD_PAUSE, CMD_END }; @@ -1365,7 +1435,7 @@ typedef struct msgline { #define ALLOC_MSGLINE 8192 #define LIMIT_MSGLINE 0 -#define CULL_MSGLINE 8 +#define CULL_MSGLINE (8 * ALLOC_MSGLINE) #define INIT_MSGLINE(_item) INIT_GENERIC(_item, msgline) #define DATA_MSGLINE(_var, _item) DATA_GENERIC(_var, _item, msgline, true) #define DATA_MSGLINE_NULL(_var, _item) DATA_GENERIC(_var, _item, msgline, false) @@ -1391,7 +1461,7 @@ typedef struct breakqueue { #define ALLOC_BREAKQUEUE 16384 #define LIMIT_BREAKQUEUE 0 -#define CULL_BREAKQUEUE 4 +#define CULL_BREAKQUEUE (4 * ALLOC_BREAKQUEUE) #define INIT_BREAKQUEUE(_item) INIT_GENERIC(_item, breakqueue) #define DATA_BREAKQUEUE(_var, _item) DATA_GENERIC(_var, _item, breakqueue, true) @@ -1427,6 +1497,8 @@ extern int reload_processing; extern int cmd_processing; extern int sockd_count; extern int max_sockd_count; +extern ts_t breaker_sleep_stt; +extern int breaker_sleep_ms; // Trigger breaker() processing extern mutex_t bq_reload_waitlock; @@ -1458,7 +1530,7 @@ typedef struct workqueue { #define ALLOC_WORKQUEUE 1024 #define LIMIT_WORKQUEUE 0 -#define CULL_WORKQUEUE 32 +#define CULL_WORKQUEUE (32 * ALLOC_WORKQUEUE) #define INIT_WORKQUEUE(_item) INIT_GENERIC(_item, workqueue) #define DATA_WORKQUEUE(_var, _item) DATA_GENERIC(_var, _item, workqueue, true) @@ -1558,12 +1630,18 @@ typedef struct transfer { // Suggest malloc use MMAP = largest under 2MB #define ALLOC_TRANSFER ((int)(2*1024*1024/sizeof(TRANSFER))) #define LIMIT_TRANSFER 0 -#define CULL_TRANSFER 16 +// ALLOC_TRANSFER is ~14k, allocated often is 3 +#define CULL_TRANSFER (4 * ALLOC_TRANSFER) #define INIT_TRANSFER(_item) INIT_GENERIC(_item, transfer) #define DATA_TRANSFER(_var, _item) DATA_GENERIC(_var, _item, transfer, true) extern K_LIST *transfer_free; +/* Allow defining and adjusting it on a running system + * cull_limit is set to the optionvalue * ALLOC_TRANSFER */ +#define CULL_TRANSFER_NAME "CullTransfer" +extern int cull_transfer; + #define transfer_data(_item) _transfer_data(_item, WHERE_FFL_HERE) extern const char Transfer[]; @@ -1782,7 +1860,7 @@ extern K_LIST *seqtrans_free; #define ALLOC_SEQTRANS 1024 #define LIMIT_SEQTRANS 0 -#define CULL_SEQTRANS 64 +#define CULL_SEQTRANS (16 * ALLOC_SEQTRANS) #define INIT_SEQTRANS(_item) INIT_GENERIC(_item, seqtrans) #define DATA_SEQTRANS(_var, _item) DATA_GENERIC(_var, _item, seqtrans, true) #define DATA_SEQTRANS_NULL(_var, _item) DATA_GENERIC(_var, _item, seqtrans, false) @@ -1839,6 +1917,9 @@ typedef struct users { // Address account, not a username account #define USER_ADDRESS 0x1 +// Username created due to a share that had an unknown username +#define USER_MISSING 0x2 + // 16 x base 32 (5 bits) = 10 bytes (8 bits) #define TOTPAUTH_KEYSIZE 10 #define TOTPAUTH_DSP_KEYSIZE 16 @@ -2030,7 +2111,7 @@ extern K_STORE *accountadjustment_store; typedef struct idcontrol { char idname[TXT_SML+1]; int64_t lastid; - MODIFYDATECONTROLFIELDS; + MODIFYDATECONTROLIN; } IDCONTROL; #define ALLOC_IDCONTROL 16 @@ -2038,8 +2119,7 @@ typedef struct idcontrol { #define INIT_IDCONTROL(_item) INIT_GENERIC(_item, idcontrol) #define DATA_IDCONTROL(_var, _item) DATA_GENERIC(_var, _item, idcontrol, true) -// These are only used for db access - not stored in memory -//extern K_TREE *idcontrol_root; +extern K_TREE *idcontrol_root; extern K_LIST *idcontrol_free; extern K_STORE *idcontrol_store; @@ -2846,6 +2926,11 @@ extern K_STORE *userstats_eos_store; // newer OR equal #define tv_newer_eq(_old, _new) (!(tv_newer(_new, _old))) +#define copy_ts(_dest, _src) do { \ + (_dest)->tv_sec = (_src)->tv_sec; \ + (_dest)->tv_nsec = (_src)->tv_nsec; \ + } while(0) + // WORKERSTATUS from various incoming data typedef struct workerstatus { int64_t userid; @@ -3219,7 +3304,7 @@ extern void sequence_report(bool lock); #define FREE_ITEM(item) do { } while(0) // TODO: make a macro for all other to use above macro extern void free_transfer_data(TRANSFER *transfer); -extern void free_msgline_data(K_ITEM *item, bool t_lock, bool t_cull); +extern void free_msgline_data(K_ITEM *item, bool t_lock); extern void free_users_data(K_ITEM *item); extern void free_workinfo_data(K_ITEM *item); #define free_sharesummary_data(_i) FREE_ITEM(_i) @@ -3325,6 +3410,7 @@ extern INTRANSIENT *_get_intransient(const char *fldnam, char *value, #define intransient_str(_fld, _val) \ _intransient_str(_fld, _val, WHERE_FFL_HERE) extern char *_intransient_str(char *fldnam, char *value, WHERE_FFL_ARGS); +extern void dsp_msgline(K_ITEM *item, FILE *stream); extern char *_transfer_data(K_ITEM *item, WHERE_FFL_ARGS); extern void dsp_transfer(K_ITEM *item, FILE *stream); extern cmp_t cmp_transfer(K_ITEM *a, K_ITEM *b); @@ -3427,6 +3513,9 @@ extern K_ITEM *find_first_payments(int64_t userid, K_TREE_CTX *ctx); extern K_ITEM *find_first_paypayid(int64_t userid, int64_t payoutid, K_TREE_CTX *ctx); extern cmp_t cmp_accountbalance(K_ITEM *a, K_ITEM *b); extern K_ITEM *find_accountbalance(int64_t userid); +extern void dsp_idcontrol(K_ITEM *item, FILE *stream); +extern cmp_t cmp_idcontrol(K_ITEM *a, K_ITEM *b); +extern K_ITEM *find_idcontrol(char *idname); extern cmp_t cmp_optioncontrol(K_ITEM *a, K_ITEM *b); extern K_ITEM *find_optioncontrol(char *optionname, const tv_t *now, int32_t height); #define sys_setting(_name, _def, _now) user_sys_setting(0, _name, _def, _now) @@ -3603,12 +3692,12 @@ extern void userinfo_block(BLOCKS *blocks, enum info_type isnew, int delta); #define CKPQ_READ true #define CKPQ_WRITE false -#define CKPQexec(_conn, _qry, _isread) _CKPQexec(_conn, _qry, _isread, WHERE_FFL_HERE) -extern PGresult *_CKPQexec(PGconn *conn, const char *qry, bool isread, WHERE_FFL_ARGS); -#define CKPQexecParams(_conn, _qry, _p1, _p2, _p3, _p4, _p5, _p6, _isread) \ - _CKPQexecParams(_conn, _qry, _p1, _p2, _p3, _p4, _p5, _p6, \ +#define CKPQExec(_conn, _qry, _isread) _CKPQExec(_conn, _qry, _isread, WHERE_FFL_HERE) +extern PGresult *_CKPQExec(PGconn *conn, const char *qry, bool isread, WHERE_FFL_ARGS); +#define CKPQExecParams(_conn, _qry, _p1, _p2, _p3, _p4, _p5, _p6, _isread) \ + _CKPQExecParams(_conn, _qry, _p1, _p2, _p3, _p4, _p5, _p6, \ _isread, WHERE_FFL_HERE) -extern PGresult *_CKPQexecParams(PGconn *conn, const char *qry, +extern PGresult *_CKPQExecParams(PGconn *conn, const char *qry, int nParams, const Oid *paramTypes, const char *const * paramValues, @@ -3616,10 +3705,10 @@ extern PGresult *_CKPQexecParams(PGconn *conn, const char *qry, const int *paramFormats, int resultFormat, bool isread, WHERE_FFL_ARGS); - -// Force use CKPQ... for PQ functions in use -#define PQexec CKPQexec -#define PQexecParams CKPQexecParams +extern ExecStatusType _CKPQResultStatus(PGresult *res, WHERE_FFL_ARGS); +#define CKPQResultStatus(_res) _CKPQResultStatus(_res, WHERE_FFL_HERE) +extern void _CKPQClear(PGresult *res, WHERE_FFL_ARGS); +#define CKPQClear(_res) _CKPQClear(_res, WHERE_FFL_HERE) #define PGLOG(__LOG, __str, __rescode, __conn) do { \ char *__buf = pqerrmsg(__conn); \ @@ -3630,14 +3719,23 @@ extern PGresult *_CKPQexecParams(PGconn *conn, const char *qry, #define PGLOGERR(_str, _rescode, _conn) PGLOG(LOGERR, _str, _rescode, _conn) #define PGLOGEMERG(_str, _rescode, _conn) PGLOG(LOGEMERG, _str, _rescode, _conn) +#define PGLOGNOTICE(_str, _rescode, _conn) PGLOG(LOGNOTICE, _str, _rescode, _conn) +extern void _pause_read_lock(WHERE_FFL_ARGS); +#define pause_read_lock() _pause_read_lock(WHERE_FFL_HERE) +extern void _pause_read_unlock(WHERE_FFL_ARGS); +#define pause_read_unlock() _pause_read_unlock(WHERE_FFL_HERE) extern char *pqerrmsg(PGconn *conn); -extern bool CKPQConn(PGconn **conn); -extern void CKPQDisco(PGconn **conn, bool conned); +extern bool _CKPQConn(PGconn **conn, WHERE_FFL_ARGS); +#define CKPQConn(_conn) _CKPQConn(_conn, WHERE_FFL_HERE) +extern bool _CKPQDisco(PGconn **conn, bool conned, WHERE_FFL_ARGS); +#define CKPQDisco(_conn, _conned) _CKPQDisco(_conn, _conned, WHERE_FFL_HERE) +#define CKPQFinish(_conn) CKPQDisco(_conn, true) extern bool _CKPQBegin(PGconn *conn, WHERE_FFL_ARGS); #define CKPQBegin(_conn) _CKPQBegin(conn, WHERE_FFL_HERE) extern void _CKPQEnd(PGconn *conn, bool commit, WHERE_FFL_ARGS); #define CKPQEnd(_conn, _commit) _CKPQEnd(_conn, _commit, WHERE_FFL_HERE) +#define CKPQCommit(_conn) _CKPQEnd(_conn, true, WHERE_FFL_HERE) extern int64_t nextid(PGconn *conn, char *idname, int64_t increment, tv_t *cd, char *by, char *code, char *inet); @@ -3647,11 +3745,14 @@ extern bool users_update(PGconn *conn, K_ITEM *u_item, char *oldhash, int *event); extern K_ITEM *users_add(PGconn *conn, INTRANSIENT *in_username, char *emailaddress, char *passwordhash, - int64_t userbits, char *by, char *code, char *inet, - tv_t *cd, K_TREE *trf_root); + char *secondaryuserid, int64_t userbits, char *by, + char *code, char *inet, tv_t *cd, K_TREE *trf_root); extern bool users_replace(PGconn *conn, K_ITEM *u_item, K_ITEM *old_u_item, char *by, char *code, char *inet, tv_t *cd, K_TREE *trf_root); +extern K_ITEM *create_missing_user(PGconn *conn, char *username, + char *secondaryuserid, char *by, char *code, + char *inet, tv_t *cd, K_TREE *trf_root); extern bool users_fill(PGconn *conn); extern bool useratts_item_add(PGconn *conn, K_ITEM *ua_item, tv_t *cd, bool begun); @@ -3685,6 +3786,7 @@ extern bool payments_add(PGconn *conn, bool add, K_ITEM *p_item, extern bool payments_fill(PGconn *conn); extern bool idcontrol_add(PGconn *conn, char *idname, char *idvalue, char *by, char *code, char *inet, tv_t *cd, K_TREE *trf_root); +extern bool idcontrol_fill(PGconn *conn); extern K_ITEM *optioncontrol_item_add(PGconn *conn, K_ITEM *oc_item, tv_t *cd, bool begun); extern K_ITEM *optioncontrol_add(PGconn *conn, char *optionname, char *optionvalue, char *activationdate, char *activationheight, diff --git a/src/ckdb.php b/src/ckdb.php index 9ae4b1a9..8fa2b97e 100644 --- a/src/ckdb.php +++ b/src/ckdb.php @@ -19,10 +19,12 @@ function getsock2($fun, $tmo) return _getsock($fun, "$socket_dir$socket_name/$socket_file", $tmo); } # -function msg($line, $tmo = false) +function msg($line, $tabs, $tmo = false) { global $fld_sep, $val_sep; + if ($tabs) + $line = str_replace("TAB", "\t", $line); $fun = 'stdin'; $ret = false; $socket = getsock2($fun, $tmo); @@ -40,7 +42,8 @@ function msg($line, $tmo = false) function usAge($a0) { global $socket_name_def, $socket_dir_def, $socket_file_def; - echo "usAge: php $a0 [name [dir [socket]]]\n"; + echo "usAge: php $a0 [-t] [name [dir [socket]]]\n"; + echo " -t = don't convert 'TAB' to a tab character\n"; echo " default name = $socket_name_def\n"; echo " default dir = $socket_dir_def\n"; echo " default socket = $socket_file_def\n"; @@ -48,18 +51,26 @@ function usAge($a0) exit(1); } # +$tabs = true; +# if (count($argv) > 1) { if ($argv[1] == '-?' || $argv[1] == '-h' || $argv[1] == '-help' || $argv[1] == '--help') usAge($argv[0]); - $socket_name = $argv[1]; - if (count($argv) > 2) + $a = 1; + if ($argv[$a] == '-t') + { + $tabs = false; + $a++; + } + $socket_name = $argv[$a++]; + if (count($argv) > $a) { - $socket_dir = $argv[2]; - if (count($argv) > 3) - $socket_file = $argv[3]; + $socket_dir = $argv[$a++]; + if (count($argv) > $a) + $socket_file = $argv[$a]; } } # @@ -68,7 +79,7 @@ while ($line = fgets(STDIN)) $line = trim($line); if (strlen($line) > 0) { - $rep = msg($line); + $rep = msg($line, $tabs); if ($rep === false) echo "Failed\n"; else diff --git a/src/ckdb_cmd.c b/src/ckdb_cmd.c index 8754dddf..b64f2e43 100644 --- a/src/ckdb_cmd.c +++ b/src/ckdb_cmd.c @@ -72,8 +72,9 @@ static char *cmd_adduser(PGconn *conn, char *cmd, char *id, tv_t *now, char *by, if (event == EVENT_OK) { u_item = users_add(conn, in_username, transfer_data(i_emailaddress), - transfer_data(i_passwordhash), 0, - by, code, inet, now, trf_root); + transfer_data(i_passwordhash), + NULL, 0, by, code, inet, now, + trf_root); } } @@ -3116,7 +3117,7 @@ static char *cmd_auth_do(PGconn *conn, char *cmd, char *id, char *by, DATA_OPTIONCONTROL(optioncontrol, oc_item); u_item = users_add(conn, in_username, EMPTY, optioncontrol->optionvalue, - 0, by, code, inet, cd, + NULL, 0, by, code, inet, cd, trf_root); } else ok = false; @@ -3347,7 +3348,7 @@ static char *cmd_heartbeat(__maybe_unused PGconn *conn, char *cmd, char *id, goto pulse; } - hq_store = k_new_store(heartbeatqueue_free); + hq_store = k_new_store_locked(heartbeatqueue_free); k_list_transfer_to_head(heartbeatqueue_store, hq_store); K_WUNLOCK(heartbeatqueue_free); @@ -3871,8 +3872,6 @@ static char *cmd_setatts(PGconn *conn, char *cmd, char *id, __maybe_unused tv_t *notcd, K_TREE *trf_root, __maybe_unused bool reload_data) { - ExecStatusType rescode; - PGresult *res; bool conned = false; K_ITEM *t_item, *u_item, *ua_item = NULL; INTRANSIENT *in_username; @@ -3920,21 +3919,14 @@ static char *cmd_setatts(PGconn *conn, char *cmd, char *id, *(dot++) = '\0'; // If we already had a different one, save it to the DB if (ua_item && strcmp(useratts->attname, attname) != 0) { - if (conn == NULL) { - conn = dbconnect(); + if (CKPQConn(&conn)) conned = true; - } if (!begun) { - // Beginning of a write txn - res = PQexec(conn, "Begin", CKPQ_WRITE); - rescode = PQresultStatus(res); - PQclear(res); - if (!PGOK(rescode)) { - PGLOGERR("Begin", rescode, conn); + begun = CKPQBegin(conn); + if (!begun) { reason = "DBERR"; goto bats; } - begun = true; } if (useratts_item_add(conn, ua_item, now, begun)) { ua_item = NULL; @@ -3982,21 +3974,14 @@ static char *cmd_setatts(PGconn *conn, char *cmd, char *id, t_item = next_in_ktree(ctx); } if (ua_item) { - if (conn == NULL) { - conn = dbconnect(); + if (CKPQConn(&conn)) conned = true; - } if (!begun) { - // Beginning of a write txn - res = PQexec(conn, "Begin", CKPQ_WRITE); - rescode = PQresultStatus(res); - PQclear(res); - if (!PGOK(rescode)) { - PGLOGERR("Begin", rescode, conn); + begun = CKPQBegin(conn); + if (!begun) { reason = "DBERR"; goto bats; } - begun = true; } if (!useratts_item_add(conn, ua_item, now, begun)) { reason = "DBERR"; @@ -4006,15 +3991,11 @@ static char *cmd_setatts(PGconn *conn, char *cmd, char *id, } } rollback: - if (!reason) - res = PQexec(conn, "Commit", CKPQ_WRITE); - else - res = PQexec(conn, "Rollback", CKPQ_WRITE); - PQclear(res); + CKPQEnd(conn, (reason == NULL)); + bats: - if (conned) - PQfinish(conn); + CKPQDisco(&conn, conned); if (reason) { if (ua_item) { K_WLOCK(useratts_free); @@ -4206,8 +4187,6 @@ static char *cmd_setopts(PGconn *conn, char *cmd, char *id, __maybe_unused tv_t *notcd, K_TREE *trf_root, __maybe_unused bool reload_data) { - ExecStatusType rescode; - PGresult *res; bool conned = false; K_ITEM *t_item, *oc_item = NULL, *ok = NULL; K_TREE_CTX ctx[1]; @@ -4241,21 +4220,14 @@ static char *cmd_setopts(PGconn *conn, char *cmd, char *id, reason = "Missing value"; goto rollback; } - if (conn == NULL) { - conn = dbconnect(); + if (CKPQConn(&conn)) conned = true; - } if (!begun) { - // Beginning of a write txn - res = PQexec(conn, "Begin", CKPQ_WRITE); - rescode = PQresultStatus(res); - PQclear(res); - if (!PGOK(rescode)) { - PGLOGERR("Begin", rescode, conn); + begun = CKPQBegin(conn); + if (!begun) { reason = "DBERR"; goto rollback; } - begun = true; } ok = optioncontrol_item_add(conn, oc_item, now, begun); oc_item = NULL; @@ -4298,21 +4270,14 @@ static char *cmd_setopts(PGconn *conn, char *cmd, char *id, reason = "Missing value"; goto rollback; } - if (conn == NULL) { - conn = dbconnect(); + if (CKPQConn(&conn)) conned = true; - } if (!begun) { - // Beginning of a write txn - res = PQexec(conn, "Begin", CKPQ_WRITE); - rescode = PQresultStatus(res); - PQclear(res); - if (!PGOK(rescode)) { - PGLOGERR("Begin", rescode, conn); + begun = CKPQBegin(conn); + if (!begun) { reason = "DBERR"; goto rollback; } - begun = true; } ok = optioncontrol_item_add(conn, oc_item, now, begun); oc_item = NULL; @@ -4324,17 +4289,10 @@ static char *cmd_setopts(PGconn *conn, char *cmd, char *id, } } rollback: - if (begun) { - if (reason) - res = PQexec(conn, "Rollback", CKPQ_WRITE); - else - res = PQexec(conn, "Commit", CKPQ_WRITE); - - PQclear(res); - } + if (begun) + CKPQEnd(conn, (reason == NULL)); - if (conned) - PQfinish(conn); + CKPQDisco(&conn, conned); if (reason) { snprintf(reply, siz, "ERR.%s", reason); LOGERR("%s.%s.%s", cmd, id, reply); @@ -5903,45 +5861,116 @@ static char *cmd_dsp(__maybe_unused PGconn *conn, __maybe_unused char *cmd, __maybe_unused K_TREE *trf_root, __maybe_unused bool reload_data) { - __maybe_unused K_ITEM *i_file; - __maybe_unused char reply[1024] = ""; - __maybe_unused size_t siz = sizeof(reply); - LOGDEBUG("%s(): cmd '%s'", __func__, cmd); - // WARNING: This is a gaping security hole - only use in development +#if 1 LOGDEBUG("%s.disabled.dsp", id); return strdup("disabled.dsp"); -/* +#else + // WARNING: This is a gaping security hole - only use in development + K_ITEM *i_file, *i_name, *i_type; + char reply[1024] = "", *fil, *name, *typ; + size_t siz = sizeof(reply); + K_STORE *store = NULL; + K_TREE *tree = NULL; + bool unknown_typ = true, unknown_name = true, msg = false; + i_file = require_name(trf_root, "file", 1, NULL, reply, siz); if (!i_file) return strdup(reply); - dsp_ktree(blocks_free, blocks_root, transfer_data(i_file), NULL); + i_name = require_name(trf_root, "name", 1, NULL, reply, siz); + if (!i_name) + return strdup(reply); + + i_type = optional_name(trf_root, "type", 1, NULL, reply, siz); + if (*reply) + return strdup(reply); + + fil = transfer_data(i_file); + name = transfer_data(i_name); + if (i_type) + typ = transfer_data(i_type); + else + typ = "tree"; + + if (strcasecmp(typ, "tree") == 0) { + unknown_typ = false; - dsp_ktree(transfer_free, trf_root, transfer_data(i_file), NULL); + if (strcasecmp(name, "blocks") == 0) + tree = blocks_root; - dsp_ktree(paymentaddresses_free, paymentaddresses_root, - transfer_data(i_file), NULL); + if (strcasecmp(name, "transfer") == 0) + tree = trf_root; - dsp_ktree(paymentaddresses_create_free, paymentaddresses_root, - transfer_data(i_file), NULL); + if (strcasecmp(name, "paymentaddresses") == 0) + tree = paymentaddresses_root; + + if (strcasecmp(name, "paymentaddresses_create") == 0) + tree = paymentaddresses_create_root; + + if (strcasecmp(name, "sharesummary") == 0) + tree = sharesummary_root; + + if (strcasecmp(name, "userstats") == 0) + tree = userstats_root; + + if (strcasecmp(name, "markersummary") == 0) + tree = markersummary_root; + + if (strcasecmp(name, "workmarkers") == 0) + tree = workmarkers_root; + + if (strcasecmp(name, "idcontrol") == 0) + tree = idcontrol_root; + + if (tree) { + unknown_name = false; + if (tree->master->dsp_func) + dsp_ktree(tree, fil, NULL); + else { + snprintf(reply, siz, + "%s %s has no dsp_func", + typ, name); + msg = true; + } + } + } else if (strcasecmp(typ, "store") == 0) { + unknown_typ = false; - dsp_ktree(sharesummary_free, sharesummary_root, - transfer_data(i_file), NULL); + if (strcasecmp(name, "blocks") == 0) + store = blocks_store; - dsp_ktree(userstats_free, userstats_root, - transfer_data(i_file), NULL); + if (strcasecmp(name, "markersummary") == 0) + store = markersummary_store; - dsp_ktree(markersummary_free, markersummary_root, - transfer_data(i_file), NULL); + if (strcasecmp(name, "msgline") == 0) + store = msgline_store; - dsp_ktree(workmarkers_free, workmarkers_root, - transfer_data(i_file), NULL); + if (store) { + unknown_name = false; + if (store->master->dsp_func) + dsp_kstore(store, fil, NULL); + else { + snprintf(reply, siz, + "%s %s has no dsp_func", + typ, name); + msg = true; + } + } + } - LOGDEBUG("%s.ok.dsp.file='%s'", id, transfer_data(i_file)); - return strdup("ok.dsp"); -*/ + if (unknown_typ) { + snprintf(reply, siz, "unknown typ '%s'", typ); + } else if (unknown_name) { + snprintf(reply, siz, "unknown name '%s' for '%s'", name, typ); + } else { + if (!msg) + snprintf(reply, siz, "ok.dsp.file='%s'", fil); + } + LOGDEBUG("%s.%s'", id, reply); + return strdup(reply); +#endif } static char *cmd_stats(__maybe_unused PGconn *conn, char *cmd, char *id, @@ -5954,7 +5983,7 @@ static char *cmd_stats(__maybe_unused PGconn *conn, char *cmd, char *id, char tmp[1024], *buf; const char *name; size_t len, off; - uint64_t ram, ram2, tot = 0; + int64_t ram, ram2, tot = 0; K_LIST *klist; K_LISTS *klists; int rows = 0; @@ -5999,8 +6028,8 @@ static char *cmd_stats(__maybe_unused PGconn *conn, char *cmd, char *id, snprintf(tmp, sizeof(tmp), "name:%d=%s%s%s%cinitial:%d=%d%callocated:%d=%d%c" - "instore:%d=%d%cram:%d=%"PRIu64"%c" - "ram2:%d=%"PRIu64"%ccull:%d=%d%c", + "instore:%d=%d%cram:%d=%"PRId64"%c" + "ram2:%d=%"PRId64"%ccull:%d=%d%ccull_limit:%d=%d%c", rows, name, istree ? " (tree)" : "", klist->is_lock_only ? " (lock)" : "", FLDSEP, rows, klist->allocate, FLDSEP, @@ -6008,7 +6037,8 @@ static char *cmd_stats(__maybe_unused PGconn *conn, char *cmd, char *id, rows, klist->total - klist->count, FLDSEP, rows, ram, FLDSEP, rows, ram2, FLDSEP, - rows, klist->cull_count, FLDSEP); + rows, klist->cull_count, FLDSEP, + rows, klist->cull_limit, FLDSEP); APPEND_REALLOC(buf, off, len, tmp); tot += ram + ram2; @@ -6018,13 +6048,13 @@ static char *cmd_stats(__maybe_unused PGconn *conn, char *cmd, char *id, } ck_wunlock(&lock_check_lock); - snprintf(tmp, sizeof(tmp), "totalram=%"PRIu64"%c", tot, FLDSEP); + snprintf(tmp, sizeof(tmp), "totalram=%"PRId64"%c", tot, FLDSEP); APPEND_REALLOC(buf, off, len, tmp); snprintf(tmp, sizeof(tmp), "rows=%d%cflds=%s%c", rows, FLDSEP, - "name,initial,allocated,instore,ram,cull", FLDSEP); + "name,initial,allocated,instore,ram,cull,cull_limit", FLDSEP); APPEND_REALLOC(buf, off, len, tmp); snprintf(tmp, sizeof(tmp), "arn=%s%carp=%s", "Stats", FLDSEP, ""); @@ -7030,7 +7060,7 @@ static char *cmd_query(__maybe_unused PGconn *conn, char *cmd, char *id, 1, (char *)intpatt, reply, siz); if (!i_height) - return strdup(reply); + goto badreply; TXT_TO_INT("height", transfer_data(i_height), height); i_expired = optional_name(trf_root, "expired", @@ -7116,7 +7146,7 @@ static char *cmd_query(__maybe_unused PGconn *conn, char *cmd, char *id, 1, (char *)intpatt, reply, siz); if (!i_wid) - return strdup(reply); + goto badreply; TXT_TO_BIGINT("wid", transfer_data(i_wid), wid); i_expired = optional_name(trf_root, "expired", @@ -7252,7 +7282,7 @@ static char *cmd_query(__maybe_unused PGconn *conn, char *cmd, char *id, 1, (char *)intpatt, reply, siz); if (!i_height) - return strdup(reply); + goto badreply; TXT_TO_INT("height", transfer_data(i_height), height); int_to_buf(height, reply, sizeof(reply)); @@ -7319,7 +7349,7 @@ static char *cmd_query(__maybe_unused PGconn *conn, char *cmd, char *id, 1, (char *)intpatt, reply, siz); if (!i_height) - return strdup(reply); + goto badreply; TXT_TO_INT("height", transfer_data(i_height), height); int_to_buf(height, reply, sizeof(reply)); @@ -7450,7 +7480,7 @@ static char *cmd_query(__maybe_unused PGconn *conn, char *cmd, char *id, 1, (char *)intpatt, reply, siz); if (!i_height) - return strdup(reply); + goto badreply; TXT_TO_INT("height", transfer_data(i_height), height); int_to_buf(height, reply, sizeof(reply)); @@ -7571,7 +7601,7 @@ static char *cmd_query(__maybe_unused PGconn *conn, char *cmd, char *id, 1, (char *)intpatt, reply, siz); if (!i_wid) - return strdup(reply); + goto badreply; TXT_TO_BIGINT("wid", transfer_data(i_wid), selwid); INIT_SHARES(&s_look); @@ -7703,21 +7733,117 @@ static char *cmd_query(__maybe_unused PGconn *conn, char *cmd, char *id, APPEND_REALLOC(buf, off, len, tmp); ok = true; + } else if (strcasecmp(request, "pg") == 0) { + K_RLOCK(pgdb_free); + snprintf(tmp, sizeof(tmp), "connections=%d%c", + pgdb_count, FLDSEP); + APPEND_REALLOC(buf, off, len, tmp); + K_RUNLOCK(pgdb_free); + rows++; + + ok = true; +#if 0 + } else if (strcasecmp(request, "transfer") == 0) { + /* Code for debugging the transfer stores + * limit is set to avoid a very large reply, + * since transfer can be millions of items during a reload */ + TRANSFER *trf = NULL; + K_STORE *trf_store; + K_ITEM *trf_item, *i_limit; + int stores = 0, limit = 20, tot_stores = 0; + bool exceeded = false; + + i_limit = optional_name(trf_root, "limit", + 1, (char *)intpatt, + reply, siz); + if (*reply) { + LOGERR("%s() %s.%s", __func__, id, reply); + goto badreply; + } + if (i_limit) + limit = atoi(transfer_data(i_limit)); + + snprintf(tmp, sizeof(tmp), "limit=%d%c", limit, FLDSEP); + APPEND_REALLOC(buf, off, len, tmp); + + K_RLOCK(transfer_free); + trf_store = transfer_free->next_store; + while (!exceeded && trf_store) { + trf_item = trf_store->head; + while (trf_item) { + if (rows >= limit) { + exceeded = true; + break; + } + DATA_TRANSFER(trf, trf_item); + snprintf(tmp, sizeof(tmp), "store:%d=%d%c", + rows, stores, FLDSEP); + APPEND_REALLOC(buf, off, len, tmp); + snprintf(tmp, sizeof(tmp), "storename:%d=%s%c", + rows, trf_store->name, FLDSEP); + APPEND_REALLOC(buf, off, len, tmp); + snprintf(tmp, sizeof(tmp), "name:%d=%s%c", + rows, trf->name, FLDSEP); + APPEND_REALLOC(buf, off, len, tmp); + snprintf(tmp, sizeof(tmp), "mvalue:%d=%s%c", + rows, trf->mvalue, FLDSEP); + APPEND_REALLOC(buf, off, len, tmp); + snprintf(tmp, sizeof(tmp), + "malloc:%d=%"PRIu64"%c", + rows, trf->msiz, FLDSEP); + APPEND_REALLOC(buf, off, len, tmp); + snprintf(tmp, sizeof(tmp), "intrans:%d=%c%c", + rows, trf->intransient ? 'Y' : 'N', + FLDSEP); + APPEND_REALLOC(buf, off, len, tmp); + + trf_item = trf_item->next; + rows++; + } + trf_store = trf_store->next_store; + stores++; + } + tot_stores = stores; + if (exceeded) { + while (trf_store) { + trf_store = trf_store->next_store; + tot_stores++; + } + } + K_RUNLOCK(transfer_free); + + snprintf(tmp, sizeof(tmp), "rowstores=%d%c", + stores, FLDSEP); + APPEND_REALLOC(buf, off, len, tmp); + snprintf(tmp, sizeof(tmp), "totstores=%d%c", + tot_stores, FLDSEP); + APPEND_REALLOC(buf, off, len, tmp); + snprintf(tmp, sizeof(tmp), "limitexceeded=%c%c", + exceeded ? 'Y' : 'N', FLDSEP); + APPEND_REALLOC(buf, off, len, tmp); + + snprintf(tmp, sizeof(tmp), "flds=%s%c", + "store,storename,name,mvalue,malloc,intrans", FLDSEP); + APPEND_REALLOC(buf, off, len, tmp); + snprintf(tmp, sizeof(tmp), "arn=%s%carp=%s%c", + transfer_free->name, FLDSEP, "", FLDSEP); + APPEND_REALLOC(buf, off, len, tmp); + + ok = true; +#endif } else { - free(buf); snprintf(reply, siz, "unknown request '%s'", request); LOGERR("%s() %s.%s", __func__, id, reply); - return strdup(reply); + goto badreply; } if (!ok) { - free(buf); snprintf(reply, siz, "failed.%s%s%s", request, msg[0] ? " " : "", msg[0] ? msg : ""); LOGERR("%s() %s.%s", __func__, id, reply); - return strdup(reply); + goto badreply; } snprintf(tmp, sizeof(tmp), "rows=%d", rows); @@ -7726,6 +7852,10 @@ static char *cmd_query(__maybe_unused PGconn *conn, char *cmd, char *id, msg[0] ? " " : "", msg[0] ? msg : ""); return buf; + +badreply: + free(buf); + return strdup(reply); } // Query and disable internal lock detection code @@ -8353,10 +8483,8 @@ static char *cmd_high(PGconn *conn, char *cmd, char *id, if (strcasecmp(action, "store") == 0) { /* Store the shares_hi_root list in the db now, * rather than wait for a shift process to do it */ - if (!conn) { - conn = dbconnect(); + if (CKPQConn(&conn)) conned = true; - } count = 0; do { did = false; @@ -8371,8 +8499,7 @@ static char *cmd_high(PGconn *conn, char *cmd, char *id, count++; } } while (did); - if (conned) - PQfinish(conn); + CKPQDisco(&conn, conned); if (count) { LOGWARNING("%s() Stored: %d high shares", __func__, count); @@ -8463,6 +8590,22 @@ static char *cmd_threads(__maybe_unused PGconn *conn, char *cmd, char *id, K_WUNLOCK(breakqueue_free); snprintf(reply, siz, "ok.delta %d request sent", delta_value); return strdup(reply); + } else if (strcasecmp(name, "cl") == 0 || + strcasecmp(name, "cmd_listener") == 0) { + K_WLOCK(workqueue_free); + // Just overwrite whatever's there + cmd_listener_threads_delta = delta_value; + K_WUNLOCK(workqueue_free); + snprintf(reply, siz, "ok.delta %d request sent", delta_value); + return strdup(reply); + } else if (strcasecmp(name, "bl") == 0 || + strcasecmp(name, "btc_listener") == 0) { + K_WLOCK(workqueue_free); + // Just overwrite whatever's there + btc_listener_threads_delta = delta_value; + K_WUNLOCK(workqueue_free); + snprintf(reply, siz, "ok.delta %d request sent", delta_value); + return strdup(reply); } else { snprintf(reply, siz, "unknown name '%s'", name); LOGERR("%s() %s.%s", __func__, id, reply); @@ -8472,6 +8615,71 @@ static char *cmd_threads(__maybe_unused PGconn *conn, char *cmd, char *id, return buf; } +static char *cmd_pause(__maybe_unused PGconn *conn, char *cmd, char *id, + __maybe_unused tv_t *now, __maybe_unused char *by, + __maybe_unused char *code, __maybe_unused char *inet, + __maybe_unused tv_t *cd, K_TREE *trf_root, + __maybe_unused bool reload_data) +{ + K_ITEM *i_name; + char reply[1024] = ""; + size_t siz = sizeof(reply); + char *name; + + LOGDEBUG("%s(): cmd '%s'", __func__, cmd); + + i_name = require_name(trf_root, "name", 1, NULL, reply, siz); + if (!i_name) + return strdup(reply); + name = transfer_data(i_name); + + /* Pause the breaker threads to help culling to take place for some + * tables that can be culled but 'never' empty due to threads always + * creating new data before the old data has finished being processed + * N.B. this should only be needed on a sizeable pool, once after + * the reload completes ... and even 499ms would be a long time to + * pause in the case of a sizeable pool ... DANGER, WILL ROBINSON! */ + if (strcasecmp(name, "breaker") == 0) { + K_ITEM *i_ms; + int ms = 100; + + i_ms = optional_name(trf_root, "ms", 1, NULL, reply, siz); + if (*reply) + return strdup(reply); + if (i_ms) { + ms = atoi(transfer_data(i_ms)); + // 4999 is too long, don't do it! + if (ms < 10 || ms > 4999) { + snprintf(reply, siz, + "%s ms %d outside range 10-4999", + name, ms); + goto out; + } + } + + if (!reload_queue_complete && !key_update) { + snprintf(reply, siz, + "no point pausing %s before reload completes", + name); + goto out; + } + + /* Use an absolute start time to try to get all threads asleep + * at the same time */ + K_WLOCK(breakqueue_free); + cksleep_prepare_r(&breaker_sleep_stt); + breaker_sleep_ms = ms; + K_WUNLOCK(breakqueue_free); + snprintf(reply, siz, "ok.%s %s%dms pause sent", name, + ms > 499 ? "ALERT!!! " : EMPTY, ms); + } else + snprintf(reply, siz, "unknown name '%s'", name); + +out: + LOGWARNING("%s() %s.%s", __func__, id, reply); + return strdup(reply); +} + /* The socket command format is as follows: * Basic structure: * cmd.ID.fld1=value1 FLDSEP fld2=value2 FLDSEP fld3=... @@ -8585,5 +8793,6 @@ struct CMDS ckdb_cmds[] = { { CMD_EVENTS, "events", false, false, cmd_events, SEQ_NONE, ACCESS_SYSTEM | ACCESS_WEB }, { CMD_HIGH, "high", false, false, cmd_high, SEQ_NONE, ACCESS_SYSTEM }, { CMD_THREADS, "threads", false, false, cmd_threads, SEQ_NONE, ACCESS_SYSTEM }, + { CMD_PAUSE, "pause", false, false, cmd_pause, SEQ_NONE, ACCESS_SYSTEM }, { CMD_END, NULL, false, false, NULL, SEQ_NONE, 0 } }; diff --git a/src/ckdb_data.c b/src/ckdb_data.c index d4bc4461..d7eba801 100644 --- a/src/ckdb_data.c +++ b/src/ckdb_data.c @@ -18,7 +18,7 @@ void free_transfer_data(TRANSFER *transfer) FREENULL(transfer->mvalue); } -void free_msgline_data(K_ITEM *item, bool t_lock, bool t_cull) +void free_msgline_data(K_ITEM *item, bool t_lock) { K_ITEM *t_item = NULL; TRANSFER *transfer; @@ -40,11 +40,6 @@ void free_msgline_data(K_ITEM *item, bool t_lock, bool t_cull) K_WLOCK(transfer_free); transfer_free->ram -= ram2; k_list_transfer_to_head(msgline->trf_store, transfer_free); - if (t_cull) { - if (transfer_free->count == transfer_free->total && - transfer_free->total >= ALLOC_TRANSFER * CULL_TRANSFER) - k_cull_list(transfer_free); - } if (t_lock) K_WUNLOCK(transfer_free); msgline->trf_store = k_free_store(msgline->trf_store); @@ -510,9 +505,27 @@ void _txt_to_double(char *nam, char *fld, double *data, size_t siz, WHERE_FFL_AR char *_data_to_buf(enum data_type typ, void *data, char *buf, size_t siz, WHERE_FFL_ARGS) { + static bool had_null = false; struct tm tm; double d; + // Return an empty string but only log a console message the first time + if (!data) { + // locking doesn't matter - if we get extra messages + if (!had_null) { + had_null = true; + LOGEMERG("%s() BUG - called with null data - check" + " log file" WHERE_FFL, + __func__, WHERE_FFL_PASS); + } + LOGNOTICE("%s() BUG - called with null data typ=%d" WHERE_FFL, + __func__, (int)typ, WHERE_FFL_PASS); + if (!buf) + buf = malloc(1); + *buf = '\0'; + return buf; + } + if (!buf) { switch (typ) { case TYPE_STR: @@ -816,6 +829,37 @@ char *_intransient_str(char *fldnam, char *value, WHERE_FFL_ARGS) return in->str; } +void dsp_msgline(K_ITEM *item, FILE *stream) +{ + K_ITEM *t_item; + MSGLINE *m; + int c; + + if (!item) + fprintf(stream, "%s() called with (null) item\n", __func__); + else { + DATA_MSGLINE(m, item); + if (m->trf_store) + c = m->trf_store->count; + else + c = 0; + + fprintf(stream, " which=%d id='%s' cmd='%s' msg='%.42s' " + "trf_store=%c count=%d\n", + m->which_cmds, m->id, m->cmd, m->msg, + m->trf_store ? 'Y' : 'N', c); + + if (m->trf_store) { + t_item = m->trf_store->head; + while (t_item) { + fputc(' ', stream); + dsp_transfer(t_item, stream); + t_item = t_item->next; + } + } + } +} + // For mutiple variable function calls that need the data char *_transfer_data(K_ITEM *item, WHERE_FFL_ARGS) { @@ -856,8 +900,10 @@ void dsp_transfer(K_ITEM *item, FILE *stream) fprintf(stream, "%s() called with (null) item\n", __func__); else { DATA_TRANSFER(t, item); - fprintf(stream, " name='%s' mvalue='%s' malloc=%"PRIu64"\n", - t->name, t->mvalue, t->msiz); + fprintf(stream, " name='%s' mvalue='%s' malloc=%"PRIu64 + " intransient=%c\n", + t->name, t->mvalue, t->msiz, + t->intransient ? 'Y' : 'N'); } } @@ -2141,6 +2187,52 @@ K_ITEM *find_accountbalance(int64_t userid) return item; } +void dsp_idcontrol(K_ITEM *item, FILE *stream) +{ + char createdate_buf[DATE_BUFSIZ], modifydate_buf[DATE_BUFSIZ]; + IDCONTROL *i; + + if (!item) + fprintf(stream, "%s() called with (null) item\n", __func__); + else { + DATA_IDCONTROL(i, item); + tv_to_buf(&(i->createdate), createdate_buf, sizeof(createdate_buf)); + tv_to_buf(&(i->modifydate), modifydate_buf, sizeof(modifydate_buf)); + fprintf(stream, " idname='%s' lastid=%"PRId64" cdate='%s'" + " cby='%s' ccode='%s' cinet='%s' mdate='%s'" + " mby='%s' mcode='%s' minet='%s'\n", + i->idname, i->lastid, createdate_buf, + i->in_createby, i->in_createcode, + i->in_createinet, modifydate_buf, + i->in_modifyby, i->in_modifycode, + i->in_modifyinet); + } +} + +// order by idname asc +cmp_t cmp_idcontrol(K_ITEM *a, K_ITEM *b) +{ + IDCONTROL *ida, *idb; + DATA_IDCONTROL(ida, a); + DATA_IDCONTROL(idb, b); + return CMP_STR(ida->idname, idb->idname); +} + +// idcontrol must be R or W locked +K_ITEM *find_idcontrol(char *idname) +{ + IDCONTROL idcontrol; + K_TREE_CTX ctx[1]; + K_ITEM look, *item; + + STRNCPY(idcontrol.idname, idname); + + INIT_IDCONTROL(&look); + look.data = (void *)(&idcontrol); + item = find_in_ktree(idcontrol_root, &look, ctx); + return item; +} + // order by optionname asc,activationdate asc,activationheight asc,expirydate desc cmp_t cmp_optioncontrol(K_ITEM *a, K_ITEM *b) { @@ -4881,7 +4973,8 @@ bool process_pplns(int32_t height, char *blockhash, tv_t *addr_cd) FLDSEP, cd_buf); DUP_POINTER(payouts_free, payouts->stats, &buf[0]); - conned = CKPQConn(&conn); + if (CKPQConn(&conn)) + conned = true; begun = CKPQBegin(conn); if (!begun) goto shazbot; @@ -4984,6 +5077,9 @@ bool process_pplns(int32_t height, char *blockhash, tv_t *addr_cd) (double)(pa->payratio) / (double)paytotal; used += d64; + payments->in_originaltxn = + payments->in_committxn = + payments->in_commitblockhash = EMPTY; k_add_tail_nolock(pay_store, pay_item); ok = payments_add(conn, true, pay_item, &(payments->old_item), @@ -5013,6 +5109,9 @@ bool process_pplns(int32_t height, char *blockhash, tv_t *addr_cd) payments->amount = amount; payments->diffacc = miningpayouts->diffacc; used = amount; + payments->in_originaltxn = + payments->in_committxn = + payments->in_commitblockhash = EMPTY; k_add_tail_nolock(pay_store, pay_item); ok = payments_add(conn, true, pay_item, &(payments->old_item), @@ -6345,7 +6444,7 @@ K_ITEM *_find_markersummary(int64_t markerid, int64_t workinfoid, bool make_markersummaries(bool msg, char *by, char *code, char *inet, tv_t *cd, K_TREE *trf_root) { - PGconn *conn; + PGconn *conn = NULL; K_TREE_CTX ctx[1]; WORKMARKERS *workmarkers; K_ITEM *wm_item, *wm_last = NULL, *s_item = NULL; @@ -6375,7 +6474,7 @@ bool make_markersummaries(bool msg, char *by, char *code, char *inet, return false; } - conn = dbconnect(); + CKPQConn(&conn); /* Store all shares in the DB before processing the workmarker * This way we know that the high shares in the DB will match the start @@ -6434,7 +6533,7 @@ bool make_markersummaries(bool msg, char *by, char *code, char *inet, tvdiff(&proc_lock_fin, &proc_lock_got)); flailed: - PQfinish(conn); + CKPQDisco(&conn, true); if (count > 0) { LOGWARNING("%s() Stored: %d high shares %.3fs", diff --git a/src/ckdb_dbio.c b/src/ckdb_dbio.c index e422f628..8726892b 100644 --- a/src/ckdb_dbio.c +++ b/src/ckdb_dbio.c @@ -9,10 +9,60 @@ #include "ckdb.h" +void _pause_read_lock(WHERE_FFL_ARGS) +{ + if (pgdb_pause_disabled) + return; + + if (pause_read_count > 0) { + LOGEMERG("%s() ERR lock >0 (%d) (%s/%s/%d/%c) pause disabled" + WHERE_FFL, + __func__, pause_read_count, pause_read_file, + pause_read_func, pause_read_line, + pause_read_unlock ? 'U' : 'L', WHERE_FFL_PASS); + pgdb_pause_disabled = true; + return; + } + + ck_rlock(&pgdb_pause_lock); + pause_read_count++; + pause_read_file = (char *)file; + pause_read_func = (char *)func; + pause_read_line = line; + pause_read_unlock = false; +} + +void _pause_read_unlock(WHERE_FFL_ARGS) +{ + if (pgdb_pause_disabled) + return; + + if (pause_read_count != 1) { + LOGEMERG("%s() ERR lock !=1 (%d) (%s/%s/%d/%c) pause disabled" + WHERE_FFL, + __func__, pause_read_count, pause_read_file, + pause_read_func, pause_read_line, + pause_read_unlock ? 'U' : 'L', WHERE_FFL_PASS); + pgdb_pause_disabled = true; + return; + } + + ck_runlock(&pgdb_pause_lock); + pause_read_count--; + pause_read_file = (char *)file; + pause_read_func = (char *)func; + pause_read_line = line; + pause_read_unlock = true; +} + char *pqerrmsg(PGconn *conn) { - char *ptr, *buf = strdup(PQerrorMessage(conn)); + char *ptr, *buf; + if (pgdb_paused) + return strdup("pgdb_paused"); + + buf = strdup(PQerrorMessage(conn)); if (!buf) quithere(1, "malloc OOM"); ptr = buf + strlen(buf) - 1; @@ -25,40 +75,78 @@ char *pqerrmsg(PGconn *conn) return buf; } -#define PQ_GET_FLD(__res, __row, __name, __fld, __ok) do { \ - int __col = PQfnumber(__res, __name); \ - if (__col == -1) { \ - LOGERR("%s(): Unknown field '%s' row %d", __func__, __name, __row); \ - __ok = false; \ - } else { \ - __fld = PQgetvalue(__res, __row, __col); \ - if (__fld == NULL) { \ - LOGERR("%s(): Invalid field '%s' or row %d", __func__, __name, __row); \ +/* *** WARNING: each field used in PQ_VAL_FLD() must (of course) have __num + * defined, but must also (re)initialise it to -1 with each call to PQexec + * N.B. it may? not be necessary to reinitialise them with each subsequent + * fetch, however I can't find that clearly documented anywhere, it may change + * in the future, and once per fetch reduces the number of times by a rather + * large factor of 9999 so it doesn't really matter repeating it each fetch */ +#define CKPQFETCHSIZ 9999 +#define CKPQFETCHSTR STRINT(CKPQFETCHSIZ) +#define FETCHTICK 100000 +#define CKPQFUNDEF -1 +#define CKPQ_VAL_FLD(__res, __row, __num, __name, __fld, __ok) do { \ + if (pgdb_paused) \ + break; \ + if (__num == CKPQFUNDEF) { \ + __num = PQfnumber(__res, __name); \ + if (__num == CKPQFUNDEF) { \ + LOGERR("%s():%d: Unknown field '%s' row %d", \ + __func__, __LINE__, __name, __row); \ __ok = false; \ - }\ + } \ } \ + __fld = PQgetvalue(__res, __row, __num); \ + if (__fld == NULL) { \ + LOGERR("%s():%d Invalid field '%s' or row %d", \ + __func__, __LINE__, __name, __row); \ + __ok = false; \ + }\ } while (0) +// Allow params to be macros + +#define CKPQ_VAL_FLD_tail2(__res, __row, __name, __fld, __ok) \ + CKPQ_VAL_FLD(__res, __row, __name ## _num__, #__name, __fld, __ok) +#define CKPQ_VAL_FLD_tail(__res, __row, __name, __fld, __ok) \ + CKPQ_VAL_FLD_tail2(__res, __row, __name, __fld, __ok) +#define CKPQ_VAL_FLD_num2(__res, __row, __name, __fld, __ok) \ + CKPQ_VAL_FLD(__res, __row, __name ## _num, #__name, __fld, __ok) +#define CKPQ_VAL_FLD_num(__res, __row, __name, __fld, __ok) \ + CKPQ_VAL_FLD_num2(__res, __row, __name, __fld, __ok) +#define CKPQADDNUM2(__name) __name ## _num__ +#define CKPQADDNUM(__name) CKPQADDNUM2(__name) + // HISTORY FIELDS +#define HISTORYDATE_num \ + int CKPQADDNUM(_CDDB), CKPQADDNUM(_BYDB), CKPQADDNUM(_CODEDB), \ + CKPQADDNUM(_INETDB), CKPQADDNUM(_EDDB) + +#define HISTORYDATE_init \ + CKPQADDNUM(_CDDB) = CKPQADDNUM(_BYDB) = CKPQADDNUM(_CODEDB) = \ + CKPQADDNUM(_INETDB) = CKPQADDNUM(_EDDB) = CKPQFUNDEF + #define HISTORYDATEFLDS(_res, _row, _data, _ok) do { \ char *_fld; \ - PQ_GET_FLD(_res, _row, CDDB, _fld, _ok); \ + if (pgdb_paused) \ + break; \ + CKPQ_VAL_FLD_tail(_res, _row, _CDDB, _fld, _ok); \ if (!_ok) \ break; \ TXT_TO_TVDB(CDDB, _fld, (_data)->createdate); \ - PQ_GET_FLD(_res, _row, BYDB, _fld, _ok); \ + CKPQ_VAL_FLD_tail(_res, _row, _BYDB, _fld, _ok); \ if (!_ok) \ break; \ TXT_TO_STR(BYDB, _fld, (_data)->createby); \ - PQ_GET_FLD(_res, _row, CODEDB, _fld, _ok); \ + CKPQ_VAL_FLD_tail(_res, _row, _CODEDB, _fld, _ok); \ if (!_ok) \ break; \ TXT_TO_STR(CODEDB, _fld, (_data)->createcode); \ - PQ_GET_FLD(_res, _row, INETDB, _fld, _ok); \ + CKPQ_VAL_FLD_tail(_res, _row, _INETDB, _fld, _ok); \ if (!_ok) \ break; \ TXT_TO_STR(INETDB, _fld, (_data)->createinet); \ - PQ_GET_FLD(_res, _row, EDDB, _fld, _ok); \ + CKPQ_VAL_FLD_tail(_res, _row, _EDDB, _fld, _ok); \ if (!_ok) \ break; \ TXT_TO_TVDB(EDDB, _fld, (_data)->expirydate); \ @@ -67,23 +155,25 @@ char *pqerrmsg(PGconn *conn) #define HISTORYDATEIN(_res, _row, _data, _ok) do { \ char *_fld; \ - PQ_GET_FLD(_res, _row, CDDB, _fld, _ok); \ + if (pgdb_paused) \ + break; \ + CKPQ_VAL_FLD_tail(_res, _row, _CDDB, _fld, _ok); \ if (!_ok) \ break; \ TXT_TO_TVDB(CDDB, _fld, (_data)->createdate); \ - PQ_GET_FLD(_res, _row, BYDB, _fld, _ok); \ + CKPQ_VAL_FLD_tail(_res, _row, _BYDB, _fld, _ok); \ if (!_ok) \ break; \ (_data)->in_createby = intransient_str(BYDB, _fld); \ - PQ_GET_FLD(_res, _row, CODEDB, _fld, _ok); \ + CKPQ_VAL_FLD_tail(_res, _row, _CODEDB, _fld, _ok); \ if (!_ok) \ break; \ (_data)->in_createcode = intransient_str(CODEDB, _fld); \ - PQ_GET_FLD(_res, _row, INETDB, _fld, _ok); \ + CKPQ_VAL_FLD_tail(_res, _row, _INETDB, _fld, _ok); \ if (!_ok) \ break; \ (_data)->in_createinet = intransient_str(INETDB, _fld); \ - PQ_GET_FLD(_res, _row, EDDB, _fld, _ok); \ + CKPQ_VAL_FLD_tail(_res, _row, _EDDB, _fld, _ok); \ if (!_ok) \ break; \ TXT_TO_TVDB(EDDB, _fld, (_data)->expirydate); \ @@ -107,74 +197,49 @@ char *pqerrmsg(PGconn *conn) } while (0) // MODIFY FIELDS -#define MODIFYDATEFLDPOINTERS(_list, _res, _row, _data, _ok) do { \ - char *_fld; \ - PQ_GET_FLD(_res, _row, CDDB, _fld, _ok); \ - if (!_ok) \ - break; \ - TXT_TO_TVDB(CDDB, _fld, (_data)->createdate); \ - PQ_GET_FLD(_res, _row, BYDB, _fld, _ok); \ - if (!_ok) \ - break; \ - SET_CREATEBY(_list, (_data)->createby, _fld); \ - PQ_GET_FLD(_res, _row, CODEDB, _fld, _ok); \ - if (!_ok) \ - break; \ - SET_CREATECODE(_list, (_data)->createcode, _fld); \ - PQ_GET_FLD(_res, _row, INETDB, _fld, _ok); \ - if (!_ok) \ - break; \ - SET_CREATEINET(_list, (_data)->createinet, _fld); \ - PQ_GET_FLD(_res, _row, MDDB, _fld, _ok); \ - if (!_ok) \ - break; \ - TXT_TO_TVDB(MDDB, _fld, (_data)->modifydate); \ - PQ_GET_FLD(_res, _row, MBYDB, _fld, _ok); \ - if (!_ok) \ - break; \ - SET_MODIFYBY(_list, (_data)->modifyby, _fld); \ - PQ_GET_FLD(_res, _row, MCODEDB, _fld, _ok); \ - if (!_ok) \ - break; \ - SET_MODIFYCODE(_list, (_data)->modifycode, _fld); \ - PQ_GET_FLD(_res, _row, MINETDB, _fld, _ok); \ - if (!_ok) \ - break; \ - SET_MODIFYINET(_list, (_data)->modifyinet, _fld); \ - (_data)->pointers = (_data)->pointers; \ - } while (0) +#define MODIFYDATE_num \ + int CKPQADDNUM(_CDDB), CKPQADDNUM(_BYDB), CKPQADDNUM(_CODEDB), \ + CKPQADDNUM(_INETDB), CKPQADDNUM(_MDDB), CKPQADDNUM(_MBYDB), \ + CKPQADDNUM(_MCODEDB), CKPQADDNUM(_MINETDB) + +#define MODIFYDATE_init \ + CKPQADDNUM(_CDDB) = CKPQADDNUM(_BYDB) = CKPQADDNUM(_CODEDB) = \ + CKPQADDNUM(_INETDB) = CKPQADDNUM(_MDDB) = CKPQADDNUM(_MBYDB) = \ + CKPQADDNUM(_MCODEDB) = CKPQADDNUM(_MINETDB) = CKPQFUNDEF #define MODIFYDATEIN(_res, _row, _data, _ok) do { \ char *_fld; \ - PQ_GET_FLD(_res, _row, CDDB, _fld, _ok); \ + if (pgdb_paused) \ + break; \ + CKPQ_VAL_FLD_tail(_res, _row, _CDDB, _fld, _ok); \ if (!_ok) \ break; \ TXT_TO_TVDB(CDDB, _fld, (_data)->createdate); \ - PQ_GET_FLD(_res, _row, BYDB, _fld, _ok); \ + CKPQ_VAL_FLD_tail(_res, _row, _BYDB, _fld, _ok); \ if (!_ok) \ break; \ (_data)->in_createby = intransient_str(BYDB, _fld); \ - PQ_GET_FLD(_res, _row, CODEDB, _fld, _ok); \ + CKPQ_VAL_FLD_tail(_res, _row, _CODEDB, _fld, _ok); \ if (!_ok) \ break; \ (_data)->in_createcode = intransient_str(CODEDB, _fld); \ - PQ_GET_FLD(_res, _row, INETDB, _fld, _ok); \ + CKPQ_VAL_FLD_tail(_res, _row, _INETDB, _fld, _ok); \ if (!_ok) \ break; \ (_data)->in_createinet = intransient_str(INETDB, _fld); \ - PQ_GET_FLD(_res, _row, MDDB, _fld, _ok); \ + CKPQ_VAL_FLD_tail(_res, _row, _MDDB, _fld, _ok); \ if (!_ok) \ break; \ TXT_TO_TVDB(MDDB, _fld, (_data)->modifydate); \ - PQ_GET_FLD(_res, _row, MBYDB, _fld, _ok); \ + CKPQ_VAL_FLD_tail(_res, _row, _MBYDB, _fld, _ok); \ if (!_ok) \ break; \ (_data)->in_modifyby = intransient_str(MBYDB, _fld); \ - PQ_GET_FLD(_res, _row, MCODEDB, _fld, _ok); \ + CKPQ_VAL_FLD_tail(_res, _row, _MCODEDB, _fld, _ok); \ if (!_ok) \ break; \ (_data)->in_modifycode = intransient_str(MCODEDB, _fld); \ - PQ_GET_FLD(_res, _row, MINETDB, _fld, _ok); \ + CKPQ_VAL_FLD_tail(_res, _row, _MINETDB, _fld, _ok); \ if (!_ok) \ break; \ (_data)->in_modifyinet = intransient_str(MINETDB, _fld); \ @@ -211,21 +276,31 @@ char *pqerrmsg(PGconn *conn) } while (0) // SIMPLE FIELDS +#define SIMPLEDATE_num \ + int CKPQADDNUM(_CDDB), CKPQADDNUM(_BYDB), CKPQADDNUM(_CODEDB), \ + CKPQADDNUM(_INETDB) + +#define SIMPLEDATE_init \ + CKPQADDNUM(_CDDB) = CKPQADDNUM(_BYDB) = CKPQADDNUM(_CODEDB) = \ + CKPQADDNUM(_INETDB) = CKPQFUNDEF + #define SIMPLEDATEFLDS(_res, _row, _data, _ok) do { \ char *_fld; \ - PQ_GET_FLD(_res, _row, CDDB, _fld, _ok); \ + if (pgdb_paused) \ + break; \ + CKPQ_VAL_FLD_tail(_res, _row, _CDDB, _fld, _ok); \ if (!_ok) \ break; \ TXT_TO_TVDB(CDDB, _fld, (_data)->createdate); \ - PQ_GET_FLD(_res, _row, BYDB, _fld, _ok); \ + CKPQ_VAL_FLD_tail(_res, _row, _BYDB, _fld, _ok); \ if (!_ok) \ break; \ TXT_TO_STR(BYDB, _fld, (_data)->createby); \ - PQ_GET_FLD(_res, _row, CODEDB, _fld, _ok); \ + CKPQ_VAL_FLD_tail(_res, _row, _CODEDB, _fld, _ok); \ if (!_ok) \ break; \ TXT_TO_STR(CODEDB, _fld, (_data)->createcode); \ - PQ_GET_FLD(_res, _row, INETDB, _fld, _ok); \ + CKPQ_VAL_FLD_tail(_res, _row, _INETDB, _fld, _ok); \ if (!_ok) \ break; \ TXT_TO_STR(INETDB, _fld, (_data)->createinet); \ @@ -291,9 +366,6 @@ char *pqerrmsg(PGconn *conn) } \ } while (0) -#undef PQexec -#undef PQexecParams - /* Debug level to display write transactions - 0 removes the code * Also enables checking the isread flag */ #define CKPQ_SHOW_WRITE 0 @@ -306,8 +378,11 @@ char *pqerrmsg(PGconn *conn) #define CKPQ_ISREAD3LEN (sizeof(CKPQ_ISREAD3)-1) // Bug check to ensure no unexpected write txns occur -PGresult *_CKPQexec(PGconn *conn, const char *qry, bool isread, WHERE_FFL_ARGS) +PGresult *_CKPQExec(PGconn *conn, const char *qry, bool isread, WHERE_FFL_ARGS) { + if (pgdb_paused) + return NULL; + // It would slow it down, but could check qry for insert/update/... if (!isread && confirm_sharesummary) quitfrom(1, file, func, line, "BUG: write txn during confirm"); @@ -349,7 +424,7 @@ PGresult *_CKPQexec(PGconn *conn, const char *qry, bool isread, WHERE_FFL_ARGS) return PQexec(conn, qry); } -PGresult *_CKPQexecParams(PGconn *conn, const char *qry, +PGresult *_CKPQExecParams(PGconn *conn, const char *qry, int nParams, const Oid *paramTypes, const char *const * paramValues, @@ -358,6 +433,9 @@ PGresult *_CKPQexecParams(PGconn *conn, const char *qry, int resultFormat, bool isread, WHERE_FFL_ARGS) { + if (pgdb_paused) + return NULL; + // It would slow it down, but could check qry for insert/update/... if (!isread && confirm_sharesummary) quitfrom(1, file, func, line, "BUG: write txn during confirm"); @@ -407,38 +485,79 @@ PGresult *_CKPQexecParams(PGconn *conn, const char *qry, paramFormats, resultFormat); } -#define PQexec CKPQexec -#define PQexecParams CKPQexecParams +ExecStatusType _CKPQResultStatus(PGresult *res, WHERE_FFL_ARGS) +{ + if (pgdb_paused) + return PGRES_COMMAND_OK; -// TODO: switch all to use this -bool CKPQConn(PGconn **conn) + return PQresultStatus(res); +} + +void _CKPQClear(PGresult *res, WHERE_FFL_ARGS) +{ + if (!pgdb_paused) + PQclear(res); +} + +bool _CKPQConn(PGconn **conn, WHERE_FFL_ARGS) { if (*conn == NULL) { - LOGDEBUG("%s(): connecting", __func__); - *conn = dbconnect(); + if (connect_dis == false) { + LOGEMERG("%s() ERR already (%s/%s/%d)" WHERE_FFL +#if LOCK_CHECK + " @%s" +#endif + , __func__, connect_file, connect_func, + connect_line, WHERE_FFL_PASS +#if LOCK_CHECK + , my_thread_name +#endif + ); + } + if (!pgdb_paused) { + LOGDEBUG("%s(): connecting", __func__); + *conn = dbconnect(); + K_WLOCK(pgdb_free); + pgdb_count++; + K_WUNLOCK(pgdb_free); + connect_file = (char *)file; + connect_func = (char *)func; + connect_line = line; + connect_dis = false; + } return true; } return false; } -// TODO: switch all to use this -void CKPQDisco(PGconn **conn, bool conned) +bool _CKPQDisco(PGconn **conn, bool conned, WHERE_FFL_ARGS) { - if (conned) { + if (conned && *conn) { LOGDEBUG("%s(): disco", __func__); PQfinish(*conn); + *conn = NULL; + K_WLOCK(pgdb_free); + pgdb_count--; + K_WUNLOCK(pgdb_free); + connect_file = (char *)file; + connect_func = (char *)func; + connect_line = line; + connect_dis = true; } + return false; } -// TODO: switch all to use this bool _CKPQBegin(PGconn *conn, WHERE_FFL_ARGS) { ExecStatusType rescode; PGresult *res; - res = PQexec(conn, "Begin", CKPQ_WRITE); - rescode = PQresultStatus(res); - PQclear(res); + if (pgdb_paused) + return true; + + res = _CKPQExec(conn, "Begin", CKPQ_WRITE, WHERE_FFL_PASS); + rescode = _CKPQResultStatus(res, WHERE_FFL_PASS); + _CKPQClear(res, WHERE_FFL_PASS); if (!PGOK(rescode)) { char *buf = pqerrmsg(conn); LOGEMERG("%s(): Begin failed (%d) '%s'" WHERE_FFL, @@ -450,21 +569,23 @@ bool _CKPQBegin(PGconn *conn, WHERE_FFL_ARGS) return true; } -// TODO: switch all to use this void _CKPQEnd(PGconn *conn, bool commit, WHERE_FFL_ARGS) { ExecStatusType rescode; PGresult *res; + if (pgdb_paused) + return; + if (commit) { LOGDEBUG("%s(): commit", __func__); - res = PQexec(conn, "Commit", CKPQ_WRITE); + res = _CKPQExec(conn, "Commit", CKPQ_WRITE, WHERE_FFL_PASS); } else { LOGDEBUG("%s(): rollback", __func__); - res = PQexec(conn, "Rollback", CKPQ_WRITE); + res = _CKPQExec(conn, "Rollback", CKPQ_WRITE, WHERE_FFL_PASS); } - rescode = PQresultStatus(res); - PQclear(res); + rescode = _CKPQResultStatus(res, WHERE_FFL_PASS); + _CKPQClear(res, WHERE_FFL_PASS); if (!PGOK(rescode)) { char *buf = pqerrmsg(conn); LOGEMERG("%s(): %s failed (%d) '%s'" WHERE_FFL, @@ -479,27 +600,34 @@ int64_t nextid(PGconn *conn, char *idname, int64_t increment, { ExecStatusType rescode; bool conned = false; + IDCONTROL *idcontrol; + K_ITEM *item; PGresult *res; char qry[1024]; char *params[5]; - int n, par = 0; + int n, f, par = 0; int64_t lastid; char *field; bool ok; lastid = 0; + K_WLOCK(idcontrol_free); + item = find_idcontrol(idname); + if (!item) + { + LOGERR("%s(): No matching idname='%s' in tree", __func__, idname); + goto cleanup; + } + snprintf(qry, sizeof(qry), "select lastid from idcontrol " "where idname='%s' for update", idname); - if (conn == NULL) { - conn = dbconnect(); + if (CKPQConn(&conn)) conned = true; - } - - res = PQexec(conn, qry, CKPQ_WRITE); - rescode = PQresultStatus(res); + res = CKPQExec(conn, qry, CKPQ_WRITE); + rescode = CKPQResultStatus(res); if (!PGOK(rescode)) { PGLOGERR("Select", rescode, conn); goto cleanup; @@ -519,12 +647,13 @@ int64_t nextid(PGconn *conn, char *idname, int64_t increment, } ok = true; - PQ_GET_FLD(res, 0, "lastid", field, ok); + f = CKPQFUNDEF; + CKPQ_VAL_FLD(res, 0, f, "lastid", field, ok); if (!ok) goto cleanup; TXT_TO_BIGINT("lastid", field, lastid); - PQclear(res); + CKPQClear(res); lastid += increment; snprintf(qry, sizeof(qry), "update idcontrol set " @@ -541,19 +670,26 @@ int64_t nextid(PGconn *conn, char *idname, int64_t increment, params[par++] = str_to_buf(inet, NULL, 0); PARCHK(par, params); - res = PQexecParams(conn, qry, par, NULL, (const char **)params, NULL, NULL, 0, CKPQ_WRITE); - rescode = PQresultStatus(res); + res = CKPQExecParams(conn, qry, par, NULL, (const char **)params, NULL, NULL, 0, CKPQ_WRITE); + rescode = CKPQResultStatus(res); + CKPQClear(res); if (!PGOK(rescode)) { PGLOGERR("Update", rescode, conn); lastid = 0; + } else { + DATA_IDCONTROL(idcontrol, item); + idcontrol->lastid = lastid; + copy_tv(&(idcontrol->modifydate), cd); + idcontrol->in_modifyby = intransient_str(MBYDB, by); + idcontrol->in_modifycode = intransient_str(MCODEDB, code); + idcontrol->in_modifyinet = intransient_str(MINETDB, inet); } for (n = 0; n < par; n++) free(params[n]); cleanup: - PQclear(res); - if (conned) - PQfinish(conn); + K_WUNLOCK(idcontrol_free); + CKPQDisco(&conn, conned); return lastid; } @@ -575,6 +711,9 @@ bool users_update(PGconn *conn, K_ITEM *u_item, char *oldhash, LOGDEBUG("%s(): change", __func__); + if (pgdb_paused) + LOGEMERG("ERR: %s() called when paused - data lost", __func__); + if (oldhash != NULL) hash = true; else @@ -617,23 +756,14 @@ bool users_update(PGconn *conn, K_ITEM *u_item, char *oldhash, params[par++] = tv_to_buf((tv_t *)&default_expiry, NULL, 0); PARCHKVAL(par, 3, params); - if (conn == NULL) { - conn = dbconnect(); + if (CKPQConn(&conn)) conned = true; - } - - // Beginning of a write txn - res = PQexec(conn, "Begin", CKPQ_WRITE); - rescode = PQresultStatus(res); - PQclear(res); - if (!PGOK(rescode)) { - PGLOGERR("Begin", rescode, conn); + if (!CKPQBegin(conn)) goto unparam; - } - res = PQexecParams(conn, upd, par, NULL, (const char **)params, NULL, NULL, 0, CKPQ_WRITE); - rescode = PQresultStatus(res); - PQclear(res); + res = CKPQExecParams(conn, upd, par, NULL, (const char **)params, NULL, NULL, 0, CKPQ_WRITE); + rescode = CKPQResultStatus(res); + CKPQClear(res); if (!PGOK(rescode)) { PGLOGERR("Update", rescode, conn); goto rollback; @@ -663,9 +793,9 @@ bool users_update(PGconn *conn, K_ITEM *u_item, char *oldhash, "$7,$8,$9,$10,$11 from users where " "userid=$1 and "EDDB"=$2"; - res = PQexecParams(conn, ins, par, NULL, (const char **)params, NULL, NULL, 0, CKPQ_WRITE); - rescode = PQresultStatus(res); - PQclear(res); + res = CKPQExecParams(conn, ins, par, NULL, (const char **)params, NULL, NULL, 0, CKPQ_WRITE); + rescode = CKPQResultStatus(res); + CKPQClear(res); if (!PGOK(rescode)) { PGLOGERR("Insert", rescode, conn); goto rollback; @@ -673,15 +803,11 @@ bool users_update(PGconn *conn, K_ITEM *u_item, char *oldhash, ok = true; rollback: - if (ok) - res = PQexec(conn, "Commit", CKPQ_WRITE); - else - res = PQexec(conn, "Rollback", CKPQ_WRITE); - PQclear(res); + CKPQEnd(conn, ok); + unparam: - if (conned) - PQfinish(conn); + CKPQDisco(&conn, conned); for (n = 0; n < par; n++) free(params[n]); @@ -706,8 +832,9 @@ unparam: } K_ITEM *users_add(PGconn *conn, INTRANSIENT *in_username, char *emailaddress, - char *passwordhash, int64_t userbits, char *by, - char *code, char *inet, tv_t *cd, K_TREE *trf_root) + char *passwordhash, char *secondaryuserid, + int64_t userbits, char *by, char *code, char *inet, + tv_t *cd, K_TREE *trf_root) { ExecStatusType rescode; bool conned = false; @@ -724,6 +851,9 @@ K_ITEM *users_add(PGconn *conn, INTRANSIENT *in_username, char *emailaddress, LOGDEBUG("%s(): add", __func__); + if (pgdb_paused && userbits != USER_MISSING) + LOGEMERG("ERR: %s() called when paused - data lost", __func__); + /* 2 attempts to add the same user at the same time will only do it once * The 2nd attempt will get back the data provided by the 1st * and thus throw away any differences in the 2nd */ @@ -771,9 +901,14 @@ K_ITEM *users_add(PGconn *conn, INTRANSIENT *in_username, char *emailaddress, row->status[0] = '\0'; STRNCPY(row->emailaddress, emailaddress); - snprintf(tohash, sizeof(tohash), "%s&#%s", in_username->str, emailaddress); - HASH_BER(tohash, strlen(tohash), 1, hash, tmp); - __bin2hex(row->secondaryuserid, (void *)(&hash), sizeof(hash)); + if (secondaryuserid == NULL) { + snprintf(tohash, sizeof(tohash), "%s&#%s", + in_username->str, + emailaddress); + HASH_BER(tohash, strlen(tohash), 1, hash, tmp); + __bin2hex(row->secondaryuserid, (void *)(&hash), sizeof(hash)); + } else + STRNCPY(row->secondaryuserid, secondaryuserid); make_salt(row); if (passwordhash == EMPTY) { @@ -813,13 +948,11 @@ K_ITEM *users_add(PGconn *conn, INTRANSIENT *in_username, char *emailaddress, "secondaryuserid,salt,userdata,userbits" HISTORYDATECONTROL ") values (" PQPARAM15 ")"; - if (!conn) { - conn = dbconnect(); + if (CKPQConn(&conn)) conned = true; - } - - res = PQexecParams(conn, ins, par, NULL, (const char **)params, NULL, NULL, 0, CKPQ_WRITE); - rescode = PQresultStatus(res); + res = CKPQExecParams(conn, ins, par, NULL, (const char **)params, NULL, NULL, 0, CKPQ_WRITE); + rescode = CKPQResultStatus(res); + CKPQClear(res); if (!PGOK(rescode)) { PGLOGERR("Insert", rescode, conn); goto unparam; @@ -827,9 +960,7 @@ K_ITEM *users_add(PGconn *conn, INTRANSIENT *in_username, char *emailaddress, ok = true; unparam: - PQclear(res); - if (conned) - PQfinish(conn); + CKPQDisco(&conn, conned); for (n = 0; n < par; n++) free(params[n]); unitem: @@ -869,6 +1000,9 @@ bool users_replace(PGconn *conn, K_ITEM *u_item, K_ITEM *old_u_item, char *by, LOGDEBUG("%s(): replace", __func__); + if (pgdb_paused) + LOGEMERG("ERR: %s() called when paused - data lost", __func__); + DATA_USERS(users, u_item); DATA_USERS(old_users, old_u_item); @@ -882,23 +1016,14 @@ bool users_replace(PGconn *conn, K_ITEM *u_item, K_ITEM *old_u_item, char *by, params[par++] = tv_to_buf((tv_t *)&default_expiry, NULL, 0); PARCHKVAL(par, 3, params); - if (conn == NULL) { - conn = dbconnect(); + if (CKPQConn(&conn)) conned = true; - } - - // Beginning of a write txn - res = PQexec(conn, "Begin", CKPQ_WRITE); - rescode = PQresultStatus(res); - PQclear(res); - if (!PGOK(rescode)) { - PGLOGERR("Begin", rescode, conn); + if (!CKPQBegin(conn)) goto unparam; - } - res = PQexecParams(conn, upd, par, NULL, (const char **)params, NULL, NULL, 0, CKPQ_WRITE); - rescode = PQresultStatus(res); - PQclear(res); + res = CKPQExecParams(conn, upd, par, NULL, (const char **)params, NULL, NULL, 0, CKPQ_WRITE); + rescode = CKPQResultStatus(res); + CKPQClear(res); if (!PGOK(rescode)) { PGLOGERR("Update", rescode, conn); goto rollback; @@ -926,9 +1051,9 @@ bool users_replace(PGconn *conn, K_ITEM *u_item, K_ITEM *old_u_item, char *by, "passwordhash,secondaryuserid,salt,userdata,userbits" HISTORYDATECONTROL ") values (" PQPARAM15 ")"; - res = PQexecParams(conn, ins, par, NULL, (const char **)params, NULL, NULL, 0, CKPQ_WRITE); - rescode = PQresultStatus(res); - PQclear(res); + res = CKPQExecParams(conn, ins, par, NULL, (const char **)params, NULL, NULL, 0, CKPQ_WRITE); + rescode = CKPQResultStatus(res); + CKPQClear(res); if (!PGOK(rescode)) { PGLOGERR("Insert", rescode, conn); goto rollback; @@ -936,15 +1061,11 @@ bool users_replace(PGconn *conn, K_ITEM *u_item, K_ITEM *old_u_item, char *by, ok = true; rollback: - if (ok) - res = PQexec(conn, "Commit", CKPQ_WRITE); - else - res = PQexec(conn, "Rollback", CKPQ_WRITE); - PQclear(res); + CKPQEnd(conn, ok); + unparam: - if (conned) - PQfinish(conn); + CKPQDisco(&conn, conned); for (n = 0; n < par; n++) free(params[n]); @@ -969,8 +1090,30 @@ unparam: return ok; } +/* If a share contains an unknown username then it will most likely be from + * reloading a data file when the users table is incomplete or corrupt + * Since the share is valid, it should be counted against the pool stats and + * creating a user also means the payout rewards are calculated correctly + * This also gives 2 options to resolve it later: + * 1) Correct the database/redistribute the rewards to the correct user + * 2) Rollback the database and reload it with a corrected users table + * Option 1) gives a simpler solution vs option 2) if 2) is too far in + * the past to easily do a reload */ +K_ITEM *create_missing_user(PGconn *conn, char *username, char *secondaryuserid, + char *by, char *code, char *inet, tv_t *cd, + K_TREE *trf_root) +{ + INTRANSIENT *in_username; + + in_username = get_intransient("username", username); + + return users_add(conn, in_username, EMPTY, EMPTY, secondaryuserid, + USER_MISSING, by, code, inet, cd, trf_root); +} + bool users_fill(PGconn *conn) { + char pcombuf[64]; ExecStatusType rescode; PGresult *res; K_ITEM *item; @@ -983,16 +1126,21 @@ bool users_fill(PGconn *conn) LOGDEBUG("%s(): select", __func__); + int userid_num, username_num, status_num, emailaddress_num; + int joineddate_num, passwordhash_num, secondaryuserid_num, salt_num; + int userdata_num, userbits_num; + HISTORYDATE_num; + sel = "select " "userid,username,status,emailaddress,joineddate," "passwordhash,secondaryuserid,salt,userdata,userbits" HISTORYDATECONTROL " from users"; - res = PQexec(conn, sel, CKPQ_READ); - rescode = PQresultStatus(res); + res = CKPQExec(conn, sel, CKPQ_READ); + rescode = CKPQResultStatus(res); if (!PGOK(rescode)) { PGLOGERR("Select", rescode, conn); - PQclear(res); + CKPQClear(res); return false; } @@ -1000,13 +1148,17 @@ bool users_fill(PGconn *conn) if (n != (fields + HISTORYDATECOUNT)) { LOGERR("%s(): Invalid field count - should be %d, but is %d", __func__, fields + HISTORYDATECOUNT, n); - PQclear(res); + CKPQClear(res); return false; } n = PQntuples(res); LOGDEBUG("%s(): tree build count %d", __func__, n); ok = true; + userid_num = username_num = status_num = emailaddress_num = + joineddate_num = passwordhash_num = secondaryuserid_num = salt_num = + userdata_num = userbits_num = CKPQFUNDEF; + HISTORYDATE_init; K_WLOCK(users_free); for (i = 0; i < n; i++) { item = k_unlink_head(users_free); @@ -1018,55 +1170,55 @@ bool users_fill(PGconn *conn) break; } - PQ_GET_FLD(res, i, "userid", field, ok); + CKPQ_VAL_FLD_num(res, i, userid, field, ok); if (!ok) break; TXT_TO_BIGINT("userid", field, row->userid); - PQ_GET_FLD(res, i, "username", field, ok); + CKPQ_VAL_FLD_num(res, i, username, field, ok); if (!ok) break; row->in_username = intransient_str("username", field); - PQ_GET_FLD(res, i, "status", field, ok); + CKPQ_VAL_FLD_num(res, i, status, field, ok); if (!ok) break; TXT_TO_STR("status", field, row->status); - PQ_GET_FLD(res, i, "emailaddress", field, ok); + CKPQ_VAL_FLD_num(res, i, emailaddress, field, ok); if (!ok) break; TXT_TO_STR("emailaddress", field, row->emailaddress); - PQ_GET_FLD(res, i, "joineddate", field, ok); + CKPQ_VAL_FLD_num(res, i, joineddate, field, ok); if (!ok) break; TXT_TO_TVDB("joineddate", field, row->joineddate); - PQ_GET_FLD(res, i, "passwordhash", field, ok); + CKPQ_VAL_FLD_num(res, i, passwordhash, field, ok); if (!ok) break; TXT_TO_STR("passwordhash", field, row->passwordhash); - PQ_GET_FLD(res, i, "secondaryuserid", field, ok); + CKPQ_VAL_FLD_num(res, i, secondaryuserid, field, ok); if (!ok) break; TXT_TO_STR("secondaryuserid", field, row->secondaryuserid); - PQ_GET_FLD(res, i, "salt", field, ok); + CKPQ_VAL_FLD_num(res, i, salt, field, ok); if (!ok) break; TXT_TO_STR("salt", field, row->salt); // TODO: good case for invariant - PQ_GET_FLD(res, i, "userdata", field, ok); + CKPQ_VAL_FLD_num(res, i, userdata, field, ok); if (!ok) break; TXT_TO_PTR("userdata", field, row->userdata); LIST_MEM_ADD(users_free, row->userdata); users_databits(row); - PQ_GET_FLD(res, i, "userbits", field, ok); + CKPQ_VAL_FLD_num(res, i, userbits, field, ok); if (!ok) break; TXT_TO_BIGINT("userbits", field, row->userbits); @@ -1087,11 +1239,13 @@ bool users_fill(PGconn *conn) } K_WUNLOCK(users_free); - PQclear(res); + CKPQClear(res); if (ok) { LOGDEBUG("%s(): built", __func__); - LOGWARNING("%s(): loaded %d users records", __func__, n); + pcom(n, pcombuf, sizeof(pcombuf)); + LOGWARNING("%s(): loaded %s users records", + __func__, pcombuf); } return ok; @@ -1111,6 +1265,9 @@ bool useratts_item_add(PGconn *conn, K_ITEM *ua_item, tv_t *cd, bool begun) LOGDEBUG("%s(): add", __func__); + if (pgdb_paused) + LOGEMERG("ERR: %s() called when paused - data lost", __func__); + DATA_USERATTS(useratts, ua_item); K_RLOCK(useratts_free); @@ -1121,20 +1278,11 @@ bool useratts_item_add(PGconn *conn, K_ITEM *ua_item, tv_t *cd, bool begun) /* N.B. the values of the old ua_item record, if it exists, * are completely ignored i.e. you must provide all values required */ - if (!conn) { - conn = dbconnect(); + if (CKPQConn(&conn)) conned = true; - } - if (!begun) { - // Beginning of a write txn - res = PQexec(conn, "Begin", CKPQ_WRITE); - rescode = PQresultStatus(res); - PQclear(res); - if (!PGOK(rescode)) { - PGLOGERR("Begin", rescode, conn); + if (!CKPQBegin(conn)) goto unparam; - } } if (old_item) { @@ -1147,9 +1295,9 @@ bool useratts_item_add(PGconn *conn, K_ITEM *ua_item, tv_t *cd, bool begun) params[par++] = tv_to_buf((tv_t *)&default_expiry, NULL, 0); PARCHKVAL(par, 4, params); - res = PQexecParams(conn, upd, par, NULL, (const char **)params, NULL, NULL, 0, CKPQ_WRITE); - rescode = PQresultStatus(res); - PQclear(res); + res = CKPQExecParams(conn, upd, par, NULL, (const char **)params, NULL, NULL, 0, CKPQ_WRITE); + rescode = CKPQResultStatus(res); + CKPQClear(res); if (!PGOK(rescode)) { PGLOGERR("Update", rescode, conn); goto unparam; @@ -1177,9 +1325,9 @@ bool useratts_item_add(PGconn *conn, K_ITEM *ua_item, tv_t *cd, bool begun) "attdate,attdate2" HISTORYDATECONTROL ") values (" PQPARAM14 ")"; - res = PQexecParams(conn, ins, par, NULL, (const char **)params, NULL, NULL, 0, CKPQ_WRITE); - rescode = PQresultStatus(res); - PQclear(res); + res = CKPQExecParams(conn, ins, par, NULL, (const char **)params, NULL, NULL, 0, CKPQ_WRITE); + rescode = CKPQResultStatus(res); + CKPQClear(res); if (!PGOK(rescode)) { PGLOGERR("Insert", rescode, conn); goto rollback; @@ -1187,17 +1335,11 @@ bool useratts_item_add(PGconn *conn, K_ITEM *ua_item, tv_t *cd, bool begun) ok = true; rollback: - if (!begun) { - if (ok) - res = PQexec(conn, "Commit", CKPQ_WRITE); - else - res = PQexec(conn, "Rollback", CKPQ_WRITE); + if (!begun) + CKPQEnd(conn, ok); - PQclear(res); - } unparam: - if (conned) - PQfinish(conn); + CKPQDisco(&conn, conned); for (n = 0; n < par; n++) free(params[n]); @@ -1219,7 +1361,7 @@ unparam: K_ITEM *useratts_add(PGconn *conn, char *username, char *attname, char *status, char *attstr, char *attstr2, - char *attnum, char *attnum2, char *attdate, + char *attnum, char *attnum2, char *attdate, char *attdate2, char *by, char *code, char *inet, tv_t *cd, K_TREE *trf_root, bool begun) @@ -1232,6 +1374,9 @@ K_ITEM *useratts_add(PGconn *conn, char *username, char *attname, LOGDEBUG("%s(): add", __func__); + if (pgdb_paused) + LOGEMERG("ERR: %s() called when paused - data lost", __func__); + K_WLOCK(useratts_free); item = k_unlink_head(useratts_free); K_WUNLOCK(useratts_free); @@ -1312,6 +1457,9 @@ bool useratts_item_expire(PGconn *conn, K_ITEM *ua_item, tv_t *cd) LOGDEBUG("%s(): add", __func__); + if (pgdb_paused) + LOGEMERG("ERR: %s() called when paused - data lost", __func__); + DATA_USERATTS(useratts, ua_item); /* This is pointless if ua_item is part of the tree, however, @@ -1323,11 +1471,8 @@ bool useratts_item_expire(PGconn *conn, K_ITEM *ua_item, tv_t *cd) if (item) { DATA_USERATTS(useratts, item); - if (!conn) { - conn = dbconnect(); + if (CKPQConn(&conn)) conned = true; - } - upd = "update useratts set "EDDB"=$1 where userid=$2 and " "attname=$3 and "EDDB"=$4"; par = 0; @@ -1337,8 +1482,9 @@ bool useratts_item_expire(PGconn *conn, K_ITEM *ua_item, tv_t *cd) params[par++] = tv_to_buf((tv_t *)&default_expiry, NULL, 0); PARCHKVAL(par, 4, params); - res = PQexecParams(conn, upd, par, NULL, (const char **)params, NULL, NULL, 0, CKPQ_WRITE); - rescode = PQresultStatus(res); + res = CKPQExecParams(conn, upd, par, NULL, (const char **)params, NULL, NULL, 0, CKPQ_WRITE); + rescode = CKPQResultStatus(res); + CKPQClear(res); if (!PGOK(rescode)) { PGLOGERR("Update", rescode, conn); goto unparam; @@ -1347,9 +1493,7 @@ bool useratts_item_expire(PGconn *conn, K_ITEM *ua_item, tv_t *cd) ok = true; unparam: if (par) { - PQclear(res); - if (conned) - PQfinish(conn); + CKPQDisco(&conn, conned); for (n = 0; n < par; n++) free(params[n]); } @@ -1367,6 +1511,7 @@ unparam: bool useratts_fill(PGconn *conn) { + char pcombuf[64]; ExecStatusType rescode; PGresult *res; K_ITEM *item; @@ -1379,16 +1524,20 @@ bool useratts_fill(PGconn *conn) LOGDEBUG("%s(): select", __func__); + int userid_num, attname_num, status_num, attstr_num, attstr2_num; + int attnum_num, attnum2_num, attdate_num, attdate2_num; + HISTORYDATE_num; + sel = "select " "userid,attname,status,attstr,attstr2,attnum,attnum2" ",attdate,attdate2" HISTORYDATECONTROL " from useratts"; - res = PQexec(conn, sel, CKPQ_READ); - rescode = PQresultStatus(res); + res = CKPQExec(conn, sel, CKPQ_READ); + rescode = CKPQResultStatus(res); if (!PGOK(rescode)) { PGLOGERR("Select", rescode, conn); - PQclear(res); + CKPQClear(res); return false; } @@ -1396,13 +1545,16 @@ bool useratts_fill(PGconn *conn) if (n != (fields + HISTORYDATECOUNT)) { LOGERR("%s(): Invalid field count - should be %d, but is %d", __func__, fields + HISTORYDATECOUNT, n); - PQclear(res); + CKPQClear(res); return false; } n = PQntuples(res); LOGDEBUG("%s(): tree build count %d", __func__, n); ok = true; + userid_num = attname_num = status_num = attstr_num = attstr2_num = + attnum_num = attnum2_num = attdate_num = attdate2_num = CKPQFUNDEF; + HISTORYDATE_init; K_WLOCK(useratts_free); for (i = 0; i < n; i++) { item = k_unlink_head(useratts_free); @@ -1414,47 +1566,47 @@ bool useratts_fill(PGconn *conn) break; } - PQ_GET_FLD(res, i, "userid", field, ok); + CKPQ_VAL_FLD_num(res, i, userid, field, ok); if (!ok) break; TXT_TO_BIGINT("userid", field, row->userid); - PQ_GET_FLD(res, i, "attname", field, ok); + CKPQ_VAL_FLD_num(res, i, attname, field, ok); if (!ok) break; TXT_TO_STR("attname", field, row->attname); - PQ_GET_FLD(res, i, "status", field, ok); + CKPQ_VAL_FLD_num(res, i, status, field, ok); if (!ok) break; TXT_TO_STR("status", field, row->status); - PQ_GET_FLD(res, i, "attstr", field, ok); + CKPQ_VAL_FLD_num(res, i, attstr, field, ok); if (!ok) break; TXT_TO_STR("attstr", field, row->attstr); - PQ_GET_FLD(res, i, "attstr2", field, ok); + CKPQ_VAL_FLD_num(res, i, attstr2, field, ok); if (!ok) break; TXT_TO_STR("attstr2", field, row->attstr2); - PQ_GET_FLD(res, i, "attnum", field, ok); + CKPQ_VAL_FLD_num(res, i, attnum, field, ok); if (!ok) break; TXT_TO_BIGINT("attnum", field, row->attnum); - PQ_GET_FLD(res, i, "attnum2", field, ok); + CKPQ_VAL_FLD_num(res, i, attnum2, field, ok); if (!ok) break; TXT_TO_BIGINT("attnum2", field, row->attnum2); - PQ_GET_FLD(res, i, "attdate", field, ok); + CKPQ_VAL_FLD_num(res, i, attdate, field, ok); if (!ok) break; TXT_TO_TVDB("attdate", field, row->attdate); - PQ_GET_FLD(res, i, "attdate2", field, ok); + CKPQ_VAL_FLD_num(res, i, attdate2, field, ok); if (!ok) break; TXT_TO_TVDB("attdate2", field, row->attdate2); @@ -1470,11 +1622,13 @@ bool useratts_fill(PGconn *conn) k_add_head(useratts_free, item); K_WUNLOCK(useratts_free); - PQclear(res); + CKPQClear(res); if (ok) { LOGDEBUG("%s(): built", __func__); - LOGWARNING("%s(): loaded %d useratts records", __func__, n); + pcom(n, pcombuf, sizeof(pcombuf)); + LOGWARNING("%s(): loaded %s useratts records", + __func__, pcombuf); } return ok; @@ -1518,11 +1672,8 @@ K_ITEM *workers_add(PGconn *conn, int64_t userid, char *workername, bool add_ws, DATA_WORKERS(row, item); - if (conn == NULL) { - conn = dbconnect(); + if (CKPQConn(&conn)) conned = true; - } - bzero(row, sizeof(*row)); row->workerid = nextid(conn, "workerid", (int64_t)1, cd, by, code, inet); if (row->workerid == 0) @@ -1589,8 +1740,9 @@ K_ITEM *workers_add(PGconn *conn, int64_t userid, char *workername, bool add_ws, "idlenotificationenabled,idlenotificationtime,workerbits" HISTORYDATECONTROL ") values (" PQPARAM12 ")"; - res = PQexecParams(conn, ins, par, NULL, (const char **)params, NULL, NULL, 0, CKPQ_WRITE); - rescode = PQresultStatus(res); + res = CKPQExecParams(conn, ins, par, NULL, (const char **)params, NULL, NULL, 0, CKPQ_WRITE); + rescode = CKPQResultStatus(res); + CKPQClear(res); if (!PGOK(rescode)) { PGLOGERR("Insert", rescode, conn); goto unparam; @@ -1598,12 +1750,10 @@ K_ITEM *workers_add(PGconn *conn, int64_t userid, char *workername, bool add_ws, ret = item; unparam: - PQclear(res); for (n = 0; n < par; n++) free(params[n]); unitem: - if (conned) - PQfinish(conn); + CKPQDisco(&conn, conned); K_WLOCK(workers_free); if (!ret) k_add_head(workers_free, item); @@ -1652,6 +1802,9 @@ bool workers_update(PGconn *conn, K_ITEM *item, char *difficultydefault, LOGDEBUG("%s(): update", __func__); + if (pgdb_paused) + LOGEMERG("ERR: %s() called when paused - data lost", __func__); + /* Two attempts to update the same worker at the same time * will determine the final state based on which gets the lock last, * i.e. randomly, but without overwriting at the same time */ @@ -1709,22 +1862,14 @@ bool workers_update(PGconn *conn, K_ITEM *item, char *difficultydefault, params[par++] = tv_to_buf((tv_t *)&default_expiry, NULL, 0); PARCHKVAL(par, 3, params); - if (conn == NULL) { - conn = dbconnect(); + if (CKPQConn(&conn)) conned = true; - } - - res = PQexec(conn, "Begin", CKPQ_WRITE); - rescode = PQresultStatus(res); - PQclear(res); - if (!PGOK(rescode)) { - PGLOGERR("Begin", rescode, conn); + if (!CKPQBegin(conn)) goto unparam; - } - res = PQexecParams(conn, upd, par, NULL, (const char **)params, NULL, NULL, 0, CKPQ_WRITE); - rescode = PQresultStatus(res); - PQclear(res); + res = CKPQExecParams(conn, upd, par, NULL, (const char **)params, NULL, NULL, 0, CKPQ_WRITE); + rescode = CKPQResultStatus(res); + CKPQClear(res); if (!PGOK(rescode)) { PGLOGERR("Update", rescode, conn); goto rollback; @@ -1749,9 +1894,9 @@ bool workers_update(PGconn *conn, K_ITEM *item, char *difficultydefault, HISTORYDATEPARAMS(params, par, row); PARCHK(par, params); - res = PQexecParams(conn, ins, par, NULL, (const char **)params, NULL, NULL, 0, CKPQ_WRITE); - rescode = PQresultStatus(res); - PQclear(res); + res = CKPQExecParams(conn, ins, par, NULL, (const char **)params, NULL, NULL, 0, CKPQ_WRITE); + rescode = CKPQResultStatus(res); + CKPQClear(res); if (!PGOK(rescode)) { PGLOGERR("Insert", rescode, conn); goto rollback; @@ -1759,15 +1904,11 @@ bool workers_update(PGconn *conn, K_ITEM *item, char *difficultydefault, ok = true; rollback: - if (ok) - res = PQexec(conn, "Commit", CKPQ_WRITE); - else - res = PQexec(conn, "Rollback", CKPQ_WRITE); - PQclear(res); + CKPQEnd(conn, ok); + unparam: - if (conned) - PQfinish(conn); + CKPQDisco(&conn, conned); for (n = 0; n < par; n++) free(params[n]); early: @@ -1779,6 +1920,7 @@ early: bool workers_fill(PGconn *conn) { + char pcombuf[64]; ExecStatusType rescode; PGresult *res; K_ITEM *item = NULL; @@ -1791,25 +1933,25 @@ bool workers_fill(PGconn *conn) LOGDEBUG("%s(): select", __func__); + int userid_num, workername_num, difficultydefault_num; + int idlenotificationenabled_num, idlenotificationtime_num; + int workerbits_num, workerid_num; + HISTORYDATE_num; + sel = "declare wk cursor for select " "userid,workername,difficultydefault," "idlenotificationenabled,idlenotificationtime,workerbits" HISTORYDATECONTROL ",workerid from workers"; - res = PQexec(conn, "Begin", CKPQ_READ); - rescode = PQresultStatus(res); - PQclear(res); - if (!PGOK(rescode)) { - PGLOGERR("Begin", rescode, conn); + if (!CKPQBegin(conn)) return false; - } // See workers_add() about this lock K_WLOCK(workers_db_free); - res = PQexec(conn, sel, CKPQ_READ); - rescode = PQresultStatus(res); - PQclear(res); + res = CKPQExec(conn, sel, CKPQ_READ); + rescode = CKPQResultStatus(res); + CKPQClear(res); if (!PGOK(rescode)) { PGLOGERR("Declare", rescode, conn); goto flail; @@ -1817,11 +1959,11 @@ bool workers_fill(PGconn *conn) LOGDEBUG("%s(): fetching ...", __func__); - res = PQexec(conn, "fetch 1 in wk", CKPQ_READ); - rescode = PQresultStatus(res); + res = CKPQExec(conn, "fetch 1 in wk", CKPQ_READ); + rescode = CKPQResultStatus(res); if (!PGOK(rescode)) { PGLOGERR("Fetch first", rescode, conn); - PQclear(res); + CKPQClear(res); goto flail; } @@ -1829,12 +1971,16 @@ bool workers_fill(PGconn *conn) if (n != (fields + HISTORYDATECOUNT)) { LOGERR("%s(): Invalid field count - should be %d, but is %d", __func__, fields + HISTORYDATECOUNT, n); - PQclear(res); + CKPQClear(res); goto flail; } n = 0; ok = true; + userid_num = workername_num = difficultydefault_num = + idlenotificationenabled_num = idlenotificationtime_num = + workerbits_num = workerid_num = CKPQFUNDEF; + HISTORYDATE_init; while ((t = PQntuples(res)) > 0) { for (i = 0; i < t; i++) { K_WLOCK(workers_free); @@ -1848,32 +1994,32 @@ bool workers_fill(PGconn *conn) break; } - PQ_GET_FLD(res, i, "userid", field, ok); + CKPQ_VAL_FLD_num(res, i, userid, field, ok); if (!ok) break; TXT_TO_BIGINT("userid", field, row->userid); - PQ_GET_FLD(res, i, "workername", field, ok); + CKPQ_VAL_FLD_num(res, i, workername, field, ok); if (!ok) break; row->in_workername = intransient_str("workername", field); - PQ_GET_FLD(res, i, "difficultydefault", field, ok); + CKPQ_VAL_FLD_num(res, i, difficultydefault, field, ok); if (!ok) break; TXT_TO_INT("difficultydefault", field, row->difficultydefault); - PQ_GET_FLD(res, i, "idlenotificationenabled", field, ok); + CKPQ_VAL_FLD_num(res, i, idlenotificationenabled, field, ok); if (!ok) break; TXT_TO_STR("idlenotificationenabled", field, row->idlenotificationenabled); - PQ_GET_FLD(res, i, "idlenotificationtime", field, ok); + CKPQ_VAL_FLD_num(res, i, idlenotificationtime, field, ok); if (!ok) break; TXT_TO_INT("idlenotificationtime", field, row->idlenotificationtime); - PQ_GET_FLD(res, i, "workerbits", field, ok); + CKPQ_VAL_FLD_num(res, i, workerbits, field, ok); if (!ok) break; TXT_TO_BIGINT("workerbits", field, row->workerbits); @@ -1882,7 +2028,7 @@ bool workers_fill(PGconn *conn) if (!ok) break; - PQ_GET_FLD(res, i, "workerid", field, ok); + CKPQ_VAL_FLD_num(res, i, workerid, field, ok); if (!ok) break; TXT_TO_BIGINT("workerid", field, row->workerid); @@ -1899,28 +2045,33 @@ bool workers_fill(PGconn *conn) tick(); n++; } - PQclear(res); - res = PQexec(conn, "fetch 9999 in wk", CKPQ_READ); - rescode = PQresultStatus(res); + CKPQClear(res); + res = CKPQExec(conn, "fetch "CKPQFETCHSTR" in wk", CKPQ_READ); + rescode = CKPQResultStatus(res); if (!PGOK(rescode)) { PGLOGERR("Fetch next", rescode, conn); ok = false; break; } + userid_num = workername_num = difficultydefault_num = + idlenotificationenabled_num = idlenotificationtime_num = + workerbits_num = workerid_num = CKPQFUNDEF; + HISTORYDATE_init; } if (!ok) k_add_head(workers_free, item); - PQclear(res); + CKPQClear(res); flail: - res = PQexec(conn, "Commit", CKPQ_READ); - PQclear(res); + CKPQCommit(conn); K_WUNLOCK(workers_db_free); if (ok) { LOGDEBUG("%s(): built", __func__); - LOGWARNING("%s(): fetched %d workers records", __func__, n); + pcom(n, pcombuf, sizeof(pcombuf)); + LOGWARNING("%s(): fetched %s workers records", + __func__, pcombuf); } return ok; @@ -1951,24 +2102,19 @@ bool paymentaddresses_set(PGconn *conn, int64_t userid, K_STORE *pa_store, LOGDEBUG("%s(): add", __func__); + if (pgdb_paused) + LOGEMERG("ERR: %s() called when paused - data lost", __func__); + // Quick early abort if (pa_store->count > ABS_ADDR_LIMIT) return false; - if (conn == NULL) { - conn = dbconnect(); + if (CKPQConn(&conn)) conned = true; - } - /* This means the nextid updates will rollback on an error, but also * means that it will lock the nextid record for the whole update */ - res = PQexec(conn, "Begin", CKPQ_WRITE); - rescode = PQresultStatus(res); - if (!PGOK(rescode)) { - PGLOGERR("Begin", rescode, conn); + if (!CKPQBegin(conn)) goto unparam; - } - PQclear(res); // First step - DB expire all the old/changed records in RAM LOGDEBUG("%s(): Step 1 userid=%"PRId64, __func__, userid); @@ -2037,9 +2183,9 @@ bool paymentaddresses_set(PGconn *conn, int64_t userid, K_STORE *pa_store, } else { APPEND_REALLOC(upd, off, len, ")"); PARCHKVAL(par, par, params); - res = PQexecParams(conn, upd, par, NULL, (const char **)params, NULL, NULL, 0, CKPQ_WRITE); - rescode = PQresultStatus(res); - PQclear(res); + res = CKPQExecParams(conn, upd, par, NULL, (const char **)params, NULL, NULL, 0, CKPQ_WRITE); + rescode = CKPQResultStatus(res); + CKPQClear(res); if (!PGOK(rescode)) { PGLOGERR("Update", rescode, conn); goto rollback; @@ -2082,10 +2228,10 @@ bool paymentaddresses_set(PGconn *conn, int64_t userid, K_STORE *pa_store, HISTORYDATEPARAMSIN(params, par, row); PARCHKVAL(par, 10, params); // As per PQPARAM10 above - res = PQexecParams(conn, ins, par, NULL, (const char **)params, + res = CKPQExecParams(conn, ins, par, NULL, (const char **)params, NULL, NULL, 0, CKPQ_WRITE); - rescode = PQresultStatus(res); - PQclear(res); + rescode = CKPQResultStatus(res); + CKPQClear(res); if (!PGOK(rescode)) { PGLOGERR("Insert", rescode, conn); goto rollback; @@ -2103,15 +2249,11 @@ bool paymentaddresses_set(PGconn *conn, int64_t userid, K_STORE *pa_store, ok = true; rollback: - if (ok) - res = PQexec(conn, "Commit", CKPQ_WRITE); - else - res = PQexec(conn, "Rollback", CKPQ_WRITE); - PQclear(res); + CKPQEnd(conn, ok); + unparam: - if (conned) - PQfinish(conn); + CKPQDisco(&conn, conned); for (n = 0; n < par; n++) free(params[n]); FREENULL(upd); @@ -2176,6 +2318,7 @@ unparam: bool paymentaddresses_fill(PGconn *conn) { + char pcombuf[64]; ExecStatusType rescode; PGresult *res; K_ITEM *item; @@ -2188,15 +2331,19 @@ bool paymentaddresses_fill(PGconn *conn) LOGDEBUG("%s(): select", __func__); + int paymentaddressid_num, userid_num, payaddress_num, payratio_num; + int payname_num; + HISTORYDATE_num; + sel = "select " "paymentaddressid,userid,payaddress,payratio,payname" HISTORYDATECONTROL " from paymentaddresses"; - res = PQexec(conn, sel, CKPQ_READ); - rescode = PQresultStatus(res); + res = CKPQExec(conn, sel, CKPQ_READ); + rescode = CKPQResultStatus(res); if (!PGOK(rescode)) { PGLOGERR("Select", rescode, conn); - PQclear(res); + CKPQClear(res); return false; } @@ -2204,13 +2351,16 @@ bool paymentaddresses_fill(PGconn *conn) if (n != (fields + HISTORYDATECOUNT)) { LOGERR("%s(): Invalid field count - should be %d, but is %d", __func__, fields + HISTORYDATECOUNT, n); - PQclear(res); + CKPQClear(res); return false; } n = PQntuples(res); LOGDEBUG("%s(): tree build count %d", __func__, n); ok = true; + paymentaddressid_num = userid_num = payaddress_num = payratio_num = + payname_num = CKPQFUNDEF; + HISTORYDATE_init; K_WLOCK(paymentaddresses_free); for (i = 0; i < n; i++) { item = k_unlink_head(paymentaddresses_free); @@ -2222,27 +2372,27 @@ bool paymentaddresses_fill(PGconn *conn) break; } - PQ_GET_FLD(res, i, "paymentaddressid", field, ok); + CKPQ_VAL_FLD_num(res, i, paymentaddressid, field, ok); if (!ok) break; TXT_TO_BIGINT("paymentaddressid", field, row->paymentaddressid); - PQ_GET_FLD(res, i, "userid", field, ok); + CKPQ_VAL_FLD_num(res, i, userid, field, ok); if (!ok) break; TXT_TO_BIGINT("userid", field, row->userid); - PQ_GET_FLD(res, i, "payaddress", field, ok); + CKPQ_VAL_FLD_num(res, i, payaddress, field, ok); if (!ok) break; row->in_payaddress = intransient_str("payaddress", field); - PQ_GET_FLD(res, i, "payratio", field, ok); + CKPQ_VAL_FLD_num(res, i, payratio, field, ok); if (!ok) break; TXT_TO_INT("payratio", field, row->payratio); - PQ_GET_FLD(res, i, "payname", field, ok); + CKPQ_VAL_FLD_num(res, i, payname, field, ok); if (!ok) break; TXT_TO_STR("payname", field, row->payname); @@ -2259,11 +2409,13 @@ bool paymentaddresses_fill(PGconn *conn) k_add_head(paymentaddresses_free, item); K_WUNLOCK(paymentaddresses_free); - PQclear(res); + CKPQClear(res); if (ok) { LOGDEBUG("%s(): built", __func__); - LOGWARNING("%s(): loaded %d paymentaddresses records", __func__, n); + pcom(n, pcombuf, sizeof(pcombuf)); + LOGWARNING("%s(): loaded %s paymentaddresses records", + __func__, pcombuf); } return ok; @@ -2319,7 +2471,8 @@ bool payments_add(PGconn *conn, bool add, K_ITEM *p_item, K_ITEM **old_p_item, *old_p_item = find_payments(row->payoutid, row->userid, row->in_subname); K_RUNLOCK(payments_free); - conned = CKPQConn(&conn); + if (CKPQConn(&conn)) + conned = true; if (!already) { begun = CKPQBegin(conn); if (!begun) @@ -2339,9 +2492,9 @@ bool payments_add(PGconn *conn, bool add, K_ITEM *p_item, K_ITEM **old_p_item, params[par++] = tv_to_buf((tv_t *)&default_expiry, NULL, 0); PARCHKVAL(par, 3, params); - res = PQexecParams(conn, upd, par, NULL, (const char **)params, NULL, NULL, 0, CKPQ_WRITE); - rescode = PQresultStatus(res); - PQclear(res); + res = CKPQExecParams(conn, upd, par, NULL, (const char **)params, NULL, NULL, 0, CKPQ_WRITE); + rescode = CKPQResultStatus(res); + CKPQClear(res); if (!PGOK(rescode)) { PGLOGERR("Update", rescode, conn); goto rollback; @@ -2388,8 +2541,9 @@ bool payments_add(PGconn *conn, bool add, K_ITEM *p_item, K_ITEM **old_p_item, "originaltxn,amount,diffacc,committxn,commitblockhash" HISTORYDATECONTROL ") values (" PQPARAM16 ")"; - res = PQexecParams(conn, ins, par, NULL, (const char **)params, NULL, NULL, 0, CKPQ_WRITE); - rescode = PQresultStatus(res); + res = CKPQExecParams(conn, ins, par, NULL, (const char **)params, NULL, NULL, 0, CKPQ_WRITE); + rescode = CKPQResultStatus(res); + CKPQClear(res); if (!PGOK(rescode)) { PGLOGERR("Insert", rescode, conn); goto unparam; @@ -2430,22 +2584,22 @@ bool payments_fill(PGconn *conn) STRNCPY(tickbuf, TICK_PREFIX"pm 0"); cr_msg(false, tickbuf); + int paymentid_num, payoutid_num, userid_num, subname_num, paydate_num; + int payaddress_num, originaltxn_num, amount_num, diffacc_num; + int committxn_num, commitblockhash_num; + HISTORYDATE_num; + sel = "declare ps cursor for select " "paymentid,payoutid,userid,subname,paydate,payaddress," "originaltxn,amount,diffacc,committxn,commitblockhash" HISTORYDATECONTROL " from payments"; - res = PQexec(conn, "Begin", CKPQ_READ); - rescode = PQresultStatus(res); - PQclear(res); - if (!PGOK(rescode)) { - PGLOGERR("Begin", rescode, conn); + if (!CKPQBegin(conn)) return false; - } - res = PQexec(conn, sel, CKPQ_READ); - rescode = PQresultStatus(res); - PQclear(res); + res = CKPQExec(conn, sel, CKPQ_READ); + rescode = CKPQResultStatus(res); + CKPQClear(res); if (!PGOK(rescode)) { PGLOGERR("Declare", rescode, conn); goto flail; @@ -2453,11 +2607,11 @@ bool payments_fill(PGconn *conn) LOGDEBUG("%s(): fetching ...", __func__); - res = PQexec(conn, "fetch 1 in ps", CKPQ_READ); - rescode = PQresultStatus(res); + res = CKPQExec(conn, "fetch 1 in ps", CKPQ_READ); + rescode = CKPQResultStatus(res); if (!PGOK(rescode)) { PGLOGERR("Fetch first", rescode, conn); - PQclear(res); + CKPQClear(res); goto flail; } @@ -2465,12 +2619,16 @@ bool payments_fill(PGconn *conn) if (n != (fields + HISTORYDATECOUNT)) { LOGERR("%s(): Invalid field count - should be %d, but is %d", __func__, fields + HISTORYDATECOUNT, n); - PQclear(res); + CKPQClear(res); goto flail; } n = 0; ok = true; + paymentid_num = payoutid_num = userid_num = subname_num = paydate_num = + payaddress_num = originaltxn_num = amount_num = diffacc_num = + committxn_num = commitblockhash_num = CKPQFUNDEF; + HISTORYDATE_init; K_WLOCK(payments_free); while ((t = PQntuples(res)) > 0) { for (i = 0; i < t; i++) { @@ -2483,57 +2641,57 @@ bool payments_fill(PGconn *conn) break; } - PQ_GET_FLD(res, i, "paymentid", field, ok); + CKPQ_VAL_FLD_num(res, i, paymentid, field, ok); if (!ok) break; TXT_TO_BIGINT("paymentid", field, row->paymentid); - PQ_GET_FLD(res, i, "payoutid", field, ok); + CKPQ_VAL_FLD_num(res, i, payoutid, field, ok); if (!ok) break; TXT_TO_BIGINT("payoutid", field, row->payoutid); - PQ_GET_FLD(res, i, "userid", field, ok); + CKPQ_VAL_FLD_num(res, i, userid, field, ok); if (!ok) break; TXT_TO_BIGINT("userid", field, row->userid); - PQ_GET_FLD(res, i, "subname", field, ok); + CKPQ_VAL_FLD_num(res, i, subname, field, ok); if (!ok) break; row->in_subname = intransient_str("subname", field); - PQ_GET_FLD(res, i, "paydate", field, ok); + CKPQ_VAL_FLD_num(res, i, paydate, field, ok); if (!ok) break; TXT_TO_TVDB("paydate", field, row->paydate); - PQ_GET_FLD(res, i, "payaddress", field, ok); + CKPQ_VAL_FLD_num(res, i, payaddress, field, ok); if (!ok) break; row->in_payaddress = intransient_str("payaddress", field); - PQ_GET_FLD(res, i, "originaltxn", field, ok); + CKPQ_VAL_FLD_num(res, i, originaltxn, field, ok); if (!ok) break; row->in_originaltxn = intransient_str("originaltxn", field); - PQ_GET_FLD(res, i, "amount", field, ok); + CKPQ_VAL_FLD_num(res, i, amount, field, ok); if (!ok) break; TXT_TO_BIGINT("amount", field, row->amount); - PQ_GET_FLD(res, i, "diffacc", field, ok); + CKPQ_VAL_FLD_num(res, i, diffacc, field, ok); if (!ok) break; TXT_TO_DOUBLE("diffacc", field, row->diffacc); - PQ_GET_FLD(res, i, "committxn", field, ok); + CKPQ_VAL_FLD_num(res, i, committxn, field, ok); if (!ok) break; row->in_committxn = intransient_str("committxn", field); - PQ_GET_FLD(res, i, "commitblockhash", field, ok); + CKPQ_VAL_FLD_num(res, i, commitblockhash, field, ok); if (!ok) break; row->in_commitblockhash = intransient_str("commitblockhash", field); @@ -2545,7 +2703,7 @@ bool payments_fill(PGconn *conn) add_to_ktree(payments_root, item); k_add_head(payments_store, item); - if (n == 0 || ((n+1) % 100000) == 0) { + if (n == 0 || ((n+1) % FETCHTICK) == 0) { pcom(n+1, pcombuf, sizeof(pcombuf)); snprintf(tickbuf, sizeof(tickbuf), TICK_PREFIX"pm %s", pcombuf); @@ -2554,27 +2712,32 @@ bool payments_fill(PGconn *conn) tick(); n++; } - PQclear(res); - res = PQexec(conn, "fetch 9999 in ps", CKPQ_READ); - rescode = PQresultStatus(res); + CKPQClear(res); + res = CKPQExec(conn, "fetch "CKPQFETCHSTR" in ps", CKPQ_READ); + rescode = CKPQResultStatus(res); if (!PGOK(rescode)) { PGLOGERR("Fetch next", rescode, conn); ok = false; break; } + paymentid_num = payoutid_num = userid_num = subname_num = + paydate_num = payaddress_num = originaltxn_num = amount_num = + diffacc_num = committxn_num = commitblockhash_num = CKPQFUNDEF; + HISTORYDATE_init; } if (!ok) k_add_head(payments_free, item); K_WUNLOCK(payments_free); - PQclear(res); + CKPQClear(res); flail: - res = PQexec(conn, "Commit", CKPQ_READ); - PQclear(res); + CKPQCommit(conn); if (ok) { LOGDEBUG("%s(): built", __func__); - LOGWARNING("%s(): fetched %d payments records", __func__, n); + pcom(n, pcombuf, sizeof(pcombuf)); + LOGWARNING("%s(): fetched %s payments records", + __func__, pcombuf); } return ok; @@ -2584,7 +2747,7 @@ bool idcontrol_add(PGconn *conn, char *idname, char *idvalue, char *by, char *code, char *inet, tv_t *cd, __maybe_unused K_TREE *trf_root) { - K_ITEM *look; + K_ITEM *item; IDCONTROL *row; char *params[2 + MODIFYDATECOUNT]; int n, par = 0; @@ -2597,32 +2760,29 @@ bool idcontrol_add(PGconn *conn, char *idname, char *idvalue, char *by, LOGDEBUG("%s(): add", __func__); K_WLOCK(idcontrol_free); - look = k_unlink_head(idcontrol_free); + item = k_unlink_head(idcontrol_free); K_WUNLOCK(idcontrol_free); - DATA_IDCONTROL(row, look); + DATA_IDCONTROL(row, item); STRNCPY(row->idname, idname); TXT_TO_BIGINT("idvalue", idvalue, row->lastid); - MODIFYDATEINIT(row, cd, by, code, inet); + MODIFYDATEINTRANS(row, cd, by, code, inet); par = 0; params[par++] = str_to_buf(row->idname, NULL, 0); params[par++] = bigint_to_buf(row->lastid, NULL, 0); - MODIFYDATEPARAMS(params, par, row); + MODIFYDATEPARAMSIN(params, par, row); PARCHK(par, params); ins = "insert into idcontrol " "(idname,lastid" MODIFYDATECONTROL ") values (" PQPARAM10 ")"; - if (!conn) { - conn = dbconnect(); + if (CKPQConn(&conn)) conned = true; - } - - res = PQexecParams(conn, ins, par, NULL, (const char **)params, NULL, NULL, 0, CKPQ_WRITE); - rescode = PQresultStatus(res); - PQclear(res); + res = CKPQExecParams(conn, ins, par, NULL, (const char **)params, NULL, NULL, 0, CKPQ_WRITE); + rescode = CKPQResultStatus(res); + CKPQClear(res); if (!PGOK(rescode)) { PGLOGERR("Insert", rescode, conn); goto foil; @@ -2630,14 +2790,108 @@ bool idcontrol_add(PGconn *conn, char *idname, char *idvalue, char *by, ok = true; foil: - if (conned) - PQfinish(conn); + CKPQDisco(&conn, conned); for (n = 0; n < par; n++) free(params[n]); + /* N.B. The DB key matches the tree key, + * the tree depends on this to be valid */ K_WLOCK(idcontrol_free); - k_add_head(idcontrol_free, look); + if (ok) { + add_to_ktree(idcontrol_root, item); + k_add_head(idcontrol_store, item); + } else + k_add_head(idcontrol_free, item); + K_WUNLOCK(idcontrol_free); + + return ok; +} + +bool idcontrol_fill(PGconn *conn) +{ + char pcombuf[64]; + ExecStatusType rescode; + PGresult *res; + K_ITEM *item; + int n, i; + IDCONTROL *row; + char *field; + char *sel; + int fields = 2; + bool ok; + + LOGDEBUG("%s(): select", __func__); + + int idname_num, lastid_num; + MODIFYDATE_num; + + sel = "select " + "idname,lastid" + MODIFYDATECONTROL + " from idcontrol"; + res = CKPQExec(conn, sel, CKPQ_READ); + rescode = CKPQResultStatus(res); + if (!PGOK(rescode)) { + PGLOGERR("Select", rescode, conn); + CKPQClear(res); + return false; + } + + n = PQnfields(res); + if (n != (fields + MODIFYDATECOUNT)) { + LOGERR("%s(): Invalid field count - should be %d, but is %d", + __func__, fields + MODIFYDATECOUNT, n); + CKPQClear(res); + return false; + } + + n = PQntuples(res); + LOGDEBUG("%s(): tree build count %d", __func__, n); + ok = true; + idname_num = lastid_num = CKPQFUNDEF; + MODIFYDATE_init; + K_WLOCK(idcontrol_free); + for (i = 0; i < n; i++) { + item = k_unlink_head(idcontrol_free); + DATA_IDCONTROL(row, item); + bzero(row, sizeof(*row)); + + if (everyone_die) { + ok = false; + break; + } + + CKPQ_VAL_FLD_num(res, i, idname, field, ok); + if (!ok) + break; + TXT_TO_STR("idname", field, row->idname); + + CKPQ_VAL_FLD_num(res, i, lastid, field, ok); + if (!ok) + break; + TXT_TO_BIGINT("lastid", field, row->lastid); + + MODIFYDATEIN(res, i, row, ok); + if (!ok) + break; + + /* N.B. The DB key matches the tree key, + * the tree depends on this to be valid */ + add_to_ktree(idcontrol_root, item); + k_add_head(idcontrol_store, item); + } + if (!ok) + k_add_head(idcontrol_free, item); + K_WUNLOCK(idcontrol_free); + CKPQClear(res); + + if (ok) { + LOGDEBUG("%s(): built", __func__); + pcom(n, pcombuf, sizeof(pcombuf)); + LOGWARNING("%s(): loaded %s idcontrol records", + __func__, pcombuf); + } return ok; } @@ -2863,18 +3117,45 @@ void oc_ips(OPTIONCONTROL *oc, const char *from) } } +void oc_trf(OPTIONCONTROL *oc, const char *from) +{ + int ct; + + ct = atoi(oc->optionvalue); + if (ct < 0 || ct > 64) { + LOGERR("%s(%s) ERR set cull_transfer ignored '%s' (%d)" + " must be 0..64", + from, __func__, oc->optionvalue, ct); + } else { + K_WLOCK(transfer_free); + // ct isn't the value, it's the multiplier + cull_transfer = ct * ALLOC_TRANSFER; + transfer_free->cull_limit = cull_transfer; + K_WUNLOCK(transfer_free); + LOGWARNING("%s(%s) set cull_transfer to %d->%d", + from, __func__, ct, cull_transfer); + } +} + +/* Trigger functions shouldn't touch expirydate since the optioncontrol data + * isn't locked - that's the only field that could change */ OC_TRIGGER oc_trigger[] = { { SWITCH_STATE_NAME, true, oc_switch_state }, { DIFF_PERCENT_NAME, true, oc_diff_percent }, { OC_LIMITS, false, oc_event_limits }, { OC_OLIMITS, false, oc_ovent_limits }, { OC_IPS, false, oc_ips }, + { CULL_TRANSFER_NAME, true, oc_trf }, { NULL, 0, NULL } }; /* For oc items that aren't date/height controlled, and use global variables * rather than having to look up the value every time it's needed - * Called from within the write lock that loaded/added the oc_item */ + * Called from within the write lock that loaded/added the oc_item + * The write lock is released before calling the trigger function, + * and regained after it returns, thus expirydate shouldn't be accessed + * since it's value could be changed by another thread, under lock + * None of the other oc field values will change */ static void optioncontrol_trigger(K_ITEM *oc_item, const char *from) { char cd_buf[DATE_BUFSIZ], cd2_buf[DATE_BUFSIZ]; @@ -2900,6 +3181,9 @@ static void optioncontrol_trigger(K_ITEM *oc_item, const char *from) } } if (got > -1) { + /* Don't hold the lock during debug messages or + * during the call to the trigger function */ + K_WUNLOCK(optioncontrol_free); // If it's date/height controlled, display an ERR if (oc->activationheight != OPTIONCONTROL_HEIGHT || tv_newer(&date_begin, &(oc->activationdate))) @@ -2916,6 +3200,7 @@ static void optioncontrol_trigger(K_ITEM *oc_item, const char *from) OPTIONCONTROL_HEIGHT, cd2_buf); } else oc_trigger[i].func(oc, from); + K_WLOCK(optioncontrol_free); } } } @@ -2927,7 +3212,7 @@ K_ITEM *optioncontrol_item_add(PGconn *conn, K_ITEM *oc_item, tv_t *cd, bool beg K_TREE_CTX ctx[1]; PGresult *res; K_ITEM *old_item, look; - OPTIONCONTROL *row; + OPTIONCONTROL *row, *oc; char *upd, *ins; bool ok = false; char *params[4 + HISTORYDATECOUNT]; @@ -2935,6 +3220,9 @@ K_ITEM *optioncontrol_item_add(PGconn *conn, K_ITEM *oc_item, tv_t *cd, bool beg LOGDEBUG("%s(): add", __func__); + if (pgdb_paused) + LOGEMERG("ERR: %s() called when paused - data lost", __func__); + DATA_OPTIONCONTROL(row, oc_item); // Enforce the rule that switch_state isn't date/height controlled @@ -2950,19 +3238,11 @@ K_ITEM *optioncontrol_item_add(PGconn *conn, K_ITEM *oc_item, tv_t *cd, bool beg old_item = find_in_ktree(optioncontrol_root, &look, ctx); K_RUNLOCK(optioncontrol_free); - if (!conn) { - conn = dbconnect(); + if (CKPQConn(&conn)) conned = true; - } - if (!begun) { - res = PQexec(conn, "Begin", CKPQ_WRITE); - rescode = PQresultStatus(res); - PQclear(res); - if (!PGOK(rescode)) { - PGLOGERR("Begin", rescode, conn); + if (!CKPQBegin(conn)) goto nostart; - } } if (old_item) { @@ -2979,9 +3259,9 @@ K_ITEM *optioncontrol_item_add(PGconn *conn, K_ITEM *oc_item, tv_t *cd, bool beg params[par++] = tv_to_buf((tv_t *)&default_expiry, NULL, 0); PARCHKVAL(par, 5, params); - res = PQexecParams(conn, upd, par, NULL, (const char **)params, NULL, NULL, 0, CKPQ_WRITE); - rescode = PQresultStatus(res); - PQclear(res); + res = CKPQExecParams(conn, upd, par, NULL, (const char **)params, NULL, NULL, 0, CKPQ_WRITE); + rescode = CKPQResultStatus(res); + CKPQClear(res); if (!PGOK(rescode)) { PGLOGERR("Update", rescode, conn); goto rollback; @@ -3003,9 +3283,9 @@ K_ITEM *optioncontrol_item_add(PGconn *conn, K_ITEM *oc_item, tv_t *cd, bool beg "(optionname,optionvalue,activationdate,activationheight" HISTORYDATECONTROL ") values (" PQPARAM9 ")"; - res = PQexecParams(conn, ins, par, NULL, (const char **)params, NULL, NULL, 0, CKPQ_WRITE); - rescode = PQresultStatus(res); - PQclear(res); + res = CKPQExecParams(conn, ins, par, NULL, (const char **)params, NULL, NULL, 0, CKPQ_WRITE); + rescode = CKPQResultStatus(res); + CKPQClear(res); if (!PGOK(rescode)) { PGLOGERR("Insert", rescode, conn); goto rollback; @@ -3013,17 +3293,11 @@ K_ITEM *optioncontrol_item_add(PGconn *conn, K_ITEM *oc_item, tv_t *cd, bool beg ok = true; rollback: - if (!begun) { - if (ok) - res = PQexec(conn, "Commit", CKPQ_WRITE); - else - res = PQexec(conn, "Rollback", CKPQ_WRITE); + if (!begun) + CKPQEnd(conn, ok); - PQclear(res); - } nostart: - if (conned) - PQfinish(conn); + CKPQDisco(&conn, conned); for (n = 0; n < par; n++) free(params[n]); @@ -3035,12 +3309,12 @@ nostart: free_optioncontrol_data(oc_item); k_add_head(optioncontrol_free, oc_item); } else { - // Discard old + // Keep old to ensure the new item can be read outside lock if (old_item) { remove_from_ktree(optioncontrol_root, old_item); - k_unlink_item(optioncontrol_store, old_item); - free_optioncontrol_data(old_item); - k_add_head(optioncontrol_free, old_item); + DATA_OPTIONCONTROL(oc, old_item); + copy_tv(&(oc->expirydate), cd); + add_to_ktree(optioncontrol_root, old_item); } add_to_ktree(optioncontrol_root, oc_item); k_add_head(optioncontrol_store, oc_item); @@ -3065,6 +3339,9 @@ K_ITEM *optioncontrol_add(PGconn *conn, char *optionname, char *optionvalue, LOGDEBUG("%s(): add", __func__); + if (pgdb_paused) + LOGEMERG("ERR: %s() called when paused - data lost", __func__); + K_WLOCK(optioncontrol_free); item = k_unlink_head(optioncontrol_free); K_WUNLOCK(optioncontrol_free); @@ -3095,6 +3372,7 @@ K_ITEM *optioncontrol_add(PGconn *conn, char *optionname, char *optionvalue, bool optioncontrol_fill(PGconn *conn) { + char pcombuf[64]; ExecStatusType rescode; PGresult *res; K_ITEM *item; @@ -3110,6 +3388,10 @@ bool optioncontrol_fill(PGconn *conn) LOGDEBUG("%s(): select", __func__); + int optionname_num, optionvalue_num, activationdate_num; + int activationheight_num; + HISTORYDATE_num; + // No need to keep old versions in ram for now ... sel = "select " "optionname,optionvalue,activationdate,activationheight" @@ -3118,11 +3400,11 @@ bool optioncontrol_fill(PGconn *conn) par = 0; params[par++] = tv_to_buf((tv_t *)(&default_expiry), NULL, 0); PARCHK(par, params); - res = PQexecParams(conn, sel, par, NULL, (const char **)params, NULL, NULL, 0, CKPQ_READ); - rescode = PQresultStatus(res); + res = CKPQExecParams(conn, sel, par, NULL, (const char **)params, NULL, NULL, 0, CKPQ_READ); + rescode = CKPQResultStatus(res); if (!PGOK(rescode)) { PGLOGERR("Select", rescode, conn); - PQclear(res); + CKPQClear(res); return false; } @@ -3130,13 +3412,16 @@ bool optioncontrol_fill(PGconn *conn) if (n != (fields + HISTORYDATECOUNT)) { LOGERR("%s(): Invalid field count - should be %d, but is %d", __func__, fields + HISTORYDATECOUNT, n); - PQclear(res); + CKPQClear(res); return false; } n = PQntuples(res); LOGDEBUG("%s(): tree build count %d", __func__, n); ok = true; + optionname_num = optionvalue_num = activationdate_num = + activationheight_num = CKPQFUNDEF; + HISTORYDATE_init; K_WLOCK(optioncontrol_free); for (i = 0; i < n; i++) { item = k_unlink_head(optioncontrol_free); @@ -3148,23 +3433,23 @@ bool optioncontrol_fill(PGconn *conn) break; } - PQ_GET_FLD(res, i, "optionname", field, ok); + CKPQ_VAL_FLD_num(res, i, optionname, field, ok); if (!ok) break; TXT_TO_STR("optionname", field, row->optionname); - PQ_GET_FLD(res, i, "optionvalue", field, ok); + CKPQ_VAL_FLD_num(res, i, optionvalue, field, ok); if (!ok) break; TXT_TO_BLOB("optionvalue", field, row->optionvalue); LIST_MEM_ADD(optioncontrol_free, row->optionvalue); - PQ_GET_FLD(res, i, "activationdate", field, ok); + CKPQ_VAL_FLD_num(res, i, activationdate, field, ok); if (!ok) break; TXT_TO_TVDB("activationdate", field, row->activationdate); - PQ_GET_FLD(res, i, "activationheight", field, ok); + CKPQ_VAL_FLD_num(res, i, activationheight, field, ok); if (!ok) break; TXT_TO_INT("activationheight", field, row->activationheight); @@ -3184,13 +3469,15 @@ bool optioncontrol_fill(PGconn *conn) } K_WUNLOCK(optioncontrol_free); - PQclear(res); + CKPQClear(res); for (n = 0; n < par; n++) free(params[n]); if (ok) { LOGDEBUG("%s(): built", __func__); - LOGWARNING("%s(): loaded %d optioncontrol records", __func__, n); + pcom(n, pcombuf, sizeof(pcombuf)); + LOGWARNING("%s(): loaded %s optioncontrol records", + __func__, pcombuf); LOGWARNING("%s() switch_state initially %d", __func__, switch_state); @@ -3303,13 +3590,11 @@ int64_t workinfo_add(PGconn *conn, char *workinfoidstr, "prevhash,coinbase1,coinbase2,version,bits,ntime,reward" HISTORYDATECONTROL ") values (" PQPARAM16 ")"; - if (!conn) { - conn = dbconnect(); + if (CKPQConn(&conn)) conned = true; - } - - res = PQexecParams(conn, ins, par, NULL, (const char **)params, NULL, NULL, 0, CKPQ_WRITE); - rescode = PQresultStatus(res); + res = CKPQExecParams(conn, ins, par, NULL, (const char **)params, NULL, NULL, 0, CKPQ_WRITE); + rescode = CKPQResultStatus(res); + CKPQClear(res); if (!PGOK(rescode)) { PGLOGERR("Insert", rescode, conn); goto unparam; @@ -3320,9 +3605,7 @@ int64_t workinfo_add(PGconn *conn, char *workinfoidstr, unparam: if (par) { - PQclear(res); - if (conned) - PQfinish(conn); + CKPQDisco(&conn, conned); for (n = 0; n < par; n++) free(params[n]); } @@ -3386,6 +3669,11 @@ bool workinfo_fill(PGconn *conn) STRNCPY(tickbuf, TICK_PREFIX"wi 0"); cr_msg(false, tickbuf); + int workinfoid_num, poolinstance_num, merklehash_num, prevhash_num; + int coinbase1_num, coinbase2_num, version_num, bits_num, ntime_num; + int reward_num; + HISTORYDATE_num; + APPEND_REALLOC_INIT(sel, off, len); APPEND_REALLOC(sel, off, len, "declare wi cursor for select " @@ -3419,27 +3707,22 @@ bool workinfo_fill(PGconn *conn) params[par++] = bigint_to_buf(dbload_workinfoid_finish, NULL, 0); PARCHK(par, params); - res = PQexec(conn, "Begin", CKPQ_READ); - rescode = PQresultStatus(res); - PQclear(res); - if (!PGOK(rescode)) { - PGLOGERR("Begin", rescode, conn); + if (!CKPQBegin(conn)) return false; - } if (exclusive_db) { - res = PQexec(conn, "Lock table workinfo in access exclusive mode", CKPQ_READ); - rescode = PQresultStatus(res); - PQclear(res); + res = CKPQExec(conn, "Lock table workinfo in access exclusive mode", CKPQ_READ); + rescode = CKPQResultStatus(res); + CKPQClear(res); if (!PGOK(rescode)) { PGLOGERR("Lock", rescode, conn); goto flail; } } - res = PQexecParams(conn, sel, par, NULL, (const char **)params, NULL, NULL, 0, CKPQ_READ); - rescode = PQresultStatus(res); - PQclear(res); + res = CKPQExecParams(conn, sel, par, NULL, (const char **)params, NULL, NULL, 0, CKPQ_READ); + rescode = CKPQResultStatus(res); + CKPQClear(res); if (!PGOK(rescode)) { PGLOGERR("Declare", rescode, conn); goto flail; @@ -3447,11 +3730,11 @@ bool workinfo_fill(PGconn *conn) LOGDEBUG("%s(): fetching ...", __func__); - res = PQexec(conn, "fetch 1 in wi", CKPQ_READ); - rescode = PQresultStatus(res); + res = CKPQExec(conn, "fetch 1 in wi", CKPQ_READ); + rescode = CKPQResultStatus(res); if (!PGOK(rescode)) { PGLOGERR("Fetch first", rescode, conn); - PQclear(res); + CKPQClear(res); goto flail; } @@ -3459,12 +3742,16 @@ bool workinfo_fill(PGconn *conn) if (n != (fields + HISTORYDATECOUNT)) { LOGERR("%s(): Invalid field count - should be %d, but is %d", __func__, fields + HISTORYDATECOUNT, n); - PQclear(res); + CKPQClear(res); goto flail; } n = 0; ok = true; + workinfoid_num = poolinstance_num = merklehash_num = prevhash_num = + coinbase1_num = coinbase2_num = version_num = bits_num = ntime_num = + reward_num = CKPQFUNDEF; + HISTORYDATE_init; K_WLOCK(workinfo_free); while ((t = PQntuples(res)) > 0) { for (i = 0; i < t; i++) { @@ -3477,7 +3764,7 @@ bool workinfo_fill(PGconn *conn) break; } - PQ_GET_FLD(res, i, "poolinstance", field, ok); + CKPQ_VAL_FLD_num(res, i, poolinstance, field, ok); if (!ok) break; if (sys_poolinstance && strcmp(field, sys_poolinstance)) { @@ -3487,7 +3774,7 @@ bool workinfo_fill(PGconn *conn) } row->in_poolinstance = intransient_str("poolinstance", field); - PQ_GET_FLD(res, i, "workinfoid", field, ok); + CKPQ_VAL_FLD_num(res, i, workinfoid, field, ok); if (!ok) break; TXT_TO_BIGINT("workinfoid", field, row->workinfoid); @@ -3495,39 +3782,39 @@ bool workinfo_fill(PGconn *conn) row->transactiontree = EMPTY; row->merklehash = EMPTY; - PQ_GET_FLD(res, i, "prevhash", field, ok); + CKPQ_VAL_FLD_num(res, i, prevhash, field, ok); if (!ok) break; row->in_prevhash = intransient_str("prevhash", field); - PQ_GET_FLD(res, i, "coinbase1", field, ok); + CKPQ_VAL_FLD_num(res, i, coinbase1, field, ok); if (!ok) break; TXT_TO_BLOB("coinbase1", field, row->coinbase1); LIST_MEM_ADD(workinfo_free, row->coinbase1); - PQ_GET_FLD(res, i, "coinbase2", field, ok); + CKPQ_VAL_FLD_num(res, i, coinbase2, field, ok); if (!ok) break; TXT_TO_BLOB("coinbase2", field, row->coinbase2); LIST_MEM_ADD(workinfo_free, row->coinbase2); - PQ_GET_FLD(res, i, "version", field, ok); + CKPQ_VAL_FLD_num(res, i, version, field, ok); if (!ok) break; row->in_version = intransient_str("version", field); - PQ_GET_FLD(res, i, "bits", field, ok); + CKPQ_VAL_FLD_num(res, i, bits, field, ok); if (!ok) break; row->in_bits = intransient_str("bits", field); - PQ_GET_FLD(res, i, "ntime", field, ok); + CKPQ_VAL_FLD_num(res, i, ntime, field, ok); if (!ok) break; TXT_TO_STR("ntime", field, row->ntime); - PQ_GET_FLD(res, i, "reward", field, ok); + CKPQ_VAL_FLD_num(res, i, reward, field, ok); if (!ok) break; TXT_TO_BIGINT("reward", field, row->reward); @@ -3551,7 +3838,7 @@ bool workinfo_fill(PGconn *conn) dbstatus.newest_workinfoid = row->workinfoid; } - if (n == 0 || ((n+1) % 100000) == 0) { + if (n == 0 || ((n+1) % FETCHTICK) == 0) { pcom(n+1, pcombuf, sizeof(pcombuf)); snprintf(tickbuf, sizeof(tickbuf), TICK_PREFIX"wi %s", pcombuf); @@ -3560,14 +3847,18 @@ bool workinfo_fill(PGconn *conn) tick(); n++; } - PQclear(res); - res = PQexec(conn, "fetch 9999 in wi", CKPQ_READ); - rescode = PQresultStatus(res); + CKPQClear(res); + res = CKPQExec(conn, "fetch "CKPQFETCHSTR" in wi", CKPQ_READ); + rescode = CKPQResultStatus(res); if (!PGOK(rescode)) { PGLOGERR("Fetch next", rescode, conn); ok = false; break; } + workinfoid_num = poolinstance_num = merklehash_num = + prevhash_num = coinbase1_num = coinbase2_num = version_num = + bits_num = ntime_num = reward_num = CKPQFUNDEF; + HISTORYDATE_init; } if (!ok) { free_workinfo_data(item); @@ -3579,10 +3870,10 @@ bool workinfo_fill(PGconn *conn) } K_WUNLOCK(workinfo_free); - PQclear(res); + CKPQClear(res); flail: - res = PQexec(conn, "Commit", CKPQ_READ); - PQclear(res); + + CKPQCommit(conn); for (i = 0; i < par; i++) free(params[i]); par = 0; @@ -3591,7 +3882,9 @@ flail: if (ok) { LOGDEBUG("%s(): built", __func__); - LOGWARNING("%s(): fetched %d workinfo records", __func__, n); + pcom(n, pcombuf, sizeof(pcombuf)); + LOGWARNING("%s(): fetched %s workinfo records", + __func__, pcombuf); } POOLINSTANCE_DBLOAD_MSG(workinfo); @@ -3656,10 +3949,11 @@ static bool shares_process(PGconn *conn, SHARES *shares, K_ITEM *wi_item, shares->createcode, shares->createinet, &(shares->createdate), trf_root); if (!w_item) { - LOGDEBUG("%s(): new_default_worker failed %"PRId64"/%s/%ld,%ld", - __func__, shares->userid, - st = safe_text_nonull(shares->in_workername), - shares->createdate.tv_sec, shares->createdate.tv_usec); + LOGERR("%s(): ERR new_default_worker failed" + " %"PRId64"/%s/%ld,%ld", + __func__, shares->userid, + st = safe_text_nonull(shares->in_workername), + shares->createdate.tv_sec, shares->createdate.tv_usec); FREENULL(st); return false; } @@ -3671,12 +3965,12 @@ static bool shares_process(PGconn *conn, SHARES *shares, K_ITEM *wi_item, MARKER_PROCESSED, NULL); K_RUNLOCK(workmarkers_free); if (wm_item) { - LOGDEBUG("%s(): workmarker exists for wid %"PRId64 - " %"PRId64"/%s/%ld,%ld", - __func__, shares->workinfoid, shares->userid, - st = safe_text_nonull(shares->in_workername), - shares->createdate.tv_sec, - shares->createdate.tv_usec); + LOGERR("%s(): ERR workmarker exists for wid %"PRId64 + " %"PRId64"/%s/%ld,%ld", + __func__, shares->workinfoid, shares->userid, + st = safe_text_nonull(shares->in_workername), + shares->createdate.tv_sec, + shares->createdate.tv_usec); FREENULL(st); return false; } @@ -3875,18 +4169,31 @@ bool shares_add(PGconn *conn, char *workinfoid, char *username, K_RLOCK(users_free); u_item = find_users(username); K_RUNLOCK(users_free); - /* Can't change outside lock since we don't delete users + /* Won't change outside lock since we don't delete users * or change their *userid */ if (!u_item) { - btv_to_buf(cd, cd_buf, sizeof(cd_buf)); /* This should never happen unless there's a bug in ckpool or the authentication information got to ckdb after - the shares ... which shouldn't ever happen */ - LOGERR("%s() %s/%ld,%ld %s no user! Share discarded!", - __func__, st = safe_text_nonull(username), - cd->tv_sec, cd->tv_usec, cd_buf); - FREENULL(st); - goto tisbad; + the shares or the users table is missing data ... + which shouldn't ever happen + However, since it's a valid share, store it */ + u_item = create_missing_user(conn, username, secondaryuserid, + by, code, inet, cd, trf_root); + btv_to_buf(cd, cd_buf, sizeof(cd_buf)); + if (!u_item) { + LOGERR("%s() ERR %s/%ld,%ld %s no/failed user! Share" + " discarded!", + __func__, st = safe_text_nonull(username), + cd->tv_sec, cd->tv_usec, cd_buf); + FREENULL(st); + goto tisbad; + } else { + DATA_USERS(users, u_item); + LOGERR("%s() MISSING %s/%ld,%ld %s created", + __func__, st = safe_text_nonull(username), + cd->tv_sec, cd->tv_usec, cd_buf); + FREENULL(st); + } } DATA_USERS(users, u_item); shares->userid = users->userid; @@ -4066,24 +4373,26 @@ bool shares_db(PGconn *conn, K_ITEM *s_item) "diff,sdiff,errn,error,secondaryuserid,ntime,minsdiff,address," "agent" HISTORYDATECONTROL ") values (" PQPARAM21 ")"; - if (!conn) { - conn = dbconnect(); + if (CKPQConn(&conn)) conned = true; - } - - res = PQexecParams(conn, ins, par, NULL, (const char **)params, NULL, NULL, 0, CKPQ_WRITE); - rescode = PQresultStatus(res); + res = CKPQExecParams(conn, ins, par, NULL, (const char **)params, NULL, NULL, 0, CKPQ_WRITE); + rescode = CKPQResultStatus(res); if (!PGOK(rescode)) { - PGLOGERR("Insert", rescode, conn); - goto unparam; + char *code = PQresultErrorField(res, PG_DIAG_SQLSTATE); + if (!code || strcmp(code, SQL_UNIQUE_VIOLATION)) { + PGLOGERR("Insert", rescode, conn); + goto unparam; + } else { + // If the share is already in the db use NOTICE + PGLOGNOTICE("Insert", rescode, conn); + } } ok = true; unparam: if (par) { - PQclear(res); - if (conned) - PQfinish(conn); + CKPQClear(res); + CKPQDisco(&conn, conned); for (n = 0; n < par; n++) free(params[n]); } @@ -4153,6 +4462,12 @@ bool shares_fill(PGconn *conn) STRNCPY(tickbuf, TICK_PREFIX"sh 0"); cr_msg(false, tickbuf); + int workinfoid_num, userid_num, workername_num, clientid_num; + int enonce1_num, nonce2_num, nonce_num, diff_num, sdiff_num, errn_num; + int error_num, secondaryuserid_num, ntime_num, minsdiff_num, agent_num; + int address_num; + HISTORYDATE_num; + sel = "declare sh cursor for select " "workinfoid,userid,workername,clientid,enonce1,nonce2,nonce," "diff,sdiff,errn,error,secondaryuserid,ntime,minsdiff,agent," @@ -4163,27 +4478,22 @@ bool shares_fill(PGconn *conn) params[par++] = bigint_to_buf(workinfoid, NULL, 0); PARCHK(par, params); - res = PQexec(conn, "Begin", CKPQ_READ); - rescode = PQresultStatus(res); - PQclear(res); - if (!PGOK(rescode)) { - PGLOGERR("Begin", rescode, conn); + if (!CKPQBegin(conn)) return false; - } if (exclusive_db) { - res = PQexec(conn, "Lock table shares in access exclusive mode", CKPQ_READ); - rescode = PQresultStatus(res); - PQclear(res); + res = CKPQExec(conn, "Lock table shares in access exclusive mode", CKPQ_READ); + rescode = CKPQResultStatus(res); + CKPQClear(res); if (!PGOK(rescode)) { PGLOGERR("Lock", rescode, conn); goto flail; } } - res = PQexecParams(conn, sel, par, NULL, (const char **)params, NULL, NULL, 0, CKPQ_READ); - rescode = PQresultStatus(res); - PQclear(res); + res = CKPQExecParams(conn, sel, par, NULL, (const char **)params, NULL, NULL, 0, CKPQ_READ); + rescode = CKPQResultStatus(res); + CKPQClear(res); if (!PGOK(rescode)) { PGLOGERR("Declare", rescode, conn); goto flail; @@ -4191,11 +4501,11 @@ bool shares_fill(PGconn *conn) LOGDEBUG("%s(): fetching ...", __func__); - res = PQexec(conn, "fetch 1 in sh", CKPQ_READ); - rescode = PQresultStatus(res); + res = CKPQExec(conn, "fetch 1 in sh", CKPQ_READ); + rescode = CKPQResultStatus(res); if (!PGOK(rescode)) { PGLOGERR("Fetch first", rescode, conn); - PQclear(res); + CKPQClear(res); goto flail; } @@ -4203,12 +4513,17 @@ bool shares_fill(PGconn *conn) if (n != (fields + HISTORYDATECOUNT)) { LOGERR("%s(): Invalid field count - should be %d, but is %d", __func__, fields + HISTORYDATECOUNT, n); - PQclear(res); + CKPQClear(res); goto flail; } n = 0; ok = true; + workinfoid_num = userid_num = workername_num = clientid_num = + enonce1_num = nonce2_num = nonce_num = diff_num = sdiff_num = errn_num = + error_num = secondaryuserid_num = ntime_num = minsdiff_num = agent_num = + address_num = CKPQFUNDEF; + HISTORYDATE_init; K_WLOCK(shares_free); while ((t = PQntuples(res)) > 0) { for (i = 0; i < t; i++) { @@ -4221,72 +4536,72 @@ bool shares_fill(PGconn *conn) break; } - PQ_GET_FLD(res, i, "workinfoid", field, ok); + CKPQ_VAL_FLD_num(res, i, workinfoid, field, ok); if (!ok) break; TXT_TO_BIGINT("workinfoid", field, row->workinfoid); - PQ_GET_FLD(res, i, "userid", field, ok); + CKPQ_VAL_FLD_num(res, i, userid, field, ok); if (!ok) break; TXT_TO_BIGINT("userid", field, row->userid); - PQ_GET_FLD(res, i, "workername", field, ok); + CKPQ_VAL_FLD_num(res, i, workername, field, ok); if (!ok) break; row->in_workername = intransient_str("workername", field); - PQ_GET_FLD(res, i, "clientid", field, ok); + CKPQ_VAL_FLD_num(res, i, clientid, field, ok); if (!ok) break; TXT_TO_INT("clientid", field, row->clientid); - PQ_GET_FLD(res, i, "enonce1", field, ok); + CKPQ_VAL_FLD_num(res, i, enonce1, field, ok); if (!ok) break; TXT_TO_STR("enonce1", field, row->enonce1); - PQ_GET_FLD(res, i, "nonce2", field, ok); + CKPQ_VAL_FLD_num(res, i, nonce2, field, ok); if (!ok) break; TXT_TO_STR("nonce2", field, row->nonce2); - PQ_GET_FLD(res, i, "nonce", field, ok); + CKPQ_VAL_FLD_num(res, i, nonce, field, ok); if (!ok) break; TXT_TO_STR("nonce", field, row->nonce); - PQ_GET_FLD(res, i, "diff", field, ok); + CKPQ_VAL_FLD_num(res, i, diff, field, ok); if (!ok) break; TXT_TO_DOUBLE("diff", field, row->diff); - PQ_GET_FLD(res, i, "sdiff", field, ok); + CKPQ_VAL_FLD_num(res, i, sdiff, field, ok); if (!ok) break; TXT_TO_DOUBLE("sdiff", field, row->sdiff); - PQ_GET_FLD(res, i, "errn", field, ok); + CKPQ_VAL_FLD_num(res, i, errn, field, ok); if (!ok) break; TXT_TO_INT("errn", field, row->errn); - PQ_GET_FLD(res, i, "error", field, ok); + CKPQ_VAL_FLD_num(res, i, error, field, ok); if (!ok) break; TXT_TO_STR("error", field, row->error); - PQ_GET_FLD(res, i, "secondaryuserid", field, ok); + CKPQ_VAL_FLD_num(res, i, secondaryuserid, field, ok); if (!ok) break; TXT_TO_STR("secondaryuserid", field, row->secondaryuserid); - PQ_GET_FLD(res, i, "ntime", field, ok); + CKPQ_VAL_FLD_num(res, i, ntime, field, ok); if (!ok) break; TXT_TO_STR("ntime", field, row->ntime); - PQ_GET_FLD(res, i, "minsdiff", field, ok); + CKPQ_VAL_FLD_num(res, i, minsdiff, field, ok); if (!ok) break; TXT_TO_DOUBLE("minsdiff", field, row->sdiff); @@ -4295,14 +4610,14 @@ bool shares_fill(PGconn *conn) if (!ok) break; - PQ_GET_FLD(res, i, "agent", field, ok); + CKPQ_VAL_FLD_num(res, i, agent, field, ok); if (!ok) break; if (!(*field)) no_agent++; TXT_TO_STR("agent", field, row->agent); - PQ_GET_FLD(res, i, "address", field, ok); + CKPQ_VAL_FLD_num(res, i, address, field, ok); if (!ok) break; if (!(*field)) @@ -4312,7 +4627,7 @@ bool shares_fill(PGconn *conn) add_to_ktree(shares_db_root, item); k_add_head(shares_hi_store, item); - if (n == 0 || ((n+1) % 100000) == 0) { + if (n == 0 || ((n+1) % FETCHTICK) == 0) { pcom(n+1, pcombuf, sizeof(pcombuf)); snprintf(tickbuf, sizeof(tickbuf), TICK_PREFIX"sh %s", pcombuf); @@ -4321,27 +4636,33 @@ bool shares_fill(PGconn *conn) tick(); n++; } - PQclear(res); - res = PQexec(conn, "fetch 9999 in sh", CKPQ_READ); - rescode = PQresultStatus(res); + CKPQClear(res); + res = CKPQExec(conn, "fetch "CKPQFETCHSTR" in sh", CKPQ_READ); + rescode = CKPQResultStatus(res); if (!PGOK(rescode)) { PGLOGERR("Fetch next", rescode, conn); ok = false; break; } + workinfoid_num = userid_num = workername_num = clientid_num = + enonce1_num = nonce2_num = nonce_num = diff_num = sdiff_num = + errn_num = error_num = secondaryuserid_num = ntime_num = + minsdiff_num = agent_num = address_num = CKPQFUNDEF; + HISTORYDATE_init; } if (!ok) k_add_head(shares_free, item); K_WUNLOCK(shares_free); - PQclear(res); + CKPQClear(res); flail: - res = PQexec(conn, "Commit", CKPQ_READ); - PQclear(res); + CKPQCommit(conn); if (ok) { LOGDEBUG("%s(): built", __func__); - LOGWARNING("%s(): fetched %d shares records", __func__, n); + pcom(n, pcombuf, sizeof(pcombuf)); + LOGWARNING("%s(): fetched %s shares records", + __func__, pcombuf); if (no_addr || no_agent) { if (no_addr == no_agent) { LOGWARNING(" %d had no address and agent", @@ -4714,8 +5035,7 @@ bool sharesummaries_to_markersummaries(PGconn *conn, WORKMARKERS *workmarkers, static const char *shortname = "K/SS_to_K/MS"; static const char *sshortname = "SS_to_MS"; static const char *kshortname = "KSS_to_KS"; - ExecStatusType rescode; - PGresult *res; + K_TREE_CTX ss_ctx[1], kss_ctx[1], ms_ctx[1], ks_ctx[1]; SHARESUMMARY *sharesummary, looksharesummary; KEYSHARESUMMARY *keysharesummary, lookkeysharesummary; @@ -5037,16 +5357,9 @@ dokey: setnow(&kadd_fin); setnow(&db_stt); - if (conn == NULL) { - conn = dbconnect(); + if (CKPQConn(&conn)) conned = true; - } - - res = PQexec(conn, "Begin", CKPQ_WRITE); - rescode = PQresultStatus(res); - PQclear(res); - if (!PGOK(rescode)) { - PGLOGERR("Begin", rescode, conn); + if (!CKPQBegin(conn)) { setnow(&db_fin); goto flail; } @@ -5090,15 +5403,11 @@ dokey: } setnow(&kdb_fin); rollback: - if (ok) - res = PQexec(conn, "Commit", CKPQ_WRITE); - else - res = PQexec(conn, "Rollback", CKPQ_WRITE); - PQclear(res); + CKPQEnd(conn, ok); + flail: - if (conned) - PQfinish(conn); + CKPQDisco(&conn, conned); if (reason) { // already displayed the full workmarkers detail at the top @@ -5356,13 +5665,10 @@ bool delete_markersummaries(PGconn *conn, WORKMARKERS *wm) del = "delete from markersummary where markerid=$1"; - if (conn == NULL) { - conn = dbconnect(); + if (CKPQConn(&conn)) conned = true; - } - - res = PQexecParams(conn, del, par, NULL, (const char **)params, NULL, NULL, 0, CKPQ_WRITE); - rescode = PQresultStatus(res); + res = CKPQExecParams(conn, del, par, NULL, (const char **)params, NULL, NULL, 0, CKPQ_WRITE); + rescode = CKPQResultStatus(res); if (!PGOK(rescode)) { PGLOGERR("Delete", rescode, conn); reason = "db error"; @@ -5385,10 +5691,9 @@ bool delete_markersummaries(PGconn *conn, WORKMARKERS *wm) ok = true; unparam: - PQclear(res); + CKPQClear(res); flail: - if (conned) - PQfinish(conn); + CKPQDisco(&conn, conned); if (!ok) { if (del_markersummary_store && del_markersummary_store->count) { @@ -5877,22 +6182,14 @@ bool blocks_stats(PGconn *conn, int32_t height, char *blockhash, params[par++] = tv_to_buf((tv_t *)&default_expiry, NULL, 0); PARCHKVAL(par, 3, params); - if (conn == NULL) { - conn = dbconnect(); + if (CKPQConn(&conn)) conned = true; - } - - res = PQexec(conn, "Begin", CKPQ_WRITE); - rescode = PQresultStatus(res); - PQclear(res); - if (!PGOK(rescode)) { - PGLOGERR("Begin", rescode, conn); + if (!CKPQBegin(conn)) goto unparam; - } - res = PQexecParams(conn, upd, par, NULL, (const char **)params, NULL, NULL, 0, CKPQ_WRITE); - rescode = PQresultStatus(res); - PQclear(res); + res = CKPQExecParams(conn, upd, par, NULL, (const char **)params, NULL, NULL, 0, CKPQ_WRITE); + rescode = CKPQResultStatus(res); + CKPQClear(res); if (!PGOK(rescode)) { PGLOGERR("Update", rescode, conn); goto rollback; @@ -5927,9 +6224,9 @@ bool blocks_stats(PGconn *conn, int32_t height, char *blockhash, "$9,$10,$11,$12,$13 from blocks where " "blockhash=$1 and "EDDB"=$2"; - res = PQexecParams(conn, ins, par, NULL, (const char **)params, NULL, NULL, 0, CKPQ_WRITE); - rescode = PQresultStatus(res); - PQclear(res); + res = CKPQExecParams(conn, ins, par, NULL, (const char **)params, NULL, NULL, 0, CKPQ_WRITE); + rescode = CKPQResultStatus(res); + CKPQClear(res); if (!PGOK(rescode)) { PGLOGERR("Insert", rescode, conn); goto rollback; @@ -5937,19 +6234,14 @@ bool blocks_stats(PGconn *conn, int32_t height, char *blockhash, ok = true; rollback: - if (ok) - res = PQexec(conn, "Commit", CKPQ_WRITE); - else - res = PQexec(conn, "Rollback", CKPQ_WRITE); - PQclear(res); + CKPQEnd(conn, ok); + unparam: + CKPQDisco(&conn, conned); for (n = 0; n < par; n++) free(params[n]); - if (conned) - PQfinish(conn); - K_WLOCK(blocks_free); if (!ok) k_add_head(blocks_free, b_item); @@ -6099,14 +6391,11 @@ bool blocks_add(PGconn *conn, int32_t height, char *blockhash, "statsconfirmed" HISTORYDATECONTROL ") values (" PQPARAM23 ")"; - if (conn == NULL) { - conn = dbconnect(); + if (CKPQConn(&conn)) conned = true; - } - - res = PQexecParams(conn, ins, par, NULL, (const char **)params, NULL, NULL, 0, CKPQ_WRITE); - rescode = PQresultStatus(res); - PQclear(res); + res = CKPQExecParams(conn, ins, par, NULL, (const char **)params, NULL, NULL, 0, CKPQ_WRITE); + rescode = CKPQResultStatus(res); + CKPQClear(res); if (!PGOK(rescode)) { PGLOGERR("Insert", rescode, conn); goto unparam; @@ -6181,11 +6470,8 @@ bool blocks_add(PGconn *conn, int32_t height, char *blockhash, params[par++] = tv_to_buf((tv_t *)&default_expiry, NULL, 0); PARCHKVAL(par, 3, params); - if (conn == NULL) { - conn = dbconnect(); + if (CKPQConn(&conn)) conned = true; - } - // New is mostly a copy of the old copy_blocks(row, oldblocks); STRNCPY(row->confirmed, confirmed); @@ -6201,17 +6487,11 @@ bool blocks_add(PGconn *conn, int32_t height, char *blockhash, HISTORYDATEINIT(row, cd, by, code, inet); HISTORYDATETRANSFER(trf_root, row); - res = PQexec(conn, "Begin", CKPQ_WRITE); - rescode = PQresultStatus(res); - PQclear(res); - if (!PGOK(rescode)) { - PGLOGERR("Begin", rescode, conn); + if (!CKPQBegin(conn)) goto unparam; - } - - res = PQexecParams(conn, upd, par, NULL, (const char **)params, NULL, NULL, 0, CKPQ_WRITE); - rescode = PQresultStatus(res); - PQclear(res); + res = CKPQExecParams(conn, upd, par, NULL, (const char **)params, NULL, NULL, 0, CKPQ_WRITE); + rescode = CKPQResultStatus(res); + CKPQClear(res); if (!PGOK(rescode)) { PGLOGERR("Update", rescode, conn); goto rollback; @@ -6263,9 +6543,9 @@ bool blocks_add(PGconn *conn, int32_t height, char *blockhash, "blockhash=$1 and "EDDB"=$2"; } - res = PQexecParams(conn, ins, par, NULL, (const char **)params, NULL, NULL, 0, CKPQ_WRITE); - rescode = PQresultStatus(res); - PQclear(res); + res = CKPQExecParams(conn, ins, par, NULL, (const char **)params, NULL, NULL, 0, CKPQ_WRITE); + rescode = CKPQResultStatus(res); + CKPQClear(res); if (!PGOK(rescode)) { PGLOGERR("Insert", rescode, conn); goto rollback; @@ -6324,18 +6604,14 @@ bool blocks_add(PGconn *conn, int32_t height, char *blockhash, ok = true; rollback: - if (ok) - res = PQexec(conn, "Commit", CKPQ_WRITE); - else - res = PQexec(conn, "Rollback", CKPQ_WRITE); - PQclear(res); + CKPQEnd(conn, ok); + unparam: for (n = 0; n < par; n++) free(params[n]); flail: - if (conned) - PQfinish(conn); + CKPQDisco(&conn, conned); K_RLOCK(workinfo_free); K_WLOCK(blocks_free); @@ -6423,6 +6699,7 @@ flail: bool blocks_fill(PGconn *conn) { + char pcombuf[64]; ExecStatusType rescode; PGresult *res; K_TREE_CTX ctx[1]; @@ -6436,17 +6713,23 @@ bool blocks_fill(PGconn *conn) LOGDEBUG("%s(): select", __func__); + int height_num, blockhash_num, workinfoid_num, userid_num; + int workername_num, clientid_num, enonce1_num, nonce2_num, nonce_num; + int reward_num, confirmed_num, info_num, diffacc_num, diffinv_num; + int shareacc_num, shareinv_num, elapsed_num, statsconfirmed_num; + HISTORYDATE_num; + sel = "select " "height,blockhash,workinfoid,userid,workername," "clientid,enonce1,nonce2,nonce,reward,confirmed,info," "diffacc,diffinv,shareacc,shareinv,elapsed,statsconfirmed" HISTORYDATECONTROL " from blocks"; - res = PQexec(conn, sel, CKPQ_READ); - rescode = PQresultStatus(res); + res = CKPQExec(conn, sel, CKPQ_READ); + rescode = CKPQResultStatus(res); if (!PGOK(rescode)) { PGLOGERR("Select", rescode, conn); - PQclear(res); + CKPQClear(res); return false; } @@ -6454,13 +6737,19 @@ bool blocks_fill(PGconn *conn) if (n != (fields + HISTORYDATECOUNT)) { LOGERR("%s(): Invalid field count - should be %d, but is %d", __func__, fields + HISTORYDATECOUNT, n); - PQclear(res); + CKPQClear(res); return false; } n = PQntuples(res); LOGDEBUG("%s(): tree build count %d", __func__, n); ok = true; + height_num = blockhash_num = workinfoid_num = userid_num = + workername_num = clientid_num = enonce1_num = nonce2_num = nonce_num = + reward_num = confirmed_num = info_num = diffacc_num = diffinv_num = + shareacc_num = shareinv_num = elapsed_num = + statsconfirmed_num = CKPQFUNDEF; + HISTORYDATE_init; K_WLOCK(blocks_free); for (i = 0; i < n; i++) { item = k_unlink_head(blocks_free); @@ -6472,92 +6761,92 @@ bool blocks_fill(PGconn *conn) break; } - PQ_GET_FLD(res, i, "height", field, ok); + CKPQ_VAL_FLD_num(res, i, height, field, ok); if (!ok) break; TXT_TO_INT("height", field, row->height); - PQ_GET_FLD(res, i, "blockhash", field, ok); + CKPQ_VAL_FLD_num(res, i, blockhash, field, ok); if (!ok) break; TXT_TO_STR("blockhash", field, row->blockhash); - PQ_GET_FLD(res, i, "workinfoid", field, ok); + CKPQ_VAL_FLD_num(res, i, workinfoid, field, ok); if (!ok) break; TXT_TO_BIGINT("workinfoid", field, row->workinfoid); - PQ_GET_FLD(res, i, "userid", field, ok); + CKPQ_VAL_FLD_num(res, i, userid, field, ok); if (!ok) break; TXT_TO_BIGINT("userid", field, row->userid); - PQ_GET_FLD(res, i, "workername", field, ok); + CKPQ_VAL_FLD_num(res, i, workername, field, ok); if (!ok) break; row->in_workername = intransient_str("workername", field); - PQ_GET_FLD(res, i, "clientid", field, ok); + CKPQ_VAL_FLD_num(res, i, clientid, field, ok); if (!ok) break; TXT_TO_INT("clientid", field, row->clientid); - PQ_GET_FLD(res, i, "enonce1", field, ok); + CKPQ_VAL_FLD_num(res, i, enonce1, field, ok); if (!ok) break; TXT_TO_STR("enonce1", field, row->enonce1); - PQ_GET_FLD(res, i, "nonce2", field, ok); + CKPQ_VAL_FLD_num(res, i, nonce2, field, ok); if (!ok) break; TXT_TO_STR("nonce2", field, row->nonce2); - PQ_GET_FLD(res, i, "nonce", field, ok); + CKPQ_VAL_FLD_num(res, i, nonce, field, ok); if (!ok) break; TXT_TO_STR("nonce", field, row->nonce); - PQ_GET_FLD(res, i, "reward", field, ok); + CKPQ_VAL_FLD_num(res, i, reward, field, ok); if (!ok) break; TXT_TO_BIGINT("reward", field, row->reward); - PQ_GET_FLD(res, i, "confirmed", field, ok); + CKPQ_VAL_FLD_num(res, i, confirmed, field, ok); if (!ok) break; TXT_TO_STR("confirmed", field, row->confirmed); - PQ_GET_FLD(res, i, "info", field, ok); + CKPQ_VAL_FLD_num(res, i, info, field, ok); if (!ok) break; TXT_TO_STR("info", field, row->info); - PQ_GET_FLD(res, i, "diffacc", field, ok); + CKPQ_VAL_FLD_num(res, i, diffacc, field, ok); if (!ok) break; TXT_TO_DOUBLE("diffacc", field, row->diffacc); - PQ_GET_FLD(res, i, "diffinv", field, ok); + CKPQ_VAL_FLD_num(res, i, diffinv, field, ok); if (!ok) break; TXT_TO_DOUBLE("diffinv", field, row->diffinv); - PQ_GET_FLD(res, i, "shareacc", field, ok); + CKPQ_VAL_FLD_num(res, i, shareacc, field, ok); if (!ok) break; TXT_TO_DOUBLE("shareacc", field, row->shareacc); - PQ_GET_FLD(res, i, "shareinv", field, ok); + CKPQ_VAL_FLD_num(res, i, shareinv, field, ok); if (!ok) break; TXT_TO_DOUBLE("shareinv", field, row->shareinv); - PQ_GET_FLD(res, i, "elapsed", field, ok); + CKPQ_VAL_FLD_num(res, i, elapsed, field, ok); if (!ok) break; TXT_TO_BIGINT("elapsed", field, row->elapsed); - PQ_GET_FLD(res, i, "statsconfirmed", field, ok); + CKPQ_VAL_FLD_num(res, i, statsconfirmed, field, ok); if (!ok) break; TXT_TO_STR("statsconfirmed", field, row->statsconfirmed); @@ -6606,11 +6895,13 @@ bool blocks_fill(PGconn *conn) } K_WUNLOCK(blocks_free); - PQclear(res); + CKPQClear(res); if (ok) { LOGDEBUG("%s(): built", __func__); - LOGWARNING("%s(): loaded %d blocks records", __func__, n); + pcom(n, pcombuf, sizeof(pcombuf)); + LOGWARNING("%s(): loaded %s blocks records", + __func__, pcombuf); } return ok; @@ -6666,7 +6957,8 @@ bool miningpayouts_add(PGconn *conn, bool add, K_ITEM *mp_item, *old_mp_item = find_miningpayouts(row->payoutid, row->userid); K_RUNLOCK(miningpayouts_free); - conned = CKPQConn(&conn); + if (CKPQConn(&conn)) + conned = true; if (!already) { begun = CKPQBegin(conn); if (!begun) @@ -6687,9 +6979,9 @@ bool miningpayouts_add(PGconn *conn, bool add, K_ITEM *mp_item, params[par++] = tv_to_buf((tv_t *)&default_expiry, NULL, 0); PARCHKVAL(par, 4, params); - res = PQexecParams(conn, upd, par, NULL, (const char **)params, NULL, NULL, 0, CKPQ_WRITE); - rescode = PQresultStatus(res); - PQclear(res); + res = CKPQExecParams(conn, upd, par, NULL, (const char **)params, NULL, NULL, 0, CKPQ_WRITE); + rescode = CKPQResultStatus(res); + CKPQClear(res); if (!PGOK(rescode)) { PGLOGERR("Update", rescode, conn); goto rollback; @@ -6718,8 +7010,9 @@ bool miningpayouts_add(PGconn *conn, bool add, K_ITEM *mp_item, "(payoutid,userid,diffacc,amount" HISTORYDATECONTROL ") values (" PQPARAM9 ")"; - res = PQexecParams(conn, ins, par, NULL, (const char **)params, NULL, NULL, 0, CKPQ_WRITE); - rescode = PQresultStatus(res); + res = CKPQExecParams(conn, ins, par, NULL, (const char **)params, NULL, NULL, 0, CKPQ_WRITE); + rescode = CKPQResultStatus(res); + CKPQClear(res); if (!PGOK(rescode)) { PGLOGERR("Insert", rescode, conn); goto unparam; @@ -6760,31 +7053,30 @@ bool miningpayouts_fill(PGconn *conn) STRNCPY(tickbuf, TICK_PREFIX"mp 0"); cr_msg(false, tickbuf); + + int payoutid_num, userid_num, diffacc_num, amount_num; + HISTORYDATE_num; + sel = "declare mp cursor for select " "payoutid,userid,diffacc,amount" HISTORYDATECONTROL " from miningpayouts"; - res = PQexec(conn, "Begin", CKPQ_READ); - rescode = PQresultStatus(res); - PQclear(res); - if (!PGOK(rescode)) { - PGLOGERR("Begin", rescode, conn); + if (!CKPQBegin(conn)) return false; - } if (exclusive_db) { - res = PQexec(conn, "Lock table miningpayouts in access exclusive mode", CKPQ_READ); - rescode = PQresultStatus(res); - PQclear(res); + res = CKPQExec(conn, "Lock table miningpayouts in access exclusive mode", CKPQ_READ); + rescode = CKPQResultStatus(res); + CKPQClear(res); if (!PGOK(rescode)) { PGLOGERR("Lock", rescode, conn); goto flail; } } - res = PQexec(conn, sel, CKPQ_READ); - rescode = PQresultStatus(res); - PQclear(res); + res = CKPQExec(conn, sel, CKPQ_READ); + rescode = CKPQResultStatus(res); + CKPQClear(res); if (!PGOK(rescode)) { PGLOGERR("Declare", rescode, conn); goto flail; @@ -6792,11 +7084,11 @@ bool miningpayouts_fill(PGconn *conn) LOGDEBUG("%s(): fetching ...", __func__); - res = PQexec(conn, "fetch 1 in mp", CKPQ_READ); - rescode = PQresultStatus(res); + res = CKPQExec(conn, "fetch 1 in mp", CKPQ_READ); + rescode = CKPQResultStatus(res); if (!PGOK(rescode)) { PGLOGERR("Fetch first", rescode, conn); - PQclear(res); + CKPQClear(res); goto flail; } @@ -6804,12 +7096,14 @@ bool miningpayouts_fill(PGconn *conn) if (n != (fields + HISTORYDATECOUNT)) { LOGERR("%s(): Invalid field count - should be %d, but is %d", __func__, fields + HISTORYDATECOUNT, n); - PQclear(res); + CKPQClear(res); goto flail; } n = 0; ok = true; + payoutid_num = userid_num = diffacc_num = amount_num = CKPQFUNDEF; + HISTORYDATE_init; K_WLOCK(miningpayouts_free); while ((t = PQntuples(res)) > 0) { for (i = 0; i < t; i++) { @@ -6822,22 +7116,22 @@ bool miningpayouts_fill(PGconn *conn) break; } - PQ_GET_FLD(res, i, "payoutid", field, ok); + CKPQ_VAL_FLD_num(res, i, payoutid, field, ok); if (!ok) break; TXT_TO_BIGINT("payoutid", field, row->payoutid); - PQ_GET_FLD(res, i, "userid", field, ok); + CKPQ_VAL_FLD_num(res, i, userid, field, ok); if (!ok) break; TXT_TO_BIGINT("userid", field, row->userid); - PQ_GET_FLD(res, i, "diffacc", field, ok); + CKPQ_VAL_FLD_num(res, i, diffacc, field, ok); if (!ok) break; TXT_TO_DOUBLE("diffacc", field, row->diffacc); - PQ_GET_FLD(res, i, "amount", field, ok); + CKPQ_VAL_FLD_num(res, i, amount, field, ok); if (!ok) break; TXT_TO_BIGINT("amount", field, row->amount); @@ -6849,7 +7143,7 @@ bool miningpayouts_fill(PGconn *conn) add_to_ktree(miningpayouts_root, item); k_add_head(miningpayouts_store, item); - if (n == 0 || ((n+1) % 100000) == 0) { + if (n == 0 || ((n+1) % FETCHTICK) == 0) { pcom(n+1, pcombuf, sizeof(pcombuf)); snprintf(tickbuf, sizeof(tickbuf), TICK_PREFIX"mp %s", pcombuf); @@ -6858,27 +7152,30 @@ bool miningpayouts_fill(PGconn *conn) tick(); n++; } - PQclear(res); - res = PQexec(conn, "fetch 9999 in mp", CKPQ_READ); - rescode = PQresultStatus(res); + CKPQClear(res); + res = CKPQExec(conn, "fetch "CKPQFETCHSTR" in mp", CKPQ_READ); + rescode = CKPQResultStatus(res); if (!PGOK(rescode)) { PGLOGERR("Fetch next", rescode, conn); ok = false; break; } + payoutid_num = userid_num = diffacc_num = amount_num = CKPQFUNDEF; + HISTORYDATE_init; } if (!ok) k_add_head(miningpayouts_free, item); K_WUNLOCK(miningpayouts_free); - PQclear(res); + CKPQClear(res); flail: - res = PQexec(conn, "Commit", CKPQ_READ); - PQclear(res); + CKPQCommit(conn); if (ok) { LOGDEBUG("%s(): built", __func__); - LOGWARNING("%s(): fetched %d miningpayout records", __func__, n); + pcom(n, pcombuf, sizeof(pcombuf)); + LOGWARNING("%s(): fetched %s miningpayout records", + __func__, pcombuf); } return ok; @@ -6941,7 +7238,8 @@ bool payouts_add(PGconn *conn, bool add, K_ITEM *p_item, K_ITEM **old_p_item, *old_p_item = find_payouts(row->height, row->blockhash); K_RUNLOCK(payouts_free); - conned = CKPQConn(&conn); + if (CKPQConn(&conn)) + conned = true; if (!already) { begun = CKPQBegin(conn); if (!begun) @@ -6961,9 +7259,9 @@ bool payouts_add(PGconn *conn, bool add, K_ITEM *p_item, K_ITEM **old_p_item, params[par++] = tv_to_buf((tv_t *)&default_expiry, NULL, 0); PARCHKVAL(par, 3, params); - res = PQexecParams(conn, upd, par, NULL, (const char **)params, NULL, NULL, 0, CKPQ_WRITE); - rescode = PQresultStatus(res); - PQclear(res); + res = CKPQExecParams(conn, upd, par, NULL, (const char **)params, NULL, NULL, 0, CKPQ_WRITE); + rescode = CKPQResultStatus(res); + CKPQClear(res); if (!PGOK(rescode)) { PGLOGERR("Update", rescode, conn); goto rollback; @@ -7013,8 +7311,9 @@ bool payouts_add(PGconn *conn, bool add, K_ITEM *p_item, K_ITEM **old_p_item, "lastshareacc,stats" HISTORYDATECONTROL ") values (" PQPARAM18 ")"; - res = PQexecParams(conn, ins, par, NULL, (const char **)params, NULL, NULL, 0, CKPQ_WRITE); - rescode = PQresultStatus(res); + res = CKPQExecParams(conn, ins, par, NULL, (const char **)params, NULL, NULL, 0, CKPQ_WRITE); + rescode = CKPQResultStatus(res); + CKPQClear(res); if (!PGOK(rescode)) { PGLOGERR("Insert", rescode, conn); goto unparam; @@ -7071,8 +7370,8 @@ K_ITEM *payouts_full_expire(PGconn *conn, int64_t payoutid, tv_t *now, bool lock goto matane; } - conned = CKPQConn(&conn); - + if (CKPQConn(&conn)) + conned = true; begun = CKPQBegin(conn); if (!begun) goto matane; @@ -7084,8 +7383,8 @@ K_ITEM *payouts_full_expire(PGconn *conn, int64_t payoutid, tv_t *now, bool lock params[par++] = tv_to_buf((tv_t *)&default_expiry, NULL, 0); PARCHKVAL(par, 3, params); - res = PQexecParams(conn, upd, par, NULL, (const char **)params, NULL, NULL, 0, CKPQ_WRITE); - rescode = PQresultStatus(res); + res = CKPQExecParams(conn, upd, par, NULL, (const char **)params, NULL, NULL, 0, CKPQ_WRITE); + rescode = CKPQResultStatus(res); if (PGOK(rescode)) { tuples = PQcmdTuples(res); if (tuples && *tuples) { @@ -7094,11 +7393,12 @@ K_ITEM *payouts_full_expire(PGconn *conn, int64_t payoutid, tv_t *now, bool lock LOGERR("%s() updated payouts should be 1" " but updated=%d", __func__, po_upd); + CKPQClear(res); goto matane; } } } - PQclear(res); + CKPQClear(res); if (!PGOK(rescode)) { PGLOGERR("Update payouts", rescode, conn); goto matane; @@ -7114,14 +7414,14 @@ K_ITEM *payouts_full_expire(PGconn *conn, int64_t payoutid, tv_t *now, bool lock params[par++] = tv_to_buf((tv_t *)&default_expiry, NULL, 0); PARCHKVAL(par, 3, params); - res = PQexecParams(conn, upd, par, NULL, (const char **)params, NULL, NULL, 0, CKPQ_WRITE); - rescode = PQresultStatus(res); + res = CKPQExecParams(conn, upd, par, NULL, (const char **)params, NULL, NULL, 0, CKPQ_WRITE); + rescode = CKPQResultStatus(res); if (PGOK(rescode)) { tuples = PQcmdTuples(res); if (tuples && *tuples) mp_upd = atoi(tuples); } - PQclear(res); + CKPQClear(res); if (!PGOK(rescode)) { PGLOGERR("Update miningpayouts", rescode, conn); goto matane; @@ -7137,14 +7437,14 @@ K_ITEM *payouts_full_expire(PGconn *conn, int64_t payoutid, tv_t *now, bool lock params[par++] = tv_to_buf((tv_t *)&default_expiry, NULL, 0); PARCHKVAL(par, 3, params); - res = PQexecParams(conn, upd, par, NULL, (const char **)params, NULL, NULL, 0, CKPQ_WRITE); - rescode = PQresultStatus(res); + res = CKPQExecParams(conn, upd, par, NULL, (const char **)params, NULL, NULL, 0, CKPQ_WRITE); + rescode = CKPQResultStatus(res); if (PGOK(rescode)) { tuples = PQcmdTuples(res); if (tuples && *tuples) pm_upd = atoi(tuples); } - PQclear(res); + CKPQClear(res); if (!PGOK(rescode)) { PGLOGERR("Update payments", rescode, conn); goto matane; @@ -7286,6 +7586,7 @@ matane: bool payouts_fill(PGconn *conn) { + char pcombuf[64]; ExecStatusType rescode; PGresult *res; K_ITEM *item, *b_item; @@ -7300,16 +7601,22 @@ bool payouts_fill(PGconn *conn) LOGDEBUG("%s(): select", __func__); + int payoutid_num, height_num, blockhash_num, minerreward_num; + int workinfoidstart_num, workinfoidend_num, elapsed_num, status_num; + int diffwanted_num, diffused_num, shareacc_num, lastshareacc_num; + int stats_num; + HISTORYDATE_num; + sel = "select " "payoutid,height,blockhash,minerreward,workinfoidstart,workinfoidend," "elapsed,status,diffwanted,diffused,shareacc,lastshareacc,stats" HISTORYDATECONTROL " from payouts"; - res = PQexec(conn, sel, CKPQ_READ); - rescode = PQresultStatus(res); + res = CKPQExec(conn, sel, CKPQ_READ); + rescode = CKPQResultStatus(res); if (!PGOK(rescode)) { PGLOGERR("Select", rescode, conn); - PQclear(res); + CKPQClear(res); return false; } @@ -7317,13 +7624,18 @@ bool payouts_fill(PGconn *conn) if (n != (fields + HISTORYDATECOUNT)) { LOGERR("%s(): Invalid field count - should be %d, but is %d", __func__, fields + HISTORYDATECOUNT, n); - PQclear(res); + CKPQClear(res); return false; } n = PQntuples(res); LOGDEBUG("%s(): tree build count %d", __func__, n); ok = true; + payoutid_num = height_num = blockhash_num = minerreward_num = + workinfoidstart_num = workinfoidend_num = elapsed_num = status_num = + diffwanted_num = diffused_num = shareacc_num = lastshareacc_num = + stats_num = CKPQFUNDEF; + HISTORYDATE_init; K_WLOCK(payouts_free); for (i = 0; i < n; i++) { item = k_unlink_head(payouts_free); @@ -7335,67 +7647,67 @@ bool payouts_fill(PGconn *conn) break; } - PQ_GET_FLD(res, i, "payoutid", field, ok); + CKPQ_VAL_FLD_num(res, i, payoutid, field, ok); if (!ok) break; TXT_TO_BIGINT("payoutid", field, row->payoutid); - PQ_GET_FLD(res, i, "height", field, ok); + CKPQ_VAL_FLD_num(res, i, height, field, ok); if (!ok) break; TXT_TO_INT("height", field, row->height); - PQ_GET_FLD(res, i, "blockhash", field, ok); + CKPQ_VAL_FLD_num(res, i, blockhash, field, ok); if (!ok) break; TXT_TO_STR("blockhash", field, row->blockhash); - PQ_GET_FLD(res, i, "minerreward", field, ok); + CKPQ_VAL_FLD_num(res, i, minerreward, field, ok); if (!ok) break; TXT_TO_BIGINT("minerreward", field, row->minerreward); - PQ_GET_FLD(res, i, "workinfoidstart", field, ok); + CKPQ_VAL_FLD_num(res, i, workinfoidstart, field, ok); if (!ok) break; TXT_TO_BIGINT("workinfoidstart", field, row->workinfoidstart); - PQ_GET_FLD(res, i, "workinfoidend", field, ok); + CKPQ_VAL_FLD_num(res, i, workinfoidend, field, ok); if (!ok) break; TXT_TO_BIGINT("workinfoidend", field, row->workinfoidend); - PQ_GET_FLD(res, i, "elapsed", field, ok); + CKPQ_VAL_FLD_num(res, i, elapsed, field, ok); if (!ok) break; TXT_TO_BIGINT("elapsed", field, row->elapsed); - PQ_GET_FLD(res, i, "status", field, ok); + CKPQ_VAL_FLD_num(res, i, status, field, ok); if (!ok) break; TXT_TO_STR("status", field, row->status); - PQ_GET_FLD(res, i, "diffwanted", field, ok); + CKPQ_VAL_FLD_num(res, i, diffwanted, field, ok); if (!ok) break; TXT_TO_DOUBLE("diffwanted", field, row->diffwanted); - PQ_GET_FLD(res, i, "diffused", field, ok); + CKPQ_VAL_FLD_num(res, i, diffused, field, ok); if (!ok) break; TXT_TO_DOUBLE("diffused", field, row->diffused); - PQ_GET_FLD(res, i, "shareacc", field, ok); + CKPQ_VAL_FLD_num(res, i, shareacc, field, ok); if (!ok) break; TXT_TO_DOUBLE("shareacc", field, row->shareacc); - PQ_GET_FLD(res, i, "lastshareacc", field, ok); + CKPQ_VAL_FLD_num(res, i, lastshareacc, field, ok); if (!ok) break; TXT_TO_TVDB("lastshareacc", field, row->lastshareacc); - PQ_GET_FLD(res, i, "stats", field, ok); + CKPQ_VAL_FLD_num(res, i, stats, field, ok); if (!ok) break; TXT_TO_BLOB("stats", field, row->stats); @@ -7439,11 +7751,13 @@ bool payouts_fill(PGconn *conn) } K_WUNLOCK(payouts_free); - PQclear(res); + CKPQClear(res); if (ok) { LOGDEBUG("%s(): built", __func__); - LOGWARNING("%s(): loaded %d payout records", __func__, n); + pcom(n, pcombuf, sizeof(pcombuf)); + LOGWARNING("%s(): loaded %s payout records", + __func__, pcombuf); } return ok; @@ -7814,8 +8128,8 @@ bool auths_add(PGconn *conn, INTRANSIENT *in_poolinstance, if (!u_item) { if (addressuser) { u_item = users_add(conn, in_username, EMPTY, EMPTY, - USER_ADDRESS, by, code, inet, cd, - trf_root); + NULL, USER_ADDRESS, by, code, inet, + cd, trf_root); } else { LOGDEBUG("%s(): unknown user '%s'", __func__, @@ -7964,13 +8278,10 @@ bool poolstats_add(PGconn *conn, bool store, INTRANSIENT *in_poolinstance, "hashrate5m,hashrate1hr,hashrate24hr" SIMPLEDATECONTROL ") values (" PQPARAM12 ")"; - if (!conn) { - conn = dbconnect(); + if (CKPQConn(&conn)) conned = true; - } - - res = PQexecParams(conn, ins, par, NULL, (const char **)params, NULL, NULL, 0, CKPQ_WRITE); - rescode = PQresultStatus(res); + res = CKPQExecParams(conn, ins, par, NULL, (const char **)params, NULL, NULL, 0, CKPQ_WRITE); + rescode = CKPQResultStatus(res); if (!PGOK(rescode)) { bool show_msg = true; char *code; @@ -7990,9 +8301,8 @@ bool poolstats_add(PGconn *conn, bool store, INTRANSIENT *in_poolinstance, ok = true; unparam: if (store) { - PQclear(res); - if (conned) - PQfinish(conn); + CKPQClear(res); + CKPQDisco(&conn, conned); for (n = 0; n < par; n++) free(params[n]); } @@ -8012,6 +8322,7 @@ unparam: // TODO: data selection - only require ? bool poolstats_fill(PGconn *conn) { + char pcombuf[64]; ExecStatusType rescode; PGresult *res; K_ITEM *item; @@ -8060,6 +8371,10 @@ bool poolstats_fill(PGconn *conn) tm.tm_sec, tzinfo); + int poolinstance_num, elapsed_num, users_num, workers_num, hashrate_num; + int hashrate5m_num, hashrate1hr_num, hashrate24hr_num; + SIMPLEDATE_num; + APPEND_REALLOC_INIT(sel, off, len); APPEND_REALLOC(sel, off, len, "select " @@ -8069,11 +8384,11 @@ bool poolstats_fill(PGconn *conn) " from poolstats where "CDDB">"); APPEND_REALLOC(sel, off, len, stamp); - res = PQexec(conn, sel, CKPQ_READ); - rescode = PQresultStatus(res); + res = CKPQExec(conn, sel, CKPQ_READ); + rescode = CKPQResultStatus(res); if (!PGOK(rescode)) { PGLOGERR("Select", rescode, conn); - PQclear(res); + CKPQClear(res); ok = false; goto clean; } @@ -8082,7 +8397,7 @@ bool poolstats_fill(PGconn *conn) if (n != (fields + SIMPLEDATECOUNT)) { LOGERR("%s(): Invalid field count - should be %d, but is %d", __func__, fields + SIMPLEDATECOUNT, n); - PQclear(res); + CKPQClear(res); ok = false; goto clean; } @@ -8090,6 +8405,10 @@ bool poolstats_fill(PGconn *conn) n = PQntuples(res); LOGDEBUG("%s(): tree build count %d", __func__, n); ok = true; + poolinstance_num = elapsed_num = users_num = workers_num = + hashrate_num = hashrate5m_num = hashrate1hr_num = + hashrate24hr_num = CKPQFUNDEF; + SIMPLEDATE_init; K_WLOCK(poolstats_free); for (i = 0; i < n; i++) { item = k_unlink_head(poolstats_free); @@ -8103,7 +8422,7 @@ bool poolstats_fill(PGconn *conn) row->stored = true; - PQ_GET_FLD(res, i, "poolinstance", field, ok); + CKPQ_VAL_FLD_num(res, i, poolinstance, field, ok); if (!ok) break; if (sys_poolinstance && strcmp(field, sys_poolinstance)) { @@ -8113,37 +8432,37 @@ bool poolstats_fill(PGconn *conn) } row->in_poolinstance = intransient_str("poolinstance", field); - PQ_GET_FLD(res, i, "elapsed", field, ok); + CKPQ_VAL_FLD_num(res, i, elapsed, field, ok); if (!ok) break; TXT_TO_BIGINT("elapsed", field, row->elapsed); - PQ_GET_FLD(res, i, "users", field, ok); + CKPQ_VAL_FLD_num(res, i, users, field, ok); if (!ok) break; TXT_TO_INT("users", field, row->users); - PQ_GET_FLD(res, i, "workers", field, ok); + CKPQ_VAL_FLD_num(res, i, workers, field, ok); if (!ok) break; TXT_TO_INT("workers", field, row->workers); - PQ_GET_FLD(res, i, "hashrate", field, ok); + CKPQ_VAL_FLD_num(res, i, hashrate, field, ok); if (!ok) break; TXT_TO_DOUBLE("hashrate", field, row->hashrate); - PQ_GET_FLD(res, i, "hashrate5m", field, ok); + CKPQ_VAL_FLD_num(res, i, hashrate5m, field, ok); if (!ok) break; TXT_TO_DOUBLE("hashrate5m", field, row->hashrate5m); - PQ_GET_FLD(res, i, "hashrate1hr", field, ok); + CKPQ_VAL_FLD_num(res, i, hashrate1hr, field, ok); if (!ok) break; TXT_TO_DOUBLE("hashrate1hr", field, row->hashrate1hr); - PQ_GET_FLD(res, i, "hashrate24hr", field, ok); + CKPQ_VAL_FLD_num(res, i, hashrate24hr, field, ok); if (!ok) break; TXT_TO_DOUBLE("hashrate24hr", field, row->hashrate24hr); @@ -8164,11 +8483,13 @@ bool poolstats_fill(PGconn *conn) k_add_head(poolstats_free, item); K_WUNLOCK(poolstats_free); - PQclear(res); + CKPQClear(res); if (ok) { LOGDEBUG("%s(): built", __func__); - LOGWARNING("%s(): loaded %d poolstats records", __func__, n); + pcom(n, pcombuf, sizeof(pcombuf)); + LOGWARNING("%s(): loaded %s poolstats records", + __func__, pcombuf); } clean: free(sel); @@ -8416,13 +8737,11 @@ bool markersummary_add(PGconn *conn, K_ITEM *ms_item, char *by, char *code, row->diffacc); FREENULL(st); - if (!conn) { - conn = dbconnect(); + if (CKPQConn(&conn)) conned = true; - } - - res = PQexecParams(conn, ins, par, NULL, (const char **)params, NULL, NULL, 0, CKPQ_WRITE); - rescode = PQresultStatus(res); + res = CKPQExecParams(conn, ins, par, NULL, (const char **)params, NULL, NULL, 0, CKPQ_WRITE); + rescode = CKPQResultStatus(res); + CKPQClear(res); if (!PGOK(rescode)) { PGLOGERR("Insert", rescode, conn); goto unparam; @@ -8430,9 +8749,7 @@ bool markersummary_add(PGconn *conn, K_ITEM *ms_item, char *by, char *code, ok = true; unparam: - PQclear(res); - if (conned) - PQfinish(conn); + CKPQDisco(&conn, conned); for (n = 0; n < par; n++) free(params[n]); @@ -8526,6 +8843,13 @@ bool markersummary_fill(PGconn *conn) } } + int markerid_num, userid_num, workername_num, diffacc_num, diffsta_num; + int diffdup_num, diffhi_num, diffrej_num, shareacc_num, sharesta_num; + int sharedup_num, sharehi_num, sharerej_num, sharecount_num; + int errorcount_num, firstshare_num, lastshare_num, firstshareacc_num; + int lastshareacc_num, lastdiffacc_num; + MODIFYDATE_num; + // TODO: limit how far back sel = "declare ws cursor for select " "markerid,userid,workername,diffacc,diffsta,diffdup,diffhi," @@ -8548,27 +8872,22 @@ bool markersummary_fill(PGconn *conn) STRNCPY(tickbuf, TICK_PREFIX"ms 0"); cr_msg(false, tickbuf); - res = PQexec(conn, "Begin", CKPQ_READ); - rescode = PQresultStatus(res); - PQclear(res); - if (!PGOK(rescode)) { - PGLOGERR("Begin", rescode, conn); + if (!CKPQBegin(conn)) return false; - } if (exclusive_db) { - res = PQexec(conn, "Lock table markersummary in access exclusive mode", CKPQ_READ); - rescode = PQresultStatus(res); - PQclear(res); + res = CKPQExec(conn, "Lock table markersummary in access exclusive mode", CKPQ_READ); + rescode = CKPQResultStatus(res); + CKPQClear(res); if (!PGOK(rescode)) { PGLOGERR("Lock", rescode, conn); goto flail; } } - res = PQexecParams(conn, sel, par, NULL, (const char **)params, NULL, NULL, 0, CKPQ_READ); - rescode = PQresultStatus(res); - PQclear(res); + res = CKPQExecParams(conn, sel, par, NULL, (const char **)params, NULL, NULL, 0, CKPQ_READ); + rescode = CKPQResultStatus(res); + CKPQClear(res); if (!PGOK(rescode)) { PGLOGERR("Declare", rescode, conn); goto flail; @@ -8576,11 +8895,11 @@ bool markersummary_fill(PGconn *conn) LOGDEBUG("%s(): fetching ...", __func__); - res = PQexec(conn, "fetch 1 in ws", CKPQ_READ); - rescode = PQresultStatus(res); + res = CKPQExec(conn, "fetch 1 in ws", CKPQ_READ); + rescode = CKPQResultStatus(res); if (!PGOK(rescode)) { PGLOGERR("Fetch first", rescode, conn); - PQclear(res); + CKPQClear(res); goto flail; } @@ -8588,12 +8907,18 @@ bool markersummary_fill(PGconn *conn) if (n != (fields + MODIFYDATECOUNT)) { LOGERR("%s(): Invalid field count - should be %d, but is %d", __func__, fields + MODIFYDATECOUNT, n); - PQclear(res); + CKPQClear(res); goto flail; } n = 0; ok = true; + markerid_num = userid_num = workername_num = diffacc_num = diffsta_num = + diffdup_num = diffhi_num = diffrej_num = shareacc_num = sharesta_num = + sharedup_num = sharehi_num = sharerej_num = sharecount_num = + errorcount_num = firstshare_num = lastshare_num = firstshareacc_num = + lastshareacc_num = lastdiffacc_num = CKPQFUNDEF; + MODIFYDATE_init; K_WLOCK(markersummary_free); while ((t = PQntuples(res)) > 0) { // Avoid locking them too many times @@ -8609,102 +8934,102 @@ bool markersummary_fill(PGconn *conn) break; } - PQ_GET_FLD(res, i, "markerid", field, ok); + CKPQ_VAL_FLD_num(res, i, markerid, field, ok); if (!ok) break; TXT_TO_BIGINT("markerid", field, row->markerid); - PQ_GET_FLD(res, i, "userid", field, ok); + CKPQ_VAL_FLD_num(res, i, userid, field, ok); if (!ok) break; TXT_TO_BIGINT("userid", field, row->userid); - PQ_GET_FLD(res, i, "workername", field, ok); + CKPQ_VAL_FLD_num(res, i, workername, field, ok); if (!ok) break; row->in_workername = intransient_str("workername", field); - PQ_GET_FLD(res, i, "diffacc", field, ok); + CKPQ_VAL_FLD_num(res, i, diffacc, field, ok); if (!ok) break; TXT_TO_DOUBLE("diffacc", field, row->diffacc); - PQ_GET_FLD(res, i, "diffsta", field, ok); + CKPQ_VAL_FLD_num(res, i, diffsta, field, ok); if (!ok) break; TXT_TO_DOUBLE("diffsta", field, row->diffsta); - PQ_GET_FLD(res, i, "diffdup", field, ok); + CKPQ_VAL_FLD_num(res, i, diffdup, field, ok); if (!ok) break; TXT_TO_DOUBLE("diffdup", field, row->diffdup); - PQ_GET_FLD(res, i, "diffhi", field, ok); + CKPQ_VAL_FLD_num(res, i, diffhi, field, ok); if (!ok) break; TXT_TO_DOUBLE("diffhi", field, row->diffhi); - PQ_GET_FLD(res, i, "diffrej", field, ok); + CKPQ_VAL_FLD_num(res, i, diffrej, field, ok); if (!ok) break; TXT_TO_DOUBLE("diffrej", field, row->diffrej); - PQ_GET_FLD(res, i, "shareacc", field, ok); + CKPQ_VAL_FLD_num(res, i, shareacc, field, ok); if (!ok) break; TXT_TO_DOUBLE("shareacc", field, row->shareacc); - PQ_GET_FLD(res, i, "sharesta", field, ok); + CKPQ_VAL_FLD_num(res, i, sharesta, field, ok); if (!ok) break; TXT_TO_DOUBLE("sharesta", field, row->sharesta); - PQ_GET_FLD(res, i, "sharedup", field, ok); + CKPQ_VAL_FLD_num(res, i, sharedup, field, ok); if (!ok) break; TXT_TO_DOUBLE("sharedup", field, row->sharedup); - PQ_GET_FLD(res, i, "sharehi", field, ok); + CKPQ_VAL_FLD_num(res, i, sharehi, field, ok); if (!ok) break; TXT_TO_DOUBLE("sharehi", field, row->sharehi); - PQ_GET_FLD(res, i, "sharerej", field, ok); + CKPQ_VAL_FLD_num(res, i, sharerej, field, ok); if (!ok) break; TXT_TO_DOUBLE("sharerej", field, row->sharerej); - PQ_GET_FLD(res, i, "sharecount", field, ok); + CKPQ_VAL_FLD_num(res, i, sharecount, field, ok); if (!ok) break; TXT_TO_BIGINT("sharecount", field, row->sharecount); - PQ_GET_FLD(res, i, "errorcount", field, ok); + CKPQ_VAL_FLD_num(res, i, errorcount, field, ok); if (!ok) break; TXT_TO_BIGINT("errorcount", field, row->errorcount); - PQ_GET_FLD(res, i, "firstshare", field, ok); + CKPQ_VAL_FLD_num(res, i, firstshare, field, ok); if (!ok) break; TXT_TO_TVDB("firstshare", field, row->firstshare); - PQ_GET_FLD(res, i, "lastshare", field, ok); + CKPQ_VAL_FLD_num(res, i, lastshare, field, ok); if (!ok) break; TXT_TO_TVDB("lastshare", field, row->lastshare); - PQ_GET_FLD(res, i, "firstshareacc", field, ok); + CKPQ_VAL_FLD_num(res, i, firstshareacc, field, ok); if (!ok) break; TXT_TO_TVDB("firstshareacc", field, row->firstshareacc); - PQ_GET_FLD(res, i, "lastshareacc", field, ok); + CKPQ_VAL_FLD_num(res, i, lastshareacc, field, ok); if (!ok) break; TXT_TO_TVDB("lastshareacc", field, row->lastshareacc); - PQ_GET_FLD(res, i, "lastdiffacc", field, ok); + CKPQ_VAL_FLD_num(res, i, lastdiffacc, field, ok); if (!ok) break; TXT_TO_DOUBLE("lastdiffacc", field, row->lastdiffacc); @@ -8746,7 +9071,7 @@ bool markersummary_fill(PGconn *conn) userinfo_update(NULL, NULL, row, false); - if (n == 0 || ((n+1) % 100000) == 0) { + if (n == 0 || ((n+1) % FETCHTICK) == 0) { pcom(n+1, pcombuf, sizeof(pcombuf)); snprintf(tickbuf, sizeof(tickbuf), TICK_PREFIX"ms %s", pcombuf); @@ -8757,14 +9082,21 @@ bool markersummary_fill(PGconn *conn) } K_WUNLOCK(userinfo_free); K_RUNLOCK(workmarkers_free); - PQclear(res); - res = PQexec(conn, "fetch 9999 in ws", CKPQ_READ); - rescode = PQresultStatus(res); + CKPQClear(res); + res = CKPQExec(conn, "fetch "CKPQFETCHSTR" in ws", CKPQ_READ); + rescode = CKPQResultStatus(res); if (!PGOK(rescode)) { PGLOGERR("Fetch next", rescode, conn); ok = false; break; } + markerid_num = userid_num = workername_num = diffacc_num = + diffsta_num = diffdup_num = diffhi_num = diffrej_num = + shareacc_num = sharesta_num = sharedup_num = sharehi_num = + sharerej_num = sharecount_num = errorcount_num = + firstshare_num = lastshare_num = firstshareacc_num = + lastshareacc_num = lastdiffacc_num = CKPQFUNDEF; + MODIFYDATE_init; } if (!ok) { free_markersummary_data(item); @@ -8774,19 +9106,22 @@ bool markersummary_fill(PGconn *conn) p_n = markersummary_pool_store->count; K_WUNLOCK(markersummary_free); - PQclear(res); + CKPQClear(res); flail: - res = PQexec(conn, "Commit", CKPQ_READ); - PQclear(res); + CKPQCommit(conn); for (i = 0; i < par; i++) free(params[i]); par = 0; if (ok) { LOGDEBUG("%s(): built", __func__); - LOGWARNING("%s(): fetched %d markersummary records", __func__, n); - LOGWARNING("%s(): created %d markersummary pool records", __func__, p_n); + pcom(n, pcombuf, sizeof(pcombuf)); + LOGWARNING("%s(): fetched %s markersummary records", + __func__, pcombuf); + pcom(p_n, pcombuf, sizeof(pcombuf)); + LOGWARNING("%s(): created %s markersummary pool records", + __func__, pcombuf); } return ok; @@ -8848,13 +9183,11 @@ bool keysummary_add(PGconn *conn, K_ITEM *ks_item, char *by, char *code, row->diffacc); FREENULL(st); - if (!conn) { - conn = dbconnect(); + if (CKPQConn(&conn)) conned = true; - } - - res = PQexecParams(conn, ins, par, NULL, (const char **)params, NULL, NULL, 0, CKPQ_WRITE); - rescode = PQresultStatus(res); + res = CKPQExecParams(conn, ins, par, NULL, (const char **)params, NULL, NULL, 0, CKPQ_WRITE); + rescode = CKPQResultStatus(res); + CKPQClear(res); if (!PGOK(rescode)) { PGLOGERR("Insert", rescode, conn); /* Don't fail on a duplicate during key_update @@ -8866,9 +9199,7 @@ bool keysummary_add(PGconn *conn, K_ITEM *ks_item, char *by, char *code, ok = true; unparam: - PQclear(res); - if (conned) - PQfinish(conn); + CKPQDisco(&conn, conned); for (n = 0; n < par; n++) free(params[n]); @@ -8925,20 +9256,12 @@ bool _workmarkers_process(PGconn *conn, bool already, bool add, LOGDEBUG("%s(): updating old", __func__); DATA_WORKMARKERS(oldworkmarkers, old_wm_item); - if (!conn) { - conn = dbconnect(); + if (CKPQConn(&conn)) conned = true; - } if (!already) { - res = PQexec(conn, "Begin", CKPQ_WRITE); - rescode = PQresultStatus(res); - PQclear(res); - if (!PGOK(rescode)) { - PGLOGERR("Begin", rescode, conn); + begun = CKPQBegin(conn); + if (!begun) goto unparam; - } - - begun = true; } upd = "update workmarkers set "EDDB"=$1 where markerid=$2" @@ -8949,9 +9272,9 @@ bool _workmarkers_process(PGconn *conn, bool already, bool add, params[par++] = tv_to_buf((tv_t *)&default_expiry, NULL, 0); PARCHKVAL(par, 3, params); - res = PQexecParams(conn, upd, par, NULL, (const char **)params, NULL, NULL, 0, CKPQ_WRITE); - rescode = PQresultStatus(res); - PQclear(res); + res = CKPQExecParams(conn, upd, par, NULL, (const char **)params, NULL, NULL, 0, CKPQ_WRITE); + rescode = CKPQResultStatus(res); + CKPQClear(res); if (!PGOK(rescode)) { PGLOGERR("Update", rescode, conn); goto rollback; @@ -8988,20 +9311,12 @@ bool _workmarkers_process(PGconn *conn, bool already, bool add, DATA_WORKMARKERS(row, wm_item); bzero(row, sizeof(*row)); - if (conn == NULL) { - conn = dbconnect(); + if (CKPQConn(&conn)) conned = true; - } - if (!already && !begun) { - res = PQexec(conn, "Begin", CKPQ_WRITE); - rescode = PQresultStatus(res); - PQclear(res); - if (!PGOK(rescode)) { - PGLOGERR("Begin", rescode, conn); + begun = CKPQBegin(conn); + if (!begun) goto unparam; - } - begun = true; } if (old_wm_item) @@ -9039,9 +9354,9 @@ bool _workmarkers_process(PGconn *conn, bool already, bool add, HISTORYDATEPARAMS(params, par, row); PARCHK(par, params); - res = PQexecParams(conn, ins, par, NULL, (const char **)params, NULL, NULL, 0, CKPQ_WRITE); - rescode = PQresultStatus(res); - PQclear(res); + res = CKPQExecParams(conn, ins, par, NULL, (const char **)params, NULL, NULL, 0, CKPQ_WRITE); + rescode = CKPQResultStatus(res); + CKPQClear(res); if (!PGOK(rescode)) { PGLOGERR("Insert", rescode, conn); goto rollback; @@ -9051,21 +9366,14 @@ bool _workmarkers_process(PGconn *conn, bool already, bool add, ok = true; rollback: - if (begun) { - if (ok) - res = PQexec(conn, "Commit", CKPQ_WRITE); - else - res = PQexec(conn, "Rollback", CKPQ_WRITE); + if (begun) + CKPQEnd(conn, ok); - PQclear(res); - } unparam: + CKPQDisco(&conn, conned); for (n = 0; n < par; n++) free(params[n]); - if (conned) - PQfinish(conn); - if (!ok) { if (wm_item) { K_WLOCK(workmarkers_free); @@ -9098,6 +9406,7 @@ unparam: bool workmarkers_fill(PGconn *conn) { + char pcombuf[64]; ExecStatusType rescode; PGresult *res; K_ITEM *item, *wi_item; @@ -9112,6 +9421,10 @@ bool workmarkers_fill(PGconn *conn) LOGDEBUG("%s(): select", __func__); + int markerid_num, poolinstance_num, workinfoidend_num; + int workinfoidstart_num, description_num, status_num; + HISTORYDATE_num; + // Allow limiting the load for key_update if (key_update && dbload_workinfoid_start != -1) { sel = "select " @@ -9122,19 +9435,19 @@ bool workmarkers_fill(PGconn *conn) par = 0; params[par++] = bigint_to_buf(dbload_workinfoid_start, NULL, 0); PARCHK(par, params); - res = PQexecParams(conn, sel, par, NULL, (const char **)params, NULL, NULL, 0, CKPQ_READ); + res = CKPQExecParams(conn, sel, par, NULL, (const char **)params, NULL, NULL, 0, CKPQ_READ); } else { sel = "select " "markerid,poolinstance,workinfoidend,workinfoidstart," "description,status" HISTORYDATECONTROL " from workmarkers"; - res = PQexec(conn, sel, CKPQ_READ); + res = CKPQExec(conn, sel, CKPQ_READ); } - rescode = PQresultStatus(res); + rescode = CKPQResultStatus(res); if (!PGOK(rescode)) { PGLOGERR("Select", rescode, conn); - PQclear(res); + CKPQClear(res); return false; } @@ -9142,13 +9455,16 @@ bool workmarkers_fill(PGconn *conn) if (n != (fields + HISTORYDATECOUNT)) { LOGERR("%s(): Invalid field count - should be %d, but is %d", __func__, fields + HISTORYDATECOUNT, n); - PQclear(res); + CKPQClear(res); return false; } n = PQntuples(res); LOGDEBUG("%s(): tree build count %d", __func__, n); ok = true; + markerid_num = poolinstance_num = workinfoidend_num = + workinfoidstart_num = description_num = status_num = CKPQFUNDEF; + HISTORYDATE_init; K_WLOCK(workmarkers_free); for (i = 0; i < n; i++) { item = k_unlink_head(workmarkers_free); @@ -9160,7 +9476,7 @@ bool workmarkers_fill(PGconn *conn) break; } - PQ_GET_FLD(res, i, "poolinstance", field, ok); + CKPQ_VAL_FLD_num(res, i, poolinstance, field, ok); if (!ok) break; if (sys_poolinstance && strcmp(field, sys_poolinstance)) { @@ -9170,28 +9486,28 @@ bool workmarkers_fill(PGconn *conn) } row->in_poolinstance = intransient_str("poolinstance", field); - PQ_GET_FLD(res, i, "markerid", field, ok); + CKPQ_VAL_FLD_num(res, i, markerid, field, ok); if (!ok) break; TXT_TO_BIGINT("markerid", field, row->markerid); - PQ_GET_FLD(res, i, "workinfoidend", field, ok); + CKPQ_VAL_FLD_num(res, i, workinfoidend, field, ok); if (!ok) break; TXT_TO_BIGINT("workinfoidend", field, row->workinfoidend); - PQ_GET_FLD(res, i, "workinfoidstart", field, ok); + CKPQ_VAL_FLD_num(res, i, workinfoidstart, field, ok); if (!ok) break; TXT_TO_BIGINT("workinfoidstart", field, row->workinfoidstart); - PQ_GET_FLD(res, i, "description", field, ok); + CKPQ_VAL_FLD_num(res, i, description, field, ok); if (!ok) break; TXT_TO_PTR("description", field, row->description); LIST_MEM_ADD(workmarkers_free, row->description); - PQ_GET_FLD(res, i, "status", field, ok); + CKPQ_VAL_FLD_num(res, i, status, field, ok); if (!ok) break; TXT_TO_STR("status", field, row->status); @@ -9256,14 +9572,16 @@ bool workmarkers_fill(PGconn *conn) } K_WUNLOCK(workmarkers_free); - PQclear(res); + CKPQClear(res); for (i = 0; i < par; i++) free(params[i]); par = 0; if (ok) { LOGDEBUG("%s(): built", __func__); - LOGWARNING("%s(): loaded %d workmarkers records", __func__, n); + pcom(n, pcombuf, sizeof(pcombuf)); + LOGWARNING("%s(): loaded %s workmarkers records", + __func__, pcombuf); } POOLINSTANCE_DBLOAD_MSG(workmarkers); @@ -9298,19 +9616,11 @@ bool _marks_process(PGconn *conn, bool add, char *poolinstance, LOGDEBUG("%s(): updating old", __func__); DATA_MARKS(oldmarks, old_m_item); - if (!conn) { - conn = dbconnect(); + if (CKPQConn(&conn)) conned = true; - } - res = PQexec(conn, "Begin", CKPQ_WRITE); - rescode = PQresultStatus(res); - PQclear(res); - if (!PGOK(rescode)) { - PGLOGERR("Begin", rescode, conn); + begun = CKPQBegin(conn); + if (!begun) goto unparam; - } - - begun = true; upd = "update marks set "EDDB"=$1 where workinfoid=$2" " and "EDDB"=$3"; @@ -9320,9 +9630,9 @@ bool _marks_process(PGconn *conn, bool add, char *poolinstance, params[par++] = tv_to_buf((tv_t *)&default_expiry, NULL, 0); PARCHKVAL(par, 3, params); - res = PQexecParams(conn, upd, par, NULL, (const char **)params, NULL, NULL, 0, CKPQ_WRITE); - rescode = PQresultStatus(res); - PQclear(res); + res = CKPQExecParams(conn, upd, par, NULL, (const char **)params, NULL, NULL, 0, CKPQ_WRITE); + rescode = CKPQResultStatus(res); + CKPQClear(res); if (!PGOK(rescode)) { PGLOGERR("Update", rescode, conn); goto rollback; @@ -9379,25 +9689,17 @@ bool _marks_process(PGconn *conn, bool add, char *poolinstance, HISTORYDATEPARAMS(params, par, row); PARCHK(par, params); - if (conn == NULL) { - conn = dbconnect(); + if (CKPQConn(&conn)) conned = true; - } - if (!begun) { - res = PQexec(conn, "Begin", CKPQ_WRITE); - rescode = PQresultStatus(res); - PQclear(res); - if (!PGOK(rescode)) { - PGLOGERR("Begin", rescode, conn); + begun = CKPQBegin(conn); + if (!begun) goto unparam; - } - begun = true; } - res = PQexecParams(conn, ins, par, NULL, (const char **)params, NULL, NULL, 0, CKPQ_WRITE); - rescode = PQresultStatus(res); - PQclear(res); + res = CKPQExecParams(conn, ins, par, NULL, (const char **)params, NULL, NULL, 0, CKPQ_WRITE); + rescode = CKPQResultStatus(res); + CKPQClear(res); if (!PGOK(rescode)) { PGLOGERR("Insert", rescode, conn); goto rollback; @@ -9406,21 +9708,14 @@ bool _marks_process(PGconn *conn, bool add, char *poolinstance, ok = true; rollback: - if (begun) { - if (ok) - res = PQexec(conn, "Commit", CKPQ_WRITE); - else - res = PQexec(conn, "Rollback", CKPQ_WRITE); + if (begun) + CKPQEnd(conn, ok); - PQclear(res); - } unparam: + CKPQDisco(&conn, conned); for (n = 0; n < par; n++) free(params[n]); - if (conned) - PQfinish(conn); - K_WLOCK(marks_free); if (!ok) { if (m_item) { @@ -9445,6 +9740,7 @@ unparam: bool marks_fill(PGconn *conn) { + char pcombuf[64]; ExecStatusType rescode; PGresult *res; K_ITEM *item; @@ -9457,16 +9753,20 @@ bool marks_fill(PGconn *conn) LOGDEBUG("%s(): select", __func__); + int poolinstance_num, workinfoid_num, description_num, extra_num; + int marktype_num, status_num; + HISTORYDATE_num; + // TODO: limit how far back sel = "select " "poolinstance,workinfoid,description,extra,marktype,status" HISTORYDATECONTROL " from marks"; - res = PQexec(conn, sel, CKPQ_READ); - rescode = PQresultStatus(res); + res = CKPQExec(conn, sel, CKPQ_READ); + rescode = CKPQResultStatus(res); if (!PGOK(rescode)) { PGLOGERR("Select", rescode, conn); - PQclear(res); + CKPQClear(res); return false; } @@ -9474,13 +9774,16 @@ bool marks_fill(PGconn *conn) if (n != (fields + HISTORYDATECOUNT)) { LOGERR("%s(): Invalid field count - should be %d, but is %d", __func__, fields + HISTORYDATECOUNT, n); - PQclear(res); + CKPQClear(res); return false; } n = PQntuples(res); LOGDEBUG("%s(): tree build count %d", __func__, n); ok = true; + poolinstance_num = workinfoid_num = description_num = extra_num = + marktype_num = status_num = CKPQFUNDEF; + HISTORYDATE_init; K_WLOCK(marks_free); for (i = 0; i < n; i++) { item = k_unlink_head(marks_free); @@ -9492,7 +9795,7 @@ bool marks_fill(PGconn *conn) break; } - PQ_GET_FLD(res, i, "poolinstance", field, ok); + CKPQ_VAL_FLD_num(res, i, poolinstance, field, ok); if (!ok) break; if (sys_poolinstance && strcmp(field, sys_poolinstance)) { @@ -9502,29 +9805,29 @@ bool marks_fill(PGconn *conn) } row->in_poolinstance = intransient_str("poolinstance", field); - PQ_GET_FLD(res, i, "workinfoid", field, ok); + CKPQ_VAL_FLD_num(res, i, workinfoid, field, ok); if (!ok) break; TXT_TO_BIGINT("workinfoid", field, row->workinfoid); - PQ_GET_FLD(res, i, "description", field, ok); + CKPQ_VAL_FLD_num(res, i, description, field, ok); if (!ok) break; TXT_TO_PTR("description", field, row->description); LIST_MEM_ADD(marks_free, row->description); - PQ_GET_FLD(res, i, "extra", field, ok); + CKPQ_VAL_FLD_num(res, i, extra, field, ok); if (!ok) break; TXT_TO_PTR("extra", field, row->extra); LIST_MEM_ADD(marks_free, row->extra); - PQ_GET_FLD(res, i, "marktype", field, ok); + CKPQ_VAL_FLD_num(res, i, marktype, field, ok); if (!ok) break; TXT_TO_STR("marktype", field, row->marktype); - PQ_GET_FLD(res, i, "status", field, ok); + CKPQ_VAL_FLD_num(res, i, status, field, ok); if (!ok) break; TXT_TO_STR("status", field, row->status); @@ -9544,11 +9847,13 @@ bool marks_fill(PGconn *conn) } K_WUNLOCK(marks_free); - PQclear(res); + CKPQClear(res); if (ok) { LOGDEBUG("%s(): built", __func__); - LOGWARNING("%s(): loaded %d marks records", __func__, n); + pcom(n, pcombuf, sizeof(pcombuf)); + LOGWARNING("%s(): loaded %s marks records", + __func__, pcombuf); } POOLINSTANCE_DBLOAD_MSG(marks); @@ -9564,16 +9869,16 @@ bool check_db_version(PGconn *conn) char *pgv; int fields = 3; bool ok; - int n; + int n, f; LOGDEBUG("%s(): select", __func__); sel = "select version() as pgv,* from version;"; - res = PQexec(conn, sel, CKPQ_READ); - rescode = PQresultStatus(res); + res = CKPQExec(conn, sel, CKPQ_READ); + rescode = CKPQResultStatus(res); if (!PGOK(rescode)) { PGLOGEMERG("Select", rescode, conn); - PQclear(res); + CKPQClear(res); return false; } @@ -9581,7 +9886,7 @@ bool check_db_version(PGconn *conn) if (n != fields) { LOGEMERG("%s(): Invalid field count - should be %d, but is %d", __func__, fields, n); - PQclear(res); + CKPQClear(res); return false; } @@ -9589,47 +9894,51 @@ bool check_db_version(PGconn *conn) if (n != 1) { LOGEMERG("%s(): Invalid record count - should be %d, but is %d", __func__, 1, n); - PQclear(res); + CKPQClear(res); return false; } ok = true; - PQ_GET_FLD(res, 0, "vlock", field, ok); + f = CKPQFUNDEF; + CKPQ_VAL_FLD(res, 0, f, "vlock", field, ok); if (!ok) { LOGEMERG("%s(): Missing field vlock", __func__); - PQclear(res); + CKPQClear(res); return false; } if (strcmp(field, DB_VLOCK)) { LOGEMERG("%s(): incorrect vlock '%s' - should be '%s'", __func__, field, DB_VLOCK); - PQclear(res); + CKPQClear(res); return false; } ok = true; - PQ_GET_FLD(res, 0, "version", field, ok); + f = CKPQFUNDEF; + CKPQ_VAL_FLD(res, 0, f, "version", field, ok); if (!ok) { LOGEMERG("%s(): Missing field version", __func__); - PQclear(res); + CKPQClear(res); return false; } if (strcmp(field, DB_VERSION)) { LOGEMERG("%s(): incorrect version '%s' - should be '%s'", __func__, field, DB_VERSION); - PQclear(res); + CKPQClear(res); return false; } - PQ_GET_FLD(res, 0, "pgv", field, ok); + ok = true; + f = CKPQFUNDEF; + CKPQ_VAL_FLD(res, 0, f, "pgv", field, ok); if (ok) pgv = strdup(field); else pgv = strdup("Failed to get postgresql version information"); - PQclear(res); + CKPQClear(res); LOGWARNING("%s(): DB version (%s) correct (CKDB V%s)", __func__, DB_VERSION, CKDB_VERSION); @@ -9639,82 +9948,3 @@ bool check_db_version(PGconn *conn) return true; } - -char *cmd_newid(PGconn *conn, char *cmd, char *id, tv_t *now, char *by, - char *code, char *inet, __maybe_unused tv_t *cd, - K_TREE *trf_root) -{ - char reply[1024] = ""; - size_t siz = sizeof(reply); - K_ITEM *i_idname, *i_idvalue, *look; - IDCONTROL *row; - char *params[2 + MODIFYDATECOUNT]; - int n, par = 0; - bool ok = false; - ExecStatusType rescode; - bool conned = false; - PGresult *res; - char *ins; - - LOGDEBUG("%s(): cmd '%s'", __func__, cmd); - - i_idname = require_name(trf_root, "idname", 3, (char *)idpatt, reply, siz); - if (!i_idname) - return strdup(reply); - - i_idvalue = require_name(trf_root, "idvalue", 1, (char *)intpatt, reply, siz); - if (!i_idvalue) - return strdup(reply); - - K_WLOCK(idcontrol_free); - look = k_unlink_head(idcontrol_free); - K_WUNLOCK(idcontrol_free); - - DATA_IDCONTROL(row, look); - - STRNCPY(row->idname, transfer_data(i_idname)); - TXT_TO_BIGINT("idvalue", transfer_data(i_idvalue), row->lastid); - MODIFYDATEINIT(row, now, by, code, inet); - - par = 0; - params[par++] = str_to_buf(row->idname, NULL, 0); - params[par++] = bigint_to_buf(row->lastid, NULL, 0); - MODIFYDATEPARAMS(params, par, row); - PARCHK(par, params); - - ins = "insert into idcontrol " - "(idname,lastid" MODIFYDATECONTROL ") values (" PQPARAM10 ")"; - - if (!conn) { - conn = dbconnect(); - conned = true; - } - - res = PQexecParams(conn, ins, par, NULL, (const char **)params, NULL, NULL, 0, CKPQ_WRITE); - rescode = PQresultStatus(res); - if (!PGOK(rescode)) { - PGLOGERR("Insert", rescode, conn); - goto foil; - } - - ok = true; -foil: - PQclear(res); - if (conned) - PQfinish(conn); - for (n = 0; n < par; n++) - free(params[n]); - - K_WLOCK(idcontrol_free); - k_add_head(idcontrol_free, look); - K_WUNLOCK(idcontrol_free); - - if (!ok) { - LOGERR("%s() %s.failed.DBE", __func__, id); - return strdup("failed.DBE"); - } - LOGDEBUG("%s.ok.added %s %"PRId64, id, transfer_data(i_idname), row->lastid); - snprintf(reply, siz, "ok.added %s %"PRId64, - transfer_data(i_idname), row->lastid); - return strdup(reply); -} diff --git a/src/klist.c b/src/klist.c index 1741489e..105b7b4e 100644 --- a/src/klist.c +++ b/src/klist.c @@ -60,6 +60,63 @@ K_LISTS *all_klists; #define CHKITEM(__item, __list) _CHKITEM(__item, __list, "item") +void _dsp_kstore(K_STORE *store, char *filename, char *msg, KLIST_FFL_ARGS) +{ + K_ITEM *item; + FILE *stream; + struct tm tm; + time_t now_t; + char stamp[128]; + + if (!(store->master->dsp_func)) { + quithere(1, "List %s has no dsp_func" KLIST_FFL, + store->master->name, KLIST_FFL_PASS); + } + + now_t = time(NULL); + localtime_r(&now_t, &tm); + snprintf(stamp, sizeof(stamp), + "[%d-%02d-%02d %02d:%02d:%02d]", + tm.tm_year + 1900, + tm.tm_mon + 1, + tm.tm_mday, + tm.tm_hour, + tm.tm_min, + tm.tm_sec); + + stream = fopen(filename, "ae"); + if (!stream) + { + fprintf(stderr, "%s %s() failed to open '%s' (%d) %s", + stamp, __func__, filename, errno, strerror(errno)); + return; + } + + if (msg) + fprintf(stream, "%s %s\n", stamp, msg); + else + fprintf(stream, "%s Dump of store '%s':\n", stamp, store->master->name); + + if (store->count > 0) + { + K_RLOCK(store->master); + + item = store->head; + while (item) + { + store->master->dsp_func(item, stream); + item = item->next; + } + K_RUNLOCK(store->master); + + fprintf(stream, "End\n\n"); + } + else + fprintf(stream, "Empty kstore\n\n"); + + fclose(stream); +} + static void k_alloc_items(K_LIST *list, KLIST_FFL_ARGS) { K_ITEM *item; @@ -134,7 +191,7 @@ static void k_alloc_items(K_LIST *list, KLIST_FFL_ARGS) list->count_up = allocate; } -K_STORE *_k_new_store(K_LIST *list, KLIST_FFL_ARGS) +K_STORE *_k_new_store(K_LIST *list, bool gotlock, KLIST_FFL_ARGS) { K_STORE *store; @@ -149,14 +206,31 @@ K_STORE *_k_new_store(K_LIST *list, KLIST_FFL_ARGS) store->lock = NULL; store->name = list->name; store->do_tail = list->do_tail; - list->stores++; + store->prev_store = NULL; + // Only tracked for lists with a lock + if (store->master->lock == NULL) { + store->next_store = NULL; + store->master->stores++; + } else { + if (!gotlock) + K_WLOCK(list); + // In the master list, next is the head + if (list->next_store) + list->next_store->prev_store = store; + store->next_store = list->next_store; + list->next_store = store; + list->stores++; + if (!gotlock) + K_WUNLOCK(list); + } return store; } K_LIST *_k_new_list(const char *name, size_t siz, int allocate, int limit, bool do_tail, bool lock_only, bool without_lock, - bool local_list, const char *name2, KLIST_FFL_ARGS) + bool local_list, const char *name2, int cull_limit, + KLIST_FFL_ARGS) { K_LIST *list; @@ -166,6 +240,11 @@ K_LIST *_k_new_list(const char *name, size_t siz, int allocate, int limit, if (limit < 0) quithere(1, "Invalid new list %s with limit %d must be >= 0", name, limit); + /* after culling, the first block of items are again allocated, + * so there's no point culling a single block of items */ + if (cull_limit > 0 && cull_limit <= allocate) + quithere(1, "Invalid new list %s with cull_limit %d must be > allocate (%d)", name, cull_limit, allocate); + list = calloc(1, sizeof(*list)); if (!list) quithere(1, "Failed to calloc list %s", name); @@ -191,6 +270,8 @@ K_LIST *_k_new_list(const char *name, size_t siz, int allocate, int limit, list->allocate = allocate; list->limit = limit; list->do_tail = do_tail; + list->cull_limit = cull_limit; + list->next_store = list->prev_store = NULL; if (!(list->is_lock_only)) k_alloc_items(list, KLIST_FFL_PASS); @@ -303,6 +384,58 @@ K_ITEM *_k_unlink_tail(K_LIST *list, LOCK_MAYBE bool chklock, KLIST_FFL_ARGS) return item; } +#define CHKCULL(_list) \ + do { \ + if (!((_list)->is_store) && !((_list)->is_lock_only) && \ + (_list)->cull_limit > 0 && \ + (_list)->count == (_list)->total && \ + (_list)->total >= (_list)->cull_limit) { \ + k_cull_list(_list, file, func, line); \ + } \ + } while(0); + +static void k_cull_list(K_LIST *list, KLIST_FFL_ARGS) +{ + int i; + + CHKLIST(list); + _LIST_WRITE(list, true, file, func, line); + + if (list->is_store) { + quithere(1, "List %s can't %s() a store" KLIST_FFL, + list->name, __func__, KLIST_FFL_PASS); + } + + if (list->is_lock_only) { + quithere(1, "List %s can't %s() a lock_only" KLIST_FFL, + list->name, __func__, KLIST_FFL_PASS); + } + + if (list->count != list->total) { + quithere(1, "List %s can't %s() a list in use" KLIST_FFL, + list->name, __func__, KLIST_FFL_PASS); + } + + for (i = 0; i < list->item_mem_count; i++) + free(list->item_memory[i]); + free(list->item_memory); + list->item_memory = NULL; + list->item_mem_count = 0; + + for (i = 0; i < list->data_mem_count; i++) + free(list->data_memory[i]); + free(list->data_memory); + list->data_memory = NULL; + list->data_mem_count = 0; + + list->total = list->count = list->count_up = 0; + list->head = list->tail = NULL; + + list->cull_count++; + + k_alloc_items(list, KLIST_FFL_PASS); +} + void _k_add_head(K_LIST *list, K_ITEM *item, LOCK_MAYBE bool chklock, KLIST_FFL_ARGS) { CHKLS(list); @@ -333,6 +466,8 @@ void _k_add_head(K_LIST *list, K_ITEM *item, LOCK_MAYBE bool chklock, KLIST_FFL_ list->count++; list->count_up++; + + CHKCULL(list); } /* slows it down (of course) - only for debugging @@ -380,6 +515,8 @@ void _k_add_tail(K_LIST *list, K_ITEM *item, LOCK_MAYBE bool chklock, KLIST_FFL_ list->count++; list->count_up++; + + CHKCULL(list); } // Insert item into the list next after 'after' @@ -418,6 +555,8 @@ void _k_insert_after(K_LIST *list, K_ITEM *item, K_ITEM *after, LOCK_MAYBE bool list->count++; list->count_up++; + + // no point checking cull since this wouldn't be an _free list } void _k_unlink_item(K_LIST *list, K_ITEM *item, LOCK_MAYBE bool chklock, KLIST_FFL_ARGS) @@ -484,6 +623,8 @@ void _k_list_transfer_to_head(K_LIST *from, K_LIST *to, LOCK_MAYBE bool chklock, from->count = 0; to->count_up += from->count_up; from->count_up = 0; + + CHKCULL(to); } void _k_list_transfer_to_tail(K_LIST *from, K_LIST *to, LOCK_MAYBE bool chklock, KLIST_FFL_ARGS) @@ -520,6 +661,8 @@ void _k_list_transfer_to_tail(K_LIST *from, K_LIST *to, LOCK_MAYBE bool chklock, from->count = 0; to->count_up += from->count_up; from->count_up = 0; + + CHKCULL(to); } K_LIST *_k_free_list(K_LIST *list, KLIST_FFL_ARGS) @@ -592,47 +735,23 @@ K_STORE *_k_free_store(K_STORE *store, KLIST_FFL_ARGS) store->name, __func__, KLIST_FFL_PASS); } - store->master->stores--; + if (store->master->lock == NULL) + store->master->stores--; + else { + K_WLOCK(store->master); + // unlink store from the list + if (store->prev_store) + store->prev_store->next_store = store->next_store; + if (store->next_store) + store->next_store->prev_store = store->prev_store; + // correct the head if we are the head + if (store->master->next_store == store) + store->master->next_store = store->next_store; + store->master->stores--; + K_WUNLOCK(store->master); + } free(store); return NULL; } - -// Must be locked and none in use and/or unlinked -void _k_cull_list(K_LIST *list, LOCK_MAYBE bool chklock, KLIST_FFL_ARGS) -{ - int i; - - CHKLIST(list); - _LIST_WRITE(list, chklock, file, func, line); - - if (list->is_store) { - quithere(1, "List %s can't %s() a store" KLIST_FFL, - list->name, __func__, KLIST_FFL_PASS); - } - - if (list->count != list->total) { - quithere(1, "List %s can't %s() a list in use" KLIST_FFL, - list->name, __func__, KLIST_FFL_PASS); - } - - for (i = 0; i < list->item_mem_count; i++) - free(list->item_memory[i]); - free(list->item_memory); - list->item_memory = NULL; - list->item_mem_count = 0; - - for (i = 0; i < list->data_mem_count; i++) - free(list->data_memory[i]); - free(list->data_memory); - list->data_memory = NULL; - list->data_mem_count = 0; - - list->total = list->count = list->count_up = 0; - list->head = list->tail = NULL; - - list->cull_count++; - - k_alloc_items(list, KLIST_FFL_PASS); -} diff --git a/src/klist.h b/src/klist.h index d40b199b..2d9ed190 100644 --- a/src/klist.h +++ b/src/klist.h @@ -151,8 +151,11 @@ typedef struct k_list { int data_mem_count; // how many item data memory buffers have been allocated void **data_memory; // allocated item data memory buffers void (*dsp_func)(K_ITEM *, FILE *); // optional data display to a file - int cull_count; + int cull_limit; // <1 means don't cull, otherwise total to cull at + int cull_count; // number of times culled uint64_t ram; // ram allocated for data pointers - code must manage it + struct k_list *next_store; // list of all stores - the head is next_store in the list master + struct k_list *prev_store; // the stores themselves have their prev and next int stores; // how many stores it currently has #if LOCK_CHECK // Since each thread has it's own k_lock no locking is required on this @@ -663,18 +666,23 @@ static inline K_ITEM *list_rtail(K_LIST *list) #define STORE_HEAD_NOLOCK(_s) LIST_HEAD_NOLOCK(_s) #define STORE_TAIL_NOLOCK(_s) LIST_TAIL_NOLOCK(_s) -extern K_STORE *_k_new_store(K_LIST *list, KLIST_FFL_ARGS); -#define k_new_store(_list) _k_new_store(_list, KLIST_FFL_HERE) +extern void _dsp_kstore(K_STORE *store, char *filename, char *msg, KLIST_FFL_ARGS); +#define dsp_kstore(_store, _file, _msg) _dsp_kstore(_store, _file, _msg, KLIST_FFL_HERE) +extern K_STORE *_k_new_store(K_LIST *list, bool gotlock, KLIST_FFL_ARGS); +#define k_new_store(_list) _k_new_store(_list, false, KLIST_FFL_HERE) +#define k_new_store_locked(_list) _k_new_store(_list, true, KLIST_FFL_HERE) extern K_LIST *_k_new_list(const char *name, size_t siz, int allocate, int limit, bool do_tail, bool lock_only, bool without_lock, bool local_list, - const char *name2, KLIST_FFL_ARGS); + const char *name2, int cull_limit, KLIST_FFL_ARGS); #define k_new_list(_name, _siz, _allocate, _limit, _do_tail) \ - _k_new_list(_name, _siz, _allocate, _limit, _do_tail, false, false, false, NULL, KLIST_FFL_HERE) + _k_new_list(_name, _siz, _allocate, _limit, _do_tail, false, false, false, NULL, 0, KLIST_FFL_HERE) #define k_lock_only_list(_name) \ - _k_new_list(_name, 1, 1, 1, true, true, false, false, NULL, KLIST_FFL_HERE) + _k_new_list(_name, 1, 1, 1, true, true, false, false, NULL, 0, KLIST_FFL_HERE) #define k_new_tree_list(_name, _siz, _allocate, _limit, _do_tail, _local_tree, _name2) \ - _k_new_list(_name, _siz, _allocate, _limit, _do_tail, false, true, _local_tree, _name2, KLIST_FFL_HERE) + _k_new_list(_name, _siz, _allocate, _limit, _do_tail, false, true, _local_tree, _name2, 0, KLIST_FFL_HERE) +#define k_new_list_cull(_name, _siz, _allocate, _limit, _do_tail, _cull) \ + _k_new_list(_name, _siz, _allocate, _limit, _do_tail, false, false, false, NULL, _cull, KLIST_FFL_HERE) extern K_ITEM *_k_unlink_head(K_LIST *list, LOCK_MAYBE bool chklock, KLIST_FFL_ARGS); #define k_unlink_head(_list) _k_unlink_head(_list, true, KLIST_FFL_HERE) #define k_unlink_head_nolock(_list) _k_unlink_head(_list, false, KLIST_FFL_HERE) @@ -709,8 +717,5 @@ extern K_LIST *_k_free_list(K_LIST *list, KLIST_FFL_ARGS); #define k_free_list(_list) _k_free_list(_list, KLIST_FFL_HERE) extern K_STORE *_k_free_store(K_STORE *store, KLIST_FFL_ARGS); #define k_free_store(_store) _k_free_store(_store, KLIST_FFL_HERE) -extern void _k_cull_list(K_LIST *list, LOCK_MAYBE bool chklock, KLIST_FFL_ARGS); -#define k_cull_list(_list) _k_cull_list(_list, true, KLIST_FFL_HERE) -//#define k_cull_list_nolock(_list) _k_cull_list(_list, false, KLIST_FFL_HERE) #endif diff --git a/src/ktree.c b/src/ktree.c index 78b2ef98..5675a5ae 100644 --- a/src/ktree.c +++ b/src/ktree.c @@ -182,8 +182,6 @@ void _dsp_ktree(K_TREE *tree, char *filename, char *msg, KTREE_FFL_ARGS) if (!(tree->master->dsp_func)) FAIL("NULLDSP NULL dsp_func in %s", tree->master->name); - _TREE_READ(tree, true, file, func, line); - now_t = time(NULL); localtime_r(&now_t, &tm); snprintf(stamp, sizeof(stamp), @@ -210,12 +208,16 @@ void _dsp_ktree(K_TREE *tree, char *filename, char *msg, KTREE_FFL_ARGS) if (tree->root->isNil == No) { + K_RLOCK(tree->master); + item = first_in_ktree(tree, ctx); while (item) { tree->master->dsp_func(item, stream); item = next_in_ktree(ctx); } + K_RUNLOCK(tree->master); + fprintf(stream, "End\n\n"); } else