From c7f03c2a331f111696f0e8163f295461711e8eb5 Mon Sep 17 00:00:00 2001 From: kanoi Date: Sat, 5 Dec 2015 19:24:40 +1100 Subject: [PATCH] ckdb - add lock checking and deadlock prediction --- src/ckdb.c | 321 ++++++++++++++++++--------- src/ckdb.h | 81 +++---- src/ckdb_cmd.c | 155 +++++++++---- src/ckdb_data.c | 431 +++++++++++++++++------------------- src/ckdb_dbio.c | 195 +++++++++++------ src/klist.c | 131 +++++++++-- src/klist.h | 569 +++++++++++++++++++++++++++++++++++++++++++++--- src/ktree.c | 47 +++- src/ktree.h | 36 ++- 9 files changed, 1411 insertions(+), 555 deletions(-) diff --git a/src/ckdb.c b/src/ckdb.c index 44b6f2cc..6838b0f9 100644 --- a/src/ckdb.c +++ b/src/ckdb.c @@ -115,6 +115,10 @@ static bool logger_using_data; static bool listener_using_data; char *EMPTY = ""; +const char *nullstr = "(null)"; + +const char *true_str = "true"; +const char *false_str = "false"; static char *db_name; static char *db_user; @@ -312,9 +316,14 @@ K_LIST *transfer_free; // SEQSET K_LIST *seqset_free; -K_STORE *seqset_store; -char *seqnam[SEQ_MAX]; -static cklock_t seq_lock; +// each new seqset is added to the head, so head is the current one +static K_STORE *seqset_store; +// Initialised when seqset_free is allocated +static char *seqnam[SEQ_MAX]; + +// Full lock for access to sequence processing data +#define SEQLOCK() K_WLOCK(seqset_free); +#define SEQUNLOCK() K_WUNLOCK(seqset_free); // SEQTRANS K_LIST *seqtrans_free; @@ -334,6 +343,8 @@ K_STORE *useratts_store; K_TREE *workers_root; K_LIST *workers_free; K_STORE *workers_store; +// Emulate a list for lock checking +K_LIST *workers_db_free; // PAYMENTADDRESSES K_TREE *paymentaddresses_root; @@ -416,6 +427,7 @@ const char *blocks_unknown = "?Unknown?"; K_TREE *blocks_root; K_LIST *blocks_free; K_STORE *blocks_store; +// Access both under blocks_free lock tv_t blocks_stats_time; bool blocks_stats_rebuild = true; @@ -430,7 +442,8 @@ K_TREE *payouts_id_root; K_TREE *payouts_wid_root; K_LIST *payouts_free; K_STORE *payouts_store; -cklock_t process_pplns_lock; +// Emulate a list for lock checking +K_LIST *process_pplns_free; /* // EVENTLOG @@ -997,7 +1010,8 @@ static void alloc_storage() ALLOC_WORKQUEUE, LIMIT_WORKQUEUE, true); workqueue_store = k_new_store(workqueue_free); - heartbeatqueue_free = k_new_list("HeartBeatQueue", sizeof(HEARTBEATQUEUE), + heartbeatqueue_free = k_new_list("HeartBeatQueue", + sizeof(HEARTBEATQUEUE), ALLOC_HEARTBEATQUEUE, LIMIT_HEARTBEATQUEUE, true); heartbeatqueue_store = k_new_store(heartbeatqueue_free); @@ -1009,43 +1023,48 @@ static void alloc_storage() users_free = k_new_list("Users", sizeof(USERS), ALLOC_USERS, LIMIT_USERS, true); users_store = k_new_store(users_free); - users_root = new_ktree(cmp_users); - userid_root = new_ktree(cmp_userid); + users_root = new_ktree(cmp_users, users_free); + userid_root = new_ktree(cmp_userid, users_free); useratts_free = k_new_list("Useratts", sizeof(USERATTS), ALLOC_USERATTS, LIMIT_USERATTS, true); useratts_store = k_new_store(useratts_free); - useratts_root = new_ktree(cmp_useratts); + useratts_root = new_ktree(cmp_useratts, useratts_free); optioncontrol_free = k_new_list("OptionControl", sizeof(OPTIONCONTROL), ALLOC_OPTIONCONTROL, LIMIT_OPTIONCONTROL, true); optioncontrol_store = k_new_store(optioncontrol_free); - optioncontrol_root = new_ktree(cmp_optioncontrol); + optioncontrol_root = new_ktree(cmp_optioncontrol, optioncontrol_free); workers_free = k_new_list("Workers", sizeof(WORKERS), ALLOC_WORKERS, LIMIT_WORKERS, true); workers_store = k_new_store(workers_free); - workers_root = new_ktree(cmp_workers); + workers_root = new_ktree(cmp_workers, workers_free); paymentaddresses_free = k_new_list("PaymentAddresses", sizeof(PAYMENTADDRESSES), ALLOC_PAYMENTADDRESSES, LIMIT_PAYMENTADDRESSES, true); paymentaddresses_store = k_new_store(paymentaddresses_free); - paymentaddresses_root = new_ktree(cmp_paymentaddresses); - paymentaddresses_create_root = new_ktree(cmp_payaddr_create); + paymentaddresses_root = new_ktree(cmp_paymentaddresses, + paymentaddresses_free); + paymentaddresses_create_root = new_ktree(cmp_payaddr_create, + paymentaddresses_free); paymentaddresses_free->dsp_func = dsp_paymentaddresses; payments_free = k_new_list("Payments", sizeof(PAYMENTS), ALLOC_PAYMENTS, LIMIT_PAYMENTS, true); payments_store = k_new_store(payments_free); - payments_root = new_ktree(cmp_payments); + payments_root = new_ktree(cmp_payments, payments_free); - accountbalance_free = k_new_list("AccountBalance", sizeof(ACCOUNTBALANCE), - ALLOC_ACCOUNTBALANCE, LIMIT_ACCOUNTBALANCE, true); + accountbalance_free = k_new_list("AccountBalance", + sizeof(ACCOUNTBALANCE), + ALLOC_ACCOUNTBALANCE, + LIMIT_ACCOUNTBALANCE, true); accountbalance_store = k_new_store(accountbalance_free); - accountbalance_root = new_ktree(cmp_accountbalance); + accountbalance_root = new_ktree(cmp_accountbalance, + accountbalance_free); idcontrol_free = k_new_list("IDControl", sizeof(IDCONTROL), ALLOC_IDCONTROL, LIMIT_IDCONTROL, true); @@ -1054,98 +1073,157 @@ static void alloc_storage() workinfo_free = k_new_list("WorkInfo", sizeof(WORKINFO), ALLOC_WORKINFO, LIMIT_WORKINFO, true); workinfo_store = k_new_store(workinfo_free); - workinfo_root = new_ktree(cmp_workinfo); - if (!confirm_sharesummary) - workinfo_height_root = new_ktree(cmp_workinfo_height); + workinfo_root = new_ktree(cmp_workinfo, workinfo_free); + if (!confirm_sharesummary) { + workinfo_height_root = new_ktree(cmp_workinfo_height, + workinfo_free); + } shares_free = k_new_list("Shares", sizeof(SHARES), ALLOC_SHARES, LIMIT_SHARES, true); shares_store = k_new_store(shares_free); shares_early_store = k_new_store(shares_free); - shares_root = new_ktree(cmp_shares); - shares_early_root = new_ktree(cmp_shares); + shares_root = new_ktree(cmp_shares, shares_free); + shares_early_root = new_ktree(cmp_shares, shares_free); shareerrors_free = k_new_list("ShareErrors", sizeof(SHAREERRORS), ALLOC_SHAREERRORS, LIMIT_SHAREERRORS, true); shareerrors_store = k_new_store(shareerrors_free); shareerrors_early_store = k_new_store(shareerrors_free); - shareerrors_root = new_ktree(cmp_shareerrors); - shareerrors_early_root = new_ktree(cmp_shareerrors); + shareerrors_root = new_ktree(cmp_shareerrors, shareerrors_free); + shareerrors_early_root = new_ktree(cmp_shareerrors, shareerrors_free); sharesummary_free = k_new_list("ShareSummary", sizeof(SHARESUMMARY), ALLOC_SHARESUMMARY, LIMIT_SHARESUMMARY, true); sharesummary_store = k_new_store(sharesummary_free); - sharesummary_root = new_ktree(cmp_sharesummary); - sharesummary_workinfoid_root = new_ktree(cmp_sharesummary_workinfoid); + sharesummary_root = new_ktree(cmp_sharesummary, sharesummary_free); + sharesummary_workinfoid_root = new_ktree(cmp_sharesummary_workinfoid, + sharesummary_free); sharesummary_free->dsp_func = dsp_sharesummary; sharesummary_pool_store = k_new_store(sharesummary_free); - sharesummary_pool_root = new_ktree(cmp_sharesummary); + sharesummary_pool_root = new_ktree(cmp_sharesummary, sharesummary_free); blocks_free = k_new_list("Blocks", sizeof(BLOCKS), ALLOC_BLOCKS, LIMIT_BLOCKS, true); blocks_store = k_new_store(blocks_free); - blocks_root = new_ktree(cmp_blocks); + blocks_root = new_ktree(cmp_blocks, blocks_free); blocks_free->dsp_func = dsp_blocks; miningpayouts_free = k_new_list("MiningPayouts", sizeof(MININGPAYOUTS), ALLOC_MININGPAYOUTS, LIMIT_MININGPAYOUTS, true); miningpayouts_store = k_new_store(miningpayouts_free); - miningpayouts_root = new_ktree(cmp_miningpayouts); + miningpayouts_root = new_ktree(cmp_miningpayouts, miningpayouts_free); payouts_free = k_new_list("Payouts", sizeof(PAYOUTS), ALLOC_PAYOUTS, LIMIT_PAYOUTS, true); payouts_store = k_new_store(payouts_free); - payouts_root = new_ktree(cmp_payouts); - payouts_id_root = new_ktree(cmp_payouts_id); - payouts_wid_root = new_ktree(cmp_payouts_wid); + payouts_root = new_ktree(cmp_payouts, payouts_free); + payouts_id_root = new_ktree(cmp_payouts_id, payouts_free); + payouts_wid_root = new_ktree(cmp_payouts_wid, payouts_free); auths_free = k_new_list("Auths", sizeof(AUTHS), ALLOC_AUTHS, LIMIT_AUTHS, true); auths_store = k_new_store(auths_free); - auths_root = new_ktree(cmp_auths); + auths_root = new_ktree(cmp_auths, auths_free); poolstats_free = k_new_list("PoolStats", sizeof(POOLSTATS), ALLOC_POOLSTATS, LIMIT_POOLSTATS, true); poolstats_store = k_new_store(poolstats_free); - poolstats_root = new_ktree(cmp_poolstats); + poolstats_root = new_ktree(cmp_poolstats, poolstats_free); userstats_free = k_new_list("UserStats", sizeof(USERSTATS), ALLOC_USERSTATS, LIMIT_USERSTATS, true); userstats_store = k_new_store(userstats_free); userstats_eos_store = k_new_store(userstats_free); - userstats_root = new_ktree(cmp_userstats); + userstats_root = new_ktree(cmp_userstats, userstats_free); userstats_free->dsp_func = dsp_userstats; workerstatus_free = k_new_list("WorkerStatus", sizeof(WORKERSTATUS), ALLOC_WORKERSTATUS, LIMIT_WORKERSTATUS, true); workerstatus_store = k_new_store(workerstatus_free); - workerstatus_root = new_ktree(cmp_workerstatus); + workerstatus_root = new_ktree(cmp_workerstatus, workerstatus_free); markersummary_free = k_new_list("MarkerSummary", sizeof(MARKERSUMMARY), ALLOC_MARKERSUMMARY, LIMIT_MARKERSUMMARY, true); markersummary_store = k_new_store(markersummary_free); - markersummary_root = new_ktree(cmp_markersummary); - markersummary_userid_root = new_ktree(cmp_markersummary_userid); + markersummary_root = new_ktree(cmp_markersummary, markersummary_free); + markersummary_userid_root = new_ktree(cmp_markersummary_userid, + markersummary_free); markersummary_free->dsp_func = dsp_markersummary; markersummary_pool_store = k_new_store(markersummary_free); - markersummary_pool_root = new_ktree(cmp_markersummary); + markersummary_pool_root = new_ktree(cmp_markersummary, + markersummary_free); workmarkers_free = k_new_list("WorkMarkers", sizeof(WORKMARKERS), ALLOC_WORKMARKERS, LIMIT_WORKMARKERS, true); workmarkers_store = k_new_store(workmarkers_free); - workmarkers_root = new_ktree(cmp_workmarkers); - workmarkers_workinfoid_root = new_ktree(cmp_workmarkers_workinfoid); + workmarkers_root = new_ktree(cmp_workmarkers, workmarkers_free); + workmarkers_workinfoid_root = new_ktree(cmp_workmarkers_workinfoid, + workmarkers_free); workmarkers_free->dsp_func = dsp_workmarkers; marks_free = k_new_list("Marks", sizeof(MARKS), ALLOC_MARKS, LIMIT_MARKS, true); marks_store = k_new_store(marks_free); - marks_root = new_ktree(cmp_marks); + marks_root = new_ktree(cmp_marks, marks_free); userinfo_free = k_new_list("UserInfo", sizeof(USERINFO), ALLOC_USERINFO, LIMIT_USERINFO, true); userinfo_store = k_new_store(userinfo_free); - userinfo_root = new_ktree(cmp_userinfo); + userinfo_root = new_ktree(cmp_userinfo, userinfo_free); + +#if LOCK_CHECK + DLPRIO(seqset, 91); + + DLPRIO(transfer, 90); + + DLPRIO(payouts, 87); + DLPRIO(miningpayouts, 86); + DLPRIO(payments, 85); + + DLPRIO(accountbalance, 80); + + DLPRIO(workerstatus, 69); + DLPRIO(sharesummary, 68); + DLPRIO(markersummary, 67); + DLPRIO(workmarkers, 66); + + DLPRIO(marks, 60); + + DLPRIO(workinfo, 56); + + DLPRIO(blocks, 53); + + DLPRIO(userinfo, 50); + + DLPRIO(auths, 44); + DLPRIO(users, 43); + DLPRIO(useratts, 42); + + DLPRIO(shares, 31); + DLPRIO(shareerrors, 30); + + DLPRIO(seqset, 21); + DLPRIO(seqtrans, 20); + + DLPRIO(msgline, 17); + DLPRIO(workqueue, 16); + DLPRIO(heartbeatqueue, 15); + + DLPRIO(poolstats, 11); + DLPRIO(userstats, 10); + + // Don't currently nest any locks in these: + DLPRIO(workers, PRIO_TERMINAL); + DLPRIO(idcontrol, PRIO_TERMINAL); + DLPRIO(optioncontrol, PRIO_TERMINAL); + DLPRIO(paymentaddresses, PRIO_TERMINAL); + + DLPCHECK(); + + if (auto_check_deadlocks) + check_deadlocks = true; +#endif } #define SEQSETMSG(_set, _seqset, _msgtxt, _endtxt) do { \ @@ -1201,7 +1279,7 @@ static void alloc_storage() #define FREE_STORE_DATA(_list) \ if (_list ## _store) { \ - K_ITEM *_item = _list ## _store->head; \ + K_ITEM *_item = STORE_HEAD_NOLOCK(_list ## _store); \ while (_item) { \ free_ ## _list ## _data(_item); \ _item = _item->next; \ @@ -1211,7 +1289,7 @@ static void alloc_storage() #define FREE_LIST_DATA(_list) \ if (_list ## _free) { \ - K_ITEM *_item = _list ## _free->head; \ + K_ITEM *_item = LIST_HEAD_NOLOCK(_list ## _free); \ while (_item) { \ free_ ## _list ## _data(_item); \ _item = _item->next; \ @@ -1240,9 +1318,12 @@ void sequence_report(bool lock) last = false; set = 0; - if (lock) - ck_wlock(&seq_lock); - ss_item = seqset_store->head; + if (lock) { + SEQLOCK(); + ss_item = STORE_RHEAD(seqset_store); + } else { + ss_item = STORE_HEAD_NOLOCK(seqset_store); + } while (!last && ss_item) { if (!ss_item->next) last = true; @@ -1256,19 +1337,19 @@ void sequence_report(bool lock) (seqset->seqdata[SEQ_SHARES].lost > 0); if (lock) { memcpy(&seqset_copy, seqset, sizeof(seqset_copy)); - ck_wunlock(&seq_lock); + SEQUNLOCK(); seqset = &seqset_copy; } SEQSETMSG(set, seqset, miss ? "SHARES MISSING" : "status" , EMPTY); if (lock) - ck_wlock(&seq_lock); + SEQLOCK(); } ss_item = ss_item->next; set++; } if (lock) - ck_wunlock(&seq_lock); + SEQUNLOCK(); } static void dealloc_storage() @@ -1305,7 +1386,8 @@ static void dealloc_storage() LOGWARNING("%s() markersummary ...", __func__); FREE_TREE(markersummary_pool); - k_list_transfer_to_tail(markersummary_pool_store, markersummary_store); + k_list_transfer_to_tail_nolock(markersummary_pool_store, + markersummary_store); FREE_STORE(markersummary_pool); FREE_TREE(markersummary_userid); FREE_TREE(markersummary); @@ -1337,7 +1419,8 @@ static void dealloc_storage() LOGWARNING("%s() sharesummary ...", __func__); FREE_TREE(sharesummary_pool); - k_list_transfer_to_tail(sharesummary_pool_store, sharesummary_store); + k_list_transfer_to_tail_nolock(sharesummary_pool_store, + sharesummary_store); FREE_STORE(sharesummary_pool); FREE_TREE(sharesummary_workinfoid); FREE_TREE(sharesummary); @@ -1347,7 +1430,7 @@ static void dealloc_storage() if (shareerrors_early_store->count > 0) { LOGERR("%s() *** shareerrors_early count %d ***", __func__, shareerrors_early_store->count); - s_item = shareerrors_early_store->head; + s_item = STORE_HEAD_NOLOCK(shareerrors_early_store); while (s_item) { DATA_SHAREERRORS(shareerrors, s_item); LOGERR("%s(): %"PRId64"/%s/%"PRId32"/%s/%ld,%ld", @@ -1368,7 +1451,7 @@ static void dealloc_storage() if (shares_early_store->count > 0) { LOGERR("%s() *** shares_early count %d ***", __func__, shares_early_store->count); - s_item = shares_early_store->head; + s_item = STORE_HEAD_NOLOCK(shares_early_store); while (s_item) { DATA_SHARES(shares, s_item); LOGERR("%s(): %"PRId64"/%s/%s/%"PRId32"/%ld,%ld", @@ -1507,26 +1590,29 @@ static bool setup_data() LOGWARNING("reload complete %.0fm %.3fs", min, sec); // full lock access since mark processing can occur - ck_wlock(&process_pplns_lock); + K_WLOCK(process_pplns_free); + K_WLOCK(workerstatus_free); K_RLOCK(sharesummary_free); - K_RLOCK(workmarkers_free); K_RLOCK(markersummary_free); + K_RLOCK(workmarkers_free); set_block_share_counters(); if (!everyone_die) workerstatus_ready(); - K_RUNLOCK(markersummary_free); K_RUNLOCK(workmarkers_free); + K_RUNLOCK(markersummary_free); K_RUNLOCK(sharesummary_free); K_WUNLOCK(workerstatus_free); - ck_wunlock(&process_pplns_lock); + + K_WUNLOCK(process_pplns_free); if (everyone_die) return false; + K_WLOCK(workinfo_free); workinfo_current = last_in_ktree(workinfo_height_root, ctx); if (workinfo_current) { DATA_WORKINFO(wic, workinfo_current); @@ -1543,6 +1629,7 @@ static bool setup_data() // No longer needed free_ktree(workinfo_height_root, NULL); } + K_WUNLOCK(workinfo_free); return true; } @@ -1621,7 +1708,7 @@ static tv_t last_trancheck; * and we don't run trans_process() during reloading * We also only know now, not cd, for a missing item * This fills in store with a copy of the details of all the new transients - * N.B. this is called under seq_lock */ + * N.B. this is called under SEQLOCK() */ static void trans_process(SEQSET *seqset, tv_t *now, K_STORE *store) { SEQDATA *seqdata = NULL; @@ -1707,11 +1794,11 @@ static void trans_seq(tv_t *now) store = k_new_store(seqtrans_free); for (i = 0; more; i++) { - ck_wlock(&seq_lock); + SEQLOCK(); if (seqset_store->count <= i) more = false; else { - item = seqset_store->head; + item = STORE_RHEAD(seqset_store); for (j = 0; item && j < 0; j++) item = item->next; if (!item) @@ -1734,9 +1821,9 @@ static void trans_seq(tv_t *now) } if (seqset_store->count <= (i + 1)) more = false; - ck_wunlock(&seq_lock); + SEQUNLOCK(); - st_item = store->tail; + st_item = STORE_TAIL_NOLOCK(store); while (st_item) { DATA_SEQTRANS(seqtrans, st_item); btu64_to_buf(&seqstt, t_buf, sizeof(t_buf)); @@ -1749,7 +1836,7 @@ static void trans_seq(tv_t *now) t_buf2, seqtrans->entry.code); st_item = st_item->prev; } - if (store->head) { + if (store->count) { K_WLOCK(seqtrans_free); k_list_transfer_to_head(store, seqtrans_free); if (seqtrans_free->count == seqtrans_free->total && @@ -1770,9 +1857,9 @@ static void seq_reloadmax() SEQDATA *seqdata; int i; - ck_wlock(&seq_lock); + SEQLOCK(); if (seqset_store->count > 0) { - seqset_item = seqset_store->head; + seqset_item = STORE_WHEAD(seqset_store); while (seqset_item) { DATA_SEQSET(seqset, seqset_item); if (seqset->seqstt) { @@ -1786,7 +1873,7 @@ static void seq_reloadmax() seqset_item = seqset_item->next; } } - ck_wunlock(&seq_lock); + SEQUNLOCK(); } /* Most of the extra message logic in here is to avoid putting too many @@ -1823,13 +1910,13 @@ static bool update_seq(enum seq_num seq, uint64_t n_seqcmd, firstseq = newseq = expseq = gothigh = okhi = gotstale = gotstalestart = dup = wastrans = gotrecover = false; - ck_wlock(&seq_lock); + SEQLOCK(); // Get the seqset if (seqset_store->count == 0) firstseq = true; else { // Normal processing is: count=1 and head is current - seqset_item = seqset_store->head; + seqset_item = STORE_WHEAD(seqset_store); DATA_SEQSET(seqset, seqset_item); set = 0; if (n_seqstt == seqset->seqstt && n_seqpid == seqset->seqpid) @@ -1848,8 +1935,8 @@ static bool update_seq(enum seq_num seq, uint64_t n_seqcmd, // Need to setup a new seqset newseq = true; if (!firstseq) { - // If !seqset_store->head (i.e. a bug) this will quit() - DATA_SEQSET(seqset0, seqset_store->head); + // If !STORE_WHEAD(seqset_store) (i.e. a bug) this will quit() + DATA_SEQSET(seqset0, STORE_WHEAD(seqset_store)); // The current seqset (may become the previous) memcpy(&seqset_pre, seqset0, sizeof(seqset_pre)); } @@ -1934,7 +2021,7 @@ static bool update_seq(enum seq_num seq, uint64_t n_seqcmd, int s = 0; seqset = NULL; seqset_item = NULL; - ss_item = seqset_store->head; + ss_item = STORE_WHEAD(seqset_store); while (ss_item) { DATA_SEQSET(ss, ss_item); if (!seqset) { @@ -1976,7 +2063,7 @@ static bool update_seq(enum seq_num seq, uint64_t n_seqcmd, } else { // put it next after the head k_insert_after(seqset_store, seqset_item, - seqset_store->head); + STORE_WHEAD(seqset_store)); set = 1; } } @@ -2156,7 +2243,7 @@ gotseqset: * to cause this */ st_item = NULL; if (seqdata->reload_lost) { - st_item = seqdata->reload_lost->head; + st_item = STORE_WHEAD(seqdata->reload_lost); // seqnum order is not guaranteed while (st_item) { DATA_SEQTRANS(seqtrans, st_item); @@ -2228,7 +2315,7 @@ setitemdata: copy_tv(&(seqdata->lastcd), cd); } - ck_wunlock(&seq_lock); + SEQUNLOCK(); if (firstseq) { // The first ever SEQ_ALL @@ -2333,12 +2420,12 @@ setitemdata: } } - if (lost && lost->head) { + if (lost && lost->count) { int tran = 0, miss = 0; uint64_t prev = 0; char range_buf[256]; bool isrange = false; - st_item = lost->head; + st_item = STORE_HEAD_NOLOCK(lost); while (st_item) { DATA_SEQTRANS(seqtrans, st_item); st_item = st_item->next; @@ -2590,7 +2677,8 @@ static enum cmd_values breakdown(K_ITEM **ml_item, char *buf, tv_t *now, goto nogood; } - msgline->trf_root = new_ktree(cmp_transfer); + // N.B. these aren't shared so they use _nolock, below + msgline->trf_root = new_ktree(cmp_transfer, transfer_free); msgline->trf_store = k_new_store(transfer_free); next = data; if (next && strncmp(next, JSON_TRANSFER, JSON_TRANSFER_LEN) == 0) { @@ -2742,8 +2830,8 @@ static enum cmd_values breakdown(K_ITEM **ml_item, char *buf, tv_t *now, STRNCPYSIZ(transfer->svalue, next, siz+1); transfer->mvalue = transfer->svalue; } - add_to_ktree(msgline->trf_root, t_item); - k_add_head(msgline->trf_store, t_item); + add_to_ktree_nolock(msgline->trf_root, t_item); + k_add_head_nolock(msgline->trf_store, t_item); t_item = NULL; // find the separator then move to the next name @@ -2765,7 +2853,6 @@ static enum cmd_values breakdown(K_ITEM **ml_item, char *buf, tv_t *now, goto nogood; } } else { - K_WLOCK(transfer_free); while (next && *next) { data = next; next = strchr(data, FLDSEP); @@ -2778,23 +2865,24 @@ static enum cmd_values breakdown(K_ITEM **ml_item, char *buf, tv_t *now, else *(eq++) = '\0'; + K_WLOCK(transfer_free); t_item = k_unlink_head(transfer_free); + K_WUNLOCK(transfer_free); DATA_TRANSFER(transfer, t_item); STRNCPY(transfer->name, data); STRNCPY(transfer->svalue, eq); transfer->mvalue = transfer->svalue; // Discard duplicates - if (find_in_ktree(msgline->trf_root, t_item, ctx)) { + if (find_in_ktree_nolock(msgline->trf_root, t_item, ctx)) { if (transfer->mvalue != transfer->svalue) FREENULL(transfer->mvalue); k_add_head(transfer_free, t_item); } else { - add_to_ktree(msgline->trf_root, t_item); - k_add_head(msgline->trf_store, t_item); + add_to_ktree_nolock(msgline->trf_root, t_item); + k_add_head_nolock(msgline->trf_store, t_item); } } - K_WUNLOCK(transfer_free); } seqall = find_transfer(msgline->trf_root, SEQALL); @@ -2861,12 +2949,14 @@ static void check_blocks() BLOCKS *blocks; K_RLOCK(blocks_free); - // Find the oldest block BLOCKS_NEW or BLOCKS_CONFIRM + /* Find the oldest block BLOCKS_NEW or BLOCKS_CONFIRM + * ... that's summarised, so processing order is correct */ b_item = first_in_ktree(blocks_root, ctx); while (b_item) { DATA_BLOCKS(blocks, b_item); if (!blocks->ignore && CURRENT(&(blocks->expirydate)) && + blocks->statsconfirmed[0] != BLOCKS_STATSPENDING && (blocks->confirmed[0] == BLOCKS_NEW || blocks->confirmed[0] == BLOCKS_CONFIRM)) break; @@ -2979,10 +3069,13 @@ static void summarise_blocks() INIT_SHARESUMMARY(&ss_look); ss_look.data = (void *)(&looksharesummary); + // We don't want them in an indeterminate state due to pplns + K_WLOCK(process_pplns_free); + // For now, just lock all 3 K_RLOCK(sharesummary_free); - K_RLOCK(workmarkers_free); K_RLOCK(markersummary_free); + K_RLOCK(workmarkers_free); ss_item = find_before_in_ktree(sharesummary_workinfoid_root, &ss_look, ss_ctx); @@ -2990,9 +3083,11 @@ static void summarise_blocks() while (ss_item && sharesummary->workinfoid > wi_start) { if (sharesummary->complete[0] == SUMMARY_NEW) { // Not aged yet - K_RUNLOCK(markersummary_free); K_RUNLOCK(workmarkers_free); + K_RUNLOCK(markersummary_free); K_RUNLOCK(sharesummary_free); + + K_WUNLOCK(process_pplns_free); return; } has_ss = true; @@ -3075,10 +3170,12 @@ static void summarise_blocks() DATA_WORKMARKERS_NULL(workmarkers, wm_item); } - K_RUNLOCK(markersummary_free); K_RUNLOCK(workmarkers_free); + K_RUNLOCK(markersummary_free); K_RUNLOCK(sharesummary_free); + K_WUNLOCK(process_pplns_free); + if (!has_ss && !has_ms) { // This will repeat each call here until fixed ... LOGERR("%s() block %d, after block %d, no sharesummaries " @@ -3118,6 +3215,7 @@ static void *summariser(__maybe_unused void *arg) pthread_detach(pthread_self()); + LOCK_INIT("db_summariser"); rename_proc("db_summariser"); /* Don't do any summarisation until the reload queue completes coz: @@ -3674,6 +3772,7 @@ static void *marker(__maybe_unused void *arg) pthread_detach(pthread_self()); + LOCK_INIT("db_marker"); rename_proc("db_marker"); /* We want this to start during the CCL reload so that if we run a @@ -3747,6 +3846,7 @@ static void *logger(__maybe_unused void *arg) pthread_detach(pthread_self()); snprintf(buf, sizeof(buf), "db%s_logger", dbcode); + LOCK_INIT(buf); rename_proc(buf); LOGWARNING("%s() Start processing...", __func__); @@ -3790,7 +3890,7 @@ static void *logger(__maybe_unused void *arg) LOGFILE(buf, logname_io); if (count) LOGERR("%s", buf); - lq_item = logqueue_store->head; + lq_item = STORE_WHEAD(logqueue_store); copy_tv(&then, &now); while (lq_item) { DATA_LOGQUEUE(lq, lq_item); @@ -3862,6 +3962,7 @@ static void *socketer(__maybe_unused void *arg) pthread_detach(pthread_self()); + LOCK_INIT("db_socketer"); rename_proc("db_socketer"); while (!everyone_die && !db_users_complete) @@ -4085,6 +4186,7 @@ static void *socketer(__maybe_unused void *arg) case CMD_SHSTA: case CMD_USERINFO: case CMD_BTCSET: + case CMD_LOCKS: ans = ckdb_cmds[msgline->which_cmds].func(NULL, msgline->cmd, msgline->id, @@ -4420,6 +4522,7 @@ static void reload_line(PGconn *conn, char *filename, uint64_t count, char *buf) case CMD_USERINFO: case CMD_BTCSET: case CMD_QUERY: + case CMD_LOCKS: LOGERR("%s() INVALID message line %"PRIu64 " ignored '%.42s...", __func__, count, @@ -4807,10 +4910,16 @@ static void *listener(void *arg) K_ITEM *ss_item; int i; + LOCK_INIT("db_listener"); + logqueue_free = k_new_list("LogQueue", sizeof(LOGQUEUE), ALLOC_LOGQUEUE, LIMIT_LOGQUEUE, true); logqueue_store = k_new_store(logqueue_free); +#if LOCK_CHECK + DLPRIO(logqueue, 94); +#endif + create_pthread(&log_pt, logger, NULL); create_pthread(&sock_pt, socketer, arg); @@ -4832,9 +4941,9 @@ static void *listener(void *arg) } if (!everyone_die) { - K_RLOCK(workqueue_store); + K_RLOCK(workqueue_free); wqcount = workqueue_store->count; - K_RUNLOCK(workqueue_store); + K_RUNLOCK(workqueue_free); LOGWARNING("reload shares OoO %s", ooo_status(ooo_buf, sizeof(ooo_buf))); sequence_report(true); @@ -4863,10 +4972,10 @@ static void *listener(void *arg) // Process queued work while (!everyone_die) { - K_WLOCK(workqueue_store); + K_WLOCK(workqueue_free); wq_item = k_unlink_head(workqueue_store); left = workqueue_store->count; - K_WUNLOCK(workqueue_store); + K_WUNLOCK(workqueue_free); if (left == 0 && wq_stt.tv_sec != 0L) setnow(&wq_fin); @@ -4904,9 +5013,9 @@ static void *listener(void *arg) /* Cleanup all the reload_lost stores since * they should no longer be needed and the ram * they use should be freed by the next cull */ - ck_wlock(&seq_lock); + SEQLOCK(); if (seqset_store->count > 0) { - ss_item = seqset_store->head; + ss_item = STORE_WHEAD(seqset_store); while (ss_item) { DATA_SEQSET(seqset, ss_item); if (seqset->seqstt) { @@ -4920,7 +5029,7 @@ static void *listener(void *arg) } } seqdata_reload_lost = false; - ck_wunlock(&seq_lock); + SEQUNLOCK(); } if (!wq_item) { @@ -5468,8 +5577,13 @@ static void confirm_summaries() ALLOC_LOGQUEUE, LIMIT_LOGQUEUE, true); logqueue_store = k_new_store(logqueue_free); +#if LOCK_CHECK + DLPRIO(logqueue, 94); +#endif + create_pthread(&log_pt, logger, NULL); + LOCK_INIT("dby_confirmer"); rename_proc("dby_confirmer"); alloc_storage(); @@ -5726,7 +5840,9 @@ int main(int argc, char **argv) if (!ckp.name) ckp.name = "ckdb"; snprintf(buf, 15, "%s%s", ckp.name, dbcode); + FIRST_LOCK_INIT(buf); prctl(PR_SET_NAME, buf, 0, 0, 0); + memset(buf, 0, 15); check_restore_dir(ckp.name); @@ -5782,8 +5898,15 @@ int main(int argc, char **argv) cklock_init(&last_lock); cklock_init(&btc_lock); - cklock_init(&seq_lock); - cklock_init(&process_pplns_lock); + + // Emulate a list for lock checking + process_pplns_free = k_lock_only_list("ProcessPPLNS"); + workers_db_free = k_lock_only_list("WorkersDB"); + +#if LOCK_CHECK + DLPRIO(process_pplns, 99); + DLPRIO(workers_db, 98); +#endif if (confirm_sharesummary) { // TODO: add a system lock to stop running 2 at once? diff --git a/src/ckdb.h b/src/ckdb.h index 3b458a0d..f658044d 100644 --- a/src/ckdb.h +++ b/src/ckdb.h @@ -55,7 +55,7 @@ #define DB_VLOCK "1" #define DB_VERSION "1.0.4" -#define CKDB_VERSION DB_VERSION"-1.505" +#define CKDB_VERSION DB_VERSION"-1.600" #define WHERE_FFL " - from %s %s() line %d" #define WHERE_FFL_HERE __FILE__, __func__, __LINE__ @@ -112,6 +112,12 @@ extern enum free_modes free_mode; #define BLANK " " extern char *EMPTY; +extern const char *nullstr; + +extern const char *true_str; +extern const char *false_str; + +#define TFSTR(_b) ((_b) ? true_str : false_str) #define FREENULL(mem) do { \ if ((mem) && (void *)(mem) != (void *)EMPTY) { \ @@ -424,6 +430,7 @@ enum cmd_values { CMD_USERINFO, CMD_BTCSET, CMD_QUERY, + CMD_LOCKS, CMD_END }; @@ -1026,12 +1033,8 @@ typedef struct seqset { #define INIT_SEQSET(_item) INIT_GENERIC(_item, seqset) #define DATA_SEQSET(_var, _item) DATA_GENERIC(_var, _item, seqset, true) +// other variables are static in ckdb.c extern K_LIST *seqset_free; -// each new seqset is added to the head, so head is the current one -extern K_STORE *seqset_store; - -// Initialised when seqset_free is allocated -extern char *seqnam[SEQ_MAX]; // SEQTRANS also used for reload_lost typedef struct seqtrans { @@ -1168,6 +1171,8 @@ typedef struct workers { extern K_TREE *workers_root; extern K_LIST *workers_free; extern K_STORE *workers_store; +// Emulate a list for lock checking +extern K_LIST *workers_db_free; // Currently no workerbits attributes @@ -1598,6 +1603,7 @@ extern const char *blocks_unknown; extern K_TREE *blocks_root; extern K_LIST *blocks_free; extern K_STORE *blocks_store; +// Access both under blocks_free lock extern tv_t blocks_stats_time; extern bool blocks_stats_rebuild; @@ -1656,7 +1662,8 @@ extern K_TREE *payouts_id_root; extern K_TREE *payouts_wid_root; extern K_LIST *payouts_free; extern K_STORE *payouts_store; -extern cklock_t process_pplns_lock; +// Emulate a list for lock checking +extern K_LIST *process_pplns_free; // N.B. status should be checked under r/w lock #define PAYOUTS_GENERATED 'G' @@ -2156,8 +2163,8 @@ extern void free_optioncontrol_data(K_ITEM *item); extern void free_markersummary_data(K_ITEM *item); extern void free_workmarkers_data(K_ITEM *item); extern void free_marks_data(K_ITEM *item); -#define free_seqset_data(_item) _free_seqset_data(_item, false) -extern void _free_seqset_data(K_ITEM *item, bool lock); +#define free_seqset_data(_item) _free_seqset_data(_item) +extern void _free_seqset_data(K_ITEM *item); // Data copy functions #define COPY_DATA(_new, _old) memcpy(_new, _old, sizeof(*(_new))) @@ -2244,16 +2251,15 @@ extern K_ITEM *_optional_name(K_TREE *trf_root, char *name, int len, char *patt, extern K_ITEM *_require_name(K_TREE *trf_root, char *name, int len, char *patt, char *reply, size_t siz, WHERE_FFL_ARGS); extern cmp_t cmp_workerstatus(K_ITEM *a, K_ITEM *b); -extern K_ITEM *get_workerstatus(bool lock, int64_t userid, char *workername); -#define find_create_workerstatus(_l, _u, _w, _file, _func, _line) \ - _find_create_workerstatus(_l, _u, _w, true, _file, _func, _line, WHERE_FFL_HERE) -#define find_workerstatus(_l, _u, _w, _file, _func, _line) \ - _find_create_workerstatus(_l, _u, _w, false, _file, _func, _line, WHERE_FFL_HERE) - -extern K_ITEM *_find_create_workerstatus(bool lock, int64_t userid, - char *workername, bool create, - const char *file2, const char *func2, - const int line2, WHERE_FFL_ARGS); +extern K_ITEM *find_workerstatus(bool gotlock, int64_t userid, char *workername); +#define find_create_workerstatus(_gl, _ac, _u, _w, _hw, _file, _func, _line) \ + _find_create_workerstatus(_gl, _ac, _u, _w, _hw, _file, _func, _line, \ + WHERE_FFL_HERE) +extern K_ITEM *_find_create_workerstatus(bool gotlock, bool alertcreate, + int64_t userid, char *workername, + bool hasworker, const char *file2, + const char *func2, const int line2, + WHERE_FFL_ARGS); extern void zero_all_active(tv_t *when); extern void workerstatus_ready(); #define workerstatus_update(_auths, _shares, _userstats) \ @@ -2297,7 +2303,7 @@ extern void _users_userdata_add_bin(USERS *users, char *name, int64_t bit, extern cmp_t cmp_useratts(K_ITEM *a, K_ITEM *b); extern K_ITEM *find_useratts(int64_t userid, char *attname); extern cmp_t cmp_workers(K_ITEM *a, K_ITEM *b); -extern K_ITEM *find_workers(int64_t userid, char *workername); +extern K_ITEM *find_workers(bool gotlock, int64_t userid, char *workername); extern K_ITEM *first_workers(int64_t userid, K_TREE_CTX *ctx); extern K_ITEM *new_worker(PGconn *conn, bool update, int64_t userid, char *workername, char *diffdef, char *idlenotificationenabled, @@ -2327,14 +2333,15 @@ extern cmp_t cmp_workinfo(K_ITEM *a, K_ITEM *b); #define coinbase1height(_wi) _coinbase1height(_wi, WHERE_FFL_HERE) extern int32_t _coinbase1height(WORKINFO *wi, WHERE_FFL_ARGS); extern cmp_t cmp_workinfo_height(K_ITEM *a, K_ITEM *b); -extern K_ITEM *find_workinfo(int64_t workinfoid, K_TREE_CTX *ctx); +#define find_workinfo(_wid, _ctx) _find_workinfo(_wid, false, _ctx); +extern K_ITEM *_find_workinfo(int64_t workinfoid, bool gotlock, K_TREE_CTX *ctx); extern K_ITEM *next_workinfo(int64_t workinfoid, K_TREE_CTX *ctx); extern bool workinfo_age(int64_t workinfoid, char *poolinstance, char *by, char *code, char *inet, tv_t *cd, tv_t *ss_first, tv_t *ss_last, int64_t *ss_count, int64_t *s_count, int64_t *s_diff); extern double coinbase_reward(int32_t height); -extern double workinfo_pps(K_ITEM *w_item, int64_t workinfoid, bool lock); +extern double workinfo_pps(K_ITEM *w_item, int64_t workinfoid); extern cmp_t cmp_shares(K_ITEM *a, K_ITEM *b); extern cmp_t cmp_shareerrors(K_ITEM *a, K_ITEM *b); extern void dsp_sharesummary(K_ITEM *item, FILE *stream); @@ -2367,19 +2374,19 @@ extern cmp_t cmp_blocks(K_ITEM *a, K_ITEM *b); extern K_ITEM *find_blocks(int32_t height, char *blockhash, K_TREE_CTX *ctx); extern K_ITEM *find_prev_blocks(int32_t height, K_TREE_CTX *ctx); extern const char *blocks_confirmed(char *confirmed); -extern void zero_on_new_block(bool lock); +extern void zero_on_new_block(bool gotlock); extern void set_block_share_counters(); extern bool check_update_blocks_stats(tv_t *stats); #define set_blockcreatedate(_h) _set_blockcreatedate(_h, WHERE_FFL_HERE) extern bool _set_blockcreatedate(int32_t oldest_height, WHERE_FFL_ARGS); -#define set_prevcreatedate(_h) _set_prevcreatedate(_h, WHERE_FFL_HERE) +#define set_prevcreatedate(_oh) _set_prevcreatedate(_oh, WHERE_FFL_HERE) extern bool _set_prevcreatedate(int32_t oldest_height, WHERE_FFL_ARGS); extern cmp_t cmp_miningpayouts(K_ITEM *a, K_ITEM *b); extern K_ITEM *find_miningpayouts(int64_t payoutid, int64_t userid); extern K_ITEM *first_miningpayouts(int64_t payoutid, K_TREE_CTX *ctx); extern cmp_t cmp_mu(K_ITEM *a, K_ITEM *b); -extern K_TREE *upd_add_mu(K_TREE *mu_root, K_STORE *mu_store, int64_t userid, - double diffacc); +extern void upd_add_mu(K_TREE *mu_root, K_STORE *mu_store, int64_t userid, + double diffacc); extern cmp_t cmp_payouts(K_ITEM *a, K_ITEM *b); extern cmp_t cmp_payouts_id(K_ITEM *a, K_ITEM *b); extern cmp_t cmp_payouts_wid(K_ITEM *a, K_ITEM *b); @@ -2420,7 +2427,7 @@ extern K_ITEM *find_workmarkerid(int64_t markerid, bool anystatus, char status); extern bool workmarkers_generate(PGconn *conn, char *err, size_t siz, char *by, char *code, char *inet, tv_t *cd, K_TREE *trf_root, bool none_error); -extern bool reward_shifts(PAYOUTS *payouts, bool lock, int delta); +extern bool reward_shifts(PAYOUTS *payouts, int delta); extern bool shift_rewards(K_ITEM *wm_item); extern cmp_t cmp_marks(K_ITEM *a, K_ITEM *b); extern K_ITEM *find_marks(int64_t workinfoid); @@ -2432,16 +2439,12 @@ extern bool _marks_description(char *description, size_t siz, char *marktype, WHERE_FFL_ARGS); extern char *shiftcode(tv_t *createdate); extern cmp_t cmp_userinfo(K_ITEM *a, K_ITEM *b); -#define get_userinfo(_userid) _get_userinfo(_userid, true) -extern K_ITEM *_get_userinfo(int64_t userid, bool lock); -#define find_userinfo(_userid) _find_create_userinfo(_userid, true, WHERE_FFL_HERE) -#define _find_userinfo(_userid, _lock) _find_create_userinfo(_userid, _lock, WHERE_FFL_HERE) -extern K_ITEM *_find_create_userinfo(int64_t userid, bool lock, WHERE_FFL_ARGS); -#define userinfo_update(_s, _ss, _ms) _userinfo_update(_s, _ss, _ms, true, true) -extern void _userinfo_update(SHARES *shares, SHARESUMMARY *sharesummary, - MARKERSUMMARY *markersummary, bool ss_sub, bool lock); -#define userinfo_block(_blocks, _isnew, _delta) _userinfo_block(_blocks, _isnew, _delta, true) -extern void _userinfo_block(BLOCKS *blocks, enum info_type isnew, int delta, bool lock); +extern K_ITEM *get_userinfo(int64_t userid); +#define find_create_userinfo(_userid) _find_create_userinfo(_userid, WHERE_FFL_HERE) +extern K_ITEM *_find_create_userinfo(int64_t userid, WHERE_FFL_ARGS); +extern void userinfo_update(SHARES *shares, SHARESUMMARY *sharesummary, + MARKERSUMMARY *markersummary, bool ss_sub); +extern void userinfo_block(BLOCKS *blocks, enum info_type isnew, int delta); // *** // *** PostgreSQL functions ckdb_dbio.c @@ -2516,8 +2519,8 @@ extern K_ITEM *useratts_add(PGconn *conn, char *username, char *attname, bool begun); extern bool useratts_item_expire(PGconn *conn, K_ITEM *ua_item, tv_t *cd); extern bool useratts_fill(PGconn *conn); -extern K_ITEM *workers_add(PGconn *conn, bool lock, int64_t userid, - char *workername, char *difficultydefault, +extern K_ITEM *workers_add(PGconn *conn, int64_t userid, char *workername, + bool add_ws, char *difficultydefault, char *idlenotificationenabled, char *idlenotificationtime, char *by, char *code, char *inet, tv_t *cd, K_TREE *trf_root); diff --git a/src/ckdb_cmd.c b/src/ckdb_cmd.c index 9322d1ae..18c89302 100644 --- a/src/ckdb_cmd.c +++ b/src/ckdb_cmd.c @@ -393,11 +393,9 @@ static char *cmd_2fa(__maybe_unused PGconn *conn, char *cmd, char *id, APPEND_REALLOC(buf, off, len, tmp); FREENULL(keystr); - K_RLOCK(optioncontrol_free); oc_item = find_optioncontrol(TOTPAUTH_ISSUER, now, OPTIONCONTROL_HEIGHT); - K_RUNLOCK(optioncontrol_free); if (oc_item) { DATA_OPTIONCONTROL(oc, oc_item); issuer = oc->optionvalue; @@ -627,7 +625,7 @@ static char *cmd_userset(PGconn *conn, char *cmd, char *id, goto struckout; } address = transfer_data(i_address); - pa_item = pa_store->head; + pa_item = STORE_HEAD_NOLOCK(pa_store); while (pa_item) { DATA_PAYMENTADDRESSES(row, pa_item); if (strcmp(row->payaddress, address) == 0) { @@ -665,7 +663,7 @@ static char *cmd_userset(PGconn *conn, char *cmd, char *id, } if (pa_store && pa_store->count > 0) { - pa_item = pa_store->head; + pa_item = STORE_HEAD_NOLOCK(pa_store); while (pa_item) { DATA_PAYMENTADDRESSES(row, pa_item); // Only EVER validate addresses once ... for now @@ -821,7 +819,7 @@ static char *cmd_workerset(PGconn *conn, char *cmd, char *id, tv_t *now, if (!i_workername) break; - w_item = find_workers(users->userid, + w_item = find_workers(false, users->userid, transfer_data(i_workername)); // Abort if any dont exist if (!w_item) { @@ -958,7 +956,9 @@ static char *cmd_poolstats_do(PGconn *conn, char *cmd, char *id, char *by, row.createdate.tv_usec = date_eot.tv_usec; INIT_POOLSTATS(&look); look.data = (void *)(&row); + K_RLOCK(poolstats_free); ps = find_before_in_ktree(poolstats_root, &look, ctx); + K_RUNLOCK(poolstats_free); if (!ps) store = true; else { @@ -1193,9 +1193,7 @@ static char *cmd_blocklist(__maybe_unused PGconn *conn, char *cmd, char *id, APPEND_REALLOC(buf, off, len, "ok."); redo: - K_WLOCK(blocks_free); has_stats = check_update_blocks_stats(&stats_tv); - K_WUNLOCK(blocks_free); srows = rows = 0; K_RLOCK(blocks_free); @@ -1670,7 +1668,7 @@ static char *cmd_payments(__maybe_unused PGconn *conn, char *cmd, char *id, } K_WUNLOCK(payments_free); - p_item = pay_store->head; + p_item = STORE_HEAD_NOLOCK(pay_store); while (p_item) { DATA_PAYMENTS(payments, p_item); pok = false; @@ -1763,12 +1761,15 @@ static char *cmd_percent(char *cmd, char *id, tv_t *now, USERS *users) lookworkers.workername[0] = '\0'; DATE_ZERO(&(lookworkers.expirydate)); w_look.data = (void *)(&lookworkers); + K_RLOCK(workers_free); w_item = find_after_in_ktree(workers_root, &w_look, w_ctx); + K_RUNLOCK(workers_free); DATA_WORKERS_NULL(workers, w_item); while (w_item && workers->userid == users->userid) { if (CURRENT(&(workers->expirydate))) { - ws_item = get_workerstatus(true, users->userid, - workers->workername); + K_RLOCK(workerstatus_free); + ws_item = find_workerstatus(true, users->userid, + workers->workername); if (ws_item) { DATA_WORKERSTATUS(workerstatus, ws_item); t_diffacc += workerstatus->block_diffacc; @@ -1784,6 +1785,7 @@ static char *cmd_percent(char *cmd, char *id, tv_t *now, USERS *users) t_sharehi += workerstatus->block_sharehi; t_sharerej += workerstatus->block_sharerej; } + K_RUNLOCK(workerstatus_free); /* TODO: workers_root userid+worker is ordered * so no 'find' should be needed - @@ -1801,7 +1803,9 @@ static char *cmd_percent(char *cmd, char *id, tv_t *now, USERS *users) } K_RUNLOCK(userstats_free); } + K_RLOCK(workers_free); w_item = next_in_ktree(w_ctx); + K_RUNLOCK(workers_free); DATA_WORKERS_NULL(workers, w_item); } @@ -2029,21 +2033,23 @@ static char *cmd_workers(__maybe_unused PGconn *conn, char *cmd, char *id, lookworkers.workername[0] = '\0'; DATE_ZERO(&(lookworkers.expirydate)); w_look.data = (void *)(&lookworkers); + K_RLOCK(workers_free); w_item = find_after_in_ktree(workers_root, &w_look, w_ctx); + K_RUNLOCK(workers_free); DATA_WORKERS_NULL(workers, w_item); rows = 0; while (w_item && workers->userid == users->userid) { if (CURRENT(&(workers->expirydate))) { - ws_item = get_workerstatus(true, users->userid, - workers->workername); + K_RLOCK(workerstatus_free); + ws_item = find_workerstatus(true, users->userid, + workers->workername); if (ws_item) { DATA_WORKERSTATUS(workerstatus, ws_item); - K_RLOCK(workerstatus_free); // good or bad - either means active copy_tv(&last_share, &(workerstatus->last_share)); - K_RUNLOCK(workerstatus_free); } else DATE_ZERO(&last_share); + K_RUNLOCK(workerstatus_free); if (tvdiff(now, &last_share) < oldworkers) { str_to_buf(workers->workername, reply, sizeof(reply)); @@ -2223,7 +2229,9 @@ static char *cmd_workers(__maybe_unused PGconn *conn, char *cmd, char *id, rows++; } } + K_RLOCK(workers_free); w_item = next_in_ktree(w_ctx); + K_RUNLOCK(workers_free); DATA_WORKERS_NULL(workers, w_item); } snprintf(tmp, sizeof(tmp), @@ -2298,7 +2306,7 @@ static char *cmd_allusers(__maybe_unused PGconn *conn, char *cmd, char *id, APPEND_REALLOC_INIT(buf, off, len); APPEND_REALLOC(buf, off, len, "ok."); rows = 0; - usu_item = usu_store->head; + usu_item = STORE_HEAD_NOLOCK(usu_store); while (usu_item) { DATA_USERSTATS(userstats_u, usu_item); K_RLOCK(users_free); @@ -2865,9 +2873,7 @@ static char *cmd_auth_do(PGconn *conn, char *cmd, char *id, char *by, if (!i_preauth) i_preauth = &auth_preauth; - K_RLOCK(optioncontrol_free); oc_item = find_optioncontrol(OPTIONCONTROL_AUTOADDUSER, cd, pool.height); - K_RUNLOCK(optioncontrol_free); if (oc_item) { K_RLOCK(users_free); u_item = find_users(username); @@ -3096,7 +3102,7 @@ static char *cmd_heartbeat(__maybe_unused PGconn *conn, char *cmd, char *id, APPEND_REALLOC_INIT(buf, off, len); APPEND_REALLOC(buf, off, len, "ok.heartbeat={\"diffchange\":["); - hq_item = hq_store->tail; + hq_item = STORE_TAIL_NOLOCK(hq_store); first = true; while (hq_item) { DATA_HEARTBEATQUEUE(heartbeatqueue, hq_item); @@ -3159,7 +3165,7 @@ static char *cmd_homepage(__maybe_unused PGconn *conn, char *cmd, char *id, ftv_to_buf(now, reply, siz); snprintf(tmp, sizeof(tmp), "now=%s%c", reply, FLDSEP); APPEND_REALLOC(buf, off, len, tmp); - ck_wlock(&last_lock); + ck_rlock(&last_lock); ftv_to_buf(&last_heartbeat, reply, siz); snprintf(tmp, sizeof(tmp), "lasthb=%s%c", reply, FLDSEP); APPEND_REALLOC(buf, off, len, tmp); @@ -3176,7 +3182,7 @@ static char *cmd_homepage(__maybe_unused PGconn *conn, char *cmd, char *id, snprintf(tmp, sizeof(tmp), "lastshinv=%s%c", reply, FLDSEP); APPEND_REALLOC(buf, off, len, tmp); ftv_to_buf(&last_auth, reply, siz); - ck_wunlock(&last_lock); + ck_runlock(&last_lock); snprintf(tmp, sizeof(tmp), "lastau=%s%c", reply, FLDSEP); APPEND_REALLOC(buf, off, len, tmp); @@ -3249,7 +3255,9 @@ static char *cmd_homepage(__maybe_unused PGconn *conn, char *cmd, char *id, APPEND_REALLOC(buf, off, len, tmp); // TODO: assumes only one poolinstance (for now) + K_RLOCK(poolstats_free); p_item = last_in_ktree(poolstats_root, ctx); + K_RUNLOCK(poolstats_free); if (p_item) { DATA_POOLSTATS(poolstats, p_item); int_to_buf(poolstats->users, reply, siz); @@ -3830,9 +3838,7 @@ static char *cmd_getopts(__maybe_unused PGconn *conn, char *cmd, char *id, comma = strchr(ptr, ','); if (comma) *(comma++) = '\0'; - K_RLOCK(optioncontrol_free); oc_item = find_optioncontrol(ptr, now, pool.height); - K_RUNLOCK(optioncontrol_free); /* web code must check the existance of the optionname * in the reply since it will be missing if it doesn't * exist in the DB */ @@ -4175,13 +4181,14 @@ static char *cmd_pplns(__maybe_unused PGconn *conn, char *cmd, char *id, ss_count = wm_count = ms_count = 0; mu_store = k_new_store(miningpayouts_free); - mu_root = new_ktree(cmp_mu); + mu_root = new_ktree(cmp_mu, miningpayouts_free); looksharesummary.workinfoid = block_workinfoid; looksharesummary.userid = MAXID; looksharesummary.workername = EMPTY; INIT_SHARESUMMARY(&ss_look); ss_look.data = (void *)(&looksharesummary); + K_WLOCK(miningpayouts_free); K_RLOCK(sharesummary_free); K_RLOCK(workmarkers_free); K_RLOCK(markersummary_free); @@ -4215,9 +4222,8 @@ static char *cmd_pplns(__maybe_unused PGconn *conn, char *cmd, char *id, begin_workinfoid = sharesummary->workinfoid; if (tv_newer(&end_tv, &(sharesummary->lastshareacc))) copy_tv(&end_tv, &(sharesummary->lastshareacc)); - mu_root = upd_add_mu(mu_root, mu_store, - sharesummary->userid, - (int64_t)(sharesummary->diffacc)); + upd_add_mu(mu_root, mu_store, sharesummary->userid, + (int64_t)(sharesummary->diffacc)); ss_item = prev_in_ktree(ctx); DATA_SHARESUMMARY_NULL(sharesummary, ss_item); } @@ -4240,9 +4246,8 @@ static char *cmd_pplns(__maybe_unused PGconn *conn, char *cmd, char *id, total_share_count += sharesummary->sharecount; acc_share_count += sharesummary->shareacc; total_diff += (int64_t)(sharesummary->diffacc); - mu_root = upd_add_mu(mu_root, mu_store, - sharesummary->userid, - (int64_t)(sharesummary->diffacc)); + upd_add_mu(mu_root, mu_store, sharesummary->userid, + (int64_t)(sharesummary->diffacc)); ss_item = prev_in_ktree(ctx); DATA_SHARESUMMARY_NULL(sharesummary, ss_item); } @@ -4289,9 +4294,8 @@ static char *cmd_pplns(__maybe_unused PGconn *conn, char *cmd, char *id, begin_workinfoid = workmarkers->workinfoidstart; if (tv_newer(&end_tv, &(markersummary->lastshareacc))) copy_tv(&end_tv, &(markersummary->lastshareacc)); - mu_root = upd_add_mu(mu_root, mu_store, - markersummary->userid, - (int64_t)(markersummary->diffacc)); + upd_add_mu(mu_root, mu_store, markersummary->userid, + (int64_t)(markersummary->diffacc)); ms_item = prev_in_ktree(ms_ctx); DATA_MARKERSUMMARY_NULL(markersummary, ms_item); } @@ -4305,6 +4309,7 @@ static char *cmd_pplns(__maybe_unused PGconn *conn, char *cmd, char *id, K_RUNLOCK(markersummary_free); K_RUNLOCK(workmarkers_free); K_RUNLOCK(sharesummary_free); + K_WUNLOCK(miningpayouts_free); LOGDEBUG("%s(): total %.0f want %.0f", __func__, total_diff, diff_want); if (total_diff == 0.0) { @@ -4487,9 +4492,9 @@ static char *cmd_pplns(__maybe_unused PGconn *conn, char *cmd, char *id, APPEND_REALLOC(buf, off, len, "pplns_last=1"); free_ktree(mu_root, NULL); - K_WLOCK(mu_store); + K_WLOCK(miningpayouts_free); k_list_transfer_to_head(mu_store, miningpayouts_free); - K_WUNLOCK(mu_store); + K_WUNLOCK(miningpayouts_free); mu_store = k_free_store(mu_store); LOGDEBUG("%s.ok.pplns.%s", id, buf); @@ -4498,9 +4503,9 @@ static char *cmd_pplns(__maybe_unused PGconn *conn, char *cmd, char *id, shazbot: free_ktree(mu_root, NULL); - K_WLOCK(mu_store); + K_WLOCK(miningpayouts_free); k_list_transfer_to_head(mu_store, miningpayouts_free); - K_WUNLOCK(mu_store); + K_WUNLOCK(miningpayouts_free); mu_store = k_free_store(mu_store); return strdup(reply); @@ -4853,7 +4858,7 @@ static char *cmd_payouts(PGconn *conn, char *cmd, char *id, tv_t *now, return strdup(reply); } // Original wasn't generated, so reward it - reward_shifts(payouts2, true, 1); + reward_shifts(payouts2, 1); DATA_PAYOUTS(payouts2, p2_item); DATA_PAYOUTS(old_payouts2, old_p2_item); snprintf(msg, sizeof(msg), @@ -4924,7 +4929,7 @@ static char *cmd_payouts(PGconn *conn, char *cmd, char *id, tv_t *now, return strdup(reply); } // Original was generated, so undo the reward - reward_shifts(payouts2, true, -1); + reward_shifts(payouts2, -1); DATA_PAYOUTS(payouts2, p2_item); DATA_PAYOUTS(old_payouts2, old_p2_item); snprintf(msg, sizeof(msg), @@ -5064,7 +5069,7 @@ static char *cmd_mpayouts(__maybe_unused PGconn *conn, char *cmd, char *id, while (po_item) { if (CURRENT(&(payouts->expirydate)) && PAYGENERATED(payouts->status)) { - // Not locked ... for now + K_RLOCK(miningpayouts_free); mp_item = find_miningpayouts(payouts->payoutid, users->userid); if (mp_item) { @@ -5123,6 +5128,7 @@ static char *cmd_mpayouts(__maybe_unused PGconn *conn, char *cmd, char *id, rows++; } + K_RUNLOCK(miningpayouts_free); } po_item = prev_in_ktree(ctx); DATA_PAYOUTS_NULL(payouts, po_item); @@ -6324,7 +6330,9 @@ static char *cmd_pshift(__maybe_unused PGconn *conn, char *cmd, char *id, } K_RLOCK(markersummary_free); + K_RLOCK(workmarkers_free); ms_item = find_markersummary_p(wm->markerid); + K_RUNLOCK(workmarkers_free); K_RUNLOCK(markersummary_free); if (ms_item) { DATA_MARKERSUMMARY(ms, ms_item); @@ -6494,7 +6502,7 @@ static char *cmd_userinfo(__maybe_unused PGconn *conn, char *cmd, char *id, rows = 0; K_RLOCK(userinfo_free); - ui_item = userinfo_store->head; + ui_item = STORE_RHEAD(userinfo_store); while (ui_item) { DATA_USERINFO(userinfo, ui_item); @@ -6798,8 +6806,7 @@ static char *cmd_query(__maybe_unused PGconn *conn, char *cmd, char *id, snprintf(tmp, sizeof(tmp), "ppsvalue:%d=%.15f%c", rows, workinfo_pps(wi_item, - workinfo->workinfoid, - true), + workinfo->workinfoid), FLDSEP); APPEND_REALLOC(buf, off, len, tmp); K_RLOCK(workmarkers_free); @@ -7171,6 +7178,7 @@ static char *cmd_query(__maybe_unused PGconn *conn, char *cmd, char *id, p_item = next_in_ktree(ctx); DATA_PAYOUTS_NULL(payouts, p_item); } + K_RUNLOCK(payouts_free); snprintf(tmp, sizeof(tmp), "flds=%s%c", "height,payoutid,minerreward,workinfoidstart," @@ -7208,6 +7216,68 @@ static char *cmd_query(__maybe_unused PGconn *conn, char *cmd, char *id, return buf; } +// Query and disable internal lock detection code +static char *cmd_locks(__maybe_unused PGconn *conn, char *cmd, char *id, + __maybe_unused tv_t *now, __maybe_unused char *by, + __maybe_unused char *code, __maybe_unused char *inet, + __maybe_unused tv_t *cd, + __maybe_unused K_TREE *trf_root) +{ + bool code_locks = false, code_deadlocks = false; + bool was_locks = false, was_deadlocks = false; + bool new_locks = false, new_deadlocks = false; + char reply[1024] = ""; + size_t siz = sizeof(reply); +#if LOCK_CHECK + K_ITEM *i_locks, *i_deadlocks; + char *deadlocks; +#endif + + LOGDEBUG("%s(): cmd '%s'", __func__, cmd); + +#if LOCK_CHECK + code_locks = true; + was_locks = new_locks = check_locks; + + code_deadlocks = true; + was_deadlocks = new_locks = check_deadlocks; +#endif + + /* options are + * locks <- disable lock checking if it's enabled (value ignored) + * deadlocks=Y/N <- enable/disable deadlock prediction + * any word with any case starting with 'Y' means enable it + * anything else means disable it + * When you enable it, it won't re-enable it for threads that + * have failed a deadlock prediction test + * It will report the status of both */ + +#if LOCK_CHECK + i_locks = optional_name(trf_root, "locks", 0, NULL, reply, siz); + if (i_locks) + new_locks = check_locks = false; + + i_deadlocks = optional_name(trf_root, "deadlocks", 0, NULL, reply, siz); + if (i_deadlocks) { + deadlocks = transfer_data(i_deadlocks); + if (toupper(*deadlocks) == TRUE_CHR) + check_deadlocks = true; + else + check_deadlocks = false; + new_deadlocks = check_deadlocks; + } +#endif + + snprintf(reply, siz, + "code_locks=%s%cwas_locks=%s%cnew_locks=%s%c" + "code_deadlocks=%s%cwas_deadlocks=%s%cnew_deadlocks=%s", + TFSTR(code_locks), FLDSEP, TFSTR(was_locks), FLDSEP, + TFSTR(new_locks), FLDSEP, TFSTR(code_deadlocks), FLDSEP, + TFSTR(was_deadlocks), FLDSEP, TFSTR(new_deadlocks)); + LOGWARNING("%s() %s.%s", __func__, id, reply); + return strdup(reply); +} + /* The socket command format is as follows: * Basic structure: * cmd.ID.fld1=value1 FLDSEP fld2=value2 FLDSEP fld3=... @@ -7317,5 +7387,6 @@ struct CMDS ckdb_cmds[] = { { CMD_USERINFO, "userinfo", false, false, cmd_userinfo, SEQ_NONE, ACCESS_WEB }, { CMD_BTCSET, "btcset", false, false, cmd_btcset, SEQ_NONE, ACCESS_SYSTEM }, { CMD_QUERY, "query", false, false, cmd_query, SEQ_NONE, ACCESS_SYSTEM }, + { CMD_LOCKS, "locks", false, false, cmd_locks, SEQ_NONE, ACCESS_SYSTEM }, { CMD_END, NULL, false, false, NULL, SEQ_NONE, 0 } }; diff --git a/src/ckdb_data.c b/src/ckdb_data.c index 30347485..ac1aaabe 100644 --- a/src/ckdb_data.c +++ b/src/ckdb_data.c @@ -22,7 +22,7 @@ void free_msgline_data(K_ITEM *item, bool t_lock, bool t_cull) if (msgline->trf_root) free_ktree(msgline->trf_root, NULL); if (msgline->trf_store) { - t_item = msgline->trf_store->head; + t_item = STORE_HEAD_NOLOCK(msgline->trf_store); while (t_item) { DATA_TRANSFER(transfer, t_item); if (transfer->mvalue != transfer->svalue) @@ -136,7 +136,7 @@ void free_marks_data(K_ITEM *item) FREENULL(marks->extra); } -void _free_seqset_data(K_ITEM *item, bool lock) +void _free_seqset_data(K_ITEM *item) { K_STORE *reload_lost; SEQSET *seqset; @@ -147,11 +147,9 @@ void _free_seqset_data(K_ITEM *item, bool lock) for (i = 0; i < SEQ_MAX; i++) { reload_lost = seqset->seqdata[i].reload_lost; if (reload_lost) { - if (lock) - K_WLOCK(seqtrans_free); + K_WLOCK(seqtrans_free); k_list_transfer_to_head(reload_lost, seqtrans_free); - if (lock) - K_WUNLOCK(seqtrans_free); + K_WUNLOCK(seqtrans_free); k_free_store(reload_lost); seqset->seqdata[i].reload_lost = NULL; } @@ -691,7 +689,8 @@ K_ITEM *find_transfer(K_TREE *trf_root, char *name) STRNCPY(transfer.name, name); INIT_TRANSFER(&look); look.data = (void *)(&transfer); - return find_in_ktree(trf_root, &look, ctx); + // trf_root stores aren't shared + return find_in_ktree_nolock(trf_root, &look, ctx); } K_ITEM *_optional_name(K_TREE *trf_root, char *name, int len, char *patt, @@ -815,7 +814,7 @@ cmp_t cmp_workerstatus(K_ITEM *a, K_ITEM *b) /* TODO: replace a lot of the code for all data types that codes finds, * each with specific functions for finding, to centralise the finds, * with passed ctx's */ -K_ITEM *get_workerstatus(bool lock, int64_t userid, char *workername) +K_ITEM *find_workerstatus(bool gotlock, int64_t userid, char *workername) { WORKERSTATUS workerstatus; K_TREE_CTX ctx[1]; @@ -826,42 +825,44 @@ K_ITEM *get_workerstatus(bool lock, int64_t userid, char *workername) INIT_WORKERSTATUS(&look); look.data = (void *)(&workerstatus); - if (lock) + if (!gotlock) K_RLOCK(workerstatus_free); find = find_in_ktree(workerstatus_root, &look, ctx); - if (lock) + if (!gotlock) K_RUNLOCK(workerstatus_free); return find; } -/* Worker loading/creation calls this with create = true - * All others with create = false since the workerstatus should exist - * If it is missing, it will check for and create the worker if needed - * and create a new workerstatus and return it +/* workerstatus will always be created if it is missing + * alertcreate means it was not expected to be missing and log this + * hasworker means it was called from code where the worker exists + * (e.g. a loop over workers or the add_worker() function) + * so there is no need to check for the worker or create it + * this also avoids an unlikely loop of add_worker() calling back + * to add_worker() (if the add_worker() fails) * This has 2 sets of file/func/line to allow 2 levels of traceback - * to see why it happened + * in the log */ -K_ITEM *_find_create_workerstatus(bool lock, int64_t userid, char *workername, - bool create, const char *file2, +K_ITEM *_find_create_workerstatus(bool gotlock, bool alertcreate, + int64_t userid, char *workername, + bool hasworker, const char *file2, const char *func2, const int line2, WHERE_FFL_ARGS) { WORKERSTATUS *row; K_ITEM *ws_item, *w_item = NULL; - bool ws_err = false, w_err = false; + bool ws_none = false, w_none = false; tv_t now; - ws_item = get_workerstatus(lock, userid, workername); + ws_item = find_workerstatus(gotlock, userid, workername); if (!ws_item) { - if (!create) { - ws_err = true; - - w_item = find_workers(userid, workername); + if (!hasworker) { + w_item = find_workers(false, userid, workername); if (!w_item) { - w_err = true; + w_none = true; setnow(&now); - w_item = workers_add(NULL, lock, userid, - workername, + w_item = workers_add(NULL, userid, + workername, false, NULL, NULL, NULL, by_default, (char *)__func__, @@ -870,37 +871,43 @@ K_ITEM *_find_create_workerstatus(bool lock, int64_t userid, char *workername, } } - if (lock) + if (!gotlock) K_WLOCK(workerstatus_free); - ws_item = k_unlink_head(workerstatus_free); - DATA_WORKERSTATUS(row, ws_item); + ws_item = find_workerstatus(true, userid, workername); + if (!ws_item) { + ws_none = true; + ws_item = k_unlink_head(workerstatus_free); - bzero(row, sizeof(*row)); - row->userid = userid; - STRNCPY(row->workername, workername); + DATA_WORKERSTATUS(row, ws_item); + + bzero(row, sizeof(*row)); + row->userid = userid; + STRNCPY(row->workername, workername); - add_to_ktree(workerstatus_root, ws_item); - k_add_head(workerstatus_store, ws_item); - if (lock) + add_to_ktree(workerstatus_root, ws_item); + k_add_head(workerstatus_store, ws_item); + } + if (!gotlock) K_WUNLOCK(workerstatus_free); - if (ws_err) { + if (ws_none && alertcreate) { LOGNOTICE("%s(): CREATED Missing workerstatus" " %"PRId64"/%s" WHERE_FFL WHERE_FFL, __func__, userid, workername, file2, func2, line2, WHERE_FFL_PASS); - if (w_err) { - int sta = LOG_ERR; - if (w_item) - sta = LOG_NOTICE; - LOGMSG(sta, - "%s(): %s Missing worker %"PRId64"/%s", - __func__, - w_item ? "CREATED" : "FAILED TO CREATE", - userid, workername); - } + } + // Always at least log_notice worker created (for !hasworker) + if (w_none) { + int sta = LOG_ERR; + if (w_item) + sta = LOG_NOTICE; + LOGMSG(sta, + "%s(): %s Missing worker %"PRId64"/%s", + __func__, + w_item ? "CREATED" : "FAILED TO CREATE", + userid, workername); } } return ws_item; @@ -909,6 +916,7 @@ K_ITEM *_find_create_workerstatus(bool lock, int64_t userid, char *workername, // workerstatus must be locked static void zero_on_idle(tv_t *when, WORKERSTATUS *workerstatus) { + LIST_WRITE(workerstatus_free); copy_tv(&(workerstatus->active_start), when); workerstatus->active_diffacc = workerstatus->active_diffinv = workerstatus->active_diffsta = workerstatus->active_diffdup = @@ -1002,8 +1010,9 @@ void _workerstatus_update(AUTHS *auths, SHARES *shares, K_ITEM *item; if (auths) { - item = find_workerstatus(true, auths->userid, auths->workername, - file, func, line); + item = find_create_workerstatus(false, false, auths->userid, + auths->workername, false, + file, func, line); if (item) { DATA_WORKERSTATUS(row, item); K_WLOCK(workerstatus_free); @@ -1023,9 +1032,9 @@ void _workerstatus_update(AUTHS *auths, SHARES *shares, pool.diffinv += shares->diff; pool.shareinv++; } - item = find_workerstatus(true, shares->userid, - shares->workername, - file, func, line); + item = find_create_workerstatus(false, true, shares->userid, + shares->workername, false, + file, func, line); if (item) { DATA_WORKERSTATUS(row, item); K_WLOCK(workerstatus_free); @@ -1092,9 +1101,9 @@ void _workerstatus_update(AUTHS *auths, SHARES *shares, } if (startup_complete && userstats) { - item = find_workerstatus(true, userstats->userid, - userstats->workername, - file, func, line); + item = find_create_workerstatus(false, true, userstats->userid, + userstats->workername, false, + file, func, line); if (item) { DATA_WORKERSTATUS(row, item); K_WLOCK(workerstatus_free); @@ -1480,11 +1489,11 @@ cmp_t cmp_workers(K_ITEM *a, K_ITEM *b) return c; } -K_ITEM *find_workers(int64_t userid, char *workername) +K_ITEM *find_workers(bool gotlock, int64_t userid, char *workername) { WORKERS workers; K_TREE_CTX ctx[1]; - K_ITEM look; + K_ITEM look, *w_item; workers.userid = userid; STRNCPY(workers.workername, workername); @@ -1493,9 +1502,15 @@ K_ITEM *find_workers(int64_t userid, char *workername) INIT_WORKERS(&look); look.data = (void *)(&workers); - return find_in_ktree(workers_root, &look, ctx); + if (!gotlock) + K_RLOCK(workers_free); + w_item = find_in_ktree(workers_root, &look, ctx); + if (!gotlock) + K_RUNLOCK(workers_free); + return w_item; } +// Requires at least K_RLOCK K_ITEM *first_workers(int64_t userid, K_TREE_CTX *ctx) { WORKERS workers; @@ -1522,7 +1537,7 @@ K_ITEM *new_worker(PGconn *conn, bool update, int64_t userid, char *workername, { K_ITEM *item; - item = find_workers(userid, workername); + item = find_workers(false, userid, workername); if (item) { if (!confirm_sharesummary && update) { workers_update(conn, item, diffdef, idlenotificationenabled, @@ -1538,9 +1553,10 @@ K_ITEM *new_worker(PGconn *conn, bool update, int64_t userid, char *workername, } // TODO: limit how many? - item = workers_add(conn, true, userid, workername, diffdef, - idlenotificationenabled, idlenotificationtime, - by, code, inet, cd, trf_root); + item = workers_add(conn, userid, workername, true, + diffdef, idlenotificationenabled, + idlenotificationtime, by, code, inet, cd, + trf_root); } return item; } @@ -1553,30 +1569,6 @@ K_ITEM *new_default_worker(PGconn *conn, bool update, int64_t userid, char *work by, code, inet, cd, trf_root); } -/* unused -static K_ITEM *new_worker_find_user(PGconn *conn, bool update, char *username, - char *workername, char *diffdef, - char *idlenotificationenabled, - char *idlenotificationtime, - char *by, char *code, char *inet, - tv_t *cd, K_TREE *trf_root) -{ - K_ITEM *item; - USERS *users; - - K_RLOCK(users_free); - item = find_users(username); - K_RUNLOCK(users_free); - if (!item) - return NULL; - - DATA_USERS(users, item); - return new_worker(conn, update, users->userid, workername, diffdef, - idlenotificationenabled, idlenotificationtime, - by, code, inet, cd, trf_root); -} -*/ - void dsp_paymentaddresses(K_ITEM *item, FILE *stream) { char expirydate_buf[DATE_BUFSIZ], createdate_buf[DATE_BUFSIZ]; @@ -1855,7 +1847,6 @@ static bool _reward_override_name(int32_t height, char *buf, size_t siz, return true; } -// Must be R or W locked before call K_ITEM *find_optioncontrol(char *optionname, const tv_t *now, int32_t height) { OPTIONCONTROL optioncontrol, *oc, *ocbest; @@ -1889,6 +1880,7 @@ K_ITEM *find_optioncontrol(char *optionname, const tv_t *now, int32_t height) INIT_OPTIONCONTROL(&look); look.data = (void *)(&optioncontrol); + K_RLOCK(optioncontrol_free); item = find_after_in_ktree(optioncontrol_root, &look, ctx); ocbest = NULL; best = NULL; @@ -1913,6 +1905,7 @@ K_ITEM *find_optioncontrol(char *optionname, const tv_t *now, int32_t height) } item = next_in_ktree(ctx); } + K_RUNLOCK(optioncontrol_free); return best; } @@ -1942,9 +1935,7 @@ int64_t user_sys_setting(int64_t userid, char *setting_name, } } - K_RLOCK(optioncontrol_free); oc_item = find_optioncontrol(setting_name, now, pool.height); - K_RUNLOCK(optioncontrol_free); if (oc_item) { DATA_OPTIONCONTROL(optioncontrol, oc_item); return (int64_t)atol(optioncontrol->optionvalue); @@ -2019,7 +2010,7 @@ cmp_t cmp_workinfo_height(K_ITEM *a, K_ITEM *b) return c; } -K_ITEM *find_workinfo(int64_t workinfoid, K_TREE_CTX *ctx) +K_ITEM *_find_workinfo(int64_t workinfoid, bool gotlock, K_TREE_CTX *ctx) { WORKINFO workinfo; K_TREE_CTX ctx0[1]; @@ -2034,9 +2025,11 @@ K_ITEM *find_workinfo(int64_t workinfoid, K_TREE_CTX *ctx) INIT_WORKINFO(&look); look.data = (void *)(&workinfo); - K_RLOCK(workinfo_free); + if (!gotlock) + K_RLOCK(workinfo_free); item = find_in_ktree(workinfo_root, &look, ctx); - K_RUNLOCK(workinfo_free); + if (!gotlock) + K_RUNLOCK(workinfo_free); return item; } @@ -2104,11 +2097,13 @@ bool workinfo_age(int64_t workinfoid, char *poolinstance, char *by, char *code, if (strcmp(poolinstance, workinfo->poolinstance) != 0) { tv_to_buf(cd, cd_buf, sizeof(cd_buf)); LOGERR("%s() %"PRId64"/%s/%ld,%ld %.19s Poolinstance changed " - "(from %s)! Age discarded!", +// "(from %s)! Age discarded!", + "(from %s)! Age not discarded", __func__, workinfoid, poolinstance, cd->tv_sec, cd->tv_usec, cd_buf, workinfo->poolinstance); - goto bye; +// TODO: ckdb only supports one, so until multiple support is written: +// goto bye; } K_RLOCK(workmarkers_free); @@ -2256,7 +2251,7 @@ double coinbase_reward(int32_t height) } // The PPS value of a 1diff share for the given workinfoid -double workinfo_pps(K_ITEM *w_item, int64_t workinfoid, bool lock) +double workinfo_pps(K_ITEM *w_item, int64_t workinfoid) { OPTIONCONTROL *optioncontrol; K_ITEM *oc_item; @@ -2265,12 +2260,9 @@ double workinfo_pps(K_ITEM *w_item, int64_t workinfoid, bool lock) // Allow optioncontrol override for a given workinfoid snprintf(oc_name, sizeof(oc_name), PPSOVERRIDE"_%"PRId64, workinfoid); - if (lock) - K_RLOCK(optioncontrol_free); + // No time/height control is used, just find the latest record oc_item = find_optioncontrol(oc_name, &date_eot, MAX_HEIGHT); - if (lock) - K_RUNLOCK(optioncontrol_free); // Value is a floating point double of satoshi if (oc_item) { @@ -2384,6 +2376,7 @@ cmp_t cmp_sharesummary_workinfoid(K_ITEM *a, K_ITEM *b) void zero_sharesummary(SHARESUMMARY *row) { + LIST_WRITE(sharesummary_free); row->diffacc = row->diffsta = row->diffdup = row->diffhi = row->diffrej = row->shareacc = row->sharesta = row->sharedup = row->sharehi = row->sharerej = 0.0; @@ -2397,6 +2390,7 @@ void zero_sharesummary(SHARESUMMARY *row) row->complete[1] = '\0'; } +// Must be R or W locked K_ITEM *_find_sharesummary(int64_t userid, char *workername, int64_t workinfoid, bool pool) { SHARESUMMARY sharesummary; @@ -2415,6 +2409,7 @@ K_ITEM *_find_sharesummary(int64_t userid, char *workername, int64_t workinfoid, return find_in_ktree(sharesummary_root, &look, ctx); } +// Must be R or W locked K_ITEM *find_last_sharesummary(int64_t userid, char *workername) { SHARESUMMARY look_sharesummary, *sharesummary; @@ -2751,14 +2746,17 @@ const char *blocks_confirmed(char *confirmed) return blocks_unknown; } -void zero_on_new_block(bool lock) +void zero_on_new_block(bool gotlock) { WORKERSTATUS *workerstatus; K_TREE_CTX ctx[1]; K_ITEM *ws_item; - if (lock) + if (gotlock) + LIST_WRITE(workerstatus_free); + else K_WLOCK(workerstatus_free); + pool.diffacc = pool.diffinv = pool.shareacc = pool.shareinv = pool.best_sdiff = 0; ws_item = first_in_ktree(workerstatus_root, ctx); @@ -2772,7 +2770,7 @@ void zero_on_new_block(bool lock) workerstatus->block_sharehi = workerstatus->block_sharerej = 0.0; ws_item = next_in_ktree(ctx); } - if (lock) + if (!gotlock) K_WUNLOCK(workerstatus_free); } @@ -2791,7 +2789,7 @@ void set_block_share_counters() INIT_SHARESUMMARY(&ss_look); INIT_MARKERSUMMARY(&ms_look); - zero_on_new_block(false); + zero_on_new_block(true); ws_item = NULL; /* From the end backwards so we can skip the workinfoid's we don't @@ -2817,18 +2815,14 @@ void set_block_share_counters() if (!ws_item || sharesummary->userid != workerstatus->userid || strcmp(sharesummary->workername, workerstatus->workername)) { - /* This is to trigger a console error if it is missing - * since it should always exist - * However, it is simplest to simply create it - * and keep going */ - ws_item = find_workerstatus(false, sharesummary->userid, - sharesummary->workername, - __FILE__, __func__, __LINE__); - if (!ws_item) { - ws_item = find_create_workerstatus(false, sharesummary->userid, - sharesummary->workername, - __FILE__, __func__, __LINE__); - } + /* Trigger a console error if it is missing since it + * should already exist, however, it is simplest to + * create it and keep going */ + ws_item = find_create_workerstatus(true, true, + sharesummary->userid, + sharesummary->workername, + false, __FILE__, + __func__, __LINE__); DATA_WORKERSTATUS(workerstatus, ws_item); } @@ -2900,18 +2894,14 @@ void set_block_share_counters() if (!ws_item || markersummary->userid != workerstatus->userid || strcmp(markersummary->workername, workerstatus->workername)) { - /* This is to trigger a console error if it is missing - * since it should always exist - * However, it is simplest to simply create it - * and keep going */ - ws_item = find_workerstatus(false, markersummary->userid, - markersummary->workername, - __FILE__, __func__, __LINE__); - if (!ws_item) { - ws_item = find_create_workerstatus(false, markersummary->userid, - markersummary->workername, - __FILE__, __func__, __LINE__); - } + /* Trigger a console error if it is missing since it + * should already exist, however, it is simplest to + * create it and keep going */ + ws_item = find_create_workerstatus(true, true, + markersummary->userid, + markersummary->workername, + false, __FILE__, __func__, + __LINE__); DATA_WORKERSTATUS(workerstatus, ws_item); } @@ -2948,8 +2938,7 @@ void set_block_share_counters() LOGWARNING("%s(): Update block counters complete", __func__); } -/* Must be under K_WLOCK(blocks_free) when called - * Call this before using the block stats and again check (under lock) +/* Call this before using the block stats and again check (under lock) * the blocks_stats_time didn't change after you finish processing * If it has changed, redo the processing from scratch * If return is false, then stats aren't available @@ -2965,6 +2954,7 @@ bool check_update_blocks_stats(tv_t *stats) WORKINFO *workinfo; BLOCKS *blocks; double ok, diffacc, netsumm, diffmean, pending, txmean, cr; + bool ret = false; tv_t now; /* Wait for startup_complete rather than db_load_complete @@ -2972,6 +2962,8 @@ bool check_update_blocks_stats(tv_t *stats) if (!startup_complete) return false; + K_RLOCK(workinfo_free); + K_WLOCK(blocks_free); if (blocks_stats_rebuild) { /* Have to first work out the diffcalc for each block * Orphans count towards the next valid block after the orphan @@ -2993,16 +2985,13 @@ bool check_update_blocks_stats(tv_t *stats) } b_item = next_in_ktree(ctx); } - ok = diffacc = netsumm = diffmean = 0.0, txmean = 0.0; + ok = diffacc = netsumm = diffmean = txmean = 0.0; b_item = last_in_ktree(blocks_root, ctx); while (b_item) { DATA_BLOCKS(blocks, b_item); if (CURRENT(&(blocks->expirydate))) { if (blocks->netdiff == 0) { - // Deadlock alert - K_RLOCK(workinfo_free); - w_item = find_workinfo(blocks->workinfoid, NULL); - K_RUNLOCK(workinfo_free); + w_item = _find_workinfo(blocks->workinfoid, true, NULL); if (!w_item) { setnow(&now); if (blocks->workinfoid != last_missing_workinfoid || @@ -3015,7 +3004,7 @@ bool check_update_blocks_stats(tv_t *stats) } last_missing_workinfoid = blocks->workinfoid; copy_tv(&last_message, &now); - return false; + goto bailout; } DATA_WORKINFO(workinfo, w_item); blocks->netdiff = workinfo->diff_target; @@ -3076,10 +3065,14 @@ bool check_update_blocks_stats(tv_t *stats) blocks_stats_rebuild = false; } copy_tv(stats, &blocks_stats_time); - return true; + ret = true; +bailout: + K_WUNLOCK(blocks_free); + K_RUNLOCK(workinfo_free); + return ret; } -// Must be under K_WLOCK(blocks_free) when called except during DB load +// Must be under K_WLOCK(blocks_free) when called bool _set_blockcreatedate(int32_t oldest_height, WHERE_FFL_ARGS) { K_TREE_CTX ctx[1]; @@ -3091,6 +3084,8 @@ bool _set_blockcreatedate(int32_t oldest_height, WHERE_FFL_ARGS) tv_t createdate; bool ok = true; + _LIST_WRITE(blocks_free, true, file, func, line); + // No blocks? if (blocks_store->count == 0) return true; @@ -3134,7 +3129,7 @@ bool _set_blockcreatedate(int32_t oldest_height, WHERE_FFL_ARGS) return ok; } -// Must be under K_WLOCK(blocks_free) when called except during DB load +// Must be under K_RLOCK(workinfo_free) and K_WLOCK(blocks_free) when called bool _set_prevcreatedate(int32_t oldest_height, WHERE_FFL_ARGS) { K_ITEM look, *b_item = NULL, *wi_item; @@ -3189,9 +3184,7 @@ bool _set_prevcreatedate(int32_t oldest_height, WHERE_FFL_ARGS) } else { /* There's none before oldest_height, so instead use: * 'Pool Start' = first workinfo createdate */ - K_RLOCK(workinfo_free); wi_item = first_in_ktree(workinfo_root, wi_ctx); - K_RUNLOCK(workinfo_free); if (wi_item) { DATA_WORKINFO(workinfo, wi_item); copy_tv(&prev_createdate, &(workinfo->createdate)); @@ -3273,6 +3266,7 @@ cmp_t cmp_miningpayouts(K_ITEM *a, K_ITEM *b) return c; } +// Must be R or W locked K_ITEM *find_miningpayouts(int64_t payoutid, int64_t userid) { MININGPAYOUTS miningpayouts; @@ -3289,6 +3283,7 @@ K_ITEM *find_miningpayouts(int64_t payoutid, int64_t userid) return find_in_ktree(miningpayouts_root, &look, ctx); } +// Must be R or W locked K_ITEM *first_miningpayouts(int64_t payoutid, K_TREE_CTX *ctx) { MININGPAYOUTS miningpayouts; @@ -3320,8 +3315,9 @@ cmp_t cmp_mu(K_ITEM *a, K_ITEM *b) return CMP_BIGINT(ma->userid, mb->userid); } -// update the userid record or add a new one if the userid isn't already present -K_TREE *upd_add_mu(K_TREE *mu_root, K_STORE *mu_store, int64_t userid, +/* update the userid record or add a new one if the userid isn't already present + * K_WLOCK(miningpayouts_free) required before calling, for 'A*' below */ +void upd_add_mu(K_TREE *mu_root, K_STORE *mu_store, int64_t userid, double diffacc) { MININGPAYOUTS lookminingpayouts, *miningpayouts; @@ -3332,22 +3328,19 @@ K_TREE *upd_add_mu(K_TREE *mu_root, K_STORE *mu_store, int64_t userid, INIT_MININGPAYOUTS(&look); look.data = (void *)(&lookminingpayouts); // No locking required since it's not a shared tree or store - mu_item = find_in_ktree(mu_root, &look, ctx); + mu_item = find_in_ktree_nolock(mu_root, &look, ctx); if (mu_item) { DATA_MININGPAYOUTS(miningpayouts, mu_item); miningpayouts->diffacc += diffacc; } else { - K_WLOCK(mu_store); + // A* requires K_WLOCK(miningpayouts_free) mu_item = k_unlink_head(miningpayouts_free); DATA_MININGPAYOUTS(miningpayouts, mu_item); miningpayouts->userid = userid; miningpayouts->diffacc = diffacc; - add_to_ktree(mu_root, mu_item); - k_add_head(mu_store, mu_item); - K_WUNLOCK(mu_store); + add_to_ktree_nolock(mu_root, mu_item); + k_add_head_nolock(mu_store, mu_item); } - - return mu_root; } // order by height asc,blockhash asc,expirydate asc @@ -3393,6 +3386,7 @@ cmp_t cmp_payouts_wid(K_ITEM *a, K_ITEM *b) return c; } +// Must be R or W locked K_ITEM *find_payouts(int32_t height, char *blockhash) { PAYOUTS payouts; @@ -3445,6 +3439,7 @@ K_ITEM *find_last_payouts() return NULL; } +// Must be R or W locked K_ITEM *find_payoutid(int64_t payoutid) { PAYOUTS payouts; @@ -3601,7 +3596,7 @@ bool process_pplns(int32_t height, char *blockhash, tv_t *addr_cd) * and simply avoids the problems that would cause without much more * strict locking than is used already */ - ck_wlock(&process_pplns_lock); + K_WLOCK(process_pplns_free); setnow(&now); @@ -3662,10 +3657,8 @@ bool process_pplns(int32_t height, char *blockhash, tv_t *addr_cd) DATA_WORKINFO(workinfo, w_item); // Get the PPLNS N values - K_RLOCK(optioncontrol_free); oc_item = find_optioncontrol(PPLNSDIFFTIMES, &(blocks->blockcreatedate), height); - K_RUNLOCK(optioncontrol_free); if (!oc_item) { tv_to_buf(&(blocks->blockcreatedate), cd_buf, sizeof(cd_buf)); LOGEMERG("%s(): missing optioncontrol %s (%s/%"PRId32")", @@ -3675,10 +3668,8 @@ bool process_pplns(int32_t height, char *blockhash, tv_t *addr_cd) DATA_OPTIONCONTROL(optioncontrol, oc_item); diff_times = atof(optioncontrol->optionvalue); - K_RLOCK(optioncontrol_free); oc_item = find_optioncontrol(PPLNSDIFFADD, &(blocks->blockcreatedate), height); - K_RUNLOCK(optioncontrol_free); if (!oc_item) { tv_to_buf(&(blocks->blockcreatedate), cd_buf, sizeof(cd_buf)); LOGEMERG("%s(): missing optioncontrol %s (%s/%"PRId32")", @@ -3713,16 +3704,17 @@ bool process_pplns(int32_t height, char *blockhash, tv_t *addr_cd) ss_count = wm_count = ms_count = 0; mu_store = k_new_store(miningpayouts_free); - mu_root = new_ktree(cmp_mu); + mu_root = new_ktree(cmp_mu, miningpayouts_free); looksharesummary.workinfoid = blocks->workinfoid; looksharesummary.userid = MAXID; looksharesummary.workername = EMPTY; INIT_SHARESUMMARY(&ss_look); ss_look.data = (void *)(&looksharesummary); + K_WLOCK(miningpayouts_free); K_RLOCK(sharesummary_free); - K_RLOCK(workmarkers_free); K_RLOCK(markersummary_free); + K_RLOCK(workmarkers_free); ss_item = find_before_in_ktree(sharesummary_workinfoid_root, &ss_look, ss_ctx); DATA_SHARESUMMARY_NULL(sharesummary, ss_item); @@ -3740,16 +3732,17 @@ bool process_pplns(int32_t height, char *blockhash, tv_t *addr_cd) break; default: // Release ASAP - K_RUNLOCK(markersummary_free); K_RUNLOCK(workmarkers_free); + K_RUNLOCK(markersummary_free); K_RUNLOCK(sharesummary_free); + K_WUNLOCK(miningpayouts_free); LOGERR("%s(): sharesummary not ready %" PRId64"/%s/%"PRId64"/%s. allow_aged=%s", __func__, sharesummary->userid, sharesummary->workername, sharesummary->workinfoid, sharesummary->complete, - allow_aged ? "true" : "false"); + TFSTR(allow_aged)); goto shazbot; } @@ -3764,9 +3757,8 @@ bool process_pplns(int32_t height, char *blockhash, tv_t *addr_cd) begin_workinfoid = sharesummary->workinfoid; if (tv_newer(&end_tv, &(sharesummary->lastshareacc))) copy_tv(&end_tv, &(sharesummary->lastshareacc)); - mu_root = upd_add_mu(mu_root, mu_store, - sharesummary->userid, - sharesummary->diffacc); + upd_add_mu(mu_root, mu_store, sharesummary->userid, + sharesummary->diffacc); ss_item = prev_in_ktree(ss_ctx); DATA_SHARESUMMARY_NULL(sharesummary, ss_item); } @@ -3784,13 +3776,14 @@ bool process_pplns(int32_t height, char *blockhash, tv_t *addr_cd) K_RUNLOCK(markersummary_free); K_RUNLOCK(workmarkers_free); K_RUNLOCK(sharesummary_free); + K_WUNLOCK(miningpayouts_free); LOGERR("%s(): sharesummary2 not ready %" PRId64"/%s/%"PRId64"/%s. allow_aged=%s", __func__, sharesummary->userid, sharesummary->workername, sharesummary->workinfoid, sharesummary->complete, - allow_aged ? "true" : "false"); + TFSTR(allow_aged)); goto shazbot; } ss_count++; @@ -3799,9 +3792,8 @@ bool process_pplns(int32_t height, char *blockhash, tv_t *addr_cd) total_diff += sharesummary->diffacc; if (tv_newer(&end_tv, &(sharesummary->lastshareacc))) copy_tv(&end_tv, &(sharesummary->lastshareacc)); - mu_root = upd_add_mu(mu_root, mu_store, - sharesummary->userid, - sharesummary->diffacc); + upd_add_mu(mu_root, mu_store, sharesummary->userid, + sharesummary->diffacc); ss_item = prev_in_ktree(ss_ctx); DATA_SHARESUMMARY_NULL(sharesummary, ss_item); } @@ -3849,9 +3841,8 @@ bool process_pplns(int32_t height, char *blockhash, tv_t *addr_cd) begin_workinfoid = workmarkers->workinfoidstart; if (tv_newer(&end_tv, &(markersummary->lastshareacc))) copy_tv(&end_tv, &(markersummary->lastshareacc)); - mu_root = upd_add_mu(mu_root, mu_store, - markersummary->userid, - markersummary->diffacc); + upd_add_mu(mu_root, mu_store, markersummary->userid, + markersummary->diffacc); ms_item = prev_in_ktree(ms_ctx); DATA_MARKERSUMMARY_NULL(markersummary, ms_item); } @@ -3862,9 +3853,10 @@ bool process_pplns(int32_t height, char *blockhash, tv_t *addr_cd) LOGDEBUG("%s(): wm %"PRId64" ms %"PRId64" total %.1f want %.1f", __func__, wm_count, ms_count, total_diff, diff_want); } - K_RUNLOCK(markersummary_free); K_RUNLOCK(workmarkers_free); + K_RUNLOCK(markersummary_free); K_RUNLOCK(sharesummary_free); + K_WUNLOCK(miningpayouts_free); usercount = mu_store->count; @@ -3998,7 +3990,7 @@ bool process_pplns(int32_t height, char *blockhash, tv_t *addr_cd) // Update and store the miningpayouts and payments pay_store = k_new_store(payments_free); - mu_item = first_in_ktree(mu_root, mu_ctx); + mu_item = first_in_ktree_nolock(mu_root, mu_ctx); while (mu_item) { DATA_MININGPAYOUTS(miningpayouts, mu_item); @@ -4063,7 +4055,7 @@ bool process_pplns(int32_t height, char *blockhash, tv_t *addr_cd) } K_WUNLOCK(paymentaddresses_free); - pa_item = addr_store->head; + pa_item = STORE_HEAD_NOLOCK(addr_store); if (pa_item) { // Normal user with at least 1 paymentaddress while (pa_item) { @@ -4087,7 +4079,7 @@ bool process_pplns(int32_t height, char *blockhash, tv_t *addr_cd) (double)(pa->payratio) / (double)paytotal; used += d64; - k_add_tail(pay_store, pay_item); + k_add_tail_nolock(pay_store, pay_item); ok = payments_add(conn, true, pay_item, &(payments->old_item), (char *)by_default, @@ -4116,7 +4108,7 @@ bool process_pplns(int32_t height, char *blockhash, tv_t *addr_cd) payments->amount = amount; payments->diffacc = miningpayouts->diffacc; used = amount; - k_add_tail(pay_store, pay_item); + k_add_tail_nolock(pay_store, pay_item); ok = payments_add(conn, true, pay_item, &(payments->old_item), (char *)by_default, @@ -4146,11 +4138,11 @@ bool process_pplns(int32_t height, char *blockhash, tv_t *addr_cd) if (addr_store->count) { K_WLOCK(paymentaddresses_free); k_list_transfer_to_head(addr_store, paymentaddresses_free); - K_WUNLOCK(addr_store); + K_WUNLOCK(paymentaddresses_free); } addr_store = k_free_store(addr_store); - mu_item = next_in_ktree(mu_ctx); + mu_item = next_in_ktree_nolock(mu_ctx); } // begun is true @@ -4159,19 +4151,19 @@ bool process_pplns(int32_t height, char *blockhash, tv_t *addr_cd) payouts_add_ram(true, p_item, old_p_item, &now); free_ktree(mu_root, NULL); - mu_item = k_unlink_head(mu_store); + mu_item = k_unlink_head_nolock(mu_store); while (mu_item) { DATA_MININGPAYOUTS(miningpayouts, mu_item); miningpayouts_add_ram(true, mu_item, miningpayouts->old_item, &now); - mu_item = k_unlink_head(mu_store); + mu_item = k_unlink_head_nolock(mu_store); } mu_store = k_free_store(mu_store); - pay_item = k_unlink_head(pay_store); + pay_item = k_unlink_head_nolock(pay_store); while (pay_item) { DATA_PAYMENTS(payments, pay_item); payments_add_ram(true, pay_item, payments->old_item, &now); - pay_item = k_unlink_head(pay_store); + pay_item = k_unlink_head_nolock(pay_store); } pay_store = k_free_store(pay_store); @@ -4225,7 +4217,7 @@ bool process_pplns(int32_t height, char *blockhash, tv_t *addr_cd) } // Flag each shift as rewarded - reward_shifts(payouts2, true, 1); + reward_shifts(payouts2, 1); CKPQDisco(&conn, conned); @@ -4247,30 +4239,30 @@ shazbot: oku: ; - ck_wunlock(&process_pplns_lock); + K_WUNLOCK(process_pplns_free); if (mu_root) free_ktree(mu_root, NULL); if (mu_store) { if (mu_store->count) { - K_WLOCK(mu_store); + K_WLOCK(miningpayouts_free); k_list_transfer_to_head(mu_store, miningpayouts_free); - K_WUNLOCK(mu_store); + K_WUNLOCK(miningpayouts_free); } mu_store = k_free_store(mu_store); } if (pay_store) { if (pay_store->count) { - K_WLOCK(pay_store); + K_WLOCK(payments_free); k_list_transfer_to_head(pay_store, payments_free); - K_WUNLOCK(pay_store); + K_WUNLOCK(payments_free); } pay_store = k_free_store(pay_store); } if (addr_store) { if (addr_store->count) { - K_WLOCK(addr_store); + K_WLOCK(paymentaddresses_free); k_list_transfer_to_head(addr_store, paymentaddresses_free); - K_WUNLOCK(addr_store); + K_WUNLOCK(paymentaddresses_free); } addr_store = k_free_store(addr_store); } @@ -4339,6 +4331,7 @@ cmp_t cmp_userstats(K_ITEM *a, K_ITEM *b) return c; } +// Must be R or W locked K_ITEM *find_userstats(int64_t userid, char *workername) { USERSTATS userstats; @@ -4426,6 +4419,7 @@ K_ITEM *find_markersummary_userid(int64_t userid, char *workername, return ms_item; } +// Must be R or W locked K_ITEM *_find_markersummary(int64_t markerid, int64_t workinfoid, int64_t userid, char *workername, bool pool) { @@ -4517,10 +4511,10 @@ bool make_markersummaries(bool msg, char *by, char *code, char *inet, /* So we can't change any sharesummaries/markersummaries while a * payout is being generated * N.B. this is a long lock since it stores the markersummaries */ - ck_wlock(&process_pplns_lock); + K_WLOCK(process_pplns_free); ok = sharesummaries_to_markersummaries(NULL, workmarkers, by, code, inet, &now, trf_root); - ck_wunlock(&process_pplns_lock); + K_WUNLOCK(process_pplns_free); return ok; } @@ -4566,6 +4560,7 @@ cmp_t cmp_workmarkers_workinfoid(K_ITEM *a, K_ITEM *b) return c; } +// requires K_RLOCK(workmarkers_free) K_ITEM *find_workmarkers(int64_t workinfoid, bool anystatus, char status, K_TREE_CTX *ctx) { WORKMARKERS workmarkers, *wm; @@ -4593,6 +4588,7 @@ K_ITEM *find_workmarkers(int64_t workinfoid, bool anystatus, char status, K_TREE return wm_item; } +// requires K_RLOCK(workmarkers_free) K_ITEM *find_workmarkerid(int64_t markerid, bool anystatus, char status) { WORKMARKERS workmarkers, *wm; @@ -4869,7 +4865,7 @@ bool workmarkers_generate(PGconn *conn, char *err, size_t siz, char *by, } // delta = 1 or -1 i.e. reward or undo reward -bool reward_shifts(PAYOUTS *payouts, bool lock, int delta) +bool reward_shifts(PAYOUTS *payouts, int delta) { // TODO: PPS calculations K_TREE_CTX ctx[1]; @@ -4881,8 +4877,7 @@ bool reward_shifts(PAYOUTS *payouts, bool lock, int delta) payout_pps = (double)delta * (double)(payouts->minerreward) / payouts->diffused; - if (lock) - K_WLOCK(workmarkers_free); + K_WLOCK(workmarkers_free); wm_item = find_workmarkers(payouts->workinfoidstart, false, MARKER_PROCESSED, ctx); @@ -4900,8 +4895,7 @@ bool reward_shifts(PAYOUTS *payouts, bool lock, int delta) wm_item = next_in_ktree(ctx); } - if (lock) - K_WUNLOCK(workmarkers_free); + K_WUNLOCK(workmarkers_free); return did_one; } @@ -4926,7 +4920,6 @@ bool shift_rewards(K_ITEM *wm_item) DATA_WORKMARKERS(wm, wm_item); - // Deadlock risk since calling code should have workmarkers locked K_RLOCK(payouts_free); p_item = find_payouts_wid(wm->workinfoidend, ctx); DATA_PAYOUTS_NULL(payouts, p_item); @@ -5134,7 +5127,7 @@ cmp_t cmp_userinfo(K_ITEM *a, K_ITEM *b) return CMP_BIGINT(ua->userid, ub->userid); } -K_ITEM *_get_userinfo(int64_t userid, bool lock) +K_ITEM *get_userinfo(int64_t userid) { USERINFO userinfo; K_TREE_CTX ctx[1]; @@ -5144,31 +5137,23 @@ K_ITEM *_get_userinfo(int64_t userid, bool lock) INIT_USERINFO(&look); look.data = (void *)(&userinfo); - if (lock) - K_RLOCK(userinfo_free); find = find_in_ktree(userinfo_root, &look, ctx); - if (lock) - K_RUNLOCK(userinfo_free); return find; } -K_ITEM *_find_create_userinfo(int64_t userid, bool lock, WHERE_FFL_ARGS) +K_ITEM *_find_create_userinfo(int64_t userid, WHERE_FFL_ARGS) { K_ITEM *ui_item, *u_item; USERS *users = NULL; USERINFO *row; - ui_item = _get_userinfo(userid, lock); + ui_item = get_userinfo(userid); if (!ui_item) { - if (lock) - K_RLOCK(users_free); + K_RLOCK(users_free); u_item = find_userid(userid); - if (lock) - K_RUNLOCK(users_free); + K_RUNLOCK(users_free); DATA_USERS_NULL(users, u_item); - if (lock) - K_WLOCK(userinfo_free); ui_item = k_unlink_head(userinfo_free); DATA_USERINFO(row, ui_item); @@ -5181,23 +5166,20 @@ K_ITEM *_find_create_userinfo(int64_t userid, bool lock, WHERE_FFL_ARGS) add_to_ktree(userinfo_root, ui_item); k_add_head(userinfo_store, ui_item); - if (lock) - K_WUNLOCK(userinfo_free); } return ui_item; } -void _userinfo_update(SHARES *shares, SHARESUMMARY *sharesummary, - MARKERSUMMARY *markersummary, bool ss_sub, bool lock) +// Must be under K_WLOCK(userinfo_free) when called +void userinfo_update(SHARES *shares, SHARESUMMARY *sharesummary, + MARKERSUMMARY *markersummary, bool ss_sub) { USERINFO *row; K_ITEM *item; if (shares) { - item = _find_userinfo(shares->userid, lock); + item = find_create_userinfo(shares->userid); DATA_USERINFO(row, item); - if (lock) - K_WLOCK(userinfo_free); switch (shares->errn) { case SE_NONE: row->diffacc += shares->diff; @@ -5220,13 +5202,11 @@ void _userinfo_update(SHARES *shares, SHARESUMMARY *sharesummary, row->sharerej++; break; } - if (lock) - K_WUNLOCK(userinfo_free); } - // Only during db load so no locking required + // Only during db load if (sharesummary) { - item = _find_userinfo(sharesummary->userid, false); + item = find_create_userinfo(sharesummary->userid); DATA_USERINFO(row, item); if (ss_sub) { row->diffacc -= sharesummary->diffacc; @@ -5253,9 +5233,9 @@ void _userinfo_update(SHARES *shares, SHARESUMMARY *sharesummary, } } - // Only during db load so no locking required + // Only during db load if (markersummary) { - item = _find_userinfo(markersummary->userid, false); + item = find_create_userinfo(markersummary->userid); DATA_USERINFO(row, item); row->diffacc += markersummary->diffacc; row->diffsta += markersummary->diffsta; @@ -5271,15 +5251,15 @@ void _userinfo_update(SHARES *shares, SHARESUMMARY *sharesummary, } // N.B. good blocks = blocks - (orphans + rejects) -void _userinfo_block(BLOCKS *blocks, enum info_type isnew, int delta, bool lock) +void userinfo_block(BLOCKS *blocks, enum info_type isnew, int delta) { USERINFO *row; K_ITEM *item; - item = find_userinfo(blocks->userid); + K_WLOCK(userinfo_free); + + item = find_create_userinfo(blocks->userid); DATA_USERINFO(row, item); - if (lock) - K_WLOCK(userinfo_free); if (isnew == INFO_NEW) { row->blocks += delta; copy_tv(&(row->last_block), &(blocks->createdate)); @@ -5288,6 +5268,5 @@ void _userinfo_block(BLOCKS *blocks, enum info_type isnew, int delta, bool lock) else if (isnew == INFO_REJECT) row->rejects += delta; - if (lock) - K_WUNLOCK(userinfo_free); + K_WUNLOCK(userinfo_free); } diff --git a/src/ckdb_dbio.c b/src/ckdb_dbio.c index 90e16fb3..2aa6d8cf 100644 --- a/src/ckdb_dbio.c +++ b/src/ckdb_dbio.c @@ -565,7 +565,7 @@ K_ITEM *users_add(PGconn *conn, char *username, char *emailaddress, dup = false; K_RLOCK(users_free); - u_item = users_store->head; + u_item = STORE_RHEAD(users_store); while (u_item) { DATA_USERS(users, u_item); if (strcmp(row->usertrim, users->usertrim) == 0) { @@ -1292,10 +1292,10 @@ bool useratts_fill(PGconn *conn) return ok; } -K_ITEM *workers_add(PGconn *conn, bool lock, int64_t userid, char *workername, +K_ITEM *workers_add(PGconn *conn, int64_t userid, char *workername, bool add_ws, char *difficultydefault, char *idlenotificationenabled, - char *idlenotificationtime, char *by, - char *code, char *inet, tv_t *cd, K_TREE *trf_root) + char *idlenotificationtime, char *by, char *code, + char *inet, tv_t *cd, K_TREE *trf_root) { ExecStatusType rescode; bool conned = false; @@ -1310,11 +1310,23 @@ K_ITEM *workers_add(PGconn *conn, bool lock, int64_t userid, char *workername, LOGDEBUG("%s(): add", __func__); - if (lock) - K_WLOCK(workers_free); + /* Since shares can add workers and there's lotsa shares :) ... + * and workers_add() and workers_fill() are the only places where + * workers can be added, to ensure that the existence of the worker + * hasn't changed, we check it again under a lock here that's unique + * to workers_add() and workers_fill() + * i.e. multiple threads trying to add the same worker will only end + * up adding one and thus avoid wasted DB IO and avoid DB duplicate + * errors */ + K_WLOCK(workers_db_free); + + ret = find_workers(false, userid, workername); + if (ret) + goto hadit; + + K_WLOCK(workers_free); item = k_unlink_head(workers_free); - if (lock) - K_WUNLOCK(workers_free); + K_WUNLOCK(workers_free); DATA_WORKERS(row, item); @@ -1404,19 +1416,27 @@ unparam: unitem: if (conned) PQfinish(conn); - if (lock) - K_WLOCK(workers_free); + K_WLOCK(workers_free); if (!ret) k_add_head(workers_free, item); else { add_to_ktree(workers_root, item); k_add_head(workers_store, item); - // Ensure there is a matching workerstatus - find_create_workerstatus(lock, userid, workername, + } + K_WUNLOCK(workers_free); + +hadit: + ; + K_WUNLOCK(workers_db_free); + + if (ret && add_ws) { + /* Ensure there is a matching workerstatus + * WARNING - find_create_workerstatus() can call workers_add()! + * The hasworker=true argument guarantees it wont and + * add_ws=false above ensures it wont call back */ + find_create_workerstatus(false, false, userid, workername, true, __FILE__, __func__, __LINE__); } - if (lock) - K_WUNLOCK(workers_free); return ret; } @@ -1588,6 +1608,9 @@ bool workers_fill(PGconn *conn) return false; } + // See workers_add() about this lock + K_WLOCK(workers_db_free); + res = PQexec(conn, sel, CKPQ_READ); rescode = PQresultStatus(res); PQclear(res); @@ -1616,10 +1639,11 @@ bool workers_fill(PGconn *conn) n = 0; ok = true; - K_WLOCK(workers_free); while ((t = PQntuples(res)) > 0) { for (i = 0; i < t; i++) { + K_WLOCK(workers_free); item = k_unlink_head(workers_free); + K_WUNLOCK(workers_free); DATA_WORKERS(row, item); bzero(row, sizeof(*row)); @@ -1667,14 +1691,14 @@ bool workers_fill(PGconn *conn) break; TXT_TO_BIGINT("workerid", field, row->workerid); + K_WLOCK(workers_free); add_to_ktree(workers_root, item); k_add_head(workers_store, item); + K_WUNLOCK(workers_free); - /* Make sure a workerstatus exists for each worker - * This is to ensure that code can use the workerstatus tree - * to reference other tables and not miss workers in the - * other tables */ - find_create_workerstatus(false, row->userid, row->workername, + // Make sure a workerstatus exists for each worker + find_create_workerstatus(false, false, row->userid, + row->workername, true, __FILE__, __func__, __LINE__); tick(); n++; @@ -1691,12 +1715,13 @@ bool workers_fill(PGconn *conn) if (!ok) k_add_head(workers_free, item); - K_WUNLOCK(workers_free); PQclear(res); flail: res = PQexec(conn, "Commit", CKPQ_READ); PQclear(res); + K_WUNLOCK(workers_db_free); + if (ok) { LOGDEBUG("%s(): built", __func__); LOGWARNING("%s(): fetched %d workers records", __func__, n); @@ -1779,7 +1804,7 @@ bool paymentaddresses_set(PGconn *conn, int64_t userid, K_STORE *pa_store, break; // Find the RAM record in pa_store - match = pa_store->head; + match = STORE_HEAD_NOLOCK(pa_store); while (match) { DATA_PAYMENTADDRESSES(pa, match); if (strcmp(pa->payaddress, row->payaddress) == 0 && @@ -1804,7 +1829,7 @@ bool paymentaddresses_set(PGconn *conn, int64_t userid, K_STORE *pa_store, DATA_PAYMENTADDRESSES_NULL(row, item); } LOGDEBUG("%s(): Step 1 par=%d count=%d matches=%d first=%s", __func__, - par, count, matches, first ? "true" : "false"); + par, count, matches, TFSTR(first)); // Too many, or none need expiring = don't do the update if (count > ABS_ADDR_LIMIT || first == true) { for (n = 0; n < par; n++) @@ -1838,7 +1863,7 @@ bool paymentaddresses_set(PGconn *conn, int64_t userid, K_STORE *pa_store, HISTORYDATECONTROL ") values (" PQPARAM10 ")"; count = 0; - match = pa_store->head; + match = STORE_HEAD_NOLOCK(pa_store); while (match) { DATA_PAYMENTADDRESSES(row, match); if (!row->match) { @@ -1895,7 +1920,7 @@ unparam: free(params[n]); FREENULL(upd); // Third step - do step 1 and 2 to the RAM version of the DB - LOGDEBUG("%s(): Step 3, ok=%s", __func__, ok ? "true" : "false"); + LOGDEBUG("%s(): Step 3, ok=%s", __func__, TFSTR(ok)); matches = count = n = 0; if (ok) { // Change the expiry on all records that we expired in the DB @@ -1904,7 +1929,7 @@ unparam: while (item && CURRENT(&(row->expirydate)) && row->userid == userid) { prev = prev_in_ktree(ctx); // Find the RAM record in pa_store - match = pa_store->head; + match = STORE_HEAD_NOLOCK(pa_store); while (match) { DATA_PAYMENTADDRESSES(pa, match); if (strcmp(pa->payaddress, row->payaddress) == 0 && @@ -1930,7 +1955,7 @@ unparam: } // Add in all the non-matching ps_store - match = pa_store->head; + match = STORE_HEAD_NOLOCK(pa_store); while (match) { next = match->next; DATA_PAYMENTADDRESSES(pa, match); @@ -2943,7 +2968,7 @@ bool workinfo_fill(PGconn *conn) n = 0; ok = true; - //K_WLOCK(workinfo_free); + K_WLOCK(workinfo_free); while ((t = PQntuples(res)) > 0) { for (i = 0; i < t; i++) { item = k_unlink_head(workinfo_free); @@ -3055,10 +3080,13 @@ bool workinfo_fill(PGconn *conn) if (!ok) { free_workinfo_data(item); k_add_head(workinfo_free, item); - } else + } else { + K_WLOCK(blocks_free); ok = set_prevcreatedate(0); + K_WUNLOCK(blocks_free); + } - //K_WUNLOCK(workinfo_free); + K_WUNLOCK(workinfo_free); PQclear(res); flail: res = PQexec(conn, "Commit", CKPQ_READ); @@ -3140,8 +3168,10 @@ static bool shares_process(PGconn *conn, SHARES *shares, K_ITEM *wi_item, if (reloading && !confirm_sharesummary) { // We only need to know if the workmarker is processed + K_RLOCK(workmarkers_free); wm_item = find_workmarkers(shares->workinfoid, false, MARKER_PROCESSED, NULL); + K_RUNLOCK(workmarkers_free); if (wm_item) { LOGDEBUG("%s(): workmarker exists for wid %"PRId64 " %"PRId64"/%s/%ld,%ld", @@ -3153,8 +3183,10 @@ static bool shares_process(PGconn *conn, SHARES *shares, K_ITEM *wi_item, return false; } + K_RLOCK(sharesummary_free); ss_item = find_sharesummary(shares->userid, shares->workername, shares->workinfoid); + K_RUNLOCK(sharesummary_free); if (ss_item) { DATA_SHARESUMMARY(sharesummary, ss_item); if (sharesummary->complete[0] != SUMMARY_NEW) { @@ -3174,7 +3206,9 @@ static bool shares_process(PGconn *conn, SHARES *shares, K_ITEM *wi_item, if (!confirm_sharesummary) { workerstatus_update(NULL, shares, NULL); - userinfo_update(shares, NULL, NULL); + K_WLOCK(userinfo_free); + userinfo_update(shares, NULL, NULL, false); + K_WUNLOCK(userinfo_free); } sharesummary_update(shares, NULL, shares->createby, shares->createcode, @@ -3440,8 +3474,10 @@ static bool shareerrors_process(PGconn *conn, SHAREERRORS *shareerrors, if (reloading && !confirm_sharesummary) { // We only need to know if the workmarker is processed + K_RLOCK(workmarkers_free); wm_item = find_workmarkers(shareerrors->workinfoid, false, MARKER_PROCESSED, NULL); + K_RUNLOCK(workmarkers_free); if (wm_item) { LOGDEBUG("%s(): workmarker exists for wid %"PRId64 " %"PRId64"/%s/%ld,%ld", @@ -3454,9 +3490,11 @@ static bool shareerrors_process(PGconn *conn, SHAREERRORS *shareerrors, return false; } + K_RLOCK(sharesummary_free); ss_item = find_sharesummary(shareerrors->userid, shareerrors->workername, shareerrors->workinfoid); + K_RUNLOCK(sharesummary_free); if (ss_item) { DATA_SHARESUMMARY(sharesummary, ss_item); if (sharesummary->complete[0] != SUMMARY_NEW) { @@ -3775,7 +3813,7 @@ bool sharesummaries_to_markersummaries(PGconn *conn, WORKMARKERS *workmarkers, K_STORE *old_sharesummary_store = k_new_store(sharesummary_free); K_STORE *new_markersummary_store = k_new_store(markersummary_free); - K_TREE *ms_root = new_ktree(cmp_markersummary); + K_TREE *ms_root = new_ktree(cmp_markersummary, markersummary_free); if (!CURRENT(&(workmarkers->expirydate))) { reason = "unexpired"; @@ -3844,12 +3882,12 @@ bool sharesummaries_to_markersummaries(PGconn *conn, WORKMARKERS *workmarkers, lookmarkersummary.workername = sharesummary->workername; ms_look.data = (void *)(&lookmarkersummary); - ms_item = find_in_ktree(ms_root, &ms_look, ms_ctx); + ms_item = find_in_ktree_nolock(ms_root, &ms_look, ms_ctx); if (!ms_item) { K_WLOCK(markersummary_free); ms_item = k_unlink_head(markersummary_free); K_WUNLOCK(markersummary_free); - k_add_head(new_markersummary_store, ms_item); + k_add_head_nolock(new_markersummary_store, ms_item); DATA_MARKERSUMMARY(markersummary, ms_item); bzero(markersummary, sizeof(*markersummary)); markersummary->markerid = workmarkers->markerid; @@ -3857,7 +3895,7 @@ bool sharesummaries_to_markersummaries(PGconn *conn, WORKMARKERS *workmarkers, DUP_POINTER(markersummary_free, markersummary->workername, sharesummary->workername); - add_to_ktree(ms_root, ms_item); + add_to_ktree_nolock(ms_root, ms_item); LOGDEBUG("%s() new ms %"PRId64"/%"PRId64"/%s", shortname, markersummary->markerid, @@ -3900,8 +3938,10 @@ bool sharesummaries_to_markersummaries(PGconn *conn, WORKMARKERS *workmarkers, diffacc += sharesummary->diffacc; shareacc += sharesummary->shareacc; + K_WLOCK(sharesummary_free); k_unlink_item(sharesummary_store, ss_item); - k_add_head(old_sharesummary_store, ss_item); + K_WUNLOCK(sharesummary_free); + k_add_head_nolock(old_sharesummary_store, ss_item); ss_item = ss_prev; } @@ -3919,7 +3959,7 @@ bool sharesummaries_to_markersummaries(PGconn *conn, WORKMARKERS *workmarkers, goto flail; } - ms_item = new_markersummary_store->head; + ms_item = STORE_HEAD_NOLOCK(new_markersummary_store); while (ms_item) { if (!(markersummary_add(conn, ms_item, by, code, inet, cd, trf_root))) { @@ -3960,7 +4000,7 @@ flail: if (!ok) { if (new_markersummary_store->count > 0) { // Throw them away (they don't exist anywhere else) - ms_item = new_markersummary_store->head; + ms_item = STORE_HEAD_NOLOCK(new_markersummary_store); while (ms_item) { free_markersummary_data(ms_item); ms_item = ms_item->next; @@ -3978,10 +4018,11 @@ flail: } else { ms_count = new_markersummary_store->count; ss_count = old_sharesummary_store->count; - // Deadlock alert for other newer code ... + K_WLOCK(sharesummary_free); K_WLOCK(markersummary_free); - ms_item = new_markersummary_store->head; + K_RLOCK(workmarkers_free); + ms_item = STORE_HEAD_NOLOCK(new_markersummary_store); while (ms_item) { // move the new markersummaries into the trees/stores add_to_ktree(markersummary_root, ms_item); @@ -4007,7 +4048,7 @@ flail: /* For normal shift processing this wont be very quick * so it will be a 'long' LOCK */ - ss_item = old_sharesummary_store->head; + ss_item = STORE_HEAD_NOLOCK(old_sharesummary_store); while (ss_item) { // remove the old sharesummaries from the trees remove_from_ktree(sharesummary_root, ss_item); @@ -4027,6 +4068,7 @@ flail: ss_item = ss_item->next; } k_list_transfer_to_head(old_sharesummary_store, sharesummary_free); + K_RUNLOCK(workmarkers_free); K_WUNLOCK(markersummary_free); K_WUNLOCK(sharesummary_free); @@ -4088,8 +4130,8 @@ bool delete_markersummaries(PGconn *conn, WORKMARKERS *wm) INIT_MARKERSUMMARY(&ms_look); ms_look.data = (void *)(&lookmarkersummary); - K_WLOCK(workmarkers_free); K_WLOCK(markersummary_free); + K_WLOCK(workmarkers_free); ms_item = find_after_in_ktree(markersummary_root, &ms_look, ms_ctx); DATA_MARKERSUMMARY_NULL(markersummary, ms_item); @@ -4160,7 +4202,7 @@ flail: /* TODO: add a list garbage collection thread so as to not * invalidate the data immediately (free_*), rather after * some delay */ - ms_item = del_markersummary_store->head; + ms_item = STORE_HEAD_NOLOCK(del_markersummary_store); while (ms_item) { remove_from_ktree(markersummary_root, ms_item); remove_from_ktree(markersummary_userid_root, ms_item); @@ -4180,8 +4222,8 @@ flail: } } - K_WUNLOCK(markersummary_free); K_WUNLOCK(workmarkers_free); + K_WUNLOCK(markersummary_free); if (!ok) { // already displayed the full workmarkers detail at the top @@ -4209,6 +4251,8 @@ static void set_sharesummary_stats(SHARESUMMARY *row, SHARES *s_row, { tv_t *createdate; + K_WLOCK(sharesummary_free); + if (s_row) createdate = &(s_row->createdate); else @@ -4270,6 +4314,8 @@ static void set_sharesummary_stats(SHARESUMMARY *row, SHARES *s_row, *tdf = tvdiff(createdate, &(row->firstshare)); *tdl = tvdiff(createdate, &(row->lastshare)); } + + K_WUNLOCK(sharesummary_free); } /* Keep some simple stats on how often shares are out of order @@ -4978,6 +5024,7 @@ flail: if (conned) PQfinish(conn); + K_RLOCK(workinfo_free); K_WLOCK(blocks_free); if (!ok) k_add_head(blocks_free, b_item); @@ -4998,6 +5045,7 @@ flail: set_prevcreatedate(row->height); } K_WUNLOCK(blocks_free); + K_RUNLOCK(workinfo_free); if (ok) { char pct[16] = "?"; @@ -5041,7 +5089,7 @@ flail: if (pool.workinfoid < row->workinfoid) { pool.workinfoid = row->workinfoid; pool.height = row->height; - zero_on_new_block(true); + zero_on_new_block(false); } break; case BLOCKS_ORPHAN: @@ -5221,7 +5269,7 @@ bool blocks_fill(PGconn *conn) // first add all the NEW blocks if (row->confirmed[0] == BLOCKS_NEW) - _userinfo_block(row, INFO_NEW, 1, false); + userinfo_block(row, INFO_NEW, 1); } if (!ok) @@ -5236,9 +5284,9 @@ bool blocks_fill(PGconn *conn) DATA_BLOCKS(row, item); if (CURRENT(&(row->expirydate))) { if (row->confirmed[0] == BLOCKS_ORPHAN) - _userinfo_block(row, INFO_ORPHAN, 1, false); + userinfo_block(row, INFO_ORPHAN, 1); else if (row->confirmed[0] == BLOCKS_REJECT) - _userinfo_block(row, INFO_REJECT, 1, false); + userinfo_block(row, INFO_REJECT, 1); } item = next_in_ktree(ctx); } @@ -5675,7 +5723,7 @@ K_ITEM *payouts_full_expire(PGconn *conn, int64_t payoutid, tv_t *now, bool lock // If not already done before calling if (lock) - ck_wlock(&process_pplns_lock); + K_WLOCK(process_pplns_free); // This will be rare so a full lock is best K_WLOCK(payouts_free); @@ -5873,7 +5921,7 @@ K_ITEM *payouts_full_expire(PGconn *conn, int64_t payoutid, tv_t *now, bool lock if (PAYGENERATED(payouts->status)) { // Original was generated, so undo the reward - reward_shifts(payouts, true, -1); + reward_shifts(payouts, -1); } @@ -5891,7 +5939,7 @@ matane: CKPQDisco(&conn, conned); if (lock) - ck_wunlock(&process_pplns_lock); + K_WUNLOCK(process_pplns_free); for (n = 0; n < par; n++) free(params[n]); @@ -6024,7 +6072,9 @@ bool payouts_fill(PGconn *conn) break; // This also of course, verifies the payouts -> blocks reference + K_RLOCK(blocks_free); b_item = find_blocks(row->height, row->blockhash, ctx); + K_RUNLOCK(blocks_free); if (!b_item) { LOGERR("%s(): payoutid %"PRId64" references unknown " "block %"PRId32"/%s", @@ -6045,7 +6095,7 @@ bool payouts_fill(PGconn *conn) k_add_head(payouts_store, item); if (CURRENT(&(row->expirydate)) && PAYGENERATED(row->status)) - reward_shifts(row, false, 1); + reward_shifts(row, 1); tick(); } @@ -6216,12 +6266,13 @@ bool poolstats_add(PGconn *conn, bool store, char *poolinstance, SIMPLEDATEINIT(row, cd, by, code, inet); SIMPLEDATETRANSFER(trf_root, row); + K_WLOCK(poolstats_free); if (igndup && find_in_ktree(poolstats_root, p_item, ctx)) { - K_WLOCK(poolstats_free); k_add_head(poolstats_free, p_item); K_WUNLOCK(poolstats_free); return true; } + K_WUNLOCK(poolstats_free); if (store) { par = 0; @@ -6499,7 +6550,8 @@ bool userstats_add(char *poolinstance, char *elapsed, char *username, workerstatus_update(NULL, NULL, row); /* group at: userid,workername */ - us_match = userstats_eos_store->head; + K_WLOCK(userstats_free); + us_match = STORE_WHEAD(userstats_eos_store); while (us_match && cmp_userstats(us_item, us_match) != 0.0) us_match = us_match->next; @@ -6513,19 +6565,16 @@ bool userstats_add(char *poolinstance, char *elapsed, char *username, if (match->elapsed > row->elapsed) match->elapsed = row->elapsed; // Unused - K_WLOCK(userstats_free); k_add_head(userstats_free, us_item); - K_WUNLOCK(userstats_free); } else { // New user+worker - K_WLOCK(userstats_free); k_add_head(userstats_eos_store, us_item); - K_WUNLOCK(userstats_free); } + K_WUNLOCK(userstats_free); if (eos) { K_WLOCK(userstats_free); - us_next = userstats_eos_store->head; + us_next = STORE_WHEAD(userstats_eos_store); while (us_next) { us_item = find_in_ktree(userstats_root, us_next, ctx); if (!us_item) { @@ -6771,8 +6820,11 @@ bool markersummary_fill(PGconn *conn) n = 0; ok = true; - //K_WLOCK(markersummary_free); + K_WLOCK(markersummary_free); while ((t = PQntuples(res)) > 0) { + // Avoid locking them too many times + K_RLOCK(workmarkers_free); + K_WLOCK(userinfo_free); for (i = 0; i < t; i++) { item = k_unlink_head(markersummary_free); DATA_MARKERSUMMARY(row, item); @@ -6919,7 +6971,7 @@ bool markersummary_fill(PGconn *conn) markersummary_to_pool(p_row, row); - _userinfo_update(NULL, NULL, row, false, false); + userinfo_update(NULL, NULL, row, false); if (n == 0 || ((n+1) % 100000) == 0) { printf(TICK_PREFIX"ms "); @@ -6930,6 +6982,8 @@ bool markersummary_fill(PGconn *conn) tick(); n++; } + K_WUNLOCK(userinfo_free); + K_RUNLOCK(workmarkers_free); PQclear(res); res = PQexec(conn, "fetch 9999 in ws", CKPQ_READ); rescode = PQresultStatus(res); @@ -6946,7 +7000,7 @@ bool markersummary_fill(PGconn *conn) p_n = markersummary_pool_store->count; - //K_WUNLOCK(markersummary_free); + K_WUNLOCK(markersummary_free); PQclear(res); flail: res = PQexec(conn, "Commit", CKPQ_READ); @@ -7130,7 +7184,7 @@ bool _workmarkers_process(PGconn *conn, bool already, bool add, PGLOGERR("Insert", rescode, conn); goto rollback; } - row->pps_value = workinfo_pps(w_item, workinfoidend, true); + row->pps_value = workinfo_pps(w_item, workinfoidend); } ok = true; @@ -7150,14 +7204,17 @@ unparam: if (conned) PQfinish(conn); - K_WLOCK(workmarkers_free); if (!ok) { if (wm_item) { + K_WLOCK(workmarkers_free); free_workmarkers_data(wm_item); k_add_head(workmarkers_free, wm_item); + K_WUNLOCK(workmarkers_free); } - } - else { + } else { + if (wm_item) + shift_rewards(wm_item); + K_WLOCK(workmarkers_free); if (old_wm_item) { remove_from_ktree(workmarkers_root, old_wm_item); remove_from_ktree(workmarkers_workinfoid_root, @@ -7167,14 +7224,12 @@ unparam: add_to_ktree(workmarkers_workinfoid_root, old_wm_item); } if (wm_item) { - shift_rewards(wm_item); - add_to_ktree(workmarkers_root, wm_item); add_to_ktree(workmarkers_workinfoid_root, wm_item); k_add_head(workmarkers_store, wm_item); } + K_WUNLOCK(workmarkers_free); } - K_WUNLOCK(workmarkers_free); return ok; } @@ -7278,7 +7333,7 @@ bool workmarkers_fill(PGconn *conn) __func__, row->markerid, row->workinfoidend); } - row->pps_value = workinfo_pps(wi_item, row->workinfoidend, false); + row->pps_value = workinfo_pps(wi_item, row->workinfoidend); if (CURRENT(&(row->expirydate)) && !WMPROCESSED(row->status)) { diff --git a/src/klist.c b/src/klist.c index 1254628d..023f389d 100644 --- a/src/klist.c +++ b/src/klist.c @@ -1,5 +1,6 @@ /* * Copyright 2013-2014 Andrew Smith - BlackArrow Ltd + * Copyright 2015 Andrew Smith * * This program is free software; you can redistribute it and/or modify it * under the terms of the GNU General Public License as published by the Free @@ -9,6 +10,29 @@ #include "klist.h" +#if LOCK_CHECK +bool check_locks = true; +const char *thread_noname = "UNSET"; +int next_thread_id = 0; +bool lock_check_init = false; +cklock_t lock_check_lock; +__thread int my_thread_id = -1; +__thread char *my_thread_name = NULL; +__thread bool my_check_locks = true; + +bool auto_check_deadlocks = true; +// Must be false to start with +bool check_deadlocks = false; +__thread int my_locks[MAX_LOCKDEPTH]; +__thread const char *my_locks_n[MAX_LOCKDEPTH]; +__thread const char *my_locks_fl[MAX_LOCKDEPTH]; +__thread const char *my_locks_f[MAX_LOCKDEPTH]; +__thread int my_locks_l[MAX_LOCKDEPTH]; +__thread int my_lock_level = 0; +__thread bool my_check_deadlocks = true; +K_LISTS *all_klists; +#endif + #define _CHKLIST(_list, _name) do {\ if (!_list) { \ quithere(1, "%s() can't process a NULL " _name \ @@ -111,6 +135,7 @@ K_STORE *_k_new_store(K_LIST *list, KLIST_FFL_ARGS) if (!store) quithere(1, "Failed to calloc store for %s", list->name); + store->master = list; store->is_store = true; store->lock = list->lock; store->name = list->name; @@ -119,7 +144,8 @@ K_STORE *_k_new_store(K_LIST *list, KLIST_FFL_ARGS) return store; } -K_LIST *_k_new_list(const char *name, size_t siz, int allocate, int limit, bool do_tail, KLIST_FFL_ARGS) +K_LIST *_k_new_list(const char *name, size_t siz, int allocate, int limit, + bool do_tail, bool lock_only, KLIST_FFL_ARGS) { K_LIST *list; @@ -133,7 +159,9 @@ K_LIST *_k_new_list(const char *name, size_t siz, int allocate, int limit, bool if (!list) quithere(1, "Failed to calloc list %s", name); + list->master = list; list->is_store = false; + list->is_lock_only = lock_only; list->lock = calloc(1, sizeof(*(list->lock))); if (!(list->lock)) @@ -147,7 +175,29 @@ K_LIST *_k_new_list(const char *name, size_t siz, int allocate, int limit, bool list->limit = limit; list->do_tail = do_tail; - k_alloc_items(list, KLIST_FFL_PASS); + if (!(list->is_lock_only)) + k_alloc_items(list, KLIST_FFL_PASS); + +#if LOCK_CHECK + K_LISTS *klists; + + // not locked :P + if (!lock_check_init) { + quitfrom(1, file, func, line, + "in %s(), lock_check_lock has not been initialised!", + __func__); + } + + klists = calloc(1, sizeof(*klists)); + if (!klists) + quithere(1, "Failed to calloc klists %s", name); + + klists->klist = list; + ck_wlock(&lock_check_lock); + klists->next = all_klists; + all_klists = klists; + ck_wunlock(&lock_check_lock); +#endif return list; } @@ -159,11 +209,12 @@ K_LIST *_k_new_list(const char *name, size_t siz, int allocate, int limit, bool * 2) alloc a new list and return the head - * which is NULL if the list limit has been reached */ -K_ITEM *_k_unlink_head(K_LIST *list, KLIST_FFL_ARGS) +K_ITEM *_k_unlink_head(K_LIST *list, LOCK_MAYBE bool chklock, KLIST_FFL_ARGS) { K_ITEM *item; CHKLS(list); + _LIST_WRITE(list, chklock, file, func, line); if (!(list->head) && !(list->is_store)) k_alloc_items(list, KLIST_FFL_PASS); @@ -188,13 +239,14 @@ K_ITEM *_k_unlink_head(K_LIST *list, KLIST_FFL_ARGS) } // Zeros the head returned -K_ITEM *_k_unlink_head_zero(K_LIST *list, KLIST_FFL_ARGS) +K_ITEM *_k_unlink_head_zero(K_LIST *list, LOCK_MAYBE bool chklock, KLIST_FFL_ARGS) { K_ITEM *item; CHKLS(list); + _LIST_WRITE(list, chklock, file, func, line); - item = _k_unlink_head(list, KLIST_FFL_PASS); + item = _k_unlink_head(list, false, KLIST_FFL_PASS); if (item) memset(item->data, 0, list->siz); @@ -203,11 +255,12 @@ K_ITEM *_k_unlink_head_zero(K_LIST *list, KLIST_FFL_ARGS) } // Returns NULL if empty -K_ITEM *_k_unlink_tail(K_LIST *list, KLIST_FFL_ARGS) +K_ITEM *_k_unlink_tail(K_LIST *list, LOCK_MAYBE bool chklock, KLIST_FFL_ARGS) { K_ITEM *item; CHKLS(list); + _LIST_WRITE(list, chklock, file, func, line); if (!(list->do_tail)) { quithere(1, "List %s can't %s() - do_tail is false" KLIST_FFL, @@ -231,11 +284,11 @@ K_ITEM *_k_unlink_tail(K_LIST *list, KLIST_FFL_ARGS) return item; } -void _k_add_head(K_LIST *list, K_ITEM *item, KLIST_FFL_ARGS) +void _k_add_head(K_LIST *list, K_ITEM *item, LOCK_MAYBE bool chklock, KLIST_FFL_ARGS) { CHKLS(list); - CHKITEM(item, list); + _LIST_WRITE(list, chklock, file, func, line); if (item->name != list->name) { quithere(1, "List %s can't %s() a %s item" KLIST_FFL, @@ -264,22 +317,22 @@ void _k_add_head(K_LIST *list, K_ITEM *item, KLIST_FFL_ARGS) } /* slows it down (of course) - only for debugging -void _k_free_head(K_LIST *list, K_ITEM *item, KLIST_FFL_ARGS) +void _k_free_head(K_LIST *list, K_ITEM *item, LOCK_MAYBE bool chklock, KLIST_FFL_ARGS) { CHKLS(list); - CHKITEM(item, list); + _LIST_WRITE(list, chklock, file, func, line); memset(item->data, 0xff, list->siz); _k_add_head(list, item, KLIST_FFL_PASS); } */ -void _k_add_tail(K_LIST *list, K_ITEM *item, KLIST_FFL_ARGS) +void _k_add_tail(K_LIST *list, K_ITEM *item, LOCK_MAYBE bool chklock, KLIST_FFL_ARGS) { CHKLS(list); - CHKITEM(item, list); + _LIST_WRITE(list, chklock, file, func, line); if (item->name != list->name) { quithere(1, "List %s can't %s() a %s item" KLIST_FFL, @@ -311,13 +364,12 @@ void _k_add_tail(K_LIST *list, K_ITEM *item, KLIST_FFL_ARGS) } // Insert item into the list next after 'after' -void _k_insert_after(K_LIST *list, K_ITEM *item, K_ITEM *after, KLIST_FFL_ARGS) +void _k_insert_after(K_LIST *list, K_ITEM *item, K_ITEM *after, LOCK_MAYBE bool chklock, KLIST_FFL_ARGS) { CHKLS(list); - CHKITEM(item, list); - _CHKITEM(item, after, "after"); + _LIST_WRITE(list, chklock, file, func, line); if (item->name != list->name) { quithere(1, "List %s can't %s() a %s item" KLIST_FFL, @@ -349,11 +401,11 @@ void _k_insert_after(K_LIST *list, K_ITEM *item, K_ITEM *after, KLIST_FFL_ARGS) list->count_up++; } -void _k_unlink_item(K_LIST *list, K_ITEM *item, KLIST_FFL_ARGS) +void _k_unlink_item(K_LIST *list, K_ITEM *item, LOCK_MAYBE bool chklock, KLIST_FFL_ARGS) { CHKLS(list); - CHKITEM(item, list); + _LIST_WRITE(list, chklock, file, func, line); if (item->name != list->name) { quithere(1, "List %s can't %s() a %s item" KLIST_FFL, @@ -379,10 +431,9 @@ void _k_unlink_item(K_LIST *list, K_ITEM *item, KLIST_FFL_ARGS) list->count--; } -void _k_list_transfer_to_head(K_LIST *from, K_LIST *to, KLIST_FFL_ARGS) +void _k_list_transfer_to_head(K_LIST *from, K_LIST *to, LOCK_MAYBE bool chklock, KLIST_FFL_ARGS) { _CHKLIST(from, "from list/store"); - _CHKLIST(to, "to list/store"); if (from->name != to->name) { @@ -390,6 +441,9 @@ void _k_list_transfer_to_head(K_LIST *from, K_LIST *to, KLIST_FFL_ARGS) from->name, __func__, to->name, KLIST_FFL_PASS); } + // from and to are the same lock + _LIST_WRITE(to, chklock, file, func, line); + if (!(from->do_tail)) { quithere(1, "List %s can't %s() - do_tail is false" KLIST_FFL, from->name, __func__, KLIST_FFL_PASS); @@ -413,10 +467,9 @@ void _k_list_transfer_to_head(K_LIST *from, K_LIST *to, KLIST_FFL_ARGS) from->count_up = 0; } -void _k_list_transfer_to_tail(K_LIST *from, K_LIST *to, KLIST_FFL_ARGS) +void _k_list_transfer_to_tail(K_LIST *from, K_LIST *to, LOCK_MAYBE bool chklock, KLIST_FFL_ARGS) { _CHKLIST(from, "from list/store"); - _CHKLIST(to, "to list/store"); if (from->name != to->name) { @@ -424,6 +477,9 @@ void _k_list_transfer_to_tail(K_LIST *from, K_LIST *to, KLIST_FFL_ARGS) from->name, __func__, to->name, KLIST_FFL_PASS); } + // from and to are the same lock + _LIST_WRITE(to, chklock, file, func, line); + if (!(from->do_tail)) { quithere(1, "List %s can't %s() - do_tail is false" KLIST_FFL, from->name, __func__, KLIST_FFL_PASS); @@ -470,6 +526,36 @@ K_LIST *_k_free_list(K_LIST *list, KLIST_FFL_ARGS) free(list->lock); +#if LOCK_CHECK + K_LISTS *klists, *klists_prev = NULL; + + // not locked :P + if (!lock_check_init) { + quitfrom(1, file, func, line, + "in %s(), lock_check_lock has not been initialised!", + __func__); + } + + ck_wlock(&lock_check_lock); + klists = all_klists; + while (klists && klists->klist != list) { + klists_prev = klists; + klists = klists->next; + } + if (!klists) { + quitfrom(1, file, func, line, + "in %s(), list %s not in klists", + __func__, list->name); + } else { + if (klists_prev) + klists_prev->next = klists->next; + else + all_klists = klists->next; + free(klists); + } + ck_wunlock(&lock_check_lock); +#endif + free(list); return NULL; @@ -490,11 +576,12 @@ K_STORE *_k_free_store(K_STORE *store, KLIST_FFL_ARGS) } // Must be locked and none in use and/or unlinked -void _k_cull_list(K_LIST *list, KLIST_FFL_ARGS) +void _k_cull_list(K_LIST *list, LOCK_MAYBE bool chklock, KLIST_FFL_ARGS) { int i; CHKLIST(list); + _LIST_WRITE(list, chklock, file, func, line); if (list->is_store) { quithere(1, "List %s can't %s() a store" KLIST_FFL, diff --git a/src/klist.h b/src/klist.h index 602bdac1..9e5888a7 100644 --- a/src/klist.h +++ b/src/klist.h @@ -1,5 +1,6 @@ /* * Copyright 2013-2014 Andrew Smith - BlackArrow Ltd + * Copyright 2015 Andrew Smith * * This program is free software; you can redistribute it and/or modify it * under the terms of the GNU General Public License as published by the Free @@ -16,12 +17,97 @@ quitfrom(status, __FILE__, __func__, __LINE__, fmt, ##__VA_ARGS__) #define KLIST_FFL " - from %s %s() line %d" +#define KLIST_SFFL " - from %s %s():%d" +#define KLIST_AFFL "at %s %s():%d" #define KLIST_FFL_HERE __FILE__, __func__, __LINE__ #define KLIST_FFL_PASS file, func, line #define KLIST_FFL_ARGS __maybe_unused const char *file, \ __maybe_unused const char *func, \ __maybe_unused const int line +/* Code to check the state of locks being requested and also check + * the state of locks when accessing the klist or ktree + * You can disable it with ckpmsg 'locks.ID.locks' so you can compare + * CPU usage with and later without it + * (or just completely disable it by defining LOCK_CHECK 0 below) + * + * If we already hold any lock we ask for, it will report the duplication. + * Duplication of read locks wont fail the code, but they do represent + * code that should be 'fixed' + * + * If klist/ktree access expects to have a lock, but doesn't have the + * required lock, it will report this bug + * + * Any errors found will set my_check_locks false before reporting the error, + * to avoid floods of error messages or crashes due to unexpected states + * of the thread's lock info + * i.e. you can only find one bug per thread each time you run ckdb + * ... they should be rare if ever ... since I used this to attempt to + * find them :) + */ +#define LOCK_CHECK 1 + +/* Deadlock prediction is quite simple: + * alloc_storage gives each K_LIST a unique 'lock priority' order greater + * than 1 with the lowest being PRIO_TERMINAL=1 + * (thus until alloc_storage is run, no deadlock prediction is enabled) + * If any code locks a K_LIST with a 'lock priority' higher than one it + * already holds, then that means it could result in a deadlock, + * since the reverse priority order is expected + * + * This is implemented by checking the 'lock priority' + * each time we attempt to take out a lock within a lock + * It simply checks the previous lock held to see if it's 'lock priority' + * is lower than the new lock thus marginal CPU increase is only noticeable + * on multi-level locks + * + * Any deadlocks predicted will set my_check_deadlocks false before reporting + * the error, to avoid possible floods of repeated error messages + * i.e. you can only find one deadlock per thread each time you run ckdb + * ... they should be rare if ever ... since I used this to attempt to + * find them :) + */ + +/* Deadlock prediction is part of LOCK_CHECK coz it uses the CHECK_LOCK() macro + * If you want only deadlock checking, edit klist.c and set check_locks + * default to false, + * or turn off check_locks during ckdb startup with a ckpmsg 'locks.ID.locks' + * If you turn deadlock prediction on with ckpmsg 'locks.1.deadlocks=y' + * it will not re-enable it for any thread that has alread predicted + * a deadlock */ + +#if LOCK_CHECK +// We disable lock checking if an error is encountered +extern bool check_locks; +/* Maximum number of threads preallocated + * This allows access to the lock tables without + * using any locks */ +#define MAX_THREADS 128 +extern const char *thread_noname; +extern int next_thread_id; +extern bool lock_check_init; +extern cklock_t lock_check_lock; +extern __thread int my_thread_id; +extern __thread char *my_thread_name; +extern __thread bool my_check_locks; + +// This decides if alloc_storage will set 'check_deadlocks' after it's setup +extern bool auto_check_deadlocks; +// This decides if deadlock prediction is happening +extern bool check_deadlocks; +// It should never get to 16 unless there's a bug +#define MAX_LOCKDEPTH 16 +extern __thread int my_locks[MAX_LOCKDEPTH]; +extern __thread const char *my_locks_n[MAX_LOCKDEPTH]; +extern __thread const char *my_locks_fl[MAX_LOCKDEPTH]; +extern __thread const char *my_locks_f[MAX_LOCKDEPTH]; +extern __thread int my_locks_l[MAX_LOCKDEPTH]; +extern __thread int my_lock_level; +extern __thread bool my_check_deadlocks; + +extern const char *nullstr; +#endif + typedef struct k_item { const char *name; struct k_item *prev; @@ -29,9 +115,22 @@ typedef struct k_item { void *data; } K_ITEM; +#if LOCK_CHECK +typedef struct k_lock { + int r_count; + int w_count; + const char *first_held; + const char *file; + const char *func; + int line; +} K_LOCK; +#endif + typedef struct k_list { const char *name; + struct k_list *master; bool is_store; + bool is_lock_only; // a lock emulating a list for lock checking cklock_t *lock; struct k_item *head; struct k_item *tail; @@ -49,8 +148,22 @@ typedef struct k_list { void (*dsp_func)(K_ITEM *, FILE *); // optional data display to a file int cull_count; int ram; // ram allocated for data pointers - code must manage it +#if LOCK_CHECK + // Since each thread has it's own k_lock no locking is required on this + K_LOCK k_lock[MAX_THREADS]; + // 0=unset=an error, >=1 is the priority - bigger=higher priority + int deadlock_priority; +#endif } K_LIST; +#if LOCK_CHECK +typedef struct k_lists { + K_LIST *klist; + struct k_lists *next; +} K_LISTS; +extern K_LISTS *all_klists; +#endif + /* * K_STORE is for a list of items taken from a K_LIST * The restriction is, a K_STORE must not allocate new items, @@ -59,47 +172,433 @@ typedef struct k_list { */ #define K_STORE K_LIST -/* - * N.B. all locking is done in the code using the K_*LOCK macros - */ -#define K_WLOCK(_list) ck_wlock(_list->lock) -#define K_WUNLOCK(_list) ck_wunlock(_list->lock) -#define K_RLOCK(_list) ck_rlock(_list->lock) -#define K_RUNLOCK(_list) ck_runlock(_list->lock) -#define K_ILOCK(_list) ck_ilock(_list->lock) -#define K_IUNLOCK(_list) ck_uilock(_list->lock) -// Upgrade I to W -#define K_ULOCK(_list) ck_ulock(_list->lock) +#if LOCK_CHECK +#define LOCK_MAYBE +/* The simple lock_check_init check is in case someone incorrectly changes ckdb.c ... + * It's not fool proof :P */ +#define LOCK_INIT(_name) do { \ + if (!lock_check_init) { \ + quithere(1, "In thread %s, lock_check_lock has not been " \ + "initialised!", _name); \ + } \ + ck_wlock(&lock_check_lock); \ + my_thread_id = next_thread_id++; \ + ck_wunlock(&lock_check_lock); \ + my_thread_name = strdup(_name); \ + } while (0) +#define FIRST_LOCK_INIT(_name) do { \ + if (lock_check_init) { \ + quithere(1, "In thread %s, lock_check_lock has already been " \ + "initialised!", (_name)); \ + } \ + cklock_init(&lock_check_lock); \ + lock_check_init = true; \ + LOCK_INIT(_name); \ + } while (0) + +#define LOCK_MODE_LOCK 0 +#define LOCK_MODE_UNLOCK 1 +#define LOCK_TYPE_READ 0 +#define LOCK_TYPE_WRITE 1 + +// Lists with this priority cannot nest any lock inside their lock +#define PRIO_TERMINAL 1 + +#define LOCKERR(fmt, ...) LOGEMERG("***CHKLOCK %s:%d(now off) " fmt, \ + my_thread_name ? : thread_noname, \ + my_thread_id, ##__VA_ARGS__) +#define DLOCKERR(fmt, ...) LOGEMERG("***PREDLOCK %s(now off) " fmt, \ + my_thread_name ? : thread_noname, \ + ##__VA_ARGS__) +#define DLOCKOK(fmt, ...) LOGWARNING("***PREDLOCK %s " fmt, \ + my_thread_name ? : thread_noname, \ + ##__VA_ARGS__) + +// Neither test 'should' ever fail +#define DLPRIO(_list, _p) do { \ + if ((_list ## _free)->is_store) \ + quithere(1, "Can't deadlock prioritise a K_STORE"); \ + if ((_list ## _free)->master != (_list ## _free)) \ + quithere(1, "K_LIST master is not itself"); \ + (_list ## _free)->deadlock_priority = (_p); \ + } while (0) + +#define DLPCHECK() do { \ + K_LISTS *_klists; \ + if (!lock_check_init) { \ + quithere(1, "lock_check_lock has not been initialised!"); \ + } \ + ck_wlock(&lock_check_lock); \ + _klists = all_klists; \ + while (_klists) { \ + if (_klists->klist->deadlock_priority < PRIO_TERMINAL) { \ + DLOCKOK("%s priority not set (%d)", \ + _klists->klist->name, \ + _klists->klist->deadlock_priority); \ + } \ + _klists = _klists->next; \ + } \ + ck_wunlock(&lock_check_lock); \ + } while (0) + +/* Optimisation should remove the code for all but the required _mode/_type + * since all the related ifs are constants */ +#define THRLCK(_list) (((_list)->master)->k_lock[my_thread_id]) +#define CHECK_LOCK(_list, _func, _mode, _type) do { \ + static const char *_n = #_list " " #_func; \ + static const char *_fl = __FILE__; \ + static const char *_f = __func__; \ + static const int _l = __LINE__; \ + if (my_check_locks && check_locks) { \ + if (_mode == LOCK_MODE_LOCK) { \ + if (THRLCK(_list).first_held || \ + (THRLCK(_list).r_count != 0) || \ + (THRLCK(_list).w_count != 0)) { \ + my_check_locks = false; \ + LOCKERR("%s " KLIST_AFFL " invalid (r%d:w%d) " \ + "first: %s " KLIST_AFFL, \ + _n, _fl, _f, _l, \ + THRLCK(_list).r_count, \ + THRLCK(_list).w_count, \ + THRLCK(_list).first_held ? : thread_noname, \ + THRLCK(_list).file ? : nullstr, \ + THRLCK(_list).func ? : nullstr, \ + THRLCK(_list).line); \ + } else { \ + THRLCK(_list).first_held = _n; \ + THRLCK(_list).file = _fl; \ + THRLCK(_list).func = _f; \ + THRLCK(_list).line = _l; \ + if (_type == LOCK_TYPE_READ) \ + THRLCK(_list).r_count++; \ + if (_type == LOCK_TYPE_WRITE) \ + THRLCK(_list).w_count++; \ + } \ + } \ + if (_mode == LOCK_MODE_UNLOCK && _type == LOCK_TYPE_READ) { \ + if (!THRLCK(_list).first_held || \ + (THRLCK(_list).r_count != 1) || \ + (THRLCK(_list).w_count != 0)) { \ + my_check_locks = false; \ + LOCKERR("%s " KLIST_AFFL " invalid (r%d:w%d) " \ + "first: %s " KLIST_AFFL, \ + _n, _fl, _f, _l, \ + THRLCK(_list).r_count, \ + THRLCK(_list).w_count, \ + THRLCK(_list).first_held ? : thread_noname, \ + THRLCK(_list).file ? : nullstr, \ + THRLCK(_list).func ? : nullstr, \ + THRLCK(_list).line); \ + } else { \ + THRLCK(_list).first_held = NULL; \ + THRLCK(_list).file = NULL; \ + THRLCK(_list).func = NULL; \ + THRLCK(_list).line = 0; \ + THRLCK(_list).r_count--; \ + } \ + } \ + if (_mode == LOCK_MODE_UNLOCK && _type == LOCK_TYPE_WRITE) { \ + if (!THRLCK(_list).first_held || \ + (THRLCK(_list).r_count != 0) || \ + (THRLCK(_list).w_count != 1)) { \ + my_check_locks = false; \ + LOCKERR("%s " KLIST_AFFL " invalid (r%d:w%d) " \ + "first: %s " KLIST_AFFL, \ + _n, _fl, _f, _l, \ + THRLCK(_list).r_count, \ + THRLCK(_list).w_count, \ + THRLCK(_list).first_held ? : thread_noname, \ + THRLCK(_list).file ? : nullstr, \ + THRLCK(_list).func ? : nullstr, \ + THRLCK(_list).line); \ + } else { \ + THRLCK(_list).first_held = NULL; \ + THRLCK(_list).file = NULL; \ + THRLCK(_list).func = NULL; \ + THRLCK(_list).line = 0; \ + THRLCK(_list).w_count--; \ + } \ + } \ + } \ + if (check_deadlocks && my_check_deadlocks) { \ + int _dp = (_list)->deadlock_priority; \ + if (my_lock_level == 0) { \ + if (_mode == LOCK_MODE_LOCK) { \ + if (_dp < PRIO_TERMINAL) { \ + my_check_deadlocks = false; \ + DLOCKERR("%s " KLIST_AFFL \ + " bad lock prio %d", \ + _n, _fl, _f, _l, \ + _dp); \ + } else { \ + my_locks[0] = _dp; \ + my_locks_n[0] = _n; \ + my_locks_fl[0] = _fl; \ + my_locks_f[0] = _f; \ + my_locks_l[0] = _l; \ + my_lock_level = 1; \ + } \ + } \ + if (_mode == LOCK_MODE_UNLOCK) { \ + DLOCKOK("%s " KLIST_AFFL \ + " lock level was 0 - unlock" \ + " prio %d ignored", \ + _n, _fl, _f, _l, \ + _dp); \ + } \ + } else { \ + if (_mode == LOCK_MODE_UNLOCK) { \ + if (my_locks[--my_lock_level] != _dp) { \ + my_check_deadlocks = false; \ + DLOCKERR("%s " KLIST_AFFL \ + " unlock prio %d" \ + " doesn't match prev" \ + " locked prio %d", \ + _n, _fl, _f, _l, \ + _dp, \ + my_locks[my_lock_level]); \ + } \ + } \ + if (_mode == LOCK_MODE_LOCK) { \ + int _i = my_lock_level - 1; \ + if (my_locks[_i] == PRIO_TERMINAL) { \ + my_check_deadlocks = false; \ + DLOCKERR("%s " KLIST_AFFL \ + " prev lock" \ + " prio[%d]=TERMINAL" \ + " ... lock prio[%d]" \ + "=%d %s " KLIST_AFFL, \ + _n, _fl, _f, _l, _i, \ + my_lock_level, _dp, \ + my_locks_n[_i], \ + my_locks_fl[_i], \ + my_locks_f[_i], \ + my_locks_l[_i]); \ + } else if (my_locks[_i] <= _dp) { \ + my_check_deadlocks = false; \ + DLOCKERR("%s " KLIST_AFFL \ + " lock prio[%d]=%d" \ + " >= prev lock" \ + " prio[%d]=%d" \ + " %s " KLIST_AFFL, \ + _n, _fl, _f, _l, \ + my_lock_level, _dp, \ + _i, my_locks[_i], \ + my_locks_n[_i], \ + my_locks_fl[_i], \ + my_locks_f[_i], \ + my_locks_l[_i]); \ + } else { \ + my_locks[my_lock_level] = _dp; \ + my_locks_n[my_lock_level] = _n; \ + my_locks_fl[my_lock_level] = _fl; \ + my_locks_f[my_lock_level] = _f; \ + my_locks_l[my_lock_level++] = _l; \ + } \ + } \ + } \ + } \ + ck_##_func(_list->lock); \ + } while (0) + +#define CHECK_WLOCK(_list) CHECK_LOCK(_list, wlock, \ + LOCK_MODE_LOCK, LOCK_TYPE_WRITE) +#define CHECK_WUNLOCK(_list) CHECK_LOCK(_list, wunlock, \ + LOCK_MODE_UNLOCK, LOCK_TYPE_WRITE) +#define CHECK_RLOCK(_list) CHECK_LOCK(_list, rlock, \ + LOCK_MODE_LOCK, LOCK_TYPE_READ) +#define CHECK_RUNLOCK(_list) CHECK_LOCK(_list, runlock, \ + LOCK_MODE_UNLOCK, LOCK_TYPE_READ) + +#define _LIST_WRITE(_list, _chklock, _file, _func, _line) do { \ + if (my_check_locks && check_locks && _chklock) { \ + if (!THRLCK(_list).first_held || \ + (THRLCK(_list).r_count != 0) || \ + (THRLCK(_list).w_count != 1)) { \ + my_check_locks = false; \ + LOCKERR("%s " KLIST_AFFL " invalid write " \ + "access (r%d:w%d) first: %s " \ + KLIST_AFFL KLIST_SFFL, \ + (_list)->master->name ? : nullstr, \ + KLIST_FFL_HERE, \ + THRLCK(_list).r_count, \ + THRLCK(_list).w_count, \ + THRLCK(_list).first_held ? : thread_noname, \ + THRLCK(_list).file ? : nullstr, \ + THRLCK(_list).func ? : nullstr, \ + THRLCK(_list).line, \ + _file, _func, _line); \ + } \ + } \ + } while (0) +#define _LIST_WRITE2(_list, _chklock) do { \ + if (my_check_locks && check_locks && _chklock) { \ + if (!THRLCK(_list).first_held || \ + (THRLCK(_list).r_count != 0) || \ + (THRLCK(_list).w_count != 1)) { \ + my_check_locks = false; \ + LOCKERR("%s " KLIST_AFFL " invalid write " \ + "access (r%d:w%d) first: %s " \ + KLIST_AFFL, \ + (_list)->master->name ? : nullstr, \ + KLIST_FFL_HERE, \ + THRLCK(_list).r_count, \ + THRLCK(_list).w_count, \ + THRLCK(_list).first_held ? : thread_noname, \ + THRLCK(_list).file ? : nullstr, \ + THRLCK(_list).func ? : nullstr, \ + THRLCK(_list).line); \ + } \ + } \ + } while (0) +// read is ok under read or write +#define _LIST_READ(_list, _chklock, _file, _func, _line) do { \ + if (my_check_locks && check_locks && _chklock) { \ + if (!THRLCK(_list).first_held || \ + (THRLCK(_list).r_count + \ + THRLCK(_list).w_count) != 1) { \ + my_check_locks = false; \ + LOCKERR("%s " KLIST_AFFL " invalid read " \ + "access (r%d:w%d) first: %s " \ + KLIST_AFFL KLIST_SFFL, \ + (_list)->master->name ? : nullstr, \ + KLIST_FFL_HERE, \ + THRLCK(_list).r_count, \ + THRLCK(_list).w_count, \ + THRLCK(_list).first_held ? : thread_noname, \ + THRLCK(_list).file ? : nullstr, \ + THRLCK(_list).func ? : nullstr, \ + THRLCK(_list).line, \ + _file, _func, _line); \ + } \ + } \ + } while (0) +#define _LIST_READ2(_list, _chklock) do { \ + if (my_check_locks && check_locks && _chklock) { \ + if (!THRLCK(_list).first_held || \ + (THRLCK(_list).r_count + \ + THRLCK(_list).w_count) != 1) { \ + my_check_locks = false; \ + LOCKERR("%s " KLIST_AFFL " invalid read " \ + "access (r%d:w%d) first: %s " \ + KLIST_AFFL, \ + (_list)->master->name ? : nullstr, \ + KLIST_FFL_HERE, \ + THRLCK(_list).r_count, \ + THRLCK(_list).w_count, \ + THRLCK(_list).first_held ? : thread_noname, \ + THRLCK(_list).file ? : nullstr, \ + THRLCK(_list).func ? : nullstr, \ + THRLCK(_list).line); \ + } \ + } \ + } while (0) +#define LIST_WRITE(_list) _LIST_WRITE2(_list, true) +#define LIST_READ(_list) _LIST_READ2(_list, true) +static inline K_ITEM *list_whead(K_LIST *list) +{ + LIST_WRITE(list); + return list->head; +} +static inline K_ITEM *list_rhead(K_LIST *list) +{ + LIST_READ(list); + return list->head; +} +static inline K_ITEM *list_wtail(K_LIST *list) +{ + LIST_READ(list); + return list->head; +} +static inline K_ITEM *list_rtail(K_LIST *list) +{ + LIST_READ(list); + return list->head; +} +#define LIST_WHEAD(_list) list_whead(_list) +#define LIST_RHEAD(_list) list_rhead(_list) +#define LIST_WTAIL(_list) list_wtail(_list) +#define LIST_RTAIL(_list) list_rtail(_list) +#else +#define LOCK_MAYBE __maybe_unused +#define LOCK_INIT(_name) +#define FIRST_LOCK_INIT(_name) +#define CHECK_WLOCK(_list) ck_wlock((_list)->lock) +#define CHECK_WUNLOCK(_list) ck_wunlock((_list)->lock) +#define CHECK_RLOCK(_list) ck_rlock((_list)->lock) +#define CHECK_RUNLOCK(_list) ck_runlock((_list)->lock) +#define _LIST_WRITE(_list, _chklock, _file, _func, _line) +#define _LIST_READ(_list, _chklock, _file, _func, _line) +#define LIST_WRITE(_list) +#define LIST_READ(_list) +#define LIST_WHEAD(_list) (_list)->head +#define LIST_RHEAD(_list) (_list)->head +#define LIST_WTAIL(_list) (_list)->tail +#define LIST_RTAIL(_list) (_list)->tail +#endif + +#define LIST_HEAD_NOLOCK(_list) (_list)->head +#define LIST_TAIL_NOLOCK(_list) (_list)->tail + +#define K_WLOCK(_list) CHECK_WLOCK(_list) +#define K_WUNLOCK(_list) CHECK_WUNLOCK(_list) +#define K_RLOCK(_list) CHECK_RLOCK(_list) +#define K_RUNLOCK(_list) CHECK_RUNLOCK(_list) + +#define STORE_WHEAD(_s) LIST_WHEAD(_s) +#define STORE_RHEAD(_s) LIST_RHEAD(_s) +#define STORE_WTAIL(_s) LIST_WTAIL(_s) +#define STORE_RTAIL(_s) LIST_RTAIL(_s) + +// No need to lock temporary/private stores +#define STORE_HEAD_NOLOCK(_s) LIST_HEAD_NOLOCK(_s) +#define STORE_TAIL_NOLOCK(_s) LIST_TAIL_NOLOCK(_s) extern K_STORE *_k_new_store(K_LIST *list, KLIST_FFL_ARGS); #define k_new_store(_list) _k_new_store(_list, KLIST_FFL_HERE) -extern K_LIST *_k_new_list(const char *name, size_t siz, int allocate, int limit, bool do_tail, KLIST_FFL_ARGS); -#define k_new_list(_name, _siz, _allocate, _limit, _do_tail) _k_new_list(_name, _siz, _allocate, _limit, _do_tail, KLIST_FFL_HERE) -extern K_ITEM *_k_unlink_head(K_LIST *list, KLIST_FFL_ARGS); -#define k_unlink_head(_list) _k_unlink_head(_list, KLIST_FFL_HERE) -extern K_ITEM *_k_unlink_head_zero(K_LIST *list, KLIST_FFL_ARGS); -#define k_unlink_head_zero(_list) _k_unlink_head_zero(_list, KLIST_FFL_HERE) -extern K_ITEM *_k_unlink_tail(K_LIST *list, KLIST_FFL_ARGS); -#define k_unlink_tail(_list) _k_unlink_tail(_list, KLIST_FFL_HERE) -extern void _k_add_head(K_LIST *list, K_ITEM *item, KLIST_FFL_ARGS); -#define k_add_head(_list, _item) _k_add_head(_list, _item, KLIST_FFL_HERE) -// extern void k_free_head(K_LIST *list, K_ITEM *item, KLIST_FFL_ARGS); -#define k_free_head(__list, __item) _k_add_head(__list, __item, KLIST_FFL_HERE) -extern void _k_add_tail(K_LIST *list, K_ITEM *item, KLIST_FFL_ARGS); -#define k_add_tail(_list, _item) _k_add_tail(_list, _item, KLIST_FFL_HERE) -extern void _k_insert_after(K_LIST *list, K_ITEM *item, K_ITEM *after, KLIST_FFL_ARGS); -#define k_insert_after(_list, _item, _after) _k_insert_after(_list, _item, _after, KLIST_FFL_HERE) -extern void _k_unlink_item(K_LIST *list, K_ITEM *item, KLIST_FFL_ARGS); -#define k_unlink_item(_list, _item) _k_unlink_item(_list, _item, KLIST_FFL_HERE) -extern void _k_list_transfer_to_head(K_LIST *from, K_LIST *to, KLIST_FFL_ARGS); -#define k_list_transfer_to_head(_from, _to) _k_list_transfer_to_head(_from, _to, KLIST_FFL_HERE) -extern void _k_list_transfer_to_tail(K_LIST *from, K_LIST *to, KLIST_FFL_ARGS); -#define k_list_transfer_to_tail(_from, _to) _k_list_transfer_to_tail(_from, _to, KLIST_FFL_HERE) +extern K_LIST *_k_new_list(const char *name, size_t siz, int allocate, + int limit, bool do_tail, bool lock_only, + KLIST_FFL_ARGS); +#define k_new_list(_name, _siz, _allocate, _limit, _do_tail) \ + _k_new_list(_name, _siz, _allocate, _limit, _do_tail, false, KLIST_FFL_HERE) +#define k_lock_only_list(_name) \ + _k_new_list(_name, 1, 1, 1, true, true, KLIST_FFL_HERE) +extern K_ITEM *_k_unlink_head(K_LIST *list, LOCK_MAYBE bool chklock, KLIST_FFL_ARGS); +#define k_unlink_head(_list) _k_unlink_head(_list, true, KLIST_FFL_HERE) +#define k_unlink_head_nolock(_list) _k_unlink_head(_list, false, KLIST_FFL_HERE) +extern K_ITEM *_k_unlink_head_zero(K_LIST *list, LOCK_MAYBE bool chklock, KLIST_FFL_ARGS); +#define k_unlink_head_zero(_list) _k_unlink_head_zero(_list, true, KLIST_FFL_HERE) +//#define k_unlink_head_zero_nolock(_list) _k_unlink_head_zero(_list, false, KLIST_FFL_HERE) +extern K_ITEM *_k_unlink_tail(K_LIST *list, LOCK_MAYBE bool chklock, KLIST_FFL_ARGS); +#define k_unlink_tail(_list) _k_unlink_tail(_list, true, KLIST_FFL_HERE) +//#define k_unlink_tail_nolock(_list) _k_unlink_tail(_list, false, KLIST_FFL_HERE) +extern void _k_add_head(K_LIST *list, K_ITEM *item, LOCK_MAYBE bool chklock, KLIST_FFL_ARGS); +#define k_add_head(_list, _item) _k_add_head(_list, _item, true, KLIST_FFL_HERE) +#define k_add_head_nolock(_list, _item) _k_add_head(_list, _item, false, KLIST_FFL_HERE) +// extern void k_free_head(K_LIST *list, K_ITEM *item, LOCK_MAYBE bool chklock, KLIST_FFL_ARGS); +#define k_free_head(__list, __item) _k_add_head(__list, __item, true, KLIST_FFL_HERE) +//#define k_free_head_nolock(__list, __item) _k_add_head(__list, __item, false, KLIST_FFL_HERE) +extern void _k_add_tail(K_LIST *list, K_ITEM *item, LOCK_MAYBE bool chklock, KLIST_FFL_ARGS); +#define k_add_tail(_list, _item) _k_add_tail(_list, _item, true, KLIST_FFL_HERE) +#define k_add_tail_nolock(_list, _item) _k_add_tail(_list, _item, false, KLIST_FFL_HERE) +extern void _k_insert_after(K_LIST *list, K_ITEM *item, K_ITEM *after, LOCK_MAYBE bool chklock, KLIST_FFL_ARGS); +#define k_insert_after(_list, _item, _after) _k_insert_after(_list, _item, _after, true, KLIST_FFL_HERE) +//#define k_insert_after_nolock(_list, _item, _after) _k_insert_after(_list, _item, _after, false, KLIST_FFL_HERE) +extern void _k_unlink_item(K_LIST *list, K_ITEM *item, LOCK_MAYBE bool chklock, KLIST_FFL_ARGS); +#define k_unlink_item(_list, _item) _k_unlink_item(_list, _item, true, KLIST_FFL_HERE) +//#define k_unlink_item_nolock(_list, _item) _k_unlink_item(_list, _item, false, KLIST_FFL_HERE) +extern void _k_list_transfer_to_head(K_LIST *from, K_LIST *to, LOCK_MAYBE bool chklock, KLIST_FFL_ARGS); +#define k_list_transfer_to_head(_from, _to) _k_list_transfer_to_head(_from, _to, true, KLIST_FFL_HERE) +//#define k_list_transfer_to_head_nolock(_from, _to) _k_list_transfer_to_head(_from, _to, false, KLIST_FFL_HERE) +extern void _k_list_transfer_to_tail(K_LIST *from, K_LIST *to, LOCK_MAYBE bool chklock, KLIST_FFL_ARGS); +#define k_list_transfer_to_tail(_from, _to) _k_list_transfer_to_tail(_from, _to, true, KLIST_FFL_HERE) +#define k_list_transfer_to_tail_nolock(_from, _to) _k_list_transfer_to_tail(_from, _to, false, KLIST_FFL_HERE) extern K_LIST *_k_free_list(K_LIST *list, KLIST_FFL_ARGS); #define k_free_list(_list) _k_free_list(_list, KLIST_FFL_HERE) extern K_STORE *_k_free_store(K_STORE *store, KLIST_FFL_ARGS); #define k_free_store(_store) _k_free_store(_store, KLIST_FFL_HERE) -extern void _k_cull_list(K_LIST *list, KLIST_FFL_ARGS); -#define k_cull_list(_list) _k_cull_list(_list, KLIST_FFL_HERE) +extern void _k_cull_list(K_LIST *list, LOCK_MAYBE bool chklock, KLIST_FFL_ARGS); +#define k_cull_list(_list) _k_cull_list(_list, true, KLIST_FFL_HERE) +//#define k_cull_list_nolock(_list) _k_cull_list(_list, false, KLIST_FFL_HERE) #endif diff --git a/src/ktree.c b/src/ktree.c index 1181d069..c70131f2 100644 --- a/src/ktree.c +++ b/src/ktree.c @@ -43,7 +43,7 @@ static K_NODE *_new_knode(KTREE_FFL_ARGS) return node; } -K_TREE *_new_ktree(cmp_t (*cmp_funct)(K_ITEM *, K_ITEM *), KTREE_FFL_ARGS) +K_TREE *_new_ktree(cmp_t (*cmp_funct)(K_ITEM *, K_ITEM *), K_LIST *master, KTREE_FFL_ARGS) { K_TREE *tree = (K_TREE *)malloc(sizeof(*tree)); @@ -54,6 +54,8 @@ K_TREE *_new_ktree(cmp_t (*cmp_funct)(K_ITEM *, K_ITEM *), KTREE_FFL_ARGS) tree->cmp_funct = cmp_funct; + tree->master = master; + return tree; } @@ -123,6 +125,8 @@ void _dump_ktree(K_TREE *tree, char *(*dsp_funct)(K_ITEM *), KTREE_FFL_ARGS) { char buf[42424]; + _TREE_READ(tree, true, file, func, line); + printf("dump:\n"); if (tree->root->isNil == No) { @@ -134,7 +138,7 @@ void _dump_ktree(K_TREE *tree, char *(*dsp_funct)(K_ITEM *), KTREE_FFL_ARGS) printf(" Empty tree\n"); } -void _dsp_ktree(K_LIST *list, K_TREE *tree, char *filename, char *msg, KTREE_FFL_ARGS) +void _dsp_ktree(K_TREE *tree, char *filename, char *msg, KTREE_FFL_ARGS) { K_TREE_CTX ctx[1]; K_ITEM *item; @@ -143,8 +147,10 @@ void _dsp_ktree(K_LIST *list, K_TREE *tree, char *filename, char *msg, KTREE_FFL time_t now_t; char stamp[128]; - if (!list->dsp_func) - FAIL("%s", "NULLDSP NULL dsp_func"); + if (!(tree->master->dsp_func)) + FAIL("NULLDSP NULL dsp_func in %s", tree->master->name); + + _TREE_READ(tree, true, file, func, line); now_t = time(NULL); localtime_r(&now_t, &tm); @@ -168,14 +174,14 @@ void _dsp_ktree(K_LIST *list, K_TREE *tree, char *filename, char *msg, KTREE_FFL if (msg) fprintf(stream, "%s %s\n", stamp, msg); else - fprintf(stream, "%s Dump of tree '%s':\n", stamp, list->name); + fprintf(stream, "%s Dump of tree '%s':\n", stamp, tree->master->name); if (tree->root->isNil == No) { item = first_in_ktree(tree, ctx); while (item) { - list->dsp_func(item, stream); + tree->master->dsp_func(item, stream); item = next_in_ktree(ctx); } fprintf(stream, "End\n\n"); @@ -345,8 +351,10 @@ static K_ITEM *_first_in_knode(K_NODE *node, K_TREE_CTX *ctx, KTREE_FFL_ARGS) return(NULL); } -K_ITEM *_first_in_ktree(K_TREE *tree, K_TREE_CTX *ctx, KTREE_FFL_ARGS) +K_ITEM *_first_in_ktree(K_TREE *tree, K_TREE_CTX *ctx, LOCK_MAYBE bool chklock, KTREE_FFL_ARGS) { + _TREE_READ(tree, chklock, file, func, line); + return _first_in_knode(tree->root, ctx, KTREE_FFL_PASS); } @@ -367,9 +375,16 @@ static K_ITEM *_last_in_knode(K_NODE *node, K_TREE_CTX *ctx, KTREE_FFL_ARGS) K_ITEM *_last_in_ktree(K_TREE *tree, K_TREE_CTX *ctx, KTREE_FFL_ARGS) { + _TREE_READ(tree, true, file, func, line); + return _last_in_knode(tree->root, ctx, KTREE_FFL_PASS); } +/* TODO: change ctx to a structure of tree and node then can test _TREE_READ + * However, next/prev is never called before a first/last/find so it's less + * likely to see an error i.e. if code was missing a lock it would be seen + * by first/last/find - here would only see coding errors e.g. looping + * outside the lock */ K_ITEM *_next_in_ktree(K_TREE_CTX *ctx, KTREE_FFL_ARGS) { K_NODE *parent; @@ -482,7 +497,7 @@ static K_NODE *right_rotate(K_NODE *root, K_NODE *about) return(root); } -void _add_to_ktree(K_TREE *tree, K_ITEM *data, KTREE_FFL_ARGS) +void _add_to_ktree(K_TREE *tree, K_ITEM *data, LOCK_MAYBE bool chklock, KTREE_FFL_ARGS) { K_NODE *knode; K_NODE *x, *y; @@ -497,6 +512,8 @@ void _add_to_ktree(K_TREE *tree, K_ITEM *data, KTREE_FFL_ARGS) if (tree->root->parent != nil && tree->root->parent != NULL) FAIL("%s", "ADDROOT add tree->root isn't the root"); + _TREE_WRITE(tree, chklock, file, func, line); + knode = new_data(data, KTREE_FFL_PASS); if (tree->root->isNil == Yo) @@ -581,7 +598,7 @@ void _add_to_ktree(K_TREE *tree, K_ITEM *data, KTREE_FFL_ARGS) //check_ktree(tree, "root == NULL) FAIL("%s", "FINDNULL find tree->root is NULL"); + if (chklock) { + _TREE_READ(tree, true, file, func, line); + } + knode = tree->root; while (knode->isNil == No && cmp != 0) @@ -617,7 +638,7 @@ K_ITEM *_find_in_ktree(K_TREE *tree, K_ITEM *data, K_TREE_CTX *ctx, KTREE_FFL_AR } } -K_ITEM *_find_after_in_ktree(K_TREE *tree, K_ITEM *data, K_TREE_CTX *ctx, KTREE_FFL_ARGS) +K_ITEM *_find_after_in_ktree(K_TREE *tree, K_ITEM *data, K_TREE_CTX *ctx, LOCK_MAYBE bool chklock, KTREE_FFL_ARGS) { K_NODE *knode, *old = NULL; cmp_t cmp = -1, oldcmp = -1; @@ -628,6 +649,8 @@ K_ITEM *_find_after_in_ktree(K_TREE *tree, K_ITEM *data, K_TREE_CTX *ctx, KTREE_ if (tree->root == NULL) FAIL("%s", "FINDNULL find_after tree->root is NULL"); + _TREE_READ(tree, chklock, file, func, line); + knode = tree->root; while (knode->isNil == No && cmp != 0) @@ -678,6 +701,8 @@ K_ITEM *_find_before_in_ktree(K_TREE *tree, K_ITEM *data, K_TREE_CTX *ctx, KTREE if (tree->root == NULL) FAIL("%s", "FINDNULL find_before tree->root is NULL"); + _TREE_READ(tree, true, file, func, line); + knode = tree->root; while (knode->isNil == No && cmp != 0) @@ -815,6 +840,8 @@ void _remove_from_ktree(K_TREE *tree, K_ITEM *data, K_TREE_CTX *ctx, KTREE_FFL_A if (tree->root == NULL) FAIL("%s", "REMNULL remove tree->root is NULL"); + _TREE_WRITE(tree, true, file, func, line); + if (tree->root->isNil == Yo) { *ctx = NULL; diff --git a/src/ktree.h b/src/ktree.h index bf388c92..c1108be6 100644 --- a/src/ktree.h +++ b/src/ktree.h @@ -34,6 +34,9 @@ #define CMP_BIGINT CMP_BIG #define CMP_DOUBLE CMP_BIG +#define _TREE_WRITE(_t, _c, _fl, _f, _l) _LIST_WRITE(_t, _c, _fl, _f, _l) +#define _TREE_READ(_t, _c, _fl, _f, _l) _LIST_READ(_t, _c, _fl, _f, _l) + typedef struct knode { bool isNil; @@ -49,30 +52,39 @@ typedef struct ktree { K_NODE *root; cmp_t (*cmp_funct)(K_ITEM *, K_ITEM *); + K_LIST *master; } K_TREE; typedef void *K_TREE_CTX; -extern K_TREE *_new_ktree(cmp_t (*cmp_funct)(K_ITEM *, K_ITEM *), KTREE_FFL_ARGS); -#define new_ktree(_cmp_funct) _new_ktree(_cmp_funct, KLIST_FFL_HERE) +extern K_TREE *_new_ktree(cmp_t (*cmp_funct)(K_ITEM *, K_ITEM *), K_LIST *master, KTREE_FFL_ARGS); +#define new_ktree(_cmp_funct, _master) _new_ktree(_cmp_funct, _master, KLIST_FFL_HERE) extern void _dump_ktree(K_TREE *tree, char *(*dsp_funct)(K_ITEM *), KTREE_FFL_ARGS); #define dump_ktree(_tree, _dsp_funct) _dump_ktree(_tree, _dsp_funct, KLIST_FFL_HERE) -extern void _dsp_ktree(K_LIST *list, K_TREE *tree, char *filename, char *msg, KTREE_FFL_ARGS); -#define dsp_ktree(_list, _tree, _filename, _msg) _dsp_ktree(_list, _tree, _filename, _msg, KLIST_FFL_HERE) -extern K_ITEM *_first_in_ktree(K_TREE *tree, K_TREE_CTX *ctx, KTREE_FFL_ARGS); -#define first_in_ktree(_tree, _ctx) _first_in_ktree(_tree, _ctx, KLIST_FFL_HERE) +extern void _dsp_ktree(K_TREE *tree, char *filename, char *msg, KTREE_FFL_ARGS); +#define dsp_ktree(_tree, _filename, _msg) _dsp_ktree(_tree, _filename, _msg, KLIST_FFL_HERE) +extern K_ITEM *_first_in_ktree(K_TREE *tree, K_TREE_CTX *ctx, LOCK_MAYBE bool chklock, KTREE_FFL_ARGS); +#define first_in_ktree(_tree, _ctx) _first_in_ktree(_tree, _ctx, true, KLIST_FFL_HERE) +#define first_in_ktree_nolock(_ktree, _ctx) _first_in_ktree(_ktree, _ctx, false, KLIST_FFL_HERE) extern K_ITEM *_last_in_ktree(K_TREE *tree, K_TREE_CTX *ctx, KTREE_FFL_ARGS); #define last_in_ktree(_tree, _ctx) _last_in_ktree(_tree, _ctx, KLIST_FFL_HERE) extern K_ITEM *_next_in_ktree(K_TREE_CTX *ctx, KTREE_FFL_ARGS); #define next_in_ktree(_ctx) _next_in_ktree(_ctx, KLIST_FFL_HERE) +// No difference for now +#define next_in_ktree_nolock(_ctx) _next_in_ktree(_ctx, KLIST_FFL_HERE) extern K_ITEM *_prev_in_ktree(K_TREE_CTX *ctx, KTREE_FFL_ARGS); #define prev_in_ktree(_ctx) _prev_in_ktree(_ctx, KLIST_FFL_HERE) -extern void _add_to_ktree(K_TREE *tree, K_ITEM *data, KTREE_FFL_ARGS); -#define add_to_ktree(_tree, _data) _add_to_ktree(_tree, _data, KLIST_FFL_HERE) -extern K_ITEM *_find_in_ktree(K_TREE *tree, K_ITEM *data, K_TREE_CTX *ctx, KTREE_FFL_ARGS); -#define find_in_ktree(_tree, _data, _ctx) _find_in_ktree(_tree, _data, _ctx, KLIST_FFL_HERE) -extern K_ITEM *_find_after_in_ktree(K_TREE *ktree, K_ITEM *data, K_TREE_CTX *ctx, KTREE_FFL_ARGS); -#define find_after_in_ktree(_ktree, _data, _ctx) _find_after_in_ktree(_ktree, _data, _ctx, KLIST_FFL_HERE) +// No difference for now +#define prev_in_ktree_nolock(_ctx) _prev_in_ktree(_ctx, KLIST_FFL_HERE) +extern void _add_to_ktree(K_TREE *tree, K_ITEM *data, LOCK_MAYBE bool chklock, KTREE_FFL_ARGS); +#define add_to_ktree(_tree, _data) _add_to_ktree(_tree, _data, true, KLIST_FFL_HERE) +#define add_to_ktree_nolock(_tree, _data) _add_to_ktree(_tree, _data, false, KLIST_FFL_HERE) +extern K_ITEM *_find_in_ktree(K_TREE *tree, K_ITEM *data, K_TREE_CTX *ctx, bool chklock, KTREE_FFL_ARGS); +#define find_in_ktree(_tree, _data, _ctx) _find_in_ktree(_tree, _data, _ctx, true, KLIST_FFL_HERE) +#define find_in_ktree_nolock(_tree, _data, _ctx) _find_in_ktree(_tree, _data, _ctx, false, KLIST_FFL_HERE) +extern K_ITEM *_find_after_in_ktree(K_TREE *ktree, K_ITEM *data, K_TREE_CTX *ctx, LOCK_MAYBE bool chklock, KTREE_FFL_ARGS); +#define find_after_in_ktree(_ktree, _data, _ctx) _find_after_in_ktree(_ktree, _data, _ctx, true, KLIST_FFL_HERE) +//#define find_after_in_ktree_nolock(_ktree, _data, _ctx) _find_after_in_ktree(_ktree, _data, _ctx, false, KLIST_FFL_HERE) extern K_ITEM *_find_before_in_ktree(K_TREE *ktree, K_ITEM *data, K_TREE_CTX *ctx, KTREE_FFL_ARGS); #define find_before_in_ktree(_ktree, _data, _ctx) _find_before_in_ktree(_ktree, _data, _ctx, KLIST_FFL_HERE) extern void _remove_from_ktree(K_TREE *tree, K_ITEM *data, K_TREE_CTX *ctx, KTREE_FFL_ARGS);