From f4d00f83e516b0d6099b64852151ebd815ff2f09 Mon Sep 17 00:00:00 2001 From: kanoi Date: Wed, 5 Aug 2015 11:51:49 +1000 Subject: [PATCH] ckp - make blob ram alloc/free/accounting more consistent --- src/ckdb.c | 8 +++- src/ckdb.h | 38 ++++++++++------- src/ckdb_cmd.c | 6 ++- src/ckdb_crypt.c | 2 + src/ckdb_data.c | 64 ++++++++++++++++++---------- src/ckdb_dbio.c | 109 ++++++++++++++--------------------------------- 6 files changed, 109 insertions(+), 118 deletions(-) diff --git a/src/ckdb.c b/src/ckdb.c index 041e3cc6..c0c78bdd 100644 --- a/src/ckdb.c +++ b/src/ckdb.c @@ -1320,7 +1320,9 @@ static void dealloc_storage() FREE_ALL(auths); FREE_TREE(payouts_id); - FREE_ALL(payouts); + FREE_TREE(payouts); + FREE_STORE_DATA(payouts); + FREE_LIST_DATA(payouts); FREE_ALL(miningpayouts); FREE_ALL(blocks); @@ -1400,7 +1402,9 @@ static void dealloc_storage() FREE_ALL(useratts); FREE_TREE(userid); - FREE_ALL(users); + FREE_TREE(users); + FREE_STORE_DATA(users); + FREE_LIST_DATA(users); LOGWARNING("%s() transfer/heartbeatqueue/workqueue ...", __func__); diff --git a/src/ckdb.h b/src/ckdb.h index 76a45dd6..a26155ec 100644 --- a/src/ckdb.h +++ b/src/ckdb.h @@ -55,7 +55,7 @@ #define DB_VLOCK "1" #define DB_VERSION "1.0.1" -#define CKDB_VERSION DB_VERSION"-1.200" +#define CKDB_VERSION DB_VERSION"-1.210" #define WHERE_FFL " - from %s %s() line %d" #define WHERE_FFL_HERE __FILE__, __func__, __LINE__ @@ -67,13 +67,6 @@ #define STRINT(x) STRINT2(x) #define STRINT2(x) #x -#define FREENULL(mem) do { \ - if (mem) { \ - free(mem); \ - mem = NULL; \ - } \ - } while (0) - // So they can fit into a 1 byte flag field #define TRUE_STR "Y" #define FALSE_STR "N" @@ -105,6 +98,13 @@ extern int switch_state; #define BLANK " " extern char *EMPTY; +#define FREENULL(mem) do { \ + if ((mem) && (void *)(mem) != (void *)EMPTY) { \ + free(mem); \ + mem = NULL; \ + } \ + } while (0) + // To ensure there's space for the ticker #define TICK_PREFIX " " @@ -450,17 +450,21 @@ enum cmd_values { } while (0) #define LIST_MEM_ADD(_list, _fld) do { \ - size_t __siz; \ - __siz = strlen(_fld) + 1; \ - LIST_MEM_ADD_SIZ(_list, __siz); \ + if ((_fld) && (_fld) != EMPTY) { \ + size_t __siz; \ + __siz = strlen(_fld) + 1; \ + LIST_MEM_ADD_SIZ(_list, __siz); \ + } \ } while (0) #define LIST_MEM_SUB(_list, _fld) do { \ - size_t __siz; \ - __siz = strlen(_fld) + 1; \ - if (__siz % MEMBASE) \ - __siz += MEMBASE - (__siz % MEMBASE); \ - _list->ram -= (int)__siz; \ + if ((_fld) && (_fld) != EMPTY) { \ + size_t __siz; \ + __siz = strlen(_fld) + 1; \ + if (__siz % MEMBASE) \ + __siz += MEMBASE - (__siz % MEMBASE); \ + _list->ram -= (int)__siz; \ + } \ } while (0) #define SET_POINTER(_list, _fld, _val, _def) do { \ @@ -2029,8 +2033,10 @@ extern void sequence_report(bool lock); // Data free functions (first) extern void free_msgline_data(K_ITEM *item, bool t_lock, bool t_cull); +extern void free_users_data(K_ITEM *item); extern void free_workinfo_data(K_ITEM *item); extern void free_sharesummary_data(K_ITEM *item); +extern void free_payouts_data(K_ITEM *item); extern void free_optioncontrol_data(K_ITEM *item); extern void free_markersummary_data(K_ITEM *item); extern void free_workmarkers_data(K_ITEM *item); diff --git a/src/ckdb_cmd.c b/src/ckdb_cmd.c index 798cc6c4..00d787c3 100644 --- a/src/ckdb_cmd.c +++ b/src/ckdb_cmd.c @@ -3797,6 +3797,7 @@ static char *cmd_setopts(PGconn *conn, char *cmd, char *id, optioncontrol->optionvalue = strdup(data); if (!(optioncontrol->optionvalue)) quithere(1, "malloc (%d) OOM", (int)strlen(data)); + LIST_MEM_ADD(optioncontrol_free, optioncontrol->optionvalue); gotvalue = true; } else if (strcmp(dot, "date") == 0) { att_to_date(&(optioncontrol->activationdate), data, now); @@ -4695,6 +4696,7 @@ static char *cmd_payouts(PGconn *conn, char *cmd, char *id, tv_t *now, payouts2->shareacc = payouts->shareacc; copy_tv(&(payouts2->lastshareacc), &(payouts->lastshareacc)); payouts2->stats = strdup(payouts->stats); + LIST_MEM_ADD(payouts_free, payouts2->stats); ok = payouts_add(conn, true, p2_item, &old_p2_item, by, code, inet, now, NULL, false); @@ -4759,6 +4761,7 @@ static char *cmd_payouts(PGconn *conn, char *cmd, char *id, tv_t *now, payouts2->shareacc = payouts->shareacc; copy_tv(&(payouts2->lastshareacc), &(payouts->lastshareacc)); payouts2->stats = strdup(payouts->stats); + LIST_MEM_ADD(payouts_free, payouts2->stats); ok = payouts_add(conn, true, p2_item, &old_p2_item, by, code, inet, now, NULL, false); @@ -5410,8 +5413,7 @@ static char *cmd_stats(__maybe_unused PGconn *conn, char *cmd, char *id, APPEND_REALLOC_INIT(buf, off, len); APPEND_REALLOC(buf, off, len, "ok."); -// Doesn't include blob memory -// - average transactiontree length of ~119k I have is ~28k (>3.3GB) +// FYI average transactiontree length of the ~119k I have is ~28k (>3.3GB) #define USEINFO(_obj, _stores, _trees) \ klist = _obj ## _free; \ ram = sizeof(K_LIST) + _stores * sizeof(K_STORE) + \ diff --git a/src/ckdb_crypt.c b/src/ckdb_crypt.c index 360ba93a..42e7f43c 100644 --- a/src/ckdb_crypt.c +++ b/src/ckdb_crypt.c @@ -122,6 +122,7 @@ K_ITEM *gen_2fa_key(K_ITEM *old_u_item, int32_t entropy, char *by, char *code, users->userdata = strdup(users->userdata); if (!users->userdata) quithere(1, "strdup OOM"); + LIST_MEM_ADD(users_free, users->userdata); } users_userdata_add_bin(users, USER_TOTPAUTH_NAME, USER_TOTPAUTH, key, sizeof(key)); @@ -229,6 +230,7 @@ bool tst_2fa(K_ITEM *old_u_item, int32_t value, char *by, char *code, users->userdata = strdup(users->userdata); if (!users->userdata) quithere(1, "strdup OOM"); + LIST_MEM_ADD(users_free, users->userdata); } users_userdata_del(users, USER_TEST2FA_NAME, USER_TEST2FA); ok = users_replace(NULL, u_item, old_u_item, by, code, inet, cd, diff --git a/src/ckdb_data.c b/src/ckdb_data.c index 18e780fd..d44d870d 100644 --- a/src/ckdb_data.c +++ b/src/ckdb_data.c @@ -44,15 +44,33 @@ void free_msgline_data(K_ITEM *item, bool t_lock, bool t_cull) FREENULL(msgline->msg); } +void free_users_data(K_ITEM *item) +{ + USERS *users; + + DATA_USERS(users, item); + LIST_MEM_SUB(users_free, users->userdata); + FREENULL(users->userdata); +} + void free_workinfo_data(K_ITEM *item) { WORKINFO *workinfo; DATA_WORKINFO(workinfo, item); - if (workinfo->transactiontree) - FREENULL(workinfo->transactiontree); - if (workinfo->merklehash) - FREENULL(workinfo->merklehash); + LIST_MEM_SUB(workinfo_free, workinfo->transactiontree); + FREENULL(workinfo->transactiontree); + LIST_MEM_SUB(workinfo_free, workinfo->merklehash); + FREENULL(workinfo->merklehash); +} + +void free_payouts_data(K_ITEM *item) +{ + PAYOUTS *payouts; + + DATA_PAYOUTS(payouts, item); + LIST_MEM_SUB(payouts_free, payouts->stats); + FREENULL(payouts->stats); } void free_sharesummary_data(K_ITEM *item) @@ -60,10 +78,8 @@ void free_sharesummary_data(K_ITEM *item) SHARESUMMARY *sharesummary; DATA_SHARESUMMARY(sharesummary, item); - if (sharesummary->workername) { - LIST_MEM_SUB(sharesummary_free, sharesummary->workername); - FREENULL(sharesummary->workername); - } + LIST_MEM_SUB(sharesummary_free, sharesummary->workername); + FREENULL(sharesummary->workername); SET_CREATEBY(sharesummary_free, sharesummary->createby, EMPTY); SET_CREATECODE(sharesummary_free, sharesummary->createcode, EMPTY); SET_CREATEINET(sharesummary_free, sharesummary->createinet, EMPTY); @@ -77,8 +93,8 @@ void free_optioncontrol_data(K_ITEM *item) OPTIONCONTROL *optioncontrol; DATA_OPTIONCONTROL(optioncontrol, item); - if (optioncontrol->optionvalue) - FREENULL(optioncontrol->optionvalue); + LIST_MEM_SUB(optioncontrol_free, optioncontrol->optionvalue); + FREENULL(optioncontrol->optionvalue); } void free_markersummary_data(K_ITEM *item) @@ -86,8 +102,8 @@ void free_markersummary_data(K_ITEM *item) MARKERSUMMARY *markersummary; DATA_MARKERSUMMARY(markersummary, item); - if (markersummary->workername) - FREENULL(markersummary->workername); + LIST_MEM_SUB(markersummary_free, markersummary->workername); + FREENULL(markersummary->workername); SET_CREATEBY(markersummary_free, markersummary->createby, EMPTY); SET_CREATECODE(markersummary_free, markersummary->createcode, EMPTY); SET_CREATEINET(markersummary_free, markersummary->createinet, EMPTY); @@ -101,10 +117,10 @@ void free_workmarkers_data(K_ITEM *item) WORKMARKERS *workmarkers; DATA_WORKMARKERS(workmarkers, item); - if (workmarkers->poolinstance) - FREENULL(workmarkers->poolinstance); - if (workmarkers->description) - FREENULL(workmarkers->description); + LIST_MEM_SUB(workmarkers_free, workmarkers->poolinstance); + FREENULL(workmarkers->poolinstance); + LIST_MEM_SUB(workmarkers_free, workmarkers->description); + FREENULL(workmarkers->description); } void free_marks_data(K_ITEM *item) @@ -112,12 +128,12 @@ void free_marks_data(K_ITEM *item) MARKS *marks; DATA_MARKS(marks, item); - if (marks->poolinstance && marks->poolinstance != EMPTY) - FREENULL(marks->poolinstance); - if (marks->description && marks->description != EMPTY) - FREENULL(marks->description); - if (marks->extra && marks->extra != EMPTY) - FREENULL(marks->extra); + LIST_MEM_SUB(marks_free, marks->poolinstance); + FREENULL(marks->poolinstance); + LIST_MEM_SUB(marks_free, marks->description); + FREENULL(marks->description); + LIST_MEM_SUB(marks_free, marks->extra); + FREENULL(marks->extra); } void _free_seqset_data(K_ITEM *item, bool lock) @@ -3657,6 +3673,7 @@ bool process_pplns(int32_t height, char *blockhash, tv_t *addr_cd) FLDSEP, ss_count, FLDSEP, wm_count, FLDSEP, ms_count, FLDSEP, cd_buf); payouts->stats = buf; + LIST_MEM_ADD(payouts_free, payouts->stats); conned = CKPQConn(&conn); begun = CKPQBegin(conn); @@ -3872,6 +3889,7 @@ bool process_pplns(int32_t height, char *blockhash, tv_t *addr_cd) // convert the stack memory to heap memeory payouts->stats = strdup(payouts->stats); + LIST_MEM_ADD(payouts_free, payouts->stats); K_WLOCK(payouts_free); p2_item = k_unlink_head(payouts_free); @@ -3891,6 +3909,7 @@ bool process_pplns(int32_t height, char *blockhash, tv_t *addr_cd) payouts2->shareacc = payouts->shareacc; copy_tv(&(payouts2->lastshareacc), &(payouts->lastshareacc)); payouts2->stats = strdup(payouts->stats); + LIST_MEM_ADD(payouts_free, payouts2->stats); setnow(&now); /* N.B. the PROCESSING payouts could have expirydate = createdate @@ -3918,6 +3937,7 @@ shazbot: if (p_item) { K_WLOCK(payouts_free); + free_payouts_data(p_item); k_add_head(payouts_free, p_item); K_WUNLOCK(payouts_free); } diff --git a/src/ckdb_dbio.c b/src/ckdb_dbio.c index 07117a82..3b131b55 100644 --- a/src/ckdb_dbio.c +++ b/src/ckdb_dbio.c @@ -432,6 +432,7 @@ bool users_update(PGconn *conn, K_ITEM *u_item, char *oldhash, row->userdata = strdup(users->userdata); if (!row->userdata) quithere(1, "strdup OOM"); + LIST_MEM_ADD(users_free, row->userdata); } HISTORYDATEINIT(row, cd, by, code, inet); @@ -514,8 +515,7 @@ unparam: K_WLOCK(users_free); if (!ok) { - if (row->userdata != EMPTY) - FREENULL(row->userdata); + free_users_data(item); k_add_head(users_free, item); } else { users_root = remove_from_ktree(users_root, u_item, cmp_users); @@ -649,9 +649,10 @@ unparam: free(params[n]); unitem: K_WLOCK(users_free); - if (!ok) + if (!ok) { + free_users_data(item); k_add_head(users_free, item); - else { + } else { users_root = add_to_ktree(users_root, item, cmp_users); userid_root = add_to_ktree(userid_root, item, cmp_userid); k_add_head(users_store, item); @@ -761,8 +762,7 @@ unparam: K_WLOCK(users_free); if (!ok) { // cleanup done here - if (users->userdata != EMPTY) - FREENULL(users->userdata); + free_users_data(u_item); k_add_head(users_free, u_item); } else { users_root = remove_from_ktree(users_root, old_u_item, cmp_users); @@ -892,8 +892,10 @@ bool users_fill(PGconn *conn) userid_root = add_to_ktree(userid_root, item, cmp_userid); k_add_head(users_store, item); } - if (!ok) + if (!ok) { + free_users_data(item); k_add_head(users_free, item); + } K_WUNLOCK(users_free); PQclear(res); @@ -2341,7 +2343,7 @@ K_ITEM *optioncontrol_item_add(PGconn *conn, K_ITEM *oc_item, tv_t *cd, bool beg K_TREE_CTX ctx[1]; PGresult *res; K_ITEM *old_item, look; - OPTIONCONTROL *row, *optioncontrol; + OPTIONCONTROL *row; char *upd, *ins; bool ok = false; char *params[4 + HISTORYDATECOUNT]; @@ -2444,16 +2446,15 @@ nostart: K_WLOCK(optioncontrol_free); if (!ok) { // Cleanup item passed in - FREENULL(row->optionvalue); + free_optioncontrol_data(oc_item); k_add_head(optioncontrol_free, oc_item); } else { // Discard old if (old_item) { - DATA_OPTIONCONTROL(optioncontrol, old_item); optioncontrol_root = remove_from_ktree(optioncontrol_root, old_item, cmp_optioncontrol); k_unlink_item(optioncontrol_store, old_item); - FREENULL(optioncontrol->optionvalue); + free_optioncontrol_data(old_item); k_add_head(optioncontrol_free, old_item); } optioncontrol_root = add_to_ktree(optioncontrol_root, oc_item, cmp_optioncontrol); @@ -2600,7 +2601,7 @@ bool optioncontrol_fill(PGconn *conn) } } if (!ok) { - FREENULL(row->optionvalue); + free_optioncontrol_data(item); k_add_head(optioncontrol_free, item); } @@ -2670,11 +2671,8 @@ int64_t workinfo_add(PGconn *conn, char *workinfoidstr, char *poolinstance, K_WLOCK(workinfo_free); if (find_in_ktree(workinfo_root, item, cmp_workinfo, ctx)) { - LIST_MEM_SUB(workinfo_free, row->transactiontree); - FREENULL(row->transactiontree); - LIST_MEM_SUB(workinfo_free, row->merklehash); - FREENULL(row->merklehash); workinfoid = row->workinfoid; + free_workinfo_data(item); k_add_head(workinfo_free, item); K_WUNLOCK(workinfo_free); @@ -2735,19 +2733,12 @@ unparam: K_WLOCK(workinfo_free); if (workinfoid == -1) { - LIST_MEM_SUB(workinfo_free, row->transactiontree); - FREENULL(row->transactiontree); - LIST_MEM_SUB(workinfo_free, row->merklehash); - FREENULL(row->merklehash); + free_workinfo_data(item); k_add_head(workinfo_free, item); } else { - if (row->transactiontree && *(row->transactiontree)) { - // Not currently needed in RAM - LIST_MEM_SUB(workinfo_free, row->transactiontree); - free(row->transactiontree); - row->transactiontree = strdup(EMPTY); - LIST_MEM_ADD(workinfo_free, row->transactiontree); - } + // Not currently needed in RAM + LIST_MEM_SUB(workinfo_free, row->transactiontree); + FREENULL(row->transactiontree); hex2bin(ndiffbin, row->bits, 4); current_ndiff = diff_from_nbits(ndiffbin); @@ -2870,9 +2861,9 @@ bool workinfo_fill(PGconn *conn) if (!ok) break; TXT_TO_BLOB("transactiontree", field, row->transactiontree); -*/ - row->transactiontree = strdup(EMPTY); LIST_MEM_ADD(workinfo_free, row->transactiontree); +*/ + row->transactiontree = EMPTY; PQ_GET_FLD(res, i, "merklehash", field, ok); if (!ok) @@ -2940,8 +2931,7 @@ bool workinfo_fill(PGconn *conn) tick(); } if (!ok) { - //FREENULL(row->transactiontree); - FREENULL(row->merklehash); + free_workinfo_data(item); k_add_head(workinfo_free, item); } @@ -5097,6 +5087,7 @@ void payouts_add_ram(bool ok, K_ITEM *p_item, K_ITEM *old_p_item, tv_t *cd) K_WLOCK(payouts_free); if (!ok) { // Cleanup for the calling function + free_payouts_data(p_item); k_add_head(payouts_free, p_item); } else { if (old_p_item) { @@ -5601,7 +5592,7 @@ bool payouts_fill(PGconn *conn) tick(); } if (!ok) { - FREENULL(row->stats); + free_payouts_data(item); k_add_head(payouts_free, item); } @@ -6448,7 +6439,7 @@ bool markersummary_fill(PGconn *conn) tick(); } if (!ok) { - FREENULL(row->workername); + free_markersummary_data(item); k_add_head(markersummary_free, item); } @@ -6658,23 +6649,7 @@ unparam: K_WLOCK(workmarkers_free); if (!ok) { if (wm_item) { - DATA_WORKMARKERS(row, wm_item); - if (row->poolinstance) { - if (row->poolinstance != EMPTY) { - LIST_MEM_SUB(workmarkers_free, - row->poolinstance); - free(row->poolinstance); - } - row->poolinstance = NULL; - } - if (row->description) { - if (row->description != EMPTY) { - LIST_MEM_SUB(workmarkers_free, - row->description); - free(row->description); - } - row->description = NULL; - } + free_workmarkers_data(wm_item); k_add_head(workmarkers_free, wm_item); } } @@ -6820,8 +6795,10 @@ bool workmarkers_fill(PGconn *conn) tick(); } - if (!ok) + if (!ok) { + free_workmarkers_data(item); k_add_head(workmarkers_free, item); + } K_WUNLOCK(workmarkers_free); PQclear(res); @@ -6991,32 +6968,10 @@ unparam: K_WLOCK(marks_free); if (!ok) { if (m_item) { - DATA_MARKS(row, m_item); - if (row->poolinstance) { - if (row->poolinstance != EMPTY) { - LIST_MEM_SUB(marks_free, row->poolinstance); - free(row->poolinstance); - } - row->poolinstance = NULL; - } - if (row->description) { - if (row->description != EMPTY) { - LIST_MEM_SUB(marks_free, row->description); - free(row->description); - } - row->description = NULL; - } - if (row->extra) { - if (row->extra != EMPTY) { - LIST_MEM_SUB(marks_free, row->extra); - free(row->extra); - } - row->extra = NULL; - } + free_marks_data(m_item); k_add_head(marks_free, m_item); } - } - else { + } else { if (old_m_item) { marks_root = remove_from_ktree(marks_root, old_m_item, cmp_marks); copy_tv(&(oldmarks->expirydate), cd); @@ -7123,8 +7078,10 @@ bool marks_fill(PGconn *conn) tick(); } - if (!ok) + if (!ok) { + free_marks_data(item); k_add_head(marks_free, item); + } K_WUNLOCK(marks_free); PQclear(res);