ckp - make blob ram alloc/free/accounting more consistent

src/ckdb.c

@@ -1320,7 +1320,9 @@ static void dealloc_storage()
 	FREE_ALL(auths);
 
 	FREE_TREE(payouts_id);
-	FREE_ALL(payouts);
+	FREE_TREE(payouts);
+	FREE_STORE_DATA(payouts);
+	FREE_LIST_DATA(payouts);
 
 	FREE_ALL(miningpayouts);
 	FREE_ALL(blocks);
@@ -1400,7 +1402,9 @@ static void dealloc_storage()
 	FREE_ALL(useratts);
 
 	FREE_TREE(userid);
-	FREE_ALL(users);
+	FREE_TREE(users);
+	FREE_STORE_DATA(users);
+	FREE_LIST_DATA(users);
 
 	LOGWARNING("%s() transfer/heartbeatqueue/workqueue ...", __func__);
 

src/ckdb.h

@@ -55,7 +55,7 @@
 
 #define DB_VLOCK "1"
 #define DB_VERSION "1.0.1"
-#define CKDB_VERSION DB_VERSION"-1.200"
+#define CKDB_VERSION DB_VERSION"-1.210"
 
 #define WHERE_FFL " - from %s %s() line %d"
 #define WHERE_FFL_HERE __FILE__, __func__, __LINE__
@@ -67,13 +67,6 @@
 #define STRINT(x) STRINT2(x)
 #define STRINT2(x) #x
 
-#define FREENULL(mem) do { \
-		if (mem) { \
-			free(mem); \
-			mem = NULL; \
-		} \
-	} while (0)
-
 // So they can fit into a 1 byte flag field
 #define TRUE_STR "Y"
 #define FALSE_STR "N"
@@ -105,6 +98,13 @@ extern int switch_state;
 #define BLANK " "
 extern char *EMPTY;
 
+#define FREENULL(mem) do { \
+		if ((mem) && (void *)(mem) != (void *)EMPTY) { \
+			free(mem); \
+			mem = NULL; \
+		} \
+	} while (0)
+
 // To ensure there's space for the ticker
 #define TICK_PREFIX "  "
 
@@ -450,17 +450,21 @@ enum cmd_values {
 	} while (0)
 
 #define LIST_MEM_ADD(_list, _fld) do { \
-		size_t __siz; \
-		__siz = strlen(_fld) + 1; \
-		LIST_MEM_ADD_SIZ(_list, __siz); \
+		if ((_fld) && (_fld) != EMPTY) { \
+			size_t __siz; \
+			__siz = strlen(_fld) + 1; \
+			LIST_MEM_ADD_SIZ(_list, __siz); \
+		} \
 	} while (0)
 
 #define LIST_MEM_SUB(_list, _fld) do { \
-		size_t __siz; \
-		__siz = strlen(_fld) + 1; \
-		if (__siz % MEMBASE) \
-			__siz += MEMBASE - (__siz % MEMBASE); \
-		_list->ram -= (int)__siz; \
+		if ((_fld) && (_fld) != EMPTY) { \
+			size_t __siz; \
+			__siz = strlen(_fld) + 1; \
+			if (__siz % MEMBASE) \
+				__siz += MEMBASE - (__siz % MEMBASE); \
+			_list->ram -= (int)__siz; \
+		} \
 	} while (0)
 
 #define SET_POINTER(_list, _fld, _val, _def) do { \
@@ -2029,8 +2033,10 @@ extern void sequence_report(bool lock);
 
 // Data free functions (first)
 extern void free_msgline_data(K_ITEM *item, bool t_lock, bool t_cull);
+extern void free_users_data(K_ITEM *item);
 extern void free_workinfo_data(K_ITEM *item);
 extern void free_sharesummary_data(K_ITEM *item);
+extern void free_payouts_data(K_ITEM *item);
 extern void free_optioncontrol_data(K_ITEM *item);
 extern void free_markersummary_data(K_ITEM *item);
 extern void free_workmarkers_data(K_ITEM *item);

src/ckdb_cmd.c

@@ -3797,6 +3797,7 @@ static char *cmd_setopts(PGconn *conn, char *cmd, char *id,
 				optioncontrol->optionvalue = strdup(data);
 				if (!(optioncontrol->optionvalue))
 					quithere(1, "malloc (%d) OOM", (int)strlen(data));
+				LIST_MEM_ADD(optioncontrol_free, optioncontrol->optionvalue);
 				gotvalue = true;
 			} else if (strcmp(dot, "date") == 0) {
 				att_to_date(&(optioncontrol->activationdate), data, now);
@@ -4695,6 +4696,7 @@ static char *cmd_payouts(PGconn *conn, char *cmd, char *id, tv_t *now,
 		payouts2->shareacc = payouts->shareacc;
 		copy_tv(&(payouts2->lastshareacc), &(payouts->lastshareacc));
 		payouts2->stats = strdup(payouts->stats);
+		LIST_MEM_ADD(payouts_free, payouts2->stats);
 
 		ok = payouts_add(conn, true, p2_item, &old_p2_item,
 				 by, code, inet, now, NULL, false);
@@ -4759,6 +4761,7 @@ static char *cmd_payouts(PGconn *conn, char *cmd, char *id, tv_t *now,
 		payouts2->shareacc = payouts->shareacc;
 		copy_tv(&(payouts2->lastshareacc), &(payouts->lastshareacc));
 		payouts2->stats = strdup(payouts->stats);
+		LIST_MEM_ADD(payouts_free, payouts2->stats);
 
 		ok = payouts_add(conn, true, p2_item, &old_p2_item,
 				 by, code, inet, now, NULL, false);
@@ -5410,8 +5413,7 @@ static char *cmd_stats(__maybe_unused PGconn *conn, char *cmd, char *id,
 	APPEND_REALLOC_INIT(buf, off, len);
 	APPEND_REALLOC(buf, off, len, "ok.");
 
-// Doesn't include blob memory
-// - average transactiontree length of ~119k I have is ~28k (>3.3GB)
+// FYI average transactiontree length of the ~119k I have is ~28k (>3.3GB)
 #define USEINFO(_obj, _stores, _trees) \
 	klist = _obj ## _free; \
 	ram = sizeof(K_LIST) + _stores * sizeof(K_STORE) + \

src/ckdb_crypt.c

@@ -122,6 +122,7 @@ K_ITEM *gen_2fa_key(K_ITEM *old_u_item, int32_t entropy, char *by, char *code,
 			users->userdata = strdup(users->userdata);
 			if (!users->userdata)
 				quithere(1, "strdup OOM");
+			LIST_MEM_ADD(users_free, users->userdata);
 		}
 		users_userdata_add_bin(users, USER_TOTPAUTH_NAME,
 					USER_TOTPAUTH, key, sizeof(key));
@@ -229,6 +230,7 @@ bool tst_2fa(K_ITEM *old_u_item, int32_t value, char *by, char *code,
 			users->userdata = strdup(users->userdata);
 			if (!users->userdata)
 				quithere(1, "strdup OOM");
+			LIST_MEM_ADD(users_free, users->userdata);
 		}
 		users_userdata_del(users, USER_TEST2FA_NAME, USER_TEST2FA);
 		ok = users_replace(NULL, u_item, old_u_item, by, code, inet, cd,

src/ckdb_data.c

@@ -44,15 +44,33 @@ void free_msgline_data(K_ITEM *item, bool t_lock, bool t_cull)
 	FREENULL(msgline->msg);
 }
 
+void free_users_data(K_ITEM *item)
+{
+	USERS *users;
+
+	DATA_USERS(users, item);
+	LIST_MEM_SUB(users_free, users->userdata);
+	FREENULL(users->userdata);
+}
+
 void free_workinfo_data(K_ITEM *item)
 {
 	WORKINFO *workinfo;
 
 	DATA_WORKINFO(workinfo, item);
-	if (workinfo->transactiontree)
-		FREENULL(workinfo->transactiontree);
-	if (workinfo->merklehash)
-		FREENULL(workinfo->merklehash);
+	LIST_MEM_SUB(workinfo_free, workinfo->transactiontree);
+	FREENULL(workinfo->transactiontree);
+	LIST_MEM_SUB(workinfo_free, workinfo->merklehash);
+	FREENULL(workinfo->merklehash);
+}
+
+void free_payouts_data(K_ITEM *item)
+{
+	PAYOUTS *payouts;
+
+	DATA_PAYOUTS(payouts, item);
+	LIST_MEM_SUB(payouts_free, payouts->stats);
+	FREENULL(payouts->stats);
 }
 
 void free_sharesummary_data(K_ITEM *item)
@@ -60,10 +78,8 @@ void free_sharesummary_data(K_ITEM *item)
 	SHARESUMMARY *sharesummary;
 
 	DATA_SHARESUMMARY(sharesummary, item);
-	if (sharesummary->workername) {
-		LIST_MEM_SUB(sharesummary_free, sharesummary->workername);
-		FREENULL(sharesummary->workername);
-	}
+	LIST_MEM_SUB(sharesummary_free, sharesummary->workername);
+	FREENULL(sharesummary->workername);
 	SET_CREATEBY(sharesummary_free, sharesummary->createby, EMPTY);
 	SET_CREATECODE(sharesummary_free, sharesummary->createcode, EMPTY);
 	SET_CREATEINET(sharesummary_free, sharesummary->createinet, EMPTY);
@@ -77,8 +93,8 @@ void free_optioncontrol_data(K_ITEM *item)
 	OPTIONCONTROL *optioncontrol;
 
 	DATA_OPTIONCONTROL(optioncontrol, item);
-	if (optioncontrol->optionvalue)
-		FREENULL(optioncontrol->optionvalue);
+	LIST_MEM_SUB(optioncontrol_free, optioncontrol->optionvalue);
+	FREENULL(optioncontrol->optionvalue);
 }
 
 void free_markersummary_data(K_ITEM *item)
@@ -86,8 +102,8 @@ void free_markersummary_data(K_ITEM *item)
 	MARKERSUMMARY *markersummary;
 
 	DATA_MARKERSUMMARY(markersummary, item);
-	if (markersummary->workername)
-		FREENULL(markersummary->workername);
+	LIST_MEM_SUB(markersummary_free, markersummary->workername);
+	FREENULL(markersummary->workername);
 	SET_CREATEBY(markersummary_free, markersummary->createby, EMPTY);
 	SET_CREATECODE(markersummary_free, markersummary->createcode, EMPTY);
 	SET_CREATEINET(markersummary_free, markersummary->createinet, EMPTY);
@@ -101,10 +117,10 @@ void free_workmarkers_data(K_ITEM *item)
 	WORKMARKERS *workmarkers;
 
 	DATA_WORKMARKERS(workmarkers, item);
-	if (workmarkers->poolinstance)
-		FREENULL(workmarkers->poolinstance);
-	if (workmarkers->description)
-		FREENULL(workmarkers->description);
+	LIST_MEM_SUB(workmarkers_free, workmarkers->poolinstance);
+	FREENULL(workmarkers->poolinstance);
+	LIST_MEM_SUB(workmarkers_free, workmarkers->description);
+	FREENULL(workmarkers->description);
 }
 
 void free_marks_data(K_ITEM *item)
@@ -112,12 +128,12 @@ void free_marks_data(K_ITEM *item)
 	MARKS *marks;
 
 	DATA_MARKS(marks, item);
-	if (marks->poolinstance && marks->poolinstance != EMPTY)
-		FREENULL(marks->poolinstance);
-	if (marks->description && marks->description != EMPTY)
-		FREENULL(marks->description);
-	if (marks->extra && marks->extra != EMPTY)
-		FREENULL(marks->extra);
+	LIST_MEM_SUB(marks_free, marks->poolinstance);
+	FREENULL(marks->poolinstance);
+	LIST_MEM_SUB(marks_free, marks->description);
+	FREENULL(marks->description);
+	LIST_MEM_SUB(marks_free, marks->extra);
+	FREENULL(marks->extra);
 }
 
 void _free_seqset_data(K_ITEM *item, bool lock)
@@ -3657,6 +3673,7 @@ bool process_pplns(int32_t height, char *blockhash, tv_t *addr_cd)
 		 FLDSEP, ss_count, FLDSEP, wm_count, FLDSEP, ms_count,
 		 FLDSEP, cd_buf);
 	payouts->stats = buf;
+	LIST_MEM_ADD(payouts_free, payouts->stats);
 
 	conned = CKPQConn(&conn);
 	begun = CKPQBegin(conn);
@@ -3872,6 +3889,7 @@ bool process_pplns(int32_t height, char *blockhash, tv_t *addr_cd)
 
 	// convert the stack memory to heap memeory
 	payouts->stats = strdup(payouts->stats);
+	LIST_MEM_ADD(payouts_free, payouts->stats);
 
 	K_WLOCK(payouts_free);
 	p2_item = k_unlink_head(payouts_free);
@@ -3891,6 +3909,7 @@ bool process_pplns(int32_t height, char *blockhash, tv_t *addr_cd)
 	payouts2->shareacc = payouts->shareacc;
 	copy_tv(&(payouts2->lastshareacc), &(payouts->lastshareacc));
 	payouts2->stats = strdup(payouts->stats);
+	LIST_MEM_ADD(payouts_free, payouts2->stats);
 
 	setnow(&now);
 	/* N.B. the PROCESSING payouts could have expirydate = createdate
@@ -3918,6 +3937,7 @@ shazbot:
 
 	if (p_item) {
 		K_WLOCK(payouts_free);
+		free_payouts_data(p_item);
 		k_add_head(payouts_free, p_item);
 		K_WUNLOCK(payouts_free);
 	}

src/ckdb_dbio.c

@@ -432,6 +432,7 @@ bool users_update(PGconn *conn, K_ITEM *u_item, char *oldhash,
 		row->userdata = strdup(users->userdata);
 		if (!row->userdata)
 			quithere(1, "strdup OOM");
+		LIST_MEM_ADD(users_free, row->userdata);
 	}
 
 	HISTORYDATEINIT(row, cd, by, code, inet);
@@ -514,8 +515,7 @@ unparam:
 
 	K_WLOCK(users_free);
 	if (!ok) {
-		if (row->userdata != EMPTY)
-			FREENULL(row->userdata);
+		free_users_data(item);
 		k_add_head(users_free, item);
 	} else {
 		users_root = remove_from_ktree(users_root, u_item, cmp_users);
@@ -649,9 +649,10 @@ unparam:
 		free(params[n]);
 unitem:
 	K_WLOCK(users_free);
-	if (!ok)
+	if (!ok) {
+		free_users_data(item);
 		k_add_head(users_free, item);
-	else {
+	} else {
 		users_root = add_to_ktree(users_root, item, cmp_users);
 		userid_root = add_to_ktree(userid_root, item, cmp_userid);
 		k_add_head(users_store, item);
@@ -761,8 +762,7 @@ unparam:
 	K_WLOCK(users_free);
 	if (!ok) {
 		// cleanup done here
-		if (users->userdata != EMPTY)
-			FREENULL(users->userdata);
+		free_users_data(u_item);
 		k_add_head(users_free, u_item);
 	} else {
 		users_root = remove_from_ktree(users_root, old_u_item, cmp_users);
@@ -892,8 +892,10 @@ bool users_fill(PGconn *conn)
 		userid_root = add_to_ktree(userid_root, item, cmp_userid);
 		k_add_head(users_store, item);
 	}
-	if (!ok)
+	if (!ok) {
+		free_users_data(item);
 		k_add_head(users_free, item);
+	}
 
 	K_WUNLOCK(users_free);
 	PQclear(res);
@@ -2341,7 +2343,7 @@ K_ITEM *optioncontrol_item_add(PGconn *conn, K_ITEM *oc_item, tv_t *cd, bool beg
 	K_TREE_CTX ctx[1];
 	PGresult *res;
 	K_ITEM *old_item, look;
-	OPTIONCONTROL *row, *optioncontrol;
+	OPTIONCONTROL *row;
 	char *upd, *ins;
 	bool ok = false;
 	char *params[4 + HISTORYDATECOUNT];
@@ -2444,16 +2446,15 @@ nostart:
 	K_WLOCK(optioncontrol_free);
 	if (!ok) {
 		// Cleanup item passed in
-		FREENULL(row->optionvalue);
+		free_optioncontrol_data(oc_item);
 		k_add_head(optioncontrol_free, oc_item);
 	} else {
 		// Discard old
 		if (old_item) {
-			DATA_OPTIONCONTROL(optioncontrol, old_item);
 			optioncontrol_root = remove_from_ktree(optioncontrol_root, old_item,
 							       cmp_optioncontrol);
 			k_unlink_item(optioncontrol_store, old_item);
-			FREENULL(optioncontrol->optionvalue);
+			free_optioncontrol_data(old_item);
 			k_add_head(optioncontrol_free, old_item);
 		}
 		optioncontrol_root = add_to_ktree(optioncontrol_root, oc_item, cmp_optioncontrol);
@@ -2600,7 +2601,7 @@ bool optioncontrol_fill(PGconn *conn)
 		}
 	}
 	if (!ok) {
-		FREENULL(row->optionvalue);
+		free_optioncontrol_data(item);
 		k_add_head(optioncontrol_free, item);
 	}
 
@@ -2670,11 +2671,8 @@ int64_t workinfo_add(PGconn *conn, char *workinfoidstr, char *poolinstance,
 
 	K_WLOCK(workinfo_free);
 	if (find_in_ktree(workinfo_root, item, cmp_workinfo, ctx)) {
-		LIST_MEM_SUB(workinfo_free, row->transactiontree);
-		FREENULL(row->transactiontree);
-		LIST_MEM_SUB(workinfo_free, row->merklehash);
-		FREENULL(row->merklehash);
 		workinfoid = row->workinfoid;
+		free_workinfo_data(item);
 		k_add_head(workinfo_free, item);
 		K_WUNLOCK(workinfo_free);
 
@@ -2735,19 +2733,12 @@ unparam:
 
 	K_WLOCK(workinfo_free);
 	if (workinfoid == -1) {
-		LIST_MEM_SUB(workinfo_free, row->transactiontree);
-		FREENULL(row->transactiontree);
-		LIST_MEM_SUB(workinfo_free, row->merklehash);
-		FREENULL(row->merklehash);
+		free_workinfo_data(item);
 		k_add_head(workinfo_free, item);
 	} else {
-		if (row->transactiontree && *(row->transactiontree)) {
-			// Not currently needed in RAM
-			LIST_MEM_SUB(workinfo_free, row->transactiontree);
-			free(row->transactiontree);
-			row->transactiontree = strdup(EMPTY);
-			LIST_MEM_ADD(workinfo_free, row->transactiontree);
-		}
+		// Not currently needed in RAM
+		LIST_MEM_SUB(workinfo_free, row->transactiontree);
+		FREENULL(row->transactiontree);
 
 		hex2bin(ndiffbin, row->bits, 4);
 		current_ndiff = diff_from_nbits(ndiffbin);
@@ -2870,9 +2861,9 @@ bool workinfo_fill(PGconn *conn)
 		if (!ok)
 			break;
 		TXT_TO_BLOB("transactiontree", field, row->transactiontree);
-*/
-		row->transactiontree = strdup(EMPTY);
 		LIST_MEM_ADD(workinfo_free, row->transactiontree);
+*/
+		row->transactiontree = EMPTY;
 
 		PQ_GET_FLD(res, i, "merklehash", field, ok);
 		if (!ok)
@@ -2940,8 +2931,7 @@ bool workinfo_fill(PGconn *conn)
 		tick();
 	}
 	if (!ok) {
-		//FREENULL(row->transactiontree);
-		FREENULL(row->merklehash);
+		free_workinfo_data(item);
 		k_add_head(workinfo_free, item);
 	}
 
@@ -5097,6 +5087,7 @@ void payouts_add_ram(bool ok, K_ITEM *p_item, K_ITEM *old_p_item, tv_t *cd)
 	K_WLOCK(payouts_free);
 	if (!ok) {
 		// Cleanup for the calling function
+		free_payouts_data(p_item);
 		k_add_head(payouts_free, p_item);
 	} else {
 		if (old_p_item) {
@@ -5601,7 +5592,7 @@ bool payouts_fill(PGconn *conn)
 		tick();
 	}
 	if (!ok) {
-		FREENULL(row->stats);
+		free_payouts_data(item);
 		k_add_head(payouts_free, item);
 	}
 
@@ -6448,7 +6439,7 @@ bool markersummary_fill(PGconn *conn)
 		tick();
 	}
 	if (!ok) {
-		FREENULL(row->workername);
+		free_markersummary_data(item);
 		k_add_head(markersummary_free, item);
 	}
 
@@ -6658,23 +6649,7 @@ unparam:
 	K_WLOCK(workmarkers_free);
 	if (!ok) {
 		if (wm_item) {
-			DATA_WORKMARKERS(row, wm_item);
-			if (row->poolinstance) {
-				if (row->poolinstance != EMPTY) {
-					LIST_MEM_SUB(workmarkers_free,
-						     row->poolinstance);
-					free(row->poolinstance);
-				}
-				row->poolinstance = NULL;
-			}
-			if (row->description) {
-				if (row->description != EMPTY) {
-					LIST_MEM_SUB(workmarkers_free,
-						     row->description);
-					free(row->description);
-				}
-				row->description = NULL;
-			}
+			free_workmarkers_data(wm_item);
 			k_add_head(workmarkers_free, wm_item);
 		}
 	}
@@ -6820,8 +6795,10 @@ bool workmarkers_fill(PGconn *conn)
 
 		tick();
 	}
-	if (!ok)
+	if (!ok) {
+		free_workmarkers_data(item);
 		k_add_head(workmarkers_free, item);
+	}
 
 	K_WUNLOCK(workmarkers_free);
 	PQclear(res);
@@ -6991,32 +6968,10 @@ unparam:
 	K_WLOCK(marks_free);
 	if (!ok) {
 		if (m_item) {
-			DATA_MARKS(row, m_item);
-			if (row->poolinstance) {
-				if (row->poolinstance != EMPTY) {
-					LIST_MEM_SUB(marks_free, row->poolinstance);
-					free(row->poolinstance);
-				}
-				row->poolinstance = NULL;
-			}
-			if (row->description) {
-				if (row->description != EMPTY) {
-					LIST_MEM_SUB(marks_free, row->description);
-					free(row->description);
-				}
-				row->description = NULL;
-			}
-			if (row->extra) {
-				if (row->extra != EMPTY) {
-					LIST_MEM_SUB(marks_free, row->extra);
-					free(row->extra);
-				}
-				row->extra = NULL;
-			}
+			free_marks_data(m_item);
 			k_add_head(marks_free, m_item);
 		}
-	}
-	else {
+	} else {
 		if (old_m_item) {
 			marks_root = remove_from_ktree(marks_root, old_m_item, cmp_marks);
 			copy_tv(&(oldmarks->expirydate), cd);
@@ -7123,8 +7078,10 @@ bool marks_fill(PGconn *conn)
 
 		tick();
 	}
-	if (!ok)
+	if (!ok) {
+		free_marks_data(item);
 		k_add_head(marks_free, item);
+	}
 
 	K_WUNLOCK(marks_free);
 	PQclear(res);