ckdb - reduce the size of workinfo and make some intransient

src/ckdb.c

@@ -469,6 +469,8 @@ char *intransient_fields[] = {
 	"useragent",
 	"address",
 	"prevhash",
+	"bits",
+	"version",
 	"poolinstance",
 	"payaddress",
 	"originaltxn",
@@ -2729,7 +2731,7 @@ static bool setup_data()
 	workinfo_current = last_in_ktree(workinfo_height_root, ctx);
 	if (workinfo_current) {
 		DATA_WORKINFO(wic, workinfo_current);
-		STRNCPY(wi.coinbase1, wic->coinbase1);
+		wi.coinbase1 = wic->coinbase1;
 		DATE_ZERO(&(wi.createdate));
 		INIT_WORKINFO(&look);
 		look.data = (void *)(&wi);
@@ -8953,7 +8955,7 @@ static struct option long_options[] = {
 	{ "btc-user",		required_argument,	0,	'U' },
 	{ "version",		no_argument,		0,	'v' },
 	{ "workinfoid",		required_argument,	0,	'w' },
-	// Disable writing to the db-log file
+	// Disable writing to the db-log file (for testing environments)
 	{ "no-db-log",		no_argument,		0,	'x' },
 	{ "confirm",		no_argument,		0,	'y' },
 	{ "confirmrange",	required_argument,	0,	'Y' },

src/ckdb.h

@@ -58,7 +58,7 @@
 
 #define DB_VLOCK "1"
 #define DB_VERSION "1.0.7"
-#define CKDB_VERSION DB_VERSION"-2.446"
+#define CKDB_VERSION DB_VERSION"-2.447"
 
 #define WHERE_FFL " - from %s %s() line %d"
 #define WHERE_FFL_HERE __FILE__, __func__, __LINE__
@@ -1156,8 +1156,7 @@ typedef struct intransient {
 	char *str;
 } INTRANSIENT;
 
-/* Items never to be deleted and list never to be culled
- * They are all created by breakdown */
+// Items never to be deleted and list never to be culled
 #define ALLOC_INTRANSIENT 1024
 #define LIMIT_INTRANSIENT 0
 #define INIT_INTRANSIENT(_item) INIT_GENERIC(_item, intransient)
@@ -1952,14 +1951,14 @@ extern K_STORE *esm_store;
 // WORKINFO workinfo.id.json={...}
 typedef struct workinfo {
 	int64_t workinfoid;
-	char poolinstance[TXT_BIG+1];
+	char *poolinstance;
 	char *transactiontree;
 	char *merklehash;
-	char prevhash[TXT_BIG+1];
-	char coinbase1[TXT_BIG+1];
-	char coinbase2[TXT_BIG+1];
-	char version[TXT_SML+1];
-	char bits[TXT_SML+1];
+	char *prevhash;
+	char *coinbase1;
+	char *coinbase2;
+	char *version;
+	char *bits;
 	char ntime[TXT_SML+1];
 	int64_t reward;
 	int32_t height; // non-DB field
@@ -3152,6 +3151,9 @@ extern cmp_t cmp_intransient(K_ITEM *a, K_ITEM *b);
 	_get_intransient(_fld, _val , _siz, WHERE_FFL_HERE)
 extern INTRANSIENT *_get_intransient(char *fldnam, char *value, size_t siz,
 					WHERE_FFL_ARGS);
+#define intransient_str(_fld, _val) \
+	_intransient_str(_fld, _val, WHERE_FFL_HERE)
+extern char *_intransient_str(char *fldnam, char *value, WHERE_FFL_ARGS);
 extern char *_transfer_data(K_ITEM *item, WHERE_FFL_ARGS);
 extern void dsp_transfer(K_ITEM *item, FILE *stream);
 extern cmp_t cmp_transfer(K_ITEM *a, K_ITEM *b);
@@ -3168,6 +3170,11 @@ extern K_ITEM *_optional_name(K_TREE *trf_root, char *name, int len, char *patt,
 				_siz, WHERE_FFL_HERE)
 extern K_ITEM *_require_name(K_TREE *trf_root, char *name, int len, char *patt,
 				char *reply, size_t siz, WHERE_FFL_ARGS);
+#define require_in(_root, _name, _len, _patt, _reply, _siz) \
+		_require_in(_root, _name, _len, _patt, _reply, \
+				_siz, WHERE_FFL_HERE)
+extern INTRANSIENT *_require_in(K_TREE *trf_root, char *name, int len, char *patt,
+				char *reply, size_t siz, WHERE_FFL_ARGS);
 extern cmp_t cmp_workerstatus(K_ITEM *a, K_ITEM *b);
 extern K_ITEM *find_workerstatus(bool gotlock, int64_t userid, char *workername);
 #define find_create_workerstatus(_gl, _ac, _u, _w, _hw, _file, _func, _line) \
@@ -3504,10 +3511,11 @@ extern K_ITEM *optioncontrol_add(PGconn *conn, char *optionname, char *optionval
 				 char *by, char *code, char *inet, tv_t *cd,
 				 K_TREE *trf_root, bool begun);
 extern bool optioncontrol_fill(PGconn *conn);
-extern int64_t workinfo_add(PGconn *conn, char *workinfoidstr, char *poolinstance,
-				char *transactiontree, char *merklehash, char *prevhash,
-				char *coinbase1, char *coinbase2, char *version,
-				char *bits, char *ntime, char *reward, char *by,
+extern int64_t workinfo_add(PGconn *conn, char *workinfoidstr,
+				INTRANSIENT *in_poolinstance, char *transactiontree,
+				char *merklehash, INTRANSIENT *in_prevhash,
+				char *coinbase1, char *coinbase2, INTRANSIENT *in_version,
+				INTRANSIENT *in_bits, char *ntime, char *reward, char *by,
 				char *code, char *inet, tv_t *cd, bool igndup,
 				K_TREE *trf_root);
 extern bool workinfo_fill(PGconn *conn);

src/ckdb_cmd.c

@@ -2522,9 +2522,9 @@ static char *cmd_sharelog(PGconn *conn, char *cmd, char *id,
 	LOGDEBUG("%s(): cmd '%s'", __func__, cmd);
 
 	if (strcasecmp(cmd, STR_WORKINFO) == 0) {
-		K_ITEM *i_workinfoid, *i_poolinstance, *i_transactiontree, *i_merklehash;
-		K_ITEM *i_prevhash, *i_coinbase1, *i_coinbase2, *i_version, *i_bits;
-		K_ITEM *i_ntime, *i_reward;
+		K_ITEM *i_workinfoid, *i_transactiontree, *i_merklehash;
+		K_ITEM *i_coinbase1, *i_coinbase2, *i_ntime, *i_reward;
+		INTRANSIENT *in_poolinstance, *in_prevhash, *in_version, *in_bits;
 		bool igndup = false;
 		char *txn_tree;
 
@@ -2532,12 +2532,12 @@ static char *cmd_sharelog(PGconn *conn, char *cmd, char *id,
 		if (key_update)
 			goto wiconf;
 
-		i_poolinstance = require_name(trf_root, "poolinstance", 1, NULL, reply, siz);
-		if (!i_poolinstance)
+		in_poolinstance = require_in(trf_root, "poolinstance", 1, NULL, reply, siz);
+		if (!in_poolinstance)
 			return strdup(reply);
 
-		if (poolinstance && strcmp(poolinstance, transfer_data(i_poolinstance))){
-			POOLINSTANCE_DATA_SET(workinfo, transfer_data(i_poolinstance));
+		if (poolinstance && strcmp(poolinstance, in_poolinstance->str)){
+			POOLINSTANCE_DATA_SET(workinfo, in_poolinstance->str);
 			return strdup(FAILED_PI);
 		}
 
@@ -2570,8 +2570,8 @@ static char *cmd_sharelog(PGconn *conn, char *cmd, char *id,
 		if (!i_merklehash)
 			return strdup(reply);
 
-		i_prevhash = require_name(trf_root, "prevhash", 1, NULL, reply, siz);
-		if (!i_prevhash)
+		in_prevhash = require_in(trf_root, "prevhash", 1, NULL, reply, siz);
+		if (!in_prevhash)
 			return strdup(reply);
 
 		i_coinbase1 = require_name(trf_root, "coinbase1", 1, NULL, reply, siz);
@@ -2582,12 +2582,12 @@ static char *cmd_sharelog(PGconn *conn, char *cmd, char *id,
 		if (!i_coinbase2)
 			return strdup(reply);
 
-		i_version = require_name(trf_root, "version", 1, NULL, reply, siz);
-		if (!i_version)
+		in_version = require_in(trf_root, "version", 1, NULL, reply, siz);
+		if (!in_version)
 			return strdup(reply);
 
-		i_bits = require_name(trf_root, "bits", 1, NULL, reply, siz);
-		if (!i_bits)
+		in_bits = require_in(trf_root, "bits", 1, NULL, reply, siz);
+		if (!in_bits)
 			return strdup(reply);
 
 		i_ntime = require_name(trf_root, "ntime", 1, NULL, reply, siz);
@@ -2599,14 +2599,14 @@ static char *cmd_sharelog(PGconn *conn, char *cmd, char *id,
 			return strdup(reply);
 
 		workinfoid = workinfo_add(conn, transfer_data(i_workinfoid),
-						transfer_data(i_poolinstance),
+						in_poolinstance,
 						txn_tree,
 						transfer_data(i_merklehash),
-						transfer_data(i_prevhash),
+						in_prevhash,
 						transfer_data(i_coinbase1),
 						transfer_data(i_coinbase2),
-						transfer_data(i_version),
-						transfer_data(i_bits),
+						in_version,
+						in_bits,
 						transfer_data(i_ntime),
 						transfer_data(i_reward),
 						by, code, inet, cd, igndup, trf_root);

src/ckdb_data.c

@@ -67,6 +67,10 @@ void free_workinfo_data(K_ITEM *item)
 	FREENULL(workinfo->transactiontree);
 	LIST_MEM_SUB(workinfo_free, workinfo->merklehash);
 	FREENULL(workinfo->merklehash);
+	LIST_MEM_SUB(workinfo_free, workinfo->coinbase1);
+	FREENULL(workinfo->coinbase1);
+	LIST_MEM_SUB(workinfo_free, workinfo->coinbase2);
+	FREENULL(workinfo->coinbase2);
 }
 
 void free_payouts_data(K_ITEM *item)
@@ -792,6 +796,14 @@ INTRANSIENT *_get_intransient(char *fldnam, char *value, size_t siz, WHERE_FFL_A
 	return in;
 }
 
+char *_intransient_str(char *fldnam, char *value, WHERE_FFL_ARGS)
+{
+	INTRANSIENT *in;
+
+	in = _get_intransient(fldnam, value, 0, WHERE_FFL_PASS);
+	return in->str;
+}
+
 // For mutiple variable function calls that need the data
 char *_transfer_data(K_ITEM *item, WHERE_FFL_ARGS)
 {
@@ -970,6 +982,75 @@ K_ITEM *_require_name(K_TREE *trf_root, char *name, int len, char *patt,
 	return item;
 }
 
+INTRANSIENT *_require_in(K_TREE *trf_root, char *name, int len, char *patt,
+			 char *reply, size_t siz, WHERE_FFL_ARGS)
+{
+	INTRANSIENT *in;
+	TRANSFER *trf;
+	K_ITEM *item;
+	char *mvalue;
+	regex_t re;
+	size_t dlen;
+	int ret;
+
+	reply[0] = '\0';
+
+	item = find_transfer(trf_root, name);
+	if (!item) {
+		LOGERR("%s(): failed, field '%s' missing from %s():%d",
+			__func__, name, func, line);
+		snprintf(reply, siz, "failed.missing %s", name);
+		return NULL;
+	}
+
+	DATA_TRANSFER(trf, item);
+	if (!(in = trf->intransient)) {
+		LOGERR("%s(): failed, field '%s' is not intransient %s():%d",
+			__func__, name, func, line);
+		snprintf(reply, siz, "failed.transient %s", name);
+		return NULL;
+	}
+
+	mvalue = trf->mvalue;
+	if (mvalue)
+		dlen = strlen(mvalue);
+	else
+		dlen = 0;
+	if (!mvalue || (int)dlen < len) {
+		LOGERR("%s(): failed, field '%s' short (%s%d<%d) from %s():%d",
+			__func__, name, mvalue ? EMPTY : "null ",
+			(int)dlen, len, func, line);
+		snprintf(reply, siz, "failed.short %s", name);
+		return NULL;
+	}
+
+	if (patt) {
+		if (regcomp(&re, patt, REG_NOSUB) != 0) {
+			LOGERR("%s(): failed, field '%s' failed to"
+				" compile patt from %s():%d",
+				__func__, name, func, line);
+			snprintf(reply, siz, "failed.REG %s", name);
+			return NULL;
+		}
+
+		ret = regexec(&re, mvalue, (size_t)0, NULL, 0);
+		regfree(&re);
+
+		if (ret != 0) {
+			char *st = NULL;
+			LOGERR("%s(): failed, field '%s'='%.20s%s' invalid "
+				"from %s():%d",
+				__func__, name, st = safe_text_nonull(mvalue),
+				(dlen > 20) ? "..." : EMPTY, func, line);
+			FREENULL(st);
+			snprintf(reply, siz, "failed.invalid %s", name);
+			return NULL;
+		}
+	}
+
+	return in;
+}
+
 // order by userid asc,workername asc
 cmp_t cmp_workerstatus(K_ITEM *a, K_ITEM *b)
 {

src/ckdb_dbio.c

@@ -3126,12 +3126,13 @@ bool optioncontrol_fill(PGconn *conn)
 	return ok;
 }
 
-int64_t workinfo_add(PGconn *conn, char *workinfoidstr, char *poolinstance,
-			char *transactiontree, char *merklehash, char *prevhash,
-			char *coinbase1, char *coinbase2, char *version,
-			char *bits, char *ntime, char *reward, char *by,
-			char *code, char *inet, tv_t *cd, bool igndup,
-			K_TREE *trf_root)
+int64_t workinfo_add(PGconn *conn, char *workinfoidstr,
+			INTRANSIENT *in_poolinstance, char *transactiontree,
+			char *merklehash, INTRANSIENT *in_prevhash,
+			char *coinbase1, char *coinbase2,
+			INTRANSIENT *in_version, INTRANSIENT *in_bits,
+			char *ntime, char *reward, char *by, char *code,
+			char *inet, tv_t *cd, bool igndup, K_TREE *trf_root)
 {
 	ExecStatusType rescode;
 	bool conned = false;
@@ -3157,14 +3158,14 @@ int64_t workinfo_add(PGconn *conn, char *workinfoidstr, char *poolinstance,
 	bzero(row, sizeof(*row));
 
 	TXT_TO_BIGINT("workinfoid", workinfoidstr, row->workinfoid);
-	STRNCPY(row->poolinstance, poolinstance);
+	row->poolinstance = in_poolinstance->str;
 	DUP_POINTER(workinfo_free, row->transactiontree, transactiontree);
 	DUP_POINTER(workinfo_free, row->merklehash, merklehash);
-	STRNCPY(row->prevhash, prevhash);
-	STRNCPY(row->coinbase1, coinbase1);
-	STRNCPY(row->coinbase2, coinbase2);
-	STRNCPY(row->version, version);
-	STRNCPY(row->bits, bits);
+	row->prevhash = in_prevhash->str;
+	DUP_POINTER(workinfo_free, row->coinbase1, coinbase1);
+	DUP_POINTER(workinfo_free, row->coinbase2, coinbase2);
+	row->version = in_version->str;
+	row->bits = in_bits->str;
 	STRNCPY(row->ntime, ntime);
 	TXT_TO_BIGINT("reward", reward, row->reward);
 	pool.reward = row->reward;
@@ -3242,6 +3243,8 @@ unparam:
 		// Not currently needed in RAM
 		LIST_MEM_SUB(workinfo_free, row->transactiontree);
 		FREENULL(row->transactiontree);
+		LIST_MEM_SUB(workinfo_free, row->merklehash);
+		FREENULL(row->merklehash);
 
 		row->height = coinbase1height(row);
 		hex2bin(ndiffbin, row->bits, 4);
@@ -3389,7 +3392,7 @@ bool workinfo_fill(PGconn *conn)
 				POOLINSTANCE_DBLOAD_SET(workinfo, field);
 				continue;
 			}
-			TXT_TO_STR("poolinstance", field, row->poolinstance);
+			row->poolinstance = intransient_str("poolinstance", field);
 
 			PQ_GET_FLD(res, i, "workinfoid", field, ok);
 			if (!ok)
@@ -3397,37 +3400,34 @@ bool workinfo_fill(PGconn *conn)
 			TXT_TO_BIGINT("workinfoid", field, row->workinfoid);
 
 			row->transactiontree = EMPTY;
-
-			PQ_GET_FLD(res, i, "merklehash", field, ok);
-			if (!ok)
-				break;
-			TXT_TO_BLOB("merklehash", field, row->merklehash);
-			LIST_MEM_ADD(workinfo_free, row->merklehash);
+			row->merklehash = EMPTY;
 
 			PQ_GET_FLD(res, i, "prevhash", field, ok);
 			if (!ok)
 				break;
-			TXT_TO_STR("prevhash", field, row->prevhash);
+			row->prevhash = intransient_str("prevhash", field);
 
 			PQ_GET_FLD(res, i, "coinbase1", field, ok);
 			if (!ok)
 				break;
-			TXT_TO_STR("coinbase1", field, row->coinbase1);
+			TXT_TO_BLOB("coinbase1", field, row->coinbase1);
+			LIST_MEM_ADD(workinfo_free, row->coinbase1);
 
 			PQ_GET_FLD(res, i, "coinbase2", field, ok);
 			if (!ok)
 				break;
-			TXT_TO_STR("coinbase2", field, row->coinbase2);
+			TXT_TO_BLOB("coinbase2", field, row->coinbase2);
+			LIST_MEM_ADD(workinfo_free, row->coinbase2);
 
 			PQ_GET_FLD(res, i, "version", field, ok);
 			if (!ok)
 				break;
-			TXT_TO_STR("version", field, row->version);
+			row->version = intransient_str("version", field);
 
 			PQ_GET_FLD(res, i, "bits", field, ok);
 			if (!ok)
 				break;
-			TXT_TO_STR("bits", field, row->bits);
+			row->bits = intransient_str("bits", field);
 
 			PQ_GET_FLD(res, i, "ntime", field, ok);
 			if (!ok)