ckdb - intransient store of common strings to reduce ram - only used in transfer so far

src/ckdb.c

@@ -208,24 +208,24 @@ const char *strpatt = "^[ -<>-~]*$";
 const char Transfer[] = "Transfer";
 
 // older version missing field defaults
-static TRANSFER auth_1 = { "poolinstance", "", auth_1.svalue };
+static TRANSFER auth_1 = { "poolinstance", "", auth_1.svalue, NULL };
 K_ITEM auth_poolinstance = { Transfer, NULL, NULL, (void *)(&auth_1) };
-static TRANSFER auth_2 = { "preauth", FALSE_STR, auth_2.svalue };
+static TRANSFER auth_2 = { "preauth", FALSE_STR, auth_2.svalue, NULL };
 K_ITEM auth_preauth = { Transfer, NULL, NULL, (void *)(&auth_2) };
-static TRANSFER poolstats_1 = { "elapsed", "0", poolstats_1.svalue };
+static TRANSFER poolstats_1 = { "elapsed", "0", poolstats_1.svalue, NULL };
 K_ITEM poolstats_elapsed = { Transfer, NULL, NULL, (void *)(&poolstats_1) };
-static TRANSFER userstats_1 = { "elapsed", "0", userstats_1.svalue };
+static TRANSFER userstats_1 = { "elapsed", "0", userstats_1.svalue, NULL };
 K_ITEM userstats_elapsed = { Transfer, NULL, NULL, (void *)(&userstats_1) };
-static TRANSFER userstats_2 = { "workername", "all", userstats_2.svalue };
+static TRANSFER userstats_2 = { "workername", "all", userstats_2.svalue, NULL };
 K_ITEM userstats_workername = { Transfer, NULL, NULL, (void *)(&userstats_2) };
-static TRANSFER userstats_3 = { "idle", FALSE_STR, userstats_3.svalue };
+static TRANSFER userstats_3 = { "idle", FALSE_STR, userstats_3.svalue, NULL };
 K_ITEM userstats_idle = { Transfer, NULL, NULL, (void *)(&userstats_3) };
-static TRANSFER userstats_4 = { "eos", TRUE_STR, userstats_4.svalue };
+static TRANSFER userstats_4 = { "eos", TRUE_STR, userstats_4.svalue, NULL };
 K_ITEM userstats_eos = { Transfer, NULL, NULL, (void *)(&userstats_4) };
 
-static TRANSFER shares_1 = { "secondaryuserid", TRUE_STR, shares_1.svalue };
+static TRANSFER shares_1 = { "secondaryuserid", TRUE_STR, shares_1.svalue, NULL };
 K_ITEM shares_secondaryuserid = { Transfer, NULL, NULL, (void *)(&shares_1) };
-static TRANSFER shareerrors_1 = { "secondaryuserid", TRUE_STR, shareerrors_1.svalue };
+static TRANSFER shareerrors_1 = { "secondaryuserid", TRUE_STR, shareerrors_1.svalue, NULL };
 K_ITEM shareerrors_secondaryuserid = { Transfer, NULL, NULL, (void *)(&shareerrors_1) };
 // Time limit that this problem occurred
 // 24-Aug-2014 05:20+00 (1st one shortly after this)
@@ -447,6 +447,34 @@ static pthread_cond_t f_ioqueue_waitcond;
 K_LIST *logqueue_free;
 K_STORE *logqueue_store;
 
+// NAMERAM
+K_LIST *nameram_free;
+K_STORE *nameram_store;
+
+// INTRANSIENT
+K_TREE *intransient_root;
+K_LIST *intransient_free;
+K_STORE *intransient_store;
+
+char *intransient_fields[] = {
+	"workername",
+	"blockhash",
+	BYTRF,
+	CODETRF,
+	INETTRF,
+	"username",
+	"agent",
+	"useragent",
+	"address",
+	"prevhash",
+	"poolinstance",
+	"payaddress",
+	"originaltxn",
+	"committxn",
+	"commitblockhash",
+	NULL
+};
+
 // MSGLINE
 K_LIST *msgline_free;
 K_STORE *msgline_store;
@@ -1880,15 +1908,28 @@ static void alloc_storage()
 	logqueue_store = k_new_store(logqueue_free);
 
 	breakqueue_free = k_new_list("BreakQueue", sizeof(BREAKQUEUE),
-					ALLOC_BREAKQUEUE, LIMIT_BREAKQUEUE, true);
+				     ALLOC_BREAKQUEUE, LIMIT_BREAKQUEUE, true);
 	reload_breakqueue_store = k_new_store(breakqueue_free);
 	reload_done_breakqueue_store = k_new_store(breakqueue_free);
 	cmd_breakqueue_store = k_new_store(breakqueue_free);
 	cmd_done_breakqueue_store = k_new_store(breakqueue_free);
 
+	intransient_free = k_new_list("Intransient", sizeof(INTRANSIENT),
+					ALLOC_INTRANSIENT, LIMIT_INTRANSIENT,
+					true);
+	intransient_store = k_new_store(intransient_free);
+	intransient_root = new_ktree(NULL, cmp_intransient, intransient_free);
+
+	nameram_free = k_new_list("NameRAM", sizeof(NAMERAM), ALLOC_NAMERAM,
+				  LIMIT_NAMERAM, true);
+	nameram_store = k_new_store(nameram_free);
+
 #if LOCK_CHECK
 	DLPRIO(logqueue, 94);
 	DLPRIO(breakqueue, PRIO_TERMINAL);
+
+	DLPRIO(intransient, 2);
+	DLPRIO(nameram, 1);
 #endif
 
 	seqset_free = k_new_list("SeqSet", sizeof(SEQSET),
@@ -3910,8 +3951,9 @@ static enum cmd_values breakdown(K_ITEM **ml_item, char *buf, tv_t *now,
 	K_ITEM *t_item = NULL, *cd_item = NULL, *seqall;
 	char *cmdptr, *idptr, *next, *eq, *end, *was;
 	char *data = NULL, *st = NULL, *st2 = NULL, *ip = NULL;
-	bool noid = false;
+	bool noid = false, intrans;
 	size_t siz;
+	int i;
 
 	K_WLOCK(msgline_free);
 	*ml_item = k_unlink_head_zero(msgline_free);
@@ -4033,6 +4075,7 @@ static enum cmd_values breakdown(K_ITEM **ml_item, char *buf, tv_t *now,
 				FREENULL(st2);
 				goto nogood;
 			}
+			// LOGERR of buf could be truncated
 			*(end++) = '\0';
 			K_WLOCK(transfer_free);
 			t_item = k_unlink_head(transfer_free);
@@ -4131,12 +4174,29 @@ static enum cmd_values breakdown(K_ITEM **ml_item, char *buf, tv_t *now,
 				}
 				siz = end - next;
 			}
-			if (siz >= sizeof(transfer->svalue)) {
+			intrans = false;
-				transfer->mvalue = malloc(siz+1);
+			for (i = 0; intransient_fields[i]; i++) {
-				STRNCPYSIZ(transfer->mvalue, next, siz+1);
+				if (strcmp(transfer->name,
-			} else {
+					   intransient_fields[i]) == 0) {
-				STRNCPYSIZ(transfer->svalue, next, siz+1);
+					intrans = true;
-				transfer->mvalue = transfer->svalue;
+					// This will affect 'was' if JSON_END is missing
+					*(next + siz) = '\0';
+					transfer->intransient = get_intransient_siz(next, siz+1);
+					transfer->mvalue = transfer->intransient->str;
+					break;
+				}
+			}
+			if (!intrans) {
+				transfer->intransient = NULL;
+				if (siz >= sizeof(transfer->svalue)) {
+					transfer->mvalue = malloc(siz+1);
+					STRNCPYSIZ(transfer->mvalue, next,
+						   siz+1);
+				} else {
+					STRNCPYSIZ(transfer->svalue, next,
+						   siz+1);
+					transfer->mvalue = transfer->svalue;
+				}
 			}
 			add_to_ktree_nolock(msgline->trf_root, t_item);
 			k_add_head_nolock(msgline->trf_store, t_item);
@@ -4179,12 +4239,25 @@ static enum cmd_values breakdown(K_ITEM **ml_item, char *buf, tv_t *now,
 			DATA_TRANSFER(transfer, t_item);
 			STRNCPY(transfer->name, data);
 			siz = strlen(eq) + 1;
-			if (siz > sizeof(transfer->svalue)) {
+			intrans = false;
-				transfer->mvalue = malloc(siz);
+			for (i = 0; intransient_fields[i]; i++) {
-				STRNCPYSIZ(transfer->mvalue, eq, siz);
+				if (strcmp(transfer->name,
-			} else {
+					   intransient_fields[i]) == 0) {
-				STRNCPYSIZ(transfer->svalue, eq, siz);
+					intrans = true;
-				transfer->mvalue = transfer->svalue;
+					transfer->intransient = get_intransient_siz(eq, siz);
+					transfer->mvalue = transfer->intransient->str;
+					break;
+				}
+			}
+			if (!intrans) {
+				transfer->intransient = NULL;
+				if (siz > sizeof(transfer->svalue)) {
+					transfer->mvalue = malloc(siz);
+					STRNCPYSIZ(transfer->mvalue, eq, siz);
+				} else {
+					STRNCPYSIZ(transfer->svalue, eq, siz);
+					transfer->mvalue = transfer->svalue;
+				}
 			}
 
 			// Discard duplicates

src/ckdb.h

@@ -58,7 +58,7 @@
 
 #define DB_VLOCK "1"
 #define DB_VERSION "1.0.7"
-#define CKDB_VERSION DB_VERSION"-2.438"
+#define CKDB_VERSION DB_VERSION"-2.440"
 
 #define WHERE_FFL " - from %s %s() line %d"
 #define WHERE_FFL_HERE __FILE__, __func__, __LINE__
@@ -1131,6 +1131,41 @@ typedef struct logqueue {
 extern K_LIST *logqueue_free;
 extern K_STORE *logqueue_store;
 
+// NAMERAM - RAM for INTRANSIENT names - 2M per allocation
+typedef struct nameram {
+	char rem[2*1024*1024-sizeof(void *)-sizeof(size_t)];
+	void *next;
+	size_t left;
+} NAMERAM;
+
+// Items never to be deleted and list never to be culled
+#define ALLOC_NAMERAM 1
+#define LIMIT_NAMERAM 0
+#define INIT_NAMERAM(_item) INIT_GENERIC(_item, nameram)
+#define DATA_NAMERAM(_var, _item) DATA_GENERIC(_var, _item, nameram, true)
+
+extern K_LIST *nameram_free;
+extern K_STORE *nameram_store;
+
+// INTRANSIENT - a list of common strings, to avoid wasting RAM
+typedef struct intransient {
+	char *str;
+} INTRANSIENT;
+
+/* Items never to be deleted and list never to be culled
+ * They are all created by the db load functions and breakdown */
+#define ALLOC_INTRANSIENT 1024
+#define LIMIT_INTRANSIENT 0
+#define INIT_INTRANSIENT(_item) INIT_GENERIC(_item, intransient)
+#define DATA_INTRANSIENT(_var, _item) DATA_GENERIC(_var, _item, intransient, true)
+#define DATA_INTRANSIENT_NULL(_var, _item) DATA_GENERIC(_var, _item, intransient, false)
+
+extern K_TREE *intransient_root;
+extern K_LIST *intransient_free;
+extern K_STORE *intransient_store;
+
+extern char *intransient_fields[];
+
 // MSGLINE
 typedef struct msgline {
 	int which_cmds;
@@ -1343,6 +1378,7 @@ typedef struct transfer {
 	char name[NAME_SIZE+1];
 	char svalue[VALUE_SIZE+1];
 	char *mvalue;
+	INTRANSIENT *intransient;
 } TRANSFER;
 
 // Suggest malloc use MMAP = largest under 2MB
@@ -3074,6 +3110,11 @@ extern char *_hms_to_buf(time_t *data, char *buf, size_t siz, WHERE_FFL_ARGS);
 // Convert to MM:SS
 extern char *_ms_to_buf(time_t *data, char *buf, size_t siz, WHERE_FFL_ARGS);
 
+extern cmp_t cmp_intransient(K_ITEM *a, K_ITEM *b);
+#define get_intransient(_name) _get_intransient(_name, 0, WHERE_FFL_HERE)
+#define get_intransient_siz(_name, _siz) \
+	_get_intransient(_name, _siz, WHERE_FFL_HERE)
+extern INTRANSIENT *_get_intransient(char *name, size_t siz, WHERE_FFL_ARGS);
 extern char *_transfer_data(K_ITEM *item, WHERE_FFL_ARGS);
 extern void dsp_transfer(K_ITEM *item, FILE *stream);
 extern cmp_t cmp_transfer(K_ITEM *a, K_ITEM *b);

src/ckdb_data.c

@@ -25,8 +25,13 @@ void free_msgline_data(K_ITEM *item, bool t_lock, bool t_cull)
 		t_item = STORE_HEAD_NOLOCK(msgline->trf_store);
 		while (t_item) {
 			DATA_TRANSFER(transfer, t_item);
-			if (transfer->mvalue != transfer->svalue)
+			if (transfer->mvalue != transfer->svalue) {
-				FREENULL(transfer->mvalue);
+				if (transfer->intransient) {
+					transfer->intransient = NULL;
+					transfer->mvalue = NULL;
+				} else
+					FREENULL(transfer->mvalue);
+			}
 			t_item = t_item->next;
 		}
 		if (t_lock)
@@ -701,6 +706,91 @@ char *_ms_to_buf(time_t *data, char *buf, size_t siz, WHERE_FFL_ARGS)
 	return _data_to_buf(TYPE_MS, (void *)data, buf, siz, WHERE_FFL_PASS);
 }
 
+// order by name asc
+cmp_t cmp_intransient(K_ITEM *a, K_ITEM *b)
+{
+	INTRANSIENT *ia, *ib;
+	DATA_INTRANSIENT(ia, a);
+	DATA_INTRANSIENT(ib, b);
+	return CMP_STR(ia->str, ib->str);
+}
+
+INTRANSIENT *_get_intransient(char *name, size_t siz, WHERE_FFL_ARGS)
+{
+	INTRANSIENT intransient, *in = NULL;
+	K_ITEM look, *i_item, *n_item;
+	NAMERAM *nameram = NULL;
+	K_TREE_CTX ctx[1];
+	char *buf;
+	bool new;
+
+	if (siz == 0)
+		siz = strlen(name) + 1;
+
+	if (siz > sizeof(nameram->rem)) {
+		char *st = NULL;
+		LOGEMERG("%s() ERR '%10s...' discarded - siz %d>%d" WHERE_FFL,
+			 __func__, st = safe_text_nonull(name), (int)siz,
+			 (int)sizeof(nameram->rem), WHERE_FFL_PASS);
+		name = EMPTY;
+		siz = 1;
+	}
+
+	intransient.str = name;
+	INIT_INTRANSIENT(&look);
+	look.data = (void *)(&intransient);
+	K_RLOCK(intransient_free);
+	i_item = _find_in_ktree(intransient_root, &look, ctx, false,
+				WHERE_FFL_PASS);
+	K_RUNLOCK(intransient_free);
+
+	if (i_item) {
+		DATA_INTRANSIENT(in, i_item);
+		return in;
+	}
+
+	K_WLOCK(intransient_free);
+	// Search again, to be thread safe
+	i_item = _find_in_ktree(intransient_root, &look, ctx, false,
+				WHERE_FFL_PASS);
+	if (i_item) {
+		DATA_INTRANSIENT(in, i_item);
+	} else {
+		new = false;
+		buf = NULL;
+		K_WLOCK(nameram_free);
+		if (nameram_store->count == 0)
+			new = true;
+		else {
+			n_item = STORE_WHEAD(nameram_store);
+			DATA_NAMERAM(nameram, n_item);
+			if (nameram->left < siz)
+				new = true;
+		}
+		if (new) {
+			n_item = k_unlink_head(nameram_free);
+			DATA_NAMERAM(nameram, n_item);
+			nameram->next = nameram->rem;
+			nameram->left = sizeof(nameram->rem);
+			k_add_head(nameram_store, n_item);
+
+		}
+		buf = nameram->next;
+		nameram->next += siz;
+		nameram->left -= siz;
+		K_WUNLOCK(nameram_free);
+		strcpy(buf, name);
+		i_item = k_unlink_head(intransient_free);
+		DATA_INTRANSIENT(in, i_item);
+		in->str = buf;
+		k_add_tail(intransient_store, i_item);
+		add_to_ktree(intransient_root, i_item);
+	}
+	K_WUNLOCK(intransient_free);
+
+	return in;
+}
+
 // For mutiple variable function calls that need the data
 char *_transfer_data(K_ITEM *item, WHERE_FFL_ARGS)
 {