ckdb - speed up data allocation and update the memory ckp web page

pool/page_ckp.php

@@ -31,7 +31,7 @@ function dockp($data, $user)
  $pg .= '<td class=dl>Name</td>';
  $pg .= '<td class=dr>Initial</td>';
  $pg .= '<td class=dr>Allocated</td>';
- $pg .= '<td class=dr>Store</td>';
+ $pg .= '<td class=dr>In&nbsp;Store</td>';
  $pg .= '<td class=dr>RAM</td>';
  $pg .= '<td class=dr>RAM2</td>';
  $pg .= '<td class=dr>Cull</td>';
@@ -50,7 +50,7 @@ function dockp($data, $user)
 		$pg .= '<td class=dl>'.$ans['name:'.$i].'</td>';
 		$pg .= '<td class=dr>'.stnum($ans['initial:'.$i]).'</td>';
 		$pg .= '<td class=dr>'.stnum($ans['allocated:'.$i]).'</td>';
-		$pg .= '<td class=dr>'.stnum($ans['store:'.$i]).'</td>';
+		$pg .= '<td class=dr>'.stnum($ans['instore:'.$i]).'</td>';
 		$pg .= '<td class=dr>'.stnum($ans['ram:'.$i]).'</td>';
 		$pg .= '<td class=dr>'.stnum($ans['ram2:'.$i]).'</td>';
 		$pg .= '<td class=dr>'.stnum($ans['cull:'.$i]).'</td>';

src/ckdb.h

@@ -51,7 +51,7 @@
 
 #define DB_VLOCK "1"
 #define DB_VERSION "1.0.4"
-#define CKDB_VERSION DB_VERSION"-1.801"
+#define CKDB_VERSION DB_VERSION"-1.900"
 
 #define WHERE_FFL " - from %s %s() line %d"
 #define WHERE_FFL_HERE __FILE__, __func__, __LINE__

src/ckdb_cmd.c

@@ -5631,73 +5631,71 @@ static char *cmd_stats(__maybe_unused PGconn *conn, char *cmd, char *id,
 			__maybe_unused tv_t *notcd, __maybe_unused K_TREE *trf_root)
 {
 	char tmp[1024], *buf;
+	const char *name;
 	size_t len, off;
 	uint64_t ram, ram2, tot = 0;
 	K_LIST *klist;
+	K_LISTS *klists;
 	int rows = 0;
+	bool istree;
 
 	LOGDEBUG("%s(): cmd '%s'", __func__, cmd);
 
 	APPEND_REALLOC_INIT(buf, off, len);
 	APPEND_REALLOC(buf, off, len, "ok.");
 
-// FYI average transactiontree length of the ~119k I have is ~28k (>3.3GB)
-#define USEINFO(_obj, _stores, _trees) \
-	klist = _obj ## _free; \
-	ram = sizeof(K_LIST) + _stores * sizeof(K_STORE) + \
-		klist->allocate * klist->item_mem_count * klist->siz + \
-		sizeof(K_TREE) * (klist->total - klist->count) * _trees; \
-	ram2 = klist->ram; \
-	snprintf(tmp, sizeof(tmp), \
-		 "name:%d=" #_obj "%cinitial:%d=%d%callocated:%d=%d%c" \
-		 "store:%d=%d%ctrees:%d=%d%cram:%d=%"PRIu64"%c" \
-		 "ram2:%d=%"PRIu64"%ccull:%d=%d%c", \
-		 rows, FLDSEP, \
-		 rows, klist->allocate, FLDSEP, \
-		 rows, klist->total, FLDSEP, \
-		 rows, klist->total - klist->count, FLDSEP, \
-		 rows, _trees, FLDSEP, \
-		 rows, ram, FLDSEP, \
-		 rows, ram2, FLDSEP, \
-		 rows, klist->cull_count, FLDSEP); \
-	APPEND_REALLOC(buf, off, len, tmp); \
-	tot += ram + ram2; \
-	rows++;
-
-	USEINFO(users, 1, 2);
-	USEINFO(useratts, 1, 1);
-	USEINFO(workers, 1, 1);
-	USEINFO(paymentaddresses, 1, 2);
-	USEINFO(payments, 1, 1);
-	USEINFO(accountbalance, 1, 1);
-	USEINFO(idcontrol, 1, 0);
-	USEINFO(optioncontrol, 1, 1);
-	USEINFO(workinfo, 1, 1);
-	// Trees don't share items so count as 1 tree
-	USEINFO(shares, 2, 1);
-	// Trees don't share items so count as 1 tree
-	USEINFO(shareerrors, 2, 1);
-	// _pool doesn't share items so is included
-	USEINFO(sharesummary, 1, 2);
-	USEINFO(workmarkers, 1, 2);
-	// _pool doesn't share items so is included
-	USEINFO(markersummary, 1, 2);
-	USEINFO(marks, 1, 1);
-	USEINFO(blocks, 1, 1);
-	USEINFO(miningpayouts, 1, 1);
-	USEINFO(payouts, 1, 3);
-	USEINFO(auths, 1, 1);
-	USEINFO(poolstats, 1, 1);
-	USEINFO(userstats, 2, 1);
-	USEINFO(workerstatus, 1, 1);
-	USEINFO(userinfo, 1, 1);
-	USEINFO(msgline, 1, 0);
-	USEINFO(workqueue, 3, 0);
-	USEINFO(transfer, 0, 0);
-	USEINFO(heartbeatqueue, 1, 0);
-	USEINFO(logqueue, 1, 0);
-	USEINFO(seqset, 1, 0);
-	USEINFO(seqtrans, 0, 0);
+	/* All but temporary lists are in klist_all
+	 * All trees are there also since all trees have a node klist */
+	ck_wlock(&lock_check_lock);
+	klists = all_klists;
+	while (klists) {
+		klist = klists->klist;
+
+		ram = sizeof(*klist);
+		if (klist->name == tree_node_list_name) {
+			ram += sizeof(K_TREE);
+			istree = true;
+			name = klist->name2;
+		} else {
+			istree = false;
+			name = klist->name;
+		}
+		if (klist->lock)
+			ram += sizeof(*(klist->lock));
+		// List of item lists
+		ram += klist->item_mem_count * sizeof(*(klist->item_memory));
+		// items
+		ram += klist->total * sizeof(K_ITEM);
+		// List of data lists
+		ram += klist->data_mem_count * sizeof(*(klist->data_memory));
+		// data
+		ram += klist->total * klist->siz;
+
+		// stores
+		ram += klist->stores * sizeof(K_STORE);
+
+		ram2 = klist->ram;
+
+		snprintf(tmp, sizeof(tmp),
+			 "name:%d=%s%s%s%cinitial:%d=%d%callocated:%d=%d%c"
+			 "instore:%d=%d%cram:%d=%"PRIu64"%c"
+			 "ram2:%d=%"PRIu64"%ccull:%d=%d%c",
+			 rows, name, istree ? " (tree)" : "",
+			 klist->is_lock_only ? " (lock)" : "", FLDSEP,
+			 rows, klist->allocate, FLDSEP,
+			 rows, klist->total, FLDSEP,
+			 rows, klist->total - klist->count, FLDSEP,
+			 rows, ram, FLDSEP,
+			 rows, ram2, FLDSEP,
+			 rows, klist->cull_count, FLDSEP);
+		APPEND_REALLOC(buf, off, len, tmp);
+
+		tot += ram + ram2;
+		rows++;
+
+		klists = klists->next;
+	}
+	ck_wunlock(&lock_check_lock);
 
 	snprintf(tmp, sizeof(tmp), "totalram=%"PRIu64"%c", tot, FLDSEP);
 	APPEND_REALLOC(buf, off, len, tmp);
@@ -5705,7 +5703,7 @@ static char *cmd_stats(__maybe_unused PGconn *conn, char *cmd, char *id,
 	snprintf(tmp, sizeof(tmp),
 		 "rows=%d%cflds=%s%c",
 		 rows, FLDSEP,
-		 "name,initial,allocated,store,trees,ram,cull", FLDSEP);
+		 "name,initial,allocated,instore,ram,cull", FLDSEP);
 	APPEND_REALLOC(buf, off, len, tmp);
 
 	snprintf(tmp, sizeof(tmp), "arn=%s%carp=%s", "Stats", FLDSEP, "");

src/klist.c

@@ -16,8 +16,6 @@ const char *tree_node_list_name = "TreeNodes";
 bool check_locks = true;
 const char *thread_noname = "UNSET";
 int next_thread_id = 0;
-bool lock_check_init = false;
-cklock_t lock_check_lock;
 __thread int my_thread_id = -1;
 __thread char *my_thread_name = NULL;
 __thread bool my_check_locks = true;
@@ -32,8 +30,11 @@ __thread const char *my_locks_f[MAX_LOCKDEPTH];
 __thread int my_locks_l[MAX_LOCKDEPTH];
 __thread int my_lock_level = 0;
 __thread bool my_check_deadlocks = true;
-K_LISTS *all_klists;
 #endif
+// Required for cmd_stats
+bool lock_check_init = false;
+cklock_t lock_check_lock;
+K_LISTS *all_klists;
 
 #define _CHKLIST(_list, _name) do {\
 		if (!_list) { \
@@ -61,6 +62,7 @@ K_LISTS *all_klists;
 static void k_alloc_items(K_LIST *list, KLIST_FFL_ARGS)
 {
 	K_ITEM *item;
+	void *data;
 	int allocate, i;
 
 	CHKLIST(list);
@@ -90,10 +92,6 @@ static void k_alloc_items(K_LIST *list, KLIST_FFL_ARGS)
 	}
 	list->item_memory[list->item_mem_count - 1] = (void *)item;
 
-	list->total += allocate;
-	list->count = allocate;
-	list->count_up = allocate;
-
 	item[0].name = list->name;
 	item[0].prev = NULL;
 	item[0].next = &(item[1]);
@@ -110,21 +108,29 @@ static void k_alloc_items(K_LIST *list, KLIST_FFL_ARGS)
 	if (list->do_tail)
 		list->tail = &(item[allocate-1]);
 
+	list->data_mem_count++;
+	if (!(list->data_memory = realloc(list->data_memory,
+					  list->data_mem_count * sizeof(*(list->data_memory))))) {
+		quithere(1, "List %s data_memory failed to realloc count=%d",
+				list->name, list->data_mem_count);
+	}
+	data = calloc(allocate, list->siz);
+	if (!data) {
+		quithere(1, "List %s failed to calloc %d new data - total was %d, limit was %d",
+				list->name, allocate, list->total, list->limit);
+	}
+	list->data_memory[list->data_mem_count - 1] = data;
+
 	item = list->head;
 	while (item) {
-		list->data_mem_count++;
-		if (!(list->data_memory = realloc(list->data_memory,
-						  list->data_mem_count *
-						  sizeof(*(list->data_memory))))) {
-			quithere(1, "List %s data_memory failed to realloc count=%d",
-					list->name, list->data_mem_count);
-		}
-		item->data = calloc(1, list->siz);
-		if (!(item->data))
-			quithere(1, "List %s failed to calloc item data", list->name);
-		list->data_memory[list->data_mem_count - 1] = (void *)(item->data);
+		item->data = data;
+		data += list->siz;
 		item = item->next;
 	}
+
+	list->total += allocate;
+	list->count = allocate;
+	list->count_up = allocate;
 }
 
 K_STORE *_k_new_store(K_LIST *list, KLIST_FFL_ARGS)
@@ -142,6 +148,7 @@ K_STORE *_k_new_store(K_LIST *list, KLIST_FFL_ARGS)
 	store->lock = list->lock;
 	store->name = list->name;
 	store->do_tail = list->do_tail;
+	list->stores++;
 
 	return store;
 }
@@ -187,7 +194,6 @@ K_LIST *_k_new_list(const char *name, size_t siz, int allocate, int limit,
 	if (!(list->is_lock_only))
 		k_alloc_items(list, KLIST_FFL_PASS);
 
-#if LOCK_CHECK
 	/* Don't want to keep track of short lived (tree) lists
 	 * since they wont use locking anyway */
 	if (!list->local_list) {
@@ -210,7 +216,6 @@ K_LIST *_k_new_list(const char *name, size_t siz, int allocate, int limit,
 		all_klists = klists;
 		ck_wunlock(&lock_check_lock);
 	}
-#endif
 
 	return list;
 }
@@ -541,7 +546,6 @@ K_LIST *_k_free_list(K_LIST *list, KLIST_FFL_ARGS)
 		free(list->lock);
 	}
 
-#if LOCK_CHECK
 	// local_list lists are not stored in all_klists
 	if (!list->local_list) {
 		K_LISTS *klists, *klists_prev = NULL;
@@ -572,7 +576,6 @@ K_LIST *_k_free_list(K_LIST *list, KLIST_FFL_ARGS)
 		}
 		ck_wunlock(&lock_check_lock);
 	}
-#endif
 
 	free(list);
 
@@ -588,6 +591,8 @@ K_STORE *_k_free_store(K_STORE *store, KLIST_FFL_ARGS)
 				store->name, __func__, KLIST_FFL_PASS);
 	}
 
+	store->master->stores--;
+
 	free(store);
 
 	return NULL;

src/klist.h

@@ -87,8 +87,6 @@ extern bool check_locks;
 #define MAX_THREADS 128
 extern const char *thread_noname;
 extern int next_thread_id;
-extern bool lock_check_init;
-extern cklock_t lock_check_lock;
 extern __thread int my_thread_id;
 extern __thread char *my_thread_name;
 extern __thread bool my_check_locks;
@@ -152,6 +150,7 @@ typedef struct k_list {
 	void (*dsp_func)(K_ITEM *, FILE *); // optional data display to a file
 	int cull_count;
 	int ram;		// ram allocated for data pointers - code must manage it
+	int stores;		// how many stores it currently has
 #if LOCK_CHECK
 	// Since each thread has it's own k_lock no locking is required on this
 	K_LOCK k_lock[MAX_THREADS];
@@ -160,13 +159,14 @@ typedef struct k_list {
 #endif
 } K_LIST;
 
-#if LOCK_CHECK
+// Required for cmd_stats
+extern bool lock_check_init;
+extern cklock_t lock_check_lock;
 typedef struct k_lists {
 	K_LIST *klist;
 	struct k_lists *next;
 } K_LISTS;
 extern K_LISTS *all_klists;
-#endif
 
 /*
  * K_STORE is for a list of items taken from a K_LIST
@@ -526,7 +526,14 @@ static inline K_ITEM *list_rtail(K_LIST *list)
 #else
 #define LOCK_MAYBE __maybe_unused
 #define LOCK_INIT(_name)
-#define FIRST_LOCK_INIT(_name)
+#define FIRST_LOCK_INIT(_ignore) do { \
+		if (lock_check_init) { \
+			quithere(1, "lock_check_lock has already been " \
+				 "initialised!"); \
+		} \
+		cklock_init(&lock_check_lock); \
+		lock_check_init = true; \
+	} while (0)
 #define CHECK_WLOCK(_list) ck_wlock((_list)->lock)
 #define CHECK_WUNLOCK(_list) ck_wunlock((_list)->lock)
 #define CHECK_RLOCK(_list) ck_rlock((_list)->lock)