From 1ae5e57fc4cd9df43688a71096b101ea2cebe965 Mon Sep 17 00:00:00 2001
From: Ben U <theullrich@gmail.com>
Date: Sat, 8 Nov 2014 14:17:15 -0700
Subject: [PATCH 01/21] Fixed mobile menu bar with proper spacing.

---
 pool/page.php | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/pool/page.php b/pool/page.php
index d1183b4b..5d0d7d44 100644
--- a/pool/page.php
+++ b/pool/page.php
@@ -110,8 +110,8 @@ span.err {color:red; font-weight:bold; font-size:120%;}
 span.alert {color:red; font-weight:bold; font-size:250%;}
 input.tiny {width: 0px; height: 0px; margin: 0px; padding: 0px; outline: none; border: 0px;}
 #n42 {margin:0; position: relative; color:#fff; background:#07e;}
-#n42 a {color:#fff; text-decoration:none; margin: 4px;}
-#n42 td {min-width: 100px; float: left; vertical-align: top; padding: 2px;}
+#n42 a {color:#fff; text-decoration:none; padding: 8px; display:block;}
+#n42 td {min-width: 100px; float: left; vertical-align: top; padding: 0px 2px;}
 #n42 td.navboxr {float: right;}
 #n42 td.nav {position: relative;}
 #n42 div.sub {left: 0px; z-index: 42; position: absolute; visibility: hidden;}

From 03efdd03323660138b537d9ae3fef591f0c51986 Mon Sep 17 00:00:00 2001
From: kanoi <gitkanoi@k1k2.com>
Date: Tue, 18 Nov 2014 12:29:10 +1100
Subject: [PATCH 02/21] ckdb - disallow web usernames like BTC addresses

---
 src/ckdb.c      |  8 ++++++--
 src/ckdb.h      | 11 ++++++++++-
 src/ckdb_cmd.c  | 37 +++++++++++++++++++++++++------------
 src/ckdb_data.c | 49 +++++++++++++++++++++++++++++++++++++++++++++++++
 4 files changed, 90 insertions(+), 15 deletions(-)

diff --git a/src/ckdb.c b/src/ckdb.c
index 3554a11e..c43ae064 100644
--- a/src/ckdb.c
+++ b/src/ckdb.c
@@ -174,8 +174,12 @@ const char *mailpatt = "^[A-Za-z0-9_-][A-Za-z0-9_\\.-]*@[A-Za-z0-9][A-Za-z0-9\\.
 const char *idpatt = "^[_A-Za-z][_A-Za-z0-9]*$";
 const char *intpatt = "^[0-9][0-9]*$";
 const char *hashpatt = "^[A-Fa-f0-9]*$";
-// TODO: bitcoind will check it properly
-const char *addrpatt = "^[A-Za-z0-9]*$";
+/* BTC addresses start with '1' (one) or '3' (three),
+ *  exclude: capital 'I' (eye), capital 'O' (oh),
+ *  lowercase 'l' (elle) and '0' (zero)
+ *  and with a simple test must be ADDR_MIN_LEN to ADDR_MAX_LEN (ckdb.h)
+ * bitcoind is used to fully validate them when required */
+const char *addrpatt = "^[13][A-HJ-NP-Za-km-z1-9]*$";
 
 // So the records below have the same 'name' as the klist
 const char Transfer[] = "Transfer";
diff --git a/src/ckdb.h b/src/ckdb.h
index dd245ef3..f9a216c0 100644
--- a/src/ckdb.h
+++ b/src/ckdb.h
@@ -52,7 +52,7 @@
 
 #define DB_VLOCK "1"
 #define DB_VERSION "0.9.4"
-#define CKDB_VERSION DB_VERSION"-0.631"
+#define CKDB_VERSION DB_VERSION"-0.640"
 
 #define WHERE_FFL " - from %s %s() line %d"
 #define WHERE_FFL_HERE __FILE__, __func__, __LINE__
@@ -80,6 +80,14 @@ extern const char *intpatt;
 extern const char *hashpatt;
 extern const char *addrpatt;
 
+/* If a trimmed username is like an address but this many or more characters,
+ * disallow it */
+#define ADDR_USER_CHECK 16
+
+// BTC address size
+#define ADDR_MIN_LEN 26
+#define ADDR_MAX_LEN 34
+
 typedef struct loadstatus {
 	tv_t oldest_sharesummary_firstshare_n;
 	tv_t newest_sharesummary_firstshare_a;
@@ -1334,6 +1342,7 @@ extern PGconn *dbconnect();
 
 extern char *safe_text(char *txt);
 extern void username_trim(USERS *users);
+extern bool like_address(char *username);
 
 extern void _txt_to_data(enum data_type typ, char *nam, char *fld, void *data, size_t siz, WHERE_FFL_ARGS);
 
diff --git a/src/ckdb_cmd.c b/src/ckdb_cmd.c
index c7c580c8..e3443858 100644
--- a/src/ckdb_cmd.c
+++ b/src/ckdb_cmd.c
@@ -15,7 +15,7 @@ static char *cmd_adduser(PGconn *conn, char *cmd, char *id, tv_t *now, char *by,
 {
 	char reply[1024] = "";
 	size_t siz = sizeof(reply);
-	K_ITEM *i_username, *i_emailaddress, *i_passwordhash, *u_item;
+	K_ITEM *i_username, *i_emailaddress, *i_passwordhash, *u_item = NULL;
 
 	LOGDEBUG("%s(): cmd '%s'", __func__, cmd);
 
@@ -23,18 +23,30 @@ static char *cmd_adduser(PGconn *conn, char *cmd, char *id, tv_t *now, char *by,
 	if (!i_username)
 		return strdup(reply);
 
-	i_emailaddress = require_name(trf_root, "emailaddress", 7, (char *)mailpatt, reply, siz);
-	if (!i_emailaddress)
-		return strdup(reply);
+	/* If a username added from the web site looks like an address
+	 *  then disallow it - a false positive is not an issue
+	 * Allowing it will create a security issue - someone could create
+	 *  an account with someone else's, as yet unused, payout address
+	 *  and redirect the payout to another payout address.
+	 *  ... and the person who owns the payout address can't check that
+	 *  in advance, they'll just find out with their first payout not
+	 *  arriving at their payout address */
+	if (!like_address(transfer_data(i_username))) {
+		i_emailaddress = require_name(trf_root, "emailaddress", 7,
+					      (char *)mailpatt, reply, siz);
+		if (!i_emailaddress)
+			return strdup(reply);
 
-	i_passwordhash = require_name(trf_root, "passwordhash", 64, (char *)hashpatt, reply, siz);
-	if (!i_passwordhash)
-		return strdup(reply);
+		i_passwordhash = require_name(trf_root, "passwordhash", 64,
+					      (char *)hashpatt, reply, siz);
+		if (!i_passwordhash)
+			return strdup(reply);
 
-	u_item = users_add(conn, transfer_data(i_username),
-				 transfer_data(i_emailaddress),
-				 transfer_data(i_passwordhash),
-				 by, code, inet, now, trf_root);
+		u_item = users_add(conn, transfer_data(i_username),
+					 transfer_data(i_emailaddress),
+					 transfer_data(i_passwordhash),
+					 by, code, inet, now, trf_root);
+	}
 
 	if (!u_item) {
 		LOGERR("%s() %s.failed.DBE", __func__, id);
@@ -223,7 +235,8 @@ static char *cmd_userset(PGconn *conn, char *cmd, char *id,
 				email = NULL;
 			}
 			i_address = optional_name(trf_root, "address",
-						  27, (char *)addrpatt,
+						  ADDR_MIN_LEN,
+						  (char *)addrpatt,
 						  reply, siz);
 			if (i_address)
 				address = transfer_data(i_address);
diff --git a/src/ckdb_data.c b/src/ckdb_data.c
index e712f520..b799648d 100644
--- a/src/ckdb_data.c
+++ b/src/ckdb_data.c
@@ -67,6 +67,55 @@ void username_trim(USERS *users)
 	}
 }
 
+/* Is the trimmed username like an address?
+ * False positive is OK (i.e. 'like')
+ * Before checking, it is trimmed to avoid web display confusion
+ * Length check is done before trimming - this may give a false
+ *  positive on any username with lots of trim characters ... which is OK */
+bool like_address(char *username)
+{
+	char *tmp, *front, *trail;
+	size_t len;
+	regex_t re;
+	int ret;
+
+	len = strlen(username);
+	if (len < ADDR_USER_CHECK)
+		return false;
+
+	tmp = strdup(username);
+	front = tmp;
+	while (*front && TRIM_IGNORE(*front))
+		front++;
+
+	trail = front + strlen(front) - 1;
+	while (trail >= front) {
+		if (TRIM_IGNORE(*trail))
+			*(trail--) = '\0';
+		else
+			break;
+	}
+
+	if (regcomp(&re, addrpatt, REG_NOSUB) != 0) {
+		LOGEMERG("%s(): failed to compile addrpatt '%s'",
+			 __func__, addrpatt);
+		free(tmp);
+		// This will disable adding any new usernames ...
+		return true;
+	}
+
+	ret = regexec(&re, front, (size_t)0, NULL, 0);
+	regfree(&re);
+
+	if (ret == 0) {
+		free(tmp);
+		return true;
+	}
+
+	free(tmp);
+	return false;
+}
+
 void _txt_to_data(enum data_type typ, char *nam, char *fld, void *data, size_t siz, WHERE_FFL_ARGS)
 {
 	char *tmp;

From 7a8392bb1fbebe1e278f9a9df831e7dcd67042ff Mon Sep 17 00:00:00 2001
From: kanoi <gitkanoi@k1k2.com>
Date: Tue, 18 Nov 2014 12:35:41 +1100
Subject: [PATCH 03/21] ckdb - suggest to malloc to use mmap for transfer

---
 src/ckdb.h | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/src/ckdb.h b/src/ckdb.h
index f9a216c0..61848f18 100644
--- a/src/ckdb.h
+++ b/src/ckdb.h
@@ -52,7 +52,7 @@
 
 #define DB_VLOCK "1"
 #define DB_VERSION "0.9.4"
-#define CKDB_VERSION DB_VERSION"-0.640"
+#define CKDB_VERSION DB_VERSION"-0.641"
 
 #define WHERE_FFL " - from %s %s() line %d"
 #define WHERE_FFL_HERE __FILE__, __func__, __LINE__
@@ -611,7 +611,8 @@ typedef struct transfer {
 	char *mvalue;
 } TRANSFER;
 
-#define ALLOC_TRANSFER 64
+// Suggest malloc use MMAP - 1913 = largest under 2MB
+#define ALLOC_TRANSFER 1913
 #define LIMIT_TRANSFER 0
 #define CULL_TRANSFER 1024
 #define INIT_TRANSFER(_item) INIT_GENERIC(_item, transfer)

From 5d522ddec836b6d208780dac45f4a37bee12acab Mon Sep 17 00:00:00 2001
From: kanoi <gitkanoi@k1k2.com>
Date: Tue, 18 Nov 2014 12:49:29 +1100
Subject: [PATCH 04/21] ckdb - fail auth if users.status isn't empty

---
 src/ckdb.h      | 2 +-
 src/ckdb_dbio.c | 4 ++++
 2 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/src/ckdb.h b/src/ckdb.h
index 61848f18..bb35f06d 100644
--- a/src/ckdb.h
+++ b/src/ckdb.h
@@ -52,7 +52,7 @@
 
 #define DB_VLOCK "1"
 #define DB_VERSION "0.9.4"
-#define CKDB_VERSION DB_VERSION"-0.641"
+#define CKDB_VERSION DB_VERSION"-0.642"
 
 #define WHERE_FFL " - from %s %s() line %d"
 #define WHERE_FFL_HERE __FILE__, __func__, __LINE__
diff --git a/src/ckdb_dbio.c b/src/ckdb_dbio.c
index 8c0b3d52..f837e78b 100644
--- a/src/ckdb_dbio.c
+++ b/src/ckdb_dbio.c
@@ -4055,6 +4055,10 @@ bool auths_add(PGconn *conn, char *poolinstance, char *username,
 	}
 	DATA_USERS(*users, u_item);
 
+	// Any status content means disallow mining
+	if ((*users)->status[0])
+		goto unitem;
+
 	STRNCPY(row->poolinstance, poolinstance);
 	row->userid = (*users)->userid;
 	// since update=false, a dup will be ok and do nothing when igndup=true

From 27b722e66e4aa4601747cd1c69646cb5fd6ce897 Mon Sep 17 00:00:00 2001
From: kanoi <gitkanoi@k1k2.com>
Date: Tue, 18 Nov 2014 18:24:19 +1100
Subject: [PATCH 05/21] ckdb - allow disabling/enabling users

---
 src/ckdb.c      |  9 +++++
 src/ckdb.h      | 11 +++---
 src/ckdb_cmd.c  | 91 ++++++++++++++++++++++++++++++++++++++++++-------
 src/ckdb_dbio.c | 28 +++++++++------
 4 files changed, 111 insertions(+), 28 deletions(-)

diff --git a/src/ckdb.c b/src/ckdb.c
index c43ae064..11a0274f 100644
--- a/src/ckdb.c
+++ b/src/ckdb.c
@@ -2052,6 +2052,7 @@ static void *socketer(__maybe_unused void *arg)
 	char *last_newid = NULL, *reply_newid = NULL;
 	char *last_setatts = NULL, *reply_setatts = NULL;
 	char *last_setopts = NULL, *reply_setopts = NULL;
+	char *last_userstatus = NULL, *reply_userstatus = NULL;
 	char *last_web = NULL, *reply_web = NULL;
 	char *reply_last, duptype[CMD_SIZ+1];
 	enum cmd_values cmdnum;
@@ -2155,6 +2156,9 @@ static void *socketer(__maybe_unused void *arg)
 			} else if (last_setopts && strcmp(last_setopts, buf) == 0) {
 				reply_last = reply_setopts;
 				dup = true;
+			} else if (last_userstatus && strcmp(last_userstatus, buf) == 0) {
+				reply_last = reply_userstatus;
+				dup = true;
 			} else if (last_web && strcmp(last_web, buf) == 0) {
 				reply_last = reply_web;
 				dup = true;
@@ -2255,6 +2259,7 @@ static void *socketer(__maybe_unused void *arg)
 					case CMD_BLOCKLIST:
 					case CMD_NEWID:
 					case CMD_STATS:
+					case CMD_USERSTATUS:
 						ans = ckdb_cmds[which_cmds].func(NULL, cmd, id, &now,
 										 by_default,
 										 (char *)__func__,
@@ -2297,6 +2302,9 @@ static void *socketer(__maybe_unused void *arg)
 							case CMD_SETOPTS:
 								STORELASTREPLY(setopts);
 								break;
+							case CMD_USERSTATUS:
+								STORELASTREPLY(userstatus);
+								break;
 							// The rest
 							default:
 								free(rep);
@@ -2503,6 +2511,7 @@ static bool reload_line(PGconn *conn, char *filename, uint64_t count, char *buf)
 			case CMD_DSP:
 			case CMD_STATS:
 			case CMD_PPLNS:
+			case CMD_USERSTATUS:
 				LOGERR("%s() Message line %"PRIu64" '%s' - invalid - ignored",
 					__func__, count, cmd);
 				break;
diff --git a/src/ckdb.h b/src/ckdb.h
index bb35f06d..e7341af0 100644
--- a/src/ckdb.h
+++ b/src/ckdb.h
@@ -52,7 +52,7 @@
 
 #define DB_VLOCK "1"
 #define DB_VERSION "0.9.4"
-#define CKDB_VERSION DB_VERSION"-0.642"
+#define CKDB_VERSION DB_VERSION"-0.643"
 
 #define WHERE_FFL " - from %s %s() line %d"
 #define WHERE_FFL_HERE __FILE__, __func__, __LINE__
@@ -303,6 +303,7 @@ enum cmd_values {
 	CMD_DSP,
 	CMD_STATS,
 	CMD_PPLNS,
+	CMD_USERSTATUS,
 	CMD_END
 };
 
@@ -641,7 +642,7 @@ typedef struct users {
 	int64_t userid;
 	char username[TXT_BIG+1];
 	char usertrim[TXT_BIG+1]; // Non DB field
-	// TODO: Anything in 'status' disables the account
+	// Anything in 'status' fails mining authentication
 	char status[TXT_BIG+1];
 	char emailaddress[TXT_BIG+1];
 	tv_t joineddate;
@@ -1544,9 +1545,9 @@ extern char *pqerrmsg(PGconn *conn);
 
 extern int64_t nextid(PGconn *conn, char *idname, int64_t increment,
 			tv_t *cd, char *by, char *code, char *inet);
-extern bool users_pass_email(PGconn *conn, K_ITEM *u_item, char *oldhash,
-			     char *newhash, char *email, char *by, char *code,
-			     char *inet, tv_t *cd, K_TREE *trf_root);
+extern bool users_update(PGconn *conn, K_ITEM *u_item, char *oldhash,
+			 char *newhash, char *email, char *by, char *code,
+			 char *inet, tv_t *cd, K_TREE *trf_root, char *status);
 extern K_ITEM *users_add(PGconn *conn, char *username, char *emailaddress,
 			char *passwordhash, char *by, char *code, char *inet,
 			tv_t *cd, K_TREE *trf_root);
diff --git a/src/ckdb_cmd.c b/src/ckdb_cmd.c
index e3443858..53d9cbef 100644
--- a/src/ckdb_cmd.c
+++ b/src/ckdb_cmd.c
@@ -97,12 +97,13 @@ static char *cmd_newpass(__maybe_unused PGconn *conn, char *cmd, char *id,
 		K_RUNLOCK(users_free);
 
 		if (u_item) {
-			ok = users_pass_email(NULL, u_item,
-						    oldhash,
-						    transfer_data(i_newhash),
-						    NULL,
-						    by, code, inet, now,
-						    trf_root);
+			ok = users_update(NULL, u_item,
+						oldhash,
+						transfer_data(i_newhash),
+						NULL,
+						by, code, inet, now,
+						trf_root,
+						NULL);
 		} else
 			ok = false;
 	}
@@ -261,10 +262,12 @@ static char *cmd_userset(PGconn *conn, char *cmd, char *id,
 			}
 
 			if (email && *email) {
-				ok = users_pass_email(conn, u_item, NULL,
-							    NULL, email,
-							    by, code, inet,
-							    now, trf_root);
+				ok = users_update(conn, u_item,
+							NULL, NULL,
+							email,
+							by, code, inet, now,
+							trf_root,
+							NULL);
 				if (!ok) {
 					reason = "email error";
 					goto struckout;
@@ -3392,6 +3395,69 @@ static char *cmd_stats(__maybe_unused PGconn *conn, char *cmd, char *id,
 	return buf;
 }
 
+// TODO: add to heartbeat to disable the miner if active and status != ""
+static char *cmd_userstatus(PGconn *conn, char *cmd, char *id, tv_t *now, char *by,
+			  char *code, char *inet, __maybe_unused tv_t *cd,
+			  K_TREE *trf_root)
+{
+	char reply[1024] = "";
+	size_t siz = sizeof(reply);
+	K_ITEM *i_username, *i_userid, *i_status, *u_item;
+	int64_t userid;
+	char *status;
+	USERS *users;
+	bool ok;
+
+	LOGDEBUG("%s(): cmd '%s'", __func__, cmd);
+
+	i_username = optional_name(trf_root, "username", 3, (char *)userpatt, reply, siz);
+	i_userid = optional_name(trf_root, "userid", 1, (char *)intpatt, reply, siz);
+	// Either username or userid
+	if (!i_username && !i_userid) {
+		snprintf(reply, siz, "failed.invalid/missing userinfo");
+		LOGERR("%s.%s", id, reply);
+		return strdup(reply);
+	}
+
+	// A zero length status re-enables it
+	i_status = require_name(trf_root, "status", 0, NULL, reply, siz);
+	if (!i_status)
+		return strdup(reply);
+	status = transfer_data(i_status);
+
+	K_RLOCK(users_free);
+	if (i_username)
+		u_item = find_users(transfer_data(i_username));
+	else {
+		TXT_TO_BIGINT("userid", transfer_data(i_userid), userid);
+		u_item = find_userid(userid);
+	}
+	K_RUNLOCK(users_free);
+
+	if (!u_item)
+		ok = false;
+	else {
+		ok = users_update(conn, u_item,
+					NULL, NULL,
+					NULL,
+					by, code, inet, now,
+					trf_root,
+					status);
+	}
+
+	if (!ok) {
+		LOGERR("%s() %s.failed.DBE", __func__, id);
+		return strdup("failed.DBE");
+	}
+	DATA_USERS(users, u_item);
+	snprintf(reply, siz, "ok.updated %"PRId64" %s status %s",
+			     users->userid,
+			     users->username,
+			     status[0] ? "disabled" : "enabled");
+	LOGWARNING("%s.%s", id, reply);
+	return strdup(reply);
+}
+
 // TODO: limit access by having seperate sockets for each
 #define ACCESS_POOL	"p"
 #define ACCESS_SYSTEM	"s"
@@ -3492,7 +3558,8 @@ struct CMDS ckdb_cmds[] = {
 	{ CMD_GETOPTS,	"getopts",	false,	false,	cmd_getopts,	ACCESS_WEB },
 	{ CMD_SETOPTS,	"setopts",	false,	false,	cmd_setopts,	ACCESS_WEB },
 	{ CMD_DSP,	"dsp",		false,	false,	cmd_dsp,	ACCESS_SYSTEM },
-	{ CMD_STATS,	"stats",	true,	false,	cmd_stats,	ACCESS_SYSTEM },
-	{ CMD_PPLNS,	"pplns",	false,	false,	cmd_pplns,	ACCESS_SYSTEM },
+	{ CMD_STATS,	"stats",	true,	false,	cmd_stats,	ACCESS_SYSTEM ACCESS_WEB },
+	{ CMD_PPLNS,	"pplns",	false,	false,	cmd_pplns,	ACCESS_SYSTEM ACCESS_WEB },
+	{ CMD_USERSTATUS,"userstatus",	false,	false,	cmd_userstatus,	ACCESS_SYSTEM ACCESS_WEB },
 	{ CMD_END,	NULL,		false,	false,	NULL,		NULL }
 };
diff --git a/src/ckdb_dbio.c b/src/ckdb_dbio.c
index f837e78b..c8d1d864 100644
--- a/src/ckdb_dbio.c
+++ b/src/ckdb_dbio.c
@@ -304,9 +304,10 @@ cleanup:
 	return lastid;
 }
 
-bool users_pass_email(PGconn *conn, K_ITEM *u_item, char *oldhash,
-			char *newhash, char *email, char *by, char *code,
-			char *inet, tv_t *cd, K_TREE *trf_root)
+// status was added to the end so type checking intercepts new mistakes
+bool users_update(PGconn *conn, K_ITEM *u_item, char *oldhash,
+		  char *newhash, char *email, char *by, char *code,
+		  char *inet, tv_t *cd, K_TREE *trf_root, char *status)
 {
 	ExecStatusType rescode;
 	bool conned = false;
@@ -315,7 +316,7 @@ bool users_pass_email(PGconn *conn, K_ITEM *u_item, char *oldhash,
 	USERS *row, *users;
 	char *upd, *ins;
 	bool ok = false;
-	char *params[5 + HISTORYDATECOUNT];
+	char *params[6 + HISTORYDATECOUNT];
 	bool hash;
 	int n, par = 0;
 
@@ -337,14 +338,18 @@ bool users_pass_email(PGconn *conn, K_ITEM *u_item, char *oldhash,
 
 	DATA_USERS(row, item);
 	memcpy(row, users, sizeof(*row));
-	// Update one, leave the other
+
+	// Update each one supplied
 	if (hash) {
 		// New salt each password change
 		make_salt(row);
 		password_hash(row->username, newhash, row->salt,
 			      row->passwordhash, sizeof(row->passwordhash));
-	} else
+	}
+	if (email)
 		STRNCPY(row->emailaddress, email);
+	if (status)
+		STRNCPY(row->status, status);
 
 	HISTORYDATEINIT(row, cd, by, code, inet);
 	HISTORYDATETRANSFER(trf_root, row);
@@ -384,21 +389,22 @@ bool users_pass_email(PGconn *conn, K_ITEM *u_item, char *oldhash,
 	par = 0;
 	params[par++] = bigint_to_buf(row->userid, NULL, 0);
 	params[par++] = tv_to_buf(cd, NULL, 0);
-	// Copy them both in - one will be new and one will be old
+	// Copy them all in - at least one will be new
+	params[par++] = str_to_buf(row->status, NULL, 0);
 	params[par++] = str_to_buf(row->emailaddress, NULL, 0);
 	params[par++] = str_to_buf(row->passwordhash, NULL, 0);
 	// New salt for each password change (or recopy old)
 	params[par++] = str_to_buf(row->salt, NULL, 0);
 	HISTORYDATEPARAMS(params, par, row);
-	PARCHKVAL(par, 5 + HISTORYDATECOUNT, params); // 10 as per ins
+	PARCHKVAL(par, 6 + HISTORYDATECOUNT, params); // 11 as per ins
 
 	ins = "insert into users "
 		"(userid,username,status,emailaddress,joineddate,"
 		"passwordhash,secondaryuserid,salt"
 		HISTORYDATECONTROL ") select "
-		"userid,username,status,$3,joineddate,"
-		"$4,secondaryuserid,$5,"
-		"$6,$7,$8,$9,$10 from users where "
+		"userid,username,$3,$4,joineddate,"
+		"$5,secondaryuserid,$6,"
+		"$7,$8,$9,$10,$11 from users where "
 		"userid=$1 and expirydate=$2";
 
 	res = PQexecParams(conn, ins, par, NULL, (const char **)params, NULL, NULL, 0, CKPQ_WRITE);

From 180410913c14d74449824fa0007ae23680e79294 Mon Sep 17 00:00:00 2001
From: kanoi <gitkanoi@k1k2.com>
Date: Tue, 18 Nov 2014 22:51:11 +1100
Subject: [PATCH 06/21] ckdb - include workmarkers in old pplns calculations

---
 src/ckdb.h     |  2 +-
 src/ckdb_cmd.c | 64 ++++++++++++++++++++++++++++++++++++++++++++++----
 2 files changed, 60 insertions(+), 6 deletions(-)

diff --git a/src/ckdb.h b/src/ckdb.h
index e7341af0..57279994 100644
--- a/src/ckdb.h
+++ b/src/ckdb.h
@@ -52,7 +52,7 @@
 
 #define DB_VLOCK "1"
 #define DB_VERSION "0.9.4"
-#define CKDB_VERSION DB_VERSION"-0.643"
+#define CKDB_VERSION DB_VERSION"-0.644"
 
 #define WHERE_FFL " - from %s %s() line %d"
 #define WHERE_FFL_HERE __FILE__, __func__, __LINE__
diff --git a/src/ckdb_cmd.c b/src/ckdb_cmd.c
index 53d9cbef..5d2119b9 100644
--- a/src/ckdb_cmd.c
+++ b/src/ckdb_cmd.c
@@ -2940,8 +2940,8 @@ static K_TREE *upd_add_mu(K_TREE *mu_root, K_STORE *mu_store, int64_t userid, in
 	diffacc_user / diffacc_total
 */
 
-/* TODO: redesign to include workmarkers
- * ... before next payout that extends into a markersummary ... */
+/* TODO: doesn't work if there are no sharesummaries,
+ *	 i.e. only markersummaries */
 static char *cmd_pplns(__maybe_unused PGconn *conn, char *cmd, char *id,
 			  __maybe_unused tv_t *now, __maybe_unused char *by,
 			  __maybe_unused char *code, __maybe_unused char *inet,
@@ -2951,8 +2951,11 @@ static char *cmd_pplns(__maybe_unused PGconn *conn, char *cmd, char *id,
 	size_t siz = sizeof(reply);
 	K_ITEM *i_height, *i_difftimes, *i_diffadd, *i_allowaged;
 	K_ITEM b_look, ss_look, *b_item, *w_item, *ss_item;
+	K_ITEM wm_look, *wm_item, ms_look, *ms_item;
 	K_ITEM *mu_item, *wb_item, *u_item;
 	SHARESUMMARY looksharesummary, *sharesummary;
+	WORKMARKERS lookworkmarkers, *workmarkers;
+	MARKERSUMMARY lookmarkersummary, *markersummary;
 	MININGPAYOUTS *miningpayouts;
 	WORKINFO *workinfo;
 	TRANSFER *transfer;
@@ -2966,7 +2969,7 @@ static char *cmd_pplns(__maybe_unused PGconn *conn, char *cmd, char *id,
 	int64_t share_count;
 	char tv_buf[DATE_BUFSIZ];
 	tv_t cd, begin_tv, block_tv, end_tv;
-	K_TREE_CTX ctx[1];
+	K_TREE_CTX ctx[1], wm_ctx[1], ms_ctx[1];
 	double ndiff, total, elapsed;
 	double diff_times = 1.0;
 	double diff_add = 0.0;
@@ -3053,9 +3056,13 @@ static char *cmd_pplns(__maybe_unused PGconn *conn, char *cmd, char *id,
 	INIT_SHARESUMMARY(&ss_look);
 	ss_look.data = (void *)(&looksharesummary);
 	K_RLOCK(sharesummary_free);
+	K_RLOCK(workmarkers_free);
+	K_RLOCK(markersummary_free);
 	ss_item = find_before_in_ktree(sharesummary_workinfoid_root, &ss_look,
 					cmp_sharesummary_workinfoid, ctx);
 	if (!ss_item) {
+		K_RUNLOCK(markersummary_free);
+		K_RUNLOCK(workmarkers_free);
 		K_RUNLOCK(sharesummary_free);
 		snprintf(reply, siz,
 			 "ERR.no shares found with or before "
@@ -3071,7 +3078,7 @@ static char *cmd_pplns(__maybe_unused PGconn *conn, char *cmd, char *id,
 	/* add up all sharesummaries until >= diff_want
 	 * also record the latest lastshare - that will be the end pplns time
 	 *  which will be >= block_tv */
-	while (ss_item && total < diff_want) {
+	while (total < diff_want && ss_item) {
 		switch (sharesummary->complete[0]) {
 			case SUMMARY_CONFIRM:
 				break;
@@ -3079,6 +3086,8 @@ static char *cmd_pplns(__maybe_unused PGconn *conn, char *cmd, char *id,
 				if (allow_aged)
 					break;
 			default:
+				K_RUNLOCK(markersummary_free);
+				K_RUNLOCK(workmarkers_free);
 				K_RUNLOCK(sharesummary_free);
 				snprintf(reply, siz,
 					 "ERR.sharesummary1 not ready in "
@@ -3086,7 +3095,7 @@ static char *cmd_pplns(__maybe_unused PGconn *conn, char *cmd, char *id,
 					 sharesummary->workinfoid);
 				goto shazbot;
 		}
-		share_count++;
+		share_count += sharesummary->sharecount;
 		total += (int64_t)(sharesummary->diffacc);
 		begin_workinfoid = sharesummary->workinfoid;
 		if (tv_newer(&end_tv, &(sharesummary->lastshare)))
@@ -3107,6 +3116,8 @@ static char *cmd_pplns(__maybe_unused PGconn *conn, char *cmd, char *id,
 				if (allow_aged)
 					break;
 			default:
+				K_RUNLOCK(markersummary_free);
+				K_RUNLOCK(workmarkers_free);
 				K_RUNLOCK(sharesummary_free);
 				snprintf(reply, siz,
 					 "ERR.sharesummary2 not ready in "
@@ -3122,6 +3133,49 @@ static char *cmd_pplns(__maybe_unused PGconn *conn, char *cmd, char *id,
 		ss_item = prev_in_ktree(ctx);
 		DATA_SHARESUMMARY_NULL(sharesummary, ss_item);
 	}
+
+	/* If we haven't met or exceeded the required N,
+	 * move on to the markersummaries */
+	if (total < diff_want) {
+		lookworkmarkers.expirydate.tv_sec = default_expiry.tv_sec;
+		lookworkmarkers.expirydate.tv_usec = default_expiry.tv_usec;
+		lookworkmarkers.workinfoidend = begin_workinfoid;
+		INIT_WORKMARKERS(&wm_look);
+		wm_look.data = (void *)(&lookworkmarkers);
+		wm_item = find_before_in_ktree(workmarkers_workinfoid_root, &wm_look,
+					       cmp_workmarkers_workinfoid, wm_ctx);
+		DATA_WORKMARKERS_NULL(workmarkers, wm_item);
+		while (total < diff_want && wm_item && CURRENT(&(workmarkers->expirydate))) {
+			if (WMREADY(workmarkers->status)) {
+				lookmarkersummary.markerid = workmarkers->markerid;
+				lookmarkersummary.userid = MAXID;
+				lookmarkersummary.workername[0] = '\0';
+				INIT_MARKERSUMMARY(&ms_look);
+				ms_look.data = (void *)(&lookmarkersummary);
+				ms_item = find_before_in_ktree(markersummary_root, &ms_look,
+							       cmp_markersummary, ms_ctx);
+				DATA_MARKERSUMMARY_NULL(markersummary, ms_item);
+				// add the whole markerid
+				while (ms_item && markersummary->markerid == workmarkers->markerid) {
+					share_count += markersummary->sharecount;
+					total += (int64_t)(markersummary->diffacc);
+					begin_workinfoid = workmarkers->workinfoidstart;
+					if (tv_newer(&end_tv, &(markersummary->lastshare)))
+						copy_tv(&end_tv, &(markersummary->lastshare));
+					mu_root = upd_add_mu(mu_root, mu_store,
+							     markersummary->userid,
+							     (int64_t)(markersummary->diffacc));
+					ms_item = prev_in_ktree(ms_ctx);
+					DATA_MARKERSUMMARY_NULL(markersummary, ms_item);
+				}
+			}
+			wm_item = prev_in_ktree(wm_ctx);
+			DATA_WORKMARKERS_NULL(workmarkers, wm_item);
+		}
+	}
+
+	K_RUNLOCK(markersummary_free);
+	K_RUNLOCK(workmarkers_free);
 	K_RUNLOCK(sharesummary_free);
 
 	if (total == 0.0) {

From b231b158e61f3bc6b84e14d511e639852ce46c5a Mon Sep 17 00:00:00 2001
From: kanoi <gitkanoi@k1k2.com>
Date: Wed, 19 Nov 2014 08:47:59 +1100
Subject: [PATCH 07/21] ckdb/php - allow pplns to show blocks that can't be or
 aren't ready to be paid out

---
 pool/page_pplns.php | 18 +++++++++-
 src/ckdb.c          |  2 +-
 src/ckdb.h          |  2 +-
 src/ckdb_cmd.c      | 87 ++++++++++++++++++++++++---------------------
 4 files changed, 65 insertions(+), 44 deletions(-)

diff --git a/pool/page_pplns.php b/pool/page_pplns.php
index 1b46dd0e..cbd76be8 100644
--- a/pool/page_pplns.php
+++ b/pool/page_pplns.php
@@ -14,7 +14,7 @@ function stnum($num)
  return $b4.$fmt.$af;
 }
 #
-# ... Of course ... check the output and add the txin
+# ... Of course ... check the output and add the txin ... etc.
 function calctx($ans, $count, $miner_sat, $diffacc_total)
 {
  $pg = '<br><table cellpadding=0 cellspacing=0 border=0>';
@@ -159,6 +159,7 @@ Block: <input type=text name=blk size=10 value='$blkuse'>
 	$ans['miner_sat'] = $miner_sat;
 
 	$data = array(	'Block' => 'block',
+			'Block Status' => 'block_status',
 			'Block Hash' => 'block_hash',
 			'Block Reward (Satoshis)' => 'block_reward',
 			'Miner Reward (Satoshis)' => 'miner_sat',
@@ -182,6 +183,21 @@ Block: <input type=text name=blk size=10 value='$blkuse'>
 
 	$pg = '<br><a href=https://blockchain.info/block-height/';
 	$pg .= $ans['block'].'>Blockchain '.$ans['block']."</a><br>\n";
+
+	if (strlen($ans['block_extra']) > 0)
+	{
+		$pg .= '<br><span class=err>';
+		$msg = $ans['block_status'].' - '.$ans['block_extra'];
+		$pg .= str_replace(' ', '&nbsp;', $msg)."</span><br>\n";
+	}
+
+	if (strlen($ans['share_status']) > 0)
+	{
+		$pg .= '<br><span class=err>';
+		$msg = $ans['share_status']." - Can't be paid out yet";
+		$pg .= str_replace(' ', '&nbsp;', $msg)."</span><br>\n";
+	}
+
 	$pg .= "<br><table callpadding=0 cellspacing=0 border=0>\n";
 	$pg .= '<tr class=title>';
 	$pg .= '<td class=dl>Name</td>';
diff --git a/src/ckdb.c b/src/ckdb.c
index 11a0274f..8f3357b5 100644
--- a/src/ckdb.c
+++ b/src/ckdb.c
@@ -1656,7 +1656,7 @@ static void summarise_blocks()
 		if (WMREADY(workmarkers->status)) {
 			lookmarkersummary.markerid = workmarkers->markerid;
 			lookmarkersummary.userid = MAXID;
-			lookmarkersummary.workername[0] = '\0';
+			lookmarkersummary.workername = EMPTY;
 			INIT_MARKERSUMMARY(&ms_look);
 			ms_look.data = (void *)(&lookmarkersummary);
 			ms_item = find_before_in_ktree(markersummary_root, &ms_look,
diff --git a/src/ckdb.h b/src/ckdb.h
index 57279994..7767835f 100644
--- a/src/ckdb.h
+++ b/src/ckdb.h
@@ -52,7 +52,7 @@
 
 #define DB_VLOCK "1"
 #define DB_VERSION "0.9.4"
-#define CKDB_VERSION DB_VERSION"-0.644"
+#define CKDB_VERSION DB_VERSION"-0.645"
 
 #define WHERE_FFL " - from %s %s() line %d"
 #define WHERE_FFL_HERE __FILE__, __func__, __LINE__
diff --git a/src/ckdb_cmd.c b/src/ckdb_cmd.c
index 5d2119b9..13761c58 100644
--- a/src/ckdb_cmd.c
+++ b/src/ckdb_cmd.c
@@ -2925,6 +2925,9 @@ static K_TREE *upd_add_mu(K_TREE *mu_root, K_STORE *mu_store, int64_t userid, in
      (also summarising diffacc per user)
     then keep stepping back until we complete the current begin_workinfoid
      (also summarising diffacc per user)
+   While we are still below diff_want
+    find each workmarker and add on the full set of worksummary
+     diffacc shares (also summarising diffacc per user)
    This will give us the total number of diff1 shares (diffacc_total)
     to use for the payment calculations
    The value of diff_want defaults to the block's network difficulty
@@ -2938,16 +2941,18 @@ static K_TREE *upd_add_mu(K_TREE *mu_root, K_STORE *mu_store, int64_t userid, in
 	diffacc_user * 2^32 / pplns_elapsed
    PPLNS fraction of the payout would be:
 	diffacc_user / diffacc_total
+
+   N.B. 'begin' means the oldest back in time and 'end' means the newest
+	'end' should usually be the info of the found block with the pplns
+	data going back in time to 'begin'
 */
 
-/* TODO: doesn't work if there are no sharesummaries,
- *	 i.e. only markersummaries */
 static char *cmd_pplns(__maybe_unused PGconn *conn, char *cmd, char *id,
 			  __maybe_unused tv_t *now, __maybe_unused char *by,
 			  __maybe_unused char *code, __maybe_unused char *inet,
 			  __maybe_unused tv_t *notcd, K_TREE *trf_root)
 {
-	char reply[1024], tmp[1024], *buf;
+	char reply[1024], tmp[1024], *buf, *block_extra, *share_status = EMPTY;
 	size_t siz = sizeof(reply);
 	K_ITEM *i_height, *i_difftimes, *i_diffadd, *i_allowaged;
 	K_ITEM b_look, ss_look, *b_item, *w_item, *ss_item;
@@ -2964,7 +2969,7 @@ static char *cmd_pplns(__maybe_unused PGconn *conn, char *cmd, char *id,
 	K_STORE *mu_store;
 	USERS *users;
 	int32_t height;
-	int64_t workinfoid, end_workinfoid;
+	int64_t workinfoid, end_workinfoid = 0;
 	int64_t begin_workinfoid;
 	int64_t share_count;
 	char tv_buf[DATE_BUFSIZ];
@@ -3015,16 +3020,28 @@ static char *cmd_pplns(__maybe_unused PGconn *conn, char *cmd, char *id,
 	}
 	DATA_BLOCKS_NULL(blocks, b_item);
 	while (b_item && blocks->height == height) {
-		if (blocks->confirmed[0] == BLOCKS_CONFIRM)
+		// Allow any state, but report it
+		if (CURRENT(&(blocks->expirydate)))
 			break;
 		b_item = prev_in_ktree(ctx);
 		DATA_BLOCKS_NULL(blocks, b_item);
 	}
 	K_RUNLOCK(blocks_free);
 	if (!b_item || blocks->height != height) {
-		snprintf(reply, siz, "ERR.unconfirmed block %d", height);
+		snprintf(reply, siz, "ERR.no current block %d", height);
 		return strdup(reply);
 	}
+	switch (blocks->confirmed[0]) {
+		case BLOCKS_NEW:
+			block_extra = "Can't be paid out yet";
+			break;
+		case BLOCKS_ORPHAN:
+			block_extra = "Can't be paid out";
+			break;
+		default:
+			block_extra = EMPTY;
+			break;
+	}
 	workinfoid = blocks->workinfoid;
 	copy_tv(&block_tv, &(blocks->createdate));
 	copy_tv(&end_tv, &(blocks->createdate));
@@ -3050,6 +3067,9 @@ static char *cmd_pplns(__maybe_unused PGconn *conn, char *cmd, char *id,
 	share_count = 0;
 	total = 0;
 
+	mu_store = k_new_store(miningpayouts_free);
+	mu_root = new_ktree();
+
 	looksharesummary.workinfoid = workinfoid;
 	looksharesummary.userid = MAXID;
 	looksharesummary.workername[0] = '\0';
@@ -3060,21 +3080,9 @@ static char *cmd_pplns(__maybe_unused PGconn *conn, char *cmd, char *id,
 	K_RLOCK(markersummary_free);
 	ss_item = find_before_in_ktree(sharesummary_workinfoid_root, &ss_look,
 					cmp_sharesummary_workinfoid, ctx);
-	if (!ss_item) {
-		K_RUNLOCK(markersummary_free);
-		K_RUNLOCK(workmarkers_free);
-		K_RUNLOCK(sharesummary_free);
-		snprintf(reply, siz,
-			 "ERR.no shares found with or before "
-			 "workinfo %"PRId64,
-			 workinfoid);
-		return strdup(reply);
-	}
-	DATA_SHARESUMMARY(sharesummary, ss_item);
-
-	mu_store = k_new_store(miningpayouts_free);
-	mu_root = new_ktree();
-	end_workinfoid = sharesummary->workinfoid;
+	DATA_SHARESUMMARY_NULL(sharesummary, ss_item);
+	if (ss_item)
+		end_workinfoid = sharesummary->workinfoid;
 	/* add up all sharesummaries until >= diff_want
 	 * also record the latest lastshare - that will be the end pplns time
 	 *  which will be >= block_tv */
@@ -3086,14 +3094,7 @@ static char *cmd_pplns(__maybe_unused PGconn *conn, char *cmd, char *id,
 				if (allow_aged)
 					break;
 			default:
-				K_RUNLOCK(markersummary_free);
-				K_RUNLOCK(workmarkers_free);
-				K_RUNLOCK(sharesummary_free);
-				snprintf(reply, siz,
-					 "ERR.sharesummary1 not ready in "
-					 "workinfo %"PRId64,
-					 sharesummary->workinfoid);
-				goto shazbot;
+				share_status = "Not ready1";
 		}
 		share_count += sharesummary->sharecount;
 		total += (int64_t)(sharesummary->diffacc);
@@ -3116,14 +3117,10 @@ static char *cmd_pplns(__maybe_unused PGconn *conn, char *cmd, char *id,
 				if (allow_aged)
 					break;
 			default:
-				K_RUNLOCK(markersummary_free);
-				K_RUNLOCK(workmarkers_free);
-				K_RUNLOCK(sharesummary_free);
-				snprintf(reply, siz,
-					 "ERR.sharesummary2 not ready in "
-					 "workinfo %"PRId64,
-					 sharesummary->workinfoid);
-				goto shazbot;
+				if (share_status == EMPTY)
+					share_status = "Not ready2";
+				else
+					share_status = "Not ready1+2";
 		}
 		share_count++;
 		total += (int64_t)(sharesummary->diffacc);
@@ -3142,14 +3139,14 @@ static char *cmd_pplns(__maybe_unused PGconn *conn, char *cmd, char *id,
 		lookworkmarkers.workinfoidend = begin_workinfoid;
 		INIT_WORKMARKERS(&wm_look);
 		wm_look.data = (void *)(&lookworkmarkers);
-		wm_item = find_before_in_ktree(workmarkers_workinfoid_root, &wm_look,
-					       cmp_workmarkers_workinfoid, wm_ctx);
+		wm_item = find_before_in_ktree(workmarkers_root, &wm_look,
+					       cmp_workmarkers, wm_ctx);
 		DATA_WORKMARKERS_NULL(workmarkers, wm_item);
 		while (total < diff_want && wm_item && CURRENT(&(workmarkers->expirydate))) {
 			if (WMREADY(workmarkers->status)) {
 				lookmarkersummary.markerid = workmarkers->markerid;
 				lookmarkersummary.userid = MAXID;
-				lookmarkersummary.workername[0] = '\0';
+				lookmarkersummary.workername = EMPTY;
 				INIT_MARKERSUMMARY(&ms_look);
 				ms_look.data = (void *)(&lookmarkersummary);
 				ms_item = find_before_in_ktree(markersummary_root, &ms_look,
@@ -3157,6 +3154,8 @@ static char *cmd_pplns(__maybe_unused PGconn *conn, char *cmd, char *id,
 				DATA_MARKERSUMMARY_NULL(markersummary, ms_item);
 				// add the whole markerid
 				while (ms_item && markersummary->markerid == workmarkers->markerid) {
+					if (end_workinfoid == 0)
+						end_workinfoid = workmarkers->workinfoidend;
 					share_count += markersummary->sharecount;
 					total += (int64_t)(markersummary->diffacc);
 					begin_workinfoid = workmarkers->workinfoidstart;
@@ -3173,7 +3172,6 @@ static char *cmd_pplns(__maybe_unused PGconn *conn, char *cmd, char *id,
 			DATA_WORKMARKERS_NULL(workmarkers, wm_item);
 		}
 	}
-
 	K_RUNLOCK(markersummary_free);
 	K_RUNLOCK(workmarkers_free);
 	K_RUNLOCK(sharesummary_free);
@@ -3210,6 +3208,13 @@ static char *cmd_pplns(__maybe_unused PGconn *conn, char *cmd, char *id,
 	APPEND_REALLOC(buf, off, len, tmp);
 	snprintf(tmp, sizeof(tmp), "block_reward=%"PRId64"%c", blocks->reward, FLDSEP);
 	APPEND_REALLOC(buf, off, len, tmp);
+	snprintf(tmp, sizeof(tmp), "block_status=%s%c",
+				   blocks_confirmed(blocks->confirmed), FLDSEP);
+	APPEND_REALLOC(buf, off, len, tmp);
+	snprintf(tmp, sizeof(tmp), "block_extra=%s%c", block_extra, FLDSEP);
+	APPEND_REALLOC(buf, off, len, tmp);
+	snprintf(tmp, sizeof(tmp), "share_status=%s%c", share_status, FLDSEP);
+	APPEND_REALLOC(buf, off, len, tmp);
 	snprintf(tmp, sizeof(tmp), "workername=%s%c", blocks->workername, FLDSEP);
 	APPEND_REALLOC(buf, off, len, tmp);
 	snprintf(tmp, sizeof(tmp), "nonce=%s%c", blocks->nonce, FLDSEP);

From c05098f226857de07b7845966949a554905d8bb0 Mon Sep 17 00:00:00 2001
From: kanoi <gitkanoi@k1k2.com>
Date: Wed, 19 Nov 2014 10:42:07 +1100
Subject: [PATCH 08/21] php - since menu hitbox is now the full box, reduce
 back to 6

---
 pool/page.php | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pool/page.php b/pool/page.php
index 5d0d7d44..2f335fcf 100644
--- a/pool/page.php
+++ b/pool/page.php
@@ -110,7 +110,7 @@ span.err {color:red; font-weight:bold; font-size:120%;}
 span.alert {color:red; font-weight:bold; font-size:250%;}
 input.tiny {width: 0px; height: 0px; margin: 0px; padding: 0px; outline: none; border: 0px;}
 #n42 {margin:0; position: relative; color:#fff; background:#07e;}
-#n42 a {color:#fff; text-decoration:none; padding: 8px; display:block;}
+#n42 a {color:#fff; text-decoration:none; padding: 6px; display:block;}
 #n42 td {min-width: 100px; float: left; vertical-align: top; padding: 0px 2px;}
 #n42 td.navboxr {float: right;}
 #n42 td.nav {position: relative;}

From 2ddfc892b4e01518b045c674428852ac2d2a3217 Mon Sep 17 00:00:00 2001
From: kanoi <gitkanoi@k1k2.com>
Date: Wed, 19 Nov 2014 11:03:35 +1100
Subject: [PATCH 09/21] php - help menu changes

---
 pool/prime.php | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/pool/prime.php b/pool/prime.php
index b4bd1dee..c50c69f7 100644
--- a/pool/prime.php
+++ b/pool/prime.php
@@ -69,8 +69,9 @@ function check()
 			'API' => 'api'
 	),
 	'Help' => array(
-		'Help' => 'help',
-		'Payouts' => 'payout'
+		'Payouts' => 'payout',
+		'Workers ' => 'workers',
+		'Blocks' => 'blocks'
 	)
  );
  tryLogInOut();

From c9dd84fac29bc16ecdbb1c9181b2aae2cd52e08f Mon Sep 17 00:00:00 2001
From: Con Kolivas <kernel@kolivas.org>
Date: Wed, 19 Nov 2014 21:22:31 +1100
Subject: [PATCH 10/21] Handle invalidated socket fds with an error in the wait
 read/write select functions

---
 src/libckpool.c | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/src/libckpool.c b/src/libckpool.c
index 8fb55173..747ac02d 100644
--- a/src/libckpool.c
+++ b/src/libckpool.c
@@ -711,6 +711,8 @@ int wait_read_select(int sockd, int timeout)
 {
 	struct pollfd sfd;
 
+	if (unlikely(sockd < 0))
+		return -1;
 	sfd.fd = sockd;
 	sfd.events = POLLIN;
 	sfd.revents = 0;
@@ -784,6 +786,8 @@ int wait_write_select(int sockd, int timeout)
 {
 	struct pollfd sfd;
 
+	if (unlikely(sockd < 0))
+		return -1;
 	sfd.fd = sockd;
 	sfd.events = POLLOUT;
 	sfd.revents = 0;

From 06507de1fcf8aec776ab3e5f1a711a6442ea7ecf Mon Sep 17 00:00:00 2001
From: Con Kolivas <kernel@kolivas.org>
Date: Wed, 19 Nov 2014 21:25:33 +1100
Subject: [PATCH 11/21] Handle read and write_length functions receiving an
 invalidated fd

---
 src/libckpool.c | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/src/libckpool.c b/src/libckpool.c
index 747ac02d..860b537e 100644
--- a/src/libckpool.c
+++ b/src/libckpool.c
@@ -728,6 +728,8 @@ int read_length(int sockd, void *buf, int len)
 		LOGWARNING("Invalid read length of %d requested in read_length", len);
 		return -1;
 	}
+	if (unlikely(sockd < 0))
+		return -1;
 	while (len) {
 		ret = recv(sockd, buf + ofs, len, MSG_WAITALL);
 		if (unlikely(ret < 1))
@@ -803,6 +805,8 @@ int write_length(int sockd, const void *buf, int len)
 		LOGWARNING("Invalid write length of %d requested in write_length", len);
 		return -1;
 	}
+	if (unlikely(sockd < 0))
+		return -1;
 	while (len) {
 		ret = write(sockd, buf + ofs, len);
 		if (unlikely(ret < 0))

From 61300493e79a206d82d4633e0e69cfca69675153 Mon Sep 17 00:00:00 2001
From: Con Kolivas <kernel@kolivas.org>
Date: Wed, 19 Nov 2014 21:29:18 +1100
Subject: [PATCH 12/21] Empty buffers of servers and proxy when killing them

---
 src/ckpool.h    | 1 +
 src/generator.c | 2 ++
 2 files changed, 3 insertions(+)

diff --git a/src/ckpool.h b/src/ckpool.h
index d246e97f..6c7743a2 100644
--- a/src/ckpool.h
+++ b/src/ckpool.h
@@ -188,6 +188,7 @@ bool ckmsgq_empty(ckmsgq_t *ckmsgq);
 ckpool_t *global_ckp;
 
 bool ping_main(ckpool_t *ckp);
+void empty_buffer(connsock_t *cs);
 int read_socket_line(connsock_t *cs, int timeout);
 bool _send_proc(proc_instance_t *pi, const char *msg, const char *file, const char *func, const int line);
 #define send_proc(pi, msg) _send_proc(pi, msg, __FILE__, __func__, __LINE__)
diff --git a/src/generator.c b/src/generator.c
index f9791257..0fd73d55 100644
--- a/src/generator.c
+++ b/src/generator.c
@@ -225,6 +225,7 @@ static void kill_server(server_instance_t *si)
 	LOGNOTICE("Killing server");
 	cs = &si->cs;
 	Close(cs->fd);
+	empty_buffer(cs);
 	dealloc(cs->url);
 	dealloc(cs->port);
 	dealloc(cs->auth);
@@ -1428,6 +1429,7 @@ static void kill_proxy(ckpool_t *ckp, proxy_instance_t *proxi)
 	LOGNOTICE("Killing proxy");
 	cs = proxi->cs;
 	Close(cs->fd);
+	empty_buffer(cs);
 
 	/* All our notify data is invalid if we reconnect so discard them */
 	mutex_lock(&proxi->notify_lock);

From 1d0d914d52e7d46fcb2836d6703159be004542f2 Mon Sep 17 00:00:00 2001
From: Con Kolivas <kernel@kolivas.org>
Date: Wed, 19 Nov 2014 21:34:40 +1100
Subject: [PATCH 13/21] Store the fd in read_socket_line to not have it change
 under us, and empty the cs buffer on failure

---
 src/ckpool.c | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/src/ckpool.c b/src/ckpool.c
index e1d1186b..abff3050 100644
--- a/src/ckpool.c
+++ b/src/ckpool.c
@@ -367,11 +367,11 @@ void empty_buffer(connsock_t *cs)
  * of the buffer for use on the next receive. */
 int read_socket_line(connsock_t *cs, int timeout)
 {
+	int fd = cs->fd, ret = -1;
 	char *eom = NULL;
 	size_t buflen;
-	int ret = -1;
 
-	if (unlikely(cs->fd < 0))
+	if (unlikely(fd < 0))
 		goto out;
 
 	if (unlikely(!cs->buf))
@@ -388,7 +388,7 @@ int read_socket_line(connsock_t *cs, int timeout)
 	while (42) {
 		char readbuf[PAGESIZE] = {};
 
-		ret = wait_read_select(cs->fd, eom ? 0 : timeout);
+		ret = wait_read_select(fd, eom ? 0 : timeout);
 		if (eom && !ret)
 			break;
 		if (ret < 1) {
@@ -398,7 +398,7 @@ int read_socket_line(connsock_t *cs, int timeout)
 				LOGERR("Select failed in read_socket_line");
 			goto out;
 		}
-		ret = recv(cs->fd, readbuf, PAGESIZE - 4, 0);
+		ret = recv(fd, readbuf, PAGESIZE - 4, 0);
 		if (ret < 1) {
 			LOGERR("Failed to recv in read_socket_line");
 			ret = -1;
@@ -423,6 +423,7 @@ int read_socket_line(connsock_t *cs, int timeout)
 	*eom = '\0';
 out:
 	if (ret < 0) {
+		empty_buffer(cs);
 		dealloc(cs->buf);
 		Close(cs->fd);
 	}

From 5a57d01911766025a0725e69a2fe542fefb92d16 Mon Sep 17 00:00:00 2001
From: kanoi <gitkanoi@k1k2.com>
Date: Wed, 19 Nov 2014 22:49:45 +1100
Subject: [PATCH 14/21] ckdb - correct pplns workmarkers sort

---
 src/ckdb.h     | 2 +-
 src/ckdb_cmd.c | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/ckdb.h b/src/ckdb.h
index 7767835f..d6d3c49d 100644
--- a/src/ckdb.h
+++ b/src/ckdb.h
@@ -52,7 +52,7 @@
 
 #define DB_VLOCK "1"
 #define DB_VERSION "0.9.4"
-#define CKDB_VERSION DB_VERSION"-0.645"
+#define CKDB_VERSION DB_VERSION"-0.646"
 
 #define WHERE_FFL " - from %s %s() line %d"
 #define WHERE_FFL_HERE __FILE__, __func__, __LINE__
diff --git a/src/ckdb_cmd.c b/src/ckdb_cmd.c
index 13761c58..b9bddb32 100644
--- a/src/ckdb_cmd.c
+++ b/src/ckdb_cmd.c
@@ -3139,8 +3139,8 @@ static char *cmd_pplns(__maybe_unused PGconn *conn, char *cmd, char *id,
 		lookworkmarkers.workinfoidend = begin_workinfoid;
 		INIT_WORKMARKERS(&wm_look);
 		wm_look.data = (void *)(&lookworkmarkers);
-		wm_item = find_before_in_ktree(workmarkers_root, &wm_look,
-					       cmp_workmarkers, wm_ctx);
+		wm_item = find_before_in_ktree(workmarkers_workinfoid_root, &wm_look,
+					       cmp_workmarkers_workinfoid, wm_ctx);
 		DATA_WORKMARKERS_NULL(workmarkers, wm_item);
 		while (total < diff_want && wm_item && CURRENT(&(workmarkers->expirydate))) {
 			if (WMREADY(workmarkers->status)) {

From 71eac4e579bd38d89b3d71dd8a67a44516845ad4 Mon Sep 17 00:00:00 2001
From: kanoi <gitkanoi@k1k2.com>
Date: Thu, 20 Nov 2014 08:02:24 +1100
Subject: [PATCH 15/21] ckdb/sql - add sql/code definition for Marks - not
 implemented yet

---
 sql/ckdb.sql          | 19 +++++++++++++--
 sql/v0.9.4-v0.9.5.sql | 41 ++++++++++++++++++++++++++++++++
 src/ckdb.c            |  5 ++++
 src/ckdb.h            | 54 ++++++++++++++++++++++++++++++++++++++++++-
 4 files changed, 116 insertions(+), 3 deletions(-)
 create mode 100644 sql/v0.9.4-v0.9.5.sql

diff --git a/sql/ckdb.sql b/sql/ckdb.sql
index f9bf6d73..aaa6eca7 100644
--- a/sql/ckdb.sql
+++ b/sql/ckdb.sql
@@ -251,6 +251,21 @@ CREATE TABLE sharesummary ( -- per workinfo for each user+worker
 );
 
 
+CREATE TABLE marks ( -- workinfoids to make workmarkers
+    poolinstance character varying(256) NOT NULL,
+    workinfoid bigint NOT NULL,
+    description character varying(256) DEFAULT ''::character varying NOT NULL,
+    marktype char NOT NULL, -- 'b'lock 'p'plns-begin 's'hift-begin 'e'=shift-end
+    status char NOT NULL,
+    createdate timestamp with time zone NOT NULL,
+    createby character varying(64) DEFAULT ''::character varying NOT NULL,
+    createcode character varying(128) DEFAULT ''::character varying NOT NULL,
+    createinet character varying(128) DEFAULT ''::character varying NOT NULL,
+    expirydate timestamp with time zone DEFAULT '6666-06-06 06:06:06+00',
+    PRIMARY KEY (poolinstance, workinfoid, expirydate)
+);
+
+
 CREATE TABLE workmarkers ( -- range of workinfo for share accounting
     markerid bigint NOT NULL,
     poolinstance character varying(256) NOT NULL,
@@ -263,7 +278,7 @@ CREATE TABLE workmarkers ( -- range of workinfo for share accounting
     createcode character varying(128) DEFAULT ''::character varying NOT NULL,
     createinet character varying(128) DEFAULT ''::character varying NOT NULL,
     expirydate timestamp with time zone DEFAULT '6666-06-06 06:06:06+00',
-    PRIMARY KEY (markerid)
+    PRIMARY KEY (markerid, expirydate)
 );
 
 
@@ -417,4 +432,4 @@ CREATE TABLE version (
     PRIMARY KEY (vlock)
 );
 
-insert into version (vlock,version) values (1,'0.9.4');
+insert into version (vlock,version) values (1,'0.9.5');
diff --git a/sql/v0.9.4-v0.9.5.sql b/sql/v0.9.4-v0.9.5.sql
new file mode 100644
index 00000000..c61a4e9c
--- /dev/null
+++ b/sql/v0.9.4-v0.9.5.sql
@@ -0,0 +1,41 @@
+SET SESSION AUTHORIZATION 'postgres';
+
+BEGIN transaction;
+
+DO $$
+DECLARE ver TEXT;
+BEGIN
+
+ UPDATE version set version='0.9.5' where vlock=1 and version='0.9.4';
+
+ IF found THEN
+  RETURN;
+ END IF;
+
+ SELECT version into ver from version
+  WHERE vlock=1;
+
+ RAISE EXCEPTION 'Wrong DB version - expect "0.9.4" - found "%"', ver;
+
+END $$;
+
+CREATE TABLE marks ( -- workinfoids to make workmarkers
+    poolinstance character varying(256) NOT NULL,
+    workinfoid bigint NOT NULL,
+    description character varying(256) DEFAULT ''::character varying NOT NULL,
+    marktype char NOT NULL, -- 'b'lock(end) 'p'plns-begin 's'hift-begin 'e'=shift-end
+    status char NOT NULL,
+    createdate timestamp with time zone NOT NULL,
+    createby character varying(64) DEFAULT ''::character varying NOT NULL,
+    createcode character varying(128) DEFAULT ''::character varying NOT NULL,
+    createinet character varying(128) DEFAULT ''::character varying NOT NULL,
+    expirydate timestamp with time zone DEFAULT '6666-06-06 06:06:06+00',
+    PRIMARY KEY (poolinstance, workinfoid, expirydate)
+);
+
+ALTER TABLE workmarkers DROP CONSTRAINT workmarkers_pkey;
+
+ALTER TABLE workmarkers ADD CONSTRAINT workmarkers_pkey
+ PRIMARY KEY (markerid, expirydate);
+
+END transaction;
diff --git a/src/ckdb.c b/src/ckdb.c
index 8f3357b5..11cead07 100644
--- a/src/ckdb.c
+++ b/src/ckdb.c
@@ -453,6 +453,11 @@ K_TREE *workmarkers_workinfoid_root;
 K_LIST *workmarkers_free;
 K_STORE *workmarkers_store;
 
+// MARKS
+K_TREE *marks_root;
+K_LIST *marks_free;
+K_STORE *marks_store;
+
 static char logname[512];
 static char *dbcode;
 
diff --git a/src/ckdb.h b/src/ckdb.h
index d6d3c49d..ab4fded8 100644
--- a/src/ckdb.h
+++ b/src/ckdb.h
@@ -51,7 +51,7 @@
  */
 
 #define DB_VLOCK "1"
-#define DB_VERSION "0.9.4"
+#define DB_VERSION "0.9.5"
 #define CKDB_VERSION DB_VERSION"-0.646"
 
 #define WHERE_FFL " - from %s %s() line %d"
@@ -1333,6 +1333,58 @@ extern K_STORE *workmarkers_store;
 #define MARKER_COMPLETE 'x'
 #define WMREADY(_status) (tolower(_status[0]) == MARKER_COMPLETE)
 
+// MARKS
+// TODO: implement
+typedef struct marks {
+	char *poolinstance;
+	int64_t workinfoid;
+	char *description;
+	char marktype[TXT_FLAG+1];
+	char status[TXT_FLAG+1];
+	HISTORYDATECONTROLFIELDS;
+} MARKS;
+
+/* Marks:
+ *  marktype is one of:
+ *   b - block end
+ *   p - pplns begin
+ *   s - shift begin (not yet used)
+ *   e - shift end (not yet used)
+ *  description should one one of
+ *   b - Block NNN stt
+ *   p - Payout NNN fin (where NNN is the block number of the payout)
+ *   s/e - to be decided
+ *
+ * WorkMarkers are from a begin workinfoid to an end workinfoid
+ *  the "-1" and "+1" below mean adding to or subtracting from
+ *  the workinfoid number
+ *
+ * Until we start using shifts:
+ *  WorkMarkers can be created up to ending in the largest 'p' "-1"
+ *  WorkMarkers will always be the smallest of:
+ *   Block NNN-1 "+1" to Block NNN
+ *   Block NNN "+1" to Payout MMM "-1"
+ *   Payout MMM to Block NNN
+ *   Payout MMM-1 to Payout MMM "-1"
+ * Thus to generate the WorkMarkers from the Marks:
+ *  Find the last 'p' with no matching workinfoidbegin
+ *  Then determine each previous WorkMarker based on each previous
+ *  mark, using the above rules and stop when we find one that already exists
+ */
+
+#define ALLOC_MARKS 1000
+#define LIMIT_MARKS 0
+#define INIT_MARKS(_item) INIT_GENERIC(_item, marks)
+#define DATA_MARKS(_var, _item) DATA_GENERIC(_var, _item, marks, true)
+#define DATA_MARKS_NULL(_var, _item) DATA_GENERIC(_var, _item, marks, false)
+
+extern K_TREE *marks_root;
+extern K_LIST *marks_free;
+extern K_STORE *marks_store;
+
+#define MARK_READY 'x'
+#define MREADY(_status) (tolower(_status[0]) == MARK_READY)
+
 extern void logmsg(int loglevel, const char *fmt, ...);
 extern void setnow(tv_t *now);
 extern void tick();

From 5fe3dd77e22478c8e90f4cc026793252bd281d4c Mon Sep 17 00:00:00 2001
From: kanoi <gitkanoi@k1k2.com>
Date: Thu, 20 Nov 2014 08:57:07 +1100
Subject: [PATCH 16/21] ckdb/php - add more stats in pplns about where the
 shares come from

---
 pool/page_pplns.php |  6 +++++-
 src/ckdb.h          |  2 +-
 src/ckdb_cmd.c      | 33 +++++++++++++++++++++++++++------
 3 files changed, 33 insertions(+), 8 deletions(-)

diff --git a/pool/page_pplns.php b/pool/page_pplns.php
index cbd76be8..116763aa 100644
--- a/pool/page_pplns.php
+++ b/pool/page_pplns.php
@@ -179,7 +179,11 @@ Block: <input type=text name=blk size=10 value='$blkuse'>
 			'Network Difficulty' => 'block_ndiff',
 			'PPLNS Factor' => 'diff_times',
 			'PPLNS Added' => 'diff_add',
-			'Share Count' => 'share_count');
+			'Accepted Share Count' => 'acc_share_count',
+			'Total Share Count' => 'total_share_count',
+			'ShareSummary Count' => 'ss_count',
+			'WorkMarkers Count' => 'wm_count',
+			'MarkerSummary Count' => 'ms_count');
 
 	$pg = '<br><a href=https://blockchain.info/block-height/';
 	$pg .= $ans['block'].'>Blockchain '.$ans['block']."</a><br>\n";
diff --git a/src/ckdb.h b/src/ckdb.h
index ab4fded8..b81f98c8 100644
--- a/src/ckdb.h
+++ b/src/ckdb.h
@@ -52,7 +52,7 @@
 
 #define DB_VLOCK "1"
 #define DB_VERSION "0.9.5"
-#define CKDB_VERSION DB_VERSION"-0.646"
+#define CKDB_VERSION DB_VERSION"-0.647"
 
 #define WHERE_FFL " - from %s %s() line %d"
 #define WHERE_FFL_HERE __FILE__, __func__, __LINE__
diff --git a/src/ckdb_cmd.c b/src/ckdb_cmd.c
index b9bddb32..c331eca2 100644
--- a/src/ckdb_cmd.c
+++ b/src/ckdb_cmd.c
@@ -2971,7 +2971,8 @@ static char *cmd_pplns(__maybe_unused PGconn *conn, char *cmd, char *id,
 	int32_t height;
 	int64_t workinfoid, end_workinfoid = 0;
 	int64_t begin_workinfoid;
-	int64_t share_count;
+	int64_t total_share_count, acc_share_count;
+	int64_t ss_count, wm_count, ms_count;
 	char tv_buf[DATE_BUFSIZ];
 	tv_t cd, begin_tv, block_tv, end_tv;
 	K_TREE_CTX ctx[1], wm_ctx[1], ms_ctx[1];
@@ -3044,6 +3045,8 @@ static char *cmd_pplns(__maybe_unused PGconn *conn, char *cmd, char *id,
 	}
 	workinfoid = blocks->workinfoid;
 	copy_tv(&block_tv, &(blocks->createdate));
+	/* Last share should be the share that found it and match the createdate,
+	 * or not long after it for shares accepted until the workinfo switch */
 	copy_tv(&end_tv, &(blocks->createdate));
 
 	w_item = find_workinfo(workinfoid);
@@ -3064,8 +3067,9 @@ static char *cmd_pplns(__maybe_unused PGconn *conn, char *cmd, char *id,
 	}
 
 	begin_workinfoid = 0;
-	share_count = 0;
+	total_share_count = acc_share_count = 0;
 	total = 0;
+	ss_count = wm_count = ms_count = 0;
 
 	mu_store = k_new_store(miningpayouts_free);
 	mu_root = new_ktree();
@@ -3096,7 +3100,9 @@ static char *cmd_pplns(__maybe_unused PGconn *conn, char *cmd, char *id,
 			default:
 				share_status = "Not ready1";
 		}
-		share_count += sharesummary->sharecount;
+		ss_count++;
+		total_share_count += sharesummary->sharecount;
+		acc_share_count += sharesummary->shareacc;
 		total += (int64_t)(sharesummary->diffacc);
 		begin_workinfoid = sharesummary->workinfoid;
 		if (tv_newer(&end_tv, &(sharesummary->lastshare)))
@@ -3122,7 +3128,9 @@ static char *cmd_pplns(__maybe_unused PGconn *conn, char *cmd, char *id,
 				else
 					share_status = "Not ready1+2";
 		}
-		share_count++;
+		ss_count++;
+		total_share_count += sharesummary->sharecount;
+		acc_share_count += sharesummary->shareacc;
 		total += (int64_t)(sharesummary->diffacc);
 		mu_root = upd_add_mu(mu_root, mu_store,
 				     sharesummary->userid,
@@ -3144,6 +3152,7 @@ static char *cmd_pplns(__maybe_unused PGconn *conn, char *cmd, char *id,
 		DATA_WORKMARKERS_NULL(workmarkers, wm_item);
 		while (total < diff_want && wm_item && CURRENT(&(workmarkers->expirydate))) {
 			if (WMREADY(workmarkers->status)) {
+				wm_count++;
 				lookmarkersummary.markerid = workmarkers->markerid;
 				lookmarkersummary.userid = MAXID;
 				lookmarkersummary.workername = EMPTY;
@@ -3156,7 +3165,9 @@ static char *cmd_pplns(__maybe_unused PGconn *conn, char *cmd, char *id,
 				while (ms_item && markersummary->markerid == workmarkers->markerid) {
 					if (end_workinfoid == 0)
 						end_workinfoid = workmarkers->workinfoidend;
-					share_count += markersummary->sharecount;
+					ms_count++;
+					total_share_count += markersummary->sharecount;
+					acc_share_count += markersummary->shareacc;
 					total += (int64_t)(markersummary->diffacc);
 					begin_workinfoid = workmarkers->workinfoidstart;
 					if (tv_newer(&end_tv, &(markersummary->lastshare)))
@@ -3308,7 +3319,17 @@ static char *cmd_pplns(__maybe_unused PGconn *conn, char *cmd, char *id,
 	APPEND_REALLOC(buf, off, len, tmp);
 	snprintf(tmp, sizeof(tmp), "diff_want=%f%c", diff_want, FLDSEP);
 	APPEND_REALLOC(buf, off, len, tmp);
-	snprintf(tmp, sizeof(tmp), "share_count=%"PRId64, share_count);
+	snprintf(tmp, sizeof(tmp), "acc_share_count=%"PRId64"%c",
+				   acc_share_count, FLDSEP);
+	APPEND_REALLOC(buf, off, len, tmp);
+	snprintf(tmp, sizeof(tmp), "total_share_count=%"PRId64"%c",
+				   total_share_count, FLDSEP);
+	APPEND_REALLOC(buf, off, len, tmp);
+	snprintf(tmp, sizeof(tmp), "ss_count=%"PRId64"%c", ss_count, FLDSEP);
+	APPEND_REALLOC(buf, off, len, tmp);
+	snprintf(tmp, sizeof(tmp), "wm_count=%"PRId64"%c", wm_count, FLDSEP);
+	APPEND_REALLOC(buf, off, len, tmp);
+	snprintf(tmp, sizeof(tmp), "ms_count=%"PRId64, ms_count);
 	APPEND_REALLOC(buf, off, len, tmp);
 
 	mu_root = free_ktree(mu_root, NULL);

From 941cba9b3bf31b23351281e040fc1b992dfeeb6d Mon Sep 17 00:00:00 2001
From: kanoi <gitkanoi@k1k2.com>
Date: Thu, 20 Nov 2014 10:15:33 +1100
Subject: [PATCH 17/21] ckdb - convert sharesummary workername to a ptr -
 reduce ram ... lots

---
 src/ckdb.c      |  9 +++++++--
 src/ckdb.h      | 37 ++++++++++++++++++++++---------------
 src/ckdb_cmd.c  |  2 +-
 src/ckdb_data.c | 11 +++++------
 src/ckdb_dbio.c | 20 +++++++++++++++-----
 5 files changed, 50 insertions(+), 29 deletions(-)

diff --git a/src/ckdb.c b/src/ckdb.c
index 11cead07..ffede0a4 100644
--- a/src/ckdb.c
+++ b/src/ckdb.c
@@ -1057,6 +1057,11 @@ static void free_sharesummary_data(K_ITEM *item)
 	SHARESUMMARY *sharesummary;
 
 	DATA_SHARESUMMARY(sharesummary, item);
+	if (sharesummary->workername) {
+		LIST_MEM_SUB(sharesummary_free, sharesummary->workername);
+		free(sharesummary->workername);
+		sharesummary->workername = NULL;
+	}
 	SET_CREATEBY(sharesummary_free, sharesummary->createby, EMPTY);
 	SET_CREATECODE(sharesummary_free, sharesummary->createcode, EMPTY);
 	SET_CREATEINET(sharesummary_free, sharesummary->createinet, EMPTY);
@@ -1595,7 +1600,7 @@ static void summarise_blocks()
 	// Add up the sharesummaries, abort if any SUMMARY_NEW
 	looksharesummary.workinfoid = wi_finish;
 	looksharesummary.userid = MAXID;
-	looksharesummary.workername[0] = '\0';
+	looksharesummary.workername = EMPTY;
 	INIT_SHARESUMMARY(&ss_look);
 	ss_look.data = (void *)(&looksharesummary);
 
@@ -2872,7 +2877,7 @@ static void compare_summaries(K_TREE *leftsum, char *leftname,
 
 	looksharesummary.workinfoid = confirm_first_workinfoid;
 	looksharesummary.userid = -1;
-	looksharesummary.workername[0] = '\0';
+	looksharesummary.workername = EMPTY;
 	INIT_SHARESUMMARY(&look);
 	look.data = (void *)(&looksharesummary);
 
diff --git a/src/ckdb.h b/src/ckdb.h
index b81f98c8..bb269752 100644
--- a/src/ckdb.h
+++ b/src/ckdb.h
@@ -52,7 +52,7 @@
 
 #define DB_VLOCK "1"
 #define DB_VERSION "0.9.5"
-#define CKDB_VERSION DB_VERSION"-0.647"
+#define CKDB_VERSION DB_VERSION"-0.648"
 
 #define WHERE_FFL " - from %s %s() line %d"
 #define WHERE_FFL_HERE __FILE__, __func__, __LINE__
@@ -347,15 +347,26 @@ enum cmd_values {
 // The size strdup will allocate multiples of
 #define MEMBASE 4
 
+#define LIST_MEM_ADD(_list, _fld) do { \
+		size_t __siz; \
+		__siz = strlen(_fld) + 1; \
+		if (__siz % MEMBASE) \
+			__siz += MEMBASE - (__siz % MEMBASE); \
+		_list->ram += (int)__siz; \
+	} while (0)
+
+#define LIST_MEM_SUB(_list, _fld) do { \
+		size_t __siz; \
+		__siz = strlen(_fld) + 1; \
+		if (__siz % MEMBASE) \
+			__siz += MEMBASE - (__siz % MEMBASE); \
+		_list->ram -= (int)__siz; \
+	} while (0)
+
 #define SET_POINTER(_list, _fld, _val, _def) do { \
-		size_t _siz; \
 		if ((_fld) && ((_fld) != EMPTY) && ((_fld) != (_def))) { \
-			if (_list) { \
-				_siz = strlen(_fld) + 1; \
-				if (_siz % MEMBASE) \
-					_siz += MEMBASE - (_siz % MEMBASE); \
-				_list->ram -= (int)_siz; \
-			} \
+			if (_list) \
+				LIST_MEM_SUB(_list, _fld); \
 			free(_fld); \
 		} \
 		if (!(_val) || !(*(_val))) \
@@ -364,12 +375,8 @@ enum cmd_values {
 			if (((_val) == (_def)) || (strcmp(_val, _def) == 0)) \
 				(_fld) = (_def); \
 			else { \
-				if (_list) { \
-					_siz = strlen(_val) + 1; \
-					if (_siz % MEMBASE) \
-						_siz += MEMBASE - (_siz % MEMBASE); \
-					_list->ram += (int)_siz; \
-				} \
+				if (_list) \
+					LIST_MEM_ADD(_list, _val); \
 				_fld = strdup(_val); \
 				if (!(_fld)) \
 					quithere(1, "malloc OOM"); \
@@ -936,7 +943,7 @@ extern K_STORE *shareerrors_store;
 // SHARESUMMARY
 typedef struct sharesummary {
 	int64_t userid;
-	char workername[TXT_BIG+1];
+	char *workername;
 	int64_t workinfoid;
 	double diffacc;
 	double diffsta;
diff --git a/src/ckdb_cmd.c b/src/ckdb_cmd.c
index c331eca2..e51db263 100644
--- a/src/ckdb_cmd.c
+++ b/src/ckdb_cmd.c
@@ -3076,7 +3076,7 @@ static char *cmd_pplns(__maybe_unused PGconn *conn, char *cmd, char *id,
 
 	looksharesummary.workinfoid = workinfoid;
 	looksharesummary.userid = MAXID;
-	looksharesummary.workername[0] = '\0';
+	looksharesummary.workername = EMPTY;
 	INIT_SHARESUMMARY(&ss_look);
 	ss_look.data = (void *)(&looksharesummary);
 	K_RLOCK(sharesummary_free);
diff --git a/src/ckdb_data.c b/src/ckdb_data.c
index b799648d..e176acfe 100644
--- a/src/ckdb_data.c
+++ b/src/ckdb_data.c
@@ -710,7 +710,7 @@ void workerstatus_ready()
 
 		// The last one
 		looksharesummary.userid = workerstatus->userid;
-		STRNCPY(looksharesummary.workername, workerstatus->workername);
+		looksharesummary.workername = workerstatus->workername;
 		looksharesummary.workinfoid = MAXID;
 		ss_look.data = (void *)(&looksharesummary);
 		K_RLOCK(sharesummary_free);
@@ -1376,7 +1376,7 @@ bool workinfo_age(PGconn *conn, int64_t workinfoid, char *poolinstance,
 	// Find the first matching sharesummary
 	looksharesummary.workinfoid = workinfoid;
 	looksharesummary.userid = -1;
-	looksharesummary.workername[0] = '\0';
+	looksharesummary.workername = EMPTY;
 
 	ok = true;
 	ss_tot = ss_already = ss_failed = shares_tot = shares_dumped = 0;
@@ -1607,7 +1607,7 @@ K_ITEM *find_sharesummary(int64_t userid, char *workername, int64_t workinfoid)
 	K_ITEM look;
 
 	sharesummary.userid = userid;
-	STRNCPY(sharesummary.workername, workername);
+	sharesummary.workername = workername;
 	sharesummary.workinfoid = workinfoid;
 
 	INIT_SHARESUMMARY(&look);
@@ -1643,7 +1643,7 @@ void auto_age_older(PGconn *conn, int64_t workinfoid, char *poolinstance,
 	// Find the oldest 'unaged' sharesummary < workinfoid and >= prev_found
 	looksharesummary.workinfoid = prev_found;
 	looksharesummary.userid = -1;
-	looksharesummary.workername[0] = '\0';
+	looksharesummary.workername = EMPTY;
 	INIT_SHARESUMMARY(&look);
 	look.data = (void *)(&looksharesummary);
 
@@ -1956,8 +1956,7 @@ void set_block_share_counters()
 		if (sharesummary->workinfoid <= pool.workinfoid) {
 			// Skip back to the next worker
 			looksharesummary.userid = sharesummary->userid;
-			STRNCPY(looksharesummary.workername,
-				sharesummary->workername);
+			looksharesummary.workername = sharesummary->workername;
 			looksharesummary.workinfoid = -1;
 			ss_look.data = (void *)(&looksharesummary);
 			ss_item = find_before_in_ktree(sharesummary_root, &ss_look,
diff --git a/src/ckdb_dbio.c b/src/ckdb_dbio.c
index c8d1d864..f4e4681a 100644
--- a/src/ckdb_dbio.c
+++ b/src/ckdb_dbio.c
@@ -2719,7 +2719,8 @@ bool _sharesummary_update(PGconn *conn, SHARES *s_row, SHAREERRORS *e_row, K_ITE
 			K_WUNLOCK(sharesummary_free);
 			DATA_SHARESUMMARY(row, item);
 			row->userid = userid;
-			STRNCPY(row->workername, workername);
+			row->workername = strdup(workername);
+			LIST_MEM_ADD(sharesummary_free, row->workername);
 			row->workinfoid = workinfoid;
 			zero_sharesummary(row, sharecreatedate, diff);
 			row->inserted = false;
@@ -2948,15 +2949,15 @@ late:
 		PQfinish(conn);
 
 	// We keep the new item no matter what 'ok' is, since it will be inserted later
-	K_WLOCK(sharesummary_free);
 	if (new) {
+		K_WLOCK(sharesummary_free);
 		sharesummary_root = add_to_ktree(sharesummary_root, item, cmp_sharesummary);
 		sharesummary_workinfoid_root = add_to_ktree(sharesummary_workinfoid_root,
 							    item,
 							    cmp_sharesummary_workinfoid);
 		k_add_head(sharesummary_store, item);
+		K_WUNLOCK(sharesummary_free);
 	}
-	K_WUNLOCK(sharesummary_free);
 
 	return ok;
 }
@@ -3010,6 +3011,7 @@ bool sharesummary_fill(PGconn *conn)
 	for (i = 0; i < n; i++) {
 		item = k_unlink_head(sharesummary_free);
 		DATA_SHARESUMMARY(row, item);
+		row->workername = NULL;
 
 		if (everyone_die) {
 			ok = false;
@@ -3026,7 +3028,8 @@ bool sharesummary_fill(PGconn *conn)
 		PQ_GET_FLD(res, i, "workername", field, ok);
 		if (!ok)
 			break;
-		TXT_TO_STR("workername", field, row->workername);
+		row->workername = strdup(field);
+		LIST_MEM_ADD(sharesummary_free, row->workername);
 
 		PQ_GET_FLD(res, i, "workinfoid", field, ok);
 		if (!ok)
@@ -3153,8 +3156,15 @@ bool sharesummary_fill(PGconn *conn)
 
 		tick();
 	}
-	if (!ok)
+	if (!ok) {
+		DATA_SHARESUMMARY(row, item);
+		if (row->workername) {
+			LIST_MEM_SUB(sharesummary_free, row->workername);
+			free(row->workername);
+			row->workername = NULL;
+		}
 		k_add_head(sharesummary_free, item);
+	}
 
 	PQclear(res);
 

From 61318d94383d9f2263a16d8efd6bf8bd064a40b1 Mon Sep 17 00:00:00 2001
From: kanoi <gitkanoi@k1k2.com>
Date: Thu, 20 Nov 2014 23:35:21 +1100
Subject: [PATCH 18/21] ckdb - pplns set block createdate to first block record

---
 src/ckdb.h     |  2 +-
 src/ckdb_cmd.c | 38 ++++++++++++++++++++++----------------
 2 files changed, 23 insertions(+), 17 deletions(-)

diff --git a/src/ckdb.h b/src/ckdb.h
index bb269752..c05e0cc4 100644
--- a/src/ckdb.h
+++ b/src/ckdb.h
@@ -52,7 +52,7 @@
 
 #define DB_VLOCK "1"
 #define DB_VERSION "0.9.5"
-#define CKDB_VERSION DB_VERSION"-0.648"
+#define CKDB_VERSION DB_VERSION"-0.649"
 
 #define WHERE_FFL " - from %s %s() line %d"
 #define WHERE_FFL_HERE __FILE__, __func__, __LINE__
diff --git a/src/ckdb_cmd.c b/src/ckdb_cmd.c
index e51db263..1dcd7b6b 100644
--- a/src/ckdb_cmd.c
+++ b/src/ckdb_cmd.c
@@ -2976,7 +2976,7 @@ static char *cmd_pplns(__maybe_unused PGconn *conn, char *cmd, char *id,
 	char tv_buf[DATE_BUFSIZ];
 	tv_t cd, begin_tv, block_tv, end_tv;
 	K_TREE_CTX ctx[1], wm_ctx[1], ms_ctx[1];
-	double ndiff, total, elapsed;
+	double ndiff, total_diff, elapsed;
 	double diff_times = 1.0;
 	double diff_add = 0.0;
 	double diff_want;
@@ -3007,6 +3007,7 @@ static char *cmd_pplns(__maybe_unused PGconn *conn, char *cmd, char *id,
 			allow_aged = true;
 	}
 
+	block_tv.tv_sec = block_tv.tv_usec = 0L;
 	cd.tv_sec = cd.tv_usec = 0L;
 	lookblocks.height = height + 1;
 	lookblocks.blockhash[0] = '\0';
@@ -3021,6 +3022,10 @@ static char *cmd_pplns(__maybe_unused PGconn *conn, char *cmd, char *id,
 	}
 	DATA_BLOCKS_NULL(blocks, b_item);
 	while (b_item && blocks->height == height) {
+		if (blocks->confirmed[0] == BLOCKS_NEW) {
+			copy_tv(&block_tv, &(blocks->createdate));
+			copy_tv(&end_tv, &(blocks->createdate));
+		}
 		// Allow any state, but report it
 		if (CURRENT(&(blocks->expirydate)))
 			break;
@@ -3029,7 +3034,13 @@ static char *cmd_pplns(__maybe_unused PGconn *conn, char *cmd, char *id,
 	}
 	K_RUNLOCK(blocks_free);
 	if (!b_item || blocks->height != height) {
-		snprintf(reply, siz, "ERR.no current block %d", height);
+		snprintf(reply, siz, "ERR.no CURRENT block %d", height);
+		return strdup(reply);
+	}
+	if (block_tv.tv_sec == 0) {
+		snprintf(reply, siz, "ERR.block %d missing '%s' record",
+				     height,
+				     blocks_confirmed(BLOCKS_NEW_STR));
 		return strdup(reply);
 	}
 	switch (blocks->confirmed[0]) {
@@ -3044,11 +3055,6 @@ static char *cmd_pplns(__maybe_unused PGconn *conn, char *cmd, char *id,
 			break;
 	}
 	workinfoid = blocks->workinfoid;
-	copy_tv(&block_tv, &(blocks->createdate));
-	/* Last share should be the share that found it and match the createdate,
-	 * or not long after it for shares accepted until the workinfo switch */
-	copy_tv(&end_tv, &(blocks->createdate));
-
 	w_item = find_workinfo(workinfoid);
 	if (!w_item) {
 		snprintf(reply, siz, "ERR.missing workinfo %"PRId64, workinfoid);
@@ -3068,7 +3074,7 @@ static char *cmd_pplns(__maybe_unused PGconn *conn, char *cmd, char *id,
 
 	begin_workinfoid = 0;
 	total_share_count = acc_share_count = 0;
-	total = 0;
+	total_diff = 0;
 	ss_count = wm_count = ms_count = 0;
 
 	mu_store = k_new_store(miningpayouts_free);
@@ -3090,7 +3096,7 @@ static char *cmd_pplns(__maybe_unused PGconn *conn, char *cmd, char *id,
 	/* add up all sharesummaries until >= diff_want
 	 * also record the latest lastshare - that will be the end pplns time
 	 *  which will be >= block_tv */
-	while (total < diff_want && ss_item) {
+	while (total_diff < diff_want && ss_item) {
 		switch (sharesummary->complete[0]) {
 			case SUMMARY_CONFIRM:
 				break;
@@ -3103,7 +3109,7 @@ static char *cmd_pplns(__maybe_unused PGconn *conn, char *cmd, char *id,
 		ss_count++;
 		total_share_count += sharesummary->sharecount;
 		acc_share_count += sharesummary->shareacc;
-		total += (int64_t)(sharesummary->diffacc);
+		total_diff += (int64_t)(sharesummary->diffacc);
 		begin_workinfoid = sharesummary->workinfoid;
 		if (tv_newer(&end_tv, &(sharesummary->lastshare)))
 			copy_tv(&end_tv, &(sharesummary->lastshare));
@@ -3131,7 +3137,7 @@ static char *cmd_pplns(__maybe_unused PGconn *conn, char *cmd, char *id,
 		ss_count++;
 		total_share_count += sharesummary->sharecount;
 		acc_share_count += sharesummary->shareacc;
-		total += (int64_t)(sharesummary->diffacc);
+		total_diff += (int64_t)(sharesummary->diffacc);
 		mu_root = upd_add_mu(mu_root, mu_store,
 				     sharesummary->userid,
 				     (int64_t)(sharesummary->diffacc));
@@ -3141,7 +3147,7 @@ static char *cmd_pplns(__maybe_unused PGconn *conn, char *cmd, char *id,
 
 	/* If we haven't met or exceeded the required N,
 	 * move on to the markersummaries */
-	if (total < diff_want) {
+	if (total_diff < diff_want) {
 		lookworkmarkers.expirydate.tv_sec = default_expiry.tv_sec;
 		lookworkmarkers.expirydate.tv_usec = default_expiry.tv_usec;
 		lookworkmarkers.workinfoidend = begin_workinfoid;
@@ -3150,7 +3156,7 @@ static char *cmd_pplns(__maybe_unused PGconn *conn, char *cmd, char *id,
 		wm_item = find_before_in_ktree(workmarkers_workinfoid_root, &wm_look,
 					       cmp_workmarkers_workinfoid, wm_ctx);
 		DATA_WORKMARKERS_NULL(workmarkers, wm_item);
-		while (total < diff_want && wm_item && CURRENT(&(workmarkers->expirydate))) {
+		while (total_diff < diff_want && wm_item && CURRENT(&(workmarkers->expirydate))) {
 			if (WMREADY(workmarkers->status)) {
 				wm_count++;
 				lookmarkersummary.markerid = workmarkers->markerid;
@@ -3168,7 +3174,7 @@ static char *cmd_pplns(__maybe_unused PGconn *conn, char *cmd, char *id,
 					ms_count++;
 					total_share_count += markersummary->sharecount;
 					acc_share_count += markersummary->shareacc;
-					total += (int64_t)(markersummary->diffacc);
+					total_diff += (int64_t)(markersummary->diffacc);
 					begin_workinfoid = workmarkers->workinfoidstart;
 					if (tv_newer(&end_tv, &(markersummary->lastshare)))
 						copy_tv(&end_tv, &(markersummary->lastshare));
@@ -3187,7 +3193,7 @@ static char *cmd_pplns(__maybe_unused PGconn *conn, char *cmd, char *id,
 	K_RUNLOCK(workmarkers_free);
 	K_RUNLOCK(sharesummary_free);
 
-	if (total == 0.0) {
+	if (total_diff == 0.0) {
 		snprintf(reply, siz,
 			 "ERR.total share diff 0 before workinfo %"PRId64,
 			 workinfoid);
@@ -3236,7 +3242,7 @@ static char *cmd_pplns(__maybe_unused PGconn *conn, char *cmd, char *id,
 	APPEND_REALLOC(buf, off, len, tmp);
 	snprintf(tmp, sizeof(tmp), "end_workinfoid=%"PRId64"%c", end_workinfoid, FLDSEP);
 	APPEND_REALLOC(buf, off, len, tmp);
-	snprintf(tmp, sizeof(tmp), "diffacc_total=%.0f%c", total, FLDSEP);
+	snprintf(tmp, sizeof(tmp), "diffacc_total=%.0f%c", total_diff, FLDSEP);
 	APPEND_REALLOC(buf, off, len, tmp);
 	snprintf(tmp, sizeof(tmp), "pplns_elapsed=%f%c", elapsed, FLDSEP);
 	APPEND_REALLOC(buf, off, len, tmp);

From 98b52140b7c7674f264d5a90295ff4a2d21e6989 Mon Sep 17 00:00:00 2001
From: kanoi <gitkanoi@k1k2.com>
Date: Fri, 21 Nov 2014 07:22:11 +1100
Subject: [PATCH 19/21] ckdb - load the marks table

---
 src/ckdb.c      |  9 +++++
 src/ckdb.h      |  4 ++-
 src/ckdb_data.c | 13 +++++++
 src/ckdb_dbio.c | 94 +++++++++++++++++++++++++++++++++++++++++++++++++
 4 files changed, 119 insertions(+), 1 deletion(-)

diff --git a/src/ckdb.c b/src/ckdb.c
index ffede0a4..07e8b056 100644
--- a/src/ckdb.c
+++ b/src/ckdb.c
@@ -737,6 +737,8 @@ static bool getdata3()
 	}
 	if (!(ok = workinfo_fill(conn)) || everyone_die)
 		goto sukamudai;
+	if (!(ok = marks_fill(conn)) || everyone_die)
+		goto sukamudai;
 	if (!(ok = workmarkers_fill(conn)) || everyone_die)
 		goto sukamudai;
 	if (!(ok = markersummary_fill(conn)) || everyone_die)
@@ -1039,6 +1041,11 @@ static void alloc_storage()
 	workmarkers_root = new_ktree();
 	workmarkers_workinfoid_root = new_ktree();
 	workmarkers_free->dsp_func = dsp_workmarkers;
+
+	marks_free = k_new_list("Marks", sizeof(MARKS),
+				ALLOC_MARKS, LIMIT_MARKS, true);
+	marks_store = k_new_store(workmarkers_free);
+	marks_root = new_ktree();
 }
 
 static void free_workinfo_data(K_ITEM *item)
@@ -1145,6 +1152,8 @@ static void dealloc_storage()
 {
 	FREE_LISTS(logqueue);
 
+	FREE_ALL(marks);
+
 	FREE_TREE(workmarkers_workinfoid);
 	FREE_TREE(workmarkers);
 	FREE_STORE_DATA(workmarkers);
diff --git a/src/ckdb.h b/src/ckdb.h
index c05e0cc4..b25150c2 100644
--- a/src/ckdb.h
+++ b/src/ckdb.h
@@ -52,7 +52,7 @@
 
 #define DB_VLOCK "1"
 #define DB_VERSION "0.9.5"
-#define CKDB_VERSION DB_VERSION"-0.649"
+#define CKDB_VERSION DB_VERSION"-0.650"
 
 #define WHERE_FFL " - from %s %s() line %d"
 #define WHERE_FFL_HERE __FILE__, __func__, __LINE__
@@ -1557,6 +1557,7 @@ extern cmp_t cmp_workmarkers(K_ITEM *a, K_ITEM *b);
 extern cmp_t cmp_workmarkers_workinfoid(K_ITEM *a, K_ITEM *b);
 extern K_ITEM *find_workmarkers(int64_t workinfoid);
 extern K_ITEM *find_workmarkerid(int64_t markerid);
+extern cmp_t cmp_marks(K_ITEM *a, K_ITEM *b);
 
 // ***
 // *** PostgreSQL functions ckdb_dbio.c
@@ -1700,6 +1701,7 @@ extern bool userstats_add(char *poolinstance, char *elapsed, char *username,
 extern bool userstats_fill(PGconn *conn);
 extern bool markersummary_fill(PGconn *conn);
 extern bool workmarkers_fill(PGconn *conn);
+extern bool marks_fill(PGconn *conn);
 extern bool check_db_version(PGconn *conn);
 
 // ***
diff --git a/src/ckdb_data.c b/src/ckdb_data.c
index e176acfe..241fdac3 100644
--- a/src/ckdb_data.c
+++ b/src/ckdb_data.c
@@ -2434,3 +2434,16 @@ K_ITEM *find_workmarkerid(int64_t markerid)
 	return wm_item;
 }
 
+// order by expirydate asc,workinfoid asc
+// TODO: add poolinstance
+cmp_t cmp_marks(K_ITEM *a, K_ITEM *b)
+{
+	MARKS *ma, *mb;
+	DATA_MARKS(ma, a);
+	DATA_MARKS(mb, b);
+	cmp_t c = CMP_TV(ma->expirydate, mb->expirydate);
+	if (c == 0)
+		c = CMP_BIGINT(ma->workinfoid, mb->workinfoid);
+	return c;
+}
+
diff --git a/src/ckdb_dbio.c b/src/ckdb_dbio.c
index f4e4681a..c5a23fda 100644
--- a/src/ckdb_dbio.c
+++ b/src/ckdb_dbio.c
@@ -5108,6 +5108,100 @@ bool workmarkers_fill(PGconn *conn)
 	return ok;
 }
 
+bool marks_fill(PGconn *conn)
+{
+	ExecStatusType rescode;
+	PGresult *res;
+	K_ITEM *item;
+	int n, i;
+	MARKS *row;
+	char *field;
+	char *sel;
+	int fields = 5;
+	bool ok;
+
+	LOGDEBUG("%s(): select", __func__);
+
+	// TODO: limit how far back
+	sel = "select "
+		"poolinstance,workinfoid,description,marktype,status"
+		HISTORYDATECONTROL
+		" from marks";
+	res = PQexec(conn, sel, CKPQ_READ);
+	rescode = PQresultStatus(res);
+	if (!PGOK(rescode)) {
+		PGLOGERR("Select", rescode, conn);
+		PQclear(res);
+		return false;
+	}
+
+	n = PQnfields(res);
+	if (n != (fields + HISTORYDATECOUNT)) {
+		LOGERR("%s(): Invalid field count - should be %d, but is %d",
+			__func__, fields + HISTORYDATECOUNT, n);
+		PQclear(res);
+		return false;
+	}
+
+	n = PQntuples(res);
+	LOGDEBUG("%s(): tree build count %d", __func__, n);
+	ok = true;
+	for (i = 0; i < n; i++) {
+		item = k_unlink_head(marks_free);
+		DATA_MARKS(row, item);
+
+		if (everyone_die) {
+			ok = false;
+			break;
+		}
+
+		PQ_GET_FLD(res, i, "poolinstance", field, ok);
+		if (!ok)
+			break;
+		TXT_TO_PTR("poolinstance", field, row->poolinstance);
+
+		PQ_GET_FLD(res, i, "workinfoid", field, ok);
+		if (!ok)
+			break;
+		TXT_TO_BIGINT("workinfoid", field, row->workinfoid);
+
+		PQ_GET_FLD(res, i, "description", field, ok);
+		if (!ok)
+			break;
+		TXT_TO_PTR("description", field, row->description);
+
+		PQ_GET_FLD(res, i, "marktype", field, ok);
+		if (!ok)
+			break;
+		TXT_TO_STR("marktype", field, row->marktype);
+
+		PQ_GET_FLD(res, i, "status", field, ok);
+		if (!ok)
+			break;
+		TXT_TO_STR("status", field, row->status);
+
+		HISTORYDATEFLDS(res, i, row, ok);
+		if (!ok)
+			break;
+
+		marks_root = add_to_ktree(marks_root, item, cmp_marks);
+		k_add_head(marks_store, item);
+
+		tick();
+	}
+	if (!ok)
+		k_add_head(marks_free, item);
+
+	PQclear(res);
+
+	if (ok) {
+		LOGDEBUG("%s(): built", __func__);
+		LOGWARNING("%s(): loaded %d marks records", __func__, n);
+	}
+
+	return ok;
+}
+
 bool check_db_version(PGconn *conn)
 {
 	ExecStatusType rescode;

From a4fb2ca9278c4cdc399ad3e273e5163578cbdcdf Mon Sep 17 00:00:00 2001
From: kanoi <gitkanoi@k1k2.com>
Date: Fri, 21 Nov 2014 10:34:42 +1100
Subject: [PATCH 20/21] ckdb - allow the -w option to only affect
 sharesummaries

---
 src/ckdb.c      | 14 +++++++++++++-
 src/ckdb.h      |  4 +++-
 src/ckdb_dbio.c |  5 ++++-
 3 files changed, 20 insertions(+), 3 deletions(-)

diff --git a/src/ckdb.c b/src/ckdb.c
index 07e8b056..c0305a38 100644
--- a/src/ckdb.c
+++ b/src/ckdb.c
@@ -272,6 +272,8 @@ static tv_t reload_timestamp;
  * workinfo and sharesummary */
 int64_t dbload_workinfoid_start = -1;
 int64_t dbload_workinfoid_finish = MAXID;
+// Only restrict sharesummary, not workinfo
+bool dbload_only_sharesummary = false;
 
 // DB users,workers,auth load is complete
 bool db_auths_complete = false;
@@ -1235,6 +1237,8 @@ static bool setup_data()
 	if (dbload_workinfoid_start != -1) {
 		LOGWARNING("WARNING: dbload starting at workinfoid %"PRId64,
 			   dbload_workinfoid_start);
+		if (dbload_only_sharesummary)
+			LOGWARNING("NOTICE: dbload only restricting sharesummary");
 	}
 
 	if (!getdata3() || everyone_die)
@@ -3567,7 +3571,15 @@ int main(int argc, char **argv)
 			case 'w':
 				// Don't use this :)
 				{
-					int64_t start = atoll(optarg);
+					char *ptr = optarg;
+					int64_t start;
+
+					if (*ptr == 's') {
+						dbload_only_sharesummary = true;
+						ptr++;
+					}
+
+					start = atoll(ptr);
 					if (start < 0) {
 						quit(1, "Invalid workinfoid start"
 						     " %"PRId64" - must be >= 0",
diff --git a/src/ckdb.h b/src/ckdb.h
index b25150c2..ef97eb84 100644
--- a/src/ckdb.h
+++ b/src/ckdb.h
@@ -52,7 +52,7 @@
 
 #define DB_VLOCK "1"
 #define DB_VERSION "0.9.5"
-#define CKDB_VERSION DB_VERSION"-0.650"
+#define CKDB_VERSION DB_VERSION"-0.651"
 
 #define WHERE_FFL " - from %s %s() line %d"
 #define WHERE_FFL_HERE __FILE__, __func__, __LINE__
@@ -240,6 +240,8 @@ extern int64_t confirm_last_workinfoid;
  * workinfo and sharesummary */
 extern int64_t dbload_workinfoid_start;
 extern int64_t dbload_workinfoid_finish;
+// Only restrict sharesummary, not workinfo
+extern bool dbload_only_sharesummary;
 
 // DB users,workers,auth load is complete
 extern bool db_auths_complete;
diff --git a/src/ckdb_dbio.c b/src/ckdb_dbio.c
index c5a23fda..7d5bddd7 100644
--- a/src/ckdb_dbio.c
+++ b/src/ckdb_dbio.c
@@ -2274,7 +2274,10 @@ bool workinfo_fill(PGconn *conn)
 		"  workinfoid in (select workinfoid from blocks) )";
 	par = 0;
 	params[par++] = tv_to_buf((tv_t *)(&default_expiry), NULL, 0);
-	params[par++] = bigint_to_buf(dbload_workinfoid_start, NULL, 0);
+	if (dbload_only_sharesummary)
+		params[par++] = bigint_to_buf(-1, NULL, 0);
+	else
+		params[par++] = bigint_to_buf(dbload_workinfoid_start, NULL, 0);
 	params[par++] = bigint_to_buf(dbload_workinfoid_finish, NULL, 0);
 	PARCHK(par, params);
 	res = PQexecParams(conn, sel, par, NULL, (const char **)params, NULL, NULL, 0, CKPQ_READ);

From 0c9ac579e5c812a3b413377fac8179c6b7f52019 Mon Sep 17 00:00:00 2001
From: kanoi <gitkanoi@k1k2.com>
Date: Fri, 21 Nov 2014 10:37:48 +1100
Subject: [PATCH 21/21] ckdb - reduce transfer cull to compensate for the alloc
 increase

---
 src/ckdb.h | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/ckdb.h b/src/ckdb.h
index ef97eb84..47c1b9cc 100644
--- a/src/ckdb.h
+++ b/src/ckdb.h
@@ -52,7 +52,7 @@
 
 #define DB_VLOCK "1"
 #define DB_VERSION "0.9.5"
-#define CKDB_VERSION DB_VERSION"-0.651"
+#define CKDB_VERSION DB_VERSION"-0.652"
 
 #define WHERE_FFL " - from %s %s() line %d"
 #define WHERE_FFL_HERE __FILE__, __func__, __LINE__
@@ -624,7 +624,7 @@ typedef struct transfer {
 // Suggest malloc use MMAP - 1913 = largest under 2MB
 #define ALLOC_TRANSFER 1913
 #define LIMIT_TRANSFER 0
-#define CULL_TRANSFER 1024
+#define CULL_TRANSFER 16
 #define INIT_TRANSFER(_item) INIT_GENERIC(_item, transfer)
 #define DATA_TRANSFER(_var, _item) DATA_GENERIC(_var, _item, transfer, true)