ckdb - set confirmer to only load data near the range required

src/ckdb.c

@@ -257,6 +257,11 @@ static tv_t confirm_finish;
 
 static tv_t reload_timestamp;
 
+/* Allow overriding the workinfoid range used in the db load of
+ * workinfo and sharesummary */
+int64_t dbload_workinfoid_start = -1;
+int64_t dbload_workinfoid_finish = MAXID;
+
 // DB users,workers,auth load is complete
 bool db_auths_complete = false;
 // DB load is complete
@@ -678,13 +683,24 @@ matane:
 	return ok;
 }
 
+/* Load blocks first to allow data range settings to know
+ * the blocks info for setting limits for tables in getdata3()
+ */
 static bool getdata2()
+{
+	PGconn *conn = dbconnect();
+	bool ok = blocks_fill(conn);
+
+	PQfinish(conn);
+
+	return ok;
+}
+
+static bool getdata3()
 {
 	PGconn *conn = dbconnect();
 	bool ok = true;
 
-	if (!(ok = blocks_fill(conn)) || everyone_die)
-		goto sukamudai;
 	if (!confirm_sharesummary) {
 		if (!(ok = paymentaddresses_fill(conn)) || everyone_die)
 			goto sukamudai;
@@ -1001,6 +1017,9 @@ static bool setup_data()
 	if (!getdata2() || everyone_die)
 		return false;
 
+	if (!getdata3() || everyone_die)
+		return false;
+
 	db_load_complete = true;
 
 	if (!reload() || everyone_die)
@@ -2834,8 +2853,17 @@ static void confirm_reload()
 				   __func__, workinfo->workinfoid);
 		}
 	} else {
-		start.tv_sec = start.tv_usec = 0;
+		if (confirm_first_workinfoid == 0) {
-		LOGWARNING("%s() no start workinfo found ... using time 0", __func__);
+			start.tv_sec = start.tv_usec = 0;
+			LOGWARNING("%s() no start workinfo found ... "
+				   "using time 0", __func__);
+		} else {
+			// Abort, otherwise reload will reload all log files
+			LOGERR("%s(): Start workinfoid doesn't exist, "
+			       "use 0 to mean from the beginning of time",
+			       __func__);
+			return;
+		}
 	}
 
 	/* Find the workinfo after confirm_last_workinfoid-1
@@ -2995,6 +3023,8 @@ static void confirm_summaries()
 					return;
 				}
 				free(range);
+				dbload_workinfoid_start = confirm_range_start - 1;
+				dbload_workinfoid_finish = confirm_range_finish + 1;
 				break;
 			case 'w':
 				confirm_range_start = atoll(confirm_range+1);
@@ -3026,6 +3056,11 @@ static void confirm_summaries()
 		return;
 	}
 
+	if (!getdata3()) {
+		LOGEMERG("%s() ABORTING from getdata3()", __func__);
+		return;
+	}
+
 	confirm_reload();
 }
 

src/ckdb.h

@@ -52,7 +52,7 @@
 
 #define DB_VLOCK "1"
 #define DB_VERSION "0.9.2"
-#define CKDB_VERSION DB_VERSION"-0.572"
+#define CKDB_VERSION DB_VERSION"-0.580"
 
 #define WHERE_FFL " - from %s %s() line %d"
 #define WHERE_FFL_HERE __FILE__, __func__, __LINE__
@@ -225,6 +225,11 @@ extern int64_t confirm_last_workinfoid;
  * ckpool uses 10min - but add 1min to be sure */
 #define WORKINFO_AGE 660
 
+/* Allow defining the workinfoid range used in the db load of
+ * workinfo and sharesummary */
+extern int64_t dbload_workinfoid_start;
+extern int64_t dbload_workinfoid_finish;
+
 // DB users,workers,auth load is complete
 extern bool db_auths_complete;
 // DB load is complete

src/ckdb_dbio.c

@@ -2240,7 +2240,7 @@ bool workinfo_fill(PGconn *conn)
 	PGresult *res;
 	K_ITEM *item;
 	WORKINFO *row;
-	char *params[1];
+	char *params[3];
 	int n, i, par = 0;
 	char *field;
 	char *sel;
@@ -2257,8 +2257,10 @@ bool workinfo_fill(PGconn *conn)
 		"workinfoid,poolinstance,merklehash,prevhash,"
 		"coinbase1,coinbase2,version,bits,ntime,reward"
 		HISTORYDATECONTROL
-		" from workinfo where expirydate=$1";
+		" from workinfo where workinfoid>=$1 and workinfoid<=$2 and expirydate=$3";
 	par = 0;
+	params[par++] = bigint_to_buf(dbload_workinfoid_start, NULL, 0);
+	params[par++] = bigint_to_buf(dbload_workinfoid_finish, NULL, 0);
 	params[par++] = tv_to_buf((tv_t *)(&default_expiry), NULL, 0);
 	PARCHK(par, params);
 	res = PQexecParams(conn, sel, par, NULL, (const char **)params, NULL, NULL, 0, CKPQ_READ);
@@ -2915,8 +2917,9 @@ bool sharesummary_fill(PGconn *conn)
 	ExecStatusType rescode;
 	PGresult *res;
 	K_ITEM *item;
-	int n, i;
+	int n, i, par = 0;
 	SHARESUMMARY *row;
+	char *params[2];
 	char *field;
 	char *sel;
 	int fields = 19;
@@ -2931,8 +2934,12 @@ bool sharesummary_fill(PGconn *conn)
 		"sharecount,errorcount,firstshare,lastshare,"
 		"lastdiffacc,complete"
 		MODIFYDATECONTROL
-		" from sharesummary";
+		" from sharesummary where workinfoid>=$1 and workinfoid<=$2";
-	res = PQexec(conn, sel, CKPQ_READ);
+	par = 0;
+	params[par++] = bigint_to_buf(dbload_workinfoid_start, NULL, 0);
+	params[par++] = bigint_to_buf(dbload_workinfoid_finish, NULL, 0);
+	PARCHK(par, params);
+	res = PQexecParams(conn, sel, par, NULL, (const char **)params, NULL, NULL, 0, CKPQ_READ);
 	rescode = PQresultStatus(res);
 	if (!PGOK(rescode)) {
 		PGLOGERR("Select", rescode, conn);