Implement fallback to higher priority server by using a ckmsgq to inform a server watchdog thread of what the current server is and testing if they're alive no more than every 5 seconds

src/generator.c

@@ -122,6 +122,8 @@ struct proxy_instance {
 
 typedef struct proxy_instance proxy_instance_t;
 
+static ckmsgq_t *srvchk;	// Server check message queue
+
 static bool server_alive(ckpool_t *ckp, server_instance_t *si, bool pinging)
 {
 	char *userpass = NULL;
@@ -252,6 +254,8 @@ reconnect:
 	}
 
 retry:
+	ckmsgq_add(srvchk, si);
+
 	do {
 		selret = wait_read_select(us->sockd, 5);
 		if (!selret && !ping_main(ckp)) {
@@ -344,6 +348,8 @@ retry:
 			send_unix_msg(sockd, "true");
 		else
 			send_unix_msg(sockd, "false");
+	} else if (cmdmatch(buf, "fallback")) {
+		goto reconnect;
 	} else if (cmdmatch(buf, "loglevel")) {
 		sscanf(buf, "loglevel=%d", &ckp->loglevel);
 	} else if (cmdmatch(buf, "ping")) {
@@ -1464,6 +1470,8 @@ retry:
 				LOGWARNING("Block rejected locally.");
 		} else
 			LOGNOTICE("No backup btcd to send block to ourselves");
+	} else if (cmdmatch(buf, "fallback")) {
+		goto reconnect;
 	} else if (cmdmatch(buf, "loglevel")) {
 		sscanf(buf, "loglevel=%d", &ckp->loglevel);
 	} else if (cmdmatch(buf, "ping")) {
@@ -1617,6 +1625,51 @@ static int proxy_mode(ckpool_t *ckp, proc_instance_t *pi)
 	return ret;
 }
 
+/* Tell the watchdog what the current server instance is and decide if we
+ * should check to see if the higher priority servers are alive and fallback */
+static void server_watchdog(ckpool_t *ckp, server_instance_t *cursi)
+{
+	static time_t last_t = 0;
+	bool alive = false;
+	time_t now_t;
+	int i, srvs;
+
+	/* Rate limit to checking only once every 5 seconds */
+	now_t = time(NULL);
+	if (now_t <= last_t + 5)
+		return;
+
+	last_t = now_t;
+
+	/* Is this the highest priority server already? */
+	if (cursi == ckp->servers[0])
+		return;
+
+	if (ckp->proxy)
+		srvs = ckp->proxies;
+	else
+		srvs = ckp->btcds;
+	for (i = 0; i < srvs; i++) {
+		server_instance_t *si = ckp->servers[i];
+
+		/* Have we reached the current server? */
+		if (si == cursi)
+			return;
+
+		if (ckp->proxy) {
+			proxy_instance_t *proxi = si->data;
+			connsock_t *cs = proxi->cs;
+
+			alive = proxy_alive(ckp, si, proxi, cs, true);
+		} else
+			alive = server_alive(ckp, si, true);
+		if (alive)
+			break;
+	}
+	if (alive)
+		send_proc(ckp->generator, "fallback");
+}
+
 int generator(proc_instance_t *pi)
 {
 	ckpool_t *ckp = pi->ckp;
@@ -1624,6 +1677,8 @@ int generator(proc_instance_t *pi)
 
 	LOGWARNING("%s generator starting", ckp->name);
 
+	srvchk = create_ckmsgq(ckp, "srvchk", &server_watchdog);
+
 	if (ckp->proxy)
 		ret = proxy_mode(ckp, pi);
 	else