From eaf0bb4499a1aba6f4865c23f8db3de01be02445 Mon Sep 17 00:00:00 2001 From: Con Kolivas Date: Mon, 27 Nov 2017 11:15:35 +1100 Subject: [PATCH] Back off increasingly more on failing to connect to proxies instead of ever disabling them and add status to the proxy stats. Set status only on the parent proxy. --- src/generator.c | 61 +++++++++++++++++++++++++++++++++---------------- 1 file changed, 41 insertions(+), 20 deletions(-) diff --git a/src/generator.c b/src/generator.c index d72dd448..56853f12 100644 --- a/src/generator.c +++ b/src/generator.c @@ -77,7 +77,7 @@ struct pass_msg { typedef struct pass_msg pass_msg_t; typedef struct cs_msg cs_msg_t; -/* States of various proxy statuses - connect, subscribe and auth */ +/* Statuses of various proxy states - connect, subscribe and auth */ enum proxy_stat { STATUS_INIT = 0, STATUS_SUCCESS, @@ -155,6 +155,9 @@ struct proxy_instance { enum proxy_stat subscribe_status; enum proxy_stat auth_status; + /* Back off from retrying if we fail one of the above */ + int backoff; + /* Are we in the middle of a blocked write of this message? */ cs_msg_t *sending; @@ -1461,8 +1464,13 @@ static bool auth_stratum(ckpool_t *ckp, connsock_t *cs, proxy_instance_t *proxi) if (res_val) { ret = json_is_true(res_val); if (!ret) { - LOGWARNING("Proxy %d:%d %s failed to authorise in auth_stratum, got: %s", - proxi->id, proxi->subid, proxi->url, buf); + if (proxi->global) { + LOGWARNING("Proxy %d:%d %s failed to authorise in auth_stratum, got: %s", + proxi->id, proxi->subid, proxi->url, buf); + } else { + LOGNOTICE("Proxy %d:%d %s failed to authorise in auth_stratum, got: %s", + proxi->id, proxi->subid, proxi->url, buf); + } goto out; } } else { @@ -1482,11 +1490,6 @@ out: break; parse_method(ckp, proxi, buf); }; - } else if (!proxi->global) { - LOGNOTICE("Disabling userproxy %d:%d %s that failed authorisation as %s", - proxi->id, proxi->subid, proxi->url, proxi->auth); - proxi->disabled = true; - disable_subproxy(ckp->gdata, proxi->parent, proxi); } /* Jump here if we failed to connect properly and didn't even get to * try authorising. */ @@ -2035,9 +2038,19 @@ static void suggest_diff(ckpool_t *ckp, connsock_t *cs, proxy_instance_t *proxy) } +/* Upon failing connnect, subscribe, or auth, back off on the next attempt. + * This function should be called on the parent proxy */ +static void proxy_backoff(proxy_instance_t *proxy) +{ + /* Add 5 seconds with each backoff, up to maximum of 1 minute */ + if (proxy->backoff < 60) + proxy->backoff += 5; +} + static bool proxy_alive(ckpool_t *ckp, proxy_instance_t *proxi, connsock_t *cs, bool pinging) { + proxy_instance_t *parent = proxi->parent; bool ret = false; /* Has this proxy already been reconnected? */ @@ -2062,10 +2075,11 @@ static bool proxy_alive(ckpool_t *ckp, proxy_instance_t *proxi, connsock_t *cs, LOGINFO("Failed to connect to %s:%s in proxy_mode!", cs->url, cs->port); } - proxi->connect_status = STATUS_FAIL; + parent->connect_status = STATUS_FAIL; + proxy_backoff(parent); goto out; } - proxi->connect_status = STATUS_SUCCESS; + parent->connect_status = STATUS_SUCCESS; if (ckp->node) { if (!node_stratum(cs, proxi)) { @@ -2091,10 +2105,11 @@ static bool proxy_alive(ckpool_t *ckp, proxy_instance_t *proxi, connsock_t *cs, LOGWARNING("Failed initial subscribe to %s:%s !", cs->url, cs->port); } - proxi->subscribe_status = STATUS_FAIL; + parent->subscribe_status = STATUS_FAIL; + proxy_backoff(parent); goto out; } - proxi->subscribe_status = STATUS_SUCCESS; + parent->subscribe_status = STATUS_SUCCESS; if (!ckp->passthrough) send_subscribe(ckp, proxi); @@ -2103,11 +2118,13 @@ static bool proxy_alive(ckpool_t *ckp, proxy_instance_t *proxi, connsock_t *cs, LOGWARNING("Failed initial authorise to %s:%s with %s:%s !", cs->url, cs->port, proxi->auth, proxi->pass); } - proxi->auth_status = STATUS_FAIL; + parent->auth_status = STATUS_FAIL; + proxy_backoff(parent); goto out; } - proxi->auth_status = STATUS_SUCCESS; + parent->auth_status = STATUS_SUCCESS; proxi->authorised = ret = true; + parent->backoff = 0; if (ckp->mindiff > 1) suggest_diff(ckp, cs, proxi); out: @@ -2146,6 +2163,10 @@ static void *proxy_recruit(void *arg) pthread_detach(pthread_self()); + /* We do this in a separate thread so it's okay to sleep here */ + if (parent->backoff) + sleep(parent->backoff); + retry: recruit = false; proxy = create_subproxy(ckp, gdata, parent, parent->url, parent->baseurl); @@ -2208,10 +2229,9 @@ static void *proxy_reconnect(void *arg) ckpool_t *ckp = proxy->ckp; pthread_detach(pthread_self()); + if (proxy->parent->backoff) + sleep(proxy->parent->backoff); proxy_alive(ckp, proxy, cs, true); - sleep(3); - /* Delay resetting this flag to throttle how frequently we can create - * this thread and check proxy_alive */ proxy->reconnecting = false; return NULL; } @@ -3014,9 +3034,10 @@ static json_t *__proxystats(proxy_instance_t *proxy, proxy_instance_t *parent, b json_set_double(val, "accepted", proxy->diff_accepted); json_set_double(val, "rejected", proxy->diff_rejected); } - json_set_string(val, "connect", proxy_status[proxy->connect_status]); - json_set_string(val, "subscribe", proxy_status[proxy->subscribe_status]); - json_set_string(val, "authorise", proxy_status[proxy->auth_status]); + json_set_string(val, "connect", proxy_status[parent->connect_status]); + json_set_string(val, "subscribe", proxy_status[parent->subscribe_status]); + json_set_string(val, "authorise", proxy_status[parent->auth_status]); + json_set_int(val, "backoff", parent->backoff); json_set_int(val, "lastshare", proxy->last_share.tv_sec); json_set_bool(val, "global", proxy->global); json_set_bool(val, "disabled", proxy->disabled);