Browse Source

ckdb - don't store auths in the db

master
kanoi 10 years ago
parent
commit
2c35501d6d
  1. 37
      src/ckdb.c
  2. 10
      src/ckdb.h
  3. 32
      src/ckdb_cmd.c
  4. 197
      src/ckdb_dbio.c

37
src/ckdb.c

@ -31,18 +31,11 @@
* with an ok.queued reply to ckpool, to be processed after the reload * with an ok.queued reply to ckpool, to be processed after the reload
* completes and just process authorise messages immediately while the * completes and just process authorise messages immediately while the
* reload runs * reload runs
* This can't cause a duplicate process of an authorise message since a * We start the ckpool message queue after loading
* reload will ignore any messages before the last DB auths message, * the users, idcontrol and workers DB tables, before loading the
* however, if ckdb and ckpool get out of sync due to ckpool starting
* during the reload (as mentioned below) it is possible for ckdb to
* find an authorise message in the CCLs that was processed in the
* message queue and thus is already in the DB.
* This error would be very rare and also not an issue
* To avoid this, we start the ckpool message queue after loading
* the users, auths, idcontrol and workers DB tables, before loading the
* much larger DB tables so that ckdb is effectively ready for messages * much larger DB tables so that ckdb is effectively ready for messages
* almost immediately * almost immediately
* The first ckpool message also allows us to know where ckpool is up to * The first ckpool message allows us to know where ckpool is up to
* in the CCLs and thus where to stop processing the CCLs to stay in * in the CCLs and thus where to stop processing the CCLs to stay in
* sync with ckpool * sync with ckpool
* If ckpool isn't running, then the reload will complete at the end of * If ckpool isn't running, then the reload will complete at the end of
@ -89,13 +82,14 @@
* TODO: Verify that all DB sharesummaries with complete='n' * TODO: Verify that all DB sharesummaries with complete='n'
* have done this * have done this
* DB+RAM workinfo: start from newest DB createdate workinfo * DB+RAM workinfo: start from newest DB createdate workinfo
* DB+RAM auths: start from newest DB createdate auths * RAM auths: none (we store them in RAM only)
* DB+RAM poolstats: newest createdate poolstats * DB+RAM poolstats: newest createdate poolstats
* TODO: subtract how much we need in RAM of the 'between' * TODO: subtract how much we need in RAM of the 'between'
* non db records - will depend on TODO: pool stats reporting * non db records - will depend on TODO: pool stats reporting
* requirements * requirements
* RAM userstats: none (we simply store the last one found) * RAM userstats: none (we simply store the last one found)
* DB+RAM workers: created by auths so auths will resolve it * DB+RAM workers: created by auths so are simply ignore if they
* already exist
* DB+RAM blocks: resolved by workinfo - any unsaved blocks (if any) * DB+RAM blocks: resolved by workinfo - any unsaved blocks (if any)
* will be after the last DB workinfo * will be after the last DB workinfo
* DB+RAM accountbalance (TODO): resolved by shares/workinfo/blocks * DB+RAM accountbalance (TODO): resolved by shares/workinfo/blocks
@ -269,8 +263,8 @@ bool dbload_only_sharesummary = false;
* markersummaries and pplns payouts may not be correct */ * markersummaries and pplns payouts may not be correct */
bool sharesummary_marks_limit = false; bool sharesummary_marks_limit = false;
// DB users,workers,auth load is complete // DB users,workers load is complete
bool db_auths_complete = false; bool db_users_complete = false;
// DB load is complete // DB load is complete
bool db_load_complete = false; bool db_load_complete = false;
// Different input data handling // Different input data handling
@ -717,10 +711,7 @@ static bool getdata1()
goto matane; goto matane;
if (!(ok = users_fill(conn))) if (!(ok = users_fill(conn)))
goto matane; goto matane;
if (!(ok = workers_fill(conn))) ok = workers_fill(conn);
goto matane;
if (!confirm_sharesummary)
ok = auths_fill(conn);
matane: matane:
@ -792,8 +783,6 @@ static bool reload()
LOGWARNING("%s(): %s newest DB complete sharesummary", __func__, buf); LOGWARNING("%s(): %s newest DB complete sharesummary", __func__, buf);
tv_to_buf(&(dbstatus.newest_createdate_workinfo), buf, sizeof(buf)); tv_to_buf(&(dbstatus.newest_createdate_workinfo), buf, sizeof(buf));
LOGWARNING("%s(): %s newest DB workinfo", __func__, buf); LOGWARNING("%s(): %s newest DB workinfo", __func__, buf);
tv_to_buf(&(dbstatus.newest_createdate_auths), buf, sizeof(buf));
LOGWARNING("%s(): %s newest DB auths", __func__, buf);
tv_to_buf(&(dbstatus.newest_createdate_poolstats), buf, sizeof(buf)); tv_to_buf(&(dbstatus.newest_createdate_poolstats), buf, sizeof(buf));
LOGWARNING("%s(): %s newest DB poolstats", __func__, buf); LOGWARNING("%s(): %s newest DB poolstats", __func__, buf);
tv_to_buf(&(dbstatus.newest_createdate_blocks), buf, sizeof(buf)); tv_to_buf(&(dbstatus.newest_createdate_blocks), buf, sizeof(buf));
@ -810,10 +799,6 @@ static bool reload()
copy_tv(&start, &(dbstatus.newest_createdate_workinfo)); copy_tv(&start, &(dbstatus.newest_createdate_workinfo));
reason = "workinfo"; reason = "workinfo";
} }
if (!tv_newer(&start, &(dbstatus.newest_createdate_auths))) {
copy_tv(&start, &(dbstatus.newest_createdate_auths));
reason = "auths";
}
if (!tv_newer(&start, &(dbstatus.newest_createdate_poolstats))) { if (!tv_newer(&start, &(dbstatus.newest_createdate_poolstats))) {
copy_tv(&start, &(dbstatus.newest_createdate_poolstats)); copy_tv(&start, &(dbstatus.newest_createdate_poolstats));
reason = "poolstats"; reason = "poolstats";
@ -1184,7 +1169,7 @@ static bool setup_data()
if (!getdata1() || everyone_die) if (!getdata1() || everyone_die)
return false; return false;
db_auths_complete = true; db_users_complete = true;
cksem_post(&socketer_sem); cksem_post(&socketer_sem);
if (!getdata2() || everyone_die) if (!getdata2() || everyone_die)
@ -2383,7 +2368,7 @@ static void *socketer(__maybe_unused void *arg)
rename_proc("db_socketer"); rename_proc("db_socketer");
while (!everyone_die && !db_auths_complete) while (!everyone_die && !db_users_complete)
cksem_mswait(&socketer_sem, 420); cksem_mswait(&socketer_sem, 420);
socketer_using_data = true; socketer_using_data = true;

10
src/ckdb.h

@ -52,7 +52,7 @@
#define DB_VLOCK "1" #define DB_VLOCK "1"
#define DB_VERSION "0.9.6" #define DB_VERSION "0.9.6"
#define CKDB_VERSION DB_VERSION"-0.910" #define CKDB_VERSION DB_VERSION"-0.920"
#define WHERE_FFL " - from %s %s() line %d" #define WHERE_FFL " - from %s %s() line %d"
#define WHERE_FFL_HERE __FILE__, __func__, __LINE__ #define WHERE_FFL_HERE __FILE__, __func__, __LINE__
@ -103,7 +103,6 @@ typedef struct loadstatus {
tv_t oldest_sharesummary_firstshare_a; tv_t oldest_sharesummary_firstshare_a;
tv_t newest_sharesummary_firstshare_y; tv_t newest_sharesummary_firstshare_y;
tv_t newest_createdate_workinfo; tv_t newest_createdate_workinfo;
tv_t newest_createdate_auths;
tv_t newest_createdate_poolstats; tv_t newest_createdate_poolstats;
tv_t newest_starttimeband_userstats; tv_t newest_starttimeband_userstats;
tv_t newest_createdate_blocks; tv_t newest_createdate_blocks;
@ -256,8 +255,8 @@ extern bool dbload_only_sharesummary;
* markersummaries and pplns payouts may not be correct */ * markersummaries and pplns payouts may not be correct */
extern bool sharesummary_marks_limit; extern bool sharesummary_marks_limit;
// DB users,workers,auth load is complete // DB users,workers load is complete
extern bool db_auths_complete; extern bool db_users_complete;
// DB load is complete // DB load is complete
extern bool db_load_complete; extern bool db_load_complete;
// Different input data handling // Different input data handling
@ -1835,9 +1834,8 @@ extern bool miningpayouts_fill(PGconn *conn);
extern bool auths_add(PGconn *conn, char *poolinstance, char *username, extern bool auths_add(PGconn *conn, char *poolinstance, char *username,
char *workername, char *clientid, char *enonce1, char *workername, char *clientid, char *enonce1,
char *useragent, char *preauth, char *by, char *code, char *useragent, char *preauth, char *by, char *code,
char *inet, tv_t *cd, bool igndup, K_TREE *trf_root, char *inet, tv_t *cd, K_TREE *trf_root,
bool addressuser, USERS **users, WORKERS **workers); bool addressuser, USERS **users, WORKERS **workers);
extern bool auths_fill(PGconn *conn);
extern bool poolstats_add(PGconn *conn, bool store, char *poolinstance, extern bool poolstats_add(PGconn *conn, bool store, char *poolinstance,
char *elapsed, char *users, char *workers, char *elapsed, char *users, char *workers,
char *hashrate, char *hashrate5m, char *hashrate, char *hashrate5m,

32
src/ckdb_cmd.c

@ -2174,7 +2174,7 @@ static char *cmd_blocks(PGconn *conn, char *cmd, char *id,
} }
static char *cmd_auth_do(PGconn *conn, char *cmd, char *id, char *by, static char *cmd_auth_do(PGconn *conn, char *cmd, char *id, char *by,
char *code, char *inet, tv_t *cd, bool igndup, char *code, char *inet, tv_t *cd,
K_TREE *trf_root) K_TREE *trf_root)
{ {
char reply[1024] = ""; char reply[1024] = "";
@ -2241,7 +2241,7 @@ static char *cmd_auth_do(PGconn *conn, char *cmd, char *id, char *by,
transfer_data(i_enonce1), transfer_data(i_enonce1),
transfer_data(i_useragent), transfer_data(i_useragent),
transfer_data(i_preauth), transfer_data(i_preauth),
by, code, inet, cd, igndup, trf_root, false, by, code, inet, cd, trf_root, false,
&users, &workers); &users, &workers);
if (!ok) { if (!ok) {
@ -2266,21 +2266,11 @@ static char *cmd_auth(PGconn *conn, char *cmd, char *id,
char *code, char *inet, tv_t *cd, char *code, char *inet, tv_t *cd,
K_TREE *trf_root) K_TREE *trf_root)
{ {
bool igndup = false; return cmd_auth_do(conn, cmd, id, by, code, inet, cd, trf_root);
// confirm_summaries() doesn't call this
if (reloading) {
if (tv_equal(cd, &(dbstatus.newest_createdate_auths)))
igndup = true;
else if (tv_newer(cd, &(dbstatus.newest_createdate_auths)))
return NULL;
}
return cmd_auth_do(conn, cmd, id, by, code, inet, cd, igndup, trf_root);
} }
static char *cmd_addrauth_do(PGconn *conn, char *cmd, char *id, char *by, static char *cmd_addrauth_do(PGconn *conn, char *cmd, char *id, char *by,
char *code, char *inet, tv_t *cd, bool igndup, char *code, char *inet, tv_t *cd,
K_TREE *trf_root) K_TREE *trf_root)
{ {
char reply[1024] = ""; char reply[1024] = "";
@ -2329,7 +2319,7 @@ static char *cmd_addrauth_do(PGconn *conn, char *cmd, char *id, char *by,
transfer_data(i_enonce1), transfer_data(i_enonce1),
transfer_data(i_useragent), transfer_data(i_useragent),
transfer_data(i_preauth), transfer_data(i_preauth),
by, code, inet, cd, igndup, trf_root, true, by, code, inet, cd, trf_root, true,
&users, &workers); &users, &workers);
if (!ok) { if (!ok) {
@ -2354,17 +2344,7 @@ static char *cmd_addrauth(PGconn *conn, char *cmd, char *id,
char *code, char *inet, tv_t *cd, char *code, char *inet, tv_t *cd,
K_TREE *trf_root) K_TREE *trf_root)
{ {
bool igndup = false; return cmd_addrauth_do(conn, cmd, id, by, code, inet, cd, trf_root);
// confirm_summaries() doesn't call this
if (reloading) {
if (tv_equal(cd, &(dbstatus.newest_createdate_auths)))
igndup = true;
else if (tv_newer(cd, &(dbstatus.newest_createdate_auths)))
return NULL;
}
return cmd_addrauth_do(conn, cmd, id, by, code, inet, cd, igndup, trf_root);
} }
static char *cmd_heartbeat(__maybe_unused PGconn *conn, char *cmd, char *id, static char *cmd_heartbeat(__maybe_unused PGconn *conn, char *cmd, char *id,

197
src/ckdb_dbio.c

@ -4490,22 +4490,17 @@ bool miningpayouts_fill(PGconn *conn)
return ok; return ok;
} }
// TODO: discard them from RAM
bool auths_add(PGconn *conn, char *poolinstance, char *username, bool auths_add(PGconn *conn, char *poolinstance, char *username,
char *workername, char *clientid, char *enonce1, char *workername, char *clientid, char *enonce1,
char *useragent, char *preauth, char *by, char *code, char *useragent, char *preauth, char *by, char *code,
char *inet, tv_t *cd, bool igndup, K_TREE *trf_root, char *inet, tv_t *cd, K_TREE *trf_root,
bool addressuser, USERS **users, WORKERS **workers) bool addressuser, USERS **users, WORKERS **workers)
{ {
ExecStatusType rescode;
bool conned = false;
PGresult *res;
K_TREE_CTX ctx[1]; K_TREE_CTX ctx[1];
K_ITEM *a_item, *u_item, *w_item; K_ITEM *a_item, *u_item, *w_item;
char cd_buf[DATE_BUFSIZ]; char cd_buf[DATE_BUFSIZ];
AUTHS *row; AUTHS *row;
char *ins;
char *params[8 + HISTORYDATECOUNT];
int n, par = 0;
bool ok = false; bool ok = false;
LOGDEBUG("%s(): add", __func__); LOGDEBUG("%s(): add", __func__);
@ -4521,10 +4516,6 @@ bool auths_add(PGconn *conn, char *poolinstance, char *username,
K_RUNLOCK(users_free); K_RUNLOCK(users_free);
if (!u_item) { if (!u_item) {
if (addressuser) { if (addressuser) {
if (conn == NULL) {
conn = dbconnect();
conned = true;
}
u_item = users_add(conn, username, EMPTY, EMPTY, u_item = users_add(conn, username, EMPTY, EMPTY,
by, code, inet, cd, trf_root); by, code, inet, cd, trf_root);
} else { } else {
@ -4569,14 +4560,10 @@ bool auths_add(PGconn *conn, char *poolinstance, char *username,
k_add_head(auths_free, a_item); k_add_head(auths_free, a_item);
K_WUNLOCK(auths_free); K_WUNLOCK(auths_free);
if (conned) // Shouldn't actually be possible unless twice in the logs
PQfinish(conn);
if (!igndup) {
tv_to_buf(cd, cd_buf, sizeof(cd_buf)); tv_to_buf(cd, cd_buf, sizeof(cd_buf));
LOGERR("%s(): Duplicate auths ignored %s/%s/%s", LOGERR("%s(): Duplicate auths ignored %s/%s/%s",
__func__, poolinstance, workername, cd_buf); __func__, poolinstance, workername, cd_buf);
}
/* Let them mine, that's what matters :) /* Let them mine, that's what matters :)
* though this would normally only be during a reload */ * though this would normally only be during a reload */
@ -4587,45 +4574,10 @@ bool auths_add(PGconn *conn, char *poolinstance, char *username,
// Update even if DB fails // Update even if DB fails
workerstatus_update(row, NULL, NULL); workerstatus_update(row, NULL, NULL);
if (conn == NULL) { row->authid = 1;
conn = dbconnect();
conned = true;
}
row->authid = nextid(conn, "authid", (int64_t)1, cd, by, code, inet);
if (row->authid == 0)
goto unitem;
par = 0;
params[par++] = bigint_to_buf(row->authid, NULL, 0);
params[par++] = str_to_buf(row->poolinstance, NULL, 0);
params[par++] = bigint_to_buf(row->userid, NULL, 0);
params[par++] = str_to_buf(row->workername, NULL, 0);
params[par++] = int_to_buf(row->clientid, NULL, 0);
params[par++] = str_to_buf(row->enonce1, NULL, 0);
params[par++] = str_to_buf(row->useragent, NULL, 0);
params[par++] = str_to_buf(row->preauth, NULL, 0);
HISTORYDATEPARAMS(params, par, row);
PARCHK(par, params);
ins = "insert into auths "
"(authid,poolinstance,userid,workername,clientid,enonce1,useragent,preauth"
HISTORYDATECONTROL ") values (" PQPARAM13 ")";
res = PQexecParams(conn, ins, par, NULL, (const char **)params, NULL, NULL, 0, CKPQ_WRITE);
rescode = PQresultStatus(res);
if (!PGOK(rescode)) {
PGLOGERR("Insert", rescode, conn);
goto unparam;
}
ok = true; ok = true;
unparam:
PQclear(res);
for (n = 0; n < par; n++)
free(params[n]);
unitem: unitem:
if (conned)
PQfinish(conn);
K_WLOCK(auths_free); K_WLOCK(auths_free);
#if 1 #if 1
/* To save ram for now, don't store them, /* To save ram for now, don't store them,
@ -4644,147 +4596,6 @@ unitem:
return ok; return ok;
} }
bool auths_fill(PGconn *conn)
{
ExecStatusType rescode;
PGresult *res;
K_ITEM *item;
AUTHS *row;
char *params[2];
int n, i;
int par = 0;
char *field;
char *sel;
int fields = 7;
bool ok;
tv_t now;
LOGDEBUG("%s(): select", __func__);
// TODO: add/update a (single) fake auth every ~10min or 10min after the last one?
sel = "select "
"authid,userid,workername,clientid,enonce1,useragent,preauth"
HISTORYDATECONTROL
" from auths where expirydate=$1 and createdate>=$2";
setnow(&now);
now.tv_sec -= (24 * 60 * 60); // last day worth
par = 0;
params[par++] = tv_to_buf((tv_t *)(&default_expiry), NULL, 0);
params[par++] = tv_to_buf((tv_t *)(&now), NULL, 0);
PARCHK(par, params);
res = PQexecParams(conn, sel, par, NULL, (const char **)params, NULL, NULL, 0, CKPQ_READ);
rescode = PQresultStatus(res);
if (!PGOK(rescode)) {
PGLOGERR("Select", rescode, conn);
PQclear(res);
return false;
}
#if 0
// Only load the last record for each workername
sel = "with last as ("
"select authid,userid,workername,clientid,enonce1,useragent,preauth"
HISTORYDATECONTROL
",row_number() over(partition by userid,workername "
"order by expirydate desc, createdate desc)"
" as best from auths"
") select * from last where best=1";
res = PQexec(conn, sel, CKPQ_READ);
rescode = PQresultStatus(res);
if (!PGOK(rescode)) {
PGLOGERR("Select", rescode, conn);
PQclear(res);
return false;
}
#endif
n = PQnfields(res);
if (n != (fields + HISTORYDATECOUNT)) {
LOGERR("%s(): Invalid field count - should be %d, but is %d",
__func__, fields + HISTORYDATECOUNT, n);
PQclear(res);
return false;
}
n = PQntuples(res);
LOGDEBUG("%s(): tree build count %d", __func__, n);
ok = true;
K_WLOCK(auths_free);
for (i = 0; i < n; i++) {
item = k_unlink_head(auths_free);
DATA_AUTHS(row, item);
if (everyone_die) {
ok = false;
break;
}
PQ_GET_FLD(res, i, "authid", field, ok);
if (!ok)
break;
TXT_TO_BIGINT("authid", field, row->authid);
PQ_GET_FLD(res, i, "userid", field, ok);
if (!ok)
break;
TXT_TO_BIGINT("userid", field, row->userid);
PQ_GET_FLD(res, i, "workername", field, ok);
if (!ok)
break;
TXT_TO_STR("workername", field, row->workername);
PQ_GET_FLD(res, i, "clientid", field, ok);
if (!ok)
break;
TXT_TO_INT("clientid", field, row->clientid);
PQ_GET_FLD(res, i, "enonce1", field, ok);
if (!ok)
break;
TXT_TO_STR("enonce1", field, row->enonce1);
PQ_GET_FLD(res, i, "useragent", field, ok);
if (!ok)
break;
TXT_TO_STR("useragent", field, row->useragent);
PQ_GET_FLD(res, i, "preauth", field, ok);
if (!ok)
break;
TXT_TO_STR("preauth", field, row->preauth);
HISTORYDATEFLDS(res, i, row, ok);
if (!ok)
break;
auths_root = add_to_ktree(auths_root, item, cmp_auths);
k_add_head(auths_store, item);
workerstatus_update(row, NULL, NULL);
if (tv_newer(&(dbstatus.newest_createdate_auths), &(row->createdate)))
copy_tv(&(dbstatus.newest_createdate_auths), &(row->createdate));
tick();
}
if (!ok)
k_add_head(auths_free, item);
K_WUNLOCK(auths_free);
PQclear(res);
if (ok) {
LOGDEBUG("%s(): built", __func__);
LOGWARNING("%s(): loaded %d auth records", __func__, n);
}
return ok;
}
bool poolstats_add(PGconn *conn, bool store, char *poolinstance, bool poolstats_add(PGconn *conn, bool store, char *poolinstance,
char *elapsed, char *users, char *workers, char *elapsed, char *users, char *workers,
char *hashrate, char *hashrate5m, char *hashrate, char *hashrate5m,

Loading…
Cancel
Save