From 00b5a6b1bef0626be85846636e08fe0ff424827f Mon Sep 17 00:00:00 2001 From: kanoi Date: Thu, 13 Oct 2016 09:44:30 +1100 Subject: [PATCH] ckdb - track idcontrol in ram --- src/ckdb.c | 14 ++-- src/ckdb.h | 11 ++- src/ckdb_cmd.c | 70 +++++++++++++---- src/ckdb_data.c | 46 +++++++++++ src/ckdb_dbio.c | 199 +++++++++++++++++++++++++++++------------------- 5 files changed, 236 insertions(+), 104 deletions(-) diff --git a/src/ckdb.c b/src/ckdb.c index b3c6cb0e..046df82f 100644 --- a/src/ckdb.c +++ b/src/ckdb.c @@ -688,8 +688,7 @@ K_LIST *accountadjustment_free; K_STORE *accountadjustment_store; // IDCONTROL -// These are only used for db access - not stored in memory -//K_TREE *idcontrol_root; +K_TREE *idcontrol_root; K_LIST *idcontrol_free; K_STORE *idcontrol_store; @@ -1711,7 +1710,6 @@ PGconn *dbconnect() } /* Load tables required to support auths,adduser,chkpass and newid - * N.B. idcontrol is DB internal so is always ready * OptionControl is loaded first in case it is needed by other loads * (though not yet) */ @@ -1726,6 +1724,8 @@ static bool getdata1() goto matane; if (!(ok = optioncontrol_fill(conn))) goto matane; + if (!(ok = idcontrol_fill(conn))) + goto matane; if (!(ok = users_fill(conn))) goto matane; if (!(ok = workers_fill(conn))) @@ -2113,6 +2113,8 @@ static void alloc_storage() idcontrol_free = k_new_list("IDControl", sizeof(IDCONTROL), ALLOC_IDCONTROL, LIMIT_IDCONTROL, true); idcontrol_store = k_new_store(idcontrol_free); + idcontrol_root = new_ktree(NULL, cmp_idcontrol, idcontrol_free); + idcontrol_free->dsp_func = dsp_idcontrol; esm_free = k_new_list("ESM", sizeof(ESM), ALLOC_ESM, LIMIT_ESM, true); esm_store = k_new_store(esm_free); @@ -2331,10 +2333,12 @@ static void alloc_storage() DLPRIO(paymentaddresses, 5); + // Must be above instransient + DLPRIO(idcontrol, 3); + // Don't currently nest any locks in these: DLPRIO(esm, PRIO_TERMINAL); DLPRIO(workers, PRIO_TERMINAL); - DLPRIO(idcontrol, PRIO_TERMINAL); DLPRIO(ips, PRIO_TERMINAL); DLPRIO(replies, PRIO_TERMINAL); @@ -2674,7 +2678,7 @@ static void dealloc_storage() esm_report(); FREE_ALL(esm); - FREE_LISTS(idcontrol); + FREE_ALL(idcontrol); FREE_ALL(accountbalance); FREE_ALL(payments); diff --git a/src/ckdb.h b/src/ckdb.h index c7a20e50..4a9b9560 100644 --- a/src/ckdb.h +++ b/src/ckdb.h @@ -58,7 +58,7 @@ #define DB_VLOCK "1" #define DB_VERSION "1.0.7" -#define CKDB_VERSION DB_VERSION"-2.702" +#define CKDB_VERSION DB_VERSION"-2.703" #define WHERE_FFL " - from %s %s() line %d" #define WHERE_FFL_HERE __FILE__, __func__, __LINE__ @@ -2062,7 +2062,7 @@ extern K_STORE *accountadjustment_store; typedef struct idcontrol { char idname[TXT_SML+1]; int64_t lastid; - MODIFYDATECONTROLFIELDS; + MODIFYDATECONTROLIN; } IDCONTROL; #define ALLOC_IDCONTROL 16 @@ -2070,8 +2070,7 @@ typedef struct idcontrol { #define INIT_IDCONTROL(_item) INIT_GENERIC(_item, idcontrol) #define DATA_IDCONTROL(_var, _item) DATA_GENERIC(_var, _item, idcontrol, true) -// These are only used for db access - not stored in memory -//extern K_TREE *idcontrol_root; +extern K_TREE *idcontrol_root; extern K_LIST *idcontrol_free; extern K_STORE *idcontrol_store; @@ -3460,6 +3459,9 @@ extern K_ITEM *find_first_payments(int64_t userid, K_TREE_CTX *ctx); extern K_ITEM *find_first_paypayid(int64_t userid, int64_t payoutid, K_TREE_CTX *ctx); extern cmp_t cmp_accountbalance(K_ITEM *a, K_ITEM *b); extern K_ITEM *find_accountbalance(int64_t userid); +extern void dsp_idcontrol(K_ITEM *item, FILE *stream); +extern cmp_t cmp_idcontrol(K_ITEM *a, K_ITEM *b); +extern K_ITEM *find_idcontrol(char *idname); extern cmp_t cmp_optioncontrol(K_ITEM *a, K_ITEM *b); extern K_ITEM *find_optioncontrol(char *optionname, const tv_t *now, int32_t height); #define sys_setting(_name, _def, _now) user_sys_setting(0, _name, _def, _now) @@ -3726,6 +3728,7 @@ extern bool payments_add(PGconn *conn, bool add, K_ITEM *p_item, extern bool payments_fill(PGconn *conn); extern bool idcontrol_add(PGconn *conn, char *idname, char *idvalue, char *by, char *code, char *inet, tv_t *cd, K_TREE *trf_root); +extern bool idcontrol_fill(PGconn *conn); extern K_ITEM *optioncontrol_item_add(PGconn *conn, K_ITEM *oc_item, tv_t *cd, bool begun); extern K_ITEM *optioncontrol_add(PGconn *conn, char *optionname, char *optionvalue, char *activationdate, char *activationheight, diff --git a/src/ckdb_cmd.c b/src/ckdb_cmd.c index 2dbd3eab..5eb52d35 100644 --- a/src/ckdb_cmd.c +++ b/src/ckdb_cmd.c @@ -5867,6 +5867,9 @@ static char *cmd_dsp(__maybe_unused PGconn *conn, __maybe_unused char *cmd, K_ITEM *i_file, *i_name, *i_type; char reply[1024] = "", *fil, *name, *typ; size_t siz = sizeof(reply); + K_STORE *store = NULL; + K_TREE *tree = NULL; + bool unknown_typ = true, unknown_name = true, msg = false; i_file = require_name(trf_root, "file", 1, NULL, reply, siz); if (!i_file) @@ -5888,44 +5891,81 @@ static char *cmd_dsp(__maybe_unused PGconn *conn, __maybe_unused char *cmd, typ = "tree"; if (strcasecmp(typ, "tree") == 0) { + unknown_typ = false; + if (strcasecmp(name, "blocks") == 0) - dsp_ktree(blocks_root, fil, NULL); + tree = blocks_root; if (strcasecmp(name, "transfer") == 0) - dsp_ktree(trf_root, fil, NULL); + tree = trf_root; if (strcasecmp(name, "paymentaddresses") == 0) - dsp_ktree(paymentaddresses_root, fil, NULL); + tree = paymentaddresses_root; if (strcasecmp(name, "paymentaddresses_create") == 0) - dsp_ktree(paymentaddresses_root, fil, NULL); + tree = paymentaddresses_create_root; if (strcasecmp(name, "sharesummary") == 0) - dsp_ktree(sharesummary_root, fil, NULL); + tree = sharesummary_root; if (strcasecmp(name, "userstats") == 0) - dsp_ktree(userstats_root, fil, NULL); + tree = userstats_root; if (strcasecmp(name, "markersummary") == 0) - dsp_ktree(markersummary_root, fil, NULL); + tree = markersummary_root; if (strcasecmp(name, "workmarkers") == 0) - dsp_ktree(workmarkers_root, fil, NULL); - } + tree = workmarkers_root; + + if (strcasecmp(name, "idcontrol") == 0) + tree = idcontrol_root; + + if (tree) { + unknown_name = false; + if (tree->master->dsp_func) + dsp_ktree(tree, fil, NULL); + else { + snprintf(reply, siz, + "%s %s has no dsp_func", + typ, name); + msg = true; + } + } + } else if (strcasecmp(typ, "store") == 0) { + unknown_typ = false; - if (strcasecmp(typ, "store") == 0) { if (strcasecmp(name, "blocks") == 0) - dsp_kstore(blocks_store, fil, NULL); + store = blocks_store; if (strcasecmp(name, "markersummary") == 0) - dsp_kstore(markersummary_store, fil, NULL); + store = markersummary_store; if (strcasecmp(name, "msgline") == 0) - dsp_kstore(msgline_store, fil, NULL); + store = msgline_store; + + if (store) { + unknown_name = false; + if (store->master->dsp_func) + dsp_kstore(store, fil, NULL); + else { + snprintf(reply, siz, + "%s %s has no dsp_func", + typ, name); + msg = true; + } + } } - LOGDEBUG("%s.ok.dsp.file='%s'", id, fil); - return strdup("ok.dsp"); + if (unknown_typ) { + snprintf(reply, siz, "unknown typ '%s'", typ); + } else if (unknown_name) { + snprintf(reply, siz, "unknown name '%s' for '%s'", name, typ); + } else { + if (!msg) + snprintf(reply, siz, "ok.dsp.file='%s'", fil); + } + LOGDEBUG("%s.%s'", id, reply); + return strdup(reply); #endif } diff --git a/src/ckdb_data.c b/src/ckdb_data.c index 8a1755a1..9a7323d5 100644 --- a/src/ckdb_data.c +++ b/src/ckdb_data.c @@ -2187,6 +2187,52 @@ K_ITEM *find_accountbalance(int64_t userid) return item; } +void dsp_idcontrol(K_ITEM *item, FILE *stream) +{ + char createdate_buf[DATE_BUFSIZ], modifydate_buf[DATE_BUFSIZ]; + IDCONTROL *i; + + if (!item) + fprintf(stream, "%s() called with (null) item\n", __func__); + else { + DATA_IDCONTROL(i, item); + tv_to_buf(&(i->createdate), createdate_buf, sizeof(createdate_buf)); + tv_to_buf(&(i->modifydate), modifydate_buf, sizeof(modifydate_buf)); + fprintf(stream, " idname='%s' lastid=%"PRId64" cdate='%s'" + " cby='%s' ccode='%s' cinet='%s' mdate='%s'" + " mby='%s' mcode='%s' minet='%s'\n", + i->idname, i->lastid, createdate_buf, + i->in_createby, i->in_createcode, + i->in_createinet, modifydate_buf, + i->in_modifyby, i->in_modifycode, + i->in_modifyinet); + } +} + +// order by idname asc +cmp_t cmp_idcontrol(K_ITEM *a, K_ITEM *b) +{ + IDCONTROL *ida, *idb; + DATA_IDCONTROL(ida, a); + DATA_IDCONTROL(idb, b); + return CMP_STR(ida->idname, idb->idname); +} + +// idcontrol must be R or W locked +K_ITEM *find_idcontrol(char *idname) +{ + IDCONTROL idcontrol; + K_TREE_CTX ctx[1]; + K_ITEM look, *item; + + STRNCPY(idcontrol.idname, idname); + + INIT_IDCONTROL(&look); + look.data = (void *)(&idcontrol); + item = find_in_ktree(idcontrol_root, &look, ctx); + return item; +} + // order by optionname asc,activationdate asc,activationheight asc,expirydate desc cmp_t cmp_optioncontrol(K_ITEM *a, K_ITEM *b) { diff --git a/src/ckdb_dbio.c b/src/ckdb_dbio.c index aff62a08..3d53c11b 100644 --- a/src/ckdb_dbio.c +++ b/src/ckdb_dbio.c @@ -496,6 +496,8 @@ int64_t nextid(PGconn *conn, char *idname, int64_t increment, { ExecStatusType rescode; bool conned = false; + IDCONTROL *idcontrol; + K_ITEM *item; PGresult *res; char qry[1024]; char *params[5]; @@ -506,6 +508,14 @@ int64_t nextid(PGconn *conn, char *idname, int64_t increment, lastid = 0; + K_WLOCK(idcontrol_free); + item = find_idcontrol(idname); + if (!item) + { + LOGERR("%s(): No matching idname='%s' in tree", __func__, idname); + goto cleanup; + } + snprintf(qry, sizeof(qry), "select lastid from idcontrol " "where idname='%s' for update", idname); @@ -561,11 +571,19 @@ int64_t nextid(PGconn *conn, char *idname, int64_t increment, if (!PGOK(rescode)) { PGLOGERR("Update", rescode, conn); lastid = 0; + } else { + DATA_IDCONTROL(idcontrol, item); + idcontrol->lastid = lastid; + copy_tv(&(idcontrol->modifydate), cd); + idcontrol->in_modifyby = intransient_str(MBYDB, by); + idcontrol->in_modifycode = intransient_str(MCODEDB, code); + idcontrol->in_modifyinet = intransient_str(MINETDB, inet); } for (n = 0; n < par; n++) free(params[n]); cleanup: + K_WUNLOCK(idcontrol_free); conned = CKPQDisco(&conn, conned); return lastid; } @@ -2591,7 +2609,7 @@ bool idcontrol_add(PGconn *conn, char *idname, char *idvalue, char *by, char *code, char *inet, tv_t *cd, __maybe_unused K_TREE *trf_root) { - K_ITEM *look; + K_ITEM *item; IDCONTROL *row; char *params[2 + MODIFYDATECOUNT]; int n, par = 0; @@ -2604,19 +2622,19 @@ bool idcontrol_add(PGconn *conn, char *idname, char *idvalue, char *by, LOGDEBUG("%s(): add", __func__); K_WLOCK(idcontrol_free); - look = k_unlink_head(idcontrol_free); + item = k_unlink_head(idcontrol_free); K_WUNLOCK(idcontrol_free); - DATA_IDCONTROL(row, look); + DATA_IDCONTROL(row, item); STRNCPY(row->idname, idname); TXT_TO_BIGINT("idvalue", idvalue, row->lastid); - MODIFYDATEINIT(row, cd, by, code, inet); + MODIFYDATEINTRANS(row, cd, by, code, inet); par = 0; params[par++] = str_to_buf(row->idname, NULL, 0); params[par++] = bigint_to_buf(row->lastid, NULL, 0); - MODIFYDATEPARAMS(params, par, row); + MODIFYDATEPARAMSIN(params, par, row); PARCHK(par, params); ins = "insert into idcontrol " @@ -2637,13 +2655,108 @@ foil: for (n = 0; n < par; n++) free(params[n]); + /* N.B. The DB key matches the tree key, + * the tree depends on this to be valid */ K_WLOCK(idcontrol_free); - k_add_head(idcontrol_free, look); + if (ok) { + add_to_ktree(idcontrol_root, item); + k_add_head(idcontrol_store, item); + } else + k_add_head(idcontrol_free, item); K_WUNLOCK(idcontrol_free); return ok; } +bool idcontrol_fill(PGconn *conn) +{ + char pcombuf[64]; + ExecStatusType rescode; + PGresult *res; + K_ITEM *item; + int n, i; + IDCONTROL *row; + char *field; + char *sel; + int fields = 2; + bool ok; + + LOGDEBUG("%s(): select", __func__); + + int idname_num, lastid_num; + MODIFYDATE_num; + + sel = "select " + "idname,lastid" + MODIFYDATECONTROL + " from idcontrol"; + res = CKPQExec(conn, sel, CKPQ_READ); + rescode = CKPQResultStatus(res); + if (!PGOK(rescode)) { + PGLOGERR("Select", rescode, conn); + CKPQClear(res); + return false; + } + + n = PQnfields(res); + if (n != (fields + MODIFYDATECOUNT)) { + LOGERR("%s(): Invalid field count - should be %d, but is %d", + __func__, fields + MODIFYDATECOUNT, n); + CKPQClear(res); + return false; + } + + n = PQntuples(res); + LOGDEBUG("%s(): tree build count %d", __func__, n); + ok = true; + idname_num = lastid_num = CKPQFUNDEF; + MODIFYDATE_init; + K_WLOCK(idcontrol_free); + for (i = 0; i < n; i++) { + item = k_unlink_head(idcontrol_free); + DATA_IDCONTROL(row, item); + bzero(row, sizeof(*row)); + + if (everyone_die) { + ok = false; + break; + } + + CKPQ_VAL_FLD_num(res, i, idname, field, ok); + if (!ok) + break; + TXT_TO_STR("idname", field, row->idname); + + CKPQ_VAL_FLD_num(res, i, lastid, field, ok); + if (!ok) + break; + TXT_TO_BIGINT("lastid", field, row->lastid); + + MODIFYDATEIN(res, i, row, ok); + if (!ok) + break; + + /* N.B. The DB key matches the tree key, + * the tree depends on this to be valid */ + add_to_ktree(idcontrol_root, item); + k_add_head(idcontrol_store, item); + } + if (!ok) + k_add_head(idcontrol_free, item); + + K_WUNLOCK(idcontrol_free); + CKPQClear(res); + + if (ok) { + LOGDEBUG("%s(): built", __func__); + pcom(n, pcombuf, sizeof(pcombuf)); + LOGWARNING("%s(): loaded %s idcontrol records", + __func__, pcombuf); + } + + return ok; +} + void oc_switch_state(OPTIONCONTROL *oc, const char *from) { switch_state = atoi(oc->optionvalue); @@ -9672,77 +9785,3 @@ bool check_db_version(PGconn *conn) return true; } - -char *cmd_newid(PGconn *conn, char *cmd, char *id, tv_t *now, char *by, - char *code, char *inet, __maybe_unused tv_t *cd, - K_TREE *trf_root) -{ - char reply[1024] = ""; - size_t siz = sizeof(reply); - K_ITEM *i_idname, *i_idvalue, *look; - IDCONTROL *row; - char *params[2 + MODIFYDATECOUNT]; - int n, par = 0; - bool ok = false; - ExecStatusType rescode; - bool conned = false; - PGresult *res; - char *ins; - - LOGDEBUG("%s(): cmd '%s'", __func__, cmd); - - i_idname = require_name(trf_root, "idname", 3, (char *)idpatt, reply, siz); - if (!i_idname) - return strdup(reply); - - i_idvalue = require_name(trf_root, "idvalue", 1, (char *)intpatt, reply, siz); - if (!i_idvalue) - return strdup(reply); - - K_WLOCK(idcontrol_free); - look = k_unlink_head(idcontrol_free); - K_WUNLOCK(idcontrol_free); - - DATA_IDCONTROL(row, look); - - STRNCPY(row->idname, transfer_data(i_idname)); - TXT_TO_BIGINT("idvalue", transfer_data(i_idvalue), row->lastid); - MODIFYDATEINIT(row, now, by, code, inet); - - par = 0; - params[par++] = str_to_buf(row->idname, NULL, 0); - params[par++] = bigint_to_buf(row->lastid, NULL, 0); - MODIFYDATEPARAMS(params, par, row); - PARCHK(par, params); - - ins = "insert into idcontrol " - "(idname,lastid" MODIFYDATECONTROL ") values (" PQPARAM10 ")"; - - conned = CKPQConn(&conn); - res = CKPQExecParams(conn, ins, par, NULL, (const char **)params, NULL, NULL, 0, CKPQ_WRITE); - rescode = CKPQResultStatus(res); - CKPQClear(res); - if (!PGOK(rescode)) { - PGLOGERR("Insert", rescode, conn); - goto foil; - } - - ok = true; -foil: - conned = CKPQDisco(&conn, conned); - for (n = 0; n < par; n++) - free(params[n]); - - K_WLOCK(idcontrol_free); - k_add_head(idcontrol_free, look); - K_WUNLOCK(idcontrol_free); - - if (!ok) { - LOGERR("%s() %s.failed.DBE", __func__, id); - return strdup("failed.DBE"); - } - LOGDEBUG("%s.ok.added %s %"PRId64, id, transfer_data(i_idname), row->lastid); - snprintf(reply, siz, "ok.added %s %"PRId64, - transfer_data(i_idname), row->lastid); - return strdup(reply); -}