From 1d47cfbaad0cf94dff295d276b9ba855de37e140 Mon Sep 17 00:00:00 2001 From: kanoi Date: Fri, 2 Sep 2016 01:03:50 +1000 Subject: [PATCH] ckdb - reduce the size of workinfo and make some intransient --- src/ckdb.c | 6 ++-- src/ckdb.h | 34 +++++++++++++-------- src/ckdb_cmd.c | 34 ++++++++++----------- src/ckdb_data.c | 81 +++++++++++++++++++++++++++++++++++++++++++++++++ src/ckdb_dbio.c | 48 ++++++++++++++--------------- 5 files changed, 147 insertions(+), 56 deletions(-) diff --git a/src/ckdb.c b/src/ckdb.c index b9ce3ec6..0a692dfa 100644 --- a/src/ckdb.c +++ b/src/ckdb.c @@ -469,6 +469,8 @@ char *intransient_fields[] = { "useragent", "address", "prevhash", + "bits", + "version", "poolinstance", "payaddress", "originaltxn", @@ -2729,7 +2731,7 @@ static bool setup_data() workinfo_current = last_in_ktree(workinfo_height_root, ctx); if (workinfo_current) { DATA_WORKINFO(wic, workinfo_current); - STRNCPY(wi.coinbase1, wic->coinbase1); + wi.coinbase1 = wic->coinbase1; DATE_ZERO(&(wi.createdate)); INIT_WORKINFO(&look); look.data = (void *)(&wi); @@ -8953,7 +8955,7 @@ static struct option long_options[] = { { "btc-user", required_argument, 0, 'U' }, { "version", no_argument, 0, 'v' }, { "workinfoid", required_argument, 0, 'w' }, - // Disable writing to the db-log file + // Disable writing to the db-log file (for testing environments) { "no-db-log", no_argument, 0, 'x' }, { "confirm", no_argument, 0, 'y' }, { "confirmrange", required_argument, 0, 'Y' }, diff --git a/src/ckdb.h b/src/ckdb.h index ca026664..cafc6d55 100644 --- a/src/ckdb.h +++ b/src/ckdb.h @@ -58,7 +58,7 @@ #define DB_VLOCK "1" #define DB_VERSION "1.0.7" -#define CKDB_VERSION DB_VERSION"-2.446" +#define CKDB_VERSION DB_VERSION"-2.447" #define WHERE_FFL " - from %s %s() line %d" #define WHERE_FFL_HERE __FILE__, __func__, __LINE__ @@ -1156,8 +1156,7 @@ typedef struct intransient { char *str; } INTRANSIENT; -/* Items never to be deleted and list never to be culled - * They are all created by breakdown */ +// Items never to be deleted and list never to be culled #define ALLOC_INTRANSIENT 1024 #define LIMIT_INTRANSIENT 0 #define INIT_INTRANSIENT(_item) INIT_GENERIC(_item, intransient) @@ -1952,14 +1951,14 @@ extern K_STORE *esm_store; // WORKINFO workinfo.id.json={...} typedef struct workinfo { int64_t workinfoid; - char poolinstance[TXT_BIG+1]; + char *poolinstance; char *transactiontree; char *merklehash; - char prevhash[TXT_BIG+1]; - char coinbase1[TXT_BIG+1]; - char coinbase2[TXT_BIG+1]; - char version[TXT_SML+1]; - char bits[TXT_SML+1]; + char *prevhash; + char *coinbase1; + char *coinbase2; + char *version; + char *bits; char ntime[TXT_SML+1]; int64_t reward; int32_t height; // non-DB field @@ -3152,6 +3151,9 @@ extern cmp_t cmp_intransient(K_ITEM *a, K_ITEM *b); _get_intransient(_fld, _val , _siz, WHERE_FFL_HERE) extern INTRANSIENT *_get_intransient(char *fldnam, char *value, size_t siz, WHERE_FFL_ARGS); +#define intransient_str(_fld, _val) \ + _intransient_str(_fld, _val, WHERE_FFL_HERE) +extern char *_intransient_str(char *fldnam, char *value, WHERE_FFL_ARGS); extern char *_transfer_data(K_ITEM *item, WHERE_FFL_ARGS); extern void dsp_transfer(K_ITEM *item, FILE *stream); extern cmp_t cmp_transfer(K_ITEM *a, K_ITEM *b); @@ -3168,6 +3170,11 @@ extern K_ITEM *_optional_name(K_TREE *trf_root, char *name, int len, char *patt, _siz, WHERE_FFL_HERE) extern K_ITEM *_require_name(K_TREE *trf_root, char *name, int len, char *patt, char *reply, size_t siz, WHERE_FFL_ARGS); +#define require_in(_root, _name, _len, _patt, _reply, _siz) \ + _require_in(_root, _name, _len, _patt, _reply, \ + _siz, WHERE_FFL_HERE) +extern INTRANSIENT *_require_in(K_TREE *trf_root, char *name, int len, char *patt, + char *reply, size_t siz, WHERE_FFL_ARGS); extern cmp_t cmp_workerstatus(K_ITEM *a, K_ITEM *b); extern K_ITEM *find_workerstatus(bool gotlock, int64_t userid, char *workername); #define find_create_workerstatus(_gl, _ac, _u, _w, _hw, _file, _func, _line) \ @@ -3504,10 +3511,11 @@ extern K_ITEM *optioncontrol_add(PGconn *conn, char *optionname, char *optionval char *by, char *code, char *inet, tv_t *cd, K_TREE *trf_root, bool begun); extern bool optioncontrol_fill(PGconn *conn); -extern int64_t workinfo_add(PGconn *conn, char *workinfoidstr, char *poolinstance, - char *transactiontree, char *merklehash, char *prevhash, - char *coinbase1, char *coinbase2, char *version, - char *bits, char *ntime, char *reward, char *by, +extern int64_t workinfo_add(PGconn *conn, char *workinfoidstr, + INTRANSIENT *in_poolinstance, char *transactiontree, + char *merklehash, INTRANSIENT *in_prevhash, + char *coinbase1, char *coinbase2, INTRANSIENT *in_version, + INTRANSIENT *in_bits, char *ntime, char *reward, char *by, char *code, char *inet, tv_t *cd, bool igndup, K_TREE *trf_root); extern bool workinfo_fill(PGconn *conn); diff --git a/src/ckdb_cmd.c b/src/ckdb_cmd.c index e49d628a..3930b126 100644 --- a/src/ckdb_cmd.c +++ b/src/ckdb_cmd.c @@ -2522,9 +2522,9 @@ static char *cmd_sharelog(PGconn *conn, char *cmd, char *id, LOGDEBUG("%s(): cmd '%s'", __func__, cmd); if (strcasecmp(cmd, STR_WORKINFO) == 0) { - K_ITEM *i_workinfoid, *i_poolinstance, *i_transactiontree, *i_merklehash; - K_ITEM *i_prevhash, *i_coinbase1, *i_coinbase2, *i_version, *i_bits; - K_ITEM *i_ntime, *i_reward; + K_ITEM *i_workinfoid, *i_transactiontree, *i_merklehash; + K_ITEM *i_coinbase1, *i_coinbase2, *i_ntime, *i_reward; + INTRANSIENT *in_poolinstance, *in_prevhash, *in_version, *in_bits; bool igndup = false; char *txn_tree; @@ -2532,12 +2532,12 @@ static char *cmd_sharelog(PGconn *conn, char *cmd, char *id, if (key_update) goto wiconf; - i_poolinstance = require_name(trf_root, "poolinstance", 1, NULL, reply, siz); - if (!i_poolinstance) + in_poolinstance = require_in(trf_root, "poolinstance", 1, NULL, reply, siz); + if (!in_poolinstance) return strdup(reply); - if (poolinstance && strcmp(poolinstance, transfer_data(i_poolinstance))){ - POOLINSTANCE_DATA_SET(workinfo, transfer_data(i_poolinstance)); + if (poolinstance && strcmp(poolinstance, in_poolinstance->str)){ + POOLINSTANCE_DATA_SET(workinfo, in_poolinstance->str); return strdup(FAILED_PI); } @@ -2570,8 +2570,8 @@ static char *cmd_sharelog(PGconn *conn, char *cmd, char *id, if (!i_merklehash) return strdup(reply); - i_prevhash = require_name(trf_root, "prevhash", 1, NULL, reply, siz); - if (!i_prevhash) + in_prevhash = require_in(trf_root, "prevhash", 1, NULL, reply, siz); + if (!in_prevhash) return strdup(reply); i_coinbase1 = require_name(trf_root, "coinbase1", 1, NULL, reply, siz); @@ -2582,12 +2582,12 @@ static char *cmd_sharelog(PGconn *conn, char *cmd, char *id, if (!i_coinbase2) return strdup(reply); - i_version = require_name(trf_root, "version", 1, NULL, reply, siz); - if (!i_version) + in_version = require_in(trf_root, "version", 1, NULL, reply, siz); + if (!in_version) return strdup(reply); - i_bits = require_name(trf_root, "bits", 1, NULL, reply, siz); - if (!i_bits) + in_bits = require_in(trf_root, "bits", 1, NULL, reply, siz); + if (!in_bits) return strdup(reply); i_ntime = require_name(trf_root, "ntime", 1, NULL, reply, siz); @@ -2599,14 +2599,14 @@ static char *cmd_sharelog(PGconn *conn, char *cmd, char *id, return strdup(reply); workinfoid = workinfo_add(conn, transfer_data(i_workinfoid), - transfer_data(i_poolinstance), + in_poolinstance, txn_tree, transfer_data(i_merklehash), - transfer_data(i_prevhash), + in_prevhash, transfer_data(i_coinbase1), transfer_data(i_coinbase2), - transfer_data(i_version), - transfer_data(i_bits), + in_version, + in_bits, transfer_data(i_ntime), transfer_data(i_reward), by, code, inet, cd, igndup, trf_root); diff --git a/src/ckdb_data.c b/src/ckdb_data.c index 8e24aa0c..6855ac63 100644 --- a/src/ckdb_data.c +++ b/src/ckdb_data.c @@ -67,6 +67,10 @@ void free_workinfo_data(K_ITEM *item) FREENULL(workinfo->transactiontree); LIST_MEM_SUB(workinfo_free, workinfo->merklehash); FREENULL(workinfo->merklehash); + LIST_MEM_SUB(workinfo_free, workinfo->coinbase1); + FREENULL(workinfo->coinbase1); + LIST_MEM_SUB(workinfo_free, workinfo->coinbase2); + FREENULL(workinfo->coinbase2); } void free_payouts_data(K_ITEM *item) @@ -792,6 +796,14 @@ INTRANSIENT *_get_intransient(char *fldnam, char *value, size_t siz, WHERE_FFL_A return in; } +char *_intransient_str(char *fldnam, char *value, WHERE_FFL_ARGS) +{ + INTRANSIENT *in; + + in = _get_intransient(fldnam, value, 0, WHERE_FFL_PASS); + return in->str; +} + // For mutiple variable function calls that need the data char *_transfer_data(K_ITEM *item, WHERE_FFL_ARGS) { @@ -970,6 +982,75 @@ K_ITEM *_require_name(K_TREE *trf_root, char *name, int len, char *patt, return item; } +INTRANSIENT *_require_in(K_TREE *trf_root, char *name, int len, char *patt, + char *reply, size_t siz, WHERE_FFL_ARGS) +{ + INTRANSIENT *in; + TRANSFER *trf; + K_ITEM *item; + char *mvalue; + regex_t re; + size_t dlen; + int ret; + + reply[0] = '\0'; + + item = find_transfer(trf_root, name); + if (!item) { + LOGERR("%s(): failed, field '%s' missing from %s():%d", + __func__, name, func, line); + snprintf(reply, siz, "failed.missing %s", name); + return NULL; + } + + DATA_TRANSFER(trf, item); + if (!(in = trf->intransient)) { + LOGERR("%s(): failed, field '%s' is not intransient %s():%d", + __func__, name, func, line); + snprintf(reply, siz, "failed.transient %s", name); + return NULL; + } + + mvalue = trf->mvalue; + if (mvalue) + dlen = strlen(mvalue); + else + dlen = 0; + if (!mvalue || (int)dlen < len) { + LOGERR("%s(): failed, field '%s' short (%s%d<%d) from %s():%d", + __func__, name, mvalue ? EMPTY : "null ", + (int)dlen, len, func, line); + snprintf(reply, siz, "failed.short %s", name); + return NULL; + } + + if (patt) { + if (regcomp(&re, patt, REG_NOSUB) != 0) { + LOGERR("%s(): failed, field '%s' failed to" + " compile patt from %s():%d", + __func__, name, func, line); + snprintf(reply, siz, "failed.REG %s", name); + return NULL; + } + + ret = regexec(&re, mvalue, (size_t)0, NULL, 0); + regfree(&re); + + if (ret != 0) { + char *st = NULL; + LOGERR("%s(): failed, field '%s'='%.20s%s' invalid " + "from %s():%d", + __func__, name, st = safe_text_nonull(mvalue), + (dlen > 20) ? "..." : EMPTY, func, line); + FREENULL(st); + snprintf(reply, siz, "failed.invalid %s", name); + return NULL; + } + } + + return in; +} + // order by userid asc,workername asc cmp_t cmp_workerstatus(K_ITEM *a, K_ITEM *b) { diff --git a/src/ckdb_dbio.c b/src/ckdb_dbio.c index 41b43ef7..d29b08f6 100644 --- a/src/ckdb_dbio.c +++ b/src/ckdb_dbio.c @@ -3126,12 +3126,13 @@ bool optioncontrol_fill(PGconn *conn) return ok; } -int64_t workinfo_add(PGconn *conn, char *workinfoidstr, char *poolinstance, - char *transactiontree, char *merklehash, char *prevhash, - char *coinbase1, char *coinbase2, char *version, - char *bits, char *ntime, char *reward, char *by, - char *code, char *inet, tv_t *cd, bool igndup, - K_TREE *trf_root) +int64_t workinfo_add(PGconn *conn, char *workinfoidstr, + INTRANSIENT *in_poolinstance, char *transactiontree, + char *merklehash, INTRANSIENT *in_prevhash, + char *coinbase1, char *coinbase2, + INTRANSIENT *in_version, INTRANSIENT *in_bits, + char *ntime, char *reward, char *by, char *code, + char *inet, tv_t *cd, bool igndup, K_TREE *trf_root) { ExecStatusType rescode; bool conned = false; @@ -3157,14 +3158,14 @@ int64_t workinfo_add(PGconn *conn, char *workinfoidstr, char *poolinstance, bzero(row, sizeof(*row)); TXT_TO_BIGINT("workinfoid", workinfoidstr, row->workinfoid); - STRNCPY(row->poolinstance, poolinstance); + row->poolinstance = in_poolinstance->str; DUP_POINTER(workinfo_free, row->transactiontree, transactiontree); DUP_POINTER(workinfo_free, row->merklehash, merklehash); - STRNCPY(row->prevhash, prevhash); - STRNCPY(row->coinbase1, coinbase1); - STRNCPY(row->coinbase2, coinbase2); - STRNCPY(row->version, version); - STRNCPY(row->bits, bits); + row->prevhash = in_prevhash->str; + DUP_POINTER(workinfo_free, row->coinbase1, coinbase1); + DUP_POINTER(workinfo_free, row->coinbase2, coinbase2); + row->version = in_version->str; + row->bits = in_bits->str; STRNCPY(row->ntime, ntime); TXT_TO_BIGINT("reward", reward, row->reward); pool.reward = row->reward; @@ -3242,6 +3243,8 @@ unparam: // Not currently needed in RAM LIST_MEM_SUB(workinfo_free, row->transactiontree); FREENULL(row->transactiontree); + LIST_MEM_SUB(workinfo_free, row->merklehash); + FREENULL(row->merklehash); row->height = coinbase1height(row); hex2bin(ndiffbin, row->bits, 4); @@ -3389,7 +3392,7 @@ bool workinfo_fill(PGconn *conn) POOLINSTANCE_DBLOAD_SET(workinfo, field); continue; } - TXT_TO_STR("poolinstance", field, row->poolinstance); + row->poolinstance = intransient_str("poolinstance", field); PQ_GET_FLD(res, i, "workinfoid", field, ok); if (!ok) @@ -3397,37 +3400,34 @@ bool workinfo_fill(PGconn *conn) TXT_TO_BIGINT("workinfoid", field, row->workinfoid); row->transactiontree = EMPTY; - - PQ_GET_FLD(res, i, "merklehash", field, ok); - if (!ok) - break; - TXT_TO_BLOB("merklehash", field, row->merklehash); - LIST_MEM_ADD(workinfo_free, row->merklehash); + row->merklehash = EMPTY; PQ_GET_FLD(res, i, "prevhash", field, ok); if (!ok) break; - TXT_TO_STR("prevhash", field, row->prevhash); + row->prevhash = intransient_str("prevhash", field); PQ_GET_FLD(res, i, "coinbase1", field, ok); if (!ok) break; - TXT_TO_STR("coinbase1", field, row->coinbase1); + TXT_TO_BLOB("coinbase1", field, row->coinbase1); + LIST_MEM_ADD(workinfo_free, row->coinbase1); PQ_GET_FLD(res, i, "coinbase2", field, ok); if (!ok) break; - TXT_TO_STR("coinbase2", field, row->coinbase2); + TXT_TO_BLOB("coinbase2", field, row->coinbase2); + LIST_MEM_ADD(workinfo_free, row->coinbase2); PQ_GET_FLD(res, i, "version", field, ok); if (!ok) break; - TXT_TO_STR("version", field, row->version); + row->version = intransient_str("version", field); PQ_GET_FLD(res, i, "bits", field, ok); if (!ok) break; - TXT_TO_STR("bits", field, row->bits); + row->bits = intransient_str("bits", field); PQ_GET_FLD(res, i, "ntime", field, ok); if (!ok)