From 69cf35313dbaffcc5e2e923192f8015609b4483f Mon Sep 17 00:00:00 2001 From: kanoi Date: Wed, 31 Aug 2016 22:12:55 +1000 Subject: [PATCH] ckdb - intransient store of common strings to reduce ram - only used in transfer so far --- src/ckdb.c | 119 ++++++++++++++++++++++++++++++++++++++---------- src/ckdb.h | 43 ++++++++++++++++- src/ckdb_data.c | 94 +++++++++++++++++++++++++++++++++++++- 3 files changed, 230 insertions(+), 26 deletions(-) diff --git a/src/ckdb.c b/src/ckdb.c index c9219873..4e58a238 100644 --- a/src/ckdb.c +++ b/src/ckdb.c @@ -208,24 +208,24 @@ const char *strpatt = "^[ -<>-~]*$"; const char Transfer[] = "Transfer"; // older version missing field defaults -static TRANSFER auth_1 = { "poolinstance", "", auth_1.svalue }; +static TRANSFER auth_1 = { "poolinstance", "", auth_1.svalue, NULL }; K_ITEM auth_poolinstance = { Transfer, NULL, NULL, (void *)(&auth_1) }; -static TRANSFER auth_2 = { "preauth", FALSE_STR, auth_2.svalue }; +static TRANSFER auth_2 = { "preauth", FALSE_STR, auth_2.svalue, NULL }; K_ITEM auth_preauth = { Transfer, NULL, NULL, (void *)(&auth_2) }; -static TRANSFER poolstats_1 = { "elapsed", "0", poolstats_1.svalue }; +static TRANSFER poolstats_1 = { "elapsed", "0", poolstats_1.svalue, NULL }; K_ITEM poolstats_elapsed = { Transfer, NULL, NULL, (void *)(&poolstats_1) }; -static TRANSFER userstats_1 = { "elapsed", "0", userstats_1.svalue }; +static TRANSFER userstats_1 = { "elapsed", "0", userstats_1.svalue, NULL }; K_ITEM userstats_elapsed = { Transfer, NULL, NULL, (void *)(&userstats_1) }; -static TRANSFER userstats_2 = { "workername", "all", userstats_2.svalue }; +static TRANSFER userstats_2 = { "workername", "all", userstats_2.svalue, NULL }; K_ITEM userstats_workername = { Transfer, NULL, NULL, (void *)(&userstats_2) }; -static TRANSFER userstats_3 = { "idle", FALSE_STR, userstats_3.svalue }; +static TRANSFER userstats_3 = { "idle", FALSE_STR, userstats_3.svalue, NULL }; K_ITEM userstats_idle = { Transfer, NULL, NULL, (void *)(&userstats_3) }; -static TRANSFER userstats_4 = { "eos", TRUE_STR, userstats_4.svalue }; +static TRANSFER userstats_4 = { "eos", TRUE_STR, userstats_4.svalue, NULL }; K_ITEM userstats_eos = { Transfer, NULL, NULL, (void *)(&userstats_4) }; -static TRANSFER shares_1 = { "secondaryuserid", TRUE_STR, shares_1.svalue }; +static TRANSFER shares_1 = { "secondaryuserid", TRUE_STR, shares_1.svalue, NULL }; K_ITEM shares_secondaryuserid = { Transfer, NULL, NULL, (void *)(&shares_1) }; -static TRANSFER shareerrors_1 = { "secondaryuserid", TRUE_STR, shareerrors_1.svalue }; +static TRANSFER shareerrors_1 = { "secondaryuserid", TRUE_STR, shareerrors_1.svalue, NULL }; K_ITEM shareerrors_secondaryuserid = { Transfer, NULL, NULL, (void *)(&shareerrors_1) }; // Time limit that this problem occurred // 24-Aug-2014 05:20+00 (1st one shortly after this) @@ -447,6 +447,34 @@ static pthread_cond_t f_ioqueue_waitcond; K_LIST *logqueue_free; K_STORE *logqueue_store; +// NAMERAM +K_LIST *nameram_free; +K_STORE *nameram_store; + +// INTRANSIENT +K_TREE *intransient_root; +K_LIST *intransient_free; +K_STORE *intransient_store; + +char *intransient_fields[] = { + "workername", + "blockhash", + BYTRF, + CODETRF, + INETTRF, + "username", + "agent", + "useragent", + "address", + "prevhash", + "poolinstance", + "payaddress", + "originaltxn", + "committxn", + "commitblockhash", + NULL +}; + // MSGLINE K_LIST *msgline_free; K_STORE *msgline_store; @@ -1880,15 +1908,28 @@ static void alloc_storage() logqueue_store = k_new_store(logqueue_free); breakqueue_free = k_new_list("BreakQueue", sizeof(BREAKQUEUE), - ALLOC_BREAKQUEUE, LIMIT_BREAKQUEUE, true); + ALLOC_BREAKQUEUE, LIMIT_BREAKQUEUE, true); reload_breakqueue_store = k_new_store(breakqueue_free); reload_done_breakqueue_store = k_new_store(breakqueue_free); cmd_breakqueue_store = k_new_store(breakqueue_free); cmd_done_breakqueue_store = k_new_store(breakqueue_free); + intransient_free = k_new_list("Intransient", sizeof(INTRANSIENT), + ALLOC_INTRANSIENT, LIMIT_INTRANSIENT, + true); + intransient_store = k_new_store(intransient_free); + intransient_root = new_ktree(NULL, cmp_intransient, intransient_free); + + nameram_free = k_new_list("NameRAM", sizeof(NAMERAM), ALLOC_NAMERAM, + LIMIT_NAMERAM, true); + nameram_store = k_new_store(nameram_free); + #if LOCK_CHECK DLPRIO(logqueue, 94); DLPRIO(breakqueue, PRIO_TERMINAL); + + DLPRIO(intransient, 2); + DLPRIO(nameram, 1); #endif seqset_free = k_new_list("SeqSet", sizeof(SEQSET), @@ -3910,8 +3951,9 @@ static enum cmd_values breakdown(K_ITEM **ml_item, char *buf, tv_t *now, K_ITEM *t_item = NULL, *cd_item = NULL, *seqall; char *cmdptr, *idptr, *next, *eq, *end, *was; char *data = NULL, *st = NULL, *st2 = NULL, *ip = NULL; - bool noid = false; + bool noid = false, intrans; size_t siz; + int i; K_WLOCK(msgline_free); *ml_item = k_unlink_head_zero(msgline_free); @@ -4033,6 +4075,7 @@ static enum cmd_values breakdown(K_ITEM **ml_item, char *buf, tv_t *now, FREENULL(st2); goto nogood; } + // LOGERR of buf could be truncated *(end++) = '\0'; K_WLOCK(transfer_free); t_item = k_unlink_head(transfer_free); @@ -4131,12 +4174,29 @@ static enum cmd_values breakdown(K_ITEM **ml_item, char *buf, tv_t *now, } siz = end - next; } - if (siz >= sizeof(transfer->svalue)) { - transfer->mvalue = malloc(siz+1); - STRNCPYSIZ(transfer->mvalue, next, siz+1); - } else { - STRNCPYSIZ(transfer->svalue, next, siz+1); - transfer->mvalue = transfer->svalue; + intrans = false; + for (i = 0; intransient_fields[i]; i++) { + if (strcmp(transfer->name, + intransient_fields[i]) == 0) { + intrans = true; + // This will affect 'was' if JSON_END is missing + *(next + siz) = '\0'; + transfer->intransient = get_intransient_siz(next, siz+1); + transfer->mvalue = transfer->intransient->str; + break; + } + } + if (!intrans) { + transfer->intransient = NULL; + if (siz >= sizeof(transfer->svalue)) { + transfer->mvalue = malloc(siz+1); + STRNCPYSIZ(transfer->mvalue, next, + siz+1); + } else { + STRNCPYSIZ(transfer->svalue, next, + siz+1); + transfer->mvalue = transfer->svalue; + } } add_to_ktree_nolock(msgline->trf_root, t_item); k_add_head_nolock(msgline->trf_store, t_item); @@ -4179,12 +4239,25 @@ static enum cmd_values breakdown(K_ITEM **ml_item, char *buf, tv_t *now, DATA_TRANSFER(transfer, t_item); STRNCPY(transfer->name, data); siz = strlen(eq) + 1; - if (siz > sizeof(transfer->svalue)) { - transfer->mvalue = malloc(siz); - STRNCPYSIZ(transfer->mvalue, eq, siz); - } else { - STRNCPYSIZ(transfer->svalue, eq, siz); - transfer->mvalue = transfer->svalue; + intrans = false; + for (i = 0; intransient_fields[i]; i++) { + if (strcmp(transfer->name, + intransient_fields[i]) == 0) { + intrans = true; + transfer->intransient = get_intransient_siz(eq, siz); + transfer->mvalue = transfer->intransient->str; + break; + } + } + if (!intrans) { + transfer->intransient = NULL; + if (siz > sizeof(transfer->svalue)) { + transfer->mvalue = malloc(siz); + STRNCPYSIZ(transfer->mvalue, eq, siz); + } else { + STRNCPYSIZ(transfer->svalue, eq, siz); + transfer->mvalue = transfer->svalue; + } } // Discard duplicates diff --git a/src/ckdb.h b/src/ckdb.h index 0ec8ecc4..1136c632 100644 --- a/src/ckdb.h +++ b/src/ckdb.h @@ -58,7 +58,7 @@ #define DB_VLOCK "1" #define DB_VERSION "1.0.7" -#define CKDB_VERSION DB_VERSION"-2.438" +#define CKDB_VERSION DB_VERSION"-2.440" #define WHERE_FFL " - from %s %s() line %d" #define WHERE_FFL_HERE __FILE__, __func__, __LINE__ @@ -1131,6 +1131,41 @@ typedef struct logqueue { extern K_LIST *logqueue_free; extern K_STORE *logqueue_store; +// NAMERAM - RAM for INTRANSIENT names - 2M per allocation +typedef struct nameram { + char rem[2*1024*1024-sizeof(void *)-sizeof(size_t)]; + void *next; + size_t left; +} NAMERAM; + +// Items never to be deleted and list never to be culled +#define ALLOC_NAMERAM 1 +#define LIMIT_NAMERAM 0 +#define INIT_NAMERAM(_item) INIT_GENERIC(_item, nameram) +#define DATA_NAMERAM(_var, _item) DATA_GENERIC(_var, _item, nameram, true) + +extern K_LIST *nameram_free; +extern K_STORE *nameram_store; + +// INTRANSIENT - a list of common strings, to avoid wasting RAM +typedef struct intransient { + char *str; +} INTRANSIENT; + +/* Items never to be deleted and list never to be culled + * They are all created by the db load functions and breakdown */ +#define ALLOC_INTRANSIENT 1024 +#define LIMIT_INTRANSIENT 0 +#define INIT_INTRANSIENT(_item) INIT_GENERIC(_item, intransient) +#define DATA_INTRANSIENT(_var, _item) DATA_GENERIC(_var, _item, intransient, true) +#define DATA_INTRANSIENT_NULL(_var, _item) DATA_GENERIC(_var, _item, intransient, false) + +extern K_TREE *intransient_root; +extern K_LIST *intransient_free; +extern K_STORE *intransient_store; + +extern char *intransient_fields[]; + // MSGLINE typedef struct msgline { int which_cmds; @@ -1343,6 +1378,7 @@ typedef struct transfer { char name[NAME_SIZE+1]; char svalue[VALUE_SIZE+1]; char *mvalue; + INTRANSIENT *intransient; } TRANSFER; // Suggest malloc use MMAP = largest under 2MB @@ -3074,6 +3110,11 @@ extern char *_hms_to_buf(time_t *data, char *buf, size_t siz, WHERE_FFL_ARGS); // Convert to MM:SS extern char *_ms_to_buf(time_t *data, char *buf, size_t siz, WHERE_FFL_ARGS); +extern cmp_t cmp_intransient(K_ITEM *a, K_ITEM *b); +#define get_intransient(_name) _get_intransient(_name, 0, WHERE_FFL_HERE) +#define get_intransient_siz(_name, _siz) \ + _get_intransient(_name, _siz, WHERE_FFL_HERE) +extern INTRANSIENT *_get_intransient(char *name, size_t siz, WHERE_FFL_ARGS); extern char *_transfer_data(K_ITEM *item, WHERE_FFL_ARGS); extern void dsp_transfer(K_ITEM *item, FILE *stream); extern cmp_t cmp_transfer(K_ITEM *a, K_ITEM *b); diff --git a/src/ckdb_data.c b/src/ckdb_data.c index bc95a1c1..c3531eb1 100644 --- a/src/ckdb_data.c +++ b/src/ckdb_data.c @@ -25,8 +25,13 @@ void free_msgline_data(K_ITEM *item, bool t_lock, bool t_cull) t_item = STORE_HEAD_NOLOCK(msgline->trf_store); while (t_item) { DATA_TRANSFER(transfer, t_item); - if (transfer->mvalue != transfer->svalue) - FREENULL(transfer->mvalue); + if (transfer->mvalue != transfer->svalue) { + if (transfer->intransient) { + transfer->intransient = NULL; + transfer->mvalue = NULL; + } else + FREENULL(transfer->mvalue); + } t_item = t_item->next; } if (t_lock) @@ -701,6 +706,91 @@ char *_ms_to_buf(time_t *data, char *buf, size_t siz, WHERE_FFL_ARGS) return _data_to_buf(TYPE_MS, (void *)data, buf, siz, WHERE_FFL_PASS); } +// order by name asc +cmp_t cmp_intransient(K_ITEM *a, K_ITEM *b) +{ + INTRANSIENT *ia, *ib; + DATA_INTRANSIENT(ia, a); + DATA_INTRANSIENT(ib, b); + return CMP_STR(ia->str, ib->str); +} + +INTRANSIENT *_get_intransient(char *name, size_t siz, WHERE_FFL_ARGS) +{ + INTRANSIENT intransient, *in = NULL; + K_ITEM look, *i_item, *n_item; + NAMERAM *nameram = NULL; + K_TREE_CTX ctx[1]; + char *buf; + bool new; + + if (siz == 0) + siz = strlen(name) + 1; + + if (siz > sizeof(nameram->rem)) { + char *st = NULL; + LOGEMERG("%s() ERR '%10s...' discarded - siz %d>%d" WHERE_FFL, + __func__, st = safe_text_nonull(name), (int)siz, + (int)sizeof(nameram->rem), WHERE_FFL_PASS); + name = EMPTY; + siz = 1; + } + + intransient.str = name; + INIT_INTRANSIENT(&look); + look.data = (void *)(&intransient); + K_RLOCK(intransient_free); + i_item = _find_in_ktree(intransient_root, &look, ctx, false, + WHERE_FFL_PASS); + K_RUNLOCK(intransient_free); + + if (i_item) { + DATA_INTRANSIENT(in, i_item); + return in; + } + + K_WLOCK(intransient_free); + // Search again, to be thread safe + i_item = _find_in_ktree(intransient_root, &look, ctx, false, + WHERE_FFL_PASS); + if (i_item) { + DATA_INTRANSIENT(in, i_item); + } else { + new = false; + buf = NULL; + K_WLOCK(nameram_free); + if (nameram_store->count == 0) + new = true; + else { + n_item = STORE_WHEAD(nameram_store); + DATA_NAMERAM(nameram, n_item); + if (nameram->left < siz) + new = true; + } + if (new) { + n_item = k_unlink_head(nameram_free); + DATA_NAMERAM(nameram, n_item); + nameram->next = nameram->rem; + nameram->left = sizeof(nameram->rem); + k_add_head(nameram_store, n_item); + + } + buf = nameram->next; + nameram->next += siz; + nameram->left -= siz; + K_WUNLOCK(nameram_free); + strcpy(buf, name); + i_item = k_unlink_head(intransient_free); + DATA_INTRANSIENT(in, i_item); + in->str = buf; + k_add_tail(intransient_store, i_item); + add_to_ktree(intransient_root, i_item); + } + K_WUNLOCK(intransient_free); + + return in; +} + // For mutiple variable function calls that need the data char *_transfer_data(K_ITEM *item, WHERE_FFL_ARGS) {