markow
/
ckpool
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity
Browse Source

ckdb - intransient store of common strings to reduce ram - only used in transfer so far

master
kanoi 8 years ago
parent
f96cf93aa6
commit
69cf35313d
3 changed files with 230 additions and 26 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 119
      src/ckdb.c
  2. 43
      src/ckdb.h
  3. 94
      src/ckdb_data.c

119
src/ckdb.c
Unescape View File

@ -208,24 +208,24 @@ const char *strpatt = "^[ -<>-~]*$";
const char Transfer[] = "Transfer";
// older version missing field defaults
static TRANSFER auth_1 = { "poolinstance", "", auth_1.svalue };
static TRANSFER auth_1 = { "poolinstance", "", auth_1.svalue, NULL };
K_ITEM auth_poolinstance = { Transfer, NULL, NULL, (void *)(&auth_1) };
static TRANSFER auth_2 = { "preauth", FALSE_STR, auth_2.svalue };
static TRANSFER auth_2 = { "preauth", FALSE_STR, auth_2.svalue, NULL };
K_ITEM auth_preauth = { Transfer, NULL, NULL, (void *)(&auth_2) };
static TRANSFER poolstats_1 = { "elapsed", "0", poolstats_1.svalue };
static TRANSFER poolstats_1 = { "elapsed", "0", poolstats_1.svalue, NULL };
K_ITEM poolstats_elapsed = { Transfer, NULL, NULL, (void *)(&poolstats_1) };
static TRANSFER userstats_1 = { "elapsed", "0", userstats_1.svalue };
static TRANSFER userstats_1 = { "elapsed", "0", userstats_1.svalue, NULL };
K_ITEM userstats_elapsed = { Transfer, NULL, NULL, (void *)(&userstats_1) };
static TRANSFER userstats_2 = { "workername", "all", userstats_2.svalue };
static TRANSFER userstats_2 = { "workername", "all", userstats_2.svalue, NULL };
K_ITEM userstats_workername = { Transfer, NULL, NULL, (void *)(&userstats_2) };
static TRANSFER userstats_3 = { "idle", FALSE_STR, userstats_3.svalue };
static TRANSFER userstats_3 = { "idle", FALSE_STR, userstats_3.svalue, NULL };
K_ITEM userstats_idle = { Transfer, NULL, NULL, (void *)(&userstats_3) };
static TRANSFER userstats_4 = { "eos", TRUE_STR, userstats_4.svalue };
static TRANSFER userstats_4 = { "eos", TRUE_STR, userstats_4.svalue, NULL };
K_ITEM userstats_eos = { Transfer, NULL, NULL, (void *)(&userstats_4) };
static TRANSFER shares_1 = { "secondaryuserid", TRUE_STR, shares_1.svalue };
static TRANSFER shares_1 = { "secondaryuserid", TRUE_STR, shares_1.svalue, NULL };
K_ITEM shares_secondaryuserid = { Transfer, NULL, NULL, (void *)(&shares_1) };
static TRANSFER shareerrors_1 = { "secondaryuserid", TRUE_STR, shareerrors_1.svalue };
static TRANSFER shareerrors_1 = { "secondaryuserid", TRUE_STR, shareerrors_1.svalue, NULL };
K_ITEM shareerrors_secondaryuserid = { Transfer, NULL, NULL, (void *)(&shareerrors_1) };
// Time limit that this problem occurred
// 24-Aug-2014 05:20+00 (1st one shortly after this)
@ -447,6 +447,34 @@ static pthread_cond_t f_ioqueue_waitcond;
K_LIST *logqueue_free;
K_STORE *logqueue_store;
// NAMERAM
K_LIST *nameram_free;
K_STORE *nameram_store;
// INTRANSIENT
K_TREE *intransient_root;
K_LIST *intransient_free;
K_STORE *intransient_store;
char *intransient_fields[] = {
"workername",
"blockhash",
BYTRF,
CODETRF,
INETTRF,
"username",
"agent",
"useragent",
"address",
"prevhash",
"poolinstance",
"payaddress",
"originaltxn",
"committxn",
"commitblockhash",
NULL
};
// MSGLINE
K_LIST *msgline_free;
K_STORE *msgline_store;
@ -1880,15 +1908,28 @@ static void alloc_storage()
logqueue_store = k_new_store(logqueue_free);
breakqueue_free = k_new_list("BreakQueue", sizeof(BREAKQUEUE),
ALLOC_BREAKQUEUE, LIMIT_BREAKQUEUE, true);
ALLOC_BREAKQUEUE, LIMIT_BREAKQUEUE, true);
reload_breakqueue_store = k_new_store(breakqueue_free);
reload_done_breakqueue_store = k_new_store(breakqueue_free);
cmd_breakqueue_store = k_new_store(breakqueue_free);
cmd_done_breakqueue_store = k_new_store(breakqueue_free);
intransient_free = k_new_list("Intransient", sizeof(INTRANSIENT),
ALLOC_INTRANSIENT, LIMIT_INTRANSIENT,
true);
intransient_store = k_new_store(intransient_free);
intransient_root = new_ktree(NULL, cmp_intransient, intransient_free);
nameram_free = k_new_list("NameRAM", sizeof(NAMERAM), ALLOC_NAMERAM,
LIMIT_NAMERAM, true);
nameram_store = k_new_store(nameram_free);
#if LOCK_CHECK
DLPRIO(logqueue, 94);
DLPRIO(breakqueue, PRIO_TERMINAL);
DLPRIO(intransient, 2);
DLPRIO(nameram, 1);
#endif
seqset_free = k_new_list("SeqSet", sizeof(SEQSET),
@ -3910,8 +3951,9 @@ static enum cmd_values breakdown(K_ITEM **ml_item, char *buf, tv_t *now,
K_ITEM *t_item = NULL, *cd_item = NULL, *seqall;
char *cmdptr, *idptr, *next, *eq, *end, *was;
char *data = NULL, *st = NULL, *st2 = NULL, *ip = NULL;
bool noid = false;
bool noid = false, intrans;
size_t siz;
int i;
K_WLOCK(msgline_free);
*ml_item = k_unlink_head_zero(msgline_free);
@ -4033,6 +4075,7 @@ static enum cmd_values breakdown(K_ITEM **ml_item, char *buf, tv_t *now,
FREENULL(st2);
goto nogood;
}
// LOGERR of buf could be truncated
*(end++) = '\0';
K_WLOCK(transfer_free);
t_item = k_unlink_head(transfer_free);
@ -4131,12 +4174,29 @@ static enum cmd_values breakdown(K_ITEM **ml_item, char *buf, tv_t *now,
}
siz = end - next;
}
if (siz >= sizeof(transfer->svalue)) {
transfer->mvalue = malloc(siz+1);
STRNCPYSIZ(transfer->mvalue, next, siz+1);
} else {
STRNCPYSIZ(transfer->svalue, next, siz+1);
transfer->mvalue = transfer->svalue;
intrans = false;
for (i = 0; intransient_fields[i]; i++) {
if (strcmp(transfer->name,
intransient_fields[i]) == 0) {
intrans = true;
// This will affect 'was' if JSON_END is missing
*(next + siz) = '\0';
transfer->intransient = get_intransient_siz(next, siz+1);
transfer->mvalue = transfer->intransient->str;
break;
}
}
if (!intrans) {
transfer->intransient = NULL;
if (siz >= sizeof(transfer->svalue)) {
transfer->mvalue = malloc(siz+1);
STRNCPYSIZ(transfer->mvalue, next,
siz+1);
} else {
STRNCPYSIZ(transfer->svalue, next,
siz+1);
transfer->mvalue = transfer->svalue;
}
}
add_to_ktree_nolock(msgline->trf_root, t_item);
k_add_head_nolock(msgline->trf_store, t_item);
@ -4179,12 +4239,25 @@ static enum cmd_values breakdown(K_ITEM **ml_item, char *buf, tv_t *now,
DATA_TRANSFER(transfer, t_item);
STRNCPY(transfer->name, data);
siz = strlen(eq) + 1;
if (siz > sizeof(transfer->svalue)) {
transfer->mvalue = malloc(siz);
STRNCPYSIZ(transfer->mvalue, eq, siz);
} else {
STRNCPYSIZ(transfer->svalue, eq, siz);
transfer->mvalue = transfer->svalue;
intrans = false;
for (i = 0; intransient_fields[i]; i++) {
if (strcmp(transfer->name,
intransient_fields[i]) == 0) {
intrans = true;
transfer->intransient = get_intransient_siz(eq, siz);
transfer->mvalue = transfer->intransient->str;
break;
}
}
if (!intrans) {
transfer->intransient = NULL;
if (siz > sizeof(transfer->svalue)) {
transfer->mvalue = malloc(siz);
STRNCPYSIZ(transfer->mvalue, eq, siz);
} else {
STRNCPYSIZ(transfer->svalue, eq, siz);
transfer->mvalue = transfer->svalue;
}
}
// Discard duplicates

43
src/ckdb.h
Unescape View File

@ -58,7 +58,7 @@
#define DB_VLOCK "1"
#define DB_VERSION "1.0.7"
#define CKDB_VERSION DB_VERSION"-2.438"
#define CKDB_VERSION DB_VERSION"-2.440"
#define WHERE_FFL " - from %s %s() line %d"
#define WHERE_FFL_HERE __FILE__, __func__, __LINE__
@ -1131,6 +1131,41 @@ typedef struct logqueue {
extern K_LIST *logqueue_free;
extern K_STORE *logqueue_store;
// NAMERAM - RAM for INTRANSIENT names - 2M per allocation
typedef struct nameram {
char rem[2*1024*1024-sizeof(void *)-sizeof(size_t)];
void *next;
size_t left;
} NAMERAM;
// Items never to be deleted and list never to be culled
#define ALLOC_NAMERAM 1
#define LIMIT_NAMERAM 0
#define INIT_NAMERAM(_item) INIT_GENERIC(_item, nameram)
#define DATA_NAMERAM(_var, _item) DATA_GENERIC(_var, _item, nameram, true)
extern K_LIST *nameram_free;
extern K_STORE *nameram_store;
// INTRANSIENT - a list of common strings, to avoid wasting RAM
typedef struct intransient {
char *str;
} INTRANSIENT;
/* Items never to be deleted and list never to be culled
* They are all created by the db load functions and breakdown */
#define ALLOC_INTRANSIENT 1024
#define LIMIT_INTRANSIENT 0
#define INIT_INTRANSIENT(_item) INIT_GENERIC(_item, intransient)
#define DATA_INTRANSIENT(_var, _item) DATA_GENERIC(_var, _item, intransient, true)
#define DATA_INTRANSIENT_NULL(_var, _item) DATA_GENERIC(_var, _item, intransient, false)
extern K_TREE *intransient_root;
extern K_LIST *intransient_free;
extern K_STORE *intransient_store;
extern char *intransient_fields[];
// MSGLINE
typedef struct msgline {
int which_cmds;
@ -1343,6 +1378,7 @@ typedef struct transfer {
char name[NAME_SIZE+1];
char svalue[VALUE_SIZE+1];
char *mvalue;
INTRANSIENT *intransient;
} TRANSFER;
// Suggest malloc use MMAP = largest under 2MB
@ -3074,6 +3110,11 @@ extern char *_hms_to_buf(time_t *data, char *buf, size_t siz, WHERE_FFL_ARGS);
// Convert to MM:SS
extern char *_ms_to_buf(time_t *data, char *buf, size_t siz, WHERE_FFL_ARGS);
extern cmp_t cmp_intransient(K_ITEM *a, K_ITEM *b);
#define get_intransient(_name) _get_intransient(_name, 0, WHERE_FFL_HERE)
#define get_intransient_siz(_name, _siz) \
_get_intransient(_name, _siz, WHERE_FFL_HERE)
extern INTRANSIENT *_get_intransient(char *name, size_t siz, WHERE_FFL_ARGS);
extern char *_transfer_data(K_ITEM *item, WHERE_FFL_ARGS);
extern void dsp_transfer(K_ITEM *item, FILE *stream);
extern cmp_t cmp_transfer(K_ITEM *a, K_ITEM *b);

94
src/ckdb_data.c
Unescape View File

@ -25,8 +25,13 @@ void free_msgline_data(K_ITEM *item, bool t_lock, bool t_cull)
t_item = STORE_HEAD_NOLOCK(msgline->trf_store);
while (t_item) {
DATA_TRANSFER(transfer, t_item);
if (transfer->mvalue != transfer->svalue)
FREENULL(transfer->mvalue);
if (transfer->mvalue != transfer->svalue) {
if (transfer->intransient) {
transfer->intransient = NULL;
transfer->mvalue = NULL;
} else
FREENULL(transfer->mvalue);
}
t_item = t_item->next;
}
if (t_lock)
@ -701,6 +706,91 @@ char *_ms_to_buf(time_t *data, char *buf, size_t siz, WHERE_FFL_ARGS)
return _data_to_buf(TYPE_MS, (void *)data, buf, siz, WHERE_FFL_PASS);
}
// order by name asc
cmp_t cmp_intransient(K_ITEM *a, K_ITEM *b)
{
INTRANSIENT *ia, *ib;
DATA_INTRANSIENT(ia, a);
DATA_INTRANSIENT(ib, b);
return CMP_STR(ia->str, ib->str);
}
INTRANSIENT *_get_intransient(char *name, size_t siz, WHERE_FFL_ARGS)
{
INTRANSIENT intransient, *in = NULL;
K_ITEM look, *i_item, *n_item;
NAMERAM *nameram = NULL;
K_TREE_CTX ctx[1];
char *buf;
bool new;
if (siz == 0)
siz = strlen(name) + 1;
if (siz > sizeof(nameram->rem)) {
char *st = NULL;
LOGEMERG("%s() ERR '%10s...' discarded - siz %d>%d" WHERE_FFL,
__func__, st = safe_text_nonull(name), (int)siz,
(int)sizeof(nameram->rem), WHERE_FFL_PASS);
name = EMPTY;
siz = 1;
}
intransient.str = name;
INIT_INTRANSIENT(&look);
look.data = (void *)(&intransient);
K_RLOCK(intransient_free);
i_item = _find_in_ktree(intransient_root, &look, ctx, false,
WHERE_FFL_PASS);
K_RUNLOCK(intransient_free);
if (i_item) {
DATA_INTRANSIENT(in, i_item);
return in;
}
K_WLOCK(intransient_free);
// Search again, to be thread safe
i_item = _find_in_ktree(intransient_root, &look, ctx, false,
WHERE_FFL_PASS);
if (i_item) {
DATA_INTRANSIENT(in, i_item);
} else {
new = false;
buf = NULL;
K_WLOCK(nameram_free);
if (nameram_store->count == 0)
new = true;
else {
n_item = STORE_WHEAD(nameram_store);
DATA_NAMERAM(nameram, n_item);
if (nameram->left < siz)
new = true;
}
if (new) {
n_item = k_unlink_head(nameram_free);
DATA_NAMERAM(nameram, n_item);
nameram->next = nameram->rem;
nameram->left = sizeof(nameram->rem);
k_add_head(nameram_store, n_item);
}
buf = nameram->next;
nameram->next += siz;
nameram->left -= siz;
K_WUNLOCK(nameram_free);
strcpy(buf, name);
i_item = k_unlink_head(intransient_free);
DATA_INTRANSIENT(in, i_item);
in->str = buf;
k_add_tail(intransient_store, i_item);
add_to_ktree(intransient_root, i_item);
}
K_WUNLOCK(intransient_free);
return in;
}
// For mutiple variable function calls that need the data
char *_transfer_data(K_ITEM *item, WHERE_FFL_ARGS)
{

Write Preview
Loading…
Cancel
Save