From 9aa281539d5b3cc2c4424f2d61c56ef625b04345 Mon Sep 17 00:00:00 2001 From: kanoi Date: Sat, 24 Jan 2015 13:28:55 +1100 Subject: [PATCH 1/4] ckdb - direct code a json decode replacement for breakdown() --- src/ckdb.c | 287 +++++++++++++++++++++++++++++++++-------------------- src/ckdb.h | 11 +- 2 files changed, 190 insertions(+), 108 deletions(-) diff --git a/src/ckdb.c b/src/ckdb.c index 6337d190..1158e248 100644 --- a/src/ckdb.c +++ b/src/ckdb.c @@ -1233,9 +1233,10 @@ static enum cmd_values breakdown(K_TREE **trf_root, K_STORE **trf_store, TRANSFER *transfer; K_TREE_CTX ctx[1]; K_ITEM *item; - char *cmdptr, *idptr, *next, *eq; - char *data = NULL; + char *cmdptr, *idptr, *next, *eq, *end, *was; + char *data = NULL, *tmp; bool noid = false; + size_t siz; *trf_root = NULL; *trf_store = NULL; @@ -1284,122 +1285,192 @@ static enum cmd_values breakdown(K_TREE **trf_root, K_STORE **trf_store, *trf_store = k_new_store(transfer_free); next = data; if (next && strncmp(next, JSON_TRANSFER, JSON_TRANSFER_LEN) == 0) { - json_t *json_data; - json_error_t err_val; - void *json_iter; - const char *json_key, *json_str; - json_t *json_value; - int json_typ; - size_t siz; - bool ok; - + // It's json next += JSON_TRANSFER_LEN; - json_data = json_loads(next, JSON_DISABLE_EOF_CHECK, &err_val); - if (!json_data) { - /* This REALLY shouldn't ever get an error since the input - * is a json generated string - * If that happens then dump lots of information */ - char *text = safe_text(next); - LOGERR("Json decode error from command: '%s' " - "json_err=(%d:%d:%d)%s:%s input='%s'", - cmd, err_val.line, err_val.column, - err_val.position, err_val.source, - err_val.text, text); - free(text); + was = next; + while (*next == ' ') + next++; + if (*next != JSON_BEGIN) { + LOGERR("JSON_BEGIN '%c' was: %.32s...", + JSON_BEGIN, tmp = safe_text(was)); + free(tmp); free(cmdptr); return CMD_REPLY; } - json_iter = json_object_iter(json_data); - K_WLOCK(transfer_free); - while (json_iter) { - json_key = json_object_iter_key(json_iter); - json_value = json_object_iter_value(json_iter); + next++; + // while we have a new quoted name + while (*next == JSON_STR) { + was = next; + end = ++next; + // look for the end quote + while (*end && *end != JSON_STR) { + if (*(end++) == JSON_ESC) + end++; + } + if (!*end) { + LOGERR("JSON name no trailing '%c' was: %.32s...", + JSON_STR, tmp = safe_text(was)); + free(tmp); + free(cmdptr); + return CMD_REPLY; + } + if (next == end) { + LOGERR("JSON zero length name was: %.32s...", + tmp = safe_text(was)); + free(tmp); + free(cmdptr); + return CMD_REPLY; + } + *(end++) = '\0'; + K_WLOCK(transfer_free); item = k_unlink_head(transfer_free); + K_WUNLOCK(transfer_free); DATA_TRANSFER(transfer, item); - ok = true; - json_typ = json_typeof(json_value); - switch (json_typ) { - case JSON_STRING: - json_str = json_string_value(json_value); - siz = strlen(json_str); - if (siz >= sizeof(transfer->svalue)) - transfer->mvalue = strdup(json_str); - else { - STRNCPY(transfer->svalue, json_str); - transfer->mvalue = transfer->svalue; + STRNCPY(transfer->name, next); + was = next = end; + while (*next == ' ') + next++; + // we have a name, now expect a value after it + if (*next != JSON_VALUE) { + LOGERR("JSON_VALUE '%c' '%s' was: %.32s...", + JSON_VALUE, transfer->name, + tmp = safe_text(was)); + free(tmp); + free(cmdptr); + K_WLOCK(transfer_free); + k_add_head(transfer_free, item); + K_WUNLOCK(transfer_free); + return CMD_REPLY; + } + was = ++next; + while (*next == ' ') + next++; + if (*next == JSON_STR) { + end = ++next; + // A quoted value must have a terminating quote + while (*end && *end != JSON_STR) { + if (*(end++) == JSON_ESC) + end++; } - break; - case JSON_REAL: - snprintf(transfer->svalue, - sizeof(transfer->svalue), - "%f", json_real_value(json_value)); - transfer->mvalue = transfer->svalue; - break; - case JSON_INTEGER: - snprintf(transfer->svalue, - sizeof(transfer->svalue), - "%"PRId64, - (int64_t)json_integer_value(json_value)); - transfer->mvalue = transfer->svalue; - break; - case JSON_TRUE: - case JSON_FALSE: - snprintf(transfer->svalue, - sizeof(transfer->svalue), - "%s", (json_typ == JSON_TRUE) ? - TRUE_STR : FALSE_STR); - transfer->mvalue = transfer->svalue; - break; - case JSON_ARRAY: - { - /* only one level array of strings for now (merkletree) - * ignore other data */ - size_t i, len, off, count = json_array_size(json_value); - json_t *json_element; - bool first = true; - - APPEND_REALLOC_INIT(transfer->mvalue, off, len); - for (i = 0; i < count; i++) { - json_element = json_array_get(json_value, i); - if (json_is_string(json_element)) { - json_str = json_string_value(json_element); - siz = strlen(json_str); - if (first) - first = false; - else { - APPEND_REALLOC(transfer->mvalue, - off, len, " "); - } - APPEND_REALLOC(transfer->mvalue, - off, len, json_str); - } else - LOGERR("%s() unhandled json type %d in array %s" - " in cmd %s", __func__, - json_typ, json_key, cmd); - } + if (!*end) { + LOGERR("JSON '%s' value was: %.32s...", + transfer->name, + tmp = safe_text(was)); + free(tmp); + free(cmdptr); + K_WLOCK(transfer_free); + k_add_head(transfer_free, item); + K_WUNLOCK(transfer_free); + return CMD_REPLY; } - break; - default: - LOGERR("%s() unhandled json type %d in cmd %s", - __func__, json_typ, cmd); - ok = false; - break; + if (end <= next+1) { + LOGERR("JSON '%s' zero length value " + "was: %.32s...", + transfer->name, + tmp = safe_text(was)); + free(tmp); + free(cmdptr); + K_WLOCK(transfer_free); + k_add_head(transfer_free, item); + K_WUNLOCK(transfer_free); + return CMD_REPLY; + } + siz = end - next; + end++; + } else if (*next == JSON_ARRAY) { + // Only merklehash for now + if (strcmp(transfer->name, "merklehash")) { + LOGERR("JSON '%s' can't be an array", + transfer->name); + free(cmdptr); + K_WLOCK(transfer_free); + k_add_head(transfer_free, item); + K_WUNLOCK(transfer_free); + return CMD_REPLY; + } + end = ++next; + /* No structure testing for now since we + * don't expect merklehash to get it wrong, + * and if it does, it will show up as some + * other error anyway */ + while (*end && *end != JSON_ARRAY_END) + end++; + if (end < next+1) { + LOGERR("JSON '%s' zero length value " + "was: %.32s...", + transfer->name, + tmp = safe_text(was)); + free(tmp); + free(cmdptr); + K_WLOCK(transfer_free); + k_add_head(transfer_free, item); + K_WUNLOCK(transfer_free); + return CMD_REPLY; + } + siz = end - next; + end++; + } else { + end = next; + // A non quoted value ends on SEP, END or space + while (*end && *end != JSON_SEP && + *end != JSON_END && *end != ' ') { + end++; + } + if (!*end) { + LOGERR("JSON '%s' value was: %.32s...", + transfer->name, + tmp = safe_text(was)); + free(tmp); + free(cmdptr); + K_WLOCK(transfer_free); + k_add_head(transfer_free, item); + K_WUNLOCK(transfer_free); + return CMD_REPLY; + } + if (next == end) { + LOGERR("JSON '%s' zero length value " + "was: %.32s...", + transfer->name, + tmp = safe_text(was)); + free(tmp); + free(cmdptr); + K_WLOCK(transfer_free); + k_add_head(transfer_free, item); + K_WUNLOCK(transfer_free); + return CMD_REPLY; + } + siz = end - next; } - - if (ok) - STRNCPY(transfer->name, json_key); - if (!ok || find_in_ktree(*trf_root, item, cmp_transfer, ctx)) { - if (transfer->mvalue != transfer->svalue) - FREENULL(transfer->mvalue); - k_add_head(transfer_free, item); + if (siz >= sizeof(transfer->svalue)) { + transfer->mvalue = malloc(siz+1); + STRNCPYSIZ(transfer->mvalue, next, siz+1); } else { - *trf_root = add_to_ktree(*trf_root, item, cmp_transfer); - k_add_head(*trf_store, item); + STRNCPYSIZ(transfer->svalue, next, siz+1); + transfer->mvalue = transfer->svalue; + } + *trf_root = add_to_ktree(*trf_root, item, cmp_transfer); + k_add_head(*trf_store, item); + + // find the separator then move to the next name + next = end; + while (*next == ' ') + next++; + if (*next == JSON_SEP) { + next++; + while (*next == ' ') + next++; } - json_iter = json_object_iter_next(json_data, json_iter); } - K_WUNLOCK(transfer_free); - json_decref(json_data); + if (*next != JSON_END) { + LOGERR("JSON_END '%c' was: %.32s...", + JSON_END, tmp = safe_text(next)); + free(tmp); + free(cmdptr); + K_WLOCK(transfer_free); + k_add_head(transfer_free, item); + K_WUNLOCK(transfer_free); + return CMD_REPLY; + } } else { K_WLOCK(transfer_free); while (next && *next) { @@ -1433,8 +1504,10 @@ static enum cmd_values breakdown(K_TREE **trf_root, K_STORE **trf_store, } if (ckdb_cmds[*which_cmds].createdate) { item = require_name(*trf_root, "createdate", 10, NULL, reply, sizeof(reply)); - if (!item) + if (!item) { + free(cmdptr); return CMD_REPLY; + } DATA_TRANSFER(transfer, item); txt_to_ctv("createdate", transfer->mvalue, cd, sizeof(*cd)); diff --git a/src/ckdb.h b/src/ckdb.h index 27c3a55a..f50ea45d 100644 --- a/src/ckdb.h +++ b/src/ckdb.h @@ -52,7 +52,7 @@ #define DB_VLOCK "1" #define DB_VERSION "0.9.6" -#define CKDB_VERSION DB_VERSION"-0.841" +#define CKDB_VERSION DB_VERSION"-0.900" #define WHERE_FFL " - from %s %s() line %d" #define WHERE_FFL_HERE __FILE__, __func__, __LINE__ @@ -268,6 +268,15 @@ extern bool everyone_die; #define JSON_TRANSFER "json=" #define JSON_TRANSFER_LEN (sizeof(JSON_TRANSFER)-1) +#define JSON_BEGIN '{' +// Arrays have limited support in breakdown() +#define JSON_ARRAY '[' +#define JSON_ARRAY_END ']' +#define JSON_STR '"' +#define JSON_VALUE ':' +#define JSON_SEP ',' +#define JSON_END '}' +#define JSON_ESC '\\' // Methods for sharelog (common function for all) #define STR_WORKINFO "workinfo" From 373bf7749fd7feceeda4f19e9db91317373d5f2d Mon Sep 17 00:00:00 2001 From: kanoi Date: Sat, 24 Jan 2015 15:07:23 +1100 Subject: [PATCH 2/4] ckdb - report reload total lines per second --- src/ckdb.c | 13 ++++++++++--- src/ckdb.h | 2 +- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/src/ckdb.c b/src/ckdb.c index 1158e248..770a8c5a 100644 --- a/src/ckdb.c +++ b/src/ckdb.c @@ -2995,7 +2995,8 @@ static bool reload_from(tv_t *start) bool finished = false, matched = false, ret = true, ok, apipe = false; char *filename = NULL; uint64_t count, total; - tv_t now; + tv_t now, begin; + double diff; FILE *fp = NULL; reload_buf = malloc(MAX_READ); @@ -3016,6 +3017,7 @@ static bool reload_from(tv_t *start) quithere(1, "Failed to open '%s'", filename); setnow(&now); + copy_tv(&begin, &now); tvs_to_buf(&now, run, sizeof(run)); snprintf(reload_buf, MAX_READ, "reload.%s.s0", run); LOGQUE(reload_buf); @@ -3099,12 +3101,17 @@ static bool reload_from(tv_t *start) PQfinish(conn); + setnow(&now); + diff = tvdiff(&now, &begin); + if (diff == 0) + diff = 1; + snprintf(reload_buf, MAX_READ, "reload.%s.%"PRIu64, run, total); LOGQUE(reload_buf); - LOGWARNING("%s(): read %d file%s, total %"PRIu64" line%s", + LOGWARNING("%s(): read %d file%s, total %"PRIu64" line%s %.2f/s", __func__, processing, processing == 1 ? "" : "s", - total, total == 1 ? "" : "s"); + total, total == 1 ? "" : "s", (total / diff)); if (everyone_die) return true; diff --git a/src/ckdb.h b/src/ckdb.h index f50ea45d..dd85c418 100644 --- a/src/ckdb.h +++ b/src/ckdb.h @@ -52,7 +52,7 @@ #define DB_VLOCK "1" #define DB_VERSION "0.9.6" -#define CKDB_VERSION DB_VERSION"-0.900" +#define CKDB_VERSION DB_VERSION"-0.901" #define WHERE_FFL " - from %s %s() line %d" #define WHERE_FFL_HERE __FILE__, __func__, __LINE__ From 73cc2eb40591aa02c69fbd798ffffe4d384f3708 Mon Sep 17 00:00:00 2001 From: kanoi Date: Sat, 24 Jan 2015 15:21:39 +1100 Subject: [PATCH 3/4] ckdb - breakdown() allow zero length string values --- src/ckdb.c | 12 ------------ src/ckdb.h | 2 +- 2 files changed, 1 insertion(+), 13 deletions(-) diff --git a/src/ckdb.c b/src/ckdb.c index 770a8c5a..70e1380a 100644 --- a/src/ckdb.c +++ b/src/ckdb.c @@ -1363,18 +1363,6 @@ static enum cmd_values breakdown(K_TREE **trf_root, K_STORE **trf_store, K_WUNLOCK(transfer_free); return CMD_REPLY; } - if (end <= next+1) { - LOGERR("JSON '%s' zero length value " - "was: %.32s...", - transfer->name, - tmp = safe_text(was)); - free(tmp); - free(cmdptr); - K_WLOCK(transfer_free); - k_add_head(transfer_free, item); - K_WUNLOCK(transfer_free); - return CMD_REPLY; - } siz = end - next; end++; } else if (*next == JSON_ARRAY) { diff --git a/src/ckdb.h b/src/ckdb.h index dd85c418..4944ff81 100644 --- a/src/ckdb.h +++ b/src/ckdb.h @@ -52,7 +52,7 @@ #define DB_VLOCK "1" #define DB_VERSION "0.9.6" -#define CKDB_VERSION DB_VERSION"-0.901" +#define CKDB_VERSION DB_VERSION"-0.902" #define WHERE_FFL " - from %s %s() line %d" #define WHERE_FFL_HERE __FILE__, __func__, __LINE__ From 55595f3855c962c9ed51fccf4ee5c503a3719f24 Mon Sep 17 00:00:00 2001 From: kanoi Date: Sat, 24 Jan 2015 17:57:38 +1100 Subject: [PATCH 4/4] ckdb/php - put the pool block # in the web block information --- pool/page_blocks.php | 5 ++++- src/ckdb.h | 2 +- src/ckdb_cmd.c | 15 +++++++++++++-- 3 files changed, 18 insertions(+), 4 deletions(-) diff --git a/pool/page_blocks.php b/pool/page_blocks.php index 8e860ef2..a0d6e92f 100644 --- a/pool/page_blocks.php +++ b/pool/page_blocks.php @@ -96,10 +96,10 @@ function doblocks($data, $user) $csv = "Sequence,Height,Status,Timestamp,DiffAcc,NetDiff,Hash\n"; if ($ans['STATUS'] == 'ok') { + $tot = $ans['tot']; $count = $ans['rows']; for ($i = 0; $i < $count; $i++) { - $seq = $i + 1; if (($i % 2) == 0) $row = 'even'; else @@ -114,7 +114,10 @@ function doblocks($data, $user) { $ex = 's'; $orph = true; + $seq = ''; } + else + $seq = $tot--; if ($stat == '1-Confirm') { if (isset($data['info']['lastheight'])) diff --git a/src/ckdb.h b/src/ckdb.h index 4944ff81..56272881 100644 --- a/src/ckdb.h +++ b/src/ckdb.h @@ -52,7 +52,7 @@ #define DB_VLOCK "1" #define DB_VERSION "0.9.6" -#define CKDB_VERSION DB_VERSION"-0.902" +#define CKDB_VERSION DB_VERSION"-0.903" #define WHERE_FFL " - from %s %s() line %d" #define WHERE_FFL_HERE __FILE__, __func__, __LINE__ diff --git a/src/ckdb_cmd.c b/src/ckdb_cmd.c index 02ebd6b0..08001bdf 100644 --- a/src/ckdb_cmd.c +++ b/src/ckdb_cmd.c @@ -819,7 +819,7 @@ static char *cmd_blocklist(__maybe_unused PGconn *conn, char *cmd, char *id, size_t len, off; int32_t height = -1; tv_t first_cd = {0,0}; - int rows; + int rows, tot; LOGDEBUG("%s(): cmd '%s'", __func__, cmd); @@ -828,6 +828,16 @@ static char *cmd_blocklist(__maybe_unused PGconn *conn, char *cmd, char *id, rows = 0; K_RLOCK(blocks_free); b_item = last_in_ktree(blocks_root, ctx); + tot = 0; + while (b_item) { + DATA_BLOCKS(blocks, b_item); + if (CURRENT(&(blocks->expirydate))) { + if (blocks->confirmed[0] != BLOCKS_ORPHAN) + tot++; + } + b_item = prev_in_ktree(ctx); + } + b_item = last_in_ktree(blocks_root, ctx); while (b_item && rows < 42) { DATA_BLOCKS(blocks, b_item); if (height != blocks->height) { @@ -914,7 +924,8 @@ static char *cmd_blocklist(__maybe_unused PGconn *conn, char *cmd, char *id, } K_RUNLOCK(blocks_free); snprintf(tmp, sizeof(tmp), - "rows=%d%cflds=%s%c", + "tot=%d%crows=%d%cflds=%s%c", + tot, FLDSEP, rows, FLDSEP, "height,blockhash,nonce,reward,workername,firstcreatedate," "createdate,status,diffacc,diffinv,shareacc,shareinv,elapsed,"