From f2a0d3636f8107244bc2251514ab9d0c1aef997e Mon Sep 17 00:00:00 2001 From: ckolivas Date: Wed, 3 Feb 2016 13:49:06 +1100 Subject: [PATCH 1/5] Rename transactions to txns to not clash with the transactions value returned by getblocktemplate --- src/bitcoin.c | 22 +++++++++++----------- src/bitcoin.h | 2 +- src/stratifier.c | 30 +++++++++++++++--------------- 3 files changed, 27 insertions(+), 27 deletions(-) diff --git a/src/bitcoin.c b/src/bitcoin.c index 7f9b5f23..92bb7919 100644 --- a/src/bitcoin.c +++ b/src/bitcoin.c @@ -89,18 +89,18 @@ static bool gbt_merkle_bins(gbtbase_t *gbt, json_t *transaction_arr) dealloc(gbt->txn_data); dealloc(gbt->txn_hashes); - gbt->transactions = 0; + gbt->txns = 0; gbt->merkles = 0; - gbt->transactions = json_array_size(transaction_arr); - binlen = gbt->transactions * 32 + 32; + gbt->txns = json_array_size(transaction_arr); + binlen = gbt->txns * 32 + 32; hashbin = alloca(binlen + 32); memset(hashbin, 0, 32); binleft = binlen / 32; - if (gbt->transactions) { + if (gbt->txns) { int len = 1, ofs = 0; const char *txn; - for (i = 0; i < gbt->transactions; i++) { + for (i = 0; i < gbt->txns; i++) { arr_val = json_array_get(transaction_arr, i); txn = json_string_value(json_object_get(arr_val, "data")); if (!txn) { @@ -111,10 +111,10 @@ static bool gbt_merkle_bins(gbtbase_t *gbt, json_t *transaction_arr) } gbt->txn_data = ckzalloc(len + 1); - gbt->txn_hashes = ckzalloc(gbt->transactions * 65 + 1); - memset(gbt->txn_hashes, 0x20, gbt->transactions * 65); // Spaces + gbt->txn_hashes = ckzalloc(gbt->txns * 65 + 1); + memset(gbt->txn_hashes, 0x20, gbt->txns * 65); // Spaces - for (i = 0; i < gbt->transactions; i++) { + for (i = 0; i < gbt->txns; i++) { char binswap[32]; const char *hash; @@ -168,7 +168,7 @@ static bool gbt_merkle_bins(gbtbase_t *gbt, json_t *transaction_arr) binlen = binleft * 32; } } - LOGINFO("Stored %d transactions", gbt->transactions); + LOGINFO("Stored %d transactions", gbt->txns); return true; } @@ -259,8 +259,8 @@ bool gen_gbtbase(connsock_t *cs, gbtbase_t *gbt) json_object_set_new_nocheck(gbt->json, "flags", json_string_nocheck(gbt->flags)); gbt_merkle_bins(gbt, transaction_arr); - json_object_set_new_nocheck(gbt->json, "transactions", json_integer(gbt->transactions)); - if (gbt->transactions) { + json_object_set_new_nocheck(gbt->json, "txns", json_integer(gbt->txns)); + if (gbt->txns) { json_object_set_new_nocheck(gbt->json, "txn_data", json_string_nocheck(gbt->txn_data)); json_object_set_new_nocheck(gbt->json, "txn_hashes", json_string_nocheck(gbt->txn_hashes)); } diff --git a/src/bitcoin.h b/src/bitcoin.h index cb51457f..1404c538 100644 --- a/src/bitcoin.h +++ b/src/bitcoin.h @@ -22,7 +22,7 @@ struct gbtbase { uint64_t coinbasevalue; int height; char *flags; - int transactions; + int txns; char *txn_data; char *txn_hashes; int merkles; diff --git a/src/stratifier.c b/src/stratifier.c index cd21449e..7c73a09b 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -106,7 +106,7 @@ struct workbase { uint64_t coinbasevalue; int height; char *flags; - int transactions; + int txns; char *txn_data; char *txn_hashes; int merkles; @@ -877,8 +877,8 @@ static void send_node_workinfo(sdata_t *sdata, const workbase_t *wb) json_set_int(wb_val, "coinbasevalue", wb->coinbasevalue); json_set_int(wb_val, "height", wb->height); json_set_string(wb_val, "flags", wb->flags); - json_set_int(wb_val, "transactions", wb->transactions); - if (likely(wb->transactions)) + json_set_int(wb_val, "txns", wb->txns); + if (likely(wb->txns)) json_set_string(wb_val, "txn_data", wb->txn_data); /* We don't need txn_hashes */ json_set_int(wb_val, "merkles", wb->merkles); @@ -1158,8 +1158,8 @@ retry: json_uint64cpy(&wb->coinbasevalue, val, "coinbasevalue"); json_intcpy(&wb->height, val, "height"); json_strdup(&wb->flags, val, "flags"); - json_intcpy(&wb->transactions, val, "transactions"); - if (wb->transactions) { + json_intcpy(&wb->txns, val, "txns"); + if (wb->txns) { json_strdup(&wb->txn_data, val, "txn_data"); json_strdup(&wb->txn_hashes, val, "txn_hashes"); } else @@ -1230,8 +1230,8 @@ static void add_node_base(ckpool_t *ckp, json_t *val) json_uint64cpy(&wb->coinbasevalue, val, "coinbasevalue"); json_intcpy(&wb->height, val, "height"); json_strdup(&wb->flags, val, "flags"); - json_intcpy(&wb->transactions, val, "transactions"); - if (wb->transactions) + json_intcpy(&wb->txns, val, "txns"); + if (wb->txns) json_strdup(&wb->txn_data, val, "txn_data"); json_intcpy(&wb->merkles, val, "merkles"); wb->merkle_array = json_object_dup(val, "merklehash"); @@ -1433,7 +1433,7 @@ static void process_block(ckpool_t *ckp, const workbase_t *wb, const char *coinbase, const int cblen, const uchar *data, const uchar *hash, uchar *swap32, char *blockhash) { - int transactions = wb->transactions + 1; + int txns = wb->txns + 1; char *gbt_block, varint[12]; char hexcoinbase[1024]; @@ -1444,17 +1444,17 @@ process_block(ckpool_t *ckp, const workbase_t *wb, const char *coinbase, const i /* Message format: "submitblock:hash,data" */ sprintf(gbt_block, "submitblock:%s,", blockhash); __bin2hex(gbt_block + 12 + 64 + 1, data, 80); - if (transactions < 0xfd) { - uint8_t val8 = transactions; + if (txns < 0xfd) { + uint8_t val8 = txns; __bin2hex(varint, (const unsigned char *)&val8, 1); - } else if (transactions <= 0xffff) { - uint16_t val16 = htole16(transactions); + } else if (txns <= 0xffff) { + uint16_t val16 = htole16(txns); strcat(gbt_block, "fd"); __bin2hex(varint, (const unsigned char *)&val16, 2); } else { - uint32_t val32 = htole32(transactions); + uint32_t val32 = htole32(txns); strcat(gbt_block, "fe"); __bin2hex(varint, (const unsigned char *)&val32, 4); @@ -1462,7 +1462,7 @@ process_block(ckpool_t *ckp, const workbase_t *wb, const char *coinbase, const i strcat(gbt_block, varint); __bin2hex(hexcoinbase, coinbase, cblen); strcat(gbt_block, hexcoinbase); - if (wb->transactions) + if (wb->txns) realloc_strcat(&gbt_block, wb->txn_data); send_generator(ckp, gbt_block, GEN_PRIORITY); if (ckp->remote) @@ -6474,7 +6474,7 @@ static int transactions_by_jobid(sdata_t *sdata, const int64_t id) ck_rlock(&sdata->workbase_lock); HASH_FIND_I64(sdata->workbases, &id, wb); if (wb) - ret = wb->transactions; + ret = wb->txns; ck_runlock(&sdata->workbase_lock); return ret; From 8702c028226c33aa469f9a5e34c1801f58592068 Mon Sep 17 00:00:00 2001 From: ckolivas Date: Wed, 3 Feb 2016 15:00:06 +1100 Subject: [PATCH 2/5] Move merkle tree generation to the stratifier along with all transaction data to allow transaction data to be used directly in future code --- src/bitcoin.c | 112 ++--------------------------------------------- src/bitcoin.h | 2 +- src/stratifier.c | 112 ++++++++++++++++++++++++++++++++++++++--------- 3 files changed, 96 insertions(+), 130 deletions(-) diff --git a/src/bitcoin.c b/src/bitcoin.c index 92bb7919..016a543f 100644 --- a/src/bitcoin.c +++ b/src/bitcoin.c @@ -79,99 +79,6 @@ out: return ret; } -/* Distill down a set of transactions into an efficient tree arrangement for - * stratum messages and fast work assembly. */ -static bool gbt_merkle_bins(gbtbase_t *gbt, json_t *transaction_arr) -{ - int i, j, binleft, binlen; - json_t *arr_val; - uchar *hashbin; - - dealloc(gbt->txn_data); - dealloc(gbt->txn_hashes); - gbt->txns = 0; - gbt->merkles = 0; - gbt->txns = json_array_size(transaction_arr); - binlen = gbt->txns * 32 + 32; - hashbin = alloca(binlen + 32); - memset(hashbin, 0, 32); - binleft = binlen / 32; - if (gbt->txns) { - int len = 1, ofs = 0; - const char *txn; - - for (i = 0; i < gbt->txns; i++) { - arr_val = json_array_get(transaction_arr, i); - txn = json_string_value(json_object_get(arr_val, "data")); - if (!txn) { - LOGWARNING("json_string_value fail - cannot find transaction data"); - return false; - } - len += strlen(txn); - } - - gbt->txn_data = ckzalloc(len + 1); - gbt->txn_hashes = ckzalloc(gbt->txns * 65 + 1); - memset(gbt->txn_hashes, 0x20, gbt->txns * 65); // Spaces - - for (i = 0; i < gbt->txns; i++) { - char binswap[32]; - const char *hash; - - arr_val = json_array_get(transaction_arr, i); - hash = json_string_value(json_object_get(arr_val, "hash")); - txn = json_string_value(json_object_get(arr_val, "data")); - len = strlen(txn); - memcpy(gbt->txn_data + ofs, txn, len); - ofs += len; -#if 0 - /* In case we ever want to be a gbt poolproxy */ - if (!hash) { - char *txn_bin; - int txn_len; - - txn_len = len / 2; - txn_bin = ckalloc(txn_len); - hex2bin(txn_bin, txn, txn_len); - /* This is needed for pooled mining since only - * transaction data and not hashes are sent */ - gen_hash(txn_bin, hashbin + 32 + 32 * i, txn_len); - continue; - } -#endif - if (!hex2bin(binswap, hash, 32)) { - LOGERR("Failed to hex2bin hash in gbt_merkle_bins"); - return false; - } - memcpy(gbt->txn_hashes + i * 65, hash, 64); - bswap_256(hashbin + 32 + 32 * i, binswap); - } - } - if (binleft > 1) { - while (42) { - uchar merklebin[32]; - - if (binleft == 1) - break; - memcpy(merklebin, hashbin + 32, 32); - __bin2hex(&gbt->merklehash[gbt->merkles][0], merklebin, 32); - LOGDEBUG("MH%d %s",gbt->merkles, &gbt->merklehash[gbt->merkles][0]); - gbt->merkles++; - if (binleft % 2) { - memcpy(hashbin + binlen, hashbin + binlen - 32, 32); - binlen += 32; - binleft++; - } - for (i = 32, j = 64; j < binlen; i += 32, j += 64) - gen_hash(hashbin + j, hashbin + i, 64); - binleft /= 2; - binlen = binleft * 32; - } - } - LOGINFO("Stored %d transactions", gbt->txns); - return true; -} - static const char *gbt_req = "{\"method\": \"getblocktemplate\", \"params\": [{\"capabilities\": [\"coinbasetxn\", \"workid\", \"coinbase/append\"]}]}\n"; /* Request getblocktemplate from bitcoind already connected with a connsock_t @@ -179,7 +86,7 @@ static const char *gbt_req = "{\"method\": \"getblocktemplate\", \"params\": [{\ * required to assemble a mining template, storing it in a gbtbase_t structure */ bool gen_gbtbase(connsock_t *cs, gbtbase_t *gbt) { - json_t *transaction_arr, *coinbase_aux, *res_val, *val, *array; + json_t *txn_array, *coinbase_aux, *res_val, *val; const char *previousblockhash; char hash_swap[32], tmp[32]; uint64_t coinbasevalue; @@ -189,7 +96,6 @@ bool gen_gbtbase(connsock_t *cs, gbtbase_t *gbt) int version; int curtime; int height; - int i; bool ret = false; val = json_rpc_call(cs, gbt_req); @@ -205,7 +111,7 @@ bool gen_gbtbase(connsock_t *cs, gbtbase_t *gbt) previousblockhash = json_string_value(json_object_get(res_val, "previousblockhash")); target = json_string_value(json_object_get(res_val, "target")); - transaction_arr = json_object_get(res_val, "transactions"); + txn_array = json_object_get(res_val, "transactions"); version = json_integer_value(json_object_get(res_val, "version")); curtime = json_integer_value(json_object_get(res_val, "curtime")); bits = json_string_value(json_object_get(res_val, "bits")); @@ -258,19 +164,7 @@ bool gen_gbtbase(connsock_t *cs, gbtbase_t *gbt) gbt->flags = strdup(flags); json_object_set_new_nocheck(gbt->json, "flags", json_string_nocheck(gbt->flags)); - gbt_merkle_bins(gbt, transaction_arr); - json_object_set_new_nocheck(gbt->json, "txns", json_integer(gbt->txns)); - if (gbt->txns) { - json_object_set_new_nocheck(gbt->json, "txn_data", json_string_nocheck(gbt->txn_data)); - json_object_set_new_nocheck(gbt->json, "txn_hashes", json_string_nocheck(gbt->txn_hashes)); - } - json_object_set_new_nocheck(gbt->json, "merkles", json_integer(gbt->merkles)); - if (gbt->merkles) { - array = json_array(); - for (i = 0; i < gbt->merkles; i++) - json_array_append_new(array, json_string_nocheck(&gbt->merklehash[i][0])); - json_object_set_new_nocheck(gbt->json, "merklehash", array); - } + json_object_set_new_nocheck(gbt->json, "transactions", json_deep_copy(txn_array)); ret = true; out: diff --git a/src/bitcoin.h b/src/bitcoin.h index 1404c538..9ba4534d 100644 --- a/src/bitcoin.h +++ b/src/bitcoin.h @@ -1,5 +1,5 @@ /* - * Copyright 2014 Con Kolivas + * Copyright 2014-2016 Con Kolivas * * This program is free software; you can redistribute it and/or modify it * under the terms of the GNU General Public License as published by the Free diff --git a/src/stratifier.c b/src/stratifier.c index 7c73a09b..b3463f91 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -1104,6 +1104,95 @@ struct update_req { static void broadcast_ping(sdata_t *sdata); +/* Distill down a set of transactions into an efficient tree arrangement for + * stratum messages and fast work assembly. */ +static void wb_merkle_bins(workbase_t *wb, json_t *txn_array) +{ + int i, j, binleft, binlen; + json_t *arr_val; + uchar *hashbin; + + wb->txns = json_array_size(txn_array); + wb->merkles = 0; + binlen = wb->txns * 32 + 32; + hashbin = alloca(binlen + 32); + memset(hashbin, 0, 32); + binleft = binlen / 32; + if (wb->txns) { + int len = 1, ofs = 0; + const char *txn; + + for (i = 0; i < wb->txns; i++) { + arr_val = json_array_get(txn_array, i); + txn = json_string_value(json_object_get(arr_val, "data")); + if (!txn) { + LOGWARNING("json_string_value fail - cannot find transaction data"); + return; + } + len += strlen(txn); + } + + wb->txn_data = ckzalloc(len + 1); + wb->txn_hashes = ckzalloc(wb->txns * 65 + 1); + memset(wb->txn_hashes, 0x20, wb->txns * 65); // Spaces + + for (i = 0; i < wb->txns; i++) { + char binswap[32]; + const char *hash; + + arr_val = json_array_get(txn_array, i); + hash = json_string_value(json_object_get(arr_val, "hash")); + txn = json_string_value(json_object_get(arr_val, "data")); + len = strlen(txn); + memcpy(wb->txn_data + ofs, txn, len); + ofs += len; +#if 0 + /* In case we ever want to be a gbt poolproxy */ + if (!hash) { + char *txn_bin; + int txn_len; + + txn_len = len / 2; + txn_bin = ckalloc(txn_len); + hex2bin(txn_bin, txn, txn_len); + /* This is needed for pooled mining since only + * transaction data and not hashes are sent */ + gen_hash(txn_bin, hashbin + 32 + 32 * i, txn_len); + continue; + } +#endif + if (!hex2bin(binswap, hash, 32)) { + LOGERR("Failed to hex2bin hash in gbt_merkle_bins"); + return; + } + memcpy(wb->txn_hashes + i * 65, hash, 64); + bswap_256(hashbin + 32 + 32 * i, binswap); + } + } else + wb->txn_hashes = ckzalloc(1); + if (binleft > 1) { + while (42) { + uchar merklebin[32]; + + if (binleft == 1) + break; + memcpy(merklebin, hashbin + 32, 32); + __bin2hex(&wb->merklehash[wb->merkles][0], merklebin, 32); + LOGDEBUG("MH%d %s",wb->merkles, &wb->merklehash[wb->merkles][0]); + wb->merkles++; + if (binleft % 2) { + memcpy(hashbin + binlen, hashbin + binlen - 32, 32); + binlen += 32; + binleft++; + } + for (i = 32, j = 64; j < binlen; i += 32, j += 64) + gen_hash(hashbin + j, hashbin + i, 64); + binleft /= 2; + binlen = binleft * 32; + } + } +} + /* This function assumes it will only receive a valid json gbt base template * since checking should have been done earlier, and creates the base template * for generating work templates. */ @@ -1113,11 +1202,11 @@ static void *do_update(void *arg) int prio = ur->prio, retries = 0; ckpool_t *ckp = ur->ckp; sdata_t *sdata = ckp->data; + json_t *val, *txn_array; bool new_block = false; bool ret = false; workbase_t *wb; time_t now_t; - json_t *val; char *buf; pthread_detach(pthread_self()); @@ -1158,25 +1247,8 @@ retry: json_uint64cpy(&wb->coinbasevalue, val, "coinbasevalue"); json_intcpy(&wb->height, val, "height"); json_strdup(&wb->flags, val, "flags"); - json_intcpy(&wb->txns, val, "txns"); - if (wb->txns) { - json_strdup(&wb->txn_data, val, "txn_data"); - json_strdup(&wb->txn_hashes, val, "txn_hashes"); - } else - wb->txn_hashes = ckzalloc(1); - json_intcpy(&wb->merkles, val, "merkles"); - wb->merkle_array = json_array(); - if (wb->merkles) { - json_t *arr; - int i; - - arr = json_object_get(val, "merklehash"); - for (i = 0; i < wb->merkles; i++) { - strcpy(&wb->merklehash[i][0], json_string_value(json_array_get(arr, i))); - hex2bin(&wb->merklebin[i][0], &wb->merklehash[i][0], 32); - json_array_append_new(wb->merkle_array, json_string(&wb->merklehash[i][0])); - } - } + txn_array = json_object_get(val, "transactions"); + wb_merkle_bins(wb, txn_array); json_decref(val); generate_coinbase(ckp, wb); From c43432ea7e8a6185aa633b34c4c159352ec9b154 Mon Sep 17 00:00:00 2001 From: ckolivas Date: Wed, 3 Feb 2016 16:31:07 +1100 Subject: [PATCH 3/5] Add transactions to a hashtable to know which are new, which need propagating, and which are no longer needed --- src/stratifier.c | 66 +++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 63 insertions(+), 3 deletions(-) diff --git a/src/stratifier.c b/src/stratifier.c index b3463f91..4fc6609a 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -375,6 +375,16 @@ struct session { char address[INET6_ADDRSTRLEN]; }; +typedef struct txntable txntable_t; + +struct txntable { + UT_hash_handle hh; + int id; + char hash[68]; + const char *data; + int refcount; +}; + #define ID_AUTH 0 #define ID_WORKINFO 1 #define ID_AGEWORKINFO 2 @@ -447,6 +457,7 @@ struct stratifier_data { workbase_t *workbases; workbase_t *current_workbase; int workbases_generated; + txntable_t *txns; /* Semaphore to serialise calls to add_base */ sem_t update_sem; @@ -1104,11 +1115,37 @@ struct update_req { static void broadcast_ping(sdata_t *sdata); +/* Build a hashlist of all transactions, allowing us to compare with the list of + * existing transactions to determine which need to be propagated */ +static void add_txn(sdata_t *sdata, txntable_t *txns, const char *hash, const char *data) +{ + bool found = false; + txntable_t *txn; + + ck_rlock(&sdata->workbase_lock); + HASH_FIND_STR(sdata->txns, hash, txn); + if (txn) { + txn->refcount++; + found = true; + } + ck_runlock(&sdata->workbase_lock); + if (found) + return; + + txn = ckzalloc(sizeof(txntable_t)); + memcpy(txn->hash, hash, 65); + /* Note that data is pointing to a string in a json struture which will + * be destroyed so we can only use the data value until then. */ + txn->data = data; + HASH_ADD_STR(txns, hash, txn); +} + /* Distill down a set of transactions into an efficient tree arrangement for * stratum messages and fast work assembly. */ -static void wb_merkle_bins(workbase_t *wb, json_t *txn_array) +static void wb_merkle_bins(sdata_t *sdata, workbase_t *wb, json_t *txn_array) { - int i, j, binleft, binlen; + int i, j, binleft, binlen, added = 0, purged = 0; + txntable_t *txns = NULL, *tmp, *tmpa; json_t *arr_val; uchar *hashbin; @@ -1143,6 +1180,7 @@ static void wb_merkle_bins(workbase_t *wb, json_t *txn_array) arr_val = json_array_get(txn_array, i); hash = json_string_value(json_object_get(arr_val, "hash")); txn = json_string_value(json_object_get(arr_val, "data")); + add_txn(sdata, txns, hash, txn); len = strlen(txn); memcpy(wb->txn_data + ofs, txn, len); ofs += len; @@ -1191,6 +1229,28 @@ static void wb_merkle_bins(workbase_t *wb, json_t *txn_array) binlen = binleft * 32; } } + + ck_wlock(&sdata->workbase_lock); + HASH_ITER(hh, sdata->txns, tmp, tmpa) { + if (tmp->refcount--) + continue; + HASH_DEL(sdata->txns, tmp); + dealloc(tmp); + purged++; + } + HASH_ITER(hh, txns, tmp, tmpa) { + /* Propagate transaction here */ + /* Move to the sdata transaction table */ + HASH_DEL(txns, tmp); + HASH_ADD_STR(sdata->txns, data, tmp); + /* Empty data once used to not dereference since the json structure + * will be destroyed. */ + tmp->data = NULL; + added++; + } + ck_wunlock(&sdata->workbase_lock); + + LOGDEBUG("Stratifier added %d transactions and purged %d", added, purged); } /* This function assumes it will only receive a valid json gbt base template @@ -1248,7 +1308,7 @@ retry: json_intcpy(&wb->height, val, "height"); json_strdup(&wb->flags, val, "flags"); txn_array = json_object_get(val, "transactions"); - wb_merkle_bins(wb, txn_array); + wb_merkle_bins(sdata, wb, txn_array); json_decref(val); generate_coinbase(ckp, wb); From 147175f2955bd6ad54637ded11d1f48dc335a46b Mon Sep 17 00:00:00 2001 From: ckolivas Date: Wed, 3 Feb 2016 16:43:24 +1100 Subject: [PATCH 4/5] Revert "Add transactions to a hashtable to know which are new, which need propagating, and which are no longer needed" This reverts commit c43432ea7e8a6185aa633b34c4c159352ec9b154. --- src/stratifier.c | 66 +++--------------------------------------------- 1 file changed, 3 insertions(+), 63 deletions(-) diff --git a/src/stratifier.c b/src/stratifier.c index 4fc6609a..b3463f91 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -375,16 +375,6 @@ struct session { char address[INET6_ADDRSTRLEN]; }; -typedef struct txntable txntable_t; - -struct txntable { - UT_hash_handle hh; - int id; - char hash[68]; - const char *data; - int refcount; -}; - #define ID_AUTH 0 #define ID_WORKINFO 1 #define ID_AGEWORKINFO 2 @@ -457,7 +447,6 @@ struct stratifier_data { workbase_t *workbases; workbase_t *current_workbase; int workbases_generated; - txntable_t *txns; /* Semaphore to serialise calls to add_base */ sem_t update_sem; @@ -1115,37 +1104,11 @@ struct update_req { static void broadcast_ping(sdata_t *sdata); -/* Build a hashlist of all transactions, allowing us to compare with the list of - * existing transactions to determine which need to be propagated */ -static void add_txn(sdata_t *sdata, txntable_t *txns, const char *hash, const char *data) -{ - bool found = false; - txntable_t *txn; - - ck_rlock(&sdata->workbase_lock); - HASH_FIND_STR(sdata->txns, hash, txn); - if (txn) { - txn->refcount++; - found = true; - } - ck_runlock(&sdata->workbase_lock); - if (found) - return; - - txn = ckzalloc(sizeof(txntable_t)); - memcpy(txn->hash, hash, 65); - /* Note that data is pointing to a string in a json struture which will - * be destroyed so we can only use the data value until then. */ - txn->data = data; - HASH_ADD_STR(txns, hash, txn); -} - /* Distill down a set of transactions into an efficient tree arrangement for * stratum messages and fast work assembly. */ -static void wb_merkle_bins(sdata_t *sdata, workbase_t *wb, json_t *txn_array) +static void wb_merkle_bins(workbase_t *wb, json_t *txn_array) { - int i, j, binleft, binlen, added = 0, purged = 0; - txntable_t *txns = NULL, *tmp, *tmpa; + int i, j, binleft, binlen; json_t *arr_val; uchar *hashbin; @@ -1180,7 +1143,6 @@ static void wb_merkle_bins(sdata_t *sdata, workbase_t *wb, json_t *txn_array) arr_val = json_array_get(txn_array, i); hash = json_string_value(json_object_get(arr_val, "hash")); txn = json_string_value(json_object_get(arr_val, "data")); - add_txn(sdata, txns, hash, txn); len = strlen(txn); memcpy(wb->txn_data + ofs, txn, len); ofs += len; @@ -1229,28 +1191,6 @@ static void wb_merkle_bins(sdata_t *sdata, workbase_t *wb, json_t *txn_array) binlen = binleft * 32; } } - - ck_wlock(&sdata->workbase_lock); - HASH_ITER(hh, sdata->txns, tmp, tmpa) { - if (tmp->refcount--) - continue; - HASH_DEL(sdata->txns, tmp); - dealloc(tmp); - purged++; - } - HASH_ITER(hh, txns, tmp, tmpa) { - /* Propagate transaction here */ - /* Move to the sdata transaction table */ - HASH_DEL(txns, tmp); - HASH_ADD_STR(sdata->txns, data, tmp); - /* Empty data once used to not dereference since the json structure - * will be destroyed. */ - tmp->data = NULL; - added++; - } - ck_wunlock(&sdata->workbase_lock); - - LOGDEBUG("Stratifier added %d transactions and purged %d", added, purged); } /* This function assumes it will only receive a valid json gbt base template @@ -1308,7 +1248,7 @@ retry: json_intcpy(&wb->height, val, "height"); json_strdup(&wb->flags, val, "flags"); txn_array = json_object_get(val, "transactions"); - wb_merkle_bins(sdata, wb, txn_array); + wb_merkle_bins(wb, txn_array); json_decref(val); generate_coinbase(ckp, wb); From d0398fed4a67c5295453aedb27bc80f8f3ff7dd0 Mon Sep 17 00:00:00 2001 From: ckolivas Date: Wed, 3 Feb 2016 16:43:35 +1100 Subject: [PATCH 5/5] Revert "Move merkle tree generation to the stratifier along with all transaction data to allow transaction data to be used directly in future code" This reverts commit 8702c028226c33aa469f9a5e34c1801f58592068. --- src/bitcoin.c | 112 +++++++++++++++++++++++++++++++++++++++++++++-- src/bitcoin.h | 2 +- src/stratifier.c | 112 +++++++++-------------------------------------- 3 files changed, 130 insertions(+), 96 deletions(-) diff --git a/src/bitcoin.c b/src/bitcoin.c index 016a543f..92bb7919 100644 --- a/src/bitcoin.c +++ b/src/bitcoin.c @@ -79,6 +79,99 @@ out: return ret; } +/* Distill down a set of transactions into an efficient tree arrangement for + * stratum messages and fast work assembly. */ +static bool gbt_merkle_bins(gbtbase_t *gbt, json_t *transaction_arr) +{ + int i, j, binleft, binlen; + json_t *arr_val; + uchar *hashbin; + + dealloc(gbt->txn_data); + dealloc(gbt->txn_hashes); + gbt->txns = 0; + gbt->merkles = 0; + gbt->txns = json_array_size(transaction_arr); + binlen = gbt->txns * 32 + 32; + hashbin = alloca(binlen + 32); + memset(hashbin, 0, 32); + binleft = binlen / 32; + if (gbt->txns) { + int len = 1, ofs = 0; + const char *txn; + + for (i = 0; i < gbt->txns; i++) { + arr_val = json_array_get(transaction_arr, i); + txn = json_string_value(json_object_get(arr_val, "data")); + if (!txn) { + LOGWARNING("json_string_value fail - cannot find transaction data"); + return false; + } + len += strlen(txn); + } + + gbt->txn_data = ckzalloc(len + 1); + gbt->txn_hashes = ckzalloc(gbt->txns * 65 + 1); + memset(gbt->txn_hashes, 0x20, gbt->txns * 65); // Spaces + + for (i = 0; i < gbt->txns; i++) { + char binswap[32]; + const char *hash; + + arr_val = json_array_get(transaction_arr, i); + hash = json_string_value(json_object_get(arr_val, "hash")); + txn = json_string_value(json_object_get(arr_val, "data")); + len = strlen(txn); + memcpy(gbt->txn_data + ofs, txn, len); + ofs += len; +#if 0 + /* In case we ever want to be a gbt poolproxy */ + if (!hash) { + char *txn_bin; + int txn_len; + + txn_len = len / 2; + txn_bin = ckalloc(txn_len); + hex2bin(txn_bin, txn, txn_len); + /* This is needed for pooled mining since only + * transaction data and not hashes are sent */ + gen_hash(txn_bin, hashbin + 32 + 32 * i, txn_len); + continue; + } +#endif + if (!hex2bin(binswap, hash, 32)) { + LOGERR("Failed to hex2bin hash in gbt_merkle_bins"); + return false; + } + memcpy(gbt->txn_hashes + i * 65, hash, 64); + bswap_256(hashbin + 32 + 32 * i, binswap); + } + } + if (binleft > 1) { + while (42) { + uchar merklebin[32]; + + if (binleft == 1) + break; + memcpy(merklebin, hashbin + 32, 32); + __bin2hex(&gbt->merklehash[gbt->merkles][0], merklebin, 32); + LOGDEBUG("MH%d %s",gbt->merkles, &gbt->merklehash[gbt->merkles][0]); + gbt->merkles++; + if (binleft % 2) { + memcpy(hashbin + binlen, hashbin + binlen - 32, 32); + binlen += 32; + binleft++; + } + for (i = 32, j = 64; j < binlen; i += 32, j += 64) + gen_hash(hashbin + j, hashbin + i, 64); + binleft /= 2; + binlen = binleft * 32; + } + } + LOGINFO("Stored %d transactions", gbt->txns); + return true; +} + static const char *gbt_req = "{\"method\": \"getblocktemplate\", \"params\": [{\"capabilities\": [\"coinbasetxn\", \"workid\", \"coinbase/append\"]}]}\n"; /* Request getblocktemplate from bitcoind already connected with a connsock_t @@ -86,7 +179,7 @@ static const char *gbt_req = "{\"method\": \"getblocktemplate\", \"params\": [{\ * required to assemble a mining template, storing it in a gbtbase_t structure */ bool gen_gbtbase(connsock_t *cs, gbtbase_t *gbt) { - json_t *txn_array, *coinbase_aux, *res_val, *val; + json_t *transaction_arr, *coinbase_aux, *res_val, *val, *array; const char *previousblockhash; char hash_swap[32], tmp[32]; uint64_t coinbasevalue; @@ -96,6 +189,7 @@ bool gen_gbtbase(connsock_t *cs, gbtbase_t *gbt) int version; int curtime; int height; + int i; bool ret = false; val = json_rpc_call(cs, gbt_req); @@ -111,7 +205,7 @@ bool gen_gbtbase(connsock_t *cs, gbtbase_t *gbt) previousblockhash = json_string_value(json_object_get(res_val, "previousblockhash")); target = json_string_value(json_object_get(res_val, "target")); - txn_array = json_object_get(res_val, "transactions"); + transaction_arr = json_object_get(res_val, "transactions"); version = json_integer_value(json_object_get(res_val, "version")); curtime = json_integer_value(json_object_get(res_val, "curtime")); bits = json_string_value(json_object_get(res_val, "bits")); @@ -164,7 +258,19 @@ bool gen_gbtbase(connsock_t *cs, gbtbase_t *gbt) gbt->flags = strdup(flags); json_object_set_new_nocheck(gbt->json, "flags", json_string_nocheck(gbt->flags)); - json_object_set_new_nocheck(gbt->json, "transactions", json_deep_copy(txn_array)); + gbt_merkle_bins(gbt, transaction_arr); + json_object_set_new_nocheck(gbt->json, "txns", json_integer(gbt->txns)); + if (gbt->txns) { + json_object_set_new_nocheck(gbt->json, "txn_data", json_string_nocheck(gbt->txn_data)); + json_object_set_new_nocheck(gbt->json, "txn_hashes", json_string_nocheck(gbt->txn_hashes)); + } + json_object_set_new_nocheck(gbt->json, "merkles", json_integer(gbt->merkles)); + if (gbt->merkles) { + array = json_array(); + for (i = 0; i < gbt->merkles; i++) + json_array_append_new(array, json_string_nocheck(&gbt->merklehash[i][0])); + json_object_set_new_nocheck(gbt->json, "merklehash", array); + } ret = true; out: diff --git a/src/bitcoin.h b/src/bitcoin.h index 9ba4534d..1404c538 100644 --- a/src/bitcoin.h +++ b/src/bitcoin.h @@ -1,5 +1,5 @@ /* - * Copyright 2014-2016 Con Kolivas + * Copyright 2014 Con Kolivas * * This program is free software; you can redistribute it and/or modify it * under the terms of the GNU General Public License as published by the Free diff --git a/src/stratifier.c b/src/stratifier.c index b3463f91..7c73a09b 100644 --- a/src/stratifier.c +++ b/src/stratifier.c @@ -1104,95 +1104,6 @@ struct update_req { static void broadcast_ping(sdata_t *sdata); -/* Distill down a set of transactions into an efficient tree arrangement for - * stratum messages and fast work assembly. */ -static void wb_merkle_bins(workbase_t *wb, json_t *txn_array) -{ - int i, j, binleft, binlen; - json_t *arr_val; - uchar *hashbin; - - wb->txns = json_array_size(txn_array); - wb->merkles = 0; - binlen = wb->txns * 32 + 32; - hashbin = alloca(binlen + 32); - memset(hashbin, 0, 32); - binleft = binlen / 32; - if (wb->txns) { - int len = 1, ofs = 0; - const char *txn; - - for (i = 0; i < wb->txns; i++) { - arr_val = json_array_get(txn_array, i); - txn = json_string_value(json_object_get(arr_val, "data")); - if (!txn) { - LOGWARNING("json_string_value fail - cannot find transaction data"); - return; - } - len += strlen(txn); - } - - wb->txn_data = ckzalloc(len + 1); - wb->txn_hashes = ckzalloc(wb->txns * 65 + 1); - memset(wb->txn_hashes, 0x20, wb->txns * 65); // Spaces - - for (i = 0; i < wb->txns; i++) { - char binswap[32]; - const char *hash; - - arr_val = json_array_get(txn_array, i); - hash = json_string_value(json_object_get(arr_val, "hash")); - txn = json_string_value(json_object_get(arr_val, "data")); - len = strlen(txn); - memcpy(wb->txn_data + ofs, txn, len); - ofs += len; -#if 0 - /* In case we ever want to be a gbt poolproxy */ - if (!hash) { - char *txn_bin; - int txn_len; - - txn_len = len / 2; - txn_bin = ckalloc(txn_len); - hex2bin(txn_bin, txn, txn_len); - /* This is needed for pooled mining since only - * transaction data and not hashes are sent */ - gen_hash(txn_bin, hashbin + 32 + 32 * i, txn_len); - continue; - } -#endif - if (!hex2bin(binswap, hash, 32)) { - LOGERR("Failed to hex2bin hash in gbt_merkle_bins"); - return; - } - memcpy(wb->txn_hashes + i * 65, hash, 64); - bswap_256(hashbin + 32 + 32 * i, binswap); - } - } else - wb->txn_hashes = ckzalloc(1); - if (binleft > 1) { - while (42) { - uchar merklebin[32]; - - if (binleft == 1) - break; - memcpy(merklebin, hashbin + 32, 32); - __bin2hex(&wb->merklehash[wb->merkles][0], merklebin, 32); - LOGDEBUG("MH%d %s",wb->merkles, &wb->merklehash[wb->merkles][0]); - wb->merkles++; - if (binleft % 2) { - memcpy(hashbin + binlen, hashbin + binlen - 32, 32); - binlen += 32; - binleft++; - } - for (i = 32, j = 64; j < binlen; i += 32, j += 64) - gen_hash(hashbin + j, hashbin + i, 64); - binleft /= 2; - binlen = binleft * 32; - } - } -} - /* This function assumes it will only receive a valid json gbt base template * since checking should have been done earlier, and creates the base template * for generating work templates. */ @@ -1202,11 +1113,11 @@ static void *do_update(void *arg) int prio = ur->prio, retries = 0; ckpool_t *ckp = ur->ckp; sdata_t *sdata = ckp->data; - json_t *val, *txn_array; bool new_block = false; bool ret = false; workbase_t *wb; time_t now_t; + json_t *val; char *buf; pthread_detach(pthread_self()); @@ -1247,8 +1158,25 @@ retry: json_uint64cpy(&wb->coinbasevalue, val, "coinbasevalue"); json_intcpy(&wb->height, val, "height"); json_strdup(&wb->flags, val, "flags"); - txn_array = json_object_get(val, "transactions"); - wb_merkle_bins(wb, txn_array); + json_intcpy(&wb->txns, val, "txns"); + if (wb->txns) { + json_strdup(&wb->txn_data, val, "txn_data"); + json_strdup(&wb->txn_hashes, val, "txn_hashes"); + } else + wb->txn_hashes = ckzalloc(1); + json_intcpy(&wb->merkles, val, "merkles"); + wb->merkle_array = json_array(); + if (wb->merkles) { + json_t *arr; + int i; + + arr = json_object_get(val, "merklehash"); + for (i = 0; i < wb->merkles; i++) { + strcpy(&wb->merklehash[i][0], json_string_value(json_array_get(arr, i))); + hex2bin(&wb->merklebin[i][0], &wb->merklehash[i][0], 32); + json_array_append_new(wb->merkle_array, json_string(&wb->merklehash[i][0])); + } + } json_decref(val); generate_coinbase(ckp, wb);