markow
/
ckpool
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity
Browse Source

ckdb - speed up data allocation and update the memory ckp web page

master
kanoi 9 years ago
parent
248a70810a
commit
196f923957
5 changed files with 98 additions and 88 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Unified View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 4
      pool/page_ckp.php
  2. 2
      src/ckdb.h
  3. 114
      src/ckdb_cmd.c
  4. 49
      src/klist.c
  5. 17
      src/klist.h

4
pool/page_ckp.php
Unescape View File

@ -31,7 +31,7 @@ function dockp($data, $user)
$pg .= '<td class=dl>Name</td>'; $pg .= '<td class=dl>Name</td>';
$pg .= '<td class=dr>Initial</td>'; $pg .= '<td class=dr>Initial</td>';
$pg .= '<td class=dr>Allocated</td>'; $pg .= '<td class=dr>Allocated</td>';
$pg .= '<td class=dr>Store</td>'; $pg .= '<td class=dr>In&nbsp;Store</td>';
$pg .= '<td class=dr>RAM</td>'; $pg .= '<td class=dr>RAM</td>';
$pg .= '<td class=dr>RAM2</td>'; $pg .= '<td class=dr>RAM2</td>';
$pg .= '<td class=dr>Cull</td>'; $pg .= '<td class=dr>Cull</td>';
@ -50,7 +50,7 @@ function dockp($data, $user)
$pg .= '<td class=dl>'.$ans['name:'.$i].'</td>'; $pg .= '<td class=dl>'.$ans['name:'.$i].'</td>';
$pg .= '<td class=dr>'.stnum($ans['initial:'.$i]).'</td>'; $pg .= '<td class=dr>'.stnum($ans['initial:'.$i]).'</td>';
$pg .= '<td class=dr>'.stnum($ans['allocated:'.$i]).'</td>'; $pg .= '<td class=dr>'.stnum($ans['allocated:'.$i]).'</td>';
$pg .= '<td class=dr>'.stnum($ans['store:'.$i]).'</td>'; $pg .= '<td class=dr>'.stnum($ans['instore:'.$i]).'</td>';
$pg .= '<td class=dr>'.stnum($ans['ram:'.$i]).'</td>'; $pg .= '<td class=dr>'.stnum($ans['ram:'.$i]).'</td>';
$pg .= '<td class=dr>'.stnum($ans['ram2:'.$i]).'</td>'; $pg .= '<td class=dr>'.stnum($ans['ram2:'.$i]).'</td>';
$pg .= '<td class=dr>'.stnum($ans['cull:'.$i]).'</td>'; $pg .= '<td class=dr>'.stnum($ans['cull:'.$i]).'</td>';

2
src/ckdb.h
Unescape View File

@ -51,7 +51,7 @@
#define DB_VLOCK "1" #define DB_VLOCK "1"
#define DB_VERSION "1.0.4" #define DB_VERSION "1.0.4"
#define CKDB_VERSION DB_VERSION"-1.801" #define CKDB_VERSION DB_VERSION"-1.900"
#define WHERE_FFL " - from %s %s() line %d" #define WHERE_FFL " - from %s %s() line %d"
#define WHERE_FFL_HERE __FILE__, __func__, __LINE__ #define WHERE_FFL_HERE __FILE__, __func__, __LINE__

114
src/ckdb_cmd.c
Unescape View File

@ -5631,73 +5631,71 @@ static char *cmd_stats(__maybe_unused PGconn *conn, char *cmd, char *id,
__maybe_unused tv_t *notcd, __maybe_unused K_TREE *trf_root) __maybe_unused tv_t *notcd, __maybe_unused K_TREE *trf_root)
{ {
char tmp[1024], *buf; char tmp[1024], *buf;
const char *name;
size_t len, off; size_t len, off;
uint64_t ram, ram2, tot = 0; uint64_t ram, ram2, tot = 0;
K_LIST *klist; K_LIST *klist;
K_LISTS *klists;
int rows = 0; int rows = 0;
bool istree;
LOGDEBUG("%s(): cmd '%s'", __func__, cmd); LOGDEBUG("%s(): cmd '%s'", __func__, cmd);
APPEND_REALLOC_INIT(buf, off, len); APPEND_REALLOC_INIT(buf, off, len);
APPEND_REALLOC(buf, off, len, "ok."); APPEND_REALLOC(buf, off, len, "ok.");
// FYI average transactiontree length of the ~119k I have is ~28k (>3.3GB) /* All but temporary lists are in klist_all
#define USEINFO(_obj, _stores, _trees) \ * All trees are there also since all trees have a node klist */
klist = _obj ## _free; \ ck_wlock(&lock_check_lock);
ram = sizeof(K_LIST) + _stores * sizeof(K_STORE) + \ klists = all_klists;
klist->allocate * klist->item_mem_count * klist->siz + \ while (klists) {
sizeof(K_TREE) * (klist->total - klist->count) * _trees; \ klist = klists->klist;
ram2 = klist->ram; \
snprintf(tmp, sizeof(tmp), \ ram = sizeof(*klist);
"name:%d=" #_obj "%cinitial:%d=%d%callocated:%d=%d%c" \ if (klist->name == tree_node_list_name) {
"store:%d=%d%ctrees:%d=%d%cram:%d=%"PRIu64"%c" \ ram += sizeof(K_TREE);
"ram2:%d=%"PRIu64"%ccull:%d=%d%c", \ istree = true;
rows, FLDSEP, \ name = klist->name2;
rows, klist->allocate, FLDSEP, \ } else {
rows, klist->total, FLDSEP, \ istree = false;
rows, klist->total - klist->count, FLDSEP, \ name = klist->name;
rows, _trees, FLDSEP, \ }
rows, ram, FLDSEP, \ if (klist->lock)
rows, ram2, FLDSEP, \ ram += sizeof(*(klist->lock));
rows, klist->cull_count, FLDSEP); \ // List of item lists
APPEND_REALLOC(buf, off, len, tmp); \ ram += klist->item_mem_count * sizeof(*(klist->item_memory));
tot += ram + ram2; \ // items
rows++; ram += klist->total * sizeof(K_ITEM);
// List of data lists
USEINFO(users, 1, 2); ram += klist->data_mem_count * sizeof(*(klist->data_memory));
USEINFO(useratts, 1, 1); // data
USEINFO(workers, 1, 1); ram += klist->total * klist->siz;
USEINFO(paymentaddresses, 1, 2);
USEINFO(payments, 1, 1); // stores
USEINFO(accountbalance, 1, 1); ram += klist->stores * sizeof(K_STORE);
USEINFO(idcontrol, 1, 0);
USEINFO(optioncontrol, 1, 1); ram2 = klist->ram;
USEINFO(workinfo, 1, 1);
// Trees don't share items so count as 1 tree snprintf(tmp, sizeof(tmp),
USEINFO(shares, 2, 1); "name:%d=%s%s%s%cinitial:%d=%d%callocated:%d=%d%c"
// Trees don't share items so count as 1 tree "instore:%d=%d%cram:%d=%"PRIu64"%c"
USEINFO(shareerrors, 2, 1); "ram2:%d=%"PRIu64"%ccull:%d=%d%c",
// _pool doesn't share items so is included rows, name, istree ? " (tree)" : "",
USEINFO(sharesummary, 1, 2); klist->is_lock_only ? " (lock)" : "", FLDSEP,
USEINFO(workmarkers, 1, 2); rows, klist->allocate, FLDSEP,
// _pool doesn't share items so is included rows, klist->total, FLDSEP,
USEINFO(markersummary, 1, 2); rows, klist->total - klist->count, FLDSEP,
USEINFO(marks, 1, 1); rows, ram, FLDSEP,
USEINFO(blocks, 1, 1); rows, ram2, FLDSEP,
USEINFO(miningpayouts, 1, 1); rows, klist->cull_count, FLDSEP);
USEINFO(payouts, 1, 3); APPEND_REALLOC(buf, off, len, tmp);
USEINFO(auths, 1, 1);
USEINFO(poolstats, 1, 1); tot += ram + ram2;
USEINFO(userstats, 2, 1); rows++;
USEINFO(workerstatus, 1, 1);
USEINFO(userinfo, 1, 1); klists = klists->next;
USEINFO(msgline, 1, 0); }
USEINFO(workqueue, 3, 0); ck_wunlock(&lock_check_lock);
USEINFO(transfer, 0, 0);
USEINFO(heartbeatqueue, 1, 0);
USEINFO(logqueue, 1, 0);
USEINFO(seqset, 1, 0);
USEINFO(seqtrans, 0, 0);
snprintf(tmp, sizeof(tmp), "totalram=%"PRIu64"%c", tot, FLDSEP); snprintf(tmp, sizeof(tmp), "totalram=%"PRIu64"%c", tot, FLDSEP);
APPEND_REALLOC(buf, off, len, tmp); APPEND_REALLOC(buf, off, len, tmp);
@ -5705,7 +5703,7 @@ static char *cmd_stats(__maybe_unused PGconn *conn, char *cmd, char *id,
snprintf(tmp, sizeof(tmp), snprintf(tmp, sizeof(tmp),
"rows=%d%cflds=%s%c", "rows=%d%cflds=%s%c",
rows, FLDSEP, rows, FLDSEP,
"name,initial,allocated,store,trees,ram,cull", FLDSEP); "name,initial,allocated,instore,ram,cull", FLDSEP);
APPEND_REALLOC(buf, off, len, tmp); APPEND_REALLOC(buf, off, len, tmp);
snprintf(tmp, sizeof(tmp), "arn=%s%carp=%s", "Stats", FLDSEP, ""); snprintf(tmp, sizeof(tmp), "arn=%s%carp=%s", "Stats", FLDSEP, "");

49
src/klist.c
Unescape View File

@ -16,8 +16,6 @@ const char *tree_node_list_name = "TreeNodes";
bool check_locks = true; bool check_locks = true;
const char *thread_noname = "UNSET"; const char *thread_noname = "UNSET";
int next_thread_id = 0; int next_thread_id = 0;
bool lock_check_init = false;
cklock_t lock_check_lock;
__thread int my_thread_id = -1; __thread int my_thread_id = -1;
__thread char *my_thread_name = NULL; __thread char *my_thread_name = NULL;
__thread bool my_check_locks = true; __thread bool my_check_locks = true;
@ -32,8 +30,11 @@ __thread const char *my_locks_f[MAX_LOCKDEPTH];
__thread int my_locks_l[MAX_LOCKDEPTH]; __thread int my_locks_l[MAX_LOCKDEPTH];
__thread int my_lock_level = 0; __thread int my_lock_level = 0;
__thread bool my_check_deadlocks = true; __thread bool my_check_deadlocks = true;
K_LISTS *all_klists;
#endif #endif
// Required for cmd_stats
bool lock_check_init = false;
cklock_t lock_check_lock;
K_LISTS *all_klists;
#define _CHKLIST(_list, _name) do {\ #define _CHKLIST(_list, _name) do {\
if (!_list) { \ if (!_list) { \
@ -61,6 +62,7 @@ K_LISTS *all_klists;
static void k_alloc_items(K_LIST *list, KLIST_FFL_ARGS) static void k_alloc_items(K_LIST *list, KLIST_FFL_ARGS)
{ {
K_ITEM *item; K_ITEM *item;
void *data;
int allocate, i; int allocate, i;
CHKLIST(list); CHKLIST(list);
@ -90,10 +92,6 @@ static void k_alloc_items(K_LIST *list, KLIST_FFL_ARGS)
} }
list->item_memory[list->item_mem_count - 1] = (void *)item; list->item_memory[list->item_mem_count - 1] = (void *)item;
list->total += allocate;
list->count = allocate;
list->count_up = allocate;
item[0].name = list->name; item[0].name = list->name;
item[0].prev = NULL; item[0].prev = NULL;
item[0].next = &(item[1]); item[0].next = &(item[1]);
@ -110,21 +108,29 @@ static void k_alloc_items(K_LIST *list, KLIST_FFL_ARGS)
if (list->do_tail) if (list->do_tail)
list->tail = &(item[allocate-1]); list->tail = &(item[allocate-1]);
list->data_mem_count++;
if (!(list->data_memory = realloc(list->data_memory,
list->data_mem_count * sizeof(*(list->data_memory))))) {
quithere(1, "List %s data_memory failed to realloc count=%d",
list->name, list->data_mem_count);
}
data = calloc(allocate, list->siz);
if (!data) {
quithere(1, "List %s failed to calloc %d new data - total was %d, limit was %d",
list->name, allocate, list->total, list->limit);
}
list->data_memory[list->data_mem_count - 1] = data;
item = list->head; item = list->head;
while (item) { while (item) {
list->data_mem_count++; item->data = data;
if (!(list->data_memory = realloc(list->data_memory, data += list->siz;
list->data_mem_count *
sizeof(*(list->data_memory))))) {
quithere(1, "List %s data_memory failed to realloc count=%d",
list->name, list->data_mem_count);
}
item->data = calloc(1, list->siz);
if (!(item->data))
quithere(1, "List %s failed to calloc item data", list->name);
list->data_memory[list->data_mem_count - 1] = (void *)(item->data);
item = item->next; item = item->next;
} }
list->total += allocate;
list->count = allocate;
list->count_up = allocate;
} }
K_STORE *_k_new_store(K_LIST *list, KLIST_FFL_ARGS) K_STORE *_k_new_store(K_LIST *list, KLIST_FFL_ARGS)
@ -142,6 +148,7 @@ K_STORE *_k_new_store(K_LIST *list, KLIST_FFL_ARGS)
store->lock = list->lock; store->lock = list->lock;
store->name = list->name; store->name = list->name;
store->do_tail = list->do_tail; store->do_tail = list->do_tail;
list->stores++;
return store; return store;
} }
@ -187,7 +194,6 @@ K_LIST *_k_new_list(const char *name, size_t siz, int allocate, int limit,
if (!(list->is_lock_only)) if (!(list->is_lock_only))
k_alloc_items(list, KLIST_FFL_PASS); k_alloc_items(list, KLIST_FFL_PASS);
#if LOCK_CHECK
/* Don't want to keep track of short lived (tree) lists /* Don't want to keep track of short lived (tree) lists
* since they wont use locking anyway */ * since they wont use locking anyway */
if (!list->local_list) { if (!list->local_list) {
@ -210,7 +216,6 @@ K_LIST *_k_new_list(const char *name, size_t siz, int allocate, int limit,
all_klists = klists; all_klists = klists;
ck_wunlock(&lock_check_lock); ck_wunlock(&lock_check_lock);
} }
#endif
return list; return list;
} }
@ -541,7 +546,6 @@ K_LIST *_k_free_list(K_LIST *list, KLIST_FFL_ARGS)
free(list->lock); free(list->lock);
} }
#if LOCK_CHECK
// local_list lists are not stored in all_klists // local_list lists are not stored in all_klists
if (!list->local_list) { if (!list->local_list) {
K_LISTS *klists, *klists_prev = NULL; K_LISTS *klists, *klists_prev = NULL;
@ -572,7 +576,6 @@ K_LIST *_k_free_list(K_LIST *list, KLIST_FFL_ARGS)
} }
ck_wunlock(&lock_check_lock); ck_wunlock(&lock_check_lock);
} }
#endif
free(list); free(list);
@ -588,6 +591,8 @@ K_STORE *_k_free_store(K_STORE *store, KLIST_FFL_ARGS)
store->name, __func__, KLIST_FFL_PASS); store->name, __func__, KLIST_FFL_PASS);
} }
store->master->stores--;
free(store); free(store);
return NULL; return NULL;

17
src/klist.h
Unescape View File

@ -87,8 +87,6 @@ extern bool check_locks;
#define MAX_THREADS 128 #define MAX_THREADS 128
extern const char *thread_noname; extern const char *thread_noname;
extern int next_thread_id; extern int next_thread_id;
extern bool lock_check_init;
extern cklock_t lock_check_lock;
extern __thread int my_thread_id; extern __thread int my_thread_id;
extern __thread char *my_thread_name; extern __thread char *my_thread_name;
extern __thread bool my_check_locks; extern __thread bool my_check_locks;
@ -152,6 +150,7 @@ typedef struct k_list {
void (*dsp_func)(K_ITEM *, FILE *); // optional data display to a file void (*dsp_func)(K_ITEM *, FILE *); // optional data display to a file
int cull_count; int cull_count;
int ram; // ram allocated for data pointers - code must manage it int ram; // ram allocated for data pointers - code must manage it
int stores; // how many stores it currently has
#if LOCK_CHECK #if LOCK_CHECK
// Since each thread has it's own k_lock no locking is required on this // Since each thread has it's own k_lock no locking is required on this
K_LOCK k_lock[MAX_THREADS]; K_LOCK k_lock[MAX_THREADS];
@ -160,13 +159,14 @@ typedef struct k_list {
#endif #endif
} K_LIST; } K_LIST;
#if LOCK_CHECK // Required for cmd_stats
extern bool lock_check_init;
extern cklock_t lock_check_lock;
typedef struct k_lists { typedef struct k_lists {
K_LIST *klist; K_LIST *klist;
struct k_lists *next; struct k_lists *next;
} K_LISTS; } K_LISTS;
extern K_LISTS *all_klists; extern K_LISTS *all_klists;
#endif
/* /*
* K_STORE is for a list of items taken from a K_LIST * K_STORE is for a list of items taken from a K_LIST
@ -526,7 +526,14 @@ static inline K_ITEM *list_rtail(K_LIST *list)
#else #else
#define LOCK_MAYBE __maybe_unused #define LOCK_MAYBE __maybe_unused
#define LOCK_INIT(_name) #define LOCK_INIT(_name)
#define FIRST_LOCK_INIT(_name) #define FIRST_LOCK_INIT(_ignore) do { \
if (lock_check_init) { \
quithere(1, "lock_check_lock has already been " \
"initialised!"); \
} \
cklock_init(&lock_check_lock); \
lock_check_init = true; \
} while (0)
#define CHECK_WLOCK(_list) ck_wlock((_list)->lock) #define CHECK_WLOCK(_list) ck_wlock((_list)->lock)
#define CHECK_WUNLOCK(_list) ck_wunlock((_list)->lock) #define CHECK_WUNLOCK(_list) ck_wunlock((_list)->lock)
#define CHECK_RLOCK(_list) ck_rlock((_list)->lock) #define CHECK_RLOCK(_list) ck_rlock((_list)->lock)

Write Preview
Loading…
Cancel
Save