Browse Source

ckdb/php - more block statistics

master
kanoi 10 years ago
parent
commit
951f9019a9
  1. 7
      configure.ac
  2. 98
      pool/page_blocks.php
  3. 2
      src/Makefile.am
  4. 2
      src/ckdb.c
  5. 29
      src/ckdb.h
  6. 132
      src/ckdb_cmd.c
  7. 108
      src/ckdb_data.c
  8. 12
      src/ckdb_dbio.c

7
configure.ac

@ -38,6 +38,7 @@ AC_CHECK_HEADERS(alloca.h pthread.h stdio.h math.h signal.h sys/prctl.h)
AC_CHECK_HEADERS(sys/types.h sys/socket.h sys/stat.h linux/un.h netdb.h)
AC_CHECK_HEADERS(stdint.h netinet/in.h netinet/tcp.h sys/ioctl.h getopt.h)
AC_CHECK_HEADERS(sys/epoll.h libpq-fe.h postgresql/libpq-fe.h grp.h)
AC_CHECK_HEADERS(gsl/gsl_math.h gsl/gsl_cdf.h)
PTHREAD_LIBS="-lpthread"
MATH_LIBS="-lm"
@ -59,8 +60,12 @@ AC_ARG_WITH([ckdb],
if test "x$ckdb" != "xno"; then
AC_CHECK_LIB([pq], [main],[PQ=-lpq],echo "Error: Required library libpq-dev
not found. Install it or disable postgresql support with --without-ckdb" && exit 1)
AC_CHECK_LIB([gsl], [main],[GSL=-lgsl],echo "Error: Required library gsl-dev
not found. Install it or disable support with --without-ckdb" && exit 1)
AC_CHECK_LIB([gslcblas], [main],[GSLCBLAS=-lgslcblas],echo "Error: Required library gslcblas
not found. Install it or disable support with --without-ckdb" && exit 1)
AC_DEFINE([USE_CKDB], [1], [Defined to 1 if ckdb support required])
PQ_LIBS="-lpq"
PQ_LIBS="-lpq -lgsl -lgslcblas"
else
PQ_LIBS=""
fi

98
pool/page_blocks.php

@ -56,6 +56,43 @@ function doblocks($data, $user)
if ($wantcsv === false)
{
if ($ans['STATUS'] == 'ok' and isset($ans['s_rows']) and $ans['s_rows'] > 0)
{
$pg .= '<h1>Block Statistics</h1>';
$pg .= "<table callpadding=0 cellspacing=0 border=0>\n";
$pg .= "<tr class=title>";
$pg .= "<td class=dl>Description</td>";
$pg .= "<td class=dr>Diff%</td>";
$pg .= "<td class=dr>Mean%</td>";
$pg .= "<td class=dr>CDF[Erl]</td>";
$pg .= "<td class=dr>Luck%</td>";
$pg .= "</tr>\n";
$count = $ans['s_rows'];
for ($i = 0; $i < $count; $i++)
{
if (($i % 2) == 0)
$row = 'even';
else
$row = 'odd';
$desc = $ans['s_desc:'.$i];
$diff = number_format(100 * $ans['s_diffratio:'.$i], 2);
$mean = number_format(100 * $ans['s_diffmean:'.$i], 2);
$cdferl = number_format($ans['s_cdferl:'.$i], 4);
$luck = number_format(100 * $ans['s_luck:'.$i], 2);
$pg .= "<tr class=$row>";
$pg .= "<td class=dl>$desc Blocks</td>";
$pg .= "<td class=dr>$diff%</td>";
$pg .= "<td class=dr>$mean%</td>";
$pg .= "<td class=dr>$cdferl</td>";
$pg .= "<td class=dr>$luck%</td>";
$pg .= "</tr>\n";
}
$pg .= "</table>\n";
}
if ($ans['STATUS'] == 'ok')
{
$count = $ans['rows'];
@ -70,15 +107,15 @@ function doblocks($data, $user)
$s = 's';
}
$pg = "<h1>Last$num Block$s</h1>";
$pg .= "<h1>Last$num Block$s</h1>";
}
else
$pg = '<h1>Blocks</h1>';
$pg .= '<h1>Blocks</h1>';
list($fg, $bg) = pctcolour(25.0);
$pg .= "<span style='background:$bg; color:$fg;'>";
$pg .= "&nbsp;Green&nbsp;</span>&nbsp;";
$pg .= 'is good luck. Lower Diff% and brighter green is best luck.<br>';
$pg .= 'is good luck. Lower Diff% and brighter green is better luck.<br>';
list($fg, $bg) = pctcolour(100.0);
$pg .= "<span style='background:$bg; color:$fg;'>";
$pg .= "&nbsp;100%&nbsp;</span>&nbsp;";
@ -94,7 +131,7 @@ function doblocks($data, $user)
$pg .= "<td class=dl>Height</td>";
if ($user !== null)
$pg .= "<td class=dl>Who</td>";
$pg .= "<td class=dr>Reward</td>";
$pg .= "<td class=dr>Block Reward</td>";
$pg .= "<td class=dc>When</td>";
$pg .= "<td class=dr>Status</td>";
$pg .= "<td class=dr>Diff</td>";
@ -109,7 +146,6 @@ function doblocks($data, $user)
$csv = "Sequence,Height,Status,Timestamp,DiffAcc,NetDiff,Hash\n";
if ($ans['STATUS'] == 'ok')
{
$tot = $ans['tot'];
$count = $ans['rows'];
for ($i = 0; $i < $count; $i++)
{
@ -130,7 +166,7 @@ function doblocks($data, $user)
$seq = '';
}
else
$seq = $tot--;
$seq = $ans['seq:'.$i];
if ($stat == '1-Confirm')
{
if (isset($data['info']['lastheight']))
@ -143,37 +179,34 @@ function doblocks($data, $user)
}
$stara = '';
$starp = '';
if (isset($ans['status:'.($i+1)]))
if ($ans['status:'.($i+1)] == 'Orphan'
&& $stat != 'Orphan')
{
if ($stat == 'Orphan')
$stara = '<span class=st1>*</span>';
$starp = '<span class=st0>*</span>';
}
$diffacc = $ans['diffacc:'.$i];
$acc = number_format($diffacc, 0);
$netdiff = $ans['netdiff:'.$i];
if ($netdiff > 0)
$diffratio = $ans['diffratio:'.$i];
$cdf = $ans['cdf:'.$i];
$luck = $ans['luck:'.$i];
if ($diffratio > 0)
{
$pct = 100.0 * $diffacc / $netdiff;
$pct = 100.0 * $diffratio;
list($fg, $bg) = pctcolour($pct);
$bpct = "<font color=$fg>$starp".number_format($pct, 2).'%</font>';
$bpct = "<font color=$fg>".number_format($pct, 2).'%</font>';
$bg = " bgcolor=$bg";
$blktot += $diffacc;
if ($stat != 'Orphan')
$nettot += $netdiff;
$cdfv = 1 - exp(-1 * $diffacc / $netdiff);
$cdf = number_format($cdfv, 2);
$cdfdsp = number_format($cdf, 2);
}
else
{
$bg = '';
$bpct = '?';
$cdf = '?';
$cdfdsp = '?';
}
if ($wantcsv === false)
@ -188,7 +221,7 @@ function doblocks($data, $user)
$pg .= "<td class=dr$ex>".$stat.'</td>';
$pg .= "<td class=dr>$stara$acc</td>";
$pg .= "<td class=dr$bg>$bpct</td>";
$pg .= "<td class=dr>$cdf</td>";
$pg .= "<td class=dr>$cdfdsp</td>";
$pg .= "</tr>\n";
}
else
@ -208,28 +241,6 @@ function doblocks($data, $user)
echo $csv;
exit(0);
}
if ($nettot > 0)
{
if (($i % 2) == 0)
$row = 'even';
else
$row = 'odd';
$pct = 100.0 * $blktot / $nettot;
list($fg, $bg) = pctcolour($pct);
$bpct = "<font color=$fg>".number_format($pct, 2).'%</font>';
$bg = " bgcolor=$bg";
$pg .= "<tr class=$row>";
$pg .= '<td colspan=2 class=dr>Total:</td>';
$pg .= '<td class=dl colspan=';
if ($user === null)
$pg .= '4';
else
$pg .= '5';
$pg .= '></td>';
$pg .= "<td class=dr$bg>".$bpct.'</td>';
$pg .= "<td></td></tr>\n";
if ($orph === true)
{
$pg .= '<tr><td colspan=';
@ -238,10 +249,9 @@ function doblocks($data, $user)
else
$pg .= '8';
$pg .= ' class=dc><font size=-1><span class=st1>*</span>';
$pg .= '% total is adjusted to include orphans correctly';
$pg .= "Orphans count as shares but not as a block in calculations";
$pg .= '</font></td></tr>';
}
}
$pg .= "</table>\n";
return $pg;

2
src/Makefile.am

@ -23,5 +23,5 @@ if WANT_CKDB
bin_PROGRAMS += ckdb
ckdb_SOURCES = ckdb.c ckdb_cmd.c ckdb_data.c ckdb_dbio.c ckdb_btc.c \
ckdb.h klist.c ktree.c klist.h ktree.h
ckdb_LDADD = libckpool.la @JANSSON_LIBS@ @PQ_LIBS@
ckdb_LDADD = libckpool.la @JANSSON_LIBS@ @PQ_LIBS@ @MATH_LIBS@
endif

2
src/ckdb.c

@ -399,6 +399,8 @@ const char *blocks_unknown = "?Unknown?";
K_TREE *blocks_root;
K_LIST *blocks_free;
K_STORE *blocks_store;
tv_t blocks_stats_time;
bool blocks_stats_rebuild = true;
// MININGPAYOUTS
K_TREE *miningpayouts_root;

29
src/ckdb.h

@ -37,6 +37,9 @@
#include <postgresql/libpq-fe.h>
#endif
#include <gsl/gsl_math.h>
#include <gsl/gsl_cdf.h>
#include "ckpool.h"
#include "libckpool.h"
@ -52,7 +55,7 @@
#define DB_VLOCK "1"
#define DB_VERSION "1.0.0"
#define CKDB_VERSION DB_VERSION"-1.012"
#define CKDB_VERSION DB_VERSION"-1.020"
#define WHERE_FFL " - from %s %s() line %d"
#define WHERE_FFL_HERE __FILE__, __func__, __LINE__
@ -192,6 +195,11 @@ extern POOLSTATUS pool;
_dstoff += _srclen; \
} while(0)
#define APPEND_REALLOC_RESET(_buf, _off) do { \
(_buf)[0] = '\0'; \
_off = 0; \
} while(0)
enum data_type {
TYPE_STR,
TYPE_BIGINT,
@ -1095,6 +1103,22 @@ typedef struct blocks {
char statsconfirmed[TXT_FLAG+1];
HISTORYDATECONTROLFIELDS;
bool ignore; // Non DB field
// Calculated only when = 0
double netdiff;
/* Non DB fields for the web page
* Calculate them once off/recalc them when required */
double blockdiffratio;
double blockcdf;
double blockluck;
/* From the last found block to this one
* Orphans have these set to zero */
double diffratio;
double diffmean;
double cdferl;
double luck;
} BLOCKS;
#define ALLOC_BLOCKS 100
@ -1134,6 +1158,8 @@ extern const char *blocks_unknown;
extern K_TREE *blocks_root;
extern K_LIST *blocks_free;
extern K_STORE *blocks_store;
extern tv_t blocks_stats_time;
extern bool blocks_stats_rebuild;
// MININGPAYOUTS
typedef struct miningpayouts {
@ -1750,6 +1776,7 @@ extern K_ITEM *find_prev_blocks(int32_t height);
extern const char *blocks_confirmed(char *confirmed);
extern void zero_on_new_block();
extern void set_block_share_counters();
extern bool check_update_blocks_stats(tv_t *stats);
extern cmp_t cmp_miningpayouts(K_ITEM *a, K_ITEM *b);
extern K_ITEM *find_miningpayouts(int64_t payoutid, int64_t userid);
extern K_ITEM *first_miningpayouts(int64_t payoutid, K_TREE_CTX *ctx);

132
src/ckdb_cmd.c

@ -811,23 +811,30 @@ static char *cmd_blocklist(__maybe_unused PGconn *conn, char *cmd, char *id,
__maybe_unused K_TREE *trf_root)
{
K_TREE_CTX ctx[1];
K_ITEM *b_item, *w_item;
K_ITEM *b_item;
BLOCKS *blocks;
char reply[1024] = "";
char tmp[1024];
char *buf;
char *buf, *desc, desc_buf[64];
size_t len, off;
int32_t height = -1;
tv_t first_cd = {0,0};
int rows, tot;
tv_t first_cd = {0,0}, stats_tv = {0,0}, stats_tv2 = {0,0};
int rows, srows, tot, seq;
bool has_stats;
LOGDEBUG("%s(): cmd '%s'", __func__, cmd);
APPEND_REALLOC_INIT(buf, off, len);
APPEND_REALLOC(buf, off, len, "ok.");
rows = 0;
redo:
K_WLOCK(blocks_free);
has_stats = check_update_blocks_stats(&stats_tv);
K_WUNLOCK(blocks_free);
srows = rows = 0;
K_RLOCK(blocks_free);
b_item = last_in_ktree(blocks_root, ctx);
b_item = first_in_ktree(blocks_root, ctx);
tot = 0;
while (b_item) {
DATA_BLOCKS(blocks, b_item);
@ -835,16 +842,31 @@ static char *cmd_blocklist(__maybe_unused PGconn *conn, char *cmd, char *id,
if (blocks->confirmed[0] != BLOCKS_ORPHAN)
tot++;
}
b_item = prev_in_ktree(ctx);
b_item = next_in_ktree(ctx);
}
seq = tot;
b_item = last_in_ktree(blocks_root, ctx);
while (b_item && rows < 42) {
DATA_BLOCKS(blocks, b_item);
/* For each block remember the initial createdate
* Reverse sort order the oldest expirydate is first
* which should be the 'n' record */
if (height != blocks->height) {
height = blocks->height;
copy_tv(&first_cd, &(blocks->createdate));
}
if (CURRENT(&(blocks->expirydate))) {
if (blocks->confirmed[0] == BLOCKS_ORPHAN) {
snprintf(tmp, sizeof(tmp),
"seq:%d=o%c",
rows, FLDSEP);
APPEND_REALLOC(buf, off, len, tmp);
} else {
snprintf(tmp, sizeof(tmp),
"seq:%d=%d%c",
rows, seq--, FLDSEP);
APPEND_REALLOC(buf, off, len, tmp);
}
int_to_buf(blocks->height, reply, sizeof(reply));
snprintf(tmp, sizeof(tmp), "height:%d=%s%c", rows, reply, FLDSEP);
APPEND_REALLOC(buf, off, len, tmp);
@ -900,21 +922,21 @@ static char *cmd_blocklist(__maybe_unused PGconn *conn, char *cmd, char *id,
snprintf(tmp, sizeof(tmp), "elapsed:%d=%s%c", rows, reply, FLDSEP);
APPEND_REALLOC(buf, off, len, tmp);
w_item = find_workinfo(blocks->workinfoid, NULL);
if (w_item) {
char wdiffbin[TXT_SML+1];
double wdiff;
WORKINFO *workinfo;
DATA_WORKINFO(workinfo, w_item);
hex2bin(wdiffbin, workinfo->bits, 4);
wdiff = diff_from_nbits(wdiffbin);
if (has_stats) {
snprintf(tmp, sizeof(tmp),
"netdiff:%d=%.1f%c",
rows, wdiff, FLDSEP);
"netdiff:%d=%.8f%cdiffratio:%d=%.8f%c"
"cdf:%d=%.8f%cluck:%d=%.8f%c",
rows, blocks->netdiff, FLDSEP,
rows, blocks->blockdiffratio, FLDSEP,
rows, blocks->blockcdf, FLDSEP,
rows, blocks->blockluck, FLDSEP);
APPEND_REALLOC(buf, off, len, tmp);
} else {
snprintf(tmp, sizeof(tmp),
"netdiff:%d=?%c", rows, FLDSEP);
"netdiff:%d=?%cdiffratio:%d=?%c"
"cdf:%d=?%cluck:%d=?%c",
rows, FLDSEP, rows, FLDSEP,
rows, FLDSEP, rows, FLDSEP);
APPEND_REALLOC(buf, off, len, tmp);
}
@ -922,17 +944,81 @@ static char *cmd_blocklist(__maybe_unused PGconn *conn, char *cmd, char *id,
}
b_item = prev_in_ktree(ctx);
}
if (has_stats) {
seq = tot;
b_item = last_in_ktree(blocks_root, ctx);
while (b_item) {
DATA_BLOCKS(blocks, b_item);
if (CURRENT(&(blocks->expirydate)) &&
blocks->confirmed[0] != BLOCKS_ORPHAN) {
desc = NULL;
if (seq == 1) {
snprintf(desc_buf, sizeof(desc_buf),
"All - Last %d", tot);
desc = desc_buf;
} else if (seq == tot - 4) {
desc = "Last 5";
} else if (seq == tot - 9) {
desc = "Last 10";
} else if (seq == tot - 19) {
desc = "Last 20";
} else if (seq == tot - 41) {
desc = "Last 42";
}
if (desc) {
snprintf(tmp, sizeof(tmp),
"s_seq:%d=%d%c"
"s_desc:%d=%s%c"
"s_diffratio:%d=%.8f%c"
"s_diffmean:%d=%.8f%c"
"s_cdferl:%d=%.8f%c"
"s_luck:%d=%.8f%c",
srows, seq, FLDSEP,
srows, desc, FLDSEP,
srows, blocks->diffratio, FLDSEP,
srows, blocks->diffmean, FLDSEP,
srows, blocks->cdferl, FLDSEP,
srows, blocks->luck, FLDSEP);
APPEND_REALLOC(buf, off, len, tmp);
srows++;
}
seq--;
}
b_item = prev_in_ktree(ctx);
}
copy_tv(&stats_tv2, &blocks_stats_time);
}
K_RUNLOCK(blocks_free);
// Only check for a redo if we used the stats values
if (has_stats) {
/* If the stats changed then redo with the new corrected values
* This isn't likely at all, but it guarantees the blocks
* page shows correct information since any code that wants
* to modify the blocks table must have it under write lock
* then flag the stats as needing to be recalculated */
if (!tv_equal(&stats_tv, &stats_tv2)) {
APPEND_REALLOC_RESET(buf, off);
goto redo;
}
}
snprintf(tmp, sizeof(tmp),
"s_rows=%d%cs_flds=%s%c",
srows, FLDSEP,
"s_seq,s_desc,s_diffratio,s_diffmean,s_cdferl,s_luck",
FLDSEP);
APPEND_REALLOC(buf, off, len, tmp);
snprintf(tmp, sizeof(tmp),
"tot=%d%crows=%d%cflds=%s%c",
tot, FLDSEP,
"rows=%d%cflds=%s%c",
rows, FLDSEP,
"height,blockhash,nonce,reward,workername,firstcreatedate,"
"seq,height,blockhash,nonce,reward,workername,firstcreatedate,"
"createdate,status,diffacc,diffinv,shareacc,shareinv,elapsed,"
"netdiff", FLDSEP);
"netdiff,diffratio,cdf,luck", FLDSEP);
APPEND_REALLOC(buf, off, len, tmp);
snprintf(tmp, sizeof(tmp), "arn=%s%carp=%s", "Blocks", FLDSEP, "");
snprintf(tmp, sizeof(tmp), "arn=%s%carp=%s", "Blocks,BlockStats", FLDSEP, ",s");
APPEND_REALLOC(buf, off, len, tmp);
LOGDEBUG("%s.ok.%d_blocks", id, rows);

108
src/ckdb_data.c

@ -2399,6 +2399,114 @@ void set_block_share_counters()
LOGWARNING("%s(): Update block counters complete", __func__);
}
/* Must be under K_WLOCK(blocks_free) when called
* Call this before using the block stats and again check (under lock)
* the blocks_stats_time didn't change after you finish processing
* If it has changed, redo the processing from scratch
* If return is false, then stats aren't available
* TODO: consider storing the partial calculations in the BLOCKS structure
* and only recalc from the last block modified (remembered)
* Will be useful with a large block history */
bool check_update_blocks_stats(tv_t *stats)
{
static int64_t last_missing_workinfoid = 0;
static tv_t last_message = { 0L, 0L };
K_TREE_CTX ctx[1];
K_ITEM *b_item, *w_item;
WORKINFO *workinfo;
BLOCKS *blocks;
char ndiffbin[TXT_SML+1];
double ok, diffacc, netsumm, diffmean, pending;
tv_t now;
/* Wait for startup_complete rather than db_load_complete
* This avoids doing a 'long' lock stats update while reloading */
if (!startup_complete)
return false;
if (blocks_stats_rebuild) {
ok = diffacc = netsumm = diffmean = pending = 0.0;
b_item = last_in_ktree(blocks_root, ctx);
while (b_item) {
DATA_BLOCKS(blocks, b_item);
if (CURRENT(&(blocks->expirydate))) {
if (blocks->netdiff == 0) {
// Deadlock alert
K_RLOCK(workinfo_free);
w_item = find_workinfo(blocks->workinfoid, NULL);
K_RUNLOCK(workinfo_free);
if (!w_item) {
setnow(&now);
if (!blocks->workinfoid != last_missing_workinfoid ||
tvdiff(&now, &last_message) >= 5.0) {
LOGEMERG("%s(): missing block workinfoid %"
PRId32"/%"PRId64"/%s",
__func__, blocks->height,
blocks->workinfoid,
blocks->confirmed);
}
last_missing_workinfoid = blocks->workinfoid;
copy_tv(&last_message, &now);
return false;
}
DATA_WORKINFO(workinfo, w_item);
hex2bin(ndiffbin, workinfo->bits, 4);
blocks->netdiff = diff_from_nbits(ndiffbin);
}
pending += blocks->diffacc;
/* Stats for each blocks are independent of
* if they are orphans or not */
if (blocks->netdiff == 0.0)
blocks->blockdiffratio = 0.0;
else
blocks->blockdiffratio = blocks->diffacc / blocks->netdiff;
blocks->blockcdf = 1.0 - exp(-1.0 * blocks->blockdiffratio);
if (blocks->blockdiffratio == 0.0)
blocks->blockluck = 0.0;
else
blocks->blockluck = 1.0 / blocks->blockdiffratio;
/* Orphans are treated as +diffacc but no block
* i.e. they simply add shares to the later block
* and have running stats set to zero */
if (blocks->confirmed[0] == BLOCKS_ORPHAN) {
blocks->diffratio = 0.0;
blocks->diffmean = 0.0;
blocks->cdferl = 0.0;
blocks->luck = 0.0;
} else {
ok++;
diffacc += pending;
pending = 0.0;
netsumm += blocks->netdiff;
if (netsumm == 0.0)
blocks->diffratio = 0.0;
else
blocks->diffratio = diffacc / netsumm;
diffmean = (diffmean * (ok - 1) + blocks->blockdiffratio) / ok;
blocks->diffmean = diffmean;
if (diffmean == 0.0) {
blocks->cdferl = 0.0;
blocks->luck = 0.0;
} else {
blocks->cdferl = gsl_cdf_gamma_P(diffmean, ok, 1.0 / ok);
blocks->luck = 1.0 / diffmean;
}
}
}
b_item = prev_in_ktree(ctx);
}
setnow(&blocks_stats_time);
blocks_stats_rebuild = false;
}
copy_tv(stats, &blocks_stats_time);
return true;
}
/* order by payoutid asc,userid asc,expirydate asc
* i.e. only one payout amount per block per user */
cmp_t cmp_miningpayouts(K_ITEM *a, K_ITEM *b)

12
src/ckdb_dbio.c

@ -3996,9 +3996,13 @@ unparam:
blocks_root = remove_from_ktree(blocks_root, old_b_item, cmp_blocks);
copy_tv(&(oldblocks->expirydate), cd);
blocks_root = add_to_ktree(blocks_root, old_b_item, cmp_blocks);
}
// Copy it over to avoid having to recalculate it
row->netdiff = oldblocks->netdiff;
} else
row->netdiff = 0;
blocks_root = add_to_ktree(blocks_root, b_item, cmp_blocks);
k_add_head(blocks_store, b_item);
blocks_stats_rebuild = true;
}
K_WUNLOCK(blocks_free);
@ -4300,9 +4304,13 @@ flail:
blocks_root = remove_from_ktree(blocks_root, old_b_item, cmp_blocks);
copy_tv(&(oldblocks->expirydate), cd);
blocks_root = add_to_ktree(blocks_root, old_b_item, cmp_blocks);
}
// Copy it over to avoid having to recalculate it
row->netdiff = oldblocks->netdiff;
} else
row->netdiff = 0;
blocks_root = add_to_ktree(blocks_root, b_item, cmp_blocks);
k_add_head(blocks_store, b_item);
blocks_stats_rebuild = true;
}
K_WUNLOCK(blocks_free);

Loading…
Cancel
Save