From 857a2b5f79d2a184f73be56ab0b0083b675dd1e6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jorge=20Tim=C3=B3n?= Date: Sat, 8 Jul 2017 02:15:36 +0200 Subject: [PATCH 1/6] Avoid special case for truncated zeros with new CFeeRate::GetTruncatedFee --- src/policy/feerate.cpp | 14 ++++++++------ src/policy/feerate.h | 7 ++++++- 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/src/policy/feerate.cpp b/src/policy/feerate.cpp index a089c02284f0..c8b4f54e194d 100644 --- a/src/policy/feerate.cpp +++ b/src/policy/feerate.cpp @@ -20,20 +20,22 @@ CFeeRate::CFeeRate(const CAmount& nFeePaid, size_t nBytes_) nSatoshisPerK = 0; } -CAmount CFeeRate::GetFee(size_t nBytes_) const +CAmount CFeeRate::GetTruncatedFee(size_t bytes) const { - assert(nBytes_ <= uint64_t(std::numeric_limits::max())); - int64_t nSize = int64_t(nBytes_); + assert(bytes <= uint64_t(std::numeric_limits::max())); + return nSatoshisPerK * int64_t(bytes) / 1000; +} - CAmount nFee = nSatoshisPerK * nSize / 1000; +CAmount CFeeRate::GetFee(size_t bytes) const +{ + CAmount nFee = GetTruncatedFee(bytes); - if (nFee == 0 && nSize != 0) { + if (nFee == 0 && bytes != 0) { if (nSatoshisPerK > 0) nFee = CAmount(1); if (nSatoshisPerK < 0) nFee = CAmount(-1); } - return nFee; } diff --git a/src/policy/feerate.h b/src/policy/feerate.h index 3449cdd6990d..80b089d87821 100644 --- a/src/policy/feerate.h +++ b/src/policy/feerate.h @@ -34,7 +34,12 @@ class CFeeRate /** * Return the fee in satoshis for the given size in bytes. */ - CAmount GetFee(size_t nBytes) const; + CAmount GetTruncatedFee(size_t bytes) const; + /** + * Return the fee in satoshis for the given size in bytes. If the + * result is zero, return 1 if the fee was positive or -1 if was negative. + */ + CAmount GetFee(size_t bytes) const; /** * Return the fee in satoshis for a size of 1000 bytes */ From bd371df4c862e78abcd5f29f62d0bb4b2d8205fa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jorge=20Tim=C3=B3n?= Date: Sun, 4 Jun 2017 17:14:00 +0200 Subject: [PATCH 2/6] RPC: Separate ReadBlockCheckPruned() from getblock() --- src/rpc/blockchain.cpp | 28 +++++++++++++++++----------- 1 file changed, 17 insertions(+), 11 deletions(-) diff --git a/src/rpc/blockchain.cpp b/src/rpc/blockchain.cpp index 68af376f355b..bbad2e49e887 100644 --- a/src/rpc/blockchain.cpp +++ b/src/rpc/blockchain.cpp @@ -688,6 +688,22 @@ UniValue getblockheader(const JSONRPCRequest& request) return blockheaderToJSON(pblockindex); } +static void ReadBlockCheckPruned(CBlock& block, const CBlockIndex* pblockindex) +{ + if (fHavePruned && !(pblockindex->nStatus & BLOCK_HAVE_DATA) && pblockindex->nTx > 0) { + throw JSONRPCError(RPC_MISC_ERROR, "Block not available (pruned data)"); + } + + if (!ReadBlockFromDisk(block, pblockindex, Params().GetConsensus())) { + // Block not found on disk. This could be because we have the block + // header in our index but don't have the block (for example if a + // non-whitelisted node sends us an unrequested long chain of valid + // blocks, we add the headers to our index, but don't accept the + // block). + throw JSONRPCError(RPC_MISC_ERROR, "Block not found on disk"); + } +} + UniValue getblock(const JSONRPCRequest& request) { if (request.fHelp || request.params.size() < 1 || request.params.size() > 2) @@ -756,17 +772,7 @@ UniValue getblock(const JSONRPCRequest& request) CBlock block; CBlockIndex* pblockindex = mapBlockIndex[hash]; - - if (fHavePruned && !(pblockindex->nStatus & BLOCK_HAVE_DATA) && pblockindex->nTx > 0) - throw JSONRPCError(RPC_MISC_ERROR, "Block not available (pruned data)"); - - if (!ReadBlockFromDisk(block, pblockindex, Params().GetConsensus())) - // Block not found on disk. This could be because we have the block - // header in our index but don't have the block (for example if a - // non-whitelisted node sends us an unrequested long chain of valid - // blocks, we add the headers to our index, but don't accept the - // block). - throw JSONRPCError(RPC_MISC_ERROR, "Block not found on disk"); + ReadBlockCheckPruned(block, pblockindex); if (verbosity <= 0) { From 2497afa41126fd4e5cf17b095980c19fc1b813a5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jorge=20Tim=C3=B3n?= Date: Sun, 4 Jun 2017 00:25:55 +0200 Subject: [PATCH 3/6] RPC: Introduce getblockstats --- src/rpc/blockchain.cpp | 307 ++++++++++++++++++++++++++++++++++++++++- src/rpc/client.cpp | 2 + 2 files changed, 308 insertions(+), 1 deletion(-) diff --git a/src/rpc/blockchain.cpp b/src/rpc/blockchain.cpp index bbad2e49e887..e90e921452eb 100644 --- a/src/rpc/blockchain.cpp +++ b/src/rpc/blockchain.cpp @@ -30,6 +30,7 @@ #include +#include #include // boost::thread::interrupt #include @@ -1538,7 +1539,7 @@ UniValue getchaintxstats(const JSONRPCRequest& request) pindex = chainActive.Tip(); } } - + assert(pindex != nullptr); if (request.params[0].isNull()) { @@ -1570,6 +1571,309 @@ UniValue getchaintxstats(const JSONRPCRequest& request) return ret; } +static void RpcGetTx(const uint256& hash, CTransactionRef& tx_out) +{ + uint256 hashBlock; + if (!GetTransaction(hash, tx_out, Params().GetConsensus(), hashBlock, true)) { + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, std::string(fTxIndex ? "No such mempool or blockchain transaction" + : "No such mempool transaction. Use -txindex to enable blockchain transaction queries")); + } +} + +template +static T CalculateTruncatedMedian(std::vector& scores) +{ + size_t size = scores.size(); + if (size == 0) { + return 0; + } if (size == 1) { + return scores[0]; + } + + std::sort(scores.begin(), scores.end()); + if (size % 2 == 0) { + return (scores[size / 2 - 1] + scores[size / 2]) / 2; + } else { + return scores[size / 2]; + } +} + +// outpoint (needed for the utxo index) + nHeight + fCoinBase +static const size_t PER_UTXO_OVERHEAD = sizeof(COutPoint) + sizeof(uint32_t) + sizeof(bool); + +static void UpdateBlockStats(const CBlockIndex* pindex, std::set& stats, std::map& map_stats) +{ + int64_t inputs = 0; + int64_t outputs = 0; + int64_t swtxs = 0; + int64_t total_size = 0; + int64_t total_weight = 0; + int64_t swtotal_size = 0; + int64_t swtotal_weight = 0; + int64_t utxo_size_inc = 0; + CAmount total_out = 0; + CAmount totalfee = 0; + CAmount minfee = MAX_MONEY; + CAmount maxfee = 0; + CAmount minfeerate = MAX_MONEY; + CAmount maxfeerate = 0; + std::vector fee_array; + std::vector feerate_array; + int64_t mintxsize = MAX_BLOCK_SERIALIZED_SIZE; + int64_t maxtxsize = 0; + std::vector txsize_array; + + CBlock block; + ReadBlockCheckPruned(block, pindex); + + for (const auto& tx : block.vtx) { + outputs += tx->vout.size(); + CAmount tx_total_out = 0; + for (const CTxOut& out : tx->vout) { + utxo_size_inc += GetSerializeSize(out, SER_NETWORK, PROTOCOL_VERSION) + PER_UTXO_OVERHEAD; + tx_total_out += out.nValue; + } + + if (tx->IsCoinBase()) { + continue; + } + total_out += tx_total_out; + inputs += tx->vin.size(); // Don't count coinbase's fake input + int64_t tx_size = tx->GetTotalSize(); + txsize_array.push_back(tx_size); + total_size += tx_size; + mintxsize = std::min(mintxsize, tx_size); + maxtxsize = std::max(maxtxsize, tx_size); + int64_t weight = GetTransactionWeight(*tx); + total_weight += weight; + + if (tx->HasWitness()) { + ++swtxs; + swtotal_size += tx_size; + swtotal_weight += weight; + } + + CAmount tx_total_in = 0; + for (const CTxIn& in : tx->vin) { + CTransactionRef tx_in; + RpcGetTx(in.prevout.hash, tx_in); + CTxOut prevoutput = tx_in->vout[in.prevout.n]; + + tx_total_in += prevoutput.nValue; + utxo_size_inc -= GetSerializeSize(prevoutput, SER_NETWORK, PROTOCOL_VERSION) + PER_UTXO_OVERHEAD; + } + CAmount txfee = tx_total_in - tx_total_out; + assert(MoneyRange(txfee)); + fee_array.push_back(txfee); + totalfee += txfee; + minfee = std::min(minfee, txfee); + maxfee = std::max(maxfee, txfee); + + // New feerate uses satoshis per virtual byte instead of per serialized byte + CAmount feerate = CFeeRate(txfee, weight).GetTruncatedFee(WITNESS_SCALE_FACTOR); + feerate_array.push_back(feerate); + + minfeerate = std::min(minfeerate, feerate); + maxfeerate = std::max(maxfeerate, feerate); + } + + for (const std::string& stat : stats) { + // Update map_stats + if (stat == "height") { + map_stats[stat].push_back((int64_t)pindex->nHeight); + } else if (stat == "time") { + map_stats[stat].push_back(pindex->GetBlockTime()); + } else if (stat == "mediantime") { + map_stats[stat].push_back(pindex->GetMedianTimePast()); + } else if (stat == "subsidy") { + map_stats[stat].push_back(GetBlockSubsidy(pindex->nHeight, Params().GetConsensus())); + } else if (stat == "totalfee") { + map_stats[stat].push_back(totalfee); + } else if (stat == "txs") { + map_stats[stat].push_back((int64_t)block.vtx.size()); + } else if (stat == "swtxs") { + map_stats[stat].push_back(swtxs); + } else if (stat == "ins") { + map_stats[stat].push_back(inputs); + } else if (stat == "outs") { + map_stats[stat].push_back(outputs); + } else if (stat == "utxo_increase") { + map_stats[stat].push_back(outputs - inputs); + } else if (stat == "utxo_size_inc") { + map_stats[stat].push_back(utxo_size_inc); + } else if (stat == "total_size") { + map_stats[stat].push_back(total_size); + } else if (stat == "total_weight") { + map_stats[stat].push_back(total_weight); + } else if (stat == "swtotal_size") { + map_stats[stat].push_back(swtotal_size); + } else if (stat == "swtotal_weight") { + map_stats[stat].push_back(swtotal_weight); + } else if (stat == "total_out") { + map_stats[stat].push_back(total_out); + } else if (stat == "minfee") { + map_stats[stat].push_back((minfee == MAX_MONEY) ? 0 : minfee); + } else if (stat == "maxfee") { + map_stats[stat].push_back(maxfee); + } else if (stat == "medianfee") { + map_stats[stat].push_back(CalculateTruncatedMedian(fee_array)); + } else if (stat == "avgfee") { + map_stats[stat].push_back((block.vtx.size() > 1) ? totalfee / (block.vtx.size() - 1) : 0); + } else if (stat == "minfeerate") { + map_stats[stat].push_back((minfeerate == MAX_MONEY) ? 0 : minfeerate); + } else if (stat == "maxfeerate") { + map_stats[stat].push_back(maxfeerate); + } else if (stat == "medianfeerate") { + map_stats[stat].push_back(CalculateTruncatedMedian(feerate_array)); + } else if (stat == "avgfeerate") { + map_stats[stat].push_back(CFeeRate(totalfee, total_weight).GetTruncatedFee(WITNESS_SCALE_FACTOR)); + } else if (stat == "mintxsize") { + map_stats[stat].push_back(mintxsize == MAX_BLOCK_SERIALIZED_SIZE ? 0 : mintxsize); + } else if (stat == "maxtxsize") { + map_stats[stat].push_back(maxtxsize); + } else if (stat == "mediantxsize") { + map_stats[stat].push_back(CalculateTruncatedMedian(txsize_array)); + } else if (stat == "avgtxsize") { + map_stats[stat].push_back((block.vtx.size() > 1) ? total_size / (block.vtx.size() - 1) : 0); + } + } +} + +UniValue getblockstats(const JSONRPCRequest& request) +{ + std::set valid_stats = { + "height", + "time", + "mediantime", + "txs", + "swtxs", + "ins", + "outs", + "subsidy", + "totalfee", + "utxo_increase", + "utxo_size_inc", + "total_size", + "total_weight", + "swtotal_size", + "swtotal_weight", + "total_out", + "minfee", + "maxfee", + "medianfee", + "avgfee", + "minfeerate", + "maxfeerate", + "medianfeerate", + "avgfeerate", + "mintxsize", + "maxtxsize", + "mediantxsize", + "avgtxsize", + }; + + if (request.fHelp || request.params.size() < 1 || request.params.size() > 4) + throw std::runtime_error( + "getblockstats ( nStart nEnd stats )\n" + "\nCompute per block statistics for a given window. All amounts are in satoshis.\n" + "\nNegative values for start or end count back from the current tip.\n" + "\nIt won't work in some cases with pruning or without -txindex.\n" + "\nArguments:\n" + "1. \"start\" (numeric, required) The height of the block that starts the window.\n" + "2. \"end\" (numeric, optional) The height of the block that ends the window (default: current tip).\n" + "3. \"stats\" (string, optional) Values to plot (comma separated), default(all): " + boost::join(valid_stats, ",") + + "\nResult: (all values are in reverse order height-wise)\n" + "{ (json object)\n" + " \"height\": [], (array) The height of the blocks, ie: [end, end-1, ..., start+1, start].\n" + " \"time\": [], (array) The block time.\n" + " \"mediantime\": [], (array) The block median time past.\n" + " \"txs\": [], (array) The number of transactions (excluding coinbase).\n" + " \"swtxs\": [], (array) The number of segwit transactions.\n" + " \"ins\": [], (array) The number of inputs (excluding coinbase).\n" + " \"outs\": [], (array) The number of outputs (including coinbase).\n" + " \"subsidy\": [], (array) The block subsidy.\n" + " \"totalfee\": [], (array) The fee total.\n" + " \"utxo_increase\": [], (array) The increase/decrease in the number of unspent outputs.\n" + " \"utxo_size_inc\": [], (array) The increase/decrease in size for the utxo index (not discounting op_return and similar).\n" + " \"total_size\": [], (array) Total size of all non-coinbase transactions.\n" + " \"total_weight\": [], (array) Total weight of all non-coinbase transactions divided by segwit scale factor (4).\n" + " \"swtotal_size\": [], (array) Total size of all segwit transactions.\n" + " \"swtotal_weight\": [], (array) Total weight of all segwit transactions divided by segwit scale factor (4).\n" + " \"total_out\": [], (array) Total amount in all outputs (excluding coinbase and thus reward [ie subsidy + totalfee]).\n" + " \"minfee\": [], (array) Minimum fee in the block.\n" + " \"maxfee\": [], (array) Maximum fee in the block.\n" + " \"medianfee\": [], (array) Truncated median fee in the block.\n" + " \"avgfee\": [], (array) Average fee in the block.\n" + " \"minfeerate\": [], (array) Minimum feerate (in satoshis per virtual byte).\n" + " \"maxfeerate\": [], (array) Maximum feerate (in satoshis per virtual byte).\n" + " \"medianfeerate\": [], (array) Truncated median feerate (in satoshis per virtual byte).\n" + " \"avgfeerate\": [], (array) Average feerate (in satoshis per virtual byte).\n" + " \"mintxsize\": [], (array) Minimum transaction size.\n" + " \"maxtxsize\": [], (array) Maximum transaction size.\n" + " \"mediantxsize\": [], (array) Truncated median transaction size.\n" + " \"avgtxsize\": [], (array) Average transaction size.\n" + "}\n" + "\nExamples:\n" + + HelpExampleCli("getblockstats", "1000 1000 \"minfeerate,avgfeerate\"") + + HelpExampleRpc("getblockstats", "1000 1000 \"maxfeerate,avgfeerate\"") + ); + + LOCK(cs_main); + + int start = request.params[0].get_int(); + int current_tip = chainActive.Height(); + if (start < 0) { + start = current_tip + start; + } + if (start < 0 || start > current_tip) { + throw JSONRPCError(RPC_INVALID_PARAMETER, strprintf("Start block height %d after current tip %d", start, current_tip)); + } + + int end; + if (request.params.size() > 1) { + end = request.params[1].get_int(); + if (end < 0) { + end = current_tip + end; + } + } else { + end = current_tip; + } + if (end < 0 || end > current_tip) { + throw JSONRPCError(RPC_INVALID_PARAMETER, strprintf("End block height %d after current tip %d", end, current_tip)); + } + if (start > end) { + throw JSONRPCError(RPC_INVALID_PARAMETER, strprintf("Start block height %d higher than end %d", start, end)); + } + + std::set stats; + if (request.params.size() > 2) { + boost::split(stats, request.params[2].get_str(), boost::is_any_of(",")); + + for (const std::string& stat : stats) { + if (valid_stats.count(stat) == 0) { + throw JSONRPCError(RPC_INVALID_PARAMETER, strprintf("Invalid selected statistic %s", stat)); + } + } + } else { + stats = valid_stats; + } + + std::map map_stats; + for (const std::string& stat : stats) { + map_stats[stat] = UniValue(UniValue::VARR); + } + + for (int i = start; i <= end; ++i) { + UpdateBlockStats(chainActive[i], stats, map_stats); + } + + UniValue ret(UniValue::VOBJ); + for (const std::string stat : stats) { + ret.push_back(Pair(stat, map_stats[stat])); + } + return ret; +} + UniValue savemempool(const JSONRPCRequest& request) { if (request.fHelp || request.params.size() != 0) { @@ -1594,6 +1898,7 @@ static const CRPCCommand commands[] = // --------------------- ------------------------ ----------------------- ---------- { "blockchain", "getblockchaininfo", &getblockchaininfo, {} }, { "blockchain", "getchaintxstats", &getchaintxstats, {"nblocks", "blockhash"} }, + { "blockchain", "getblockstats", &getblockstats, {"start", "end", "stats"} }, { "blockchain", "getbestblockhash", &getbestblockhash, {} }, { "blockchain", "getblockcount", &getblockcount, {} }, { "blockchain", "getblock", &getblock, {"blockhash","verbosity|verbose"} }, diff --git a/src/rpc/client.cpp b/src/rpc/client.cpp index 721f363aeff7..0b335fe731f3 100644 --- a/src/rpc/client.cpp +++ b/src/rpc/client.cpp @@ -111,6 +111,8 @@ static const CRPCConvertParam vRPCConvertParams[] = { "importmulti", 1, "options" }, { "verifychain", 0, "checklevel" }, { "verifychain", 1, "nblocks" }, + { "getblockstats", 0, "start" }, + { "getblockstats", 1, "end" }, { "pruneblockchain", 0, "height" }, { "keypoolrefill", 0, "newsize" }, { "getrawmempool", 0, "verbose" }, From caeaf86de426f9b71c0828be23253827dd0149fb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jorge=20Tim=C3=B3n?= Date: Wed, 21 Jun 2017 03:08:05 +0200 Subject: [PATCH 4/6] QA: Test new getblockstats RPC --- test/functional/getblockstats.py | 187 +++++++++++++++++++++++++++++++ test/functional/test_runner.py | 1 + 2 files changed, 188 insertions(+) create mode 100755 test/functional/getblockstats.py diff --git a/test/functional/getblockstats.py b/test/functional/getblockstats.py new file mode 100755 index 000000000000..00fddf3a90de --- /dev/null +++ b/test/functional/getblockstats.py @@ -0,0 +1,187 @@ +#!/usr/bin/env python3 +# Copyright (c) 2017-2017 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +# +# Test getblockstats rpc call +# +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import ( + assert_equal, + assert_raises_rpc_error, +) + +def assert_contains(data, values, check_cointains=True): + for val in values: + if (check_cointains): + assert(val in data) + else: + assert(val not in data) + +class GetblockstatsTest(BitcoinTestFramework): + def set_test_params(self): + self.num_nodes = 2 + self.extra_args = [['-txindex'], ['-paytxfee=0.003']] + self.setup_clean_chain = True + + def run_test(self): + node = self.nodes[0] + node.generate(101) + + node.sendtoaddress(address=self.nodes[1].getnewaddress(), amount=10, subtractfeefromamount=True) + node.generate(1) + self.sync_all() + + node.sendtoaddress(address=node.getnewaddress(), amount=10, subtractfeefromamount=True) + node.sendtoaddress(address=node.getnewaddress(), amount=10, subtractfeefromamount=True) + self.nodes[1].sendtoaddress(address=node.getnewaddress(), amount=1, subtractfeefromamount=True) + self.sync_all() + node.generate(1) + + start_height = 101 + max_stat_pos = 2 + stats = node.getblockstats(start=start_height, end=start_height + max_stat_pos) + + all_values = [ + "height", + "time", + "mediantime", + "txs", + "swtxs", + "ins", + "outs", + "subsidy", + "totalfee", + "utxo_increase", + "utxo_size_inc", + "total_size", + "total_weight", + "swtotal_size", + "swtotal_weight", + "total_out", + "minfee", + "maxfee", + "medianfee", + "avgfee", + "minfeerate", + "maxfeerate", + "medianfeerate", + "avgfeerate", + "mintxsize", + "maxtxsize", + "mediantxsize", + "avgtxsize", + ] + assert_contains(stats, all_values) + # Make sure all valid statistics are included + assert_contains(all_values, stats.keys()) + + assert_equal(stats['height'][0], start_height) + assert_equal(stats['height'][max_stat_pos], start_height + max_stat_pos) + + assert_equal(stats['txs'][0], 1) + assert_equal(stats['swtxs'][0], 0) + assert_equal(stats['ins'][0], 0) + assert_equal(stats['outs'][0], 2) + assert_equal(stats['totalfee'][0], 0) + assert_equal(stats['utxo_increase'][0], 2) + assert_equal(stats['utxo_size_inc'][0], 173) + assert_equal(stats['total_size'][0], 0) + assert_equal(stats['total_weight'][0], 0) + assert_equal(stats['swtotal_size'][0], 0) + assert_equal(stats['swtotal_weight'][0], 0) + assert_equal(stats['total_out'][0], 0) + assert_equal(stats['minfee'][0], 0) + assert_equal(stats['maxfee'][0], 0) + assert_equal(stats['medianfee'][0], 0) + assert_equal(stats['avgfee'][0], 0) + assert_equal(stats['minfeerate'][0], 0) + assert_equal(stats['maxfeerate'][0], 0) + assert_equal(stats['medianfeerate'][0], 0) + assert_equal(stats['avgfeerate'][0], 0) + assert_equal(stats['mintxsize'][0], 0) + assert_equal(stats['maxtxsize'][0], 0) + assert_equal(stats['mediantxsize'][0], 0) + assert_equal(stats['avgtxsize'][0], 0) + + assert_equal(stats['txs'][1], 2) + assert_equal(stats['swtxs'][1], 0) + assert_equal(stats['ins'][1], 1) + assert_equal(stats['outs'][1], 4) + assert_equal(stats['totalfee'][1], 3840) + assert_equal(stats['utxo_increase'][1], 3) + assert_equal(stats['utxo_size_inc'][1], 238) + # assert_equal(stats['total_size'][1], 191) + # assert_equal(stats['total_weight'][1], 768) + assert_equal(stats['total_out'][1], 4999996160) + assert_equal(stats['minfee'][1], 3840) + assert_equal(stats['maxfee'][1], 3840) + assert_equal(stats['medianfee'][1], 3840) + assert_equal(stats['avgfee'][1], 3840) + assert_equal(stats['minfeerate'][1], 20) + assert_equal(stats['maxfeerate'][1], 20) + assert_equal(stats['medianfeerate'][1], 20) + assert_equal(stats['avgfeerate'][1], 20) + # assert_equal(stats['mintxsize'][1], 192) + # assert_equal(stats['maxtxsize'][1], 192) + # assert_equal(stats['mediantxsize'][1], 192) + # assert_equal(stats['avgtxsize'][1], 192) + + assert_equal(stats['txs'][max_stat_pos], 4) + assert_equal(stats['swtxs'][max_stat_pos], 0) + assert_equal(stats['ins'][max_stat_pos], 3) + assert_equal(stats['outs'][max_stat_pos], 8) + assert_equal(stats['totalfee'][max_stat_pos], 76160) + assert_equal(stats['utxo_increase'][max_stat_pos], 5) + assert_equal(stats['utxo_size_inc'][max_stat_pos], 388) + # assert_equal(stats['total_size'][max_stat_pos], 643) + # assert_equal(stats['total_weight'][max_stat_pos], 2572) + assert_equal(stats['total_out'][max_stat_pos], 9999920000) + assert_equal(stats['minfee'][max_stat_pos], 3840) + assert_equal(stats['maxfee'][max_stat_pos], 67800) + assert_equal(stats['medianfee'][max_stat_pos], 4520) + assert_equal(stats['avgfee'][max_stat_pos], 25386) + assert_equal(stats['minfeerate'][max_stat_pos], 20) + # assert_equal(stats['maxfeerate'][max_stat_pos], 300) + assert_equal(stats['medianfeerate'][max_stat_pos], 20) + assert_equal(stats['avgfeerate'][max_stat_pos], 118) + # assert_equal(stats['mintxsize'][max_stat_pos], 192) + # assert_equal(stats['maxtxsize'][max_stat_pos], 226) + # assert_equal(stats['mediantxsize'][max_stat_pos], 225) + # assert_equal(stats['avgtxsize'][max_stat_pos], 214) + + # Test invalid parameters raise the proper json exceptions + tip = start_height + max_stat_pos + assert_raises_rpc_error(-8, 'Start block height %d after current tip %d' % (tip+1, tip), node.getblockstats, start=tip+1) + assert_raises_rpc_error(-8, 'Start block height %d after current tip %d' % (-1, tip), node.getblockstats, start=-tip-1) + assert_raises_rpc_error(-8, 'Start block height %d higher than end %d' % (tip-1, tip-2), node.getblockstats, start=-1, end=-2) + assert_raises_rpc_error(-8, 'End block height %d after current tip %d' % (tip+1, tip), node.getblockstats, start=1, end=tip+1) + assert_raises_rpc_error(-8, 'Start block height 2 higher than end 1', node.getblockstats, start=2, end=1) + assert_raises_rpc_error(-8, 'Start block height %d higher than end %d' % (tip, tip-1), node.getblockstats, start=tip, end=tip-1) + + # Make sure not valid stats aren't allowed + inv_sel_stat = 'asdfghjkl' + inv_stats = [ + 'minfee,%s' % inv_sel_stat, + '%s,minfee' % inv_sel_stat, + 'minfee,%s,maxfee' % inv_sel_stat, + ] + for inv_stat in inv_stats: + assert_raises_rpc_error(-8, 'Invalid selected statistic %s' % inv_sel_stat, node.getblockstats, start=1, end=2, stats=inv_stat) + # Make sure we aren't always returning inv_sel_stat as the culprit stat + assert_raises_rpc_error(-8, 'Invalid selected statistic aaa%s' % inv_sel_stat, node.getblockstats, start=1, end=2, stats='minfee,aaa%s' % inv_sel_stat) + + # Make sure only the selected statistics are included + stats = node.getblockstats(start=1, end=2, stats='minfee,maxfee') + some_values = [ + 'minfee', + 'maxfee', + ] + assert_contains(stats, some_values) + # Make sure valid stats that haven't been selected don't appear + other_values = [x for x in all_values if x not in some_values] + assert_contains(stats, other_values, False) + +if __name__ == '__main__': + GetblockstatsTest().main() diff --git a/test/functional/test_runner.py b/test/functional/test_runner.py index 8c4651f6e02e..5e11167daff1 100755 --- a/test/functional/test_runner.py +++ b/test/functional/test_runner.py @@ -122,6 +122,7 @@ 'bip65-cltv-p2p.py', 'uptime.py', 'resendwallettransactions.py', + 'getblockstats.py', 'minchainwork.py', 'p2p-fingerprint.py', ] From c203b30401ee9178b9021c280815ecfe85349812 Mon Sep 17 00:00:00 2001 From: Anthony Towns Date: Thu, 19 Oct 2017 20:16:02 +1000 Subject: [PATCH 5/6] switch to decimal bitcoins rather than integer satoshis --- src/rpc/blockchain.cpp | 22 ++++++++-------- test/functional/getblockstats.py | 43 ++++++++++++++++++-------------- 2 files changed, 35 insertions(+), 30 deletions(-) diff --git a/src/rpc/blockchain.cpp b/src/rpc/blockchain.cpp index e90e921452eb..f236476559a5 100644 --- a/src/rpc/blockchain.cpp +++ b/src/rpc/blockchain.cpp @@ -1686,9 +1686,9 @@ static void UpdateBlockStats(const CBlockIndex* pindex, std::set& s } else if (stat == "mediantime") { map_stats[stat].push_back(pindex->GetMedianTimePast()); } else if (stat == "subsidy") { - map_stats[stat].push_back(GetBlockSubsidy(pindex->nHeight, Params().GetConsensus())); + map_stats[stat].push_back(ValueFromAmount(GetBlockSubsidy(pindex->nHeight, Params().GetConsensus()))); } else if (stat == "totalfee") { - map_stats[stat].push_back(totalfee); + map_stats[stat].push_back(ValueFromAmount(totalfee)); } else if (stat == "txs") { map_stats[stat].push_back((int64_t)block.vtx.size()); } else if (stat == "swtxs") { @@ -1710,23 +1710,23 @@ static void UpdateBlockStats(const CBlockIndex* pindex, std::set& s } else if (stat == "swtotal_weight") { map_stats[stat].push_back(swtotal_weight); } else if (stat == "total_out") { - map_stats[stat].push_back(total_out); + map_stats[stat].push_back(ValueFromAmount(total_out)); } else if (stat == "minfee") { - map_stats[stat].push_back((minfee == MAX_MONEY) ? 0 : minfee); + map_stats[stat].push_back(ValueFromAmount((minfee == MAX_MONEY) ? 0 : minfee)); } else if (stat == "maxfee") { - map_stats[stat].push_back(maxfee); + map_stats[stat].push_back(ValueFromAmount(maxfee)); } else if (stat == "medianfee") { - map_stats[stat].push_back(CalculateTruncatedMedian(fee_array)); + map_stats[stat].push_back(ValueFromAmount(CalculateTruncatedMedian(fee_array))); } else if (stat == "avgfee") { - map_stats[stat].push_back((block.vtx.size() > 1) ? totalfee / (block.vtx.size() - 1) : 0); + map_stats[stat].push_back(ValueFromAmount((block.vtx.size() > 1) ? totalfee / (block.vtx.size() - 1) : 0)); } else if (stat == "minfeerate") { - map_stats[stat].push_back((minfeerate == MAX_MONEY) ? 0 : minfeerate); + map_stats[stat].push_back(ValueFromAmount((minfeerate == MAX_MONEY) ? 0 : minfeerate)); } else if (stat == "maxfeerate") { - map_stats[stat].push_back(maxfeerate); + map_stats[stat].push_back(ValueFromAmount(maxfeerate)); } else if (stat == "medianfeerate") { - map_stats[stat].push_back(CalculateTruncatedMedian(feerate_array)); + map_stats[stat].push_back(ValueFromAmount(CalculateTruncatedMedian(feerate_array))); } else if (stat == "avgfeerate") { - map_stats[stat].push_back(CFeeRate(totalfee, total_weight).GetTruncatedFee(WITNESS_SCALE_FACTOR)); + map_stats[stat].push_back(ValueFromAmount(CFeeRate(totalfee, total_weight).GetTruncatedFee(WITNESS_SCALE_FACTOR))); } else if (stat == "mintxsize") { map_stats[stat].push_back(mintxsize == MAX_BLOCK_SERIALIZED_SIZE ? 0 : mintxsize); } else if (stat == "maxtxsize") { diff --git a/test/functional/getblockstats.py b/test/functional/getblockstats.py index 00fddf3a90de..e97989a8d413 100755 --- a/test/functional/getblockstats.py +++ b/test/functional/getblockstats.py @@ -12,6 +12,11 @@ assert_raises_rpc_error, ) +import decimal +decimal.getcontext().rounding = decimal.ROUND_DOWN +def D(f): + return decimal.Decimal(str(f)) + def assert_contains(data, values, check_cointains=True): for val in values: if (check_cointains): @@ -109,20 +114,20 @@ def run_test(self): assert_equal(stats['swtxs'][1], 0) assert_equal(stats['ins'][1], 1) assert_equal(stats['outs'][1], 4) - assert_equal(stats['totalfee'][1], 3840) + assert_equal(stats['totalfee'][1], D(0.00003840)) assert_equal(stats['utxo_increase'][1], 3) assert_equal(stats['utxo_size_inc'][1], 238) # assert_equal(stats['total_size'][1], 191) # assert_equal(stats['total_weight'][1], 768) - assert_equal(stats['total_out'][1], 4999996160) - assert_equal(stats['minfee'][1], 3840) - assert_equal(stats['maxfee'][1], 3840) - assert_equal(stats['medianfee'][1], 3840) - assert_equal(stats['avgfee'][1], 3840) - assert_equal(stats['minfeerate'][1], 20) - assert_equal(stats['maxfeerate'][1], 20) - assert_equal(stats['medianfeerate'][1], 20) - assert_equal(stats['avgfeerate'][1], 20) + assert_equal(stats['total_out'][1], D(49.99996160)) + assert_equal(stats['minfee'][1], D(0.00003840)) + assert_equal(stats['maxfee'][1], D(0.00003840)) + assert_equal(stats['medianfee'][1], D(0.00003840)) + assert_equal(stats['avgfee'][1], D(0.00003840)) + assert_equal(stats['minfeerate'][1], D(0.00000020)) + assert_equal(stats['maxfeerate'][1], D(0.00000020)) + assert_equal(stats['medianfeerate'][1], D(0.00000020)) + assert_equal(stats['avgfeerate'][1], D(0.00000020)) # assert_equal(stats['mintxsize'][1], 192) # assert_equal(stats['maxtxsize'][1], 192) # assert_equal(stats['mediantxsize'][1], 192) @@ -132,20 +137,20 @@ def run_test(self): assert_equal(stats['swtxs'][max_stat_pos], 0) assert_equal(stats['ins'][max_stat_pos], 3) assert_equal(stats['outs'][max_stat_pos], 8) - assert_equal(stats['totalfee'][max_stat_pos], 76160) + assert_equal(stats['totalfee'][max_stat_pos], D(0.00076160)) assert_equal(stats['utxo_increase'][max_stat_pos], 5) assert_equal(stats['utxo_size_inc'][max_stat_pos], 388) # assert_equal(stats['total_size'][max_stat_pos], 643) # assert_equal(stats['total_weight'][max_stat_pos], 2572) - assert_equal(stats['total_out'][max_stat_pos], 9999920000) - assert_equal(stats['minfee'][max_stat_pos], 3840) - assert_equal(stats['maxfee'][max_stat_pos], 67800) - assert_equal(stats['medianfee'][max_stat_pos], 4520) - assert_equal(stats['avgfee'][max_stat_pos], 25386) - assert_equal(stats['minfeerate'][max_stat_pos], 20) + assert_equal(stats['total_out'][max_stat_pos], D(99.99920000)) + assert_equal(stats['minfee'][max_stat_pos], D(0.00003840)) + assert_equal(stats['maxfee'][max_stat_pos], D(0.00067800)) + assert_equal(stats['medianfee'][max_stat_pos], D(0.00004520)) + assert_equal(stats['avgfee'][max_stat_pos], D(0.00025386)) + assert_equal(stats['minfeerate'][max_stat_pos], D(0.00000020)) # assert_equal(stats['maxfeerate'][max_stat_pos], 300) - assert_equal(stats['medianfeerate'][max_stat_pos], 20) - assert_equal(stats['avgfeerate'][max_stat_pos], 118) + assert_equal(stats['medianfeerate'][max_stat_pos], D(0.00000020)) + assert_equal(stats['avgfeerate'][max_stat_pos], D(0.00000118)) # assert_equal(stats['mintxsize'][max_stat_pos], 192) # assert_equal(stats['maxtxsize'][max_stat_pos], 226) # assert_equal(stats['mediantxsize'][max_stat_pos], 225) From 80dccacab0851ce4561b5cc1c9d89133d4a49f60 Mon Sep 17 00:00:00 2001 From: Anthony Towns Date: Thu, 19 Oct 2017 20:16:23 +1000 Subject: [PATCH 6/6] test against python reimplementation --- test/functional/getblockstats.py | 164 ++++++++++++++++++++++++------- 1 file changed, 131 insertions(+), 33 deletions(-) diff --git a/test/functional/getblockstats.py b/test/functional/getblockstats.py index e97989a8d413..875663ad2942 100755 --- a/test/functional/getblockstats.py +++ b/test/functional/getblockstats.py @@ -25,11 +25,121 @@ def assert_contains(data, values, check_cointains=True): assert(val not in data) class GetblockstatsTest(BitcoinTestFramework): + all_values = [ + "height", + "time", + "mediantime", + "txs", + "swtxs", + "ins", + "outs", + "subsidy", + "totalfee", + "utxo_increase", + "utxo_size_inc", + "total_size", + "total_weight", + "swtotal_size", + "swtotal_weight", + "total_out", + "minfee", + "maxfee", + "medianfee", + "avgfee", + "mintxsize", + "maxtxsize", + "mediantxsize", + "avgtxsize", + "minfeerate", + "maxfeerate", + "medianfeerate", + "avgfeerate", + ] + def set_test_params(self): self.num_nodes = 2 self.extra_args = [['-txindex'], ['-paytxfee=0.003']] self.setup_clean_chain = True + def utxo_size(self, txout): + s = len(txout["scriptPubKey"]["hex"])//2 + if s < 253: + s += 1 + elif s < 65536: + s += 3 + elif s < 4294967296: + s += 5 + else: + s += 9 + return 41 + 8 + s + + def expected_stats(self, height): + blk = self.nodes[0].getblock(self.nodes[0].getblockhash(height), 2) + txs = ins = outs = totalfee = total_out = swtxs = 0 + total_size = total_weight = swtotal_size = swtotal_weight = 0 + utxo_size_inc = 0 + _fees = [] + _feerates = [] + _sizes = [] + for txnum,tx in enumerate(blk["tx"]): + txs += 1 + outs += len(tx["vout"]) + for o in tx["vout"]: + utxo_size_inc += self.utxo_size(o) + if txnum == 0: + subsidy = sum(o["value"] for o in tx["vout"]) + else: + ins += len(tx["vin"]) + _spend = sum(o["value"] for o in tx["vout"]) + _receive = 0 + for i in tx["vin"]: + _prevout = self.nodes[0].getrawtransaction(i["txid"],1)["vout"][i["vout"]] + _receive += _prevout["value"] + utxo_size_inc -= self.utxo_size(_prevout) + _size = tx["size"] + _weight = tx["vsize"]*4 + _fee = _receive - _spend + _fees.append(_fee) + _feerates.append(round(_fee/tx["size"],8)) + _sizes.append(tx["vsize"]) + #Useful for debugging averages and feerates + #print(">> %d:%d %r %r %r %r %r" % (height, txnum, _fee, tx["vsize"], tx["size"], round(_fee/tx["size"], 8), round(_fee/tx["vsize"],8))) + if any("txinwitness" in i for i in tx["vin"]): + swtxs += 1 + swtotal_size += _size + swtotal_weight += _weight + totalfee += _fee + total_out += _spend + total_size += _size + total_weight += _weight + + subsidy -= totalfee + utxo_increase = outs - ins + time = blk["time"] + mediantime = blk["mediantime"] + res = {} + for a in locals().copy(): + if a in self.all_values: res[a] = locals()[a] + stats = {"fee": (_fees, 8), "feerate": (_feerates, 8), "txsize": (_sizes,0)} + for _k,(_v,_prec) in stats.items(): + if len(_v) == 0: + res["min"+_k] = res["max"+_k] = res["median"+_k] = res["avg"+_k] = 0 + else: + _v.sort() + res["min"+_k] = _v[0] + res["max"+_k] = _v[-1] + if len(_v) % 2 == 0: + _m = round((_v[len(_v)//2-1] + _v[len(_v)//2])/2,_prec) + else: + _m = _v[(len(_v)-1)//2] + res["median"+_k] = _m + if _k == "feerate": + # weighted average + res["avgfeerate"] = round(totalfee*1000/total_weight*4/1000, 8) + else: + res["avg"+_k] = round(sum(_v)/D(len(_v)), _prec) + return res + def run_test(self): node = self.nodes[0] node.generate(101) @@ -46,41 +156,29 @@ def run_test(self): start_height = 101 max_stat_pos = 2 + stats = node.getblockstats(start=start_height, end=start_height + max_stat_pos) - all_values = [ - "height", - "time", - "mediantime", - "txs", - "swtxs", - "ins", - "outs", - "subsidy", - "totalfee", - "utxo_increase", - "utxo_size_inc", - "total_size", - "total_weight", - "swtotal_size", - "swtotal_weight", - "total_out", - "minfee", - "maxfee", - "medianfee", - "avgfee", - "minfeerate", - "maxfeerate", - "medianfeerate", - "avgfeerate", - "mintxsize", - "maxtxsize", - "mediantxsize", - "avgtxsize", - ] - assert_contains(stats, all_values) + unchecked = set(self.all_values) + all_zero = set(self.all_values) + for h in range(0, max_stat_pos+1): + expected = self.expected_stats(start_height+h) + for a in expected: + unchecked.discard(a) + if expected[a] != 0: + all_zero.discard(a) + missing = [a for a in self.all_values if a not in expected] + if missing: + print("Unchecked values: %s" % (",".join(missing))) + for a in expected: + assert stats[a][h] == expected[a], "stats for %s at height %d(%d+%d) unexpected (got: %r expected: %r)" % (a, start_height+h, start_height, h, stats[a][h], expected[a]) + assert not unchecked, "Following stats unchecked: %s" % (",".join(sorted(unchecked))) + if all_zero: + print("Only ever expected 0: %s" % (",".join(sorted(all_zero)))) + + assert_contains(stats, self.all_values) # Make sure all valid statistics are included - assert_contains(all_values, stats.keys()) + assert_contains(self.all_values, stats.keys()) assert_equal(stats['height'][0], start_height) assert_equal(stats['height'][max_stat_pos], start_height + max_stat_pos) @@ -185,7 +283,7 @@ def run_test(self): ] assert_contains(stats, some_values) # Make sure valid stats that haven't been selected don't appear - other_values = [x for x in all_values if x not in some_values] + other_values = [x for x in self.all_values if x not in some_values] assert_contains(stats, other_values, False) if __name__ == '__main__':