From 857a2b5f79d2a184f73be56ab0b0083b675dd1e6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jorge=20Tim=C3=B3n?= <jtimon@jtimon.cc>
Date: Sat, 8 Jul 2017 02:15:36 +0200
Subject: [PATCH 1/6] Avoid special case for truncated zeros with new
 CFeeRate::GetTruncatedFee

---
 src/policy/feerate.cpp | 14 ++++++++------
 src/policy/feerate.h   |  7 ++++++-
 2 files changed, 14 insertions(+), 7 deletions(-)

diff --git a/src/policy/feerate.cpp b/src/policy/feerate.cpp
index a089c02284f0..c8b4f54e194d 100644
--- a/src/policy/feerate.cpp
+++ b/src/policy/feerate.cpp
@@ -20,20 +20,22 @@ CFeeRate::CFeeRate(const CAmount& nFeePaid, size_t nBytes_)
         nSatoshisPerK = 0;
 }
 
-CAmount CFeeRate::GetFee(size_t nBytes_) const
+CAmount CFeeRate::GetTruncatedFee(size_t bytes) const
 {
-    assert(nBytes_ <= uint64_t(std::numeric_limits<int64_t>::max()));
-    int64_t nSize = int64_t(nBytes_);
+    assert(bytes <= uint64_t(std::numeric_limits<int64_t>::max()));
+    return nSatoshisPerK * int64_t(bytes) / 1000;
+}
 
-    CAmount nFee = nSatoshisPerK * nSize / 1000;
+CAmount CFeeRate::GetFee(size_t bytes) const
+{
+    CAmount nFee = GetTruncatedFee(bytes);
 
-    if (nFee == 0 && nSize != 0) {
+    if (nFee == 0 && bytes != 0) {
         if (nSatoshisPerK > 0)
             nFee = CAmount(1);
         if (nSatoshisPerK < 0)
             nFee = CAmount(-1);
     }
-
     return nFee;
 }
 
diff --git a/src/policy/feerate.h b/src/policy/feerate.h
index 3449cdd6990d..80b089d87821 100644
--- a/src/policy/feerate.h
+++ b/src/policy/feerate.h
@@ -34,7 +34,12 @@ class CFeeRate
     /**
      * Return the fee in satoshis for the given size in bytes.
      */
-    CAmount GetFee(size_t nBytes) const;
+    CAmount GetTruncatedFee(size_t bytes) const;
+    /**
+     * Return the fee in satoshis for the given size in bytes. If the
+     * result is zero, return 1 if the fee was positive or -1 if was negative.
+     */
+    CAmount GetFee(size_t bytes) const;
     /**
      * Return the fee in satoshis for a size of 1000 bytes
      */

From bd371df4c862e78abcd5f29f62d0bb4b2d8205fa Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jorge=20Tim=C3=B3n?= <jtimon@jtimon.cc>
Date: Sun, 4 Jun 2017 17:14:00 +0200
Subject: [PATCH 2/6] RPC: Separate ReadBlockCheckPruned() from getblock()

---
 src/rpc/blockchain.cpp | 28 +++++++++++++++++-----------
 1 file changed, 17 insertions(+), 11 deletions(-)

diff --git a/src/rpc/blockchain.cpp b/src/rpc/blockchain.cpp
index 68af376f355b..bbad2e49e887 100644
--- a/src/rpc/blockchain.cpp
+++ b/src/rpc/blockchain.cpp
@@ -688,6 +688,22 @@ UniValue getblockheader(const JSONRPCRequest& request)
     return blockheaderToJSON(pblockindex);
 }
 
+static void ReadBlockCheckPruned(CBlock& block, const CBlockIndex* pblockindex)
+{
+    if (fHavePruned && !(pblockindex->nStatus & BLOCK_HAVE_DATA) && pblockindex->nTx > 0) {
+        throw JSONRPCError(RPC_MISC_ERROR, "Block not available (pruned data)");
+    }
+
+    if (!ReadBlockFromDisk(block, pblockindex, Params().GetConsensus())) {
+        // Block not found on disk. This could be because we have the block
+        // header in our index but don't have the block (for example if a
+        // non-whitelisted node sends us an unrequested long chain of valid
+        // blocks, we add the headers to our index, but don't accept the
+        // block).
+        throw JSONRPCError(RPC_MISC_ERROR, "Block not found on disk");
+    }
+}
+
 UniValue getblock(const JSONRPCRequest& request)
 {
     if (request.fHelp || request.params.size() < 1 || request.params.size() > 2)
@@ -756,17 +772,7 @@ UniValue getblock(const JSONRPCRequest& request)
 
     CBlock block;
     CBlockIndex* pblockindex = mapBlockIndex[hash];
-
-    if (fHavePruned && !(pblockindex->nStatus & BLOCK_HAVE_DATA) && pblockindex->nTx > 0)
-        throw JSONRPCError(RPC_MISC_ERROR, "Block not available (pruned data)");
-
-    if (!ReadBlockFromDisk(block, pblockindex, Params().GetConsensus()))
-        // Block not found on disk. This could be because we have the block
-        // header in our index but don't have the block (for example if a
-        // non-whitelisted node sends us an unrequested long chain of valid
-        // blocks, we add the headers to our index, but don't accept the
-        // block).
-        throw JSONRPCError(RPC_MISC_ERROR, "Block not found on disk");
+    ReadBlockCheckPruned(block, pblockindex);
 
     if (verbosity <= 0)
     {

From 2497afa41126fd4e5cf17b095980c19fc1b813a5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jorge=20Tim=C3=B3n?= <jtimon@jtimon.cc>
Date: Sun, 4 Jun 2017 00:25:55 +0200
Subject: [PATCH 3/6] RPC: Introduce getblockstats

---
 src/rpc/blockchain.cpp | 307 ++++++++++++++++++++++++++++++++++++++++-
 src/rpc/client.cpp     |   2 +
 2 files changed, 308 insertions(+), 1 deletion(-)

diff --git a/src/rpc/blockchain.cpp b/src/rpc/blockchain.cpp
index bbad2e49e887..e90e921452eb 100644
--- a/src/rpc/blockchain.cpp
+++ b/src/rpc/blockchain.cpp
@@ -30,6 +30,7 @@
 
 #include <univalue.h>
 
+#include <boost/algorithm/string.hpp>
 #include <boost/thread/thread.hpp> // boost::thread::interrupt
 
 #include <mutex>
@@ -1538,7 +1539,7 @@ UniValue getchaintxstats(const JSONRPCRequest& request)
             pindex = chainActive.Tip();
         }
     }
-    
+
     assert(pindex != nullptr);
 
     if (request.params[0].isNull()) {
@@ -1570,6 +1571,309 @@ UniValue getchaintxstats(const JSONRPCRequest& request)
     return ret;
 }
 
+static void RpcGetTx(const uint256& hash, CTransactionRef& tx_out)
+{
+    uint256 hashBlock;
+    if (!GetTransaction(hash, tx_out, Params().GetConsensus(), hashBlock, true)) {
+        throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, std::string(fTxIndex ? "No such mempool or blockchain transaction"
+          : "No such mempool transaction. Use -txindex to enable blockchain transaction queries"));
+    }
+}
+
+template<typename T>
+static T CalculateTruncatedMedian(std::vector<T>& scores)
+{
+    size_t size = scores.size();
+    if (size == 0) {
+        return 0;
+    } if (size == 1) {
+        return scores[0];
+    }
+
+    std::sort(scores.begin(), scores.end());
+    if (size % 2 == 0) {
+        return (scores[size / 2 - 1] + scores[size / 2]) / 2;
+    } else {
+        return scores[size / 2];
+    }
+}
+
+// outpoint (needed for the utxo index) + nHeight + fCoinBase
+static const size_t PER_UTXO_OVERHEAD = sizeof(COutPoint) + sizeof(uint32_t) + sizeof(bool);
+
+static void UpdateBlockStats(const CBlockIndex* pindex, std::set<std::string>& stats, std::map<std::string, UniValue>& map_stats)
+{
+    int64_t inputs = 0;
+    int64_t outputs = 0;
+    int64_t swtxs = 0;
+    int64_t total_size = 0;
+    int64_t total_weight = 0;
+    int64_t swtotal_size = 0;
+    int64_t swtotal_weight = 0;
+    int64_t utxo_size_inc = 0;
+    CAmount total_out = 0;
+    CAmount totalfee = 0;
+    CAmount minfee = MAX_MONEY;
+    CAmount maxfee = 0;
+    CAmount minfeerate = MAX_MONEY;
+    CAmount maxfeerate = 0;
+    std::vector<CAmount> fee_array;
+    std::vector<CAmount> feerate_array;
+    int64_t mintxsize = MAX_BLOCK_SERIALIZED_SIZE;
+    int64_t maxtxsize = 0;
+    std::vector<int64_t> txsize_array;
+
+    CBlock block;
+    ReadBlockCheckPruned(block, pindex);
+
+    for (const auto& tx : block.vtx) {
+        outputs += tx->vout.size();
+        CAmount tx_total_out = 0;
+        for (const CTxOut& out : tx->vout) {
+            utxo_size_inc += GetSerializeSize(out, SER_NETWORK, PROTOCOL_VERSION) + PER_UTXO_OVERHEAD;
+            tx_total_out += out.nValue;
+        }
+
+        if (tx->IsCoinBase()) {
+            continue;
+        }
+        total_out += tx_total_out;
+        inputs += tx->vin.size(); // Don't count coinbase's fake input
+        int64_t tx_size = tx->GetTotalSize();
+        txsize_array.push_back(tx_size);
+        total_size += tx_size;
+        mintxsize = std::min(mintxsize, tx_size);
+        maxtxsize = std::max(maxtxsize, tx_size);
+        int64_t weight = GetTransactionWeight(*tx);
+        total_weight += weight;
+
+        if (tx->HasWitness()) {
+            ++swtxs;
+            swtotal_size += tx_size;
+            swtotal_weight += weight;
+        }
+
+        CAmount tx_total_in = 0;
+        for (const CTxIn& in : tx->vin) {
+            CTransactionRef tx_in;
+            RpcGetTx(in.prevout.hash, tx_in);
+            CTxOut prevoutput = tx_in->vout[in.prevout.n];
+
+            tx_total_in += prevoutput.nValue;
+            utxo_size_inc -= GetSerializeSize(prevoutput, SER_NETWORK, PROTOCOL_VERSION) + PER_UTXO_OVERHEAD;
+        }
+        CAmount txfee = tx_total_in - tx_total_out;
+        assert(MoneyRange(txfee));
+        fee_array.push_back(txfee);
+        totalfee += txfee;
+        minfee = std::min(minfee, txfee);
+        maxfee = std::max(maxfee, txfee);
+
+        // New feerate uses satoshis per virtual byte instead of per serialized byte
+        CAmount feerate = CFeeRate(txfee, weight).GetTruncatedFee(WITNESS_SCALE_FACTOR);
+        feerate_array.push_back(feerate);
+
+        minfeerate = std::min(minfeerate, feerate);
+        maxfeerate = std::max(maxfeerate, feerate);
+    }
+
+    for (const std::string& stat : stats) {
+        // Update map_stats
+        if (stat == "height") {
+            map_stats[stat].push_back((int64_t)pindex->nHeight);
+        } else if (stat == "time") {
+            map_stats[stat].push_back(pindex->GetBlockTime());
+        } else if (stat == "mediantime") {
+            map_stats[stat].push_back(pindex->GetMedianTimePast());
+        } else if (stat == "subsidy") {
+            map_stats[stat].push_back(GetBlockSubsidy(pindex->nHeight, Params().GetConsensus()));
+        } else if (stat == "totalfee") {
+            map_stats[stat].push_back(totalfee);
+        } else if (stat == "txs") {
+            map_stats[stat].push_back((int64_t)block.vtx.size());
+        } else if (stat == "swtxs") {
+            map_stats[stat].push_back(swtxs);
+        } else if (stat == "ins") {
+            map_stats[stat].push_back(inputs);
+        } else if (stat == "outs") {
+            map_stats[stat].push_back(outputs);
+        } else if (stat == "utxo_increase") {
+            map_stats[stat].push_back(outputs - inputs);
+        } else if (stat == "utxo_size_inc") {
+            map_stats[stat].push_back(utxo_size_inc);
+        } else if (stat == "total_size") {
+            map_stats[stat].push_back(total_size);
+        } else if (stat == "total_weight") {
+            map_stats[stat].push_back(total_weight);
+        } else if (stat == "swtotal_size") {
+            map_stats[stat].push_back(swtotal_size);
+        } else if (stat == "swtotal_weight") {
+            map_stats[stat].push_back(swtotal_weight);
+        } else if (stat == "total_out") {
+            map_stats[stat].push_back(total_out);
+        } else if (stat == "minfee") {
+            map_stats[stat].push_back((minfee == MAX_MONEY) ? 0 : minfee);
+        } else if (stat == "maxfee") {
+            map_stats[stat].push_back(maxfee);
+        } else if (stat == "medianfee") {
+            map_stats[stat].push_back(CalculateTruncatedMedian(fee_array));
+        } else if (stat == "avgfee") {
+            map_stats[stat].push_back((block.vtx.size() > 1) ? totalfee / (block.vtx.size() - 1) : 0);
+        } else if (stat == "minfeerate") {
+            map_stats[stat].push_back((minfeerate == MAX_MONEY) ? 0 : minfeerate);
+        } else if (stat == "maxfeerate") {
+            map_stats[stat].push_back(maxfeerate);
+        } else if (stat == "medianfeerate") {
+            map_stats[stat].push_back(CalculateTruncatedMedian(feerate_array));
+        } else if (stat == "avgfeerate") {
+            map_stats[stat].push_back(CFeeRate(totalfee, total_weight).GetTruncatedFee(WITNESS_SCALE_FACTOR));
+        } else if (stat == "mintxsize") {
+            map_stats[stat].push_back(mintxsize == MAX_BLOCK_SERIALIZED_SIZE ? 0 : mintxsize);
+        } else if (stat == "maxtxsize") {
+            map_stats[stat].push_back(maxtxsize);
+        } else if (stat == "mediantxsize") {
+            map_stats[stat].push_back(CalculateTruncatedMedian(txsize_array));
+        } else if (stat == "avgtxsize") {
+            map_stats[stat].push_back((block.vtx.size() > 1) ? total_size / (block.vtx.size() - 1) : 0);
+        }
+    }
+}
+
+UniValue getblockstats(const JSONRPCRequest& request)
+{
+    std::set<std::string> valid_stats = {
+        "height",
+        "time",
+        "mediantime",
+        "txs",
+        "swtxs",
+        "ins",
+        "outs",
+        "subsidy",
+        "totalfee",
+        "utxo_increase",
+        "utxo_size_inc",
+        "total_size",
+        "total_weight",
+        "swtotal_size",
+        "swtotal_weight",
+        "total_out",
+        "minfee",
+        "maxfee",
+        "medianfee",
+        "avgfee",
+        "minfeerate",
+        "maxfeerate",
+        "medianfeerate",
+        "avgfeerate",
+        "mintxsize",
+        "maxtxsize",
+        "mediantxsize",
+        "avgtxsize",
+    };
+
+    if (request.fHelp || request.params.size() < 1 || request.params.size() > 4)
+        throw std::runtime_error(
+            "getblockstats ( nStart nEnd stats )\n"
+            "\nCompute per block statistics for a given window. All amounts are in satoshis.\n"
+            "\nNegative values for start or end count back from the current tip.\n"
+            "\nIt won't work in some cases with pruning or without -txindex.\n"
+            "\nArguments:\n"
+            "1. \"start\"      (numeric, required) The height of the block that starts the window.\n"
+            "2. \"end\"        (numeric, optional) The height of the block that ends the window (default: current tip).\n"
+            "3. \"stats\"      (string,  optional) Values to plot (comma separated), default(all): " + boost::join(valid_stats, ",") +
+            "\nResult: (all values are in reverse order height-wise)\n"
+            "{                           (json object)\n"
+            "  \"height\": [],           (array) The height of the blocks, ie: [end, end-1, ..., start+1, start].\n"
+            "  \"time\": [],             (array) The block time.\n"
+            "  \"mediantime\": [],       (array) The block median time past.\n"
+            "  \"txs\": [],              (array) The number of transactions (excluding coinbase).\n"
+            "  \"swtxs\": [],            (array) The number of segwit transactions.\n"
+            "  \"ins\": [],              (array) The number of inputs (excluding coinbase).\n"
+            "  \"outs\": [],             (array) The number of outputs (including coinbase).\n"
+            "  \"subsidy\": [],          (array) The block subsidy.\n"
+            "  \"totalfee\": [],         (array) The fee total.\n"
+            "  \"utxo_increase\": [],    (array) The increase/decrease in the number of unspent outputs.\n"
+            "  \"utxo_size_inc\": [],    (array) The increase/decrease in size for the utxo index (not discounting op_return and similar).\n"
+            "  \"total_size\": [],       (array) Total size of all non-coinbase transactions.\n"
+            "  \"total_weight\": [],     (array) Total weight of all non-coinbase transactions divided by segwit scale factor (4).\n"
+            "  \"swtotal_size\": [],     (array) Total size of all segwit transactions.\n"
+            "  \"swtotal_weight\": [],   (array) Total weight of all segwit transactions divided by segwit scale factor (4).\n"
+            "  \"total_out\": [],        (array) Total amount in all outputs (excluding coinbase and thus reward [ie subsidy + totalfee]).\n"
+            "  \"minfee\": [],           (array) Minimum fee in the block.\n"
+            "  \"maxfee\": [],           (array) Maximum fee in the block.\n"
+            "  \"medianfee\": [],        (array) Truncated median fee in the block.\n"
+            "  \"avgfee\": [],           (array) Average fee in the block.\n"
+            "  \"minfeerate\": [],       (array) Minimum feerate (in satoshis per virtual byte).\n"
+            "  \"maxfeerate\": [],       (array) Maximum feerate (in satoshis per virtual byte).\n"
+            "  \"medianfeerate\": [],    (array) Truncated median feerate (in satoshis per virtual byte).\n"
+            "  \"avgfeerate\": [],       (array) Average feerate (in satoshis per virtual byte).\n"
+            "  \"mintxsize\": [],        (array) Minimum transaction size.\n"
+            "  \"maxtxsize\": [],        (array) Maximum transaction size.\n"
+            "  \"mediantxsize\": [],     (array) Truncated median transaction size.\n"
+            "  \"avgtxsize\": [],        (array) Average transaction size.\n"
+            "}\n"
+            "\nExamples:\n"
+            + HelpExampleCli("getblockstats", "1000 1000 \"minfeerate,avgfeerate\"")
+            + HelpExampleRpc("getblockstats", "1000 1000 \"maxfeerate,avgfeerate\"")
+        );
+
+    LOCK(cs_main);
+
+    int start = request.params[0].get_int();
+    int current_tip = chainActive.Height();
+    if (start < 0) {
+        start = current_tip + start;
+    }
+    if (start < 0 || start > current_tip) {
+        throw JSONRPCError(RPC_INVALID_PARAMETER, strprintf("Start block height %d after current tip %d", start, current_tip));
+    }
+
+    int end;
+    if (request.params.size() > 1) {
+        end = request.params[1].get_int();
+        if (end < 0) {
+            end = current_tip + end;
+        }
+    } else {
+        end = current_tip;
+    }
+    if (end < 0 || end > current_tip) {
+        throw JSONRPCError(RPC_INVALID_PARAMETER, strprintf("End block height %d after current tip %d", end, current_tip));
+    }
+    if (start > end) {
+        throw JSONRPCError(RPC_INVALID_PARAMETER, strprintf("Start block height %d higher than end %d", start, end));
+    }
+
+    std::set<std::string> stats;
+    if (request.params.size() > 2) {
+        boost::split(stats, request.params[2].get_str(), boost::is_any_of(","));
+
+        for (const std::string& stat : stats) {
+            if (valid_stats.count(stat) == 0) {
+                throw JSONRPCError(RPC_INVALID_PARAMETER, strprintf("Invalid selected statistic %s", stat));
+            }
+        }
+    } else {
+        stats = valid_stats;
+    }
+
+    std::map<std::string, UniValue> map_stats;
+    for (const std::string& stat : stats) {
+        map_stats[stat] = UniValue(UniValue::VARR);
+    }
+
+    for (int i = start; i <= end; ++i) {
+        UpdateBlockStats(chainActive[i], stats, map_stats);
+    }
+
+    UniValue ret(UniValue::VOBJ);
+    for (const std::string stat : stats) {
+        ret.push_back(Pair(stat, map_stats[stat]));
+    }
+    return ret;
+}
+
 UniValue savemempool(const JSONRPCRequest& request)
 {
     if (request.fHelp || request.params.size() != 0) {
@@ -1594,6 +1898,7 @@ static const CRPCCommand commands[] =
   //  --------------------- ------------------------  -----------------------  ----------
     { "blockchain",         "getblockchaininfo",      &getblockchaininfo,      {} },
     { "blockchain",         "getchaintxstats",        &getchaintxstats,        {"nblocks", "blockhash"} },
+    { "blockchain",         "getblockstats",          &getblockstats,          {"start", "end", "stats"} },
     { "blockchain",         "getbestblockhash",       &getbestblockhash,       {} },
     { "blockchain",         "getblockcount",          &getblockcount,          {} },
     { "blockchain",         "getblock",               &getblock,               {"blockhash","verbosity|verbose"} },
diff --git a/src/rpc/client.cpp b/src/rpc/client.cpp
index 721f363aeff7..0b335fe731f3 100644
--- a/src/rpc/client.cpp
+++ b/src/rpc/client.cpp
@@ -111,6 +111,8 @@ static const CRPCConvertParam vRPCConvertParams[] =
     { "importmulti", 1, "options" },
     { "verifychain", 0, "checklevel" },
     { "verifychain", 1, "nblocks" },
+    { "getblockstats", 0, "start" },
+    { "getblockstats", 1, "end" },
     { "pruneblockchain", 0, "height" },
     { "keypoolrefill", 0, "newsize" },
     { "getrawmempool", 0, "verbose" },

From caeaf86de426f9b71c0828be23253827dd0149fb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jorge=20Tim=C3=B3n?= <jtimon@jtimon.cc>
Date: Wed, 21 Jun 2017 03:08:05 +0200
Subject: [PATCH 4/6] QA: Test new getblockstats RPC

---
 test/functional/getblockstats.py | 187 +++++++++++++++++++++++++++++++
 test/functional/test_runner.py   |   1 +
 2 files changed, 188 insertions(+)
 create mode 100755 test/functional/getblockstats.py

diff --git a/test/functional/getblockstats.py b/test/functional/getblockstats.py
new file mode 100755
index 000000000000..00fddf3a90de
--- /dev/null
+++ b/test/functional/getblockstats.py
@@ -0,0 +1,187 @@
+#!/usr/bin/env python3
+# Copyright (c) 2017-2017 The Bitcoin Core developers
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+#
+# Test getblockstats rpc call
+#
+from test_framework.test_framework import BitcoinTestFramework
+from test_framework.util import (
+    assert_equal,
+    assert_raises_rpc_error,
+)
+
+def assert_contains(data, values, check_cointains=True):
+    for val in values:
+        if (check_cointains):
+            assert(val in data)
+        else:
+            assert(val not in data)
+
+class GetblockstatsTest(BitcoinTestFramework):
+    def set_test_params(self):
+        self.num_nodes = 2
+        self.extra_args = [['-txindex'], ['-paytxfee=0.003']]
+        self.setup_clean_chain = True
+
+    def run_test(self):
+        node = self.nodes[0]
+        node.generate(101)
+
+        node.sendtoaddress(address=self.nodes[1].getnewaddress(), amount=10, subtractfeefromamount=True)
+        node.generate(1)
+        self.sync_all()
+
+        node.sendtoaddress(address=node.getnewaddress(), amount=10, subtractfeefromamount=True)
+        node.sendtoaddress(address=node.getnewaddress(), amount=10, subtractfeefromamount=True)
+        self.nodes[1].sendtoaddress(address=node.getnewaddress(), amount=1, subtractfeefromamount=True)
+        self.sync_all()
+        node.generate(1)
+
+        start_height = 101
+        max_stat_pos = 2
+        stats = node.getblockstats(start=start_height, end=start_height + max_stat_pos)
+
+        all_values = [
+            "height",
+            "time",
+            "mediantime",
+            "txs",
+            "swtxs",
+            "ins",
+            "outs",
+            "subsidy",
+            "totalfee",
+            "utxo_increase",
+            "utxo_size_inc",
+            "total_size",
+            "total_weight",
+            "swtotal_size",
+            "swtotal_weight",
+            "total_out",
+            "minfee",
+            "maxfee",
+            "medianfee",
+            "avgfee",
+            "minfeerate",
+            "maxfeerate",
+            "medianfeerate",
+            "avgfeerate",
+            "mintxsize",
+            "maxtxsize",
+            "mediantxsize",
+            "avgtxsize",
+        ]
+        assert_contains(stats, all_values)
+        # Make sure all valid statistics are included
+        assert_contains(all_values, stats.keys())
+
+        assert_equal(stats['height'][0], start_height)
+        assert_equal(stats['height'][max_stat_pos], start_height + max_stat_pos)
+
+        assert_equal(stats['txs'][0], 1)
+        assert_equal(stats['swtxs'][0], 0)
+        assert_equal(stats['ins'][0], 0)
+        assert_equal(stats['outs'][0], 2)
+        assert_equal(stats['totalfee'][0], 0)
+        assert_equal(stats['utxo_increase'][0], 2)
+        assert_equal(stats['utxo_size_inc'][0], 173)
+        assert_equal(stats['total_size'][0], 0)
+        assert_equal(stats['total_weight'][0], 0)
+        assert_equal(stats['swtotal_size'][0], 0)
+        assert_equal(stats['swtotal_weight'][0], 0)
+        assert_equal(stats['total_out'][0], 0)
+        assert_equal(stats['minfee'][0], 0)
+        assert_equal(stats['maxfee'][0], 0)
+        assert_equal(stats['medianfee'][0], 0)
+        assert_equal(stats['avgfee'][0], 0)
+        assert_equal(stats['minfeerate'][0], 0)
+        assert_equal(stats['maxfeerate'][0], 0)
+        assert_equal(stats['medianfeerate'][0], 0)
+        assert_equal(stats['avgfeerate'][0], 0)
+        assert_equal(stats['mintxsize'][0], 0)
+        assert_equal(stats['maxtxsize'][0], 0)
+        assert_equal(stats['mediantxsize'][0], 0)
+        assert_equal(stats['avgtxsize'][0], 0)
+
+        assert_equal(stats['txs'][1], 2)
+        assert_equal(stats['swtxs'][1], 0)
+        assert_equal(stats['ins'][1], 1)
+        assert_equal(stats['outs'][1], 4)
+        assert_equal(stats['totalfee'][1], 3840)
+        assert_equal(stats['utxo_increase'][1], 3)
+        assert_equal(stats['utxo_size_inc'][1], 238)
+        # assert_equal(stats['total_size'][1], 191)
+        # assert_equal(stats['total_weight'][1], 768)
+        assert_equal(stats['total_out'][1], 4999996160)
+        assert_equal(stats['minfee'][1], 3840)
+        assert_equal(stats['maxfee'][1], 3840)
+        assert_equal(stats['medianfee'][1], 3840)
+        assert_equal(stats['avgfee'][1], 3840)
+        assert_equal(stats['minfeerate'][1], 20)
+        assert_equal(stats['maxfeerate'][1], 20)
+        assert_equal(stats['medianfeerate'][1], 20)
+        assert_equal(stats['avgfeerate'][1], 20)
+        # assert_equal(stats['mintxsize'][1], 192)
+        # assert_equal(stats['maxtxsize'][1], 192)
+        # assert_equal(stats['mediantxsize'][1], 192)
+        # assert_equal(stats['avgtxsize'][1], 192)
+
+        assert_equal(stats['txs'][max_stat_pos], 4)
+        assert_equal(stats['swtxs'][max_stat_pos], 0)
+        assert_equal(stats['ins'][max_stat_pos], 3)
+        assert_equal(stats['outs'][max_stat_pos], 8)
+        assert_equal(stats['totalfee'][max_stat_pos], 76160)
+        assert_equal(stats['utxo_increase'][max_stat_pos], 5)
+        assert_equal(stats['utxo_size_inc'][max_stat_pos], 388)
+        # assert_equal(stats['total_size'][max_stat_pos], 643)
+        # assert_equal(stats['total_weight'][max_stat_pos], 2572)
+        assert_equal(stats['total_out'][max_stat_pos], 9999920000)
+        assert_equal(stats['minfee'][max_stat_pos], 3840)
+        assert_equal(stats['maxfee'][max_stat_pos], 67800)
+        assert_equal(stats['medianfee'][max_stat_pos], 4520)
+        assert_equal(stats['avgfee'][max_stat_pos], 25386)
+        assert_equal(stats['minfeerate'][max_stat_pos], 20)
+        # assert_equal(stats['maxfeerate'][max_stat_pos], 300)
+        assert_equal(stats['medianfeerate'][max_stat_pos], 20)
+        assert_equal(stats['avgfeerate'][max_stat_pos], 118)
+        # assert_equal(stats['mintxsize'][max_stat_pos], 192)
+        # assert_equal(stats['maxtxsize'][max_stat_pos], 226)
+        # assert_equal(stats['mediantxsize'][max_stat_pos], 225)
+        # assert_equal(stats['avgtxsize'][max_stat_pos], 214)
+
+        # Test invalid parameters raise the proper json exceptions
+        tip = start_height + max_stat_pos
+        assert_raises_rpc_error(-8, 'Start block height %d after current tip %d' % (tip+1, tip), node.getblockstats, start=tip+1)
+        assert_raises_rpc_error(-8, 'Start block height %d after current tip %d' % (-1, tip), node.getblockstats, start=-tip-1)
+        assert_raises_rpc_error(-8, 'Start block height %d higher than end %d' % (tip-1, tip-2), node.getblockstats, start=-1, end=-2)
+        assert_raises_rpc_error(-8, 'End block height %d after current tip %d' % (tip+1, tip), node.getblockstats, start=1, end=tip+1)
+        assert_raises_rpc_error(-8, 'Start block height 2 higher than end 1', node.getblockstats, start=2, end=1)
+        assert_raises_rpc_error(-8, 'Start block height %d higher than end %d' % (tip, tip-1), node.getblockstats, start=tip, end=tip-1)
+
+        # Make sure not valid stats aren't allowed
+        inv_sel_stat = 'asdfghjkl'
+        inv_stats = [
+            'minfee,%s' % inv_sel_stat,
+            '%s,minfee' % inv_sel_stat,
+            'minfee,%s,maxfee' % inv_sel_stat,
+        ]
+        for inv_stat in inv_stats:
+            assert_raises_rpc_error(-8, 'Invalid selected statistic %s' % inv_sel_stat, node.getblockstats, start=1, end=2, stats=inv_stat)
+        # Make sure we aren't always returning inv_sel_stat as the culprit stat
+        assert_raises_rpc_error(-8, 'Invalid selected statistic aaa%s' % inv_sel_stat, node.getblockstats, start=1, end=2, stats='minfee,aaa%s' % inv_sel_stat)
+
+        # Make sure only the selected statistics are included
+        stats = node.getblockstats(start=1, end=2, stats='minfee,maxfee')
+        some_values = [
+            'minfee',
+            'maxfee',
+        ]
+        assert_contains(stats, some_values)
+        # Make sure valid stats that haven't been selected don't appear
+        other_values = [x for x in all_values if x not in some_values]
+        assert_contains(stats, other_values, False)
+
+if __name__ == '__main__':
+    GetblockstatsTest().main()
diff --git a/test/functional/test_runner.py b/test/functional/test_runner.py
index 8c4651f6e02e..5e11167daff1 100755
--- a/test/functional/test_runner.py
+++ b/test/functional/test_runner.py
@@ -122,6 +122,7 @@
     'bip65-cltv-p2p.py',
     'uptime.py',
     'resendwallettransactions.py',
+    'getblockstats.py',
     'minchainwork.py',
     'p2p-fingerprint.py',
 ]

From c203b30401ee9178b9021c280815ecfe85349812 Mon Sep 17 00:00:00 2001
From: Anthony Towns <aj@erisian.com.au>
Date: Thu, 19 Oct 2017 20:16:02 +1000
Subject: [PATCH 5/6] switch to decimal bitcoins rather than integer satoshis

---
 src/rpc/blockchain.cpp           | 22 ++++++++--------
 test/functional/getblockstats.py | 43 ++++++++++++++++++--------------
 2 files changed, 35 insertions(+), 30 deletions(-)

diff --git a/src/rpc/blockchain.cpp b/src/rpc/blockchain.cpp
index e90e921452eb..f236476559a5 100644
--- a/src/rpc/blockchain.cpp
+++ b/src/rpc/blockchain.cpp
@@ -1686,9 +1686,9 @@ static void UpdateBlockStats(const CBlockIndex* pindex, std::set<std::string>& s
         } else if (stat == "mediantime") {
             map_stats[stat].push_back(pindex->GetMedianTimePast());
         } else if (stat == "subsidy") {
-            map_stats[stat].push_back(GetBlockSubsidy(pindex->nHeight, Params().GetConsensus()));
+            map_stats[stat].push_back(ValueFromAmount(GetBlockSubsidy(pindex->nHeight, Params().GetConsensus())));
         } else if (stat == "totalfee") {
-            map_stats[stat].push_back(totalfee);
+            map_stats[stat].push_back(ValueFromAmount(totalfee));
         } else if (stat == "txs") {
             map_stats[stat].push_back((int64_t)block.vtx.size());
         } else if (stat == "swtxs") {
@@ -1710,23 +1710,23 @@ static void UpdateBlockStats(const CBlockIndex* pindex, std::set<std::string>& s
         } else if (stat == "swtotal_weight") {
             map_stats[stat].push_back(swtotal_weight);
         } else if (stat == "total_out") {
-            map_stats[stat].push_back(total_out);
+            map_stats[stat].push_back(ValueFromAmount(total_out));
         } else if (stat == "minfee") {
-            map_stats[stat].push_back((minfee == MAX_MONEY) ? 0 : minfee);
+            map_stats[stat].push_back(ValueFromAmount((minfee == MAX_MONEY) ? 0 : minfee));
         } else if (stat == "maxfee") {
-            map_stats[stat].push_back(maxfee);
+            map_stats[stat].push_back(ValueFromAmount(maxfee));
         } else if (stat == "medianfee") {
-            map_stats[stat].push_back(CalculateTruncatedMedian(fee_array));
+            map_stats[stat].push_back(ValueFromAmount(CalculateTruncatedMedian(fee_array)));
         } else if (stat == "avgfee") {
-            map_stats[stat].push_back((block.vtx.size() > 1) ? totalfee / (block.vtx.size() - 1) : 0);
+            map_stats[stat].push_back(ValueFromAmount((block.vtx.size() > 1) ? totalfee / (block.vtx.size() - 1) : 0));
         } else if (stat == "minfeerate") {
-            map_stats[stat].push_back((minfeerate == MAX_MONEY) ? 0 : minfeerate);
+            map_stats[stat].push_back(ValueFromAmount((minfeerate == MAX_MONEY) ? 0 : minfeerate));
         } else if (stat == "maxfeerate") {
-            map_stats[stat].push_back(maxfeerate);
+            map_stats[stat].push_back(ValueFromAmount(maxfeerate));
         } else if (stat == "medianfeerate") {
-            map_stats[stat].push_back(CalculateTruncatedMedian(feerate_array));
+            map_stats[stat].push_back(ValueFromAmount(CalculateTruncatedMedian(feerate_array)));
         } else if (stat == "avgfeerate") {
-            map_stats[stat].push_back(CFeeRate(totalfee, total_weight).GetTruncatedFee(WITNESS_SCALE_FACTOR));
+            map_stats[stat].push_back(ValueFromAmount(CFeeRate(totalfee, total_weight).GetTruncatedFee(WITNESS_SCALE_FACTOR)));
         } else if (stat == "mintxsize") {
             map_stats[stat].push_back(mintxsize == MAX_BLOCK_SERIALIZED_SIZE ? 0 : mintxsize);
         } else if (stat == "maxtxsize") {
diff --git a/test/functional/getblockstats.py b/test/functional/getblockstats.py
index 00fddf3a90de..e97989a8d413 100755
--- a/test/functional/getblockstats.py
+++ b/test/functional/getblockstats.py
@@ -12,6 +12,11 @@
     assert_raises_rpc_error,
 )
 
+import decimal
+decimal.getcontext().rounding = decimal.ROUND_DOWN
+def D(f):
+    return decimal.Decimal(str(f))
+
 def assert_contains(data, values, check_cointains=True):
     for val in values:
         if (check_cointains):
@@ -109,20 +114,20 @@ def run_test(self):
         assert_equal(stats['swtxs'][1], 0)
         assert_equal(stats['ins'][1], 1)
         assert_equal(stats['outs'][1], 4)
-        assert_equal(stats['totalfee'][1], 3840)
+        assert_equal(stats['totalfee'][1], D(0.00003840))
         assert_equal(stats['utxo_increase'][1], 3)
         assert_equal(stats['utxo_size_inc'][1], 238)
         # assert_equal(stats['total_size'][1], 191)
         # assert_equal(stats['total_weight'][1], 768)
-        assert_equal(stats['total_out'][1], 4999996160)
-        assert_equal(stats['minfee'][1], 3840)
-        assert_equal(stats['maxfee'][1], 3840)
-        assert_equal(stats['medianfee'][1], 3840)
-        assert_equal(stats['avgfee'][1], 3840)
-        assert_equal(stats['minfeerate'][1], 20)
-        assert_equal(stats['maxfeerate'][1], 20)
-        assert_equal(stats['medianfeerate'][1], 20)
-        assert_equal(stats['avgfeerate'][1], 20)
+        assert_equal(stats['total_out'][1], D(49.99996160))
+        assert_equal(stats['minfee'][1], D(0.00003840))
+        assert_equal(stats['maxfee'][1], D(0.00003840))
+        assert_equal(stats['medianfee'][1], D(0.00003840))
+        assert_equal(stats['avgfee'][1], D(0.00003840))
+        assert_equal(stats['minfeerate'][1], D(0.00000020))
+        assert_equal(stats['maxfeerate'][1], D(0.00000020))
+        assert_equal(stats['medianfeerate'][1], D(0.00000020))
+        assert_equal(stats['avgfeerate'][1], D(0.00000020))
         # assert_equal(stats['mintxsize'][1], 192)
         # assert_equal(stats['maxtxsize'][1], 192)
         # assert_equal(stats['mediantxsize'][1], 192)
@@ -132,20 +137,20 @@ def run_test(self):
         assert_equal(stats['swtxs'][max_stat_pos], 0)
         assert_equal(stats['ins'][max_stat_pos], 3)
         assert_equal(stats['outs'][max_stat_pos], 8)
-        assert_equal(stats['totalfee'][max_stat_pos], 76160)
+        assert_equal(stats['totalfee'][max_stat_pos], D(0.00076160))
         assert_equal(stats['utxo_increase'][max_stat_pos], 5)
         assert_equal(stats['utxo_size_inc'][max_stat_pos], 388)
         # assert_equal(stats['total_size'][max_stat_pos], 643)
         # assert_equal(stats['total_weight'][max_stat_pos], 2572)
-        assert_equal(stats['total_out'][max_stat_pos], 9999920000)
-        assert_equal(stats['minfee'][max_stat_pos], 3840)
-        assert_equal(stats['maxfee'][max_stat_pos], 67800)
-        assert_equal(stats['medianfee'][max_stat_pos], 4520)
-        assert_equal(stats['avgfee'][max_stat_pos], 25386)
-        assert_equal(stats['minfeerate'][max_stat_pos], 20)
+        assert_equal(stats['total_out'][max_stat_pos], D(99.99920000))
+        assert_equal(stats['minfee'][max_stat_pos], D(0.00003840))
+        assert_equal(stats['maxfee'][max_stat_pos], D(0.00067800))
+        assert_equal(stats['medianfee'][max_stat_pos], D(0.00004520))
+        assert_equal(stats['avgfee'][max_stat_pos], D(0.00025386))
+        assert_equal(stats['minfeerate'][max_stat_pos], D(0.00000020))
         # assert_equal(stats['maxfeerate'][max_stat_pos], 300)
-        assert_equal(stats['medianfeerate'][max_stat_pos], 20)
-        assert_equal(stats['avgfeerate'][max_stat_pos], 118)
+        assert_equal(stats['medianfeerate'][max_stat_pos], D(0.00000020))
+        assert_equal(stats['avgfeerate'][max_stat_pos], D(0.00000118))
         # assert_equal(stats['mintxsize'][max_stat_pos], 192)
         # assert_equal(stats['maxtxsize'][max_stat_pos], 226)
         # assert_equal(stats['mediantxsize'][max_stat_pos], 225)

From 80dccacab0851ce4561b5cc1c9d89133d4a49f60 Mon Sep 17 00:00:00 2001
From: Anthony Towns <aj@erisian.com.au>
Date: Thu, 19 Oct 2017 20:16:23 +1000
Subject: [PATCH 6/6] test against python reimplementation

---
 test/functional/getblockstats.py | 164 ++++++++++++++++++++++++-------
 1 file changed, 131 insertions(+), 33 deletions(-)

diff --git a/test/functional/getblockstats.py b/test/functional/getblockstats.py
index e97989a8d413..875663ad2942 100755
--- a/test/functional/getblockstats.py
+++ b/test/functional/getblockstats.py
@@ -25,11 +25,121 @@ def assert_contains(data, values, check_cointains=True):
             assert(val not in data)
 
 class GetblockstatsTest(BitcoinTestFramework):
+    all_values = [
+        "height",
+        "time",
+        "mediantime",
+        "txs",
+        "swtxs",
+        "ins",
+        "outs",
+        "subsidy",
+        "totalfee",
+        "utxo_increase",
+        "utxo_size_inc",
+        "total_size",
+        "total_weight",
+        "swtotal_size",
+        "swtotal_weight",
+        "total_out",
+        "minfee",
+        "maxfee",
+        "medianfee",
+        "avgfee",
+        "mintxsize",
+        "maxtxsize",
+        "mediantxsize",
+        "avgtxsize",
+        "minfeerate",
+        "maxfeerate",
+        "medianfeerate",
+        "avgfeerate",
+    ]
+
     def set_test_params(self):
         self.num_nodes = 2
         self.extra_args = [['-txindex'], ['-paytxfee=0.003']]
         self.setup_clean_chain = True
 
+    def utxo_size(self, txout):
+        s = len(txout["scriptPubKey"]["hex"])//2
+        if s < 253:
+            s += 1
+        elif s < 65536:
+            s += 3
+        elif s < 4294967296:
+            s += 5
+        else:
+            s += 9
+        return 41 + 8 + s
+
+    def expected_stats(self, height):
+        blk = self.nodes[0].getblock(self.nodes[0].getblockhash(height), 2)
+        txs = ins = outs = totalfee = total_out = swtxs = 0
+        total_size = total_weight = swtotal_size = swtotal_weight = 0
+        utxo_size_inc = 0
+        _fees = []
+        _feerates = []
+        _sizes = []
+        for txnum,tx in enumerate(blk["tx"]):
+            txs += 1
+            outs += len(tx["vout"])
+            for o in tx["vout"]:
+                utxo_size_inc += self.utxo_size(o)
+            if txnum == 0:
+                subsidy = sum(o["value"] for o in tx["vout"])
+            else:
+                ins += len(tx["vin"])
+                _spend = sum(o["value"] for o in tx["vout"])
+                _receive = 0
+                for i in tx["vin"]:
+                    _prevout = self.nodes[0].getrawtransaction(i["txid"],1)["vout"][i["vout"]]
+                    _receive += _prevout["value"]
+                    utxo_size_inc -= self.utxo_size(_prevout)
+                _size = tx["size"]
+                _weight = tx["vsize"]*4
+                _fee = _receive - _spend
+                _fees.append(_fee)
+                _feerates.append(round(_fee/tx["size"],8))
+                _sizes.append(tx["vsize"])
+                #Useful for debugging averages and feerates
+                #print(">> %d:%d  %r %r %r %r %r" % (height, txnum, _fee, tx["vsize"], tx["size"], round(_fee/tx["size"], 8), round(_fee/tx["vsize"],8)))
+                if any("txinwitness" in i for i in tx["vin"]):
+                    swtxs += 1
+                    swtotal_size += _size
+                    swtotal_weight += _weight
+                totalfee += _fee
+                total_out += _spend
+                total_size += _size
+                total_weight += _weight
+
+        subsidy -= totalfee
+        utxo_increase = outs - ins
+        time = blk["time"]
+        mediantime = blk["mediantime"]
+        res = {}
+        for a in locals().copy():
+            if a in self.all_values: res[a] = locals()[a]
+        stats = {"fee": (_fees, 8), "feerate": (_feerates, 8), "txsize": (_sizes,0)}
+        for _k,(_v,_prec) in stats.items():
+            if len(_v) == 0:
+                res["min"+_k] = res["max"+_k] = res["median"+_k] = res["avg"+_k] = 0
+            else:
+                _v.sort()
+                res["min"+_k] = _v[0]
+                res["max"+_k] = _v[-1]
+                if len(_v) % 2 == 0:
+                    _m = round((_v[len(_v)//2-1] + _v[len(_v)//2])/2,_prec)
+                else:
+                    _m = _v[(len(_v)-1)//2]
+                res["median"+_k] = _m
+                if _k == "feerate":
+                    # weighted average
+                    res["avgfeerate"] = round(totalfee*1000/total_weight*4/1000, 8)
+                else:
+                    res["avg"+_k] = round(sum(_v)/D(len(_v)), _prec)
+        return res
+
     def run_test(self):
         node = self.nodes[0]
         node.generate(101)
@@ -46,41 +156,29 @@ def run_test(self):
 
         start_height = 101
         max_stat_pos = 2
+
         stats = node.getblockstats(start=start_height, end=start_height + max_stat_pos)
 
-        all_values = [
-            "height",
-            "time",
-            "mediantime",
-            "txs",
-            "swtxs",
-            "ins",
-            "outs",
-            "subsidy",
-            "totalfee",
-            "utxo_increase",
-            "utxo_size_inc",
-            "total_size",
-            "total_weight",
-            "swtotal_size",
-            "swtotal_weight",
-            "total_out",
-            "minfee",
-            "maxfee",
-            "medianfee",
-            "avgfee",
-            "minfeerate",
-            "maxfeerate",
-            "medianfeerate",
-            "avgfeerate",
-            "mintxsize",
-            "maxtxsize",
-            "mediantxsize",
-            "avgtxsize",
-        ]
-        assert_contains(stats, all_values)
+        unchecked = set(self.all_values)
+        all_zero = set(self.all_values)
+        for h in range(0, max_stat_pos+1):
+            expected = self.expected_stats(start_height+h)
+            for a in expected:
+                unchecked.discard(a)
+                if expected[a] != 0:
+                    all_zero.discard(a)
+            missing = [a for a in self.all_values if a not in expected]
+            if missing:
+                print("Unchecked values: %s" % (",".join(missing)))
+            for a in expected:
+                assert stats[a][h] == expected[a], "stats for %s at height %d(%d+%d) unexpected (got: %r expected: %r)" % (a, start_height+h, start_height, h, stats[a][h], expected[a])
+        assert not unchecked, "Following stats unchecked: %s" % (",".join(sorted(unchecked)))
+        if all_zero:
+            print("Only ever expected 0: %s" % (",".join(sorted(all_zero))))
+
+        assert_contains(stats, self.all_values)
         # Make sure all valid statistics are included
-        assert_contains(all_values, stats.keys())
+        assert_contains(self.all_values, stats.keys())
 
         assert_equal(stats['height'][0], start_height)
         assert_equal(stats['height'][max_stat_pos], start_height + max_stat_pos)
@@ -185,7 +283,7 @@ def run_test(self):
         ]
         assert_contains(stats, some_values)
         # Make sure valid stats that haven't been selected don't appear
-        other_values = [x for x in all_values if x not in some_values]
+        other_values = [x for x in self.all_values if x not in some_values]
         assert_contains(stats, other_values, False)
 
 if __name__ == '__main__':
