From 4865a3c6228ccb5d08865ad78f16854c25143f35 Mon Sep 17 00:00:00 2001 From: Kittywhiskers Van Gogh <63189531+kwvg@users.noreply.github.com> Date: Tue, 22 Mar 2022 14:17:51 +0530 Subject: [PATCH 01/14] merge bitcoin#24587: use MiniWallet for rpc_createmultisig.py includes: - 2726b60a3ac098b44f2970bed21147b70e12a1c2 (changes excl. wallet.py) continuation of a21dfd27 checkbalances() has to be modified as the balances are now predominantly generated by MiniWallet and the nine[1] blocks mined by the test leave behind immature balances that we must discount. [1] - 8 blocks mined by 4 calls to do_multisig() and 1 block mined by 1 call to checkbalances()) --- test/functional/rpc_createmultisig.py | 94 +++++++++++++++------------ test/functional/test_runner.py | 3 +- 2 files changed, 53 insertions(+), 44 deletions(-) diff --git a/test/functional/rpc_createmultisig.py b/test/functional/rpc_createmultisig.py index cb87eca254c6..22028e900518 100755 --- a/test/functional/rpc_createmultisig.py +++ b/test/functional/rpc_createmultisig.py @@ -17,15 +17,18 @@ assert_equal, ) from test_framework.wallet_util import bytes_to_wif +from test_framework.wallet import ( + MiniWallet, + getnewdestination, +) class RpcCreateMultiSigTest(BitcoinTestFramework): def set_test_params(self): self.setup_clean_chain = True self.num_nodes = 3 self.supports_cli = False - - def skip_test_if_missing_module(self): - self.skip_if_no_wallet() + if self.is_bdb_compiled(): + self.requires_wallet = True def get_keys(self): self.pub = [] @@ -36,29 +39,32 @@ def get_keys(self): k.generate() self.pub.append(k.get_pubkey().get_bytes().hex()) self.priv.append(bytes_to_wif(k.get_bytes(), k.is_compressed)) - self.final = node2.getnewaddress() + if self.is_bdb_compiled(): + self.final = node2.getnewaddress() + else: + self.final = getnewdestination()[2] def run_test(self): node0, node1, node2 = self.nodes + self.wallet = MiniWallet(test_node=node0) - self.check_addmultisigaddress_errors() + if self.is_bdb_compiled(): + self.check_addmultisigaddress_errors() self.log.info('Generating blocks ...') - self.generate(node0, 149) + self.generate(self.wallet, 149) self.moved = 0 for self.nkeys in [3, 5]: for self.nsigs in [2, 3]: self.get_keys() self.do_multisig() - - self.checkbalances() - - # Test mixed compressed and uncompressed pubkeys + if self.is_bdb_compiled(): + self.checkbalances() self.log.info('Mixed compressed and uncompressed multisigs are not allowed') - pk0 = node0.getaddressinfo(node0.getnewaddress())['pubkey'] - pk1 = node1.getaddressinfo(node1.getnewaddress())['pubkey'] - pk2 = node2.getaddressinfo(node2.getnewaddress())['pubkey'] + pk0 = getnewdestination()[0].hex() + pk1 = getnewdestination()[0].hex() + pk2 = getnewdestination()[0].hex() # decompress pk2 pk_obj = ECPubKey() @@ -66,14 +72,16 @@ def run_test(self): pk_obj.compressed = False pk2 = pk_obj.get_bytes().hex() - node0.createwallet(wallet_name='wmulti0', disable_private_keys=True) - wmulti0 = node0.get_wallet_rpc('wmulti0') + if self.is_bdb_compiled(): + node0.createwallet(wallet_name='wmulti0', disable_private_keys=True) + wmulti0 = node0.get_wallet_rpc('wmulti0') # Check all permutations of keys because order matters apparently for keys in itertools.permutations([pk0, pk1, pk2]): # Results should be the same as this legacy one legacy_addr = node0.createmultisig(2, keys)['address'] - assert_equal(legacy_addr, wmulti0.addmultisigaddress(2, keys, '')['address']) + if self.is_bdb_compiled(): + assert_equal(legacy_addr, wmulti0.addmultisigaddress(2, keys, '')['address']) self.log.info('Testing sortedmulti descriptors with BIP 67 test vectors') with open(os.path.join(os.path.dirname(os.path.realpath(__file__)), 'data/rpc_bip67.json'), encoding='utf-8') as f: @@ -105,27 +113,29 @@ def checkbalances(self): bal0 = node0.getbalance() bal1 = node1.getbalance() bal2 = node2.getbalance() + balw = self.wallet.get_balance() height = node0.getblockchaininfo()["blocks"] assert 150 < height < 350 - total = (height - 99 - 1) * 500 - decimal.Decimal("0.00001223") * 4 + total = (height - 9) * 500 - decimal.Decimal("0.00002000") * 4 assert bal1 == 0 assert bal2 == self.moved - - assert bal0 + bal1 + bal2 == total + assert_equal(bal0 + bal1 + bal2 + balw, total) def do_multisig(self): node0, node1, node2 = self.nodes - if 'wmulti' not in node1.listwallets(): - try: - node1.loadwallet('wmulti') - except JSONRPCException as e: - path = os.path.join(self.options.tmpdir, "node1", "regtest", "wallets", "wmulti") - if e.error['code'] == -18 and "Wallet file verification failed. Failed to load database path '{}'. Path does not exist.".format(path) in e.error['message']: - node1.createwallet(wallet_name='wmulti', disable_private_keys=True) - else: - raise - wmulti = node1.get_wallet_rpc('wmulti') + + if self.is_bdb_compiled(): + if 'wmulti' not in node1.listwallets(): + try: + node1.loadwallet('wmulti') + except JSONRPCException as e: + path = os.path.join(self.options.tmpdir, "node1", "regtest", "wallets", "wmulti") + if e.error['code'] == -18 and "Wallet file verification failed. Failed to load database path '{}'. Path does not exist.".format(path) in e.error['message']: + node1.createwallet(wallet_name='wmulti', disable_private_keys=True) + else: + raise + wmulti = node1.get_wallet_rpc('wmulti') # Construct the expected descriptor desc = 'multi({},{})'.format(self.nsigs, ','.join(self.pub)) @@ -137,17 +147,19 @@ def do_multisig(self): mredeem = msig["redeemScript"] assert_equal(desc, msig['descriptor']) - # compare against addmultisigaddress - msigw = wmulti.addmultisigaddress(self.nsigs, self.pub, None) - maddw = msigw["address"] - mredeemw = msigw["redeemScript"] - assert_equal(desc, drop_origins(msigw['descriptor'])) - # addmultisigiaddress and createmultisig work the same - assert maddw == madd - assert mredeemw == mredeem - - txid = node0.sendtoaddress(madd, 40) - + if self.is_bdb_compiled(): + # compare against addmultisigaddress + msigw = wmulti.addmultisigaddress(self.nsigs, self.pub, None) + maddw = msigw["address"] + mredeemw = msigw["redeemScript"] + assert_equal(desc, drop_origins(msigw['descriptor'])) + # addmultisigiaddress and createmultisig work the same + assert maddw == madd + assert mredeemw == mredeem + wmulti.unloadwallet() + + spk = bytes.fromhex(node0.validateaddress(madd)["scriptPubKey"]) + txid, _ = self.wallet.send_to(from_node=self.nodes[0], scriptPubKey=spk, amount=2000) tx = node0.getrawtransaction(txid, True) vout = [v["n"] for v in tx["vout"] if madd == v["scriptPubKey"]["address"]] assert len(vout) == 1 @@ -185,8 +197,6 @@ def do_multisig(self): txinfo = node0.getrawtransaction(tx, True, blk) self.log.info("n/m=%d/%d size=%d" % (self.nsigs, self.nkeys, txinfo["size"])) - wmulti.unloadwallet() - if __name__ == '__main__': RpcCreateMultiSigTest().main() diff --git a/test/functional/test_runner.py b/test/functional/test_runner.py index ff0da91247fc..963d281ed235 100755 --- a/test/functional/test_runner.py +++ b/test/functional/test_runner.py @@ -268,8 +268,7 @@ 'feature_utxo_set_hash.py', 'mempool_packages.py', 'mempool_package_onemore.py', - 'rpc_createmultisig.py --legacy-wallet', - 'rpc_createmultisig.py --descriptors', + 'rpc_createmultisig.py', 'rpc_packages.py', 'mempool_package_limits.py', 'feature_versionbits_warning.py', From 177087757dde7318421957cbaa481e861687657d Mon Sep 17 00:00:00 2001 From: Kittywhiskers Van Gogh <63189531+kwvg@users.noreply.github.com> Date: Wed, 18 Jun 2025 11:45:54 +0000 Subject: [PATCH 02/14] merge bitcoin#24637: use MiniWallet for mempool_package_onemore.py includes: - 2b6dd4e75b3ad2daff553fde018fe4c8f1187878 continuation of 46213202 --- test/functional/mempool_package_onemore.py | 52 ++++++++++------------ 1 file changed, 24 insertions(+), 28 deletions(-) diff --git a/test/functional/mempool_package_onemore.py b/test/functional/mempool_package_onemore.py index de7b20454a83..15ebf39a15ce 100755 --- a/test/functional/mempool_package_onemore.py +++ b/test/functional/mempool_package_onemore.py @@ -7,68 +7,64 @@ size. """ -from decimal import Decimal - -from test_framework.blocktools import COINBASE_MATURITY from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_equal, assert_raises_rpc_error, - chain_transaction, ) +from test_framework.wallet import MiniWallet + MAX_ANCESTORS = 25 MAX_DESCENDANTS = 25 + class MempoolPackagesTest(BitcoinTestFramework): def set_test_params(self): self.num_nodes = 1 self.extra_args = [["-maxorphantxsize=100000"]] - def skip_test_if_missing_module(self): - self.skip_if_no_wallet() + def chain_tx(self, utxos_to_spend, *, num_outputs=1): + return self.wallet.send_self_transfer_multi( + from_node=self.nodes[0], + utxos_to_spend=utxos_to_spend, + num_outputs=num_outputs)['new_utxos'] def run_test(self): - # Mine some blocks and have them mature. - self.generate(self.nodes[0], COINBASE_MATURITY + 1) - utxo = self.nodes[0].listunspent(10) - txid = utxo[0]['txid'] - vout = utxo[0]['vout'] - value = utxo[0]['amount'] + self.wallet = MiniWallet(self.nodes[0]) + self.wallet.rescan_utxos() - fee = Decimal("0.0002") # MAX_ANCESTORS transactions off a confirmed tx should be fine chain = [] + utxo = self.wallet.get_utxo() for _ in range(4): - (txid, sent_value) = chain_transaction(self.nodes[0], [txid], [vout], value, fee, 2) - vout = 0 - value = sent_value - chain.append([txid, value]) + utxo, utxo2 = self.chain_tx([utxo], num_outputs=2) + chain.append(utxo2) for _ in range(MAX_ANCESTORS - 4): - (txid, sent_value) = chain_transaction(self.nodes[0], [txid], [0], value, fee, 1) - value = sent_value - chain.append([txid, value]) - (second_chain, second_chain_value) = chain_transaction(self.nodes[0], [utxo[1]['txid']], [utxo[1]['vout']], utxo[1]['amount'], fee, 1) + utxo, = self.chain_tx([utxo]) + chain.append(utxo) + second_chain, = self.chain_tx([self.wallet.get_utxo()]) # Check mempool has MAX_ANCESTORS + 1 transactions in it assert_equal(len(self.nodes[0].getrawmempool()), MAX_ANCESTORS + 1) # Adding one more transaction on to the chain should fail. - assert_raises_rpc_error(-26, "too-long-mempool-chain, too many unconfirmed ancestors [limit: 25]", chain_transaction, self.nodes[0], [txid], [0], value, fee, 1) + assert_raises_rpc_error(-26, "too-long-mempool-chain, too many unconfirmed ancestors [limit: 25]", self.chain_tx, [utxo]) # ...even if it chains on from some point in the middle of the chain. - assert_raises_rpc_error(-26, "too-long-mempool-chain, too many descendants", chain_transaction, self.nodes[0], [chain[2][0]], [1], chain[2][1], fee, 1) - assert_raises_rpc_error(-26, "too-long-mempool-chain, too many descendants", chain_transaction, self.nodes[0], [chain[1][0]], [1], chain[1][1], fee, 1) + assert_raises_rpc_error(-26, "too-long-mempool-chain, too many descendants", self.chain_tx, [chain[2]]) + assert_raises_rpc_error(-26, "too-long-mempool-chain, too many descendants", self.chain_tx, [chain[1]]) # ...even if it chains on to two parent transactions with one in the chain. - assert_raises_rpc_error(-26, "too-long-mempool-chain, too many descendants", chain_transaction, self.nodes[0], [chain[0][0], second_chain], [1, 0], chain[0][1] + second_chain_value, fee, 1) + assert_raises_rpc_error(-26, "too-long-mempool-chain, too many descendants", self.chain_tx, [chain[0], second_chain]) # ...especially if its > 40k weight - assert_raises_rpc_error(-26, "too-long-mempool-chain, too many descendants", chain_transaction, self.nodes[0], [chain[0][0]], [1], chain[0][1], fee, 350) + assert_raises_rpc_error(-26, "too-long-mempool-chain, too many descendants", self.chain_tx, [chain[0]], num_outputs=350) # But not if it chains directly off the first transaction - chain_transaction(self.nodes[0], [chain[0][0]], [1], chain[0][1], fee, 1) + self.wallet.send_self_transfer_multi(from_node=self.nodes[0], utxos_to_spend=[chain[0]])['tx'] # and the second chain should work just fine - chain_transaction(self.nodes[0], [second_chain], [0], second_chain_value, fee, 1) + self.chain_tx([second_chain]) # Finally, check that we added two transactions assert_equal(len(self.nodes[0].getrawmempool()), MAX_ANCESTORS + 3) + if __name__ == '__main__': MempoolPackagesTest().main() From ae7e4cb592f01da1a566fb9c43b0d4437fd1d34f Mon Sep 17 00:00:00 2001 From: Kittywhiskers Van Gogh <63189531+kwvg@users.noreply.github.com> Date: Thu, 10 Sep 2020 08:06:34 +1000 Subject: [PATCH 03/14] partial bitcoin#19937: signet mining utility includes: - 95d5d5e6257825bb385cee318d5681597f7f7646 needed for backports affecting mining_prioritisetransaction.py so we don't need to manually reconnect (or keep around) nodes. --- src/rpc/mining.cpp | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/src/rpc/mining.cpp b/src/rpc/mining.cpp index 54cb60ddc7a3..a3dcb9a58023 100644 --- a/src/rpc/mining.cpp +++ b/src/rpc/mining.cpp @@ -727,14 +727,19 @@ static RPCHelpMan getblocktemplate() throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid mode"); const CConnman& connman = EnsureConnman(node); - CHECK_NONFATAL(node.sporkman); - if (connman.GetNodeCount(ConnectionDirection::Both) == 0) - throw JSONRPCError(RPC_CLIENT_NOT_CONNECTED, PACKAGE_NAME " is not connected!"); - if (active_chainstate.IsInitialBlockDownload()) - throw JSONRPCError(RPC_CLIENT_IN_INITIAL_DOWNLOAD, PACKAGE_NAME " is in initial sync and waiting for blocks..."); + if (!Params().IsTestChain()) { + if (connman.GetNodeCount(ConnectionDirection::Both) == 0) { + throw JSONRPCError(RPC_CLIENT_NOT_CONNECTED, PACKAGE_NAME " is not connected!"); + } + + if (active_chainstate.IsInitialBlockDownload()) { + throw JSONRPCError(RPC_CLIENT_IN_INITIAL_DOWNLOAD, PACKAGE_NAME " is in initial sync and waiting for blocks..."); + } + } // next bock is a superblock and we need governance info to correctly construct it + CHECK_NONFATAL(node.sporkman); if (AreSuperblocksEnabled(*node.sporkman) && !node.mn_sync->IsSynced() && CSuperblock::IsValidBlockHeight(active_chain.Height() + 1)) From 8d5883ccda353395afb7e763be265dae7c4679ca Mon Sep 17 00:00:00 2001 From: Kittywhiskers Van Gogh <63189531+kwvg@users.noreply.github.com> Date: Wed, 18 Jun 2025 11:53:44 +0000 Subject: [PATCH 04/14] merge bitcoin#24623: Add diamond-shape prioritisetransaction test includes: - 462132022b2e5b76ff4229f04de42ad532963ccf continuation of b84bcadd need to reestablish connection between nodes to prevent generate calls from flaking out (connect_nodes()) --- .../mining_prioritisetransaction.py | 85 +++++++++++++++++++ 1 file changed, 85 insertions(+) diff --git a/test/functional/mining_prioritisetransaction.py b/test/functional/mining_prioritisetransaction.py index b9a59f9f2fc7..b5b9506d81cd 100755 --- a/test/functional/mining_prioritisetransaction.py +++ b/test/functional/mining_prioritisetransaction.py @@ -4,9 +4,14 @@ # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the prioritisetransaction mining RPC.""" +from decimal import Decimal + +from test_framework.blocktools import COINBASE_MATURITY from test_framework.messages import COIN, MAX_BLOCK_SIZE from test_framework.test_framework import BitcoinTestFramework from test_framework.util import assert_equal, assert_raises_rpc_error, create_confirmed_utxos, create_lots_of_big_transactions, gen_return_txouts +from test_framework.wallet import MiniWallet + class PrioritiseTransactionTest(BitcoinTestFramework): def set_test_params(self): @@ -21,7 +26,85 @@ def set_test_params(self): def skip_test_if_missing_module(self): self.skip_if_no_wallet() + def test_diamond(self): + self.log.info("Test diamond-shape package with priority") + self.generate(self.wallet, COINBASE_MATURITY + 1) + self.nodes[0].setmocktime(self.mocktime) + + # tx_a + # / \ + # / \ + # tx_b tx_c + # \ / + # \ / + # tx_d + + tx_o_a = self.wallet.send_self_transfer_multi( + from_node=self.nodes[0], + num_outputs=2, + ) + txid_a = tx_o_a["txid"] + + tx_o_b, tx_o_c = [self.wallet.send_self_transfer( + from_node=self.nodes[0], + utxo_to_spend=u, + ) for u in tx_o_a["new_utxos"]] + txid_b = tx_o_b["txid"] + txid_c = tx_o_c["txid"] + + tx_o_d = self.wallet.send_self_transfer_multi( + from_node=self.nodes[0], + utxos_to_spend=[ + self.wallet.get_utxo(txid=txid_b), + self.wallet.get_utxo(txid=txid_c), + ], + ) + txid_d = tx_o_d["txid"] + + self.log.info("Test priority while txs are in mempool") + raw_before = self.nodes[0].getrawmempool(verbose=True) + fee_delta_b = Decimal(9999) / COIN + fee_delta_c_1 = Decimal(-1234) / COIN + fee_delta_c_2 = Decimal(8888) / COIN + self.nodes[0].prioritisetransaction(txid=txid_b, fee_delta=int(fee_delta_b * COIN)) + self.nodes[0].prioritisetransaction(txid=txid_c, fee_delta=int(fee_delta_c_1 * COIN)) + self.nodes[0].prioritisetransaction(txid=txid_c, fee_delta=int(fee_delta_c_2 * COIN)) + raw_before[txid_a]["fees"]["descendant"] += fee_delta_b + fee_delta_c_1 + fee_delta_c_2 + raw_before[txid_b]["fees"]["modified"] += fee_delta_b + raw_before[txid_b]["fees"]["ancestor"] += fee_delta_b + raw_before[txid_b]["fees"]["descendant"] += fee_delta_b + raw_before[txid_c]["fees"]["modified"] += fee_delta_c_1 + fee_delta_c_2 + raw_before[txid_c]["fees"]["ancestor"] += fee_delta_c_1 + fee_delta_c_2 + raw_before[txid_c]["fees"]["descendant"] += fee_delta_c_1 + fee_delta_c_2 + raw_before[txid_d]["fees"]["ancestor"] += fee_delta_b + fee_delta_c_1 + fee_delta_c_2 + raw_after = self.nodes[0].getrawmempool(verbose=True) + assert_equal(raw_before[txid_a], raw_after[txid_a]) + assert_equal(raw_before, raw_after) + + self.log.info("Test priority while txs are not in mempool") + self.restart_node(0, extra_args=["-nopersistmempool"]) + self.connect_nodes(0, 1) + self.nodes[0].setmocktime(self.mocktime) + assert_equal(self.nodes[0].getmempoolinfo()["size"], 0) + self.nodes[0].prioritisetransaction(txid=txid_b, fee_delta=int(fee_delta_b * COIN)) + self.nodes[0].prioritisetransaction(txid=txid_c, fee_delta=int(fee_delta_c_1 * COIN)) + self.nodes[0].prioritisetransaction(txid=txid_c, fee_delta=int(fee_delta_c_2 * COIN)) + for t in [tx_o_a["hex"], tx_o_b["hex"], tx_o_c["hex"], tx_o_d["hex"]]: + self.nodes[0].sendrawtransaction(t) + raw_after = self.nodes[0].getrawmempool(verbose=True) + assert_equal(raw_before[txid_a], raw_after[txid_a]) + assert_equal(raw_before, raw_after) + + # Clear mempool + self.generate(self.nodes[0], 1) + + # Use default extra_args + self.restart_node(0) + self.connect_nodes(0, 1) + def run_test(self): + self.wallet = MiniWallet(self.nodes[0]) + # Test `prioritisetransaction` required parameters assert_raises_rpc_error(-1, "prioritisetransaction", self.nodes[0].prioritisetransaction) assert_raises_rpc_error(-1, "prioritisetransaction", self.nodes[0].prioritisetransaction, '') @@ -38,6 +121,8 @@ def run_test(self): # Test `prioritisetransaction` invalid `fee_delta` assert_raises_rpc_error(-1, "JSON value is not an integer as expected", self.nodes[0].prioritisetransaction, txid=txid, fee_delta='foo') + self.test_diamond() + self.txouts = gen_return_txouts() self.relayfee = self.nodes[0].getnetworkinfo()['relayfee'] From 30b1637727b3b929775194d975d8c89736aef373 Mon Sep 17 00:00:00 2001 From: Kittywhiskers Van Gogh <63189531+kwvg@users.noreply.github.com> Date: Mon, 20 Sep 2021 19:22:03 +0200 Subject: [PATCH 05/14] merge bitcoin#23075: Fee estimation functional test cleanups excludes: - e50213967b6d5dda9c0acc4643c8ec67f9fd7284 (we don't support RBF) - 60ae1161a4c415cc73f47df95598f3688e8d34df (see above) --- test/functional/feature_fee_estimation.py | 170 ++++++++++------------ 1 file changed, 76 insertions(+), 94 deletions(-) diff --git a/test/functional/feature_fee_estimation.py b/test/functional/feature_fee_estimation.py index f8114b3d024e..e6299deb63da 100755 --- a/test/functional/feature_fee_estimation.py +++ b/test/functional/feature_fee_estimation.py @@ -16,7 +16,6 @@ from test_framework.script import ( CScript, OP_1, - OP_2, OP_DROP, OP_TRUE, ) @@ -35,16 +34,14 @@ # Construct 2 trivial P2SH's and the ScriptSigs that spend them # So we can create many transactions without needing to spend # time signing. -REDEEM_SCRIPT_1 = CScript([OP_1, OP_DROP]) -REDEEM_SCRIPT_2 = CScript([OP_2, OP_DROP]) -P2SH_1 = script_to_p2sh_script(REDEEM_SCRIPT_1) -P2SH_2 = script_to_p2sh_script(REDEEM_SCRIPT_2) +SCRIPT = CScript([OP_1, OP_DROP]) +P2SH = script_to_p2sh_script(SCRIPT) +REDEEM_SCRIPT = CScript([OP_TRUE, SCRIPT]) -# Associated ScriptSig's to spend satisfy P2SH_1 and P2SH_2 -SCRIPT_SIG = [CScript([OP_TRUE, REDEEM_SCRIPT_1]), CScript([OP_TRUE, REDEEM_SCRIPT_2])] - -def small_txpuzzle_randfee(from_node, conflist, unconflist, amount, min_fee, fee_increment): +def small_txpuzzle_randfee( + from_node, conflist, unconflist, amount, min_fee, fee_increment +): """Create and send a transaction with a random fee. The transaction pays to a trivial P2SH script, and assumes that its inputs @@ -65,20 +62,15 @@ def small_txpuzzle_randfee(from_node, conflist, unconflist, amount, min_fee, fee while total_in <= (amount + fee) and len(conflist) > 0: t = conflist.pop(0) total_in += t["amount"] - tx.vin.append(CTxIn(COutPoint(int(t["txid"], 16), t["vout"]), b"")) + tx.vin.append(CTxIn(COutPoint(int(t["txid"], 16), t["vout"]), REDEEM_SCRIPT)) + while total_in <= (amount + fee) and len(unconflist) > 0: + t = unconflist.pop(0) + total_in += t["amount"] + tx.vin.append(CTxIn(COutPoint(int(t["txid"], 16), t["vout"]), REDEEM_SCRIPT)) if total_in <= amount + fee: - while total_in <= (amount + fee) and len(unconflist) > 0: - t = unconflist.pop(0) - total_in += t["amount"] - tx.vin.append(CTxIn(COutPoint(int(t["txid"], 16), t["vout"]), b"")) - if total_in <= amount + fee: - raise RuntimeError(f"Insufficient funds: need {amount + fee}, have {total_in}") - tx.vout.append(CTxOut(int((total_in - amount - fee) * COIN), P2SH_1)) - tx.vout.append(CTxOut(int(amount * COIN), P2SH_2)) - # These transactions don't need to be signed, but we still have to insert - # the ScriptSig that will satisfy the ScriptPubKey. - for inp in tx.vin: - inp.scriptSig = SCRIPT_SIG[inp.prevout.n] + raise RuntimeError(f"Insufficient funds: need {amount + fee}, have {total_in}") + tx.vout.append(CTxOut(int((total_in - amount - fee) * COIN), P2SH)) + tx.vout.append(CTxOut(int(amount * COIN), P2SH)) txid = from_node.sendrawtransaction(hexstring=tx.serialize().hex(), maxfeerate=0) unconflist.append({"txid": txid, "vout": 0, "amount": total_in - amount - fee}) unconflist.append({"txid": txid, "vout": 1, "amount": amount}) @@ -86,35 +78,6 @@ def small_txpuzzle_randfee(from_node, conflist, unconflist, amount, min_fee, fee return (tx.serialize().hex(), fee) -def split_inputs(from_node, txins, txouts, initial_split=False): - """Generate a lot of inputs so we can generate a ton of transactions. - - This function takes an input from txins, and creates and sends a transaction - which splits the value into 2 outputs which are appended to txouts. - Previously this was designed to be small inputs so they wouldn't have - a high coin age when the notion of priority still existed.""" - - prevtxout = txins.pop() - tx = CTransaction() - tx.vin.append(CTxIn(COutPoint(int(prevtxout["txid"], 16), prevtxout["vout"]), b"")) - - half_change = satoshi_round(prevtxout["amount"] / 2) - rem_change = prevtxout["amount"] - half_change - Decimal("0.00001000") - tx.vout.append(CTxOut(int(half_change * COIN), P2SH_1)) - tx.vout.append(CTxOut(int(rem_change * COIN), P2SH_2)) - - # If this is the initial split we actually need to sign the transaction - # Otherwise we just need to insert the proper ScriptSig - if (initial_split): - completetx = from_node.signrawtransactionwithwallet(tx.serialize().hex())["hex"] - else: - tx.vin[0].scriptSig = SCRIPT_SIG[prevtxout["vout"]] - completetx = tx.serialize().hex() - txid = from_node.sendrawtransaction(hexstring=completetx, maxfeerate=0) - txouts.append({"txid": txid, "vout": 0, "amount": half_change}) - txouts.append({"txid": txid, "vout": 1, "amount": rem_change}) - - def check_raw_estimates(node, fees_seen): """Call estimaterawfee and verify that the estimates meet certain invariants.""" @@ -125,7 +88,10 @@ def check_raw_estimates(node, fees_seen): assert_greater_than(feerate, 0) if feerate + delta < min(fees_seen) or feerate - delta > max(fees_seen): - raise AssertionError(f"Estimated fee ({feerate}) out of range ({min(fees_seen)},{max(fees_seen)})") + raise AssertionError( + f"Estimated fee ({feerate}) out of range ({min(fees_seen)},{max(fees_seen)})" + ) + def check_smart_estimates(node, fees_seen): @@ -134,8 +100,8 @@ def check_smart_estimates(node, fees_seen): delta = 1.0e-6 # account for rounding error last_feerate = float(max(fees_seen)) all_smart_estimates = [node.estimatesmartfee(i) for i in range(1, 26)] - mempoolMinFee = node.getmempoolinfo()['mempoolminfee'] - minRelaytxFee = node.getmempoolinfo()['minrelaytxfee'] + mempoolMinFee = node.getmempoolinfo()["mempoolminfee"] + minRelaytxFee = node.getmempoolinfo()["minrelaytxfee"] for i, e in enumerate(all_smart_estimates): # estimate is for i+1 feerate = float(e["feerate"]) assert_greater_than(feerate, 0) @@ -143,9 +109,13 @@ def check_smart_estimates(node, fees_seen): assert_greater_than_or_equal(feerate, float(minRelaytxFee)) if feerate + delta < min(fees_seen) or feerate - delta > max(fees_seen): - raise AssertionError(f"Estimated fee ({feerate}) out of range ({min(fees_seen)},{max(fees_seen)})") + raise AssertionError( + f"Estimated fee ({feerate}) out of range ({min(fees_seen)},{max(fees_seen)})" + ) if feerate - delta > last_feerate: - raise AssertionError(f"Estimated fee ({feerate}) larger than last fee ({last_feerate}) for lower number of confirms") + raise AssertionError( + f"Estimated fee ({feerate}) larger than last fee ({last_feerate}) for lower number of confirms" + ) last_feerate = feerate if i == 0: @@ -161,12 +131,11 @@ def check_estimates(node, fees_seen): class EstimateFeeTest(BitcoinTestFramework): def set_test_params(self): self.num_nodes = 3 - # mine non-standard txs (e.g. txs with "dust" outputs) # Force fSendTrickle to true (via whitelist) self.extra_args = [ - ["-acceptnonstdtxn=1", "-whitelist=noban@127.0.0.1"], - ["-acceptnonstdtxn=1", "-whitelist=noban@127.0.0.1", "-blockmaxsize=17000"], - ["-acceptnonstdtxn=1", "-whitelist=noban@127.0.0.1", "-blockmaxsize=8000"] + ["-whitelist=noban@127.0.0.1"], + ["-whitelist=noban@127.0.0.1", "-blockmaxsize=17000"], + ["-whitelist=noban@127.0.0.1", "-blockmaxsize=8000"] ] def skip_test_if_missing_module(self): @@ -199,11 +168,17 @@ def transact_and_mine(self, numblocks, mining_node): random.shuffle(self.confutxo) for _ in range(random.randrange(100 - 50, 100 + 50)): from_index = random.randint(1, 2) - (txhex, fee) = small_txpuzzle_randfee(self.nodes[from_index], self.confutxo, - self.memutxo, Decimal("0.005"), min_fee, min_fee) + (txhex, fee) = small_txpuzzle_randfee( + self.nodes[from_index], + self.confutxo, + self.memutxo, + Decimal("0.005"), + min_fee, + min_fee, + ) tx_kbytes = (len(txhex) // 2) / 1000.0 self.fees_per_kb.append(float(fee) / tx_kbytes) - self.sync_mempools(wait=.1) + self.sync_mempools(wait=0.1) mined = mining_node.getblock(self.generate(mining_node, 1)[0], True)["tx"] # update which txouts are confirmed newmem = [] @@ -216,46 +191,49 @@ def transact_and_mine(self, numblocks, mining_node): def initial_split(self, node): """Split two coinbase UTxOs into many small coins""" - self.txouts = [] - self.txouts2 = [] - # Split a coinbase into two transaction puzzle outputs - split_inputs(node, node.listunspent(0), self.txouts, True) - - # Mine + utxo_count = 2048 + self.confutxo = [] + splitted_amount = Decimal("0.04") + fee = Decimal("0.0007") + # Calculate change from UTXOs instead of relying on hardcoded amounts + coinbase_utxos = node.listunspent()[:2] + total_input = sum(Decimal(str(cb["amount"])) for cb in coinbase_utxos) + change = total_input - splitted_amount * utxo_count - fee + + tx = CTransaction() + tx.vin = [ + CTxIn(COutPoint(int(cb["txid"], 16), cb["vout"])) + for cb in coinbase_utxos + ] + tx.vout = [CTxOut(int(splitted_amount * COIN), P2SH) for _ in range(utxo_count)] + tx.vout.append(CTxOut(int(change * COIN), P2SH)) + txhex = node.signrawtransactionwithwallet(tx.serialize().hex())["hex"] + txid = node.sendrawtransaction(txhex) + self.confutxo = [ + {"txid": txid, "vout": i, "amount": splitted_amount} + for i in range(utxo_count) + ] while len(node.getrawmempool()) > 0: self.generate(node, 1, sync_fun=self.no_op) - # Repeatedly split those 2 outputs, doubling twice for each rep - # Use txouts to monitor the available utxo, since these won't be tracked in wallet - reps = 0 - while reps < 5: - # Double txouts to txouts2 - while len(self.txouts) > 0: - split_inputs(node, self.txouts, self.txouts2) - while len(node.getrawmempool()) > 0: - self.generate(node, 1, sync_fun=self.no_op) - # Double txouts2 to txouts - while len(self.txouts2) > 0: - split_inputs(node, self.txouts2, self.txouts) - while len(node.getrawmempool()) > 0: - self.generate(node, 1, sync_fun=self.no_op) - reps += 1 - def sanity_check_estimates_range(self): """Populate estimation buckets, assert estimates are in a sane range and are strictly increasing as the target decreases.""" self.fees_per_kb = [] self.memutxo = [] - self.confutxo = self.txouts # Start with the set of confirmed txouts after splitting self.log.info("Will output estimates for 1/2/3/6/15/25 blocks") for _ in range(2): - self.log.info("Creating transactions and mining them with a block size that can't keep up") + self.log.info( + "Creating transactions and mining them with a block size that can't keep up" + ) # Create transactions and mine 10 small blocks with node 2, but create txs faster than we can mine self.transact_and_mine(10, self.nodes[2]) check_estimates(self.nodes[1], self.fees_per_kb) - self.log.info("Creating transactions and mining them at a block size that is just big enough") + self.log.info( + "Creating transactions and mining them at a block size that is just big enough" + ) # Generate transactions while mining 10 more blocks, this time with node1 # which mines blocks with capacity just above the rate that transactions are being created self.transact_and_mine(10, self.nodes[1]) @@ -264,12 +242,13 @@ def sanity_check_estimates_range(self): # Finish by mining a normal-sized block: while len(self.nodes[1].getrawmempool()) > 0: self.generate(self.nodes[1], 1) + self.log.info("Final estimates after emptying mempools") check_estimates(self.nodes[1], self.fees_per_kb) def test_feerate_mempoolminfee(self): - high_val = 3*self.nodes[1].estimatesmartfee(1)['feerate'] - self.restart_node(1, extra_args=[f'-minrelaytxfee={high_val}']) + high_val = 3 * self.nodes[1].estimatesmartfee(1)["feerate"] + self.restart_node(1, extra_args=[f"-minrelaytxfee={high_val}"]) check_estimates(self.nodes[1], self.fees_per_kb) self.stop_node(1, expected_stderr="Warning: -minrelaytxfee is set very high! The wallet will avoid paying less than the minimum relay fee.") @@ -295,14 +274,17 @@ def run_test(self): self.sanity_check_estimates_range() # check that the effective feerate is greater than or equal to the mempoolminfee even for high mempoolminfee - self.log.info("Test fee rate estimation after restarting node with high MempoolMinFee") + self.log.info( + "Test fee rate estimation after restarting node with high MempoolMinFee" + ) self.test_feerate_mempoolminfee() self.log.info("Testing that fee estimation is disabled in blocksonly.") self.restart_node(0, ["-blocksonly"]) - assert_raises_rpc_error(-32603, "Fee estimation disabled", - self.nodes[0].estimatesmartfee, 2) + assert_raises_rpc_error( + -32603, "Fee estimation disabled", self.nodes[0].estimatesmartfee, 2 + ) -if __name__ == '__main__': +if __name__ == "__main__": EstimateFeeTest().main() From 917f6b69bb0b821fb1c5c0af02757273a7bbc876 Mon Sep 17 00:00:00 2001 From: Kittywhiskers Van Gogh <63189531+kwvg@users.noreply.github.com> Date: Sun, 10 Apr 2022 01:44:22 +0200 Subject: [PATCH 06/14] merge bitcoin#24817: use MiniWallet for feature_fee_estimation.py includes changes to feature_fee_estimation.py from bitcoin#25445 (fa04ff61) due to OOO backport needed for correctness --- test/functional/feature_fee_estimation.py | 88 +++++++---------------- 1 file changed, 26 insertions(+), 62 deletions(-) diff --git a/test/functional/feature_fee_estimation.py b/test/functional/feature_fee_estimation.py index e6299deb63da..c9fc12470fb7 100755 --- a/test/functional/feature_fee_estimation.py +++ b/test/functional/feature_fee_estimation.py @@ -3,24 +3,12 @@ # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test fee estimation code.""" +from copy import deepcopy from decimal import Decimal import random from test_framework.messages import ( COIN, - COutPoint, - CTransaction, - CTxIn, - CTxOut, -) -from test_framework.script import ( - CScript, - OP_1, - OP_DROP, - OP_TRUE, -) -from test_framework.script_util import ( - script_to_p2sh_script, ) from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( @@ -30,22 +18,14 @@ assert_raises_rpc_error, satoshi_round, ) - -# Construct 2 trivial P2SH's and the ScriptSigs that spend them -# So we can create many transactions without needing to spend -# time signing. -SCRIPT = CScript([OP_1, OP_DROP]) -P2SH = script_to_p2sh_script(SCRIPT) -REDEEM_SCRIPT = CScript([OP_TRUE, SCRIPT]) +from test_framework.wallet import MiniWallet def small_txpuzzle_randfee( - from_node, conflist, unconflist, amount, min_fee, fee_increment + wallet, from_node, conflist, unconflist, amount, min_fee, fee_increment ): - """Create and send a transaction with a random fee. + """Create and send a transaction with a random fee using MiniWallet. - The transaction pays to a trivial P2SH script, and assumes that its inputs - are of the same form. The function takes a list of confirmed outputs and unconfirmed outputs and attempts to use the confirmed list first for its inputs. It adds the newly created outputs to the unconfirmed list. @@ -57,23 +37,28 @@ def small_txpuzzle_randfee( rand_fee = float(fee_increment) * (1.1892 ** random.randint(0, 28)) # Total fee ranges from min_fee to min_fee + 127*fee_increment fee = min_fee - fee_increment + satoshi_round(rand_fee) - tx = CTransaction() + utxos_to_spend = [] total_in = Decimal("0.00000000") while total_in <= (amount + fee) and len(conflist) > 0: t = conflist.pop(0) - total_in += t["amount"] - tx.vin.append(CTxIn(COutPoint(int(t["txid"], 16), t["vout"]), REDEEM_SCRIPT)) + total_in += t["value"] + utxos_to_spend.append(t) while total_in <= (amount + fee) and len(unconflist) > 0: t = unconflist.pop(0) - total_in += t["amount"] - tx.vin.append(CTxIn(COutPoint(int(t["txid"], 16), t["vout"]), REDEEM_SCRIPT)) + total_in += t["value"] + utxos_to_spend.append(t) if total_in <= amount + fee: raise RuntimeError(f"Insufficient funds: need {amount + fee}, have {total_in}") - tx.vout.append(CTxOut(int((total_in - amount - fee) * COIN), P2SH)) - tx.vout.append(CTxOut(int(amount * COIN), P2SH)) + tx = wallet.create_self_transfer_multi( + utxos_to_spend=utxos_to_spend, + fee_per_output=0)["tx"] + tx.vout[0].nValue = int((total_in - amount - fee) * COIN) + tx.vout.append(deepcopy(tx.vout[0])) + tx.vout[1].nValue = int(amount * COIN) + txid = from_node.sendrawtransaction(hexstring=tx.serialize().hex(), maxfeerate=0) - unconflist.append({"txid": txid, "vout": 0, "amount": total_in - amount - fee}) - unconflist.append({"txid": txid, "vout": 1, "amount": amount}) + unconflist.append({"txid": txid, "vout": 0, "value": total_in - amount - fee}) + unconflist.append({"txid": txid, "vout": 1, "value": amount}) return (tx.serialize().hex(), fee) @@ -138,9 +123,6 @@ def set_test_params(self): ["-whitelist=noban@127.0.0.1", "-blockmaxsize=8000"] ] - def skip_test_if_missing_module(self): - self.skip_if_no_wallet() - def setup_network(self): """ We'll setup the network to have 3 nodes that all mine with different parameters. @@ -154,9 +136,6 @@ def setup_network(self): # (17k is room enough for 110 or so transactions) # Node2 is a stingy miner, that # produces too small blocks (room for only 55 or so transactions) - self.start_nodes() - self.import_deterministic_coinbase_privkeys() - self.stop_nodes() def transact_and_mine(self, numblocks, mining_node): min_fee = Decimal("0.0001") @@ -169,6 +148,7 @@ def transact_and_mine(self, numblocks, mining_node): for _ in range(random.randrange(100 - 50, 100 + 50)): from_index = random.randint(1, 2) (txhex, fee) = small_txpuzzle_randfee( + self.wallet, self.nodes[from_index], self.confutxo, self.memutxo, @@ -191,28 +171,10 @@ def transact_and_mine(self, numblocks, mining_node): def initial_split(self, node): """Split two coinbase UTxOs into many small coins""" - utxo_count = 2048 - self.confutxo = [] - splitted_amount = Decimal("0.04") - fee = Decimal("0.0007") - # Calculate change from UTXOs instead of relying on hardcoded amounts - coinbase_utxos = node.listunspent()[:2] - total_input = sum(Decimal(str(cb["amount"])) for cb in coinbase_utxos) - change = total_input - splitted_amount * utxo_count - fee - - tx = CTransaction() - tx.vin = [ - CTxIn(COutPoint(int(cb["txid"], 16), cb["vout"])) - for cb in coinbase_utxos - ] - tx.vout = [CTxOut(int(splitted_amount * COIN), P2SH) for _ in range(utxo_count)] - tx.vout.append(CTxOut(int(change * COIN), P2SH)) - txhex = node.signrawtransactionwithwallet(tx.serialize().hex())["hex"] - txid = node.sendrawtransaction(txhex) - self.confutxo = [ - {"txid": txid, "vout": i, "amount": splitted_amount} - for i in range(utxo_count) - ] + self.confutxo = self.wallet.send_self_transfer_multi( + from_node=node, + utxos_to_spend=[self.wallet.get_utxo() for _ in range(2)], + num_outputs=2048)['new_utxos'] while len(node.getrawmempool()) > 0: self.generate(node, 1, sync_fun=self.no_op) @@ -250,7 +212,7 @@ def test_feerate_mempoolminfee(self): high_val = 3 * self.nodes[1].estimatesmartfee(1)["feerate"] self.restart_node(1, extra_args=[f"-minrelaytxfee={high_val}"]) check_estimates(self.nodes[1], self.fees_per_kb) - self.stop_node(1, expected_stderr="Warning: -minrelaytxfee is set very high! The wallet will avoid paying less than the minimum relay fee.") + self.stop_node(1) def run_test(self): self.log.info("This test is time consuming, please be patient") @@ -258,6 +220,8 @@ def run_test(self): # Split two coinbases into many small utxos self.start_node(0) + self.wallet = MiniWallet(self.nodes[0]) + self.wallet.rescan_utxos() self.initial_split(self.nodes[0]) self.log.info("Finished splitting") From 69abcec052ddb6c3e185be3d43272446977718d7 Mon Sep 17 00:00:00 2001 From: Kittywhiskers Van Gogh <63189531+kwvg@users.noreply.github.com> Date: Mon, 9 May 2022 00:38:17 +0200 Subject: [PATCH 07/14] merge bitcoin#25087: use MiniWallet for feature_dbcrash.py includes changes to feature_dbcrash.py from bitcoin#25445 (fa04ff61) due to OOO backport needed for correctness --- test/functional/feature_dbcrash.py | 54 ++++++++++++------------ test/functional/test_framework/wallet.py | 7 +++ 2 files changed, 33 insertions(+), 28 deletions(-) diff --git a/test/functional/feature_dbcrash.py b/test/functional/feature_dbcrash.py index 725182cb7aae..dd8230f3a1f1 100755 --- a/test/functional/feature_dbcrash.py +++ b/test/functional/feature_dbcrash.py @@ -30,17 +30,17 @@ import random import time +from test_framework.blocktools import COINBASE_MATURITY from test_framework.messages import ( COIN, - COutPoint, - CTransaction, - CTxIn, - CTxOut, ) from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_equal, - create_confirmed_utxos, +) +from test_framework.wallet import ( + MiniWallet, + getnewdestination, ) @@ -66,13 +66,9 @@ def set_test_params(self): self.node3_args = ["-dustrelayfee=0"] self.extra_args = [self.node0_args, self.node1_args, self.node2_args, self.node3_args] - def skip_test_if_missing_module(self): - self.skip_if_no_wallet() - def setup_network(self): self.add_nodes(self.num_nodes, extra_args=self.extra_args) self.start_nodes() - self.import_deterministic_coinbase_privkeys() # Leave them unconnected, we'll use submitblock directly in this test def restart_node(self, node_index, expected_tip): @@ -190,36 +186,37 @@ def generate_small_transactions(self, node, count, utxo_list): num_transactions = 0 random.shuffle(utxo_list) while len(utxo_list) >= 2 and num_transactions < count: - tx = CTransaction() - input_amount = 0 - for _ in range(2): - utxo = utxo_list.pop() - tx.vin.append(CTxIn(COutPoint(int(utxo['txid'], 16), utxo['vout']))) - input_amount += int(utxo['amount'] * COIN) - output_amount = (input_amount - FEE) // 3 - - if output_amount <= 0: + utxos_to_spend = [utxo_list.pop() for _ in range(2)] + input_amount = int(sum([utxo['value'] for utxo in utxos_to_spend]) * COIN) + if input_amount < FEE: # Sanity check -- if we chose inputs that are too small, skip continue - for _ in range(3): - tx.vout.append(CTxOut(output_amount, bytes.fromhex(utxo['scriptPubKey']))) + self.wallet.send_self_transfer_multi( + from_node=node, + utxos_to_spend=utxos_to_spend, + num_outputs=3, + fee_per_output=FEE // 3 + ) - # Sign and send the transaction to get into the mempool - tx_signed_hex = node.signrawtransactionwithwallet(tx.serialize().hex())['hex'] - node.sendrawtransaction(tx_signed_hex) num_transactions += 1 def run_test(self): + self.wallet = MiniWallet(self.nodes[3]) + self.wallet.rescan_utxos() + initial_height = self.nodes[3].getblockcount() + self.generate(self.nodes[3], COINBASE_MATURITY, sync_fun=self.no_op) + # Track test coverage statistics self.restart_counts = [0, 0, 0] # Track the restarts for nodes 0-2 self.crashed_on_restart = 0 # Track count of crashes during recovery # Start by creating a lot of utxos on node3 - initial_height = self.nodes[3].getblockcount() utxo_list = [] for _ in range(5): - utxo_list.extend(create_confirmed_utxos(self, self.nodes[3].getnetworkinfo()['relayfee'], self.nodes[3], 1000, sync_fun=self.no_op)) + utxo_list.extend(self.wallet.send_self_transfer_multi(from_node=self.nodes[3], num_outputs=1000)['new_utxos']) + self.generate(self.nodes[3], 1, sync_fun=self.no_op) + assert_equal(len(self.nodes[3].getrawmempool()), 0) self.log.info(f"Prepped {len(utxo_list)} utxo entries") # Sync these blocks with the other nodes @@ -259,13 +256,14 @@ def run_test(self): self.nodes[3], nblocks=min(10, current_height + 1 - self.nodes[3].getblockcount()), # new address to avoid mining a block that has just been invalidated - address=self.nodes[3].getnewaddress(), + address=getnewdestination()[2], sync_fun=self.no_op, )) self.log.debug(f"Syncing {len(block_hashes)} new blocks...") self.sync_node3blocks(block_hashes) - utxo_list = self.nodes[3].listunspent() - self.log.debug(f"Node3 utxo count: {len(utxo_list)}") + self.wallet.rescan_utxos() + utxo_list = self.wallet.get_utxos() + self.log.debug(f"MiniWallet utxo count: {len(utxo_list)}") # Check that the utxo hashes agree with node3 # Useful side effect: each utxo cache gets flushed here, so that we diff --git a/test/functional/test_framework/wallet.py b/test/functional/test_framework/wallet.py index e0e1b7fab8c3..b0cefa547345 100644 --- a/test/functional/test_framework/wallet.py +++ b/test/functional/test_framework/wallet.py @@ -178,6 +178,13 @@ def get_utxo(self, *, txid: str = '', vout: Optional[int] = None, mark_as_spent= else: return self._utxos[index] + def get_utxos(self, *, mark_as_spent=True): + """Returns the list of all utxos and optionally mark them as spent""" + utxos = deepcopy(self._utxos) + if mark_as_spent: + self._utxos = [] + return utxos + def send_self_transfer(self, *, from_node, **kwargs): """Call create_self_transfer and send the transaction.""" tx = self.create_self_transfer(**kwargs) From 6add1c4c814750f62be53356f4b5bd975ed6750b Mon Sep 17 00:00:00 2001 From: Kittywhiskers Van Gogh <63189531+kwvg@users.noreply.github.com> Date: Sat, 21 Jun 2025 21:49:12 +0000 Subject: [PATCH 08/14] merge bitcoin#24839: use MiniWallet for mining_prioritisetransaction.py includes changes to util.py, mining_prioritisetransaction.py from bitcoin#25435 (fa8421bc) and changes to util.py from bitcoin#25356 (fa779de66) due to OOO backport needed for correctness --- test/functional/mempool_limit.py | 14 +----- .../mining_prioritisetransaction.py | 50 ++++++++++--------- test/functional/test_framework/util.py | 40 ++++++--------- 3 files changed, 44 insertions(+), 60 deletions(-) diff --git a/test/functional/mempool_limit.py b/test/functional/mempool_limit.py index 645f1d759682..553c1cfef3a7 100755 --- a/test/functional/mempool_limit.py +++ b/test/functional/mempool_limit.py @@ -7,12 +7,12 @@ from decimal import Decimal from test_framework.blocktools import COINBASE_MATURITY -from test_framework.messages import COIN from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_equal, assert_greater_than, assert_raises_rpc_error, + create_lots_of_big_transactions, gen_return_txouts, ) from test_framework.wallet import MiniWallet @@ -29,16 +29,6 @@ def set_test_params(self): ]] self.supports_cli = False - def send_large_txs(self, node, miniwallet, txouts, fee, tx_batch_size): - for _ in range(tx_batch_size): - tx = miniwallet.create_self_transfer(fee_rate=0)['tx'] - for txout in txouts: - tx.vout.append(txout) - tx.vout[0].nValue -= int(fee * COIN) - res = node.testmempoolaccept([tx.serialize().hex()])[0] - assert_equal(res['fees']['base'], fee) - miniwallet.sendrawtransaction(from_node=node, tx_hex=tx.serialize().hex()) - def run_test(self): txouts = gen_return_txouts() node = self.nodes[0] @@ -71,7 +61,7 @@ def run_test(self): self.log.info("Fill up the mempool with txs with higher fee rate") for batch_of_txid in range(num_of_batches): fee = (batch_of_txid + 1) * base_fee - self.send_large_txs(node, miniwallet, txouts, fee, tx_batch_size) + create_lots_of_big_transactions(miniwallet, node, fee, tx_batch_size, txouts) self.log.info('The tx should be evicted by now') # The number of transactions created should be greater than the ones present in the mempool diff --git a/test/functional/mining_prioritisetransaction.py b/test/functional/mining_prioritisetransaction.py index b5b9506d81cd..6fb6e2d1bef6 100755 --- a/test/functional/mining_prioritisetransaction.py +++ b/test/functional/mining_prioritisetransaction.py @@ -6,29 +6,31 @@ from decimal import Decimal -from test_framework.blocktools import COINBASE_MATURITY -from test_framework.messages import COIN, MAX_BLOCK_SIZE +from test_framework.messages import ( + COIN, + MAX_BLOCK_SIZE, +) from test_framework.test_framework import BitcoinTestFramework -from test_framework.util import assert_equal, assert_raises_rpc_error, create_confirmed_utxos, create_lots_of_big_transactions, gen_return_txouts +from test_framework.util import ( + assert_equal, + assert_raises_rpc_error, + create_lots_of_big_transactions, + gen_return_txouts, +) from test_framework.wallet import MiniWallet class PrioritiseTransactionTest(BitcoinTestFramework): def set_test_params(self): - self.setup_clean_chain = True - self.num_nodes = 2 + self.num_nodes = 1 self.extra_args = [[ "-printpriority=1", "-datacarriersize=100000", ]] * self.num_nodes self.supports_cli = False - def skip_test_if_missing_module(self): - self.skip_if_no_wallet() - def test_diamond(self): self.log.info("Test diamond-shape package with priority") - self.generate(self.wallet, COINBASE_MATURITY + 1) self.nodes[0].setmocktime(self.mocktime) # tx_a @@ -83,7 +85,6 @@ def test_diamond(self): self.log.info("Test priority while txs are not in mempool") self.restart_node(0, extra_args=["-nopersistmempool"]) - self.connect_nodes(0, 1) self.nodes[0].setmocktime(self.mocktime) assert_equal(self.nodes[0].getmempoolinfo()["size"], 0) self.nodes[0].prioritisetransaction(txid=txid_b, fee_delta=int(fee_delta_b * COIN)) @@ -100,10 +101,10 @@ def test_diamond(self): # Use default extra_args self.restart_node(0) - self.connect_nodes(0, 1) def run_test(self): self.wallet = MiniWallet(self.nodes[0]) + self.wallet.rescan_utxos() # Test `prioritisetransaction` required parameters assert_raises_rpc_error(-1, "prioritisetransaction", self.nodes[0].prioritisetransaction) @@ -127,7 +128,10 @@ def run_test(self): self.relayfee = self.nodes[0].getnetworkinfo()['relayfee'] utxo_count = 90 - utxos = create_confirmed_utxos(self, self.relayfee, self.nodes[0], utxo_count) + utxos = self.wallet.send_self_transfer_multi(from_node=self.nodes[0], num_outputs=utxo_count)['new_utxos'] + self.generate(self.wallet, 1) + assert_equal(len(self.nodes[0].getrawmempool()), 0) + base_fee = self.relayfee*100 # our transactions are smaller than 100kb txids = [] @@ -137,7 +141,13 @@ def run_test(self): txids.append([]) start_range = i * range_size end_range = start_range + range_size - txids[i] = create_lots_of_big_transactions(self.nodes[0], self.txouts, utxos[start_range:end_range], end_range - start_range, (i+1)*base_fee) + txids[i] = create_lots_of_big_transactions( + self.wallet, + self.nodes[0], + (i+1) * base_fee, + end_range - start_range, + self.txouts, + utxos[start_range:end_range]) # Make sure that the size of each group of transactions exceeds # MAX_BLOCK_SIZE -- otherwise the test needs to be revised to @@ -196,17 +206,9 @@ def run_test(self): assert x not in mempool # Create a free transaction. Should be rejected. - utxo_list = self.nodes[0].listunspent() - assert len(utxo_list) > 0 - utxo = utxo_list[0] - - inputs = [] - outputs = {} - inputs.append({"txid" : utxo["txid"], "vout" : utxo["vout"]}) - outputs[self.nodes[0].getnewaddress()] = utxo["amount"] - raw_tx = self.nodes[0].createrawtransaction(inputs, outputs) - tx_hex = self.nodes[0].signrawtransactionwithwallet(raw_tx)["hex"] - tx_id = self.nodes[0].decoderawtransaction(tx_hex)["txid"] + tx_res = self.wallet.create_self_transfer(fee_rate=0) + tx_hex = tx_res['hex'] + tx_id = tx_res['txid'] # This will raise an exception due to min relay fee not being met assert_raises_rpc_error(-26, "min relay fee not met", self.nodes[0].sendrawtransaction, tx_hex) diff --git a/test/functional/test_framework/util.py b/test/functional/test_framework/util.py index bec80856001c..d31ab07f703b 100644 --- a/test/functional/test_framework/util.py +++ b/test/functional/test_framework/util.py @@ -614,24 +614,21 @@ def gen_return_txouts(): # Create a spend of each passed-in utxo, splicing in "txouts" to each raw # transaction to make it large. See gen_return_txouts() above. -def create_lots_of_big_transactions(node, txouts, utxos, num, fee): - addr = node.getnewaddress() +def create_lots_of_big_transactions(mini_wallet, node, fee, tx_batch_size, txouts, utxos=None): + from .messages import COIN + fee_sats = int(fee * COIN) txids = [] - from .messages import tx_from_hex - for _ in range(num): - t = utxos.pop() - inputs = [{"txid": t["txid"], "vout": t["vout"]}] - outputs = {} - change = t['amount'] - fee - outputs[addr] = satoshi_round(change) - rawtx = node.createrawtransaction(inputs, outputs) - tx = tx_from_hex(rawtx) - for txout in txouts: - tx.vout.append(txout) - newtx = tx.serialize().hex() - signresult = node.signrawtransactionwithwallet(newtx, None, "NONE") - txid = node.sendrawtransaction(signresult["hex"], 0) - txids.append(txid) + use_internal_utxos = utxos is None + for _ in range(tx_batch_size): + tx = mini_wallet.create_self_transfer( + utxo_to_spend=None if use_internal_utxos else utxos.pop(), + fee_rate=0, + )['tx'] + tx.vout[0].nValue -= fee_sats + tx.vout.extend(txouts) + res = node.testmempoolaccept([tx.serialize().hex()])[0] + assert_equal(res['fees']['base'], fee) + txids.append(node.sendrawtransaction(tx.serialize().hex())) return txids @@ -639,13 +636,8 @@ def mine_large_block(test_framework, mini_wallet, node): # generate a 66k transaction, # and 14 of them is close to the 1MB block limit txouts = gen_return_txouts() - from .messages import COIN - fee = 100 * int(node.getnetworkinfo()["relayfee"] * COIN) - for _ in range(14): - tx = mini_wallet.create_self_transfer(fee_rate=0)['tx'] - tx.vout[0].nValue -= fee - tx.vout.extend(txouts) - mini_wallet.sendrawtransaction(from_node=node, tx_hex=tx.serialize().hex()) + fee = 100 * node.getnetworkinfo()["relayfee"] + create_lots_of_big_transactions(mini_wallet, node, fee, 14, txouts) test_framework.generate(node, 1) From 9937fb86906b0a5769692931a84ed40976c4bf39 Mon Sep 17 00:00:00 2001 From: Kittywhiskers Van Gogh <63189531+kwvg@users.noreply.github.com> Date: Sat, 21 Jun 2025 21:54:49 +0000 Subject: [PATCH 09/14] merge bitcoin#20456: Fix intermittent issue in mempool_compatibility getmempoolinfo["loaded"] was introduced in bitcoin#15323 (dash#4609), shipped as part of v18.x but this version is also needed by bitcoin#26640 as v0.15 does not recognize "maxfeerate", which MiniWallet needs. We also need to delete the datadir of the v18.2.2 node to avoid problems that could arise from the EvoDb, the files needed for the test are managed by the test. --- test/functional/mempool_compatibility.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/test/functional/mempool_compatibility.py b/test/functional/mempool_compatibility.py index 91f08964d940..4980c360f086 100755 --- a/test/functional/mempool_compatibility.py +++ b/test/functional/mempool_compatibility.py @@ -11,6 +11,7 @@ """ import os +import shutil from test_framework.test_framework import BitcoinTestFramework from test_framework.wallet import MiniWallet @@ -26,13 +27,16 @@ def skip_test_if_missing_module(self): def setup_network(self): self.add_nodes(self.num_nodes, versions=[ - 150000, # oldest version supported by the test framework + 18020200, # oldest version with getmempoolinfo.loaded (used to avoid intermittent issues) None, ]) self.extra_args = [ [], [], ] + # Delete v18.2.2 cached datadir to avoid making a legacy version try to + # make sense of our current database formats + shutil.rmtree(os.path.join(self.nodes[0].datadir, self.chain)) self.start_nodes() self.import_deterministic_coinbase_privkeys() From 2c1e298ccfc68deade4afc156bad3726ee19049e Mon Sep 17 00:00:00 2001 From: Kittywhiskers Van Gogh <63189531+kwvg@users.noreply.github.com> Date: Mon, 5 Dec 2022 12:40:47 +0100 Subject: [PATCH 10/14] merge bitcoin#26640: Run mempool_compatibility.py with MiniWallet --- test/functional/mempool_compatibility.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/test/functional/mempool_compatibility.py b/test/functional/mempool_compatibility.py index 4980c360f086..b78aa3da91a6 100755 --- a/test/functional/mempool_compatibility.py +++ b/test/functional/mempool_compatibility.py @@ -20,7 +20,6 @@ class MempoolCompatibilityTest(BitcoinTestFramework): def set_test_params(self): self.num_nodes = 2 - self.wallet_names = [None] def skip_test_if_missing_module(self): self.skip_if_no_previous_releases() @@ -38,7 +37,6 @@ def setup_network(self): # make sense of our current database formats shutil.rmtree(os.path.join(self.nodes[0].datadir, self.chain)) self.start_nodes() - self.import_deterministic_coinbase_privkeys() def run_test(self): self.log.info("Test that mempool.dat is compatible between versions") @@ -52,11 +50,10 @@ def run_test(self): # unbroadcasted_tx won't pass old_node's `MemPoolAccept::PreChecks`. self.connect_nodes(0, 1) self.sync_blocks() - recipient = old_node.getnewaddress() self.stop_node(1) self.log.info("Add a transaction to mempool on old node and shutdown") - old_tx_hash = old_node.sendtoaddress(recipient, 0.0001) + old_tx_hash = new_wallet.send_self_transfer(from_node=old_node)["txid"] assert old_tx_hash in old_node.getrawmempool() self.stop_node(0) From 314c28a81dbe33fc4eb56e60e552913988cde070 Mon Sep 17 00:00:00 2001 From: Kittywhiskers Van Gogh <63189531+kwvg@users.noreply.github.com> Date: Wed, 18 Jun 2025 12:16:38 +0000 Subject: [PATCH 11/14] merge bitcoin#26892: simplify p2p_permissions.py by using MiniWallet --- test/functional/p2p_permissions.py | 25 ++++++------------------- 1 file changed, 6 insertions(+), 19 deletions(-) diff --git a/test/functional/p2p_permissions.py b/test/functional/p2p_permissions.py index 71ebfd38f9f1..53bf2e4f8a58 100755 --- a/test/functional/p2p_permissions.py +++ b/test/functional/p2p_permissions.py @@ -7,29 +7,27 @@ Test that permissions are correctly calculated and applied """ -from test_framework.address import ADDRESS_BCRT1_P2SH_OP_TRUE from test_framework.messages import ( - tx_from_hex, + SEQUENCE_FINAL, ) from test_framework.p2p import P2PDataStore -from test_framework.script import ( - CScript, - OP_TRUE, -) from test_framework.test_node import ErrorMatch from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_equal, p2p_port, ) +from test_framework.wallet import MiniWallet class P2PPermissionsTests(BitcoinTestFramework): def set_test_params(self): self.num_nodes = 2 - self.setup_clean_chain = True def run_test(self): + self.wallet = MiniWallet(self.nodes[0]) + self.wallet.rescan_utxos() + self.check_tx_relay() self.checkpermission( @@ -93,8 +91,6 @@ def run_test(self): self.nodes[1].assert_start_raises_init_error(["-whitebind=noban@127.0.0.1", "-bind=127.0.0.1", "-listen=0"], "Cannot set -bind or -whitebind together with -listen=0", match=ErrorMatch.PARTIAL_REGEX) def check_tx_relay(self): - block_op_true = self.nodes[0].getblock(self.generatetoaddress(self.nodes[0], 100, ADDRESS_BCRT1_P2SH_OP_TRUE)[0]) - self.log.debug("Create a connection from a forcerelay peer that rebroadcasts raw txs") # A test framework p2p connection is needed to send the raw transaction directly. If a full node was used, it could only # rebroadcast via the inv-getdata mechanism. However, even for forcerelay connections, a full node would @@ -103,16 +99,7 @@ def check_tx_relay(self): p2p_rebroadcast_wallet = self.nodes[1].add_p2p_connection(P2PDataStore()) self.log.debug("Send a tx from the wallet initially") - tx = tx_from_hex( - self.nodes[0].createrawtransaction( - inputs=[{ - 'txid': block_op_true['tx'][0], - 'vout': 0, - }], outputs=[{ - ADDRESS_BCRT1_P2SH_OP_TRUE: 5, - }]), - ) - tx.vin[0].scriptSig = CScript([CScript([OP_TRUE])]) + tx = self.wallet.create_self_transfer(sequence=SEQUENCE_FINAL)['tx'] txid = tx.rehash() self.log.debug("Wait until tx is in node[1]'s mempool") From 2f7b3b542a14467bb46668bc22094aee4d2a010a Mon Sep 17 00:00:00 2001 From: Kittywhiskers Van Gogh <63189531+kwvg@users.noreply.github.com> Date: Fri, 7 Oct 2022 15:05:34 +0200 Subject: [PATCH 12/14] merge bitcoin#26280: Return coinbase flag in scantxoutset needed by d0a909ae in bitcoin#26657 --- src/rpc/blockchain.cpp | 2 ++ test/functional/rpc_scantxoutset.py | 3 +++ 2 files changed, 5 insertions(+) diff --git a/src/rpc/blockchain.cpp b/src/rpc/blockchain.cpp index 0888896dd636..26ced62a4ec6 100644 --- a/src/rpc/blockchain.cpp +++ b/src/rpc/blockchain.cpp @@ -2422,6 +2422,7 @@ static RPCHelpMan scantxoutset() {RPCResult::Type::STR_HEX, "scriptPubKey", "The script key"}, {RPCResult::Type::STR, "desc", "A specialized descriptor for the matched scriptPubKey"}, {RPCResult::Type::STR_AMOUNT, "amount", "The total amount in " + CURRENCY_UNIT + " of the unspent output"}, + {RPCResult::Type::BOOL, "coinbase", "Whether this is a coinbase output"}, {RPCResult::Type::NUM, "height", "Height of the unspent transaction output"}, }}, }}, @@ -2525,6 +2526,7 @@ static RPCHelpMan scantxoutset() unspent.pushKV("scriptPubKey", HexStr(txo.scriptPubKey)); unspent.pushKV("desc", descriptors[txo.scriptPubKey]); unspent.pushKV("amount", ValueFromAmount(txo.nValue)); + unspent.pushKV("coinbase", coin.IsCoinBase()); unspent.pushKV("height", (int32_t)coin.nHeight); unspents.push_back(unspent); diff --git a/test/functional/rpc_scantxoutset.py b/test/functional/rpc_scantxoutset.py index 14c0999045e5..da0d94e36ed3 100755 --- a/test/functional/rpc_scantxoutset.py +++ b/test/functional/rpc_scantxoutset.py @@ -32,6 +32,9 @@ def run_test(self): self.wallet = MiniWallet(self.nodes[0]) self.wallet.rescan_utxos() + self.log.info("Test if we find coinbase outputs.") + assert_equal(sum(u["coinbase"] for u in self.nodes[0].scantxoutset("start", [self.wallet.get_descriptor()])["unspents"]), 49) + self.log.info("Create UTXOs...") pubk1, spk1, addr1 = getnewdestination("legacy") pubk2, spk2, addr2 = getnewdestination("legacy") From ace58fd69c54f146f6490393f338c663de614581 Mon Sep 17 00:00:00 2001 From: Kittywhiskers Van Gogh <63189531+kwvg@users.noreply.github.com> Date: Mon, 12 Dec 2022 18:22:28 +0800 Subject: [PATCH 13/14] merge bitcoin#26657: Run feature_bip68_sequence.py with MiniWallet --- test/functional/feature_bip68_sequence.py | 104 +++++++++------------- test/functional/test_framework/wallet.py | 79 +++++++++------- 2 files changed, 92 insertions(+), 91 deletions(-) diff --git a/test/functional/feature_bip68_sequence.py b/test/functional/feature_bip68_sequence.py index 9ad7e65c7349..34cb66e893d5 100755 --- a/test/functional/feature_bip68_sequence.py +++ b/test/functional/feature_bip68_sequence.py @@ -24,6 +24,7 @@ softfork_active, ) from test_framework.script_util import DUMMY_P2SH_SCRIPT +from test_framework.wallet import MiniWallet SEQUENCE_LOCKTIME_DISABLE_FLAG = (1<<31) SEQUENCE_LOCKTIME_TYPE_FLAG = (1<<22) # this means use time (0 means height) @@ -47,14 +48,12 @@ def set_test_params(self): ], ] - def skip_test_if_missing_module(self): - self.skip_if_no_wallet() - def run_test(self): self.relayfee = self.nodes[0].getnetworkinfo()["relayfee"] + self.wallet = MiniWallet(self.nodes[0]) # Generate some coins - self.generate(self.nodes[0], 110) + self.generate(self.wallet, 110) self.log.info("Running test disable flag") self.test_disable_flag() @@ -81,16 +80,10 @@ def run_test(self): # the first sequence bit is set. def test_disable_flag(self): # Create some unconfirmed inputs - new_addr = self.nodes[0].getnewaddress() - self.nodes[0].sendtoaddress(new_addr, 2) # send 2 BTC - - utxos = self.nodes[0].listunspent(0, 0) - assert len(utxos) > 0 - - utxo = utxos[0] + utxo = self.wallet.send_self_transfer(from_node=self.nodes[0])["new_utxo"] tx1 = CTransaction() - value = int((utxo["amount"] - self.relayfee) * COIN) + value = int((utxo["value"] - self.relayfee) * COIN) # Check that the disable flag disables relative locktime. # If sequence locks were used, this would require 1 block for the @@ -99,8 +92,8 @@ def test_disable_flag(self): tx1.vin = [CTxIn(COutPoint(int(utxo["txid"], 16), utxo["vout"]), nSequence=sequence_value)] tx1.vout = [CTxOut(value, DUMMY_P2SH_SCRIPT)] - tx1_signed = self.nodes[0].signrawtransactionwithwallet(tx1.serialize().hex())["hex"] - tx1_id = self.nodes[0].sendrawtransaction(tx1_signed) + self.wallet.sign_tx(tx=tx1) + tx1_id = self.wallet.sendrawtransaction(from_node=self.nodes[0], tx_hex=tx1.serialize().hex()) tx1_id = int(tx1_id, 16) # This transaction will enable sequence-locks, so this transaction should @@ -112,13 +105,13 @@ def test_disable_flag(self): tx2.vout = [CTxOut(int(value - self.relayfee * COIN), DUMMY_P2SH_SCRIPT)] tx2.rehash() - assert_raises_rpc_error(-26, NOT_FINAL_ERROR, self.nodes[0].sendrawtransaction, tx2.serialize().hex()) + assert_raises_rpc_error(-26, NOT_FINAL_ERROR, self.wallet.sendrawtransaction, from_node=self.nodes[0], tx_hex=tx2.serialize().hex()) # Setting the version back down to 1 should disable the sequence lock, # so this should be accepted. tx2.nVersion = 1 - self.nodes[0].sendrawtransaction(tx2.serialize().hex()) + self.wallet.sendrawtransaction(from_node=self.nodes[0], tx_hex=tx2.serialize().hex()) # Calculate the median time past of a prior block ("confirmations" before # the current tip). @@ -131,20 +124,13 @@ def test_sequence_lock_confirmed_inputs(self): # Create lots of confirmed utxos, and use them to generate lots of random # transactions. max_outputs = 50 - addresses = [] - while len(addresses) < max_outputs: - addresses.append(self.nodes[0].getnewaddress()) - while len(self.nodes[0].listunspent()) < 200: + while len(self.wallet.get_utxos(include_immature_coinbase=False, mark_as_spent=False)) < 200: import random - random.shuffle(addresses) num_outputs = random.randint(1, max_outputs) - outputs = {} - for i in range(num_outputs): - outputs[addresses[i]] = random.randint(1, 20)*0.01 - self.nodes[0].sendmany("", outputs) - self.generate(self.nodes[0], 1) + self.wallet.send_self_transfer_multi(from_node=self.nodes[0], num_outputs=num_outputs) + self.generate(self.wallet, 1) - utxos = self.nodes[0].listunspent() + utxos = self.wallet.get_utxos(include_immature_coinbase=False) # Try creating a lot of random transactions. # Each time, choose a random number of inputs, and randomly set @@ -201,19 +187,20 @@ def test_sequence_lock_confirmed_inputs(self): sequence_value = ((cur_time - orig_time) >> SEQUENCE_LOCKTIME_GRANULARITY)+1 sequence_value |= SEQUENCE_LOCKTIME_TYPE_FLAG tx.vin.append(CTxIn(COutPoint(int(utxos[j]["txid"], 16), utxos[j]["vout"]), nSequence=sequence_value)) - value += utxos[j]["amount"]*COIN + value += utxos[j]["value"]*COIN # Overestimate the size of the tx - signatures should be less than 120 bytes, and leave 50 for the output tx_size = len(tx.serialize().hex())//2 + 120*num_inputs + 50 tx.vout.append(CTxOut(int(value-self.relayfee*tx_size*COIN/1000), DUMMY_P2SH_SCRIPT)) - rawtx = self.nodes[0].signrawtransactionwithwallet(tx.serialize().hex())["hex"] + self.wallet.sign_tx(tx=tx) if (using_sequence_locks and not should_pass): # This transaction should be rejected - assert_raises_rpc_error(-26, NOT_FINAL_ERROR, self.nodes[0].sendrawtransaction, rawtx) + assert_raises_rpc_error(-26, NOT_FINAL_ERROR, self.wallet.sendrawtransaction, from_node=self.nodes[0], tx_hex=tx.serialize().hex()) else: # This raw transaction should be accepted - self.nodes[0].sendrawtransaction(rawtx) - utxos = self.nodes[0].listunspent() + self.wallet.sendrawtransaction(from_node=self.nodes[0], tx_hex=tx.serialize().hex()) + self.wallet.rescan_utxos() + utxos = self.wallet.get_utxos(include_immature_coinbase=False) # Test that sequence locks on unconfirmed inputs must have nSequence # height or time of 0 to be accepted. @@ -224,8 +211,8 @@ def test_sequence_lock_unconfirmed_inputs(self): cur_height = self.nodes[0].getblockcount() # Create a mempool tx. - txid = self.nodes[0].sendtoaddress(self.nodes[0].getnewaddress(), 2) - tx1 = tx_from_hex(self.nodes[0].getrawtransaction(txid)) + self.wallet.rescan_utxos() + tx1 = self.wallet.send_self_transfer(from_node=self.nodes[0])["tx"] tx1.rehash() # Anyone-can-spend mempool tx. @@ -234,11 +221,11 @@ def test_sequence_lock_unconfirmed_inputs(self): tx2.nVersion = 2 tx2.vin = [CTxIn(COutPoint(tx1.sha256, 0), nSequence=0)] tx2.vout = [CTxOut(int(tx1.vout[0].nValue - self.relayfee*COIN), DUMMY_P2SH_SCRIPT)] - tx2_raw = self.nodes[0].signrawtransactionwithwallet(tx2.serialize().hex())["hex"] - tx2 = tx_from_hex(tx2_raw) + self.wallet.sign_tx(tx=tx2) + tx2_raw = tx2.serialize().hex() tx2.rehash() - self.nodes[0].sendrawtransaction(tx2_raw) + self.wallet.sendrawtransaction(from_node=self.nodes[0], tx_hex=tx2_raw) # Create a spend of the 0th output of orig_tx with a sequence lock # of 1, and test what happens when submitting. @@ -256,10 +243,10 @@ def test_nonzero_locks(orig_tx, node, relayfee, use_height_lock): if (orig_tx.hash in node.getrawmempool()): # sendrawtransaction should fail if the tx is in the mempool - assert_raises_rpc_error(-26, NOT_FINAL_ERROR, node.sendrawtransaction, tx.serialize().hex()) + assert_raises_rpc_error(-26, NOT_FINAL_ERROR, self.wallet.sendrawtransaction, from_node=node, tx_hex=tx.serialize().hex()) else: # sendrawtransaction should succeed if the tx is not in the mempool - node.sendrawtransaction(tx.serialize().hex()) + self.wallet.sendrawtransaction(from_node=node, tx_hex=tx.serialize().hex()) return tx @@ -272,7 +259,7 @@ def test_nonzero_locks(orig_tx, node, relayfee, use_height_lock): cur_time = self.mocktime for _ in range(10): self.nodes[0].setmocktime(cur_time + 600) - self.generate(self.nodes[0], 1, sync_fun=self.no_op) + self.generate(self.wallet, 1, sync_fun=self.no_op) cur_time += 600 assert tx2.hash in self.nodes[0].getrawmempool() @@ -306,12 +293,12 @@ def test_nonzero_locks(orig_tx, node, relayfee, use_height_lock): tx5 = test_nonzero_locks(tx4, self.nodes[0], self.relayfee, use_height_lock=True) assert tx5.hash not in self.nodes[0].getrawmempool() - utxos = self.nodes[0].listunspent() - tx5.vin.append(CTxIn(COutPoint(int(utxos[0]["txid"], 16), utxos[0]["vout"]), nSequence=1)) - tx5.vout[0].nValue += int(utxos[0]["amount"]*COIN) - raw_tx5 = self.nodes[0].signrawtransactionwithwallet(tx5.serialize().hex())["hex"] + utxo = self.wallet.get_utxo() + tx5.vin.append(CTxIn(COutPoint(int(utxo["txid"], 16), utxo["vout"]), nSequence=1)) + tx5.vout[0].nValue += int(utxo["value"]*COIN) + self.wallet.sign_tx(tx=tx5) - assert_raises_rpc_error(-26, NOT_FINAL_ERROR, self.nodes[0].sendrawtransaction, raw_tx5) + assert_raises_rpc_error(-26, NOT_FINAL_ERROR, self.wallet.sendrawtransaction, from_node=self.nodes[0], tx_hex=tx5.serialize().hex()) # Test mempool-BIP68 consistency after reorg # @@ -347,7 +334,7 @@ def test_nonzero_locks(orig_tx, node, relayfee, use_height_lock): # Reset the chain and get rid of the mocktimed-blocks self.nodes[0].setmocktime(self.mocktime) self.nodes[0].invalidateblock(self.nodes[0].getblockhash(cur_height+1)) - self.generate(self.nodes[0], 10, sync_fun=self.no_op) + self.generate(self.wallet, 10, sync_fun=self.no_op) # Make sure that BIP68 isn't being used to validate blocks prior to # activation height. If more blocks are mined prior to this test @@ -355,9 +342,8 @@ def test_nonzero_locks(orig_tx, node, relayfee, use_height_lock): # this test should be moved to run earlier, or deleted. def test_bip68_not_consensus(self): assert not softfork_active(self.nodes[0], 'csv') - txid = self.nodes[0].sendtoaddress(self.nodes[0].getnewaddress(), 2) - tx1 = tx_from_hex(self.nodes[0].getrawtransaction(txid)) + tx1 = self.wallet.send_self_transfer(from_node=self.nodes[0])["tx"] tx1.rehash() # Make an anyone-can-spend transaction @@ -367,11 +353,12 @@ def test_bip68_not_consensus(self): tx2.vout = [CTxOut(int(tx1.vout[0].nValue - self.relayfee*COIN), DUMMY_P2SH_SCRIPT)] # sign tx2 - tx2_raw = self.nodes[0].signrawtransactionwithwallet(tx2.serialize().hex())["hex"] + self.wallet.sign_tx(tx=tx2) + tx2_raw = tx2.serialize().hex() tx2 = tx_from_hex(tx2_raw) tx2.rehash() - self.nodes[0].sendrawtransaction(tx2.serialize().hex()) + self.wallet.sendrawtransaction(from_node=self.nodes[0], tx_hex=tx2_raw) # Now make an invalid spend of tx2 according to BIP68 sequence_value = 100 # 100 block relative locktime @@ -382,7 +369,7 @@ def test_bip68_not_consensus(self): tx3.vout = [CTxOut(int(tx2.vout[0].nValue - self.relayfee * COIN), DUMMY_P2SH_SCRIPT)] tx3.rehash() - assert_raises_rpc_error(-26, NOT_FINAL_ERROR, self.nodes[0].sendrawtransaction, tx3.serialize().hex()) + assert_raises_rpc_error(-26, NOT_FINAL_ERROR, self.wallet.sendrawtransaction, from_node=self.nodes[0], tx_hex=tx3.serialize().hex()) # make a block that violates bip68; ensure that the tip updates block = create_block(tmpl=self.nodes[0].getblocktemplate(NORMAL_GBT_REQUEST_PARAMS)) @@ -399,22 +386,19 @@ def activateCSV(self): min_activation_height = 432 height = self.nodes[0].getblockcount() assert_greater_than(min_activation_height - height, 2) - self.generate(self.nodes[0], min_activation_height - height - 2, sync_fun=self.no_op) + self.generate(self.wallet, min_activation_height - height - 2, sync_fun=self.no_op) assert not softfork_active(self.nodes[0], 'csv') - self.generate(self.nodes[0], 1, sync_fun=self.no_op) + self.generate(self.wallet, 1, sync_fun=self.no_op) assert softfork_active(self.nodes[0], 'csv') self.sync_blocks() # Use self.nodes[1] to test that version 2 transactions are standard. def test_version2_relay(self): - inputs = [ ] - outputs = { self.nodes[1].getnewaddress() : 1.0 } - rawtx = self.nodes[1].createrawtransaction(inputs, outputs) - rawtxfund = self.nodes[1].fundrawtransaction(rawtx)['hex'] - tx = tx_from_hex(rawtxfund) + mini_wallet = MiniWallet(self.nodes[1]) + mini_wallet.rescan_utxos() + tx = mini_wallet.create_self_transfer()["tx"] tx.nVersion = 2 - tx_signed = self.nodes[1].signrawtransactionwithwallet(tx.serialize().hex())["hex"] - self.nodes[1].sendrawtransaction(tx_signed) + mini_wallet.sendrawtransaction(from_node=self.nodes[1], tx_hex=tx.serialize().hex()) if __name__ == '__main__': BIP68Test().main() diff --git a/test/functional/test_framework/wallet.py b/test/functional/test_framework/wallet.py index b0cefa547345..99cd11c7a812 100644 --- a/test/functional/test_framework/wallet.py +++ b/test/functional/test_framework/wallet.py @@ -45,6 +45,7 @@ assert_equal, assert_greater_than_or_equal, ) +from test_framework.blocktools import COINBASE_MATURITY DEFAULT_FEE = Decimal("0.0001") @@ -90,19 +91,31 @@ def __init__(self, test_node, *, mode=MiniWalletMode.ADDRESS_OP_TRUE): self._address = ADDRESS_BCRT1_P2SH_OP_TRUE self._scriptPubKey = bytes.fromhex(self._test_node.validateaddress(self._address)['scriptPubKey']) - def _create_utxo(self, *, txid, vout, value, height): - return {"txid": txid, "vout": vout, "value": value, "height": height} + def _create_utxo(self, *, txid, vout, value, height, coinbase, confirmations): + return {"txid": txid, "vout": vout, "value": value, "height": height, "coinbase": coinbase, "confirmations": confirmations} def get_balance(self): return sum(u['value'] for u in self._utxos) - def rescan_utxos(self): + def rescan_utxos(self, *, include_mempool=True): """Drop all utxos and rescan the utxo set""" self._utxos = [] res = self._test_node.scantxoutset(action="start", scanobjects=[self.get_descriptor()]) assert_equal(True, res['success']) for utxo in res['unspents']: - self._utxos.append(self._create_utxo(txid=utxo["txid"], vout=utxo["vout"], value=utxo["amount"], height=utxo["height"])) + self._utxos.append( + self._create_utxo(txid=utxo["txid"], + vout=utxo["vout"], + value=utxo["amount"], + height=utxo["height"], + coinbase=utxo["coinbase"], + confirmations=res["height"] - utxo["height"] + 1)) + if include_mempool: + mempool = self._test_node.getrawmempool(verbose=True) + # Sort tx by ancestor count. See BlockAssembler::SortForBlock in src/node/miner.cpp + sorted_mempool = sorted(mempool.items(), key=lambda item: (item[1]["ancestorcount"], int(item[0], 16))) + for txid, _ in sorted_mempool: + self.scan_tx(self._test_node.getrawtransaction(txid=txid, verbose=True)) def scan_tx(self, tx): """Scan the tx and adjust the internal list of owned utxos""" @@ -117,23 +130,30 @@ def scan_tx(self, tx): pass for out in tx['vout']: if out['scriptPubKey']['hex'] == self._scriptPubKey.hex(): - self._utxos.append(self._create_utxo(txid=tx["txid"], vout=out["n"], value=out["value"], height=0)) + self._utxos.append(self._create_utxo(txid=tx["txid"], vout=out["n"], value=out["value"], height=0, coinbase=False, confirmations=0)) def sign_tx(self, tx, fixed_length=True): - """Sign tx that has been created by MiniWallet in P2PK mode""" - assert_equal(self._mode, MiniWalletMode.RAW_P2PK) - (sighash, err) = SignatureHash(CScript(self._scriptPubKey), tx, 0, SIGHASH_ALL) - assert err is None - # for exact fee calculation, create only signatures with fixed size by default (>49.89% probability): - # 65 bytes: high-R val (33 bytes) + low-S val (32 bytes) - # with the DER header/skeleton data of 6 bytes added, this leads to a target size of 71 bytes - der_sig = b'' - while not len(der_sig) == 71: - der_sig = self._priv_key.sign_ecdsa(sighash) - if not fixed_length: - break - tx.vin[0].scriptSig = CScript([der_sig + bytes(bytearray([SIGHASH_ALL]))]) - tx.rehash() + if self._mode == MiniWalletMode.RAW_P2PK: + (sighash, err) = SignatureHash(CScript(self._scriptPubKey), tx, 0, SIGHASH_ALL) + assert err is None + # for exact fee calculation, create only signatures with fixed size by default (>49.89% probability): + # 65 bytes: high-R val (33 bytes) + low-S val (32 bytes) + # with the DER header/skeleton data of 6 bytes added, this leads to a target size of 71 bytes + der_sig = b'' + while not len(der_sig) == 71: + der_sig = self._priv_key.sign_ecdsa(sighash) + if not fixed_length: + break + tx.vin[0].scriptSig = CScript([der_sig + bytes(bytearray([SIGHASH_ALL]))]) + tx.rehash() + elif self._mode == MiniWalletMode.RAW_OP_TRUE: + for i in range(len(tx.vin)): + tx.vin[i].scriptSig = CScript([OP_NOP] * 24) # pad to identical size + elif self._mode == MiniWalletMode.ADDRESS_OP_TRUE: + for i in range(len(tx.vin)): + tx.vin[i].scriptSig = CScript([CScript([OP_TRUE])]) + else: + assert False def generate(self, num_blocks, **kwargs): """Generate blocks with coinbase outputs to the internal address, and call rescan_utxos""" @@ -178,9 +198,13 @@ def get_utxo(self, *, txid: str = '', vout: Optional[int] = None, mark_as_spent= else: return self._utxos[index] - def get_utxos(self, *, mark_as_spent=True): + def get_utxos(self, *, include_immature_coinbase=False, mark_as_spent=True): """Returns the list of all utxos and optionally mark them as spent""" - utxos = deepcopy(self._utxos) + if not include_immature_coinbase: + utxo_filter = filter(lambda utxo: not utxo['coinbase'] or COINBASE_MATURITY <= utxo['confirmations'], self._utxos) + else: + utxo_filter = self._utxos + utxos = deepcopy(list(utxo_filter)) if mark_as_spent: self._utxos = [] return utxos @@ -246,16 +270,7 @@ def create_self_transfer_multi( tx.vout = [CTxOut(amount_per_output, bytearray(self._scriptPubKey)) for _ in range(num_outputs)] tx.nLockTime = locktime - if self._mode == MiniWalletMode.RAW_P2PK: - self.sign_tx(tx) - elif self._mode == MiniWalletMode.RAW_OP_TRUE: - for i in range(len(utxos_to_spend)): - tx.vin[i].scriptSig = CScript([OP_NOP] * 24) # pad to identical size - elif self._mode == MiniWalletMode.ADDRESS_OP_TRUE: - for i in range(len(utxos_to_spend)): - tx.vin[i].scriptSig = CScript([CScript([OP_TRUE])]) - else: - assert False + self.sign_tx(tx) txid = tx.rehash() return { @@ -264,6 +279,8 @@ def create_self_transfer_multi( vout=i, value=Decimal(tx.vout[i].nValue) / COIN, height=0, + coinbase=False, + confirmations=0, ) for i in range(len(tx.vout))], "txid": txid, "hex": tx.serialize().hex(), From 54740ae4c7f926edabadc4c5b936c3bcb16a8140 Mon Sep 17 00:00:00 2001 From: Kittywhiskers Van Gogh <63189531+kwvg@users.noreply.github.com> Date: Thu, 19 Jan 2023 16:08:55 +0100 Subject: [PATCH 14/14] merge bitcoin#26923: simplify p2p_{tx_download,eviction}.py by using MiniWallet Running `rescan_utxos()` is necessary to ensure that funds are visible during the tests. --- test/functional/p2p_eviction.py | 33 +++++++++++------------------- test/functional/p2p_tx_download.py | 23 +++++++-------------- 2 files changed, 19 insertions(+), 37 deletions(-) diff --git a/test/functional/p2p_eviction.py b/test/functional/p2p_eviction.py index a8c1eb425e56..e829d4e30f13 100755 --- a/test/functional/p2p_eviction.py +++ b/test/functional/p2p_eviction.py @@ -12,10 +12,12 @@ the same local address. See Issue #14210 for more info. Therefore, this test is limited to the remaining protection criteria. """ - import time -from test_framework.blocktools import COINBASE_MATURITY, create_block, create_coinbase +from test_framework.blocktools import ( + create_block, + create_coinbase, +) from test_framework.p2p import ( P2PDataStore, P2PInterface, @@ -25,23 +27,23 @@ from test_framework.messages import ( msg_pong, msg_tx, - tx_from_hex, ) - +from test_framework.wallet import MiniWallet class SlowP2PDataStore(P2PDataStore): def on_ping(self, message): time.sleep(0.1) self.send_message(msg_pong(message.nonce)) + class SlowP2PInterface(P2PInterface): def on_ping(self, message): time.sleep(0.1) self.send_message(msg_pong(message.nonce)) + class P2PEvict(BitcoinTestFramework): def set_test_params(self): - self.setup_clean_chain = True self.disable_mocktime = True self.num_nodes = 1 # The choice of maxconnections=32 results in a maximum of 21 inbound connections @@ -53,7 +55,8 @@ def run_test(self): protected_peers = set() # peers that we expect to be protected from eviction current_peer = -1 node = self.nodes[0] - self.generatetoaddress(node, COINBASE_MATURITY + 1, node.get_deterministic_priv_key().address) + self.wallet = MiniWallet(node) + self.wallet.rescan_utxos() self.log.info("Create 4 peers and protect them from eviction by sending us a block") for _ in range(4): @@ -79,21 +82,8 @@ def run_test(self): current_peer += 1 txpeer.sync_with_ping() - prevtx = node.getblock(node.getblockhash(i + 1), 2)['tx'][0] - rawtx = node.createrawtransaction( - inputs=[{'txid': prevtx['txid'], 'vout': 0}], - outputs=[{node.get_deterministic_priv_key().address: 50 - 0.00125}], - ) - sigtx = node.signrawtransactionwithkey( - hexstring=rawtx, - privkeys=[node.get_deterministic_priv_key().key], - prevtxs=[{ - 'txid': prevtx['txid'], - 'vout': 0, - 'scriptPubKey': prevtx['vout'][0]['scriptPubKey']['hex'], - }], - )['hex'] - txpeer.send_message(msg_tx(tx_from_hex(sigtx))) + tx = self.wallet.create_self_transfer()['tx'] + txpeer.send_message(msg_tx(tx)) protected_peers.add(current_peer) self.log.info("Create 8 peers and protect them from eviction by having faster pings") @@ -133,5 +123,6 @@ def run_test(self): self.log.debug("{} protected peers: {}".format(len(protected_peers), protected_peers)) assert evicted_peers[0] not in protected_peers + if __name__ == '__main__': P2PEvict().main() diff --git a/test/functional/p2p_tx_download.py b/test/functional/p2p_tx_download.py index c0d077a94172..957305f2299f 100755 --- a/test/functional/p2p_tx_download.py +++ b/test/functional/p2p_tx_download.py @@ -12,7 +12,6 @@ MSG_TYPE_MASK, msg_inv, msg_notfound, - tx_from_hex, ) from test_framework.p2p import ( P2PInterface, @@ -22,7 +21,7 @@ from test_framework.util import ( assert_equal, ) -from test_framework.address import ADDRESS_BCRT1_UNSPENDABLE +from test_framework.wallet import MiniWallet class TestP2PConn(P2PInterface): @@ -80,19 +79,8 @@ def getdata_found(peer_index): def test_inv_block(self): self.log.info("Generate a transaction on node 0") - tx = self.nodes[0].createrawtransaction( - inputs=[{ # coinbase - "txid": self.nodes[0].getblock(self.nodes[0].getblockhash(1))['tx'][0], - "vout": 0 - }], - outputs={ADDRESS_BCRT1_UNSPENDABLE: 500 - 0.00025}, - ) - tx = self.nodes[0].signrawtransactionwithkey( - hexstring=tx, - privkeys=[self.nodes[0].get_deterministic_priv_key().key], - )['hex'] - ctx = tx_from_hex(tx) - txid = int(ctx.rehash(), 16) + tx = self.wallet.create_self_transfer() + txid = int(tx['txid'], 16) self.log.info( "Announce the transaction to all nodes from all {} incoming peers, but never send it".format(NUM_INBOUND)) @@ -103,7 +91,7 @@ def test_inv_block(self): self.bump_mocktime(1) self.log.info("Put the tx in node 0's mempool") - self.nodes[0].sendrawtransaction(tx) + self.nodes[0].sendrawtransaction(tx['hex']) # Since node 1 is connected outbound to an honest peer (node 0), it # should get the tx within a timeout. (Assuming that node 0 @@ -155,6 +143,9 @@ def test_spurious_notfound(self): self.nodes[0].p2ps[0].send_message(msg_notfound(vec=[CInv(1, 1)])) def run_test(self): + self.wallet = MiniWallet(self.nodes[0]) + self.wallet.rescan_utxos() + # Run each test against new bitcoind instances, as setting mocktimes has long-term effects on when # the next trickle relay event happens. for test in [self.test_spurious_notfound, self.test_in_flight_max, self.test_inv_block, self.test_tx_requests]: