[tests] Move stop_node and start_node methods to BitcoinTestFramework

This commit moves functions start_node, start_nodes, stop_node and
stop_nodes functions into the BitcoinTestFramework class. It also moves
the bitcoind_processes dict and coverage variables into BitcoinTestFramework.

test/functional/blockchain.py

@@ -21,15 +21,13 @@ from decimal import Decimal
 import http.client
 import subprocess
 
-from test_framework.test_framework import BitcoinTestFramework
+from test_framework.test_framework import (BitcoinTestFramework, BITCOIND_PROC_WAIT_TIMEOUT)
 from test_framework.util import (
     assert_equal,
     assert_raises,
     assert_raises_jsonrpc,
     assert_is_hex_string,
     assert_is_hash_string,
-    bitcoind_processes,
-    BITCOIND_PROC_WAIT_TIMEOUT,
 )
 
 
@@ -141,13 +139,13 @@ class BlockchainTest(BitcoinTestFramework):
         self.nodes[0].generate(6)
         assert_equal(self.nodes[0].getblockcount(), 206)
         self.log.debug('Node should not stop at this height')
-        assert_raises(subprocess.TimeoutExpired, lambda: bitcoind_processes[0].wait(timeout=3))
+        assert_raises(subprocess.TimeoutExpired, lambda: self.bitcoind_processes[0].wait(timeout=3))
         try:
             self.nodes[0].generate(1)
         except (ConnectionError, http.client.BadStatusLine):
             pass  # The node already shut down before response
         self.log.debug('Node should stop at this height...')
-        bitcoind_processes[0].wait(timeout=BITCOIND_PROC_WAIT_TIMEOUT)
+        self.bitcoind_processes[0].wait(timeout=BITCOIND_PROC_WAIT_TIMEOUT)
         self.nodes[0] = self.start_node(0, self.options.tmpdir)
         assert_equal(self.nodes[0].getblockcount(), 207)
 

test/functional/bumpfee.py

@@ -42,7 +42,7 @@ class BumpFeeTest(BitcoinTestFramework):
 
         # Encrypt wallet for test_locked_wallet_fails test
         self.nodes[1].encryptwallet(WALLET_PASSPHRASE)
-        bitcoind_processes[1].wait()
+        self.bitcoind_processes[1].wait()
         self.nodes[1] = self.start_node(1, self.options.tmpdir, extra_args[1])
         self.nodes[1].walletpassphrase(WALLET_PASSPHRASE, WALLET_PASSPHRASE_TIMEOUT)
 

test/functional/dbcrash.py

@@ -88,7 +88,7 @@ class ChainstateWriteCrashTest(BitcoinTestFramework):
                 # An exception here should mean the node is about to crash.
                 # If bitcoind exits, then try again.  wait_for_node_exit()
                 # should raise an exception if bitcoind doesn't exit.
-                wait_for_node_exit(node_index, timeout=10)
+                self.wait_for_node_exit(node_index, timeout=10)
             self.crashed_on_restart += 1
             time.sleep(1)
 
@@ -140,7 +140,7 @@ class ChainstateWriteCrashTest(BitcoinTestFramework):
                 if not self.submit_block_catch_error(i, block):
                     # TODO: more carefully check that the crash is due to -dbcrashratio
                     # (change the exit code perhaps, and check that here?)
-                    wait_for_node_exit(i, timeout=30)
+                    self.wait_for_node_exit(i, timeout=30)
                     self.log.debug("Restarting node %d after block hash %s", i, block_hash)
                     nodei_utxo_hash = self.restart_node(i, block_hash)
                     assert nodei_utxo_hash is not None

test/functional/fundrawtransaction.py

@@ -4,7 +4,7 @@
 # file COPYING or http://www.opensource.org/licenses/mit-license.php.
 """Test the fundrawtransaction RPC."""
 
-from test_framework.test_framework import BitcoinTestFramework
+from test_framework.test_framework import BitcoinTestFramework, BITCOIND_PROC_WAIT_TIMEOUT
 from test_framework.util import *
 
 
@@ -452,7 +452,7 @@ class RawTransactionsTest(BitcoinTestFramework):
         self.stop_node(2)
         self.stop_node(3)
         self.nodes[1].encryptwallet("test")
-        bitcoind_processes[1].wait(timeout=BITCOIND_PROC_WAIT_TIMEOUT)
+        self.bitcoind_processes[1].wait(timeout=BITCOIND_PROC_WAIT_TIMEOUT)
 
         self.nodes = self.start_nodes(self.num_nodes, self.options.tmpdir)
         # This test is not meant to test fee estimation and we'd like

test/functional/keypool.py

@@ -18,7 +18,7 @@ class KeyPoolTest(BitcoinTestFramework):
         
         # Encrypt wallet and wait to terminate
         nodes[0].encryptwallet('test')
-        bitcoind_processes[0].wait()
+        self.bitcoind_processes[0].wait()
         # Restart node 0
         nodes[0] = self.start_node(0, self.options.tmpdir)
         # Keep creating keys

test/functional/rpcbind_test.py

@@ -37,7 +37,7 @@ class RPCBindTest(BitcoinTestFramework):
             base_args += ['-rpcallowip=' + x for x in allow_ips]
         binds = ['-rpcbind='+addr for addr in addresses]
         self.nodes = self.start_nodes(self.num_nodes, self.options.tmpdir, [base_args + binds], connect_to)
-        pid = bitcoind_processes[0].pid
+        pid = self.bitcoind_processes[0].pid
         assert_equal(set(get_bind_addrs(pid)), set(expected))
         self.stop_nodes()
 

test/functional/test_framework/test_framework.py

@@ -5,7 +5,9 @@
 """Base class for RPC testing."""
 
 from collections import deque
+import errno
 from enum import Enum
+import http.client
 import logging
 import optparse
 import os
@@ -16,15 +18,16 @@ import tempfile
 import time
 import traceback
 
+from .authproxy import JSONRPCException
+from . import coverage
 from .util import (
-    PortSeed,
     MAX_NODES,
-    bitcoind_processes,
+    PortSeed,
+    assert_equal,
     check_json_precision,
     connect_nodes_bi,
     disable_mocktime,
     disconnect_nodes,
-    enable_coverage,
     enable_mocktime,
     get_mocktime,
     get_rpc_proxy,
@@ -34,15 +37,9 @@ from .util import (
     p2p_port,
     rpc_url,
     set_node_times,
-    _start_node,
-    _start_nodes,
-    _stop_node,
-    _stop_nodes,
     sync_blocks,
     sync_mempools,
-    wait_for_bitcoind_start,
 )
-from .authproxy import JSONRPCException
 
 class TestStatus(Enum):
     PASSED = 1
@@ -53,6 +50,8 @@ TEST_EXIT_PASSED = 0
 TEST_EXIT_FAILED = 1
 TEST_EXIT_SKIPPED = 77
 
+BITCOIND_PROC_WAIT_TIMEOUT = 60
+
 class BitcoinTestFramework(object):
     """Base class for a bitcoin test script.
 
@@ -72,7 +71,8 @@ class BitcoinTestFramework(object):
     def __init__(self):
         self.num_nodes = 4
         self.setup_clean_chain = False
-        self.nodes = None
+        self.nodes = []
+        self.bitcoind_processes = {}
 
     def add_options(self, parser):
         pass
@@ -98,7 +98,7 @@ class BitcoinTestFramework(object):
         extra_args = None
         if hasattr(self, "extra_args"):
             extra_args = self.extra_args
-        self.nodes = _start_nodes(self.num_nodes, self.options.tmpdir, extra_args)
+        self.nodes = self.start_nodes(self.num_nodes, self.options.tmpdir, extra_args)
 
     def run_test(self):
         raise NotImplementedError
@@ -130,9 +130,6 @@ class BitcoinTestFramework(object):
         self.add_options(parser)
         (self.options, self.args) = parser.parse_args()
 
-        if self.options.coveragedir:
-            enable_coverage(self.options.coveragedir)
-
         PortSeed.n = self.options.port_seed
 
         os.environ['PATH'] = self.options.srcdir + ":" + self.options.srcdir + "/qt:" + os.environ['PATH']
@@ -209,16 +206,88 @@ class BitcoinTestFramework(object):
     # Public helper methods. These can be accessed by the subclass test scripts.
 
     def start_node(self, i, dirname, extra_args=None, rpchost=None, timewait=None, binary=None, stderr=None):
-        return _start_node(i, dirname, extra_args, rpchost, timewait, binary, stderr)
+        """Start a bitcoind and return RPC connection to it"""
+
+        datadir = os.path.join(dirname, "node" + str(i))
+        if binary is None:
+            binary = os.getenv("BITCOIND", "bitcoind")
+        args = [binary, "-datadir=" + datadir, "-server", "-keypool=1", "-discover=0", "-rest", "-logtimemicros", "-debug", "-debugexclude=libevent", "-debugexclude=leveldb", "-mocktime=" + str(get_mocktime()), "-uacomment=testnode%d" % i]
+        if extra_args is not None:
+            args.extend(extra_args)
+        self.bitcoind_processes[i] = subprocess.Popen(args, stderr=stderr)
+        self.log.debug("initialize_chain: bitcoind started, waiting for RPC to come up")
+        self._wait_for_bitcoind_start(self.bitcoind_processes[i], datadir, i, rpchost)
+        self.log.debug("initialize_chain: RPC successfully started")
+        proxy = get_rpc_proxy(rpc_url(datadir, i, rpchost), i, timeout=timewait)
+
+        if self.options.coveragedir:
+            coverage.write_all_rpc_commands(self.options.coveragedir, proxy)
+
+        return proxy
 
     def start_nodes(self, num_nodes, dirname, extra_args=None, rpchost=None, timewait=None, binary=None):
-        return _start_nodes(num_nodes, dirname, extra_args, rpchost, timewait, binary)
+        """Start multiple bitcoinds, return RPC connections to them"""
 
-    def stop_node(self, num_node):
+        if extra_args is None:
-        _stop_node(self.nodes[num_node], num_node)
+            extra_args = [None] * num_nodes
+        if binary is None:
+            binary = [None] * num_nodes
+        assert_equal(len(extra_args), num_nodes)
+        assert_equal(len(binary), num_nodes)
+        rpcs = []
+        try:
+            for i in range(num_nodes):
+                rpcs.append(self.start_node(i, dirname, extra_args[i], rpchost, timewait=timewait, binary=binary[i]))
+        except:
+            # If one node failed to start, stop the others
+            # TODO: abusing self.nodes in this way is a little hacky.
+            # Eventually we should do a better job of tracking nodes
+            self.nodes.extend(rpcs)
+            self.stop_nodes()
+            self.nodes = []
+            raise
+        return rpcs
+
+    def stop_node(self, i):
+        """Stop a bitcoind test node"""
+
+        self.log.debug("Stopping node %d" % i)
+        try:
+            self.nodes[i].stop()
+        except http.client.CannotSendRequest as e:
+            self.log.exception("Unable to stop node")
+        return_code = self.bitcoind_processes[i].wait(timeout=BITCOIND_PROC_WAIT_TIMEOUT)
+        assert_equal(return_code, 0)
+        del self.bitcoind_processes[i]
 
     def stop_nodes(self):
-        _stop_nodes(self.nodes)
+        """Stop multiple bitcoind test nodes"""
+
+        for i in range(len(self.nodes)):
+            self.stop_node(i)
+        assert not self.bitcoind_processes.values()  # All connections must be gone now
+
+    def assert_start_raises_init_error(self, i, dirname, extra_args=None, expected_msg=None):
+        with tempfile.SpooledTemporaryFile(max_size=2**16) as log_stderr:
+            try:
+                self.start_node(i, dirname, extra_args, stderr=log_stderr)
+                self.stop_node(i)
+            except Exception as e:
+                assert 'bitcoind exited' in str(e)  # node must have shutdown
+                if expected_msg is not None:
+                    log_stderr.seek(0)
+                    stderr = log_stderr.read().decode('utf-8')
+                    if expected_msg not in stderr:
+                        raise AssertionError("Expected error \"" + expected_msg + "\" not found in:\n" + stderr)
+            else:
+                if expected_msg is None:
+                    assert_msg = "bitcoind should have exited with an error"
+                else:
+                    assert_msg = "bitcoind should have exited with expected error " + expected_msg
+                raise AssertionError(assert_msg)
+
+    def wait_for_node_exit(self, i, timeout):
+        self.bitcoind_processes[i].wait(timeout)
 
     def split_network(self):
         """
@@ -300,9 +369,9 @@ class BitcoinTestFramework(object):
                 args = [os.getenv("BITCOIND", "bitcoind"), "-server", "-keypool=1", "-datadir=" + datadir, "-discover=0"]
                 if i > 0:
                     args.append("-connect=127.0.0.1:" + str(p2p_port(0)))
-                bitcoind_processes[i] = subprocess.Popen(args)
+                self.bitcoind_processes[i] = subprocess.Popen(args)
                 self.log.debug("initialize_chain: bitcoind started, waiting for RPC to come up")
-                wait_for_bitcoind_start(bitcoind_processes[i], datadir, i)
+                self._wait_for_bitcoind_start(self.bitcoind_processes[i], datadir, i)
                 self.log.debug("initialize_chain: RPC successfully started")
 
             self.nodes = []
@@ -355,6 +424,30 @@ class BitcoinTestFramework(object):
         for i in range(num_nodes):
             initialize_datadir(test_dir, i)
 
+    def _wait_for_bitcoind_start(self, process, datadir, i, rpchost=None):
+        """Wait for bitcoind to start.
+
+        This means that RPC is accessible and fully initialized.
+        Raise an exception if bitcoind exits during initialization."""
+        while True:
+            if process.poll() is not None:
+                raise Exception('bitcoind exited with status %i during initialization' % process.returncode)
+            try:
+                # Check if .cookie file to be created
+                rpc = get_rpc_proxy(rpc_url(datadir, i, rpchost), i, coveragedir=self.options.coveragedir)
+                rpc.getblockcount()
+                break  # break out of loop on success
+            except IOError as e:
+                if e.errno != errno.ECONNREFUSED:  # Port not yet open?
+                    raise  # unknown IO error
+            except JSONRPCException as e:  # Initialization phase
+                if e.error['code'] != -28:  # RPC in warmup?
+                    raise  # unknown JSON RPC exception
+            except ValueError as e:  # cookie file not found and no rpcuser or rpcassword. bitcoind still starting
+                if "No RPC credentials" not in str(e):
+                    raise
+            time.sleep(0.25)
+
 class ComparisonTestFramework(BitcoinTestFramework):
     """Test framework for doing p2p comparison testing
 

test/functional/test_framework/util.py

@@ -7,22 +7,16 @@
 from base64 import b64encode
 from binascii import hexlify, unhexlify
 from decimal import Decimal, ROUND_DOWN
-import errno
-import http.client
 import json
 import logging
 import os
 import random
 import re
-import subprocess
-import tempfile
 import time
 
 from . import coverage
 from .authproxy import AuthServiceProxy, JSONRPCException
 
-COVERAGE_DIR = None
-
 logger = logging.getLogger("TestFramework.utils")
 
 # The maximum number of nodes a single test can spawn
@@ -32,9 +26,6 @@ PORT_MIN = 11000
 # The number of ports to "reserve" for p2p and rpc, each
 PORT_RANGE = 5000
 
-BITCOIND_PROC_WAIT_TIMEOUT = 60
-
-
 class PortSeed:
     # Must be initialized with a unique integer for each process
     n = None
@@ -60,13 +51,7 @@ def disable_mocktime():
 def get_mocktime():
     return MOCKTIME
 
-def enable_coverage(dirname):
+def get_rpc_proxy(url, node_number, timeout=None, coveragedir=None):
-    """Maintain a log of which RPC calls are made during testing."""
-    global COVERAGE_DIR
-    COVERAGE_DIR = dirname
-
-
-def get_rpc_proxy(url, node_number, timeout=None):
     """
     Args:
         url (str): URL of the RPC server to call
@@ -87,7 +72,7 @@ def get_rpc_proxy(url, node_number, timeout=None):
     proxy.url = url  # store URL on proxy for info
 
     coverage_logfile = coverage.get_filename(
-        COVERAGE_DIR, node_number) if COVERAGE_DIR else None
+        coveragedir, node_number) if coveragedir else None
 
     return coverage.AuthServiceProxyWrapper(proxy, coverage_logfile)
 
@@ -172,8 +157,6 @@ def sync_mempools(rpc_connections, *, wait=1, timeout=60):
         timeout -= wait
     raise AssertionError("Mempool sync failed")
 
-bitcoind_processes = {}
-
 def initialize_datadir(dirname, n):
     datadir = os.path.join(dirname, "node" + str(n))
     if not os.path.isdir(datadir):
@@ -222,121 +205,9 @@ def rpc_url(datadir, i, rpchost=None):
             host = rpchost
     return "http://%s:%s@%s:%d" % (rpc_u, rpc_p, host, int(port))
 
-def wait_for_bitcoind_start(process, datadir, i, rpchost=None):
-    '''
-    Wait for bitcoind to start. This means that RPC is accessible and fully initialized.
-    Raise an exception if bitcoind exits during initialization.
-    '''
-    while True:
-        if process.poll() is not None:
-            raise Exception('bitcoind exited with status %i during initialization' % process.returncode)
-        try:
-            # Check if .cookie file to be created
-            rpc = get_rpc_proxy(rpc_url(datadir, i, rpchost), i)
-            rpc.getblockcount()
-            break  # break out of loop on success
-        except IOError as e:
-            if e.errno != errno.ECONNREFUSED:  # Port not yet open?
-                raise  # unknown IO error
-        except JSONRPCException as e:  # Initialization phase
-            if e.error['code'] != -28:  # RPC in warmup?
-                raise  # unknown JSON RPC exception
-        except ValueError as e:  # cookie file not found and no rpcuser or rpcassword. bitcoind still starting
-            if "No RPC credentials" not in str(e):
-                raise
-        time.sleep(0.25)
-
-def wait_for_node_exit(node_index, timeout):
-    bitcoind_processes[node_index].wait(timeout)
-
-def _start_node(i, dirname, extra_args=None, rpchost=None, timewait=None, binary=None, stderr=None):
-    """Start a bitcoind and return RPC connection to it
-
-    This function should only be called from within test_framework, not by individual test scripts."""
-
-    datadir = os.path.join(dirname, "node" + str(i))
-    if binary is None:
-        binary = os.getenv("BITCOIND", "bitcoind")
-    args = [binary, "-datadir=" + datadir, "-server", "-keypool=1", "-discover=0", "-rest", "-logtimemicros", "-debug", "-debugexclude=libevent", "-debugexclude=leveldb", "-mocktime=" + str(get_mocktime()), "-uacomment=testnode%d" % i]
-    if extra_args is not None:
-        args.extend(extra_args)
-    bitcoind_processes[i] = subprocess.Popen(args, stderr=stderr)
-    logger.debug("initialize_chain: bitcoind started, waiting for RPC to come up")
-    wait_for_bitcoind_start(bitcoind_processes[i], datadir, i, rpchost)
-    logger.debug("initialize_chain: RPC successfully started")
-    proxy = get_rpc_proxy(rpc_url(datadir, i, rpchost), i, timeout=timewait)
-
-    if COVERAGE_DIR:
-        coverage.write_all_rpc_commands(COVERAGE_DIR, proxy)
-
-    return proxy
-
-def assert_start_raises_init_error(i, dirname, extra_args=None, expected_msg=None):
-    with tempfile.SpooledTemporaryFile(max_size=2**16) as log_stderr:
-        try:
-            node = _start_node(i, dirname, extra_args, stderr=log_stderr)
-            _stop_node(node, i)
-        except Exception as e:
-            assert 'bitcoind exited' in str(e)  # node must have shutdown
-            if expected_msg is not None:
-                log_stderr.seek(0)
-                stderr = log_stderr.read().decode('utf-8')
-                if expected_msg not in stderr:
-                    raise AssertionError("Expected error \"" + expected_msg + "\" not found in:\n" + stderr)
-        else:
-            if expected_msg is None:
-                assert_msg = "bitcoind should have exited with an error"
-            else:
-                assert_msg = "bitcoind should have exited with expected error " + expected_msg
-            raise AssertionError(assert_msg)
-
-def _start_nodes(num_nodes, dirname, extra_args=None, rpchost=None, timewait=None, binary=None):
-    """Start multiple bitcoinds, return RPC connections to them
-
-    This function should only be called from within test_framework, not by individual test scripts."""
-
-    if extra_args is None:
-        extra_args = [None] * num_nodes
-    if binary is None:
-        binary = [None] * num_nodes
-    assert_equal(len(extra_args), num_nodes)
-    assert_equal(len(binary), num_nodes)
-    rpcs = []
-    try:
-        for i in range(num_nodes):
-            rpcs.append(_start_node(i, dirname, extra_args[i], rpchost, timewait=timewait, binary=binary[i]))
-    except:
-        # If one node failed to start, stop the others
-        _stop_nodes(rpcs)
-        raise
-    return rpcs
-
 def log_filename(dirname, n_node, logname):
     return os.path.join(dirname, "node" + str(n_node), "regtest", logname)
 
-def _stop_node(node, i):
-    """Stop a bitcoind test node
-
-    This function should only be called from within test_framework, not by individual test scripts."""
-
-    logger.debug("Stopping node %d" % i)
-    try:
-        node.stop()
-    except http.client.CannotSendRequest as e:
-        logger.exception("Unable to stop node")
-    return_code = bitcoind_processes[i].wait(timeout=BITCOIND_PROC_WAIT_TIMEOUT)
-    del bitcoind_processes[i]
-    assert_equal(return_code, 0)
-
-def _stop_nodes(nodes):
-    """Stop multiple bitcoind test nodes
-
-    This function should only be called from within test_framework, not by individual test scripts."""
-
-    for i, node in enumerate(nodes):
-        _stop_node(node, i)
-    assert not bitcoind_processes.values()  # All connections must be gone now
-
 def set_node_times(nodes, t):
     for node in nodes:
         node.setmocktime(t)

test/functional/wallet-dump.py

@@ -7,7 +7,7 @@
 import os
 
 from test_framework.test_framework import BitcoinTestFramework
-from test_framework.util import (assert_equal, bitcoind_processes)
+from test_framework.util import assert_equal
 
 
 def read_dump(file_name, addrs, hd_master_addr_old):
@@ -95,7 +95,7 @@ class WalletDumpTest(BitcoinTestFramework):
 
         #encrypt wallet, restart, unlock and dump
         self.nodes[0].encryptwallet('test')
-        bitcoind_processes[0].wait()
+        self.bitcoind_processes[0].wait()
         self.nodes[0] = self.start_node(0, self.options.tmpdir, self.extra_args[0])
         self.nodes[0].walletpassphrase('test', 10)
         # Should be a no-op:

test/functional/wallet-encryption.py

@@ -6,12 +6,10 @@
 
 import time
 
-from test_framework.test_framework import BitcoinTestFramework
+from test_framework.test_framework import BitcoinTestFramework, BITCOIND_PROC_WAIT_TIMEOUT
 from test_framework.util import (
     assert_equal,
     assert_raises_jsonrpc,
-    bitcoind_processes,
-    BITCOIND_PROC_WAIT_TIMEOUT,
 )
 
 class WalletEncryptionTest(BitcoinTestFramework):
@@ -33,7 +31,7 @@ class WalletEncryptionTest(BitcoinTestFramework):
 
         # Encrypt the wallet
         self.nodes[0].encryptwallet(passphrase)
-        bitcoind_processes[0].wait(timeout=BITCOIND_PROC_WAIT_TIMEOUT)
+        self.bitcoind_processes[0].wait(timeout=BITCOIND_PROC_WAIT_TIMEOUT)
         self.nodes[0] = self.start_node(0, self.options.tmpdir)
 
         # Test that the wallet is encrypted

test/functional/wallet-hd.py

@@ -8,7 +8,6 @@ from test_framework.test_framework import BitcoinTestFramework
 from test_framework.util import (
     assert_equal,
     connect_nodes_bi,
-    assert_start_raises_init_error
 )
 import os
 import shutil
@@ -27,7 +26,7 @@ class WalletHDTest(BitcoinTestFramework):
 
         # Make sure can't switch off usehd after wallet creation
         self.stop_node(1)
-        assert_start_raises_init_error(1, self.options.tmpdir, ['-usehd=0'], 'already existing HD wallet')
+        self.assert_start_raises_init_error(1, self.options.tmpdir, ['-usehd=0'], 'already existing HD wallet')
         self.nodes[1] = self.start_node(1, self.options.tmpdir, self.extra_args[1])
         connect_nodes_bi(self.nodes, 0, 1)