LBRYCommunity/lbrycrd
1
0
Fork 0
Code Issues 29 Pull requests 1 Projects Releases 32 Packages Wiki Activity

test: Speed up cache creation

Browse source
This commit is contained in:
MarcoFalke 2019-05-17 13:07:07 -04:00
parent fa6ad7a5ec
commit fa47330397
No known key found for this signature in database
GPG key ID: CE2B75697E69A548
2 changed files with 35 additions and 49 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

69
test/functional/test_framework/test_framework.py
View file

@ -29,7 +29,6 @@ from .util import (
disconnect_nodes,
get_datadir_path,
initialize_datadir,
p2p_port,
sync_blocks,
sync_mempools,
)
@ -468,35 +467,23 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass):
def _initialize_chain(self):
"""Initialize a pre-mined blockchain for use by the test.
Create a cache of a 199-block-long chain (with wallet) for MAX_NODES
Create a cache of a 199-block-long chain
Afterward, create num_nodes copies from the cache."""
CACHE_NODE_ID = 0 # Use node 0 to create the cache for all other nodes
cache_node_dir = get_datadir_path(self.options.cachedir, CACHE_NODE_ID)
assert self.num_nodes <= MAX_NODES
create_cache = False
for i in range(MAX_NODES):
if not os.path.isdir(get_datadir_path(self.options.cachedir, i)):
create_cache = True
break
if create_cache:
self.log.debug("Creating data directories from cached datadir")
if not os.path.isdir(cache_node_dir):
self.log.debug("Creating cache directory {}".format(cache_node_dir))
# find and delete old cache directories if any exist
for i in range(MAX_NODES):
if os.path.isdir(get_datadir_path(self.options.cachedir, i)):
shutil.rmtree(get_datadir_path(self.options.cachedir, i))
# Create cache directories, run bitcoinds:
for i in range(MAX_NODES):
datadir = initialize_datadir(self.options.cachedir, i)
args = [self.options.bitcoind, "-datadir=" + datadir, '-disablewallet']
if i > 0:
args.append("-connect=127.0.0.1:" + str(p2p_port(0)))
self.nodes.append(TestNode(
i,
get_datadir_path(self.options.cachedir, i),
initialize_datadir(self.options.cachedir, CACHE_NODE_ID)
self.nodes.append(
TestNode(
CACHE_NODE_ID,
cache_node_dir,
extra_conf=["bind=127.0.0.1"],
extra_args=[],
extra_args=['-disablewallet'],
rpchost=None,
timewait=self.rpc_timeout,
bitcoind=self.options.bitcoind,
@ -504,12 +491,10 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass):
coverage_dir=None,
cwd=self.options.tmpdir,
))
self.nodes[i].args = args
self.start_node(i)
self.start_node(CACHE_NODE_ID)
# Wait for RPC connections to be ready
for node in self.nodes:
node.wait_for_rpc_connection()
self.nodes[CACHE_NODE_ID].wait_for_rpc_connection()
# Create a 199-block-long chain; each of the 4 first nodes
# gets 25 mature blocks and 25 immature.
@ -518,29 +503,29 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass):
# This is needed so that we are out of IBD when the test starts,
# see the tip age check in IsInitialBlockDownload().
for i in range(8):
self.nodes[0].generatetoaddress(25 if i != 7 else 24, self.nodes[i % 4].get_deterministic_priv_key().address)
self.sync_blocks()
self.nodes[CACHE_NODE_ID].generatetoaddress(
nblocks=25 if i != 7 else 24,
address=TestNode.PRIV_KEYS[i % 4].address,
)
for n in self.nodes:
assert_equal(n.getblockchaininfo()["blocks"], 199)
assert_equal(self.nodes[CACHE_NODE_ID].getblockchaininfo()["blocks"], 199)
# Shut them down, and clean up cache directories:
# Shut it down, and clean up cache directories:
self.stop_nodes()
self.nodes = []
def cache_path(n, *paths):
return os.path.join(get_datadir_path(self.options.cachedir, n), "regtest", *paths)
def cache_path(*paths):
return os.path.join(cache_node_dir, "regtest", *paths)
for i in range(MAX_NODES):
os.rmdir(cache_path(i, 'wallets')) # Remove empty wallets dir
for entry in os.listdir(cache_path(i)):
if entry not in ['chainstate', 'blocks']:
os.remove(cache_path(i, entry))
os.rmdir(cache_path('wallets')) # Remove empty wallets dir
for entry in os.listdir(cache_path()):
if entry not in ['chainstate', 'blocks']: # Only keep chainstate and blocks folder
os.remove(cache_path(entry))
for i in range(self.num_nodes):
from_dir = get_datadir_path(self.options.cachedir, i)
self.log.debug("Copy cache directory {} to node {}".format(cache_node_dir, i))
to_dir = get_datadir_path(self.options.tmpdir, i)
shutil.copytree(from_dir, to_dir)
shutil.copytree(cache_node_dir, to_dir)
initialize_datadir(self.options.tmpdir, i) # Overwrite port/rpcport in bitcoin.conf
def _initialize_chain_clean(self):

15
test/functional/test_framework/test_node.py
View file

@ -111,10 +111,8 @@ class TestNode():
self.p2ps = []
def get_deterministic_priv_key(self):
"""Return a deterministic priv key in base58, that only depends on the node's index"""
AddressKeyPair = collections.namedtuple('AddressKeyPair', ['address', 'key'])
PRIV_KEYS = [
AddressKeyPair = collections.namedtuple('AddressKeyPair', ['address', 'key'])
PRIV_KEYS = [
# address , privkey
AddressKeyPair('mjTkW3DjgyZck4KbiRusZsqTgaYTxdSz6z', 'cVpF924EspNh8KjYsfhgY96mmxvT6DgdWiTYMtMjuM74hJaU5psW'),
AddressKeyPair('msX6jQXvxiNhx3Q62PKeLPrhrqZQdSimTg', 'cUxsWyKyZ9MAQTaAhUQWJmBbSvHMwSmuv59KgxQV7oZQU3PXN3KE'),
@ -128,9 +126,12 @@ class TestNode():
AddressKeyPair('mq4fBNdckGtvY2mijd9am7DRsbRB4KjUkf', 'cN55daf1HotwBAgAKWVgDcoppmUNDtQSfb7XLutTLeAgVc3u8hik'),
AddressKeyPair('mpFAHDjX7KregM3rVotdXzQmkbwtbQEnZ6', 'cT7qK7g1wkYEMvKowd2ZrX1E5f6JQ7TM246UfqbCiyF7kZhorpX3'),
AddressKeyPair('mzRe8QZMfGi58KyWCse2exxEFry2sfF2Y7', 'cPiRWE8KMjTRxH1MWkPerhfoHFn5iHPWVK5aPqjW8NxmdwenFinJ'),
]
assert len(PRIV_KEYS) == MAX_NODES
return PRIV_KEYS[self.index]
]
def get_deterministic_priv_key(self):
"""Return a deterministic priv key in base58, that only depends on the node's index"""
assert len(self.PRIV_KEYS) == MAX_NODES
return self.PRIV_KEYS[self.index]
def get_mem_rss_kilobytes(self):
"""Get the memory usage (RSS) per `ps`.