LBRYCommunity/lbrycrd
1
0
Fork 0
Code Issues 29 Pull requests 1 Projects Releases 32 Packages Wiki Activity

Refactor rpc-tests.py

Browse source 
- add main()
- remove global variables
This commit is contained in:
John Newbery 2017-02-06 09:07:14 -05:00
parent afd38e7cc8
commit 3de3ccdfaa
1 changed files with 100 additions and 85 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

185
qa/pull-tester/rpc-tests.py
View file

@ -23,69 +23,8 @@ import subprocess
import tempfile import tempfile
import re import re
# Parse arguments and pass through unrecognised args
parser = argparse.ArgumentParser(add_help=False,
usage='%(prog)s [rpc-test.py options] [script options] [scripts]',
description=__doc__,
epilog='''
Help text and arguments for individual test script:''',
formatter_class=argparse.RawTextHelpFormatter)
parser.add_argument('--coverage', action='store_true', help='generate a basic coverage report for the RPC interface')
parser.add_argument('--extended', action='store_true', help='run the extended test suite in addition to the basic tests')
parser.add_argument('--help', '-h', '-?', action='store_true', help='print help text and exit')
parser.add_argument('--jobs', '-j', type=int, default=4, help='how many test scripts to run in parallel. Default=4.')
parser.add_argument('--nozmq', action='store_true', help='do not run the zmq tests')
parser.add_argument('--win', action='store_true', help='signal that this is running in a Windows environment and that we should run the tests')
(args, unknown_args) = parser.parse_known_args()
#Create a set to store arguments and create the passon string
tests = set(arg for arg in unknown_args if arg[:2] != "--")
passon_args = [arg for arg in unknown_args if arg[:2] == "--"]
BOLD = ("","")
if os.name == 'posix':
# primitive formatting on supported
# terminal via ANSI escape sequences:
BOLD = ('\033[0m', '\033[1m')
# Read config generated by configure.
config = configparser.ConfigParser()
config.read_file(open(os.path.dirname(__file__) + "/tests_config.ini"))
ENABLE_WALLET = config["components"]["ENABLE_WALLET"] == "True"
ENABLE_UTILS = config["components"]["ENABLE_UTILS"] == "True"
ENABLE_BITCOIND = config["components"]["ENABLE_BITCOIND"] == "True"
ENABLE_ZMQ = config["components"]["ENABLE_ZMQ"] == "True" and not args.nozmq
RPC_TESTS_DIR = config["environment"]["SRCDIR"] + '/qa/rpc-tests/'
print_help = args.help
jobs = args.jobs
#Set env vars
if "BITCOIND" not in os.environ:
os.environ["BITCOIND"] = config["environment"]["BUILDDIR"] + '/src/bitcoind' + config["environment"]["EXEEXT"]
if config["environment"]["EXEEXT"] == ".exe" and not args.win:
# https://github.com/bitcoin/bitcoin/commit/d52802551752140cf41f0d9a225a43e84404d3e9
# https://github.com/bitcoin/bitcoin/pull/5677#issuecomment-136646964
print("Win tests currently disabled by default. Use --win option to enable")
sys.exit(0)
if not (ENABLE_WALLET and ENABLE_UTILS and ENABLE_BITCOIND):
print("No rpc tests to run. Wallet, utils, and bitcoind must all be enabled")
sys.exit(0)
# python3-zmq may not be installed. Handle this gracefully and with some helpful info
if ENABLE_ZMQ:
try:
import zmq
except ImportError:
print("ERROR: \"import zmq\" failed. Use -nozmq to run without the ZMQ tests."
"To run zmq tests, see dependency info in /qa/README.md.")
raise
BASE_SCRIPTS= [ BASE_SCRIPTS= [
# Scripts that are run by the travis build process
# longest test should go first, to favor running tests in parallel # longest test should go first, to favor running tests in parallel
'wallet-hd.py', 'wallet-hd.py',
'walletbackup.py', 'walletbackup.py',
@ -142,9 +81,15 @@ BASE_SCRIPTS= [
'rpcnamedargs.py', 'rpcnamedargs.py',
'listsinceblock.py', 'listsinceblock.py',
] ]
ZMQ_SCRIPTS = ["zmq_test.py"]
ZMQ_SCRIPTS = [
# ZMQ test can only be run if bitcoin was built with zmq-enabled.
# call rpc_tests.py with -nozmq to explicitly exclude these tests.
"zmq_test.py"]
EXTENDED_SCRIPTS = [ EXTENDED_SCRIPTS = [
# These tests are not run by the travis build process.
# Longest test should go first, to favor running tests in parallel
'pruning.py', 'pruning.py',
# vv Tests less than 20m vv # vv Tests less than 20m vv
'smartfees.py', 'smartfees.py',
@ -175,7 +120,55 @@ EXTENDED_SCRIPTS = [
ALL_SCRIPTS = BASE_SCRIPTS + ZMQ_SCRIPTS + EXTENDED_SCRIPTS ALL_SCRIPTS = BASE_SCRIPTS + ZMQ_SCRIPTS + EXTENDED_SCRIPTS
def runtests(): def main():
# Parse arguments and pass through unrecognised args
parser = argparse.ArgumentParser(add_help=False,
usage='%(prog)s [rpc-test.py options] [script options] [scripts]',
description=__doc__,
epilog='''
Help text and arguments for individual test script:''',
formatter_class=argparse.RawTextHelpFormatter)
parser.add_argument('--coverage', action='store_true', help='generate a basic coverage report for the RPC interface')
parser.add_argument('--extended', action='store_true', help='run the extended test suite in addition to the basic tests')
parser.add_argument('--help', '-h', '-?', action='store_true', help='print help text and exit')
parser.add_argument('--nozmq', action='store_true', help='do not run the zmq tests')
parser.add_argument('--jobs', '-j', type=int, default=4, help='how many test scripts to run in parallel. Default=4.')
parser.add_argument('--win', action='store_true', help='signal that this is running in a Windows environment and that we should run the tests')
(args, unknown_args) = parser.parse_known_args()
# Create a set to store arguments and create the passon string
tests = set(arg for arg in unknown_args if arg[:2] != "--")
passon_args = [arg for arg in unknown_args if arg[:2] == "--"]
# Read config generated by configure.
config = configparser.ConfigParser()
config.read_file(open(os.path.dirname(__file__) + "/tests_config.ini"))
enable_wallet = config["components"]["ENABLE_WALLET"] == "True"
enable_utils = config["components"]["ENABLE_UTILS"] == "True"
enable_bitcoind = config["components"]["ENABLE_BITCOIND"] == "True"
enable_zmq = config["components"]["ENABLE_ZMQ"] == "True" and not args.nozmq
if config["environment"]["EXEEXT"] == ".exe" and not args.win:
# https://github.com/bitcoin/bitcoin/commit/d52802551752140cf41f0d9a225a43e84404d3e9
# https://github.com/bitcoin/bitcoin/pull/5677#issuecomment-136646964
print("Win tests currently disabled by default. Use --win option to enable")
sys.exit(0)
if not (enable_wallet and enable_utils and enable_bitcoind):
print("No rpc tests to run. Wallet, utils, and bitcoind must all be enabled")
print("Rerun `configure` with -enable-wallet, -with-utils and -with-daemon and rerun make")
sys.exit(0)
# python3-zmq may not be installed. Handle this gracefully and with some helpful info
if enable_zmq:
try:
import zmq
except ImportError:
print("ERROR: \"import zmq\" failed. Use -nozmq to run without the ZMQ tests."
"To run zmq tests, see dependency info in /qa/README.md.")
raise
# Build list of tests # Build list of tests
if len(tests) != 0: if len(tests) != 0:
# Individual tests have been specified. Run specified tests that exist # Individual tests have been specified. Run specified tests that exist
@ -185,12 +178,15 @@ def runtests():
if len(test_list) == 0: if len(test_list) == 0:
print("No valid test scripts specified. Check that your test is in one " print("No valid test scripts specified. Check that your test is in one "
"of the test lists in rpc-tests.py or run rpc-tests.py with no arguments to run all tests") "of the test lists in rpc-tests.py or run rpc-tests.py with no arguments to run all tests")
print("Scripts not found:")
print(tests)
sys.exit(0) sys.exit(0)
else: else:
# No individual tests have been specified. Run base tests, and # No individual tests have been specified. Run base tests, and
# optionally ZMQ tests and extended tests. # optionally ZMQ tests and extended tests.
test_list = BASE_SCRIPTS test_list = BASE_SCRIPTS
if ENABLE_ZMQ: if enable_zmq:
test_list += ZMQ_SCRIPTS test_list += ZMQ_SCRIPTS
if args.extended: if args.extended:
test_list += EXTENDED_SCRIPTS test_list += EXTENDED_SCRIPTS
@ -201,30 +197,47 @@ def runtests():
if args.help: if args.help:
# Print help for rpc-tests.py, then print help of the first script and exit. # Print help for rpc-tests.py, then print help of the first script and exit.
parser.print_help() parser.print_help()
subprocess.check_call((RPC_TESTS_DIR + test_list[0]).split() + ['-h']) subprocess.check_call((config["environment"]["SRCDIR"] + '/qa/rpc-tests/' + test_list[0]).split() + ['-h'])
sys.exit(0) sys.exit(0)
coverage = None runtests(test_list, config["environment"]["SRCDIR"], config["environment"]["BUILDDIR"], config["environment"]["EXEEXT"], args.jobs, args.coverage, passon_args)
if args.coverage: def runtests(test_list, src_dir, build_dir, exeext, jobs=1, enable_coverage=False, args=[]):
BOLD = ("","")
if os.name == 'posix':
# primitive formatting on supported
# terminal via ANSI escape sequences:
BOLD = ('\033[0m', '\033[1m')
#Set env vars
if "BITCOIND" not in os.environ:
os.environ["BITCOIND"] = build_dir + '/src/bitcoind' + exeext
tests_dir = src_dir + '/qa/rpc-tests/'
flags = ["--srcdir=" + src_dir] + args
flags.append("--cachedir=%s/qa/cache" % build_dir)
if enable_coverage:
coverage = RPCCoverage() coverage = RPCCoverage()
print("Initializing coverage directory at %s\n" % coverage.dir)
flags = ["--srcdir=%s/src" % config["environment"]["BUILDDIR"]] + passon_args
flags.append("--cachedir=%s/qa/cache" % config["environment"]["BUILDDIR"])
if coverage:
flags.append(coverage.flag) flags.append(coverage.flag)
print("Initializing coverage directory at %s\n" % coverage.dir)
else:
coverage = None
if len(test_list) > 1 and jobs > 1: if len(test_list) > 1 and jobs > 1:
# Populate cache # Populate cache
subprocess.check_output([RPC_TESTS_DIR + 'create_cache.py'] + flags) subprocess.check_output([tests_dir + 'create_cache.py'] + flags)
#Run Tests #Run Tests
max_len_name = len(max(test_list, key=len)) all_passed = True
time_sum = 0 time_sum = 0
time0 = time.time() time0 = time.time()
job_queue = RPCTestHandler(jobs, test_list, flags)
job_queue = RPCTestHandler(jobs, tests_dir, test_list, flags)
max_len_name = len(max(test_list, key=len))
results = BOLD[1] + "%s | %s | %s\n\n" % ("TEST".ljust(max_len_name), "PASSED", "DURATION") + BOLD[0] results = BOLD[1] + "%s | %s | %s\n\n" % ("TEST".ljust(max_len_name), "PASSED", "DURATION") + BOLD[0]
all_passed = True
for _ in range(len(test_list)): for _ in range(len(test_list)):
(name, stdout, stderr, passed, duration) = job_queue.get_next() (name, stdout, stderr, passed, duration) = job_queue.get_next()
all_passed = all_passed and passed all_passed = all_passed and passed
@ -233,8 +246,10 @@ def runtests():
print('\n' + BOLD[1] + name + BOLD[0] + ":") print('\n' + BOLD[1] + name + BOLD[0] + ":")
print('' if passed else stdout + '\n', end='') print('' if passed else stdout + '\n', end='')
print('' if stderr == '' else 'stderr:\n' + stderr + '\n', end='') print('' if stderr == '' else 'stderr:\n' + stderr + '\n', end='')
results += "%s | %s | %s s\n" % (name.ljust(max_len_name), str(passed).ljust(6), duration)
print("Pass: %s%s%s, Duration: %s s\n" % (BOLD[1], passed, BOLD[0], duration)) print("Pass: %s%s%s, Duration: %s s\n" % (BOLD[1], passed, BOLD[0], duration))
results += "%s | %s | %s s\n" % (name.ljust(max_len_name), str(passed).ljust(6), duration)
results += BOLD[1] + "\n%s | %s | %s s (accumulated)" % ("ALL".ljust(max_len_name), str(all_passed).ljust(6), time_sum) + BOLD[0] results += BOLD[1] + "\n%s | %s | %s s (accumulated)" % ("ALL".ljust(max_len_name), str(all_passed).ljust(6), time_sum) + BOLD[0]
print(results) print(results)
print("\nRuntime: %s s" % (int(time.time() - time0))) print("\nRuntime: %s s" % (int(time.time() - time0)))
@ -247,15 +262,15 @@ def runtests():
sys.exit(not all_passed) sys.exit(not all_passed)
class RPCTestHandler: class RPCTestHandler:
""" """
Trigger the testscrips passed in via the list. Trigger the testscrips passed in via the list.
""" """
def __init__(self, num_tests_parallel, test_list=None, flags=None): def __init__(self, num_tests_parallel, tests_dir, test_list=None, flags=None):
assert(num_tests_parallel >= 1) assert(num_tests_parallel >= 1)
self.num_jobs = num_tests_parallel self.num_jobs = num_tests_parallel
self.tests_dir = tests_dir
self.test_list = test_list self.test_list = test_list
self.flags = flags self.flags = flags
self.num_running = 0 self.num_running = 0
@ -275,7 +290,7 @@ class RPCTestHandler:
log_stderr = tempfile.SpooledTemporaryFile(max_size=2**16) log_stderr = tempfile.SpooledTemporaryFile(max_size=2**16)
self.jobs.append((t, self.jobs.append((t,
time.time(), time.time(),
subprocess.Popen((RPC_TESTS_DIR + t).split() + self.flags + port_seed, subprocess.Popen((self.tests_dir + t).split() + self.flags + port_seed,
universal_newlines=True, universal_newlines=True,
stdout=log_stdout, stdout=log_stdout,
stderr=log_stderr), stderr=log_stderr),
@ -340,10 +355,10 @@ class RPCCoverage(object):
""" """
# This is shared from `qa/rpc-tests/test-framework/coverage.py` # This is shared from `qa/rpc-tests/test-framework/coverage.py`
REFERENCE_FILENAME = 'rpc_interface.txt' reference_filename = 'rpc_interface.txt'
COVERAGE_FILE_PREFIX = 'coverage.' coverage_file_prefix = 'coverage.'
coverage_ref_filename = os.path.join(self.dir, REFERENCE_FILENAME) coverage_ref_filename = os.path.join(self.dir, reference_filename)
coverage_filenames = set() coverage_filenames = set()
all_cmds = set() all_cmds = set()
covered_cmds = set() covered_cmds = set()
@ -356,7 +371,7 @@ class RPCCoverage(object):
for root, dirs, files in os.walk(self.dir): for root, dirs, files in os.walk(self.dir):
for filename in files: for filename in files:
if filename.startswith(COVERAGE_FILE_PREFIX): if filename.startswith(coverage_file_prefix):
coverage_filenames.add(os.path.join(root, filename)) coverage_filenames.add(os.path.join(root, filename))
for filename in coverage_filenames: for filename in coverage_filenames:
@ -367,4 +382,4 @@ class RPCCoverage(object):
if __name__ == '__main__': if __name__ == '__main__':
runtests() main()