Merge #10331: Share config between util and functional tests
8ad5bdeMerge bctest.py into bitcoin-util-test.py (John Newbery)95836c5Use shared config file for functional and util tests (John Newbery)89fcd35Use an .ini config file for environment vars in bitcoin-util-test.py (John Newbery)e9265dfChange help_text in bitcoin-util-test.py to a docstring. (John Newbery)ce58e93Change bitcoin-util-test.py to use Python3 (John Newbery) Tree-SHA512: 66dab0b4a8546aee0dfaef134a165f1447aff4c0ec335754bbc7d9e55909721c62f09cdbf4b22d02ac1fcd5a9b66780f91e1cc4d8687fae7288cc9072a23a78f
This commit is contained in:
MarcoFalke
commit
75e898c094
9 changed files with 157 additions and 175 deletions
4
.gitignore
vendored
4
.gitignore
vendored
|
|
@ -80,7 +80,6 @@ Bitcoin-Qt.app
|
|||
# Unit-tests
|
||||
Makefile.test
|
||||
bitcoin-qt_test
|
||||
src/test/buildenv.py
|
||||
|
||||
# Resources cpp
|
||||
qrc_*.cpp
|
||||
|
|
@ -101,8 +100,7 @@ coverage_percent.txt
|
|||
linux-coverage-build
|
||||
linux-build
|
||||
win32-build
|
||||
test/functional/config.ini
|
||||
test/util/buildenv.py
|
||||
test/config.ini
|
||||
test/cache/*
|
||||
|
||||
!src/leveldb*/Makefile
|
||||
|
|
|
|||
|
|
@ -223,7 +223,6 @@ dist_noinst_SCRIPTS = autogen.sh
|
|||
EXTRA_DIST = $(top_srcdir)/share/genbuild.sh test/functional/test_runner.py test/functional $(DIST_CONTRIB) $(DIST_DOCS) $(WINDOWS_PACKAGING) $(OSX_PACKAGING) $(BIN_CHECKS)
|
||||
|
||||
EXTRA_DIST += \
|
||||
test/util/bctest.py \
|
||||
test/util/bitcoin-util-test.py \
|
||||
test/util/data/bitcoin-util-test.json \
|
||||
test/util/data/blanktxv1.hex \
|
||||
|
|
@ -277,9 +276,6 @@ EXTRA_DIST += \
|
|||
|
||||
CLEANFILES = $(OSX_DMG) $(BITCOIN_WIN_INSTALLER)
|
||||
|
||||
# This file is problematic for out-of-tree builds if it exists.
|
||||
DISTCLEANFILES = test/util/buildenv.pyc
|
||||
|
||||
.INTERMEDIATE: $(COVERAGE_INFO)
|
||||
|
||||
DISTCHECK_CONFIGURE_FLAGS = --enable-man
|
||||
|
|
|
|||
|
|
@ -1161,13 +1161,11 @@ AC_SUBST(EVENT_PTHREADS_LIBS)
|
|||
AC_SUBST(ZMQ_LIBS)
|
||||
AC_SUBST(PROTOBUF_LIBS)
|
||||
AC_SUBST(QR_LIBS)
|
||||
AC_CONFIG_FILES([Makefile src/Makefile doc/man/Makefile share/setup.nsi share/qt/Info.plist test/functional/config.ini])
|
||||
AC_CONFIG_FILES([test/util/buildenv.py],[chmod +x test/util/buildenv.py])
|
||||
AC_CONFIG_FILES([Makefile src/Makefile doc/man/Makefile share/setup.nsi share/qt/Info.plist test/config.ini])
|
||||
AC_CONFIG_FILES([contrib/devtools/split-debug.sh],[chmod +x contrib/devtools/split-debug.sh])
|
||||
AC_CONFIG_FILES([doc/Doxyfile])
|
||||
AC_CONFIG_LINKS([test/functional/test_runner.py:test/functional/test_runner.py])
|
||||
AC_CONFIG_LINKS([test/util/bitcoin-util-test.py:test/util/bitcoin-util-test.py])
|
||||
AC_CONFIG_LINKS([test/util/bctest.py:test/util/bctest.py])
|
||||
|
||||
dnl boost's m4 checks do something really nasty: they export these vars. As a
|
||||
dnl result, they leak into secp256k1's configure and crazy things happen.
|
||||
|
|
@ -1215,8 +1213,8 @@ esac
|
|||
dnl Replace the BUILDDIR path with the correct Windows path if compiling on Native Windows
|
||||
case ${OS} in
|
||||
*Windows*)
|
||||
sed 's/BUILDDIR="\/\([[a-z]]\)/BUILDDIR="\1:/' test/functional/config.ini > test/functional/config-2.ini
|
||||
mv test/functional/config-2.ini test/functional/config.ini
|
||||
sed 's/BUILDDIR="\/\([[a-z]]\)/BUILDDIR="\1:/' test/config.ini > test/config-2.ini
|
||||
mv test/config-2.ini test/config.ini
|
||||
;;
|
||||
esac
|
||||
|
||||
|
|
|
|||
|
|
@ -148,7 +148,7 @@ bitcoin_test_clean : FORCE
|
|||
|
||||
check-local:
|
||||
@echo "Running test/util/bitcoin-util-test.py..."
|
||||
$(PYTHON) $(top_builddir)/test/util/bitcoin-util-test.py
|
||||
$(top_builddir)/test/util/bitcoin-util-test.py
|
||||
$(AM_V_at)$(MAKE) $(AM_MAKEFLAGS) -C secp256k1 check
|
||||
if EMBEDDED_UNIVALUE
|
||||
$(AM_V_at)$(MAKE) $(AM_MAKEFLAGS) -C univalue check
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@
|
|||
# file COPYING or http://www.opensource.org/licenses/mit-license.php.
|
||||
|
||||
# These environment variables are set by the build process and read by
|
||||
# test/functional/test_runner.py
|
||||
# test/functional/test_runner.py and test/util/bitcoin-util-test.py
|
||||
|
||||
[environment]
|
||||
SRCDIR=@abs_top_srcdir@
|
||||
|
|
@ -180,7 +180,7 @@ def main():
|
|||
|
||||
# Read config generated by configure.
|
||||
config = configparser.ConfigParser()
|
||||
configfile = os.path.abspath(os.path.dirname(__file__)) + "/config.ini"
|
||||
configfile = os.path.abspath(os.path.dirname(__file__)) + "/../config.ini"
|
||||
config.read_file(open(configfile))
|
||||
|
||||
passon_args.append("--configfile=%s" % configfile)
|
||||
|
|
|
|||
|
|
@ -1,139 +0,0 @@
|
|||
# Copyright 2014 BitPay Inc.
|
||||
# Copyright 2016 The Bitcoin Core developers
|
||||
# Distributed under the MIT software license, see the accompanying
|
||||
# file COPYING or http://www.opensource.org/licenses/mit-license.php.
|
||||
from __future__ import division,print_function,unicode_literals
|
||||
import subprocess
|
||||
import os
|
||||
import json
|
||||
import sys
|
||||
import binascii
|
||||
import difflib
|
||||
import logging
|
||||
import pprint
|
||||
|
||||
def parse_output(a, fmt):
|
||||
"""Parse the output according to specified format.
|
||||
|
||||
Raise an error if the output can't be parsed."""
|
||||
if fmt == 'json': # json: compare parsed data
|
||||
return json.loads(a)
|
||||
elif fmt == 'hex': # hex: parse and compare binary data
|
||||
return binascii.a2b_hex(a.strip())
|
||||
else:
|
||||
raise NotImplementedError("Don't know how to compare %s" % fmt)
|
||||
|
||||
def bctest(testDir, testObj, buildenv):
|
||||
"""Runs a single test, comparing output and RC to expected output and RC.
|
||||
|
||||
Raises an error if input can't be read, executable fails, or output/RC
|
||||
are not as expected. Error is caught by bctester() and reported.
|
||||
"""
|
||||
# Get the exec names and arguments
|
||||
execprog = buildenv.BUILDDIR + "/src/" + testObj['exec'] + buildenv.exeext
|
||||
execargs = testObj['args']
|
||||
execrun = [execprog] + execargs
|
||||
|
||||
# Read the input data (if there is any)
|
||||
stdinCfg = None
|
||||
inputData = None
|
||||
if "input" in testObj:
|
||||
filename = testDir + "/" + testObj['input']
|
||||
inputData = open(filename).read()
|
||||
stdinCfg = subprocess.PIPE
|
||||
|
||||
# Read the expected output data (if there is any)
|
||||
outputFn = None
|
||||
outputData = None
|
||||
if "output_cmp" in testObj:
|
||||
outputFn = testObj['output_cmp']
|
||||
outputType = os.path.splitext(outputFn)[1][1:] # output type from file extension (determines how to compare)
|
||||
try:
|
||||
outputData = open(testDir + "/" + outputFn).read()
|
||||
except:
|
||||
logging.error("Output file " + outputFn + " can not be opened")
|
||||
raise
|
||||
if not outputData:
|
||||
logging.error("Output data missing for " + outputFn)
|
||||
raise Exception
|
||||
|
||||
# Run the test
|
||||
proc = subprocess.Popen(execrun, stdin=stdinCfg, stdout=subprocess.PIPE, stderr=subprocess.PIPE,universal_newlines=True)
|
||||
try:
|
||||
outs = proc.communicate(input=inputData)
|
||||
except OSError:
|
||||
logging.error("OSError, Failed to execute " + execprog)
|
||||
raise
|
||||
|
||||
if outputData:
|
||||
data_mismatch, formatting_mismatch = False, False
|
||||
# Parse command output and expected output
|
||||
try:
|
||||
a_parsed = parse_output(outs[0], outputType)
|
||||
except Exception as e:
|
||||
logging.error('Error parsing command output as %s: %s' % (outputType,e))
|
||||
raise
|
||||
try:
|
||||
b_parsed = parse_output(outputData, outputType)
|
||||
except Exception as e:
|
||||
logging.error('Error parsing expected output %s as %s: %s' % (outputFn,outputType,e))
|
||||
raise
|
||||
# Compare data
|
||||
if a_parsed != b_parsed:
|
||||
logging.error("Output data mismatch for " + outputFn + " (format " + outputType + ")")
|
||||
data_mismatch = True
|
||||
# Compare formatting
|
||||
if outs[0] != outputData:
|
||||
error_message = "Output formatting mismatch for " + outputFn + ":\n"
|
||||
error_message += "".join(difflib.context_diff(outputData.splitlines(True),
|
||||
outs[0].splitlines(True),
|
||||
fromfile=outputFn,
|
||||
tofile="returned"))
|
||||
logging.error(error_message)
|
||||
formatting_mismatch = True
|
||||
|
||||
assert not data_mismatch and not formatting_mismatch
|
||||
|
||||
# Compare the return code to the expected return code
|
||||
wantRC = 0
|
||||
if "return_code" in testObj:
|
||||
wantRC = testObj['return_code']
|
||||
if proc.returncode != wantRC:
|
||||
logging.error("Return code mismatch for " + outputFn)
|
||||
raise Exception
|
||||
|
||||
if "error_txt" in testObj:
|
||||
want_error = testObj["error_txt"]
|
||||
# Compare error text
|
||||
# TODO: ideally, we'd compare the strings exactly and also assert
|
||||
# That stderr is empty if no errors are expected. However, bitcoin-tx
|
||||
# emits DISPLAY errors when running as a windows application on
|
||||
# linux through wine. Just assert that the expected error text appears
|
||||
# somewhere in stderr.
|
||||
if want_error not in outs[1]:
|
||||
logging.error("Error mismatch:\n" + "Expected: " + want_error + "\nReceived: " + outs[1].rstrip())
|
||||
raise Exception
|
||||
|
||||
def bctester(testDir, input_basename, buildenv):
|
||||
""" Loads and parses the input file, runs all tests and reports results"""
|
||||
input_filename = testDir + "/" + input_basename
|
||||
raw_data = open(input_filename).read()
|
||||
input_data = json.loads(raw_data)
|
||||
|
||||
failed_testcases = []
|
||||
|
||||
for testObj in input_data:
|
||||
try:
|
||||
bctest(testDir, testObj, buildenv)
|
||||
logging.info("PASSED: " + testObj["description"])
|
||||
except:
|
||||
logging.info("FAILED: " + testObj["description"])
|
||||
failed_testcases.append(testObj["description"])
|
||||
|
||||
if failed_testcases:
|
||||
error_message = "FAILED_TESTCASES:\n"
|
||||
error_message += pprint.pformat(failed_testcases, width=400)
|
||||
logging.error(error_message)
|
||||
sys.exit(1)
|
||||
else:
|
||||
sys.exit(0)
|
||||
|
|
@ -1,26 +1,30 @@
|
|||
#!/usr/bin/env python
|
||||
#!/usr/bin/env python3
|
||||
# Copyright 2014 BitPay Inc.
|
||||
# Copyright 2016 The Bitcoin Core developers
|
||||
# Copyright 2016-2017 The Bitcoin Core developers
|
||||
# Distributed under the MIT software license, see the accompanying
|
||||
# file COPYING or http://www.opensource.org/licenses/mit-license.php.
|
||||
from __future__ import division,print_function,unicode_literals
|
||||
import os
|
||||
import sys
|
||||
import argparse
|
||||
import logging
|
||||
"""Test framework for bitcoin utils.
|
||||
|
||||
help_text="""Test framework for bitcoin utils.
|
||||
|
||||
Runs automatically during `make check`.
|
||||
Runs automatically during `make check`.
|
||||
|
||||
Can also be run manually."""
|
||||
|
||||
if __name__ == '__main__':
|
||||
sys.path.append(os.path.dirname(os.path.abspath(__file__)))
|
||||
import buildenv
|
||||
import bctest
|
||||
import argparse
|
||||
import binascii
|
||||
import configparser
|
||||
import difflib
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import pprint
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
parser = argparse.ArgumentParser(description=help_text)
|
||||
def main():
|
||||
config = configparser.ConfigParser()
|
||||
config.read_file(open(os.path.dirname(__file__) + "/../config.ini"))
|
||||
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument('-v', '--verbose', action='store_true')
|
||||
args = parser.parse_args()
|
||||
verbose = args.verbose
|
||||
|
|
@ -31,6 +35,135 @@ if __name__ == '__main__':
|
|||
level = logging.ERROR
|
||||
formatter = '%(asctime)s - %(levelname)s - %(message)s'
|
||||
# Add the format/level to the logger
|
||||
logging.basicConfig(format = formatter, level=level)
|
||||
logging.basicConfig(format=formatter, level=level)
|
||||
|
||||
bctest.bctester(buildenv.SRCDIR + "/test/util/data", "bitcoin-util-test.json", buildenv)
|
||||
bctester(config["environment"]["SRCDIR"] + "/test/util/data", "bitcoin-util-test.json", config["environment"])
|
||||
|
||||
def bctester(testDir, input_basename, buildenv):
|
||||
""" Loads and parses the input file, runs all tests and reports results"""
|
||||
input_filename = testDir + "/" + input_basename
|
||||
raw_data = open(input_filename).read()
|
||||
input_data = json.loads(raw_data)
|
||||
|
||||
failed_testcases = []
|
||||
|
||||
for testObj in input_data:
|
||||
try:
|
||||
bctest(testDir, testObj, buildenv)
|
||||
logging.info("PASSED: " + testObj["description"])
|
||||
except:
|
||||
logging.info("FAILED: " + testObj["description"])
|
||||
failed_testcases.append(testObj["description"])
|
||||
|
||||
if failed_testcases:
|
||||
error_message = "FAILED_TESTCASES:\n"
|
||||
error_message += pprint.pformat(failed_testcases, width=400)
|
||||
logging.error(error_message)
|
||||
sys.exit(1)
|
||||
else:
|
||||
sys.exit(0)
|
||||
|
||||
def bctest(testDir, testObj, buildenv):
|
||||
"""Runs a single test, comparing output and RC to expected output and RC.
|
||||
|
||||
Raises an error if input can't be read, executable fails, or output/RC
|
||||
are not as expected. Error is caught by bctester() and reported.
|
||||
"""
|
||||
# Get the exec names and arguments
|
||||
execprog = buildenv["BUILDDIR"] + "/src/" + testObj['exec'] + buildenv["EXEEXT"]
|
||||
execargs = testObj['args']
|
||||
execrun = [execprog] + execargs
|
||||
|
||||
# Read the input data (if there is any)
|
||||
stdinCfg = None
|
||||
inputData = None
|
||||
if "input" in testObj:
|
||||
filename = testDir + "/" + testObj['input']
|
||||
inputData = open(filename).read()
|
||||
stdinCfg = subprocess.PIPE
|
||||
|
||||
# Read the expected output data (if there is any)
|
||||
outputFn = None
|
||||
outputData = None
|
||||
if "output_cmp" in testObj:
|
||||
outputFn = testObj['output_cmp']
|
||||
outputType = os.path.splitext(outputFn)[1][1:] # output type from file extension (determines how to compare)
|
||||
try:
|
||||
outputData = open(testDir + "/" + outputFn).read()
|
||||
except:
|
||||
logging.error("Output file " + outputFn + " can not be opened")
|
||||
raise
|
||||
if not outputData:
|
||||
logging.error("Output data missing for " + outputFn)
|
||||
raise Exception
|
||||
|
||||
# Run the test
|
||||
proc = subprocess.Popen(execrun, stdin=stdinCfg, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True)
|
||||
try:
|
||||
outs = proc.communicate(input=inputData)
|
||||
except OSError:
|
||||
logging.error("OSError, Failed to execute " + execprog)
|
||||
raise
|
||||
|
||||
if outputData:
|
||||
data_mismatch, formatting_mismatch = False, False
|
||||
# Parse command output and expected output
|
||||
try:
|
||||
a_parsed = parse_output(outs[0], outputType)
|
||||
except Exception as e:
|
||||
logging.error('Error parsing command output as %s: %s' % (outputType, e))
|
||||
raise
|
||||
try:
|
||||
b_parsed = parse_output(outputData, outputType)
|
||||
except Exception as e:
|
||||
logging.error('Error parsing expected output %s as %s: %s' % (outputFn, outputType, e))
|
||||
raise
|
||||
# Compare data
|
||||
if a_parsed != b_parsed:
|
||||
logging.error("Output data mismatch for " + outputFn + " (format " + outputType + ")")
|
||||
data_mismatch = True
|
||||
# Compare formatting
|
||||
if outs[0] != outputData:
|
||||
error_message = "Output formatting mismatch for " + outputFn + ":\n"
|
||||
error_message += "".join(difflib.context_diff(outputData.splitlines(True),
|
||||
outs[0].splitlines(True),
|
||||
fromfile=outputFn,
|
||||
tofile="returned"))
|
||||
logging.error(error_message)
|
||||
formatting_mismatch = True
|
||||
|
||||
assert not data_mismatch and not formatting_mismatch
|
||||
|
||||
# Compare the return code to the expected return code
|
||||
wantRC = 0
|
||||
if "return_code" in testObj:
|
||||
wantRC = testObj['return_code']
|
||||
if proc.returncode != wantRC:
|
||||
logging.error("Return code mismatch for " + outputFn)
|
||||
raise Exception
|
||||
|
||||
if "error_txt" in testObj:
|
||||
want_error = testObj["error_txt"]
|
||||
# Compare error text
|
||||
# TODO: ideally, we'd compare the strings exactly and also assert
|
||||
# That stderr is empty if no errors are expected. However, bitcoin-tx
|
||||
# emits DISPLAY errors when running as a windows application on
|
||||
# linux through wine. Just assert that the expected error text appears
|
||||
# somewhere in stderr.
|
||||
if want_error not in outs[1]:
|
||||
logging.error("Error mismatch:\n" + "Expected: " + want_error + "\nReceived: " + outs[1].rstrip())
|
||||
raise Exception
|
||||
|
||||
def parse_output(a, fmt):
|
||||
"""Parse the output according to specified format.
|
||||
|
||||
Raise an error if the output can't be parsed."""
|
||||
if fmt == 'json': # json: compare parsed data
|
||||
return json.loads(a)
|
||||
elif fmt == 'hex': # hex: parse and compare binary data
|
||||
return binascii.a2b_hex(a.strip())
|
||||
else:
|
||||
raise NotImplementedError("Don't know how to compare %s" % fmt)
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
|
|
|
|||
|
|
@ -1,4 +0,0 @@
|
|||
#!/usr/bin/env python
|
||||
exeext="@EXEEXT@"
|
||||
SRCDIR="@abs_top_srcdir@"
|
||||
BUILDDIR="@abs_top_builddir@"
|
||||
Loading…
Reference in a new issue