Add logging to bitcoin-util-test.py
- Use the python standard logging library - Run all tests and report all failing test-cases (rather than stop after one test case fails) - If output is different from expected output, log a contextual diff.
This commit is contained in:
jnewbery
parent
a4fd8dff68
commit
32c0d6e1d2
2 changed files with 54 additions and 24 deletions
|
|
@ -7,6 +7,8 @@ import os
|
|||
import json
|
||||
import sys
|
||||
import binascii
|
||||
import difflib
|
||||
import logging
|
||||
|
||||
def parse_output(a, fmt):
|
||||
if fmt == 'json': # json: compare parsed data
|
||||
|
|
@ -33,53 +35,70 @@ def bctest(testDir, testObj, exeext):
|
|||
if "output_cmp" in testObj:
|
||||
outputFn = testObj['output_cmp']
|
||||
outputType = os.path.splitext(outputFn)[1][1:] # output type from file extension (determines how to compare)
|
||||
outputData = open(testDir + "/" + outputFn).read()
|
||||
try:
|
||||
outputData = open(testDir + "/" + outputFn).read()
|
||||
except:
|
||||
logging.error("Output file " + outputFn + " can not be opened")
|
||||
raise
|
||||
if not outputData:
|
||||
print("Output data missing for " + outputFn)
|
||||
sys.exit(1)
|
||||
logging.error("Output data missing for " + outputFn)
|
||||
raise Exception
|
||||
|
||||
proc = subprocess.Popen(execrun, stdin=stdinCfg, stdout=subprocess.PIPE, stderr=subprocess.PIPE,universal_newlines=True)
|
||||
try:
|
||||
outs = proc.communicate(input=inputData)
|
||||
except OSError:
|
||||
print("OSError, Failed to execute " + execprog)
|
||||
sys.exit(1)
|
||||
logging.error("OSError, Failed to execute " + execprog)
|
||||
raise
|
||||
|
||||
if outputData:
|
||||
try:
|
||||
a_parsed = parse_output(outs[0], outputType)
|
||||
except Exception as e:
|
||||
print('Error parsing command output as %s: %s' % (outputType,e))
|
||||
sys.exit(1)
|
||||
logging.error('Error parsing command output as %s: %s' % (outputType,e))
|
||||
raise
|
||||
try:
|
||||
b_parsed = parse_output(outputData, outputType)
|
||||
except Exception as e:
|
||||
print('Error parsing expected output %s as %s: %s' % (outputFn,outputType,e))
|
||||
sys.exit(1)
|
||||
logging.error('Error parsing expected output %s as %s: %s' % (outputFn,outputType,e))
|
||||
raise
|
||||
if a_parsed != b_parsed:
|
||||
print("Output data mismatch for " + outputFn + " (format " + outputType + ")")
|
||||
sys.exit(1)
|
||||
logging.error("Output data mismatch for " + outputFn + " (format " + outputType + ")")
|
||||
raise Exception
|
||||
if outs[0] != outputData:
|
||||
print("Output formatting mismatch for " + outputFn + " (format " + outputType + ")")
|
||||
sys.exit(1)
|
||||
error_message = "Output formatting mismatch for " + outputFn + ":\n"
|
||||
error_message += "".join(difflib.context_diff(outputData.splitlines(True),
|
||||
outs[0].splitlines(True),
|
||||
fromfile=outputFn,
|
||||
tofile="returned"))
|
||||
logging.error(error_message)
|
||||
raise Exception
|
||||
|
||||
wantRC = 0
|
||||
if "return_code" in testObj:
|
||||
wantRC = testObj['return_code']
|
||||
if proc.returncode != wantRC:
|
||||
print("Return code mismatch for " + outputFn)
|
||||
sys.exit(1)
|
||||
logging.error("Return code mismatch for " + outputFn)
|
||||
raise Exception
|
||||
|
||||
def bctester(testDir, input_basename, buildenv, verbose = False):
|
||||
def bctester(testDir, input_basename, buildenv):
|
||||
input_filename = testDir + "/" + input_basename
|
||||
raw_data = open(input_filename).read()
|
||||
input_data = json.loads(raw_data)
|
||||
|
||||
failed_testcases = []
|
||||
|
||||
for testObj in input_data:
|
||||
if verbose and "description" in testObj:
|
||||
print ("Testing: " + testObj["description"])
|
||||
bctest(testDir, testObj, buildenv.exeext)
|
||||
if verbose and "description" in testObj:
|
||||
print ("PASS")
|
||||
try:
|
||||
bctest(testDir, testObj, buildenv.exeext)
|
||||
logging.info("PASSED: " + testObj["description"])
|
||||
except:
|
||||
logging.info("FAILED: " + testObj["description"])
|
||||
failed_testcases.append(testObj["description"])
|
||||
|
||||
sys.exit(0)
|
||||
if failed_testcases:
|
||||
logging.error("FAILED TESTCASES: [" + ", ".join(failed_testcases) + "]")
|
||||
sys.exit(1)
|
||||
else:
|
||||
sys.exit(0)
|
||||
|
||||
|
|
|
|||
|
|
@ -4,9 +4,11 @@
|
|||
# file COPYING or http://www.opensource.org/licenses/mit-license.php.
|
||||
from __future__ import division,print_function,unicode_literals
|
||||
import os
|
||||
import sys
|
||||
import bctest
|
||||
import buildenv
|
||||
import argparse
|
||||
import logging
|
||||
|
||||
help_text="""Test framework for bitcoin utils.
|
||||
|
||||
|
|
@ -19,9 +21,9 @@ test/bitcoin-util-test.py --src=[srcdir]
|
|||
|
||||
|
||||
if __name__ == '__main__':
|
||||
verbose = False
|
||||
try:
|
||||
srcdir = os.environ["srcdir"]
|
||||
verbose = False
|
||||
except:
|
||||
parser = argparse.ArgumentParser(description=help_text)
|
||||
parser.add_argument('-s', '--srcdir')
|
||||
|
|
@ -29,4 +31,13 @@ if __name__ == '__main__':
|
|||
args = parser.parse_args()
|
||||
srcdir = args.srcdir
|
||||
verbose = args.verbose
|
||||
bctest.bctester(srcdir + "/test/data", "bitcoin-util-test.json", buildenv, verbose = verbose)
|
||||
|
||||
if verbose:
|
||||
level = logging.DEBUG
|
||||
else:
|
||||
level = logging.ERROR
|
||||
formatter = '%(asctime)s - %(levelname)s - %(message)s'
|
||||
# Add the format/level to the logger
|
||||
logging.basicConfig(format = formatter, level=level)
|
||||
|
||||
bctest.bctester(srcdir + "/test/data", "bitcoin-util-test.json", buildenv)
|
||||
|
|
|
|||
Loading…
Reference in a new issue