qa: Add test/fuzz/test_runner.py
This commit is contained in:
parent
ad039aa0d3
commit
fa7ca8ef58
8 changed files with 199 additions and 27 deletions
15
.travis.yml
15
.travis.yml
|
@ -15,6 +15,7 @@ env:
|
|||
- MAKEJOBS=-j3
|
||||
- RUN_UNIT_TESTS=true
|
||||
- RUN_FUNCTIONAL_TESTS=true
|
||||
- RUN_FUZZ_TESTS=false
|
||||
- DOCKER_NAME_TAG=ubuntu:18.04
|
||||
- BOOST_TEST_RANDOM=1$TRAVIS_BUILD_ID
|
||||
- CCACHE_SIZE=100M
|
||||
|
@ -100,7 +101,7 @@ jobs:
|
|||
PACKAGES="python3-zmq qtbase5-dev qttools5-dev-tools protobuf-compiler libdbus-1-dev libharfbuzz-dev libprotobuf-dev"
|
||||
DEP_OPTS="NO_QT=1 NO_UPNP=1 DEBUG=1 ALLOW_HOST_PACKAGES=1"
|
||||
GOAL="install"
|
||||
BITCOIN_CONFIG="--enable-zmq --with-gui=qt5 --enable-fuzz --enable-glibc-back-compat --enable-reduce-exports --enable-debug CXXFLAGS=\"-g0 -O2\""
|
||||
BITCOIN_CONFIG="--enable-zmq --with-gui=qt5 --enable-glibc-back-compat --enable-reduce-exports --enable-debug CXXFLAGS=\"-g0 -O2\""
|
||||
|
||||
- stage: test
|
||||
name: 'x86_64 Linux [GOAL: install] [trusty] [no functional tests, no depends, only system libs]'
|
||||
|
@ -132,6 +133,18 @@ jobs:
|
|||
GOAL="install"
|
||||
BITCOIN_CONFIG="--enable-zmq --with-incompatible-bdb --with-gui=qt5 CPPFLAGS=-DDEBUG_LOCKORDER --with-sanitizers=address,integer,undefined CC=clang CXX=clang++"
|
||||
|
||||
- stage: test
|
||||
name: 'x86_64 Linux [GOAL: install] [bionic] [no depends, only system libs, sanitizers: fuzzer,address]'
|
||||
env: >-
|
||||
HOST=x86_64-unknown-linux-gnu
|
||||
PACKAGES="clang llvm python3 libssl1.0-dev libevent-dev bsdmainutils libboost-system-dev libboost-filesystem-dev libboost-chrono-dev libboost-test-dev libboost-thread-dev"
|
||||
NO_DEPENDS=1
|
||||
RUN_UNIT_TESTS=false
|
||||
RUN_FUNCTIONAL_TESTS=false
|
||||
RUN_FUZZ_TESTS=true
|
||||
GOAL="install"
|
||||
BITCOIN_CONFIG="--disable-wallet --disable-bench --with-utils=no --with-daemon=no --with-libs=no --with-gui=no --enable-fuzz --with-sanitizers=fuzzer,address CC=clang CXX=clang++"
|
||||
|
||||
- stage: test
|
||||
name: 'x86_64 Linux [GOAL: install] [bionic] [no wallet]'
|
||||
env: >-
|
||||
|
|
|
@ -7,6 +7,11 @@
|
|||
export LC_ALL=C.UTF-8
|
||||
|
||||
travis_retry docker pull "$DOCKER_NAME_TAG"
|
||||
|
||||
export DIR_FUZZ_IN=${TRAVIS_BUILD_DIR}/qa-assets
|
||||
git clone https://github.com/bitcoin-core/qa-assets ${DIR_FUZZ_IN}
|
||||
export DIR_FUZZ_IN=${DIR_FUZZ_IN}/fuzz_seed_corpus/
|
||||
|
||||
mkdir -p "${TRAVIS_BUILD_DIR}/sanitizer-output/"
|
||||
export ASAN_OPTIONS=""
|
||||
export LSAN_OPTIONS="suppressions=${TRAVIS_BUILD_DIR}/test/sanitizer_suppressions/lsan"
|
||||
|
|
|
@ -19,3 +19,9 @@ if [ "$RUN_FUNCTIONAL_TESTS" = "true" ]; then
|
|||
DOCKER_EXEC test/functional/test_runner.py --ci --combinedlogslen=4000 --coverage --quiet --failfast
|
||||
END_FOLD
|
||||
fi
|
||||
|
||||
if [ "$RUN_FUZZ_TESTS" = "true" ]; then
|
||||
BEGIN_FOLD fuzz-tests
|
||||
DOCKER_EXEC test/fuzz/test_runner.py -l DEBUG ${DIR_FUZZ_IN}
|
||||
END_FOLD
|
||||
fi
|
||||
|
|
|
@ -220,7 +220,11 @@ endif
|
|||
|
||||
dist_noinst_SCRIPTS = autogen.sh
|
||||
|
||||
EXTRA_DIST = $(DIST_SHARE) test/functional/test_runner.py test/functional $(DIST_CONTRIB) $(DIST_DOCS) $(WINDOWS_PACKAGING) $(OSX_PACKAGING) $(BIN_CHECKS)
|
||||
EXTRA_DIST = $(DIST_SHARE) $(DIST_CONTRIB) $(DIST_DOCS) $(WINDOWS_PACKAGING) $(OSX_PACKAGING) $(BIN_CHECKS)
|
||||
|
||||
EXTRA_DIST += \
|
||||
test/functional \
|
||||
test/fuzz
|
||||
|
||||
EXTRA_DIST += \
|
||||
test/util/bitcoin-util-test.py \
|
||||
|
|
|
@ -5,6 +5,29 @@ A special test harness in `src/test/fuzz/` is provided for each fuzz target to
|
|||
provide an easy entry point for fuzzers and the like. In this document we'll
|
||||
describe how to use it with AFL and libFuzzer.
|
||||
|
||||
## Preparing fuzzing
|
||||
|
||||
AFL needs an input directory with examples, and an output directory where it
|
||||
will place examples that it found. These can be anywhere in the file system,
|
||||
we'll define environment variables to make it easy to reference them.
|
||||
|
||||
libFuzzer will use the input directory as output directory.
|
||||
|
||||
Extract the example seeds (or other starting inputs) into the inputs
|
||||
directory before starting fuzzing.
|
||||
|
||||
```
|
||||
git clone https://github.com/bitcoin-core/qa-assets
|
||||
export DIR_FUZZ_IN=$PWD/qa-assets/fuzz_seed_corpus
|
||||
```
|
||||
|
||||
Only for AFL:
|
||||
|
||||
```
|
||||
mkdir outputs
|
||||
export AFLOUT=$PWD/outputs
|
||||
```
|
||||
|
||||
## AFL
|
||||
|
||||
### Building AFL
|
||||
|
@ -23,7 +46,7 @@ export AFLPATH=$PWD
|
|||
To build Bitcoin Core using AFL instrumentation (this assumes that the
|
||||
`AFLPATH` was set as above):
|
||||
```
|
||||
./configure --disable-ccache --disable-shared --enable-tests --enable-fuzz CC=${AFLPATH}/afl-gcc CXX=${AFLPATH}/afl-g++
|
||||
./configure --disable-ccache --disable-shared --enable-tests --enable-fuzz --disable-wallet --disable-bench --with-utils=no --with-daemon=no --with-libs=no --with-gui=no CC=${AFLPATH}/afl-gcc CXX=${AFLPATH}/afl-g++
|
||||
export AFL_HARDEN=1
|
||||
cd src/
|
||||
make
|
||||
|
@ -39,31 +62,14 @@ binary will be instrumented in such a way that the AFL
|
|||
features "persistent mode" and "deferred forkserver" can be used. See
|
||||
https://github.com/mcarpenter/afl/tree/master/llvm_mode for details.
|
||||
|
||||
### Preparing fuzzing
|
||||
|
||||
AFL needs an input directory with examples, and an output directory where it
|
||||
will place examples that it found. These can be anywhere in the file system,
|
||||
we'll define environment variables to make it easy to reference them.
|
||||
|
||||
```
|
||||
mkdir inputs
|
||||
AFLIN=$PWD/inputs
|
||||
mkdir outputs
|
||||
AFLOUT=$PWD/outputs
|
||||
```
|
||||
|
||||
Example inputs are available from:
|
||||
|
||||
- https://download.visucore.com/bitcoin/bitcoin_fuzzy_in.tar.xz
|
||||
- http://strateman.ninja/fuzzing.tar.xz
|
||||
|
||||
Extract these (or other starting inputs) into the `inputs` directory before starting fuzzing.
|
||||
|
||||
### Fuzzing
|
||||
|
||||
To start the actual fuzzing use:
|
||||
|
||||
```
|
||||
$AFLPATH/afl-fuzz -i ${AFLIN} -o ${AFLOUT} -m52 -- test/fuzz/fuzz_target_foo
|
||||
export FUZZ_TARGET=fuzz_target_foo # Pick a fuzz_target
|
||||
mkdir ${AFLOUT}/${FUZZ_TARGET}
|
||||
$AFLPATH/afl-fuzz -i ${DIR_FUZZ_IN}/${FUZZ_TARGET} -o ${AFLOUT}/${FUZZ_TARGET} -m52 -- test/fuzz/${FUZZ_TARGET}
|
||||
```
|
||||
|
||||
You may have to change a few kernel parameters to test optimally - `afl-fuzz`
|
||||
|
@ -74,10 +80,10 @@ will print an error and suggestion if so.
|
|||
A recent version of `clang`, the address sanitizer and libFuzzer is needed (all
|
||||
found in the `compiler-rt` runtime libraries package).
|
||||
|
||||
To build the `test/test_bitcoin_fuzzy` executable run
|
||||
To build all fuzz targets with libFuzzer, run
|
||||
|
||||
```
|
||||
./configure --disable-ccache --enable-fuzz --with-sanitizers=fuzzer,address CC=clang CXX=clang++
|
||||
./configure --disable-ccache --disable-wallet --disable-bench --with-utils=no --with-daemon=no --with-libs=no --with-gui=no --enable-fuzz --with-sanitizers=fuzzer,address CC=clang CXX=clang++
|
||||
make
|
||||
```
|
||||
|
||||
|
@ -86,3 +92,6 @@ interchangeably between libFuzzer and AFL.
|
|||
|
||||
See https://llvm.org/docs/LibFuzzer.html#running on how to run the libFuzzer
|
||||
instrumented executable.
|
||||
|
||||
Alternatively run the script in `./test/fuzz/test_runner.py` and provide it
|
||||
with the `${DIR_FUZZ_IN}` created earlier.
|
||||
|
|
|
@ -2,7 +2,6 @@
|
|||
# Distributed under the MIT software license, see the accompanying
|
||||
# file COPYING or http://www.opensource.org/licenses/mit-license.php.
|
||||
|
||||
bin_PROGRAMS += test/test_bitcoin
|
||||
|
||||
FUZZ_TARGETS = \
|
||||
test/fuzz/address_deserialize \
|
||||
|
@ -28,6 +27,8 @@ FUZZ_TARGETS = \
|
|||
|
||||
if ENABLE_FUZZ
|
||||
noinst_PROGRAMS += $(FUZZ_TARGETS:=)
|
||||
else
|
||||
bin_PROGRAMS += test/test_bitcoin
|
||||
endif
|
||||
|
||||
TEST_SRCDIR = test
|
||||
|
|
|
@ -16,4 +16,5 @@ RPCAUTH=@abs_top_srcdir@/share/rpcauth/rpcauth.py
|
|||
@ENABLE_WALLET_TRUE@ENABLE_WALLET=true
|
||||
@BUILD_BITCOIN_CLI_TRUE@ENABLE_CLI=true
|
||||
@BUILD_BITCOIND_TRUE@ENABLE_BITCOIND=true
|
||||
@ENABLE_FUZZ_TRUE@ENABLE_FUZZ=true
|
||||
@ENABLE_ZMQ_TRUE@ENABLE_ZMQ=true
|
||||
|
|
133
test/fuzz/test_runner.py
Executable file
133
test/fuzz/test_runner.py
Executable file
|
@ -0,0 +1,133 @@
|
|||
#!/usr/bin/env python3
|
||||
# Copyright (c) 2019 The Bitcoin Core developers
|
||||
# Distributed under the MIT software license, see the accompanying
|
||||
# file COPYING or http://www.opensource.org/licenses/mit-license.php.
|
||||
"""Run fuzz test targets.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import configparser
|
||||
import os
|
||||
import sys
|
||||
import subprocess
|
||||
import logging
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter)
|
||||
parser.add_argument(
|
||||
"-l",
|
||||
"--loglevel",
|
||||
dest="loglevel",
|
||||
default="INFO",
|
||||
help="log events at this level and higher to the console. Can be set to DEBUG, INFO, WARNING, ERROR or CRITICAL. Passing --loglevel DEBUG will output all logs to console.",
|
||||
)
|
||||
parser.add_argument(
|
||||
'--export_coverage',
|
||||
action='store_true',
|
||||
help='If true, export coverage information to files in the seed corpus',
|
||||
)
|
||||
parser.add_argument(
|
||||
'seed_dir',
|
||||
help='The seed corpus to run on (must contain subfolders for each fuzz target).',
|
||||
)
|
||||
parser.add_argument(
|
||||
'target',
|
||||
nargs='*',
|
||||
help='The target(s) to run. Default is to run all targets.',
|
||||
)
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
# Set up logging
|
||||
logging.basicConfig(
|
||||
format='%(message)s',
|
||||
level=int(args.loglevel) if args.loglevel.isdigit() else args.loglevel.upper(),
|
||||
)
|
||||
|
||||
# Read config generated by configure.
|
||||
config = configparser.ConfigParser()
|
||||
configfile = os.path.abspath(os.path.dirname(__file__)) + "/../config.ini"
|
||||
config.read_file(open(configfile, encoding="utf8"))
|
||||
|
||||
if not config["components"].getboolean("ENABLE_FUZZ"):
|
||||
logging.error("Must have fuzz targets built")
|
||||
sys.exit(1)
|
||||
|
||||
# Build list of tests
|
||||
test_list_all = parse_test_list(makefile=os.path.join(config["environment"]["SRCDIR"], 'src', 'Makefile.test.include'))
|
||||
|
||||
if not test_list_all:
|
||||
logging.error("No fuzz targets found")
|
||||
sys.exit(1)
|
||||
|
||||
logging.info("Fuzz targets found: {}".format(test_list_all))
|
||||
|
||||
args.target = args.target or test_list_all # By default run all
|
||||
test_list_error = list(set(args.target).difference(set(test_list_all)))
|
||||
if test_list_error:
|
||||
logging.error("Unknown fuzz targets selected: {}".format(test_list_error))
|
||||
test_list_selection = list(set(test_list_all).intersection(set(args.target)))
|
||||
if not test_list_selection:
|
||||
logging.error("No fuzz targets selected")
|
||||
logging.info("Fuzz targets selected: {}".format(test_list_selection))
|
||||
|
||||
help_output = subprocess.run(
|
||||
args=[
|
||||
os.path.join(config["environment"]["BUILDDIR"], 'src', 'test', 'fuzz', test_list_selection[0]),
|
||||
'-help=1',
|
||||
],
|
||||
check=True,
|
||||
stderr=subprocess.PIPE,
|
||||
universal_newlines=True,
|
||||
).stderr
|
||||
if "libFuzzer" not in help_output:
|
||||
logging.error("Must be built with libFuzzer")
|
||||
sys.exit(1)
|
||||
|
||||
run_once(
|
||||
corpus=args.seed_dir,
|
||||
test_list=test_list_selection,
|
||||
build_dir=config["environment"]["BUILDDIR"],
|
||||
export_coverage=args.export_coverage,
|
||||
)
|
||||
|
||||
|
||||
def run_once(*, corpus, test_list, build_dir, export_coverage):
|
||||
for t in test_list:
|
||||
args = [
|
||||
os.path.join(build_dir, 'src', 'test', 'fuzz', t),
|
||||
'-runs=1',
|
||||
os.path.join(corpus, t),
|
||||
]
|
||||
logging.debug('Run {} with args {}'.format(t, args))
|
||||
output = subprocess.run(args, check=True, stderr=subprocess.PIPE, universal_newlines=True).stderr
|
||||
logging.debug('Output: {}'.format(output))
|
||||
if not export_coverage:
|
||||
continue
|
||||
for l in output.splitlines():
|
||||
if 'INITED' in l:
|
||||
with open(os.path.join(corpus, t + '_coverage'), 'w', encoding='utf-8') as cov_file:
|
||||
cov_file.write(l)
|
||||
break
|
||||
|
||||
|
||||
def parse_test_list(makefile):
|
||||
with open(makefile, encoding='utf-8') as makefile_test:
|
||||
test_list_all = []
|
||||
read_targets = False
|
||||
for line in makefile_test.readlines():
|
||||
line = line.strip().replace('test/fuzz/', '').replace(' \\', '')
|
||||
if read_targets:
|
||||
if not line:
|
||||
break
|
||||
test_list_all.append(line)
|
||||
continue
|
||||
|
||||
if line == 'FUZZ_TARGETS =':
|
||||
read_targets = True
|
||||
return test_list_all
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
Loading…
Reference in a new issue