Merge #14007: tests: Run functional test on Windows and enable it on Appveyor

661ac15a4a appveyor: Run functional tests on appveyor (Chun Kuan Lee)
2148c36b6e tests: Make it possible to run functional tests on Windows (Chun Kuan Lee)

Pull request description:

  This PR do the following things:
  - Make functional tests compatible with Windows
  - Print color output in functional tests for Windows 10
  - Run util and functional tests on appveyor
  - Do not run symlink tests on Windows

  Note:
  - The wallet_multiwallet.py fail is unrelated to the test framework, it's a bug related to c++ code or maybe dependencies. `bitcoind` would exit with 0xC0000005(Access violation) during shutdown occasionally. Disable this for now.
  - Not using `--failfast` because this is still in experimental. We should track if there is any other error.
  - Disable ZMQ tests because the python zmq library could cause access violation sometimes.
  - Disable `feature_notifications` because Bitcoin Core handles the command in different thread, whicha can cause a race condition.

Tree-SHA512: b76db137d264e62a5c130e1cbca7a2ca002a7a0f4153fa0b92c1ea6c9c09ef0533e11c49bdbd566c472d8ff59f245758feb5e5a6ec6cb6bb66a1c67bab5fa48a

appveyor.yml

@@ -7,6 +7,7 @@ environment:
   APPVEYOR_SAVE_CACHE_ON_ERROR: true
   CLCACHE_SERVER: 1
   PACKAGES: boost-filesystem boost-signals2 boost-interprocess boost-test libevent openssl zeromq berkeleydb secp256k1 leveldb
+  PYTHONIOENCODING: utf-8
 cache:
 - C:\tools\vcpkg\installed
 - C:\Users\appveyor\clcache
@@ -15,6 +16,8 @@ init:
 - cmd: set PATH=C:\Python36-x64;C:\Python36-x64\Scripts;%PATH%
 install:
 - cmd: pip install git+https://github.com/frerich/clcache.git
+# Disable zmq test for now since python zmq library on Windows would cause Access violation sometimes.
+# - cmd: pip install zmq
 - ps:  $packages = $env:PACKAGES -Split ' '
 - ps:  for ($i=0; $i -lt $packages.length; $i++) {
            $env:ALL_PACKAGES += $packages[$i] + ":" + $env:PLATFORM + "-windows-static "
@@ -40,6 +43,17 @@ after_build:
 - cmd: move build_msvc\%PLATFORM%\%CONFIGURATION%\*.iobj build_msvc\cache\
 - cmd: move build_msvc\%PLATFORM%\%CONFIGURATION%\*.ipdb build_msvc\cache\
 - cmd: del C:\Users\appveyor\clcache\stats.txt
+before_test:
+- ps:  ${conf_ini} = (Get-Content([IO.Path]::Combine(${env:APPVEYOR_BUILD_FOLDER}, "test", "config.ini.in")))
+- ps:  ${conf_ini} = $conf_ini.Replace("@abs_top_srcdir@", ${env:APPVEYOR_BUILD_FOLDER}).Replace("@abs_top_builddir@", ${env:APPVEYOR_BUILD_FOLDER}).Replace("@EXEEXT@", ".exe")
+- ps:  ${conf_ini} = $conf_ini.Replace("@ENABLE_WALLET_TRUE@", "").Replace("@BUILD_BITCOIN_UTILS_TRUE@", "").Replace("@BUILD_BITCOIND_TRUE@", "").Replace("@ENABLE_ZMQ_TRUE@", "")
+- ps:  ${utf8} = New-Object System.Text.UTF8Encoding ${false}
+- ps:  '[IO.File]::WriteAllLines([IO.Path]::Combine(${env:APPVEYOR_BUILD_FOLDER}, "test", "config.ini"), $conf_ini, ${utf8})'
+- ps:  move "build_msvc\${env:PLATFORM}\${env:CONFIGURATION}\*.exe" src
 test_script:
-- cmd: build_msvc\%PLATFORM%\%CONFIGURATION%\test_bitcoin.exe
+- cmd: src\test_bitcoin.exe
+- ps:  src\bench_bitcoin.exe -evals=1 -scaling=0
+- ps:  python test\util\bitcoin-util-test.py
+- cmd: python test\util\rpcauth-test.py
+- cmd: python test\functional\test_runner.py --force --quiet --combinedlogslen=4000 --exclude "feature_notifications,wallet_multiwallet,wallet_multiwallet.py --usecli"
 deploy: off

test/functional/combine_logs.py

@@ -25,10 +25,6 @@ def main():
     parser.add_argument('--html', dest='html', action='store_true', help='outputs the combined log as html. Requires jinja2. pip install jinja2')
     args, unknown_args = parser.parse_known_args()
 
-    if args.color and os.name != 'posix':
-        print("Color output requires posix terminal colors.")
-        sys.exit(1)
-
     if args.html and args.color:
         print("Only one out of --color or --html should be specified")
         sys.exit(1)

test/functional/feature_block.py

@@ -824,7 +824,7 @@ class FullBlockTest(BitcoinTestFramework):
         tx.vin.append(CTxIn(COutPoint(b64a.vtx[1].sha256, 0)))
         b64a = self.update_block("64a", [tx])
         assert_equal(len(b64a.serialize()), MAX_BLOCK_BASE_SIZE + 8)
-        self.sync_blocks([b64a], success=False, reject_reason='non-canonical ReadCompactSize(): iostream error')
+        self.sync_blocks([b64a], success=False, reject_reason='non-canonical ReadCompactSize():')
 
         # bitcoind doesn't disconnect us for sending a bloated block, but if we subsequently
         # resend the header message, it won't send us the getdata message again. Just

test/functional/test_framework/authproxy.py

@@ -38,6 +38,7 @@ import decimal
 import http.client
 import json
 import logging
+import os
 import socket
 import time
 import urllib.parse
@@ -71,19 +72,12 @@ class AuthServiceProxy():
         self._service_name = service_name
         self.ensure_ascii = ensure_ascii  # can be toggled on the fly by tests
         self.__url = urllib.parse.urlparse(service_url)
-        port = 80 if self.__url.port is None else self.__url.port
         user = None if self.__url.username is None else self.__url.username.encode('utf8')
         passwd = None if self.__url.password is None else self.__url.password.encode('utf8')
         authpair = user + b':' + passwd
         self.__auth_header = b'Basic ' + base64.b64encode(authpair)
-
-        if connection:
-            # Callables re-use the connection of the original proxy
-            self.__conn = connection
-        elif self.__url.scheme == 'https':
-            self.__conn = http.client.HTTPSConnection(self.__url.hostname, port, timeout=timeout)
-        else:
-            self.__conn = http.client.HTTPConnection(self.__url.hostname, port, timeout=timeout)
+        self.timeout = timeout
+        self._set_conn(connection)
 
     def __getattr__(self, name):
         if name.startswith('__') and name.endswith('__'):
@@ -102,6 +96,10 @@ class AuthServiceProxy():
                    'User-Agent': USER_AGENT,
                    'Authorization': self.__auth_header,
                    'Content-type': 'application/json'}
+        if os.name == 'nt':
+            # Windows somehow does not like to re-use connections
+            # TODO: Find out why the connection would disconnect occasionally and make it reusable on Windows
+            self._set_conn()
         try:
             self.__conn.request(method, path, postdata, headers)
             return self._get_response()
@@ -178,3 +176,13 @@ class AuthServiceProxy():
 
     def __truediv__(self, relative_uri):
         return AuthServiceProxy("{}/{}".format(self.__service_url, relative_uri), self._service_name, connection=self.__conn)
+
+    def _set_conn(self, connection=None):
+        port = 80 if self.__url.port is None else self.__url.port
+        if connection:
+            self.__conn = connection
+            self.timeout = connection.timeout
+        elif self.__url.scheme == 'https':
+            self.__conn = http.client.HTTPSConnection(self.__url.hostname, port, timeout=self.timeout)
+        else:
+            self.__conn = http.client.HTTPConnection(self.__url.hostname, port, timeout=self.timeout)

test/functional/test_runner.py

@@ -29,7 +29,7 @@ import re
 import logging
 
 # Formatting. Default colors to empty strings.
-BOLD, BLUE, RED, GREY = ("", ""), ("", ""), ("", ""), ("", "")
+BOLD, GREEN, RED, GREY = ("", ""), ("", ""), ("", ""), ("", "")
 try:
     # Make sure python thinks it can write unicode to its stdout
     "\u2713".encode("utf_8").decode(sys.stdout.encoding)
@@ -41,11 +41,27 @@ except UnicodeDecodeError:
     CROSS = "x "
     CIRCLE = "o "
 
-if os.name == 'posix':
+if os.name != 'nt' or sys.getwindowsversion() >= (10, 0, 14393):
+    if os.name == 'nt':
+        import ctypes
+        kernel32 = ctypes.windll.kernel32
+        ENABLE_VIRTUAL_TERMINAL_PROCESSING = 4
+        STD_OUTPUT_HANDLE = -11
+        STD_ERROR_HANDLE = -12
+        # Enable ascii color control to stdout
+        stdout = kernel32.GetStdHandle(STD_OUTPUT_HANDLE)
+        stdout_mode = ctypes.c_int32()
+        kernel32.GetConsoleMode(stdout, ctypes.byref(stdout_mode))
+        kernel32.SetConsoleMode(stdout, stdout_mode.value | ENABLE_VIRTUAL_TERMINAL_PROCESSING)
+        # Enable ascii color control to stderr
+        stderr = kernel32.GetStdHandle(STD_ERROR_HANDLE)
+        stderr_mode = ctypes.c_int32()
+        kernel32.GetConsoleMode(stderr, ctypes.byref(stderr_mode))
+        kernel32.SetConsoleMode(stderr, stderr_mode.value | ENABLE_VIRTUAL_TERMINAL_PROCESSING)
     # primitive formatting on supported
     # terminal via ANSI escape sequences:
     BOLD = ('\033[0m', '\033[1m')
-    BLUE = ('\033[0m', '\033[0;34m')
+    GREEN = ('\033[0m', '\033[0;32m')
     RED = ('\033[0m', '\033[0;31m')
     GREY = ('\033[0m', '\033[1;30m')
 
@@ -227,6 +243,11 @@ def main():
 
     # Create base test directory
     tmpdir = "%s/test_runner_₿_🏃_%s" % (args.tmpdirprefix, datetime.datetime.now().strftime("%Y%m%d_%H%M%S"))
+
+    # If we fixed the command-line and filename encoding issue on Windows, these two lines could be removed
+    if config["environment"]["EXEEXT"] == ".exe":
+        tmpdir = "%s/test_runner_%s" % (args.tmpdirprefix, datetime.datetime.now().strftime("%Y%m%d_%H%M%S"))
+
     os.makedirs(tmpdir)
 
     logging.debug("Temporary test directory at %s" % tmpdir)
@@ -264,7 +285,7 @@ def main():
 
     # Remove the test cases that the user has explicitly asked to exclude.
     if args.exclude:
-        exclude_tests = [re.sub("\.py$", "", test) + ".py" for test in args.exclude.split(',')]
+        exclude_tests = [re.sub("\.py$", "", test) + (".py" if ".py" not in test else "") for test in args.exclude.split(',')]
         for exclude_test in exclude_tests:
             if exclude_test in test_list:
                 test_list.remove(exclude_test)
@@ -359,7 +380,10 @@ def run_tests(test_list, src_dir, build_dir, tmpdir, jobs=1, enable_coverage=Fal
                 print('\n============')
                 print('{}Combined log for {}:{}'.format(BOLD[1], testdir, BOLD[0]))
                 print('============\n')
-                combined_logs, _ = subprocess.Popen([sys.executable, os.path.join(tests_dir, 'combine_logs.py'), '-c', testdir], universal_newlines=True, stdout=subprocess.PIPE).communicate()
+                combined_logs_args = [sys.executable, os.path.join(tests_dir, 'combine_logs.py'), testdir]
+                if BOLD[0]:
+                    combined_logs_args += ['--color']
+                combined_logs, _ = subprocess.Popen(combined_logs_args, universal_newlines=True, stdout=subprocess.PIPE).communicate()
                 print("\n".join(deque(combined_logs.splitlines(), combined_logs_len)))
 
             if failfast:
@@ -498,7 +522,7 @@ class TestResult():
 
     def __repr__(self):
         if self.status == "Passed":
-            color = BLUE
+            color = GREEN
             glyph = TICK
         elif self.status == "Failed":
             color = RED

test/functional/wallet_multiwallet.py

@@ -44,8 +44,9 @@ class MultiWalletTest(BitcoinTestFramework):
 
         # create symlink to verify wallet directory path can be referenced
         # through symlink
-        os.mkdir(wallet_dir('w7'))
-        os.symlink('w7', wallet_dir('w7_symlink'))
+        if os.name != 'nt':
+            os.mkdir(wallet_dir('w7'))
+            os.symlink('w7', wallet_dir('w7_symlink'))
 
         # rename wallet.dat to make sure plain wallet file paths (as opposed to
         # directory paths) can be loaded
@@ -66,6 +67,8 @@ class MultiWalletTest(BitcoinTestFramework):
         #   w8         - to verify existing wallet file is loaded correctly
         #   ''         - to verify default wallet file is created correctly
         wallet_names = ['w1', 'w2', 'w3', 'w', 'sub/w5', os.path.join(self.options.tmpdir, 'extern/w6'), 'w7_symlink', 'w8', '']
+        if os.name == 'nt':
+            wallet_names.remove('w7_symlink')
         extra_args = ['-wallet={}'.format(n) for n in wallet_names]
         self.start_node(0, extra_args)
         assert_equal(set(node.listwallets()), set(wallet_names))
@@ -76,7 +79,7 @@ class MultiWalletTest(BitcoinTestFramework):
             assert_equal(os.path.isfile(wallet_file(wallet_name)), True)
 
         # should not initialize if wallet path can't be created
-        exp_stderr = "boost::filesystem::create_directory: (The system cannot find the path specified|Not a directory):"
+        exp_stderr = "boost::filesystem::create_directory:"
         self.nodes[0].assert_start_raises_init_error(['-wallet=wallet.dat/bad'], exp_stderr, match=ErrorMatch.PARTIAL_REGEX)
 
         self.nodes[0].assert_start_raises_init_error(['-walletdir=wallets'], 'Error: Specified -walletdir "wallets" does not exist')
@@ -92,8 +95,9 @@ class MultiWalletTest(BitcoinTestFramework):
         self.nodes[0].assert_start_raises_init_error(['-wallet=w8', '-wallet=w8_copy'], exp_stderr, match=ErrorMatch.PARTIAL_REGEX)
 
         # should not initialize if wallet file is a symlink
-        os.symlink('w8', wallet_dir('w8_symlink'))
-        self.nodes[0].assert_start_raises_init_error(['-wallet=w8_symlink'], 'Error: Invalid -wallet path \'w8_symlink\'\. .*', match=ErrorMatch.FULL_REGEX)
+        if os.name != 'nt':
+            os.symlink('w8', wallet_dir('w8_symlink'))
+            self.nodes[0].assert_start_raises_init_error(['-wallet=w8_symlink'], 'Error: Invalid -wallet path \'w8_symlink\'\. .*', match=ErrorMatch.FULL_REGEX)
 
         # should not initialize if the specified walletdir does not exist
         self.nodes[0].assert_start_raises_init_error(['-walletdir=bad'], 'Error: Specified -walletdir "bad" does not exist')
@@ -220,7 +224,8 @@ class MultiWalletTest(BitcoinTestFramework):
         assert_raises_rpc_error(-1, "BerkeleyBatch: Can't open database w8_copy (duplicates fileid", self.nodes[0].loadwallet, 'w8_copy')
 
         # Fail to load if wallet file is a symlink
-        assert_raises_rpc_error(-4, "Wallet file verification failed: Invalid -wallet path 'w8_symlink'", self.nodes[0].loadwallet, 'w8_symlink')
+        if os.name != 'nt':
+            assert_raises_rpc_error(-4, "Wallet file verification failed: Invalid -wallet path 'w8_symlink'", self.nodes[0].loadwallet, 'w8_symlink')
 
         # Fail to load if a directory is specified that doesn't contain a wallet
         os.mkdir(wallet_dir('empty_wallet_dir'))