diff --git a/contrib/devtools/circular-dependencies.py b/contrib/devtools/circular-dependencies.py index d544d5c37..abfa5ed5a 100755 --- a/contrib/devtools/circular-dependencies.py +++ b/contrib/devtools/circular-dependencies.py @@ -37,7 +37,7 @@ for arg in sys.argv[1:]: # TODO: implement support for multiple include directories for arg in sorted(files.keys()): module = files[arg] - with open(arg, 'r') as f: + with open(arg, 'r', encoding="utf8") as f: for line in f: match = RE.match(line) if match: diff --git a/contrib/devtools/clang-format-diff.py b/contrib/devtools/clang-format-diff.py index 5402870fb..77e845a9b 100755 --- a/contrib/devtools/clang-format-diff.py +++ b/contrib/devtools/clang-format-diff.py @@ -152,7 +152,7 @@ def main(): sys.exit(p.returncode) if not args.i: - with open(filename) as f: + with open(filename, encoding="utf8") as f: code = f.readlines() formatted_code = io.StringIO(stdout).readlines() diff = difflib.unified_diff(code, formatted_code, diff --git a/contrib/devtools/copyright_header.py b/contrib/devtools/copyright_header.py index 82d3c1968..da7d74bdc 100755 --- a/contrib/devtools/copyright_header.py +++ b/contrib/devtools/copyright_header.py @@ -146,7 +146,7 @@ def file_has_without_c_style_copyright_for_holder(contents, holder_name): ################################################################################ def read_file(filename): - return open(os.path.abspath(filename), 'r').read() + return open(os.path.abspath(filename), 'r', encoding="utf8").read() def gather_file_info(filename): info = {} @@ -325,13 +325,13 @@ def get_most_recent_git_change_year(filename): ################################################################################ def read_file_lines(filename): - f = open(os.path.abspath(filename), 'r') + f = open(os.path.abspath(filename), 'r', encoding="utf8") file_lines = f.readlines() f.close() return file_lines def write_file_lines(filename, file_lines): - f = open(os.path.abspath(filename), 'w') + f = open(os.path.abspath(filename), 'w', encoding="utf8") f.write(''.join(file_lines)) f.close() diff --git a/contrib/devtools/github-merge.py b/contrib/devtools/github-merge.py index 187ef75fb..4e90f85f5 100755 --- a/contrib/devtools/github-merge.py +++ b/contrib/devtools/github-merge.py @@ -191,7 +191,7 @@ def main(): merge_branch = 'pull/'+pull+'/merge' local_merge_branch = 'pull/'+pull+'/local-merge' - devnull = open(os.devnull,'w') + devnull = open(os.devnull, 'w', encoding="utf8") try: subprocess.check_call([GIT,'checkout','-q',branch]) except subprocess.CalledProcessError: diff --git a/contrib/devtools/test-security-check.py b/contrib/devtools/test-security-check.py index 307e05773..9b6d6bf66 100755 --- a/contrib/devtools/test-security-check.py +++ b/contrib/devtools/test-security-check.py @@ -9,7 +9,7 @@ import subprocess import unittest def write_testcode(filename): - with open(filename, 'w') as f: + with open(filename, 'w', encoding="utf8") as f: f.write(''' #include int main() diff --git a/contrib/filter-lcov.py b/contrib/filter-lcov.py index 299377d69..df1db76e9 100755 --- a/contrib/filter-lcov.py +++ b/contrib/filter-lcov.py @@ -13,8 +13,8 @@ pattern = args.pattern outfile = args.outfile in_remove = False -with open(tracefile, 'r') as f: - with open(outfile, 'w') as wf: +with open(tracefile, 'r', encoding="utf8") as f: + with open(outfile, 'w', encoding="utf8") as wf: for line in f: for p in pattern: if line.startswith("SF:") and p in line: diff --git a/contrib/linearize/linearize-data.py b/contrib/linearize/linearize-data.py index f8aea2734..b501388fd 100755 --- a/contrib/linearize/linearize-data.py +++ b/contrib/linearize/linearize-data.py @@ -75,7 +75,7 @@ def get_blk_dt(blk_hdr): # When getting the list of block hashes, undo any byte reversals. def get_block_hashes(settings): blkindex = [] - f = open(settings['hashlist'], "r") + f = open(settings['hashlist'], "r", encoding="utf8") for line in f: line = line.rstrip() if settings['rev_hash_bytes'] == 'true': @@ -261,7 +261,7 @@ if __name__ == '__main__': print("Usage: linearize-data.py CONFIG-FILE") sys.exit(1) - f = open(sys.argv[1]) + f = open(sys.argv[1], encoding="utf8") for line in f: # skip comment lines m = re.search('^\s*#', line) diff --git a/contrib/linearize/linearize-hashes.py b/contrib/linearize/linearize-hashes.py index 8e1266ae0..bfd217194 100755 --- a/contrib/linearize/linearize-hashes.py +++ b/contrib/linearize/linearize-hashes.py @@ -96,7 +96,7 @@ def get_block_hashes(settings, max_blocks_per_call=10000): def get_rpc_cookie(): # Open the cookie file - with open(os.path.join(os.path.expanduser(settings['datadir']), '.cookie'), 'r') as f: + with open(os.path.join(os.path.expanduser(settings['datadir']), '.cookie'), 'r', encoding="ascii") as f: combined = f.readline() combined_split = combined.split(":") settings['rpcuser'] = combined_split[0] @@ -107,7 +107,7 @@ if __name__ == '__main__': print("Usage: linearize-hashes.py CONFIG-FILE") sys.exit(1) - f = open(sys.argv[1]) + f = open(sys.argv[1], encoding="utf8") for line in f: # skip comment lines m = re.search('^\s*#', line) diff --git a/contrib/seeds/generate-seeds.py b/contrib/seeds/generate-seeds.py index ace7d3534..fe7cd1d59 100755 --- a/contrib/seeds/generate-seeds.py +++ b/contrib/seeds/generate-seeds.py @@ -127,10 +127,10 @@ def main(): g.write(' * Each line contains a 16-byte IPv6 address and a port.\n') g.write(' * IPv4 as well as onion addresses are wrapped inside an IPv6 address accordingly.\n') g.write(' */\n') - with open(os.path.join(indir,'nodes_main.txt'),'r') as f: + with open(os.path.join(indir,'nodes_main.txt'), 'r', encoding="utf8") as f: process_nodes(g, f, 'pnSeed6_main', 8333) g.write('\n') - with open(os.path.join(indir,'nodes_test.txt'),'r') as f: + with open(os.path.join(indir,'nodes_test.txt'), 'r', encoding="utf8") as f: process_nodes(g, f, 'pnSeed6_test', 18333) g.write('#endif // BITCOIN_CHAINPARAMSSEEDS_H\n') diff --git a/contrib/verify-commits/verify-commits.py b/contrib/verify-commits/verify-commits.py index 80f0aa0bf..a9e497771 100755 --- a/contrib/verify-commits/verify-commits.py +++ b/contrib/verify-commits/verify-commits.py @@ -76,11 +76,11 @@ def main(): # get directory of this program and read data files dirname = os.path.dirname(os.path.abspath(__file__)) print("Using verify-commits data from " + dirname) - verified_root = open(dirname + "/trusted-git-root", "r").read().splitlines()[0] - verified_sha512_root = open(dirname + "/trusted-sha512-root-commit", "r").read().splitlines()[0] - revsig_allowed = open(dirname + "/allow-revsig-commits", "r").read().splitlines() - unclean_merge_allowed = open(dirname + "/allow-unclean-merge-commits", "r").read().splitlines() - incorrect_sha512_allowed = open(dirname + "/allow-incorrect-sha512-commits", "r").read().splitlines() + verified_root = open(dirname + "/trusted-git-root", "r", encoding="utf8").read().splitlines()[0] + verified_sha512_root = open(dirname + "/trusted-sha512-root-commit", "r", encoding="utf8").read().splitlines()[0] + revsig_allowed = open(dirname + "/allow-revsig-commits", "r", encoding="utf-8").read().splitlines() + unclean_merge_allowed = open(dirname + "/allow-unclean-merge-commits", "r", encoding="utf-8").read().splitlines() + incorrect_sha512_allowed = open(dirname + "/allow-incorrect-sha512-commits", "r", encoding="utf-8").read().splitlines() # Set commit and branch and set variables current_commit = args.commit diff --git a/share/qt/extract_strings_qt.py b/share/qt/extract_strings_qt.py index e8f0820ca..e90807132 100755 --- a/share/qt/extract_strings_qt.py +++ b/share/qt/extract_strings_qt.py @@ -63,7 +63,7 @@ child = Popen([XGETTEXT,'--output=-','-n','--keyword=_'] + files, stdout=PIPE) messages = parse_po(out.decode('utf-8')) -f = open(OUT_CPP, 'w') +f = open(OUT_CPP, 'w', encoding="utf8") f.write(""" #include diff --git a/test/functional/feature_notifications.py b/test/functional/feature_notifications.py index 8964c8d64..6d51f31e3 100755 --- a/test/functional/feature_notifications.py +++ b/test/functional/feature_notifications.py @@ -36,7 +36,7 @@ class NotificationsTest(BitcoinTestFramework): wait_until(lambda: os.path.isfile(self.block_filename) and os.stat(self.block_filename).st_size >= (block_count * 65), timeout=10) # file content should equal the generated blocks hashes - with open(self.block_filename, 'r') as f: + with open(self.block_filename, 'r', encoding="utf-8") as f: assert_equal(sorted(blocks), sorted(l.strip() for l in f.read().splitlines())) self.log.info("test -walletnotify") @@ -45,7 +45,7 @@ class NotificationsTest(BitcoinTestFramework): # file content should equal the generated transaction hashes txids_rpc = list(map(lambda t: t['txid'], self.nodes[1].listtransactions("*", block_count))) - with open(self.tx_filename, 'r') as f: + with open(self.tx_filename, 'r', encoding="ascii") as f: assert_equal(sorted(txids_rpc), sorted(l.strip() for l in f.read().splitlines())) os.remove(self.tx_filename) @@ -58,7 +58,7 @@ class NotificationsTest(BitcoinTestFramework): # file content should equal the generated transaction hashes txids_rpc = list(map(lambda t: t['txid'], self.nodes[1].listtransactions("*", block_count))) - with open(self.tx_filename, 'r') as f: + with open(self.tx_filename, 'r', encoding="ascii") as f: assert_equal(sorted(txids_rpc), sorted(l.strip() for l in f.read().splitlines())) # Mine another 41 up-version blocks. -alertnotify should trigger on the 51st. diff --git a/test/functional/rpc_getblockstats.py b/test/functional/rpc_getblockstats.py index 060a2373b..f573faaf1 100755 --- a/test/functional/rpc_getblockstats.py +++ b/test/functional/rpc_getblockstats.py @@ -81,11 +81,11 @@ class GetblockstatsTest(BitcoinTestFramework): 'mocktime': int(mocktime), 'stats': self.expected_stats, } - with open(filename, 'w') as f: + with open(filename, 'w', encoding="utf8") as f: json.dump(to_dump, f, sort_keys=True, indent=2) def load_test_data(self, filename): - with open(filename, 'r') as f: + with open(filename, 'r', encoding="utf8") as f: d = json.load(f) blocks = d['blocks'] mocktime = d['mocktime'] diff --git a/test/functional/test_framework/util.py b/test/functional/test_framework/util.py index e016148f7..5e0b61b5e 100644 --- a/test/functional/test_framework/util.py +++ b/test/functional/test_framework/util.py @@ -327,7 +327,7 @@ def get_auth_cookie(datadir): assert password is None # Ensure that there is only one rpcpassword line password = line.split("=")[1].strip("\n") if os.path.isfile(os.path.join(datadir, "regtest", ".cookie")): - with open(os.path.join(datadir, "regtest", ".cookie"), 'r') as f: + with open(os.path.join(datadir, "regtest", ".cookie"), 'r', encoding="ascii") as f: userpass = f.read() split_userpass = userpass.split(':') user = split_userpass[0] diff --git a/test/functional/test_runner.py b/test/functional/test_runner.py index 5b3a4df0f..36101d9f5 100755 --- a/test/functional/test_runner.py +++ b/test/functional/test_runner.py @@ -213,7 +213,7 @@ def main(): # Read config generated by configure. config = configparser.ConfigParser() configfile = os.path.abspath(os.path.dirname(__file__)) + "/../config.ini" - config.read_file(open(configfile)) + config.read_file(open(configfile, encoding="utf8")) passon_args.append("--configfile=%s" % configfile) @@ -590,7 +590,7 @@ class RPCCoverage(): if not os.path.isfile(coverage_ref_filename): raise RuntimeError("No coverage reference found") - with open(coverage_ref_filename, 'r') as coverage_ref_file: + with open(coverage_ref_filename, 'r', encoding="utf8") as coverage_ref_file: all_cmds.update([line.strip() for line in coverage_ref_file.readlines()]) for root, dirs, files in os.walk(self.dir): @@ -599,7 +599,7 @@ class RPCCoverage(): coverage_filenames.add(os.path.join(root, filename)) for filename in coverage_filenames: - with open(filename, 'r') as coverage_file: + with open(filename, 'r', encoding="utf8") as coverage_file: covered_cmds.update([line.strip() for line in coverage_file.readlines()]) return all_cmds - covered_cmds diff --git a/test/functional/wallet_multiwallet.py b/test/functional/wallet_multiwallet.py index 53638615f..a0fbc4a75 100755 --- a/test/functional/wallet_multiwallet.py +++ b/test/functional/wallet_multiwallet.py @@ -88,7 +88,7 @@ class MultiWalletTest(BitcoinTestFramework): self.nodes[0].assert_start_raises_init_error(['-walletdir=bad'], 'Error: Specified -walletdir "bad" does not exist') # should not initialize if the specified walletdir is not a directory not_a_dir = wallet_dir('notadir') - open(not_a_dir, 'a').close() + open(not_a_dir, 'a', encoding="utf8").close() self.nodes[0].assert_start_raises_init_error(['-walletdir=' + not_a_dir], 'Error: Specified -walletdir "' + not_a_dir + '" is not a directory') self.log.info("Do not allow -zapwallettxes with multiwallet") diff --git a/test/lint/check-rpc-mappings.py b/test/lint/check-rpc-mappings.py index 7e96852c5..c3cdeef58 100755 --- a/test/lint/check-rpc-mappings.py +++ b/test/lint/check-rpc-mappings.py @@ -44,7 +44,7 @@ def process_commands(fname): """Find and parse dispatch table in implementation file `fname`.""" cmds = [] in_rpcs = False - with open(fname, "r") as f: + with open(fname, "r", encoding="utf8") as f: for line in f: line = line.rstrip() if not in_rpcs: @@ -70,7 +70,7 @@ def process_mapping(fname): """Find and parse conversion table in implementation file `fname`.""" cmds = [] in_rpcs = False - with open(fname, "r") as f: + with open(fname, "r", encoding="utf8") as f: for line in f: line = line.rstrip() if not in_rpcs: diff --git a/test/lint/lint-python-utf8-encoding.sh b/test/lint/lint-python-utf8-encoding.sh new file mode 100755 index 000000000..ce973e710 --- /dev/null +++ b/test/lint/lint-python-utf8-encoding.sh @@ -0,0 +1,19 @@ +#!/bin/bash +# +# Copyright (c) 2018 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +# +# Make sure we explicitly open all text files using UTF-8 (or ASCII) encoding to +# avoid potential issues on the BSDs where the locale is not always set. + +EXIT_CODE=0 +OUTPUT=$(git grep " open(" -- "*.py" | grep -vE "encoding=.(ascii|utf8|utf-8)." | grep -vE "open\([^,]*, ['\"][^'\"]*b[^'\"]*['\"]") +if [[ ${OUTPUT} != "" ]]; then + echo "Python's open(...) seems to be used to open text files without explicitly" + echo "specifying encoding=\"utf8\":" + echo + echo "${OUTPUT}" + EXIT_CODE=1 +fi +exit ${EXIT_CODE} diff --git a/test/util/bitcoin-util-test.py b/test/util/bitcoin-util-test.py index 30bd13d0d..16371a623 100755 --- a/test/util/bitcoin-util-test.py +++ b/test/util/bitcoin-util-test.py @@ -28,7 +28,7 @@ import sys def main(): config = configparser.ConfigParser() config.optionxform = str - config.readfp(open(os.path.join(os.path.dirname(__file__), "../config.ini"))) + config.readfp(open(os.path.join(os.path.dirname(__file__), "../config.ini"), encoding="utf8")) env_conf = dict(config.items('environment')) parser = argparse.ArgumentParser(description=__doc__) @@ -49,7 +49,7 @@ def main(): def bctester(testDir, input_basename, buildenv): """ Loads and parses the input file, runs all tests and reports results""" input_filename = os.path.join(testDir, input_basename) - raw_data = open(input_filename).read() + raw_data = open(input_filename, encoding="utf8").read() input_data = json.loads(raw_data) failed_testcases = [] @@ -86,7 +86,7 @@ def bctest(testDir, testObj, buildenv): inputData = None if "input" in testObj: filename = os.path.join(testDir, testObj["input"]) - inputData = open(filename).read() + inputData = open(filename, encoding="utf8").read() stdinCfg = subprocess.PIPE # Read the expected output data (if there is any) @@ -97,7 +97,7 @@ def bctest(testDir, testObj, buildenv): outputFn = testObj['output_cmp'] outputType = os.path.splitext(outputFn)[1][1:] # output type from file extension (determines how to compare) try: - outputData = open(os.path.join(testDir, outputFn)).read() + outputData = open(os.path.join(testDir, outputFn), encoding="utf8").read() except: logging.error("Output file " + outputFn + " can not be opened") raise diff --git a/test/util/rpcauth-test.py b/test/util/rpcauth-test.py index 2456feb10..46e9fbc73 100755 --- a/test/util/rpcauth-test.py +++ b/test/util/rpcauth-test.py @@ -18,7 +18,7 @@ class TestRPCAuth(unittest.TestCase): config_path = os.path.abspath( os.path.join(os.sep, os.path.abspath(os.path.dirname(__file__)), "../config.ini")) - with open(config_path) as config_file: + with open(config_path, encoding="utf8") as config_file: config.read_file(config_file) sys.path.insert(0, os.path.dirname(config['environment']['RPCAUTH'])) self.rpcauth = importlib.import_module('rpcauth')