LBRYCommunity/lbrycrd
1
0
Fork 0
Code Issues 29 Pull requests 1 Projects Releases 32 Packages Wiki Activity

Merge #14128: lint: Make sure we read the command line inputs using utf-8 decoding in python

Browse source 
5d62dcf9cf lint: Make sure we read the command line inputs using utf-8 decoding in python (Chun Kuan Lee)

Pull request description:

  Make sure we read the command line inputs using utf-8 decoding in python

  occurred from travis cron job:

  contrib/verify-commits/verify-commits.py should run with utf-8, otherwise it would raise UnicodeDecodeError
  `UnicodeDecodeError: 'ascii' codec can't decode byte 0xe2 in position 744: ordinal not in range(128)`

Tree-SHA512: 90e4ad57fdbbbecb0a21fc2d2b03a04f5ef125e54124719ef36e5a85326930b732b47534757a7c3a8730096f3947b009ec898191928b5c2d38f9f4b3e37db48d
This commit is contained in:
Wladimir J. van der Laan 2018-09-04 12:31:00 +02:00
commit fa616275b2
No known key found for this signature in database
GPG key ID: 1E4AED62986CD25D
5 changed files with 20 additions and 12 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
contrib/devtools/optimize-pngs.py
View file

@ -27,7 +27,7 @@ def content_hash(filename):
pngcrush = 'pngcrush' pngcrush = 'pngcrush'
git = 'git' git = 'git'
folders = ["src/qt/res/movies", "src/qt/res/icons", "share/pixmaps"] folders = ["src/qt/res/movies", "src/qt/res/icons", "share/pixmaps"]
basePath = subprocess.check_output([git, 'rev-parse', '--show-toplevel'], universal_newlines=True).rstrip('\n') basePath = subprocess.check_output([git, 'rev-parse', '--show-toplevel'], universal_newlines=True, encoding='utf8').rstrip('\n')
totalSaveBytes = 0 totalSaveBytes = 0
noHashChange = True noHashChange = True
@ -50,7 +50,7 @@ for folder in folders:
sys.exit(0) sys.exit(0)
#verify #verify
if "Not a PNG file" in subprocess.check_output([pngcrush, "-n", "-v", file_path], stderr=subprocess.STDOUT, universal_newlines=True): if "Not a PNG file" in subprocess.check_output([pngcrush, "-n", "-v", file_path], stderr=subprocess.STDOUT, universal_newlines=True, encoding='utf8'):
print("PNG file "+file+" is corrupted after crushing, check out pngcursh version") print("PNG file "+file+" is corrupted after crushing, check out pngcursh version")
sys.exit(1) sys.exit(1)

2
contrib/gitian-build.py
View file

@ -209,7 +209,7 @@ def main():
subprocess.check_call(['git', 'fetch', args.url, 'refs/pull/'+args.version+'/merge']) subprocess.check_call(['git', 'fetch', args.url, 'refs/pull/'+args.version+'/merge'])
os.chdir('../gitian-builder/inputs/bitcoin') os.chdir('../gitian-builder/inputs/bitcoin')
subprocess.check_call(['git', 'fetch', args.url, 'refs/pull/'+args.version+'/merge']) subprocess.check_call(['git', 'fetch', args.url, 'refs/pull/'+args.version+'/merge'])
args.commit = subprocess.check_output(['git', 'show', '-s', '--format=%H', 'FETCH_HEAD'], universal_newlines=True).strip() args.commit = subprocess.check_output(['git', 'show', '-s', '--format=%H', 'FETCH_HEAD'], universal_newlines=True, encoding='utf8').strip()
args.version = 'pull-' + args.version args.version = 'pull-' + args.version
print(args.commit) print(args.commit)
subprocess.check_call(['git', 'fetch']) subprocess.check_call(['git', 'fetch'])

14
contrib/verify-commits/verify-commits.py
View file

@ -91,7 +91,7 @@ def main():
no_sha1 = True no_sha1 = True
prev_commit = "" prev_commit = ""
initial_commit = current_commit initial_commit = current_commit
branch = subprocess.check_output([GIT, 'show', '-s', '--format=%H', initial_commit], universal_newlines=True).splitlines()[0] branch = subprocess.check_output([GIT, 'show', '-s', '--format=%H', initial_commit], universal_newlines=True, encoding='utf8').splitlines()[0]
# Iterate through commits # Iterate through commits
while True: while True:
@ -112,7 +112,7 @@ def main():
if prev_commit != "": if prev_commit != "":
print("No parent of {} was signed with a trusted key!".format(prev_commit), file=sys.stderr) print("No parent of {} was signed with a trusted key!".format(prev_commit), file=sys.stderr)
print("Parents are:", file=sys.stderr) print("Parents are:", file=sys.stderr)
parents = subprocess.check_output([GIT, 'show', '-s', '--format=format:%P', prev_commit], universal_newlines=True).splitlines()[0].split(' ') parents = subprocess.check_output([GIT, 'show', '-s', '--format=format:%P', prev_commit], universal_newlines=True, encoding='utf8').splitlines()[0].split(' ')
for parent in parents: for parent in parents:
subprocess.call([GIT, 'show', '-s', parent], stdout=sys.stderr) subprocess.call([GIT, 'show', '-s', parent], stdout=sys.stderr)
else: else:
@ -122,25 +122,25 @@ def main():
# Check the Tree-SHA512 # Check the Tree-SHA512
if (verify_tree or prev_commit == "") and current_commit not in incorrect_sha512_allowed: if (verify_tree or prev_commit == "") and current_commit not in incorrect_sha512_allowed:
tree_hash = tree_sha512sum(current_commit) tree_hash = tree_sha512sum(current_commit)
if ("Tree-SHA512: {}".format(tree_hash)) not in subprocess.check_output([GIT, 'show', '-s', '--format=format:%B', current_commit], universal_newlines=True).splitlines(): if ("Tree-SHA512: {}".format(tree_hash)) not in subprocess.check_output([GIT, 'show', '-s', '--format=format:%B', current_commit], universal_newlines=True, encoding='utf8').splitlines():
print("Tree-SHA512 did not match for commit " + current_commit, file=sys.stderr) print("Tree-SHA512 did not match for commit " + current_commit, file=sys.stderr)
sys.exit(1) sys.exit(1)
# Merge commits should only have two parents # Merge commits should only have two parents
parents = subprocess.check_output([GIT, 'show', '-s', '--format=format:%P', current_commit], universal_newlines=True).splitlines()[0].split(' ') parents = subprocess.check_output([GIT, 'show', '-s', '--format=format:%P', current_commit], universal_newlines=True, encoding='utf8').splitlines()[0].split(' ')
if len(parents) > 2: if len(parents) > 2:
print("Commit {} is an octopus merge".format(current_commit), file=sys.stderr) print("Commit {} is an octopus merge".format(current_commit), file=sys.stderr)
sys.exit(1) sys.exit(1)
# Check that the merge commit is clean # Check that the merge commit is clean
commit_time = int(subprocess.check_output([GIT, 'show', '-s', '--format=format:%ct', current_commit], universal_newlines=True).splitlines()[0]) commit_time = int(subprocess.check_output([GIT, 'show', '-s', '--format=format:%ct', current_commit], universal_newlines=True, encoding='utf8').splitlines()[0])
check_merge = commit_time > time.time() - args.clean_merge * 24 * 60 * 60 # Only check commits in clean_merge days check_merge = commit_time > time.time() - args.clean_merge * 24 * 60 * 60 # Only check commits in clean_merge days
allow_unclean = current_commit in unclean_merge_allowed allow_unclean = current_commit in unclean_merge_allowed
if len(parents) == 2 and check_merge and not allow_unclean: if len(parents) == 2 and check_merge and not allow_unclean:
current_tree = subprocess.check_output([GIT, 'show', '--format=%T', current_commit], universal_newlines=True).splitlines()[0] current_tree = subprocess.check_output([GIT, 'show', '--format=%T', current_commit], universal_newlines=True, encoding='utf8').splitlines()[0]
subprocess.call([GIT, 'checkout', '--force', '--quiet', parents[0]]) subprocess.call([GIT, 'checkout', '--force', '--quiet', parents[0]])
subprocess.call([GIT, 'merge', '--no-ff', '--quiet', parents[1]], stdout=subprocess.DEVNULL) subprocess.call([GIT, 'merge', '--no-ff', '--quiet', parents[1]], stdout=subprocess.DEVNULL)
recreated_tree = subprocess.check_output([GIT, 'show', '--format=format:%T', 'HEAD'], universal_newlines=True).splitlines()[0] recreated_tree = subprocess.check_output([GIT, 'show', '--format=format:%T', 'HEAD'], universal_newlines=True, encoding='utf8').splitlines()[0]
if current_tree != recreated_tree: if current_tree != recreated_tree:
print("Merge commit {} is not clean".format(current_commit), file=sys.stderr) print("Merge commit {} is not clean".format(current_commit), file=sys.stderr)
subprocess.call([GIT, 'diff', current_commit]) subprocess.call([GIT, 'diff', current_commit])

4
test/lint/check-doc.py
View file

@ -26,8 +26,8 @@ SET_DOC_OPTIONAL = set(['-rpcssl', '-benchmark', '-h', '-help', '-socks', '-tor'
def main(): def main():
used = check_output(CMD_GREP_ARGS, shell=True, universal_newlines=True) used = check_output(CMD_GREP_ARGS, shell=True, universal_newlines=True, encoding='utf8')
docd = check_output(CMD_GREP_DOCS, shell=True, universal_newlines=True) docd = check_output(CMD_GREP_DOCS, shell=True, universal_newlines=True, encoding='utf8')
args_used = set(re.findall(re.compile(REGEX_ARG), used)) args_used = set(re.findall(re.compile(REGEX_ARG), used))
args_docd = set(re.findall(re.compile(REGEX_DOC), docd)).union(SET_DOC_OPTIONAL) args_docd = set(re.findall(re.compile(REGEX_DOC), docd)).union(SET_DOC_OPTIONAL)

8
test/lint/lint-python-utf8-encoding.sh
View file

@ -17,4 +17,12 @@ if [[ ${OUTPUT} != "" ]]; then
echo "${OUTPUT}" echo "${OUTPUT}"
EXIT_CODE=1 EXIT_CODE=1
fi fi
OUTPUT=$(git grep "check_output(" -- "*.py" | grep "universal_newlines=True" | grep -vE "encoding=.(ascii|utf8|utf-8).")
if [[ ${OUTPUT} != "" ]]; then
echo "Python's check_output(...) seems to be used to get program outputs without explicitly"
echo "specifying encoding=\"utf8\":"
echo
echo "${OUTPUT}"
EXIT_CODE=1
fi
exit ${EXIT_CODE} exit ${EXIT_CODE}