From 8416d46fd4f70da155f62640443c30cf71a0e52f Mon Sep 17 00:00:00 2001 From: Richard Larkin Date: Mon, 27 Apr 2020 07:29:11 +0200 Subject: [PATCH] Fix flake8 CI check on toolchain, remaining recipes (#456) * :fire: Fix flake8 checks for host_setuptools3 * :fire: Fix flake8 checks for kivy recipe * :sparkles: Fix flake8 for toolchain.py * :bulb: Set python3 as the default python * Add .tox and ven to .gitignore * Update toolchain.py Co-Authored-By: Andre Miras Co-authored-by: Andre Miras --- .gitignore | 2 + recipes/host_setuptools3/__init__.py | 6 +-- recipes/kivy/__init__.py | 6 +-- recipes/python.py | 11 +---- toolchain.py | 71 ++++++++++++---------------- tox.ini | 5 -- 6 files changed, 38 insertions(+), 63 deletions(-) diff --git a/.gitignore b/.gitignore index 445896d..554a380 100644 --- a/.gitignore +++ b/.gitignore @@ -19,3 +19,5 @@ src/ios/ios.c *.DS_Store* *-ios/ __pycache__ +.tox +venv diff --git a/recipes/host_setuptools3/__init__.py b/recipes/host_setuptools3/__init__.py index c7c1764..4952398 100644 --- a/recipes/host_setuptools3/__init__.py +++ b/recipes/host_setuptools3/__init__.py @@ -1,8 +1,7 @@ from toolchain import Recipe, shprint -from os.path import join, exists +from os.path import join import sh import os -import fnmatch import shutil @@ -13,7 +12,7 @@ class HostSetuptools3(Recipe): def prebuild_arch(self, arch): hostpython = sh.Command(self.ctx.hostpython) - sh.curl("-O", "https://bootstrap.pypa.io/ez_setup.py") + sh.curl("-O", "https://bootstrap.pypa.io/ez_setup.py") shprint(hostpython, "./ez_setup.py") # Extract setuptools egg and remove .pth files. Otherwise subsequent # python package installations using setuptools will raise exceptions. @@ -32,4 +31,5 @@ class HostSetuptools3(Recipe): os.remove('easy-install.pth') shutil.rmtree('EGG-INFO') + recipe = HostSetuptools3() diff --git a/recipes/kivy/__init__.py b/recipes/kivy/__init__.py index 50b4909..01fdef4 100644 --- a/recipes/kivy/__init__.py +++ b/recipes/kivy/__init__.py @@ -1,7 +1,5 @@ -from toolchain import CythonRecipe, shprint +from toolchain import CythonRecipe from os.path import join -from os import chdir, listdir, getcwd -import sh import logging import shutil @@ -39,6 +37,7 @@ class KivyRecipe(CythonRecipe): def _patch_setup(self): # patch setup to remove some functionnalities pyconfig = join(self.build_dir, "setup.py") + def _remove_line(lines, pattern): for line in lines[:]: if pattern in line: @@ -46,7 +45,6 @@ class KivyRecipe(CythonRecipe): with open(pyconfig) as fd: lines = fd.readlines() _remove_line(lines, "flags['libraries'] = ['GLESv2']") - #_remove_line(lines, "c_options['use_sdl'] = True") with open(pyconfig, "w") as fd: fd.writelines(lines) diff --git a/recipes/python.py b/recipes/python.py index 2af00b8..7159a41 100644 --- a/recipes/python.py +++ b/recipes/python.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -import sys from toolchain import Recipe import logging from os.path import join @@ -16,15 +15,9 @@ class PythonAliasRecipe(Recipe): # search in wanted_recipes if it's the first time if "python2" in ctx.wanted_recipes: python = "python2" - elif "python3" in ctx.wanted_recipes: - python = "python3" else: - logger.error("No Python version set in the build.") - logger.error("Add python2 or python3 in your recipes:") - logger.error("./toolchain.py build python3 ...") - sys.exit(1) - if python: - self.depends = [python] + python = "python3" + self.depends = [python] self.recipe_dir = join(ctx.root_dir, "recipes", python) diff --git a/toolchain.py b/toolchain.py index dcbcdd4..9c17d54 100755 --- a/toolchain.py +++ b/toolchain.py @@ -9,7 +9,7 @@ This tool intend to replace all the previous tools/ in shell script. import sys from sys import stdout from os.path import join, dirname, realpath, exists, isdir, basename -from os import listdir, unlink, makedirs, environ, chdir, getcwd, walk, remove +from os import listdir, unlink, makedirs, environ, chdir, getcwd, walk import zipfile import tarfile import importlib @@ -21,6 +21,7 @@ import tempfile import time from datetime import datetime from pprint import pformat +import logging try: from urllib.request import FancyURLopener, urlcleanup @@ -39,7 +40,6 @@ sys.path.insert(0, join(curdir, "tools", "external")) import sh -import logging # For more detailed logging, use something like # format='%(asctime)s,%(msecs)d %(levelname)-8s [%(filename)s:%(funcName)s():%(lineno)d] %(message)s' @@ -69,6 +69,7 @@ def shprint(command, *args, **kwargs): line_str = "\n".join(line.encode("ascii", "replace").decode().splitlines()) logger.debug(line_str) + def cache_execution(f): def _cache_execution(self, *args, **kwargs): state = self.ctx.state @@ -85,11 +86,13 @@ def cache_execution(f): self.update_state(key, True) return _cache_execution + class ChromeDownloader(FancyURLopener): version = ( 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 ' '(KHTML, like Gecko) Chrome/28.0.1500.71 Safari/537.36') + urlretrieve = ChromeDownloader().retrieve @@ -136,13 +139,15 @@ class JsonStore(object): self.sync() def sync(self): - # http://stackoverflow.com/questions/12309269/write-json-data-to-file-in-python/14870531#14870531 + # https://stackoverflow.com/a/14870531/185510 if IS_PY3: with open(self.filename, 'w') as fd: json.dump(self.data, fd, ensure_ascii=False) else: with io.open(self.filename, 'w', encoding='utf-8') as fd: - fd.write(unicode(json.dumps(self.data, ensure_ascii=False))) + fd.write(unicode( # noqa: F821 + json.dumps(self.data, ensure_ascii=False))) + class Arch(object): def __init__(self, ctx): @@ -161,7 +166,6 @@ class Arch(object): d.format(arch=self)) for d in self.ctx.include_dirs] - def get_env(self): include_dirs = [ "-I{}/{}".format( @@ -197,8 +201,10 @@ class Arch(object): env.update({k: v for k, v in environ.items() if k.startswith('CCACHE_')}) env.setdefault('CCACHE_MAXSIZE', '10G') env.setdefault('CCACHE_HARDLINK', 'true') - env.setdefault('CCACHE_SLOPPINESS', ('file_macro,time_macros,' - 'include_file_mtime,include_file_ctime,file_stat_matches')) + env.setdefault( + 'CCACHE_SLOPPINESS', + ('file_macro,time_macros,' + 'include_file_mtime,include_file_ctime,file_stat_matches')) if not self._ccsh: self._ccsh = tempfile.NamedTemporaryFile() @@ -365,7 +371,7 @@ class Context(object): # path to some tools self.ccache = sh.which("ccache") if not self.ccache: - #print("ccache is missing, the build will not be optimized in the future.") + # ccache is missing, the build will not be optimized pass for cython_fn in ("cython-2.7", "cython"): cython = sh.which(cython_fn) @@ -422,7 +428,6 @@ class Context(object): return "IDEBuildOperationMaxNumberOfConcurrentCompileTasks={}".format(self.num_cores) - class Recipe(object): props = { "is_alias": False, @@ -455,6 +460,7 @@ class Recipe(object): """ if not url: return + def report_hook(index, blksize, size): if size <= 0: progression = '{0} bytes'.format(index * blksize) @@ -546,8 +552,9 @@ class Recipe(object): archive = tarfile.open(filename) except tarfile.ReadError: logger.warning('Error extracting the archive {0}'.format(filename)) - logger.warning('This is usually caused by a corrupt download. The file' - ' will be removed and re-downloaded on the next run.') + logger.warning( + 'This is usually caused by a corrupt download. The file' + ' will be removed and re-downloaded on the next run.') logger.warning(filename) return @@ -605,7 +612,7 @@ class Recipe(object): """ try: unlink(join(self.build_dir, ".{}".format(marker))) - except: + except Exception: pass def get_include_dir(self): @@ -814,19 +821,6 @@ class Recipe(object): self.delete_marker("building") self.set_marker("build_done") - def update_state(self, key, value): - """Update entry in state database - - This is usually done in the @cache_execution decorator - to log an action and its time of occurrence, - but it needs to be done manually in recipes. - - sets key = value, and key.at = current_datetime - """ - key_time = "{}.at".format(key) - self.ctx.state[key] = value - self.ctx.state[key_time] = str(datetime.utcnow()) - @cache_execution def build_all(self): filtered_archs = self.filtered_archs @@ -880,9 +874,10 @@ class Recipe(object): getattr(self, postbuild)() def update_state(self, key, value): - """Update entry in state database - - Also adds the time of update + """ + Update entry in state database. This is usually done in the + @cache_execution decorator to log an action and its time of occurrence, + but it needs to be done manually in recipes. """ key_time = "{}.at".format(key) self.ctx.state[key] = value @@ -991,7 +986,7 @@ class Recipe(object): @classmethod def get_recipe(cls, name, ctx): if not hasattr(cls, "recipes"): - cls.recipes = {} + cls.recipes = {} if '==' in name: name, version = name.split('==') @@ -1070,7 +1065,6 @@ class CythonRecipe(PythonRecipe): filename = filename[len(self.build_dir) + 1:] logger.info("Cythonize {}".format(filename)) cmd = sh.Command(join(self.ctx.root_dir, "tools", "cythonize.py")) - hostpython = self.ctx.state.get("hostpython") shprint(cmd, filename) def cythonize_build(self): @@ -1106,7 +1100,7 @@ class CythonRecipe(PythonRecipe): try: shprint(hostpython, "setup.py", "build_ext", "-g", _env=build_env) - except: + except Exception: pass self.cythonize_build() shprint(hostpython, "setup.py", "build_ext", "-g", @@ -1131,8 +1125,8 @@ def build_recipes(names, ctx): logger.error("No recipe named {}".format(name)) sys.exit(1) graph.add(name, name) - logger.info("Loaded recipe {} (depends of {}, optional are {})".format(name, - recipe.depends, recipe.optional_depends)) + logger.info("Loaded recipe {} (depends of {}, optional are {})".format( + name, recipe.depends, recipe.optional_depends)) for depend in recipe.depends: graph.add(name, depend) recipe_to_load += recipe.depends @@ -1243,7 +1237,6 @@ def update_pbxproj(filename, pbx_frameworks=None): fn = join(ctx.dist_dir, "sources", name) project.add_folder(fn, parent=g_classes) - project.backup() project.save() @@ -1339,7 +1332,7 @@ Xcode: recipe = Recipe.get_recipe(name, ctx) print("{recipe.name:<12} {recipe.version:<8}".format(recipe=recipe)) - except: + except Exception: pass def clean(self): @@ -1361,9 +1354,6 @@ Xcode: shutil.rmtree(ctx.build_dir) def distclean(self): - parser = argparse.ArgumentParser( - description="Clean the build, download, and dist") - args = parser.parse_args(sys.argv[2:]) ctx = Context() if exists(ctx.build_dir): shutil.rmtree(ctx.build_dir) @@ -1373,9 +1363,6 @@ Xcode: shutil.rmtree(ctx.cache_dir) def status(self): - parser = argparse.ArgumentParser( - description="Give a status of the build") - args = parser.parse_args(sys.argv[2:]) ctx = Context() for recipe in Recipe.list_recipes(): key = "{}.build_all".format(recipe) @@ -1555,7 +1542,7 @@ Xcode: project_name = filename.split("/")[-1].replace(".xcodeproj", "") images_xcassets = realpath(join(filename, "..", project_name, - "Images.xcassets")) + "Images.xcassets")) if not exists(images_xcassets): logger.warning("Images.xcassets not found, creating it.") makedirs(images_xcassets) diff --git a/tox.ini b/tox.ini index 142527c..0f3e898 100644 --- a/tox.ini +++ b/tox.ini @@ -15,11 +15,6 @@ commands = flake8 recipes/ tools/ tests/ .ci/ toolchain.py [flake8] exclude = tools/external/, - toolchain.py, # Temporary removal: TODO: ZenCODE - recipes/kivy, # Temporary removal: TODO: ZenCODE - recipes/python3, # Temporary removal: TODO: ZenCODE - recipes/hostpython3, # Temporary removal: TODO: ZenCODE - recipes/host_setuptools3, # Temporary removal: TODO: ZenCODE ignore = E123, # Closing bracket does not match indentation of opening bracket's line