LBRYCommunity/kivy-ios
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Fix flake8 CI check on toolchain, remaining recipes (#456)

Browse source 
* 🔥 Fix flake8 checks for host_setuptools3

* 🔥 Fix flake8 checks for kivy recipe

* ✨ Fix flake8 for toolchain.py

* 💡 Set python3 as the default python

* Add .tox and ven to .gitignore

* Update toolchain.py

Co-Authored-By: Andre Miras <AndreMiras@users.noreply.github.com>

Co-authored-by: Andre Miras <AndreMiras@users.noreply.github.com>
This commit is contained in:
Richard Larkin 2020-04-27 07:29:11 +02:00 committed by GitHub
parent 9bb9efe0f7
commit 8416d46fd4
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
6 changed files with 38 additions and 63 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
.gitignore vendored
View file

@ -19,3 +19,5 @@ src/ios/ios.c
*.DS_Store* *.DS_Store*
*-ios/ *-ios/
__pycache__ __pycache__
.tox
venv

6
recipes/host_setuptools3/__init__.py
View file

@ -1,8 +1,7 @@
from toolchain import Recipe, shprint from toolchain import Recipe, shprint
from os.path import join, exists from os.path import join
import sh import sh
import os import os
import fnmatch
import shutil import shutil
@ -13,7 +12,7 @@ class HostSetuptools3(Recipe):
def prebuild_arch(self, arch): def prebuild_arch(self, arch):
hostpython = sh.Command(self.ctx.hostpython) hostpython = sh.Command(self.ctx.hostpython)
sh.curl("-O", "https://bootstrap.pypa.io/ez_setup.py") sh.curl("-O", "https://bootstrap.pypa.io/ez_setup.py")
shprint(hostpython, "./ez_setup.py") shprint(hostpython, "./ez_setup.py")
# Extract setuptools egg and remove .pth files. Otherwise subsequent # Extract setuptools egg and remove .pth files. Otherwise subsequent
# python package installations using setuptools will raise exceptions. # python package installations using setuptools will raise exceptions.
@ -32,4 +31,5 @@ class HostSetuptools3(Recipe):
os.remove('easy-install.pth') os.remove('easy-install.pth')
shutil.rmtree('EGG-INFO') shutil.rmtree('EGG-INFO')
recipe = HostSetuptools3() recipe = HostSetuptools3()

6
recipes/kivy/__init__.py
View file

@ -1,7 +1,5 @@
from toolchain import CythonRecipe, shprint from toolchain import CythonRecipe
from os.path import join from os.path import join
from os import chdir, listdir, getcwd
import sh
import logging import logging
import shutil import shutil
@ -39,6 +37,7 @@ class KivyRecipe(CythonRecipe):
def _patch_setup(self): def _patch_setup(self):
# patch setup to remove some functionnalities # patch setup to remove some functionnalities
pyconfig = join(self.build_dir, "setup.py") pyconfig = join(self.build_dir, "setup.py")
def _remove_line(lines, pattern): def _remove_line(lines, pattern):
for line in lines[:]: for line in lines[:]:
if pattern in line: if pattern in line:
@ -46,7 +45,6 @@ class KivyRecipe(CythonRecipe):
with open(pyconfig) as fd: with open(pyconfig) as fd:
lines = fd.readlines() lines = fd.readlines()
_remove_line(lines, "flags['libraries'] = ['GLESv2']") _remove_line(lines, "flags['libraries'] = ['GLESv2']")
#_remove_line(lines, "c_options['use_sdl'] = True")
with open(pyconfig, "w") as fd: with open(pyconfig, "w") as fd:
fd.writelines(lines) fd.writelines(lines)

11
recipes/python.py
View file

@ -1,5 +1,4 @@
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
import sys
from toolchain import Recipe from toolchain import Recipe
import logging import logging
from os.path import join from os.path import join
@ -16,15 +15,9 @@ class PythonAliasRecipe(Recipe):
# search in wanted_recipes if it's the first time # search in wanted_recipes if it's the first time
if "python2" in ctx.wanted_recipes: if "python2" in ctx.wanted_recipes:
python = "python2" python = "python2"
elif "python3" in ctx.wanted_recipes:
python = "python3"
else: else:
logger.error("No Python version set in the build.") python = "python3"
logger.error("Add python2 or python3 in your recipes:") self.depends = [python]
logger.error("./toolchain.py build python3 ...")
sys.exit(1)
if python:
self.depends = [python]
self.recipe_dir = join(ctx.root_dir, "recipes", python) self.recipe_dir = join(ctx.root_dir, "recipes", python)

71
toolchain.py
View file

@ -9,7 +9,7 @@ This tool intend to replace all the previous tools/ in shell script.
import sys import sys
from sys import stdout from sys import stdout
from os.path import join, dirname, realpath, exists, isdir, basename from os.path import join, dirname, realpath, exists, isdir, basename
from os import listdir, unlink, makedirs, environ, chdir, getcwd, walk, remove from os import listdir, unlink, makedirs, environ, chdir, getcwd, walk
import zipfile import zipfile
import tarfile import tarfile
import importlib import importlib
@ -21,6 +21,7 @@ import tempfile
import time import time
from datetime import datetime from datetime import datetime
from pprint import pformat from pprint import pformat
import logging
try: try:
from urllib.request import FancyURLopener, urlcleanup from urllib.request import FancyURLopener, urlcleanup
@ -39,7 +40,6 @@ sys.path.insert(0, join(curdir, "tools", "external"))
import sh import sh
import logging
# For more detailed logging, use something like # For more detailed logging, use something like
# format='%(asctime)s,%(msecs)d %(levelname)-8s [%(filename)s:%(funcName)s():%(lineno)d] %(message)s' # format='%(asctime)s,%(msecs)d %(levelname)-8s [%(filename)s:%(funcName)s():%(lineno)d] %(message)s'
@ -69,6 +69,7 @@ def shprint(command, *args, **kwargs):
line_str = "\n".join(line.encode("ascii", "replace").decode().splitlines()) line_str = "\n".join(line.encode("ascii", "replace").decode().splitlines())
logger.debug(line_str) logger.debug(line_str)
def cache_execution(f): def cache_execution(f):
def _cache_execution(self, *args, **kwargs): def _cache_execution(self, *args, **kwargs):
state = self.ctx.state state = self.ctx.state
@ -85,11 +86,13 @@ def cache_execution(f):
self.update_state(key, True) self.update_state(key, True)
return _cache_execution return _cache_execution
class ChromeDownloader(FancyURLopener): class ChromeDownloader(FancyURLopener):
version = ( version = (
'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 ' 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 '
'(KHTML, like Gecko) Chrome/28.0.1500.71 Safari/537.36') '(KHTML, like Gecko) Chrome/28.0.1500.71 Safari/537.36')
urlretrieve = ChromeDownloader().retrieve urlretrieve = ChromeDownloader().retrieve
@ -136,13 +139,15 @@ class JsonStore(object):
self.sync() self.sync()
def sync(self): def sync(self):
# http://stackoverflow.com/questions/12309269/write-json-data-to-file-in-python/14870531#14870531 # https://stackoverflow.com/a/14870531/185510
if IS_PY3: if IS_PY3:
with open(self.filename, 'w') as fd: with open(self.filename, 'w') as fd:
json.dump(self.data, fd, ensure_ascii=False) json.dump(self.data, fd, ensure_ascii=False)
else: else:
with io.open(self.filename, 'w', encoding='utf-8') as fd: with io.open(self.filename, 'w', encoding='utf-8') as fd:
fd.write(unicode(json.dumps(self.data, ensure_ascii=False))) fd.write(unicode( # noqa: F821
json.dumps(self.data, ensure_ascii=False)))
class Arch(object): class Arch(object):
def __init__(self, ctx): def __init__(self, ctx):
@ -161,7 +166,6 @@ class Arch(object):
d.format(arch=self)) d.format(arch=self))
for d in self.ctx.include_dirs] for d in self.ctx.include_dirs]
def get_env(self): def get_env(self):
include_dirs = [ include_dirs = [
"-I{}/{}".format( "-I{}/{}".format(
@ -197,8 +201,10 @@ class Arch(object):
env.update({k: v for k, v in environ.items() if k.startswith('CCACHE_')}) env.update({k: v for k, v in environ.items() if k.startswith('CCACHE_')})
env.setdefault('CCACHE_MAXSIZE', '10G') env.setdefault('CCACHE_MAXSIZE', '10G')
env.setdefault('CCACHE_HARDLINK', 'true') env.setdefault('CCACHE_HARDLINK', 'true')
env.setdefault('CCACHE_SLOPPINESS', ('file_macro,time_macros,' env.setdefault(
'include_file_mtime,include_file_ctime,file_stat_matches')) 'CCACHE_SLOPPINESS',
('file_macro,time_macros,'
'include_file_mtime,include_file_ctime,file_stat_matches'))
if not self._ccsh: if not self._ccsh:
self._ccsh = tempfile.NamedTemporaryFile() self._ccsh = tempfile.NamedTemporaryFile()
@ -365,7 +371,7 @@ class Context(object):
# path to some tools # path to some tools
self.ccache = sh.which("ccache") self.ccache = sh.which("ccache")
if not self.ccache: if not self.ccache:
#print("ccache is missing, the build will not be optimized in the future.") # ccache is missing, the build will not be optimized
pass pass
for cython_fn in ("cython-2.7", "cython"): for cython_fn in ("cython-2.7", "cython"):
cython = sh.which(cython_fn) cython = sh.which(cython_fn)
@ -422,7 +428,6 @@ class Context(object):
return "IDEBuildOperationMaxNumberOfConcurrentCompileTasks={}".format(self.num_cores) return "IDEBuildOperationMaxNumberOfConcurrentCompileTasks={}".format(self.num_cores)
class Recipe(object): class Recipe(object):
props = { props = {
"is_alias": False, "is_alias": False,
@ -455,6 +460,7 @@ class Recipe(object):
""" """
if not url: if not url:
return return
def report_hook(index, blksize, size): def report_hook(index, blksize, size):
if size <= 0: if size <= 0:
progression = '{0} bytes'.format(index * blksize) progression = '{0} bytes'.format(index * blksize)
@ -546,8 +552,9 @@ class Recipe(object):
archive = tarfile.open(filename) archive = tarfile.open(filename)
except tarfile.ReadError: except tarfile.ReadError:
logger.warning('Error extracting the archive {0}'.format(filename)) logger.warning('Error extracting the archive {0}'.format(filename))
logger.warning('This is usually caused by a corrupt download. The file' logger.warning(
' will be removed and re-downloaded on the next run.') 'This is usually caused by a corrupt download. The file'
' will be removed and re-downloaded on the next run.')
logger.warning(filename) logger.warning(filename)
return return
@ -605,7 +612,7 @@ class Recipe(object):
""" """
try: try:
unlink(join(self.build_dir, ".{}".format(marker))) unlink(join(self.build_dir, ".{}".format(marker)))
except: except Exception:
pass pass
def get_include_dir(self): def get_include_dir(self):
@ -814,19 +821,6 @@ class Recipe(object):
self.delete_marker("building") self.delete_marker("building")
self.set_marker("build_done") self.set_marker("build_done")
def update_state(self, key, value):
"""Update entry in state database
This is usually done in the @cache_execution decorator
to log an action and its time of occurrence,
but it needs to be done manually in recipes.
sets key = value, and key.at = current_datetime
"""
key_time = "{}.at".format(key)
self.ctx.state[key] = value
self.ctx.state[key_time] = str(datetime.utcnow())
@cache_execution @cache_execution
def build_all(self): def build_all(self):
filtered_archs = self.filtered_archs filtered_archs = self.filtered_archs
@ -880,9 +874,10 @@ class Recipe(object):
getattr(self, postbuild)() getattr(self, postbuild)()
def update_state(self, key, value): def update_state(self, key, value):
"""Update entry in state database """
Update entry in state database. This is usually done in the
Also adds the time of update @cache_execution decorator to log an action and its time of occurrence,
but it needs to be done manually in recipes.
""" """
key_time = "{}.at".format(key) key_time = "{}.at".format(key)
self.ctx.state[key] = value self.ctx.state[key] = value
@ -991,7 +986,7 @@ class Recipe(object):
@classmethod @classmethod
def get_recipe(cls, name, ctx): def get_recipe(cls, name, ctx):
if not hasattr(cls, "recipes"): if not hasattr(cls, "recipes"):
cls.recipes = {} cls.recipes = {}
if '==' in name: if '==' in name:
name, version = name.split('==') name, version = name.split('==')
@ -1070,7 +1065,6 @@ class CythonRecipe(PythonRecipe):
filename = filename[len(self.build_dir) + 1:] filename = filename[len(self.build_dir) + 1:]
logger.info("Cythonize {}".format(filename)) logger.info("Cythonize {}".format(filename))
cmd = sh.Command(join(self.ctx.root_dir, "tools", "cythonize.py")) cmd = sh.Command(join(self.ctx.root_dir, "tools", "cythonize.py"))
hostpython = self.ctx.state.get("hostpython")
shprint(cmd, filename) shprint(cmd, filename)
def cythonize_build(self): def cythonize_build(self):
@ -1106,7 +1100,7 @@ class CythonRecipe(PythonRecipe):
try: try:
shprint(hostpython, "setup.py", "build_ext", "-g", shprint(hostpython, "setup.py", "build_ext", "-g",
_env=build_env) _env=build_env)
except: except Exception:
pass pass
self.cythonize_build() self.cythonize_build()
shprint(hostpython, "setup.py", "build_ext", "-g", shprint(hostpython, "setup.py", "build_ext", "-g",
@ -1131,8 +1125,8 @@ def build_recipes(names, ctx):
logger.error("No recipe named {}".format(name)) logger.error("No recipe named {}".format(name))
sys.exit(1) sys.exit(1)
graph.add(name, name) graph.add(name, name)
logger.info("Loaded recipe {} (depends of {}, optional are {})".format(name, logger.info("Loaded recipe {} (depends of {}, optional are {})".format(
recipe.depends, recipe.optional_depends)) name, recipe.depends, recipe.optional_depends))
for depend in recipe.depends: for depend in recipe.depends:
graph.add(name, depend) graph.add(name, depend)
recipe_to_load += recipe.depends recipe_to_load += recipe.depends
@ -1243,7 +1237,6 @@ def update_pbxproj(filename, pbx_frameworks=None):
fn = join(ctx.dist_dir, "sources", name) fn = join(ctx.dist_dir, "sources", name)
project.add_folder(fn, parent=g_classes) project.add_folder(fn, parent=g_classes)
project.backup() project.backup()
project.save() project.save()
@ -1339,7 +1332,7 @@ Xcode:
recipe = Recipe.get_recipe(name, ctx) recipe = Recipe.get_recipe(name, ctx)
print("{recipe.name:<12} {recipe.version:<8}".format(recipe=recipe)) print("{recipe.name:<12} {recipe.version:<8}".format(recipe=recipe))
except: except Exception:
pass pass
def clean(self): def clean(self):
@ -1361,9 +1354,6 @@ Xcode:
shutil.rmtree(ctx.build_dir) shutil.rmtree(ctx.build_dir)
def distclean(self): def distclean(self):
parser = argparse.ArgumentParser(
description="Clean the build, download, and dist")
args = parser.parse_args(sys.argv[2:])
ctx = Context() ctx = Context()
if exists(ctx.build_dir): if exists(ctx.build_dir):
shutil.rmtree(ctx.build_dir) shutil.rmtree(ctx.build_dir)
@ -1373,9 +1363,6 @@ Xcode:
shutil.rmtree(ctx.cache_dir) shutil.rmtree(ctx.cache_dir)
def status(self): def status(self):
parser = argparse.ArgumentParser(
description="Give a status of the build")
args = parser.parse_args(sys.argv[2:])
ctx = Context() ctx = Context()
for recipe in Recipe.list_recipes(): for recipe in Recipe.list_recipes():
key = "{}.build_all".format(recipe) key = "{}.build_all".format(recipe)
@ -1555,7 +1542,7 @@ Xcode:
project_name = filename.split("/")[-1].replace(".xcodeproj", "") project_name = filename.split("/")[-1].replace(".xcodeproj", "")
images_xcassets = realpath(join(filename, "..", project_name, images_xcassets = realpath(join(filename, "..", project_name,
"Images.xcassets")) "Images.xcassets"))
if not exists(images_xcassets): if not exists(images_xcassets):
logger.warning("Images.xcassets not found, creating it.") logger.warning("Images.xcassets not found, creating it.")
makedirs(images_xcassets) makedirs(images_xcassets)

5
tox.ini
View file

@ -15,11 +15,6 @@ commands = flake8 recipes/ tools/ tests/ .ci/ toolchain.py
[flake8] [flake8]
exclude = tools/external/, exclude = tools/external/,
toolchain.py, # Temporary removal: TODO: ZenCODE
recipes/kivy, # Temporary removal: TODO: ZenCODE
recipes/python3, # Temporary removal: TODO: ZenCODE
recipes/hostpython3, # Temporary removal: TODO: ZenCODE
recipes/host_setuptools3, # Temporary removal: TODO: ZenCODE
ignore = ignore =
E123, # Closing bracket does not match indentation of opening bracket's line E123, # Closing bracket does not match indentation of opening bracket's line