From 0239bde32d037686d7cf5ce21c153523f0f090c3 Mon Sep 17 00:00:00 2001 From: Ryan Pessa Date: Mon, 1 May 2017 23:10:11 -0700 Subject: [PATCH] add concurrent make, xcodebuild, gzip/bzip2 --- recipes/ffmpeg/__init__.py | 2 +- recipes/freetype/__init__.py | 2 +- recipes/hostlibffi/__init__.py | 2 +- recipes/hostpython/__init__.py | 4 +-- recipes/libffi/__init__.py | 2 +- recipes/libjpeg/__init__.py | 2 +- recipes/libpng/__init__.py | 2 +- recipes/openssl/__init__.py | 2 +- recipes/python/__init__.py | 4 +-- recipes/sdl2/__init__.py | 2 +- recipes/sdl2_image/__init__.py | 2 +- recipes/sdl2_mixer/__init__.py | 2 +- toolchain.py | 50 +++++++++++++++++++++++++++++++--- 13 files changed, 60 insertions(+), 18 deletions(-) diff --git a/recipes/ffmpeg/__init__.py b/recipes/ffmpeg/__init__.py index ec32617..f4c7d74 100644 --- a/recipes/ffmpeg/__init__.py +++ b/recipes/ffmpeg/__init__.py @@ -85,7 +85,7 @@ class FFMpegRecipe(Recipe): "config.asm") """ shprint(sh.make, "clean", _env=build_env) - shprint(sh.make, "-j4", _env=build_env) + shprint(sh.make, self.ctx.concurrent_make, _env=build_env) shprint(sh.make, "install") diff --git a/recipes/freetype/__init__.py b/recipes/freetype/__init__.py index d83164f..a7b1338 100644 --- a/recipes/freetype/__init__.py +++ b/recipes/freetype/__init__.py @@ -28,7 +28,7 @@ class FreetypeRecipe(Recipe): "--enable-static=yes", "--enable-shared=no") shprint(sh.make, "clean") - shprint(sh.make) + shprint(sh.make, self.ctx.concurrent_make) recipe = FreetypeRecipe() diff --git a/recipes/hostlibffi/__init__.py b/recipes/hostlibffi/__init__.py index 7d64efa..77c2e0c 100644 --- a/recipes/hostlibffi/__init__.py +++ b/recipes/hostlibffi/__init__.py @@ -34,7 +34,7 @@ class LibffiRecipe(Recipe): self.set_marker("patched") def build_arch(self, arch): - shprint(sh.xcodebuild, + shprint(sh.xcodebuild, self.ctx.concurrent_xcodebuild, "ONLY_ACTIVE_ARCH=NO", "ARCHS={}".format(arch.arch), "-sdk", "macosx", diff --git a/recipes/hostpython/__init__.py b/recipes/hostpython/__init__.py index 0fc7274..f85e7c6 100644 --- a/recipes/hostpython/__init__.py +++ b/recipes/hostpython/__init__.py @@ -73,7 +73,7 @@ class HostpythonRecipe(Recipe): "--disable-toolbox-glue", "--without-gcc", _env=build_env) - shprint(sh.make, "-C", self.build_dir, "-j4", "python", "Parser/pgen", + shprint(sh.make, "-C", self.build_dir, self.ctx.concurrent_make, "python", "Parser/pgen", _env=build_env) shutil.move("python", "hostpython") shutil.move("Parser/pgen", "Parser/hostpgen") @@ -88,7 +88,7 @@ class HostpythonRecipe(Recipe): shprint(sh.ln, "-s", join(build_dir, "hostpython"), join(build_dir, "Python")) - shprint(sh.make, + shprint(sh.make, self.ctx.concurrent_make, "-C", build_dir, "bininstall", "inclinstall", _env=build_env) diff --git a/recipes/libffi/__init__.py b/recipes/libffi/__init__.py index 25fc0a3..d97f3f1 100644 --- a/recipes/libffi/__init__.py +++ b/recipes/libffi/__init__.py @@ -23,7 +23,7 @@ class LibffiRecipe(Recipe): self.set_marker("patched") def build_arch(self, arch): - shprint(sh.xcodebuild, + shprint(sh.xcodebuild, self.ctx.concurrent_xcodebuild, "ONLY_ACTIVE_ARCH=NO", "ARCHS={}".format(arch.arch), "-sdk", arch.sdk, diff --git a/recipes/libjpeg/__init__.py b/recipes/libjpeg/__init__.py index 730021e..1b4410b 100644 --- a/recipes/libjpeg/__init__.py +++ b/recipes/libjpeg/__init__.py @@ -29,7 +29,7 @@ class JpegRecipe(Recipe): "--host={}".format(arch.triple), "--disable-shared") shprint(sh.make, "clean") - shprint(sh.make) + shprint(sh.make, self.ctx.concurrent_make) recipe = JpegRecipe() diff --git a/recipes/libpng/__init__.py b/recipes/libpng/__init__.py index fd2e3d0..6ecdb0d 100644 --- a/recipes/libpng/__init__.py +++ b/recipes/libpng/__init__.py @@ -21,6 +21,6 @@ class PngRecipe(Recipe): "--host={}".format(arch.triple), "--disable-shared") shprint(sh.make, "clean") - shprint(sh.make, _env=build_env) + shprint(sh.make, self.ctx.concurrent_make, _env=build_env) recipe = PngRecipe() diff --git a/recipes/openssl/__init__.py b/recipes/openssl/__init__.py index f39befb..f93cbea 100644 --- a/recipes/openssl/__init__.py +++ b/recipes/openssl/__init__.py @@ -43,6 +43,6 @@ class OpensslRecipe(Recipe): sh.sed("-ie", "s!^CFLAG=!CFLAG={} !".format(build_env['CFLAGS']), "Makefile") shprint(sh.make, "clean") - shprint(sh.make, "-j4", "build_libs") + shprint(sh.make, self.ctx.concurrent_make, "build_libs") recipe = OpensslRecipe() diff --git a/recipes/python/__init__.py b/recipes/python/__init__.py index 5825d28..0b91c16 100644 --- a/recipes/python/__init__.py +++ b/recipes/python/__init__.py @@ -56,7 +56,7 @@ class PythonRecipe(Recipe): self.apply_patch("ctypes_duplicate.patch") self.apply_patch("ctypes_duplicate_longdouble.patch") - shprint(sh.make, "-j4", + shprint(sh.make, self.ctx.concurrent_make, "CROSS_COMPILE_TARGET=yes", "HOSTPYTHON={}".format(self.ctx.hostpython), "HOSTPGEN={}".format(self.ctx.hostpgen)) @@ -66,7 +66,7 @@ class PythonRecipe(Recipe): build_env = arch.get_env() build_dir = self.get_build_dir(arch.arch) build_env["PATH"] = os.environ["PATH"] - shprint(sh.make, + shprint(sh.make, self.ctx.concurrent_make, "-C", build_dir, "install", "CROSS_COMPILE_TARGET=yes", diff --git a/recipes/sdl2/__init__.py b/recipes/sdl2/__init__.py index f3040c3..6fe77cf 100644 --- a/recipes/sdl2/__init__.py +++ b/recipes/sdl2/__init__.py @@ -20,7 +20,7 @@ class LibSDL2Recipe(Recipe): def build_arch(self, arch): env = arch.get_env() - shprint(sh.xcodebuild, + shprint(sh.xcodebuild, self.ctx.concurrent_xcodebuild, "ONLY_ACTIVE_ARCH=NO", "ARCHS={}".format(arch.arch), "CC={}".format(env['CC']), diff --git a/recipes/sdl2_image/__init__.py b/recipes/sdl2_image/__init__.py index 6476551..e9725d6 100644 --- a/recipes/sdl2_image/__init__.py +++ b/recipes/sdl2_image/__init__.py @@ -12,7 +12,7 @@ class LibSDL2ImageRecipe(Recipe): pbx_frameworks = ["CoreGraphics", "MobileCoreServices"] def build_arch(self, arch): - shprint(sh.xcodebuild, + shprint(sh.xcodebuild, self.ctx.concurrent_xcodebuild, "ONLY_ACTIVE_ARCH=NO", "ARCHS={}".format(arch.arch), "HEADER_SEARCH_PATHS={}".format( diff --git a/recipes/sdl2_mixer/__init__.py b/recipes/sdl2_mixer/__init__.py index 01eb854..d1c01ac 100644 --- a/recipes/sdl2_mixer/__init__.py +++ b/recipes/sdl2_mixer/__init__.py @@ -12,7 +12,7 @@ class LibSDL2MixerRecipe(Recipe): pbx_libraries = ["libc++"] def build_arch(self, arch): - shprint(sh.xcodebuild, + shprint(sh.xcodebuild, self.ctx.concurrent_xcodebuild, "ONLY_ACTIVE_ARCH=NO", "ARCHS={}".format(arch.arch), "HEADER_SEARCH_PATHS=$HEADER_SEARCH_PATHS {}".format(" ".join(arch.include_dirs)), diff --git a/toolchain.py b/toolchain.py index 14ce8ce..70cd048 100755 --- a/toolchain.py +++ b/toolchain.py @@ -355,6 +355,15 @@ class Context(object): if not ok: sys.exit(1) + self.use_pigz = sh.which('pigz') + self.use_pbzip2 = sh.which('pbzip2') + + try: + num_cores = int(sh.sysctl('-n', 'hw.ncpu')) + except Exception: + num_cores = None + self.num_cores = num_cores if num_cores else 4 # default to 4 if we can't detect + ensure_dir(self.root_dir) ensure_dir(self.build_dir) ensure_dir(self.cache_dir) @@ -373,6 +382,14 @@ class Context(object): # set the state self.state = JsonStore(join(self.dist_dir, "state.db")) + @property + def concurrent_make(self): + return "-j{}".format(self.num_cores) + + @property + def concurrent_xcodebuild(self): + return "IDEBuildOperationMaxNumberOfConcurrentCompileTasks={}".format(self.num_cores) + class Recipe(object): version = None @@ -425,16 +442,24 @@ class Recipe(object): return print("Extract {} into {}".format(filename, cwd)) if filename.endswith(".tgz") or filename.endswith(".tar.gz"): - shprint(sh.tar, "-C", cwd, "-xvzf", filename) + if self.ctx.use_pigz: + comp = '--use-compress-program={}'.format(self.ctx.use_pigz) + else: + comp = '-z' + shprint(sh.tar, "-C", cwd, "-xv", comp, "-f", filename) elif filename.endswith(".tbz2") or filename.endswith(".tar.bz2"): - shprint(sh.tar, "-C", cwd, "-xvjf", filename) + if self.ctx.use_pbzip2: + comp = '--use-compress-program={}'.format(self.ctx.use_pbzip2) + else: + comp = '-j' + shprint(sh.tar, "-C", cwd, "-xv", comp, "-f", filename) elif filename.endswith(".zip"): shprint(sh.unzip, "-d", cwd, filename) else: - print("Error: cannot extract, unreconized extension for {}".format( + print("Error: cannot extract, unrecognized extension for {}".format( filename)) raise Exception() @@ -1088,14 +1113,20 @@ Xcode: getattr(self, args.command)() def build(self): + ctx = Context() parser = argparse.ArgumentParser( description="Build the toolchain") parser.add_argument("recipe", nargs="+", help="Recipe to compile") parser.add_argument("--arch", action="append", help="Restrict compilation to this arch") + parser.add_argument("--concurrency", type=int, default=ctx.num_cores, + help="number of concurrent build processes (where supported)") + parser.add_argument("--no-pigz", action="store_true", default=not bool(ctx.use_pigz), + help="do not use pigz for gzip decompression") + parser.add_argument("--no-pbzip2", action="store_true", default=not bool(ctx.use_pbzip2), + help="do not use pbzip2 for bzip2 decompression") args = parser.parse_args(sys.argv[2:]) - ctx = Context() if args.arch: if len(args.arch) == 1: archs = args.arch[0].split() @@ -1109,6 +1140,17 @@ Xcode: continue ctx.archs = [arch for arch in ctx.archs if arch.arch in archs] print("Architectures restricted to: {}".format(archs)) + ctx.num_cores = args.concurrency + if args.no_pigz: + ctx.use_pigz = False + if args.no_pbzip2: + ctx.use_pbzip2 = False + ctx.use_pigz = ctx.use_pbzip2 + print("Building with {} processes, where supported".format(ctx.num_cores)) + if ctx.use_pigz: + print("Using pigz to decompress gzip data") + if ctx.use_pbzip2: + print("Using pbzip2 to decompress bzip2 data") build_recipes(args.recipe, ctx) def recipes(self):