Merge pull request #249 from kived/concurrent-build

add concurrent make, xcodebuild, gzip/bzip2

recipes/ffmpeg/__init__.py

@@ -85,7 +85,7 @@ class FFMpegRecipe(Recipe):
                     "config.asm")
         """
         shprint(sh.make, "clean", _env=build_env)
-        shprint(sh.make, "-j4", _env=build_env)
+        shprint(sh.make, self.ctx.concurrent_make, _env=build_env)
         shprint(sh.make, "install")
 
 

recipes/freetype/__init__.py

@@ -28,7 +28,7 @@ class FreetypeRecipe(Recipe):
                 "--enable-static=yes",
                 "--enable-shared=no")
         shprint(sh.make, "clean")
-        shprint(sh.make)
+        shprint(sh.make, self.ctx.concurrent_make)
 
 
 recipe = FreetypeRecipe()

recipes/hostlibffi/__init__.py

@@ -34,7 +34,7 @@ class LibffiRecipe(Recipe):
         self.set_marker("patched")
 
     def build_arch(self, arch):
-        shprint(sh.xcodebuild,
+        shprint(sh.xcodebuild, self.ctx.concurrent_xcodebuild,
                 "ONLY_ACTIVE_ARCH=NO",
                 "ARCHS={}".format(arch.arch),
                 "-sdk", "macosx",

recipes/hostpython/__init__.py

@@ -73,7 +73,7 @@ class HostpythonRecipe(Recipe):
                 "--disable-toolbox-glue",
                 "--without-gcc",
                 _env=build_env)
-        shprint(sh.make, "-C", self.build_dir, "-j4", "python", "Parser/pgen",
+        shprint(sh.make, "-C", self.build_dir, self.ctx.concurrent_make, "python", "Parser/pgen",
                 _env=build_env)
         shutil.move("python", "hostpython")
         shutil.move("Parser/pgen", "Parser/hostpgen")
@@ -88,7 +88,7 @@ class HostpythonRecipe(Recipe):
         shprint(sh.ln, "-s",
                 join(build_dir, "hostpython"),
                 join(build_dir, "Python"))
-        shprint(sh.make,
+        shprint(sh.make, self.ctx.concurrent_make,
                 "-C", build_dir,
                 "bininstall", "inclinstall",
                 _env=build_env)

recipes/libffi/__init__.py

@@ -23,7 +23,7 @@ class LibffiRecipe(Recipe):
         self.set_marker("patched")
 
     def build_arch(self, arch):
-        shprint(sh.xcodebuild,
+        shprint(sh.xcodebuild, self.ctx.concurrent_xcodebuild,
                 "ONLY_ACTIVE_ARCH=NO",
                 "ARCHS={}".format(arch.arch),
                 "-sdk", arch.sdk,

recipes/libjpeg/__init__.py

@@ -29,7 +29,7 @@ class JpegRecipe(Recipe):
                 "--host={}".format(arch.triple),
                 "--disable-shared")
         shprint(sh.make, "clean")
-        shprint(sh.make)
+        shprint(sh.make, self.ctx.concurrent_make)
 
 recipe = JpegRecipe()
 

recipes/libpng/__init__.py

@@ -21,6 +21,6 @@ class PngRecipe(Recipe):
                 "--host={}".format(arch.triple),
                 "--disable-shared")
         shprint(sh.make, "clean")
-        shprint(sh.make, _env=build_env)
+        shprint(sh.make, self.ctx.concurrent_make, _env=build_env)
 
 recipe = PngRecipe()

recipes/openssl/__init__.py

@@ -43,6 +43,6 @@ class OpensslRecipe(Recipe):
             sh.sed("-ie", "s!^CFLAG=!CFLAG={} !".format(build_env['CFLAGS']),
                    "Makefile")
         shprint(sh.make, "clean")
-        shprint(sh.make, "-j4", "build_libs")
+        shprint(sh.make, self.ctx.concurrent_make, "build_libs")
 
 recipe = OpensslRecipe()

recipes/python/__init__.py

@@ -58,7 +58,7 @@ class PythonRecipe(Recipe):
         self.apply_patch("ctypes_duplicate.patch")
         self.apply_patch("ctypes_duplicate_longdouble.patch")
 
-        shprint(sh.make, "-j4",
+        shprint(sh.make, self.ctx.concurrent_make,
                 "CROSS_COMPILE_TARGET=yes",
                 "HOSTPYTHON={}".format(self.ctx.hostpython),
                 "HOSTPGEN={}".format(self.ctx.hostpgen))
@@ -68,7 +68,7 @@ class PythonRecipe(Recipe):
         build_env = arch.get_env()
         build_dir = self.get_build_dir(arch.arch)
         build_env["PATH"] = os.environ["PATH"]
-        shprint(sh.make,
+        shprint(sh.make, self.ctx.concurrent_make,
                 "-C", build_dir,
                 "install",
                 "CROSS_COMPILE_TARGET=yes",

recipes/sdl2/__init__.py

@@ -20,7 +20,7 @@ class LibSDL2Recipe(Recipe):
 
     def build_arch(self, arch):
         env = arch.get_env()
-        shprint(sh.xcodebuild,
+        shprint(sh.xcodebuild, self.ctx.concurrent_xcodebuild,
                 "ONLY_ACTIVE_ARCH=NO",
                 "ARCHS={}".format(arch.arch),
                 "CC={}".format(env['CC']),

recipes/sdl2_image/__init__.py

@@ -12,7 +12,7 @@ class LibSDL2ImageRecipe(Recipe):
     pbx_frameworks = ["CoreGraphics", "MobileCoreServices"]
 
     def build_arch(self, arch):
-        shprint(sh.xcodebuild,
+        shprint(sh.xcodebuild, self.ctx.concurrent_xcodebuild,
                 "ONLY_ACTIVE_ARCH=NO",
                 "ARCHS={}".format(arch.arch),
                 "HEADER_SEARCH_PATHS={}".format(

recipes/sdl2_mixer/__init__.py

@@ -12,7 +12,7 @@ class LibSDL2MixerRecipe(Recipe):
     pbx_libraries = ["libc++"]
 
     def build_arch(self, arch):
-        shprint(sh.xcodebuild,
+        shprint(sh.xcodebuild, self.ctx.concurrent_xcodebuild,
                 "ONLY_ACTIVE_ARCH=NO",
                 "ARCHS={}".format(arch.arch),
                 "HEADER_SEARCH_PATHS=$HEADER_SEARCH_PATHS {}".format(" ".join(arch.include_dirs)),

toolchain.py

@@ -355,6 +355,15 @@ class Context(object):
         if not ok:
             sys.exit(1)
 
+        self.use_pigz = sh.which('pigz')
+        self.use_pbzip2 = sh.which('pbzip2')
+
+        try:
+            num_cores = int(sh.sysctl('-n', 'hw.ncpu'))
+        except Exception:
+            num_cores = None
+        self.num_cores = num_cores if num_cores else 4  # default to 4 if we can't detect
+
         ensure_dir(self.root_dir)
         ensure_dir(self.build_dir)
         ensure_dir(self.cache_dir)
@@ -373,6 +382,14 @@ class Context(object):
         # set the state
         self.state = JsonStore(join(self.dist_dir, "state.db"))
 
+    @property
+    def concurrent_make(self):
+        return "-j{}".format(self.num_cores)
+
+    @property
+    def concurrent_xcodebuild(self):
+        return "IDEBuildOperationMaxNumberOfConcurrentCompileTasks={}".format(self.num_cores)
+
 
 class Recipe(object):
     version = None
@@ -425,16 +442,24 @@ class Recipe(object):
             return
         print("Extract {} into {}".format(filename, cwd))
         if filename.endswith(".tgz") or filename.endswith(".tar.gz"):
-            shprint(sh.tar, "-C", cwd, "-xvzf", filename)
+            if self.ctx.use_pigz:
+                comp = '--use-compress-program={}'.format(self.ctx.use_pigz)
+            else:
+                comp = '-z'
+            shprint(sh.tar, "-C", cwd, "-xv", comp, "-f", filename)
 
         elif filename.endswith(".tbz2") or filename.endswith(".tar.bz2"):
-            shprint(sh.tar, "-C", cwd, "-xvjf", filename)
+            if self.ctx.use_pbzip2:
+                comp = '--use-compress-program={}'.format(self.ctx.use_pbzip2)
+            else:
+                comp = '-j'
+            shprint(sh.tar, "-C", cwd, "-xv", comp, "-f", filename)
 
         elif filename.endswith(".zip"):
             shprint(sh.unzip, "-d", cwd, filename)
 
         else:
-            print("Error: cannot extract, unreconized extension for {}".format(
+            print("Error: cannot extract, unrecognized extension for {}".format(
                 filename))
             raise Exception()
 
@@ -1088,14 +1113,20 @@ Xcode:
             getattr(self, args.command)()
 
         def build(self):
+            ctx = Context()
             parser = argparse.ArgumentParser(
                     description="Build the toolchain")
             parser.add_argument("recipe", nargs="+", help="Recipe to compile")
             parser.add_argument("--arch", action="append",
                                 help="Restrict compilation to this arch")
+            parser.add_argument("--concurrency", type=int, default=ctx.num_cores,
+                                help="number of concurrent build processes (where supported)")
+            parser.add_argument("--no-pigz", action="store_true", default=not bool(ctx.use_pigz),
+                                help="do not use pigz for gzip decompression")
+            parser.add_argument("--no-pbzip2", action="store_true", default=not bool(ctx.use_pbzip2),
+                                help="do not use pbzip2 for bzip2 decompression")
             args = parser.parse_args(sys.argv[2:])
 
-            ctx = Context()
             if args.arch:
                 if len(args.arch) == 1:
                     archs = args.arch[0].split()
@@ -1109,6 +1140,17 @@ Xcode:
                         continue
                 ctx.archs = [arch for arch in ctx.archs if arch.arch in archs]
                 print("Architectures restricted to: {}".format(archs))
+            ctx.num_cores = args.concurrency
+            if args.no_pigz:
+                ctx.use_pigz = False
+            if args.no_pbzip2:
+                ctx.use_pbzip2 = False
+            ctx.use_pigz = ctx.use_pbzip2
+            print("Building with {} processes, where supported".format(ctx.num_cores))
+            if ctx.use_pigz:
+                print("Using pigz to decompress gzip data")
+            if ctx.use_pbzip2:
+                print("Using pbzip2 to decompress bzip2 data")
             build_recipes(args.recipe, ctx)
 
         def recipes(self):