from os.path import ( abspath, join, realpath, dirname, expanduser, exists ) from os import environ import copy import os import glob import re import sh import shutil import subprocess from contextlib import suppress from pythonforandroid.util import ( current_directory, ensure_dir, BuildInterruptingException, ) from pythonforandroid.logger import (info, warning, info_notify, info_main, shprint) from pythonforandroid.archs import ArchARM, ArchARMv7_a, ArchAarch_64, Archx86, Archx86_64 from pythonforandroid.pythonpackage import get_package_name from pythonforandroid.recipe import CythonRecipe, Recipe from pythonforandroid.recommendations import ( check_ndk_version, check_target_api, check_ndk_api, RECOMMENDED_NDK_API, RECOMMENDED_TARGET_API) from pythonforandroid.androidndk import AndroidNDK def get_targets(sdk_dir): if exists(join(sdk_dir, 'cmdline-tools', 'latest', 'bin', 'avdmanager')): avdmanager = sh.Command(join(sdk_dir, 'cmdline-tools', 'latest', 'bin', 'avdmanager')) targets = avdmanager('list', 'target').stdout.decode('utf-8').split('\n') elif exists(join(sdk_dir, 'tools', 'bin', 'avdmanager')): avdmanager = sh.Command(join(sdk_dir, 'tools', 'bin', 'avdmanager')) targets = avdmanager('list', 'target').stdout.decode('utf-8').split('\n') elif exists(join(sdk_dir, 'tools', 'android')): android = sh.Command(join(sdk_dir, 'tools', 'android')) targets = android('list').stdout.decode('utf-8').split('\n') else: raise BuildInterruptingException( 'Could not find `android` or `sdkmanager` binaries in Android SDK', instructions='Make sure the path to the Android SDK is correct') return targets def get_available_apis(sdk_dir): targets = get_targets(sdk_dir) apis = [s for s in targets if re.match(r'^ *API level: ', s)] apis = [re.findall(r'[0-9]+', s) for s in apis] apis = [int(s[0]) for s in apis if s] return apis class Context: '''A build context. If anything will be built, an instance this class will be instantiated and used to hold all the build state.''' # Whether to make a debug or release build build_as_debuggable = False # Whether to strip debug symbols in `.so` files with_debug_symbols = False env = environ.copy() # the filepath of toolchain.py root_dir = None # the root dir where builds and dists will be stored storage_dir = None # in which bootstraps are copied for building # and recipes are built build_dir = None distribution = None """The Distribution object representing the current build target location.""" # the Android project folder where everything ends up dist_dir = None # where Android libs are cached after build # but before being placed in dists libs_dir = None aars_dir = None # Whether setup.py or similar should be used if present: use_setup_py = False ccache = None # whether to use ccache ndk = None bootstrap = None bootstrap_build_dir = None recipe_build_order = None # Will hold the list of all built recipes symlink_bootstrap_files = False # If True, will symlink instead of copying during build java_build_tool = 'auto' @property def packages_path(self): '''Where packages are downloaded before being unpacked''' return join(self.storage_dir, 'packages') @property def templates_dir(self): return join(self.root_dir, 'templates') @property def libs_dir(self): # Was previously hardcoded as self.build_dir/libs directory = join(self.build_dir, 'libs_collections', self.bootstrap.distribution.name) ensure_dir(directory) return directory @property def javaclass_dir(self): # Was previously hardcoded as self.build_dir/java directory = join(self.build_dir, 'javaclasses', self.bootstrap.distribution.name) ensure_dir(directory) return directory @property def aars_dir(self): directory = join(self.build_dir, 'aars', self.bootstrap.distribution.name) ensure_dir(directory) return directory @property def python_installs_dir(self): directory = join(self.build_dir, 'python-installs') ensure_dir(directory) return directory def get_python_install_dir(self, arch): return join(self.python_installs_dir, self.bootstrap.distribution.name, arch) def setup_dirs(self, storage_dir): '''Calculates all the storage and build dirs, and makes sure the directories exist where necessary.''' self.storage_dir = expanduser(storage_dir) if ' ' in self.storage_dir: raise ValueError('storage dir path cannot contain spaces, please ' 'specify a path with --storage-dir') self.build_dir = join(self.storage_dir, 'build') self.dist_dir = join(self.storage_dir, 'dists') def ensure_dirs(self): ensure_dir(self.storage_dir) ensure_dir(self.build_dir) ensure_dir(self.dist_dir) ensure_dir(join(self.build_dir, 'bootstrap_builds')) ensure_dir(join(self.build_dir, 'other_builds')) @property def android_api(self): '''The Android API being targeted.''' if self._android_api is None: raise ValueError('Tried to access android_api but it has not ' 'been set - this should not happen, something ' 'went wrong!') return self._android_api @android_api.setter def android_api(self, value): self._android_api = value @property def ndk_api(self): '''The API number compile against''' if self._ndk_api is None: raise ValueError('Tried to access ndk_api but it has not ' 'been set - this should not happen, something ' 'went wrong!') return self._ndk_api @ndk_api.setter def ndk_api(self, value): self._ndk_api = value @property def sdk_dir(self): '''The path to the Android SDK.''' if self._sdk_dir is None: raise ValueError('Tried to access sdk_dir but it has not ' 'been set - this should not happen, something ' 'went wrong!') return self._sdk_dir @sdk_dir.setter def sdk_dir(self, value): self._sdk_dir = value @property def ndk_dir(self): '''The path to the Android NDK.''' if self._ndk_dir is None: raise ValueError('Tried to access ndk_dir but it has not ' 'been set - this should not happen, something ' 'went wrong!') return self._ndk_dir @ndk_dir.setter def ndk_dir(self, value): self._ndk_dir = value def prepare_build_environment(self, user_sdk_dir, user_ndk_dir, user_android_api, user_ndk_api): '''Checks that build dependencies exist and sets internal variables for the Android SDK etc. ..warning:: This *must* be called before trying any build stuff ''' self.ensure_dirs() if self._build_env_prepared: return # Work out where the Android SDK is sdk_dir = None if user_sdk_dir: sdk_dir = user_sdk_dir # This is the old P4A-specific var if sdk_dir is None: sdk_dir = environ.get('ANDROIDSDK', None) # This seems used more conventionally if sdk_dir is None: sdk_dir = environ.get('ANDROID_HOME', None) # Checks in the buildozer SDK dir, useful for debug tests of p4a if sdk_dir is None: possible_dirs = glob.glob(expanduser(join( '~', '.buildozer', 'android', 'platform', 'android-sdk-*'))) possible_dirs = [d for d in possible_dirs if not d.endswith(('.bz2', '.gz'))] if possible_dirs: info('Found possible SDK dirs in buildozer dir: {}'.format( ', '.join(d.split(os.sep)[-1] for d in possible_dirs))) info('Will attempt to use SDK at {}'.format(possible_dirs[0])) warning('This SDK lookup is intended for debug only, if you ' 'use python-for-android much you should probably ' 'maintain your own SDK download.') sdk_dir = possible_dirs[0] if sdk_dir is None: raise BuildInterruptingException('Android SDK dir was not specified, exiting.') self.sdk_dir = realpath(sdk_dir) # Check what Android API we're using android_api = None if user_android_api: android_api = user_android_api info('Getting Android API version from user argument: {}'.format(android_api)) elif 'ANDROIDAPI' in environ: android_api = environ['ANDROIDAPI'] info('Found Android API target in $ANDROIDAPI: {}'.format(android_api)) else: info('Android API target was not set manually, using ' 'the default of {}'.format(RECOMMENDED_TARGET_API)) android_api = RECOMMENDED_TARGET_API android_api = int(android_api) self.android_api = android_api for arch in self.archs: # Maybe We could remove this one in a near future (ARMv5 is definitely old) check_target_api(android_api, arch) apis = get_available_apis(self.sdk_dir) info('Available Android APIs are ({})'.format( ', '.join(map(str, apis)))) if android_api in apis: info(('Requested API target {} is available, ' 'continuing.').format(android_api)) else: raise BuildInterruptingException( ('Requested API target {} is not available, install ' 'it with the SDK android tool.').format(android_api)) # Find the Android NDK # Could also use ANDROID_NDK, but doesn't look like many tools use this ndk_dir = None if user_ndk_dir: ndk_dir = user_ndk_dir info('Getting NDK dir from from user argument') if ndk_dir is None: # The old P4A-specific dir ndk_dir = environ.get('ANDROIDNDK', None) if ndk_dir is not None: info('Found NDK dir in $ANDROIDNDK: {}'.format(ndk_dir)) if ndk_dir is None: # Apparently the most common convention ndk_dir = environ.get('NDK_HOME', None) if ndk_dir is not None: info('Found NDK dir in $NDK_HOME: {}'.format(ndk_dir)) if ndk_dir is None: # Another convention (with maven?) ndk_dir = environ.get('ANDROID_NDK_HOME', None) if ndk_dir is not None: info('Found NDK dir in $ANDROID_NDK_HOME: {}'.format(ndk_dir)) if ndk_dir is None: # Checks in the buildozer NDK dir, useful # # for debug tests of p4a possible_dirs = glob.glob(expanduser(join( '~', '.buildozer', 'android', 'platform', 'android-ndk-r*'))) if possible_dirs: info('Found possible NDK dirs in buildozer dir: {}'.format( ', '.join(d.split(os.sep)[-1] for d in possible_dirs))) info('Will attempt to use NDK at {}'.format(possible_dirs[0])) warning('This NDK lookup is intended for debug only, if you ' 'use python-for-android much you should probably ' 'maintain your own NDK download.') ndk_dir = possible_dirs[0] if ndk_dir is None: raise BuildInterruptingException('Android NDK dir was not specified') self.ndk_dir = realpath(ndk_dir) check_ndk_version(ndk_dir) ndk_api = None if user_ndk_api: ndk_api = user_ndk_api info('Getting NDK API version (i.e. minimum supported API) from user argument') elif 'NDKAPI' in environ: ndk_api = environ.get('NDKAPI', None) info('Found Android API target in $NDKAPI') else: ndk_api = min(self.android_api, RECOMMENDED_NDK_API) warning('NDK API target was not set manually, using ' 'the default of {} = min(android-api={}, default ndk-api={})'.format( ndk_api, self.android_api, RECOMMENDED_NDK_API)) ndk_api = int(ndk_api) self.ndk_api = ndk_api check_ndk_api(ndk_api, self.android_api) self.ndk = AndroidNDK(self.ndk_dir) # path to some tools self.ccache = shutil.which("ccache") if not self.ccache: info('ccache is missing, the build will not be optimized in the ' 'future.') try: subprocess.check_output([ "python3", "-m", "cython", "--help", ]) except subprocess.CalledProcessError: warning('Cython for python3 missing. If you are building for ' ' a python 3 target (which is the default)' ' then THINGS WILL BREAK.') self.env["PATH"] = ":".join( [ self.ndk.llvm_bin_dir, self.ndk_dir, f"{self.sdk_dir}/tools", environ.get("PATH"), ] ) def __init__(self): self.include_dirs = [] self._build_env_prepared = False self._sdk_dir = None self._ndk_dir = None self._android_api = None self._ndk_api = None self.ndk = None self.local_recipes = None self.copy_libs = False self.activity_class_name = u'org.kivy.android.PythonActivity' self.service_class_name = u'org.kivy.android.PythonService' # this list should contain all Archs, it is pruned later self.archs = ( ArchARM(self), ArchARMv7_a(self), Archx86(self), Archx86_64(self), ArchAarch_64(self), ) self.root_dir = realpath(dirname(__file__)) # remove the most obvious flags that can break the compilation self.env.pop("LDFLAGS", None) self.env.pop("ARCHFLAGS", None) self.env.pop("CFLAGS", None) self.python_recipe = None # Set by TargetPythonRecipe def set_archs(self, arch_names): all_archs = self.archs new_archs = set() for name in arch_names: matching = [arch for arch in all_archs if arch.arch == name] for match in matching: new_archs.add(match) self.archs = list(new_archs) if not self.archs: raise BuildInterruptingException('Asked to compile for no Archs, so failing.') info('Will compile for the following archs: {}'.format( ', '.join(arch.arch for arch in self.archs))) def prepare_bootstrap(self, bootstrap): if not bootstrap: raise TypeError("None is not allowed for bootstrap") bootstrap.ctx = self self.bootstrap = bootstrap self.bootstrap.prepare_build_dir() self.bootstrap_build_dir = self.bootstrap.build_dir def prepare_dist(self): self.bootstrap.prepare_dist_dir() def get_site_packages_dir(self, arch): '''Returns the location of site-packages in the python-install build dir. ''' return self.get_python_install_dir(arch.arch) def get_libs_dir(self, arch): '''The libs dir for a given arch.''' ensure_dir(join(self.libs_dir, arch)) return join(self.libs_dir, arch) def has_lib(self, arch, lib): return exists(join(self.get_libs_dir(arch), lib)) def has_package(self, name, arch=None): # If this is a file path, it'll need special handling: if (name.find("/") >= 0 or name.find("\\") >= 0) and \ name.find("://") < 0: # (:// would indicate an url) if not os.path.exists(name): # Non-existing dir, cannot look this up. return False try: name = get_package_name(os.path.abspath(name)) except ValueError: # Failed to look up any meaningful name. return False # Try to look up recipe by name: try: recipe = Recipe.get_recipe(name, self) except ValueError: pass else: name = getattr(recipe, 'site_packages_name', None) or name name = name.replace('.', '/') site_packages_dir = self.get_site_packages_dir(arch) return (exists(join(site_packages_dir, name)) or exists(join(site_packages_dir, name + '.py')) or exists(join(site_packages_dir, name + '.pyc')) or exists(join(site_packages_dir, name + '.so')) or glob.glob(join(site_packages_dir, name + '-*.egg'))) def not_has_package(self, name, arch=None): return not self.has_package(name, arch) def build_recipes(build_order, python_modules, ctx, project_dir, ignore_project_setup_py=False ): # Put recipes in correct build order info_notify("Recipe build order is {}".format(build_order)) if python_modules: python_modules = sorted(set(python_modules)) info_notify( ('The requirements ({}) were not found as recipes, they will be ' 'installed with pip.').format(', '.join(python_modules))) recipes = [Recipe.get_recipe(name, ctx) for name in build_order] # download is arch independent info_main('# Downloading recipes ') for recipe in recipes: recipe.download_if_necessary() for arch in ctx.archs: info_main('# Building all recipes for arch {}'.format(arch.arch)) info_main('# Unpacking recipes') for recipe in recipes: ensure_dir(recipe.get_build_container_dir(arch.arch)) recipe.prepare_build_dir(arch.arch) info_main('# Prebuilding recipes') # 2) prebuild packages for recipe in recipes: info_main('Prebuilding {} for {}'.format(recipe.name, arch.arch)) recipe.prebuild_arch(arch) recipe.apply_patches(arch) # 3) build packages info_main('# Building recipes') for recipe in recipes: info_main('Building {} for {}'.format(recipe.name, arch.arch)) if recipe.should_build(arch): recipe.build_arch(arch) else: info('{} said it is already built, skipping' .format(recipe.name)) recipe.install_libraries(arch) # 4) biglink everything info_main('# Biglinking object files') if not ctx.python_recipe: biglink(ctx, arch) else: warning( "Context's python recipe found, " "skipping biglink (will this work?)" ) # 5) postbuild packages info_main('# Postbuilding recipes') for recipe in recipes: info_main('Postbuilding {} for {}'.format(recipe.name, arch.arch)) recipe.postbuild_arch(arch) info_main('# Installing pure Python modules') for arch in ctx.archs: run_pymodules_install( ctx, arch, python_modules, project_dir, ignore_setup_py=ignore_project_setup_py ) def project_has_setup_py(project_dir): return (project_dir is not None and (exists(join(project_dir, "setup.py")) or exists(join(project_dir, "pyproject.toml")) )) def run_setuppy_install(ctx, project_dir, env=None, arch=None): env = env or {} with current_directory(project_dir): info('got setup.py or similar, running project install. ' + '(disable this behavior with --ignore-setup-py)') # Compute & output the constraints we will use: info('Contents that will be used for constraints.txt:') constraints = subprocess.check_output([ join( ctx.build_dir, "venv", "bin", "pip" ), "freeze" ], env=copy.copy(env)) with suppress(AttributeError): constraints = constraints.decode("utf-8", "replace") info(constraints) # Make sure all packages found are fixed in version # by writing a constraint file, to avoid recipes being # upgraded & reinstalled: with open('._tmp_p4a_recipe_constraints.txt', 'wb') as fileh: fileh.write(constraints.encode("utf-8", "replace")) try: info('Populating venv\'s site-packages with ' 'ctx.get_site_packages_dir()...') # Copy dist contents into site-packages for discovery. # Why this is needed: # --target is somewhat evil and messes with discovery of # packages in PYTHONPATH if that also includes the target # folder. So we need to use the regular virtualenv # site-packages folder instead. # Reference: # https://github.com/pypa/pip/issues/6223 ctx_site_packages_dir = os.path.normpath( os.path.abspath(ctx.get_site_packages_dir(arch)) ) venv_site_packages_dir = os.path.normpath(os.path.join( ctx.build_dir, "venv", "lib", [ f for f in os.listdir(os.path.join( ctx.build_dir, "venv", "lib" )) if f.startswith("python") ][0], "site-packages" )) copied_over_contents = [] for f in os.listdir(ctx_site_packages_dir): full_path = os.path.join(ctx_site_packages_dir, f) if not os.path.exists(os.path.join( venv_site_packages_dir, f )): if os.path.isdir(full_path): shutil.copytree(full_path, os.path.join( venv_site_packages_dir, f )) else: shutil.copy2(full_path, os.path.join( venv_site_packages_dir, f )) copied_over_contents.append(f) # Get listing of virtualenv's site-packages, to see the # newly added things afterwards & copy them back into # the distribution folder / build context site-packages: previous_venv_contents = os.listdir( venv_site_packages_dir ) # Actually run setup.py: info('Launching package install...') shprint(sh.bash, '-c', ( "'" + join( ctx.build_dir, "venv", "bin", "pip" ).replace("'", "'\"'\"'") + "' " + "install -c ._tmp_p4a_recipe_constraints.txt -v ." ).format(ctx.get_site_packages_dir(arch). replace("'", "'\"'\"'")), _env=copy.copy(env)) # Go over all new additions and copy them back: info('Copying additions resulting from setup.py back ' 'into ctx.get_site_packages_dir()...') new_venv_additions = [] for f in (set(os.listdir(venv_site_packages_dir)) - set(previous_venv_contents)): new_venv_additions.append(f) full_path = os.path.join(venv_site_packages_dir, f) if os.path.isdir(full_path): shutil.copytree(full_path, os.path.join( ctx_site_packages_dir, f )) else: shutil.copy2(full_path, os.path.join( ctx_site_packages_dir, f )) # Undo all the changes we did to the venv-site packages: info('Reverting additions to ' 'virtualenv\'s site-packages...') for f in set(copied_over_contents + new_venv_additions): full_path = os.path.join(venv_site_packages_dir, f) if os.path.isdir(full_path): shutil.rmtree(full_path) else: os.remove(full_path) finally: os.remove("._tmp_p4a_recipe_constraints.txt") def run_pymodules_install(ctx, arch, modules, project_dir=None, ignore_setup_py=False): """ This function will take care of all non-recipe things, by: 1. Processing them from --requirements (the modules argument) and installing them 2. Installing the user project/app itself via setup.py if ignore_setup_py=True """ info('*** PYTHON PACKAGE / PROJECT INSTALL STAGE FOR ARCH: {} ***'.format(arch)) modules = [m for m in modules if ctx.not_has_package(m, arch)] # We change current working directory later, so this has to be an absolute # path or `None` in case that we didn't supply the `project_dir` via kwargs project_dir = abspath(project_dir) if project_dir else None # Bail out if no python deps and no setup.py to process: if not modules and ( ignore_setup_py or project_dir is None or not project_has_setup_py(project_dir) ): info('No Python modules and no setup.py to process, skipping') return # Output messages about what we're going to do: if modules: info( "The requirements ({}) don\'t have recipes, attempting to " "install them with pip".format(', '.join(modules)) ) info( "If this fails, it may mean that the module has compiled " "components and needs a recipe." ) if project_dir is not None and \ project_has_setup_py(project_dir) and not ignore_setup_py: info( "Will process project install, if it fails then the " "project may not be compatible for Android install." ) # Use our hostpython to create the virtualenv host_python = sh.Command(ctx.hostpython) with current_directory(join(ctx.build_dir)): shprint(host_python, '-m', 'venv', 'venv') # Prepare base environment and upgrade pip: base_env = dict(copy.copy(os.environ)) base_env["PYTHONPATH"] = ctx.get_site_packages_dir(arch) info('Upgrade pip to latest version') shprint(sh.bash, '-c', ( "source venv/bin/activate && pip install -U pip" ), _env=copy.copy(base_env)) # Install Cython in case modules need it to build: info('Install Cython in case one of the modules needs it to build') shprint(sh.bash, '-c', ( "venv/bin/pip install --upgrade Cython" ), _env=copy.copy(base_env)) # Get environment variables for build (with CC/compiler set): standard_recipe = CythonRecipe() standard_recipe.ctx = ctx # (note: following line enables explicit -lpython... linker options) standard_recipe.call_hostpython_via_targetpython = False recipe_env = standard_recipe.get_recipe_env(ctx.archs[0]) env = copy.copy(base_env) env.update(recipe_env) # Make sure our build package dir is available, and the virtualenv # site packages come FIRST (so the proper pip version is used): env["PYTHONPATH"] += ":" + ctx.get_site_packages_dir(arch) env["PYTHONPATH"] = os.path.abspath(join( ctx.build_dir, "venv", "lib", "python" + ctx.python_recipe.major_minor_version_string, "site-packages")) + ":" + env["PYTHONPATH"] # Install the manually specified requirements first: if not modules: info('There are no Python modules to install, skipping') else: info('Creating a requirements.txt file for the Python modules') with open('requirements.txt', 'w') as fileh: for module in modules: key = 'VERSION_' + module if key in environ: line = '{}=={}\n'.format(module, environ[key]) else: line = '{}\n'.format(module) fileh.write(line) info('Installing Python modules with pip') info( "IF THIS FAILS, THE MODULES MAY NEED A RECIPE. " "A reason for this is often modules compiling " "native code that is unaware of Android cross-compilation " "and does not work without additional " "changes / workarounds." ) shprint(sh.bash, '-c', ( "venv/bin/pip " + "install -v --target '{0}' --no-deps -r requirements.txt" ).format(ctx.get_site_packages_dir(arch).replace("'", "'\"'\"'")), _env=copy.copy(env)) # Afterwards, run setup.py if present: if project_dir is not None and ( project_has_setup_py(project_dir) and not ignore_setup_py ): run_setuppy_install(ctx, project_dir, env, arch.arch) elif not ignore_setup_py: info("No setup.py found in project directory: " + str(project_dir)) # Strip object files after potential Cython or native code builds: if not ctx.with_debug_symbols: standard_recipe.strip_object_files( arch, env, build_dir=ctx.build_dir ) def biglink(ctx, arch): # First, collate object files from each recipe info('Collating object files from each recipe') obj_dir = join(ctx.bootstrap.build_dir, 'collated_objects') ensure_dir(obj_dir) recipes = [Recipe.get_recipe(name, ctx) for name in ctx.recipe_build_order] for recipe in recipes: recipe_obj_dir = join(recipe.get_build_container_dir(arch.arch), 'objects_{}'.format(recipe.name)) if not exists(recipe_obj_dir): info('{} recipe has no biglinkable files dir, skipping' .format(recipe.name)) continue files = glob.glob(join(recipe_obj_dir, '*')) if not len(files): info('{} recipe has no biglinkable files, skipping' .format(recipe.name)) continue info('{} recipe has object files, copying'.format(recipe.name)) files.append(obj_dir) shprint(sh.cp, '-r', *files) env = arch.get_env() env['LDFLAGS'] = env['LDFLAGS'] + ' -L{}'.format( join(ctx.bootstrap.build_dir, 'obj', 'local', arch.arch)) if not len(glob.glob(join(obj_dir, '*'))): info('There seem to be no libraries to biglink, skipping.') return info('Biglinking') info('target {}'.format(join(ctx.get_libs_dir(arch.arch), 'libpymodules.so'))) do_biglink = copylibs_function if ctx.copy_libs else biglink_function # Move to the directory containing crtstart_so.o and crtend_so.o # This is necessary with newer NDKs? A gcc bug? with current_directory(arch.ndk_lib_dir): do_biglink( join(ctx.get_libs_dir(arch.arch), 'libpymodules.so'), obj_dir.split(' '), extra_link_dirs=[join(ctx.bootstrap.build_dir, 'obj', 'local', arch.arch), os.path.abspath('.')], env=env) def biglink_function(soname, objs_paths, extra_link_dirs=None, env=None): if extra_link_dirs is None: extra_link_dirs = [] print('objs_paths are', objs_paths) sofiles = [] for directory in objs_paths: for fn in os.listdir(directory): fn = os.path.join(directory, fn) if not fn.endswith(".so.o"): continue if not os.path.exists(fn[:-2] + ".libs"): continue sofiles.append(fn[:-2]) # The raw argument list. args = [] for fn in sofiles: afn = fn + ".o" libsfn = fn + ".libs" args.append(afn) with open(libsfn) as fd: data = fd.read() args.extend(data.split(" ")) unique_args = [] while args: a = args.pop() if a in ('-L', ): continue if a not in unique_args: unique_args.insert(0, a) for dir in extra_link_dirs: link = '-L{}'.format(dir) if link not in unique_args: unique_args.append(link) cc_name = env['CC'] cc = sh.Command(cc_name.split()[0]) cc = cc.bake(*cc_name.split()[1:]) shprint(cc, '-shared', '-O3', '-o', soname, *unique_args, _env=env) def copylibs_function(soname, objs_paths, extra_link_dirs=None, env=None): if extra_link_dirs is None: extra_link_dirs = [] print('objs_paths are', objs_paths) re_needso = re.compile(r'^.*\(NEEDED\)\s+Shared library: \[lib(.*)\.so\]\s*$') blacklist_libs = ( 'c', 'stdc++', 'dl', 'python2.7', 'sdl', 'sdl_image', 'sdl_ttf', 'z', 'm', 'GLESv2', 'jpeg', 'png', 'log', # bootstrap takes care of sdl2 libs (if applicable) 'SDL2', 'SDL2_ttf', 'SDL2_image', 'SDL2_mixer', ) found_libs = [] sofiles = [] if env and 'READELF' in env: readelf = env['READELF'] elif 'READELF' in os.environ: readelf = os.environ['READELF'] else: readelf = shutil.which('readelf').strip() readelf = sh.Command(readelf).bake('-d') dest = dirname(soname) for directory in objs_paths: for fn in os.listdir(directory): fn = join(directory, fn) if not fn.endswith('.libs'): continue dirfn = fn[:-1] + 'dirs' if not exists(dirfn): continue with open(fn) as f: libs = f.read().strip().split(' ') needed_libs = [lib for lib in libs if lib and lib not in blacklist_libs and lib not in found_libs] while needed_libs: print('need libs:\n\t' + '\n\t'.join(needed_libs)) start_needed_libs = needed_libs[:] found_sofiles = [] with open(dirfn) as f: libdirs = f.read().split() for libdir in libdirs: if not needed_libs: break if libdir == dest: # don't need to copy from dest to dest! continue libdir = libdir.strip() print('scanning', libdir) for lib in needed_libs[:]: if lib in found_libs: continue if lib.endswith('.a'): needed_libs.remove(lib) found_libs.append(lib) continue lib_a = 'lib' + lib + '.a' libpath_a = join(libdir, lib_a) lib_so = 'lib' + lib + '.so' libpath_so = join(libdir, lib_so) plain_so = lib + '.so' plainpath_so = join(libdir, plain_so) sopath = None if exists(libpath_so): sopath = libpath_so elif exists(plainpath_so): sopath = plainpath_so if sopath: print('found', lib, 'in', libdir) found_sofiles.append(sopath) needed_libs.remove(lib) found_libs.append(lib) continue if exists(libpath_a): print('found', lib, '(static) in', libdir) needed_libs.remove(lib) found_libs.append(lib) continue for sofile in found_sofiles: print('scanning dependencies for', sofile) out = readelf(sofile) for line in out.splitlines(): needso = re_needso.match(line) if needso: lib = needso.group(1) if (lib not in needed_libs and lib not in found_libs and lib not in blacklist_libs): needed_libs.append(needso.group(1)) sofiles += found_sofiles if needed_libs == start_needed_libs: raise RuntimeError( 'Failed to locate needed libraries!\n\t' + '\n\t'.join(needed_libs)) print('Copying libraries') shprint(sh.cp, *sofiles, dest)