1003 lines
37 KiB
Python
1003 lines
37 KiB
Python
from os.path import (
|
|
abspath, join, realpath, dirname, expanduser, exists
|
|
)
|
|
from os import environ
|
|
import copy
|
|
import os
|
|
import glob
|
|
import re
|
|
import sh
|
|
import shutil
|
|
import subprocess
|
|
from contextlib import suppress
|
|
|
|
from pythonforandroid.util import (
|
|
current_directory, ensure_dir,
|
|
BuildInterruptingException,
|
|
)
|
|
from pythonforandroid.logger import (info, warning, info_notify, info_main, shprint)
|
|
from pythonforandroid.archs import ArchARM, ArchARMv7_a, ArchAarch_64, Archx86, Archx86_64
|
|
from pythonforandroid.pythonpackage import get_package_name
|
|
from pythonforandroid.recipe import CythonRecipe, Recipe
|
|
from pythonforandroid.recommendations import (
|
|
check_ndk_version, check_target_api, check_ndk_api,
|
|
RECOMMENDED_NDK_API, RECOMMENDED_TARGET_API)
|
|
from pythonforandroid.androidndk import AndroidNDK
|
|
|
|
|
|
def get_targets(sdk_dir):
|
|
if exists(join(sdk_dir, 'cmdline-tools', 'latest', 'bin', 'avdmanager')):
|
|
avdmanager = sh.Command(join(sdk_dir, 'cmdline-tools', 'latest', 'bin', 'avdmanager'))
|
|
targets = avdmanager('list', 'target').stdout.decode('utf-8').split('\n')
|
|
|
|
elif exists(join(sdk_dir, 'tools', 'bin', 'avdmanager')):
|
|
avdmanager = sh.Command(join(sdk_dir, 'tools', 'bin', 'avdmanager'))
|
|
targets = avdmanager('list', 'target').stdout.decode('utf-8').split('\n')
|
|
elif exists(join(sdk_dir, 'tools', 'android')):
|
|
android = sh.Command(join(sdk_dir, 'tools', 'android'))
|
|
targets = android('list').stdout.decode('utf-8').split('\n')
|
|
else:
|
|
raise BuildInterruptingException(
|
|
'Could not find `android` or `sdkmanager` binaries in Android SDK',
|
|
instructions='Make sure the path to the Android SDK is correct')
|
|
return targets
|
|
|
|
|
|
def get_available_apis(sdk_dir):
|
|
targets = get_targets(sdk_dir)
|
|
apis = [s for s in targets if re.match(r'^ *API level: ', s)]
|
|
apis = [re.findall(r'[0-9]+', s) for s in apis]
|
|
apis = [int(s[0]) for s in apis if s]
|
|
return apis
|
|
|
|
|
|
class Context:
|
|
'''A build context. If anything will be built, an instance this class
|
|
will be instantiated and used to hold all the build state.'''
|
|
|
|
# Whether to make a debug or release build
|
|
build_as_debuggable = False
|
|
|
|
# Whether to strip debug symbols in `.so` files
|
|
with_debug_symbols = False
|
|
|
|
env = environ.copy()
|
|
# the filepath of toolchain.py
|
|
root_dir = None
|
|
# the root dir where builds and dists will be stored
|
|
storage_dir = None
|
|
|
|
# in which bootstraps are copied for building
|
|
# and recipes are built
|
|
build_dir = None
|
|
|
|
distribution = None
|
|
"""The Distribution object representing the current build target location."""
|
|
|
|
# the Android project folder where everything ends up
|
|
dist_dir = None
|
|
|
|
# where Android libs are cached after build
|
|
# but before being placed in dists
|
|
libs_dir = None
|
|
aars_dir = None
|
|
|
|
# Whether setup.py or similar should be used if present:
|
|
use_setup_py = False
|
|
|
|
ccache = None # whether to use ccache
|
|
|
|
ndk = None
|
|
|
|
bootstrap = None
|
|
bootstrap_build_dir = None
|
|
|
|
recipe_build_order = None # Will hold the list of all built recipes
|
|
|
|
symlink_bootstrap_files = False # If True, will symlink instead of copying during build
|
|
|
|
java_build_tool = 'auto'
|
|
|
|
@property
|
|
def packages_path(self):
|
|
'''Where packages are downloaded before being unpacked'''
|
|
return join(self.storage_dir, 'packages')
|
|
|
|
@property
|
|
def templates_dir(self):
|
|
return join(self.root_dir, 'templates')
|
|
|
|
@property
|
|
def libs_dir(self):
|
|
# Was previously hardcoded as self.build_dir/libs
|
|
directory = join(self.build_dir, 'libs_collections',
|
|
self.bootstrap.distribution.name)
|
|
ensure_dir(directory)
|
|
return directory
|
|
|
|
@property
|
|
def javaclass_dir(self):
|
|
# Was previously hardcoded as self.build_dir/java
|
|
directory = join(self.build_dir, 'javaclasses',
|
|
self.bootstrap.distribution.name)
|
|
ensure_dir(directory)
|
|
return directory
|
|
|
|
@property
|
|
def aars_dir(self):
|
|
directory = join(self.build_dir, 'aars', self.bootstrap.distribution.name)
|
|
ensure_dir(directory)
|
|
return directory
|
|
|
|
@property
|
|
def python_installs_dir(self):
|
|
directory = join(self.build_dir, 'python-installs')
|
|
ensure_dir(directory)
|
|
return directory
|
|
|
|
def get_python_install_dir(self, arch):
|
|
return join(self.python_installs_dir, self.bootstrap.distribution.name, arch)
|
|
|
|
def setup_dirs(self, storage_dir):
|
|
'''Calculates all the storage and build dirs, and makes sure
|
|
the directories exist where necessary.'''
|
|
self.storage_dir = expanduser(storage_dir)
|
|
if ' ' in self.storage_dir:
|
|
raise ValueError('storage dir path cannot contain spaces, please '
|
|
'specify a path with --storage-dir')
|
|
self.build_dir = join(self.storage_dir, 'build')
|
|
self.dist_dir = join(self.storage_dir, 'dists')
|
|
|
|
def ensure_dirs(self):
|
|
ensure_dir(self.storage_dir)
|
|
ensure_dir(self.build_dir)
|
|
ensure_dir(self.dist_dir)
|
|
ensure_dir(join(self.build_dir, 'bootstrap_builds'))
|
|
ensure_dir(join(self.build_dir, 'other_builds'))
|
|
|
|
@property
|
|
def android_api(self):
|
|
'''The Android API being targeted.'''
|
|
if self._android_api is None:
|
|
raise ValueError('Tried to access android_api but it has not '
|
|
'been set - this should not happen, something '
|
|
'went wrong!')
|
|
return self._android_api
|
|
|
|
@android_api.setter
|
|
def android_api(self, value):
|
|
self._android_api = value
|
|
|
|
@property
|
|
def ndk_api(self):
|
|
'''The API number compile against'''
|
|
if self._ndk_api is None:
|
|
raise ValueError('Tried to access ndk_api but it has not '
|
|
'been set - this should not happen, something '
|
|
'went wrong!')
|
|
return self._ndk_api
|
|
|
|
@ndk_api.setter
|
|
def ndk_api(self, value):
|
|
self._ndk_api = value
|
|
|
|
@property
|
|
def sdk_dir(self):
|
|
'''The path to the Android SDK.'''
|
|
if self._sdk_dir is None:
|
|
raise ValueError('Tried to access sdk_dir but it has not '
|
|
'been set - this should not happen, something '
|
|
'went wrong!')
|
|
return self._sdk_dir
|
|
|
|
@sdk_dir.setter
|
|
def sdk_dir(self, value):
|
|
self._sdk_dir = value
|
|
|
|
@property
|
|
def ndk_dir(self):
|
|
'''The path to the Android NDK.'''
|
|
if self._ndk_dir is None:
|
|
raise ValueError('Tried to access ndk_dir but it has not '
|
|
'been set - this should not happen, something '
|
|
'went wrong!')
|
|
return self._ndk_dir
|
|
|
|
@ndk_dir.setter
|
|
def ndk_dir(self, value):
|
|
self._ndk_dir = value
|
|
|
|
def prepare_build_environment(self,
|
|
user_sdk_dir,
|
|
user_ndk_dir,
|
|
user_android_api,
|
|
user_ndk_api):
|
|
'''Checks that build dependencies exist and sets internal variables
|
|
for the Android SDK etc.
|
|
|
|
..warning:: This *must* be called before trying any build stuff
|
|
|
|
'''
|
|
|
|
self.ensure_dirs()
|
|
|
|
if self._build_env_prepared:
|
|
return
|
|
|
|
# Work out where the Android SDK is
|
|
sdk_dir = None
|
|
if user_sdk_dir:
|
|
sdk_dir = user_sdk_dir
|
|
# This is the old P4A-specific var
|
|
if sdk_dir is None:
|
|
sdk_dir = environ.get('ANDROIDSDK', None)
|
|
# This seems used more conventionally
|
|
if sdk_dir is None:
|
|
sdk_dir = environ.get('ANDROID_HOME', None)
|
|
# Checks in the buildozer SDK dir, useful for debug tests of p4a
|
|
if sdk_dir is None:
|
|
possible_dirs = glob.glob(expanduser(join(
|
|
'~', '.buildozer', 'android', 'platform', 'android-sdk-*')))
|
|
possible_dirs = [d for d in possible_dirs if not
|
|
d.endswith(('.bz2', '.gz'))]
|
|
if possible_dirs:
|
|
info('Found possible SDK dirs in buildozer dir: {}'.format(
|
|
', '.join(d.split(os.sep)[-1] for d in possible_dirs)))
|
|
info('Will attempt to use SDK at {}'.format(possible_dirs[0]))
|
|
warning('This SDK lookup is intended for debug only, if you '
|
|
'use python-for-android much you should probably '
|
|
'maintain your own SDK download.')
|
|
sdk_dir = possible_dirs[0]
|
|
if sdk_dir is None:
|
|
raise BuildInterruptingException('Android SDK dir was not specified, exiting.')
|
|
self.sdk_dir = realpath(sdk_dir)
|
|
|
|
# Check what Android API we're using
|
|
android_api = None
|
|
if user_android_api:
|
|
android_api = user_android_api
|
|
info('Getting Android API version from user argument: {}'.format(android_api))
|
|
elif 'ANDROIDAPI' in environ:
|
|
android_api = environ['ANDROIDAPI']
|
|
info('Found Android API target in $ANDROIDAPI: {}'.format(android_api))
|
|
else:
|
|
info('Android API target was not set manually, using '
|
|
'the default of {}'.format(RECOMMENDED_TARGET_API))
|
|
android_api = RECOMMENDED_TARGET_API
|
|
android_api = int(android_api)
|
|
self.android_api = android_api
|
|
|
|
for arch in self.archs:
|
|
# Maybe We could remove this one in a near future (ARMv5 is definitely old)
|
|
check_target_api(android_api, arch)
|
|
apis = get_available_apis(self.sdk_dir)
|
|
info('Available Android APIs are ({})'.format(
|
|
', '.join(map(str, apis))))
|
|
if android_api in apis:
|
|
info(('Requested API target {} is available, '
|
|
'continuing.').format(android_api))
|
|
else:
|
|
raise BuildInterruptingException(
|
|
('Requested API target {} is not available, install '
|
|
'it with the SDK android tool.').format(android_api))
|
|
|
|
# Find the Android NDK
|
|
# Could also use ANDROID_NDK, but doesn't look like many tools use this
|
|
ndk_dir = None
|
|
if user_ndk_dir:
|
|
ndk_dir = user_ndk_dir
|
|
info('Getting NDK dir from from user argument')
|
|
if ndk_dir is None: # The old P4A-specific dir
|
|
ndk_dir = environ.get('ANDROIDNDK', None)
|
|
if ndk_dir is not None:
|
|
info('Found NDK dir in $ANDROIDNDK: {}'.format(ndk_dir))
|
|
if ndk_dir is None: # Apparently the most common convention
|
|
ndk_dir = environ.get('NDK_HOME', None)
|
|
if ndk_dir is not None:
|
|
info('Found NDK dir in $NDK_HOME: {}'.format(ndk_dir))
|
|
if ndk_dir is None: # Another convention (with maven?)
|
|
ndk_dir = environ.get('ANDROID_NDK_HOME', None)
|
|
if ndk_dir is not None:
|
|
info('Found NDK dir in $ANDROID_NDK_HOME: {}'.format(ndk_dir))
|
|
if ndk_dir is None: # Checks in the buildozer NDK dir, useful
|
|
# # for debug tests of p4a
|
|
possible_dirs = glob.glob(expanduser(join(
|
|
'~', '.buildozer', 'android', 'platform', 'android-ndk-r*')))
|
|
if possible_dirs:
|
|
info('Found possible NDK dirs in buildozer dir: {}'.format(
|
|
', '.join(d.split(os.sep)[-1] for d in possible_dirs)))
|
|
info('Will attempt to use NDK at {}'.format(possible_dirs[0]))
|
|
warning('This NDK lookup is intended for debug only, if you '
|
|
'use python-for-android much you should probably '
|
|
'maintain your own NDK download.')
|
|
ndk_dir = possible_dirs[0]
|
|
if ndk_dir is None:
|
|
raise BuildInterruptingException('Android NDK dir was not specified')
|
|
self.ndk_dir = realpath(ndk_dir)
|
|
check_ndk_version(ndk_dir)
|
|
|
|
ndk_api = None
|
|
if user_ndk_api:
|
|
ndk_api = user_ndk_api
|
|
info('Getting NDK API version (i.e. minimum supported API) from user argument')
|
|
elif 'NDKAPI' in environ:
|
|
ndk_api = environ.get('NDKAPI', None)
|
|
info('Found Android API target in $NDKAPI')
|
|
else:
|
|
ndk_api = min(self.android_api, RECOMMENDED_NDK_API)
|
|
warning('NDK API target was not set manually, using '
|
|
'the default of {} = min(android-api={}, default ndk-api={})'.format(
|
|
ndk_api, self.android_api, RECOMMENDED_NDK_API))
|
|
ndk_api = int(ndk_api)
|
|
self.ndk_api = ndk_api
|
|
|
|
check_ndk_api(ndk_api, self.android_api)
|
|
|
|
self.ndk = AndroidNDK(self.ndk_dir)
|
|
|
|
# path to some tools
|
|
self.ccache = shutil.which("ccache")
|
|
if not self.ccache:
|
|
info('ccache is missing, the build will not be optimized in the '
|
|
'future.')
|
|
try:
|
|
subprocess.check_output([
|
|
"python3", "-m", "cython", "--help",
|
|
])
|
|
except subprocess.CalledProcessError:
|
|
warning('Cython for python3 missing. If you are building for '
|
|
' a python 3 target (which is the default)'
|
|
' then THINGS WILL BREAK.')
|
|
|
|
self.env["PATH"] = ":".join(
|
|
[
|
|
self.ndk.llvm_bin_dir,
|
|
self.ndk_dir,
|
|
f"{self.sdk_dir}/tools",
|
|
environ.get("PATH"),
|
|
]
|
|
)
|
|
|
|
def __init__(self):
|
|
self.include_dirs = []
|
|
|
|
self._build_env_prepared = False
|
|
|
|
self._sdk_dir = None
|
|
self._ndk_dir = None
|
|
self._android_api = None
|
|
self._ndk_api = None
|
|
self.ndk = None
|
|
|
|
self.local_recipes = None
|
|
self.copy_libs = False
|
|
|
|
self.activity_class_name = u'org.kivy.android.PythonActivity'
|
|
self.service_class_name = u'org.kivy.android.PythonService'
|
|
|
|
# this list should contain all Archs, it is pruned later
|
|
self.archs = (
|
|
ArchARM(self),
|
|
ArchARMv7_a(self),
|
|
Archx86(self),
|
|
Archx86_64(self),
|
|
ArchAarch_64(self),
|
|
)
|
|
|
|
self.root_dir = realpath(dirname(__file__))
|
|
|
|
# remove the most obvious flags that can break the compilation
|
|
self.env.pop("LDFLAGS", None)
|
|
self.env.pop("ARCHFLAGS", None)
|
|
self.env.pop("CFLAGS", None)
|
|
|
|
self.python_recipe = None # Set by TargetPythonRecipe
|
|
|
|
def set_archs(self, arch_names):
|
|
all_archs = self.archs
|
|
new_archs = set()
|
|
for name in arch_names:
|
|
matching = [arch for arch in all_archs if arch.arch == name]
|
|
for match in matching:
|
|
new_archs.add(match)
|
|
self.archs = list(new_archs)
|
|
if not self.archs:
|
|
raise BuildInterruptingException('Asked to compile for no Archs, so failing.')
|
|
info('Will compile for the following archs: {}'.format(
|
|
', '.join(arch.arch for arch in self.archs)))
|
|
|
|
def prepare_bootstrap(self, bootstrap):
|
|
if not bootstrap:
|
|
raise TypeError("None is not allowed for bootstrap")
|
|
bootstrap.ctx = self
|
|
self.bootstrap = bootstrap
|
|
self.bootstrap.prepare_build_dir()
|
|
self.bootstrap_build_dir = self.bootstrap.build_dir
|
|
|
|
def prepare_dist(self):
|
|
self.bootstrap.prepare_dist_dir()
|
|
|
|
def get_site_packages_dir(self, arch):
|
|
'''Returns the location of site-packages in the python-install build
|
|
dir.
|
|
'''
|
|
return self.get_python_install_dir(arch.arch)
|
|
|
|
def get_libs_dir(self, arch):
|
|
'''The libs dir for a given arch.'''
|
|
ensure_dir(join(self.libs_dir, arch))
|
|
return join(self.libs_dir, arch)
|
|
|
|
def has_lib(self, arch, lib):
|
|
return exists(join(self.get_libs_dir(arch), lib))
|
|
|
|
def has_package(self, name, arch=None):
|
|
# If this is a file path, it'll need special handling:
|
|
if (name.find("/") >= 0 or name.find("\\") >= 0) and \
|
|
name.find("://") < 0: # (:// would indicate an url)
|
|
if not os.path.exists(name):
|
|
# Non-existing dir, cannot look this up.
|
|
return False
|
|
try:
|
|
name = get_package_name(os.path.abspath(name))
|
|
except ValueError:
|
|
# Failed to look up any meaningful name.
|
|
return False
|
|
|
|
# Try to look up recipe by name:
|
|
try:
|
|
recipe = Recipe.get_recipe(name, self)
|
|
except ValueError:
|
|
pass
|
|
else:
|
|
name = getattr(recipe, 'site_packages_name', None) or name
|
|
name = name.replace('.', '/')
|
|
site_packages_dir = self.get_site_packages_dir(arch)
|
|
return (exists(join(site_packages_dir, name)) or
|
|
exists(join(site_packages_dir, name + '.py')) or
|
|
exists(join(site_packages_dir, name + '.pyc')) or
|
|
exists(join(site_packages_dir, name + '.so')) or
|
|
glob.glob(join(site_packages_dir, name + '-*.egg')))
|
|
|
|
def not_has_package(self, name, arch=None):
|
|
return not self.has_package(name, arch)
|
|
|
|
|
|
def build_recipes(build_order, python_modules, ctx, project_dir,
|
|
ignore_project_setup_py=False
|
|
):
|
|
# Put recipes in correct build order
|
|
info_notify("Recipe build order is {}".format(build_order))
|
|
if python_modules:
|
|
python_modules = sorted(set(python_modules))
|
|
info_notify(
|
|
('The requirements ({}) were not found as recipes, they will be '
|
|
'installed with pip.').format(', '.join(python_modules)))
|
|
|
|
recipes = [Recipe.get_recipe(name, ctx) for name in build_order]
|
|
|
|
# download is arch independent
|
|
info_main('# Downloading recipes ')
|
|
for recipe in recipes:
|
|
recipe.download_if_necessary()
|
|
|
|
for arch in ctx.archs:
|
|
info_main('# Building all recipes for arch {}'.format(arch.arch))
|
|
|
|
info_main('# Unpacking recipes')
|
|
for recipe in recipes:
|
|
ensure_dir(recipe.get_build_container_dir(arch.arch))
|
|
recipe.prepare_build_dir(arch.arch)
|
|
|
|
info_main('# Prebuilding recipes')
|
|
# 2) prebuild packages
|
|
for recipe in recipes:
|
|
info_main('Prebuilding {} for {}'.format(recipe.name, arch.arch))
|
|
recipe.prebuild_arch(arch)
|
|
recipe.apply_patches(arch)
|
|
|
|
# 3) build packages
|
|
info_main('# Building recipes')
|
|
for recipe in recipes:
|
|
info_main('Building {} for {}'.format(recipe.name, arch.arch))
|
|
if recipe.should_build(arch):
|
|
recipe.build_arch(arch)
|
|
else:
|
|
info('{} said it is already built, skipping'
|
|
.format(recipe.name))
|
|
recipe.install_libraries(arch)
|
|
|
|
# 4) biglink everything
|
|
info_main('# Biglinking object files')
|
|
if not ctx.python_recipe:
|
|
biglink(ctx, arch)
|
|
else:
|
|
warning(
|
|
"Context's python recipe found, "
|
|
"skipping biglink (will this work?)"
|
|
)
|
|
|
|
# 5) postbuild packages
|
|
info_main('# Postbuilding recipes')
|
|
for recipe in recipes:
|
|
info_main('Postbuilding {} for {}'.format(recipe.name, arch.arch))
|
|
recipe.postbuild_arch(arch)
|
|
|
|
info_main('# Installing pure Python modules')
|
|
for arch in ctx.archs:
|
|
run_pymodules_install(
|
|
ctx, arch, python_modules, project_dir,
|
|
ignore_setup_py=ignore_project_setup_py
|
|
)
|
|
|
|
|
|
def project_has_setup_py(project_dir):
|
|
return (project_dir is not None and
|
|
(exists(join(project_dir, "setup.py")) or
|
|
exists(join(project_dir, "pyproject.toml"))
|
|
))
|
|
|
|
|
|
def run_setuppy_install(ctx, project_dir, env=None, arch=None):
|
|
env = env or {}
|
|
|
|
with current_directory(project_dir):
|
|
info('got setup.py or similar, running project install. ' +
|
|
'(disable this behavior with --ignore-setup-py)')
|
|
|
|
# Compute & output the constraints we will use:
|
|
info('Contents that will be used for constraints.txt:')
|
|
constraints = subprocess.check_output([
|
|
join(
|
|
ctx.build_dir, "venv", "bin", "pip"
|
|
),
|
|
"freeze"
|
|
], env=copy.copy(env))
|
|
with suppress(AttributeError):
|
|
constraints = constraints.decode("utf-8", "replace")
|
|
info(constraints)
|
|
|
|
# Make sure all packages found are fixed in version
|
|
# by writing a constraint file, to avoid recipes being
|
|
# upgraded & reinstalled:
|
|
with open('._tmp_p4a_recipe_constraints.txt', 'wb') as fileh:
|
|
fileh.write(constraints.encode("utf-8", "replace"))
|
|
try:
|
|
|
|
info('Populating venv\'s site-packages with '
|
|
'ctx.get_site_packages_dir()...')
|
|
|
|
# Copy dist contents into site-packages for discovery.
|
|
# Why this is needed:
|
|
# --target is somewhat evil and messes with discovery of
|
|
# packages in PYTHONPATH if that also includes the target
|
|
# folder. So we need to use the regular virtualenv
|
|
# site-packages folder instead.
|
|
# Reference:
|
|
# https://github.com/pypa/pip/issues/6223
|
|
ctx_site_packages_dir = os.path.normpath(
|
|
os.path.abspath(ctx.get_site_packages_dir(arch))
|
|
)
|
|
venv_site_packages_dir = os.path.normpath(os.path.join(
|
|
ctx.build_dir, "venv", "lib", [
|
|
f for f in os.listdir(os.path.join(
|
|
ctx.build_dir, "venv", "lib"
|
|
)) if f.startswith("python")
|
|
][0], "site-packages"
|
|
))
|
|
copied_over_contents = []
|
|
for f in os.listdir(ctx_site_packages_dir):
|
|
full_path = os.path.join(ctx_site_packages_dir, f)
|
|
if not os.path.exists(os.path.join(
|
|
venv_site_packages_dir, f
|
|
)):
|
|
if os.path.isdir(full_path):
|
|
shutil.copytree(full_path, os.path.join(
|
|
venv_site_packages_dir, f
|
|
))
|
|
else:
|
|
shutil.copy2(full_path, os.path.join(
|
|
venv_site_packages_dir, f
|
|
))
|
|
copied_over_contents.append(f)
|
|
|
|
# Get listing of virtualenv's site-packages, to see the
|
|
# newly added things afterwards & copy them back into
|
|
# the distribution folder / build context site-packages:
|
|
previous_venv_contents = os.listdir(
|
|
venv_site_packages_dir
|
|
)
|
|
|
|
# Actually run setup.py:
|
|
info('Launching package install...')
|
|
shprint(sh.bash, '-c', (
|
|
"'" + join(
|
|
ctx.build_dir, "venv", "bin", "pip"
|
|
).replace("'", "'\"'\"'") + "' " +
|
|
"install -c ._tmp_p4a_recipe_constraints.txt -v ."
|
|
).format(ctx.get_site_packages_dir(arch).
|
|
replace("'", "'\"'\"'")),
|
|
_env=copy.copy(env))
|
|
|
|
# Go over all new additions and copy them back:
|
|
info('Copying additions resulting from setup.py back '
|
|
'into ctx.get_site_packages_dir()...')
|
|
new_venv_additions = []
|
|
for f in (set(os.listdir(venv_site_packages_dir)) -
|
|
set(previous_venv_contents)):
|
|
new_venv_additions.append(f)
|
|
full_path = os.path.join(venv_site_packages_dir, f)
|
|
if os.path.isdir(full_path):
|
|
shutil.copytree(full_path, os.path.join(
|
|
ctx_site_packages_dir, f
|
|
))
|
|
else:
|
|
shutil.copy2(full_path, os.path.join(
|
|
ctx_site_packages_dir, f
|
|
))
|
|
|
|
# Undo all the changes we did to the venv-site packages:
|
|
info('Reverting additions to '
|
|
'virtualenv\'s site-packages...')
|
|
for f in set(copied_over_contents + new_venv_additions):
|
|
full_path = os.path.join(venv_site_packages_dir, f)
|
|
if os.path.isdir(full_path):
|
|
shutil.rmtree(full_path)
|
|
else:
|
|
os.remove(full_path)
|
|
finally:
|
|
os.remove("._tmp_p4a_recipe_constraints.txt")
|
|
|
|
|
|
def run_pymodules_install(ctx, arch, modules, project_dir=None,
|
|
ignore_setup_py=False):
|
|
""" This function will take care of all non-recipe things, by:
|
|
|
|
1. Processing them from --requirements (the modules argument)
|
|
and installing them
|
|
|
|
2. Installing the user project/app itself via setup.py if
|
|
ignore_setup_py=True
|
|
|
|
"""
|
|
|
|
info('*** PYTHON PACKAGE / PROJECT INSTALL STAGE FOR ARCH: {} ***'.format(arch))
|
|
|
|
modules = [m for m in modules if ctx.not_has_package(m, arch)]
|
|
|
|
# We change current working directory later, so this has to be an absolute
|
|
# path or `None` in case that we didn't supply the `project_dir` via kwargs
|
|
project_dir = abspath(project_dir) if project_dir else None
|
|
|
|
# Bail out if no python deps and no setup.py to process:
|
|
if not modules and (
|
|
ignore_setup_py or
|
|
project_dir is None or
|
|
not project_has_setup_py(project_dir)
|
|
):
|
|
info('No Python modules and no setup.py to process, skipping')
|
|
return
|
|
|
|
# Output messages about what we're going to do:
|
|
if modules:
|
|
info(
|
|
"The requirements ({}) don\'t have recipes, attempting to "
|
|
"install them with pip".format(', '.join(modules))
|
|
)
|
|
info(
|
|
"If this fails, it may mean that the module has compiled "
|
|
"components and needs a recipe."
|
|
)
|
|
if project_dir is not None and \
|
|
project_has_setup_py(project_dir) and not ignore_setup_py:
|
|
info(
|
|
"Will process project install, if it fails then the "
|
|
"project may not be compatible for Android install."
|
|
)
|
|
|
|
# Use our hostpython to create the virtualenv
|
|
host_python = sh.Command(ctx.hostpython)
|
|
with current_directory(join(ctx.build_dir)):
|
|
shprint(host_python, '-m', 'venv', 'venv')
|
|
|
|
# Prepare base environment and upgrade pip:
|
|
base_env = dict(copy.copy(os.environ))
|
|
base_env["PYTHONPATH"] = ctx.get_site_packages_dir(arch)
|
|
info('Upgrade pip to latest version')
|
|
shprint(sh.bash, '-c', (
|
|
"source venv/bin/activate && pip install -U pip"
|
|
), _env=copy.copy(base_env))
|
|
|
|
# Install Cython in case modules need it to build:
|
|
info('Install Cython in case one of the modules needs it to build')
|
|
shprint(sh.bash, '-c', (
|
|
"venv/bin/pip install --upgrade Cython"
|
|
), _env=copy.copy(base_env))
|
|
|
|
# Get environment variables for build (with CC/compiler set):
|
|
standard_recipe = CythonRecipe()
|
|
standard_recipe.ctx = ctx
|
|
# (note: following line enables explicit -lpython... linker options)
|
|
standard_recipe.call_hostpython_via_targetpython = False
|
|
recipe_env = standard_recipe.get_recipe_env(ctx.archs[0])
|
|
env = copy.copy(base_env)
|
|
env.update(recipe_env)
|
|
|
|
# Make sure our build package dir is available, and the virtualenv
|
|
# site packages come FIRST (so the proper pip version is used):
|
|
env["PYTHONPATH"] += ":" + ctx.get_site_packages_dir(arch)
|
|
env["PYTHONPATH"] = os.path.abspath(join(
|
|
ctx.build_dir, "venv", "lib",
|
|
"python" + ctx.python_recipe.major_minor_version_string,
|
|
"site-packages")) + ":" + env["PYTHONPATH"]
|
|
|
|
# Install the manually specified requirements first:
|
|
if not modules:
|
|
info('There are no Python modules to install, skipping')
|
|
else:
|
|
info('Creating a requirements.txt file for the Python modules')
|
|
with open('requirements.txt', 'w') as fileh:
|
|
for module in modules:
|
|
key = 'VERSION_' + module
|
|
if key in environ:
|
|
line = '{}=={}\n'.format(module, environ[key])
|
|
else:
|
|
line = '{}\n'.format(module)
|
|
fileh.write(line)
|
|
|
|
info('Installing Python modules with pip')
|
|
info(
|
|
"IF THIS FAILS, THE MODULES MAY NEED A RECIPE. "
|
|
"A reason for this is often modules compiling "
|
|
"native code that is unaware of Android cross-compilation "
|
|
"and does not work without additional "
|
|
"changes / workarounds."
|
|
)
|
|
|
|
shprint(sh.bash, '-c', (
|
|
"venv/bin/pip " +
|
|
"install -v --target '{0}' --no-deps -r requirements.txt"
|
|
).format(ctx.get_site_packages_dir(arch).replace("'", "'\"'\"'")),
|
|
_env=copy.copy(env))
|
|
|
|
# Afterwards, run setup.py if present:
|
|
if project_dir is not None and (
|
|
project_has_setup_py(project_dir) and not ignore_setup_py
|
|
):
|
|
run_setuppy_install(ctx, project_dir, env, arch.arch)
|
|
elif not ignore_setup_py:
|
|
info("No setup.py found in project directory: " + str(project_dir))
|
|
|
|
# Strip object files after potential Cython or native code builds:
|
|
if not ctx.with_debug_symbols:
|
|
standard_recipe.strip_object_files(
|
|
arch, env, build_dir=ctx.build_dir
|
|
)
|
|
|
|
|
|
def biglink(ctx, arch):
|
|
# First, collate object files from each recipe
|
|
info('Collating object files from each recipe')
|
|
obj_dir = join(ctx.bootstrap.build_dir, 'collated_objects')
|
|
ensure_dir(obj_dir)
|
|
recipes = [Recipe.get_recipe(name, ctx) for name in ctx.recipe_build_order]
|
|
for recipe in recipes:
|
|
recipe_obj_dir = join(recipe.get_build_container_dir(arch.arch),
|
|
'objects_{}'.format(recipe.name))
|
|
if not exists(recipe_obj_dir):
|
|
info('{} recipe has no biglinkable files dir, skipping'
|
|
.format(recipe.name))
|
|
continue
|
|
files = glob.glob(join(recipe_obj_dir, '*'))
|
|
if not len(files):
|
|
info('{} recipe has no biglinkable files, skipping'
|
|
.format(recipe.name))
|
|
continue
|
|
info('{} recipe has object files, copying'.format(recipe.name))
|
|
files.append(obj_dir)
|
|
shprint(sh.cp, '-r', *files)
|
|
|
|
env = arch.get_env()
|
|
env['LDFLAGS'] = env['LDFLAGS'] + ' -L{}'.format(
|
|
join(ctx.bootstrap.build_dir, 'obj', 'local', arch.arch))
|
|
|
|
if not len(glob.glob(join(obj_dir, '*'))):
|
|
info('There seem to be no libraries to biglink, skipping.')
|
|
return
|
|
info('Biglinking')
|
|
info('target {}'.format(join(ctx.get_libs_dir(arch.arch),
|
|
'libpymodules.so')))
|
|
do_biglink = copylibs_function if ctx.copy_libs else biglink_function
|
|
|
|
# Move to the directory containing crtstart_so.o and crtend_so.o
|
|
# This is necessary with newer NDKs? A gcc bug?
|
|
with current_directory(arch.ndk_lib_dir):
|
|
do_biglink(
|
|
join(ctx.get_libs_dir(arch.arch), 'libpymodules.so'),
|
|
obj_dir.split(' '),
|
|
extra_link_dirs=[join(ctx.bootstrap.build_dir,
|
|
'obj', 'local', arch.arch),
|
|
os.path.abspath('.')],
|
|
env=env)
|
|
|
|
|
|
def biglink_function(soname, objs_paths, extra_link_dirs=None, env=None):
|
|
if extra_link_dirs is None:
|
|
extra_link_dirs = []
|
|
print('objs_paths are', objs_paths)
|
|
sofiles = []
|
|
|
|
for directory in objs_paths:
|
|
for fn in os.listdir(directory):
|
|
fn = os.path.join(directory, fn)
|
|
|
|
if not fn.endswith(".so.o"):
|
|
continue
|
|
if not os.path.exists(fn[:-2] + ".libs"):
|
|
continue
|
|
|
|
sofiles.append(fn[:-2])
|
|
|
|
# The raw argument list.
|
|
args = []
|
|
|
|
for fn in sofiles:
|
|
afn = fn + ".o"
|
|
libsfn = fn + ".libs"
|
|
|
|
args.append(afn)
|
|
with open(libsfn) as fd:
|
|
data = fd.read()
|
|
args.extend(data.split(" "))
|
|
|
|
unique_args = []
|
|
while args:
|
|
a = args.pop()
|
|
if a in ('-L', ):
|
|
continue
|
|
if a not in unique_args:
|
|
unique_args.insert(0, a)
|
|
|
|
for dir in extra_link_dirs:
|
|
link = '-L{}'.format(dir)
|
|
if link not in unique_args:
|
|
unique_args.append(link)
|
|
|
|
cc_name = env['CC']
|
|
cc = sh.Command(cc_name.split()[0])
|
|
cc = cc.bake(*cc_name.split()[1:])
|
|
|
|
shprint(cc, '-shared', '-O3', '-o', soname, *unique_args, _env=env)
|
|
|
|
|
|
def copylibs_function(soname, objs_paths, extra_link_dirs=None, env=None):
|
|
if extra_link_dirs is None:
|
|
extra_link_dirs = []
|
|
print('objs_paths are', objs_paths)
|
|
|
|
re_needso = re.compile(r'^.*\(NEEDED\)\s+Shared library: \[lib(.*)\.so\]\s*$')
|
|
blacklist_libs = (
|
|
'c',
|
|
'stdc++',
|
|
'dl',
|
|
'python2.7',
|
|
'sdl',
|
|
'sdl_image',
|
|
'sdl_ttf',
|
|
'z',
|
|
'm',
|
|
'GLESv2',
|
|
'jpeg',
|
|
'png',
|
|
'log',
|
|
|
|
# bootstrap takes care of sdl2 libs (if applicable)
|
|
'SDL2',
|
|
'SDL2_ttf',
|
|
'SDL2_image',
|
|
'SDL2_mixer',
|
|
)
|
|
found_libs = []
|
|
sofiles = []
|
|
if env and 'READELF' in env:
|
|
readelf = env['READELF']
|
|
elif 'READELF' in os.environ:
|
|
readelf = os.environ['READELF']
|
|
else:
|
|
readelf = shutil.which('readelf').strip()
|
|
readelf = sh.Command(readelf).bake('-d')
|
|
|
|
dest = dirname(soname)
|
|
|
|
for directory in objs_paths:
|
|
for fn in os.listdir(directory):
|
|
fn = join(directory, fn)
|
|
|
|
if not fn.endswith('.libs'):
|
|
continue
|
|
|
|
dirfn = fn[:-1] + 'dirs'
|
|
if not exists(dirfn):
|
|
continue
|
|
|
|
with open(fn) as f:
|
|
libs = f.read().strip().split(' ')
|
|
needed_libs = [lib for lib in libs
|
|
if lib and
|
|
lib not in blacklist_libs and
|
|
lib not in found_libs]
|
|
|
|
while needed_libs:
|
|
print('need libs:\n\t' + '\n\t'.join(needed_libs))
|
|
|
|
start_needed_libs = needed_libs[:]
|
|
found_sofiles = []
|
|
|
|
with open(dirfn) as f:
|
|
libdirs = f.read().split()
|
|
for libdir in libdirs:
|
|
if not needed_libs:
|
|
break
|
|
|
|
if libdir == dest:
|
|
# don't need to copy from dest to dest!
|
|
continue
|
|
|
|
libdir = libdir.strip()
|
|
print('scanning', libdir)
|
|
for lib in needed_libs[:]:
|
|
if lib in found_libs:
|
|
continue
|
|
|
|
if lib.endswith('.a'):
|
|
needed_libs.remove(lib)
|
|
found_libs.append(lib)
|
|
continue
|
|
|
|
lib_a = 'lib' + lib + '.a'
|
|
libpath_a = join(libdir, lib_a)
|
|
lib_so = 'lib' + lib + '.so'
|
|
libpath_so = join(libdir, lib_so)
|
|
plain_so = lib + '.so'
|
|
plainpath_so = join(libdir, plain_so)
|
|
|
|
sopath = None
|
|
if exists(libpath_so):
|
|
sopath = libpath_so
|
|
elif exists(plainpath_so):
|
|
sopath = plainpath_so
|
|
|
|
if sopath:
|
|
print('found', lib, 'in', libdir)
|
|
found_sofiles.append(sopath)
|
|
needed_libs.remove(lib)
|
|
found_libs.append(lib)
|
|
continue
|
|
|
|
if exists(libpath_a):
|
|
print('found', lib, '(static) in', libdir)
|
|
needed_libs.remove(lib)
|
|
found_libs.append(lib)
|
|
continue
|
|
|
|
for sofile in found_sofiles:
|
|
print('scanning dependencies for', sofile)
|
|
out = readelf(sofile)
|
|
for line in out.splitlines():
|
|
needso = re_needso.match(line)
|
|
if needso:
|
|
lib = needso.group(1)
|
|
if (lib not in needed_libs
|
|
and lib not in found_libs
|
|
and lib not in blacklist_libs):
|
|
needed_libs.append(needso.group(1))
|
|
|
|
sofiles += found_sofiles
|
|
|
|
if needed_libs == start_needed_libs:
|
|
raise RuntimeError(
|
|
'Failed to locate needed libraries!\n\t' +
|
|
'\n\t'.join(needed_libs))
|
|
|
|
print('Copying libraries')
|
|
shprint(sh.cp, *sofiles, dest)
|