From 5fce549f4ab002ca4201044c53da40161f92b556 Mon Sep 17 00:00:00 2001 From: Akinwale Ariwodola Date: Wed, 26 Sep 2018 01:47:42 +0100 Subject: [PATCH 01/20] add / update recipes and build changes for Python 3.6.6 compatibility --- .travis.yml | 7 +- Vagrantfile | 9 +- buildozer.spec.sample | 8 +- buildozer.spec.travis | 8 +- buildozer.spec.vagrant | 8 +- p4a/pythonforandroid/bootstrap.py | 6 +- .../bootstraps/lbry/build/jni/src/start.c | 4 +- p4a/pythonforandroid/build.py | 5 +- p4a/pythonforandroid/distribution.py | 6 +- p4a/pythonforandroid/logger.py | 7 +- p4a/pythonforandroid/patching.py | 11 +- p4a/pythonforandroid/recipe.py | 193 ++-- .../recipes/cryptography/__init__.py | 2 +- recipes/android/__init__.py | 81 ++ recipes/android/src/setup.py | 2 +- recipes/cffi/__init__.py | 76 +- recipes/cffi/disable-pkg-config.patch | 13 +- recipes/coincurve/__init__.py | 42 + recipes/coincurve/cross_compile.patch | 12 + recipes/coincurve/drop_setup_requires.patch | 12 + recipes/coincurve/find_lib.patch | 13 + recipes/coincurve/no-download.patch | 13 + recipes/coincurve/setup.py.patch | 22 + recipes/cryptography/__init__.py | 27 +- recipes/cryptography/force-urandom.patch | 11 + recipes/enum34/__init__.py | 12 + recipes/gmpy/__init__.py | 6 +- recipes/hostpython3crystax/__init__.py | 46 + recipes/idna/__init__.py | 14 + recipes/incremental/__init__.py | 2 +- recipes/ipaddress/__init__.py | 12 +- recipes/libffi/Application.mk | 3 + recipes/libffi/__init__.py | 83 ++ recipes/libffi/disable-mips-check.patch | 35 + recipes/libffi/remove-version-info.patch | 12 + recipes/libgmp/__init__.py | 15 +- recipes/libsecp256k1/__init__.py | 33 + recipes/miniupnpc/__init__.py | 4 +- recipes/netifaces/__init__.py | 18 +- recipes/netifaces/socket-ioctls.patch | 29 + recipes/openssl/__init__.py | 4 +- recipes/pyasn1/__init__.py | 5 +- recipes/pyjnius/__init__.py | 6 +- recipes/pyopenssl/__init__.py | 13 + recipes/python3crystax/__init__.py | 258 ++++++ recipes/python3crystax/patch_python3.6.patch | 89 ++ recipes/requests/__init__.py | 12 + recipes/setuptools/__init__.py | 19 +- recipes/six/__init__.py | 1 - recipes/twisted/__init__.py | 10 +- recipes/unqlite/__init__.py | 14 +- recipes/zope_interface/__init__.py | 21 + recipes/zope_interface/no_tests.patch | 13 + scripts/build-target-python.sh | 859 ++++++++++++++++++ src/main/python/lbrynetservice.py | 1 - 55 files changed, 2000 insertions(+), 247 deletions(-) create mode 100644 recipes/coincurve/__init__.py create mode 100644 recipes/coincurve/cross_compile.patch create mode 100644 recipes/coincurve/drop_setup_requires.patch create mode 100644 recipes/coincurve/find_lib.patch create mode 100644 recipes/coincurve/no-download.patch create mode 100644 recipes/coincurve/setup.py.patch create mode 100644 recipes/cryptography/force-urandom.patch create mode 100644 recipes/enum34/__init__.py create mode 100644 recipes/hostpython3crystax/__init__.py create mode 100644 recipes/idna/__init__.py create mode 100644 recipes/libffi/Application.mk create mode 100644 recipes/libffi/__init__.py create mode 100644 recipes/libffi/disable-mips-check.patch create mode 100644 recipes/libffi/remove-version-info.patch create mode 100644 recipes/libsecp256k1/__init__.py create mode 100644 recipes/netifaces/socket-ioctls.patch create mode 100644 recipes/pyopenssl/__init__.py create mode 100644 recipes/python3crystax/__init__.py create mode 100644 recipes/python3crystax/patch_python3.6.patch create mode 100644 recipes/requests/__init__.py create mode 100644 recipes/zope_interface/__init__.py create mode 100644 recipes/zope_interface/no_tests.patch create mode 100755 scripts/build-target-python.sh diff --git a/.travis.yml b/.travis.yml index 59a7e035..53def169 100644 --- a/.travis.yml +++ b/.travis.yml @@ -26,11 +26,14 @@ install: - cd .. - mv buildozer.spec.travis buildozer.spec - mkdir -p cd ~/.buildozer/android/platform/ -- wget -q 'https://dl.google.com/android/repository/android-ndk-r13b-linux-x86_64.zip' -P ~/.buildozer/android/platform/ +- wget -q 'https://us.crystax.net/download/crystax-ndk-10.3.2-linux-x86_64.tar.xz' -P ~/.buildozer/android/ - wget -q 'https://dl.google.com/android/android-sdk_r23-linux.tgz' -P ~/.buildozer/android/platform/ - wget -q 'https://dl.google.com/android/repository/platform-27_r01.zip' -P ~/.buildozer/android/platform/ - wget -q 'https://dl.google.com/android/repository/build-tools_r26.0.1-linux.zip' -P ~/.buildozer/android/platform/ -- unzip -qq ~/.buildozer/android/platform/android-ndk-r13b-linux-x86_64.zip -d ~/.buildozer/android/platform/ +- tar -xf ~/.buildozer/android/crystax-ndk-10.3.2-linux-x86_64.tar.xz -C ~/.buildozer/android/ +- cp -f $TRAVIS_BUILD_DIR/scripts/build-target-python.sh ~/.buildozer/android/crystax-ndk-10.3.2/build/tools/build-target-python.sh +- rm -rf ~/.buildozer/android/crystax-ndk-10.3.2/platforms/android-9 +- ln -s ~/.buildozer/android/crystax-ndk-10.3.2/platforms/android-21 ~/.buildozer/android/crystax-ndk-10.3.2/platforms/android-9 - tar -xf ~/.buildozer/android/platform/android-sdk_r23-linux.tgz -C ~/.buildozer/android/platform/ - mv ~/.buildozer/android/platform/android-sdk-linux ~/.buildozer/android/platform/android-sdk-23 - unzip -qq ~/.buildozer/android/platform/platform-27_r01.zip -d ~/.buildozer/android/platform/android-sdk-23/platforms diff --git a/Vagrantfile b/Vagrantfile index e3d5fc5c..254851fb 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -63,12 +63,15 @@ Vagrant.configure("2") do |config| cp $HOME/lbry-android/buildozer.spec.vagrant $HOME/lbry-android/buildozer.spec mkdir -p cd $HOME/.buildozer/android/platform/ - wget -q 'https://dl.google.com/android/repository/android-ndk-r13b-linux-x86_64.zip' -P $HOME/.buildozer/android/platform/ + wget -q 'https://us.crystax.net/download/crystax-ndk-10.3.2-linux-x86_64.tar.xz' -P $HOME/.buildozer/android/ wget -q 'https://dl.google.com/android/android-sdk_r23-linux.tgz' -P $HOME/.buildozer/android/platform/ wget -q 'https://dl.google.com/android/repository/platform-27_r01.zip' -P $HOME/.buildozer/android/platform/ wget -q 'https://dl.google.com/android/repository/build-tools_r26.0.1-linux.zip' -P $HOME/.buildozer/android/platform/ - unzip -qq $HOME/.buildozer/android/platform/android-ndk-r13b-linux-x86_64.zip -d $HOME/.buildozer/android/platform/ - rm $HOME/.buildozer/android/platform/android-ndk-r13b-linux-x86_64.zip + tar -xf ~/.buildozer/android/crystax-ndk-10.3.2-linux-x86_64.tar.xz -C $HOME/.buildozer/android/ + rm $HOME/.buildozer/android/crystax-ndk-10.3.2-linux-x86_64.tar.xz + ln -s $HOME/.buildozer/android/crystax-ndk-10.3.2/platforms/android-21 $HOME/.buildozer/android/crystax-ndk-10.3.2/platforms/android-9 + cp -f $HOME/lbry-android/scripts/build-target-python.sh $HOME/.buildozer/android/crystax-ndk-10.3.2/build/tools/build-target-python.sh + rm -rf $HOME/.buildozer/android/crystax-ndk-10.3.2/platforms/android-9 tar -xf $HOME/.buildozer/android/platform/android-sdk_r23-linux.tgz -C $HOME/.buildozer/android/platform/ rm $HOME/.buildozer/android/platform/android-sdk_r23-linux.tgz mv $HOME/.buildozer/android/platform/android-sdk-linux $HOME/.buildozer/android/platform/android-sdk-23 diff --git a/buildozer.spec.sample b/buildozer.spec.sample index 7305b759..c44ad97a 100644 --- a/buildozer.spec.sample +++ b/buildozer.spec.sample @@ -36,7 +36,7 @@ version.filename = %(source.dir)s/main.py # (list) Application requirements # comma seperated e.g. requirements = sqlite3,kivy -requirements = openssl, sqlite3, hostpython2, android, distro, pyjnius, certifi==2018.4.16, constantly, incremental, functools32, miniupnpc==1.9, gmpy==1.17, twisted==16.6.0, appdirs==1.4.3, argparse==1.2.1, docopt==0.6.2, base58==0.2.2, colorama==0.3.7, dnspython==1.12.0, ecdsa==0.13, envparse==0.2.0, jsonrpc==1.2, jsonrpclib==0.1.7, jsonschema==2.5.1, pbkdf2==1.3, pyyaml==3.12, qrcode==5.2.2, requests==2.9.1, seccure==0.3.1.3, attrs==18.1.0, pyasn1, pyasn1-modules, service_identity==16.0.0, six==1.9.0, slowaes==0.1a1, txJSON-RPC==0.5, wsgiref==0.1.2, zope.interface==4.3.3, protobuf==3.2.0, keyring==10.4.0, netifaces, txupnp==0.0.1a10, git+https://github.com/lbryio/lbryschema.git@v0.0.16#egg=lbryschema, git+https://github.com/lbryio/lbryum.git#egg=lbryum, git+https://github.com/lbryio/lbry.git@v0.21.2#egg=lbrynet, asn1crypto, cryptography==2.2.2, pyopenssl==17.4.0, treq==17.8.0, funcsigs, mock, pbr, unqlite +requirements = python3crystax, openssl, sqlite3, hostpython3crystax, android, distro, pyjnius, certifi==2018.4.16, constantly, incremental, miniupnpc==1.9, gmpy, appdirs==1.4.3, argparse==1.2.1, docopt, base58==1.0.0, colorama==0.3.7, dnspython==1.12.0, ecdsa==0.13, envparse, jsonrpclib==0.1.7, jsonschema==2.5.1, pbkdf2, pyyaml, qrcode==5.2.2, requests, seccure==0.3.1.3, attrs==18.1.0, pyasn1, pyasn1-modules, service_identity==16.0.0, six==1.9.0, txJSON-RPC, zope.interface==4.3.3, protobuf==3.2.0, keyring==10.4.0, txupnp, git+https://github.com/lbryio/lbryschema.git#egg=lbryschema, git+https://github.com/lbryio/lbryum.git#egg=lbryum, git+https://github.com/lbryio/lbry.git#egg=lbrynet, asn1crypto, treq==17.8.0, funcsigs, mock, pbr, unqlite, pyopenssl, twisted, idna, Automat, hyperlink, PyHamcrest, netifaces, cryptography, aiohttp, git+https://github.com/lbryio/torba#egg=torba, coincurve # (str) Custom source folders for requirements # Sets custom source for any requirements with recipes @@ -98,16 +98,16 @@ android.minapi = 21 android.sdk = 23 # (str) Android NDK version to use -android.ndk = 13b +#android.ndk = 13b # (bool) Use --private data storage (True) or --dir public storage (False) #android.private_storage = True # (str) Android NDK directory (if empty, it will be automatically downloaded.) -#android.ndk_path = ~/Dev/SDKs/android-ndk-r13b +android.ndk_path = ~/.buildozer/android/crystax-ndk-10.3.2 # (str) Android SDK directory (if empty, it will be automatically downloaded.) -#android.sdk_path = ~/Dev/SDKs/android +#android.sdk_path = ~/.buildozer/android # (str) ANT directory (if empty, it will be automatically downloaded.) #android.ant_path = diff --git a/buildozer.spec.travis b/buildozer.spec.travis index 7305b759..c44ad97a 100644 --- a/buildozer.spec.travis +++ b/buildozer.spec.travis @@ -36,7 +36,7 @@ version.filename = %(source.dir)s/main.py # (list) Application requirements # comma seperated e.g. requirements = sqlite3,kivy -requirements = openssl, sqlite3, hostpython2, android, distro, pyjnius, certifi==2018.4.16, constantly, incremental, functools32, miniupnpc==1.9, gmpy==1.17, twisted==16.6.0, appdirs==1.4.3, argparse==1.2.1, docopt==0.6.2, base58==0.2.2, colorama==0.3.7, dnspython==1.12.0, ecdsa==0.13, envparse==0.2.0, jsonrpc==1.2, jsonrpclib==0.1.7, jsonschema==2.5.1, pbkdf2==1.3, pyyaml==3.12, qrcode==5.2.2, requests==2.9.1, seccure==0.3.1.3, attrs==18.1.0, pyasn1, pyasn1-modules, service_identity==16.0.0, six==1.9.0, slowaes==0.1a1, txJSON-RPC==0.5, wsgiref==0.1.2, zope.interface==4.3.3, protobuf==3.2.0, keyring==10.4.0, netifaces, txupnp==0.0.1a10, git+https://github.com/lbryio/lbryschema.git@v0.0.16#egg=lbryschema, git+https://github.com/lbryio/lbryum.git#egg=lbryum, git+https://github.com/lbryio/lbry.git@v0.21.2#egg=lbrynet, asn1crypto, cryptography==2.2.2, pyopenssl==17.4.0, treq==17.8.0, funcsigs, mock, pbr, unqlite +requirements = python3crystax, openssl, sqlite3, hostpython3crystax, android, distro, pyjnius, certifi==2018.4.16, constantly, incremental, miniupnpc==1.9, gmpy, appdirs==1.4.3, argparse==1.2.1, docopt, base58==1.0.0, colorama==0.3.7, dnspython==1.12.0, ecdsa==0.13, envparse, jsonrpclib==0.1.7, jsonschema==2.5.1, pbkdf2, pyyaml, qrcode==5.2.2, requests, seccure==0.3.1.3, attrs==18.1.0, pyasn1, pyasn1-modules, service_identity==16.0.0, six==1.9.0, txJSON-RPC, zope.interface==4.3.3, protobuf==3.2.0, keyring==10.4.0, txupnp, git+https://github.com/lbryio/lbryschema.git#egg=lbryschema, git+https://github.com/lbryio/lbryum.git#egg=lbryum, git+https://github.com/lbryio/lbry.git#egg=lbrynet, asn1crypto, treq==17.8.0, funcsigs, mock, pbr, unqlite, pyopenssl, twisted, idna, Automat, hyperlink, PyHamcrest, netifaces, cryptography, aiohttp, git+https://github.com/lbryio/torba#egg=torba, coincurve # (str) Custom source folders for requirements # Sets custom source for any requirements with recipes @@ -98,16 +98,16 @@ android.minapi = 21 android.sdk = 23 # (str) Android NDK version to use -android.ndk = 13b +#android.ndk = 13b # (bool) Use --private data storage (True) or --dir public storage (False) #android.private_storage = True # (str) Android NDK directory (if empty, it will be automatically downloaded.) -#android.ndk_path = ~/Dev/SDKs/android-ndk-r13b +android.ndk_path = ~/.buildozer/android/crystax-ndk-10.3.2 # (str) Android SDK directory (if empty, it will be automatically downloaded.) -#android.sdk_path = ~/Dev/SDKs/android +#android.sdk_path = ~/.buildozer/android # (str) ANT directory (if empty, it will be automatically downloaded.) #android.ant_path = diff --git a/buildozer.spec.vagrant b/buildozer.spec.vagrant index 7305b759..c44ad97a 100644 --- a/buildozer.spec.vagrant +++ b/buildozer.spec.vagrant @@ -36,7 +36,7 @@ version.filename = %(source.dir)s/main.py # (list) Application requirements # comma seperated e.g. requirements = sqlite3,kivy -requirements = openssl, sqlite3, hostpython2, android, distro, pyjnius, certifi==2018.4.16, constantly, incremental, functools32, miniupnpc==1.9, gmpy==1.17, twisted==16.6.0, appdirs==1.4.3, argparse==1.2.1, docopt==0.6.2, base58==0.2.2, colorama==0.3.7, dnspython==1.12.0, ecdsa==0.13, envparse==0.2.0, jsonrpc==1.2, jsonrpclib==0.1.7, jsonschema==2.5.1, pbkdf2==1.3, pyyaml==3.12, qrcode==5.2.2, requests==2.9.1, seccure==0.3.1.3, attrs==18.1.0, pyasn1, pyasn1-modules, service_identity==16.0.0, six==1.9.0, slowaes==0.1a1, txJSON-RPC==0.5, wsgiref==0.1.2, zope.interface==4.3.3, protobuf==3.2.0, keyring==10.4.0, netifaces, txupnp==0.0.1a10, git+https://github.com/lbryio/lbryschema.git@v0.0.16#egg=lbryschema, git+https://github.com/lbryio/lbryum.git#egg=lbryum, git+https://github.com/lbryio/lbry.git@v0.21.2#egg=lbrynet, asn1crypto, cryptography==2.2.2, pyopenssl==17.4.0, treq==17.8.0, funcsigs, mock, pbr, unqlite +requirements = python3crystax, openssl, sqlite3, hostpython3crystax, android, distro, pyjnius, certifi==2018.4.16, constantly, incremental, miniupnpc==1.9, gmpy, appdirs==1.4.3, argparse==1.2.1, docopt, base58==1.0.0, colorama==0.3.7, dnspython==1.12.0, ecdsa==0.13, envparse, jsonrpclib==0.1.7, jsonschema==2.5.1, pbkdf2, pyyaml, qrcode==5.2.2, requests, seccure==0.3.1.3, attrs==18.1.0, pyasn1, pyasn1-modules, service_identity==16.0.0, six==1.9.0, txJSON-RPC, zope.interface==4.3.3, protobuf==3.2.0, keyring==10.4.0, txupnp, git+https://github.com/lbryio/lbryschema.git#egg=lbryschema, git+https://github.com/lbryio/lbryum.git#egg=lbryum, git+https://github.com/lbryio/lbry.git#egg=lbrynet, asn1crypto, treq==17.8.0, funcsigs, mock, pbr, unqlite, pyopenssl, twisted, idna, Automat, hyperlink, PyHamcrest, netifaces, cryptography, aiohttp, git+https://github.com/lbryio/torba#egg=torba, coincurve # (str) Custom source folders for requirements # Sets custom source for any requirements with recipes @@ -98,16 +98,16 @@ android.minapi = 21 android.sdk = 23 # (str) Android NDK version to use -android.ndk = 13b +#android.ndk = 13b # (bool) Use --private data storage (True) or --dir public storage (False) #android.private_storage = True # (str) Android NDK directory (if empty, it will be automatically downloaded.) -#android.ndk_path = ~/Dev/SDKs/android-ndk-r13b +android.ndk_path = ~/.buildozer/android/crystax-ndk-10.3.2 # (str) Android SDK directory (if empty, it will be automatically downloaded.) -#android.sdk_path = ~/Dev/SDKs/android +#android.sdk_path = ~/.buildozer/android # (str) ANT directory (if empty, it will be automatically downloaded.) #android.ant_path = diff --git a/p4a/pythonforandroid/bootstrap.py b/p4a/pythonforandroid/bootstrap.py index f7aa6c9b..ef89fef8 100644 --- a/p4a/pythonforandroid/bootstrap.py +++ b/p4a/pythonforandroid/bootstrap.py @@ -1,12 +1,12 @@ -from os.path import (join, dirname, isdir, splitext, basename, realpath) -from os import listdir, mkdir +from os.path import (join, dirname, isdir, splitext, basename) +from os import listdir import sh import glob import json import importlib from pythonforandroid.logger import (warning, shprint, info, logger, - debug, error) + debug) from pythonforandroid.util import (current_directory, ensure_dir, temp_directory, which) from pythonforandroid.recipe import Recipe diff --git a/p4a/pythonforandroid/bootstraps/lbry/build/jni/src/start.c b/p4a/pythonforandroid/bootstraps/lbry/build/jni/src/start.c index e709ca5f..80cd5ace 100644 --- a/p4a/pythonforandroid/bootstraps/lbry/build/jni/src/start.c +++ b/p4a/pythonforandroid/bootstraps/lbry/build/jni/src/start.c @@ -308,9 +308,9 @@ JNIEXPORT void JNICALL Java_org_kivy_android_PythonService_nativeStart( setenv("PYTHONHOME", python_home, 1); setenv("PYTHONPATH", python_path, 1); setenv("PYTHON_SERVICE_ARGUMENT", arg, 1); - + char ca_path[128]; - snprintf(ca_path, 128, "%s/lib/python2.7/site-packages/certifi/cacert.pem", python_home); + snprintf(ca_path, 128, "%s/crystax_python/site-packages/certifi/cacert.pem", python_home); setenv("SSL_CERT_FILE", ca_path, 1); char *argv[] = {"."}; diff --git a/p4a/pythonforandroid/build.py b/p4a/pythonforandroid/build.py index 461839c1..38766883 100644 --- a/p4a/pythonforandroid/build.py +++ b/p4a/pythonforandroid/build.py @@ -396,6 +396,7 @@ class Context(object): 'platforms', 'android-{}'.format(self.android_min_api), platform_dir) + if not exists(self.ndk_platform): warning('ndk_platform doesn\'t exist: {}'.format( self.ndk_platform)) @@ -529,7 +530,7 @@ class Context(object): if self.python_recipe.from_crystax: return self.get_python_install_dir() return join(self.get_python_install_dir(), - 'lib', 'python2.7', 'site-packages') + 'lib', 'python3.6', 'site-packages') def get_libs_dir(self, arch): '''The libs dir for a given arch.''' @@ -634,7 +635,7 @@ def run_pymodules_install(ctx, modules): venv = sh.Command(ctx.virtualenv) with current_directory(join(ctx.build_dir)): - shprint(venv, '--python=python2.7', 'venv') + shprint(venv, '--python=python3.6', 'venv') info('Creating a requirements.txt file for the Python modules') with open('requirements.txt', 'w') as fileh: diff --git a/p4a/pythonforandroid/distribution.py b/p4a/pythonforandroid/distribution.py index 0db59a28..735b64a7 100644 --- a/p4a/pythonforandroid/distribution.py +++ b/p4a/pythonforandroid/distribution.py @@ -41,7 +41,7 @@ class Distribution(object): return str(self) @classmethod - def get_distribution(cls, ctx, name=None, recipes=[], + def get_distribution(cls, ctx, name=None, recipes=[], force_build=False, extra_dist_dirs=[], require_perfect_match=False): @@ -70,10 +70,6 @@ class Distribution(object): correct set of recipes. ''' - # AND: This whole function is a bit hacky, it needs checking - # properly to make sure it follows logically correct - # possibilities - existing_dists = Distribution.get_distributions(ctx) needs_build = True # whether the dist needs building, will be returned diff --git a/p4a/pythonforandroid/logger.py b/p4a/pythonforandroid/logger.py index 93c086f8..38afba0e 100644 --- a/p4a/pythonforandroid/logger.py +++ b/p4a/pythonforandroid/logger.py @@ -1,4 +1,3 @@ - import logging import os import re @@ -16,7 +15,7 @@ if not six.PY3: stderr = codecs.getwriter('utf8')(stderr) if six.PY2: - unistr = unicode + unistr = unicode # noqa F821 else: unistr = str @@ -43,6 +42,7 @@ class LevelDifferentiatingFormatter(logging.Formatter): Err_Style.RESET_ALL) + record.msg return super(LevelDifferentiatingFormatter, self).format(record) + logger = logging.getLogger('p4a') if not hasattr(logger, 'touched'): # Necessary as importlib reloads # this, which would add a second @@ -72,6 +72,7 @@ class colorama_shim(object): def enable(self, enable): self._enabled = enable + Out_Style = colorama_shim(Colo_Style) Out_Fore = colorama_shim(Colo_Fore) Err_Style = colorama_shim(Colo_Style) @@ -173,6 +174,8 @@ def shprint(command, *args, **kwargs): msg_width = columns - len(msg_hdr) - 1 output = command(*args, **kwargs) for line in output: + if isinstance(line, bytes): + line = line.decode('utf-8', errors='replace') if logger.level > logging.DEBUG: msg = line.replace( '\n', ' ').replace( diff --git a/p4a/pythonforandroid/patching.py b/p4a/pythonforandroid/patching.py index 541bab85..2a477332 100644 --- a/p4a/pythonforandroid/patching.py +++ b/p4a/pythonforandroid/patching.py @@ -18,6 +18,7 @@ def is_platform(platform): return uname()[0] == platform return is_x + is_linux = is_platform('Linux') is_darwin = is_platform('Darwin') @@ -30,31 +31,31 @@ def is_arch(xarch): def is_api_gt(apiver): def is_x(recipe, **kwargs): - return recipe.ctx.android_min_api > apiver + return recipe.ctx.android_api > apiver return is_x def is_api_gte(apiver): def is_x(recipe, **kwargs): - return recipe.ctx.android_min_api >= apiver + return recipe.ctx.android_api >= apiver return is_x def is_api_lt(apiver): def is_x(recipe, **kwargs): - return recipe.ctx.android_min_api < apiver + return recipe.ctx.android_api < apiver return is_x def is_api_lte(apiver): def is_x(recipe, **kwargs): - return recipe.ctx.android_min_api <= apiver + return recipe.ctx.android_api <= apiver return is_x def is_api(apiver): def is_x(recipe, **kwargs): - return recipe.ctx.android_min_api == apiver + return recipe.ctx.android_api == apiver return is_x diff --git a/p4a/pythonforandroid/recipe.py b/p4a/pythonforandroid/recipe.py index d80d03bc..20bab446 100644 --- a/p4a/pythonforandroid/recipe.py +++ b/p4a/pythonforandroid/recipe.py @@ -1,11 +1,11 @@ -from os.path import join, dirname, isdir, exists, isfile, split, realpath, basename +from os.path import basename, dirname, exists, isdir, isfile, join, realpath import importlib -import zipfile import glob from shutil import rmtree from six import PY2, with_metaclass import hashlib +from re import match import sh import shutil @@ -20,7 +20,6 @@ from pythonforandroid.logger import (logger, info, warning, error, debug, shprin from pythonforandroid.util import (urlretrieve, current_directory, ensure_dir) # this import is necessary to keep imp.load_source from complaining :) -import pythonforandroid.recipes if PY2: @@ -149,7 +148,7 @@ class Recipe(with_metaclass(RecipeMeta)): urlretrieve(url, target, report_hook) return target - elif parsed_url.scheme in ('git', 'git+ssh', 'git+http', 'git+https'): + elif parsed_url.scheme in ('git', 'git+file', 'git+ssh', 'git+http', 'git+https'): if isdir(target): with current_directory(target): shprint(sh.git, 'fetch', '--tags') @@ -168,43 +167,6 @@ class Recipe(with_metaclass(RecipeMeta)): shprint(sh.git, 'submodule', 'update', '--recursive') return target - def extract_source(self, source, cwd): - """ - (internal) Extract the `source` into the directory `cwd`. - """ - if not source: - return - if isfile(source): - info("Extract {} into {}".format(source, cwd)) - - if source.endswith(".tgz") or source.endswith(".tar.gz"): - shprint(sh.tar, "-C", cwd, "-xvzf", source) - - elif source.endswith(".tbz2") or source.endswith(".tar.bz2"): - shprint(sh.tar, "-C", cwd, "-xvjf", source) - - elif source.endswith(".zip"): - zf = zipfile.ZipFile(source) - zf.extractall(path=cwd) - zf.close() - - else: - warning( - "Error: cannot extract, unrecognized extension for {}" - .format(source)) - raise Exception() - - elif isdir(source): - info("Copying {} into {}".format(source, cwd)) - - shprint(sh.cp, '-a', source, cwd) - - else: - warning( - "Error: cannot extract or copy, unrecognized path {}" - .format(source)) - raise Exception() - # def get_archive_rootdir(self, filename): # if filename.endswith(".tgz") or filename.endswith(".tar.gz") or \ # filename.endswith(".tbz2") or filename.endswith(".tar.bz2"): @@ -225,19 +187,19 @@ class Recipe(with_metaclass(RecipeMeta)): build directory. """ info("Applying patch {}".format(filename)) - filename = join(self.recipe_dir, filename) + filename = join(self.get_recipe_dir(), filename) shprint(sh.patch, "-t", "-d", self.get_build_dir(arch), "-p1", "-i", filename, _tail=10) def copy_file(self, filename, dest): info("Copy {} to {}".format(filename, dest)) - filename = join(self.recipe_dir, filename) + filename = join(self.get_recipe_dir(), filename) dest = join(self.build_dir, dest) shutil.copy(filename, dest) def append_file(self, filename, dest): info("Append {} to {}".format(filename, dest)) - filename = join(self.recipe_dir, filename) + filename = join(self.get_recipe_dir(), filename) dest = join(self.build_dir, dest) with open(filename, "rb") as fd: data = fd.read() @@ -329,7 +291,14 @@ class Recipe(with_metaclass(RecipeMeta)): return join(self.get_build_container_dir(arch), self.name) def get_recipe_dir(self): - # AND: Redundant, an equivalent property is already set by get_recipe + """ + Returns the local recipe directory or defaults to the core recipe + directory. + """ + if self.ctx.local_recipes is not None: + local_recipe_dir = join(self.ctx.local_recipes, self.name) + if exists(local_recipe_dir): + return local_recipe_dir return join(self.ctx.root_dir, 'recipes', self.name) # Public Recipe API to be subclassed if needed @@ -349,6 +318,16 @@ class Recipe(with_metaclass(RecipeMeta)): return url = self.versioned_url + ma = match(u'^(.+)#md5=([0-9a-f]{32})$', url) + if ma: # fragmented URL? + if self.md5sum: + raise ValueError( + ('Received md5sum from both the {} recipe ' + 'and its url').format(self.name)) + url = ma.group(1) + expected_md5 = ma.group(2) + else: + expected_md5 = self.md5sum shprint(sh.mkdir, '-p', join(self.ctx.packages_path, self.name)) @@ -356,44 +335,41 @@ class Recipe(with_metaclass(RecipeMeta)): filename = shprint(sh.basename, url).stdout[:-1].decode('utf-8') do_download = True - marker_filename = '.mark-{}'.format(filename) if exists(filename) and isfile(filename): if not exists(marker_filename): shprint(sh.rm, filename) - elif self.md5sum: + elif expected_md5: current_md5 = md5sum(filename) - if current_md5 == self.md5sum: - debug('Checked md5sum: downloaded expected content!') - do_download = False - else: - info('Downloaded unexpected content...') + if current_md5 != expected_md5: debug('* Generated md5sum: {}'.format(current_md5)) - debug('* Expected md5sum: {}'.format(self.md5sum)) - + debug('* Expected md5sum: {}'.format(expected_md5)) + raise ValueError( + ('Generated md5sum does not match expected md5sum ' + 'for {} recipe').format(self.name)) + do_download = False else: do_download = False - info('{} download already cached, skipping' - .format(self.name)) # If we got this far, we will download if do_download: debug('Downloading {} from {}'.format(self.name, url)) shprint(sh.rm, '-f', marker_filename) - self.download_file(url, filename) + self.download_file(self.versioned_url, filename) shprint(sh.touch, marker_filename) - if exists(filename) and isfile(filename) and self.md5sum: + if exists(filename) and isfile(filename) and expected_md5: current_md5 = md5sum(filename) - if self.md5sum is not None: - if current_md5 == self.md5sum: - debug('Checked md5sum: downloaded expected content!') - else: - info('Downloaded unexpected content...') + if expected_md5 is not None: + if current_md5 != expected_md5: debug('* Generated md5sum: {}'.format(current_md5)) - debug('* Expected md5sum: {}'.format(self.md5sum)) - exit(1) + debug('* Expected md5sum: {}'.format(expected_md5)) + raise ValueError( + ('Generated md5sum does not match expected md5sum ' + 'for {} recipe').format(self.name)) + else: + info('{} download already cached, skipping'.format(self.name)) def unpack(self, arch): info_main('Unpacking {} for {}'.format(self.name, arch)) @@ -404,8 +380,6 @@ class Recipe(with_metaclass(RecipeMeta)): if user_dir is not None: info('P4A_{}_DIR exists, symlinking instead'.format( self.name.lower())) - # AND: Currently there's something wrong if I use ln, fix this - warning('Using cp -a instead of symlink...fix this!') if exists(self.get_build_dir(arch)): return shprint(sh.rm, '-rf', build_dir) @@ -421,11 +395,13 @@ class Recipe(with_metaclass(RecipeMeta)): filename = shprint( sh.basename, self.versioned_url).stdout[:-1].decode('utf-8') + ma = match(u'^(.+)#md5=([0-9a-f]{32})$', filename) + if ma: # fragmented URL? + filename = ma.group(1) with current_directory(build_dir): directory_name = self.get_build_dir(arch) - # AND: Could use tito's get_archive_rootdir here if not exists(directory_name) or not isdir(directory_name): extraction_filename = join( self.ctx.packages_path, self.name, filename) @@ -638,7 +614,6 @@ class Recipe(with_metaclass(RecipeMeta)): if len(logger.handlers) > 1: logger.removeHandler(logger.handlers[1]) recipe = mod.recipe - recipe.recipe_dir = dirname(recipe_file) recipe.ctx = ctx cls.recipes[name] = recipe return recipe @@ -755,6 +730,10 @@ class PythonRecipe(Recipe): return join( Recipe.get_recipe('hostpython2', self.ctx).get_build_dir(), 'hostpython') + elif 'hostpython3crystax' in self.ctx.recipe_build_order: + return join( + Recipe.get_recipe('hostpython3crystax', self.ctx).get_build_dir(), + 'hostpython') else: python_recipe = self.ctx.python_recipe return 'python{}'.format(python_recipe.version) @@ -779,6 +758,34 @@ class PythonRecipe(Recipe): env['PYTHONNOUSERSITE'] = '1' if not self.call_hostpython_via_targetpython: + # sets python headers/linkages...depending on python's recipe + python_version = self.ctx.python_recipe.version + python_short_version = '.'.join(python_version.split('.')[:2]) + if 'python2' in self.ctx.recipe_build_order: + env['PYTHON_ROOT'] = self.ctx.get_python_install_dir() + env['CFLAGS'] += ' -I' + env[ + 'PYTHON_ROOT'] + '/include/python2.7' + env['LDFLAGS'] += ' -L' + env['PYTHON_ROOT'] + '/lib' + \ + ' -lpython2.7' + elif self.ctx.python_recipe.from_crystax: + ndk_dir_python = join(self.ctx.ndk_dir, 'sources', + 'python', python_version) + env['CFLAGS'] += ' -I{} '.format( + join(ndk_dir_python, 'include', + 'python')) + env['LDFLAGS'] += ' -L{}'.format( + join(ndk_dir_python, 'libs', arch.arch)) + env['LDFLAGS'] += ' -lpython{}m'.format(python_short_version) + elif 'python3' in self.ctx.recipe_build_order: + # This headers are unused cause python3 recipe was removed + # TODO: should be reviewed when python3 recipe added + env['PYTHON_ROOT'] = self.ctx.get_python_install_dir() + env['CFLAGS'] += ' -I' + env[ + 'PYTHON_ROOT'] + '/include/python{}m'.format( + python_short_version) + env['LDFLAGS'] += ' -L' + env['PYTHON_ROOT'] + '/lib' + \ + ' -lpython{}m'.format( + python_short_version) hppath = [] hppath.append(join(dirname(self.hostpython_location), 'Lib')) hppath.append(join(hppath[0], 'site-packages')) @@ -819,28 +826,12 @@ class PythonRecipe(Recipe): with current_directory(self.get_build_dir(arch.arch)): hostpython = sh.Command(self.hostpython_location) - if self.ctx.python_recipe.from_crystax: - # hppath = join(dirname(self.hostpython_location), 'Lib', - # 'site-packages') hpenv = env.copy() - # if 'PYTHONPATH' in hpenv: - # hpenv['PYTHONPATH'] = ':'.join([hppath] + - # hpenv['PYTHONPATH'].split(':')) - # else: - # hpenv['PYTHONPATH'] = hppath - # hpenv['PYTHONHOME'] = self.ctx.get_python_install_dir() - # shprint(hostpython, 'setup.py', 'build', - # _env=hpenv, *self.setup_extra_args) shprint(hostpython, 'setup.py', 'install', '-O2', '--root={}'.format(self.ctx.get_python_install_dir()), '--install-lib=.', - # AND: will need to unhardcode the 3.5 when adding 2.7 (and other crystax supported versions) _env=hpenv, *self.setup_extra_args) - # site_packages_dir = self.ctx.get_site_packages_dir() - # built_files = glob.glob(join('build', 'lib*', '*')) - # for filen in built_files: - # shprint(sh.cp, '-r', filen, join(site_packages_dir, split(filen)[-1])) elif self.call_hostpython_via_targetpython: shprint(hostpython, 'setup.py', 'install', '-O2', _env=env, *self.setup_extra_args) @@ -857,7 +848,6 @@ class PythonRecipe(Recipe): '--root={}'.format(self.ctx.get_python_install_dir()), '--install-lib=lib/python2.7/site-packages', _env=hpenv, *self.setup_extra_args) - # AND: Hardcoded python2.7 needs fixing # If asked, also install in the hostpython build dir if self.install_in_hostpython: @@ -917,6 +907,7 @@ class CompiledComponentsPythonRecipe(PythonRecipe): shprint(hostpython, 'setup.py', self.build_cmd, '-v', _env=env, *self.setup_extra_args) + class CppCompiledComponentsPythonRecipe(CompiledComponentsPythonRecipe): """ Extensions that require the cxx-stl """ call_hostpython_via_targetpython = False @@ -926,39 +917,34 @@ class CppCompiledComponentsPythonRecipe(CompiledComponentsPythonRecipe): keys = dict( ctx=self.ctx, arch=arch, - arch_noeabi=arch.arch.replace('eabi', ''), - pyroot=self.ctx.get_python_install_dir() + arch_noeabi=arch.arch.replace('eabi', '') ) env['LDSHARED'] = env['CC'] + ' -pthread -shared -Wl,-O1 -Wl,-Bsymbolic-functions' - env['CFLAGS'] += " -I{pyroot}/include/python2.7 " \ - " -I{ctx.ndk_dir}/platforms/android-{ctx.android_min_api}/arch-{arch_noeabi}/usr/include" \ + env['CFLAGS'] += " -I{ctx.ndk_dir}/platforms/android-{ctx.android_api}/arch-{arch_noeabi}/usr/include" \ " -I{ctx.ndk_dir}/sources/cxx-stl/gnu-libstdc++/{ctx.toolchain_version}/include" \ " -I{ctx.ndk_dir}/sources/cxx-stl/gnu-libstdc++/{ctx.toolchain_version}/libs/{arch.arch}/include".format(**keys) env['CXXFLAGS'] = env['CFLAGS'] + ' -frtti -fexceptions' env['LDFLAGS'] += " -L{ctx.ndk_dir}/sources/cxx-stl/gnu-libstdc++/{ctx.toolchain_version}/libs/{arch.arch}" \ - " -lpython2.7" \ " -lgnustl_shared".format(**keys) - return env - def build_compiled_components(self,arch): + def build_compiled_components(self, arch): super(CppCompiledComponentsPythonRecipe, self).build_compiled_components(arch) # Copy libgnustl_shared.so with current_directory(self.get_build_dir(arch.arch)): sh.cp( - "{ctx.ndk_dir}/sources/cxx-stl/gnu-libstdc++/{ctx.toolchain_version}/libs/{arch.arch}/libgnustl_shared.so".format(ctx=self.ctx,arch=arch), + "{ctx.ndk_dir}/sources/cxx-stl/gnu-libstdc++/{ctx.toolchain_version}/libs/{arch.arch}/libgnustl_shared.so".format(ctx=self.ctx, arch=arch), self.ctx.get_libs_dir(arch.arch) ) - - class CythonRecipe(PythonRecipe): pre_build_ext = False cythonize = True cython_args = [] + call_hostpython_via_targetpython = False def __init__(self, *args, **kwargs): super(CythonRecipe, self).__init__(*args, **kwargs) @@ -1082,21 +1068,6 @@ class CythonRecipe(PythonRecipe): env['LIBLINK_PATH'] = liblink_path ensure_dir(liblink_path) - if self.ctx.python_recipe.from_crystax: - env['CFLAGS'] = '-I{} '.format( - join(self.ctx.ndk_dir, 'sources', 'python', - self.ctx.python_recipe.version, 'include', - 'python')) + env['CFLAGS'] - - # Temporarily hardcode the -lpython3.x as this does not - # get applied automatically in some environments. This - # will need generalising, along with the other hardcoded - # py3.5 references, to support other python3 or crystax - # python versions. - python3_version = self.ctx.python_recipe.version - python3_version = '.'.join(python3_version.split('.')[:2]) - env['LDFLAGS'] = env['LDFLAGS'] + ' -lpython{}m'.format(python3_version) - return env diff --git a/p4a/pythonforandroid/recipes/cryptography/__init__.py b/p4a/pythonforandroid/recipes/cryptography/__init__.py index 91fae24c..64f79df9 100644 --- a/p4a/pythonforandroid/recipes/cryptography/__init__.py +++ b/p4a/pythonforandroid/recipes/cryptography/__init__.py @@ -5,7 +5,7 @@ class CryptographyRecipe(CompiledComponentsPythonRecipe): name = 'cryptography' version = '1.4' url = 'https://github.com/pyca/cryptography/archive/{version}.tar.gz' - depends = [('python2', 'python3crystax'), 'openssl', 'idna', 'pyasn1', 'six', 'setuptools', 'enum34', 'ipaddress', 'cffi'] + depends = [('python2', 'python3crystax'), 'openssl', 'idna', 'pyasn1', 'six', 'setuptools', 'ipaddress', 'cffi'] call_hostpython_via_targetpython = False def get_recipe_env(self, arch): diff --git a/recipes/android/__init__.py b/recipes/android/__init__.py index 87fdff01..16110958 100644 --- a/recipes/android/__init__.py +++ b/recipes/android/__init__.py @@ -1,3 +1,83 @@ +from pythonforandroid.recipe import CythonRecipe, IncludedFilesBehaviour +from pythonforandroid.util import current_directory +from pythonforandroid.patching import will_build +from pythonforandroid import logger + +from os.path import join + + +class AndroidRecipe(IncludedFilesBehaviour, CythonRecipe): + # name = 'android' + version = None + url = None + + src_filename = 'src' + + depends = [('pygame', 'sdl2', 'genericndkbuild'), ('python2', 'python3crystax')] + + config_env = {} + + def get_recipe_env(self, arch): + env = super(AndroidRecipe, self).get_recipe_env(arch) + env.update(self.config_env) + return env + + def prebuild_arch(self, arch): + super(AndroidRecipe, self).prebuild_arch(arch) + + tpxi = 'DEF {} = {}\n' + th = '#define {} {}\n' + tpy = '{} = {}\n' + + bootstrap = bootstrap_name = self.ctx.bootstrap.name + is_sdl2 = bootstrap_name in ('sdl2', 'sdl2python3', 'sdl2_gradle') + is_pygame = bootstrap_name in ('pygame',) + is_webview = bootstrap_name in ('webview',) + is_lbry = bootstrap_name in ('lbry',) + + if is_sdl2 or is_webview or is_lbry: + if is_sdl2: + bootstrap = 'sdl2' + java_ns = 'org.kivy.android' + jni_ns = 'org/kivy/android' + elif is_pygame: + java_ns = 'org.renpy.android' + jni_ns = 'org/renpy/android' + else: + logger.error('unsupported bootstrap for android recipe: {}'.format(bootstrap_name)) + exit(1) + + config = { + 'BOOTSTRAP': bootstrap, + 'IS_SDL2': int(is_sdl2), + 'IS_PYGAME': int(is_pygame), + 'PY2': int(will_build('python2')(self)), + 'JAVA_NAMESPACE': java_ns, + 'JNI_NAMESPACE': jni_ns, + } + + with current_directory(self.get_build_dir(arch.arch)): + with open(join('android', 'config.pxi'), 'w') as fpxi: + with open(join('android', 'config.h'), 'w') as fh: + with open(join('android', 'config.py'), 'w') as fpy: + for key, value in config.items(): + fpxi.write(tpxi.format(key, repr(value))) + fpy.write(tpy.format(key, repr(value))) + fh.write(th.format(key, value if isinstance(value, int) + else '"{}"'.format(value))) + self.config_env[key] = str(value) + + if is_sdl2: + fh.write('JNIEnv *SDL_AndroidGetJNIEnv(void);\n') + fh.write('#define SDL_ANDROID_GetJNIEnv SDL_AndroidGetJNIEnv\n') + elif is_pygame: + fh.write('JNIEnv *SDL_ANDROID_GetJNIEnv(void);\n') + + +recipe = AndroidRecipe() + + +''' from pythonforandroid.recipe import CythonRecipe, Recipe, IncludedFilesBehaviour from pythonforandroid.util import current_directory from pythonforandroid.patching import will_build @@ -83,3 +163,4 @@ class AndroidRecipe(IncludedFilesBehaviour, CythonRecipe): recipe = AndroidRecipe() +''' \ No newline at end of file diff --git a/recipes/android/src/setup.py b/recipes/android/src/setup.py index 8a276fe2..47ef6a94 100755 --- a/recipes/android/src/setup.py +++ b/recipes/android/src/setup.py @@ -6,7 +6,7 @@ lib_dict = { 'pygame': ['sdl'], 'sdl2': ['SDL2', 'SDL2_image', 'SDL2_mixer', 'SDL2_ttf'] } -sdl_libs = lib_dict[os.environ['BOOTSTRAP']] +sdl_libs = lib_dict[os.environ['BOOTSTRAP']] if os.environ['BOOTSTRAP'] == 'sdl2' else [] renpy_sound = Extension('android._android_sound', ['android/_android_sound.c', 'android/_android_sound_jni.c', ], diff --git a/recipes/cffi/__init__.py b/recipes/cffi/__init__.py index 0fc6018d..3fba8a18 100644 --- a/recipes/cffi/__init__.py +++ b/recipes/cffi/__init__.py @@ -1,36 +1,58 @@ -from pythonforandroid.recipe import CompiledComponentsPythonRecipe, Recipe -from os.path import join + +import os +from pythonforandroid.recipe import CompiledComponentsPythonRecipe class CffiRecipe(CompiledComponentsPythonRecipe): - name = 'cffi' - version = '1.11.0' - #url = 'https://pypi.python.org/packages/source/c/cffi/cffi-{version}.tar.gz' - url = 'https://pypi.python.org/packages/4e/32/4070bdf32812c89eb635c80880a5caa2e0189aa7999994c265577e5154f3/cffi-{version}.tar.gz' + name = 'cffi' + version = '1.4.2' + url = 'https://pypi.python.org/packages/source/c/cffi/cffi-{version}.tar.gz' - depends = [('python2', 'python3crystax'), 'setuptools', 'pycparser', 'libffi'] + depends = [('python2', 'python3crystax'), 'setuptools', 'pycparser', 'libffi'] - patches = ['disable-pkg-config.patch'] + patches = ['disable-pkg-config.patch'] - # call_hostpython_via_targetpython = False - install_in_hostpython = True + # call_hostpython_via_targetpython = False + install_in_hostpython = True - def get_recipe_env(self, arch=None): - env = super(CffiRecipe, self).get_recipe_env(arch) - libffi = self.get_recipe('libffi', self.ctx) - includes = libffi.get_include_dirs(arch) - env['CFLAGS'] = ' -I'.join([env.get('CFLAGS', '')] + includes) - env['LDFLAGS'] = (env.get('CFLAGS', '') + ' -L' + - self.ctx.get_libs_dir(arch.arch)) - env['LDSHARED'] = env['CC'] + ' -pthread -shared -Wl,-O1 -Wl,-Bsymbolic-functions' - env['PYTHONPATH'] = ':'.join([self.ctx.get_site_packages_dir(), env['BUILDLIB_PATH']]) - - target_python = Recipe.get_recipe('python2', self.ctx).get_build_dir(arch.arch) - env['PYTHON_ROOT'] = join(target_python, 'python-install') - env['CFLAGS'] += ' -I' + env['PYTHON_ROOT'] + '/include/python2.7' - env['LDFLAGS'] += ' -L' + env['PYTHON_ROOT'] + '/lib' + ' -lpython2.7' - - return env + def get_hostrecipe_env(self, arch=None): + # fixes missing ffi.h on some host systems (e.g. gentoo) + env = super(CffiRecipe, self).get_hostrecipe_env(arch) + libffi = self.get_recipe('libffi', self.ctx) + includes = libffi.get_include_dirs(arch) + env['FFI_INC'] = ",".join(includes) + return env + + def get_recipe_env(self, arch=None): + env = super(CffiRecipe, self).get_recipe_env(arch) + # sets linker to use the correct gcc (cross compiler) + env['LDSHARED'] = env['CC'] + ' -pthread -shared -Wl,-O1 -Wl,-Bsymbolic-functions' + libffi = self.get_recipe('libffi', self.ctx) + includes = libffi.get_include_dirs(arch) + env['CFLAGS'] = ' -I'.join([env.get('CFLAGS', '')] + includes) + env['LDFLAGS'] = (env.get('CFLAGS', '') + ' -L' + + self.ctx.get_libs_dir(arch.arch)) + env['LDFLAGS'] += ' -L{}'.format(os.path.join(self.ctx.bootstrap.build_dir, 'libs', arch.arch)) + + # required for libc and libdl + ndk_dir = self.ctx.ndk_platform + ndk_lib_dir = os.path.join(ndk_dir, 'usr', 'lib') + env['LDFLAGS'] += ' -L{}'.format(ndk_lib_dir) + env['LDFLAGS'] += " --sysroot={}".format(self.ctx.ndk_platform) + env['PYTHONPATH'] = ':'.join([ + self.ctx.get_site_packages_dir(), + env['BUILDLIB_PATH'], + ]) + if self.ctx.ndk == 'crystax': + # only keeps major.minor (discards patch) + python_version = self.ctx.python_recipe.version[0:3] + ndk_dir_python = os.path.join(self.ctx.ndk_dir, 'sources/python/', python_version) + env['LDFLAGS'] += ' -L{}'.format(os.path.join(ndk_dir_python, 'libs', arch.arch)) + env['LDFLAGS'] += ' -lpython{}m'.format(python_version) + # until `pythonforandroid/archs.py` gets merged upstream: + # https://github.com/kivy/python-for-android/pull/1250/files#diff-569e13021e33ced8b54385f55b49cbe6 + env['CFLAGS'] += ' -I{}/include/python/'.format(ndk_dir_python) + return env -recipe = CffiRecipe() +recipe = CffiRecipe() \ No newline at end of file diff --git a/recipes/cffi/disable-pkg-config.patch b/recipes/cffi/disable-pkg-config.patch index 56346bb7..cf2abd5b 100644 --- a/recipes/cffi/disable-pkg-config.patch +++ b/recipes/cffi/disable-pkg-config.patch @@ -1,17 +1,18 @@ -diff -Naur cffi-1.4.2/setup.py b/setup.py ---- cffi-1.4.2/setup.py 2015-12-21 12:09:47.000000000 -0600 -+++ b/setup.py 2015-12-23 10:20:40.590622524 -0600 -@@ -5,8 +5,7 @@ +diff --git a/setup.py b/setup.py +index c1db368..57311c3 100644 +--- a/setup.py ++++ b/setup.py +@@ -5,8 +5,7 @@ import errno sources = ['c/_cffi_backend.c'] libraries = ['ffi'] -include_dirs = ['/usr/include/ffi', - '/usr/include/libffi'] # may be changed by pkg-config -+include_dirs = [] ++include_dirs = os.environ['FFI_INC'].split(",") if 'FFI_INC' in os.environ else [] define_macros = [] library_dirs = [] extra_compile_args = [] -@@ -67,14 +66,7 @@ +@@ -67,14 +66,7 @@ def ask_supports_thread(): sys.stderr.write("The above error message can be safely ignored\n") def use_pkg_config(): diff --git a/recipes/coincurve/__init__.py b/recipes/coincurve/__init__.py new file mode 100644 index 00000000..921655e5 --- /dev/null +++ b/recipes/coincurve/__init__.py @@ -0,0 +1,42 @@ +import os +from pythonforandroid.recipe import PythonRecipe, CompiledComponentsPythonRecipe + + +class CoincurveRecipe(CompiledComponentsPythonRecipe): + version = '7.1.0' + url = 'https://github.com/ofek/coincurve/archive/{version}.tar.gz' + call_hostpython_via_targetpython = False + depends = [('python2', 'python3crystax'), 'setuptools', + 'libffi', 'cffi', 'libsecp256k1'] + patches = [ + "cross_compile.patch", "drop_setup_requires.patch", + "find_lib.patch", "no-download.patch", "setup.py.patch"] + + def get_recipe_env(self, arch=None, with_flags_in_cc=True): + env = super(CoincurveRecipe, self).get_recipe_env(arch, with_flags_in_cc) + # sets linker to use the correct gcc (cross compiler) + env['LDSHARED'] = env['CC'] + ' -pthread -shared -Wl,-O1 -Wl,-Bsymbolic-functions' + libsecp256k1 = self.get_recipe('libsecp256k1', self.ctx) + libsecp256k1_dir = libsecp256k1.get_build_dir(arch.arch) + env['LDFLAGS'] += ' -L{}'.format(os.path.join(libsecp256k1_dir, '.libs')) + env['CFLAGS'] += ' -I' + os.path.join(libsecp256k1_dir, 'include') + # only keeps major.minor (discards patch) + python_version = self.ctx.python_recipe.version[0:3] + # required additional library and path for Crystax + if self.ctx.ndk == 'crystax': + ndk_dir_python = os.path.join(self.ctx.ndk_dir, 'sources/python/', python_version) + env['LDFLAGS'] += ' -L{}'.format(os.path.join(ndk_dir_python, 'libs', arch.arch)) + env['LDFLAGS'] += ' -lpython{}m'.format(python_version) + # until `pythonforandroid/archs.py` gets merged upstream: + # https://github.com/kivy/python-for-android/pull/1250/files#diff-569e13021e33ced8b54385f55b49cbe6 + env['CFLAGS'] += ' -I{}/include/python/'.format(ndk_dir_python) + else: + env['PYTHON_ROOT'] = self.ctx.get_python_install_dir() + env['CFLAGS'] += ' -I' + env['PYTHON_ROOT'] + '/include/python{}'.format(python_version) + env['LDFLAGS'] += " -lpython{}".format(python_version) + env['LDFLAGS'] += " -lsecp256k1" + return env + + +recipe = CoincurveRecipe() + diff --git a/recipes/coincurve/cross_compile.patch b/recipes/coincurve/cross_compile.patch new file mode 100644 index 00000000..fbbdd49b --- /dev/null +++ b/recipes/coincurve/cross_compile.patch @@ -0,0 +1,12 @@ +diff --git a/setup.py b/setup.py +index c224fb2..bf925bd 100644 +--- a/setup.py ++++ b/setup.py +@@ -182,6 +182,7 @@ class build_clib(_build_clib): + '--disable-dependency-tracking', + '--with-pic', + '--enable-module-recovery', ++ "--host=%s" % os.environ['TOOLCHAIN_PREFIX'], + '--disable-jni', + '--prefix', + os.path.abspath(self.build_clib), diff --git a/recipes/coincurve/drop_setup_requires.patch b/recipes/coincurve/drop_setup_requires.patch new file mode 100644 index 00000000..9994b3fa --- /dev/null +++ b/recipes/coincurve/drop_setup_requires.patch @@ -0,0 +1,12 @@ +diff --git a/setup.py b/setup.py +index c224fb2..466e789 100644 +--- a/setup.py ++++ b/setup.py +@@ -250,7 +250,6 @@ else: + def has_c_libraries(self): + return not has_system_lib() + setup_kwargs = dict( +- setup_requires=['cffi>=1.3.0', 'pytest-runner>=2.6.2'], + ext_package='coincurve', + cffi_modules=['_cffi_build/build.py:ffi'], + cmdclass={ diff --git a/recipes/coincurve/find_lib.patch b/recipes/coincurve/find_lib.patch new file mode 100644 index 00000000..3d3c41d9 --- /dev/null +++ b/recipes/coincurve/find_lib.patch @@ -0,0 +1,13 @@ +diff --git a/setup_support.py b/setup_support.py +index e7a4f2e..72f0c4d 100644 +--- a/setup_support.py ++++ b/setup_support.py +@@ -68,6 +69,8 @@ def build_flags(library, type_, path): + + + def _find_lib(): ++ # we're picking up the recipe one ++ return True + from cffi import FFI + ffi = FFI() + try: diff --git a/recipes/coincurve/no-download.patch b/recipes/coincurve/no-download.patch new file mode 100644 index 00000000..fcf4d204 --- /dev/null +++ b/recipes/coincurve/no-download.patch @@ -0,0 +1,13 @@ +diff --git a/setup.py b/setup.py +index c224fb2..d5f6d1a 100644 +--- a/setup.py ++++ b/setup.py +@@ -51,6 +51,8 @@ if [int(i) for i in setuptools_version.split('.', 2)[:2]] < [3, 3]: + + + def download_library(command): ++ # we will use the custom libsecp256k1 recipe ++ return + if command.dry_run: + return + libdir = absolute('libsecp256k1') diff --git a/recipes/coincurve/setup.py.patch b/recipes/coincurve/setup.py.patch new file mode 100644 index 00000000..bdc7ab4d --- /dev/null +++ b/recipes/coincurve/setup.py.patch @@ -0,0 +1,22 @@ +From bf3a0684e9b0af29d9777f61d6e7e1d3cc0f2803 Mon Sep 17 00:00:00 2001 +From: Kieran Prasch +Date: Thu, 19 Jul 2018 14:11:48 -0700 +Subject: [PATCH] Exclude tests in setup.py + +--- + setup.py | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/setup.py b/setup.py +index 0b579f1..0a793ed 100644 +--- a/setup.py ++++ b/setup.py +@@ -277,7 +277,7 @@ def has_c_libraries(self): + install_requires=['asn1crypto', 'cffi>=1.3.0'], + tests_require=['pytest>=2.8.7'], + +- packages=find_packages(exclude=('_cffi_build', '_cffi_build.*', 'libsecp256k1')), ++ packages=find_packages(exclude=('_cffi_build', '_cffi_build.*', 'libsecp256k1', 'tests')), + + distclass=Distribution, + zip_safe=False, diff --git a/recipes/cryptography/__init__.py b/recipes/cryptography/__init__.py index d286924a..7d03a3df 100644 --- a/recipes/cryptography/__init__.py +++ b/recipes/cryptography/__init__.py @@ -1,30 +1,37 @@ from pythonforandroid.recipe import CompiledComponentsPythonRecipe, Recipe from os.path import dirname, join +import os class CryptographyRecipe(CompiledComponentsPythonRecipe): name = 'cryptography' version = '2.2.2' url = 'https://github.com/pyca/cryptography/archive/{version}.tar.gz' - depends = [('python2', 'python3crystax'), 'openssl', 'idna', 'pyasn1', 'six', 'setuptools', 'enum34', 'ipaddress', 'cffi'] + depends = [('python2', 'python3crystax'), 'openssl', 'idna', 'pyasn1', 'six', 'setuptools', 'ipaddress', 'cffi'] call_hostpython_via_targetpython = False + patches = ['force-urandom.patch'] def get_recipe_env(self, arch): env = super(CryptographyRecipe, self).get_recipe_env(arch) r = self.get_recipe('openssl', self.ctx) openssl_dir = r.get_build_dir(arch.arch) - target_python = Recipe.get_recipe('python2', self.ctx).get_build_dir(arch.arch) - env['PYTHON_ROOT'] = self.ctx.get_python_install_dir() - env['CFLAGS'] += ' -I' + env['PYTHON_ROOT'] + '/include/python2.7' + \ - ' -I' + join(openssl_dir, 'include') + \ - ' -I' + join(target_python, 'Modules/_ctypes/libffi_arm_wince') + env['CFLAGS'] += ' -I' + join(openssl_dir, 'include') + ' -w' # Set linker to use the correct gcc env['LDSHARED'] = env['CC'] + ' -pthread -shared -Wl,-O1 -Wl,-Bsymbolic-functions' - env['LDFLAGS'] += ' -L' + env['PYTHON_ROOT'] + '/lib' + \ - ' -L' + openssl_dir + \ - ' -lpython2.7' + \ + env['LDFLAGS'] += ' -L' + openssl_dir + \ ' -lssl' + r.version + \ ' -lcrypto' + r.version - + + if self.ctx.ndk == 'crystax': + # only keeps major.minor (discards patch) + python_version = self.ctx.python_recipe.version[0:3] + ndk_dir_python = os.path.join(self.ctx.ndk_dir, 'sources/python/', python_version) + env['LDFLAGS'] += ' -L{}'.format(os.path.join(ndk_dir_python, 'libs', arch.arch)) + env['LDFLAGS'] += ' -lpython{}m'.format(python_version) + # until `pythonforandroid/archs.py` gets merged upstream: + # https://github.com/kivy/python-for-android/pull/1250/files#diff-569e13021e33ced8b54385f55b49cbe6 + env['CFLAGS'] += ' -I{}/include/python/'.format(ndk_dir_python) + ' -L{}'.format(os.path.join(ndk_dir_python, 'libs', arch.arch)) + + return env recipe = CryptographyRecipe() diff --git a/recipes/cryptography/force-urandom.patch b/recipes/cryptography/force-urandom.patch new file mode 100644 index 00000000..dceeab34 --- /dev/null +++ b/recipes/cryptography/force-urandom.patch @@ -0,0 +1,11 @@ +--- x/src/_cffi_src/openssl/src/osrandom_engine.h 2018-09-25 23:19:08.303397032 +0100 ++++ y/src/_cffi_src/openssl/src/osrandom_engine.h 2018-09-25 23:21:45.945980819 +0100 +@@ -42,7 +42,7 @@ + #define CRYPTOGRAPHY_OSRANDOM_ENGINE CRYPTOGRAPHY_OSRANDOM_ENGINE_GETENTROPY + #elif defined(__linux__) && defined(SYS_getrandom) + /* Linux 3.4.17+ */ +- #define CRYPTOGRAPHY_OSRANDOM_ENGINE CRYPTOGRAPHY_OSRANDOM_ENGINE_GETRANDOM ++ #define CRYPTOGRAPHY_OSRANDOM_ENGINE CRYPTOGRAPHY_OSRANDOM_ENGINE_DEV_URANDOM // force /dev/urandom + #else + /* Keep this as last entry, fall back to /dev/urandom */ + #define CRYPTOGRAPHY_OSRANDOM_ENGINE CRYPTOGRAPHY_OSRANDOM_ENGINE_DEV_URANDOM diff --git a/recipes/enum34/__init__.py b/recipes/enum34/__init__.py new file mode 100644 index 00000000..a54129fd --- /dev/null +++ b/recipes/enum34/__init__.py @@ -0,0 +1,12 @@ +from pythonforandroid.recipe import PythonRecipe + + +class Enum34Recipe(PythonRecipe): + version = '1.1.3' + url = 'https://pypi.python.org/packages/source/e/enum34/enum34-{version}.tar.gz' + depends = [('python2', 'python3crystax'), 'setuptools'] + site_packages_name = 'enum' + call_hostpython_via_targetpython = False + + +recipe = Enum34Recipe() \ No newline at end of file diff --git a/recipes/gmpy/__init__.py b/recipes/gmpy/__init__.py index 23c60538..d96bdd09 100644 --- a/recipes/gmpy/__init__.py +++ b/recipes/gmpy/__init__.py @@ -27,10 +27,10 @@ class GmpyRecipe(CythonRecipe): libgmp_build_dir = Recipe.get_recipe('libgmp', self.ctx).get_build_dir(arch.arch) env['CFLAGS'] += ' -I%s' % (join(libgmp_build_dir, 'include')) - target_python = Recipe.get_recipe('python2', self.ctx).get_build_dir(arch.arch) + target_python = Recipe.get_recipe('python3crystax', self.ctx).get_build_dir(arch.arch) env['PYTHON_ROOT'] = join(target_python, 'python-install') - env['CFLAGS'] += ' -I' + env['PYTHON_ROOT'] + '/include/python2.7' - env['LDFLAGS'] += ' -L' + env['PYTHON_ROOT'] + '/lib' + ' -lpython2.7' + env['CFLAGS'] += ' -I' + env['PYTHON_ROOT'] + '/include/python3.6' + env['LDFLAGS'] += ' -L' + env['PYTHON_ROOT'] + '/lib' + ' -lpython3.6m' return env diff --git a/recipes/hostpython3crystax/__init__.py b/recipes/hostpython3crystax/__init__.py new file mode 100644 index 00000000..1559df2c --- /dev/null +++ b/recipes/hostpython3crystax/__init__.py @@ -0,0 +1,46 @@ +from pythonforandroid.toolchain import Recipe, shprint +from os.path import join +import sh + + +class Hostpython3Recipe(Recipe): + version = 'auto' # the version is taken from the python3crystax recipe + name = 'hostpython3crystax' + + conflicts = ['hostpython2'] + + def get_build_container_dir(self, arch=None): + choices = self.check_recipe_choices() + dir_name = '-'.join([self.name] + choices) + return join(self.ctx.build_dir, 'other_builds', dir_name, 'desktop') + + # def prebuild_armeabi(self): + # # Override hostpython Setup? + # shprint(sh.cp, join(self.get_recipe_dir(), 'Setup'), + # join(self.get_build_dir('armeabi'), 'Modules', 'Setup')) + + def get_build_dir(self, arch=None): + return join(self.get_build_container_dir(), self.name) + + def build_arch(self, arch): + """ + Creates expected build and symlinks system Python version. + """ + self.ctx.hostpython = '/usr/bin/false' + self.ctx.hostpgen = '/usr/bin/false' + # creates the sub buildir (used by other recipes) + # https://github.com/kivy/python-for-android/issues/1154 + sub_build_dir = join(self.get_build_dir(), 'build') + shprint(sh.mkdir, '-p', sub_build_dir) + python3crystax = self.get_recipe('python3crystax', self.ctx) + system_python = sh.which("python" + python3crystax.version) + if system_python is None: + raise OSError( + ('Trying to use python3crystax=={} but this Python version ' + 'is not installed locally.').format(python3crystax.version)) + link_dest = join(self.get_build_dir(), 'hostpython') + shprint(sh.ln, '-sf', system_python, link_dest) + + +recipe = Hostpython3Recipe() + diff --git a/recipes/idna/__init__.py b/recipes/idna/__init__.py new file mode 100644 index 00000000..a79c0f6d --- /dev/null +++ b/recipes/idna/__init__.py @@ -0,0 +1,14 @@ +from pythonforandroid.recipe import PythonRecipe + + +class IdnaRecipe(PythonRecipe): + name = 'idna' + version = '2.6' + url = 'https://github.com/kjd/idna/archive/v{version}.tar.gz' + + depends = [('python2', 'python3crystax'), 'setuptools'] + + call_hostpython_via_targetpython = False + + +recipe = IdnaRecipe() \ No newline at end of file diff --git a/recipes/incremental/__init__.py b/recipes/incremental/__init__.py index 4a61083a..10e6b10f 100644 --- a/recipes/incremental/__init__.py +++ b/recipes/incremental/__init__.py @@ -7,7 +7,7 @@ class IncrementalRecipe(PythonRecipe): version = '17.5.0' url = 'https://pypi.python.org/packages/8f/26/02c4016aa95f45479eea37c90c34f8fab6775732ae62587a874b619ca097/incremental-{version}.tar.gz' - depends = ['python2', 'setuptools'] + depends = [('python2', 'python3crystax'), 'setuptools'] call_hostpython_via_targetpython = False install_in_hostpython = True diff --git a/recipes/ipaddress/__init__.py b/recipes/ipaddress/__init__.py index 12a042a1..509ecc50 100644 --- a/recipes/ipaddress/__init__.py +++ b/recipes/ipaddress/__init__.py @@ -2,13 +2,11 @@ from pythonforandroid.recipe import PythonRecipe class IpaddressRecipe(PythonRecipe): - name = 'ipaddress' - version = '1.0.16' - url = 'https://pypi.python.org/packages/source/i/ipaddress/ipaddress-{version}.tar.gz' + name = 'ipaddress' + version = '1.0.16' + url = 'https://pypi.python.org/packages/source/i/ipaddress/ipaddress-{version}.tar.gz' - depends = ['python2'] - - call_hostpython_via_targetpython = False + depends = [('python2', 'python3crystax')] -recipe = IpaddressRecipe() +recipe = IpaddressRecipe() \ No newline at end of file diff --git a/recipes/libffi/Application.mk b/recipes/libffi/Application.mk new file mode 100644 index 00000000..599da111 --- /dev/null +++ b/recipes/libffi/Application.mk @@ -0,0 +1,3 @@ +APP_OPTIM := release +APP_ABI := all # or armeabi +APP_MODULES := libffi \ No newline at end of file diff --git a/recipes/libffi/__init__.py b/recipes/libffi/__init__.py new file mode 100644 index 00000000..6d808407 --- /dev/null +++ b/recipes/libffi/__init__.py @@ -0,0 +1,83 @@ +from os.path import exists, join +from pythonforandroid.recipe import Recipe +from pythonforandroid.logger import info, shprint +from pythonforandroid.util import current_directory +import sh + + +class LibffiRecipe(Recipe): + name = 'libffi' + version = 'v3.2.1' + url = 'https://github.com/atgreen/libffi/archive/{version}.zip' + + patches = ['remove-version-info.patch'] + + def get_host(self, arch): + with current_directory(self.get_build_dir(arch.arch)): + host = None + with open('Makefile') as f: + for line in f: + if line.startswith('host = '): + host = line.strip()[7:] + break + + if not host or not exists(host): + raise RuntimeError('failed to find build output! ({})' + .format(host)) + + return host + + def should_build(self, arch): + return not exists(join(self.ctx.get_libs_dir(arch.arch), 'libffi.so')) + + def build_arch(self, arch): + env = self.get_recipe_env(arch) + with current_directory(self.get_build_dir(arch.arch)): + if not exists('configure'): + shprint(sh.Command('./autogen.sh'), _env=env) + shprint(sh.Command('autoreconf'), '-vif', _env=env) + shprint(sh.Command('./configure'), + '--host=' + arch.toolchain_prefix, + '--prefix=' + self.ctx.get_python_install_dir(), + '--enable-shared', _env=env) + #'--with-sysroot={}'.format(self.ctx.ndk_platform), + #'--target={}'.format(arch.toolchain_prefix), + + # ndk 15 introduces unified headers required --sysroot and + # -isysroot for libraries and headers. libtool's head explodes + # trying to weave them into it's own magic. The result is a link + # failure tryng to link libc. We call make to compile the bits + # and manually link... + + try: + shprint(sh.make, '-j5', 'libffi.la', _env=env) + except sh.ErrorReturnCode_2: + info("make libffi.la failed as expected") + cc = sh.Command(env['CC'].split()[0]) + cflags = env['CC'].split()[1:] + + cflags.extend(['-march=armv7-a', '-mfloat-abi=softfp', '-mfpu=vfp', + '-mthumb', '-shared', '-fPIC', '-DPIC', + 'src/.libs/prep_cif.o', 'src/.libs/types.o', + 'src/.libs/raw_api.o', 'src/.libs/java_raw_api.o', + 'src/.libs/closures.o', 'src/arm/.libs/sysv.o', + 'src/arm/.libs/ffi.o', ] + ) + + ldflags = env['LDFLAGS'].split() + cflags.extend(ldflags) + cflags.extend(['-Wl,-soname', '-Wl,libffi.so', '-o', + '.libs/libffi.so']) + + with current_directory(self.get_host(arch)): + shprint(cc, *cflags, _env=env) + + shprint(sh.cp, '-t', self.ctx.get_libs_dir(arch.arch), + join(self.get_host(arch), '.libs', 'libffi.so')) + + def get_include_dirs(self, arch): + return [join(self.get_build_dir(arch.arch), self.get_host(arch), + 'include')] + + +recipe = LibffiRecipe() diff --git a/recipes/libffi/disable-mips-check.patch b/recipes/libffi/disable-mips-check.patch new file mode 100644 index 00000000..0f727ba4 --- /dev/null +++ b/recipes/libffi/disable-mips-check.patch @@ -0,0 +1,35 @@ +diff -Naur libffi/Android.mk b/Android.mk +--- libffi/Android.mk 2015-12-22 17:00:48.025478556 -0600 ++++ b/Android.mk 2015-12-22 17:02:23.999249390 -0600 +@@ -23,23 +23,20 @@ + # Build rules for the target. + # + +-# We only build ffi for mips. +-ifeq ($(TARGET_ARCH),mips) + +- include $(CLEAR_VARS) ++include $(CLEAR_VARS) + +- ffi_arch := $(TARGET_ARCH) +- ffi_os := $(TARGET_OS) ++ffi_arch := $(TARGET_ARCH) ++ffi_os := $(TARGET_OS) + +- # This include just keeps the nesting a bit saner. +- include $(LOCAL_PATH)/Libffi.mk ++# This include just keeps the nesting a bit saner. ++include $(LOCAL_PATH)/Libffi.mk + +- LOCAL_MODULE_TAGS := optional +- LOCAL_MODULE := libffi ++LOCAL_MODULE_TAGS := optional ++LOCAL_MODULE := libffi + +- include $(BUILD_SHARED_LIBRARY) ++include $(BUILD_SHARED_LIBRARY) + +-endif + + # Also include the rules for the test suite. + include external/libffi/testsuite/Android.mk diff --git a/recipes/libffi/remove-version-info.patch b/recipes/libffi/remove-version-info.patch new file mode 100644 index 00000000..7bdc11a6 --- /dev/null +++ b/recipes/libffi/remove-version-info.patch @@ -0,0 +1,12 @@ +diff -Naur libffi/Makefile.am b/Makefile.am +--- libffi/Makefile.am 2014-11-12 06:00:59.000000000 -0600 ++++ b/Makefile.am 2015-12-23 15:57:10.363148806 -0600 +@@ -249,7 +249,7 @@ + AM_CFLAGS += -DFFI_DEBUG + endif + +-libffi_la_LDFLAGS = -no-undefined -version-info `grep -v '^\#' $(srcdir)/libtool-version` $(LTLDFLAGS) $(AM_LTLDFLAGS) ++libffi_la_LDFLAGS = -no-undefined -avoid-version $(LTLDFLAGS) $(AM_LTLDFLAGS) + + AM_CPPFLAGS = -I. -I$(top_srcdir)/include -Iinclude -I$(top_srcdir)/src + AM_CCASFLAGS = $(AM_CPPFLAGS) diff --git a/recipes/libgmp/__init__.py b/recipes/libgmp/__init__.py index a974f04e..7e5ba596 100644 --- a/recipes/libgmp/__init__.py +++ b/recipes/libgmp/__init__.py @@ -3,6 +3,7 @@ from os.path import exists, join, realpath from os import uname import glob import sh +import os class LibGMPRecipe(Recipe): @@ -18,8 +19,20 @@ class LibGMPRecipe(Recipe): env = super(LibGMPRecipe, self).get_recipe_env(arch) env['LIBGMP_LDFLAGS'] = '-avoid-version' - return env + ndk_dir = self.ctx.ndk_platform + ndk_lib_dir = os.path.join(ndk_dir, 'usr', 'lib') + if self.ctx.ndk == 'crystax': + crystax_lib_dir = os.path.join(self.ctx.ndk_dir, 'sources/crystax/libs', arch.arch) + env['CFLAGS'] = '{} -L{} -L{}'.format(env.get('CFLAGS'), crystax_lib_dir, ndk_lib_dir); + env['LDFLAGS'] += ' -L{}'.format(ndk_lib_dir) + env['LDFLAGS'] += " --sysroot={}".format(self.ctx.ndk_platform) + env['PYTHONPATH'] = ':'.join([ + self.ctx.get_site_packages_dir(), + env['BUILDLIB_PATH'], + ]) + + return env def build_arch(self, arch): with current_directory(self.get_build_dir(arch.arch)): diff --git a/recipes/libsecp256k1/__init__.py b/recipes/libsecp256k1/__init__.py new file mode 100644 index 00000000..69349d99 --- /dev/null +++ b/recipes/libsecp256k1/__init__.py @@ -0,0 +1,33 @@ +from pythonforandroid.toolchain import shprint, current_directory +from pythonforandroid.recipe import Recipe +from multiprocessing import cpu_count +from os.path import exists +import sh + + +class LibSecp256k1Recipe(Recipe): + + version = 'b0452e6' + url = 'https://github.com/bitcoin-core/secp256k1/archive/{version}.zip' + + def build_arch(self, arch): + super(LibSecp256k1Recipe, self).build_arch(arch) + env = self.get_recipe_env(arch) + with current_directory(self.get_build_dir(arch.arch)): + if not exists('configure'): + shprint(sh.Command('./autogen.sh'), _env=env) + shprint( + sh.Command('./configure'), + '--host=' + arch.toolchain_prefix, + '--prefix=' + self.ctx.get_python_install_dir(), + '--enable-shared', + '--enable-module-recovery', + '--enable-experimental', + '--enable-module-ecdh', + _env=env) + shprint(sh.make, '-j' + str(cpu_count()), _env=env) + libs = ['.libs/libsecp256k1.so'] + self.install_libs(arch, *libs) + + +recipe = LibSecp256k1Recipe() diff --git a/recipes/miniupnpc/__init__.py b/recipes/miniupnpc/__init__.py index 821ac7cc..a4b3a54d 100644 --- a/recipes/miniupnpc/__init__.py +++ b/recipes/miniupnpc/__init__.py @@ -24,8 +24,8 @@ class MiniupnpcRecipe(CythonRecipe): target_python = Recipe.get_recipe('python2', self.ctx).get_build_dir(arch.arch) env['PYTHON_ROOT'] = join(target_python, 'python-install') - env['CFLAGS'] += ' -I' + env['PYTHON_ROOT'] + '/include/python2.7' - env['LDFLAGS'] += ' -L' + env['PYTHON_ROOT'] + '/lib' + ' -lpython2.7' + env['CFLAGS'] += ' -I' + env['PYTHON_ROOT'] + '/include/python3.6' + env['LDFLAGS'] += ' -L' + env['PYTHON_ROOT'] + '/lib' + ' -lpython3.6m' return env diff --git a/recipes/netifaces/__init__.py b/recipes/netifaces/__init__.py index 0c651887..b34fc7bc 100644 --- a/recipes/netifaces/__init__.py +++ b/recipes/netifaces/__init__.py @@ -2,23 +2,31 @@ import glob from pythonforandroid.toolchain import CompiledComponentsPythonRecipe, Recipe from os.path import join +import os import sh class NetifacesRecipe(CompiledComponentsPythonRecipe): version = '0.10.7' url = 'https://files.pythonhosted.org/packages/81/39/4e9a026265ba944ddf1fea176dbb29e0fe50c43717ba4fcf3646d099fe38/netifaces-{version}.tar.gz' - depends = ['python2', 'setuptools'] + depends = [('python2', 'python3crystax'), 'setuptools'] call_hostpython_via_targetpython = False + patches = ['socket-ioctls.patch'] def get_recipe_env(self, arch): env = super(NetifacesRecipe, self).get_recipe_env(arch) - target_python = Recipe.get_recipe('python2', self.ctx).get_build_dir(arch.arch) - env['PYTHON_ROOT'] = join(target_python, 'python-install') env['LDSHARED'] = env['CC'] + ' -pthread -shared -Wl,-O1 -Wl,-Bsymbolic-functions' - env['CFLAGS'] += ' -I' + env['PYTHON_ROOT'] + '/include/python2.7' - env['LDFLAGS'] += ' -L' + env['PYTHON_ROOT'] + '/lib' + ' -lpython2.7' + + if self.ctx.ndk == 'crystax': + # only keeps major.minor (discards patch) + python_version = self.ctx.python_recipe.version[0:3] + ndk_dir_python = os.path.join(self.ctx.ndk_dir, 'sources/python/', python_version) + env['LDFLAGS'] += ' -L{}'.format(os.path.join(ndk_dir_python, 'libs', arch.arch)) + env['LDFLAGS'] += ' -lpython{}m'.format(python_version) + # until `pythonforandroid/archs.py` gets merged upstream: + # https://github.com/kivy/python-for-android/pull/1250/files#diff-569e13021e33ced8b54385f55b49cbe6 + env['CFLAGS'] += ' -I{}/include/python/'.format(ndk_dir_python) return env diff --git a/recipes/netifaces/socket-ioctls.patch b/recipes/netifaces/socket-ioctls.patch new file mode 100644 index 00000000..4e6da707 --- /dev/null +++ b/recipes/netifaces/socket-ioctls.patch @@ -0,0 +1,29 @@ +--- a/netifaces.c 2018-09-25 21:26:48.238476102 +0100 ++++ b/netifaces.c 2018-09-25 21:55:14.201995669 +0100 +@@ -22,6 +22,10 @@ + obj = Py_InitModule3((name), (methods), (doc)); + #endif + ++#ifndef HAVE_SOCKET_IOCTLS ++#define HAVE_SOCKET_IOCTLS 1 ++#endif ++ + #ifndef WIN32 + + # include +@@ -178,12 +182,12 @@ + # include + # endif /* HAVE_GETIFADDRS */ + +-# if !HAVE_GETIFADDRS && (!HAVE_SOCKET_IOCTLS || !HAVE_SIOCGIFCONF) ++//# if !HAVE_GETIFADDRS && (!HAVE_SOCKET_IOCTLS || !HAVE_SIOCGIFCONF) + /* If the platform doesn't define, what we need, barf. If you're seeing this, + it means you need to write suitable code to retrieve interface information + on your system. */ +-# error You need to add code for your platform. +-# endif ++//# error You need to add code for your platform. ++//# endif + + #else /* defined(WIN32) */ + diff --git a/recipes/openssl/__init__.py b/recipes/openssl/__init__.py index 19084691..4152b04f 100644 --- a/recipes/openssl/__init__.py +++ b/recipes/openssl/__init__.py @@ -57,7 +57,7 @@ class OpenSSLRecipe(Recipe): break shprint(sh.make, 'clean', _env=env) - self.install_libs(arch, 'libssl' + self.version + '.so', - 'libcrypto' + self.version + '.so') + self.install_libs(arch, 'libssl.a', 'libssl' + self.version + '.so', + 'libcrypto.a', 'libcrypto' + self.version + '.so') recipe = OpenSSLRecipe() diff --git a/recipes/pyasn1/__init__.py b/recipes/pyasn1/__init__.py index 140a3bda..a10f7ca9 100644 --- a/recipes/pyasn1/__init__.py +++ b/recipes/pyasn1/__init__.py @@ -5,8 +5,7 @@ from pythonforandroid.toolchain import PythonRecipe class PyASN1Recipe(PythonRecipe): version = '0.4.2' url = 'https://pypi.python.org/packages/source/p/pyasn1/pyasn1-{version}.tar.gz' - depends = ['python2'] - - call_hostpython_via_targetpython = False + depends = [('python2', 'python3crystax')] + recipe = PyASN1Recipe() diff --git a/recipes/pyjnius/__init__.py b/recipes/pyjnius/__init__.py index d9eaf420..c336e8a3 100644 --- a/recipes/pyjnius/__init__.py +++ b/recipes/pyjnius/__init__.py @@ -18,10 +18,10 @@ class PyjniusRecipe(CythonRecipe): def get_recipe_env(self, arch): env = super(PyjniusRecipe, self).get_recipe_env(arch) - target_python = Recipe.get_recipe('python2', self.ctx).get_build_dir(arch.arch) + target_python = Recipe.get_recipe('python3crystax', self.ctx).get_build_dir(arch.arch) env['PYTHON_ROOT'] = join(target_python, 'python-install') - env['CFLAGS'] += ' -I' + env['PYTHON_ROOT'] + '/include/python2.7' - env['LDFLAGS'] += ' -L' + env['PYTHON_ROOT'] + '/lib' + ' -lpython2.7' + env['CFLAGS'] += ' -I' + env['PYTHON_ROOT'] + '/include/python3.6' + env['LDFLAGS'] += ' -L' + env['PYTHON_ROOT'] + '/lib' + ' -lpython3.6m' return env diff --git a/recipes/pyopenssl/__init__.py b/recipes/pyopenssl/__init__.py new file mode 100644 index 00000000..c3284705 --- /dev/null +++ b/recipes/pyopenssl/__init__.py @@ -0,0 +1,13 @@ +from pythonforandroid.recipe import PythonRecipe + + +class PyOpenSSLRecipe(PythonRecipe): + version = '17.4.0' + url = 'https://pypi.python.org/packages/source/p/pyOpenSSL/pyOpenSSL-{version}.tar.gz' + depends = [('python2', 'python3crystax'), 'openssl', 'setuptools'] + site_packages_name = 'OpenSSL' + + call_hostpython_via_targetpython = False + + +recipe = PyOpenSSLRecipe() \ No newline at end of file diff --git a/recipes/python3crystax/__init__.py b/recipes/python3crystax/__init__.py new file mode 100644 index 00000000..9cc6814c --- /dev/null +++ b/recipes/python3crystax/__init__.py @@ -0,0 +1,258 @@ +from pythonforandroid.recipe import TargetPythonRecipe +from pythonforandroid.toolchain import shprint, current_directory, ArchARM +from pythonforandroid.logger import info, error +from pythonforandroid.util import ensure_dir, temp_directory +from os.path import exists, join +import os +import glob +import sh +from sh import Command + +# This is the content of opensslconf.h taken from +# ndkdir/build/tools/build-target-openssl.sh +OPENSSLCONF = """#if defined(__ARM_ARCH_5TE__) +#include "opensslconf_armeabi.h" +#elif defined(__ARM_ARCH_7A__) && !defined(__ARM_PCS_VFP) +#include "opensslconf_armeabi_v7a.h" +#elif defined(__ARM_ARCH_7A__) && defined(__ARM_PCS_VFP) +#include "opensslconf_armeabi_v7a_hard.h" +#elif defined(__aarch64__) +#include "opensslconf_arm64_v8a.h" +#elif defined(__i386__) +#include "opensslconf_x86.h" +#elif defined(__x86_64__) +#include "opensslconf_x86_64.h" +#elif defined(__mips__) && !defined(__mips64) +#include "opensslconf_mips.h" +#elif defined(__mips__) && defined(__mips64) +#include "opensslconf_mips64.h" +#else +#error "Unsupported ABI" +#endif +""" +LATEST_FULL_VERSION = { + '3.5': '3.5.1', + '3.6': '3.6.6', + '3.7': '3.7.0' +} + +def realpath(fname): + """ + Own implementation of os.realpath which may be broken in some python versions + Returns: the absolute path o + + """ + + if not os.path.islink(fname): + return os.path.abspath(fname) + + abs_path = os.path.abspath(fname).split(os.sep)[:-1] + rel_path = os.readlink(fname) + + if os.path.abspath(rel_path) == rel_path: + return rel_path + + rel_path = rel_path.split(os.sep) + for folder in rel_path: + if folder == '..': + abs_path.pop() + else: + abs_path.append(folder) + return os.sep.join(abs_path) + +class Python3Recipe(TargetPythonRecipe): + version = '3.6' + url = '' + name = 'python3crystax' + + depends = ['hostpython3crystax'] + conflicts = ['python2', 'python3'] + + from_crystax = True + + def download_if_necessary(self): + if 'openssl' in self.ctx.recipe_build_order or self.version == '3.6': + full_version = LATEST_FULL_VERSION[self.version] + Python3Recipe.url = 'https://www.python.org/ftp/python/{0}.{1}.{2}/Python-{0}.{1}.{2}.tgz'.format(*full_version.split('.')) + super(Python3Recipe, self).download_if_necessary() + + def get_dir_name(self): + name = super(Python3Recipe, self).get_dir_name() + name += '-version{}'.format(self.version) + return name + + def copy_include_dir(self, source, target): + ensure_dir(target) + for fname in os.listdir(source): + sh.ln('-sf', realpath(join(source, fname)), join(target, fname)) + + def _patch_dev_defaults(self, fp, target_ver): + for line in fp: + if 'OPENSSL_VERSIONS=' in line: + versions = line.split('"')[1].split(' ') + if versions[0] == target_ver: + raise ValueError('Patch not needed') + + if target_ver in versions: + versions.remove(target_ver) + + versions.insert(0, target_ver) + + yield 'OPENSSL_VERSIONS="{}"\n'.format(' '.join(versions)) + else: + yield line + + def patch_dev_defaults(self, ssl_recipe): + def_fname = join(self.ctx.ndk_dir, 'build', 'tools', 'dev-defaults.sh') + try: + with open(def_fname, 'r') as fp: + s = ''.join(self._patch_dev_defaults(fp, + str(ssl_recipe.version))) + with open(def_fname, 'w') as fp: + fp.write(s) + + except ValueError: + pass + + def check_for_sslso(self, ssl_recipe, arch): + # type: (Recipe, str) + dynlib_dir = join(self.ctx.ndk_dir, 'sources', 'python', self.version, + 'libs', arch.arch, 'modules') + + if os.path.exists(join(dynlib_dir, '_ssl.so')): + return 10, 'Shared object exists in ndk' + + # find out why _ssl.so is missing + + source_dir = join(self.ctx.ndk_dir, 'sources', 'openssl', ssl_recipe.version) + if not os.path.exists(source_dir): + return 0, 'Openssl version not present' + + # these two path checks are lifted straight from: + # crystax-ndk/build/tools/build-target-python.sh + if not os.path.exists(join(source_dir, 'Android.mk')): + return 1.1, 'Android.mk is missing in openssl source' + + include_dir = join(source_dir, 'include','openssl') + if not os.path.exists(join(include_dir, 'opensslconf.h')): + return 1.2, 'Openssl include dir missing' + + under_scored_arch = arch.arch.replace('-', '_') + if not os.path.lexists(join(include_dir, + 'opensslconf_{}.h'.format(under_scored_arch))): + return 1.3, 'Opensslconf arch header missing from include' + + + + # lastly a check to see whether shared objects for the correct arch + # is present in the ndk + if not os.path.exists(join(source_dir, 'libs', arch.arch)): + return 2, 'Openssl libs for this arch is missing in ndk' + + return 5, 'Ready to recompile python' + + def find_Android_mk(self): + openssl_dir = join(self.ctx.ndk_dir, 'sources', 'openssl') + for version in os.listdir(openssl_dir): + mk_path = join(openssl_dir, version, 'Android.mk') + if os.path.exists(mk_path): + return mk_path + + def prebuild_arch(self, arch): + super(Python3Recipe, self).prebuild_arch(arch) + if self.version == '3.6': + Python3Recipe.patches = ['patch_python3.6.patch'] + build_dir = self.get_build_dir(arch.arch) + shprint(sh.ln, '-sf', + realpath(join(build_dir, 'Lib/site-packages/README.txt')), + join(build_dir, 'Lib/site-packages/README')) + python_build_files = ['android.mk', 'config.c', 'interpreter.c'] + ndk_build_tools_python_dir = join(self.ctx.ndk_dir, 'build', 'tools', 'build-target-python') + for python_build_file in python_build_files: + shprint(sh.cp, join(ndk_build_tools_python_dir, python_build_file+'.3.5'), + join(ndk_build_tools_python_dir, python_build_file+'.3.6')) + ndk_sources_python_dir = join(self.ctx.ndk_dir, 'sources', 'python') + if not os.path.exists(join(ndk_sources_python_dir, '3.6')): + os.mkdir(join(ndk_sources_python_dir, '3.6')) + sh.sed('s#3.5#3.6#', + join(ndk_sources_python_dir, '3.5/Android.mk'), + _out=join(ndk_sources_python_dir, '3.6/Android.mk')) + + def build_arch(self, arch): + # If openssl is needed we may have to recompile cPython to get the + # ssl.py module working properly + if self.from_crystax and 'openssl' in self.ctx.recipe_build_order: + info('Openssl and crystax-python combination may require ' + 'recompilation of python...') + ssl_recipe = self.get_recipe('openssl', self.ctx) + stage, msg = self.check_for_sslso(ssl_recipe, arch) + stage = 0 if stage < 5 else stage + info(msg) + openssl_build_dir = ssl_recipe.get_build_dir(arch.arch) + openssl_ndk_dir = join(self.ctx.ndk_dir, 'sources', 'openssl', + ssl_recipe.version) + + if stage < 2: + info('Copying openssl headers and Android.mk to ndk') + ensure_dir(openssl_ndk_dir) + if stage < 1.2: + # copy include folder and Android.mk to ndk + mk_path = self.find_Android_mk() + if mk_path is None: + raise IOError('Android.mk file could not be found in ' + 'any versions in ndk->sources->openssl') + shprint(sh.cp, mk_path, openssl_ndk_dir) + + include_dir = join(openssl_build_dir, 'include') + if stage < 1.3: + ndk_include_dir = join(openssl_ndk_dir, 'include', 'openssl') + self.copy_include_dir(join(include_dir, 'openssl'), ndk_include_dir) + + target_conf = join(openssl_ndk_dir, 'include', 'openssl', + 'opensslconf.h') + shprint(sh.rm, '-f', target_conf) + # overwrite opensslconf.h + with open(target_conf, 'w') as fp: + fp.write(OPENSSLCONF) + + if stage < 1.4: + # move current conf to arch specific conf in ndk + under_scored_arch = arch.arch.replace('-', '_') + shprint(sh.ln, '-sf', + realpath(join(include_dir, 'openssl', 'opensslconf.h')), + join(openssl_ndk_dir, 'include', 'openssl', + 'opensslconf_{}.h'.format(under_scored_arch)) + ) + + if stage < 3: + info('Copying openssl libs to ndk') + arch_ndk_lib = join(openssl_ndk_dir, 'libs', arch.arch) + ensure_dir(arch_ndk_lib) + shprint(sh.ln, '-sf', + realpath(join(openssl_build_dir, 'libcrypto{}.so'.format(ssl_recipe.version))), + join(openssl_build_dir, 'libcrypto.so')) + shprint(sh.ln, '-sf', + realpath(join(openssl_build_dir, 'libssl{}.so'.format(ssl_recipe.version))), + join(openssl_build_dir, 'libssl.so')) + libs = ['libcrypto.a', 'libcrypto.so', 'libssl.a', 'libssl.so'] + cmd = [join(openssl_build_dir, lib) for lib in libs] + [arch_ndk_lib] + shprint(sh.cp, '-f', *cmd) + + if stage < 10: + info('Recompiling python-crystax') + self.patch_dev_defaults(ssl_recipe) + build_script = join(self.ctx.ndk_dir, 'build', 'tools', + 'build-target-python.sh') + + shprint(Command(build_script), + '--ndk-dir={}'.format(self.ctx.ndk_dir), + '--abis={}'.format(arch.arch), + '-j5', '--verbose', + self.get_build_dir(arch.arch)) + + info('Extracting CrystaX python3 from NDK package') + dirn = self.ctx.get_python_install_dir() + ensure_dir(dirn) + self.ctx.hostpython = 'python{}'.format(self.version) + +recipe = Python3Recipe() diff --git a/recipes/python3crystax/patch_python3.6.patch b/recipes/python3crystax/patch_python3.6.patch new file mode 100644 index 00000000..4402d1bd --- /dev/null +++ b/recipes/python3crystax/patch_python3.6.patch @@ -0,0 +1,89 @@ +diff --git a/Modules/expat/xmlparse.c b/Modules/expat/xmlparse.c +--- a/Modules/expat/xmlparse.c ++++ b/Modules/expat/xmlparse.c +@@ -84,6 +84,8 @@ + # define LOAD_LIBRARY_SEARCH_SYSTEM32 0x00000800 + #endif + ++#define XML_POOR_ENTROPY 1 ++ + #if !defined(HAVE_GETRANDOM) && !defined(HAVE_SYSCALL_GETRANDOM) \ + && !defined(HAVE_ARC4RANDOM_BUF) && !defined(HAVE_ARC4RANDOM) \ + && !defined(XML_DEV_URANDOM) \ +diff --git a/Modules/getpath.c b/Modules/getpath.c +--- a/Modules/getpath.c ++++ b/Modules/getpath.c +@@ -101,8 +101,35 @@ + #endif + + +-#if !defined(PREFIX) || !defined(EXEC_PREFIX) || !defined(VERSION) || !defined(VPATH) +-#error "PREFIX, EXEC_PREFIX, VERSION, and VPATH must be constant defined" ++ /* These variables were set this way in old versions of Python, but ++ changed somewhere between 3.5.0 and 3.5.3. Here we just force ++ the old way again. A better solution would be to work out where ++ they should be defined, and make the CrystaX build scripts do ++ so. */ ++ ++/* #if !defined(PREFIX) || !defined(EXEC_PREFIX) || !defined(VERSION) || !defined(VPATH) */ ++/* #error "PREFIX, EXEC_PREFIX, VERSION, and VPATH must be constant defined" */ ++/* #endif */ ++ ++#ifndef VERSION ++#define VERSION "2.1" ++#endif ++ ++#ifndef VPATH ++#define VPATH "." ++#endif ++ ++#ifndef PREFIX ++# define PREFIX "/usr/local" ++#endif ++ ++#ifndef EXEC_PREFIX ++#define EXEC_PREFIX PREFIX ++#endif ++ ++#ifndef PYTHONPATH ++#define PYTHONPATH PREFIX "/lib/python" VERSION ":" \ ++ EXEC_PREFIX "/lib/python" VERSION "/lib-dynload" + #endif + + #ifndef LANDMARK +diff --git a/Modules/timemodule.c b/Modules/timemodule.c +--- a/Modules/timemodule.c ++++ b/Modules/timemodule.c +@@ -358,18 +358,20 @@ time_gmtime(PyObject *self, PyObject *args) + #endif + } + +-#ifndef HAVE_TIMEGM +-static time_t +-timegm(struct tm *p) +-{ +- /* XXX: the following implementation will not work for tm_year < 1970. +- but it is likely that platforms that don't have timegm do not support +- negative timestamps anyways. */ +- return p->tm_sec + p->tm_min*60 + p->tm_hour*3600 + p->tm_yday*86400 + +- (p->tm_year-70)*31536000 + ((p->tm_year-69)/4)*86400 - +- ((p->tm_year-1)/100)*86400 + ((p->tm_year+299)/400)*86400; +-} +-#endif ++/* In the Android build, HAVE_TIMEGM apparently should be defined but isn't. A better fix would be to work out why and fix that. */ ++ ++/* #ifndef HAVE_TIMEGM */ ++/* static time_t */ ++/* timegm(struct tm *p) */ ++/* { */ ++/* /\* XXX: the following implementation will not work for tm_year < 1970. */ ++/* but it is likely that platforms that don't have timegm do not support */ ++/* negative timestamps anyways. *\/ */ ++/* return p->tm_sec + p->tm_min*60 + p->tm_hour*3600 + p->tm_yday*86400 + */ ++/* (p->tm_year-70)*31536000 + ((p->tm_year-69)/4)*86400 - */ ++/* ((p->tm_year-1)/100)*86400 + ((p->tm_year+299)/400)*86400; */ ++/* } */ ++/* #endif */ + + PyDoc_STRVAR(gmtime_doc, + "gmtime([seconds]) -> (tm_year, tm_mon, tm_mday, tm_hour, tm_min,\n\ diff --git a/recipes/requests/__init__.py b/recipes/requests/__init__.py new file mode 100644 index 00000000..f7a0290f --- /dev/null +++ b/recipes/requests/__init__.py @@ -0,0 +1,12 @@ +from pythonforandroid.recipe import PythonRecipe + + +class RequestsRecipe(PythonRecipe): + version = '2.13.0' + url = 'https://github.com/kennethreitz/requests/archive/v{version}.tar.gz' + depends = [('hostpython2', 'hostpython3crystax'), 'setuptools'] + site_packages_name = 'requests' + call_hostpython_via_targetpython = False + + +recipe = RequestsRecipe() \ No newline at end of file diff --git a/recipes/setuptools/__init__.py b/recipes/setuptools/__init__.py index c67d434a..21e89b17 100644 --- a/recipes/setuptools/__init__.py +++ b/recipes/setuptools/__init__.py @@ -1,23 +1,14 @@ - -from pythonforandroid.toolchain import ( - PythonRecipe, - Recipe, - current_directory, - info, - shprint, -) -from os.path import join -import sh +from pythonforandroid.recipe import PythonRecipe class SetuptoolsRecipe(PythonRecipe): - version = '18.5' - url = 'https://pypi.python.org/packages/source/s/setuptools/setuptools-{version}.tar.gz' + version = '40.0.0' + url = 'https://pypi.python.org/packages/source/s/setuptools/setuptools-{version}.zip' - depends = ['python2'] + depends = [('python2', 'python3crystax')] call_hostpython_via_targetpython = False install_in_hostpython = True -recipe = SetuptoolsRecipe() +recipe = SetuptoolsRecipe() \ No newline at end of file diff --git a/recipes/six/__init__.py b/recipes/six/__init__.py index e03eeb05..92b266f4 100644 --- a/recipes/six/__init__.py +++ b/recipes/six/__init__.py @@ -6,6 +6,5 @@ class SixRecipe(PythonRecipe): version = '1.9.0' url = 'https://pypi.python.org/packages/source/s/six/six-{version}.tar.gz' depends = [('python2', 'python3crystax')] - call_hostpython_via_targetpython = False recipe = SixRecipe() diff --git a/recipes/twisted/__init__.py b/recipes/twisted/__init__.py index 753a107f..852291ac 100644 --- a/recipes/twisted/__init__.py +++ b/recipes/twisted/__init__.py @@ -12,10 +12,10 @@ import sh class TwistedRecipe(CythonRecipe): - version = '16.6.0' + version = '18.7.0' url = 'https://github.com/twisted/twisted/archive/twisted-{version}.tar.gz' - depends = ['setuptools', 'zope_interface'] + depends = ['setuptools', 'zope_interface', 'incremental', 'constantly'] call_hostpython_via_targetpython = False install_in_hostpython = True @@ -29,10 +29,10 @@ class TwistedRecipe(CythonRecipe): env = super(TwistedRecipe, self).get_recipe_env(arch) # TODO: Move this and others to base Recipe class for Cython and CompiledComponent recipes - target_python = Recipe.get_recipe('python2', self.ctx).get_build_dir(arch.arch) + target_python = Recipe.get_recipe('python3crystax', self.ctx).get_build_dir(arch.arch) env['PYTHON_ROOT'] = join(target_python, 'python-install') - env['CFLAGS'] += ' -I' + env['PYTHON_ROOT'] + '/include/python2.7' - env['LDFLAGS'] += ' -L' + env['PYTHON_ROOT'] + '/lib' + ' -lpython2.7' + env['CFLAGS'] += ' -I' + env['PYTHON_ROOT'] + '/include/python3.6' + env['LDFLAGS'] += ' -L' + env['PYTHON_ROOT'] + '/lib' + ' -lpython3.6m' # We add BUILDLIB_PATH to PYTHONPATH so twisted can find _io.so env['PYTHONPATH'] = ':'.join([ diff --git a/recipes/unqlite/__init__.py b/recipes/unqlite/__init__.py index a47be77a..13306caf 100644 --- a/recipes/unqlite/__init__.py +++ b/recipes/unqlite/__init__.py @@ -13,20 +13,20 @@ import sh class UnqliteRecipe(CythonRecipe): version = '0.6.0' url = 'https://pypi.python.org/packages/cb/4e/e1f64a3d0f6462167805940b4c72f47bafc1129e363fc4c0f79a1cdc5dd1/unqlite-{version}.tar.gz' - depends = ['python2', 'setuptools'] + depends = [('python2', 'python3crystax'), 'setuptools'] call_hostpython_via_targetpython = False patches = ['setup.patch'] - + def get_recipe_env(self, arch): env = super(UnqliteRecipe, self).get_recipe_env(arch) - target_python = Recipe.get_recipe('python2', self.ctx).get_build_dir(arch.arch) + target_python = Recipe.get_recipe('python3crystax', self.ctx).get_build_dir(arch.arch) env['PYTHON_ROOT'] = join(target_python, 'python-install') - env['CFLAGS'] += ' -I' + env['PYTHON_ROOT'] + '/include/python2.7' - env['LDFLAGS'] += ' -L' + env['PYTHON_ROOT'] + '/lib' + ' -lpython2.7' - + env['CFLAGS'] += ' -I' + env['PYTHON_ROOT'] + '/include/python3.6' + env['LDFLAGS'] += ' -L' + env['PYTHON_ROOT'] + '/lib' + ' -lpython3.6m' + return env - + recipe = UnqliteRecipe() diff --git a/recipes/zope_interface/__init__.py b/recipes/zope_interface/__init__.py new file mode 100644 index 00000000..7684ca91 --- /dev/null +++ b/recipes/zope_interface/__init__.py @@ -0,0 +1,21 @@ +from pythonforandroid.recipe import PythonRecipe +from pythonforandroid.toolchain import current_directory +import sh + + +class ZopeInterfaceRecipe(PythonRecipe): + name = 'zope_interface' + version = '4.1.3' + url = 'https://pypi.python.org/packages/source/z/zope.interface/zope.interface-{version}.tar.gz' + site_packages_name = 'zope.interface' + + depends = [('python2', 'python3crystax')] + #patches = ['no_tests.patch'] + + #def prebuild_arch(self, arch): + #super(ZopeInterfaceRecipe, self).prebuild_arch(arch) + #with current_directory(self.get_build_dir(arch.arch)): + # sh.rm('-rf', 'src/zope/interface/tests', 'src/zope/interface/common/tests') + + +recipe = ZopeInterfaceRecipe() \ No newline at end of file diff --git a/recipes/zope_interface/no_tests.patch b/recipes/zope_interface/no_tests.patch new file mode 100644 index 00000000..81a0e09e --- /dev/null +++ b/recipes/zope_interface/no_tests.patch @@ -0,0 +1,13 @@ +--- zope_interface/setup.py 2015-10-05 09:35:14.000000000 +0200 ++++ b/setup.py 2016-06-15 17:44:35.108263993 +0200 +@@ -139,9 +139,8 @@ + "Topic :: Software Development :: Libraries :: Python Modules", + ], + +- packages = ['zope', 'zope.interface', 'zope.interface.tests'], ++ packages = ['zope', 'zope.interface'], + package_dir = {'': 'src'}, + cmdclass = {'build_ext': optional_build_ext, + }, +- test_suite = 'zope.interface.tests', + **extra) \ No newline at end of file diff --git a/scripts/build-target-python.sh b/scripts/build-target-python.sh new file mode 100755 index 00000000..817b7522 --- /dev/null +++ b/scripts/build-target-python.sh @@ -0,0 +1,859 @@ +#!/bin/bash + +# Copyright (c) 2011-2015 CrystaX. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without modification, are +# permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, this list of +# conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, this list +# of conditions and the following disclaimer in the documentation and/or other materials +# provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY CrystaX ''AS IS'' AND ANY EXPRESS OR IMPLIED +# WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND +# FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL CrystaX OR +# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +# ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF +# ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +# The views and conclusions contained in the software and documentation are those of the +# authors and should not be interpreted as representing official policies, either expressed +# or implied, of CrystaX. + +# include common function and variable definitions +. `dirname $0`/prebuilt-common.sh + +PROGRAM_PARAMETERS="" + +PROGRAM_DESCRIPTION=\ +"Rebuild python libraries for the CrystaX NDK. + +This requires a temporary NDK installation containing +toolchain binaries for all target architectures. + +By default, this will try with the current NDK directory, unless +you use the --ndk-dir= option. + +The output will be placed in appropriate sub-directories of +/$PYTHON_SUBDIR, but you can override this with the --out-dir= +option. +" + +PACKAGE_DIR= +register_var_option "--package-dir=" PACKAGE_DIR "Put prebuilt tarballs into " + +NDK_DIR=$ANDROID_NDK_ROOT +register_var_option "--ndk-dir=" NDK_DIR "Specify NDK root path for the build" + +BUILD_DIR= +OPTION_BUILD_DIR= +register_var_option "--build-dir=" OPTION_BUILD_DIR "Specify temporary build dir" + +ABIS=$PREBUILT_ABIS +register_var_option "--abis=" ABIS "Specify list of target ABIs" + +register_jobs_option + +extract_parameters "$@" + +PYTHON_SRCDIR=$(echo $PARAMETERS | sed 1q) +if [ -z "$PYTHON_SRCDIR" ]; then + echo "ERROR: Please provide the path to the python source tree. See --help" + exit 1 +fi + +if [ ! -d "$PYTHON_SRCDIR" ]; then + echo "ERROR: No such directory: '$PYTHON_SRCDIR'" + exit 1 +fi + +PYTHON_SRCDIR=$(cd $PYTHON_SRCDIR && pwd) + +PYTHON_MAJOR_VERSION=\ +$(cat $PYTHON_SRCDIR/Include/patchlevel.h | sed -n 's/#define[ \t]*PY_MAJOR_VERSION[ \t]*\([0-9]*\).*/\1/p') + +PYTHON_MINOR_VERSION=\ +$(cat $PYTHON_SRCDIR/Include/patchlevel.h | sed -n 's/#define[ \t]*PY_MINOR_VERSION[ \t]*\([0-9]*\).*/\1/p') + +if [ -z "$PYTHON_MAJOR_VERSION" ]; then + echo "ERROR: Can't detect python major version." 1>&2 + exit 1 +fi + +if [ -z "$PYTHON_MINOR_VERSION" ]; then + echo "ERROR: Can't detect python minor version." 1>&2 + exit 1 +fi + +PYTHON_ABI="$PYTHON_MAJOR_VERSION"'.'"$PYTHON_MINOR_VERSION" +PYTHON_DSTDIR=$NDK_DIR/$PYTHON_SUBDIR/$PYTHON_ABI +mkdir -p $PYTHON_DSTDIR +fail_panic "Can't create python destination directory: $PYTHON_DSTDIR" + +PYTHON_BUILD_UTILS_DIR=$(cd $(dirname $0)/build-target-python && pwd) +if [ ! -d "$PYTHON_BUILD_UTILS_DIR" ]; then + echo "ERROR: No such directory: '$PYTHON_BUILD_UTILS_DIR'" + exit 1 +fi + +PY_C_CONFIG_FILE="$PYTHON_BUILD_UTILS_DIR/config.c.$PYTHON_ABI" +if [ ! -f "$PY_C_CONFIG_FILE" ]; then + echo "ERROR: Build of python $PYTHON_ABI is not supported, no such file: $PY_C_CONFIG_FILE" + exit 1 +fi + +PY_C_INTERPRETER_FILE="$PYTHON_BUILD_UTILS_DIR/interpreter.c.$PYTHON_ABI" +if [ ! -f "$PY_C_INTERPRETER_FILE" ]; then + echo "ERROR: Build of python $PYTHON_ABI is not supported, no such file: $PY_C_INTERPRETER_FILE" + exit 1 +fi + +PY_ANDROID_MK_TEMPLATE_FILE="$PYTHON_BUILD_UTILS_DIR/android.mk.$PYTHON_ABI" +if [ ! -f "$PY_ANDROID_MK_TEMPLATE_FILE" ]; then + echo "ERROR: Build of python $PYTHON_ABI is not supported, no such file: $PY_ANDROID_MK_TEMPLATE_FILE" + exit 1 +fi + +ABIS=$(commas_to_spaces $ABIS) + +if [ -z "$OPTION_BUILD_DIR" ]; then + BUILD_DIR=$NDK_TMPDIR/build-python +else + eval BUILD_DIR=$OPTION_BUILD_DIR +fi + +rm -rf "$BUILD_DIR" +mkdir -p "$BUILD_DIR" +fail_panic "Can't create build directory: $BUILD_DIR" + +OPENSSL_HOME='' +if [ -n "$DEFAULT_OPENSSL_VERSION" ]; then + if [ -f "$NDK_DIR/$OPENSSL_SUBDIR/$DEFAULT_OPENSSL_VERSION/Android.mk" \ + -a -f "$NDK_DIR/$OPENSSL_SUBDIR/$DEFAULT_OPENSSL_VERSION/include/openssl/opensslconf.h" ]; then + OPENSSL_HOME="openssl/$DEFAULT_OPENSSL_VERSION" + fi +fi + +# $1: ABI +# $2: build directory +build_python_for_abi () +{ + local ABI="$1" + local BUILDDIR="$2" + local PYBIN_INSTALLDIR=$PYTHON_DSTDIR/libs/$ABI + local PYBIN_INSTALLDIR_MODULES="$PYBIN_INSTALLDIR/modules" + if [ -n "$OPENSSL_HOME" ]; then + log "Building python$PYTHON_ABI for $ABI (with OpenSSL-$DEFAULT_OPENSSL_VERSION)" + else + log "Building python$PYTHON_ABI for $ABI (without OpenSSL support)" + fi + +# Step 1: configure + local BUILDDIR_CONFIG="$BUILDDIR/config" + local BUILDDIR_CORE="$BUILDDIR/core" + local OBJDIR_CORE="$BUILDDIR_CORE/obj/local/$ABI" + + run mkdir -p $BUILDDIR_CONFIG + fail_panic "Can't create directory: $BUILDDIR_CONFIG" + run mkdir -p $BUILDDIR_CORE + fail_panic "Can't create directory: $BUILDDIR_CORE" + + local BUILD_ON_PLATFORM=$($PYTHON_SRCDIR/config.guess) + if [ -z "$BUILD_ON_PLATFORM" ]; then + echo "ERROR: Can't resolve platform being built python on." 1>&2 + exit 1 + fi + + local ARCH + case $ABI in + armeabi*) + ARCH=arm + ;; + arm64*) + ARCH=arm64 + ;; + x86|x86_64|mips|mips64) + ARCH=$ABI + ;; + *) + echo "ERROR: Unknown ABI: '$ABI'" 1>&2 + exit 1 + esac + + local HOST + case $ABI in + armeabi*) + HOST=arm-linux-androideabi + ;; + arm64*) + HOST=aarch64-linux-android + ;; + x86) + HOST=i686-linux-android + ;; + x86_64) + HOST=x86_64-linux-android + ;; + mips) + HOST=mipsel-linux-android + ;; + mips64) + HOST=mips64el-linux-android + ;; + *) + echo "ERROR: Unknown ABI: '$ABI'" 1>&2 + exit 1 + esac + + local APILEVEL + case $ABI in + armeabi*|x86|mips) + APILEVEL=21 + ;; + arm64*|x86_64|mips64) + APILEVEL=21 + ;; + *) + echo "ERROR: Unknown ABI: '$ABI'" 1>&2 + exit 1 + esac + + local TOOLCHAIN + case $ABI in + armeabi*) + TOOLCHAIN=arm-linux-androideabi + ;; + x86) + TOOLCHAIN=x86 + ;; + mips) + TOOLCHAIN=mipsel-linux-android + ;; + arm64-v8a) + TOOLCHAIN=aarch64-linux-android + ;; + x86_64) + TOOLCHAIN=x86_64 + ;; + mips64) + TOOLCHAIN=mips64el-linux-android + ;; + *) + echo "ERROR: Unknown ABI: '$ABI'" 1>&2 + exit 1 + esac + + case $ABI in + armeabi) + CFLAGS="-march=armv5te -mtune=xscale -msoft-float" + ;; + armeabi-v7a) + CFLAGS="-march=armv7-a -mfpu=vfpv3-d16 -mfloat-abi=softfp" + ;; + armeabi-v7a-hard) + CFLAGS="-march=armv7-a -mfpu=vfpv3-d16 -mhard-float" + ;; + *) + CFLAGS="" + esac + + case $ABI in + armeabi*) + CFLAGS="$CFLAGS -mthumb" + esac + + local CFLAGS="$CFLAGS --sysroot=$NDK_DIR/platforms/android-$APILEVEL/arch-$ARCH" + + local LDFLAGS="" + if [ "$ABI" = "armeabi-v7a-hard" ]; then + LDFLAGS="$LDFLAGS -Wl,--no-warn-mismatch" + fi + LDFLAGS="$LDFLAGS -L$NDK_DIR/sources/crystax/libs/$ABI --sysroot=$NDK_DIR/platforms/android-$APILEVEL/arch-$ARCH" + + local TCPREFIX=$NDK_DIR/toolchains/${TOOLCHAIN}-4.9/prebuilt/$HOST_TAG + + + local CC=$TCPREFIX/bin/${HOST}-gcc + local CPP="$CC $CFLAGS -E" + local AR=$TCPREFIX/bin/${HOST}-ar + local RANLIB=$TCPREFIX/bin/${HOST}-ranlib + local READELF=$TCPREFIX/bin/${HOST}-readelf + local PYTHON_FOR_BUILD=$NDK_DIR/prebuilt/$HOST_TAG/bin/python + local CONFIG_SITE=$PYTHON_TOOLS_DIR/config.site + + local CONFIG_SITE=$BUILDDIR_CONFIG/config.site + { + echo 'ac_cv_file__dev_ptmx=no' + echo 'ac_cv_file__dev_ptc=no' + echo 'ac_cv_func_gethostbyname_r=no' + if [ "$PYTHON_MAJOR_VERSION" == "3" ]; then + echo 'ac_cv_func_faccessat=no' + fi + } >$CONFIG_SITE + fail_panic "Can't create config.site wrapper" + + local CONFIGURE_WRAPPER=$BUILDDIR_CONFIG/configure.sh + { + echo "#!/bin/bash -e" + echo '' + echo "export CC=\"$CC\"" + echo "export CPP=\"$CPP\"" + echo "export AR=\"$AR\"" + echo "export CFLAGS=\"$CFLAGS\"" + echo "export LDFLAGS=\"$LDFLAGS\"" + echo "export RANLIB=\"$RANLIB\"" + echo "export READELF=\"$READELF\"" + echo "export PYTHON_FOR_BUILD=\"$PYTHON_FOR_BUILD\"" + echo "export CONFIG_SITE=\"$CONFIG_SITE\"" + echo '' + echo 'cd $(dirname $0)' + echo '' + if [ "$PYTHON_MAJOR_VERSION" = "2" ]; then + echo "exec $PYTHON_SRCDIR/configure \\" + echo " --host=$HOST \\" + echo " --build=$BUILD_ON_PLATFORM \\" + echo " --prefix=$BUILDDIR_CONFIG/install \\" + echo " --enable-shared \\" + echo " --with-threads \\" + echo " --enable-ipv6 \\" + echo " --enable-unicode=ucs4 \\" + echo " --without-ensurepip" + else + echo "exec $PYTHON_SRCDIR/configure \\" + echo " --host=$HOST \\" + echo " --build=$BUILD_ON_PLATFORM \\" + echo " --prefix=$BUILDDIR_CONFIG/install \\" + echo " --enable-shared \\" + echo " --with-threads \\" + echo " --enable-ipv6 \\" + echo " --with-computed-gotos \\" + echo " --without-ensurepip" + fi + } >$CONFIGURE_WRAPPER + fail_panic "Can't create configure wrapper" + + chmod +x $CONFIGURE_WRAPPER + fail_panic "Can't chmod +x configure wrapper" + + run $CONFIGURE_WRAPPER + fail_panic "Can't configure python$PYTHON_ABI for $ABI" + +# Step 2: build python-core + run mkdir -p $BUILDDIR_CORE/jni + fail_panic "Can't create directory: $BUILDDIR_CORE/jni" + + run cp -p -T $PY_C_CONFIG_FILE "$BUILDDIR_CORE/jni/config.c" && \ + cp -p -t "$BUILDDIR_CORE/jni" "$PYTHON_BUILD_UTILS_DIR/pyconfig.h" + fail_panic "Can't copy config.c pyconfig.h to $BUILDDIR_CORE/jni" + + if [ "$PYTHON_MAJOR_VERSION" = "2" ]; then + local PY_C_GETPATH="$PYTHON_BUILD_UTILS_DIR/getpath.c.$PYTHON_ABI" + run cp -p -T $PY_C_GETPATH "$BUILDDIR_CORE/jni/getpath.c" + fail_panic "Can't copy $PY_C_GETPATH to $BUILDDIR_CORE/jni" + fi + + local PYCONFIG_FOR_ABI="$BUILDDIR_CORE/jni/pyconfig_$(echo $ABI | tr '-' '_').h" + run cp -p -T $BUILDDIR_CONFIG/pyconfig.h $PYCONFIG_FOR_ABI + fail_panic "Can't copy $BUILDDIR_CONFIG/pyconfig.h to $PYCONFIG_FOR_ABI" + + if [ "$PYTHON_MAJOR_VERSION" = "2" ]; then + local PYTHON_CORE_MODULE_NAME='python'"$PYTHON_ABI" + else + local PYTHON_CORE_MODULE_NAME='python'"$PYTHON_ABI"'m' + local PYTHON_SOABI='cpython-'"$PYTHON_ABI"'m' + fi + { + echo 'LOCAL_PATH := $(call my-dir)' + echo 'include $(CLEAR_VARS)' + echo "LOCAL_MODULE := $PYTHON_CORE_MODULE_NAME" + echo "MY_PYTHON_SRC_ROOT := $PYTHON_SRCDIR" + echo 'LOCAL_C_INCLUDES := $(MY_PYTHON_SRC_ROOT)/Include' + if [ "$PYTHON_MAJOR_VERSION" = "2" ]; then + echo "LOCAL_CFLAGS := -DPy_BUILD_CORE -DPy_ENABLE_SHARED -DPLATFORM=\\\"linux\\\"" + else + echo "LOCAL_CFLAGS := -DSOABI=\\\"$PYTHON_SOABI\\\" -DPy_BUILD_CORE -DPy_ENABLE_SHARED -DPLATFORM=\\\"linux\\\"" + fi + echo 'LOCAL_LDLIBS := -lz' + cat $PY_ANDROID_MK_TEMPLATE_FILE + echo 'include $(BUILD_SHARED_LIBRARY)' + } >$BUILDDIR_CORE/jni/Android.mk + fail_panic "Can't generate $BUILDDIR_CORE/jni/Android.mk" + + run $NDK_DIR/ndk-build -C $BUILDDIR_CORE -j$NUM_JOBS APP_ABI=$ABI V=1 + fail_panic "Can't build python$PYTHON_ABI-$ABI core" + + if [ "$PYTHON_HEADERS_INSTALLED" != "yes" ]; then + log "Install python$PYTHON_ABI headers into $PYTHON_DSTDIR" + run rm -Rf $PYTHON_DSTDIR/include + run mkdir -p $PYTHON_DSTDIR/include/python && \ + run cp -p $PYTHON_BUILD_UTILS_DIR/pyconfig.h $PYTHON_SRCDIR/Include/*.h $PYTHON_DSTDIR/include/python + fail_panic "Can't install python$PYTHON_ABI headers" + PYTHON_HEADERS_INSTALLED=yes + export PYTHON_HEADERS_INSTALLED + fi + log "Install $(basename $PYCONFIG_FOR_ABI) into $PYTHON_DSTDIR" + run cp -p $PYCONFIG_FOR_ABI $PYTHON_DSTDIR/include/python + fail_panic "Can't install $PYCONFIG_FOR_ABI" + + run mkdir -p $PYBIN_INSTALLDIR + fail_panic "Can't create $PYBIN_INSTALLDIR" + run mkdir -p $PYBIN_INSTALLDIR_MODULES + fail_panic "Can't create directory: $PYBIN_INSTALLDIR_MODULES" + + log "Install python$PYTHON_ABI-$ABI core in $PYBIN_INSTALLDIR" + run cp -fpH $OBJDIR_CORE/lib$PYTHON_CORE_MODULE_NAME.so $PYBIN_INSTALLDIR + fail_panic "Can't install python$PYTHON_ABI-$ABI core in $PYBIN_INSTALLDIR" + +# Step 3: build python-interpreter + local BUILDDIR_INTERPRETER="$BUILDDIR/interpreter" + local OBJDIR_INTERPRETER="$BUILDDIR_INTERPRETER/obj/local/$ABI" + + run mkdir -p $BUILDDIR_INTERPRETER/jni + fail_panic "Can't create directory: $BUILDDIR_INTERPRETER/jni" + + run cp -p -T $PY_C_INTERPRETER_FILE $BUILDDIR_INTERPRETER/jni/interpreter.c + fail_panic "Can't copy $PY_C_INTERPRETER_FILE to $BUILDDIR_INTERPRETER/jni" + + { + echo 'LOCAL_PATH := $(call my-dir)' + echo 'include $(CLEAR_VARS)' + echo 'LOCAL_MODULE := python' + echo 'LOCAL_SRC_FILES := interpreter.c' + echo 'include $(BUILD_EXECUTABLE)' + } >$BUILDDIR_INTERPRETER/jni/Android.mk + fail_panic "Can't generate $BUILDDIR_INTERPRETER/jni/Android.mk" + + run $NDK_DIR/ndk-build -C $BUILDDIR_INTERPRETER -j$NUM_JOBS APP_ABI=$ABI V=1 + fail_panic "Can't build python$PYTHON_ABI-$ABI interpreter" + + log "Install python$PYTHON_ABI-$ABI interpreter in $PYBIN_INSTALLDIR" + run cp -fpH $OBJDIR_INTERPRETER/python $PYBIN_INSTALLDIR + fail_panic "Can't install python$PYTHON_ABI-$ABI interpreter in $PYBIN_INSTALLDIR" + +# Step 4: build python stdlib + local PYSTDLIB_ZIPFILE="$PYBIN_INSTALLDIR/stdlib.zip" + log "Install python$PYTHON_ABI-$ABI stdlib as $PYSTDLIB_ZIPFILE" + if [ "$PYTHON_MAJOR_VERSION" = "2" ]; then + run $PYTHON_FOR_BUILD $PYTHON_BUILD_UTILS_DIR/build_stdlib.py --py2 --pysrc-root $PYTHON_SRCDIR --output-zip $PYSTDLIB_ZIPFILE + fail_panic "Can't install python$PYTHON_ABI-$ABI stdlib" + else + run $PYTHON_FOR_BUILD $PYTHON_BUILD_UTILS_DIR/build_stdlib.py --pysrc-root $PYTHON_SRCDIR --output-zip $PYSTDLIB_ZIPFILE + fail_panic "Can't install python$PYTHON_ABI-$ABI stdlib" + fi + +# Step 5: site-packages + local SITE_README_SRCDIR="$PYTHON_SRCDIR/Lib/site-packages" + local SITE_README_DSTDIR="$PYBIN_INSTALLDIR/site-packages" + log "Install python$PYTHON_ABI-$ABI site-packages" + run mkdir -p $SITE_README_DSTDIR && cp -fpH $SITE_README_SRCDIR/README $SITE_README_DSTDIR + fail_panic "Can't install python$PYTHON_ABI-$ABI site-packages" + +# Step 6: build python modules +# _ctypes + local BUILDDIR_CTYPES="$BUILDDIR/ctypes" + local OBJDIR_CTYPES="$BUILDDIR_CTYPES/obj/local/$ABI" + local BUILDDIR_CTYPES_CONFIG="$BUILDDIR_CTYPES/config" + run mkdir -p $BUILDDIR_CTYPES_CONFIG + fail_panic "Can't create directory: $BUILDDIR_CTYPES_CONFIG" + + local LIBFFI_CONFIGURE_WRAPPER=$BUILDDIR_CTYPES_CONFIG/configure.sh + { + echo "#!/bin/bash -e" + echo '' + echo "export CC=\"$CC\"" + echo "export CFLAGS=\"$CFLAGS\"" + echo "export LDFLAGS=\"$LDFLAGS\"" + echo "export CPP=\"$CPP\"" + echo "export AR=\"$AR\"" + echo "export RANLIB=\"$RANLIB\"" + echo '' + echo 'cd $(dirname $0)' + echo '' + echo "exec $PYTHON_SRCDIR/Modules/_ctypes/libffi/configure \\" + echo " --host=$HOST \\" + echo " --build=$BUILD_ON_PLATFORM \\" + echo " --prefix=$BUILDDIR_CTYPES_CONFIG/install \\" + } >$LIBFFI_CONFIGURE_WRAPPER + fail_panic "Can't create configure wrapper for libffi" + + chmod +x $LIBFFI_CONFIGURE_WRAPPER + fail_panic "Can't chmod +x configure wrapper for libffi" + + run $LIBFFI_CONFIGURE_WRAPPER + fail_panic "Can't configure libffi for $ABI" + + run mkdir -p "$BUILDDIR_CTYPES/jni" + fail_panic "Can't create directory: $BUILDDIR_CTYPES/jni" + + run mkdir -p "$BUILDDIR_CTYPES/jni/include" + fail_panic "Can't create directory: $BUILDDIR_CTYPES/jni/include" + + run cp -p $BUILDDIR_CTYPES_CONFIG/fficonfig.h $BUILDDIR_CTYPES_CONFIG/include/*.h $BUILDDIR_CTYPES/jni/include + fail_panic "Can't copy configured libffi headers" + + local FFI_SRC_LIST + case $ABI in + x86) + FFI_SRC_LIST="src/x86/ffi.c src/x86/sysv.S src/x86/win32.S" + ;; + x86_64) + FFI_SRC_LIST="src/x86/ffi64.c src/x86/unix64.S" + ;; + armeabi*) + FFI_SRC_LIST="src/arm/ffi.c src/arm/sysv.S" + ;; + arm64-v8a) + FFI_SRC_LIST="src/aarch64/ffi.c src/aarch64/sysv.S" + ;; + mips) + FFI_SRC_LIST="src/mips/ffi.c src/mips/o32.S" + ;; + mips64) + FFI_SRC_LIST="src/mips/ffi.c src/mips/o32.S src/mips/n32.S" + ;; + *) + echo "ERROR: Unknown ABI: '$ABI'" 1>&2 + exit 1 + esac + FFI_SRC_LIST="$FFI_SRC_LIST src/prep_cif.c" + { + echo 'LOCAL_PATH := $(call my-dir)' + echo 'include $(CLEAR_VARS)' + echo 'LOCAL_MODULE := _ctypes' + echo 'LOCAL_C_INCLUDES := $(LOCAL_PATH)/include' + echo "MY_PYTHON_SRC_ROOT := $PYTHON_SRCDIR" + echo 'LOCAL_SRC_FILES := \' + for ffi_src in $FFI_SRC_LIST; do + echo " \$(MY_PYTHON_SRC_ROOT)/Modules/_ctypes/libffi/$ffi_src \\" + done + echo ' $(MY_PYTHON_SRC_ROOT)/Modules/_ctypes/callbacks.c \' + echo ' $(MY_PYTHON_SRC_ROOT)/Modules/_ctypes/callproc.c \' + echo ' $(MY_PYTHON_SRC_ROOT)/Modules/_ctypes/cfield.c \' + echo ' $(MY_PYTHON_SRC_ROOT)/Modules/_ctypes/malloc_closure.c \' + echo ' $(MY_PYTHON_SRC_ROOT)/Modules/_ctypes/stgdict.c \' + echo ' $(MY_PYTHON_SRC_ROOT)/Modules/_ctypes/_ctypes.c' + echo 'LOCAL_STATIC_LIBRARIES := python_shared' + echo 'include $(BUILD_SHARED_LIBRARY)' + echo "\$(call import-module,python/$PYTHON_ABI)" + } >$BUILDDIR_CTYPES/jni/Android.mk + fail_panic "Can't generate $BUILDDIR_CTYPES/jni/Android.mk" + + run $NDK_DIR/ndk-build -C $BUILDDIR_CTYPES -j$NUM_JOBS APP_ABI=$ABI V=1 + fail_panic "Can't build python$PYTHON_ABI-$ABI module '_ctypes'" + + log "Install python$PYTHON_ABI-$ABI module '_ctypes' in $PYBIN_INSTALLDIR_MODULES" + run cp -p -T $OBJDIR_CTYPES/lib_ctypes.so $PYBIN_INSTALLDIR_MODULES/_ctypes.so + fail_panic "Can't install python$PYTHON_ABI-$ABI module '_ctypes' in $PYBIN_INSTALLDIR_MODULES" + +# _multiprocessing + local BUILDDIR_MULTIPROCESSING="$BUILDDIR/multiprocessing" + local OBJDIR_MULTIPROCESSING="$BUILDDIR_MULTIPROCESSING/obj/local/$ABI" + + run mkdir -p "$BUILDDIR_MULTIPROCESSING/jni" + fail_panic "Can't create directory: $BUILDDIR_MULTIPROCESSING/jni" + + { + echo 'LOCAL_PATH := $(call my-dir)' + echo 'include $(CLEAR_VARS)' + echo 'LOCAL_MODULE := _multiprocessing' + echo "MY_PYTHON_SRC_ROOT := $PYTHON_SRCDIR" + echo 'LOCAL_SRC_FILES := \' + if [ "$PYTHON_MAJOR_VERSION" = "2" ]; then + echo ' $(MY_PYTHON_SRC_ROOT)/Modules/_multiprocessing/socket_connection.c \' + fi + echo ' $(MY_PYTHON_SRC_ROOT)/Modules/_multiprocessing/multiprocessing.c \' + echo ' $(MY_PYTHON_SRC_ROOT)/Modules/_multiprocessing/semaphore.c' + echo 'LOCAL_STATIC_LIBRARIES := python_shared' + echo 'include $(BUILD_SHARED_LIBRARY)' + echo "\$(call import-module,python/$PYTHON_ABI)" + } >$BUILDDIR_MULTIPROCESSING/jni/Android.mk + fail_panic "Can't generate $BUILDDIR_MULTIPROCESSING/jni/Android.mk" + + run $NDK_DIR/ndk-build -C $BUILDDIR_MULTIPROCESSING -j$NUM_JOBS APP_ABI=$ABI V=1 + fail_panic "Can't build python$PYTHON_ABI-$ABI module '_multiprocessing'" + + log "Install python$PYTHON_ABI-$ABI module '_multiprocessing' in $PYBIN_INSTALLDIR_MODULES" + run cp -p -T $OBJDIR_MULTIPROCESSING/lib_multiprocessing.so $PYBIN_INSTALLDIR_MODULES/_multiprocessing.so + fail_panic "Can't install python$PYTHON_ABI-$ABI module '_multiprocessing' in $PYBIN_INSTALLDIR_MODULES" + +# _socket + local BUILDDIR_SOCKET="$BUILDDIR/socket" + local OBJDIR_SOCKET="$BUILDDIR_SOCKET/obj/local/$ABI" + + run mkdir -p "$BUILDDIR_SOCKET/jni" + fail_panic "Can't create directory: $BUILDDIR_SOCKET/jni" + + { + echo 'LOCAL_PATH := $(call my-dir)' + echo 'include $(CLEAR_VARS)' + echo 'LOCAL_MODULE := _socket' + echo "MY_PYTHON_SRC_ROOT := $PYTHON_SRCDIR" + echo 'LOCAL_SRC_FILES := \' + echo ' $(MY_PYTHON_SRC_ROOT)/Modules/socketmodule.c' + echo 'LOCAL_STATIC_LIBRARIES := python_shared' + echo 'include $(BUILD_SHARED_LIBRARY)' + echo "\$(call import-module,python/$PYTHON_ABI)" + } >$BUILDDIR_SOCKET/jni/Android.mk + fail_panic "Can't generate $BUILDDIR_SOCKET/jni/Android.mk" + + run $NDK_DIR/ndk-build -C $BUILDDIR_SOCKET -j$NUM_JOBS APP_ABI=$ABI V=1 + fail_panic "Can't build python$PYTHON_ABI-$ABI module '_socket'" + + log "Install python$PYTHON_ABI-$ABI module '_socket' in $PYBIN_INSTALLDIR_MODULES" + run cp -p -T $OBJDIR_SOCKET/lib_socket.so $PYBIN_INSTALLDIR_MODULES/_socket.so + fail_panic "Can't install python$PYTHON_ABI-$ABI module '_socket' in $PYBIN_INSTALLDIR_MODULES" + +#_ssl + if [ -n "$OPENSSL_HOME" ]; then + local BUILDDIR_SSL="$BUILDDIR/ssl" + local OBJDIR_SSL="$BUILDDIR_SSL/obj/local/$ABI" + + run mkdir -p "$BUILDDIR_SSL/jni" + fail_panic "Can't create directory: $BUILDDIR_SSL/jni" + + { + echo 'LOCAL_PATH := $(call my-dir)' + echo 'include $(CLEAR_VARS)' + echo 'LOCAL_MODULE := _ssl' + echo "MY_PYTHON_SRC_ROOT := $PYTHON_SRCDIR" + echo 'LOCAL_SRC_FILES := \' + echo ' $(MY_PYTHON_SRC_ROOT)/Modules/_ssl.c' + echo 'LOCAL_STATIC_LIBRARIES := python_shared openssl_static opencrypto_static' + echo 'include $(BUILD_SHARED_LIBRARY)' + echo "\$(call import-module,python/$PYTHON_ABI)" + echo "\$(call import-module,$OPENSSL_HOME)" + } >$BUILDDIR_SSL/jni/Android.mk + fail_panic "Can't generate $BUILDDIR_SSL/jni/Android.mk" + + run $NDK_DIR/ndk-build -C $BUILDDIR_SSL -j$NUM_JOBS APP_ABI=$ABI V=1 + fail_panic "Can't build python$PYTHON_ABI-$ABI module '_ssl'" + + log "Install python$PYTHON_ABI-$ABI module '_ssl' in $PYBIN_INSTALLDIR_MODULES" + run cp -p -T $OBJDIR_SSL/lib_ssl.so $PYBIN_INSTALLDIR_MODULES/_ssl.so + fail_panic "Can't install python$PYTHON_ABI-$ABI module '_ssl' in $PYBIN_INSTALLDIR_MODULES" + fi + +# _hashlib + if [ -n "$OPENSSL_HOME" ]; then + local BUILDDIR_HASHLIB="$BUILDDIR/hashlib" + local OBJDIR_HASHLIB="$BUILDDIR_HASHLIB/obj/local/$ABI" + + run mkdir -p "$BUILDDIR_HASHLIB/jni" + fail_panic "Can't create directory: $BUILDDIR_HASHLIB/jni" + + { + echo 'LOCAL_PATH := $(call my-dir)' + echo 'include $(CLEAR_VARS)' + echo 'LOCAL_MODULE := _hashlib' + echo "MY_PYTHON_SRC_ROOT := $PYTHON_SRCDIR" + echo 'LOCAL_SRC_FILES := \' + echo ' $(MY_PYTHON_SRC_ROOT)/Modules/_hashopenssl.c' + echo 'LOCAL_STATIC_LIBRARIES := python_shared openssl_static opencrypto_static' + echo 'include $(BUILD_SHARED_LIBRARY)' + echo "\$(call import-module,python/$PYTHON_ABI)" + echo "\$(call import-module,$OPENSSL_HOME)" + } >$BUILDDIR_HASHLIB/jni/Android.mk + fail_panic "Can't generate $BUILDDIR_HASHLIB/jni/Android.mk" + + run $NDK_DIR/ndk-build -C $BUILDDIR_HASHLIB -j$NUM_JOBS APP_ABI=$ABI V=1 + fail_panic "Can't build python$PYTHON_ABI-$ABI module '_hashlib'" + + log "Install python$PYTHON_ABI-$ABI module '_hashlib' in $PYBIN_INSTALLDIR_MODULES" + run cp -p -T $OBJDIR_HASHLIB/lib_hashlib.so $PYBIN_INSTALLDIR_MODULES/_hashlib.so + fail_panic "Can't install python$PYTHON_ABI-$ABI module '_hashlib' in $PYBIN_INSTALLDIR_MODULES" + fi + +# _sqlite3 + local BUILDDIR_SQLITE3="$BUILDDIR/sqlite3" + local OBJDIR_SQLITE3="$BUILDDIR_SQLITE3/obj/local/$ABI" + + run mkdir -p "$BUILDDIR_SQLITE3/jni" + fail_panic "Can't create directory: $BUILDDIR_SQLITE3/jni" + + { + echo 'LOCAL_PATH := $(call my-dir)' + echo 'include $(CLEAR_VARS)' + echo 'LOCAL_MODULE := _sqlite3' + echo 'LOCAL_CFLAGS := -DMODULE_NAME=\"sqlite3\"' + echo "MY_PYTHON_SRC_ROOT := $PYTHON_SRCDIR" + echo 'LOCAL_SRC_FILES := \' + echo ' $(MY_PYTHON_SRC_ROOT)/Modules/_sqlite/cache.c \' + echo ' $(MY_PYTHON_SRC_ROOT)/Modules/_sqlite/connection.c \' + echo ' $(MY_PYTHON_SRC_ROOT)/Modules/_sqlite/cursor.c \' + echo ' $(MY_PYTHON_SRC_ROOT)/Modules/_sqlite/microprotocols.c \' + echo ' $(MY_PYTHON_SRC_ROOT)/Modules/_sqlite/module.c \' + echo ' $(MY_PYTHON_SRC_ROOT)/Modules/_sqlite/prepare_protocol.c \' + echo ' $(MY_PYTHON_SRC_ROOT)/Modules/_sqlite/row.c \' + echo ' $(MY_PYTHON_SRC_ROOT)/Modules/_sqlite/statement.c \' + echo ' $(MY_PYTHON_SRC_ROOT)/Modules/_sqlite/util.c' + echo 'LOCAL_STATIC_LIBRARIES := python_shared sqlite3_static' + echo 'include $(BUILD_SHARED_LIBRARY)' + echo "\$(call import-module,python/$PYTHON_ABI)" + echo '$(call import-module,sqlite/3)' + } >$BUILDDIR_SQLITE3/jni/Android.mk + fail_panic "Can't generate $BUILDDIR_SQLITE3/jni/Android.mk" + + run $NDK_DIR/ndk-build -C $BUILDDIR_SQLITE3 -j$NUM_JOBS APP_ABI=$ABI V=1 + fail_panic "Can't build python$PYTHON_ABI-$ABI module '_sqlite3'" + + log "Install python$PYTHON_ABI-$ABI module '_sqlite3' in $PYBIN_INSTALLDIR_MODULES" + run cp -p -T $OBJDIR_SQLITE3/lib_sqlite3.so $PYBIN_INSTALLDIR_MODULES/_sqlite3.so + fail_panic "Can't install python$PYTHON_ABI-$ABI module '_sqlite3' in $PYBIN_INSTALLDIR_MODULES" + +#pyexpat + local BUILDDIR_PYEXPAT="$BUILDDIR/pyexpat" + local OBJDIR_PYEXPAT="$BUILDDIR_PYEXPAT/obj/local/$ABI" + + run mkdir -p "$BUILDDIR_PYEXPAT/jni" + fail_panic "Can't create directory: $BUILDDIR_PYEXPAT/jni" + + { + echo 'LOCAL_PATH := $(call my-dir)' + echo 'include $(CLEAR_VARS)' + echo 'LOCAL_MODULE := pyexpat' + echo 'LOCAL_CFLAGS := -DHAVE_EXPAT_CONFIG_H -DXML_STATIC' + echo "MY_PYTHON_SRC_ROOT := $PYTHON_SRCDIR" + echo "LOCAL_C_INCLUDES := \$(MY_PYTHON_SRC_ROOT)/Modules/expat" + echo 'LOCAL_SRC_FILES := \' + echo ' $(MY_PYTHON_SRC_ROOT)/Modules/expat/xmlparse.c \' + echo ' $(MY_PYTHON_SRC_ROOT)/Modules/expat/xmlrole.c \' + echo ' $(MY_PYTHON_SRC_ROOT)/Modules/expat/xmltok.c \' + echo ' $(MY_PYTHON_SRC_ROOT)/Modules/pyexpat.c' + echo 'LOCAL_STATIC_LIBRARIES := python_shared' + echo 'include $(BUILD_SHARED_LIBRARY)' + echo "\$(call import-module,python/$PYTHON_ABI)" + } >$BUILDDIR_PYEXPAT/jni/Android.mk + fail_panic "Can't generate $BUILDDIR_PYEXPAT/jni/Android.mk" + + run $NDK_DIR/ndk-build -C $BUILDDIR_PYEXPAT -j$NUM_JOBS APP_ABI=$ABI V=1 + fail_panic "Can't build python$PYTHON_ABI-$ABI module 'pyexpat'" + + log "Install python$PYTHON_ABI-$ABI module 'pyexpat' in $PYBIN_INSTALLDIR_MODULES" + run cp -p -T $OBJDIR_PYEXPAT/libpyexpat.so $PYBIN_INSTALLDIR_MODULES/pyexpat.so + fail_panic "Can't install python$PYTHON_ABI-$ABI module 'pyexpat' in $PYBIN_INSTALLDIR_MODULES" + +# select + local BUILDDIR_SELECT="$BUILDDIR/select" + local OBJDIR_SELECT="$BUILDDIR_SELECT/obj/local/$ABI" + + run mkdir -p "$BUILDDIR_SELECT/jni" + fail_panic "Can't create directory: $BUILDDIR_SELECT/jni" + + { + echo 'LOCAL_PATH := $(call my-dir)' + echo 'include $(CLEAR_VARS)' + echo 'LOCAL_MODULE := select' + echo "MY_PYTHON_SRC_ROOT := $PYTHON_SRCDIR" + echo 'LOCAL_SRC_FILES := \' + echo ' $(MY_PYTHON_SRC_ROOT)/Modules/selectmodule.c' + echo 'LOCAL_STATIC_LIBRARIES := python_shared' + echo 'include $(BUILD_SHARED_LIBRARY)' + echo "\$(call import-module,python/$PYTHON_ABI)" + } >$BUILDDIR_SELECT/jni/Android.mk + fail_panic "Can't generate $BUILDDIR_SELECT/jni/Android.mk" + + run $NDK_DIR/ndk-build -C $BUILDDIR_SELECT -j$NUM_JOBS APP_ABI=$ABI V=1 + fail_panic "Can't build python$PYTHON_ABI-$ABI module 'select'" + + log "Install python$PYTHON_ABI-$ABI module 'select' in $PYBIN_INSTALLDIR_MODULES" + run cp -p -T $OBJDIR_SELECT/libselect.so $PYBIN_INSTALLDIR_MODULES/select.so + fail_panic "Can't install python$PYTHON_ABI-$ABI module 'select' in $PYBIN_INSTALLDIR_MODULES" + +# unicodedata + local BUILDDIR_UNICODEDATA="$BUILDDIR/unicodedata" + local OBJDIR_UNICODEDATA="$BUILDDIR_UNICODEDATA/obj/local/$ABI" + + run mkdir -p "$BUILDDIR_UNICODEDATA/jni" + fail_panic "Can't create directory: $BUILDDIR_UNICODEDATA/jni" + + { + echo 'LOCAL_PATH := $(call my-dir)' + echo 'include $(CLEAR_VARS)' + echo 'LOCAL_MODULE := unicodedata' + echo "MY_PYTHON_SRC_ROOT := $PYTHON_SRCDIR" + echo 'LOCAL_SRC_FILES := \' + echo ' $(MY_PYTHON_SRC_ROOT)/Modules/unicodedata.c' + echo 'LOCAL_STATIC_LIBRARIES := python_shared' + echo 'include $(BUILD_SHARED_LIBRARY)' + echo "\$(call import-module,python/$PYTHON_ABI)" + } >$BUILDDIR_UNICODEDATA/jni/Android.mk + fail_panic "Can't generate $BUILDDIR_UNICODEDATA/jni/Android.mk" + + run $NDK_DIR/ndk-build -C $BUILDDIR_UNICODEDATA -j$NUM_JOBS APP_ABI=$ABI V=1 + fail_panic "Can't build python$PYTHON_ABI-$ABI module 'unicodedata'" + + log "Install python$PYTHON_ABI-$ABI module 'unicodedata' in $PYBIN_INSTALLDIR_MODULES" + run cp -p -T $OBJDIR_UNICODEDATA/libunicodedata.so $PYBIN_INSTALLDIR_MODULES/unicodedata.so + fail_panic "Can't install python$PYTHON_ABI-$ABI module 'unicodedata' in $PYBIN_INSTALLDIR_MODULES" +} + +if [ -n "$PACKAGE_DIR" ]; then + PACKAGE_NAME="python${PYTHON_MAJOR_VERSION}.${PYTHON_MINOR_VERSION}-headers.tar.xz" + echo "Look for: $PACKAGE_NAME" + try_cached_package "$PACKAGE_DIR" "$PACKAGE_NAME" no_exit + if [ $? -eq 0 ]; then + PYTHON_HEADERS_NEED_PACKAGE=no + else + PYTHON_HEADERS_NEED_PACKAGE=yes + fi +fi + +BUILT_ABIS="" +for ABI in $ABIS; do + DO_BUILD_PACKAGE="yes" + if [ -n "$PACKAGE_DIR" ]; then + PACKAGE_NAME="python${PYTHON_MAJOR_VERSION}.${PYTHON_MINOR_VERSION}-libs-${ABI}.tar.xz" + echo "Look for: $PACKAGE_NAME" + try_cached_package "$PACKAGE_DIR" "$PACKAGE_NAME" no_exit + if [ $? -eq 0 ]; then + if [ "$PYTHON_HEADERS_NEED_PACKAGE" = "yes" -a -z "$BUILT_ABIS" ]; then + BUILT_ABIS="$BUILT_ABIS $ABI" + else + DO_BUILD_PACKAGE="no" + fi + else + BUILT_ABIS="$BUILT_ABIS $ABI" + fi + fi + if [ "$DO_BUILD_PACKAGE" = "yes" ]; then + build_python_for_abi $ABI "$BUILD_DIR/$ABI" + fi +done + +if [ -n "$PACKAGE_DIR" ]; then + if [ "$PYTHON_HEADERS_NEED_PACKAGE" = "yes" ]; then + FILES="$PYTHON_SUBDIR/$PYTHON_ABI/include" + PACKAGE_NAME="python${PYTHON_MAJOR_VERSION}.${PYTHON_MINOR_VERSION}-headers.tar.xz" + PACKAGE="$PACKAGE_DIR/$PACKAGE_NAME" + dump "Packaging: $PACKAGE" + pack_archive "$PACKAGE" "$NDK_DIR" "$FILES" + fail_panic "Can't package python headers" + cache_package "$PACKAGE_DIR" "$PACKAGE_NAME" + fi + + for ABI in $BUILT_ABIS; do + FILES="$PYTHON_SUBDIR/$PYTHON_ABI/libs/$ABI" + PACKAGE_NAME="python${PYTHON_MAJOR_VERSION}.${PYTHON_MINOR_VERSION}-libs-${ABI}.tar.xz" + PACKAGE="$PACKAGE_DIR/$PACKAGE_NAME" + dump "Packaging: $PACKAGE" + pack_archive "$PACKAGE" "$NDK_DIR" "$FILES" + fail_panic "Can't package python $ABI libs" + cache_package "$PACKAGE_DIR" "$PACKAGE_NAME" + done +fi + +if [ -z "$OPTION_BUILD_DIR" ]; then + log "Cleaning up..." + rm -rf $BUILD_DIR +else + log "Don't forget to cleanup: $BUILD_DIR" +fi + +log "Done!" diff --git a/src/main/python/lbrynetservice.py b/src/main/python/lbrynetservice.py index d156f440..6aaff229 100644 --- a/src/main/python/lbrynetservice.py +++ b/src/main/python/lbrynetservice.py @@ -76,7 +76,6 @@ keyring.set_keyring(LbryAndroidKeyring()) import logging.handlers from lbrynet.core import log_support from twisted.internet import defer, reactor -from jsonrpc.proxy import JSONRPCProxy from lbrynet import analytics from lbrynet import conf -- 2.45.3 From c8a583cc03e308fdab91114218917c645eb2559d Mon Sep 17 00:00:00 2001 From: Akinwale Ariwodola Date: Wed, 26 Sep 2018 01:52:49 +0100 Subject: [PATCH 02/20] include Python 3 apt packages in travis build script --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 53def169..ab62bda2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,7 +8,7 @@ install: - export PATH=/usr/bin:$PATH - sudo dpkg --add-architecture i386 - sudo apt-get -qq update -- sudo apt-get -qq install build-essential ccache git libncurses5:i386 libstdc++6:i386 libgtk2.0-0:i386 libpangox-1.0-0:i386 libpangoxft-1.0-0:i386 libidn11:i386 python2.7 python2.7-dev openjdk-8-jdk unzip zlib1g-dev zlib1g:i386 m4 libc6-dev-i386 +- sudo apt-get -qq install build-essential python3.6 python3.6-dev python3-pip ccache git libncurses5:i386 libstdc++6:i386 libgtk2.0-0:i386 libpangox-1.0-0:i386 libpangoxft-1.0-0:i386 libidn11:i386 python2.7 python2.7-dev openjdk-8-jdk unzip zlib1g-dev zlib1g:i386 m4 libc6-dev-i386 - sudo pip install --upgrade cython==0.25.2 pip setuptools - wget -q https://nodejs.org/dist/v8.11.1/node-v8.11.1-linux-x64.tar.xz - tar -xf node-v8.11.1-linux-x64.tar.xz -- 2.45.3 From 9e0a7b7690ad5b9e8b525d9d2e61d90bbaac07fe Mon Sep 17 00:00:00 2001 From: Akinwale Ariwodola Date: Wed, 26 Sep 2018 02:00:50 +0100 Subject: [PATCH 03/20] use Python 3.6 in Travis --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index ab62bda2..20264bf5 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,13 +2,13 @@ sudo: required dist: trusty language: python python: -- '2.7' +- '3.6' install: - deactivate - export PATH=/usr/bin:$PATH - sudo dpkg --add-architecture i386 - sudo apt-get -qq update -- sudo apt-get -qq install build-essential python3.6 python3.6-dev python3-pip ccache git libncurses5:i386 libstdc++6:i386 libgtk2.0-0:i386 libpangox-1.0-0:i386 libpangoxft-1.0-0:i386 libidn11:i386 python2.7 python2.7-dev openjdk-8-jdk unzip zlib1g-dev zlib1g:i386 m4 libc6-dev-i386 +- sudo apt-get -qq install build-essential python3-pip ccache git libncurses5:i386 libstdc++6:i386 libgtk2.0-0:i386 libpangox-1.0-0:i386 libpangoxft-1.0-0:i386 libidn11:i386 python2.7 python2.7-dev openjdk-8-jdk unzip zlib1g-dev zlib1g:i386 m4 libc6-dev-i386 - sudo pip install --upgrade cython==0.25.2 pip setuptools - wget -q https://nodejs.org/dist/v8.11.1/node-v8.11.1-linux-x64.tar.xz - tar -xf node-v8.11.1-linux-x64.tar.xz -- 2.45.3 From 2f0ccb449f888c27620ffcb118f6c89955e02f68 Mon Sep 17 00:00:00 2001 From: Akinwale Ariwodola Date: Wed, 26 Sep 2018 08:58:07 +0100 Subject: [PATCH 04/20] attempt to use symlink for python3.6 --- .travis.yml | 1 + Vagrantfile | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 20264bf5..f7722e89 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,6 +9,7 @@ install: - sudo dpkg --add-architecture i386 - sudo apt-get -qq update - sudo apt-get -qq install build-essential python3-pip ccache git libncurses5:i386 libstdc++6:i386 libgtk2.0-0:i386 libpangox-1.0-0:i386 libpangoxft-1.0-0:i386 libidn11:i386 python2.7 python2.7-dev openjdk-8-jdk unzip zlib1g-dev zlib1g:i386 m4 libc6-dev-i386 +- sudo ln -s /usr/bin/python3 /usr/bin/python3.6 - sudo pip install --upgrade cython==0.25.2 pip setuptools - wget -q https://nodejs.org/dist/v8.11.1/node-v8.11.1-linux-x64.tar.xz - tar -xf node-v8.11.1-linux-x64.tar.xz diff --git a/Vagrantfile b/Vagrantfile index 254851fb..713a2efa 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -43,7 +43,7 @@ Vagrant.configure("2") do |config| dpkg --add-architecture i386 apt-get update apt-get install -y libssl-dev - apt-get install -y autoconf libffi-dev pkg-config libtool build-essential ccache git libncurses5:i386 libstdc++6:i386 libgtk2.0-0:i386 libpangox-1.0-0:i386 libpangoxft-1.0-0:i386 libidn11:i386 python2.7 python2.7-dev openjdk-8-jdk unzip zlib1g-dev zlib1g:i386 m4 libc6-dev-i386 python-pip + apt-get install -y python3.6 python3.6-dev python3-pip autoconf libffi-dev pkg-config libtool build-essential ccache git libncurses5:i386 libstdc++6:i386 libgtk2.0-0:i386 libpangox-1.0-0:i386 libpangoxft-1.0-0:i386 libidn11:i386 python2.7 python2.7-dev openjdk-8-jdk unzip zlib1g-dev zlib1g:i386 m4 libc6-dev-i386 python-pip pip install -f --upgrade setuptools pyopenssl git clone https://github.com/lbryio/buildozer.git cd buildozer -- 2.45.3 From eab5c5f1a04fe63ed6c9158aea64b9e13fc9790c Mon Sep 17 00:00:00 2001 From: Akinwale Ariwodola Date: Wed, 26 Sep 2018 09:24:22 +0100 Subject: [PATCH 05/20] try the xenial distribution --- .travis.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index f7722e89..f8138c0c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,5 +1,5 @@ sudo: required -dist: trusty +dist: xenial language: python python: - '3.6' @@ -8,8 +8,7 @@ install: - export PATH=/usr/bin:$PATH - sudo dpkg --add-architecture i386 - sudo apt-get -qq update -- sudo apt-get -qq install build-essential python3-pip ccache git libncurses5:i386 libstdc++6:i386 libgtk2.0-0:i386 libpangox-1.0-0:i386 libpangoxft-1.0-0:i386 libidn11:i386 python2.7 python2.7-dev openjdk-8-jdk unzip zlib1g-dev zlib1g:i386 m4 libc6-dev-i386 -- sudo ln -s /usr/bin/python3 /usr/bin/python3.6 +- sudo apt-get -qq install build-essential python3.6 python3.6-dev python3-pip ccache git libncurses5:i386 libstdc++6:i386 libgtk2.0-0:i386 libpangox-1.0-0:i386 libpangoxft-1.0-0:i386 libidn11:i386 python2.7 python2.7-dev openjdk-8-jdk unzip zlib1g-dev zlib1g:i386 m4 libc6-dev-i386 - sudo pip install --upgrade cython==0.25.2 pip setuptools - wget -q https://nodejs.org/dist/v8.11.1/node-v8.11.1-linux-x64.tar.xz - tar -xf node-v8.11.1-linux-x64.tar.xz -- 2.45.3 From a3376fd5986b299cb3aabbd0e20c2c60bd159c37 Mon Sep 17 00:00:00 2001 From: Akinwale Ariwodola Date: Wed, 26 Sep 2018 09:32:25 +0100 Subject: [PATCH 06/20] trying to get python 3.6 installed in Travis --- .travis.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index f8138c0c..22f4bb2c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,8 +7,9 @@ install: - deactivate - export PATH=/usr/bin:$PATH - sudo dpkg --add-architecture i386 +- sudo add-apt-repository ppa:jonathonf/python-3.6 - sudo apt-get -qq update -- sudo apt-get -qq install build-essential python3.6 python3.6-dev python3-pip ccache git libncurses5:i386 libstdc++6:i386 libgtk2.0-0:i386 libpangox-1.0-0:i386 libpangoxft-1.0-0:i386 libidn11:i386 python2.7 python2.7-dev openjdk-8-jdk unzip zlib1g-dev zlib1g:i386 m4 libc6-dev-i386 +- sudo apt-get -qq install build-essential python3.6 python3.6-dev python3.6-venv python3-pip ccache git libncurses5:i386 libstdc++6:i386 libgtk2.0-0:i386 libpangox-1.0-0:i386 libpangoxft-1.0-0:i386 libidn11:i386 python2.7 python2.7-dev openjdk-8-jdk unzip zlib1g-dev zlib1g:i386 m4 libc6-dev-i386 - sudo pip install --upgrade cython==0.25.2 pip setuptools - wget -q https://nodejs.org/dist/v8.11.1/node-v8.11.1-linux-x64.tar.xz - tar -xf node-v8.11.1-linux-x64.tar.xz -- 2.45.3 From 528bb95e68b08f95dd36cfbc6800de2042fd19a1 Mon Sep 17 00:00:00 2001 From: Akinwale Ariwodola Date: Wed, 26 Sep 2018 09:56:37 +0100 Subject: [PATCH 07/20] small tweak --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 22f4bb2c..0b7b450c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,7 +7,7 @@ install: - deactivate - export PATH=/usr/bin:$PATH - sudo dpkg --add-architecture i386 -- sudo add-apt-repository ppa:jonathonf/python-3.6 +- sudo add-apt-repository ppa:jonathonf/python-3.6 -y - sudo apt-get -qq update - sudo apt-get -qq install build-essential python3.6 python3.6-dev python3.6-venv python3-pip ccache git libncurses5:i386 libstdc++6:i386 libgtk2.0-0:i386 libpangox-1.0-0:i386 libpangoxft-1.0-0:i386 libidn11:i386 python2.7 python2.7-dev openjdk-8-jdk unzip zlib1g-dev zlib1g:i386 m4 libc6-dev-i386 - sudo pip install --upgrade cython==0.25.2 pip setuptools -- 2.45.3 From 7c969cbe933095113b28038810407e70232382e1 Mon Sep 17 00:00:00 2001 From: Akinwale Ariwodola Date: Wed, 26 Sep 2018 17:55:03 +0100 Subject: [PATCH 08/20] fix for red screen during phone number verification --- app/src/component/phoneNumberRewardSubcard/view.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/src/component/phoneNumberRewardSubcard/view.js b/app/src/component/phoneNumberRewardSubcard/view.js index 5f2bb1fb..7856c4fb 100644 --- a/app/src/component/phoneNumberRewardSubcard/view.js +++ b/app/src/component/phoneNumberRewardSubcard/view.js @@ -44,7 +44,7 @@ class PhoneNumberRewardSubcard extends React.PureComponent { DeviceEventEmitter.addListener('onVerificationCodeReceived', this.receiveVerificationCode); const { phone } = this.props; - if (phone && String(phone).trim().length() > 0) { + if (phone && String(phone).trim().length > 0) { this.setState({ newPhoneAdded: true }); } -- 2.45.3 From 8b8cf9cd7afb1239945fb660957825e7df95e76a Mon Sep 17 00:00:00 2001 From: Akinwale Ariwodola Date: Wed, 26 Sep 2018 21:11:55 +0100 Subject: [PATCH 09/20] improve stop download experience --- app/src/component/fileDownloadButton/view.js | 2 +- app/src/page/file/index.js | 12 +++--- app/src/page/file/view.js | 40 +++++++++++++++----- app/src/redux/actions/file.js | 14 +++---- 4 files changed, 46 insertions(+), 22 deletions(-) diff --git a/app/src/component/fileDownloadButton/view.js b/app/src/component/fileDownloadButton/view.js index 827f2f13..70e6f507 100644 --- a/app/src/component/fileDownloadButton/view.js +++ b/app/src/component/fileDownloadButton/view.js @@ -47,7 +47,7 @@ class FileDownloadButton extends React.PureComponent { onStartDownloadFailed } = this.props; - if (loading || downloading) { + if ((fileInfo && !fileInfo.stopped) || loading || downloading) { const progress = fileInfo && fileInfo.written_bytes ? fileInfo.written_bytes / fileInfo.total_bytes * 100 : 0, label = fileInfo ? progress.toFixed(0) + '% complete' : 'Connecting...'; diff --git a/app/src/page/file/index.js b/app/src/page/file/index.js index e653fff0..d909afb5 100644 --- a/app/src/page/file/index.js +++ b/app/src/page/file/index.js @@ -1,8 +1,9 @@ import { connect } from 'react-redux'; import { doFetchFileInfo, - doResolveUri, doFetchCostInfoForUri, + doResolveUri, + doNotify, makeSelectIsUriResolving, makeSelectCostInfoForUri, makeSelectFileInfoForUri, @@ -32,13 +33,14 @@ const select = (state, props) => { }; const perform = dispatch => ({ - fetchFileInfo: uri => dispatch(doFetchFileInfo(uri)), - fetchCostInfo: uri => dispatch(doFetchCostInfoForUri(uri)), - resolveUri: uri => dispatch(doResolveUri(uri)), - stopDownload: (uri, fileInfo) => dispatch(doStopDownloadingFile(uri, fileInfo)), deleteFile: (fileInfo, deleteFromDevice, abandonClaim) => { dispatch(doDeleteFile(fileInfo, deleteFromDevice, abandonClaim)); }, + fetchFileInfo: uri => dispatch(doFetchFileInfo(uri)), + fetchCostInfo: uri => dispatch(doFetchCostInfoForUri(uri)), + notify: data => dispatch(doNotify(data)), + resolveUri: uri => dispatch(doResolveUri(uri)), + stopDownload: (uri, fileInfo) => dispatch(doStopDownloadingFile(uri, fileInfo)), }); export default connect(select, perform)(FilePage); diff --git a/app/src/page/file/view.js b/app/src/page/file/view.js index 5e399d61..c617c6ea 100644 --- a/app/src/page/file/view.js +++ b/app/src/page/file/view.js @@ -57,7 +57,8 @@ class FilePage extends React.PureComponent { showWebView: false, playerBgHeight: 0, playerHeight: 0, - uri: null + uri: null, + stopDownloadConfirmed: false }; } @@ -155,7 +156,12 @@ class FilePage extends React.PureComponent { { text: 'No' }, { text: 'Yes', onPress: () => { deleteFile(fileInfo.outpoint, true); - this.setState({ downloadPressed: false, fileViewLogged: false, mediaLoaded: false }); + this.setState({ + downloadPressed: false, + fileViewLogged: false, + mediaLoaded: false, + stopDownloadConfirmed: false + }); }} ], { cancelable: true } @@ -163,7 +169,7 @@ class FilePage extends React.PureComponent { } onStopDownloadPressed = () => { - const { deleteFile, stopDownload, fileInfo, navigation } = this.props; + const { fileInfo, navigation, notify, stopDownload } = this.props; Alert.alert( 'Stop download', @@ -172,8 +178,20 @@ class FilePage extends React.PureComponent { { text: 'No' }, { text: 'Yes', onPress: () => { stopDownload(navigation.state.params.uri, fileInfo); - this.setState({ downloadPressed: false, fileViewLogged: false, mediaLoaded: false }); - } } + this.setState({ + downloadPressed: false, + fileViewLogged: false, + mediaLoaded: false, + stopDownloadConfirmed: true + }); + + // there can be a bit of lag between the user pressing Yes and the UI being updated + // after the file_set_status and file_delete operations, so let the user know + notify({ + message: 'The download will stop momentarily. You do not need to wait to discover something else.', + displayType: ['toast'] + }); + }} ], { cancelable: true } ); @@ -361,7 +379,9 @@ class FilePage extends React.PureComponent { const mediaType = Lbry.getMediaType(contentType); const isPlayable = mediaType === 'video' || mediaType === 'audio'; const { height, channel_name: channelName, value } = claim; - const showActions = !this.state.fullscreenMode && !this.state.showImageViewer && !this.state.showWebView && + const showActions = !this.state.fullscreenMode && + !this.state.showImageViewer && + !this.state.showWebView && (completed || (fileInfo && !fileInfo.stopped && fileInfo.written_bytes < fileInfo.total_bytes)); const channelClaimId = value && value.publisherSignature && value.publisherSignature.certificateId; @@ -420,7 +440,7 @@ class FilePage extends React.PureComponent { isPlayable={isPlayable} onPlay={() => { this.startTime = Date.now(); - this.setState({ downloadPressed: true, autoPlayMedia: true }); + this.setState({ downloadPressed: true, autoPlayMedia: true, stopDownloadConfirmed: false }); }} onButtonLayout={() => this.setState({ downloadButtonShown: true })} onStartDownloadFailed={() => { @@ -454,14 +474,16 @@ class FilePage extends React.PureComponent { onPlaybackStarted={this.onPlaybackStarted} />} - { showActions && + {fileInfo && showActions && {completed &&