Compare commits

...

12 commits

Author SHA1 Message Date
zeppi 2271aede1f debug 2022-12-06 12:49:03 -05:00
zeppi ed8ecd505f try debug 2022-12-05 11:55:23 -05:00
zeppi eea6a92f71 shprint debug 2022-12-05 11:46:27 -05:00
zeppi e06d2ffcb3 print debug 2022-12-05 11:25:15 -05:00
zeppi f511189275 remove toolchain flag 2022-12-05 11:09:33 -05:00
zeppi 6317070f19 try without --use-llvm flag 2022-12-05 10:36:30 -05:00
zeppi f3cde00406 add boost and libtorrent recipes 2022-12-05 10:07:01 -05:00
zeppi b19ec084a1 deploy 2 2022-12-05 09:48:06 -05:00
zeppi 11ac25e04c deploy? 2022-12-05 09:45:51 -05:00
zeppi cab8b423da deploy 2022-12-05 09:44:13 -05:00
zeppi b0e8785c75 add libtorrent 2022-12-05 09:37:38 -05:00
zeppi c41a17f943 filetype recipe 2022-12-02 15:05:00 -05:00
14 changed files with 473 additions and 2 deletions

View file

@ -2,7 +2,7 @@ name: Publish Assets
on:
push:
branches: [master]
branches: [master, update-deps-incremental-2]
jobs:
build_arm64_aar:

1
.gitignore vendored
View file

@ -19,3 +19,4 @@ p4a/pythonforandroid/bootstraps/lbry/build/templates/google-services.json
p4a/*.apk
p4a/*.aar
venv

View file

@ -39,7 +39,7 @@ version.filename = %(source.dir)s/main.py
# (list) Application requirements
# comma seperated e.g. requirements = sqlite3,kivy
requirements = python3crystax, openssl, sqlite3, hostpython3crystax, android, distro==1.4.0, pyjnius, certifi==2021.5.30, appdirs==1.4.3, docopt==0.6.2, base58==1.0.0, colorama==0.3.7, ecdsa==0.13.3, jsonschema==2.6.0, pbkdf2==1.3, pyyaml, protobuf==3.6.1, keyring==21.0.0, defusedxml, netifaces, aioupnp==0.0.17, asn1crypto, mock, cryptography, aiohttp==3.6.0, multidict==4.5.2, idna, yarl==1.3.0, chardet==3.0.4, async_timeout==3.0.1, coincurve, msgpack==0.6.1, six, attrs==18.2.0, pylru, hachoir, prometheus_client==0.8.0, "git+https://github.com/lbryio/lbry-sdk@v0.112.0#egg=lbry"
requirements = python3crystax, openssl, sqlite3, hostpython3crystax, android, distro==1.4.0, pyjnius, certifi==2021.5.30, appdirs==1.4.3, docopt==0.6.2, base58==1.0.0, colorama==0.3.7, ecdsa==0.13.3, jsonschema==2.6.0, pbkdf2==1.3, pyyaml, protobuf==3.6.1, keyring==21.0.0, defusedxml, filetype, netifaces, aioupnp==0.0.17, asn1crypto, mock, cryptography, aiohttp==3.6.0, multidict==4.5.2, idna, yarl==1.3.0, chardet==3.0.4, async_timeout==3.0.1, coincurve, libtorrent, msgpack==0.6.1, six, attrs==18.2.0, pylru, hachoir, prometheus_client==0.8.0, "git+https://github.com/lbryio/lbry-sdk@v0.112.0#egg=lbry"
# (str) Custom source folders for requirements
# Sets custom source for any requirements with recipes

106
recipes/boost/__init__.py Normal file
View file

@ -0,0 +1,106 @@
from pythonforandroid.toolchain import Recipe, shprint, shutil, current_directory
from os.path import join, exists
from os import environ
import sh
"""
This recipe creates a custom toolchain and bootstraps Boost from source to build Boost.Build
including python bindings
"""
class BoostRecipe(Recipe):
# Todo: make recipe compatible with all p4a architectures
'''
.. note:: This recipe can be built only against API 21+ and arch armeabi-v7a
.. versionchanged:: 0.6.0
Rewrote recipe to support clang's build. The following changes has
been made:
- Bumped version number to 1.68.0
- Better version handling for url
- Added python 3 compatibility
- Default compiler for ndk's toolchain set to clang
- Python version will be detected via user-config.jam
- Changed stl's lib from ``gnustl_shared`` to ``c++_shared``
'''
version = '1.68.0'
url = 'http://downloads.sourceforge.net/project/boost/' \
'boost/{version}/boost_{version_underscore}.tar.bz2'
depends = [('python2', 'python3crystax')]
patches = ['disable-so-version.patch',
'use-android-libs.patch',
'fix-android-issues.patch']
@property
def versioned_url(self):
if self.url is None:
return None
return self.url.format(
version=self.version,
version_underscore=self.version.replace('.', '_'))
def should_build(self, arch):
return not exists(join(self.get_build_dir(arch.arch), 'b2'))
def prebuild_arch(self, arch):
super(BoostRecipe, self).prebuild_arch(arch)
env = self.get_recipe_env(arch)
with current_directory(self.get_build_dir(arch.arch)):
if not exists(env['CROSSHOME']):
# Make custom toolchain
bash = sh.Command('bash')
shprint(bash, join('echo crosshost', env['CROSSHOST']))
shprint(bash, join('echo crosshome', env['CROSSHOME']))
shprint(bash, join(self.ctx.ndk_dir, 'build/tools/make-standalone-toolchain.sh'),
'--arch=' + arch.arch,
'--platform=android-' + str(self.ctx.android_api),
'--toolchain=' + env['CROSSHOST'] + '-' + self.ctx.toolchain_version + ':-llvm',
'--stl=libc++',
'--install-dir=' + env['CROSSHOME']
)
# Set custom configuration
shutil.copyfile(join(self.get_recipe_dir(), 'user-config.jam'),
join(env['BOOST_BUILD_PATH'], 'user-config.jam'))
def build_arch(self, arch):
super(BoostRecipe, self).build_arch(arch)
env = self.get_recipe_env(arch)
env['PYTHON_HOST'] = self.ctx.hostpython
with current_directory(self.get_build_dir(arch.arch)):
# Compile Boost.Build engine with this custom toolchain
bash = sh.Command('bash')
shprint(bash, 'bootstrap.sh') # Do not pass env
# Install app stl
shutil.copyfile(
join(self.ctx.ndk_dir, 'sources/cxx-stl/llvm-libc++/libs/'
'armeabi-v7a/libc++_shared.so'),
join(self.ctx.get_libs_dir(arch.arch), 'libc++_shared.so'))
def select_build_arch(self, arch):
return arch.arch.replace('eabi-v7a', '').replace('eabi', '')
def get_recipe_env(self, arch):
# We don't use the normal env because we
# are building with a standalone toolchain
env = environ.copy()
env['BOOST_BUILD_PATH'] = self.get_build_dir(arch.arch) # find user-config.jam
env['BOOST_ROOT'] = env['BOOST_BUILD_PATH'] # find boost source
env['PYTHON_ROOT'] = self.ctx.python_recipe.link_root(arch.arch)
env['PYTHON_INCLUDE'] = self.ctx.python_recipe.include_root(arch.arch)
env['PYTHON_MAJOR_MINOR'] = self.ctx.python_recipe.version[:3]
env['PYTHON_LINK_VERSION'] = self.ctx.python_recipe.major_minor_version_string
if 'python3' in self.ctx.python_recipe.name:
env['PYTHON_LINK_VERSION'] += 'm'
env['ARCH'] = self.select_build_arch(arch)
env['CROSSHOST'] = env['ARCH'] + '-linux-androideabi'
env['CROSSHOME'] = join(env['BOOST_ROOT'], 'standalone-' + env['ARCH'] + '-toolchain')
return env
recipe = BoostRecipe()

View file

@ -0,0 +1,12 @@
--- boost/boostcpp.jam 2015-12-14 03:30:09.000000000 +0100
+++ boost-patch/boostcpp.jam 2016-02-08 16:38:40.510859612 +0100
@@ -155,8 +155,9 @@
if $(type) = SHARED_LIB &&
! [ $(property-set).get <target-os> ] in windows cygwin darwin aix &&
! [ $(property-set).get <toolset> ] in pgi
{
+ return $(result) ; # disable version suffix for android
result = $(result).$(BOOST_VERSION) ;
}
return $(result) ;

View file

@ -0,0 +1,68 @@
diff -u -r boost_1_68_0.orig/boost/config/user.hpp boost_1_68_0/boost/config/user.hpp
--- boost_1_68_0.orig/boost/config/user.hpp 2018-08-01 22:50:46.000000000 +0200
+++ boost_1_68_0/boost/config/user.hpp 2018-08-27 15:43:38.000000000 +0200
@@ -13,6 +13,12 @@
// configuration policy:
//
+// Android defines
+// There is problem with std::atomic on android (and some other platforms).
+// See this link for more info:
+// https://code.google.com/p/android/issues/detail?id=42735#makechanges
+#define BOOST_ASIO_DISABLE_STD_ATOMIC 1
+
// define this to locate a compiler config file:
// #define BOOST_COMPILER_CONFIG <myheader>
diff -u -r boost_1_68_0.orig/boost/asio/detail/config.hpp boost_1_68_0/boost/asio/detail/config.hpp
--- boost_1_68_0.orig/boost/asio/detail/config.hpp 2018-08-01 22:50:46.000000000 +0200
+++ boost_1_68_0/boost/asio/detail/config.hpp 2018-09-19 12:39:56.000000000 +0200
@@ -804,7 +804,11 @@
# if defined(__clang__)
# if (__cplusplus >= 201402)
# if __has_include(<experimental/string_view>)
-# define BOOST_ASIO_HAS_STD_EXPERIMENTAL_STRING_VIEW 1
+# if __clang_major__ >= 7
+# undef BOOST_ASIO_HAS_STD_EXPERIMENTAL_STRING_VIEW
+# else
+# define BOOST_ASIO_HAS_STD_EXPERIMENTAL_STRING_VIEW 1
+# endif // __clang_major__ >= 7
# endif // __has_include(<experimental/string_view>)
# endif // (__cplusplus >= 201402)
# endif // defined(__clang__)
diff -u -r boost_1_68_0.orig/boost/system/error_code.hpp boost_1_68_0/boost/system/error_code.hpp
--- boost_1_68_0.orig/boost/system/error_code.hpp 2018-08-01 22:50:53.000000000 +0200
+++ boost_1_68_0/boost/system/error_code.hpp 2018-08-27 15:44:29.000000000 +0200
@@ -17,6 +17,7 @@
#include <boost/assert.hpp>
#include <boost/noncopyable.hpp>
#include <boost/utility/enable_if.hpp>
+#include <stdio.h>
#include <ostream>
#include <string>
#include <stdexcept>
diff -u -r boost_1_68_0.orig/libs/filesystem/src/operations.cpp boost_1_68_0/libs/filesystem/src/operations.cpp
--- boost_1_68_0.orig/libs/filesystem/src/operations.cpp 2018-08-01 22:50:47.000000000 +0200
+++ boost_1_68_0/libs/filesystem/src/operations.cpp 2018-08-27 15:47:15.000000000 +0200
@@ -232,6 +232,21 @@
# if defined(BOOST_POSIX_API)
+# if defined(__ANDROID__)
+# define truncate libboost_truncate_wrapper
+// truncate() is present in Android libc only starting from ABI 21, so here's a simple wrapper
+static int libboost_truncate_wrapper(const char *path, off_t length)
+{
+ int fd = open(path, O_WRONLY);
+ if (fd == -1) {
+ return -1;
+ }
+ int status = ftruncate(fd, length);
+ close(fd);
+ return status;
+}
+# endif
+
typedef int err_t;
// POSIX uses a 0 return to indicate success

View file

@ -0,0 +1,10 @@
--- boost/tools/build/src/tools/python.jam 2015-10-16 20:55:36.000000000 +0200
+++ boost-patch/tools/build/src/tools/python.jam 2016-02-09 13:16:09.519261546 +0100
@@ -646,6 +646,7 @@
case aix : return <library>pthread <library>dl ;
+ case * : return ; # use Android builtin libs
case * : return <library>pthread <library>dl
<toolset>gcc:<library>util <toolset-intel:platform>linux:<library>util ;
}

View file

@ -0,0 +1,61 @@
import os ;
local ARCH = [ os.environ ARCH ] ;
local CROSSHOME = [ os.environ CROSSHOME ] ;
local PYTHON_HOST = [ os.environ PYTHON_HOST ] ;
local PYTHON_ROOT = [ os.environ PYTHON_ROOT ] ;
local PYTHON_INCLUDE = [ os.environ PYTHON_INCLUDE ] ;
local PYTHON_LINK_VERSION = [ os.environ PYTHON_LINK_VERSION ] ;
local PYTHON_MAJOR_MINOR = [ os.environ PYTHON_MAJOR_MINOR ] ;
using clang : $(ARCH) : $(CROSSHOME)/bin/arm-linux-androideabi-clang++ :
<archiver>$(CROSSHOME)/bin/arm-linux-androideabi-ar
<root>$(CROSSHOME)/sysroot
<architecture>$(ARCH)
<compileflags>-fexceptions
<compileflags>-frtti
<compileflags>-fpic
<compileflags>-ffunction-sections
<compileflags>-funwind-tables
<compileflags>-march=armv7-a
<compileflags>-msoft-float
<compileflags>-mfpu=neon
<compileflags>-mthumb
<linkflags>-march=armv7-a
<linkflags>-Wl,--fix-cortex-a8
<compileflags>-Os
<compileflags>-fomit-frame-pointer
<compileflag>-fno-strict-aliasing
<compileflags>-DANDROID
<compileflags>-D__ANDROID__
<compileflags>-DANDROID_TOOLCHAIN=clang
<compileflags>-DANDROID_ABI=armv7-a
<compileflags>-DANDROID_STL=c++_shared
<compileflags>-DBOOST_ALL_NO_LIB
#<compileflags>-DNDEBUG
<compileflags>-O2
<compileflags>-g
<compileflags>-fvisibility=hidden
<compileflags>-fvisibility-inlines-hidden
<compileflags>-fdata-sections
<cxxflags>-D__arm__
<cxxflags>-D_REENTRANT
<cxxflags>-D_GLIBCXX__PTHREADS
<compileflags>-Wno-long-long
<compileflags>-Wno-missing-field-initializers
<compileflags>-Wno-unused-variable
<linkflags>-Wl,-z,relro
<linkflags>-Wl,-z,now
<linkflags>-lc++_shared
<linkflags>-L$(PYTHON_ROOT)
<linkflags>-lpython$(PYTHON_LINK_VERSION)
<linkflags>-Wl,-O1
<linkflags>-Wl,-Bsymbolic-functions
;
using python : $(PYTHON_MAJOR_MINOR)
: $(PYTHON_host)
: $(PYTHON_ROOT) $(PYTHON_INCLUDE)
: $(PYTHON_ROOT)/libpython$(PYTHON_LINK_VERSION).so
: #<define>BOOST_ALL_DYN_LINK
;

View file

@ -0,0 +1,13 @@
from pythonforandroid.recipe import PythonRecipe
class FileTypeRecipe(PythonRecipe):
# TODO: version
url = 'https://github.com/h2non/filetype.py/archive/refs/tags/v1.2.0.zip'
depends = ['setuptools']
call_hostpython_via_targetpython = False
recipe = FileTypeRecipe()

View file

@ -0,0 +1,138 @@
from pythonforandroid.toolchain import Recipe, shprint, shutil, current_directory
from multiprocessing import cpu_count
from os.path import join, basename
from os import listdir, walk
import sh
# This recipe builds libtorrent with Python bindings
# It depends on Boost.Build and the source of several Boost libraries present in BOOST_ROOT,
# which is all provided by the boost recipe
def get_lib_from(search_directory, lib_extension='.so'):
'''Scan directories recursively until find any file with the given
extension. The default extension to search is ``.so``.'''
for root, dirs, files in walk(search_directory):
for file in files:
if file.endswith(lib_extension):
print('get_lib_from: {}\n\t- {}'.format(
search_directory, join(root, file)))
return join(root, file)
return None
class LibtorrentRecipe(Recipe):
# Todo: make recipe compatible with all p4a architectures
'''
.. note:: This recipe can be built only against API 21+ and arch armeabi-v7a
.. versionchanged:: 0.6.0
Rewrote recipe to support clang's build and boost 1.68. The following
changes has been made:
- Bumped version number to 1.2.0
- added python 3 compatibility
- new system to detect/copy generated libraries
'''
version = '1_2_0'
url = 'https://github.com/arvidn/libtorrent/archive/libtorrent_{version}.tar.gz'
depends = ['boost']
opt_depends = ['openssl']
patches = ['disable-so-version.patch',
'use-soname-python.patch',
'setup-lib-name.patch']
# libtorrent.so is not included because is not a system library
generated_libraries = [
'boost_system', 'boost_python{py_version}', 'torrent_rasterbar']
def should_build(self, arch):
python_version = self.ctx.python_recipe.version[:3].replace('.', '')
libs = ['lib' + lib_name.format(py_version=python_version) +
'.so' for lib_name in self.generated_libraries]
return not (self.has_libs(arch, *libs) and
self.ctx.has_package('libtorrent', arch.arch))
def prebuild_arch(self, arch):
super(LibtorrentRecipe, self).prebuild_arch(arch)
if 'openssl' in recipe.ctx.recipe_build_order:
# Patch boost user-config.jam to use openssl
self.get_recipe('boost', self.ctx).apply_patch(
join(self.get_recipe_dir(), 'user-config-openssl.patch'), arch.arch)
def build_arch(self, arch):
super(LibtorrentRecipe, self).build_arch(arch)
env = self.get_recipe_env(arch)
env['PYTHON_HOST'] = self.ctx.hostpython
# Define build variables
build_dir = self.get_build_dir(arch.arch)
ctx_libs_dir = self.ctx.get_libs_dir(arch.arch)
encryption = 'openssl' if 'openssl' in recipe.ctx.recipe_build_order else 'built-in'
build_args = [
'-q',
# '-a', # force build, useful to debug the build
'-j' + str(cpu_count()),
'--debug-configuration', # so we know if our python is detected
# '--deprecated-functions=off',
'toolset=clang-arm',
'abi=aapcs',
'binary-format=elf',
'cxxflags=-std=c++11',
'target-os=android',
'threading=multi',
'link=shared',
'boost-link=shared',
'libtorrent-link=shared',
'runtime-link=shared',
'encryption={}'.format('on' if encryption == 'openssl' else 'off'),
'crypto=' + encryption
]
crypto_folder = 'encryption-off'
if encryption == 'openssl':
crypto_folder = 'crypto-openssl'
build_args.extend(['openssl-lib=' + env['OPENSSL_BUILD_PATH'],
'openssl-include=' + env['OPENSSL_INCLUDE']
])
build_args.append('release')
# Compile libtorrent with boost libraries and python bindings
with current_directory(join(build_dir, 'bindings/python')):
b2 = sh.Command(join(env['BOOST_ROOT'], 'b2'))
shprint(b2, *build_args, _env=env)
# Copy only the boost shared libraries into the libs folder. Because
# boost build two boost_python libraries, we force to search the lib
# into the corresponding build path.
b2_build_dir = 'build/clang-linux-arm/release/{encryption}/' \
'lt-visibility-hidden/'.format(encryption=crypto_folder)
boost_libs_dir = join(env['BOOST_BUILD_PATH'], 'bin.v2/libs')
for boost_lib in listdir(boost_libs_dir):
lib_path = get_lib_from(join(boost_libs_dir, boost_lib, b2_build_dir))
if lib_path:
lib_name = basename(lib_path)
shutil.copyfile(lib_path, join(ctx_libs_dir, lib_name))
# Copy libtorrent shared libraries into the right places
system_libtorrent = get_lib_from(join(build_dir, 'bin'))
if system_libtorrent:
shutil.copyfile(system_libtorrent,
join(ctx_libs_dir, 'libtorrent_rasterbar.so'))
python_libtorrent = get_lib_from(join(build_dir, 'bindings/python/bin'))
shutil.copyfile(python_libtorrent,
join(self.ctx.get_site_packages_dir(arch.arch), 'libtorrent.so'))
def get_recipe_env(self, arch):
# Use environment from boost recipe, cause we use b2 tool from boost
env = self.get_recipe('boost', self.ctx).get_recipe_env(arch)
if 'openssl' in recipe.ctx.recipe_build_order:
r = self.get_recipe('openssl', self.ctx)
env['OPENSSL_BUILD_PATH'] = r.get_build_dir(arch.arch)
env['OPENSSL_INCLUDE'] = join(r.get_build_dir(arch.arch), 'include')
env['OPENSSL_VERSION'] = r.version
return env
recipe = LibtorrentRecipe()

View file

@ -0,0 +1,10 @@
--- libtorrent/Jamfile 2016-01-17 23:52:45.000000000 +0100
+++ libtorrent-patch/Jamfile 2016-02-09 13:37:57.499561750 +0100
@@ -325,6 +325,7 @@
if $(type) = SHARED_LIB &&
( ! ( [ $(property-set).get <target-os> ] in windows cygwin ) )
{
+ return "libtorrent_rasterbar.so" ; # linked by python bindings .so
name = $(name).$(VERSION) ;
}

View file

@ -0,0 +1,20 @@
--- libtorrent/bindings/python/setup.py.orig 2018-11-26 22:21:48.772142135 +0100
+++ libtorrent/bindings/python/setup.py 2018-11-26 22:23:23.092141235 +0100
@@ -167,7 +167,7 @@
extra_compile = flags.parse(extra_cmd)
ext = [Extension(
- 'libtorrent',
+ 'libtorrent_rasterbar',
sources=sorted(source_list),
language='c++',
include_dirs=flags.include_dirs,
@@ -178,7 +178,7 @@
]
setup(
- name='python-libtorrent',
+ name='libtorrent',
version='1.2.0',
author='Arvid Norberg',
author_email='arvid@libtorrent.org',

View file

@ -0,0 +1,11 @@
--- libtorrent/bindings/python/Jamfile.orig 2018-12-07 16:46:50.851838981 +0100
+++ libtorrent/bindings/python/Jamfile 2018-12-07 16:49:09.099837663 +0100
@@ -113,7 +113,7 @@
if ( <toolset>gcc in $(properties) )
{
- result += <linkflags>-Wl,-Bsymbolic ;
+ result += <linkflags>-Wl,-soname=libtorrent.so,-Bsymbolic ;
}
}

View file

@ -0,0 +1,21 @@
--- boost/user-config.jam.orig 2018-12-07 14:16:45.911924859 +0100
+++ boost/user-config.jam 2018-12-07 14:20:16.243922853 +0100
@@ -9,6 +9,8 @@
local PYTHON_INCLUDE = [ os.environ PYTHON_INCLUDE ] ;
local PYTHON_LINK_VERSION = [ os.environ PYTHON_LINK_VERSION ] ;
local PYTHON_MAJOR_MINOR = [ os.environ PYTHON_MAJOR_MINOR ] ;
+local OPENSSL_BUILD_PATH = [ os.environ OPENSSL_BUILD_PATH ] ;
+local OPENSSL_VERSION = [ os.environ OPENSSL_VERSION ] ;
#using clang : $(ARCH) : $(ANDROID_BINARIES_PATH)/clang++ :
#<archiver>$(ANDROID_BINARIES_PATH)/llvm-ar
@@ -56,6 +58,9 @@
<linkflags>-Wl,-z,relro
<linkflags>-Wl,-z,now
<linkflags>-lc++_shared
+<linkflags>-L$(OPENSSL_BUILD_PATH)
+<linkflags>-lcrypto$(OPENSSL_VERSION)
+<linkflags>-lssl$(OPENSSL_VERSION)
<linkflags>-L$(PYTHON_ROOT)
<linkflags>-lpython$(PYTHON_LINK_VERSION)
<linkflags>-Wl,-O1