From b0ddacbe89efb268fe1e278eebcc9793edb804a1 Mon Sep 17 00:00:00 2001 From: Ryan McGuire Date: Wed, 10 Jul 2019 21:30:41 -0400 Subject: [PATCH] DOCKER-DEV: Local development in a docker container (#595) * DOCKER-DEV: Local lbry-android development via docker. * DOCKER-DEV: Add git submodule init --- .dockerignore | 3 + .gitignore | 2 + DOCKER-DEV.md | 156 +++++++++++++++++++++++++++++++++++++++ Dockerfile | 32 ++++++++ scripts/docker-build.sh | 95 ++++++++++++++++++++++++ scripts/docker-setup.sh | 27 +++++++ scripts/lbry-android.sh | 157 ++++++++++++++++++++++++++++++++++++++++ 7 files changed, 472 insertions(+) create mode 100644 .dockerignore create mode 100644 DOCKER-DEV.md create mode 100644 Dockerfile create mode 100755 scripts/docker-build.sh create mode 100755 scripts/docker-setup.sh create mode 100755 scripts/lbry-android.sh diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..be80dfc --- /dev/null +++ b/.dockerignore @@ -0,0 +1,3 @@ +.buildozer +.buildozer-downloads +.gradle diff --git a/.gitignore b/.gitignore index 47509c2..ed133ab 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,6 @@ .buildozer +.buildozer-downloads +.gradle app/node_modules/ bin buildozer.spec diff --git a/DOCKER-DEV.md b/DOCKER-DEV.md new file mode 100644 index 0000000..e865191 --- /dev/null +++ b/DOCKER-DEV.md @@ -0,0 +1,156 @@ +# lbry-android development environment inside docker + +[scripts/lbry-android.sh](scripts/lbry-android.sh) is a bash script to create a +docker container for lbry-android development. + +This is a hybrid approach where the apk is built inside docker, but you run +Android Studio, `adb`, and the app hot-reload bundler, directly on your host. + +## Features + + * Clones lbry-android source code to a directory on the host, and mounts it + inside the container. + * Installs all build dependencies inside the container, leaving your host + computer clean. + * Mounted `.buildozer` directory to save container space. (Docker containers + should stay under 10GB, and `.buildozer` is too big, so it has to stay in a + mounted host directory instead.) + * The biggest downloads are cached in `.buildozer-downloads` directory so you + can easily remove `.buildozer` and not have to re-download the large cryostax + NDK more than once. + * Instructions for installing on a real Android device, and for setting up + hot-reload. + * Only a handful of commands to go from zero->hero. + +## Requirements + +Install all of the following on your host computer: + + * Tested on Linux x86_64 with BASH. + * [Install Android Studio](https://developer.android.com/studio/). + * The normal install auto-creates a directory in `$HOME/Android/Sdk` to store + SDK downloads in your home directory. + * [Install nodejs](https://nodejs.org/en/download/package-manager/). + * [Install yarn](https://yarnpkg.com/lang/en/docs/install). + * [Install Docker](https://docs.docker.com/install/). + * Install `sudo` and give your user account access to run `sudo docker`. + +## Install + +Clone `lbry-android`: + +``` +LBRY_GIT=$HOME/git/vendor/lbryio/ +mkdir -p $LBRY_GIT +git clone https://github.com/lbryio/lbry-android.git $LBRY_GIT +cd $LBRY_GIT/lbry-android +git submodule update --init --recursive +``` + +Install a bash alias to the [scripts/lbry-android.sh](scripts/lbry-android.sh) +script: + +``` +echo "alias lbry-android=$LBRY_GIT/lbry-android/scripts/lbry-android.sh" >> $HOME/.bashrc + +source ~/.bashrc +``` + +## Usage + + * First create the base docker image: + + ``` + lbry-android docker-build + ``` + +(If anytime you change the docker build scripts in the [scripts](scripts) +subdirectory, you should rebuild the image again.) + + * Setup buildozer and install dependencies: + + ``` + lbry-android setup + ``` + + * Build the apk: + + ``` + lbry-android build + ``` + +The apk will be built and end up in the `lbry-android/bin/` subdirectory. + +## Running on a real Android device + +Once you have the apk built, you can install it on a real Android device to +start testing. + +Follow the Android documentation for [enabling USB +debugging](https://developer.android.com/studio/command-line/adb#Enabling) on +your device. + +Once you have enabled debugging, do the following: + + * Plug your device into a USB port on the host computer. + * Run: + + ```~/Android/Sdk/platform-tools/adb devices``` + + * ADB should list your device like so: + + ``` + [ryan@t440s lbry-android]$ adb devices + List of devices attached + HT71R0000000 device + ``` + + * The first time you connect, you should see a modal dialog on the device + asking to confirm the host id. You must approve it, or adb will fail to + connect. + + * If after trying several times, adb is still not connecting to your device, + and adb lists your device as `unauthorized`, you may need to [follow this + advice](https://stackoverflow.com/a/38380384/56560) and delete your + `$HOME/.android` directory. + + * Install the apk (whatever the current version is called) to the device: + + ``` + ~/Android/Sdk/platform-tools/adb install ./bin/browser-0.7.5-debug.apk + ``` + + * Open the app on your device, and follow the initial steps to get to the main + lbry-android browser page. + + * Create a tcp tunnel over the adb bridge, for the app to handle live-reload: + + ``` + ~/Android/Sdk/platform-tools/adb reverse tcp:8081 tcp:8081 + ``` + + * Start the live-reload server on the host: + + ``` + cd app/ + yarn install + yarn start + ``` + + * With your device currently running the lbry-android app, shake the device + from side to side, and a menu will appear. Choose `Enable Hot Reloading`. + + * The app should reload itself. + + * Now make a change to the source code to test hot reload. From the host, open + up the main page view source code: `app/src/component/uriBar/view.js`. Find + the line that reads `Search movies, music, and more` (This is the main search + bar at the top of the app.) - Change some of the text and save the file. + + * If hot-reload is working, within a few seconds the phone should reload the + page automatically and show the change that you made. + + * If the hot-reload does not work, try closing the app (dismiss the window from + the tab overview, via android square button) and reload the app. + + diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..87f7895 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,32 @@ +FROM thyrlian/android-sdk + +## Dependencies to run as root: +ENV DEBIAN_FRONTEND=noninteractive +RUN dpkg --add-architecture i386 && \ + apt-get -y update && \ + apt-get install -y \ + curl ca-certificates software-properties-common gpg-agent wget \ + python3.7 python3.7-dev python3-pip python2.7 python2.7-dev python3.7-venv \ + python-pip zlib1g-dev m4 zlib1g:i386 libc6-dev-i386 gawk nodejs npm unzip openjdk-8-jdk \ + autoconf autogen automake libtool libffi-dev build-essential \ + ccache git libncurses5:i386 libstdc++6:i386 \ + libgtk2.0-0:i386 libpangox-1.0-0:i386 libpangoxft-1.0-0:i386 libidn11:i386 && \ + npm install -g yarn react-native-cli && \ + pip2 install --upgrade cython setuptools && \ + pip2 install git+https://github.com/lbryio/buildozer.git@master && \ + ln -s /src/scripts/build-docker.sh /usr/local/bin/build && \ + adduser lbry-android --gecos GECOS --shell /bin/bash --disabled-password --home /home/lbry-android && \ + mkdir /home/lbry-android/.npm-packages && \ + echo "prefix=/home/lbry-android/.npm-packages" > /home/lbry-android/.npmrc && \ + chown -R lbry-android:lbry-android /home/lbry-android && \ + mkdir /src && \ + chown lbry-android:lbry-android /src && \ + mkdir /dist && \ + chown lbry-android:lbry-android /dist + +## Further setup done by lbry-android user: +USER lbry-android + +COPY scripts/docker-build.sh /home/lbry-android/bin/build +COPY scripts/docker-setup.sh /home/lbry-android/bin/setup +CMD ["/home/lbry-android/bin/build"] diff --git a/scripts/docker-build.sh b/scripts/docker-build.sh new file mode 100755 index 0000000..f1c9e06 --- /dev/null +++ b/scripts/docker-build.sh @@ -0,0 +1,95 @@ +#!/bin/bash +set -e + +exe() { ( echo "## $*"; $*; ) } + +ANDROID_SDK_LICENSE=/home/lbry-android/.buildozer/android/platform/android-sdk-23/licenses/android-sdk-license +## VERSION and REPO variables are optional: +## Use 'none' as a way to detect that none was provided by the user: +VERSION=${VERSION:-none} +REPO=${REPO:-none} + +if [ ! -f $ANDROID_SDK_LICENSE ]; then + echo "No Android SDK License provided." + echo "1) Download android-studio." + echo "2) Go to Settings -> Android SDKs -> Select Android 6.0 (Marshmellow)" + echo " and click the little download button and then agree to the license." + echo "3) Retry the build again." + exit 1 +fi + +## Two options for where the lbry-android source code comes from: +## 1) Clone directly from git via provided VERSION and REPO environment variables. +## 2) User may mount their own lbry-android source tree at /src +## Only one of these two options can be used at a time. +## VERSION is any valid git reference: commit, branch, or tag. +## REPO is the git repository URL to clone. + +## User must create their own buildozer.spec and google-services.json +## This may be done in their own fork of lbry-android, +## or done in their own clone mounted to /src + +if [ $VERSION != "none" ] || [ $REPO != "none" ]; then + # Build from a fresh git clone + + # No /src should be mounted if VERSION or REPO specified: + if mount | grep " /src"; then + echo "Cannot mount /src when VERSION and/or REPO variables are used." + echo "Aborting." + exit 1 + fi + + # A /dist directory should exist to copy final apk to: + if ! mount | grep " /dist"; then + echo "When using VERSION or REPO you must mount a /dist directory to put the final apk" + echo "Aborting." + exit 1 + fi + if [ $VERSION == "none" ]; then + VERSION=master + fi + if [ $REPO == "none" ]; then + REPO="https://github.com/lbryio/lbry-android.git" + fi + + ## Clone from $REPO and checkout $VERSION: + exe git clone $REPO /src + cd /src + exe git checkout $VERSION + + ## Create config from samples if none exists: + if [ ! -f /src/buildozer.spec ]; then + exe cp /src/buildozer.spec.sample /src/buildozer.spec + fi + if [ ! -f /src/p4a/pythonforandroid/bootstraps/lbry/build/templates/google-services.json ]; then + exe cp /src/p4a/pythonforandroid/bootstraps/lbry/build/templates/google-services.sample.json /src/p4a/pythonforandroid/bootstraps/lbry/build/templates/google-services.json + fi +fi + +if [ ! -f /src/buildozer.spec ]; then + echo "You must create a buildozer.spec file (See buildozer.spec.sample)" + echo "Aborting." + exit 1 +elif [ ! -f /src/p4a/pythonforandroid/bootstraps/lbry/build/templates/google-services.json ]; then + echo "You must create p4a/pythonforandroid/bootstraps/lbry/build/templates/google-services.json " + echo " (See p4a/pythonforandroid/bootstraps/lbry/build/templates/google-services.sample.json)" + echo "Aborting." + exit 1 +fi + +## Setup npm for non-root user: +NPM_PACKAGES="${HOME}/.npm-packages" +NODE_PATH="$NPM_PACKAGES/lib/node_modules:$NODE_PATH" +PATH="$NPM_PACKAGES/bin:$PATH" + +## Build: +cd /src/app +exe npm install +exe /src/app/bundle.sh + +cd /src +exe buildozer android debug + +if mount | grep " /dist"; then + exe cp /src/bin/* /dist +fi diff --git a/scripts/docker-setup.sh b/scripts/docker-setup.sh new file mode 100755 index 0000000..3b84887 --- /dev/null +++ b/scripts/docker-setup.sh @@ -0,0 +1,27 @@ +#!/bin/bash +set -e + +exe() { ( echo "## $*"; $*; ) } + +BUILDOZER_HOME=$HOME/.buildozer +BUILDOZER_DOWNLOADS=$HOME/.buildozer-downloads + +exe mkdir -p $BUILDOZER_HOME/android/platform +exe wget -c 'https://dl.google.com/android/android-sdk_r23-linux.tgz' -P $BUILDOZER_DOWNLOADS +exe wget -c 'https://dl.google.com/android/repository/platform-28_r06.zip' -P $BUILDOZER_DOWNLOADS +exe wget -c 'https://dl.google.com/android/repository/build-tools_r26.0.2-linux.zip' -P $BUILDOZER_DOWNLOADS +exe tar -xvf $BUILDOZER_DOWNLOADS/android-sdk_r23-linux.tgz -C $BUILDOZER_HOME/android/platform/ +exe mv $BUILDOZER_HOME/android/platform/android-sdk-linux $BUILDOZER_HOME/android/platform/android-sdk-23 +exe unzip $BUILDOZER_DOWNLOADS/platform-28_r06.zip -d $BUILDOZER_HOME/android/platform/android-sdk-23/platforms +exe mv $BUILDOZER_HOME/android/platform/android-sdk-23/platforms/android-9 $BUILDOZER_HOME/android/platform/android-sdk-23/platforms/android-28 +exe mkdir -p $BUILDOZER_HOME/android/platform/android-sdk-23/build-tools +exe unzip $BUILDOZER_DOWNLOADS/build-tools_r26.0.2-linux.zip -d $BUILDOZER_HOME/android/platform/android-sdk-23/build-tools +exe wget -c 'https://www.crystax.net/download/crystax-ndk-10.3.2-linux-x86_64.tar.xz' -P $BUILDOZER_DOWNLOADS +exe tar -xvf $BUILDOZER_DOWNLOADS/crystax-ndk-10.3.2-linux-x86_64.tar.xz -C $BUILDOZER_HOME/android/ +exe rm -rf $BUILDOZER_HOME/android/crystax-ndk-10.3.2/platforms/android-9 +exe ln -s $BUILDOZER_HOME/android/crystax-ndk-10.3.2/platforms/android-21 $BUILDOZER_HOME/android/crystax-ndk-10.3.2/platforms/android-9 +exe mkdir -p $BUILDOZER_HOME/android/crystax-ndk-10.3.2/build/tools/ +exe mkdir -p $BUILDOZER_HOME/android/crystax-ndk-10.3.2/platforms/android-21/arch-arm/usr/include/crystax/bionic/libc/include/sys/ +exe cp /src/scripts/build-target-python.sh ~/.buildozer/android/crystax-ndk-10.3.2/build/tools/build-target-python.sh +exe cp /src/scripts/mangled-glibc-syscalls.h ~/.buildozer/android/crystax-ndk-10.3.2/platforms/android-21/arch-arm/usr/include/crystax/bionic/libc/include/sys/mangled-glibc-syscalls.h +exe mv $BUILDOZER_HOME/android/platform/android-sdk-23/build-tools/android-8.1.0 $BUILDOZER_HOME/android/platform/android-sdk-23/build-tools/26.0.2 diff --git a/scripts/lbry-android.sh b/scripts/lbry-android.sh new file mode 100755 index 0000000..53f2d91 --- /dev/null +++ b/scripts/lbry-android.sh @@ -0,0 +1,157 @@ +#!/bin/bash + +( + ANDROID_STUDIO_SDK=${ANDROID_STUDIO_SDK:-$HOME/Android/Sdk} + LBRY_ANDROID_HOME=${LBRY_ANDROID_HOME:-$HOME/git/vendor/lbryio/lbry-android} + LBRY_ANDROID_BUILDOZER_HOME=${LBRY_ANDROID_BUILDOZER_HOME:-$LBRY_ANDROID_HOME/.buildozer} + LBRY_ANDROID_BUILDOZER_DOWNLOADS=${LBRY_ANDROID_BUILDOZER_DOWNLOADS:-$LBRY_ANDROID_HOME/.buildozer-downloads} + LBRY_ANDROID_REPO=${LBRY_ANDROID_REPO:-https://www.github.com/lbryio/lbry-android} + LBRY_ANDROID_IMAGE=${LBRY_ANDROID_IMAGE:-lbry-android:local} + + ## Logger utility: + exe() { ( echo "## $*"; $*; ) } + + ## Confirmation dialog: + ## usage: prompt_confirm "Overwrite File?" || return 1 + prompt-confirm() { + while true; do + read -r -n 1 -p "${1:-Continue?} [y/n]: " REPLY + case $REPLY in + [yY]) echo ; return 0 ;; + [nN]) echo ; return 1 ;; + *) printf " \033[31m %s \n\033[0m" "invalid input" + esac + done + } + + check-dependencies() { + if [ ! -d $ANDROID_STUDIO_SDK ]; then + echo "Error: $ANDROID_STUDIO_SDK not found." + echo "You must download and install Android Studio:" + echo "https://developer.android.com/studio/" + return 1 + elif ! which docker > /dev/null; then + echo "You must install docker and setup sudo to access it as regular user." + return 1 + elif ! which sudo > /dev/null; then + echo "You must install sudo and setup user account for sudo priviliges." + return 1 + fi + } + + check-src-dir() { + if [ ! -d $LBRY_ANDROID_HOME ]; then + echo "Cannot find lbry-android source code:" + echo "$LBRY_ANDROID_HOME" + echo "" + echo "Clone the lbry-android repository first, run:" + echo " lbry-android clone" + echo "" + echo "To use a different path set the LBRY_ANDROID_HOME variable." + return 1 + fi + } + + setup() { + set -e + check-src-dir || return 1 + exe $HOME/Android/Sdk/tools/bin/sdkmanager "platforms;android-27" + if [ -d $LBRY_ANDROID_BUILDOZER_HOME ]; then + echo "Buildozer path already exists: $LBRY_ANDROID_BUILDOZER_HOME" + echo "If you would like to re-install from scratch, delete that directory first." + else + mkdir -p $LBRY_ANDROID_BUILDOZER_HOME + mkdir -p $LBRY_ANDROID_BUILDOZER_DOWNLOADS + exe sudo docker run --rm -it \ + -v $LBRY_ANDROID_HOME:/src \ + -v $LBRY_ANDROID_BUILDOZER_HOME:/home/lbry-android/.buildozer/ \ + -v $LBRY_ANDROID_BUILDOZER_DOWNLOADS:/home/lbry-android/.buildozer-downloads/ \ + $LBRY_ANDROID_IMAGE \ + /home/lbry-android/bin/setup + fi + } + + ## Build lbry-android docker image + docker-build(){ + check-src-dir || return 1 + sudo docker build -t $LBRY_ANDROID_IMAGE $LBRY_ANDROID_HOME + } + + ## Build lbry-android apk + ANDROID_SDK_LICENSE=$ANDROID_STUDIO_SDK/licenses/android-sdk-license + build(){ + if [ ! -f $ANDROID_SDK_LICENSE ]; then + echo "Android SDK license file not found:" + echo $ANDROID_SDK_LICENSE + echo "Open Android-Studio, download the SDK and accept the license agreement." + return 1 + fi + if [ ! -d $LBRY_ANDROID_BUILDOZER_HOME ]; then + echo "Buildozer root not found: $LBRY_ANDROID_BUILDOZER_HOME" + echo "Run: lbry-android setup" + return 1 + fi + check-src-dir || return 1 + mkdir -p $LBRY_ANDROID_HOME/.gradle + exe sudo docker run --rm -it \ + -v $LBRY_ANDROID_HOME:/src \ + -v $LBRY_ANDROID_BUILDOZER_HOME:/home/lbry-android/.buildozer/ \ + -v $LBRY_ANDROID_HOME/.gradle:/home/lbry-android/.gradle/ \ + -v $ANDROID_SDK_LICENSE:/home/lbry-android/.buildozer/android/platform/android-sdk-23/licenses/android-sdk-license \ + $LBRY_ANDROID_IMAGE + } + + clone() { + if [ -d $LBRY_ANDROID_HOME ]; then + echo "$LBRY_ANDROID_HOME already exists." + echo "If you wish to use a different path set the LBRY_ANDROID_HOME variable." + return 1 + else + exe git clone $LBRY_ANDROID_REPO $LBRY_ANDROID_HOME + ( + cd $LBRY_ANDROID_HOME + exe git submodule update --init --recursive + ) + echo "" + echo "lbry-android clone complete." + echo "LBRY_ANDROID_HOME=$LBRY_ANDROID_HOME" + fi + } + + SUBCOMMANDS_NO_ARGS=(setup clone docker-build build) + SUBCOMMANDS_PASS_ARGS=(none) + + check-dependencies || return 1 + + if printf '%s\n' ${SUBCOMMANDS_NO_ARGS[@]} | grep -q -P "^$1$"; then + ## Subcommands that take no arguments: + ( + set -e + if [ "$#" -eq 1 ]; then + $* + else + echo "$1 does not take any additional arguments" + fi + ) + elif printf '%s\n' ${SUBCOMMANDS_PASS_ARGS[@]} | grep -q -P "^$1$"; then + ## Subcommands that pass all arguments: + ( + set -e + $* + ) + else + if [[ $# -gt 0 ]]; then + echo "## Invalid command: $1" + else + echo "## Must specify a command:" + fi + echo "" + echo "## lbry-android setup" + echo "## - Sets up buildozer and downloads dependencies" + echo "## lbry-android docker-build" + echo "## - Builds the lbry-android docker container" + echo "## lbry-android build" + echo "## - Builds the lbry-android apk" + fi +) +