diff --git a/.appveyor.yml b/.appveyor.yml index 54dd0cdcd..acd3fadb0 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -1,13 +1,37 @@ version: 1.0.{build} +environment: + GH_TOKEN: + secure: LiI5jyuHUw6XbH4kC3gP1HX4P/v4rwD/gCNtaFhQu2AvJz1/1wALkp5ECnIxRyS +# key_pass: +# secure: u6DydPcdrUJlxGL9uc7yQRYG8+5rY6aAEE9nfCSzFyNzZlX9NniOp8Uh5ZKQqX7bGEngLI6ipbLfiJvn0XFnhbn2iTkOuMqOXVJVOehvwlQ= +# pfx_key: +# secure: 1mwqyRy7hDqDjDK+TIAoaXyXzpNgwruFNA6TPkinUcVM7A+NLD33RQLnfnwVy+R5ovD2pUfhQ6+N0Fqebv6tZh436LIEsock+6IOdpgFwrg= -notifications: - - provider: Slack - incoming_webhook: - secure: LuxwG5OZnnA//gmSXzCKu8/FRqYjCgGfVFqajSsGHeQ1HQNp7rYNhQpsO8/3PK63xKJj3wzt86DJekf9q9Q5OcHa9AHXUQbEveX0psd7elw= +# +#notifications: +# - provider: Slack +# incoming_webhook: +# secure: LuxwG5OZnnA//gmSXzCKu8/FRqYjCgGfVFqajSsGHeQ1HQNp7rYNhQpsO8/3PK63xKJj3wzt86DJekf9q9Q5OcHa9AHXUQbEveX0psd7elw= clone_folder: c:\projects\lbry build_script: - - echo "Not currently building on appveyor" +- cd C:\projects\lbry\build\ +- python set_version.py +- python set_build.py +- ps: .\build.ps1 +- python zip_daemon.py +- python release_on_tag.py +- dir .\build +- dir .\dist + + +artifacts: +- path: build\dist\*.zip + name: lbrynet-daemon +#- path: build/exe.win32-2.7/ +# name: lbry-portable +#- path: packaging/windows/lbry-win32-app/LBRY-URI.reg +# name: LBRY-URI diff --git a/.gitignore b/.gitignore index 74027cd71..e3d0a36f1 100644 --- a/.gitignore +++ b/.gitignore @@ -9,8 +9,10 @@ *.prof .#* -/build -/dist +/build/build +/build/dist +/bulid/requirements_base.txt + /lbrynet.egg-info /docs_build /lbry-venv diff --git a/build/build.ps1 b/build/build.ps1 new file mode 100644 index 000000000..07aeeaaf6 --- /dev/null +++ b/build/build.ps1 @@ -0,0 +1,24 @@ +$env:Path += ";C:\MinGW\bin\" + +$env:Path += ";C:\Program Files (x86)\Windows Kits\10\bin\x86\" +gcc --version +mingw32-make --version + +# build/install miniupnpc manually +tar zxf miniupnpc-1.9.tar.gz +cd miniupnpc-1.9 +mingw32-make.exe -f Makefile.mingw +python.exe setupmingw32.py build --compiler=mingw32 +python.exe setupmingw32.py install +cd ..\ +Remove-Item -Recurse -Force miniupnpc-1.9 + +# copy requirements from lbry, but remove gmpy and miniupnpc (installed manually) +Get-Content ..\requirements.txt | Select-String -Pattern 'gmpy|miniupnpc' -NotMatch | Out-File requirements_base.txt +# add in gmpy wheel +Add-Content requirements.txt "./gmpy-1.17-cp27-none-win32.whl" + +pip.exe install -r requirements.txt + +pyinstaller -y daemon.onefile.spec +pyinstaller -y cli.onefile.spec \ No newline at end of file diff --git a/build/build.sh b/build/build.sh new file mode 100755 index 000000000..75247ae4d --- /dev/null +++ b/build/build.sh @@ -0,0 +1,61 @@ +#!/bin/bash + +set -euo pipefail +set -x + +ROOT="$( cd "$( dirname "${BASH_SOURCE[0]}" )/.." && pwd )" +cd "$ROOT" +BUILD_DIR="$ROOT/build" + +FULL_BUILD="${FULL_BUILD:-false}" +if [ -n "${TEAMCITY_VERSION:-}" -o -n "${APPVEYOR:-}" ]; then + FULL_BUILD="true" +fi + +[ -d "$BUILD_DIR/bulid" ] && rm -rf "$BUILD_DIR/build" +[ -d "$BUILD_DIR/dist" ] && rm -rf "$BUILD_DIR/dist" + +if [ "$FULL_BUILD" == "true" ]; then + # install dependencies + $BUILD_DIR/prebuild.sh + + VENV="$BUILD_DIR/venv" + if [ -d "$VENV" ]; then + rm -rf "$VENV" + fi + virtualenv "$VENV" + set +u + source "$VENV/bin/activate" + set -u +fi + + +cp "$ROOT/requirements.txt" "$BUILD_DIR/requirements_base.txt" +( + cd "$BUILD_DIR" + pip install -r requirements.txt +) + +if [ "$FULL_BUILD" == "true" ]; then + python "$BUILD_DIR/set_version.py" + python "$BUILD_DIR/set_build.py" +fi + +( + cd "$BUILD_DIR" + pyinstaller -y daemon.onefile.spec + pyinstaller -y cli.onefile.spec +) + +python "$BUILD_DIR/zip_daemon.py" + +if [ "$FULL_BUILD" == "true" ]; then + # electron-build has a publish feature, but I had a hard time getting + # it to reliably work and it also seemed difficult to configure. Not proud of + # this, but it seemed better to write my own. + python "$BUILD_DIR/release_on_tag.py" + + deactivate +fi + +echo 'Build complete.' diff --git a/build/cli.onefile.spec b/build/cli.onefile.spec new file mode 100644 index 000000000..43ada14b7 --- /dev/null +++ b/build/cli.onefile.spec @@ -0,0 +1,59 @@ +# -*- mode: python -*- +import platform +import os + + +dir = 'build'; +cwd = os.getcwd() +if os.path.basename(cwd) != dir: + raise Exception('pyinstaller build needs to be run from the ' + dir + ' directory') +repo_base = os.path.abspath(os.path.join(cwd, '..')) + + +system = platform.system() +if system == 'Darwin': + icns = os.path.join(repo_base, 'build', 'icon.icns') +elif system == 'Linux': + icns = os.path.join(repo_base, 'build', 'icons', '256x256.png') +elif system == 'Windows': + icns = os.path.join(repo_base, 'build', 'icons', 'lbry256.ico') +else: + print 'Warning: System {} has no icons'.format(system) + icns = None + +block_cipher = None + + +a = Analysis( + ['cli.py'], + pathex=[cwd], + binaries=None, + datas=[], + hiddenimports=[], + hookspath=[], + runtime_hooks=[], + excludes=[], + win_no_prefer_redirects=False, + win_private_assemblies=False, + cipher=block_cipher +) + +pyz = PYZ( + a.pure, + a.zipped_data, + cipher=block_cipher +) + +exe = EXE( + pyz, + a.scripts, + a.binaries, + a.zipfiles, + a.datas, + name='lbrynet-cli', + debug=False, + strip=False, + upx=True, + console=True, + icon=icns +) diff --git a/build/cli.py b/build/cli.py new file mode 100644 index 000000000..075c13b7f --- /dev/null +++ b/build/cli.py @@ -0,0 +1,7 @@ +from lbrynet.lbrynet_daemon import DaemonCLI +import logging + +logging.basicConfig() + +if __name__ == '__main__': + DaemonCLI.main() diff --git a/build/daemon.onefile.spec b/build/daemon.onefile.spec new file mode 100644 index 000000000..80e9e7226 --- /dev/null +++ b/build/daemon.onefile.spec @@ -0,0 +1,78 @@ +# -*- mode: python -*- +import platform +import os + +import lbryum + + +dir = 'build'; +cwd = os.getcwd() +if os.path.basename(cwd) != dir: + raise Exception('pyinstaller build needs to be run from the ' + dir + ' directory') +repo_base = os.path.abspath(os.path.join(cwd, '..')) + + +system = platform.system() +if system == 'Darwin': + icns = os.path.join(repo_base, 'build', 'icon.icns') +elif system == 'Linux': + icns = os.path.join(repo_base, 'build', 'icons', '256x256.png') +elif system == 'Windows': + icns = os.path.join(repo_base, 'build', 'icons', 'lbry256.ico') +else: + print 'Warning: System {} has no icons'.format(system) + icns = None + + +block_cipher = None + + +languages = ( + 'chinese_simplified.txt', 'japanese.txt', 'spanish.txt', + 'english.txt', 'portuguese.txt' +) + + +datas = [ + ( + os.path.join(os.path.dirname(lbryum.__file__), 'wordlist', language), + 'lbryum/wordlist' + ) + for language in languages +] + + +a = Analysis( + ['daemon.py'], + pathex=[cwd], + binaries=None, + datas=datas, + hiddenimports=[], + hookspath=[], + runtime_hooks=[], + excludes=[], + win_no_prefer_redirects=False, + win_private_assemblies=False, + cipher=block_cipher +) + + +pyz = PYZ( + a.pure, a.zipped_data, + cipher=block_cipher +) + + +exe = EXE( + pyz, + a.scripts, + a.binaries, + a.zipfiles, + a.datas, + name='lbrynet-daemon', + debug=False, + strip=False, + upx=True, + console=True, + icon=icns +) diff --git a/build/daemon.py b/build/daemon.py new file mode 100644 index 000000000..2ed0360ab --- /dev/null +++ b/build/daemon.py @@ -0,0 +1,4 @@ +from lbrynet.lbrynet_daemon import DaemonControl + +if __name__ == '__main__': + DaemonControl.start() diff --git a/packaging/windows/libs/gmpy-1.17-cp27-none-win32.whl b/build/gmpy-1.17-cp27-none-win32.whl similarity index 100% rename from packaging/windows/libs/gmpy-1.17-cp27-none-win32.whl rename to build/gmpy-1.17-cp27-none-win32.whl diff --git a/build/icons/128x128.png b/build/icons/128x128.png new file mode 100644 index 000000000..de083fcbc Binary files /dev/null and b/build/icons/128x128.png differ diff --git a/build/icons/256x256.png b/build/icons/256x256.png new file mode 100644 index 000000000..659957406 Binary files /dev/null and b/build/icons/256x256.png differ diff --git a/build/icons/32x32.png b/build/icons/32x32.png new file mode 100644 index 000000000..ece10998c Binary files /dev/null and b/build/icons/32x32.png differ diff --git a/build/icons/48x48.png b/build/icons/48x48.png new file mode 100644 index 000000000..bba3a8e9d Binary files /dev/null and b/build/icons/48x48.png differ diff --git a/build/icons/96x96.png b/build/icons/96x96.png new file mode 100644 index 000000000..195c31ea7 Binary files /dev/null and b/build/icons/96x96.png differ diff --git a/build/icons/lbry128.ico b/build/icons/lbry128.ico new file mode 100644 index 000000000..3cb6f992d Binary files /dev/null and b/build/icons/lbry128.ico differ diff --git a/build/icons/lbry16.ico b/build/icons/lbry16.ico new file mode 100644 index 000000000..40d849628 Binary files /dev/null and b/build/icons/lbry16.ico differ diff --git a/build/icons/lbry256.ico b/build/icons/lbry256.ico new file mode 100644 index 000000000..f8a33ff7c Binary files /dev/null and b/build/icons/lbry256.ico differ diff --git a/build/icons/lbry32.ico b/build/icons/lbry32.ico new file mode 100644 index 000000000..6a6219b50 Binary files /dev/null and b/build/icons/lbry32.ico differ diff --git a/build/icons/lbry48.ico b/build/icons/lbry48.ico new file mode 100644 index 000000000..95c0947fa Binary files /dev/null and b/build/icons/lbry48.ico differ diff --git a/build/icons/lbry96.ico b/build/icons/lbry96.ico new file mode 100644 index 000000000..25572bc9c Binary files /dev/null and b/build/icons/lbry96.ico differ diff --git a/build/miniupnpc-1.9.tar.gz b/build/miniupnpc-1.9.tar.gz new file mode 100644 index 000000000..85deda499 Binary files /dev/null and b/build/miniupnpc-1.9.tar.gz differ diff --git a/build/prebuild.sh b/build/prebuild.sh new file mode 100755 index 000000000..3f6e7a2b3 --- /dev/null +++ b/build/prebuild.sh @@ -0,0 +1,81 @@ +#!/bin/bash + +set -euo pipefail +set -x + + +LINUX=false +OSX=false + +if [ "$(uname)" == "Darwin" ]; then + OSX=true +elif [ "$(expr substr $(uname -s) 1 5)" == "Linux" ]; then + LINUX=true +else + echo "Platform detection failed" + exit 1 +fi + + +SUDO='' +if $LINUX && (( $EUID != 0 )); then + SUDO='sudo' +fi + +cmd_exists() { + command -v "$1" >/dev/null 2>&1 + return $? +} + +set +eu +GITUSERNAME=$(git config --global --get user.name) +if [ -z "$GITUSERNAME" ]; then + git config --global user.name "$(whoami)" +fi +GITEMAIL=$(git config --global --get user.email) +if [ -z "$GITEMAIL" ]; then + git config --global user.email "$(whoami)@lbry.io" +fi +set -eu + + +if $LINUX; then + INSTALL="$SUDO apt-get install --no-install-recommends -y" + $INSTALL build-essential libssl-dev libffi-dev libgmp3-dev python2.7-dev +elif $OSX && ! cmd_exists brew ; then + /usr/bin/ruby -e "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)" +fi + + +if ! cmd_exists python; then + if $LINUX; then + $INSTALL python2.7 + elif $OSX; then + brew install python + fi +fi + +PYTHON_VERSION=$(python -c 'import sys; print(".".join(map(str, sys.version_info[:2])))') +if [ "$PYTHON_VERSION" != "2.7" ]; then + echo "Python 2.7 required" + exit 1 +fi + +if ! cmd_exists pip; then + if $LINUX; then + $INSTALL python-pip + $SUDO pip install --upgrade pip + else + echo "Pip required" + exit 1 + fi +fi + +if $LINUX && [ "$(pip list --format=columns | grep setuptools | wc -l)" -ge 1 ]; then + #$INSTALL python-setuptools + $SUDO pip install setuptools +fi + +if ! cmd_exists virtualenv; then + $SUDO pip install virtualenv +fi diff --git a/build/release_on_tag.py b/build/release_on_tag.py new file mode 100644 index 000000000..fa8537f8d --- /dev/null +++ b/build/release_on_tag.py @@ -0,0 +1,128 @@ +import glob +import json +import os +import subprocess +import sys + +import github +import requests +import uritemplate + + +def main(): + this_dir = os.path.dirname(os.path.realpath(__file__)) + try: + current_tag = subprocess.check_output( + ['git', 'describe', '--exact-match', 'HEAD']).strip() + except subprocess.CalledProcessError: + print 'Stopping as we are not currently on a tag' + return + + if 'GH_TOKEN' not in os.environ: + print 'Must set GH_TOKEN in order to publish assets to a release' + return + + gh_token = os.environ['GH_TOKEN'] + auth = github.Github(gh_token) + repo = auth.get_repo('lbryio/lbry') + + if not check_repo_has_tag(repo, current_tag): + print 'Tag {} is not in repo {}'.format(current_tag, repo) + # TODO: maybe this should be an error + return + + daemon_zip = glob.glob(this_dir + '/dist/*.zip')[0] + release = get_release(repo, current_tag) + upload_asset(release, daemon_zip, gh_token) + + +def check_repo_has_tag(repo, target_tag): + tags = repo.get_tags().get_page(0) + for tag in tags: + if tag.name == target_tag: + return True + return False + + +def get_release(current_repo, current_tag): + for release in current_repo.get_releases(): + if release.tag_name == current_tag: + return release + raise Exception('No release for {} was found'.format(current_tag)) + + +def upload_asset(release, asset_to_upload, token): + basename = os.path.basename(asset_to_upload) + if is_asset_already_uploaded(release, basename): + return + count = 0 + while count < 10: + try: + return _upload_asset(release, asset_to_upload, token, _requests_uploader) + except Exception: + print 'Failed uploading on attempt {}'.format(count + 1) + count += 1 + + +def _upload_asset(release, asset_to_upload, token, uploader): + basename = os.path.basename(asset_to_upload) + upload_uri = uritemplate.expand(release.upload_url, {'name': basename}) + output = uploader(upload_uri, asset_to_upload, token) + if 'errors' in output: + raise Exception(output) + else: + print 'Successfully uploaded to {}'.format(output['browser_download_url']) + + +# requests doesn't work on windows / linux / osx. +def _requests_uploader(upload_uri, asset_to_upload, token): + print 'Using requests to upload {} to {}'.format(asset_to_upload, upload_uri) + with open(asset_to_upload, 'rb') as f: + response = requests.post(upload_uri, data=f, auth=('', token)) + return response.json() + + +# curl -H "Content-Type: application/json" -X POST -d '{"username":"xyz","password":"xyz"}' http://localhost:3000/api/login + + +def _curl_uploader(upload_uri, asset_to_upload, token): + # using requests.post fails miserably with SSL EPIPE errors. I spent + # half a day trying to debug before deciding to switch to curl. + # + # TODO: actually set the content type + print 'Using curl to upload {} to {}'.format(asset_to_upload, upload_uri) + cmd = [ + 'curl', + '-sS', + '-X', 'POST', + '-u', ':{}'.format(os.environ['GH_TOKEN']), + '--header', 'Content-Type: application/octet-stream', + '--data-binary', '@-', + upload_uri + ] + # '-d', '{"some_key": "some_value"}', + print 'Calling curl:' + print cmd + print + with open(asset_to_upload, 'rb') as fp: + p = subprocess.Popen(cmd, stdin=fp, stderr=subprocess.PIPE, stdout=subprocess.PIPE) + stdout, stderr = p.communicate() + print 'curl return code:', p.returncode + if stderr: + print 'stderr output from curl:' + print stderr + print 'stdout from curl:' + print stdout + return json.loads(stdout) + + +def is_asset_already_uploaded(release, basename): + for asset in release.raw_data['assets']: + if asset['name'] == basename: + print 'File {} has already been uploaded to {}'.format(basename, release.tag_name) + return True + return False + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/build/requirements.txt b/build/requirements.txt new file mode 100644 index 000000000..43275963b --- /dev/null +++ b/build/requirements.txt @@ -0,0 +1,13 @@ +# install daemon requirements (copied from root, with possible modifications. see build.sh, build.ps1) +-r requirements_base.txt + +# install daemon itself. make sure you run `pip install` from this dir. this is how you do relative file paths with pip +file:../. + +# install other build requirements +PyInstaller==3.2.1 +requests[security]==2.13.0 +GitPython==2.1.1 +PyGithub==1.32 +uritemplate==3.0.0 +git+https://github.com/lbryio/bumpversion.git diff --git a/build/requirements_base.txt b/build/requirements_base.txt new file mode 100644 index 000000000..c942e551b --- /dev/null +++ b/build/requirements_base.txt @@ -0,0 +1 @@ +THIS FILE GETS FILLED IN BY BUILD SCRIPT AND INCLUDED IN requirements.txt diff --git a/build/set_build.py b/build/set_build.py new file mode 100644 index 000000000..bd508345e --- /dev/null +++ b/build/set_build.py @@ -0,0 +1,29 @@ +"""Set the build version to be 'dev', 'qa', 'rc', 'release'""" + +import os.path +import re +import subprocess +import sys + + +def main(): + build = get_build() + root_dir = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) + with open(os.path.join(root_dir, 'lbrynet', 'build_type.py'), 'w') as f: + f.write('BUILD = "{}"'.format(build)) + + +def get_build(): + try: + tag = subprocess.check_output(['git', 'describe', '--exact-match']).strip() + if re.match('v\d+\.\d+\.\d+rc\d+', tag): + return 'rc' + else: + return 'release' + except subprocess.CalledProcessError: + # if the build doesn't have a tag + return 'qa' + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/build/set_version.py b/build/set_version.py new file mode 100644 index 000000000..a2b69fb9a --- /dev/null +++ b/build/set_version.py @@ -0,0 +1,52 @@ +"""Set the package version to the output of `git describe`""" + +import argparse +import os.path +import re +import subprocess +import sys + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('--version', help="defaults to the output of `git describe`") + args = parser.parse_args() + if args.version: + version = args.version + else: + tag = subprocess.check_output(['git', 'describe']).strip() + try: + version = get_version_from_tag(tag) + except InvalidVersionTag: + # this should be an error but its easier to handle here + # than in the calling scripts. + print 'Tag cannot be converted to a version. Exiting.' + return + set_version(version) + + +class InvalidVersionTag(Exception): + pass + + +def get_version_from_tag(tag): + match = re.match('v([\d.]+)', tag) + if match: + return match.group(1) + else: + raise InvalidVersionTag('Failed to parse version from tag {}'.format(tag)) + + +def set_version(version): + root_dir = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) + with open(os.path.join(root_dir, 'lbrynet', '__init__.py'), 'w') as fp: + fp.write(LBRYNET_TEMPLATE.format(version=version)) + + +LBRYNET_TEMPLATE = """ +__version__ = "{version}" +version = tuple(__version__.split('.')) +""" + +if __name__ == '__main__': + sys.exit(main()) diff --git a/build/zip_daemon.py b/build/zip_daemon.py new file mode 100644 index 000000000..ef3f5e222 --- /dev/null +++ b/build/zip_daemon.py @@ -0,0 +1,29 @@ +import os +import platform +import subprocess +import sys +import zipfile + + +def main(): + this_dir = os.path.dirname(os.path.realpath(__file__)) + tag = subprocess.check_output(['git', 'describe']).strip() + zipfilename = 'lbrynet-daemon-{}-{}.zip'.format(tag, get_system_label()) + full_filename = os.path.join(this_dir, 'dist', zipfilename) + executables = ['lbrynet-daemon', 'lbrynet-cli'] + ext = '.exe' if platform.system() == 'Windows' else '' + with zipfile.ZipFile(full_filename, 'w') as myzip: + for executable in executables: + myzip.write(os.path.join(this_dir, 'dist', executable + ext), executable + ext) + + +def get_system_label(): + system = platform.system() + if system == 'Darwin': + return 'macos' + else: + return system.lower() + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/setup.py b/setup.py index c16604770..921c5e6e3 100644 --- a/setup.py +++ b/setup.py @@ -11,7 +11,9 @@ from setuptools import setup, find_packages # dependencies of the lbrynet library. requirements.txt includes # dependencies of dependencies and specific versions that we know # all work together. -# See https://packaging.python.org/requirements/ for more details. +# +# See https://packaging.python.org/requirements/ and +# https://caremad.io/posts/2013/07/setup-vs-requirement/ for more details. requires = [ 'Twisted', 'appdirs',