build daemon separately in teamcity

.appveyor.yml

@@ -1,13 +1,37 @@
 version: 1.0.{build}
 
+environment:
+  GH_TOKEN:
+    secure: LiI5jyuHUw6XbH4kC3gP1HX4P/v4rwD/gCNtaFhQu2AvJz1/1wALkp5ECnIxRyS
+#  key_pass:
+#    secure: u6DydPcdrUJlxGL9uc7yQRYG8+5rY6aAEE9nfCSzFyNzZlX9NniOp8Uh5ZKQqX7bGEngLI6ipbLfiJvn0XFnhbn2iTkOuMqOXVJVOehvwlQ=
+#  pfx_key:
+#    secure: 1mwqyRy7hDqDjDK+TIAoaXyXzpNgwruFNA6TPkinUcVM7A+NLD33RQLnfnwVy+R5ovD2pUfhQ6+N0Fqebv6tZh436LIEsock+6IOdpgFwrg=
 
-notifications:
-  - provider: Slack
-    incoming_webhook:
-      secure: LuxwG5OZnnA//gmSXzCKu8/FRqYjCgGfVFqajSsGHeQ1HQNp7rYNhQpsO8/3PK63xKJj3wzt86DJekf9q9Q5OcHa9AHXUQbEveX0psd7elw=
+#
+#notifications:
+#  - provider: Slack
+#    incoming_webhook:
+#      secure: LuxwG5OZnnA//gmSXzCKu8/FRqYjCgGfVFqajSsGHeQ1HQNp7rYNhQpsO8/3PK63xKJj3wzt86DJekf9q9Q5OcHa9AHXUQbEveX0psd7elw=
 
 
 clone_folder: c:\projects\lbry
 
 build_script:
-  - echo "Not currently building on appveyor"
+- cd C:\projects\lbry\build\
+- python set_version.py
+- python set_build.py
+- ps: .\build.ps1
+- python zip_daemon.py
+- python release_on_tag.py
+- dir .\build
+- dir .\dist
+
+
+artifacts:
+- path: build\dist\*.zip
+  name: lbrynet-daemon
+#- path: build/exe.win32-2.7/
+#  name: lbry-portable
+#- path: packaging/windows/lbry-win32-app/LBRY-URI.reg
+#  name: LBRY-URI

.gitignore

@@ -9,8 +9,10 @@
 *.prof
 .#*
 
-/build
-/dist
+/build/build
+/build/dist
+/bulid/requirements_base.txt
+
 /lbrynet.egg-info
 /docs_build
 /lbry-venv

build/build.ps1

@@ -0,0 +1,24 @@
+$env:Path += ";C:\MinGW\bin\"
+
+$env:Path += ";C:\Program Files (x86)\Windows Kits\10\bin\x86\"
+gcc --version
+mingw32-make --version
+
+# build/install miniupnpc manually
+tar zxf miniupnpc-1.9.tar.gz
+cd miniupnpc-1.9
+mingw32-make.exe -f Makefile.mingw
+python.exe setupmingw32.py build --compiler=mingw32
+python.exe setupmingw32.py install
+cd ..\
+Remove-Item -Recurse -Force miniupnpc-1.9
+
+# copy requirements from lbry, but remove gmpy and miniupnpc (installed manually)
+Get-Content ..\requirements.txt | Select-String -Pattern 'gmpy|miniupnpc' -NotMatch | Out-File requirements_base.txt
+# add in gmpy wheel
+Add-Content requirements.txt "./gmpy-1.17-cp27-none-win32.whl"
+
+pip.exe install -r requirements.txt
+
+pyinstaller -y daemon.onefile.spec
+pyinstaller -y cli.onefile.spec

build/build.sh

@@ -0,0 +1,61 @@
+#!/bin/bash
+
+set -euo pipefail
+set -x
+
+ROOT="$( cd "$( dirname "${BASH_SOURCE[0]}" )/.." && pwd )"
+cd "$ROOT"
+BUILD_DIR="$ROOT/build"
+
+FULL_BUILD="${FULL_BUILD:-false}"
+if [ -n "${TEAMCITY_VERSION:-}" -o -n "${APPVEYOR:-}" ]; then
+  FULL_BUILD="true"
+fi
+
+[ -d "$BUILD_DIR/bulid" ] && rm -rf "$BUILD_DIR/build"
+[ -d "$BUILD_DIR/dist" ] && rm -rf "$BUILD_DIR/dist"
+
+if [ "$FULL_BUILD" == "true" ]; then
+  # install dependencies
+  $BUILD_DIR/prebuild.sh
+
+  VENV="$BUILD_DIR/venv"
+  if [ -d "$VENV" ]; then
+    rm -rf "$VENV"
+  fi
+  virtualenv "$VENV"
+  set +u
+  source "$VENV/bin/activate"
+  set -u
+fi
+
+
+cp "$ROOT/requirements.txt" "$BUILD_DIR/requirements_base.txt"
+(
+  cd "$BUILD_DIR"
+  pip install -r requirements.txt
+)
+
+if [ "$FULL_BUILD" == "true" ]; then
+  python "$BUILD_DIR/set_version.py"
+  python "$BUILD_DIR/set_build.py"
+fi
+
+(
+  cd "$BUILD_DIR"
+  pyinstaller -y daemon.onefile.spec
+  pyinstaller -y cli.onefile.spec
+)
+
+python "$BUILD_DIR/zip_daemon.py"
+
+if [ "$FULL_BUILD" == "true" ]; then
+  # electron-build has a publish feature, but I had a hard time getting
+  # it to reliably work and it also seemed difficult to configure. Not proud of
+  # this, but it seemed better to write my own.
+  python "$BUILD_DIR/release_on_tag.py"
+
+  deactivate
+fi
+
+echo 'Build complete.'

build/cli.onefile.spec

@@ -0,0 +1,59 @@
+# -*- mode: python -*-
+import platform
+import os
+
+
+dir = 'build';
+cwd = os.getcwd()
+if os.path.basename(cwd) != dir:
+    raise Exception('pyinstaller build needs to be run from the ' + dir + ' directory')
+repo_base = os.path.abspath(os.path.join(cwd, '..'))
+
+
+system = platform.system()
+if system == 'Darwin':
+    icns = os.path.join(repo_base, 'build', 'icon.icns')
+elif system == 'Linux':
+    icns = os.path.join(repo_base, 'build', 'icons', '256x256.png')
+elif system == 'Windows':
+    icns = os.path.join(repo_base, 'build', 'icons', 'lbry256.ico')
+else:
+    print 'Warning: System {} has no icons'.format(system)
+    icns = None
+
+block_cipher = None
+
+
+a = Analysis(
+    ['cli.py'],
+    pathex=[cwd],
+    binaries=None,
+    datas=[],
+    hiddenimports=[],
+    hookspath=[],
+    runtime_hooks=[],
+    excludes=[],
+    win_no_prefer_redirects=False,
+    win_private_assemblies=False,
+    cipher=block_cipher
+)
+
+pyz = PYZ(
+    a.pure,
+    a.zipped_data,
+    cipher=block_cipher
+)
+
+exe = EXE(
+    pyz,
+    a.scripts,
+    a.binaries,
+    a.zipfiles,
+    a.datas,
+    name='lbrynet-cli',
+    debug=False,
+    strip=False,
+    upx=True,
+    console=True,
+    icon=icns
+)

build/cli.py

@@ -0,0 +1,7 @@
+from lbrynet.lbrynet_daemon import DaemonCLI
+import logging
+
+logging.basicConfig()
+
+if __name__ == '__main__':
+    DaemonCLI.main()

build/daemon.onefile.spec

@@ -0,0 +1,78 @@
+# -*- mode: python -*-
+import platform
+import os
+
+import lbryum
+
+
+dir = 'build';
+cwd = os.getcwd()
+if os.path.basename(cwd) != dir:
+    raise Exception('pyinstaller build needs to be run from the ' + dir + ' directory')
+repo_base = os.path.abspath(os.path.join(cwd, '..'))
+
+
+system = platform.system()
+if system == 'Darwin':
+    icns = os.path.join(repo_base, 'build', 'icon.icns')
+elif system == 'Linux':
+    icns = os.path.join(repo_base, 'build', 'icons', '256x256.png')
+elif system == 'Windows':
+    icns = os.path.join(repo_base, 'build', 'icons', 'lbry256.ico')
+else:
+    print 'Warning: System {} has no icons'.format(system)
+    icns = None
+
+
+block_cipher = None
+
+
+languages = (
+    'chinese_simplified.txt', 'japanese.txt', 'spanish.txt',
+    'english.txt', 'portuguese.txt'
+)
+
+
+datas = [
+    (
+        os.path.join(os.path.dirname(lbryum.__file__), 'wordlist', language),
+        'lbryum/wordlist'
+    )
+    for language in languages
+]
+
+
+a = Analysis(
+    ['daemon.py'],
+    pathex=[cwd],
+    binaries=None,
+    datas=datas,
+    hiddenimports=[],
+    hookspath=[],
+    runtime_hooks=[],
+    excludes=[],
+    win_no_prefer_redirects=False,
+    win_private_assemblies=False,
+    cipher=block_cipher
+)
+
+
+pyz = PYZ(
+    a.pure, a.zipped_data,
+    cipher=block_cipher
+)
+
+
+exe = EXE(
+    pyz,
+    a.scripts,
+    a.binaries,
+    a.zipfiles,
+    a.datas,
+    name='lbrynet-daemon',
+    debug=False,
+    strip=False,
+    upx=True,
+    console=True,
+    icon=icns
+)

build/daemon.py

@@ -0,0 +1,4 @@
+from lbrynet.lbrynet_daemon import DaemonControl
+
+if __name__ == '__main__':
+    DaemonControl.start()

build/prebuild.sh

@@ -0,0 +1,81 @@
+#!/bin/bash
+
+set -euo pipefail
+set -x
+
+
+LINUX=false
+OSX=false
+
+if [ "$(uname)" == "Darwin" ]; then
+  OSX=true
+elif [ "$(expr substr $(uname -s) 1 5)" == "Linux" ]; then
+  LINUX=true
+else
+  echo "Platform detection failed"
+  exit 1
+fi
+
+
+SUDO=''
+if $LINUX && (( $EUID != 0 )); then
+    SUDO='sudo'
+fi
+
+cmd_exists() {
+  command -v "$1" >/dev/null 2>&1
+  return $?
+}
+
+set +eu
+GITUSERNAME=$(git config --global --get user.name)
+if [ -z "$GITUSERNAME" ]; then
+  git config --global user.name "$(whoami)"
+fi
+GITEMAIL=$(git config --global --get user.email)
+if [ -z "$GITEMAIL" ]; then
+  git config --global user.email "$(whoami)@lbry.io"
+fi
+set -eu
+
+
+if $LINUX; then
+  INSTALL="$SUDO apt-get install --no-install-recommends -y"
+  $INSTALL build-essential libssl-dev libffi-dev libgmp3-dev python2.7-dev
+elif $OSX && ! cmd_exists brew ; then
+  /usr/bin/ruby -e "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)"
+fi
+
+
+if ! cmd_exists python; then
+  if $LINUX; then
+    $INSTALL python2.7
+  elif $OSX; then
+    brew install python
+  fi
+fi
+
+PYTHON_VERSION=$(python -c 'import sys; print(".".join(map(str, sys.version_info[:2])))')
+if [ "$PYTHON_VERSION" != "2.7" ]; then
+  echo "Python 2.7 required"
+  exit 1
+fi
+
+if ! cmd_exists pip; then
+  if $LINUX; then
+    $INSTALL python-pip
+    $SUDO pip install --upgrade pip
+  else
+    echo "Pip required"
+    exit 1
+  fi
+fi
+
+if $LINUX && [ "$(pip list --format=columns | grep setuptools | wc -l)" -ge 1 ]; then
+  #$INSTALL python-setuptools
+  $SUDO pip install setuptools
+fi
+
+if ! cmd_exists virtualenv; then
+  $SUDO pip install virtualenv
+fi

build/release_on_tag.py

@@ -0,0 +1,128 @@
+import glob
+import json
+import os
+import subprocess
+import sys
+
+import github
+import requests
+import uritemplate
+
+
+def main():
+    this_dir = os.path.dirname(os.path.realpath(__file__))
+    try:
+        current_tag = subprocess.check_output(
+            ['git', 'describe', '--exact-match', 'HEAD']).strip()
+    except subprocess.CalledProcessError:
+        print 'Stopping as we are not currently on a tag'
+        return
+
+    if 'GH_TOKEN' not in os.environ:
+        print 'Must set GH_TOKEN in order to publish assets to a release'
+        return
+
+    gh_token = os.environ['GH_TOKEN']
+    auth = github.Github(gh_token)
+    repo = auth.get_repo('lbryio/lbry')
+
+    if not check_repo_has_tag(repo, current_tag):
+        print 'Tag {} is not in repo {}'.format(current_tag, repo)
+        # TODO: maybe this should be an error
+        return
+
+    daemon_zip = glob.glob(this_dir + '/dist/*.zip')[0]
+    release = get_release(repo, current_tag)
+    upload_asset(release, daemon_zip, gh_token)
+
+
+def check_repo_has_tag(repo, target_tag):
+    tags = repo.get_tags().get_page(0)
+    for tag in tags:
+        if tag.name == target_tag:
+            return True
+    return False
+
+
+def get_release(current_repo, current_tag):
+    for release in current_repo.get_releases():
+        if release.tag_name == current_tag:
+            return release
+    raise Exception('No release for {} was found'.format(current_tag))
+
+
+def upload_asset(release, asset_to_upload, token):
+    basename = os.path.basename(asset_to_upload)
+    if is_asset_already_uploaded(release, basename):
+        return
+    count = 0
+    while count < 10:
+        try:
+            return _upload_asset(release, asset_to_upload, token, _requests_uploader)
+        except Exception:
+            print 'Failed uploading on attempt {}'.format(count + 1)
+            count += 1
+
+
+def _upload_asset(release, asset_to_upload, token, uploader):
+    basename = os.path.basename(asset_to_upload)
+    upload_uri = uritemplate.expand(release.upload_url, {'name': basename})
+    output = uploader(upload_uri, asset_to_upload, token)
+    if 'errors' in output:
+        raise Exception(output)
+    else:
+        print 'Successfully uploaded to {}'.format(output['browser_download_url'])
+
+
+# requests doesn't work on windows / linux / osx.
+def _requests_uploader(upload_uri, asset_to_upload, token):
+    print 'Using requests to upload {} to {}'.format(asset_to_upload, upload_uri)
+    with open(asset_to_upload, 'rb') as f:
+        response = requests.post(upload_uri, data=f, auth=('', token))
+    return response.json()
+
+
+# curl -H "Content-Type: application/json" -X POST -d '{"username":"xyz","password":"xyz"}' http://localhost:3000/api/login
+
+
+def _curl_uploader(upload_uri, asset_to_upload, token):
+    # using requests.post fails miserably with SSL EPIPE errors. I spent
+    # half a day trying to debug before deciding to switch to curl.
+    #
+    # TODO: actually set the content type
+    print 'Using curl to upload {} to {}'.format(asset_to_upload, upload_uri)
+    cmd = [
+        'curl',
+        '-sS',
+        '-X', 'POST',
+        '-u', ':{}'.format(os.environ['GH_TOKEN']),
+        '--header', 'Content-Type: application/octet-stream',
+        '--data-binary', '@-',
+        upload_uri
+    ]
+    # '-d', '{"some_key": "some_value"}',
+    print 'Calling curl:'
+    print cmd
+    print
+    with open(asset_to_upload, 'rb') as fp:
+        p = subprocess.Popen(cmd, stdin=fp, stderr=subprocess.PIPE, stdout=subprocess.PIPE)
+    stdout, stderr = p.communicate()
+    print 'curl return code:', p.returncode
+    if stderr:
+        print 'stderr output from curl:'
+        print stderr
+    print 'stdout from curl:'
+    print stdout
+    return json.loads(stdout)
+
+
+def is_asset_already_uploaded(release, basename):
+    for asset in release.raw_data['assets']:
+        if asset['name'] == basename:
+            print 'File {} has already been uploaded to {}'.format(basename, release.tag_name)
+            return True
+    return False
+
+
+if __name__ == '__main__':
+    sys.exit(main())

build/requirements.txt

@@ -0,0 +1,13 @@
+# install daemon requirements (copied from root, with possible modifications. see build.sh, build.ps1)
+-r requirements_base.txt
+
+# install daemon itself. make sure you run `pip install` from this dir. this is how you do relative file paths with pip
+file:../.
+
+# install other build requirements
+PyInstaller==3.2.1
+requests[security]==2.13.0
+GitPython==2.1.1
+PyGithub==1.32
+uritemplate==3.0.0
+git+https://github.com/lbryio/bumpversion.git

build/requirements_base.txt

@@ -0,0 +1 @@
+THIS FILE GETS FILLED IN BY BUILD SCRIPT AND INCLUDED IN requirements.txt

build/set_build.py

@@ -0,0 +1,29 @@
+"""Set the build version to be 'dev', 'qa', 'rc', 'release'"""
+
+import os.path
+import re
+import subprocess
+import sys
+
+
+def main():
+    build = get_build()
+    root_dir = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
+    with open(os.path.join(root_dir, 'lbrynet', 'build_type.py'), 'w') as f:
+        f.write('BUILD = "{}"'.format(build))
+
+
+def get_build():
+    try:
+        tag = subprocess.check_output(['git', 'describe', '--exact-match']).strip()
+        if re.match('v\d+\.\d+\.\d+rc\d+', tag):
+            return 'rc'
+        else:
+            return 'release'
+    except subprocess.CalledProcessError:
+        # if the build doesn't have a tag
+        return 'qa'
+
+
+if __name__ == '__main__':
+    sys.exit(main())

build/set_version.py

@@ -0,0 +1,52 @@
+"""Set the package version to the output of `git describe`"""
+
+import argparse
+import os.path
+import re
+import subprocess
+import sys
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--version', help="defaults to the output of `git describe`")
+    args = parser.parse_args()
+    if args.version:
+        version = args.version
+    else:
+        tag = subprocess.check_output(['git', 'describe']).strip()
+        try:
+            version = get_version_from_tag(tag)
+        except InvalidVersionTag:
+            # this should be an error but its easier to handle here
+            # than in the calling scripts.
+            print 'Tag cannot be converted to a version. Exiting.'
+            return
+    set_version(version)
+
+
+class InvalidVersionTag(Exception):
+    pass
+
+
+def get_version_from_tag(tag):
+    match = re.match('v([\d.]+)', tag)
+    if match:
+        return match.group(1)
+    else:
+        raise InvalidVersionTag('Failed to parse version from tag {}'.format(tag))
+
+
+def set_version(version):
+    root_dir = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
+    with open(os.path.join(root_dir, 'lbrynet', '__init__.py'), 'w') as fp:
+        fp.write(LBRYNET_TEMPLATE.format(version=version))
+
+
+LBRYNET_TEMPLATE = """
+__version__ = "{version}"
+version = tuple(__version__.split('.'))
+"""
+
+if __name__ == '__main__':
+    sys.exit(main())

build/zip_daemon.py

@@ -0,0 +1,29 @@
+import os
+import platform
+import subprocess
+import sys
+import zipfile
+
+
+def main():
+    this_dir = os.path.dirname(os.path.realpath(__file__))
+    tag = subprocess.check_output(['git', 'describe']).strip()
+    zipfilename = 'lbrynet-daemon-{}-{}.zip'.format(tag, get_system_label())
+    full_filename = os.path.join(this_dir, 'dist', zipfilename)
+    executables = ['lbrynet-daemon', 'lbrynet-cli']
+    ext = '.exe' if platform.system() == 'Windows' else ''
+    with zipfile.ZipFile(full_filename, 'w') as myzip:
+        for executable in executables:
+            myzip.write(os.path.join(this_dir, 'dist', executable + ext), executable + ext)
+
+
+def get_system_label():
+    system = platform.system()
+    if system == 'Darwin':
+        return 'macos'
+    else:
+        return system.lower()
+
+
+if __name__ == '__main__':
+    sys.exit(main())

setup.py

@@ -11,7 +11,9 @@ from setuptools import setup, find_packages
 # dependencies of the lbrynet library.  requirements.txt includes
 # dependencies of dependencies and specific versions that we know
 # all work together.
-# See https://packaging.python.org/requirements/ for more details.
+#
+# See https://packaging.python.org/requirements/  and
+# https://caremad.io/posts/2013/07/setup-vs-requirement/  for more details.
 requires = [
     'Twisted',
     'appdirs',