Merge branch 'build'

* build:
  external daemon, no more submodules

.appveyor.yml

@@ -0,0 +1,18 @@
+# Test against the latest version of this Node.js version
+environment:
+  nodejs_version: "7"
+  GH_TOKEN:
+    secure: LiI5jyuHUw6XbH4kC3gP1HX4P/v4rwD/gCNtaFhQu2AvJz1/1wALkp5ECnIxRySN
+
+skip_branch_with_pr: true
+
+clone_folder: C:\projects\lbry-app
+
+build_script:
+  - ps: build\build.ps1
+
+test: off
+
+artifacts:
+  - path: dist\*.exe
+    name: LBRY

.gitmodules

@@ -1,6 +0,0 @@
-[submodule "lbry"]
-	path = lbry
-	url = https://github.com/lbryio/lbry.git
-[submodule "lbryum"]
-	path = lbryum
-	url = https://github.com/lbryio/lbryum.git

appveyor.yml

@@ -1,55 +0,0 @@
-# Test against the latest version of this Node.js version
-environment:
-  nodejs_version: "6"
-  GH_TOKEN:
-    secure: LiI5jyuHUw6XbH4kC3gP1HX4P/v4rwD/gCNtaFhQu2AvJz1/1wALkp5ECnIxRySN
-
-skip_branch_with_pr: true
-
-clone_folder: C:\projects\lbry-electron
-
-# Install scripts. (runs after repo cloning)
-install:
-  # needed to deal with submodules
-  - git submodule update --init --recursive
-  - python build\set_version.py
-  - python build\set_build.py
-  # Get the latest stable version of Node.js or io.js
-  - ps: Install-Product node $env:nodejs_version
-  # install modules
-  - npm install
-  - cd app
-  - npm install
-  - cd ..
-  # create daemon and cli executable
-  - cd daemon
-  - ps: .\build.ps1
-  - cd ..
-  # build ui
-  - cd ui
-  - npm install
-  - node_modules\.bin\node-sass --output dist\css --sourcemap=none scss\
-  - node_modules\.bin\webpack
-  - ps: Copy-Item dist ..\app\ -recurse
-  - cd ..
-  # copy executables into ui
-  - ps: Copy-Item daemon\dist\lbrynet-daemon.exe app\dist
-  - ps: Copy-Item daemon\dist\lbrynet-cli.exe app\dist
-
-build_script:
-  # build electron app
-  - node_modules\.bin\build -p never
-  # for debugging, see what was built
-  - python build\zip_daemon.py
-  - dir dist
-  - pip install -r build\requirements.txt
-  - python build\release_on_tag.py
-
-test: off
-
-artifacts:
-  - path: dist\*.exe
-    name: LBRY
-
-  - path: dist\*.zip
-    name: lbrynet-daemon

build/DAEMON_URL

@@ -0,0 +1 @@
+https://github.com/lbryio/lbry/releases/download/v0.9.2rc3/lbrynet-daemon-v0.9.2rc3-OSNAME.zip

build/build.ps1

@@ -0,0 +1,34 @@
+pip install -r build\requirements.txt
+python build\set_version.py
+
+# Get the latest stable version of Node.js or io.js
+Install-Product node $env:nodejs_version
+
+# install node modules
+npm install
+cd app
+npm install
+cd ..
+
+# build ui
+cd ui
+npm install
+node_modules\.bin\node-sass --output dist\css --sourcemap=none scss\
+node_modules\.bin\webpack
+Copy-Item dist ..\app\ -recurse
+cd ..
+
+# get daemon and cli executable
+$daemon_url = (Get-Content build\DAEMON_URL -Raw).replace("OSNAME", "windows")
+Invoke-WebRequest -Uri $daemon_url -OutFile daemon.zip
+Expand-Archive daemon.zip -DestinationPath app\dist\
+dir app\dist\ # verify that daemon binary is there
+rm daemon.zip
+
+# build electron app
+node_modules\.bin\build -p never
+$binary_name = Get-ChildItem -Path dist -Filter '*.exe' -Name
+$new_name = $binary_name -replace '^LBRY Setup (.*)\.exe$', 'LBRY_$1.exe'
+Rename-Item -Path "dist\$binary_name" -NewName $new_name
+dir dist # verify that binary was built/named correctly
+python build\release_on_tag.py

build/build.sh

@@ -7,7 +7,18 @@ ROOT="$( cd "$( dirname "${BASH_SOURCE[0]}" )/.." && pwd )"
 cd "$ROOT"
 BUILD_DIR="$ROOT/build"
 
+LINUX=false
+OSX=false
 if [ "$(uname)" == "Darwin" ]; then
+  OSX=true
+elif [ "$(expr substr $(uname -s) 1 5)" == "Linux" ]; then
+  LINUX=true
+else
+  echo "Platform detection failed"
+  exit 1
+fi
+
+if $OSX; then
     ICON="$BUILD_DIR/icon.icns"
 else
     ICON="$BUILD_DIR/icons/lbry48.png"
@@ -32,7 +43,6 @@ if [ "$FULL_BUILD" == "true" ]; then
   set -u
   pip install -r "$BUILD_DIR/requirements.txt"
   python "$BUILD_DIR/set_version.py"
-  python "$BUILD_DIR/set_build.py"
 fi
 
 [ -d "$ROOT/dist" ] && rm -rf "$ROOT/dist"
@@ -62,24 +72,17 @@ npm install
 #  daemon and cli  #
 ####################
 
-(
-  cd "$ROOT/daemon"
-
-  # copy requirements from lbry, but remove lbryum (we'll add it back in below)
-  grep -v lbryum "$ROOT/lbry/requirements.txt" > requirements.txt
-  # for electron, we install lbryum and lbry using submodules
-  echo "../lbryum" >> requirements.txt
-  echo "../lbry" >> requirements.txt
-  # also add pyinstaller
-  echo "PyInstaller==3.2.1" >> requirements.txt
-
-  pip install -r requirements.txt
-  pyinstaller -y daemon.onefile.spec
-  pyinstaller -y cli.onefile.spec
-  mv dist/lbrynet-daemon dist/lbrynet-cli "$ROOT/app/dist/"
-)
-python "$BUILD_DIR/zip_daemon.py"
-
+if [ "$FULL_BUILD" == "true" ]; then
+  if $OSX; then
+    OSNAME="macos"
+  else
+    OSNAME="linux"
+  fi
+  DAEMON_URL="$(cat "$BUILD_DIR/DAEMON_URL" | sed "s/OSNAME/${OSNAME}/")"
+  wget --quiet "$DAEMON_URL" -O "$BUILD_DIR/daemon.zip"
+  unzip "$BUILD_DIR/daemon.zip" -d "$ROOT/app/dist/"
+  rm "$BUILD_DIR/daemon.zip"
+fi
 
 ###################
 #  Build the app  #
@@ -91,12 +94,18 @@ python "$BUILD_DIR/zip_daemon.py"
 )
 
 if [ "$FULL_BUILD" == "true" ]; then
-  if [ "$(uname)" == "Darwin" ]; then
+  if $OSX; then
     security unlock-keychain -p ${KEYCHAIN_PASSWORD} osx-build.keychain
   fi
 
   node_modules/.bin/build -p never
 
+  if $OSX; then
+    binary_name=$(find "$ROOT/dist" -iname "*dmg")
+    new_name=$(basename "$binary_name" | sed 's/-/_/')
+    mv "$binary_name" "$(dirname "$binary_name")/$new_name"
+  fi
+
   # electron-build has a publish feature, but I had a hard time getting
   # it to reliably work and it also seemed difficult to configure. Not proud of
   # this, but it seemed better to write my own.

build/fix_submodule_urls.sh

@@ -1,13 +0,0 @@
-#!/bin/bash
-
-# https://github.com/lbryio/lbry-app/commit/4386102ba3bf8c731a075797756111d73c31a47a
-# https://github.com/lbryio/lbry-app/commit/a3a376922298b94615f7514ca59988b73a522f7f
-
-# Appveyor and Teamcity struggle with SSH urls in submodules, so we use HTTPS
-# But locally, SSH urls are way better since they dont require a password
-
-DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
-cd "DIR"
-
-git config submodule.lbry.url git@github.com:lbryio/lbry.git
-git config submodule.lbryum.url git@github.com:lbryio/lbryum.git

build/prebuild.sh

@@ -72,7 +72,6 @@ if ! cmd_exists pip; then
 fi
 
 if $LINUX && [ "$(pip list --format=columns | grep setuptools | wc -l)" -ge 1 ]; then
-  #$INSTALL python-setuptools
   $SUDO pip install setuptools
 fi
 
@@ -88,3 +87,14 @@ if ! cmd_exists node; then
     brew install node
   fi
 fi
+
+if ! cmd_exists unzip; then
+  if $LINUX; then
+    $INSTALL unzip
+  elif $OSX; then
+    echo "unzip required"
+    exit 1
+    # not sure this works, but OSX should come with unzip
+    # brew install unzip
+  fi
+fi

build/release_on_tag.py

@@ -1,6 +1,5 @@
 import glob
 import json
-import logging
 import os
 import platform
 import subprocess
@@ -10,15 +9,13 @@ import github
 import requests
 import uritemplate
 
-from lbrynet.core import log_support
-
 
 def main():
     try:
         current_tag = subprocess.check_output(
             ['git', 'describe', '--exact-match', 'HEAD']).strip()
     except subprocess.CalledProcessError:
-        log.info('Stopping as we are not currently on a tag')
+        print 'Stopping as we are not currently on a tag'
         return
 
     if 'GH_TOKEN' not in os.environ:
@@ -27,20 +24,15 @@ def main():
 
     gh_token = os.environ['GH_TOKEN']
     auth = github.Github(gh_token)
-    app_repo = auth.get_repo('lbryio/lbry-app')
-    daemon_repo = auth.get_repo('lbryio/lbry')
+    repo = auth.get_repo('lbryio/lbry-app')
 
-    if not check_repo_has_tag(app_repo, current_tag):
-        log.info('Tag %s is not in repo %s', current_tag, app_repo)
+    if not check_repo_has_tag(repo, current_tag):
+        print 'Tag {} is not in repo {}'.format(current_tag, repo)
         # TODO: maybe this should be an error
         return
 
-    daemon = get_daemon_artifact()
-    release = get_release(daemon_repo, current_tag)
-    upload_asset(release, daemon, gh_token)
-
     app = get_app_artifact()
-    release = get_release(app_repo, current_tag)
+    release = get_release(repo, current_tag)
     upload_asset(release, app, gh_token)
 
 
@@ -60,21 +52,18 @@ def get_release(current_repo, current_tag):
 
 
 def get_app_artifact():
+    this_dir = os.path.dirname(os.path.realpath(__file__))
     system = platform.system()
     if system == 'Darwin':
-        return glob.glob('dist/mac/LBRY*.dmg')[0]
+        return glob.glob(this_dir + '/../dist/mac/LBRY*.dmg')[0]
     elif system == 'Linux':
-        return glob.glob('dist/LBRY*.deb')[0]
+        return glob.glob(this_dir + '/../dist/LBRY*.deb')[0]
     elif system == 'Windows':
-        return glob.glob('dist/LBRY*.exe')[0]
+        return glob.glob(this_dir + '/../dist/LBRY*.exe')[0]
     else:
         raise Exception("I don't know about any artifact on {}".format(system))
 
 
-def get_daemon_artifact():
-    return glob.glob('dist/*.zip')[0]
-
-
 def upload_asset(release, asset_to_upload, token):
     basename = os.path.basename(asset_to_upload)
     if is_asset_already_uploaded(release, basename):
@@ -84,30 +73,26 @@ def upload_asset(release, asset_to_upload, token):
         try:
             return _upload_asset(release, asset_to_upload, token, _curl_uploader)
         except Exception:
-            log.exception('Failed to upload')
+            print 'Failed uploading on attempt {}'.format(count + 1)
             count += 1
 
 
 def _upload_asset(release, asset_to_upload, token, uploader):
     basename = os.path.basename(asset_to_upload)
-    upload_uri = uritemplate.expand(
-        release.upload_url,
-        {'name': basename}
-    )
+    upload_uri = uritemplate.expand(release.upload_url, {'name': basename})
     output = uploader(upload_uri, asset_to_upload, token)
     if 'errors' in output:
         raise Exception(output)
     else:
-        log.info('Successfully uploaded to %s', output['browser_download_url'])
+        print 'Successfully uploaded to {}'.format(output['browser_download_url'])
 
 
 # requests doesn't work on windows / linux / osx.
 def _requests_uploader(upload_uri, asset_to_upload, token):
-    log.info('Using requests to upload %s to %s', asset_to_upload, upload_uri)
+    print 'Using requests to upload {} to {}'.format(asset_to_upload, upload_uri)
     with open(asset_to_upload, 'rb') as f:
         response = requests.post(upload_uri, data=f, auth=('', token))
-    output = response.json()
-    return output
+    return response.json()
 
 
 # curl -H "Content-Type: application/json" -X POST -d '{"username":"xyz","password":"xyz"}' http://localhost:3000/api/login
@@ -118,7 +103,7 @@ def _curl_uploader(upload_uri, asset_to_upload, token):
     # half a day trying to debug before deciding to switch to curl.
     #
     # TODO: actually set the content type
-    log.info('Using curl to upload %s to %s', asset_to_upload, upload_uri)
+    print 'Using curl to upload {} to {}'.format(asset_to_upload, upload_uri)
     cmd = [
         'curl',
         '-sS',
@@ -141,21 +126,16 @@ def _curl_uploader(upload_uri, asset_to_upload, token):
         print stderr
     print 'stdout from curl:'
     print stdout
-    output = json.loads(stdout)
-    return output
+    return json.loads(stdout)
 
 
 def is_asset_already_uploaded(release, basename):
     for asset in release.raw_data['assets']:
         if asset['name'] == basename:
-            log.info('File %s has already been uploaded to %s', basename, release.tag_name)
+            print 'File {} has already been uploaded to {}'.format(basename, release.tag_name)
             return True
     return False
 
 
 if __name__ == '__main__':
-    log = logging.getLogger('release-on-tag')
-    log_support.configure_console(level='INFO')
     sys.exit(main())
-else:
-    log = logging.getLogger(__name__)

build/reset.sh

@@ -1,21 +0,0 @@
-#!/bin/bash
-
-set -euxo pipefail
-
-ROOT="$( cd "$( dirname "${BASH_SOURCE[0]}" )/.." && pwd )"
-cd "$ROOT"
-
-(
-  cd lbry
-  git tag -d $(git describe)
-  git reset --hard origin/master
-)
-
-(
-  cd lbryum
-  git tag -d $(git describe)
-  git reset --hard origin/master
-)
-
-git tag -d $(git describe)
-git reset --hard HEAD~1

build/set_version.py

@@ -39,20 +39,13 @@ def get_version_from_tag(tag):
 
 
 def set_version(version):
-    package_file = os.path.join('app', 'package.json')
+    root_dir = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
+    package_file = os.path.join(root_dir, 'app', 'package.json')
     with open(package_file) as fp:
         package_data = json.load(fp)
     package_data['version'] = version
     with open(package_file, 'w') as fp:
         json.dump(package_data, fp, indent=2, separators=(',', ': '))
-    with open(os.path.join('lbry', 'lbrynet', '__init__.py'), 'w') as fp:
-        fp.write(LBRYNET_TEMPLATE.format(version=version))
-
-
-LBRYNET_TEMPLATE = """
-__version__ = "{version}"
-version = tuple(__version__.split('.'))
-"""
 
 
 if __name__ == '__main__':

build/zip_daemon.py

@@ -1,28 +0,0 @@
-import os
-import platform
-import subprocess
-import sys
-import zipfile
-
-
-def main():
-    tag = subprocess.check_output(['git', 'describe']).strip()
-    zipfilename = 'lbrynet-daemon-{}-{}.zip'.format(tag, get_system_label())
-    full_filename = os.path.join('dist', zipfilename)
-    executables = ['lbrynet-daemon', 'lbrynet-cli']
-    ext = '.exe' if platform.system() == 'Windows' else ''
-    with zipfile.ZipFile(full_filename, 'w') as myzip:
-        for executable in executables:
-            myzip.write(os.path.join('app', 'dist', executable + ext), executable + ext)
-
-
-def get_system_label():
-    system = platform.system()
-    if system == 'Darwin':
-        return 'macos'
-    else:
-        return system.lower()
-
-
-if __name__ == '__main__':
-    sys.exit(main())

daemon/build.ps1

@@ -1,28 +0,0 @@
-$env:Path += ";C:\MinGW\bin\"
-
-$env:Path += ";C:\Program Files (x86)\Windows Kits\10\bin\x86\"
-gcc --version
-mingw32-make --version
-
-# build/install miniupnpc manually
-tar zxf miniupnpc-1.9.tar.gz
-cd miniupnpc-1.9
-mingw32-make.exe -f Makefile.mingw
-python.exe setupmingw32.py build --compiler=mingw32
-python.exe setupmingw32.py install
-cd ..\
-Remove-Item -Recurse -Force miniupnpc-1.9
-
-# copy requirements from lbry, but remove lbryum (we'll add it back in below) and gmpy and miniupnpc (installed manually)
-Get-Content ..\lbry\requirements.txt | Select-String -Pattern 'lbryum|gmpy|miniupnpc' -NotMatch | Out-File requirements.txt
-# add in gmpy wheel
-Add-Content requirements.txt "./gmpy-1.17-cp27-none-win32.whl"
-# for electron, we install lbryum and lbry using submodules
-Add-Content requirements.txt "../lbryum"
-Add-Content requirements.txt "../lbry"
-
-pip.exe install pyinstaller
-pip.exe install -r requirements.txt
-
-pyinstaller -y daemon.onefile.spec
-pyinstaller -y cli.onefile.spec

daemon/cli.onefile.spec

@@ -1,58 +0,0 @@
-# -*- mode: python -*-
-import platform
-import os
-
-
-cwd = os.getcwd()
-if os.path.basename(cwd) != 'daemon':
-    raise Exception('The build needs to be run from the same directory as the spec file')
-repo_base = os.path.abspath(os.path.join(cwd, '..'))
-
-
-system = platform.system()
-if system == 'Darwin':
-    icns = os.path.join(repo_base, 'build', 'icon.icns')
-elif system == 'Linux':
-    icns = os.path.join(repo_base, 'build', 'icons', '256x256.png')
-elif system == 'Windows':
-    icns = os.path.join(repo_base, 'build', 'icons', 'lbry256.ico')
-else:
-    print 'Warning: System {} has no icons'.format(system)
-    icns = None
-
-block_cipher = None
-
-
-a = Analysis(
-    ['cli.py'],
-    pathex=[cwd],
-    binaries=None,
-    datas=[],
-    hiddenimports=[],
-    hookspath=[],
-    runtime_hooks=[],
-    excludes=[],
-    win_no_prefer_redirects=False,
-    win_private_assemblies=False,
-    cipher=block_cipher
-)
-
-pyz = PYZ(
-    a.pure,
-    a.zipped_data,
-    cipher=block_cipher
-)
-
-exe = EXE(
-    pyz,
-    a.scripts,
-    a.binaries,
-    a.zipfiles,
-    a.datas,
-    name='lbrynet-cli',
-    debug=False,
-    strip=False,
-    upx=True,
-    console=True,
-    icon=icns
-)

daemon/cli.py

@@ -1,7 +0,0 @@
-from lbrynet.lbrynet_daemon import DaemonCLI
-import logging
-
-logging.basicConfig()
-
-if __name__ == '__main__':
-    DaemonCLI.main()

daemon/daemon.onefile.spec

@@ -1,77 +0,0 @@
-# -*- mode: python -*-
-import platform
-import os
-
-import lbryum
-
-
-cwd = os.getcwd()
-if os.path.basename(cwd) != 'daemon':
-    raise Exception('The build needs to be run from the same directory as the spec file')
-repo_base = os.path.abspath(os.path.join(cwd, '..'))
-
-
-system = platform.system()
-if system == 'Darwin':
-    icns = os.path.join(repo_base, 'build', 'icon.icns')
-elif system == 'Linux':
-    icns = os.path.join(repo_base, 'build', 'icons', '256x256.png')
-elif system == 'Windows':
-    icns = os.path.join(repo_base, 'build', 'icons', 'lbry256.ico')
-else:
-    print 'Warning: System {} has no icons'.format(system)
-    icns = None
-
-
-block_cipher = None
-
-
-languages = (
-    'chinese_simplified.txt', 'japanese.txt', 'spanish.txt',
-    'english.txt', 'portuguese.txt'
-)
-
-
-datas = [
-    (
-        os.path.join(os.path.dirname(lbryum.__file__), 'wordlist', language),
-        'lbryum/wordlist'
-    )
-    for language in languages
-]
-
-
-a = Analysis(
-    ['daemon.py'],
-    pathex=[cwd],
-    binaries=None,
-    datas=datas,
-    hiddenimports=[],
-    hookspath=[],
-    runtime_hooks=[],
-    excludes=[],
-    win_no_prefer_redirects=False,
-    win_private_assemblies=False,
-    cipher=block_cipher
-)
-
-
-pyz = PYZ(
-    a.pure, a.zipped_data,
-    cipher=block_cipher
-)
-
-
-exe = EXE(
-    pyz,
-    a.scripts,
-    a.binaries,
-    a.zipfiles,
-    a.datas,
-    name='lbrynet-daemon',
-    debug=False,
-    strip=False,
-    upx=True,
-    console=True,
-    icon=icns
-)

daemon/daemon.py

@@ -1,4 +0,0 @@
-from lbrynet.lbrynet_daemon import DaemonControl
-
-if __name__ == '__main__':
-    DaemonControl.start()

lbry

@@ -1 +1 @@
-Subproject commit 52a42ce2fcdd1ec6ddca84c4fa754c3d318d2258
+Subproject commit 2303f88bd4482e1f0b2f2357516d2c547ccd64ee

lbryum

@@ -1 +1 @@
-Subproject commit 39ace3737509ff2b09fabaaa64d1525843de1325
+Subproject commit 71cf689c450bbe696fb1fd51d562b195b2f6854a