From cb7c84927368a46644f70ac284ad5052c069240a Mon Sep 17 00:00:00 2001 From: Job Evers-Meltzer Date: Wed, 9 Nov 2016 09:28:18 -0600 Subject: [PATCH 1/4] bug fix: actually read the environment variable --- packaging/windows/install.ps1 | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packaging/windows/install.ps1 b/packaging/windows/install.ps1 index 714851fdc..044c3af2d 100644 --- a/packaging/windows/install.ps1 +++ b/packaging/windows/install.ps1 @@ -3,12 +3,12 @@ C:\Python27\python.exe setup.py install # If this is a build because of a tag, make sure that # its either a testing tag or a tag that matches the version # specified in the source code. -If ($(APPVEYOR_REPO_TAG) -Match "true") { - If ($(APPVEYOR_REPO_TAG_NAME) -Like "test*") { +If (${Env:APPVEYOR_REPO_TAG} -Match "true") { + If (${Env:APPVEYOR_REPO_TAG_NAME} -Like "test*") { exit 0 } # non-testing tags should be in the form v1.2.3 - If ("v$(C:\Python27\python.exe setup.py -V)" -Match $(APPVEYOR_REPO_TAG_NAME)) { + If ("v$(C:\Python27\python.exe setup.py -V)" -Match ${Env:APPVEYOR_REPO_TAG_NAME}) { exit 0 } exit 1 From 694db88ac107aa73d6813872c48d2700651f4b2d Mon Sep 17 00:00:00 2001 From: Job Evers-Meltzer Date: Wed, 9 Nov 2016 09:28:47 -0600 Subject: [PATCH 2/4] bug fix: fail build if tests fail --- packaging/windows/test.ps1 | 1 + 1 file changed, 1 insertion(+) diff --git a/packaging/windows/test.ps1 b/packaging/windows/test.ps1 index fa12e8602..a6f317694 100644 --- a/packaging/windows/test.ps1 +++ b/packaging/windows/test.ps1 @@ -1,5 +1,6 @@ C:\Python27\Scripts\pip.exe install mock C:\Python27\Scripts\pip.exe install pylint C:\Python27\python.exe C:\Python27\Scripts\trial.py C:\projects\lbry\tests\unit +if ($LastExitCode -ne 0) { $host.SetShouldExit($LastExitCode) } C:\Python27\Scripts\pylint.exe -E --disable=inherit-non-class --disable=no-member --ignored-modules=distutils --enable=unused-import lbrynet packaging/windows/lbry-win32-app/LBRYWin32App.py if ($LastExitCode -ne 0) { $host.SetShouldExit($LastExitCode) } From 838508f342d96bac0e3a3b7458c12b7bbac10963 Mon Sep 17 00:00:00 2001 From: Job Evers-Meltzer Date: Wed, 9 Nov 2016 09:29:39 -0600 Subject: [PATCH 3/4] Bundle the UI into non-tagged builds. Updates the UIManager to check if a bundled UI was included in the package and preferentially use that. --- .appveyor.yml | 33 ++++++---- .travis.yml | 24 ++++++- MANIFEST.in | 1 - lbrynet/lbrynet_daemon/UIManager.py | 97 +++++++++++++++++++++++++---- packaging/travis/setup_osx.sh | 1 - packaging/windows/init.ps1 | 3 +- setup.py | 51 +++++++-------- 7 files changed, 153 insertions(+), 57 deletions(-) delete mode 100644 MANIFEST.in diff --git a/.appveyor.yml b/.appveyor.yml index 1c74b30c2..8fd49c589 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -26,8 +26,7 @@ install: build_script: -- ps: C:\Python27\python.exe setup.py build bdist_msi -- signtool.exe sign /f packaging\windows\certs\lbry2.pfx /p %key_pass% /tr http://tsa.starfieldtech.com /td SHA256 /fd SHA256 dist\*.msi +- ps: .\packaging\windows\build.ps1 test_script: @@ -44,13 +43,23 @@ artifacts: deploy: - release: $(APPVEYOR_REPO_TAG_NAME) - description: 'Release' - provider: GitHub - auth_token: - secure: 28gMVxQkXr2iXP4F+5kVwefUtKCfS1ePZ97PVfaSR8UDupjAvKhSJx862TnEjukb - artifact: /.*\.msi/ - draft: false - prerelease: true - on: - appveyor_repo_tag: true # deploy on tag push only + - provider: GitHub + release: $(APPVEYOR_REPO_TAG_NAME) + description: 'Release' + auth_token: + secure: 28gMVxQkXr2iXP4F+5kVwefUtKCfS1ePZ97PVfaSR8UDupjAvKhSJx862TnEjukb + artifact: /.*\.msi/ + draft: false + prerelease: true + on: + appveyor_repo_tag: true # deploy on tag push only + - provider: S3 + access_key_id: + secure: E25iHvmHiJP56GWFTe14L3pS8QKb5ZcyWZLY0zqh6BA= + secret_access_key: + secure: YwZe6fCv29akFYjMCCBJBF7LtI6mxPrxbq7SSoAbn1BPdqjATFegeteGAsqHur/C + bucket: lbrynet-master + region: us-west-2 + artifact: /.*\.msi/ + on: + branch: bundled-ui diff --git a/.travis.yml b/.travis.yml index 2ccba1e61..d4c382810 100644 --- a/.travis.yml +++ b/.travis.yml @@ -26,6 +26,8 @@ cache: before_install: - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then ./packaging/travis/setup_osx.sh; fi + - mkdir -p lbrynet/resources/ui + - if [[ -z "$TRAVIS_TAG" ]]; then ./packaging/travis/setup_qa.sh; fi install: - if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then ./packaging/travis/install_dependencies_and_run_tests.sh; fi @@ -41,9 +43,15 @@ script: # fail the build if this is a build for a tag and we don't have the versions matching; allow tags that start with 'test' to pass - if [[ -n "${TRAVIS_TAG}" ]]; then if [[ "${TRAVIS_TAG}" == test* ]] || [[ "v`python setup.py -V`" = "${TRAVIS_TAG}" ]]; then true; else false; fi; fi +before_deploy: + # s3 release can only upload a folder so move the package into an upload folder + - mkdir "${TRAVIS_BUILD_DIR}/upload" + - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then mv "${TRAVIS_BUILD_DIR}/packaging/osx/lbry-osx-app/`python setup.py --name`.`python setup.py -V`.dmg" "${TRAVIS_BUILD_DIR}/upload"; fi + - if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then mv "${TRAVIS_BUILD_DIR}/`python setup.py --name`_`python setup.py -V`_amd64.deb" "${TRAVIS_BUILD_DIR}/upload"; fi + deploy: - provider: releases - file: "${TRAVIS_BUILD_DIR}/`python setup.py --name`_`python setup.py -V`_amd64.deb" + file: "${TRAVIS_BUILD_DIR}/upload/`python setup.py --name`_`python setup.py -V`_amd64.deb" skip_cleanup: true prerelease: true on: @@ -53,7 +61,7 @@ deploy: api_key: secure: nKdWGROnLNodx9k9nWvq2wezkPvSVL8zF63qjPXjhdOe9kCUbcLp7WnFDYpn9EJj4Pofq/ejeCHwjA+5x7JUP3Szk7SlJV61B4c/5hl64rl7oSKOoKskjdE2jaOG3CJuOUrh0yQ59U3vMMABcsnw/wJaCIuu/erdPIm8g8R+stu1YHOGtl5Y9WiW+zLJn2vc3GooV1TWtki9EnrmFfw0Vrqc4RMVMFB1ojE7ggrK1LIwcmGSbLIYzker1ZRz8SCy+84sGk4//V+2i2NNiz5AkPuG7BBGrU2twE9nD23IlruJAdVdi71P3ytAmi0kKyvxIU4VeNaqyTk9zeL5IB9J5IIgvekHgKcsKhFUZ6QcXT1Xfxl4ELftvWCTHWiewnXFdqLcG9GZiUaE6+7wdalwDAP3tqS2emiibetlBZERHR+RMR00ej+1MBYWGMlTse/0Tglndv0a2qqgAJYLKPRT02hTRYGxZ1MrJe+WGnChRmzwgLVTIgZuiDciFOahN0TYGSORk6OpnZBsxvpzSqDw5UDJx0BmbJ1xMNDFbOs8ubZ9yIpB9yNMGw66FPacOF61XNYnmA68ILC28UtOFKuuHLrUPbM5JmQkDVhtTfFbBnyHefyCLAL4MHvJJKGi1oaOXjYaJ/J095h636/kQ0cHHuVMgoWUQZOQ44xRAz7tMuc= - provider: releases - file: "${TRAVIS_BUILD_DIR}/packaging/osx/lbry-osx-app/`python setup.py --name`.`python setup.py -V`.dmg" + file: "${TRAVIS_BUILD_DIR}/upload/`python setup.py --name`.`python setup.py -V`.dmg" skip_cleanup: true prerelease: true on: @@ -62,6 +70,18 @@ deploy: # this is the oauth token for the lbry-ci user api_key: secure: nKdWGROnLNodx9k9nWvq2wezkPvSVL8zF63qjPXjhdOe9kCUbcLp7WnFDYpn9EJj4Pofq/ejeCHwjA+5x7JUP3Szk7SlJV61B4c/5hl64rl7oSKOoKskjdE2jaOG3CJuOUrh0yQ59U3vMMABcsnw/wJaCIuu/erdPIm8g8R+stu1YHOGtl5Y9WiW+zLJn2vc3GooV1TWtki9EnrmFfw0Vrqc4RMVMFB1ojE7ggrK1LIwcmGSbLIYzker1ZRz8SCy+84sGk4//V+2i2NNiz5AkPuG7BBGrU2twE9nD23IlruJAdVdi71P3ytAmi0kKyvxIU4VeNaqyTk9zeL5IB9J5IIgvekHgKcsKhFUZ6QcXT1Xfxl4ELftvWCTHWiewnXFdqLcG9GZiUaE6+7wdalwDAP3tqS2emiibetlBZERHR+RMR00ej+1MBYWGMlTse/0Tglndv0a2qqgAJYLKPRT02hTRYGxZ1MrJe+WGnChRmzwgLVTIgZuiDciFOahN0TYGSORk6OpnZBsxvpzSqDw5UDJx0BmbJ1xMNDFbOs8ubZ9yIpB9yNMGw66FPacOF61XNYnmA68ILC28UtOFKuuHLrUPbM5JmQkDVhtTfFbBnyHefyCLAL4MHvJJKGi1oaOXjYaJ/J095h636/kQ0cHHuVMgoWUQZOQ44xRAz7tMuc= + - provider: s3 + access_key_id: + secure: "gmJNW8bda2snpA2F+0gucjgO/orvZL0a348QmiffYdtXleIseyY+C4ZI9llWm+s8n2TxiqBYpc/A3Bv7JM0yIccjF0CWQb8pu4HkVT2xLT/1p28EdfxKhR0H5sLtkxOCvLIuf3ppZWac8IzRWvh9TDER1TVxbIGvbOJCtKJ+sUKGCWVkxJY6lzAy7+YHwY9sLV6GvarbkfcnbwNh0qPEROcc6JXUQRKCjWOoZLSZHx4dwxlDiXaG1GyUSm2bwtB11VbMZqaP5eO2zypcUr24J0WkzuAOFIUMDERQnY6eSR62T88BjVjL+07kQufSTtnHC49uHsylyQLwkphkwi2Ei3c/fJ5XI1gY0Med/WOHuy99LAaaHQvHxVa9Zxvrgivvu1oa2QdLir05asMCB0G6Yjgz38Fl8jzdKN/PLHs/lhgBGOJfMN+0UR7hOrLeXdJcMriVfwzYrCl6KkFgTe50rcMx8vv6SH15AqZ5sceksy+maA4O8Mxq1hEe2qJABgRdRZK3FnFKNnQOxfrlU6N9zMNsicVcRwH6WNBtoL4BK3KeMproWnorh9rXmdoSG2Fh5X636cudYhLrAbO5yaoZAXFxHqOV1n63v3tgv4MKrB999OF9V9HyYXv3ro9WZTTsH7LskLA7YmTvXtpMVgBK8SfnvqCjLim5qpVvIVPQIOY=" + secret_access_key: + secure: "pS28jMSNV8HUSa8P8pReXgBtHW0OSWDZMAlZhBrtUI05goUnK6buXadvzCynZWgDiXPHuEuLZOLDVj5yL8N3eCa+sG1tEDcCjPxrBFxHisng8atgn3FaL15MBOdXn0qOBUr+vQ4jbUozsBYGqmoZQV0/Af53tU55KK5HJrotkd++SNTEQrFbWZavM+Wvhw4yJN2hmRer9II72DCZkayUjpvBq+irnGeU0g9SrX8VULCg+sqcnbqdLWd+VALHzQr+O74MaWE8mu2PA8A1GKWZZ/a+LoTh1T2bArpX94q0ueea1eEcUA154vC1azkKK5vy9hzENt5qiiEleBDE31DhbGaeEoakEXvHWrdI3DYiakD6prj0necD8aAK6VhPtqwiTtTf7E4BE9hxY4nJRDuZckvFI4LYFMvHlbdqBieeZPflStkws7VgsfrGV7EberYIbSEohUGGUBg3rdaCIPsn8W4nQAixq39Dcfm0orGo4BgargT6FFtAeDzsfYdbKBk/F8BOPMDJz9P1utAZ/ZeBexic16aeIbk7+wtaQUd0Hswm+2QUHddshvp9IzNniFb1BLvRR/lOVl0HdUkJj/haR2M8XhwcRFIKpL2oMZHwRmLW4c1kkEx+KSfAB77arjgrCzq3y5MlKUSliL5qZZMIJX0ReRQ2dcy3DCY0Fxdj014=" + bucket: lbrynet-master + skip_cleanup: true + local_dir: "${TRAVIS_BUILD_DIR}/upload" + region: us-west-2 + on: + branch: bundled-ui + env: global: diff --git a/MANIFEST.in b/MANIFEST.in deleted file mode 100644 index 1dbbdac95..000000000 --- a/MANIFEST.in +++ /dev/null @@ -1 +0,0 @@ -include lbrynet/lbrynet_console/plugins/blindrepeater.yapsy-plugin diff --git a/lbrynet/lbrynet_daemon/UIManager.py b/lbrynet/lbrynet_daemon/UIManager.py index 19740a6e2..3a0133f7c 100644 --- a/lbrynet/lbrynet_daemon/UIManager.py +++ b/lbrynet/lbrynet_daemon/UIManager.py @@ -2,16 +2,19 @@ import os import logging import shutil import json - from urllib2 import urlopen from StringIO import StringIO +from zipfile import ZipFile + +import pkg_resources from twisted.internet import defer from twisted.internet.task import LoopingCall + from lbrynet.conf import settings from lbrynet.lbrynet_daemon.Resources import NoCacheStaticFile from lbrynet import __version__ as lbrynet_version from lbryum.version import LBRYUM_VERSION as lbryum_version -from zipfile import ZipFile + log = logging.getLogger(__name__) @@ -62,6 +65,9 @@ class UIManager(object): self.branch = settings.ui_branch or branch self.check_requirements = settings.check_ui_requirements or check_requirements + if self._check_for_bundled_ui(): + return defer.succeed(True) + if local_ui_path: if os.path.isdir(local_ui_path): log.info("Checking user specified UI directory: " + str(local_ui_path)) @@ -74,7 +80,7 @@ class UIManager(object): log.info("User specified UI directory doesn't exist, using " + self.branch) elif self.loaded_branch == "user-specified": log.info("Loading user provided UI") - d = self._load_ui() + d = defer.maybeDeferred(self._load_ui()) return d else: log.info("Checking for updates for UI branch: " + self.branch) @@ -85,12 +91,16 @@ class UIManager(object): d.addCallback(lambda r: self._download_ui() if not r else self._load_ui()) return d + def _check_for_bundled_ui(self): + bundle_manager = BundledUIManager(self.root, self.active_dir, get_bundled_ui_path()) + return bundle_manager.setup() + def _up_to_date(self): def _get_git_info(): try: + # TODO: should this be switched to the non-blocking getPage? response = urlopen(self._git_url) - data = json.loads(response.read()) - return defer.succeed(data['sha']) + return defer.succeed(read_sha(response)) except Exception: return defer.fail() @@ -177,9 +187,7 @@ class UIManager(object): return defer.succeed(False) def _do_migrate(): - if os.path.isdir(self.active_dir): - shutil.rmtree(self.active_dir) - shutil.copytree(source_dir, self.active_dir) + replace_dir(self.active_dir, source_dir) if delete_source: shutil.rmtree(source_dir) @@ -220,6 +228,73 @@ class UIManager(object): return d def _load_ui(self): - for d in [i[0] for i in os.walk(self.active_dir) if os.path.dirname(i[0]) == self.active_dir]: - self.root.putChild(os.path.basename(d), NoCacheStaticFile(d)) - return defer.succeed(True) + return load_ui(self.root, self.active_dir) + + +class BundledUIManager(object): + """Copies the UI bundled with lbrynet, if available. + + For the QA and nightly builds, we include a copy of the most + recent checkout of the development UI. For production builds + nothing is bundled. + + n.b: For QA and nightly builds the update check is skipped. + """ + def __init__(self, root, active_dir, bundled_ui_path): + self.root = root + self.active_dir = active_dir + self.bundled_ui_path = bundled_ui_path + self.data_path = os.path.join(bundled_ui_path, 'data.json') + + def bundle_is_available(self): + return os.path.exists(self.data_path) + + def setup(self): + """Load the bundled UI if possible and necessary + + Returns True if there is a bundled UI, False otherwise + """ + if not self.bundle_is_available(): + return False + if self.is_active_already_bundled_ui(): + return True + log.info('Using bundled UI') + replace_dir(self.active_dir, self.bundled_ui_path) + load_ui(self.root, self.active_dir) + return True + + def is_active_already_bundled_ui(self): + target_data_path = os.path.join(self.active_dir, 'data.json') + if os.path.exists(target_data_path): + if are_same_version(self.data_path, target_data_path): + return True + return False + + +def get_bundled_ui_path(): + return pkg_resources.resource_filename('lbrynet', 'resources/ui') + + +def are_same_version(data_a, data_b): + """Compare two data files and return True if they are the same version""" + with open(data_a) as a: + with open(data_b) as b: + return read_sha(a) == read_sha(b) + + +def read_sha(filelike): + data = json.load(filelike) + return data['sha'] + + +def replace_dir(active_dir, source_dir): + if os.path.isdir(active_dir): + shutil.rmtree(active_dir) + shutil.copytree(source_dir, active_dir) + + +def load_ui(root, active_dir): + for name in os.listdir(active_dir): + entry = os.path.join(active_dir, name) + if os.path.isdir(entry): + root.putChild(os.path.basename(entry), NoCacheStaticFile(entry)) diff --git a/packaging/travis/setup_osx.sh b/packaging/travis/setup_osx.sh index aeae7824b..9a1448856 100755 --- a/packaging/travis/setup_osx.sh +++ b/packaging/travis/setup_osx.sh @@ -7,4 +7,3 @@ wget https://www.python.org/ftp/python/2.7.11/python-2.7.11-macosx10.6.pkg sudo installer -pkg python-2.7.11-macosx10.6.pkg -target / pip install -U pip brew install gmp - diff --git a/packaging/windows/init.ps1 b/packaging/windows/init.ps1 index 37be0343c..34becaa96 100644 --- a/packaging/windows/init.ps1 +++ b/packaging/windows/init.ps1 @@ -36,7 +36,8 @@ C:\Python27\Scripts\pip.exe install requests==2.9.1 C:\Python27\Scripts\pip.exe install zope.interface==4.1.3 -C:\Python27\Scripts\pip.exe install cx-freeze==4.3.3 +# this includes a patch to allow version numbers with non-integer values +C:\Python27\Scripts\pip.exe install https://bitbucket.org/jobevers/cx_freeze/get/tip.tar.gz C:\Python27\Scripts\pip.exe install cython==0.24.1 diff --git a/setup.py b/setup.py index 353200305..5182359e3 100644 --- a/setup.py +++ b/setup.py @@ -64,6 +64,13 @@ console_scripts = [ 'lbrynet-cli = lbrynet.lbrynet_daemon.DaemonCLI:main' ] + +def package_files(directory): + for path, _, filenames in os.walk(directory): + for filename in filenames: + yield os.path.join('..', path, filename) + + if platform == LINUX: import ez_setup ez_setup.use_setuptools() @@ -82,16 +89,10 @@ if platform == LINUX: packages=find_packages(base_dir), install_requires=requires, entry_points={'console_scripts': console_scripts}, - data_files=[ - ('lbrynet/lbrynet_console/plugins', - [ - os.path.join(base_dir, 'lbrynet', 'lbrynet_console', 'plugins', - 'blindrepeater.yapsy-plugin') - ] - ), - ], - dependency_links=['https://github.com/lbryio/lbryum/tarball/master/#egg=lbryum'], - ) + package_data={ + package_name: list(package_files('lbrynet/resources/ui')) + } + ) elif platform == DARWIN: import ez_setup @@ -110,16 +111,10 @@ elif platform == DARWIN: packages=find_packages(base_dir), install_requires=requires, entry_points={'console_scripts': console_scripts}, - data_files=[ - ('lbrynet/lbrynet_console/plugins', - [ - os.path.join(base_dir, 'lbrynet', 'lbrynet_console', 'plugins', - 'blindrepeater.yapsy-plugin') - ] - ), - ], - dependency_links=['https://github.com/lbryio/lbryum/tarball/master/#egg=lbryum'], - ) + package_data={ + package_name: list(package_files('lbrynet/resources/ui')) + } + ) elif platform == WINDOWS: import opcode @@ -304,25 +299,18 @@ elif platform == WINDOWS: script=os.path.join(app_dir, 'LBRYWin32App.py'), base='Win32GUI', icon=win_icon, - compress=True, - # shortcutName=dist_name, - # shortcutDir='DesktopFolder', targetName='{0}.exe'.format(dist_name) ) daemon_exe = Executable( script=os.path.join(daemon_dir, 'DaemonControl.py'), icon=win_icon, - # shortcutName="lbrynet-daemon", - # shortcutDir='DesktopFolder', targetName='lbrynet-daemon.exe' ) cli_exe = Executable( script=os.path.join(daemon_dir, 'DaemonCLI.py'), icon=win_icon, - # shortcutName="lbrynet-cli", - # shortcutDir='DesktopFolder', targetName='lbrynet-cli.exe' ) @@ -336,11 +324,16 @@ elif platform == WINDOWS: author=author, keywords=keywords, data_files=[], - options={'build_exe': build_exe_options, - 'bdist_msi': bdist_msi_options}, + options={ + 'build_exe': build_exe_options, + 'bdist_msi': bdist_msi_options + }, executables=[ tray_app, daemon_exe, cli_exe ], + package_data={ + package_name: list(package_files('lbrynet/resources/ui')) + } ) From 694b5b9ee53d205c954bf345db9b913c6728e77f Mon Sep 17 00:00:00 2001 From: Job Evers-Meltzer Date: Wed, 9 Nov 2016 11:02:00 -0600 Subject: [PATCH 4/4] switch branch to master --- .appveyor.yml | 2 +- .travis.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.appveyor.yml b/.appveyor.yml index 8fd49c589..819c63e43 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -62,4 +62,4 @@ deploy: region: us-west-2 artifact: /.*\.msi/ on: - branch: bundled-ui + branch: master diff --git a/.travis.yml b/.travis.yml index d4c382810..fb2a8ca0b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -80,7 +80,7 @@ deploy: local_dir: "${TRAVIS_BUILD_DIR}/upload" region: us-west-2 on: - branch: bundled-ui + branch: master env: