Bundle the UI into non-tagged builds.

Updates the UIManager to check if a bundled UI was included
in the package and preferentially use that.

.appveyor.yml

@@ -26,8 +26,7 @@ install:
 
 
 build_script:
-- ps: C:\Python27\python.exe setup.py build bdist_msi
+- ps: .\packaging\windows\build.ps1
-- signtool.exe sign /f packaging\windows\certs\lbry2.pfx /p %key_pass% /tr http://tsa.starfieldtech.com /td SHA256 /fd SHA256 dist\*.msi
 
 
 test_script:
@@ -44,9 +43,9 @@ artifacts:
 
 
 deploy:
+  - provider: GitHub
     release: $(APPVEYOR_REPO_TAG_NAME)
     description: 'Release'
-  provider: GitHub
     auth_token:
       secure: 28gMVxQkXr2iXP4F+5kVwefUtKCfS1ePZ97PVfaSR8UDupjAvKhSJx862TnEjukb
     artifact: /.*\.msi/
@@ -54,3 +53,13 @@ deploy:
     prerelease: true
     on:
       appveyor_repo_tag: true        # deploy on tag push only
+  - provider: S3
+    access_key_id:
+      secure: E25iHvmHiJP56GWFTe14L3pS8QKb5ZcyWZLY0zqh6BA=
+    secret_access_key:
+      secure: YwZe6fCv29akFYjMCCBJBF7LtI6mxPrxbq7SSoAbn1BPdqjATFegeteGAsqHur/C
+    bucket: lbrynet-master
+    region: us-west-2
+    artifact: /.*\.msi/
+    on:
+      branch: bundled-ui

.travis.yml

@@ -26,6 +26,8 @@ cache:
 
 before_install:
   - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then ./packaging/travis/setup_osx.sh; fi
+  - mkdir -p lbrynet/resources/ui
+  - if [[ -z "$TRAVIS_TAG" ]]; then ./packaging/travis/setup_qa.sh; fi
   
 install:
   - if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then ./packaging/travis/install_dependencies_and_run_tests.sh; fi
@@ -41,9 +43,15 @@ script:
   # fail the build if this is a build for a tag and we don't have the versions matching; allow tags that start with 'test' to pass
   - if [[ -n "${TRAVIS_TAG}" ]]; then if [[ "${TRAVIS_TAG}" == test* ]] || [[ "v`python setup.py -V`" = "${TRAVIS_TAG}" ]]; then true; else false; fi; fi
 
+before_deploy:
+  # s3 release can only upload a folder so move the package into an upload folder
+  - mkdir "${TRAVIS_BUILD_DIR}/upload"
+  - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then mv "${TRAVIS_BUILD_DIR}/packaging/osx/lbry-osx-app/`python setup.py --name`.`python setup.py -V`.dmg" "${TRAVIS_BUILD_DIR}/upload"; fi
+  - if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then mv "${TRAVIS_BUILD_DIR}/`python setup.py --name`_`python setup.py -V`_amd64.deb" "${TRAVIS_BUILD_DIR}/upload"; fi
+
 deploy:
   - provider: releases
-    file: "${TRAVIS_BUILD_DIR}/`python setup.py --name`_`python setup.py -V`_amd64.deb"
+    file: "${TRAVIS_BUILD_DIR}/upload/`python setup.py --name`_`python setup.py -V`_amd64.deb"
     skip_cleanup: true
     prerelease: true
     on:
@@ -53,7 +61,7 @@ deploy:
     api_key:
       secure: nKdWGROnLNodx9k9nWvq2wezkPvSVL8zF63qjPXjhdOe9kCUbcLp7WnFDYpn9EJj4Pofq/ejeCHwjA+5x7JUP3Szk7SlJV61B4c/5hl64rl7oSKOoKskjdE2jaOG3CJuOUrh0yQ59U3vMMABcsnw/wJaCIuu/erdPIm8g8R+stu1YHOGtl5Y9WiW+zLJn2vc3GooV1TWtki9EnrmFfw0Vrqc4RMVMFB1ojE7ggrK1LIwcmGSbLIYzker1ZRz8SCy+84sGk4//V+2i2NNiz5AkPuG7BBGrU2twE9nD23IlruJAdVdi71P3ytAmi0kKyvxIU4VeNaqyTk9zeL5IB9J5IIgvekHgKcsKhFUZ6QcXT1Xfxl4ELftvWCTHWiewnXFdqLcG9GZiUaE6+7wdalwDAP3tqS2emiibetlBZERHR+RMR00ej+1MBYWGMlTse/0Tglndv0a2qqgAJYLKPRT02hTRYGxZ1MrJe+WGnChRmzwgLVTIgZuiDciFOahN0TYGSORk6OpnZBsxvpzSqDw5UDJx0BmbJ1xMNDFbOs8ubZ9yIpB9yNMGw66FPacOF61XNYnmA68ILC28UtOFKuuHLrUPbM5JmQkDVhtTfFbBnyHefyCLAL4MHvJJKGi1oaOXjYaJ/J095h636/kQ0cHHuVMgoWUQZOQ44xRAz7tMuc=
   - provider: releases
-    file: "${TRAVIS_BUILD_DIR}/packaging/osx/lbry-osx-app/`python setup.py --name`.`python setup.py -V`.dmg"
+    file: "${TRAVIS_BUILD_DIR}/upload/`python setup.py --name`.`python setup.py -V`.dmg"
     skip_cleanup: true
     prerelease: true
     on:
@@ -62,6 +70,18 @@ deploy:
     # this is the oauth token for the lbry-ci user
     api_key:
       secure: nKdWGROnLNodx9k9nWvq2wezkPvSVL8zF63qjPXjhdOe9kCUbcLp7WnFDYpn9EJj4Pofq/ejeCHwjA+5x7JUP3Szk7SlJV61B4c/5hl64rl7oSKOoKskjdE2jaOG3CJuOUrh0yQ59U3vMMABcsnw/wJaCIuu/erdPIm8g8R+stu1YHOGtl5Y9WiW+zLJn2vc3GooV1TWtki9EnrmFfw0Vrqc4RMVMFB1ojE7ggrK1LIwcmGSbLIYzker1ZRz8SCy+84sGk4//V+2i2NNiz5AkPuG7BBGrU2twE9nD23IlruJAdVdi71P3ytAmi0kKyvxIU4VeNaqyTk9zeL5IB9J5IIgvekHgKcsKhFUZ6QcXT1Xfxl4ELftvWCTHWiewnXFdqLcG9GZiUaE6+7wdalwDAP3tqS2emiibetlBZERHR+RMR00ej+1MBYWGMlTse/0Tglndv0a2qqgAJYLKPRT02hTRYGxZ1MrJe+WGnChRmzwgLVTIgZuiDciFOahN0TYGSORk6OpnZBsxvpzSqDw5UDJx0BmbJ1xMNDFbOs8ubZ9yIpB9yNMGw66FPacOF61XNYnmA68ILC28UtOFKuuHLrUPbM5JmQkDVhtTfFbBnyHefyCLAL4MHvJJKGi1oaOXjYaJ/J095h636/kQ0cHHuVMgoWUQZOQ44xRAz7tMuc=
+  - provider: s3
+    access_key_id:
+      secure: "gmJNW8bda2snpA2F+0gucjgO/orvZL0a348QmiffYdtXleIseyY+C4ZI9llWm+s8n2TxiqBYpc/A3Bv7JM0yIccjF0CWQb8pu4HkVT2xLT/1p28EdfxKhR0H5sLtkxOCvLIuf3ppZWac8IzRWvh9TDER1TVxbIGvbOJCtKJ+sUKGCWVkxJY6lzAy7+YHwY9sLV6GvarbkfcnbwNh0qPEROcc6JXUQRKCjWOoZLSZHx4dwxlDiXaG1GyUSm2bwtB11VbMZqaP5eO2zypcUr24J0WkzuAOFIUMDERQnY6eSR62T88BjVjL+07kQufSTtnHC49uHsylyQLwkphkwi2Ei3c/fJ5XI1gY0Med/WOHuy99LAaaHQvHxVa9Zxvrgivvu1oa2QdLir05asMCB0G6Yjgz38Fl8jzdKN/PLHs/lhgBGOJfMN+0UR7hOrLeXdJcMriVfwzYrCl6KkFgTe50rcMx8vv6SH15AqZ5sceksy+maA4O8Mxq1hEe2qJABgRdRZK3FnFKNnQOxfrlU6N9zMNsicVcRwH6WNBtoL4BK3KeMproWnorh9rXmdoSG2Fh5X636cudYhLrAbO5yaoZAXFxHqOV1n63v3tgv4MKrB999OF9V9HyYXv3ro9WZTTsH7LskLA7YmTvXtpMVgBK8SfnvqCjLim5qpVvIVPQIOY="
+    secret_access_key:
+      secure: "pS28jMSNV8HUSa8P8pReXgBtHW0OSWDZMAlZhBrtUI05goUnK6buXadvzCynZWgDiXPHuEuLZOLDVj5yL8N3eCa+sG1tEDcCjPxrBFxHisng8atgn3FaL15MBOdXn0qOBUr+vQ4jbUozsBYGqmoZQV0/Af53tU55KK5HJrotkd++SNTEQrFbWZavM+Wvhw4yJN2hmRer9II72DCZkayUjpvBq+irnGeU0g9SrX8VULCg+sqcnbqdLWd+VALHzQr+O74MaWE8mu2PA8A1GKWZZ/a+LoTh1T2bArpX94q0ueea1eEcUA154vC1azkKK5vy9hzENt5qiiEleBDE31DhbGaeEoakEXvHWrdI3DYiakD6prj0necD8aAK6VhPtqwiTtTf7E4BE9hxY4nJRDuZckvFI4LYFMvHlbdqBieeZPflStkws7VgsfrGV7EberYIbSEohUGGUBg3rdaCIPsn8W4nQAixq39Dcfm0orGo4BgargT6FFtAeDzsfYdbKBk/F8BOPMDJz9P1utAZ/ZeBexic16aeIbk7+wtaQUd0Hswm+2QUHddshvp9IzNniFb1BLvRR/lOVl0HdUkJj/haR2M8XhwcRFIKpL2oMZHwRmLW4c1kkEx+KSfAB77arjgrCzq3y5MlKUSliL5qZZMIJX0ReRQ2dcy3DCY0Fxdj014="
+    bucket: lbrynet-master
+    skip_cleanup: true
+    local_dir: "${TRAVIS_BUILD_DIR}/upload"
+    region: us-west-2
+    on:
+      branch: bundled-ui
+
 
 env:
   global:

MANIFEST.in

@@ -1 +0,0 @@
-include lbrynet/lbrynet_console/plugins/blindrepeater.yapsy-plugin

lbrynet/lbrynet_daemon/UIManager.py

@@ -2,16 +2,19 @@ import os
 import logging
 import shutil
 import json
-
 from urllib2 import urlopen
 from StringIO import StringIO
+from zipfile import ZipFile
+
+import pkg_resources
 from twisted.internet import defer
 from twisted.internet.task import LoopingCall
+
 from lbrynet.conf import settings
 from lbrynet.lbrynet_daemon.Resources import NoCacheStaticFile
 from lbrynet import __version__ as lbrynet_version
 from lbryum.version import LBRYUM_VERSION as lbryum_version
-from zipfile import ZipFile
+
 
 
 log = logging.getLogger(__name__)
@@ -62,6 +65,9 @@ class UIManager(object):
         self.branch = settings.ui_branch or branch
         self.check_requirements = settings.check_ui_requirements or check_requirements
 
+        if self._check_for_bundled_ui():
+            return defer.succeed(True)
+
         if local_ui_path:
             if os.path.isdir(local_ui_path):
                 log.info("Checking user specified UI directory: " + str(local_ui_path))
@@ -74,7 +80,7 @@ class UIManager(object):
                 log.info("User specified UI directory doesn't exist, using " + self.branch)
         elif self.loaded_branch == "user-specified":
             log.info("Loading user provided UI")
-            d = self._load_ui()
+            d = defer.maybeDeferred(self._load_ui())
             return d
         else:
             log.info("Checking for updates for UI branch: " + self.branch)
@@ -85,12 +91,16 @@ class UIManager(object):
         d.addCallback(lambda r: self._download_ui() if not r else self._load_ui())
         return d
 
+    def _check_for_bundled_ui(self):
+        bundle_manager = BundledUIManager(self.root, self.active_dir, get_bundled_ui_path())
+        return bundle_manager.setup()
+
     def _up_to_date(self):
         def _get_git_info():
             try:
+                # TODO: should this be switched to the non-blocking getPage?
                 response = urlopen(self._git_url)
-                data = json.loads(response.read())
+                return defer.succeed(read_sha(response))
-                return defer.succeed(data['sha'])
             except Exception:
                 return defer.fail()
 
@@ -177,9 +187,7 @@ class UIManager(object):
             return defer.succeed(False)
 
         def _do_migrate():
-            if os.path.isdir(self.active_dir):
+            replace_dir(self.active_dir, source_dir)
-                shutil.rmtree(self.active_dir)
-            shutil.copytree(source_dir, self.active_dir)
             if delete_source:
                 shutil.rmtree(source_dir)
 
@@ -220,6 +228,73 @@ class UIManager(object):
         return d
 
     def _load_ui(self):
-        for d in [i[0] for i in os.walk(self.active_dir) if os.path.dirname(i[0]) == self.active_dir]:
+        return load_ui(self.root, self.active_dir)
-            self.root.putChild(os.path.basename(d), NoCacheStaticFile(d))
+
-        return defer.succeed(True)
+
+class BundledUIManager(object):
+    """Copies the UI bundled with lbrynet, if available.
+
+    For the QA and nightly builds, we include a copy of the most
+    recent checkout of the development UI. For production builds
+    nothing is bundled.
+
+    n.b: For QA and nightly builds the update check is skipped.
+    """
+    def __init__(self, root, active_dir, bundled_ui_path):
+        self.root = root
+        self.active_dir = active_dir
+        self.bundled_ui_path = bundled_ui_path
+        self.data_path = os.path.join(bundled_ui_path, 'data.json')
+
+    def bundle_is_available(self):
+        return os.path.exists(self.data_path)
+
+    def setup(self):
+        """Load the bundled UI if possible and necessary
+
+        Returns True if there is a bundled UI, False otherwise
+        """
+        if not self.bundle_is_available():
+            return False
+        if self.is_active_already_bundled_ui():
+            return True
+        log.info('Using bundled UI')
+        replace_dir(self.active_dir, self.bundled_ui_path)
+        load_ui(self.root, self.active_dir)
+        return True
+
+    def is_active_already_bundled_ui(self):
+        target_data_path = os.path.join(self.active_dir, 'data.json')
+        if os.path.exists(target_data_path):
+            if are_same_version(self.data_path, target_data_path):
+                return True
+        return False
+
+
+def get_bundled_ui_path():
+    return pkg_resources.resource_filename('lbrynet', 'resources/ui')
+
+
+def are_same_version(data_a, data_b):
+    """Compare two data files and return True if they are the same version"""
+    with open(data_a) as a:
+        with open(data_b) as b:
+            return read_sha(a) == read_sha(b)
+
+
+def read_sha(filelike):
+    data = json.load(filelike)
+    return data['sha']
+
+
+def replace_dir(active_dir, source_dir):
+    if os.path.isdir(active_dir):
+        shutil.rmtree(active_dir)
+    shutil.copytree(source_dir, active_dir)
+
+
+def load_ui(root, active_dir):
+    for name in os.listdir(active_dir):
+        entry = os.path.join(active_dir, name)
+        if os.path.isdir(entry):
+            root.putChild(os.path.basename(entry), NoCacheStaticFile(entry))

packaging/travis/setup_osx.sh

@@ -7,4 +7,3 @@ wget https://www.python.org/ftp/python/2.7.11/python-2.7.11-macosx10.6.pkg
 sudo installer -pkg python-2.7.11-macosx10.6.pkg -target /
 pip install -U pip
 brew install gmp
-

packaging/windows/init.ps1

@@ -36,7 +36,8 @@ C:\Python27\Scripts\pip.exe install requests==2.9.1
 
 C:\Python27\Scripts\pip.exe install zope.interface==4.1.3
 
-C:\Python27\Scripts\pip.exe install cx-freeze==4.3.3
+# this includes a patch to allow version numbers with non-integer values
+C:\Python27\Scripts\pip.exe install https://bitbucket.org/jobevers/cx_freeze/get/tip.tar.gz
 
 C:\Python27\Scripts\pip.exe install cython==0.24.1
 

setup.py

@@ -64,6 +64,13 @@ console_scripts = [
     'lbrynet-cli = lbrynet.lbrynet_daemon.DaemonCLI:main'
 ]
 
+
+def package_files(directory):
+    for path, _, filenames in os.walk(directory):
+        for filename in filenames:
+            yield os.path.join('..', path, filename)
+
+
 if platform == LINUX:
     import ez_setup
     ez_setup.use_setuptools()
@@ -82,15 +89,9 @@ if platform == LINUX:
           packages=find_packages(base_dir),
           install_requires=requires,
           entry_points={'console_scripts': console_scripts},
-          data_files=[
+          package_data={
-              ('lbrynet/lbrynet_console/plugins',
+              package_name: list(package_files('lbrynet/resources/ui'))
-               [
+          }
-                   os.path.join(base_dir, 'lbrynet', 'lbrynet_console', 'plugins',
-                                'blindrepeater.yapsy-plugin')
-               ]
-               ),
-          ],
-          dependency_links=['https://github.com/lbryio/lbryum/tarball/master/#egg=lbryum'],
     )
 
 elif platform == DARWIN:
@@ -110,15 +111,9 @@ elif platform == DARWIN:
           packages=find_packages(base_dir),
           install_requires=requires,
           entry_points={'console_scripts': console_scripts},
-          data_files=[
+          package_data={
-              ('lbrynet/lbrynet_console/plugins',
+              package_name: list(package_files('lbrynet/resources/ui'))
-               [
+          }
-                   os.path.join(base_dir, 'lbrynet', 'lbrynet_console', 'plugins',
-                                'blindrepeater.yapsy-plugin')
-               ]
-               ),
-          ],
-          dependency_links=['https://github.com/lbryio/lbryum/tarball/master/#egg=lbryum'],
     )
 
 elif platform == WINDOWS:
@@ -304,25 +299,18 @@ elif platform == WINDOWS:
         script=os.path.join(app_dir, 'LBRYWin32App.py'),
         base='Win32GUI',
         icon=win_icon,
-        compress=True,
-        # shortcutName=dist_name,
-        # shortcutDir='DesktopFolder',
         targetName='{0}.exe'.format(dist_name)
     )
 
     daemon_exe = Executable(
         script=os.path.join(daemon_dir, 'DaemonControl.py'),
         icon=win_icon,
-        # shortcutName="lbrynet-daemon",
-        # shortcutDir='DesktopFolder',
         targetName='lbrynet-daemon.exe'
     )
 
     cli_exe = Executable(
         script=os.path.join(daemon_dir, 'DaemonCLI.py'),
         icon=win_icon,
-        # shortcutName="lbrynet-cli",
-        # shortcutDir='DesktopFolder',
         targetName='lbrynet-cli.exe'
     )
 
@@ -336,11 +324,16 @@ elif platform == WINDOWS:
         author=author,
         keywords=keywords,
         data_files=[],
-        options={'build_exe': build_exe_options,
+        options={
-                 'bdist_msi': bdist_msi_options},
+            'build_exe': build_exe_options,
+            'bdist_msi': bdist_msi_options
+        },
         executables=[
             tray_app,
             daemon_exe,
             cli_exe
         ],
+        package_data={
+            package_name: list(package_files('lbrynet/resources/ui'))
+        }
     )