Merge pull request #5 from lbryio/test-electron-release

Add release script

.bumpversion.cfg

@@ -0,0 +1,17 @@
+[bumpversion]
+current_version = 0.9.0rc7
+commit = True
+tag = True
+parse = (?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)((?P<release>[a-z]+)(?P<candidate>\d+))?
+serialize = 
+	{major}.{minor}.{patch}{release}{candidate}
+	{major}.{minor}.{patch}
+
+[bumpversion:part:release]
+optional_value = production
+values = 
+	rc
+	production
+
+[bumpversion:file:app/package.json]
+

app/package.json

@@ -1,6 +1,6 @@
 {
   "name": "LBRY",
-  "version": "0.0.1",
+  "version": "0.9.0rc7",
   "main": "main.js",
   "description": "LBRY is a fully decentralized, open-source protocol facilitating the discovery, access, and (sometimes) purchase of data.",
   "author": {

appveyor.yml

@@ -1,6 +1,8 @@
 # Test against the latest version of this Node.js version
 environment:
   nodejs_version: "6"
+  GH_TOKEN:
+    secure: LiI5jyuHUw6XbH4kC3gP1HX4P/v4rwD/gCNtaFhQu2AvJz1/1wALkp5ECnIxRySN
 
 skip_branch_with_pr: true
 
@@ -37,21 +39,12 @@ build_script:
   - node_modules\.bin\build -p never
   # for debugging, see what was built
   - dir dist
+  - pip install PyGithub uritemplate
+  - python release_on_tag.py
 
 test: off
 
+
 artifacts:
   - path: dist\*.exe
     name: LBRY
-
-deploy:
-  - provider: GitHub
-    release: $(APPVEYOR_REPO_TAG_NAME)
-    description: 'Release'
-    auth_token:
-      secure: LiI5jyuHUw6XbH4kC3gP1HX4P/v4rwD/gCNtaFhQu2AvJz1/1wALkp5ECnIxRySN
-    artifact: LBRY
-    draft: false
-    prerelease: true
-    on:
-      appveyor_repo_tag: true        # deploy on tag push only

build.sh

@@ -83,6 +83,6 @@ if [ "$FULL_BUILD" == "true" ]; then
   # it to reliably work and it also seemed difficult to configure. Not proud of
   # this, but it seemed better to write my own.
   pip install PyGithub uritemplate
-  python release-on-tag.py
+  python release_on_tag.py
   deactivate
 fi

lbry

@@ -1 +1 @@
-Subproject commit 1d30ae447a897b1e1543a6a4cda5dbd242d2967d
+Subproject commit 04b36e2252612047e82ade94b0ca6c8af538d45b

lbry-web-ui

@@ -1 +1 @@
-Subproject commit 55b246512585e9d3abc4d6bcbc08367df9c2b879
+Subproject commit fe954ed17d461a1760e3051984eb7d48c6238cfe

lbryum

@@ -1 +1 @@
-Subproject commit 8e9362567d01732da06972c66f507df228f9817d
+Subproject commit 8834b60c7cdd9c0c31b51be87e8bcf1b21dbdbed

release-on-tag.py

@@ -1,161 +0,0 @@
-import argparse
-import glob
-import json
-import logging
-import os
-import platform
-import re
-import subprocess
-import sys
-import zipfile
-
-import github
-import requests
-import uritemplate
-
-from lbrynet.core import log_support
-
-
-def main(args=None):
-    parser = argparse.ArgumentParser()
-    parser.add_argument('--file', help='artifact to publish')
-    parser.add_argument('--label', help='text to append to `file`')
-    parser.add_argument('--zip', action='store_true')
-    parser.add_argument(
-        '--force', action='store_true',
-        help='ignores whether the repo is currently tagged, publishes a draft release')
-    args = parser.parse_args(args)
-
-    gh_token = os.environ['GH_TOKEN']
-    auth = github.Github(gh_token)
-    current_repo = auth.get_repo(current_repo_name())
-
-    if args.file:
-        artifact = args.file
-    else:
-        artifact = get_artifact()
-
-    current_tag = None
-    if not args.force:
-        try:
-            current_tag = subprocess.check_output(
-                ['git', 'describe', '--exact-match', 'HEAD']).strip()
-        except subprocess.CalledProcessError:
-            log.info('Stopping as we are not currently on a tag')
-            return
-        if not check_repo_has_tag(current_repo, current_tag):
-            log.info('Tag %s is not in repo %s', current_tag, current_repo)
-            # TODO: maybe this should be an error
-            return
-
-    release = get_release(current_repo, current_tag, args.force)
-    asset_to_upload = get_asset(artifact, args.label, args.zip)
-    upload_asset(release, asset_to_upload, gh_token)
-
-
-def current_repo_name():
-    pattern = 'github.com[:/](.*)\.git'
-    remote = subprocess.check_output('git remote -v'.split())
-    m = re.search(pattern, remote)
-    if not m:
-        raise Exception('Unable to parse repo name from remote: {}'.format(remote))
-    return m.group(1)
-
-
-def check_repo_has_tag(repo, target_tag):
-    tags = repo.get_tags().get_page(0)
-    for tag in tags:
-        if tag.name == target_tag:
-            return True
-    return False
-
-
-def get_release(current_repo, current_tag=None, draft=False):
-    assert current_tag or draft, 'either current_tag or draft must be set'
-    need_new_release = False
-    if not draft and current_tag:
-        try:
-            release = current_repo.get_release(current_tag)
-            log.info('Using an existing release for %s', current_tag)
-        except github.UnknownObjectException:
-            need_new_release = True
-    if draft or need_new_release:
-        log.info('Creating a new release for %s:%s', current_repo, current_tag)
-        tag = current_tag or 'draft'
-        release_name = current_tag or 'draft'
-        msg = 'Release' # TODO: parse changelogs to get a better message
-        try:
-            # we have a race condition where its possible that between checking
-            # for the release and now, another build agent has come along and already
-            # created a release
-            release = current_repo.create_git_release(tag, release_name, msg, draft)
-        except github.GithubException:
-            log.info('Failed to create a release, maybe somebody already has', exc_info=True)
-            release = current_repo.get_release(current_tag)
-    return release
-
-
-def get_artifact():
-    system = platform.system()
-    if system == 'Darwin':
-        return glob.glob('dist/mac/LBRY*.dmg')[0]
-    elif system == 'Linux':
-        return glob.glob('dist/LBRY*.deb')[0]
-    else:
-        raise Exception("I don't know about any artifact on {}".format(system))
-
-
-def get_asset(filename, label=None, use_zip=False):
-    if label:
-        label = '-{}'.format(label)
-    else:
-        label = ''
-    base, ext = os.path.splitext(filename)
-    if use_zip:
-        # TODO: probably want to clean this up
-        zipfilename = '{}{}.zip'.format(base, label)
-        with zipfile.ZipFile(zipfilename, 'w') as myzip:
-            myzip.write(filename)
-        asset_to_uplaod = zipfilename
-    else:
-        asset_to_upload = '{}{}{}'.format(base, label, ext)
-    return asset_to_upload
-
-
-def upload_asset(release, asset_to_upload, token):
-    basename = os.path.basename(asset_to_upload)
-
-    if is_asset_already_uploaded(release, basename):
-        return
-
-    upload_uri = uritemplate.expand(
-        release.upload_url, {'name': basename})
-    # using requests.post fails miserably with SSL EPIPE errors. I spent
-    # half a day trying to debug before deciding to switch to curl.
-    cmd = [
-        'curl', '-sS', '-X', 'POST', '-u', ':{}'.format(os.environ['GH_TOKEN']),
-        '--header', 'Content-Type:application/zip',
-        '--data-binary', '@{}'.format(asset_to_upload), upload_uri
-    ]
-    raw_output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
-    output = json.loads(raw_output)
-    if 'errors' in output:
-        raise Exception(output)
-    else:
-        log.info('Successfully uploaded to %s', output['browser_download_url'])
-
-
-def is_asset_already_uploaded(release, basename):
-    for asset in release.raw_data['assets']:
-        if asset['name'] == basename:
-            log.info('File %s has already been uploaded to %s', basename, release.tag_name)
-            return True
-    return False
-
-
-if __name__ == '__main__':
-    log = logging.getLogger('release-on-tag')
-    log_support.configure_console(level='DEBUG')
-    sys.exit(main())
-else:
-    log = logging.getLogger(__name__)

release.py

@@ -0,0 +1,205 @@
+"""Trigger a release.
+
+This script is to be run locally (not on a build server).
+"""
+import argparse
+import contextlib
+import logging
+import os
+import re
+import subprocess
+import sys
+
+import git
+import github
+import requests
+
+
+CHANGELOG_START_RE = re.compile(r'^\#\# \[Unreleased\]')
+CHANGELOG_END_RE = re.compile(r'^\#\# \[.*\] - \d{4}-\d{2}-\d{2}')
+# if we come across a section header between two release section headers
+# then we probably have an improperly formatted changelog
+CHANGELOG_ERROR_RE = re.compile(r'^\#\# ')
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("lbry_part", help="part of lbry version to bump")
+    parser.add_argument("--lbryum_part", help="part of lbryum version to bump")
+    parser.add_argument("--ui-part", help="part of the ui to bump")
+    parser.add_argument("--branch", default='master', help="branch to use for each repo")
+    parser.add_argument("--last-release")
+    parser.add_argument("--skip-sanity-checks", action="store_true")
+    args = parser.parse_args()
+
+    branch = args.branch
+
+    base = git.Repo(os.getcwd())
+
+    if not args.skip_sanity_checks:
+        run_sanity_checks(base, args)
+
+    if args.last_release:
+        last_release = args.last_release
+    else:
+        response = requests.get('https://api.github.com/repos/lbryio/lbry-app/releases/latest')
+        data = response.json()
+        last_release = data['tag_name']
+
+    gh_token = os.environ['GH_TOKEN']
+    auth = github.Github(gh_token)
+    github_repo = auth.get_repo('lbryio/lbry-app')
+
+    names = ['lbry', 'lbry-web-ui', 'lbryum']
+    repos = [Repo(name) for name in names]
+
+    # in order to see if we've had any change in the submodule, we need to checkout
+    # our last release, see what commit we were on, and then compare that to
+    # current
+    base.git.checkout(last_release)
+    base.submodule_update()
+    for repo in repos:
+        repo.save_commit()
+
+    base.git.checkout(branch)
+    base.submodule_update()
+
+    changelogs = {}
+
+    # ensure that we have changelog entries for each part
+    for repo in repos:
+        if not repo.has_changes():
+            continue
+        entry = repo.get_changelog_entry().strip()
+        if not entry:
+            raise Exception('Changelog is missing for {}'.format(repo.name))
+        changelogs[repo.name] = entry
+        part = get_part(args, repo.name)
+        if not part:
+            raise Exception('Cannot bump version for {}: no part specified'.format(repo.name))
+        repo.bumpversion(part)
+
+    release_msg = get_release_msg(changelogs, names)
+
+    for name in names:
+        base.git.add(name)
+    subprocess.check_call(['bumpversion', args.lbry_part, '--allow-dirty'])
+
+    current_tag = base.git.describe()
+
+    github_repo.create_git_release(current_tag, current_tag, release_msg, draft=True)
+    auth.get_repo('lbryio/lbrynet-daemon').create_git_release(
+        current_tag, current_tag, changelogs['lbry'], draft=True)
+    for repo in repos:
+        repo.git.push(follow_tags=True)
+    base.git.push(follow_tags=True, recurse_submodules='check')
+
+
+def get_release_msg(changelogs, names):
+    lines = []
+    for name in names:
+        entry = changelogs.get(name)
+        if not entry:
+            continue
+        lines.append('## {}\n'.format(name))
+        lines.append('{}\n'.format(entry))
+    return '\n'.join(lines)
+
+
+def run_sanity_checks(base, args):
+    branch = args.branch
+    if base.is_dirty():
+        print 'Cowardly refusing to release a dirty repo'
+        sys.exit(1)
+    if base.active_branch.name != branch:
+        print 'Cowardly refusing to release when not on the {} branch'.format(branch)
+        sys.exit(1)
+    origin = base.remotes.origin
+    origin.fetch()
+    if base.commit() != origin.refs[branch].commit:
+        print 'Cowardly refusing to release when not synced with origin'
+        sys.exit(1)
+    check_bumpversion()
+
+
+def check_bumpversion():
+
+    def requireNewVersion():
+        print 'Install bumpversion: pip install -U git+https://github.com/lbryio/bumpversion.git'
+        sys.exit(1)
+
+    try:
+        output = subprocess.check_output(['bumpversion', '-v'], stderr=subprocess.STDOUT)
+        output = output.strip()
+        if output != 'bumpversion 0.5.4-lbry':
+            requireNewVersion()
+    except subprocess.CalledProcessError:
+        requireNewVersion()
+
+def get_part(args, name):
+    if name == 'lbry-web-ui':
+        part = getattr(args, 'ui_part')
+        return part or args.lbry_part
+    else:
+        return getattr(args, name + '_part')
+
+
+class Repo(object):
+    def __init__(self, name):
+        self.name = name
+        self.directory = os.path.join(os.getcwd(), name)
+        self.git_repo = git.Repo(self.directory)
+        self.saved_commit = None
+
+    def has_changes(self):
+        return self.git_repo.commit() == self.saved_commit
+
+    def save_commit(self):
+        self.saved_commit = self.git_repo.commit()
+
+    def checkout(self, branch):
+        self.git_repo.git.checkout(branch)
+
+    def get_changelog_entry(self):
+        filename = os.path.join(self.directory, 'CHANGELOG.md')
+        err = 'Had trouble parsing changelog {}: {}'
+        output = []
+        start_found = False
+        with open(filename) as fp:
+            for line in fp:
+                if not start_found:
+                    if CHANGELOG_START_RE.search(line):
+                        start_found = True
+                    continue
+                if CHANGELOG_END_RE.search(line):
+                    return ''.join(output)
+                if CHANGELOG_ERROR_RE.search(line):
+                    raise Exception(err.format(filename, 'unexpected section header found'))
+                output.append(line)
+        # if we get here there was no previous release section, which is a problem
+        if start_found:
+            # TODO: once the lbry-web-ui has a released entry, uncomment this error
+            # raise Exception(err.format(filename, 'Reached end of file'))
+            return ''.join(output)
+        else:
+            raise Exception(err.format(filename, 'Unreleased section not found'))
+
+    def bumpversion(self, part):
+        with pushd(self.directory):
+            subprocess.check_call(['bumpversion', part])
+
+    @property
+    def git(self):
+        return self.git_repo.git
+
+
+@contextlib.contextmanager
+def pushd(new_dir):
+    previous_dir = os.getcwd()
+    os.chdir(new_dir)
+    yield
+    os.chdir(previous_dir)
+
+
+if __name__ == '__main__':
+    sys.exit(main())

release_on_tag.py

@@ -0,0 +1,134 @@
+import argparse
+import glob
+import json
+import logging
+import os
+import platform
+import re
+import subprocess
+import sys
+import zipfile
+
+import github
+import requests
+import uritemplate
+
+from lbrynet.core import log_support
+
+
+def main(args=None):
+    current_tag = None
+    try:
+        current_tag = subprocess.check_output(
+            ['git', 'describe', '--exact-match', 'HEAD']).strip()
+    except subprocess.CalledProcessError:
+        log.info('Stopping as we are not currently on a tag')
+        return
+
+    if 'GH_TOKEN' not in os.environ:
+        print 'Must set GH_TOKEN in order to publish assets to a release'
+        return
+
+    gh_token = os.environ['GH_TOKEN']
+    auth = github.Github(gh_token)
+    app_repo = auth.get_repo('lbryio/lbry-app')
+    # TODO: switch lbryio/lbrynet-daemon to lbryio/lbry
+    daemon_repo = auth.get_repo('lbryio/lbrynet-daemon')
+
+    if not check_repo_has_tag(app_repo, current_tag):
+        log.info('Tag %s is not in repo %s', current_tag, app_repo)
+        # TODO: maybe this should be an error
+        return
+
+    artifact = get_artifact()
+    release = get_release(app_repo, current_tag)
+    upload_asset(release, artifact, gh_token)
+
+    release = get_release(daemon_repo, current_tag)
+    artifact = os.path.join('app', 'dist', 'lbrynet-daemon')
+    if platform.system() == 'Windows':
+        artifact += '.exe'
+    asset_to_upload = get_asset(artifact, get_system_label())
+    upload_asset(release, asset_to_upload, gh_token)
+
+
+def check_repo_has_tag(repo, target_tag):
+    tags = repo.get_tags().get_page(0)
+    for tag in tags:
+        if tag.name == target_tag:
+            return True
+    return False
+
+
+def get_release(current_repo, current_tag):
+    for release in current_repo.get_releases():
+        if release.tag_name == current_tag:
+            return release
+    raise Exception('No release for {} was found'.format(current_tag))
+
+
+def get_artifact():
+    system = platform.system()
+    if system == 'Darwin':
+        return glob.glob('dist/mac/LBRY*.dmg')[0]
+    elif system == 'Linux':
+        return glob.glob('dist/LBRY*.deb')[0]
+    elif system == 'Windows':
+        return glob.glob('dist/LBRY*.exe')[0]
+    else:
+        raise Exception("I don't know about any artifact on {}".format(system))
+
+
+def get_system_label():
+    system = platform.system()
+    if system == 'Darwin':
+        return 'macOS'
+    else:
+        return system
+
+
+def get_asset(filename, label):
+    label = '-{}'.format(label)
+    base, ext = os.path.splitext(filename)
+    # TODO: probably want to clean this up
+    zipfilename = '{}{}.zip'.format(base, label)
+    with zipfile.ZipFile(zipfilename, 'w') as myzip:
+        myzip.write(filename)
+    return zipfilename
+
+
+def upload_asset(release, asset_to_upload, token):
+    basename = os.path.basename(asset_to_upload)
+    if is_asset_already_uploaded(release, basename):
+        return
+    upload_uri = uritemplate.expand(
+        release.upload_url, {'name': basename})
+    # using requests.post fails miserably with SSL EPIPE errors. I spent
+    # half a day trying to debug before deciding to switch to curl.
+    cmd = [
+        'curl', '-sS', '-X', 'POST', '-u', ':{}'.format(os.environ['GH_TOKEN']),
+        '--header', 'Content-Type:application/zip',
+        '--data-binary', '@{}'.format(asset_to_upload), upload_uri
+    ]
+    raw_output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
+    output = json.loads(raw_output)
+    if 'errors' in output:
+        raise Exception(output)
+    else:
+        log.info('Successfully uploaded to %s', output['browser_download_url'])
+
+
+def is_asset_already_uploaded(release, basename):
+    for asset in release.raw_data['assets']:
+        if asset['name'] == basename:
+            log.info('File %s has already been uploaded to %s', basename, release.tag_name)
+            return True
+    return False
+
+
+if __name__ == '__main__':
+    log = logging.getLogger('release-on-tag')
+    log_support.configure_console(level='DEBUG')
+    sys.exit(main())
+else:
+    log = logging.getLogger(__name__)