diff --git a/.bumpversion.cfg b/.bumpversion.cfg new file mode 100644 index 000000000..e06aa82be --- /dev/null +++ b/.bumpversion.cfg @@ -0,0 +1,17 @@ +[bumpversion] +current_version = 0.9.0rc7 +commit = True +tag = True +parse = (?P\d+)\.(?P\d+)\.(?P\d+)((?P[a-z]+)(?P\d+))? +serialize = + {major}.{minor}.{patch}{release}{candidate} + {major}.{minor}.{patch} + +[bumpversion:part:release] +optional_value = production +values = + rc + production + +[bumpversion:file:app/package.json] + diff --git a/app/package.json b/app/package.json index bcb9ec7f7..ffa895c22 100644 --- a/app/package.json +++ b/app/package.json @@ -1,6 +1,6 @@ { "name": "LBRY", - "version": "0.0.1", + "version": "0.9.0rc7", "main": "main.js", "description": "LBRY is a fully decentralized, open-source protocol facilitating the discovery, access, and (sometimes) purchase of data.", "author": { diff --git a/appveyor.yml b/appveyor.yml index 3c63834d3..255b66dda 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -1,6 +1,8 @@ # Test against the latest version of this Node.js version environment: nodejs_version: "6" + GH_TOKEN: + secure: LiI5jyuHUw6XbH4kC3gP1HX4P/v4rwD/gCNtaFhQu2AvJz1/1wALkp5ECnIxRySN skip_branch_with_pr: true @@ -37,21 +39,12 @@ build_script: - node_modules\.bin\build -p never # for debugging, see what was built - dir dist + - pip install PyGithub uritemplate + - python release_on_tag.py test: off + artifacts: - path: dist\*.exe name: LBRY - -deploy: - - provider: GitHub - release: $(APPVEYOR_REPO_TAG_NAME) - description: 'Release' - auth_token: - secure: LiI5jyuHUw6XbH4kC3gP1HX4P/v4rwD/gCNtaFhQu2AvJz1/1wALkp5ECnIxRySN - artifact: LBRY - draft: false - prerelease: true - on: - appveyor_repo_tag: true # deploy on tag push only diff --git a/build.sh b/build.sh index e6435b27a..2113aa673 100755 --- a/build.sh +++ b/build.sh @@ -83,6 +83,6 @@ if [ "$FULL_BUILD" == "true" ]; then # it to reliably work and it also seemed difficult to configure. Not proud of # this, but it seemed better to write my own. pip install PyGithub uritemplate - python release-on-tag.py + python release_on_tag.py deactivate fi diff --git a/lbry b/lbry index 1d30ae447..04b36e225 160000 --- a/lbry +++ b/lbry @@ -1 +1 @@ -Subproject commit 1d30ae447a897b1e1543a6a4cda5dbd242d2967d +Subproject commit 04b36e2252612047e82ade94b0ca6c8af538d45b diff --git a/lbry-web-ui b/lbry-web-ui index 55b246512..fe954ed17 160000 --- a/lbry-web-ui +++ b/lbry-web-ui @@ -1 +1 @@ -Subproject commit 55b246512585e9d3abc4d6bcbc08367df9c2b879 +Subproject commit fe954ed17d461a1760e3051984eb7d48c6238cfe diff --git a/lbryum b/lbryum index 8e9362567..8834b60c7 160000 --- a/lbryum +++ b/lbryum @@ -1 +1 @@ -Subproject commit 8e9362567d01732da06972c66f507df228f9817d +Subproject commit 8834b60c7cdd9c0c31b51be87e8bcf1b21dbdbed diff --git a/release-on-tag.py b/release-on-tag.py deleted file mode 100644 index 6d913a653..000000000 --- a/release-on-tag.py +++ /dev/null @@ -1,161 +0,0 @@ -import argparse -import glob -import json -import logging -import os -import platform -import re -import subprocess -import sys -import zipfile - -import github -import requests -import uritemplate - -from lbrynet.core import log_support - - -def main(args=None): - parser = argparse.ArgumentParser() - parser.add_argument('--file', help='artifact to publish') - parser.add_argument('--label', help='text to append to `file`') - parser.add_argument('--zip', action='store_true') - parser.add_argument( - '--force', action='store_true', - help='ignores whether the repo is currently tagged, publishes a draft release') - args = parser.parse_args(args) - - gh_token = os.environ['GH_TOKEN'] - auth = github.Github(gh_token) - current_repo = auth.get_repo(current_repo_name()) - - if args.file: - artifact = args.file - else: - artifact = get_artifact() - - current_tag = None - if not args.force: - try: - current_tag = subprocess.check_output( - ['git', 'describe', '--exact-match', 'HEAD']).strip() - except subprocess.CalledProcessError: - log.info('Stopping as we are not currently on a tag') - return - if not check_repo_has_tag(current_repo, current_tag): - log.info('Tag %s is not in repo %s', current_tag, current_repo) - # TODO: maybe this should be an error - return - - release = get_release(current_repo, current_tag, args.force) - asset_to_upload = get_asset(artifact, args.label, args.zip) - upload_asset(release, asset_to_upload, gh_token) - - -def current_repo_name(): - pattern = 'github.com[:/](.*)\.git' - remote = subprocess.check_output('git remote -v'.split()) - m = re.search(pattern, remote) - if not m: - raise Exception('Unable to parse repo name from remote: {}'.format(remote)) - return m.group(1) - - -def check_repo_has_tag(repo, target_tag): - tags = repo.get_tags().get_page(0) - for tag in tags: - if tag.name == target_tag: - return True - return False - - -def get_release(current_repo, current_tag=None, draft=False): - assert current_tag or draft, 'either current_tag or draft must be set' - need_new_release = False - if not draft and current_tag: - try: - release = current_repo.get_release(current_tag) - log.info('Using an existing release for %s', current_tag) - except github.UnknownObjectException: - need_new_release = True - if draft or need_new_release: - log.info('Creating a new release for %s:%s', current_repo, current_tag) - tag = current_tag or 'draft' - release_name = current_tag or 'draft' - msg = 'Release' # TODO: parse changelogs to get a better message - try: - # we have a race condition where its possible that between checking - # for the release and now, another build agent has come along and already - # created a release - release = current_repo.create_git_release(tag, release_name, msg, draft) - except github.GithubException: - log.info('Failed to create a release, maybe somebody already has', exc_info=True) - release = current_repo.get_release(current_tag) - return release - - -def get_artifact(): - system = platform.system() - if system == 'Darwin': - return glob.glob('dist/mac/LBRY*.dmg')[0] - elif system == 'Linux': - return glob.glob('dist/LBRY*.deb')[0] - else: - raise Exception("I don't know about any artifact on {}".format(system)) - - -def get_asset(filename, label=None, use_zip=False): - if label: - label = '-{}'.format(label) - else: - label = '' - base, ext = os.path.splitext(filename) - if use_zip: - # TODO: probably want to clean this up - zipfilename = '{}{}.zip'.format(base, label) - with zipfile.ZipFile(zipfilename, 'w') as myzip: - myzip.write(filename) - asset_to_uplaod = zipfilename - else: - asset_to_upload = '{}{}{}'.format(base, label, ext) - return asset_to_upload - - -def upload_asset(release, asset_to_upload, token): - basename = os.path.basename(asset_to_upload) - - if is_asset_already_uploaded(release, basename): - return - - upload_uri = uritemplate.expand( - release.upload_url, {'name': basename}) - # using requests.post fails miserably with SSL EPIPE errors. I spent - # half a day trying to debug before deciding to switch to curl. - cmd = [ - 'curl', '-sS', '-X', 'POST', '-u', ':{}'.format(os.environ['GH_TOKEN']), - '--header', 'Content-Type:application/zip', - '--data-binary', '@{}'.format(asset_to_upload), upload_uri - ] - raw_output = subprocess.check_output(cmd, stderr=subprocess.STDOUT) - output = json.loads(raw_output) - if 'errors' in output: - raise Exception(output) - else: - log.info('Successfully uploaded to %s', output['browser_download_url']) - - -def is_asset_already_uploaded(release, basename): - for asset in release.raw_data['assets']: - if asset['name'] == basename: - log.info('File %s has already been uploaded to %s', basename, release.tag_name) - return True - return False - - -if __name__ == '__main__': - log = logging.getLogger('release-on-tag') - log_support.configure_console(level='DEBUG') - sys.exit(main()) -else: - log = logging.getLogger(__name__) diff --git a/release.py b/release.py new file mode 100644 index 000000000..210d352e1 --- /dev/null +++ b/release.py @@ -0,0 +1,205 @@ +"""Trigger a release. + +This script is to be run locally (not on a build server). +""" +import argparse +import contextlib +import logging +import os +import re +import subprocess +import sys + +import git +import github +import requests + + +CHANGELOG_START_RE = re.compile(r'^\#\# \[Unreleased\]') +CHANGELOG_END_RE = re.compile(r'^\#\# \[.*\] - \d{4}-\d{2}-\d{2}') +# if we come across a section header between two release section headers +# then we probably have an improperly formatted changelog +CHANGELOG_ERROR_RE = re.compile(r'^\#\# ') + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("lbry_part", help="part of lbry version to bump") + parser.add_argument("--lbryum_part", help="part of lbryum version to bump") + parser.add_argument("--ui-part", help="part of the ui to bump") + parser.add_argument("--branch", default='master', help="branch to use for each repo") + parser.add_argument("--last-release") + parser.add_argument("--skip-sanity-checks", action="store_true") + args = parser.parse_args() + + branch = args.branch + + base = git.Repo(os.getcwd()) + + if not args.skip_sanity_checks: + run_sanity_checks(base, args) + + if args.last_release: + last_release = args.last_release + else: + response = requests.get('https://api.github.com/repos/lbryio/lbry-app/releases/latest') + data = response.json() + last_release = data['tag_name'] + + gh_token = os.environ['GH_TOKEN'] + auth = github.Github(gh_token) + github_repo = auth.get_repo('lbryio/lbry-app') + + names = ['lbry', 'lbry-web-ui', 'lbryum'] + repos = [Repo(name) for name in names] + + # in order to see if we've had any change in the submodule, we need to checkout + # our last release, see what commit we were on, and then compare that to + # current + base.git.checkout(last_release) + base.submodule_update() + for repo in repos: + repo.save_commit() + + base.git.checkout(branch) + base.submodule_update() + + changelogs = {} + + # ensure that we have changelog entries for each part + for repo in repos: + if not repo.has_changes(): + continue + entry = repo.get_changelog_entry().strip() + if not entry: + raise Exception('Changelog is missing for {}'.format(repo.name)) + changelogs[repo.name] = entry + part = get_part(args, repo.name) + if not part: + raise Exception('Cannot bump version for {}: no part specified'.format(repo.name)) + repo.bumpversion(part) + + release_msg = get_release_msg(changelogs, names) + + for name in names: + base.git.add(name) + subprocess.check_call(['bumpversion', args.lbry_part, '--allow-dirty']) + + current_tag = base.git.describe() + + github_repo.create_git_release(current_tag, current_tag, release_msg, draft=True) + auth.get_repo('lbryio/lbrynet-daemon').create_git_release( + current_tag, current_tag, changelogs['lbry'], draft=True) + for repo in repos: + repo.git.push(follow_tags=True) + base.git.push(follow_tags=True, recurse_submodules='check') + + +def get_release_msg(changelogs, names): + lines = [] + for name in names: + entry = changelogs.get(name) + if not entry: + continue + lines.append('## {}\n'.format(name)) + lines.append('{}\n'.format(entry)) + return '\n'.join(lines) + + +def run_sanity_checks(base, args): + branch = args.branch + if base.is_dirty(): + print 'Cowardly refusing to release a dirty repo' + sys.exit(1) + if base.active_branch.name != branch: + print 'Cowardly refusing to release when not on the {} branch'.format(branch) + sys.exit(1) + origin = base.remotes.origin + origin.fetch() + if base.commit() != origin.refs[branch].commit: + print 'Cowardly refusing to release when not synced with origin' + sys.exit(1) + check_bumpversion() + + +def check_bumpversion(): + + def requireNewVersion(): + print 'Install bumpversion: pip install -U git+https://github.com/lbryio/bumpversion.git' + sys.exit(1) + + try: + output = subprocess.check_output(['bumpversion', '-v'], stderr=subprocess.STDOUT) + output = output.strip() + if output != 'bumpversion 0.5.4-lbry': + requireNewVersion() + except subprocess.CalledProcessError: + requireNewVersion() + +def get_part(args, name): + if name == 'lbry-web-ui': + part = getattr(args, 'ui_part') + return part or args.lbry_part + else: + return getattr(args, name + '_part') + + +class Repo(object): + def __init__(self, name): + self.name = name + self.directory = os.path.join(os.getcwd(), name) + self.git_repo = git.Repo(self.directory) + self.saved_commit = None + + def has_changes(self): + return self.git_repo.commit() == self.saved_commit + + def save_commit(self): + self.saved_commit = self.git_repo.commit() + + def checkout(self, branch): + self.git_repo.git.checkout(branch) + + def get_changelog_entry(self): + filename = os.path.join(self.directory, 'CHANGELOG.md') + err = 'Had trouble parsing changelog {}: {}' + output = [] + start_found = False + with open(filename) as fp: + for line in fp: + if not start_found: + if CHANGELOG_START_RE.search(line): + start_found = True + continue + if CHANGELOG_END_RE.search(line): + return ''.join(output) + if CHANGELOG_ERROR_RE.search(line): + raise Exception(err.format(filename, 'unexpected section header found')) + output.append(line) + # if we get here there was no previous release section, which is a problem + if start_found: + # TODO: once the lbry-web-ui has a released entry, uncomment this error + # raise Exception(err.format(filename, 'Reached end of file')) + return ''.join(output) + else: + raise Exception(err.format(filename, 'Unreleased section not found')) + + def bumpversion(self, part): + with pushd(self.directory): + subprocess.check_call(['bumpversion', part]) + + @property + def git(self): + return self.git_repo.git + + +@contextlib.contextmanager +def pushd(new_dir): + previous_dir = os.getcwd() + os.chdir(new_dir) + yield + os.chdir(previous_dir) + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/release_on_tag.py b/release_on_tag.py new file mode 100644 index 000000000..6dd9d74ac --- /dev/null +++ b/release_on_tag.py @@ -0,0 +1,134 @@ +import argparse +import glob +import json +import logging +import os +import platform +import re +import subprocess +import sys +import zipfile + +import github +import requests +import uritemplate + +from lbrynet.core import log_support + + +def main(args=None): + current_tag = None + try: + current_tag = subprocess.check_output( + ['git', 'describe', '--exact-match', 'HEAD']).strip() + except subprocess.CalledProcessError: + log.info('Stopping as we are not currently on a tag') + return + + if 'GH_TOKEN' not in os.environ: + print 'Must set GH_TOKEN in order to publish assets to a release' + return + + gh_token = os.environ['GH_TOKEN'] + auth = github.Github(gh_token) + app_repo = auth.get_repo('lbryio/lbry-app') + # TODO: switch lbryio/lbrynet-daemon to lbryio/lbry + daemon_repo = auth.get_repo('lbryio/lbrynet-daemon') + + if not check_repo_has_tag(app_repo, current_tag): + log.info('Tag %s is not in repo %s', current_tag, app_repo) + # TODO: maybe this should be an error + return + + artifact = get_artifact() + release = get_release(app_repo, current_tag) + upload_asset(release, artifact, gh_token) + + release = get_release(daemon_repo, current_tag) + artifact = os.path.join('app', 'dist', 'lbrynet-daemon') + if platform.system() == 'Windows': + artifact += '.exe' + asset_to_upload = get_asset(artifact, get_system_label()) + upload_asset(release, asset_to_upload, gh_token) + + +def check_repo_has_tag(repo, target_tag): + tags = repo.get_tags().get_page(0) + for tag in tags: + if tag.name == target_tag: + return True + return False + + +def get_release(current_repo, current_tag): + for release in current_repo.get_releases(): + if release.tag_name == current_tag: + return release + raise Exception('No release for {} was found'.format(current_tag)) + + +def get_artifact(): + system = platform.system() + if system == 'Darwin': + return glob.glob('dist/mac/LBRY*.dmg')[0] + elif system == 'Linux': + return glob.glob('dist/LBRY*.deb')[0] + elif system == 'Windows': + return glob.glob('dist/LBRY*.exe')[0] + else: + raise Exception("I don't know about any artifact on {}".format(system)) + + +def get_system_label(): + system = platform.system() + if system == 'Darwin': + return 'macOS' + else: + return system + + +def get_asset(filename, label): + label = '-{}'.format(label) + base, ext = os.path.splitext(filename) + # TODO: probably want to clean this up + zipfilename = '{}{}.zip'.format(base, label) + with zipfile.ZipFile(zipfilename, 'w') as myzip: + myzip.write(filename) + return zipfilename + + +def upload_asset(release, asset_to_upload, token): + basename = os.path.basename(asset_to_upload) + if is_asset_already_uploaded(release, basename): + return + upload_uri = uritemplate.expand( + release.upload_url, {'name': basename}) + # using requests.post fails miserably with SSL EPIPE errors. I spent + # half a day trying to debug before deciding to switch to curl. + cmd = [ + 'curl', '-sS', '-X', 'POST', '-u', ':{}'.format(os.environ['GH_TOKEN']), + '--header', 'Content-Type:application/zip', + '--data-binary', '@{}'.format(asset_to_upload), upload_uri + ] + raw_output = subprocess.check_output(cmd, stderr=subprocess.STDOUT) + output = json.loads(raw_output) + if 'errors' in output: + raise Exception(output) + else: + log.info('Successfully uploaded to %s', output['browser_download_url']) + + +def is_asset_already_uploaded(release, basename): + for asset in release.raw_data['assets']: + if asset['name'] == basename: + log.info('File %s has already been uploaded to %s', basename, release.tag_name) + return True + return False + + +if __name__ == '__main__': + log = logging.getLogger('release-on-tag') + log_support.configure_console(level='DEBUG') + sys.exit(main()) +else: + log = logging.getLogger(__name__)