Add release script #5

Merged
jobevers merged 12 commits from test-electron-release into master 2017-02-21 20:28:00 +01:00
10 changed files with 366 additions and 178 deletions

17
.bumpversion.cfg Normal file
View file

@ -0,0 +1,17 @@
[bumpversion]
current_version = 0.9.0rc7
commit = True
tag = True
parse = (?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)((?P<release>[a-z]+)(?P<candidate>\d+))?
serialize =
{major}.{minor}.{patch}{release}{candidate}
{major}.{minor}.{patch}
[bumpversion:part:release]
optional_value = production
values =
rc
production
[bumpversion:file:app/package.json]

View file

@ -1,6 +1,6 @@
{ {
"name": "LBRY", "name": "LBRY",
"version": "0.0.1", "version": "0.9.0rc7",
"main": "main.js", "main": "main.js",
"description": "LBRY is a fully decentralized, open-source protocol facilitating the discovery, access, and (sometimes) purchase of data.", "description": "LBRY is a fully decentralized, open-source protocol facilitating the discovery, access, and (sometimes) purchase of data.",
"author": { "author": {

View file

@ -1,6 +1,8 @@
# Test against the latest version of this Node.js version # Test against the latest version of this Node.js version
environment: environment:
nodejs_version: "6" nodejs_version: "6"
GH_TOKEN:
secure: LiI5jyuHUw6XbH4kC3gP1HX4P/v4rwD/gCNtaFhQu2AvJz1/1wALkp5ECnIxRySN
skip_branch_with_pr: true skip_branch_with_pr: true
@ -37,21 +39,12 @@ build_script:
- node_modules\.bin\build -p never - node_modules\.bin\build -p never
# for debugging, see what was built # for debugging, see what was built
- dir dist - dir dist
- pip install PyGithub uritemplate
- python release_on_tag.py
test: off test: off
artifacts: artifacts:
- path: dist\*.exe - path: dist\*.exe
name: LBRY name: LBRY
deploy:
- provider: GitHub
release: $(APPVEYOR_REPO_TAG_NAME)
description: 'Release'
auth_token:
secure: LiI5jyuHUw6XbH4kC3gP1HX4P/v4rwD/gCNtaFhQu2AvJz1/1wALkp5ECnIxRySN
artifact: LBRY
draft: false
prerelease: true
on:
appveyor_repo_tag: true # deploy on tag push only

View file

@ -83,6 +83,6 @@ if [ "$FULL_BUILD" == "true" ]; then
# it to reliably work and it also seemed difficult to configure. Not proud of # it to reliably work and it also seemed difficult to configure. Not proud of
# this, but it seemed better to write my own. # this, but it seemed better to write my own.
pip install PyGithub uritemplate pip install PyGithub uritemplate
python release-on-tag.py python release_on_tag.py
deactivate deactivate
fi fi

2
lbry

@ -1 +1 @@
Subproject commit 1d30ae447a897b1e1543a6a4cda5dbd242d2967d Subproject commit 04b36e2252612047e82ade94b0ca6c8af538d45b

@ -1 +1 @@
Subproject commit 55b246512585e9d3abc4d6bcbc08367df9c2b879 Subproject commit fe954ed17d461a1760e3051984eb7d48c6238cfe

2
lbryum

@ -1 +1 @@
Subproject commit 8e9362567d01732da06972c66f507df228f9817d Subproject commit 8834b60c7cdd9c0c31b51be87e8bcf1b21dbdbed

View file

@ -1,161 +0,0 @@
import argparse
import glob
import json
import logging
import os
import platform
import re
import subprocess
import sys
import zipfile
import github
import requests
import uritemplate
from lbrynet.core import log_support
def main(args=None):
parser = argparse.ArgumentParser()
parser.add_argument('--file', help='artifact to publish')
parser.add_argument('--label', help='text to append to `file`')
parser.add_argument('--zip', action='store_true')
parser.add_argument(
'--force', action='store_true',
help='ignores whether the repo is currently tagged, publishes a draft release')
args = parser.parse_args(args)
gh_token = os.environ['GH_TOKEN']
auth = github.Github(gh_token)
current_repo = auth.get_repo(current_repo_name())
if args.file:
artifact = args.file
else:
artifact = get_artifact()
current_tag = None
if not args.force:
try:
current_tag = subprocess.check_output(
['git', 'describe', '--exact-match', 'HEAD']).strip()
except subprocess.CalledProcessError:
log.info('Stopping as we are not currently on a tag')
return
if not check_repo_has_tag(current_repo, current_tag):
log.info('Tag %s is not in repo %s', current_tag, current_repo)
# TODO: maybe this should be an error
return
release = get_release(current_repo, current_tag, args.force)
asset_to_upload = get_asset(artifact, args.label, args.zip)
upload_asset(release, asset_to_upload, gh_token)
def current_repo_name():
pattern = 'github.com[:/](.*)\.git'
remote = subprocess.check_output('git remote -v'.split())
m = re.search(pattern, remote)
if not m:
raise Exception('Unable to parse repo name from remote: {}'.format(remote))
return m.group(1)
def check_repo_has_tag(repo, target_tag):
tags = repo.get_tags().get_page(0)
for tag in tags:
if tag.name == target_tag:
return True
return False
def get_release(current_repo, current_tag=None, draft=False):
assert current_tag or draft, 'either current_tag or draft must be set'
need_new_release = False
if not draft and current_tag:
try:
release = current_repo.get_release(current_tag)
log.info('Using an existing release for %s', current_tag)
except github.UnknownObjectException:
need_new_release = True
if draft or need_new_release:
log.info('Creating a new release for %s:%s', current_repo, current_tag)
tag = current_tag or 'draft'
release_name = current_tag or 'draft'
msg = 'Release' # TODO: parse changelogs to get a better message
try:
# we have a race condition where its possible that between checking
# for the release and now, another build agent has come along and already
# created a release
release = current_repo.create_git_release(tag, release_name, msg, draft)
except github.GithubException:
log.info('Failed to create a release, maybe somebody already has', exc_info=True)
release = current_repo.get_release(current_tag)
return release
def get_artifact():
system = platform.system()
if system == 'Darwin':
return glob.glob('dist/mac/LBRY*.dmg')[0]
elif system == 'Linux':
return glob.glob('dist/LBRY*.deb')[0]
else:
raise Exception("I don't know about any artifact on {}".format(system))
def get_asset(filename, label=None, use_zip=False):
if label:
label = '-{}'.format(label)
else:
label = ''
base, ext = os.path.splitext(filename)
if use_zip:
# TODO: probably want to clean this up
zipfilename = '{}{}.zip'.format(base, label)
with zipfile.ZipFile(zipfilename, 'w') as myzip:
myzip.write(filename)
asset_to_uplaod = zipfilename
else:
asset_to_upload = '{}{}{}'.format(base, label, ext)
return asset_to_upload
def upload_asset(release, asset_to_upload, token):
basename = os.path.basename(asset_to_upload)
if is_asset_already_uploaded(release, basename):
return
upload_uri = uritemplate.expand(
release.upload_url, {'name': basename})
# using requests.post fails miserably with SSL EPIPE errors. I spent
# half a day trying to debug before deciding to switch to curl.
cmd = [
'curl', '-sS', '-X', 'POST', '-u', ':{}'.format(os.environ['GH_TOKEN']),
'--header', 'Content-Type:application/zip',
'--data-binary', '@{}'.format(asset_to_upload), upload_uri
]
raw_output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
output = json.loads(raw_output)
if 'errors' in output:
raise Exception(output)
else:
log.info('Successfully uploaded to %s', output['browser_download_url'])
def is_asset_already_uploaded(release, basename):
for asset in release.raw_data['assets']:
if asset['name'] == basename:
log.info('File %s has already been uploaded to %s', basename, release.tag_name)
return True
return False
if __name__ == '__main__':
log = logging.getLogger('release-on-tag')
log_support.configure_console(level='DEBUG')
sys.exit(main())
else:
log = logging.getLogger(__name__)

205
release.py Normal file
View file

@ -0,0 +1,205 @@
"""Trigger a release.
This script is to be run locally (not on a build server).
"""
import argparse
import contextlib
import logging
import os
import re
import subprocess
import sys
import git
import github
import requests
CHANGELOG_START_RE = re.compile(r'^\#\# \[Unreleased\]')
CHANGELOG_END_RE = re.compile(r'^\#\# \[.*\] - \d{4}-\d{2}-\d{2}')
# if we come across a section header between two release section headers
# then we probably have an improperly formatted changelog
CHANGELOG_ERROR_RE = re.compile(r'^\#\# ')
def main():
parser = argparse.ArgumentParser()
parser.add_argument("lbry_part", help="part of lbry version to bump")
parser.add_argument("--lbryum_part", help="part of lbryum version to bump")
parser.add_argument("--ui-part", help="part of the ui to bump")
parser.add_argument("--branch", default='master', help="branch to use for each repo")
parser.add_argument("--last-release")
parser.add_argument("--skip-sanity-checks", action="store_true")
args = parser.parse_args()
branch = args.branch
base = git.Repo(os.getcwd())
if not args.skip_sanity_checks:
run_sanity_checks(base, args)
if args.last_release:
last_release = args.last_release
else:
response = requests.get('https://api.github.com/repos/lbryio/lbry-app/releases/latest')
data = response.json()
last_release = data['tag_name']
gh_token = os.environ['GH_TOKEN']
auth = github.Github(gh_token)
github_repo = auth.get_repo('lbryio/lbry-app')
names = ['lbry', 'lbry-web-ui', 'lbryum']
repos = [Repo(name) for name in names]
# in order to see if we've had any change in the submodule, we need to checkout
# our last release, see what commit we were on, and then compare that to
# current
base.git.checkout(last_release)
base.submodule_update()
for repo in repos:
repo.save_commit()
base.git.checkout(branch)
base.submodule_update()
changelogs = {}
# ensure that we have changelog entries for each part
for repo in repos:
if not repo.has_changes():
continue
entry = repo.get_changelog_entry().strip()
if not entry:
raise Exception('Changelog is missing for {}'.format(repo.name))
changelogs[repo.name] = entry
part = get_part(args, repo.name)
if not part:
raise Exception('Cannot bump version for {}: no part specified'.format(repo.name))
repo.bumpversion(part)
release_msg = get_release_msg(changelogs, names)
for name in names:
base.git.add(name)
subprocess.check_call(['bumpversion', args.lbry_part, '--allow-dirty'])
current_tag = base.git.describe()
github_repo.create_git_release(current_tag, current_tag, release_msg, draft=True)
auth.get_repo('lbryio/lbrynet-daemon').create_git_release(
current_tag, current_tag, changelogs['lbry'], draft=True)
for repo in repos:
repo.git.push(follow_tags=True)
base.git.push(follow_tags=True, recurse_submodules='check')
def get_release_msg(changelogs, names):
lines = []
for name in names:
entry = changelogs.get(name)
if not entry:
continue
lines.append('## {}\n'.format(name))
lines.append('{}\n'.format(entry))
return '\n'.join(lines)
def run_sanity_checks(base, args):
branch = args.branch
if base.is_dirty():
print 'Cowardly refusing to release a dirty repo'
sys.exit(1)
if base.active_branch.name != branch:
print 'Cowardly refusing to release when not on the {} branch'.format(branch)
sys.exit(1)
origin = base.remotes.origin
origin.fetch()
if base.commit() != origin.refs[branch].commit:
print 'Cowardly refusing to release when not synced with origin'
sys.exit(1)
check_bumpversion()
def check_bumpversion():
def requireNewVersion():
print 'Install bumpversion: pip install -U git+https://github.com/lbryio/bumpversion.git'
sys.exit(1)
try:
output = subprocess.check_output(['bumpversion', '-v'], stderr=subprocess.STDOUT)
output = output.strip()
if output != 'bumpversion 0.5.4-lbry':
requireNewVersion()
except subprocess.CalledProcessError:
requireNewVersion()
def get_part(args, name):
if name == 'lbry-web-ui':
part = getattr(args, 'ui_part')
return part or args.lbry_part
else:
return getattr(args, name + '_part')
class Repo(object):
def __init__(self, name):
self.name = name
self.directory = os.path.join(os.getcwd(), name)
self.git_repo = git.Repo(self.directory)
self.saved_commit = None
def has_changes(self):
return self.git_repo.commit() == self.saved_commit
def save_commit(self):
self.saved_commit = self.git_repo.commit()
def checkout(self, branch):
self.git_repo.git.checkout(branch)
def get_changelog_entry(self):
filename = os.path.join(self.directory, 'CHANGELOG.md')
err = 'Had trouble parsing changelog {}: {}'
output = []
start_found = False
with open(filename) as fp:
for line in fp:
if not start_found:
if CHANGELOG_START_RE.search(line):
start_found = True
continue
if CHANGELOG_END_RE.search(line):
return ''.join(output)
if CHANGELOG_ERROR_RE.search(line):
raise Exception(err.format(filename, 'unexpected section header found'))
output.append(line)
# if we get here there was no previous release section, which is a problem
if start_found:
# TODO: once the lbry-web-ui has a released entry, uncomment this error
# raise Exception(err.format(filename, 'Reached end of file'))
return ''.join(output)
else:
raise Exception(err.format(filename, 'Unreleased section not found'))
def bumpversion(self, part):
with pushd(self.directory):
subprocess.check_call(['bumpversion', part])
@property
def git(self):
return self.git_repo.git
@contextlib.contextmanager
def pushd(new_dir):
previous_dir = os.getcwd()
os.chdir(new_dir)
yield
os.chdir(previous_dir)
if __name__ == '__main__':
sys.exit(main())

134
release_on_tag.py Normal file
View file

@ -0,0 +1,134 @@
import argparse
import glob
import json
import logging
import os
import platform
import re
import subprocess
import sys
import zipfile
import github
import requests
import uritemplate
from lbrynet.core import log_support
def main(args=None):
current_tag = None
try:
current_tag = subprocess.check_output(
['git', 'describe', '--exact-match', 'HEAD']).strip()
except subprocess.CalledProcessError:
log.info('Stopping as we are not currently on a tag')
return
if 'GH_TOKEN' not in os.environ:
print 'Must set GH_TOKEN in order to publish assets to a release'
return
gh_token = os.environ['GH_TOKEN']
auth = github.Github(gh_token)
app_repo = auth.get_repo('lbryio/lbry-app')
# TODO: switch lbryio/lbrynet-daemon to lbryio/lbry
daemon_repo = auth.get_repo('lbryio/lbrynet-daemon')
if not check_repo_has_tag(app_repo, current_tag):
log.info('Tag %s is not in repo %s', current_tag, app_repo)
# TODO: maybe this should be an error
return
artifact = get_artifact()
release = get_release(app_repo, current_tag)
upload_asset(release, artifact, gh_token)
release = get_release(daemon_repo, current_tag)
artifact = os.path.join('app', 'dist', 'lbrynet-daemon')
if platform.system() == 'Windows':
artifact += '.exe'
asset_to_upload = get_asset(artifact, get_system_label())
upload_asset(release, asset_to_upload, gh_token)
def check_repo_has_tag(repo, target_tag):
tags = repo.get_tags().get_page(0)
for tag in tags:
if tag.name == target_tag:
return True
return False
def get_release(current_repo, current_tag):
for release in current_repo.get_releases():
if release.tag_name == current_tag:
return release
raise Exception('No release for {} was found'.format(current_tag))
def get_artifact():
system = platform.system()
if system == 'Darwin':
return glob.glob('dist/mac/LBRY*.dmg')[0]
elif system == 'Linux':
return glob.glob('dist/LBRY*.deb')[0]
elif system == 'Windows':
return glob.glob('dist/LBRY*.exe')[0]
else:
raise Exception("I don't know about any artifact on {}".format(system))
def get_system_label():
system = platform.system()
if system == 'Darwin':
return 'macOS'
else:
return system
def get_asset(filename, label):
label = '-{}'.format(label)
base, ext = os.path.splitext(filename)
# TODO: probably want to clean this up
zipfilename = '{}{}.zip'.format(base, label)
with zipfile.ZipFile(zipfilename, 'w') as myzip:
myzip.write(filename)
return zipfilename
def upload_asset(release, asset_to_upload, token):
basename = os.path.basename(asset_to_upload)
if is_asset_already_uploaded(release, basename):
return
upload_uri = uritemplate.expand(
release.upload_url, {'name': basename})
# using requests.post fails miserably with SSL EPIPE errors. I spent
# half a day trying to debug before deciding to switch to curl.
cmd = [
'curl', '-sS', '-X', 'POST', '-u', ':{}'.format(os.environ['GH_TOKEN']),
'--header', 'Content-Type:application/zip',
'--data-binary', '@{}'.format(asset_to_upload), upload_uri
]
raw_output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
output = json.loads(raw_output)
if 'errors' in output:
raise Exception(output)
else:
log.info('Successfully uploaded to %s', output['browser_download_url'])
def is_asset_already_uploaded(release, basename):
for asset in release.raw_data['assets']:
if asset['name'] == basename:
log.info('File %s has already been uploaded to %s', basename, release.tag_name)
return True
return False
if __name__ == '__main__':
log = logging.getLogger('release-on-tag')
log_support.configure_console(level='DEBUG')
sys.exit(main())
else:
log = logging.getLogger(__name__)