rewrite release.py script

This commit is contained in:
Alex Grintsvayg 2017-04-12 18:46:32 -04:00 committed by Alex Grintsvayg
parent 2e9e0f129b
commit 38fdf56583
2 changed files with 208 additions and 313 deletions

View file

@ -1,8 +1,5 @@
import argparse
import datetime
import re
import sys
CHANGELOG_START_RE = re.compile(r'^\#\# \[Unreleased\]')
CHANGELOG_END_RE = re.compile(r'^\#\# \[.*\] - \d{4}-\d{2}-\d{2}')
@ -14,118 +11,110 @@ EMPTY_RE = re.compile(r'^\w*\*\w*$')
ENTRY_RE = re.compile(r'\* (.*)')
VALID_SECTIONS = ['Added', 'Changed', 'Deprecated', 'Removed', 'Fixed', 'Security']
# allocate some entries to cut-down on merge conflicts
TEMPLATE = """### Added
*
*
*
### Changed
*
*
*
### Fixed
*
*
*
"""
def main():
print "i am broken"
return 1
parser = argparse.ArgumentParser()
parser.add_argument('changelog')
parser.add_argument('version')
args = parser.parse_args()
bump(changelog, version)
class Changelog(object):
def __init__(self, path):
self.path = path
self.start = []
self.unreleased = []
self.rest = []
self._parse()
def _parse(self):
with open(self.path) as fp:
lines = fp.readlines()
def bump(changelog, version):
with open(changelog) as fp:
lines = fp.readlines()
unreleased_start_found = False
unreleased_end_found = False
start = []
unreleased = []
rest = []
unreleased_start_found = False
unreleased_end_found = False
for line in lines:
if not unreleased_start_found:
start.append(line)
if CHANGELOG_START_RE.search(line):
unreleased_start_found = True
continue
if unreleased_end_found:
rest.append(line)
continue
if CHANGELOG_END_RE.search(line):
rest.append(line)
unreleased_end_found = True
continue
if CHANGELOG_ERROR_RE.search(line):
raise Exception(
'Failed to parse {}: {}'.format(changelog, 'unexpected section header found'))
unreleased.append(line)
for line in lines:
if not unreleased_start_found:
self.start.append(line)
if CHANGELOG_START_RE.search(line):
unreleased_start_found = True
continue
if unreleased_end_found:
self.rest.append(line)
continue
if CHANGELOG_END_RE.search(line):
self.rest.append(line)
unreleased_end_found = True
continue
if CHANGELOG_ERROR_RE.search(line):
raise Exception(
'Failed to parse {}: {}'.format(self.path, 'unexpected section header found'))
self.unreleased.append(line)
today = datetime.datetime.today()
header = '## [{}] - {}\n'.format(version, today.strftime('%Y-%m-%d'))
released = normalize(unreleased)
if not released:
# If we don't have anything in the Unreleased section, then leave the
# changelog as it is and return None
return
self.unreleased = self._normalize_section(self.unreleased)
changelog_data = (
''.join(start) +
TEMPLATE +
header +
'\n'.join(released) + '\n\n'
+ ''.join(rest)
)
with open(changelog, 'w') as fp:
fp.write(changelog_data)
return '\n'.join(released) + '\n\n'
@staticmethod
def _normalize_section(lines):
"""Parse a changelog entry and output a normalized form"""
sections = {}
current_section_name = None
current_section_contents = []
for line in lines:
line = line.strip()
if not line or EMPTY_RE.match(line):
continue
match = SECTION_RE.match(line)
if match:
if current_section_contents:
sections[current_section_name] = current_section_contents
current_section_contents = []
current_section_name = match.group(1)
if current_section_name not in VALID_SECTIONS:
raise ValueError("Section '{}' is not valid".format(current_section_name))
continue
match = ENTRY_RE.match(line)
if match:
current_section_contents.append(match.group(1))
continue
raise Exception('Something is wrong with line: {}'.format(line))
if current_section_contents:
sections[current_section_name] = current_section_contents
output = []
for section in VALID_SECTIONS:
if section not in sections:
continue
output.append('### {}'.format(section))
for entry in sections[section]:
output.append(' * {}'.format(entry))
return output
def normalize(lines):
"""Parse a changelog entry and output a normalized form"""
sections = {}
current_section_name = None
current_section_contents = []
for line in lines:
line = line.strip()
if not line or EMPTY_RE.match(line):
continue
match = SECTION_RE.match(line)
if match:
if current_section_contents:
sections[current_section_name] = current_section_contents
current_section_contents = []
current_section_name = match.group(1)
if current_section_name not in VALID_SECTIONS:
raise ValueError("Section '{}' is not valid".format(current_section_name))
continue
match = ENTRY_RE.match(line)
if match:
current_section_contents.append(match.group(1))
continue
raise Exception('Something is wrong with line: {}'.format(line))
if current_section_contents:
sections[current_section_name] = current_section_contents
def get_unreleased(self):
return '\n'.join(self.unreleased) if self.unreleased else None
output = []
for section in VALID_SECTIONS:
if section not in sections:
continue
output.append('### {}'.format(section))
for entry in sections[section]:
output.append(' * {}'.format(entry))
return output
def bump(self, version):
if not self.unreleased:
return
today = datetime.datetime.today()
header = '## [{}] - {}\n'.format(version, today.strftime('%Y-%m-%d'))
if __name__ == '__main__':
sys.exit(main())
changelog_data = (
''.join(self.start) +
TEMPLATE +
header +
'\n'.join(self.unreleased) + '\n\n'
+ ''.join(self.rest)
)
with open(self.path, 'w') as fp:
fp.write(changelog_data)

View file

@ -1,13 +1,11 @@
"""Trigger a release.
"""Bump version and create Github release
This script is to be run locally (not on a build server).
This script should be run locally, not on a build server.
"""
import argparse
import contextlib
import logging
import os
import re
import string
import subprocess
import sys
@ -16,122 +14,131 @@ import github
import changelog
# TODO: ask bumpversion for these
LBRY_PARTS = ('major', 'minor', 'patch', 'release', 'candidate')
LBRYUM_PARTS = ('major', 'minor', 'patch')
ROOT = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
def main():
parser = argparse.ArgumentParser()
parser.add_argument(
"lbry_part", help="part of lbry version to bump",
choices=LBRY_PARTS
)
parser.add_argument(
"--skip-lbryum", help="skip bumping lbryum, even if there are changes",
action="store_true",
)
parser.add_argument(
"--lbryum-part", help="part of lbryum version to bump",
choices=LBRYUM_PARTS
)
parser.add_argument(
"--last-release",
help=("manually set the last release version. The default is to query and parse the"
" value from the release page.")
)
parser.add_argument(
"--skip-sanity-checks", action="store_true")
parser.add_argument(
"--require-changelog", action="store_true",
help=("Set this flag to raise an exception if a submodules has changes without a"
" corresponding changelog entry. The default is to log a warning")
)
parser.add_argument(
"--skip-push", action="store_true",
help="Set to not push changes to remote repo"
)
bumpversion_parts = get_bumpversion_parts()
parser = argparse.ArgumentParser()
parser.add_argument("part", choices=bumpversion_parts, help="part of version to bump")
parser.add_argument("--skip-sanity-checks", action="store_true")
parser.add_argument("--skip-push", action="store_true")
parser.add_argument("--dry-run", action="store_true")
parser.add_argument("--confirm", action="store_true")
args = parser.parse_args()
base = git.Repo(os.getcwd())
if args.dry_run:
print "DRY RUN. Nothing will be committed/pushed."
repo = Repo('lbry-app', args.part, ROOT)
branch = 'master'
print 'Current version: {}'.format(repo.current_version)
print 'New version: {}'.format(repo.new_version)
if not args.confirm and not confirm():
print "Aborting"
return 1
if not args.skip_sanity_checks:
run_sanity_checks(base, branch)
run_sanity_checks(repo, branch)
repo.assert_new_tag_is_absent()
base_repo = Repo('lbry-app', args.lbry_part, os.getcwd())
base_repo.assert_new_tag_is_absent()
is_rc = re.search('\drc\d+$', repo.new_version) is not None
# only have a release message for real releases, not for RCs
release_msg = '' if is_rc else repo.get_unreleased_changelog()
last_release = args.last_release or base_repo.get_last_tag()
logging.info('Last release: %s', last_release)
if args.dry_run:
print "rc: " + ("yes" if is_rc else "no")
print "release message: \n" + (release_msg or " NO MESSAGE FOR RCs")
return
gh_token = get_gh_token()
auth = github.Github(gh_token)
github_repo = auth.get_repo('lbryio/lbry-app')
names = ['lbryum', 'lbry']
repos = {name: Repo(name, get_part(args, name)) for name in names}
if not is_rc:
repo.bump_changelog()
repo.bumpversion()
changelogs = {}
new_tag = repo.get_new_tag()
github_repo.create_git_release(new_tag, new_tag, release_msg, draft=True, prerelease=is_rc)
for repo in repos.values():
logging.info('Processing repo: %s', repo.name)
repo.checkout(branch)
last_submodule_hash = base_repo.get_submodule_hash(last_release, repo.name)
if repo.has_changes_from_revision(last_submodule_hash):
if repo.name == 'lbryum':
if args.skip_lbryum:
continue
if not repo.part:
repo.part = get_lbryum_part()
entry = repo.get_changelog_entry()
if entry:
changelogs[repo.name] = entry.strip()
repo.add_changelog()
else:
msg = 'Changelog entry is missing for {}'.format(repo.name)
if args.require_changelog:
raise Exception(msg)
else:
logging.warning(msg)
else:
logging.warning('Submodule %s has no changes.', repo.name)
if repo.name == 'lbryum':
# The other repos have their version track each other so need to bump
# them even if there aren't any changes, but lbryum should only be
# bumped if it has changes
continue
# bumpversion will fail if there is already the tag we want in the repo
repo.assert_new_tag_is_absent()
repo.bumpversion()
release_msg = get_release_msg(changelogs, names)
for name in names:
base.git.add(name)
base_repo.bumpversion()
current_tag = base.git.describe()
is_rc = re.match('\drc\d+$', current_tag) is not None
github_repo.create_git_release(current_tag, current_tag, release_msg, draft=True,
prerelease=is_rc)
no_change_msg = ('No change since the last release. This release is simply a placeholder'
' so that LBRY and LBRY App track the same version')
lbrynet_daemon_release_msg = changelogs.get('lbry', no_change_msg)
auth.get_repo('lbryio/lbry').create_git_release(
current_tag, current_tag, lbrynet_daemon_release_msg, draft=True)
if not args.skip_push:
for repo in repos.values():
repo.git.push(follow_tags=True)
base.git.push(follow_tags=True, recurse_submodules='check')
if args.skip_push:
print (
'Skipping push; you will have to reset and delete tags if '
'you want to run this script again.'
)
else:
logging.info('Skipping push; you will have to reset and delete tags if '
'you want to run this script again. Take a look at reset.sh; '
'it probably does what you want.')
repo.git_repo.git.push(follow_tags=True, recurse_submodules='check')
class Repo(object):
def __init__(self, name, part, directory):
self.name = name
self.part = part
if not self.part:
raise Exception('Part required')
self.directory = directory
self.git_repo = git.Repo(self.directory)
self._bumped = False
self.current_version = self._get_current_version()
self.new_version = self._get_new_version()
self._changelog = changelog.Changelog(os.path.join(self.directory, 'CHANGELOG.md'))
def get_new_tag(self):
return 'v' + self.new_version
def get_unreleased_changelog(self):
return self._changelog.get_unreleased()
def bump_changelog(self):
self._changelog.bump(self.new_version)
with pushd(self.directory):
self.git_repo.git.add(os.path.basename(self._changelog.path))
def _get_current_version(self):
with pushd(self.directory):
output = subprocess.check_output(
['bumpversion', '--dry-run', '--list', '--allow-dirty', self.part])
return re.search('^current_version=(.*)$', output, re.M).group(1)
def _get_new_version(self):
with pushd(self.directory):
output = subprocess.check_output(
['bumpversion', '--dry-run', '--list', '--allow-dirty', self.part])
return re.search('^new_version=(.*)$', output, re.M).group(1)
def bumpversion(self):
if self._bumped:
raise Exception('Cowardly refusing to bump a repo twice')
with pushd(self.directory):
subprocess.check_call(['bumpversion', '--allow-dirty', self.part])
self._bumped = True
def assert_new_tag_is_absent(self):
new_tag = self.get_new_tag()
tags = self.git_repo.git.tag()
if new_tag in tags.split('\n'):
raise Exception('Tag {} is already present in repo {}.'.format(new_tag, self.name))
def is_behind(self, branch):
self.git_repo.remotes.origin.fetch()
rev_list = '{branch}...origin/{branch}'.format(branch=branch)
commits_behind = self.git_repo.git.rev_list(rev_list, right_only=True, count=True)
commits_behind = int(commits_behind)
return commits_behind > 0
def get_bumpversion_parts():
with pushd(ROOT):
output = subprocess.check_output([
'bumpversion', '--dry-run', '--list', '--allow-dirty', 'fake-part',
])
parse_line = re.search('^parse=(.*)$', output, re.M).group(1)
return tuple(re.findall('<([^>]+)>', parse_line))
def get_gh_token():
@ -148,131 +155,36 @@ in the future"""
return raw_input('token: ').strip()
def get_lbryum_part():
print """The lbryum repo has changes but you didn't specify how to bump the
version. Please enter one of: {}""".format(', '.join(LBRYUM_PARTS))
while True:
part = raw_input('part: ').strip()
if part in LBRYUM_PARTS:
return part
print 'Invalid part. Enter one of: {}'.format(', '.join(LBRYUM_PARTS))
def confirm():
return raw_input('Is this what you want? [y/N] ').strip().lower() == 'y'
def get_release_msg(changelogs, names):
lines = []
for name in names:
entry = changelogs.get(name)
if not entry:
continue
lines.append('## {}\n'.format(name))
lines.append('{}\n'.format(entry))
return '\n'.join(lines)
def run_sanity_checks(base, branch):
if base.is_dirty():
def run_sanity_checks(repo, branch):
if repo.git_repo.is_dirty():
print 'Cowardly refusing to release a dirty repo'
sys.exit(1)
if base.active_branch.name != branch:
if repo.git_repo.active_branch.name != branch:
print 'Cowardly refusing to release when not on the {} branch'.format(branch)
sys.exit(1)
if is_behind(base, branch):
if repo.is_behind(branch):
print 'Cowardly refusing to release when behind origin'
sys.exit(1)
check_bumpversion()
def is_behind(base, branch):
base.remotes.origin.fetch()
rev_list = '{branch}...origin/{branch}'.format(branch=branch)
commits_behind = base.git.rev_list(rev_list, right_only=True, count=True)
commits_behind = int(commits_behind)
return commits_behind > 0
def check_bumpversion():
def require_new_version():
print 'Install bumpversion: pip install -U git+https://github.com/lbryio/bumpversion.git'
if not is_custom_bumpversion_version():
print (
'Install LBRY\'s fork of bumpversion: '
'pip install -U git+https://github.com/lbryio/bumpversion.git'
)
sys.exit(1)
def is_custom_bumpversion_version():
try:
output = subprocess.check_output(['bumpversion', '-v'], stderr=subprocess.STDOUT)
output = output.strip()
if output != 'bumpversion 0.5.4-lbry':
require_new_version()
except (subprocess.CalledProcessError, OSError) as err:
require_new_version()
def get_part(args, name):
return getattr(args, name + '_part') or args.lbry_part
class Repo(object):
def __init__(self, name, part, directory=None):
self.name = name
self.part = part
self.directory = directory or os.path.join(os.getcwd(), name)
self.git_repo = git.Repo(self.directory)
self.saved_commit = None
self._bumped = False
def get_last_tag(self):
return string.split(self.git_repo.git.describe(tags=True), '-')[0]
def get_submodule_hash(self, revision, submodule_path):
line = getattr(self.git_repo.git, 'ls-tree')(revision, submodule_path)
return string.split(line)[2] if line else None
def has_changes_from_revision(self, revision):
commit = str(self.git_repo.commit())
logging.info('%s =? %s', commit, revision)
return commit != revision
def save_commit(self):
self.saved_commit = self.git_repo.commit()
logging.info('Saved ', self.git_repo.commit(), self.saved_commit)
def checkout(self, branch):
self.git_repo.git.checkout(branch)
self.git_repo.git.pull(rebase=True)
def get_changelog_entry(self):
filename = os.path.join(self.directory, 'CHANGELOG.md')
return changelog.bump(filename, self.new_version())
def add_changelog(self):
with pushd(self.directory):
self.git_repo.git.add('CHANGELOG.md')
def new_version(self):
if self._bumped:
raise Exception('Cannot calculate a new version on an already bumped repo')
if not self.part:
raise Exception('Cannot calculate a new version without a part')
with pushd(self.directory):
output = subprocess.check_output(
['bumpversion', '--dry-run', '--list', '--allow-dirty', self.part])
return re.search('^new_version=(.*)$', output, re.M).group(1)
def bumpversion(self):
if self._bumped:
raise Exception('Cowardly refusing to bump a repo twice')
if not self.part:
raise Exception('Cannot bump version for {}: no part specified'.format(repo.name))
with pushd(self.directory):
subprocess.check_call(['bumpversion', '--allow-dirty', self.part])
self._bumped = True
def assert_new_tag_is_absent(self):
new_tag = 'v' + self.new_version()
tags = self.git_repo.git.tag()
if new_tag in tags.split('\n'):
raise Exception('Tag {} is already present in repo {}.'.format(new_tag, self.name))
@property
def git(self):
return self.git_repo.git
output = subprocess.check_output(['bumpversion', '-v'], stderr=subprocess.STDOUT).strip()
if output == 'bumpversion 0.5.4-lbry':
return True
except (subprocess.CalledProcessError, OSError):
pass
return False
@contextlib.contextmanager
@ -284,10 +196,4 @@ def pushd(new_dir):
if __name__ == '__main__':
logging.basicConfig(
format="%(asctime)s %(levelname)-8s %(name)s:%(lineno)d: %(message)s",
level='INFO'
)
sys.exit(main())
else:
log = logging.getLogger('__name__')