lbry-desktop/build/upload_assets.py

import glob
import json
import os
import platform
import subprocess
import sys

import github
import uritemplate
import boto3

S3_BUCKET = 'releases.lbry.io'
RELEASES_S3_PATH = 'app'
LATEST_S3_PATH = 'app/latest'

def main():
    upload_to_github_if_tagged('lbryio/lbry-app')
    upload_to_s3(RELEASES_S3_PATH)


def get_asset_path():
    this_dir = os.path.dirname(os.path.realpath(__file__))
    system = platform.system()
    if system == 'Darwin':
        suffix = 'dmg'
    elif system == 'Linux':
        suffix = 'deb'
    elif system == 'Windows':
        suffix = 'exe'
    else:
        raise Exception("I don't know about any artifact on {}".format(system))

    return os.path.realpath(glob.glob(this_dir + '/../dist/LBRY*.' + suffix)[0])

def get_update_asset_path():
    # Get the asset used used for updates. On Mac, this is a .zip; on
    # Windows it's just the installer file.
    if platform.system() == 'Darwin':
        this_dir = os.path.dirname(os.path.realpath(__file__))
        return os.path.realpath(glob.glob(this_dir + '/../dist/LBRY*.zip')[0])
    else:
        return get_asset_path()


def get_latest_file_path():
  # The update metadata file is called latest.yml on Windows, latest-mac.yml on
  # Mac, latest-linux.yml on Linux
  this_dir = os.path.dirname(os.path.realpath(__file__))

  latestfilematches = glob.glob(this_dir + '/../dist/latest*.yml')

  return latestfilematches[0] if latestfilematches else None


def upload_to_s3(folder):
    asset_path = get_asset_path()

    if 'AWS_ACCESS_KEY_ID' not in os.environ or 'AWS_SECRET_ACCESS_KEY' not in os.environ:
        print 'Must set AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY to publish assets to s3'
        return 1

    asset_filename = os.path.basename(asset_path)

    tag = subprocess.check_output(['git', 'describe', '--always', '--abbrev=8', 'HEAD']).strip()
    commit_date = subprocess.check_output([
        'git', 'show', '-s', '--format=%cd', '--date=format:%Y%m%d-%H%I%S', 'HEAD']).strip()
    key = folder + '/' + commit_date + '-' + tag + '/' + asset_filename

    print "Uploading asset file at " + asset_path + " to s3://" + S3_BUCKET + '/' + key

    s3 = boto3.resource(
        's3',
        aws_access_key_id=os.environ['AWS_ACCESS_KEY_ID'],
        aws_secret_access_key=os.environ['AWS_SECRET_ACCESS_KEY'],
        config=boto3.session.Config(signature_version='s3v4')
    )

    s3.Object(S3_BUCKET, key).upload_file(asset_path)

    # Populate the update bucket (update.lbry.io)

    update_asset_path = get_update_asset_path()
    if asset_path == update_asset_path:
        # If the update asset and the regular built file are the same, we can
        # just copy over.
        print "Copying asset file to s3://" + S3_BUCKET + "/" + LATEST_S3_PATH + "/" + asset_filename
        s3.Object(S3_BUCKET, LATEST_S3_PATH + "/" + asset_filename).copy_from(CopySource={
            'Bucket': S3_BUCKET,
            'Key': key
        })
    else:
        update_asset_filename = os.path.basename(update_asset_path)
        print "Uploading update asset file at", update_asset_path, \
              "to s3://" + S3_BUCKET + "/" + LATEST_S3_PATH + "/" + update_asset_filename
        s3.Object(S3_BUCKET, LATEST_S3_PATH + "/" + update_asset_filename).upload_file(update_asset_path)

    metadatafilepath = get_latest_file_path()

    if metadatafilepath is not None:
        # For some reason latest-linux.yml isn't being created, but it's OK because updates don't
        # work on Linux yet anyway.
        metadatafilename = os.path.basename(metadatafilepath)

        print "Uploading update metadata file at", metadatafilepath, "to S3"
        s3.Object(S3_BUCKET, LATEST_S3_PATH + "/" + metadatafilename).upload_file(metadatafilepath)


def upload_to_github_if_tagged(repo_name):
    try:
        current_tag = subprocess.check_output(
            ['git', 'describe', '--exact-match', 'HEAD']).strip()
    except subprocess.CalledProcessError:
        print 'Not uploading to GitHub as we are not currently on a tag'
        return 1

    print "Current tag: " + current_tag

    if 'GH_TOKEN' not in os.environ:
        print 'Must set GH_TOKEN in order to publish assets to a release'
        return 1

    gh_token = os.environ['GH_TOKEN']
    auth = github.Github(gh_token)
    repo = auth.get_repo(repo_name)

    if not check_repo_has_tag(repo, current_tag):
        print 'Tag {} is not in repo {}'.format(current_tag, repo)
        # TODO: maybe this should be an error
        return 1

    asset_path = get_asset_path()
    print "Uploading " + asset_path + " to Github tag " + current_tag
    release = get_github_release(repo, current_tag)
    upload_asset_to_github(release, asset_path, gh_token)


def check_repo_has_tag(repo, target_tag):
    tags = repo.get_tags().get_page(0)
    for tag in tags:
        if tag.name == target_tag:
            return True
    return False


def get_github_release(repo, current_tag):
    for release in repo.get_releases():
        if release.tag_name == current_tag:
            return release
    raise Exception('No release for {} was found'.format(current_tag))


def upload_asset_to_github(release, asset_to_upload, token):
    basename = os.path.basename(asset_to_upload)
    for asset in release.raw_data['assets']:
        if asset['name'] == basename:
            print 'File {} has already been uploaded to {}'.format(basename, release.tag_name)
            return

    upload_uri = uritemplate.expand(release.upload_url, {'name': basename})
    count = 0
    while count < 10:
        try:
            output = _curl_uploader(upload_uri, asset_to_upload, token)
            if 'errors' in output:
                raise Exception(output)
            else:
                print 'Successfully uploaded to {}'.format(output['browser_download_url'])
        except Exception:
            print 'Failed uploading on attempt {}'.format(count + 1)
            count += 1


def _curl_uploader(upload_uri, asset_to_upload, token):
    # using requests.post fails miserably with SSL EPIPE errors. I spent
    # half a day trying to debug before deciding to switch to curl.
    #
    # TODO: actually set the content type
    print 'Using curl to upload {} to {}'.format(asset_to_upload, upload_uri)
    cmd = [
        'curl',
        '-sS',
        '-X', 'POST',
        '-u', ':{}'.format(os.environ['GH_TOKEN']),
        '--header', 'Content-Type: application/octet-stream',
        '--data-binary', '@-',
        upload_uri
    ]
    # '-d', '{"some_key": "some_value"}',
    print 'Calling curl:'
    print cmd
    print
    with open(asset_to_upload, 'rb') as fp:
        p = subprocess.Popen(cmd, stdin=fp, stderr=subprocess.PIPE, stdout=subprocess.PIPE)
    stdout, stderr = p.communicate()
    print 'curl return code:', p.returncode
    if stderr:
        print 'stderr output from curl:'
        print stderr
    print 'stdout from curl:'
    print stdout
    return json.loads(stdout)


if __name__ == '__main__':
    sys.exit(main())