upload builds to s3

.appveyor.yml

@@ -9,6 +9,10 @@ environment:
     secure: 1mwqyRy7hDqDjDK+TIAoaXyXzpNgwruFNA6TPkinUcVM7A+NLD33RQLnfnwVy+R5ovD2pUfhQ6+N0Fqebv6tZh436LIEsock+6IOdpgFwrg=
   # find with: Get-Childitem –Path "C:\Program Files (x86)\Microsoft SDKs\Windows\" -Include *signtool* -File -Recurse -ErrorAction SilentlyContinue
   SIGNTOOL_PATH: C:\Program Files (x86)\Microsoft SDKs\Windows\v7.1A\Bin\signtool.exe
+  AWS_ACCESS_KEY_ID:
+    secure: iVGwoJ7ogspjSmuqr+haVPLglSgQsp6tUZx6mIlKH7Q=
+  AWS_SECRET_ACCESS_KEY:
+    secure: zKaqdZGPl0exDL5YhJkb33prSemC9Rzg9S7Lw2wFy1WnJ6ffgl6mQH7jqJDUTqsY
 
 skip_branch_with_pr: true
 

build/build.ps1

@@ -37,4 +37,4 @@ nuget install secure-file -ExcludeVersion
 secure-file\tools\secure-file -decrypt build\lbry2.pfx.enc -secret "$env:pfx_key"
 & ${env:SIGNTOOL_PATH} sign /f build\lbry2.pfx /p "$env:key_pass" /tr http://tsa.starfieldtech.com /td SHA256 /fd SHA256 dist\*.exe
 
-python build\release_on_tag.py
+python build\upload_assets.py

build/build.sh

@@ -107,7 +107,7 @@ if [ "$FULL_BUILD" == "true" ]; then
   # electron-build has a publish feature, but I had a hard time getting
   # it to reliably work and it also seemed difficult to configure. Not proud of
   # this, but it seemed better to write my own.
-  python "$BUILD_DIR/release_on_tag.py"
+  python "$BUILD_DIR/upload_assets.py"
 
   deactivate
 

build/requirements.txt

@@ -4,3 +4,4 @@ requests[security]==2.13.0
 PyInstaller==3.2.1
 uritemplate==3.0.0
 git+https://github.com/lbryio/bumpversion.git
+boto3==1.4.4

build/upload_assets.py

@@ -6,52 +6,16 @@ import subprocess
 import sys
 
 import github
-import requests
 import uritemplate
+import boto3
 
 
 def main():
-    try:
+    upload_to_github_if_tagged('lbryio/lbry-app')
-        current_tag = subprocess.check_output(
+    upload_to_s3('app')
-            ['git', 'describe', '--exact-match', 'HEAD']).strip()
-    except subprocess.CalledProcessError:
-        print 'Stopping as we are not currently on a tag'
-        return
-
-    if 'GH_TOKEN' not in os.environ:
-        print 'Must set GH_TOKEN in order to publish assets to a release'
-        return
-
-    gh_token = os.environ['GH_TOKEN']
-    auth = github.Github(gh_token)
-    repo = auth.get_repo('lbryio/lbry-app')
-
-    if not check_repo_has_tag(repo, current_tag):
-        print 'Tag {} is not in repo {}'.format(current_tag, repo)
-        # TODO: maybe this should be an error
-        return
-
-    app = get_app_artifact()
-    release = get_release(repo, current_tag)
-    upload_asset(release, app, gh_token)
 
 
-def check_repo_has_tag(repo, target_tag):
+def get_asset_filename():
-    tags = repo.get_tags().get_page(0)
-    for tag in tags:
-        if tag.name == target_tag:
-            return True
-    return False
-
-
-def get_release(current_repo, current_tag):
-    for release in current_repo.get_releases():
-        if release.tag_name == current_tag:
-            return release
-    raise Exception('No release for {} was found'.format(current_tag))
-
-
-def get_app_artifact():
     this_dir = os.path.dirname(os.path.realpath(__file__))
     system = platform.system()
     if system == 'Darwin':
@@ -64,40 +28,95 @@ def get_app_artifact():
         raise Exception("I don't know about any artifact on {}".format(system))
 
 
-def upload_asset(release, asset_to_upload, token):
+def upload_to_s3(folder):
+    tag = subprocess.check_output(['git', 'describe', '--always', 'HEAD']).strip()
+    commit_date = subprocess.check_output([
+        'git', 'show', '-s', '--format=%cd', '--date=format:%Y%m%d-%H%I%S', 'HEAD']).strip()
+
+    asset_path = get_asset_filename()
+    bucket = 'releases.lbry.io'
+    key = folder + '/' + commit_date + '-' + tag + '/' + os.path.basename(asset_path)
+
+    print "Uploading " + asset_path + " to s3://" + bucket + '/' + key + ''
+
+    if 'AWS_ACCESS_KEY_ID' not in os.environ or 'AWS_SECRET_ACCESS_KEY' not in os.environ:
+        print 'Must set AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY to publish assets to s3'
+        return 1
+
+    s3 = boto3.resource(
+        's3',
+        aws_access_key_id=os.environ['AWS_ACCESS_KEY_ID'],
+        aws_secret_access_key=os.environ['AWS_SECRET_ACCESS_KEY'],
+        config=boto3.session.Config(signature_version='s3v4')
+    )
+    s3.meta.client.upload_file(asset_path, bucket, key)
+
+
+def upload_to_github_if_tagged(repo_name):
+    try:
+        current_tag = subprocess.check_output(
+            ['git', 'describe', '--exact-match', 'HEAD']).strip()
+    except subprocess.CalledProcessError:
+        print 'Not uploading to GitHub as we are not currently on a tag'
+        return 1
+
+    print "Current tag: " + current_tag
+
+    if 'GH_TOKEN' not in os.environ:
+        print 'Must set GH_TOKEN in order to publish assets to a release'
+        return 1
+
+    gh_token = os.environ['GH_TOKEN']
+    auth = github.Github(gh_token)
+    repo = auth.get_repo(repo_name)
+
+    if not check_repo_has_tag(repo, current_tag):
+        print 'Tag {} is not in repo {}'.format(current_tag, repo)
+        # TODO: maybe this should be an error
+        return 1
+
+    asset_path = get_asset_filename()
+    print "Uploading " + asset_path + " to Github tag " + current_tag
+    release = get_github_release(repo, current_tag)
+    upload_asset_to_github(release, asset_path, gh_token)
+
+
+def check_repo_has_tag(repo, target_tag):
+    tags = repo.get_tags().get_page(0)
+    for tag in tags:
+        if tag.name == target_tag:
+            return True
+    return False
+
+
+def get_github_release(repo, current_tag):
+    for release in repo.get_releases():
+        if release.tag_name == current_tag:
+            return release
+    raise Exception('No release for {} was found'.format(current_tag))
+
+
+def upload_asset_to_github(release, asset_to_upload, token):
     basename = os.path.basename(asset_to_upload)
-    if is_asset_already_uploaded(release, basename):
+    for asset in release.raw_data['assets']:
-        return
+        if asset['name'] == basename:
+            print 'File {} has already been uploaded to {}'.format(basename, release.tag_name)
+            return
+
+    upload_uri = uritemplate.expand(release.upload_url, {'name': basename})
     count = 0
     while count < 10:
         try:
-            return _upload_asset(release, asset_to_upload, token, _curl_uploader)
+            output = _curl_uploader(upload_uri, asset_to_upload, token)
+            if 'errors' in output:
+                raise Exception(output)
+            else:
+                print 'Successfully uploaded to {}'.format(output['browser_download_url'])
         except Exception:
             print 'Failed uploading on attempt {}'.format(count + 1)
             count += 1
 
 
-def _upload_asset(release, asset_to_upload, token, uploader):
-    basename = os.path.basename(asset_to_upload)
-    upload_uri = uritemplate.expand(release.upload_url, {'name': basename})
-    output = uploader(upload_uri, asset_to_upload, token)
-    if 'errors' in output:
-        raise Exception(output)
-    else:
-        print 'Successfully uploaded to {}'.format(output['browser_download_url'])
-
-
-# requests doesn't work on windows / linux / osx.
-def _requests_uploader(upload_uri, asset_to_upload, token):
-    print 'Using requests to upload {} to {}'.format(asset_to_upload, upload_uri)
-    with open(asset_to_upload, 'rb') as f:
-        response = requests.post(upload_uri, data=f, auth=('', token))
-    return response.json()
-
-
-# curl -H "Content-Type: application/json" -X POST -d '{"username":"xyz","password":"xyz"}' http://localhost:3000/api/login
-
-
 def _curl_uploader(upload_uri, asset_to_upload, token):
     # using requests.post fails miserably with SSL EPIPE errors. I spent
     # half a day trying to debug before deciding to switch to curl.
@@ -129,13 +148,5 @@ def _curl_uploader(upload_uri, asset_to_upload, token):
     return json.loads(stdout)
 
 
-def is_asset_already_uploaded(release, basename):
-    for asset in release.raw_data['assets']:
-        if asset['name'] == basename:
-            print 'File {} has already been uploaded to {}'.format(basename, release.tag_name)
-            return True
-    return False
-
-
 if __name__ == '__main__':
     sys.exit(main())