upload builds to s3

.appveyor.yml

@@ -7,7 +7,10 @@ environment:
     secure: u6DydPcdrUJlxGL9uc7yQRYG8+5rY6aAEE9nfCSzFyNzZlX9NniOp8Uh5ZKQqX7bGEngLI6ipbLfiJvn0XFnhbn2iTkOuMqOXVJVOehvwlQ=
   pfx_key:
     secure: 1mwqyRy7hDqDjDK+TIAoaXyXzpNgwruFNA6TPkinUcVM7A+NLD33RQLnfnwVy+R5ovD2pUfhQ6+N0Fqebv6tZh436LIEsock+6IOdpgFwrg=
-
+  AWS_ACCESS_KEY_ID:
+    secure: iVGwoJ7ogspjSmuqr+haVPLglSgQsp6tUZx6mIlKH7Q=
+  AWS_SECRET_ACCESS_KEY:
+    secure: zKaqdZGPl0exDL5YhJkb33prSemC9Rzg9S7Lw2wFy1WnJ6ffgl6mQH7jqJDUTqsY
 
 #notifications:
 #  - provider: Slack

build/build.ps1

@@ -31,4 +31,4 @@ secure-file\tools\secure-file -decrypt .\lbry2.pfx.enc -secret "$env:pfx_key"
 signtool.exe sign /f .\lbry2.pfx /p "$env:key_pass" /tr http://tsa.starfieldtech.com /td SHA256 /fd SHA256 dist\*.exe
 
 python zip_daemon.py
-python release_on_tag.py
+python upload_assets.py

build/build.sh

@@ -50,7 +50,7 @@ if [ "$FULL_BUILD" == "true" ]; then
   # electron-build has a publish feature, but I had a hard time getting
   # it to reliably work and it also seemed difficult to configure. Not proud of
   # this, but it seemed better to write my own.
-  python "$BUILD_DIR/release_on_tag.py"
+  python "$BUILD_DIR/upload_assets.py"
 
   deactivate
 fi

build/requirements.txt

@@ -1,4 +1,4 @@
-# install daemon requirements (copied from root, with possible modifications. see build.sh, build.ps1)
+# install daemon requirements (created by build script. see build.sh, build.ps1)
 -r requirements_base.txt
 
 # install daemon itself. make sure you run `pip install` from this dir. this is how you do relative file paths with pip
@@ -8,3 +8,4 @@ file:../.
 PyInstaller==3.2.1
 requests[security]==2.13.0
 uritemplate==3.0.0
+boto3==1.4.4

build/upload_assets.py

@@ -5,18 +5,52 @@ import subprocess
 import sys
 
 import github
-import requests
 import uritemplate
+import boto3
+
+this_dir = os.path.dirname(os.path.realpath(__file__))
 
 
 def main():
-    this_dir = os.path.dirname(os.path.realpath(__file__))
+    upload_to_github_if_tagged('lbryio/lbry')
+    upload_to_s3('daemon')
+
+
+def get_asset_filename():
+    return glob.glob(this_dir + '/dist/*.zip')[0]
+
+
+def upload_to_s3(folder):
+    tag = subprocess.check_output(['git', 'describe', '--always', 'HEAD']).strip()
+    commit_date = subprocess.check_output([
+        'git', 'show', '-s', '--format=%cd', '--date=format:%Y%m%d-%H%I%S', 'HEAD']).strip()
+
+    asset_path = get_asset_filename()
+    bucket = 'releases.lbry.io'
+    key = folder + '/' + commit_date + '-' + tag + '/' + os.path.basename(asset_path)
+
+    print "Uploading " + asset_path + " to s3://" + bucket + '/' + key + ''
+
+    if 'AWS_ACCESS_KEY_ID' not in os.environ or 'AWS_SECRET_ACCESS_KEY' not in os.environ:
+        print 'Must set AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY to publish assets to s3'
+        return 1
+
+    s3 = boto3.resource(
+        's3',
+        aws_access_key_id=os.environ['AWS_ACCESS_KEY_ID'],
+        aws_secret_access_key=os.environ['AWS_SECRET_ACCESS_KEY'],
+        config=boto3.session.Config(signature_version='s3v4')
+    )
+    s3.meta.client.upload_file(asset_path, bucket, key)
+
+
+def upload_to_github_if_tagged(repo_name):
     try:
         current_tag = subprocess.check_output(
             ['git', 'describe', '--exact-match', 'HEAD']).strip()
     except subprocess.CalledProcessError:
-        print 'Stopping as we are not currently on a tag'
+        print 'Not uploading to GitHub as we are not currently on a tag'
-        return
+        return 1
 
     print "Current tag: " + current_tag
 
@@ -26,17 +60,17 @@ def main():
 
     gh_token = os.environ['GH_TOKEN']
     auth = github.Github(gh_token)
-    repo = auth.get_repo('lbryio/lbry')
+    repo = auth.get_repo(repo_name)
 
     if not check_repo_has_tag(repo, current_tag):
         print 'Tag {} is not in repo {}'.format(current_tag, repo)
         # TODO: maybe this should be an error
         return 1
 
-    daemon_zip = glob.glob(this_dir + '/dist/*.zip')[0]
+    asset_path = get_asset_filename()
-    print "Uploading " + daemon_zip
+    print "Uploading " + asset_path + " to Github tag " + current_tag
-    release = get_release(repo, current_tag)
+    release = get_github_release(repo, current_tag)
-    upload_asset(release, daemon_zip, gh_token)
+    upload_asset_to_github(release, asset_path, gh_token)
 
 
 def check_repo_has_tag(repo, target_tag):
@@ -47,47 +81,34 @@ def check_repo_has_tag(repo, target_tag):
     return False
 
 
-def get_release(current_repo, current_tag):
+def get_github_release(repo, current_tag):
-    for release in current_repo.get_releases():
+    for release in repo.get_releases():
         if release.tag_name == current_tag:
             return release
     raise Exception('No release for {} was found'.format(current_tag))
 
 
-def upload_asset(release, asset_to_upload, token):
+def upload_asset_to_github(release, asset_to_upload, token):
     basename = os.path.basename(asset_to_upload)
-    if is_asset_already_uploaded(release, basename):
+    for asset in release.raw_data['assets']:
-        return
+        if asset['name'] == basename:
+            print 'File {} has already been uploaded to {}'.format(basename, release.tag_name)
+            return
+
+    upload_uri = uritemplate.expand(release.upload_url, {'name': basename})
     count = 0
     while count < 10:
         try:
-            return _upload_asset(release, asset_to_upload, token, _curl_uploader)
+            output = _curl_uploader(upload_uri, asset_to_upload, token)
+            if 'errors' in output:
+                raise Exception(output)
+            else:
+                print 'Successfully uploaded to {}'.format(output['browser_download_url'])
         except Exception:
             print 'Failed uploading on attempt {}'.format(count + 1)
             count += 1
 
 
-def _upload_asset(release, asset_to_upload, token, uploader):
-    basename = os.path.basename(asset_to_upload)
-    upload_uri = uritemplate.expand(release.upload_url, {'name': basename})
-    output = uploader(upload_uri, asset_to_upload, token)
-    if 'errors' in output:
-        raise Exception(output)
-    else:
-        print 'Successfully uploaded to {}'.format(output['browser_download_url'])
-
-
-# requests doesn't work on windows / linux / osx.
-def _requests_uploader(upload_uri, asset_to_upload, token):
-    print 'Using requests to upload {} to {}'.format(asset_to_upload, upload_uri)
-    with open(asset_to_upload, 'rb') as f:
-        response = requests.post(upload_uri, data=f, auth=('', token))
-    return response.json()
-
-
-# curl -H "Content-Type: application/json" -X POST -d '{"username":"xyz","password":"xyz"}' http://localhost:3000/api/login
-
-
 def _curl_uploader(upload_uri, asset_to_upload, token):
     # using requests.post fails miserably with SSL EPIPE errors. I spent
     # half a day trying to debug before deciding to switch to curl.
@@ -119,13 +140,5 @@ def _curl_uploader(upload_uri, asset_to_upload, token):
     return json.loads(stdout)
 
 
-def is_asset_already_uploaded(release, basename):
-    for asset in release.raw_data['assets']:
-        if asset['name'] == basename:
-            print 'File {} has already been uploaded to {}'.format(basename, release.tag_name)
-            return True
-    return False
-
-
 if __name__ == '__main__':
     sys.exit(main())