gitlab ci setup

what works:
- tests
- linux and mac builds
- uploading builds to s3

what's left to do:
- uploading coverage results to codecov.io
- windows build
- github releases
- what directories to cache between builds

.gitlab-ci.yml

@@ -0,0 +1,159 @@
+default:
+  image: python:3.7
+
+
+#cache:
+#  directories:
+#    - $HOME/venv
+#    - $HOME/.cache/pip
+#    - $HOME/Library/Caches/pip
+#    - $HOME/Library/Caches/Homebrew
+#    - $TRAVIS_BUILD_DIR/.tox
+
+
+stages:
+  - test
+  - build
+  - release
+
+
+# could also use extends: instead of yaml anchors (https://docs.gitlab.com/ee/ci/yaml/README.html#extends)
+.codecov.io: &codecovio
+  after_script:
+    - coverage combine lbry/
+    - bash <(curl -s https://codecov.io/bash)
+
+test:lint:
+  stage: test
+  script:
+    - make install
+    - make lint
+
+test:lbry-unit:
+  stage: test
+  script:
+    - make install
+    - cd lbry && HOME=/tmp coverage run -p --source=lbry -m unittest discover -vv tests.unit
+#  <<: *codecovio
+
+
+test:lbry-integ:
+  stage: test
+  script:
+    - pip install coverage tox-travis
+    - cd lbry && tox
+#  <<: *codecovio
+
+
+.torba-tests: &torba_tests
+  stage: test
+  script:
+    - pip install coverage tox-travis
+    - cd torba && tox
+#  <<: *codecovio
+
+test:torba-unit:
+  before_script:
+    - export TESTTYPE=unit
+  <<: *torba_tests
+
+test:torba-integ:
+  before_script:
+    - export TESTTYPE=integration
+  <<: *torba_tests
+
+test:json-api:
+  stage: test
+  script:
+    - make install
+    - cd lbry && HOME=/tmp coverage run -p --source=lbry scripts/generate_json_api.py
+#  <<: *codecovio
+
+
+
+.build:
+  stage: build
+  artifacts:
+    expire_in: 1 day
+    paths:
+      - lbrynet-${OS}.zip
+  script:
+    - pip install pyinstaller
+    - pip install -e torba/.
+    - cd lbry
+    - python3.7 scripts/set_build.py
+    - pip install -e .
+    - pyinstaller --onefile --name lbrynet lbry/extras/cli.py
+    - cd dist
+    - chmod +x lbrynet
+    - zip --junk-paths ${CI_PROJECT_DIR}/lbrynet-${OS}.zip lbrynet  # gitlab expects artifacts to be in $CI_PROJECT_DIR
+    - openssl dgst -sha256 ${CI_PROJECT_DIR}/lbrynet-${OS}.zip | egrep -o [0-9a-f]+$ # get sha256 of asset. works on mac and ubuntu
+    - ./lbrynet --version
+
+build:linux:
+  extends: .build
+  image: ubuntu:18.04  # cant use python3.7 image - binary won't run on ubuntu
+  variables:
+    OS: linux
+  before_script:
+    - apt-get update
+    - apt-get install -y --no-install-recommends software-properties-common zip curl build-essential
+    - add-apt-repository -y ppa:deadsnakes/ppa
+    - apt-get install -y --no-install-recommends python3.7-dev python3-setuptools python3-wheel
+    - python3.7 <(curl -q https://bootstrap.pypa.io/get-pip.py) # make sure we get pip with python3.7
+
+build:mac:
+  extends: .build
+  tags: [macos] # makes gitlab use the mac runner
+  variables:
+    OS: mac
+    GIT_DEPTH: 5
+    VENV: /tmp/gitlab-lbry-sdk-venv
+  before_script:
+#    - brew upgrade python || true
+    - python3 --version | grep -q '^Python 3\.7\.' # dont upgrade python on every run. just make sure we're on the right Python
+#    - pip3 install --user --upgrade pip virtualenv
+    - pip3 --version | grep -q '\(python 3\.7\)'
+    - virtualenv --python=python3.7 "${VENV}"
+    - source "${VENV}/bin/activate"
+  after_script:
+    - rm -rf "${VENV}"
+
+
+
+# upload could be done by making it a yaml alias and putting it right into the build step. that way if one OS fails, the others still get uploaded
+.upload:
+  stage: release
+  variables:
+    GIT_STRATEGY: none
+  script:
+    - pip install awscli
+    - S3_PATH="daemon/gitlab-build-${CI_PIPELINE_ID}_commit-${CI_COMMIT_SHA:0:7}$( if [ ! -z ${CI_COMMIT_TAG} ]; then echo _tag-${CI_COMMIT_TAG}; else echo _branch-${CI_COMMIT_REF_NAME}; fi )"
+    - AWS_ACCESS_KEY_ID=${ARTIFACTS_KEY} AWS_SECRET_ACCESS_KEY=${ARTIFACTS_SECRET} AWS_REGION=${ARTIFACTS_REGION}
+      aws s3 cp lbrynet-${OS}.zip s3://${ARTIFACTS_BUCKET}/${S3_PATH}/lbrynet-${OS}.zip
+
+s3:linux:
+  extends: .upload
+  variables:
+    OS: linux
+  dependencies:
+    - build:linux
+
+s3:mac:
+  extends: .upload
+  variables:
+    OS: mac
+  dependencies:
+    - build:mac
+
+#release:linux:
+#  stage: release
+#  only: [tags]
+#  variables:
+#    OS: linux
+#    GIT_STRATEGY: none
+#  dependencies: [build:linux]
+#  script:
+#    - pip install githubrelease
+#    - githubrelease asset lbryio/lbry-sdk upload ${CI_COMMIT_TAG} lbrynet-${OS}.zip
+

.travis.yml

@@ -115,6 +115,7 @@ jobs:
         - 7z a -tzip lbrynet-windows.zip lbrynet.exe
         - sha256sum -b lbrynet-windows.zip
         - ./lbrynet.exe --version
+
     - if: tag IS present
       stage: build
       name: "Wallet Server Docker Image - Tagged Release"

lbry/scripts/set_build.py

@@ -10,13 +10,13 @@ log.addHandler(logging.StreamHandler())
 log.setLevel(logging.DEBUG)
 
 
-def get_build_type(travis_tag=None):
+def get_build_type(ci_tag=None):
-    if not travis_tag:
+    if not ci_tag:
         return "qa"
-    log.debug("getting build type for tag: \"%s\"", travis_tag)
+    log.debug("getting build type for tag: \"%s\"", ci_tag)
-    if re.match(r'v\d+\.\d+\.\d+rc\d+$', travis_tag):
+    if re.match(r'v\d+\.\d+\.\d+rc\d+$', ci_tag):
         return 'rc'
-    elif re.match(r'v\d+\.\d+\.\d+$', travis_tag):
+    elif re.match(r'v\d+\.\d+\.\d+$', ci_tag):
         return 'release'
     return 'qa'
 
@@ -25,11 +25,11 @@ def main():
     root_dir = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
     build_type_path = os.path.join(root_dir, 'lbry', 'build_type.py')
     log.debug("configuring build type file: %s", build_type_path)
-    travis_commit = os.environ['TRAVIS_COMMIT'][:6]
+    commit_hash = os.getenv('CI_COMMIT_SHA', os.getenv('TRAVIS_COMMIT', None))[:6]
-    build_type = get_build_type(os.environ.get('TRAVIS_TAG', None))
+    build_type = get_build_type(os.getenv('CI_COMMIT_TAG', os.getenv('TRAVIS_TAG', None)))
-    log.debug("setting build type=%s, build commit=%s", build_type, travis_commit)
+    log.debug("setting build type=%s, build commit=%s", build_type, commit_hash)
     with open(build_type_path, 'w') as f:
-        f.write(f"BUILD = \"{build_type}\"\nBUILD_COMMIT = \"{travis_commit}\"\n")
+        f.write(f"BUILD = \"{build_type}\"\nBUILD_COMMIT = \"{commit_hash}\"\n")
 
 
 if __name__ == '__main__':