From 6112f60681195fde435c32b6f06873758f4904aa Mon Sep 17 00:00:00 2001 From: Alex Grintsvayg Date: Tue, 20 Aug 2019 15:11:52 -0400 Subject: [PATCH] gitlab ci setup what works: - tests - linux and mac builds - uploading builds to s3 what's left to do: - uploading coverage results to codecov.io - windows build - github releases - what directories to cache between builds --- .gitlab-ci.yml | 159 ++++++++++++++++++++++++++++++++++++++ .travis.yml | 1 + lbry/scripts/set_build.py | 18 ++--- 3 files changed, 169 insertions(+), 9 deletions(-) create mode 100644 .gitlab-ci.yml diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml new file mode 100644 index 000000000..adf080991 --- /dev/null +++ b/.gitlab-ci.yml @@ -0,0 +1,159 @@ +default: + image: python:3.7 + + +#cache: +# directories: +# - $HOME/venv +# - $HOME/.cache/pip +# - $HOME/Library/Caches/pip +# - $HOME/Library/Caches/Homebrew +# - $TRAVIS_BUILD_DIR/.tox + + +stages: + - test + - build + - release + + +# could also use extends: instead of yaml anchors (https://docs.gitlab.com/ee/ci/yaml/README.html#extends) +.codecov.io: &codecovio + after_script: + - coverage combine lbry/ + - bash <(curl -s https://codecov.io/bash) + +test:lint: + stage: test + script: + - make install + - make lint + +test:lbry-unit: + stage: test + script: + - make install + - cd lbry && HOME=/tmp coverage run -p --source=lbry -m unittest discover -vv tests.unit +# <<: *codecovio + + +test:lbry-integ: + stage: test + script: + - pip install coverage tox-travis + - cd lbry && tox +# <<: *codecovio + + +.torba-tests: &torba_tests + stage: test + script: + - pip install coverage tox-travis + - cd torba && tox +# <<: *codecovio + +test:torba-unit: + before_script: + - export TESTTYPE=unit + <<: *torba_tests + +test:torba-integ: + before_script: + - export TESTTYPE=integration + <<: *torba_tests + +test:json-api: + stage: test + script: + - make install + - cd lbry && HOME=/tmp coverage run -p --source=lbry scripts/generate_json_api.py +# <<: *codecovio + + + +.build: + stage: build + artifacts: + expire_in: 1 day + paths: + - lbrynet-${OS}.zip + script: + - pip install pyinstaller + - pip install -e torba/. + - cd lbry + - python3.7 scripts/set_build.py + - pip install -e . + - pyinstaller --onefile --name lbrynet lbry/extras/cli.py + - cd dist + - chmod +x lbrynet + - zip --junk-paths ${CI_PROJECT_DIR}/lbrynet-${OS}.zip lbrynet # gitlab expects artifacts to be in $CI_PROJECT_DIR + - openssl dgst -sha256 ${CI_PROJECT_DIR}/lbrynet-${OS}.zip | egrep -o [0-9a-f]+$ # get sha256 of asset. works on mac and ubuntu + - ./lbrynet --version + +build:linux: + extends: .build + image: ubuntu:18.04 # cant use python3.7 image - binary won't run on ubuntu + variables: + OS: linux + before_script: + - apt-get update + - apt-get install -y --no-install-recommends software-properties-common zip curl build-essential + - add-apt-repository -y ppa:deadsnakes/ppa + - apt-get install -y --no-install-recommends python3.7-dev python3-setuptools python3-wheel + - python3.7 <(curl -q https://bootstrap.pypa.io/get-pip.py) # make sure we get pip with python3.7 + +build:mac: + extends: .build + tags: [macos] # makes gitlab use the mac runner + variables: + OS: mac + GIT_DEPTH: 5 + VENV: /tmp/gitlab-lbry-sdk-venv + before_script: +# - brew upgrade python || true + - python3 --version | grep -q '^Python 3\.7\.' # dont upgrade python on every run. just make sure we're on the right Python +# - pip3 install --user --upgrade pip virtualenv + - pip3 --version | grep -q '\(python 3\.7\)' + - virtualenv --python=python3.7 "${VENV}" + - source "${VENV}/bin/activate" + after_script: + - rm -rf "${VENV}" + + + +# upload could be done by making it a yaml alias and putting it right into the build step. that way if one OS fails, the others still get uploaded +.upload: + stage: release + variables: + GIT_STRATEGY: none + script: + - pip install awscli + - S3_PATH="daemon/gitlab-build-${CI_PIPELINE_ID}_commit-${CI_COMMIT_SHA:0:7}$( if [ ! -z ${CI_COMMIT_TAG} ]; then echo _tag-${CI_COMMIT_TAG}; else echo _branch-${CI_COMMIT_REF_NAME}; fi )" + - AWS_ACCESS_KEY_ID=${ARTIFACTS_KEY} AWS_SECRET_ACCESS_KEY=${ARTIFACTS_SECRET} AWS_REGION=${ARTIFACTS_REGION} + aws s3 cp lbrynet-${OS}.zip s3://${ARTIFACTS_BUCKET}/${S3_PATH}/lbrynet-${OS}.zip + +s3:linux: + extends: .upload + variables: + OS: linux + dependencies: + - build:linux + +s3:mac: + extends: .upload + variables: + OS: mac + dependencies: + - build:mac + +#release:linux: +# stage: release +# only: [tags] +# variables: +# OS: linux +# GIT_STRATEGY: none +# dependencies: [build:linux] +# script: +# - pip install githubrelease +# - githubrelease asset lbryio/lbry-sdk upload ${CI_COMMIT_TAG} lbrynet-${OS}.zip + diff --git a/.travis.yml b/.travis.yml index a727aa8d5..a0b2ed9bc 100644 --- a/.travis.yml +++ b/.travis.yml @@ -115,6 +115,7 @@ jobs: - 7z a -tzip lbrynet-windows.zip lbrynet.exe - sha256sum -b lbrynet-windows.zip - ./lbrynet.exe --version + - if: tag IS present stage: build name: "Wallet Server Docker Image - Tagged Release" diff --git a/lbry/scripts/set_build.py b/lbry/scripts/set_build.py index 1382e742d..eaf3ab475 100644 --- a/lbry/scripts/set_build.py +++ b/lbry/scripts/set_build.py @@ -10,13 +10,13 @@ log.addHandler(logging.StreamHandler()) log.setLevel(logging.DEBUG) -def get_build_type(travis_tag=None): - if not travis_tag: +def get_build_type(ci_tag=None): + if not ci_tag: return "qa" - log.debug("getting build type for tag: \"%s\"", travis_tag) - if re.match(r'v\d+\.\d+\.\d+rc\d+$', travis_tag): + log.debug("getting build type for tag: \"%s\"", ci_tag) + if re.match(r'v\d+\.\d+\.\d+rc\d+$', ci_tag): return 'rc' - elif re.match(r'v\d+\.\d+\.\d+$', travis_tag): + elif re.match(r'v\d+\.\d+\.\d+$', ci_tag): return 'release' return 'qa' @@ -25,11 +25,11 @@ def main(): root_dir = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) build_type_path = os.path.join(root_dir, 'lbry', 'build_type.py') log.debug("configuring build type file: %s", build_type_path) - travis_commit = os.environ['TRAVIS_COMMIT'][:6] - build_type = get_build_type(os.environ.get('TRAVIS_TAG', None)) - log.debug("setting build type=%s, build commit=%s", build_type, travis_commit) + commit_hash = os.getenv('CI_COMMIT_SHA', os.getenv('TRAVIS_COMMIT', None))[:6] + build_type = get_build_type(os.getenv('CI_COMMIT_TAG', os.getenv('TRAVIS_TAG', None))) + log.debug("setting build type=%s, build commit=%s", build_type, commit_hash) with open(build_type_path, 'w') as f: - f.write(f"BUILD = \"{build_type}\"\nBUILD_COMMIT = \"{travis_commit}\"\n") + f.write(f"BUILD = \"{build_type}\"\nBUILD_COMMIT = \"{commit_hash}\"\n") if __name__ == '__main__':