From 44f07fb81d6d02989e54c6a3284da9a2747c8737 Mon Sep 17 00:00:00 2001 From: Alex Grintsvayg Date: Thu, 21 Nov 2019 14:16:10 -0500 Subject: [PATCH] move build-related files into top-level dir --- .gitlab-ci.yml | 30 ++++++++----------- .travis.yml | 22 +++++++------- .../Dockerfile.wallet_server | 11 +++---- {build => docker}/install_choco.ps1 | 0 {lbry/scripts => docker}/set_build.py | 9 ++++-- .../wallet_server_entrypoint.sh | 0 lbry/.gitignore | 1 + 7 files changed, 36 insertions(+), 37 deletions(-) rename {lbry/scripts => docker}/Dockerfile.wallet_server (78%) rename {build => docker}/install_choco.ps1 (100%) rename {lbry/scripts => docker}/set_build.py (81%) rename {lbry/scripts => docker}/wallet_server_entrypoint.sh (100%) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 3cdfeacd9..65533fbce 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -70,15 +70,13 @@ test:json-api: script: - pip install pyinstaller - pip install -e torba/. - - cd lbry - - python3.7 scripts/set_build.py - - pip install -e . - - pyinstaller --onefile --name lbrynet lbry/extras/cli.py - - cd dist - - chmod +x lbrynet - - zip --junk-paths ${CI_PROJECT_DIR}/lbrynet-${OS}.zip lbrynet # gitlab expects artifacts to be in $CI_PROJECT_DIR + - python3.7 docker/set_build.py + - pip install -e lbry/. + - pyinstaller --onefile --name lbrynet lbry/lbry/extras/cli.py + - chmod +x dist/lbrynet + - zip --junk-paths ${CI_PROJECT_DIR}/lbrynet-${OS}.zip dist/lbrynet # gitlab expects artifacts to be in $CI_PROJECT_DIR - openssl dgst -sha256 ${CI_PROJECT_DIR}/lbrynet-${OS}.zip | egrep -o [0-9a-f]+$ # get sha256 of asset. works on mac and ubuntu - - ./lbrynet --version + - dist/lbrynet --version build:linux: extends: .build @@ -116,7 +114,7 @@ build:windows: OS: windows GIT_DEPTH: 5 before_script: - - ./build/install_choco.ps1 + - ./docker/install_choco.ps1 - choco install -y --x86 python3 7zip checksum # - python --version | findstr /B "Python 3\.7\." # dont upgrade python on every run. just make sure we're on the right Python # - pip --version | findstr /E '\(python 3\.7\)' @@ -127,15 +125,13 @@ build:windows: - rmdir -Recurse venv script: - pip install pyinstaller - - pip install -e torba\. - - cd lbry - - python scripts\set_build.py - - pip install -e . - - pyinstaller --additional-hooks-dir=scripts/. --icon=icons/lbry256.ico -F -n lbrynet lbry/extras/cli.py - - cd dist - - 7z a -tzip $env:CI_PROJECT_DIR/lbrynet-${OS}.zip lbrynet.exe + - pip install -e torba/. + - python docker/set_build.py + - pip install -e lbry/. + - pyinstaller --additional-hooks-dir=lbry/scripts/. --icon=lbry/icons/lbry256.ico -F -n lbrynet lbry/lbry/extras/cli.py + - 7z a -tzip $env:CI_PROJECT_DIR/lbrynet-${OS}.zip dist/lbrynet.exe - checksum --type=sha256 --file=$env:CI_PROJECT_DIR/lbrynet-${OS}.zip - - ./lbrynet.exe --version + - dist/lbrynet.exe --version # upload could be done by making it a yaml alias and putting it right into the build step. that way if one OS fails, the others still get uploaded diff --git a/.travis.yml b/.travis.yml index a0b2ed9bc..844235a43 100644 --- a/.travis.yml +++ b/.travis.yml @@ -55,17 +55,15 @@ jobs: env: OS=linux install: - pip install pyinstaller awscli - - cd torba && pip install -e . && cd .. - - cd lbry - - python scripts/set_build.py - - pip install -e . + - python docker/set_build.py + - pip install -e torba/. + - pip install -e lbry/. script: - - pyinstaller -F -n lbrynet lbry/extras/cli.py - - cd dist - - chmod +x lbrynet - - zip -j lbrynet-${OS}.zip lbrynet + - pyinstaller -F -n lbrynet lbry/lbry/extras/cli.py + - chmod +x dist/lbrynet + - zip --junk-paths lbrynet-${OS}.zip dist/lbrynet - shasum -a 256 -b lbrynet-${OS}.zip - - ./lbrynet --version + - dist/lbrynet --version after_success: - aws configure set aws_access_key_id $ARTIFACTS_KEY - aws configure set aws_secret_access_key $ARTIFACTS_SECRET @@ -110,7 +108,7 @@ jobs: - python -m pip install --upgrade pip - pip install pywin32 script: - - pyinstaller --additional-hooks-dir=scripts/. --icon=icons/lbry256.ico -F -n lbrynet lbry/extras/cli.py + - pyinstaller --additional-hooks-dir=lbry/scripts/. --icon=lbry/icons/lbry256.ico -F -n lbrynet lbry/lbry/extras/cli.py - cd dist - 7z a -tzip lbrynet-windows.zip lbrynet.exe - sha256sum -b lbrynet-windows.zip @@ -122,7 +120,7 @@ jobs: script: - set -e - echo "$DOCKER_PASSWORD" | docker login --username "$DOCKER_USERNAME" --password-stdin - - travis_retry docker build -t lbry/wallet-server:$TRAVIS_TAG -t lbry/wallet-server:latest-release -f lbry/scripts/Dockerfile.wallet_server . + - travis_retry docker build -t lbry/wallet-server:$TRAVIS_TAG -t lbry/wallet-server:latest-release -f docker/Dockerfile.wallet_server . - docker push lbry/wallet-server:$TRAVIS_TAG - docker push lbry/wallet-server:latest-release @@ -132,7 +130,7 @@ jobs: script: - set -e - echo "$DOCKER_PASSWORD" | docker login --username "$DOCKER_USERNAME" --password-stdin - - travis_retry docker build -t lbry/wallet-server:master -f lbry/scripts/Dockerfile.wallet_server . + - travis_retry docker build -t lbry/wallet-server:master -f docker/Dockerfile.wallet_server . - docker push lbry/wallet-server:master cache: diff --git a/lbry/scripts/Dockerfile.wallet_server b/docker/Dockerfile.wallet_server similarity index 78% rename from lbry/scripts/Dockerfile.wallet_server rename to docker/Dockerfile.wallet_server index 80c29be52..141f696da 100644 --- a/lbry/scripts/Dockerfile.wallet_server +++ b/docker/Dockerfile.wallet_server @@ -8,7 +8,9 @@ RUN mkdir -p /home/$user/ RUN chown -R $user:$user /home/$user/ # install python, pip, git and clean up -RUN apt-get update && apt-get -y --no-install-recommends install build-essential git tar wget python3.7 python3.7-dev python3-pip && rm -rf /var/lib/apt/lists/* +RUN apt-get update && \ + apt-get -y --no-install-recommends install build-essential git tar wget python3.7 python3.7-dev python3-pip && \ + rm -rf /var/lib/apt/lists/* # create and chown database dir ARG db_dir=/database @@ -26,8 +28,7 @@ RUN python3.7 -m pip install --upgrade pip setuptools RUN python3.7 -m pip install --user uvloop # copy lbrynet -ARG projects_dir=/home/$user/projects -RUN mkdir $projects_dir +ARG projects_dir=/home/$user COPY . $projects_dir USER root RUN chown -R $user:$user . @@ -35,7 +36,7 @@ USER $user # install torba & lbry WORKDIR $projects_dir/torba -RUN python3.7 -m pip install --user . +RUN python3.7 -m pip install --user -e . WORKDIR $projects_dir/lbry RUN python3.7 -m pip install --user -e . RUN rm ~/.cache -rf @@ -53,5 +54,5 @@ ENV BANDWIDTH_LIMIT=1000000000000000000000000000000000000000000 ENV MAX_SESSIONS=1000000000 ENV MAX_SEND=1000000000000000000 ENV EVENT_LOOP_POLICY=uvloop -COPY ./lbry/scripts/wallet_server_entrypoint.sh /entrypoint.sh +COPY ./docker/wallet_server_entrypoint.sh /entrypoint.sh ENTRYPOINT ["/entrypoint.sh"] diff --git a/build/install_choco.ps1 b/docker/install_choco.ps1 similarity index 100% rename from build/install_choco.ps1 rename to docker/install_choco.ps1 diff --git a/lbry/scripts/set_build.py b/docker/set_build.py similarity index 81% rename from lbry/scripts/set_build.py rename to docker/set_build.py index eaf3ab475..bc2bd3d97 100644 --- a/lbry/scripts/set_build.py +++ b/docker/set_build.py @@ -23,10 +23,13 @@ def get_build_type(ci_tag=None): def main(): root_dir = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) - build_type_path = os.path.join(root_dir, 'lbry', 'build_type.py') + build_type_path = os.path.join(root_dir, 'lbry', 'lbry', 'build_type.py') log.debug("configuring build type file: %s", build_type_path) - commit_hash = os.getenv('CI_COMMIT_SHA', os.getenv('TRAVIS_COMMIT', None))[:6] - build_type = get_build_type(os.getenv('CI_COMMIT_TAG', os.getenv('TRAVIS_TAG', None))) + commit_hash = os.getenv('CI_COMMIT_SHA', os.getenv('TRAVIS_COMMIT')) + if commit_hash is None: + raise ValueError("Commit hash not found in env vars") + commit_hash = commit_hash[:6] + build_type = get_build_type(os.getenv('CI_COMMIT_TAG', os.getenv('TRAVIS_TAG'))) log.debug("setting build type=%s, build commit=%s", build_type, commit_hash) with open(build_type_path, 'w') as f: f.write(f"BUILD = \"{build_type}\"\nBUILD_COMMIT = \"{commit_hash}\"\n") diff --git a/lbry/scripts/wallet_server_entrypoint.sh b/docker/wallet_server_entrypoint.sh similarity index 100% rename from lbry/scripts/wallet_server_entrypoint.sh rename to docker/wallet_server_entrypoint.sh diff --git a/lbry/.gitignore b/lbry/.gitignore index 08588a3df..cfc004a50 100644 --- a/lbry/.gitignore +++ b/lbry/.gitignore @@ -11,3 +11,4 @@ __pycache__ _trial_temp/ /tests/integration/files +/tests/.coverage.*