diff --git a/.travis.yml b/.travis.yml index e46781e58..57408b0a5 100644 --- a/.travis.yml +++ b/.travis.yml @@ -29,6 +29,7 @@ jobs: - mkdir -p dist - sudo zip -Xj dist/lbrycrd-${NAME}.zip src/lbrycrdd${EXT} src/lbrycrd-cli${EXT} src/lbrycrd-tx${EXT} - sudo zip -Xj dist/lbrycrd-${NAME}-test.zip src/test/test_lbrycrd${EXT} src/test/test_lbrycrd_fuzzy${EXT} + - sudo cp dist/lbrycrd-${NAME}.zip packaging/docker-for-binary/lbrycrd-${NAME}.zip - sha256sum dist/lbrycrd-${NAME}.zip - sha256sum dist/lbrycrd-${NAME}-test.zip deploy: @@ -44,6 +45,13 @@ jobs: on: repo: lbryio/lbrycrd all_branches: true + - provider: script + script: bash packaging/docker-for-binary/auto_deploy.sh ${TRAVIS_BRANCH} + skip_cleanup: true + on: + repo: lbryio/lbrycrd + all_branches: true + condition: $NAME = linux - <<: *build-template name: windows diff --git a/packaging/docker-for-binary/Dockerfile b/packaging/docker-for-binary/Dockerfile new file mode 100644 index 000000000..4e0e29450 --- /dev/null +++ b/packaging/docker-for-binary/Dockerfile @@ -0,0 +1,36 @@ +FROM ubuntu:18.04 as prep +LABEL MAINTAINER="leopere [at] nixc [dot] us" +## TODO: Implement version pinning. `apt-get install curl=` +RUN apt-get update && \ + apt-get -y install unzip curl build-essential && \ + apt-get autoclean -y && \ + rm -rf /var/lib/apt/lists/* +WORKDIR / +SHELL ["/bin/bash", "-o", "pipefail", "-c"] +COPY ./stuff/start.sh start +COPY ./stuff/healthcheck.sh healthcheck +COPY ./stuff/advance_blocks.sh advance +COPY ./stuff/fix-permissions.c fix-permissions.c +ARG release_url +# require that release_url is set +RUN test -n "$release_url" +RUN curl --progress-bar -L -o ./lbrycrd-linux.zip "$release_url" && \ + unzip ./lbrycrd-linux.zip && \ + gcc fix-permissions.c -o fix-permissions && \ + chmod +x ./lbrycrdd ./lbrycrd-cli ./lbrycrd-tx ./start ./healthcheck ./fix-permissions ./advance + +FROM ubuntu:18.04 as app +COPY --from=prep /lbrycrdd /lbrycrd-cli /lbrycrd-tx /start /healthcheck /fix-permissions /advance /usr/bin/ +RUN addgroup --gid 1000 lbrycrd && \ + adduser lbrycrd --uid 1000 --gid 1000 --gecos GECOS --shell /bin/bash --disabled-password --home /data && \ + mkdir /etc/lbry && \ + chown lbrycrd /etc/lbry && \ + chmod a+s /usr/bin/fix-permissions +RUN apt-get update && apt-get -y install wget && apt-get autoclean -y && rm -rf /var/lib/apt/lists/* +VOLUME ["/data"] +WORKDIR /data +HEALTHCHECK CMD /usr/bin/healthcheck +EXPOSE 9246 9245 11337 29245 + +USER lbrycrd +CMD ["start"] diff --git a/packaging/docker-for-binary/Dockerfile.Auto b/packaging/docker-for-binary/Dockerfile.Auto new file mode 100644 index 000000000..36a968731 --- /dev/null +++ b/packaging/docker-for-binary/Dockerfile.Auto @@ -0,0 +1,34 @@ +FROM ubuntu:18.04 as prep +LABEL MAINTAINER="leopere [at] nixc [dot] us" +## TODO: Implement version pinning. `apt-get install curl=` +RUN apt-get update && \ + apt-get -y install unzip curl build-essential && \ + apt-get autoclean -y && \ + rm -rf /var/lib/apt/lists/* +WORKDIR / +SHELL ["/bin/bash", "-o", "pipefail", "-c"] +COPY ./stuff/start.sh start +COPY ./stuff/healthcheck.sh healthcheck +COPY ./stuff/advance_blocks.sh advance +COPY ./stuff/fix-permissions.c fix-permissions.c +COPY ./lbrycrd-linux.zip lbrycrd-linux.zip + +RUN unzip ./lbrycrd-linux.zip && \ + gcc fix-permissions.c -o fix-permissions && \ + chmod +x ./lbrycrdd ./lbrycrd-cli ./lbrycrd-tx ./start ./healthcheck ./fix-permissions ./advance + +FROM ubuntu:18.04 as app +COPY --from=prep /lbrycrdd /lbrycrd-cli /lbrycrd-tx /start /healthcheck /fix-permissions /advance /usr/bin/ +RUN addgroup --gid 1000 lbrycrd && \ + adduser lbrycrd --uid 1000 --gid 1000 --gecos GECOS --shell /bin/bash --disabled-password --home /data && \ + mkdir /etc/lbry && \ + chown lbrycrd /etc/lbry && \ + chmod a+s /usr/bin/fix-permissions +RUN apt-get update && apt-get -y install wget && apt-get autoclean -y && rm -rf /var/lib/apt/lists/* +VOLUME ["/data"] +WORKDIR /data +HEALTHCHECK CMD /usr/bin/healthcheck +EXPOSE 9246 9245 11337 29245 + +USER lbrycrd +CMD ["start"] diff --git a/packaging/docker-for-binary/README.md b/packaging/docker-for-binary/README.md new file mode 100644 index 000000000..4f2d37282 --- /dev/null +++ b/packaging/docker-for-binary/README.md @@ -0,0 +1,76 @@ +# lbrycrd Docker image + +## Scripts + +There are two scripts `deploy.sh` and `auto_deploy.sh` These are used to create +docker images to push to lbry's Docker Hub. + +### `auto_deploy.sh` + +This script is used by the LBRYcrd's CI. When a branch or tag is built +it will create a docker image for it and push it to DockerHub. This should +not be used outside of the CI environment. In addition to this, the +`Dockerfile.Auto` is associated with this script. This will only run on the +Linux build job. + +#### Requirements + +- You would need to build lbrycrd with the Docker image for reproducible +builds. https://hub.docker.com/repository/docker/lbry/build_lbrycrd +- You will need DockerHub credentials to run this locally. +- When the script is executed you will need the parameter as + `./auto_deploy.sh ${tag_name}` + +### `deploy.sh` + +This can be used locally to manually create a docker image based on a +release. `release_url` is requested as a parameter to the script when +run locally. This will grab the binary from github releases inside the +image build. + +#### Requirements + +- You will need DockerHub credentials to run this locally. + +## Configuration + +The lbrycrd container comes with a default configuration you can use for +production. Extra configuration is optional. + +The container includes a `start` script that offers a flexible configuration +style. It allows you to mount your own `lbrycrd.conf` file, or use environment +variables, or a mix of both. + +### Environment variables + +The environment variables override the values in the mounted config file. If no +mounted config file exists, these variables are used to create a fresh config. + + * `PORT` - The main lbrycrd port + * `RPC_USER` - The rpc user + * `RPC_PASSWORD` - The rpc user's password + * `RPC_ALLOW_IP` - the subnet that is allowed rpc access + * `RPC_PORT` - The port to bind the rpc service to + * `RPC_BIND` - The ip address to bind the rpc service to + + +### Example run commands + +Running the default configuration: + +``` +docker run --rm -it -e RUN_MODE=default -e SNAPSHOT_URL="https://lbry.com/snapshot/blockchain" lbry/lbrycrd:latest-release +``` + +Running with RPC password changed: + +``` +docker run --rm -it -e RUN_MODE=default -e RPC_PASSWORD=hunter2 lbry/lbrycrd:latest-release +``` + +Running with a config file but with the RPC password still overridden: + +``` +docker run --rm -it -v /path/to/lbrycrd.conf:/etc/lbry/lbrycrd.conf -e RUN_MODE=default -e RPC_PASSWORD=hunter2 lbry/lbrycrd:latest-release +``` + diff --git a/packaging/docker-for-binary/auto_deploy.sh b/packaging/docker-for-binary/auto_deploy.sh new file mode 100755 index 000000000..3bc0c45ff --- /dev/null +++ b/packaging/docker-for-binary/auto_deploy.sh @@ -0,0 +1,26 @@ +#!/bin/bash + + +set -euo pipefail +hash docker 2>/dev/null || { echo >&2 'Make sure Docker is installed'; exit 1; } + +set +eo pipefail +docker version | grep -q Server +ret=$? +set -eo pipefail + +if [ $ret -ne 0 ]; then + echo "Cannot connect to Docker server. Is it running? Do you have the right user permissions?" + exit 1 +fi + +if [ -z "$1" ]; then + echo "Docker tag parameter cannot be empty" + exit 1 +fi + +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +cd packaging/docker-for-binary +docker build --tag "lbry/lbrycrd:$1" -f Dockerfile.Auto "$DIR" +echo "$DOCKER_PASSWORD" | docker login --username "$DOCKER_USERNAME" --password-stdin +docker push "lbry/lbrycrd:$1" diff --git a/packaging/docker-for-binary/compose/docker-compose.yml-prod-example b/packaging/docker-for-binary/compose/docker-compose.yml-prod-example new file mode 100644 index 000000000..5e0172638 --- /dev/null +++ b/packaging/docker-for-binary/compose/docker-compose.yml-prod-example @@ -0,0 +1,26 @@ +version: '3.4' + +networks: + lbry-network: + external: true + +services: +############# +## Lbrycrd ## +############# + lbrycrd: + build: . + restart: always + networks: + lbry-network: + ipv4_address: 10.6.1.2 + environment: + RUN_MODE: default + env_file: + - env + expose: + - 9245 + - 9246 + ## host volumes for persistent data such as wallet private keys. + volumes: + - "../persist/data:/data" diff --git a/packaging/docker-for-binary/compose/docker-compose.yml-regtest b/packaging/docker-for-binary/compose/docker-compose.yml-regtest new file mode 100644 index 000000000..d2ea47971 --- /dev/null +++ b/packaging/docker-for-binary/compose/docker-compose.yml-regtest @@ -0,0 +1,28 @@ +version: '3.4' + +networks: + lbry-network: + external: true + +services: +############# +## Lbrycrd ## +############# + lbrycrd: + build: + context: . + dockerfile: Dockerfile + restart: always + ports: + - "11336:29246" + - "11337:29245" + ## host volumes for persistent data such as wallet private keys. + volumes: + - "../persist/data:/data" + networks: + lbry-network: + ipv4_address: 10.6.1.2 + environment: + - RUN_MODE=regtest + - PORT=29245 + - AUTO_ADVANCE=1 diff --git a/packaging/docker-for-binary/compose/docker-compose.yml-testnet b/packaging/docker-for-binary/compose/docker-compose.yml-testnet new file mode 100644 index 000000000..5653ab726 --- /dev/null +++ b/packaging/docker-for-binary/compose/docker-compose.yml-testnet @@ -0,0 +1,17 @@ +version: '3.4' + +services: +############# +## Lbrycrd ## +############# + lbrycrd: + image: lbry/lbrycrd:latest-release + restart: always + ports: + - "11336:9246" + - "11337:11337" + ## host volumes for persistent data such as wallet private keys. + volumes: + - "../persist/data:/data" + environment: + - RUN_MODE=testnet diff --git a/packaging/docker-for-binary/deploy.sh b/packaging/docker-for-binary/deploy.sh new file mode 100755 index 000000000..092fd7fb4 --- /dev/null +++ b/packaging/docker-for-binary/deploy.sh @@ -0,0 +1,33 @@ +#!/bin/bash + +set -euo pipefail + +hash docker 2>/dev/null || { echo >&2 'Make sure Docker is installed'; exit 1; } + +set +eo pipefail +docker version | grep -q Server +ret=$? +set -eo pipefail + +if [ $ret -ne 0 ]; then + echo "Cannot connect to Docker server. Is it running? Do you have the right user permissions?" + exit 1 +fi + +echo "This will build the docker image using the latest lbrycrd release and push that image to Docker Hub" +echo "" + +echo "What Docker tag should I use? It's the part that goes here: lbry/lbrycrd:TAG" + +read -p "Docker tag: " docker_tag +if [ -z "$docker_tag" ]; then + echo "Docker tag cannot be empty" + exit 1 +fi + +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +release_url=$(curl -s https://api.github.com/repos/lbryio/lbrycrd/releases | grep -F 'lbrycrd-linux' | grep download | head -n 1 | cut -d'"' -f4) + +docker build --build-arg "release_url=$release_url" --tag "lbry/lbrycrd:${docker_tag}" -f Dockerfile "$DIR" +docker push "lbry/lbrycrd:${docker_tag}" diff --git a/packaging/docker-for-binary/stuff/advance_blocks.sh b/packaging/docker-for-binary/stuff/advance_blocks.sh new file mode 100755 index 000000000..ebbe99336 --- /dev/null +++ b/packaging/docker-for-binary/stuff/advance_blocks.sh @@ -0,0 +1,4 @@ +#!/usr/bin/env bash +while sleep 2; do + lbrycrd-cli -conf=/etc/lbry/lbrycrd.conf generate 1 +done diff --git a/packaging/docker-for-binary/stuff/env-example b/packaging/docker-for-binary/stuff/env-example new file mode 100644 index 000000000..751a05ca8 --- /dev/null +++ b/packaging/docker-for-binary/stuff/env-example @@ -0,0 +1,12 @@ +COMPOSE_PROJECT_NAME=lbrycrd + +############# +## Lbrycrd ## +############# +## TODO: The credentials are a formality but we should randomize these with magic. +RPC_USER=${RPC_USER=lbryrpc} +RPC_PASSWORD=${RPC_PASSWORD:-changeme} + +## This should be the internal container IP from which you'll be calling the RPC for Lbrycrdd from. +## TODO: make this more dynamic before we move to scalability +RPC_ALLOW_IP=${RPC_ALLOW_IP:-10.6.1.3} diff --git a/packaging/docker-for-binary/stuff/fix-permissions.c b/packaging/docker-for-binary/stuff/fix-permissions.c new file mode 100644 index 000000000..d287c14e6 --- /dev/null +++ b/packaging/docker-for-binary/stuff/fix-permissions.c @@ -0,0 +1,9 @@ +#include +int main() { + // This program needs to run with setuid == root + // This needs to be in a compiled language because you cannot setuid bash scripts + setuid(0); + execle("/bin/bash", "bash", "-c", + "/bin/chown -R lbrycrd:lbrycrd /data && /bin/chmod -R 755 /data/", + (char*) NULL, (char*) NULL); +} diff --git a/packaging/docker-for-binary/stuff/healthcheck.sh b/packaging/docker-for-binary/stuff/healthcheck.sh new file mode 100755 index 000000000..51e49024d --- /dev/null +++ b/packaging/docker-for-binary/stuff/healthcheck.sh @@ -0,0 +1 @@ +/usr/bin/lbrycrd-cli -conf=/etc/lbry/lbrycrd.conf getnetworkinfo | grep -q '"networkactive": true,' diff --git a/packaging/docker-for-binary/stuff/start.sh b/packaging/docker-for-binary/stuff/start.sh new file mode 100755 index 000000000..cfcabd429 --- /dev/null +++ b/packaging/docker-for-binary/stuff/start.sh @@ -0,0 +1,139 @@ +#!/usr/bin/env bash + +CONFIG_PATH=/etc/lbry/lbrycrd.conf + + +function override_config_option() { + # Remove existing config line from a config file + # and replace with environment fed value. + # Does nothing if the variable does not exist. + # var Name of ENV variable + # option Name of config option + # config Path of config file + local var=$1 option=$2 config=$3 + if [[ -v $var ]]; then + # Remove the existing config option: + sed -i "/^$option\W*=/d" $config + # Add the value from the environment: + echo "$option=${!var}" >> $config + fi +} + + +function set_config() { + if [ -d "$CONFIG_PATH" ]; then + echo "$CONFIG_PATH is a directory when it should be a file." + exit 1 + elif [ -f "$CONFIG_PATH" ]; then + echo "Merging the mounted config file with environment variables." + local MERGED_CONFIG=/tmp/lbrycrd_merged.conf + cp $CONFIG_PATH $MERGED_CONFIG + echo "" >> $MERGED_CONFIG + override_config_option PORT port $MERGED_CONFIG + override_config_option RPC_USER rpcuser $MERGED_CONFIG + override_config_option RPC_PASSWORD rpcpassword $MERGED_CONFIG + override_config_option RPC_ALLOW_IP rpcallowip $MERGED_CONFIG + override_config_option RPC_PORT rpcport $MERGED_CONFIG + override_config_option RPC_BIND rpcbind $MERGED_CONFIG + override_config_option TX_INDEX txindex $MERGED_CONFIG + override_config_option MAX_TX_FEE maxtxfee $MERGED_CONFIG + override_config_option DUST_RELAY_FEE dustrelayfee $MERGED_CONFIG + # Make the new merged config file the new CONFIG_PATH + # This ensures that the original file the user mounted remains unmodified + CONFIG_PATH=$MERGED_CONFIG + else + echo "Creating a fresh config file from environment variables." + cat << EOF > $CONFIG_PATH +server=1 +printtoconsole=1 + +port=${PORT:-9246} +rpcuser=${RPC_USER:-lbry} +rpcpassword=${RPC_PASSWORD:-lbry} +rpcallowip=${RPC_ALLOW_IP:-127.0.0.1} +rpcport=${RPC_PORT:-9245} +rpcbind=${RPC_BIND:-"0.0.0.0"} + +txindex=${TX_INDEX:-"1"} +maxtxfee=${MAX_TX_FEE:-"0.5"} +dustrelayfee=${DUST_RELAY_FEE:-"0.00000001"} +EOF + fi + echo "Config: " + cat $CONFIG_PATH +} + + +function download_snapshot() { + local url="${SNAPSHOT_URL:-}" #off by default. latest snapshot at https://lbry.com/snapshot/blockchain + if [[ -n "$url" ]] && [[ ! -d ./.lbrycrd/blocks ]]; then + echo "Downloading blockchain snapshot from $url" + wget --no-verbose -O snapshot.tar.bz2 "$url" + echo "Extracting snapshot..." + mkdir -p ./.lbrycrd + tar xvjf snapshot.tar.bz2 --directory ./.lbrycrd + rm snapshot.tar.bz2 + fi +} + + +## Ensure perms are correct prior to running main binary +/usr/bin/fix-permissions + + +## You can optionally specify a run mode if you want to use lbry defined presets for compatibility. +case $RUN_MODE in + default ) + set_config + download_snapshot + exec lbrycrdd -conf=$CONFIG_PATH + ;; + ## If it's a first run you need to do a full index including all transactions + ## tx index creates an index of every single transaction in the block history if + ## not specified it will only create an index for transactions that are related to the wallet or have unspent outputs. + ## This is generally specific to chainquery. + reindex ) + ## Apply this RUN_MODE in the case you need to update a dataset. NOTE: you do not need to use `RUN_MODE reindex` for more than one complete run. + set_config + exec lbrycrdd -conf=$CONFIG_PATH -reindex + ;; + regtest ) + ## Set config params + ## TODO: Make this more automagic in the future. + mkdir -p `dirname $CONFIG_PATH` + echo "rpcuser=lbry" > $CONFIG_PATH + echo "rpcpassword=lbry" >> $CONFIG_PATH + echo "rpcport=29245" >> $CONFIG_PATH + echo "rpcbind=0.0.0.0" >> $CONFIG_PATH + echo "rpcallowip=0.0.0.0/0" >> $CONFIG_PATH + echo "regtest=1" >> $CONFIG_PATH + echo "txindex=1" >> $CONFIG_PATH + echo "server=1" >> $CONFIG_PATH + echo "printtoconsole=1" >> $CONFIG_PATH + + [ "${AUTO_ADVANCE:-0}" == "1" ] && nohup advance &>/dev/null & + + exec lbrycrdd -conf=$CONFIG_PATH $1 + ;; + testnet ) + ## Set config params + ## TODO: Make this more automagic in the future. + mkdir -p `dirname $CONFIG_PATH` + echo "rpcuser=lbry" > $CONFIG_PATH + echo "rpcpassword=lbry" >> $CONFIG_PATH + echo "rpcport=29245" >> $CONFIG_PATH + echo "rpcbind=0.0.0.0" >> $CONFIG_PATH + echo "rpcallowip=0.0.0.0/0" >> $CONFIG_PATH + echo "testnet=1" >> $CONFIG_PATH + echo "txindex=1" >> $CONFIG_PATH + echo "server=1" >> $CONFIG_PATH + echo "printtoconsole=1" >> $CONFIG_PATH + + #nohup advance &>/dev/null & + exec lbrycrdd -conf=$CONFIG_PATH $1 + ;; + * ) + echo "Error, you must define a RUN_MODE environment variable." + echo "Available options are testnet, regtest, chainquery, default, and reindex" + ;; +esac diff --git a/src/claimtrie/trie.cpp b/src/claimtrie/trie.cpp index cef67efe3..4a74c6b82 100644 --- a/src/claimtrie/trie.cpp +++ b/src/claimtrie/trie.cpp @@ -471,8 +471,10 @@ bool CClaimTrieCacheBase::checkConsistency() int takeoverHeight; row >> name >> hash >> takeoverHeight; auto computedHash = computeNodeHash(name, takeoverHeight); - if (computedHash != hash) + if (computedHash != hash) { + logPrint << "Invalid hash at " << name << Clog::endl; return false; + } } return true; }