move in changes from master

.travis.yml

@@ -29,6 +29,7 @@ jobs:
         - mkdir -p dist
         - sudo zip -Xj dist/lbrycrd-${NAME}.zip src/lbrycrdd${EXT} src/lbrycrd-cli${EXT} src/lbrycrd-tx${EXT}
         - sudo zip -Xj dist/lbrycrd-${NAME}-test.zip src/test/test_lbrycrd${EXT} src/test/test_lbrycrd_fuzzy${EXT}
+        - sudo cp dist/lbrycrd-${NAME}.zip packaging/docker-for-binary/lbrycrd-${NAME}.zip
         - sha256sum dist/lbrycrd-${NAME}.zip
         - sha256sum dist/lbrycrd-${NAME}-test.zip
       deploy:
@@ -44,6 +45,13 @@ jobs:
           on:
             repo: lbryio/lbrycrd
             all_branches: true
+        - provider: script
+          script: bash packaging/docker-for-binary/auto_deploy.sh ${TRAVIS_BRANCH}
+          skip_cleanup: true
+          on:
+            repo: lbryio/lbrycrd
+            all_branches: true
+            condition: $NAME = linux
 
     - <<: *build-template
       name: windows

packaging/docker-for-binary/Dockerfile

@@ -0,0 +1,36 @@
+FROM ubuntu:18.04 as prep
+LABEL MAINTAINER="leopere [at] nixc [dot] us"
+## TODO: Implement version pinning. `apt-get install curl=<version>`
+RUN apt-get update && \
+  apt-get -y install unzip curl build-essential && \
+  apt-get autoclean -y && \
+  rm -rf /var/lib/apt/lists/*
+WORKDIR /
+SHELL ["/bin/bash", "-o", "pipefail", "-c"]
+COPY ./stuff/start.sh start
+COPY ./stuff/healthcheck.sh healthcheck
+COPY ./stuff/advance_blocks.sh advance
+COPY ./stuff/fix-permissions.c fix-permissions.c
+ARG release_url
+# require that release_url is set
+RUN test -n "$release_url"
+RUN curl --progress-bar -L -o ./lbrycrd-linux.zip "$release_url" && \
+  unzip ./lbrycrd-linux.zip && \
+  gcc fix-permissions.c -o fix-permissions && \
+  chmod +x ./lbrycrdd ./lbrycrd-cli ./lbrycrd-tx ./start ./healthcheck ./fix-permissions ./advance
+
+FROM ubuntu:18.04 as app
+COPY --from=prep /lbrycrdd /lbrycrd-cli /lbrycrd-tx /start /healthcheck /fix-permissions /advance /usr/bin/
+RUN addgroup --gid 1000 lbrycrd && \
+  adduser lbrycrd --uid 1000 --gid 1000 --gecos GECOS --shell /bin/bash --disabled-password --home /data && \
+  mkdir /etc/lbry && \
+  chown lbrycrd /etc/lbry && \
+  chmod a+s /usr/bin/fix-permissions
+RUN apt-get update && apt-get -y install wget && apt-get autoclean -y && rm -rf /var/lib/apt/lists/*
+VOLUME ["/data"]
+WORKDIR /data
+HEALTHCHECK CMD /usr/bin/healthcheck
+EXPOSE 9246 9245 11337 29245
+
+USER lbrycrd
+CMD ["start"]

packaging/docker-for-binary/Dockerfile.Auto

@@ -0,0 +1,34 @@
+FROM ubuntu:18.04 as prep
+LABEL MAINTAINER="leopere [at] nixc [dot] us"
+## TODO: Implement version pinning. `apt-get install curl=<version>`
+RUN apt-get update && \
+  apt-get -y install unzip curl build-essential && \
+  apt-get autoclean -y && \
+  rm -rf /var/lib/apt/lists/*
+WORKDIR /
+SHELL ["/bin/bash", "-o", "pipefail", "-c"]
+COPY ./stuff/start.sh start
+COPY ./stuff/healthcheck.sh healthcheck
+COPY ./stuff/advance_blocks.sh advance
+COPY ./stuff/fix-permissions.c fix-permissions.c
+COPY ./lbrycrd-linux.zip lbrycrd-linux.zip
+
+RUN unzip ./lbrycrd-linux.zip && \
+  gcc fix-permissions.c -o fix-permissions && \
+  chmod +x ./lbrycrdd ./lbrycrd-cli ./lbrycrd-tx ./start ./healthcheck ./fix-permissions ./advance
+
+FROM ubuntu:18.04 as app
+COPY --from=prep /lbrycrdd /lbrycrd-cli /lbrycrd-tx /start /healthcheck /fix-permissions /advance /usr/bin/
+RUN addgroup --gid 1000 lbrycrd && \
+  adduser lbrycrd --uid 1000 --gid 1000 --gecos GECOS --shell /bin/bash --disabled-password --home /data && \
+  mkdir /etc/lbry && \
+  chown lbrycrd /etc/lbry && \
+  chmod a+s /usr/bin/fix-permissions
+RUN apt-get update && apt-get -y install wget && apt-get autoclean -y && rm -rf /var/lib/apt/lists/*
+VOLUME ["/data"]
+WORKDIR /data
+HEALTHCHECK CMD /usr/bin/healthcheck
+EXPOSE 9246 9245 11337 29245
+
+USER lbrycrd
+CMD ["start"]

packaging/docker-for-binary/README.md

@@ -0,0 +1,76 @@
+# lbrycrd Docker image
+
+## Scripts
+
+There are two scripts `deploy.sh` and `auto_deploy.sh` These are used to create 
+docker images to push to lbry's Docker Hub.
+
+### `auto_deploy.sh`
+
+This script is used by the LBRYcrd's CI. When a branch or tag is built 
+it will create a docker image for it and push it to DockerHub. This should
+not be used outside of the CI environment. In addition to this, the 
+`Dockerfile.Auto` is associated with this script. This will only run on the
+Linux build job. 
+
+#### Requirements
+
+- You would need to build lbrycrd with the Docker image for reproducible 
+builds. https://hub.docker.com/repository/docker/lbry/build_lbrycrd
+- You will need DockerHub credentials to run this locally. 
+- When the script is executed you will need the parameter as
+ `./auto_deploy.sh ${tag_name}`
+
+### `deploy.sh`
+
+This can be used locally to manually create a docker image based on a 
+release. `release_url` is requested as a parameter to the script when 
+run locally. This will grab the binary from github releases inside the
+image build. 
+
+#### Requirements
+
+- You will need DockerHub credentials to run this locally. 
+
+## Configuration
+
+The lbrycrd container comes with a default configuration you can use for
+production. Extra configuration is optional.
+
+The container includes a `start` script that offers a flexible configuration
+style. It allows you to mount your own `lbrycrd.conf` file, or use environment
+variables, or a mix of both.
+
+### Environment variables
+
+The environment variables override the values in the mounted config file. If no
+mounted config file exists, these variables are used to create a fresh config.
+
+ * `PORT` - The main lbrycrd port
+ * `RPC_USER` - The rpc user
+ * `RPC_PASSWORD` - The rpc user's password
+ * `RPC_ALLOW_IP` - the subnet that is allowed rpc access
+ * `RPC_PORT` - The port to bind the rpc service to
+ * `RPC_BIND` - The ip address to bind the rpc service to
+ 
+
+### Example run commands
+
+Running the default configuration:
+
+```
+docker run --rm -it -e RUN_MODE=default -e SNAPSHOT_URL="https://lbry.com/snapshot/blockchain" lbry/lbrycrd:latest-release
+```
+
+Running with RPC password changed:
+
+```
+docker run --rm -it -e RUN_MODE=default -e RPC_PASSWORD=hunter2 lbry/lbrycrd:latest-release
+```
+
+Running with a config file but with the RPC password still overridden:
+
+```
+docker run --rm -it -v /path/to/lbrycrd.conf:/etc/lbry/lbrycrd.conf -e RUN_MODE=default -e RPC_PASSWORD=hunter2 lbry/lbrycrd:latest-release
+```
+

packaging/docker-for-binary/auto_deploy.sh

@@ -0,0 +1,26 @@
+#!/bin/bash
+
+
+set -euo pipefail
+hash docker 2>/dev/null || { echo >&2 'Make sure Docker is installed'; exit 1;  }
+
+set +eo pipefail
+docker version | grep -q Server
+ret=$?
+set -eo pipefail
+
+if [ $ret -ne 0 ]; then
+  echo "Cannot connect to Docker server. Is it running? Do you have the right user permissions?"
+  exit 1
+fi
+
+if [ -z "$1" ]; then
+  echo "Docker tag parameter cannot be empty"
+  exit 1
+fi
+
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+cd packaging/docker-for-binary
+docker build --tag "lbry/lbrycrd:$1" -f Dockerfile.Auto "$DIR"
+echo "$DOCKER_PASSWORD" | docker login --username "$DOCKER_USERNAME" --password-stdin
+docker push "lbry/lbrycrd:$1"

packaging/docker-for-binary/compose/docker-compose.yml-prod-example

@@ -0,0 +1,26 @@
+version: '3.4'
+
+networks:
+  lbry-network:
+    external: true
+
+services:
+#############
+## Lbrycrd ##
+#############
+  lbrycrd:
+    build: .
+    restart: always
+    networks:
+      lbry-network:
+        ipv4_address: 10.6.1.2
+    environment:
+      RUN_MODE: default
+    env_file:
+     - env
+    expose:
+      - 9245
+      - 9246
+    ## host volumes for persistent data such as wallet private keys.
+    volumes:
+      - "../persist/data:/data"

packaging/docker-for-binary/compose/docker-compose.yml-regtest

@@ -0,0 +1,28 @@
+version: '3.4'
+
+networks:
+  lbry-network:
+    external: true
+
+services:
+#############
+## Lbrycrd ##
+#############
+  lbrycrd:
+    build:
+      context: .
+      dockerfile: Dockerfile
+    restart: always
+    ports:
+      - "11336:29246"
+      - "11337:29245"
+    ## host volumes for persistent data such as wallet private keys.
+    volumes:
+      - "../persist/data:/data"
+    networks:
+      lbry-network:
+        ipv4_address: 10.6.1.2
+    environment:
+      - RUN_MODE=regtest
+      - PORT=29245
+      - AUTO_ADVANCE=1

packaging/docker-for-binary/compose/docker-compose.yml-testnet

@@ -0,0 +1,17 @@
+version: '3.4'
+
+services:
+#############
+## Lbrycrd ##
+#############
+  lbrycrd:
+    image: lbry/lbrycrd:latest-release
+    restart: always
+    ports:
+      - "11336:9246"
+      - "11337:11337"
+    ## host volumes for persistent data such as wallet private keys.
+    volumes:
+      - "../persist/data:/data"
+    environment:
+      - RUN_MODE=testnet

packaging/docker-for-binary/deploy.sh

@@ -0,0 +1,33 @@
+#!/bin/bash
+
+set -euo pipefail
+
+hash docker 2>/dev/null || { echo >&2 'Make sure Docker is installed'; exit 1;  }
+
+set +eo pipefail
+docker version | grep -q Server
+ret=$?
+set -eo pipefail
+
+if [ $ret -ne 0 ]; then
+  echo "Cannot connect to Docker server. Is it running? Do you have the right user permissions?"
+  exit 1
+fi
+
+echo "This will build the docker image using the latest lbrycrd release and push that image to Docker Hub"
+echo ""
+
+echo "What Docker tag should I use? It's the part that goes here: lbry/lbrycrd:TAG"
+
+read -p "Docker tag: " docker_tag
+if [ -z "$docker_tag" ]; then
+  echo "Docker tag cannot be empty"
+  exit 1
+fi
+
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+release_url=$(curl -s https://api.github.com/repos/lbryio/lbrycrd/releases | grep -F 'lbrycrd-linux' | grep download | head -n 1 | cut -d'"' -f4)
+
+docker build --build-arg "release_url=$release_url" --tag "lbry/lbrycrd:${docker_tag}" -f Dockerfile "$DIR"
+docker push "lbry/lbrycrd:${docker_tag}"

packaging/docker-for-binary/stuff/advance_blocks.sh

@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+while sleep 2; do
+    lbrycrd-cli -conf=/etc/lbry/lbrycrd.conf generate 1
+done

packaging/docker-for-binary/stuff/env-example

@@ -0,0 +1,12 @@
+COMPOSE_PROJECT_NAME=lbrycrd
+
+#############
+## Lbrycrd ##
+#############
+## TODO: The credentials are a formality but we should randomize these with magic.
+RPC_USER=${RPC_USER=lbryrpc}
+RPC_PASSWORD=${RPC_PASSWORD:-changeme}
+
+## This should be the internal container IP from which you'll be calling the RPC for Lbrycrdd from.
+## TODO: make this more dynamic before we move to scalability
+RPC_ALLOW_IP=${RPC_ALLOW_IP:-10.6.1.3}

packaging/docker-for-binary/stuff/fix-permissions.c

@@ -0,0 +1,9 @@
+#include <unistd.h>
+int main() {
+  // This program needs to run with setuid == root
+  // This needs to be in a compiled language because you cannot setuid bash scripts
+  setuid(0);
+  execle("/bin/bash", "bash", "-c",
+         "/bin/chown -R lbrycrd:lbrycrd /data && /bin/chmod -R 755 /data/",
+         (char*) NULL, (char*) NULL);
+}

packaging/docker-for-binary/stuff/healthcheck.sh

@@ -0,0 +1 @@
+/usr/bin/lbrycrd-cli -conf=/etc/lbry/lbrycrd.conf getnetworkinfo | grep -q '"networkactive": true,'

packaging/docker-for-binary/stuff/start.sh

@@ -0,0 +1,139 @@
+#!/usr/bin/env bash
+
+CONFIG_PATH=/etc/lbry/lbrycrd.conf
+
+
+function override_config_option() {
+    # Remove existing config line from a config file
+    #  and replace with environment fed value.
+    # Does nothing if the variable does not exist.
+    #  var     Name of ENV variable
+    #  option  Name of config option
+    #  config  Path of config file
+    local var=$1 option=$2 config=$3
+    if [[ -v $var ]]; then
+        # Remove the existing config option:
+        sed -i "/^$option\W*=/d" $config
+        # Add the value from the environment:
+        echo "$option=${!var}" >> $config
+    fi
+}
+
+
+function set_config() {
+  if [ -d "$CONFIG_PATH" ]; then
+      echo "$CONFIG_PATH is a directory when it should be a file."
+      exit 1
+  elif [ -f "$CONFIG_PATH" ]; then
+      echo "Merging the mounted config file with environment variables."
+      local MERGED_CONFIG=/tmp/lbrycrd_merged.conf
+      cp $CONFIG_PATH $MERGED_CONFIG
+      echo "" >> $MERGED_CONFIG
+      override_config_option PORT port $MERGED_CONFIG
+      override_config_option RPC_USER rpcuser $MERGED_CONFIG
+      override_config_option RPC_PASSWORD rpcpassword $MERGED_CONFIG
+      override_config_option RPC_ALLOW_IP rpcallowip $MERGED_CONFIG
+      override_config_option RPC_PORT rpcport $MERGED_CONFIG
+      override_config_option RPC_BIND rpcbind $MERGED_CONFIG
+      override_config_option TX_INDEX txindex $MERGED_CONFIG
+      override_config_option MAX_TX_FEE maxtxfee $MERGED_CONFIG
+      override_config_option DUST_RELAY_FEE dustrelayfee $MERGED_CONFIG
+      # Make the new merged config file the new CONFIG_PATH
+      # This ensures that the original file the user mounted remains unmodified
+      CONFIG_PATH=$MERGED_CONFIG
+  else
+      echo "Creating a fresh config file from environment variables."
+      cat << EOF > $CONFIG_PATH
+server=1
+printtoconsole=1
+
+port=${PORT:-9246}
+rpcuser=${RPC_USER:-lbry}
+rpcpassword=${RPC_PASSWORD:-lbry}
+rpcallowip=${RPC_ALLOW_IP:-127.0.0.1}
+rpcport=${RPC_PORT:-9245}
+rpcbind=${RPC_BIND:-"0.0.0.0"}
+
+txindex=${TX_INDEX:-"1"}
+maxtxfee=${MAX_TX_FEE:-"0.5"}
+dustrelayfee=${DUST_RELAY_FEE:-"0.00000001"}
+EOF
+  fi
+  echo "Config: "
+  cat $CONFIG_PATH
+}
+
+
+function download_snapshot() {
+  local url="${SNAPSHOT_URL:-}" #off by default. latest snapshot at https://lbry.com/snapshot/blockchain
+  if [[ -n "$url" ]] && [[ ! -d ./.lbrycrd/blocks ]]; then
+    echo "Downloading blockchain snapshot from $url"
+    wget --no-verbose -O snapshot.tar.bz2 "$url"
+    echo "Extracting snapshot..."
+    mkdir -p ./.lbrycrd
+    tar xvjf snapshot.tar.bz2 --directory ./.lbrycrd
+    rm snapshot.tar.bz2
+  fi
+}
+
+
+## Ensure perms are correct prior to running main binary
+/usr/bin/fix-permissions
+
+
+## You can optionally specify a run mode if you want to use lbry defined presets for compatibility.
+case $RUN_MODE in
+  default )
+    set_config
+    download_snapshot
+    exec lbrycrdd -conf=$CONFIG_PATH
+    ;;
+  ## If it's a first run you need to do a full index including all transactions
+  ## tx index creates an index of every single transaction in the block history if
+  ## not specified it will only create an index for transactions that are related to the wallet or have unspent outputs.
+  ## This is generally specific to chainquery.
+  reindex )
+    ## Apply this RUN_MODE in the case you need to update a dataset.  NOTE: you do not need to use `RUN_MODE reindex` for more than one complete run.
+    set_config
+    exec lbrycrdd -conf=$CONFIG_PATH -reindex
+    ;;
+  regtest )
+    ## Set config params
+    ## TODO: Make this more automagic in the future.
+    mkdir -p `dirname $CONFIG_PATH`
+    echo "rpcuser=lbry" >           $CONFIG_PATH
+    echo "rpcpassword=lbry" >>      $CONFIG_PATH
+    echo "rpcport=29245" >>         $CONFIG_PATH
+    echo "rpcbind=0.0.0.0" >>       $CONFIG_PATH
+    echo "rpcallowip=0.0.0.0/0" >>  $CONFIG_PATH
+    echo "regtest=1" >>             $CONFIG_PATH
+    echo "txindex=1" >>             $CONFIG_PATH
+    echo "server=1" >>              $CONFIG_PATH
+    echo "printtoconsole=1" >>      $CONFIG_PATH
+
+    [ "${AUTO_ADVANCE:-0}" == "1" ] && nohup advance &>/dev/null &
+
+    exec lbrycrdd -conf=$CONFIG_PATH $1
+    ;;
+  testnet )
+    ## Set config params
+    ## TODO: Make this more automagic in the future.
+    mkdir -p `dirname $CONFIG_PATH`
+    echo "rpcuser=lbry" >           $CONFIG_PATH
+    echo "rpcpassword=lbry" >>      $CONFIG_PATH
+    echo "rpcport=29245" >>         $CONFIG_PATH
+    echo "rpcbind=0.0.0.0" >>       $CONFIG_PATH
+    echo "rpcallowip=0.0.0.0/0" >>  $CONFIG_PATH
+    echo "testnet=1" >>             $CONFIG_PATH
+    echo "txindex=1" >>             $CONFIG_PATH
+    echo "server=1" >>              $CONFIG_PATH
+    echo "printtoconsole=1" >>      $CONFIG_PATH
+
+    #nohup advance &>/dev/null &
+    exec lbrycrdd -conf=$CONFIG_PATH $1
+    ;;
+  * )
+    echo "Error, you must define a RUN_MODE environment variable."
+    echo "Available options are testnet, regtest, chainquery, default, and reindex"
+    ;;
+esac

src/claimtrie/trie.cpp

@@ -471,8 +471,10 @@ bool CClaimTrieCacheBase::checkConsistency()
         int takeoverHeight;
         row >> name >> hash >> takeoverHeight;
         auto computedHash = computeNodeHash(name, takeoverHeight);
-        if (computedHash != hash)
+        if (computedHash != hash) {
+            logPrint << "Invalid hash at " << name << Clog::endl;
             return false;
+        }
     }
     return true;
 }