Added e2e test docker-compose and supporting work so that ytsync can run independently of the mainnet for testing/CI

.gitignore

@@ -1 +1,2 @@
 bin/
+e2e/persist

e2e/daemon_settings.yml

@@ -0,0 +1,12 @@
+#blockchain_name: lbrycrd_main
+#blockchain_name: lbrycrd_testnet
+blockchain_name: lbrycrd_regtest
+lbryum_servers:
+#  - spv1.lbry.com:50001 #Production Wallet Server
+  - walletserver:50001
+save_blobs: true
+save_files: false
+share_usage_data: false
+tcp_port: 3333
+udp_port: 4444
+use_upnp: true

e2e/docker-compose.yml

@@ -0,0 +1,53 @@
+version: "3"
+
+services:
+  #############
+  ## Lbrycrd ##
+  #############
+  lbrycrd:
+    image: lbry/lbrycrd:v0.12.4.1
+    restart: always
+    ports:
+      - "15201:29246"
+      - "15200:29245"
+    ## host volumes for persistent data such as wallet private keys.
+    volumes:
+      - "./persist:/data"
+    environment:
+      - RUN_MODE=regtest
+  ###################
+  ## Wallet Server ##
+  ###################
+  walletserver:
+    image: lbry/wallet-server:v0.38.5
+    restart: always
+    volumes:
+      - "./persist/.walletserver/database:/database"
+    environment:
+      - DB_DIRECTORY=/database
+      - MAX_SEND=1000000000000000000000
+      - DAEMON_URL=http://lbry:lbry@lbrycrd:29245/
+      - MAX_SUBS=1000000000000
+      - BANDWIDTH_LIMIT=80000000000
+      - SESSION_TIMEOUT=10000000000000000000000000
+      - TCP_PORT=50001
+    ports:
+      - "15300:50001"
+    ulimits:
+      nofile: 90000
+    #    command: lbry.wallet.server.coin.LBC
+    command: lbry.wallet.server.coin.LBCRegTest
+  #############
+  ## Lbrynet ##
+  #############
+  lbrynet:
+    image: lbry/lbrynet:v0.38.5
+    restart: "no"
+    ports:
+      - "15100:5279"
+      - "15101:5280"
+    environment:
+      - LBRY_STREAMING_SERVER=0.0.0.0:5280
+    volumes:
+      - "./persist/.lbrynet:/home/lbrynet"
+      - ".:/etc/lbry" #Put your daemon_settings.yml here

e2e/lbrycrd/docker-compose.yml

@@ -0,0 +1,26 @@
+version: "3"
+networks:
+  lbry-network:
+    external: true
+
+services:
+  #############
+  ## Lbrycrd ##
+  #############
+  lbrycrd:
+    image: lbry/lbrycrd:v0.12.4.1
+    restart: always
+    networks:
+      lbry-network:
+        ipv4_address: 10.6.1.1
+    ports:
+      - "15201:29246"
+      - "15200:29245"
+    expose:
+      - "29246"
+      - "29245"
+    ## host volumes for persistent data such as wallet private keys.
+    volumes:
+      - "../persist/data:/data"
+    environment:
+      - RUN_MODE=regtest

e2e/lbrycrd/docker/Dockerfile

@@ -0,0 +1,37 @@
+FROM ubuntu:18.04 as prep
+LABEL MAINTAINER="leopere [at] nixc [dot] us"
+## TODO: Implement version pinning. `apt-get install curl=<version>`
+RUN apt-get update && \
+  apt-get -y install unzip curl build-essential && \
+  apt-get autoclean -y && \
+  rm -rf /var/lib/apt/lists/*
+WORKDIR /
+SHELL ["/bin/bash", "-o", "pipefail", "-c"]
+COPY ./start.sh start
+COPY ./healthcheck.sh healthcheck
+COPY ./advance_blocks.sh advance
+COPY ./fix-permissions.c fix-permissions.c
+
+## Add lbrycrd - Change the version below to create an image for a different tag/version
+ARG VERSION="v0.12.4.1"
+RUN URL=$(curl -s https://api.github.com/repos/lbryio/lbrycrd/releases/$(if [ "${VERSION}" = 'latest' ]; then echo "latest"; else echo "tags/${VERSION}"; fi) | grep browser_download_url | grep lbrycrd-linux.zip | cut -d'"' -f4) && echo $URL && curl -L -o /lbrycrd-linux.zip $URL
+
+RUN unzip ./lbrycrd-linux.zip && \
+  gcc fix-permissions.c -o fix-permissions && \
+  chmod +x ./lbrycrdd ./lbrycrd-cli ./lbrycrd-tx ./start ./healthcheck ./fix-permissions ./advance
+
+FROM ubuntu:18.04 as app
+COPY --from=prep /lbrycrdd /lbrycrd-cli /lbrycrd-tx /start /healthcheck /fix-permissions /advance /usr/bin/
+RUN addgroup --gid 1000 lbrycrd && \
+  adduser lbrycrd --uid 1000 --gid 1000 --gecos GECOS --shell /bin/bash --disabled-password --home /data && \
+  mkdir /etc/lbry && \
+  chown lbrycrd /etc/lbry && \
+  chmod a+s /usr/bin/fix-permissions
+VOLUME ["/data"]
+WORKDIR /data
+## TODO: Implement healthcheck.
+# HEALTHCHECK ["healthcheck"]
+EXPOSE 9246 9245 11337 29245
+
+USER lbrycrd
+CMD ["start"]

e2e/lbrycrd/docker/advance_blocks.sh

@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+while true; do
+        lbrycrd-cli -conf=/data/.lbrycrd/lbrycrd.conf generate 1 >> /tmp/output.log
+        sleep 2
+done

e2e/lbrycrd/docker/fix-permissions.c

@@ -0,0 +1,9 @@
+#include <unistd.h>
+int main() {
+  // This program needs to run with setuid == root
+  // This needs to be in a compiled language because you cannot setuid bash scripts
+  setuid(0);
+  execle("/bin/bash", "bash", "-c",
+         "/bin/chown -R lbrycrd:lbrycrd /data && /bin/chmod -R 755 /data/",
+         (char*) NULL, (char*) NULL);
+}

e2e/lbrycrd/docker/healthcheck.sh

@@ -0,0 +1,4 @@
+## TODO: Implement a healthcheck for lbrycrd.
+curl --data-binary '{"jsonrpc":"1.0","id":"curltext","method":"getinfo","params":[]}' -H 'content-type:text/plain;'  http://$RPC_USER:$RPC_PASSWORD@127.0.0.1:9246
+## OR
+lbrycrd-cli getinfo

e2e/lbrycrd/docker/start.sh

@@ -0,0 +1,114 @@
+#!/usr/bin/env bash
+CONFIG_PATH=/etc/lbry/lbrycrd.conf
+
+function override_config_option() {
+    # Remove existing config line from a config file
+    #  and replace with environment fed value.
+    # Does nothing if the variable does not exist.
+    #  var     Name of ENV variable
+    #  option  Name of config option
+    #  config  Path of config file
+    local var=$1 option=$2 config=$3
+    if [[ -v $var ]]; then
+        # Remove the existing config option:
+        sed -i "/^$option\W*=/d" $config
+        # Add the value from the environment:
+        echo "$option=${!var}" >> $config
+    fi
+}
+
+function set_config() {
+  if [ -d "$CONFIG_PATH" ]; then
+      echo "$CONFIG_PATH is a directory when it should be a file."
+      exit 1
+  elif [ -f "$CONFIG_PATH" ]; then
+      echo "Merging the mounted config file with environment variables."
+      local MERGED_CONFIG=/tmp/lbrycrd_merged.conf
+      cat $CONFIG_PATH > $MERGED_CONFIG
+      echo "" >> $MERGED_CONFIG
+      override_config_option PORT port $MERGED_CONFIG
+      override_config_option RPC_USER rpcuser $MERGED_CONFIG
+      override_config_option RPC_PASSWORD rpcpassword $MERGED_CONFIG
+      override_config_option RPC_ALLOW_IP rpcallowip $MERGED_CONFIG
+      override_config_option RPC_PORT rpcport $MERGED_CONFIG
+      override_config_option RPC_BIND rpcbind $MERGED_CONFIG
+      # Make the new merged config file the new CONFIG_PATH
+      # This ensures that the original file the user mounted remains unmodified
+      CONFIG_PATH=$MERGED_CONFIG
+  else
+      echo "Creating a fresh config file from environment variables."
+      ## Set config params
+      echo "port=${PORT=9246}" > $CONFIG_PATH
+      echo "rpcuser=${RPC_USER=lbry}" >> $CONFIG_PATH
+      echo "rpcpassword=${RPC_PASSWORD=lbry}" >> $CONFIG_PATH
+      echo "rpcallowip=${RPC_ALLOW_IP=127.0.0.1/24}" >> $CONFIG_PATH
+      echo "rpcport=${RPC_PORT=9245}" >> $CONFIG_PATH
+      echo "rpcbind=${RPC_BIND=0.0.0.0}" >> $CONFIG_PATH
+  fi
+  echo "Config: "
+  cat $CONFIG_PATH
+}
+
+## Ensure perms are correct prior to running main binary
+/usr/bin/fix-permissions
+
+## You can optionally specify a run mode if you want to use lbry defined presets for compatibility.
+case $RUN_MODE in
+  default )
+    set_config
+    lbrycrdd -server -conf=$CONFIG_PATH -printtoconsole
+    ;;
+  ## If it's a first run you need to do a full index including all transactions
+  ## tx index creates an index of every single transaction in the block history if
+  ## not specified it will only create an index for transactions that are related to the wallet or have unspent outputs.
+  ## This is generally specific to chainquery.
+  reindex )
+    ## Apply this RUN_MODE in the case you need to update a dataset.  NOTE: you do not need to use `RUN_MODE reindex` for more than one complete run.
+    set_config
+    lbrycrdd -server -txindex -reindex -conf=$CONFIG_PATH -printtoconsole
+    ;;
+  chainquery )
+    ## If your only goal is to run Chainquery against this instance of lbrycrd and you're starting a
+    ## fresh local dataset use this run mode.
+    set_config
+    lbrycrdd -server -txindex -conf=$CONFIG_PATH -printtoconsole
+    ;;
+  regtest )
+    ## Set config params
+    ## TODO: Make this more automagic in the future.
+    mkdir -p `dirname $CONFIG_PATH`
+    echo "rpcuser=lbry" >           $CONFIG_PATH
+    echo "rpcpassword=lbry" >>      $CONFIG_PATH
+    echo "rpcport=29245" >>         $CONFIG_PATH
+    echo "rpcbind=0.0.0.0" >>       $CONFIG_PATH
+    echo "rpcallowip=0.0.0.0/0" >>  $CONFIG_PATH
+    echo "regtest=1" >>             $CONFIG_PATH
+    echo "txindex=1" >>             $CONFIG_PATH
+    echo "server=1" >>              $CONFIG_PATH
+    echo "printtoconsole=1" >>      $CONFIG_PATH
+
+    #nohup advance &>/dev/null &
+    lbrycrdd -conf=$CONFIG_PATH $1
+    ;;
+  testnet )
+    ## Set config params
+    ## TODO: Make this more automagic in the future.
+    mkdir -p `dirname $CONFIG_PATH`
+    echo "rpcuser=lbry" >           $CONFIG_PATH
+    echo "rpcpassword=lbry" >>      $CONFIG_PATH
+    echo "rpcport=29245" >>         $CONFIG_PATH
+    echo "rpcbind=0.0.0.0" >>       $CONFIG_PATH
+    echo "rpcallowip=0.0.0.0/0" >>  $CONFIG_PATH
+    echo "testnet=1" >>             $CONFIG_PATH
+    echo "txindex=1" >>             $CONFIG_PATH
+    echo "server=1" >>              $CONFIG_PATH
+    echo "printtoconsole=1" >>      $CONFIG_PATH
+
+    #nohup advance &>/dev/null &
+    lbrycrdd -conf=$CONFIG_PATH $1
+    ;;
+  * )
+    echo "Error, you must define a RUN_MODE environment variable."
+    echo "Available options are testnet, regtest, chainquery, default, and reindex"
+    ;;
+esac

e2e/lbrynet/docker-compose.yml

@@ -0,0 +1,23 @@
+version: "3"
+networks:
+  lbry-network:
+    external: true
+
+services:
+  #############
+  ## Lbrynet ##
+  #############
+  lbrynet:
+    image: lbry/lbrynet:v0.38.5
+    restart: "no"
+    networks:
+      lbry-network:
+        ipv4_address: 10.6.1.3
+    ports:
+      - "15100:5279"
+      - "15101:5280"
+    environment:
+      - LBRY_STREAMING_SERVER=0.0.0.0:5280
+    volumes:
+      - "../persist/data/.lbrynet:/home/lbrynet"
+      - "./settings:/etc/lbry" #Put your daemon_settings.yml here

e2e/lbrynet/docker/Dockerfile

@@ -0,0 +1,27 @@
+## This base image is for running the latest lbrynet-daemon release.
+FROM ubuntu:18.04 as prep
+LABEL MAINTAINER="leopere [at] nixc [dot] us"
+RUN apt-get update && apt-get -y install unzip curl telnet
+
+## Add lbrynet
+ARG VERSION="v0.38.5"
+RUN URL=$(curl -s https://api.github.com/repos/lbryio/lbry-sdk/releases/$(if [ "${VERSION}" = 'latest' ]; then echo "latest"; else echo "tags/${VERSION}"; fi) | grep browser_download_url | grep lbrynet-linux.zip | cut -d'"' -f4) && echo $URL && curl -L -o /lbrynet.linux.zip $URL
+
+COPY start.sh /usr/bin/start
+COPY checkmount.sh /usr/bin/checkmount
+RUN unzip /lbrynet.linux.zip -d /lbrynet/ && \
+    mv /lbrynet/lbrynet /usr/bin && \
+    chmod a+x /usr/bin/checkmount /usr/bin/start /usr/bin/lbrynet
+
+FROM ubuntu:18.04 as app
+COPY --from=prep /usr/bin/start /usr/bin/checkmount /usr/bin/lbrynet /usr/bin/
+RUN adduser lbrynet --gecos GECOS --shell /bin/bash --disabled-password --home /home/lbrynet
+## Daemon port [Intended for internal use]
+## LBRYNET talks to peers on port 3333 [Intended for external use] this port is used to discover other lbrynet daemons with blobs.
+## Expose 5566 Reflector port to listen on
+## Expose 5279 Port the daemon API will listen on
+## the lbryumx aka Wallet port [Intended for internal use]
+#EXPOSE 4444 3333 5566 5279 50001
+USER lbrynet
+ENTRYPOINT ["/usr/bin/checkmount"]
+CMD ["start"]

e2e/lbrynet/docker/checkmount.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+
+## TODO: Make a bit more aware of the run mode of this appliance in case there is ever a test mode enabled in the start.sh
+mountpoint=/home/lbrynet
+
+if ! grep -qs ".* $mountpoint " /proc/mounts; then
+    echo "$mountpoint not mounted, refusing to run."
+    ## TODO: We should have documentation that this error references directly with a URL as to why it won't run without a volume.
+    exit 1
+else
+    bash -c "$*"
+fi

e2e/lbrynet/docker/start.sh

@@ -0,0 +1,4 @@
+#!/bin/bash
+lbrynet start \
+  --api="${API_BIND_IP:-0.0.0.0}":"${API_PORT:-5279}" \
+  --config="${CONFIG_PATH:-/etc/lbry/daemon_settings.yml}"

e2e/lbrynet/settings/daemon_settings.yml

@@ -0,0 +1,12 @@
+#blockchain_name: lbrycrd_main
+#blockchain_name: lbrycrd_testnet
+blockchain_name: lbrycrd_regtest
+lbryum_servers:
+#  - spv1.lbry.com:50001 #Production Wallet Server
+  - walletserver:50001
+save_blobs: true
+save_files: false
+share_usage_data: false
+tcp_port: 3333
+udp_port: 4444
+use_upnp: true

e2e/walletserver/docker-compose.yml

@@ -0,0 +1,35 @@
+version: "3"
+networks:
+  lbry-network:
+    external: true
+
+services:
+  ###################
+  ## Wallet Server ##
+  ###################
+  walletserver:
+    image: lbry/wallet-server:v0.38.5
+    restart: always
+    networks:
+      lbry-network:
+        ipv4_address: 10.6.1.2
+    volumes:
+      - "../persist/data/.walletserver/database:/database"
+    environment:
+      - DB_DIRECTORY=/database
+      - MAX_SEND=1000000000000000000000
+      - DAEMON_URL=http://lbry:lbry@lbrycrd:29245/
+      - MAX_SUBS=1000000000000
+      - BANDWIDTH_LIMIT=80000000000
+      - SESSION_TIMEOUT=10000000000000000000000000
+      - TCP_PORT=50001
+    #network_mode: host
+    #network_mode: bridge
+    ports:
+      - "50001:50001"
+    expose:
+      - "50001"
+    ulimits:
+      nofile: 90000
+#    command: lbry.wallet.server.coin.LBC
+    command: lbry.wallet.server.coin.LBCRegTest