lbrynet: Follow redirects in curl to github API

Follow redirects

chainquery/.dockerignore

@@ -1,2 +0,0 @@
-chainquery.z*
-data/*

chainquery/Dockerfile-linux-x86_64-production

@@ -3,21 +3,22 @@
 FROM ubuntu:18.04 as prep
 LABEL MAINTAINER="leopere [at] nixc [dot] us"
 RUN apt-get update && \
-  apt-get -y install unzip curl busybox-static && \
+  apt-get -y install unzip curl && \
   apt-get autoclean -y && \
   rm -rf /var/lib/apt/lists/*
 WORKDIR /
 SHELL ["/bin/bash", "-o", "pipefail", "-c"]
 COPY ./stuff/start.sh start
 COPY ./stuff/healthcheck.sh healthcheck
-RUN curl --progress-bar -L -o /chainquery https://build.lbry.io/chainquery/branch-master/chainquery && \
-    chmod +x /chainquery /start /healthcheck
+RUN curl -L -o /chainquery.zip $(curl -s https://api.github.com/repos/lbryio/chainquery/releases | grep -F 'Linux_x86_64.zip' | grep download | head -n 1 | cut -d'"' -f4) && \
+  unzip /chainquery.zip && \
+  rm /chainquery.zip
 
 FROM ubuntu:18.04 as app
 ADD https://raw.githubusercontent.com/lbryio/chainquery/master/config/default/chainqueryconfig.toml /etc/lbry/chainqueryconfig.toml.orig
 RUN adduser chainquery --gecos GECOS --shell /bin/bash --disabled-password --home /home/chainquery && \
   chown -R chainquery:chainquery /etc/lbry
-COPY --from=prep ./healthcheck /chainquery /start /bin/busybox /usr/bin/
+COPY --from=prep ./healthcheck /chainquery /start /usr/bin/
 HEALTHCHECK --interval=1m --timeout=30s \
   CMD healthcheck
 EXPOSE 6300

chainquery/compose/docker-compose.yml-prod-example

@@ -8,13 +8,16 @@ services:
 ###########
 ## MYSQL ##
 ###########
-## Work is underway to support percona
+## MariaDB is currently not supported and neither is later versions of MySQL this may change.
+## https://hub.docker.com/r/_/mariadb/
   mysql:
     image: mysql:5.7.23
     restart: always
     networks:
       lbry-network:
         ipv4_address: 10.6.1.10
+        aliases:
+          - mysql
     env_file:
       - ../environment/chainquery.env
     expose:
@@ -23,10 +26,6 @@ services:
     ## and not share it with everyone at least eventually.
     ports:
       - 3306:3306
-    healthcheck:
-      test: ["CMD-SHELL", 'mysql --database=$$MYSQL_DATABASE --password=$$MYSQL_ROOT_PASSWORD --execute="SELECT count(table_name) > 0 FROM information_schema.tables;" --skip-column-names -B']
-      interval: 5s
-      retries: 12
     volumes:
       - ../persist/chainquery/db:/var/lib/mysql
       ## This one may need to be tweaked based on where you run this docker-compose from.
@@ -36,16 +35,14 @@ services:
 ## Chainquery ##
 ################
   chainquery:
-    build:
-      context: .
-      dockerfile: Dockerfile-linux-x86_64-production
+    image: lbryio/lbry-docker:linux-x86_64-production
     restart: always
     networks:
       lbry-network:
         ipv4_address: 10.6.1.3
     env_file:
       - env
-      - ../environment/chainquery.env
+      - ../environment/lbrycrd.env
     labels:
       - "traefik.expose=false"
     expose:
@@ -55,5 +52,5 @@ services:
     depends_on:
       - mysql
     ## TODO: Uncomment this in a docker-compose.override.yml to allow for external configurations.
-    #volumes:
-      #  - ../persist/chainquery/config/chainqueryconfig.toml:/etc/chainquery/chainqueryconfig.toml
+    volumes:
+      - ../persist/chainquery/config/chainqueryconfig.toml:/etc/chainquery/chainqueryconfig.toml

chainquery/stuff/healthcheck.sh

@@ -1,2 +1,2 @@
-#!/bin/sh
-busybox wget -O- http://localhost:6300/api/status
+#!/usr/bin/env bash
+curl --fail http://localhost:6300/api/status || exit 1  

chainquery/stuff/start.sh

@@ -1,32 +1,51 @@
 #!/usr/bin/env bash
 
-# Override defaults with environment variables
-CONFIGFILE="${CONFIGFILE:-/etc/lbry/chainqueryconfig.toml}"
-DEBUGMODE="${DEBUGMODE:-false}"
-RPC_USER="${RPC_USER:-lbry}"
-RPC_PASSWORD="${RPC_PASSWORD:-lbry}"
-RPC_HOST="${RPC_HOST:-localhost}"
-RPC_PORT="${RPC_PORT:-9245}"
-MYSQL_SERVER="${MYSQL_SERVER:-localhost}"
-MYSQL_PORT="${MYSQL_PORT:-3306}"
-MYSQL_USER="${MYSQL_USER:-lbry}"
-MYSQL_PASSWORD="${MYSQL_PASSWORD:-lbry}"
-MYSQL_DATABASE="${MYSQL_DATABASE:-chainquery}"
+## Config setup
 
-exec_chainquery() {
-    CONFIG_DIR=$(dirname "${CONFIGFILE}")
-    exec chainquery serve --configpath "$CONFIG_DIR"
+## Setup Values
+DEBUGMODE=$(echo "debugmode=$DEBUGMODE")
+LBRYCRDURL=$(echo "lbrycrdurl=\"rpc://$RPC_USER:$RPC_PASSWORD@10.5.1.2:9245\"")
+MYSQLDSN=$(echo "mysqldsn=\"$MYSQL_USER:$MYSQL_PASSWORD@tcp($MYSQL_SERVER:3306)/$MYSQL_DATABASE\"")
+APIMYSQLDSN=$(echo "apimysqldsn=\"$MYSQL_USER:$MYSQL_PASSWORD@tcp($MYSQL_SERVER:3306)/$MYSQL_DATABASE\"")
+
+## Setup Defaults
+DEBUGMODE_DEFAULT='#DEFAULT-debugmode=false'
+LBRYCRDURL_DEFAULT='#DEFAULT-lbrycrdurl="rpc://lbry:lbry@localhost:9245"'
+MYSQLDSN_DEFAULT='#DEFAULT-mysqldsn="lbry:lbry@tcp(localhost:3306)/chainquery"'
+APIMYSQLDSN_DEFAULT='#DEFAULT-apimysqldsn="lbry:lbry@tcp(localhost:3306)/chainquery"'
+
+## Add setup value variable name to this list to get processed on container start
+CONFIG_SETTINGS=(
+  DEBUGMODE
+  LBRYCRDURL
+  MYSQLDSN
+  APIMYSQLDSN
+)
+
+function set_configs() {
+  ## Set configs on container start if not already set.
+  for i in "${!CONFIG_SETTINGS[@]}"; do
+    ## Indirect references http://tldp.org/LDP/abs/html/ivr.html
+    eval FROM_STRING=\$"${CONFIG_SETTINGS[$i]}_DEFAULT"
+    eval TO_STRING=\$${CONFIG_SETTINGS[$i]}
+    ## TODO: Add a bit more magic to make sure that you're only configuring things if not set by config mounts.
+    sed -i "s~$FROM_STRING~"$TO_STRING"~g" /etc/lbry/chainqueryconfig.toml
+  done
+  echo "Reading config for debugging."
+  cat /etc/lbry/chainqueryconfig.toml
 }
 
-if [[ -f "$CONFIGFILE" ]]; then
+if [[ ! -f /etc/lbry/chainqueryconfig.toml ]]; then
+  echo "[INFO]: Did not find chainqueryconfig.toml"
+  echo "        Installing default and configuring with provided environment variables if any."
+  ## Install fresh copy of config file.
+  echo "cp -v /etc/lbry/chainqueryconfig.toml.orig /etc/lbry/chainqueryconfig.toml"
+  cp -v /etc/lbry/chainqueryconfig.toml.orig /etc/lbry/chainqueryconfig.toml
+  ls -lAh /etc/lbry/
+  set_configs
+else
   echo "[INFO]: Found a copy of chainqueryconfig.toml in /etc/lbry"
-  exec_chainquery
 fi
 
-cat << EOF >> "${CONFIGFILE}"
-DEBUGMODE="${DEBUGMODE}"
-LBRYCRDURL="rpc://${RPC_USER}:${RPC_PASSWORD}@${RPC_HOST}:${RPC_PORT}"
-MYSQLDSN="${MYSQL_USER}:${MYSQL_PASSWORD}@tcp(${MYSQL_SERVER}:${MYSQL_PORT})/$MYSQL_DATABASE"
-APIMYSQLDSN="${MYSQL_USER}:${MYSQL_PASSWORD}@tcp(${MYSQL_SERVER}:${MYSQL_PORT})/$MYSQL_DATABASE"
-EOF
-exec_chainquery
+## For now keeping this simple. Potentially eventually add all command args as envvars for the Dockerfile or use safe way to add args via docker-compose.yml
+chainquery serve --configpath "/etc/lbry/"

environment/chainquery.env

@@ -3,14 +3,14 @@
 #########################
 ## Chainquery Settings ##
 #########################
-RPC_ALLOW_IP=10.6.1.3
+RPC_ALLOW_IP=10.5.1.3
 DEBUGMODE=false
 
 #################
 ## Mysql Creds ##
 #################
-MYSQL_SERVER=10.6.1.10
-MYSQL_USER=lbry
-MYSQL_PASSWORD=lbry
+MYSQL_SERVER=10.5.1.10
+MYSQL_USER=chainquery
+MYSQL_PASSWORD=changeme
 MYSQL_DATABASE=chainquery
-MYSQL_ROOT_PASSWORD=lbry
+MYSQL_ROOT_PASSWORD=changeme

environment/lbrycrd.env

@@ -10,4 +10,4 @@ GID=$UID
 
 ## This should be the internal container IP from which you'll be calling the RPC for Lbrycrdd from.
 ## TODO: make this more dynamic before we move to scalability
-RPC_ALLOW_IP=10.6.1.3
+RPC_ALLOW_IP=10.5.1.3

lbrycrd/Dockerfile-linux-x86_64-production

@@ -0,0 +1,33 @@
+FROM ubuntu:18.04 as prep
+LABEL MAINTAINER="leopere [at] nixc [dot] us"
+## TODO: Implement version pinning. `apt-get install curl=<version>`
+RUN apt-get update && \
+  apt-get -y install unzip curl build-essential && \
+  apt-get autoclean -y && \
+  rm -rf /var/lib/apt/lists/*
+WORKDIR /
+SHELL ["/bin/bash", "-o", "pipefail", "-c"]
+COPY ./stuff/start.sh start
+COPY ./stuff/healthcheck.sh healthcheck
+COPY ./stuff/advance_blocks.sh advance
+COPY ./stuff/fix-permissions.c fix-permissions.c
+RUN curl -L -o ./lbrycrd-linux.zip $(curl -s https://api.github.com/repos/lbryio/lbrycrd/releases | grep -F 'lbrycrd-linux.zip' | grep download | head -n 1 | cut -d'"' -f4) && \
+  unzip ./lbrycrd-linux.zip && \
+  gcc fix-permissions.c -o fix-permissions && \
+  chmod +x ./lbrycrdd ./lbrycrd-cli ./lbrycrd-tx ./start ./healthcheck ./fix-permissions ./advance
+
+FROM ubuntu:18.04 as app
+COPY --from=prep /lbrycrdd /lbrycrd-cli /lbrycrd-tx /start /healthcheck /fix-permissions /advance /usr/bin/
+RUN addgroup --gid 1000 lbrycrd && \
+  adduser lbrycrd --uid 1000 --gid 1000 --gecos GECOS --shell /bin/bash --disabled-password --home /data && \
+  mkdir /etc/lbry && \
+  chown lbrycrd /etc/lbry && \
+  chmod a+s /usr/bin/fix-permissions
+VOLUME ["/data"]
+WORKDIR /data
+## TODO: Implement healthcheck.
+# HEALTHCHECK ["healthcheck"]
+EXPOSE 9246 9245 11337 29245
+
+USER lbrycrd
+CMD ["start"]

lbrycrd/README.md

@@ -1 +1,48 @@
-# moved to https://github.com/lbryio/lbrycrd/tree/master/packaging/docker-for-binary
+# lbrycrd
+# Docker image tags
+`lbry/lbrycrd`
+`[linux-x86_64-production](https://github.com/lbryio/lbry-docker/blob/master/lbrycrd/Dockerfile-linux-x86_64-production)` (Latest release)
+
+
+## Configuration
+
+The lbrycrd container comes with a default configuration you can use for
+production. Extra configuration is optional.
+
+The container includes a `start` script that offers a flexible configuration
+style. It allows you to mount your own `lbrycrd.conf` file, or use environment
+variables, or a mix of both.
+
+### Environment variables
+
+The environment variables override the values in the mounted config file. If no
+mounted config file exists, these variables are used to create a fresh config.
+
+ * `PORT` - The main lbrycrd port
+ * `RPC_USER` - The rpc user
+ * `RPC_PASSWORD` - The rpc user's password
+ * `RPC_ALLOW_IP` - the subnet that is allowed rpc access
+ * `RPC_PORT` - The port to bind the rpc service to
+ * `RPC_BIND` - The ip address to bind the rpc service to
+ 
+
+### Example run commands
+
+Running the default configuration:
+
+```
+docker run --rm -it -e RUN_MODE=default lbry/lbrycrd:linux-x86_64-production
+```
+
+Running with RPC password changed:
+
+```
+docker run --rm -it -e RUN_MODE=default -e RPC_PASSWORD=hunter2 lbry/lbrycrd:linux-x86_64-production
+```
+
+Running with a config file but with the RPC password still overridden:
+
+```
+docker run --rm -it -v /path/to/lbrycrd.conf:/etc/lbry/lbrycrd.conf -e RUN_MODE=default -e RPC_PASSWORD=hunter2 lbry/lbrycrd:linux-x86_64-production
+```
+

lbrycrd/compose/docker-compose.yml-prod-example

@@ -0,0 +1,26 @@
+version: '3.4'
+
+networks:
+  lbry-network:
+    external: true
+
+services:
+#############
+## Lbrycrd ##
+#############
+  lbrycrd:
+    build: .
+    restart: always
+    networks:
+      lbry-network:
+        ipv4_address: 10.6.1.2
+    environment:
+      RUN_MODE: default
+    env_file:
+     - env
+    expose:
+      - 9245
+      - 9246
+    ## host volumes for persistent data such as wallet private keys.
+    volumes:
+      - "../persist/data:/data"

lbrycrd/compose/docker-compose.yml-regtest

@@ -0,0 +1,17 @@
+version: '3.4'
+
+services:
+#############
+## Lbrycrd ##
+#############
+  lbrycrd:
+    image: lbry/lbrycrd:linux-x86_64-production
+    restart: always
+    ports:
+      - "11336:29246"
+      - "11337:29245"
+    ## host volumes for persistent data such as wallet private keys.
+    volumes:
+      - "../persist/data:/data"
+    environment:
+      - RUN_MODE=regtest

lbrycrd/compose/docker-compose.yml-testnet

@@ -0,0 +1,17 @@
+version: '3.4'
+
+services:
+#############
+## Lbrycrd ##
+#############
+  lbrycrd:
+    image: lbry/lbrycrd:linux-x86_64-production
+    restart: always
+    ports:
+      - "11336:9246"
+      - "11337:11337"
+    ## host volumes for persistent data such as wallet private keys.
+    volumes:
+      - "../persist/data:/data"
+    environment:
+      - RUN_MODE=testnet

lbrycrd/stuff/advance_blocks.sh

@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+while true; do
+        lbrycrd-cli -conf=/data/.lbrycrd/lbrycrd.conf generate 1 >> /tmp/output.log
+        sleep 2
+done

lbrycrd/stuff/env-example

@@ -0,0 +1,12 @@
+COMPOSE_PROJECT_NAME=lbrycrd
+
+#############
+## Lbrycrd ##
+#############
+## TODO: The credentials are a formality but we should randomize these with magic.
+RPC_USER=${RPC_USER=lbryrpc}
+RPC_PASSWORD=${RPC_PASSWORD:-changeme}
+
+## This should be the internal container IP from which you'll be calling the RPC for Lbrycrdd from.
+## TODO: make this more dynamic before we move to scalability
+RPC_ALLOW_IP=${RPC_ALLOW_IP:-10.6.1.3}

lbrycrd/stuff/fix-permissions.c

@@ -0,0 +1,9 @@
+#include <unistd.h>
+int main() {
+  // This program needs to run with setuid == root
+  // This needs to be in a compiled language because you cannot setuid bash scripts
+  setuid(0);
+  execle("/bin/bash", "bash", "-c",
+         "/bin/chown -R lbrycrd:lbrycrd /data && /bin/chmod -R 755 /data/",
+         (char*) NULL, (char*) NULL);
+}

lbrycrd/stuff/healthcheck.sh

@@ -0,0 +1,4 @@
+## TODO: Implement a healthcheck for lbrycrd.
+curl --data-binary '{"jsonrpc":"1.0","id":"curltext","method":"getinfo","params":[]}' -H 'content-type:text/plain;'  http://$RPC_USER:$RPC_PASSWORD@127.0.0.1:9246
+## OR
+lbrycrd-cli getinfo

lbrycrd/stuff/start.sh

@@ -0,0 +1,114 @@
+#!/usr/bin/env bash
+CONFIG_PATH=/etc/lbry/lbrycrd.conf
+
+function override_config_option() {
+    # Remove existing config line from a config file
+    #  and replace with environment fed value.
+    # Does nothing if the variable does not exist.
+    #  var     Name of ENV variable
+    #  option  Name of config option
+    #  config  Path of config file
+    local var=$1 option=$2 config=$3
+    if [[ -v $var ]]; then
+        # Remove the existing config option:
+        sed -i "/^$option\W*=/d" $config
+        # Add the value from the environment:
+        echo "$option=${!var}" >> $config
+    fi
+}
+
+function set_config() {
+  if [ -d "$CONFIG_PATH" ]; then
+      echo "$CONFIG_PATH is a directory when it should be a file."
+      exit 1
+  elif [ -f "$CONFIG_PATH" ]; then
+      echo "Merging the mounted config file with environment variables."
+      local MERGED_CONFIG=/tmp/lbrycrd_merged.conf
+      cat $CONFIG_PATH > $MERGED_CONFIG
+      echo "" >> $MERGED_CONFIG
+      override_config_option PORT port $MERGED_CONFIG
+      override_config_option RPC_USER rpcuser $MERGED_CONFIG
+      override_config_option RPC_PASSWORD rpcpassword $MERGED_CONFIG
+      override_config_option RPC_ALLOW_IP rpcallowip $MERGED_CONFIG
+      override_config_option RPC_PORT rpcport $MERGED_CONFIG
+      override_config_option RPC_BIND rpcbind $MERGED_CONFIG
+      # Make the new merged config file the new CONFIG_PATH
+      # This ensures that the original file the user mounted remains unmodified
+      CONFIG_PATH=$MERGED_CONFIG
+  else
+      echo "Creating a fresh config file from environment variables."
+      ## Set config params
+      echo "port=${PORT=9246}" > $CONFIG_PATH
+      echo "rpcuser=${RPC_USER=lbry}" >> $CONFIG_PATH
+      echo "rpcpassword=${RPC_PASSWORD=lbry}" >> $CONFIG_PATH
+      echo "rpcallowip=${RPC_ALLOW_IP=127.0.0.1/24}" >> $CONFIG_PATH
+      echo "rpcport=${RPC_PORT=9245}" >> $CONFIG_PATH
+      echo "rpcbind=${RPC_BIND=0.0.0.0}" >> $CONFIG_PATH
+  fi
+  echo "Config: "
+  cat $CONFIG_PATH
+}
+
+## Ensure perms are correct prior to running main binary
+/usr/bin/fix-permissions
+
+## You can optionally specify a run mode if you want to use lbry defined presets for compatibility.
+case $RUN_MODE in
+  default )
+    set_config
+    lbrycrdd -server -conf=$CONFIG_PATH -printtoconsole
+    ;;
+  ## If it's a first run you need to do a full index including all transactions
+  ## tx index creates an index of every single transaction in the block history if
+  ## not specified it will only create an index for transactions that are related to the wallet or have unspent outputs.
+  ## This is generally specific to chainquery.
+  reindex )
+    ## Apply this RUN_MODE in the case you need to update a dataset.  NOTE: you do not need to use `RUN_MODE reindex` for more than one complete run.
+    set_config
+    lbrycrdd -server -txindex -reindex -conf=$CONFIG_PATH -printtoconsole
+    ;;
+  chainquery )
+    ## If your only goal is to run Chainquery against this instance of lbrycrd and you're starting a
+    ## fresh local dataset use this run mode.
+    set_config
+    lbrycrdd -server -txindex -conf=$CONFIG_PATH -printtoconsole
+    ;;
+  regtest )
+    ## Set config params
+    ## TODO: Make this more automagic in the future.
+    mkdir -p `dirname $CONFIG_PATH`
+    echo "rpcuser=lbry" >           $CONFIG_PATH
+    echo "rpcpassword=lbry" >>      $CONFIG_PATH
+    echo "rpcport=29245" >>         $CONFIG_PATH
+    echo "rpcbind=0.0.0.0" >>       $CONFIG_PATH
+    echo "rpcallowip=0.0.0.0/0" >>  $CONFIG_PATH
+    echo "regtest=1" >>             $CONFIG_PATH
+    echo "txindex=1" >>             $CONFIG_PATH
+    echo "server=1" >>              $CONFIG_PATH
+    echo "printtoconsole=1" >>      $CONFIG_PATH
+
+    #nohup advance &>/dev/null &
+    lbrycrdd -conf=$CONFIG_PATH $1
+    ;;
+  testnet )
+    ## Set config params
+    ## TODO: Make this more automagic in the future.
+    mkdir -p `dirname $CONFIG_PATH`
+    echo "rpcuser=lbry" >           $CONFIG_PATH
+    echo "rpcpassword=lbry" >>      $CONFIG_PATH
+    echo "rpcport=29245" >>         $CONFIG_PATH
+    echo "rpcbind=0.0.0.0" >>       $CONFIG_PATH
+    echo "rpcallowip=0.0.0.0/0" >>  $CONFIG_PATH
+    echo "testnet=1" >>             $CONFIG_PATH
+    echo "txindex=1" >>             $CONFIG_PATH
+    echo "server=1" >>              $CONFIG_PATH
+    echo "printtoconsole=1" >>      $CONFIG_PATH
+
+    #nohup advance &>/dev/null &
+    lbrycrdd -conf=$CONFIG_PATH $1
+    ;;
+  * )
+    echo "Error, you must define a RUN_MODE environment variable."
+    echo "Available options are testnet, regtest, chainquery, default, and reindex"
+    ;;
+esac