Add chainquery to youtube sync repo

e2e/chainquery/docker-compose.yml

@@ -0,0 +1,28 @@
+version: '3.4'
+
+services:
+  ###########
+  ## MYSQL ##
+  ###########
+  mysql:
+    image: mysql:5.7.23
+    restart: "no"
+    ports:
+      - 3306:3306
+    volumes:
+      - "../persist/chainquery/db:/var/lib/mysql"
+      ## This one may need to be tweaked based on where you run this docker-compose from.
+      - "../stuff/my.cnf:/etc/mysql/conf.d/chainquery-optimizations.cnf"
+  ################
+  ## Chainquery ##
+  ################
+  chainquery:
+    image: lbry/chainquery:v1.8.1
+    restart: "no"
+    ports:
+      - 6300:6300
+    depends_on:
+      - mysql
+    ## TODO: Uncomment this in a docker-compose.override.yml to allow for external configurations.
+    volumes:
+      - "../persist/chainquery/config/chainqueryconfig.toml:/etc/chainquery/chainqueryconfig.toml"

e2e/chainquery/docker/Dockerfile

@@ -0,0 +1,29 @@
+## Get the latest source and extract it for the app container.
+## Design choices, two RUN layers intended to keep builds faster, the zipped
+FROM ubuntu:18.04 as prep
+LABEL MAINTAINER="leopere [at] nixc [dot] us"
+RUN apt-get update && \
+  apt-get -y install unzip curl && \
+  apt-get autoclean -y && \
+  rm -rf /var/lib/apt/lists/*
+WORKDIR /
+SHELL ["/bin/bash", "-o", "pipefail", "-c"]
+COPY ./start.sh start
+COPY ./healthcheck.sh healthcheck
+ARG VERSION="master"
+RUN curl -s -o /chainquery http://build.lbry.io/chainquery/branch-"${VERSION}"/chainquery && \
+    chmod +x /chainquery
+
+
+FROM ubuntu:18.04 as app
+ARG VERSION="master"
+ADD https://raw.githubusercontent.com/lbryio/chainquery/"${VERSION}"/config/default/chainqueryconfig.toml /etc/lbry/chainqueryconfig.toml.orig
+RUN adduser chainquery --gecos GECOS --shell /bin/bash --disabled-password --home /home/chainquery && \
+  chown -R chainquery:chainquery /etc/lbry
+COPY --from=prep ./healthcheck /chainquery /start /usr/bin/
+HEALTHCHECK --interval=1m --timeout=30s \
+  CMD healthcheck
+EXPOSE 6300
+USER chainquery
+STOPSIGNAL SIGINT
+CMD ["start"]

e2e/chainquery/docker/build.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+if [ $# -eq 0 ]
+  then
+    echo "No docker tag argument supplied. Use './build.sh <tag>'"
+    exit 1
+fi
+docker build --build-arg VERSION=$1 --tag lbry/chainquery:$1 .
+docker push lbry/chainquery:$1

e2e/chainquery/docker/healthcheck.sh

@@ -0,0 +1,2 @@
+#!/usr/bin/env bash
+curl --fail http://localhost:6300/api/status || exit 1

e2e/chainquery/docker/my.cnf

@@ -0,0 +1,9 @@
+# Default Homebrew MySQL server config
+[mysqld]
+# Only allow connections from localhost
+innodb_log_file_size=5G
+key_buffer_size=1G
+innodb_flush_log_at_trx_commit = 0
+innodb_autoinc_lock_mode=2
+innodb_buffer_pool_size=1G
+innodb_log_buffer_size=1G

e2e/chainquery/docker/start.sh

@@ -0,0 +1,51 @@
+#!/usr/bin/env bash
+
+## Config setup
+
+## Setup Values
+DEBUGMODE=$(echo "debugmode=$DEBUGMODE")
+LBRYCRDURL=$(echo "lbrycrdurl=\"rpc://$RPC_USER:$RPC_PASSWORD@10.5.1.2:9245\"")
+MYSQLDSN=$(echo "mysqldsn=\"$MYSQL_USER:$MYSQL_PASSWORD@tcp($MYSQL_SERVER:3306)/$MYSQL_DATABASE\"")
+APIMYSQLDSN=$(echo "apimysqldsn=\"$MYSQL_USER:$MYSQL_PASSWORD@tcp($MYSQL_SERVER:3306)/$MYSQL_DATABASE\"")
+
+## Setup Defaults
+DEBUGMODE_DEFAULT='#DEFAULT-debugmode=false'
+LBRYCRDURL_DEFAULT='#DEFAULT-lbrycrdurl="rpc://lbry:lbry@localhost:9245"'
+MYSQLDSN_DEFAULT='#DEFAULT-mysqldsn="lbry:lbry@tcp(localhost:3306)/chainquery"'
+APIMYSQLDSN_DEFAULT='#DEFAULT-apimysqldsn="lbry:lbry@tcp(localhost:3306)/chainquery"'
+
+## Add setup value variable name to this list to get processed on container start
+CONFIG_SETTINGS=(
+  DEBUGMODE
+  LBRYCRDURL
+  MYSQLDSN
+  APIMYSQLDSN
+)
+
+function set_configs() {
+  ## Set configs on container start if not already set.
+  for i in "${!CONFIG_SETTINGS[@]}"; do
+    ## Indirect references http://tldp.org/LDP/abs/html/ivr.html
+    eval FROM_STRING=\$"${CONFIG_SETTINGS[$i]}_DEFAULT"
+    eval TO_STRING=\$${CONFIG_SETTINGS[$i]}
+    ## TODO: Add a bit more magic to make sure that you're only configuring things if not set by config mounts.
+    sed -i "s~$FROM_STRING~"$TO_STRING"~g" /etc/lbry/chainqueryconfig.toml
+  done
+  echo "Reading config for debugging."
+  cat /etc/lbry/chainqueryconfig.toml
+}
+
+if [[ ! -f /etc/lbry/chainqueryconfig.toml ]]; then
+  echo "[INFO]: Did not find chainqueryconfig.toml"
+  echo "        Installing default and configuring with provided environment variables if any."
+  ## Install fresh copy of config file.
+  echo "cp -v /etc/lbry/chainqueryconfig.toml.orig /etc/lbry/chainqueryconfig.toml"
+  cp -v /etc/lbry/chainqueryconfig.toml.orig /etc/lbry/chainqueryconfig.toml
+  ls -lAh /etc/lbry/
+  set_configs
+else
+  echo "[INFO]: Found a copy of chainqueryconfig.toml in /etc/lbry"
+fi
+
+## For now keeping this simple. Potentially eventually add all command args as envvars for the Dockerfile or use safe way to add args via docker-compose.yml
+chainquery serve --configpath "/etc/lbry/"

e2e/chainqueryconfig.toml

@@ -0,0 +1,99 @@
+#Debug mode outputs specific information to the console
+#DEFAULT: false
+#debugmode=
+
+#DebugQueryMode outputs SQL Boiler queries to the console.
+#DEFAULT: false
+#debugquerymode=
+
+#LBRYcrd URL is required for chainquery to query the blockchain
+#DEFAULT: "rpc://lbry:lbry@localhost:9245"
+lbrycrdurl="rpc://lbry:lbry@lbrycrd:29245"
+
+#MySQL DSN is required for chainquery to store information.
+#DEFAULT: "lbry:lbry@tcp(localhost:3306)/chainquery"
+#SUGGESTED: "lbry:lbry@unix(/var/run/mysqld/mysqld.sock)/chainquery"
+mysqldsn="lbry:lbry@tcp(mysqlCQ:3306)/chainquery"
+
+#API MySQL DSN is required for chainquery to expose a SQL query service
+#DEFAULT: "lbry:lbry@tcp(localhost:3306)/chainquery"
+#SUGGESTED: "lbry:lbry@unix(/var/run/mysqld/mysqld.sock)/chainquery"
+#apimysqldsn=
+
+#API Host and Port is required for the API Server to bind and listen on.
+#DEFAULT: "0.0.0.0:6300"
+#apihostport=
+
+#Profile mode enables and disables the reporting of a profile for chainquery
+#DEFAULT: false
+#profilemode=
+
+#Daemon mode tells chainquery how hard it should work catch up processing the blockchain
+#deamonmode=0 #BeastMode it continuously process block after block until caughtup.
+#daemonmode=1 #SlowAndSteadyMode it will process block with a frequency of 1 block every 100ms
+#daemonmode=2 #DelayMode it will process a block with a configured delay frequency (set via 'processingdelay')
+#daemonmode=3 #DaemonMode it will process a block every iteration of the daemon.
+#DEFAULT: 0
+#deamonmode=
+
+#Default client timeout is for communication with the api of chainquery
+#DEFAULT: 20 #Measured in seconds
+#defaultclienttimeout=
+
+#Processing delay is used to determine how frequently chainquery should process a block
+# It is only used if Daemon mode is set to delay mode
+#DEFAULT: 100 #Measured in milliseconds
+#processingdelay=
+
+#Daemon delay is the frequency at which chainquery checks for work to do.
+#DEFAULT: 1 #Measured in seconds
+#daemondelay=
+
+#Profiling options - will output the time take for certain opertions related to the below category
+#DEFAULT: false (for all 3 params)
+#daemonprofile=
+#lbrycrdprofile=
+#mysqlprofile=
+
+#Slack Hook URL allows slack integration. All logging info level and above is posted to a slack channel.
+#DEFAULT: ""
+#slackhookurl=
+
+#Slack Channel is the channel that you want the messages to appear. Works together with the hook url.
+#DEFAULT: ""
+#slackchannel=
+
+#Slack Log Level tells chainquery what level of logging will be sent to the slack channel. It will log all levels below
+# it as well. Panic=0,Fatal=1,Error=2,Warning=3,Info=4,Debug=5
+#DEFAULT: 0
+#slackloglevel=
+
+#The command that should be executed to trigger a self update of the software. For linux, for example, `<yourscript>.sh`
+#DEFAULT: ""
+#autoupdatecommand=
+
+#Twilio service of chainquery to send specifically important information to key users of the Chainquery install.
+#DEFAULT:
+##twiliosid=""
+##twilioauthtoken=""
+##smsrecipients=["",""]
+##smsfromphonenumber=""
+#twiliosid=
+#twilioauthtoken=
+#smsrecipients=
+#smsfromphonenumber=
+
+#API Keys - Disallowed by default unless keys are entered.
+#DEFAULT: []
+#apikeys=
+
+#Max Failures - Specifies the  number of failures that can happen in processing a transaction. This is for parallel
+#transaction processing which puts a transaction to the back of the processing queue if it fails. It can fail say if its
+#source output to spend is not already processed.
+#DEFAULT: 1000
+#maxfailures=
+
+#Block Chain Name - Specifies the chain params for parsing blocks, transactions, claims, and addresses. valid choices are
+#lbrycrd_main, lbrycrd_testnet, and lbrycrd_regtest.
+#DEFAULT: "lbrycrd_main"
+blockchainname="lbrycrd_regtest"

e2e/docker-compose.yml

@@ -101,3 +101,34 @@ services:
       - MYSQL_PASS=lbry
       - MYSQL_DATABASE=lbry
     entrypoint: wait-for-it mysql:3306 -- wait-for-it lbrynet:5279 -- ./latest serve
+  ######################
+  ## MySQL Chainquery ##
+  ######################
+  mysqlCQ:
+    image: mysql/mysql-server:5.7.27
+    restart: "no"
+    ports:
+      - "15600:3306"
+    expose:
+      - "3306"
+    environment:
+      - MYSQL_ALLOW_EMPTY_PASSWORD=true
+      - MYSQL_DATABASE=chainquery
+      - MYSQL_USER=lbry
+      - MYSQL_PASSWORD=lbry
+      - MYSQL_LOG_CONSOLE=true
+    volumes:
+      - "./chainquery/docker/my.cnf:/etc/mysql/conf.d/chainquery-optimizations.cnf"
+  ################
+  ## Chainquery ##
+  ################
+  chainquery:
+    image: lbry/chainquery:v1.8.1
+    restart: "no"
+    ports:
+      - 6300:6300
+    depends_on:
+      - mysql
+    ## TODO: Uncomment this in a docker-compose.override.yml to allow for external configurations.
+    volumes:
+      - ./chainqueryconfig.toml:/etc/lbry/chainqueryconfig.toml

e2e/lbrynet/docker/Dockerfile

@@ -4,7 +4,7 @@ LABEL MAINTAINER="leopere [at] nixc [dot] us"
 RUN apt-get update && apt-get -y install unzip curl telnet wait-for-it
 
 ## Add lbrynet
-ARG VERSION="v0.38.6"
+ARG VERSION="latest"
 RUN URL=$(curl -s https://api.github.com/repos/lbryio/lbry-sdk/releases/$(if [ "${VERSION}" = 'latest' ]; then echo "latest"; else echo "tags/${VERSION}"; fi) | grep browser_download_url | grep lbrynet-linux.zip | cut -d'"' -f4) && echo $URL && curl -L -o /lbrynet.linux.zip $URL
 
 COPY start.sh /usr/bin/start