diff --git a/e2e/chainquery/docker-compose.yml b/e2e/chainquery/docker-compose.yml new file mode 100644 index 0000000..452c8ff --- /dev/null +++ b/e2e/chainquery/docker-compose.yml @@ -0,0 +1,28 @@ +version: '3.4' + +services: + ########### + ## MYSQL ## + ########### + mysql: + image: mysql:5.7.23 + restart: "no" + ports: + - 3306:3306 + volumes: + - "../persist/chainquery/db:/var/lib/mysql" + ## This one may need to be tweaked based on where you run this docker-compose from. + - "../stuff/my.cnf:/etc/mysql/conf.d/chainquery-optimizations.cnf" + ################ + ## Chainquery ## + ################ + chainquery: + image: lbry/chainquery:v1.8.1 + restart: "no" + ports: + - 6300:6300 + depends_on: + - mysql + ## TODO: Uncomment this in a docker-compose.override.yml to allow for external configurations. + volumes: + - "../persist/chainquery/config/chainqueryconfig.toml:/etc/chainquery/chainqueryconfig.toml" \ No newline at end of file diff --git a/e2e/chainquery/docker/Dockerfile b/e2e/chainquery/docker/Dockerfile new file mode 100644 index 0000000..efeb65f --- /dev/null +++ b/e2e/chainquery/docker/Dockerfile @@ -0,0 +1,29 @@ +## Get the latest source and extract it for the app container. +## Design choices, two RUN layers intended to keep builds faster, the zipped +FROM ubuntu:18.04 as prep +LABEL MAINTAINER="leopere [at] nixc [dot] us" +RUN apt-get update && \ + apt-get -y install unzip curl && \ + apt-get autoclean -y && \ + rm -rf /var/lib/apt/lists/* +WORKDIR / +SHELL ["/bin/bash", "-o", "pipefail", "-c"] +COPY ./start.sh start +COPY ./healthcheck.sh healthcheck +ARG VERSION="master" +RUN curl -s -o /chainquery http://build.lbry.io/chainquery/branch-"${VERSION}"/chainquery && \ + chmod +x /chainquery + + +FROM ubuntu:18.04 as app +ARG VERSION="master" +ADD https://raw.githubusercontent.com/lbryio/chainquery/"${VERSION}"/config/default/chainqueryconfig.toml /etc/lbry/chainqueryconfig.toml.orig +RUN adduser chainquery --gecos GECOS --shell /bin/bash --disabled-password --home /home/chainquery && \ + chown -R chainquery:chainquery /etc/lbry +COPY --from=prep ./healthcheck /chainquery /start /usr/bin/ +HEALTHCHECK --interval=1m --timeout=30s \ + CMD healthcheck +EXPOSE 6300 +USER chainquery +STOPSIGNAL SIGINT +CMD ["start"] \ No newline at end of file diff --git a/e2e/chainquery/docker/build.sh b/e2e/chainquery/docker/build.sh new file mode 100755 index 0000000..61a6dfe --- /dev/null +++ b/e2e/chainquery/docker/build.sh @@ -0,0 +1,8 @@ +#!/bin/bash +if [ $# -eq 0 ] + then + echo "No docker tag argument supplied. Use './build.sh '" + exit 1 +fi +docker build --build-arg VERSION=$1 --tag lbry/chainquery:$1 . +docker push lbry/chainquery:$1 \ No newline at end of file diff --git a/e2e/chainquery/docker/healthcheck.sh b/e2e/chainquery/docker/healthcheck.sh new file mode 100755 index 0000000..6ea1b90 --- /dev/null +++ b/e2e/chainquery/docker/healthcheck.sh @@ -0,0 +1,2 @@ +#!/usr/bin/env bash +curl --fail http://localhost:6300/api/status || exit 1 \ No newline at end of file diff --git a/e2e/chainquery/docker/my.cnf b/e2e/chainquery/docker/my.cnf new file mode 100644 index 0000000..920ac23 --- /dev/null +++ b/e2e/chainquery/docker/my.cnf @@ -0,0 +1,9 @@ +# Default Homebrew MySQL server config +[mysqld] +# Only allow connections from localhost +innodb_log_file_size=5G +key_buffer_size=1G +innodb_flush_log_at_trx_commit = 0 +innodb_autoinc_lock_mode=2 +innodb_buffer_pool_size=1G +innodb_log_buffer_size=1G \ No newline at end of file diff --git a/e2e/chainquery/docker/start.sh b/e2e/chainquery/docker/start.sh new file mode 100755 index 0000000..2bd5d08 --- /dev/null +++ b/e2e/chainquery/docker/start.sh @@ -0,0 +1,51 @@ +#!/usr/bin/env bash + +## Config setup + +## Setup Values +DEBUGMODE=$(echo "debugmode=$DEBUGMODE") +LBRYCRDURL=$(echo "lbrycrdurl=\"rpc://$RPC_USER:$RPC_PASSWORD@10.5.1.2:9245\"") +MYSQLDSN=$(echo "mysqldsn=\"$MYSQL_USER:$MYSQL_PASSWORD@tcp($MYSQL_SERVER:3306)/$MYSQL_DATABASE\"") +APIMYSQLDSN=$(echo "apimysqldsn=\"$MYSQL_USER:$MYSQL_PASSWORD@tcp($MYSQL_SERVER:3306)/$MYSQL_DATABASE\"") + +## Setup Defaults +DEBUGMODE_DEFAULT='#DEFAULT-debugmode=false' +LBRYCRDURL_DEFAULT='#DEFAULT-lbrycrdurl="rpc://lbry:lbry@localhost:9245"' +MYSQLDSN_DEFAULT='#DEFAULT-mysqldsn="lbry:lbry@tcp(localhost:3306)/chainquery"' +APIMYSQLDSN_DEFAULT='#DEFAULT-apimysqldsn="lbry:lbry@tcp(localhost:3306)/chainquery"' + +## Add setup value variable name to this list to get processed on container start +CONFIG_SETTINGS=( + DEBUGMODE + LBRYCRDURL + MYSQLDSN + APIMYSQLDSN +) + +function set_configs() { + ## Set configs on container start if not already set. + for i in "${!CONFIG_SETTINGS[@]}"; do + ## Indirect references http://tldp.org/LDP/abs/html/ivr.html + eval FROM_STRING=\$"${CONFIG_SETTINGS[$i]}_DEFAULT" + eval TO_STRING=\$${CONFIG_SETTINGS[$i]} + ## TODO: Add a bit more magic to make sure that you're only configuring things if not set by config mounts. + sed -i "s~$FROM_STRING~"$TO_STRING"~g" /etc/lbry/chainqueryconfig.toml + done + echo "Reading config for debugging." + cat /etc/lbry/chainqueryconfig.toml +} + +if [[ ! -f /etc/lbry/chainqueryconfig.toml ]]; then + echo "[INFO]: Did not find chainqueryconfig.toml" + echo " Installing default and configuring with provided environment variables if any." + ## Install fresh copy of config file. + echo "cp -v /etc/lbry/chainqueryconfig.toml.orig /etc/lbry/chainqueryconfig.toml" + cp -v /etc/lbry/chainqueryconfig.toml.orig /etc/lbry/chainqueryconfig.toml + ls -lAh /etc/lbry/ + set_configs +else + echo "[INFO]: Found a copy of chainqueryconfig.toml in /etc/lbry" +fi + +## For now keeping this simple. Potentially eventually add all command args as envvars for the Dockerfile or use safe way to add args via docker-compose.yml +chainquery serve --configpath "/etc/lbry/" \ No newline at end of file diff --git a/e2e/chainqueryconfig.toml b/e2e/chainqueryconfig.toml new file mode 100644 index 0000000..300bdb5 --- /dev/null +++ b/e2e/chainqueryconfig.toml @@ -0,0 +1,99 @@ +#Debug mode outputs specific information to the console +#DEFAULT: false +#debugmode= + +#DebugQueryMode outputs SQL Boiler queries to the console. +#DEFAULT: false +#debugquerymode= + +#LBRYcrd URL is required for chainquery to query the blockchain +#DEFAULT: "rpc://lbry:lbry@localhost:9245" +lbrycrdurl="rpc://lbry:lbry@lbrycrd:29245" + +#MySQL DSN is required for chainquery to store information. +#DEFAULT: "lbry:lbry@tcp(localhost:3306)/chainquery" +#SUGGESTED: "lbry:lbry@unix(/var/run/mysqld/mysqld.sock)/chainquery" +mysqldsn="lbry:lbry@tcp(mysqlCQ:3306)/chainquery" + +#API MySQL DSN is required for chainquery to expose a SQL query service +#DEFAULT: "lbry:lbry@tcp(localhost:3306)/chainquery" +#SUGGESTED: "lbry:lbry@unix(/var/run/mysqld/mysqld.sock)/chainquery" +#apimysqldsn= + +#API Host and Port is required for the API Server to bind and listen on. +#DEFAULT: "0.0.0.0:6300" +#apihostport= + +#Profile mode enables and disables the reporting of a profile for chainquery +#DEFAULT: false +#profilemode= + +#Daemon mode tells chainquery how hard it should work catch up processing the blockchain +#deamonmode=0 #BeastMode it continuously process block after block until caughtup. +#daemonmode=1 #SlowAndSteadyMode it will process block with a frequency of 1 block every 100ms +#daemonmode=2 #DelayMode it will process a block with a configured delay frequency (set via 'processingdelay') +#daemonmode=3 #DaemonMode it will process a block every iteration of the daemon. +#DEFAULT: 0 +#deamonmode= + +#Default client timeout is for communication with the api of chainquery +#DEFAULT: 20 #Measured in seconds +#defaultclienttimeout= + +#Processing delay is used to determine how frequently chainquery should process a block +# It is only used if Daemon mode is set to delay mode +#DEFAULT: 100 #Measured in milliseconds +#processingdelay= + +#Daemon delay is the frequency at which chainquery checks for work to do. +#DEFAULT: 1 #Measured in seconds +#daemondelay= + +#Profiling options - will output the time take for certain opertions related to the below category +#DEFAULT: false (for all 3 params) +#daemonprofile= +#lbrycrdprofile= +#mysqlprofile= + +#Slack Hook URL allows slack integration. All logging info level and above is posted to a slack channel. +#DEFAULT: "" +#slackhookurl= + +#Slack Channel is the channel that you want the messages to appear. Works together with the hook url. +#DEFAULT: "" +#slackchannel= + +#Slack Log Level tells chainquery what level of logging will be sent to the slack channel. It will log all levels below +# it as well. Panic=0,Fatal=1,Error=2,Warning=3,Info=4,Debug=5 +#DEFAULT: 0 +#slackloglevel= + +#The command that should be executed to trigger a self update of the software. For linux, for example, `.sh` +#DEFAULT: "" +#autoupdatecommand= + +#Twilio service of chainquery to send specifically important information to key users of the Chainquery install. +#DEFAULT: +##twiliosid="" +##twilioauthtoken="" +##smsrecipients=["",""] +##smsfromphonenumber="" +#twiliosid= +#twilioauthtoken= +#smsrecipients= +#smsfromphonenumber= + +#API Keys - Disallowed by default unless keys are entered. +#DEFAULT: [] +#apikeys= + +#Max Failures - Specifies the number of failures that can happen in processing a transaction. This is for parallel +#transaction processing which puts a transaction to the back of the processing queue if it fails. It can fail say if its +#source output to spend is not already processed. +#DEFAULT: 1000 +#maxfailures= + +#Block Chain Name - Specifies the chain params for parsing blocks, transactions, claims, and addresses. valid choices are +#lbrycrd_main, lbrycrd_testnet, and lbrycrd_regtest. +#DEFAULT: "lbrycrd_main" +blockchainname="lbrycrd_regtest" \ No newline at end of file diff --git a/e2e/docker-compose.yml b/e2e/docker-compose.yml index b25dea7..233c99d 100644 --- a/e2e/docker-compose.yml +++ b/e2e/docker-compose.yml @@ -100,4 +100,35 @@ services: - MYSQL_USER=lbry - MYSQL_PASS=lbry - MYSQL_DATABASE=lbry - entrypoint: wait-for-it mysql:3306 -- wait-for-it lbrynet:5279 -- ./latest serve \ No newline at end of file + entrypoint: wait-for-it mysql:3306 -- wait-for-it lbrynet:5279 -- ./latest serve + ###################### + ## MySQL Chainquery ## + ###################### + mysqlCQ: + image: mysql/mysql-server:5.7.27 + restart: "no" + ports: + - "15600:3306" + expose: + - "3306" + environment: + - MYSQL_ALLOW_EMPTY_PASSWORD=true + - MYSQL_DATABASE=chainquery + - MYSQL_USER=lbry + - MYSQL_PASSWORD=lbry + - MYSQL_LOG_CONSOLE=true + volumes: + - "./chainquery/docker/my.cnf:/etc/mysql/conf.d/chainquery-optimizations.cnf" + ################ + ## Chainquery ## + ################ + chainquery: + image: lbry/chainquery:v1.8.1 + restart: "no" + ports: + - 6300:6300 + depends_on: + - mysql + ## TODO: Uncomment this in a docker-compose.override.yml to allow for external configurations. + volumes: + - ./chainqueryconfig.toml:/etc/lbry/chainqueryconfig.toml \ No newline at end of file diff --git a/e2e/lbrynet/docker/Dockerfile b/e2e/lbrynet/docker/Dockerfile index 258309f..35c752a 100644 --- a/e2e/lbrynet/docker/Dockerfile +++ b/e2e/lbrynet/docker/Dockerfile @@ -4,7 +4,7 @@ LABEL MAINTAINER="leopere [at] nixc [dot] us" RUN apt-get update && apt-get -y install unzip curl telnet wait-for-it ## Add lbrynet -ARG VERSION="v0.38.6" +ARG VERSION="latest" RUN URL=$(curl -s https://api.github.com/repos/lbryio/lbry-sdk/releases/$(if [ "${VERSION}" = 'latest' ]; then echo "latest"; else echo "tags/${VERSION}"; fi) | grep browser_download_url | grep lbrynet-linux.zip | cut -d'"' -f4) && echo $URL && curl -L -o /lbrynet.linux.zip $URL COPY start.sh /usr/bin/start