From cf486bed931791b4623bf5f6ca43637f698da043 Mon Sep 17 00:00:00 2001 From: Leopere Date: Sun, 11 Nov 2018 21:11:09 -0500 Subject: [PATCH 01/17] Started working through configuration steps --- chainquery/stuff/start.sh | 67 ++++++++++++++++++++++++--------------- 1 file changed, 42 insertions(+), 25 deletions(-) diff --git a/chainquery/stuff/start.sh b/chainquery/stuff/start.sh index c9a7f16..0954254 100755 --- a/chainquery/stuff/start.sh +++ b/chainquery/stuff/start.sh @@ -1,31 +1,48 @@ #!/usr/bin/env bash -# TODO: Remove this notes section. -## Keeping this here as notes for later sed magic. -# ######################### -# ## Chainquery Settings ## -# ######################### -# RPC_USER=lbryrpc ## Not super necessery to change this. -# RPC_PASSWORD=changeme ## Please replace changeme. -# RPC_ALLOW_IP=10.6.1.3 ## You're better off not changing this. -# -# ################# -# ## Mysql Creds ## -# ################# -# MYSQL_SERVER=10.6.1.10 ## You're better off not changing this. -# MYSQL_USER=changeme ## This could be changed. -# MYSQL_PASSWORD=changeme ## This could be set to something random it sets this string for both Mysql's main user and Chainquery's MysqlDSN. -# MYSQL_DATABASE=chainquery ## This can stay the same. -# MYSQL_ROOT_PASSWORD=changeme ## Set this to something random and obnoxious we're not using it. +## Config setup -# TODO: Add chainquery startup magic for configuration. -# sed -i '' -# -# -# debugmode=${DEBUGMODE:-false} -# lbrycrdurl="rpc://${RPC_USER:-lbryrpc}:${RPC_PASSWORD:-changeme}@10.6.1.2:9245" -# mysqldsn="${MYSQL_USER:-changeme}:${MYSQL_PASSWORD:-changeme}@tcp(${MYSQL_SERVER:-10.6.1.10}:3306)/${MYSQL_DATABASE:-chainquery}" -# apimysqldsn="${MYSQL_USER:-changeme}:${MYSQL_PASSWORD:-changeme}@tcp(${MYSQL_SERVER:-10.6.1.10}:3306)/${MYSQL_DATABASE:-chainquery}" +## Setup Values +DEBUGMODE=${DEBUGMODE:-false} +LBRYCRDURL="rpc://${RPC_USER:-lbryrpc}:${RPC_PASSWORD:-changeme}@10.5.1.2:9245" +MYSQLDSN="${MYSQL_USER:-changeme}:${MYSQL_PASSWORD:-changeme}@tcp(${MYSQL_SERVER:-10.5.1.10}:3306)/${MYSQL_DATABASE:-chainquery}" +APIMYSQLDSN="${MYSQL_USER:-changeme}:${MYSQL_PASSWORD:-changeme}@tcp(${MYSQL_SERVER:-10.5.1.10}:3306)/${MYSQL_DATABASE:-chainquery}" + +## Setup Defaults +DEBUGMODE_DEFAULT='#DEFAULT-debugquerymode=false' +LBRYCRDURL_DEFAULT='#DEFAULT-lbrycrdurl="rpc://lbry:lbry@localhost:9245"' +MYSQLDSN_EFAULT='#DEFAULT-mysqldsn="lbry:lbry@tcp(localhost:3306)/chainquery"' +APIMYSQLDSN_DEFAULT='#DEFAULT-apihostport="0.0.0.0:6300"' + +## Add setup value variable name to this list to get processed on container start +CONFIG_SETTINGS=( + DEBUGMODE + LBRYCRDURL + MYSQLDSN_ + APIMYSQLDSN +) + +function set_configs(parameter) { + ## Set configs on container start if not already set. + for i in "${!CONFIG_SETTINGS[@]}"; do + echo ${CONFIG_SETTINGS[$i]}"_KEY" + ## Indirect references http://tldp.org/LDP/abs/html/ivr.html + eval FROM_STRING=\$"${CONFIG_SETTINGS[$i]}_DEFAULT" + eval TO_STRING=\$${CONFIG_SETTINGS[$i]} + ## TODO: Add a bit more magic to make sure that you're only configuring things if not set by config mounts. + sed -i '' "s/$FROM_STRING/$TO_STRING/g" /etc/chainquery/chainqueryconfig.toml + done +} + +if [[ ! -f /etc/chainquery/chainqueryconfig.toml ]]; then + echo "[INFO]: Found no chainqueryconfig.toml" + echo " Installing default and configuring with provided environment variables if any." + set_configs +else + echo "[INFO]: Found a copy of chainqueryconfig.toml in /etc/chainquery" + echo " Attempting to non destructively install any new environment configurations." + set_configs +fi ## For now keeping this simple. Potentially eventually add all command args as envvars for the Dockerfile or use safe way to add args via docker-compose.yml chainquery serve -c "/etc/chainquery/" -- 2.45.2 From 11e055fe39dd4c7cf64ec2ff7fb8bd73ec656e6b Mon Sep 17 00:00:00 2001 From: Leopere Date: Sun, 11 Nov 2018 21:13:20 -0500 Subject: [PATCH 02/17] Add config if no config --- chainquery/stuff/start.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/chainquery/stuff/start.sh b/chainquery/stuff/start.sh index 0954254..32a816d 100755 --- a/chainquery/stuff/start.sh +++ b/chainquery/stuff/start.sh @@ -37,6 +37,8 @@ function set_configs(parameter) { if [[ ! -f /etc/chainquery/chainqueryconfig.toml ]]; then echo "[INFO]: Found no chainqueryconfig.toml" echo " Installing default and configuring with provided environment variables if any." + ## Install fresh copy of config file. + cp /etc/chainquery/chainqueryconfig.toml.orig /etc/chainquery/chainqueryconfig.toml set_configs else echo "[INFO]: Found a copy of chainqueryconfig.toml in /etc/chainquery" -- 2.45.2 From 933a61cc4c3e20d0403593244729069da7cdfaa8 Mon Sep 17 00:00:00 2001 From: Leopere Date: Sun, 11 Nov 2018 21:14:36 -0500 Subject: [PATCH 03/17] Add default config to image --- chainquery/Dockerfile | 2 +- chainquery/stuff/start.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/chainquery/Dockerfile b/chainquery/Dockerfile index ad32a32..163f405 100644 --- a/chainquery/Dockerfile +++ b/chainquery/Dockerfile @@ -34,7 +34,7 @@ COPY stuff/chainqueryconfig.toml /etc/chainquery/ ## Get latest chainqueryconfig.toml from master on github as template. # TODO: add magic which pattern matches for chainqueryconfig.toml DEFAULT strings and replaces them with configured settings on container start. -# ADD --chown=1000:1000 https://raw.githubusercontent.com/lbryio/chainquery/master/config/default/chainqueryconfig.toml /etc/chainquery/chainqueryconfig.toml.orig +ADD --chown=1000:1000 https://raw.githubusercontent.com/lbryio/chainquery/master/config/default/chainqueryconfig.toml /etc/chainquery/chainqueryconfig.toml.orig ## Install start.sh as executable script in container $PATH COPY stuff/start.sh /usr/local/bin/start diff --git a/chainquery/stuff/start.sh b/chainquery/stuff/start.sh index 32a816d..6a46040 100755 --- a/chainquery/stuff/start.sh +++ b/chainquery/stuff/start.sh @@ -22,7 +22,7 @@ CONFIG_SETTINGS=( APIMYSQLDSN ) -function set_configs(parameter) { +function set_configs() { ## Set configs on container start if not already set. for i in "${!CONFIG_SETTINGS[@]}"; do echo ${CONFIG_SETTINGS[$i]}"_KEY" -- 2.45.2 From 74d6cc34fee9642c82edab5bcefef200e797d894 Mon Sep 17 00:00:00 2001 From: Leopere Date: Sun, 11 Nov 2018 20:51:41 -0500 Subject: [PATCH 04/17] Started working on start config magic --- chainquery/stuff/start.sh | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/chainquery/stuff/start.sh b/chainquery/stuff/start.sh index 6a46040..0954254 100755 --- a/chainquery/stuff/start.sh +++ b/chainquery/stuff/start.sh @@ -22,7 +22,7 @@ CONFIG_SETTINGS=( APIMYSQLDSN ) -function set_configs() { +function set_configs(parameter) { ## Set configs on container start if not already set. for i in "${!CONFIG_SETTINGS[@]}"; do echo ${CONFIG_SETTINGS[$i]}"_KEY" @@ -37,8 +37,6 @@ function set_configs() { if [[ ! -f /etc/chainquery/chainqueryconfig.toml ]]; then echo "[INFO]: Found no chainqueryconfig.toml" echo " Installing default and configuring with provided environment variables if any." - ## Install fresh copy of config file. - cp /etc/chainquery/chainqueryconfig.toml.orig /etc/chainquery/chainqueryconfig.toml set_configs else echo "[INFO]: Found a copy of chainqueryconfig.toml in /etc/chainquery" -- 2.45.2 From 97b29f354c0e5ded09200a2f02f3226e212f6de9 Mon Sep 17 00:00:00 2001 From: Leopere Date: Sun, 11 Nov 2018 21:20:04 -0500 Subject: [PATCH 05/17] removed unknown flag --- chainquery/Dockerfile | 2 +- chainquery/stuff/start.sh | 7 ++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/chainquery/Dockerfile b/chainquery/Dockerfile index 163f405..c469a95 100644 --- a/chainquery/Dockerfile +++ b/chainquery/Dockerfile @@ -34,7 +34,7 @@ COPY stuff/chainqueryconfig.toml /etc/chainquery/ ## Get latest chainqueryconfig.toml from master on github as template. # TODO: add magic which pattern matches for chainqueryconfig.toml DEFAULT strings and replaces them with configured settings on container start. -ADD --chown=1000:1000 https://raw.githubusercontent.com/lbryio/chainquery/master/config/default/chainqueryconfig.toml /etc/chainquery/chainqueryconfig.toml.orig +ADD https://raw.githubusercontent.com/lbryio/chainquery/master/config/default/chainqueryconfig.toml /etc/chainquery/chainqueryconfig.toml.orig ## Install start.sh as executable script in container $PATH COPY stuff/start.sh /usr/local/bin/start diff --git a/chainquery/stuff/start.sh b/chainquery/stuff/start.sh index 0954254..de75642 100755 --- a/chainquery/stuff/start.sh +++ b/chainquery/stuff/start.sh @@ -22,21 +22,22 @@ CONFIG_SETTINGS=( APIMYSQLDSN ) -function set_configs(parameter) { +function set_configs() { ## Set configs on container start if not already set. for i in "${!CONFIG_SETTINGS[@]}"; do - echo ${CONFIG_SETTINGS[$i]}"_KEY" ## Indirect references http://tldp.org/LDP/abs/html/ivr.html eval FROM_STRING=\$"${CONFIG_SETTINGS[$i]}_DEFAULT" eval TO_STRING=\$${CONFIG_SETTINGS[$i]} ## TODO: Add a bit more magic to make sure that you're only configuring things if not set by config mounts. - sed -i '' "s/$FROM_STRING/$TO_STRING/g" /etc/chainquery/chainqueryconfig.toml + sed -i '' "s~$FROM_STRING~$TO_STRING~g" /etc/chainquery/chainqueryconfig.toml done } if [[ ! -f /etc/chainquery/chainqueryconfig.toml ]]; then echo "[INFO]: Found no chainqueryconfig.toml" echo " Installing default and configuring with provided environment variables if any." + ## Install fresh copy of config file. + cp /etc/chainquery/chainqueryconfig.toml.orig /etc/chainquery/chainqueryconfig.toml set_configs else echo "[INFO]: Found a copy of chainqueryconfig.toml in /etc/chainquery" -- 2.45.2 From d9aa1d93a59527317218c854b3b4df7faab4a66e Mon Sep 17 00:00:00 2001 From: Leopere Date: Sun, 11 Nov 2018 21:30:45 -0500 Subject: [PATCH 06/17] Run start as root and chainquery as not Removed default config --- chainquery/Dockerfile | 6 +++--- chainquery/stuff/start.sh | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/chainquery/Dockerfile b/chainquery/Dockerfile index c469a95..5df53a6 100644 --- a/chainquery/Dockerfile +++ b/chainquery/Dockerfile @@ -30,7 +30,7 @@ RUN adduser chainquery --gecos GECOS --shell /bin/bash --disabled-password --hom COPY --from=0 /download/chainquery /usr/bin # ADD --chown=1000:1000 https://github.com/lbryio/chainquery/releases/download/latest/chainquery_latest_Linux_x86_64.zip /usr/bin -COPY stuff/chainqueryconfig.toml /etc/chainquery/ +# COPY stuff/chainqueryconfig.toml /etc/chainquery/ ## Get latest chainqueryconfig.toml from master on github as template. # TODO: add magic which pattern matches for chainqueryconfig.toml DEFAULT strings and replaces them with configured settings on container start. @@ -42,8 +42,8 @@ COPY stuff/start.sh /usr/local/bin/start # TODO: Implement docker-entrypoint if later required this might be handy if we need certain maintainence tasks to be executed in the live container. # COPY docker-entrypoint.sh /usr/local/bin/docker-entrypoint -## From here onward we're doing this with no root. -USER chainquery +# ## From here onward we're doing this with no root. +# USER chainquery ## Expose Chainquery API port EXPOSE 6300 diff --git a/chainquery/stuff/start.sh b/chainquery/stuff/start.sh index de75642..4b85bcc 100755 --- a/chainquery/stuff/start.sh +++ b/chainquery/stuff/start.sh @@ -46,4 +46,4 @@ else fi ## For now keeping this simple. Potentially eventually add all command args as envvars for the Dockerfile or use safe way to add args via docker-compose.yml -chainquery serve -c "/etc/chainquery/" +su -c "chainquery serve -c "/etc/chainquery/"" chainquery -- 2.45.2 From 23e06883835a6cb53827cdeb5f87292ced9712b2 Mon Sep 17 00:00:00 2001 From: Leopere Date: Sun, 11 Nov 2018 21:32:36 -0500 Subject: [PATCH 07/17] Leave notes about how to mount an external config --- chainquery/docker-compose.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/chainquery/docker-compose.yml b/chainquery/docker-compose.yml index 83fbad3..dbf6260 100644 --- a/chainquery/docker-compose.yml +++ b/chainquery/docker-compose.yml @@ -52,3 +52,6 @@ services: - 6300:6300 depends_on: - mysql + ## TODO: Uncomment this in a docker-compose.override.yml to allow for external configurations. + # volumes: + # - ./data/config/chainqueryconfig.toml:/etc/chainquery/chainqueryconfig.toml -- 2.45.2 From 9f123bb25801927fdcda4cae1ec2db93854c4eda Mon Sep 17 00:00:00 2001 From: Leopere Date: Sun, 11 Nov 2018 21:37:39 -0500 Subject: [PATCH 08/17] Simplified feedback Strings were expanding at the wrong location. --- chainquery/stuff/start.sh | 26 ++++++++++++++++++-------- 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/chainquery/stuff/start.sh b/chainquery/stuff/start.sh index 4b85bcc..0c04ca8 100755 --- a/chainquery/stuff/start.sh +++ b/chainquery/stuff/start.sh @@ -3,15 +3,20 @@ ## Config setup ## Setup Values -DEBUGMODE=${DEBUGMODE:-false} -LBRYCRDURL="rpc://${RPC_USER:-lbryrpc}:${RPC_PASSWORD:-changeme}@10.5.1.2:9245" -MYSQLDSN="${MYSQL_USER:-changeme}:${MYSQL_PASSWORD:-changeme}@tcp(${MYSQL_SERVER:-10.5.1.10}:3306)/${MYSQL_DATABASE:-chainquery}" -APIMYSQLDSN="${MYSQL_USER:-changeme}:${MYSQL_PASSWORD:-changeme}@tcp(${MYSQL_SERVER:-10.5.1.10}:3306)/${MYSQL_DATABASE:-chainquery}" +echo FINDME +DEBUGMODE=$(echo "debugquerymode=${DEBUGMODE:-false}") +echo $DEBUGMODE +LBRYCRDURL=$(echo "rpc://${RPC_USER:-lbryrpc}:${RPC_PASSWORD:-changeme}@10.5.1.2:9245") +echo $LBRYCRDURL +MYSQLDSN=$(echo "${MYSQL_USER:-chainquery}:${MYSQL_PASSWORD:-changeme}@tcp(${MYSQL_SERVER:-10.5.1.10}:3306)/${MYSQL_DATABASE:-chainquery}") +echo $MYSQLDSN +APIMYSQLDSN=$(echo "${MYSQL_USER:-chainquery}:${MYSQL_PASSWORD:-changeme}@tcp(${MYSQL_SERVER:-10.5.1.10}:3306)/${MYSQL_DATABASE:-chainquery}") +echo $APIMYSQLDSN ## Setup Defaults DEBUGMODE_DEFAULT='#DEFAULT-debugquerymode=false' LBRYCRDURL_DEFAULT='#DEFAULT-lbrycrdurl="rpc://lbry:lbry@localhost:9245"' -MYSQLDSN_EFAULT='#DEFAULT-mysqldsn="lbry:lbry@tcp(localhost:3306)/chainquery"' +MYSQLDSN_DEFAULT='#DEFAULT-mysqldsn="lbry:lbry@tcp(localhost:3306)/chainquery"' APIMYSQLDSN_DEFAULT='#DEFAULT-apihostport="0.0.0.0:6300"' ## Add setup value variable name to this list to get processed on container start @@ -29,15 +34,20 @@ function set_configs() { eval FROM_STRING=\$"${CONFIG_SETTINGS[$i]}_DEFAULT" eval TO_STRING=\$${CONFIG_SETTINGS[$i]} ## TODO: Add a bit more magic to make sure that you're only configuring things if not set by config mounts. - sed -i '' "s~$FROM_STRING~$TO_STRING~g" /etc/chainquery/chainqueryconfig.toml + sed -i "s~$FROM_STRING~"$TO_STRING"~g" /etc/chainquery/chainqueryconfig.toml done + echo "Reading config for debugging." + cat /etc/chainquery/chainqueryconfig.toml } if [[ ! -f /etc/chainquery/chainqueryconfig.toml ]]; then - echo "[INFO]: Found no chainqueryconfig.toml" + echo "[INFO]: Did not find chainqueryconfig.toml" echo " Installing default and configuring with provided environment variables if any." ## Install fresh copy of config file. - cp /etc/chainquery/chainqueryconfig.toml.orig /etc/chainquery/chainqueryconfig.toml + echo "cp -v /etc/chainquery/chainqueryconfig.toml.orig /etc/chainquery/chainqueryconfig.toml" + cp -v /etc/chainquery/chainqueryconfig.toml.orig /etc/chainquery/chainqueryconfig.toml + chmod 755 /etc/chainquery/chainqueryconfig.toml + ls -lAh /etc/chainquery/ set_configs else echo "[INFO]: Found a copy of chainqueryconfig.toml in /etc/chainquery" -- 2.45.2 From c7c4966291daa5e1ca97d6095f1e37ab9baf7932 Mon Sep 17 00:00:00 2001 From: Leopere Date: Sun, 11 Nov 2018 22:16:22 -0500 Subject: [PATCH 09/17] Removed config notes --- chainquery/.env | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/chainquery/.env b/chainquery/.env index 7ed2b53..0a52862 100644 --- a/chainquery/.env +++ b/chainquery/.env @@ -3,17 +3,13 @@ COMPOSE_PROJECT_NAME=chainquery ######################### ## Chainquery Settings ## ######################### -## TODO: Test to ensure these vars can be pulled from the lbrycrd .env -#RPC_USER=lbryrpc ## Not super necessery to change this. -#RPC_PASSWORD=changeme ## Please replace changeme. - -RPC_ALLOW_IP=10.5.1.3 ## You're better off not changing this. +RPC_ALLOW_IP=10.5.1.3 ################# ## Mysql Creds ## ################# -MYSQL_SERVER=10.5.1.10 ## You're better off not changing this. -MYSQL_USER=changeme ## This could be changed. -MYSQL_PASSWORD=changeme ## This could be set to something random it sets this string for both Mysql's main user and Chainquery's MysqlDSN. -MYSQL_DATABASE=chainquery ## This can stay the same. -MYSQL_ROOT_PASSWORD=changeme ## Set this to something random and obnoxious we're not using it. +MYSQL_SERVER=10.5.1.10 +MYSQL_USER=changeme +MYSQL_PASSWORD=changeme +MYSQL_DATABASE=chainquery +MYSQL_ROOT_PASSWORD=changeme -- 2.45.2 From 0748221f386e646488b48616656c230bac310612 Mon Sep 17 00:00:00 2001 From: Leopere Date: Sun, 11 Nov 2018 22:20:38 -0500 Subject: [PATCH 10/17] Setup value strings fix applied --- chainquery/stuff/start.sh | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/chainquery/stuff/start.sh b/chainquery/stuff/start.sh index 0c04ca8..042ebba 100755 --- a/chainquery/stuff/start.sh +++ b/chainquery/stuff/start.sh @@ -4,24 +4,23 @@ ## Setup Values echo FINDME -DEBUGMODE=$(echo "debugquerymode=${DEBUGMODE:-false}") -echo $DEBUGMODE -LBRYCRDURL=$(echo "rpc://${RPC_USER:-lbryrpc}:${RPC_PASSWORD:-changeme}@10.5.1.2:9245") +# DEBUGMODE=$(echo "debugquerymode=$DEBUGMODE") +# echo $DEBUGMODE +LBRYCRDURL=$(echo "lbrycrdurl=\"rpc://$RPC_USER:$RPC_PASSWORD@10.5.1.2:9245\"") echo $LBRYCRDURL -MYSQLDSN=$(echo "${MYSQL_USER:-chainquery}:${MYSQL_PASSWORD:-changeme}@tcp(${MYSQL_SERVER:-10.5.1.10}:3306)/${MYSQL_DATABASE:-chainquery}") +MYSQLDSN=$(echo "mysqldsn=\"$MYSQL_USER:$MYSQL_PASSWORD@tcp($MYSQL_SERVER:3306)/$MYSQL_DATABASE\"") echo $MYSQLDSN -APIMYSQLDSN=$(echo "${MYSQL_USER:-chainquery}:${MYSQL_PASSWORD:-changeme}@tcp(${MYSQL_SERVER:-10.5.1.10}:3306)/${MYSQL_DATABASE:-chainquery}") +APIMYSQLDSN=$(echo "apimysqldsn=\"$MYSQL_USER:$MYSQL_PASSWORD@tcp($MYSQL_SERVER:3306)/$MYSQL_DATABASE\"") echo $APIMYSQLDSN ## Setup Defaults -DEBUGMODE_DEFAULT='#DEFAULT-debugquerymode=false' +# DEBUGMODE_DEFAULT='#DEFAULT-debugquerymode=false' LBRYCRDURL_DEFAULT='#DEFAULT-lbrycrdurl="rpc://lbry:lbry@localhost:9245"' MYSQLDSN_DEFAULT='#DEFAULT-mysqldsn="lbry:lbry@tcp(localhost:3306)/chainquery"' APIMYSQLDSN_DEFAULT='#DEFAULT-apihostport="0.0.0.0:6300"' ## Add setup value variable name to this list to get processed on container start CONFIG_SETTINGS=( - DEBUGMODE LBRYCRDURL MYSQLDSN_ APIMYSQLDSN -- 2.45.2 From f43ce4b4431b79e755d16aebd8f8722be7d4da40 Mon Sep 17 00:00:00 2001 From: Leopere Date: Sun, 11 Nov 2018 22:35:37 -0500 Subject: [PATCH 11/17] Ensuring variables are consistent --- lbrycrd/.env | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lbrycrd/.env b/lbrycrd/.env index a4e3c49..57647ae 100644 --- a/lbrycrd/.env +++ b/lbrycrd/.env @@ -4,11 +4,11 @@ COMPOSE_PROJECT_NAME=lbrycrd ## Lbrycrd ## ############# ## TODO: The credentials are a formality but we should randomize these with magic. -RPC_USER=${RPC_USER=lbryrpc} -RPC_PASSWORD=${RPC_PASSWORD:-changeme} +RPC_USER=lbryrpc +RPC_PASSWORD=changeme UID=$UID GID=$UID ## This should be the internal container IP from which you'll be calling the RPC for Lbrycrdd from. ## TODO: make this more dynamic before we move to scalability -RPC_ALLOW_IP=${RPC_ALLOW_IP:-10.5.1.3} +RPC_ALLOW_IP=10.5.1.3 -- 2.45.2 From 484683055bccb05566f66c2f88b481bd9c88c217 Mon Sep 17 00:00:00 2001 From: Leopere Date: Sun, 11 Nov 2018 22:38:20 -0500 Subject: [PATCH 12/17] Removed some debugging and swapped debug mode --- chainquery/stuff/start.sh | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/chainquery/stuff/start.sh b/chainquery/stuff/start.sh index 042ebba..3bdfa70 100755 --- a/chainquery/stuff/start.sh +++ b/chainquery/stuff/start.sh @@ -3,18 +3,13 @@ ## Config setup ## Setup Values -echo FINDME -# DEBUGMODE=$(echo "debugquerymode=$DEBUGMODE") -# echo $DEBUGMODE +DEBUGMODE=$(echo "debugmode=$DEBUGMODE") LBRYCRDURL=$(echo "lbrycrdurl=\"rpc://$RPC_USER:$RPC_PASSWORD@10.5.1.2:9245\"") -echo $LBRYCRDURL MYSQLDSN=$(echo "mysqldsn=\"$MYSQL_USER:$MYSQL_PASSWORD@tcp($MYSQL_SERVER:3306)/$MYSQL_DATABASE\"") -echo $MYSQLDSN APIMYSQLDSN=$(echo "apimysqldsn=\"$MYSQL_USER:$MYSQL_PASSWORD@tcp($MYSQL_SERVER:3306)/$MYSQL_DATABASE\"") -echo $APIMYSQLDSN ## Setup Defaults -# DEBUGMODE_DEFAULT='#DEFAULT-debugquerymode=false' +DEBUGMODE_DEFAULT='#DEFAULT-debugmode=false' LBRYCRDURL_DEFAULT='#DEFAULT-lbrycrdurl="rpc://lbry:lbry@localhost:9245"' MYSQLDSN_DEFAULT='#DEFAULT-mysqldsn="lbry:lbry@tcp(localhost:3306)/chainquery"' APIMYSQLDSN_DEFAULT='#DEFAULT-apihostport="0.0.0.0:6300"' -- 2.45.2 From cf106a8a34a1d11a74cdaad2310b7fe5e924678e Mon Sep 17 00:00:00 2001 From: Leopere Date: Sun, 11 Nov 2018 22:40:27 -0500 Subject: [PATCH 13/17] Added variable for debugmode in .env --- chainquery/.env | 1 + chainquery/stuff/start.sh | 5 +++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/chainquery/.env b/chainquery/.env index 0a52862..2ab5aea 100644 --- a/chainquery/.env +++ b/chainquery/.env @@ -4,6 +4,7 @@ COMPOSE_PROJECT_NAME=chainquery ## Chainquery Settings ## ######################### RPC_ALLOW_IP=10.5.1.3 +DEBUGMODE=false ################# ## Mysql Creds ## diff --git a/chainquery/stuff/start.sh b/chainquery/stuff/start.sh index 3bdfa70..6b96167 100755 --- a/chainquery/stuff/start.sh +++ b/chainquery/stuff/start.sh @@ -12,12 +12,13 @@ APIMYSQLDSN=$(echo "apimysqldsn=\"$MYSQL_USER:$MYSQL_PASSWORD@tcp($MYSQL_SERVER: DEBUGMODE_DEFAULT='#DEFAULT-debugmode=false' LBRYCRDURL_DEFAULT='#DEFAULT-lbrycrdurl="rpc://lbry:lbry@localhost:9245"' MYSQLDSN_DEFAULT='#DEFAULT-mysqldsn="lbry:lbry@tcp(localhost:3306)/chainquery"' -APIMYSQLDSN_DEFAULT='#DEFAULT-apihostport="0.0.0.0:6300"' +APIMYSQLDSN_DEFAULT='#DEFAULT-apimysqldsn="lbry:lbry@tcp(localhost:3306)/chainquery"' ## Add setup value variable name to this list to get processed on container start CONFIG_SETTINGS=( + DEBUGMODE LBRYCRDURL - MYSQLDSN_ + MYSQLDSN APIMYSQLDSN ) -- 2.45.2 From 773a2b44bf951a3a6cd857f546fe5cdb966989d2 Mon Sep 17 00:00:00 2001 From: Leopere Date: Mon, 12 Nov 2018 20:27:11 -0500 Subject: [PATCH 14/17] Cleaning up variables --- lbrycrd/stuff/start.sh | 18 ++++-------------- 1 file changed, 4 insertions(+), 14 deletions(-) diff --git a/lbrycrd/stuff/start.sh b/lbrycrd/stuff/start.sh index e2d6ab5..c14a62c 100755 --- a/lbrycrd/stuff/start.sh +++ b/lbrycrd/stuff/start.sh @@ -19,12 +19,12 @@ rm -f /var/run/lbrycrd.pid ## Set config params ## TODO: Make this more automagic in the future. -echo "rpcuser=lbryrpc\nrpcpassword=${RPC_PASSWORD:-changeme}" > /data/.lbrycrd/lbrycrd.conf -echo "rpcallowip=${RPC_ALLOW_IP:-10.6.1.3}" >> /data/.lbrycrd/lbrycrd.conf -echo "rpcuser=${RPC_USER:-lbryrpc}" >> /data/.lbrycrd/lbrycrd.conf +echo "rpcuser=lbryrpc\nrpcpassword=$RPC_PASSWORD" > /data/.lbrycrd/lbrycrd.conf +echo "rpcallowip=$RPC_ALLOW_IP" >> /data/.lbrycrd/lbrycrd.conf +echo "rpcuser=$RPC_USER" >> /data/.lbrycrd/lbrycrd.conf ## Control this invocation through envvar. -case ${RUN_MODE:-default} in +case $RUN_MODE in default ) su -c "lbrycrdd -server -conf=/data/.lbrycrd/ -printtoconsole" lbrycrd ;; @@ -35,13 +35,3 @@ case ${RUN_MODE:-default} in su -c "lbrycrdd -server -txindex -conf=/data/.lbrycrd/ -printtoconsole" lbrycrd ;; esac - -## TODO: Look into what we can do with these launch params -## We were unsure if these function as intended so they were disabled for the time being. -# -port=${PORT:-9246} \ -# -data=${DATA_DIR:-/data/} \ -# -pid=${PID_FILE:/var/run/lbrycrdd.pid} \ -# -rpcport=${RPC_PORT:-9245} \ -# -rpcpassword=${RPC_PASSWORD:-changeme} \ -# -rpcuser=${RPC_USER:-lbryrpc} \ -# -rpcallowip=${RPC_ALLOW_IP:-10.6.1.3} -- 2.45.2 From 8354921ead10aa835e9bcac6cf38fd5a330a79a7 Mon Sep 17 00:00:00 2001 From: Leopere Date: Mon, 12 Nov 2018 22:43:42 -0500 Subject: [PATCH 15/17] Starting work on chainquery compile --- chainquery/.dockerignore | 1 + 1 file changed, 1 insertion(+) diff --git a/chainquery/.dockerignore b/chainquery/.dockerignore index 8031321..5d1735a 100644 --- a/chainquery/.dockerignore +++ b/chainquery/.dockerignore @@ -1,3 +1,4 @@ data/ data.z* chainquery.z* +compile/ -- 2.45.2 From 81b9613208128b33b40e652aa0b2c2f79f446c3d Mon Sep 17 00:00:00 2001 From: Leopere Date: Mon, 12 Nov 2018 22:45:29 -0500 Subject: [PATCH 16/17] Adding compile subdir to start on Chainquery compiler container --- chainquery/.env | 2 +- chainquery/compile/.dockerignore | 4 + chainquery/compile/.env | 16 ++ chainquery/compile/.gitignore | 4 + chainquery/compile/Dockerfile | 52 ++++++ chainquery/compile/docker-compose.yml | 57 +++++++ chainquery/compile/quick-bootstrap.sh | 148 ++++++++++++++++++ .../compile/stuff/chainqueryconfig.toml | 16 ++ chainquery/compile/stuff/debugpaste-it.sh | 1 + chainquery/compile/stuff/docker-entrypoint.sh | 13 ++ chainquery/compile/stuff/env-example | 19 +++ chainquery/compile/stuff/healthcheck.sh | 3 + chainquery/compile/stuff/my.cnf | 9 ++ chainquery/compile/stuff/start.sh | 54 +++++++ 14 files changed, 397 insertions(+), 1 deletion(-) create mode 100644 chainquery/compile/.dockerignore create mode 100644 chainquery/compile/.env create mode 100644 chainquery/compile/.gitignore create mode 100644 chainquery/compile/Dockerfile create mode 100644 chainquery/compile/docker-compose.yml create mode 100644 chainquery/compile/quick-bootstrap.sh create mode 100644 chainquery/compile/stuff/chainqueryconfig.toml create mode 100644 chainquery/compile/stuff/debugpaste-it.sh create mode 100644 chainquery/compile/stuff/docker-entrypoint.sh create mode 100644 chainquery/compile/stuff/env-example create mode 100644 chainquery/compile/stuff/healthcheck.sh create mode 100644 chainquery/compile/stuff/my.cnf create mode 100644 chainquery/compile/stuff/start.sh diff --git a/chainquery/.env b/chainquery/.env index 2ab5aea..ca64fcf 100644 --- a/chainquery/.env +++ b/chainquery/.env @@ -10,7 +10,7 @@ DEBUGMODE=false ## Mysql Creds ## ################# MYSQL_SERVER=10.5.1.10 -MYSQL_USER=changeme +MYSQL_USER=chainquery MYSQL_PASSWORD=changeme MYSQL_DATABASE=chainquery MYSQL_ROOT_PASSWORD=changeme diff --git a/chainquery/compile/.dockerignore b/chainquery/compile/.dockerignore new file mode 100644 index 0000000..5d1735a --- /dev/null +++ b/chainquery/compile/.dockerignore @@ -0,0 +1,4 @@ +data/ +data.z* +chainquery.z* +compile/ diff --git a/chainquery/compile/.env b/chainquery/compile/.env new file mode 100644 index 0000000..2ab5aea --- /dev/null +++ b/chainquery/compile/.env @@ -0,0 +1,16 @@ +COMPOSE_PROJECT_NAME=chainquery + +######################### +## Chainquery Settings ## +######################### +RPC_ALLOW_IP=10.5.1.3 +DEBUGMODE=false + +################# +## Mysql Creds ## +################# +MYSQL_SERVER=10.5.1.10 +MYSQL_USER=changeme +MYSQL_PASSWORD=changeme +MYSQL_DATABASE=chainquery +MYSQL_ROOT_PASSWORD=changeme diff --git a/chainquery/compile/.gitignore b/chainquery/compile/.gitignore new file mode 100644 index 0000000..5d1735a --- /dev/null +++ b/chainquery/compile/.gitignore @@ -0,0 +1,4 @@ +data/ +data.z* +chainquery.z* +compile/ diff --git a/chainquery/compile/Dockerfile b/chainquery/compile/Dockerfile new file mode 100644 index 0000000..5df53a6 --- /dev/null +++ b/chainquery/compile/Dockerfile @@ -0,0 +1,52 @@ +## This base image is for running latest chainquery +FROM ubuntu:18.04 as prep +LABEL MAINTAINER="leopere [at] nixc [dot] us" + +## Install everything needed to unzip the file containing Chainquery +RUN apt-get update && \ + apt-get -y install unzip curl && \ + apt-get autoclean -y && \ + rm -rf /var/lib/apt/lists/* && \ + mkdir /download + +## Download and extract the latest Zip containing Linux binary for Chainquery. +RUN curl -L -o /download/chainquery.zip $(curl -s https://api.github.com/repos/lbryio/chainquery/releases | fgrep 'Linux_x86_64.zip' | grep download | head -n 1 | cut -d'"' -f4) && \ + cd /download && \ + ls -lAh /download && pwd && \ + unzip chainquery.zip && \ + rm chainquery.zip + +## TODO: Use this instead of everything above here at some point. To do this we will need the LBRY team to host all of their release binaries in their latest form internally I guess since github doesn't support latest tags. +# ADD --chown=1000:1000 https://github.com/lbryio/chainquery/releases/download/latest/chainquery_latest_Linux_x86_64.zip /download + + +# TODO: Switch to Alpine eventually we might need to compile in Alpine for this. +FROM ubuntu:18.04 as app + +## Run as unprivileged user. +RUN adduser chainquery --gecos GECOS --shell /bin/bash --disabled-password --home /home/chainquery + +# Pull chainquery executable from prep container. +COPY --from=0 /download/chainquery /usr/bin + +# ADD --chown=1000:1000 https://github.com/lbryio/chainquery/releases/download/latest/chainquery_latest_Linux_x86_64.zip /usr/bin +# COPY stuff/chainqueryconfig.toml /etc/chainquery/ + +## Get latest chainqueryconfig.toml from master on github as template. +# TODO: add magic which pattern matches for chainqueryconfig.toml DEFAULT strings and replaces them with configured settings on container start. +ADD https://raw.githubusercontent.com/lbryio/chainquery/master/config/default/chainqueryconfig.toml /etc/chainquery/chainqueryconfig.toml.orig + +## Install start.sh as executable script in container $PATH +COPY stuff/start.sh /usr/local/bin/start + +# TODO: Implement docker-entrypoint if later required this might be handy if we need certain maintainence tasks to be executed in the live container. +# COPY docker-entrypoint.sh /usr/local/bin/docker-entrypoint + +# ## From here onward we're doing this with no root. +# USER chainquery + +## Expose Chainquery API port +EXPOSE 6300 + +## Execute start script earlier installed into $PATH +CMD ["start"] diff --git a/chainquery/compile/docker-compose.yml b/chainquery/compile/docker-compose.yml new file mode 100644 index 0000000..a101711 --- /dev/null +++ b/chainquery/compile/docker-compose.yml @@ -0,0 +1,57 @@ +version: '3.4' + +networks: + lbrynet: + external: true + +services: +########### +## MYSQL ## +########### +## MariaDB is currently not supported and neither is later versions of MySQL this may change. +## https://hub.docker.com/r/_/mariadb/ + mysql: + image: mysql:5.7.23 + restart: always + networks: + lbrynet: + ipv4_address: 10.5.1.10 + aliases: + - mysql + env_file: + - .env + expose: + - 3306 + ## TODO: I want to find a way that is acceptable to everyone to lock this up + ## and not share it with everyone at least eventually. + ports: + - 3306:3306 + volumes: + - ./data/db:/var/lib/mysql + - ./stuff/my.cnf:/etc/mysql/conf.d/chainquery-optimizations.cnf + +################ +## Chainquery ## +################ + chainquery: + build: + context: . + target: app + restart: always + networks: + lbrynet: + ipv4_address: 10.5.1.3 + env_file: + - .env + - ../lbrycrd/.env + labels: + - "traefik.expose=false" + expose: + - 6300 + ports: + - 6300:6300 + depends_on: + - mysql + ## TODO: Uncomment this in a docker-compose.override.yml to allow for external configurations. + # volumes: + # - ./data/config/chainqueryconfig.toml:/etc/chainquery/chainqueryconfig.toml diff --git a/chainquery/compile/quick-bootstrap.sh b/chainquery/compile/quick-bootstrap.sh new file mode 100644 index 0000000..953e821 --- /dev/null +++ b/chainquery/compile/quick-bootstrap.sh @@ -0,0 +1,148 @@ +#!/usr/bin/env bash + +## TODO: Be Polite and ask for confirmation. +function QandA() { + read -r -p "Continue with $1 [y/N] " response + response=${response,,} # tolower + if [[ "$response" =~ ^(yes|y)$ ]]; then + echo "Continuing with this." + eval $1 + else + echo "Skipping the $1." + fi +} + +## Check your $PATH for required dependencies. +## Stop and complain for now, later automagically install them. +## TODO: Add dependency checker. +function test_for_deps() { + ## Test for Command + if ! [ -x "$(command -v $1)" ]; then + echo "Error: $1 is not installed." >&2 + echo "You must have $1 installed." + else + echo "Info: $1 is installed." + fi +} + +## Declare Linux app dependencies to check for. +DEPENDENCIES=( + docker + docker-compose +) + +## TODO: Check for docker and docker-compose +function check_deps() { +for i in "${!DEPENDENCIES[@]}"; do + echo ${DEPENDENCIES[$i]}"_KEY" + ## Indirect references http://tldp.org/LDP/abs/html/ivr.html + eval TESTDEP=\$"${DEPENDENCIES[$i]}" + test_for_deps $TESTDEP +done +} + +function get_checkpoint() { + ## Get DB Checkpoint data. + echo Asked to get the latest checkpoint data, downloading latest checkpoint. + echo This data is fairly large so this saves you a few days of parsing the LBRY blockchain. + docker run -v $(pwd)/:/download --rm leopere/axel-docker http://chainquery-data.s3.amazonaws.com/chainquery-data.zip -o ./chainquery.zip +} + +################################# +## The real action begins here ## +################################# +## TODO: Add ways to get into and out of a bind here. +case $1 in + getdata ) + if [[ -f ./chainquery.zip ]]; then + echo "Found a copy of ./chainquery.zip already in your system." + echo "We recommend that you delete this data before proceeding and grab a fresh copy." + QandA "rm -f ./chainquery.zip" + get_checkpoint + else + get_checkpoint + fi + ;; + extract ) + ## Unpack the data again if need be. + echo Asked to unpack chainquery.zip if downloaded. + if [[ -f ./chainquery.zip ]]; then + docker run -v $(pwd)/:/data --rm leopere/unzip-docker ./chainquery.zip + else + echo "Could not extractas chainquery.zip did not exist." + echo "Feel free to execute './quick-bootstrap.sh getdata' first next time." + fi + ;; + cleanup ) + ## Remove any junk here. + echo Asked to clean up leftover chainquery.zip to save on disk space. + rm chainquery.zip + ;; + reset ) + ## Give up on everything and try again. + ## TODO: Make it very obvious with a nice little Y/N prompt that you're about to trash your settings and start over. + echo "Agressively Killing all chainquery and dependency containers." + echo "executing: docker-compose kill" + docker-compose kill + echo "Cleaning up stopped containers." + echo "executing: docker-compose rm -f" + docker-compose rm -f + rm -Rf ./data + rm -f ./chainquery.zip + ;; + start ) + ## Unsupported start command to start containers. + ## You can use this if you want to start this thing gracefully. + ## Ideally you would not use this in production. + echo "Asked to start chainquery gracefully for you." + echo "executing: docker-compose up -d mysql" + docker-compose up -d mysql + echo "giving mysql some time to establish schema, crypto, users, permissions, and tables" + sleep 30 + echo "Starting Chainquery" + echo "executing: docker-compose up -d chainquery" + docker-compose up -d chainquery + ## TODO: verify chainquery instance is up and healthy, this requires a functional HEALTHCHECK + echo "This should have chainquery up and running, currently theres no checks in this function to verify this however." + echo "Do feel free to execute 'docker-compose ps' to verify if its running and not restarting or exited." + echo "Final Note: You should try to use the docker-compose commands in the tutorial at https://github.com/lbryio/lbry-docker/blob/master/chainquery/README.md" + ;; + compress-latest-checkpoint-data ) + ## This is not intended for public use. + docker-compose stop chainquery + docker-compose stop mysql + sudo zip -r chainquery-data.zip data + docker-compose up -d mysql + sleep 30 + docker-compose up -d chainquery + ;; + upload-latest-checkpoint-data ) + ## This is not intended for public use. + aws s3 cp ./chainquery-data.zip s3://chainquery-data/chainquery-data.new + aws s3 rm s3://chainquery-data/chainquery-data.zip + aws s3 mv s3://chainquery-data/chainquery-data.new s3://chainquery-data/chainquery-data.zip + ;; + * ) + echo "==================================================" + echo "You look like you need usage examples let me help." + echo "==================================================" + echo "./quick-boostrap.sh {Parameter}" + echo "Example: ./quick-bootstrap.sh getdata # Downloads the latest Chainquery checkpoint data from a LBRYio official aws instance." + echo "" + echo "" + echo "==================================================" + echo "Usage example and available parameters" + echo "==================================================" + echo "" + echo "getdata # This function grabs the latest Chainquery checkpoint data." + echo "extract # Unpacks the chainquery data into the correct directory. ./data/" + echo "cleanup # Removes chainquery.zip" + echo "reset # Reset the state of these containers entirely, use if all else fails." + echo "" + echo "" + echo "==================================================" + echo "==================================================" + echo "Any other functions that are not documented here are not intended for public use." + echo " These functions are included in this repository to keep things in one place." + ;; +esac diff --git a/chainquery/compile/stuff/chainqueryconfig.toml b/chainquery/compile/stuff/chainqueryconfig.toml new file mode 100644 index 0000000..21f0832 --- /dev/null +++ b/chainquery/compile/stuff/chainqueryconfig.toml @@ -0,0 +1,16 @@ +## TODO: Don't hardcode this stuff for production + +#Debug mode outputs specific information to the console +debugmode=false + +#LBRYcrd URL is required for chainquery to query the blockchain +lbrycrdurl="rpc://lbryrpc:changeme@10.5.1.2:9245" + +#MySQL DSN is required for chainquery to store information. +mysqldsn="changeme:changeme@tcp(10.5.1.10:3306)/chainquery" + +#API MySQL DSN is required for chainquery to expose a SQL query service +apimysqldsn="changeme:changeme@tcp(10.5.1.10:3306)/chainquery" + +#The command that should be executed to trigger a self update of the software. For linux, for example, `.sh` +#DEFAULT-autoupdatecommand=[unset] diff --git a/chainquery/compile/stuff/debugpaste-it.sh b/chainquery/compile/stuff/debugpaste-it.sh new file mode 100644 index 0000000..f1f641a --- /dev/null +++ b/chainquery/compile/stuff/debugpaste-it.sh @@ -0,0 +1 @@ +#!/usr/bin/env bash diff --git a/chainquery/compile/stuff/docker-entrypoint.sh b/chainquery/compile/stuff/docker-entrypoint.sh new file mode 100644 index 0000000..77e798a --- /dev/null +++ b/chainquery/compile/stuff/docker-entrypoint.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash + +# default to run whatever the user wanted like "/bin/bash" +## If user runs no need to run any more of the entrypoint script. +if [[ -z "$@" ]]; then + echo "User did not attempt input. Now executing docker-entrypoint." +else + echo "Running $@." + exec "$@" + exit 1 +fi + +/bin/bash diff --git a/chainquery/compile/stuff/env-example b/chainquery/compile/stuff/env-example new file mode 100644 index 0000000..7ed2b53 --- /dev/null +++ b/chainquery/compile/stuff/env-example @@ -0,0 +1,19 @@ +COMPOSE_PROJECT_NAME=chainquery + +######################### +## Chainquery Settings ## +######################### +## TODO: Test to ensure these vars can be pulled from the lbrycrd .env +#RPC_USER=lbryrpc ## Not super necessery to change this. +#RPC_PASSWORD=changeme ## Please replace changeme. + +RPC_ALLOW_IP=10.5.1.3 ## You're better off not changing this. + +################# +## Mysql Creds ## +################# +MYSQL_SERVER=10.5.1.10 ## You're better off not changing this. +MYSQL_USER=changeme ## This could be changed. +MYSQL_PASSWORD=changeme ## This could be set to something random it sets this string for both Mysql's main user and Chainquery's MysqlDSN. +MYSQL_DATABASE=chainquery ## This can stay the same. +MYSQL_ROOT_PASSWORD=changeme ## Set this to something random and obnoxious we're not using it. diff --git a/chainquery/compile/stuff/healthcheck.sh b/chainquery/compile/stuff/healthcheck.sh new file mode 100644 index 0000000..24cd27b --- /dev/null +++ b/chainquery/compile/stuff/healthcheck.sh @@ -0,0 +1,3 @@ +#!/usr/bin/env bash +## TODO: Implement this at some point it requires CURL in the container. +curl http://localhost:6300/api/status diff --git a/chainquery/compile/stuff/my.cnf b/chainquery/compile/stuff/my.cnf new file mode 100644 index 0000000..cb0dbed --- /dev/null +++ b/chainquery/compile/stuff/my.cnf @@ -0,0 +1,9 @@ +# Default Homebrew MySQL server config +[mysqld] +# Only allow connections from localhost +innodb_log_file_size=5G +key_buffer_size=1G +innodb_flush_log_at_trx_commit = 0 +innodb_autoinc_lock_mode=2 +innodb_buffer_pool_size=1G +innodb_log_buffer_size=1G diff --git a/chainquery/compile/stuff/start.sh b/chainquery/compile/stuff/start.sh new file mode 100644 index 0000000..6b96167 --- /dev/null +++ b/chainquery/compile/stuff/start.sh @@ -0,0 +1,54 @@ +#!/usr/bin/env bash + +## Config setup + +## Setup Values +DEBUGMODE=$(echo "debugmode=$DEBUGMODE") +LBRYCRDURL=$(echo "lbrycrdurl=\"rpc://$RPC_USER:$RPC_PASSWORD@10.5.1.2:9245\"") +MYSQLDSN=$(echo "mysqldsn=\"$MYSQL_USER:$MYSQL_PASSWORD@tcp($MYSQL_SERVER:3306)/$MYSQL_DATABASE\"") +APIMYSQLDSN=$(echo "apimysqldsn=\"$MYSQL_USER:$MYSQL_PASSWORD@tcp($MYSQL_SERVER:3306)/$MYSQL_DATABASE\"") + +## Setup Defaults +DEBUGMODE_DEFAULT='#DEFAULT-debugmode=false' +LBRYCRDURL_DEFAULT='#DEFAULT-lbrycrdurl="rpc://lbry:lbry@localhost:9245"' +MYSQLDSN_DEFAULT='#DEFAULT-mysqldsn="lbry:lbry@tcp(localhost:3306)/chainquery"' +APIMYSQLDSN_DEFAULT='#DEFAULT-apimysqldsn="lbry:lbry@tcp(localhost:3306)/chainquery"' + +## Add setup value variable name to this list to get processed on container start +CONFIG_SETTINGS=( + DEBUGMODE + LBRYCRDURL + MYSQLDSN + APIMYSQLDSN +) + +function set_configs() { + ## Set configs on container start if not already set. + for i in "${!CONFIG_SETTINGS[@]}"; do + ## Indirect references http://tldp.org/LDP/abs/html/ivr.html + eval FROM_STRING=\$"${CONFIG_SETTINGS[$i]}_DEFAULT" + eval TO_STRING=\$${CONFIG_SETTINGS[$i]} + ## TODO: Add a bit more magic to make sure that you're only configuring things if not set by config mounts. + sed -i "s~$FROM_STRING~"$TO_STRING"~g" /etc/chainquery/chainqueryconfig.toml + done + echo "Reading config for debugging." + cat /etc/chainquery/chainqueryconfig.toml +} + +if [[ ! -f /etc/chainquery/chainqueryconfig.toml ]]; then + echo "[INFO]: Did not find chainqueryconfig.toml" + echo " Installing default and configuring with provided environment variables if any." + ## Install fresh copy of config file. + echo "cp -v /etc/chainquery/chainqueryconfig.toml.orig /etc/chainquery/chainqueryconfig.toml" + cp -v /etc/chainquery/chainqueryconfig.toml.orig /etc/chainquery/chainqueryconfig.toml + chmod 755 /etc/chainquery/chainqueryconfig.toml + ls -lAh /etc/chainquery/ + set_configs +else + echo "[INFO]: Found a copy of chainqueryconfig.toml in /etc/chainquery" + echo " Attempting to non destructively install any new environment configurations." + set_configs +fi + +## For now keeping this simple. Potentially eventually add all command args as envvars for the Dockerfile or use safe way to add args via docker-compose.yml +su -c "chainquery serve -c "/etc/chainquery/"" chainquery -- 2.45.2 From c30c2d3840424068a5d8f67847ff5cb6f5395b69 Mon Sep 17 00:00:00 2001 From: root on chainquery dev Date: Thu, 13 Dec 2018 04:15:59 +0000 Subject: [PATCH 17/17] work from Lenny and Beamer for Chainquery/Lbrycrd --- .gitignore | 5 ++++- chainquery/.dockerignore | 1 + chainquery/.env | 2 +- chainquery/.gitignore | 1 + chainquery/stuff/chainqueryconfig.toml | 16 ---------------- lbrycrd/docker-compose.yml | 2 ++ lbrycrd/stuff/start.sh | 13 ++++++++----- 7 files changed, 17 insertions(+), 23 deletions(-) delete mode 100644 chainquery/stuff/chainqueryconfig.toml diff --git a/.gitignore b/.gitignore index 5453892..f8f1aa3 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,7 @@ +.gitlab-ci.yml +.DS_Store */docker-compose.override.yml docker-compose.yml */data/ - +./docker-compose.yml +*.zip \ No newline at end of file diff --git a/chainquery/.dockerignore b/chainquery/.dockerignore index 5d1735a..b683caa 100644 --- a/chainquery/.dockerignore +++ b/chainquery/.dockerignore @@ -1,4 +1,5 @@ data/ data.z* chainquery.z* +*.zip compile/ diff --git a/chainquery/.env b/chainquery/.env index ca64fcf..817badd 100644 --- a/chainquery/.env +++ b/chainquery/.env @@ -10,7 +10,7 @@ DEBUGMODE=false ## Mysql Creds ## ################# MYSQL_SERVER=10.5.1.10 -MYSQL_USER=chainquery +MYSQL_USER=root MYSQL_PASSWORD=changeme MYSQL_DATABASE=chainquery MYSQL_ROOT_PASSWORD=changeme diff --git a/chainquery/.gitignore b/chainquery/.gitignore index 8031321..67a2f9c 100644 --- a/chainquery/.gitignore +++ b/chainquery/.gitignore @@ -1,3 +1,4 @@ data/ data.z* chainquery.z* +*.zip diff --git a/chainquery/stuff/chainqueryconfig.toml b/chainquery/stuff/chainqueryconfig.toml deleted file mode 100644 index e28e642..0000000 --- a/chainquery/stuff/chainqueryconfig.toml +++ /dev/null @@ -1,16 +0,0 @@ -## TODO: Don't hardcode this stuff for production - -#Debug mode outputs specific information to the console -debugmode=false - -#LBRYcrd URL is required for chainquery to query the blockchain -lbrycrdurl="rpc://lbryrpc:changeme@10.6.1.2:9245" - -#MySQL DSN is required for chainquery to store information. -mysqldsn="changeme:changeme@tcp(10.6.1.10:3306)/chainquery" - -#API MySQL DSN is required for chainquery to expose a SQL query service -apimysqldsn="changeme:changeme@tcp(10.6.1.10:3306)/chainquery" - -#The command that should be executed to trigger a self update of the software. For linux, for example, `.sh` -#DEFAULT-autoupdatecommand=[unset] diff --git a/lbrycrd/docker-compose.yml b/lbrycrd/docker-compose.yml index a7601e5..2b52991 100644 --- a/lbrycrd/docker-compose.yml +++ b/lbrycrd/docker-compose.yml @@ -18,6 +18,8 @@ services: - "traefik.expose=false" environment: RUN_MODE: chainquery + env_file: + - .env expose: - 9245 - 9246 diff --git a/lbrycrd/stuff/start.sh b/lbrycrd/stuff/start.sh index c14a62c..e25ded5 100755 --- a/lbrycrd/stuff/start.sh +++ b/lbrycrd/stuff/start.sh @@ -19,19 +19,22 @@ rm -f /var/run/lbrycrd.pid ## Set config params ## TODO: Make this more automagic in the future. -echo "rpcuser=lbryrpc\nrpcpassword=$RPC_PASSWORD" > /data/.lbrycrd/lbrycrd.conf +echo "rpcuser=$RPC_USER" > /data/.lbrycrd/lbrycrd.conf +echo "rpcpassword=$RPC_PASSWORD" >> /data/.lbrycrd/lbrycrd.conf echo "rpcallowip=$RPC_ALLOW_IP" >> /data/.lbrycrd/lbrycrd.conf -echo "rpcuser=$RPC_USER" >> /data/.lbrycrd/lbrycrd.conf +echo "rpcport=9245" >> /data/.lbrycrd/lbrycrd.conf +echo "rpcbind=0.0.0.0" >> /data/.lbrycrd/lbrycrd.conf +#echo "bind=0.0.0.0" >> /data/.lbrycrd/lbrycrd.conf ## Control this invocation through envvar. case $RUN_MODE in default ) - su -c "lbrycrdd -server -conf=/data/.lbrycrd/ -printtoconsole" lbrycrd + su -c "lbrycrdd -server -conf=/data/.lbrycrd/lbrycrd.conf -printtoconsole" lbrycrd ;; reindex ) - su -c "lbrycrdd -server -txindex -reindex -conf=/data/.lbrycrd/ -printtoconsole" lbrycrd + su -c "lbrycrdd -server -txindex -reindex -conf=/data/.lbrycrd/lbrycrd.conf -printtoconsole" lbrycrd ;; chainquery ) - su -c "lbrycrdd -server -txindex -conf=/data/.lbrycrd/ -printtoconsole" lbrycrd + su -c "lbrycrdd -server -txindex -conf=/data/.lbrycrd/lbrycrd.conf -printtoconsole" lbrycrd ;; esac -- 2.45.2