Merge branch 'leopere2'

chainquery/Dockerfile

@@ -2,37 +2,51 @@
 FROM ubuntu:18.04 as prep
 LABEL MAINTAINER="leopere [at] nixc [dot] us"
 
+## Install everything needed to unzip the file containing Chainquery
 RUN apt-get update && \
     apt-get -y install unzip curl && \
     apt-get autoclean -y && \
     rm -rf /var/lib/apt/lists/* && \
     mkdir /download
 
+## Download and extract the latest Zip containing Linux binary for Chainquery.
 RUN curl -L -o /download/chainquery.zip $(curl -s https://api.github.com/repos/lbryio/chainquery/releases | fgrep 'Linux_x86_64.zip' | grep download | head -n 1 | cut -d'"' -f4) && \
     cd /download && \
     ls -lAh /download && pwd && \
     unzip chainquery.zip && \
     rm chainquery.zip
-## I really want to use this instead of all the curl stuff at one point.
+
+## TODO: Use this instead of everything above here at some point. To do this we will need the LBRY team to host all of their release binaries in their latest form internally I guess since github doesn't support latest tags.
 # ADD --chown=1000:1000 https://github.com/lbryio/chainquery/releases/download/latest/chainquery_latest_Linux_x86_64.zip /download
 
 
-## Switch to Alpine eventually we might need to compile in Alpine for this.
+# TODO: Switch to Alpine eventually we might need to compile in Alpine for this.
 FROM ubuntu:18.04 as app
 
+## Run as unprivileged user.
 RUN adduser chainquery --gecos GECOS --shell /bin/bash --disabled-password --home /home/chainquery
 
+# Pull chainquery executable from prep container.
 COPY --from=0 /download/chainquery /usr/bin
 
 # ADD --chown=1000:1000 https://github.com/lbryio/chainquery/releases/download/latest/chainquery_latest_Linux_x86_64.zip /usr/bin
 COPY chainqueryconfig.toml /etc/chainquery/
-ADD --chown=1000:1000 https://raw.githubusercontent.com/lbryio/chainquery/master/config/default/chainqueryconfig.toml /etc/chainquery/chainqueryconfig.toml.orig
-COPY start.sh /usr/local/bin/start
-COPY docker-entrypoint.sh /usr/local/bin/docker-entrypoint
 
+## Get latest chainqueryconfig.toml from master on github as template.
+# TODO: add magic which pattern matches for chainqueryconfig.toml DEFAULT strings and replaces them with configured settings on container start.
+# ADD --chown=1000:1000 https://raw.githubusercontent.com/lbryio/chainquery/master/config/default/chainqueryconfig.toml /etc/chainquery/chainqueryconfig.toml.orig
+
+## Install start.sh as executable script in container $PATH
+COPY start.sh /usr/local/bin/start
+
+# TODO: Implement docker-entrypoint if later required this might be handy if we need certain maintainence tasks to be executed in the live container.
+# COPY docker-entrypoint.sh /usr/local/bin/docker-entrypoint
+
+## From here onward we're doing this with no root.
 USER chainquery
 
 ## Expose Chainquery API port
 EXPOSE 6300
 
+## Execute start script earlier installed into $PATH
 CMD ["start"]

chainquery/README.md

@@ -0,0 +1,52 @@
+## Get the lbry-docker repository
+
+`git clone https://github.com/lbryio/lbry-docker.git`
+
+## Setup networking
+
+You only need external networking if you plan on keeping your docker-compose files separate.
+For the sake of modularity in the design of this git repository the plan is to give you examples to try and then you're supposed to move towards your own custom docker-compose configuration.  We're going to create
+a docker bridge network that is going to be managed externally to your usual docker-compose networks which are compose internal.
+
+`docker network create -d bridge --subnet=10.5.1.0/16 lbrynet`
+
+## Make directories and set permissions
+
+This is only required on host mounted volumes. (the default settings)
+
+`mkdir -p ./lbry-docker/lbrycrd/data`
+
+`chmod -R 755 ./lbry-docker/lbrycrd/data`
+
+## Setup lbrycrd
+
+`cd lbry-docker/lbrycrd`
+
+`docker-compose up -d && docker-compose logs -f`
+
+Wait for lbrycrd to reach the top of the blockchain (console output should noticeably slow down)
+
+Once you've reached the top of the blockchain you can press `CTRL+C` to exit back to the linux shell.
+
+## Setup chainquery
+
+Now that you're done syncing your own copy of the lbry blockchain into the `lbrycrd` instance you can start spinning up Chainquery and it's dependencies.  Since Chainquery is parsing copious amounts of unstructured data stored in the `lbry` blockchain you have two routes to go.
+
+1. Route number one [Recommended]: You use some variant of this README.md's instructions to start your chainquery instance using a LBRY.io provided database checkpoint snapshot.  This should be basically copy paste-able series of commands to get your own staging instance up asap if you want Route number one, follow along with the rest of this README.md
+2. Route number two: You can your own fresh copy of the chainquery database indexing times may vary from hours to days depending on your hardware.
+
+`cd ../chainquery/`
+
+`./quick-bootstrap.sh getdata`
+
+`./quick-bootstrap.sh extract`
+
+## docker-compose.override.yml
+
+Add a PORTS directive for binding chainquery to the host and then run.
+
+`./quick-bootstrap.sh start`
+
+*OR*
+
+`docker-compose up -d mysql && sleep 30 && docker-compose up -d chainquery`

chainquery/docker-compose.yml

@@ -22,6 +22,8 @@ services:
       - .env
     expose:
       - 3306
+    ports:
+      - 3306:3306
     volumes:
       - ./data/db:/var/lib/mysql
       - ./my.cnf:/etc/mysql/conf.d/chainquery-optimizations.cnf

chainquery/quick-bootstrap.sh

@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 
-## Be Polite and ask for permission.
+## TODO: Be Polite and ask for confirmation.
 function QandA() {
   read -r -p "Continue with $1 [y/N] " response
   response=${response,,}    # tolower
@@ -14,6 +14,7 @@ function QandA() {
 
 ## Check your $PATH for required dependencies.
 ## Stop and complain for now, later automagically install them.
+## TODO: Add dependency checker.
 function test_for_deps() {
   ## Test for Command
   if ! [ -x "$(command -v $1)" ]; then
@@ -26,15 +27,11 @@ function test_for_deps() {
 
 ## Declare Linux app dependencies to check for.
 DEPENDENCIES=(
-  wget
-  unzip
   docker
-)
-## Recommended.
-BONUS_DEPENDENCIES=(
   docker-compose
 )
 
+## TODO: Check for docker and docker-compose
 function check_deps() {
 for i in "${!DEPENDENCIES[@]}"; do
   echo ${DEPENDENCIES[$i]}"_KEY"
@@ -48,25 +45,79 @@ done
 case $1 in
   getdata )
     ## Get DB Checkpoint data.
-    axel -a -n 6 http://chainquery-data.s3.amazonaws.com/data.zip -o ./chainquery.zip
+    echo Asked to get the latest checkpoint data from
+    docker run -v $(pwd)/:/download --rm leopere/axel-docker http://chainquery-data.s3.amazonaws.com/chainquery-data.zip -o ./chainquery.zip
     ;;
   extract )
     ## Unpack the data again if need be.
-    unzip ./chainquery.zip
+    echo Asked to unpack chainquery.zip if downloaded.
+    # TODO: add some magic here which will check for the presence of chainquery.zip and notify if its already gone.
+    docker run -v $(pwd)/:/data --rm leopere/unzip-docker ./chainquery.zip
     ;;
   cleanup )
     ## Remove any junk here.
+    echo Asked to clean up leftover chainquery.zip to save on disk space.
     rm chainquery.zip
     ;;
   reset )
     ## Give up on everything and try again.
-    # rm -Rf ./data
+    ## TODO: Make it very obvious with a nice little Y/N prompt that you're about to trash your settings and start over.
-    # rm -f ./chainquery.zip
+    docker-compose kill
+    docker-compose rm -f
+    rm -Rf ./data
+    rm -f ./chainquery.zip
+    ## TODO: Consider moving this somewhere as a function.
+    # docker-compose up -d mysql
+    # sleep 30
+    # docker-compose up -d chainquery
+    ;;
+  start )
+    ## Unsupported start command to start containers.
+    ## You can use this if you want to start this thing gracefully.
+    ## Ideally you would not use this in production.
+    echo "Asked to start chainquery gracefully for you."
+    docker-compose up -d mysql
+    echo "giving mysql some time to establish schema, crypto, users, permissions, and tables"
+    sleep 30
+    echo "Starting Chainquery"
+    docker-compose up -d chainquery
+    ;;
+  compress-latest-checkpoint-data )
+    ## This is not intended for public use.
+    docker-compose stop chainquery
+    docker-compose stop mysql
+    sudo zip -r chainquery-data.zip data
+    docker-compose up -d mysql
+    sleep 30
+    docker-compose up -d chainquery
+    ;;
+  upload-latest-checkpoint-data )
+    ## This is not intended for public use.
+    aws s3 cp ./chainquery-data.zip s3://chainquery-data/chainquery-data.new
+    aws s3 rm s3://chainquery-data/chainquery-data.zip
+    aws s3 mv s3://chainquery-data/chainquery-data.new s3://chainquery-data/chainquery-data.zip
     ;;
   * )
     echo "=================================================="
     echo "You look like you need usage examples let me help."
     echo "=================================================="
-    echo "Add documentation on script params HERE"
+    echo "./quick-boostrap.sh {Parameter}"
+    echo "Example: ./quick-bootstrap.sh getdata # Downloads the latest Chainquery checkpoint data from a LBRYio official aws instance."
+    echo ""
+    echo ""
+    echo "=================================================="
+    echo "Usage example and available parameters"
+    echo "=================================================="
+    echo ""
+    echo "getdata # This function grabs the latest Chainquery checkpoint data."
+    echo "extract # Unpacks the chainquery data into the correct directory. ./data/"
+    echo "cleanup # Removes chainquery.zip"
+    echo "reset   # Reset the state of these containers entirely, use if all else fails."
+    echo ""
+    echo ""
+    echo "=================================================="
+    echo "=================================================="
+    echo "Any other functions that are not documented here are not intended for public use."
+    echo "    These functions are included in this repository to keep things in one place."
     ;;
 esac

chainquery/start.sh

@@ -1,28 +1,31 @@
 #!/usr/bin/env bash
 
-#########################
+# TODO: Remove this notes section.
-## Chainquery Settings ##
+## Keeping this here as notes for later sed magic.
-#########################
+# #########################
-RPC_USER=lbryrpc                ## Not super necessery to change this.
+# ## Chainquery Settings ##
-RPC_PASSWORD=changeme           ## Please replace changeme.
+# #########################
-RPC_ALLOW_IP=10.5.1.3           ## You're better off not changing this.
+# RPC_USER=lbryrpc                ## Not super necessery to change this.
+# RPC_PASSWORD=changeme           ## Please replace changeme.
+# RPC_ALLOW_IP=10.5.1.3           ## You're better off not changing this.
+#
+# #################
+# ## Mysql Creds ##
+# #################
+# MYSQL_SERVER=10.5.1.10          ## You're better off not changing this.
+# MYSQL_USER=changeme             ## This could be changed.
+# MYSQL_PASSWORD=changeme         ## This could be set to something random it sets this string for both Mysql's main user and Chainquery's MysqlDSN.
+# MYSQL_DATABASE=chainquery       ## This can stay the same.
+# MYSQL_ROOT_PASSWORD=changeme    ## Set this to something random and obnoxious we're not using it.
 
-#################
+# TODO: Add chainquery startup magic for configuration.
-## Mysql Creds ##
+# sed -i ''
-#################
+#
-MYSQL_SERVER=10.5.1.10          ## You're better off not changing this.
+#
-MYSQL_USER=changeme             ## This could be changed.
+# debugmode=${DEBUGMODE:-false}
-MYSQL_PASSWORD=changeme         ## This could be set to something random it sets this string for both Mysql's main user and Chainquery's MysqlDSN.
+# lbrycrdurl="rpc://${RPC_USER:-lbryrpc}:${RPC_PASSWORD:-changeme}@10.5.1.2:9245"
-MYSQL_DATABASE=chainquery       ## This can stay the same.
+# mysqldsn="${MYSQL_USER:-changeme}:${MYSQL_PASSWORD:-changeme}@tcp(${MYSQL_SERVER:-10.5.1.10}:3306)/${MYSQL_DATABASE:-chainquery}"
-MYSQL_ROOT_PASSWORD=changeme    ## Set this to something random and obnoxious we're not using it.
+# apimysqldsn="${MYSQL_USER:-changeme}:${MYSQL_PASSWORD:-changeme}@tcp(${MYSQL_SERVER:-10.5.1.10}:3306)/${MYSQL_DATABASE:-chainquery}"
-
-sed -i '' 
-
-
-debugmode=${DEBUGMODE:-false}
-lbrycrdurl="rpc://${RPC_USER:-lbryrpc}:${RPC_PASSWORD:-changeme}@10.5.1.2:9245"
-mysqldsn="${MYSQL_USER:-changeme}:${MYSQL_PASSWORD:-changeme}@tcp(${MYSQL_SERVER:-10.5.1.10}:3306)/${MYSQL_DATABASE:-chainquery}"
-apimysqldsn="${MYSQL_USER:-changeme}:${MYSQL_PASSWORD:-changeme}@tcp(${MYSQL_SERVER:-10.5.1.10}:3306)/${MYSQL_DATABASE:-chainquery}"
 
 ## For now keeping this simple. Potentially eventually add all command args as envvars for the Dockerfile or use safe way to add args via docker-compose.yml
 chainquery serve -c "/etc/chainquery/"

lbrycrd/.gitignore

@@ -1 +1 @@
-data/
+data/.lbrycrd

lbrycrd/Dockerfile

@@ -5,14 +5,15 @@ LABEL MAINTAINER="leopere [at] nixc [dot] us"
 
 RUN adduser lbrycrdd --gecos GECOS --shell /bin/bash --disabled-password --home /data/ && \
     apt-get update && \
-    apt-get -y install unzip wget && \
+    apt-get -y install unzip wget curl && \
     apt-get autoclean -y && \
     rm -rf /var/lib/apt/lists/* && \
     mkdir -p /etc/lbrycrdd && \
     chown -R 1000:1000 /etc/lbrycrdd
 
-RUN wget -O /usr/bin/debugpaste https://github.com/nixc-us/debugpaste-it/raw/master/bin/debugpaste_64 && \
+## TODO: Consider adding debugpaste or variant
-    chmod +x /usr/bin/debugpaste
+# RUN wget -O /usr/bin/debugpaste https://github.com/nixc-us/debugpaste-it/raw/master/bin/debugpaste_64 && \
+#     chmod +x /usr/bin/debugpaste
 
 RUN wget -O /usr/bin/lbrycrd-linux.zip https://github.com/lbryio/lbrycrd/releases/download/v0.12.2.2/lbrycrd-linux.zip && \
     cd /usr/bin/ && \
@@ -23,6 +24,7 @@ RUN wget -O /usr/bin/lbrycrd-linux.zip https://github.com/lbryio/lbrycrd/release
 COPY debugpaste-it.sh /usr/local/bin/debugpaste-it
 COPY start.sh /usr/local/bin/start
 COPY docker-entrypoint.sh /usr/local/bin/docker-entrypoint
+COPY healthcheck.sh /usr/local/bin/healthcheck
 
 USER lbrycrdd
 
@@ -31,6 +33,9 @@ RUN mkdir -p /data/
 VOLUME /data/
 VOLUME /etc/lbrycrdd/
 
+## TODO: Implement healthcheck.
+# HEALTHCHECK ["healthcheck"]
+
 ## Exposing daemon port and RPC port
 EXPOSE 9245 9246
 

lbrycrd/compiler/.gitignore

@@ -1 +0,0 @@
-data/

lbrycrd/docker-compose.yml

@@ -24,5 +24,4 @@ services:
       - 9246
     ## host volumes for persistent data such as wallet private keys.
     volumes:
-      - ./data/:/data
+      - ./data:/data
-      - ./config:/etc/lbrycrdd

lbrycrd/healthcheck.sh

@@ -0,0 +1,4 @@
+## TODO: Implement a healthcheck for lbrycrd.
+curl --data-binary '{"jsonrpc":"1.0","id":"curltext","method":"getinfo","params":[]}' -H 'content-type:text/plain;'  http://$RPC_USER:$RPC_PASSWORD@127.0.0.1:9246
+## OR
+lbrycrd-cli getinfo

lbrycrd/start.sh

@@ -1,7 +1,6 @@
 #!/usr/bin/env bash
 
-# ## ToDo:
+# ## ToDo: Get a test case to see if this is the first run or a repeat run.
-# ## Get a test case to see if this is the first run or a repeat run
 # ## If it's a first run you need to do a full index including all transactions
 # ## tx index creates an index of every single transaction in the block history if
 # ## not specified it will only create an index for transactions that are related to the wallet or have unspent outputs.
@@ -10,22 +9,25 @@
 ## Ensure perms are correct prior to running main binary
 chown -R 1000:1000 /data
 chmod -R 755 /data
-chown -R 1000:1000 /etc/lbrycrdd
+
-chmod -R 755 /etc/lbrycrdd
+## TODO: Consider a config directory for future magic.
-rm -f /var/run/lbrycrdd.pid
+# chown -R 1000:1000 /etc/lbrycrd
+# chmod -R 755 /etc/lbrycrd
+rm -f /var/run/lbrycrd.pid
 mkdir -p ~/.lbrycrd
 
 ## Set config params
-echo -e "rpcuser=lbryrpc\nrpcpassword=${RPC_PASSWORD:-changeme}" > ~/.lbrycrd/lbrycrd.conf
+## TODO: Make this more automagic in the future.
-echo -e "rpcallowip=${RPC_ALLOW_IP:-10.5.1.3}" >> ~/.lbrycrd/lbrycrd.conf
+echo -e "rpcuser=lbryrpc\nrpcpassword=${RPC_PASSWORD:-changeme}" > $HOME/.lbrycrd/lbrycrd.conf
-echo -e "rpcuser=${RPC_USER:-lbryrpc}" >> ~/.lbrycrd/lbrycrd.conf
+echo -e "rpcallowip=${RPC_ALLOW_IP:-10.5.1.3}" >> $HOME/.lbrycrd/lbrycrd.conf
+echo -e "rpcuser=${RPC_USER:-lbryrpc}" >> $HOME/.lbrycrd/lbrycrd.conf
 
 ## Control this invocation through envvar.
 case ${RUN_MODE:-default} in
   default )
     lbrycrdd \
       -server \
-      -conf=$HOME/.lbrycrd/lbrycrd.conf \
+      -conf=$HOME.lbrycrd/lbrycrd.conf \
       -printtoconsole
     ;;
   reindex )
@@ -33,19 +35,19 @@ case ${RUN_MODE:-default} in
       -server \
       -txindex \
       -reindex \
-      -conf=$HOME/.lbrycrd/lbrycrd.conf \
+      -conf=$HOME.lbrycrd/lbrycrd.conf \
       -printtoconsole
     ;;
   chainquery )
     lbrycrdd \
       -server \
       -txindex \
-      -conf=$HOME/.lbrycrd/lbrycrd.conf \
+      -conf=$HOME.lbrycrd/lbrycrd.conf \
       -printtoconsole
     ;;
 esac
 
-
+## TODO: Look into what we can do with these launch params
 ## We were unsure if these function as intended so they were disabled for the time being.
 #  -port=${PORT:-9246} \
 #  -data=${DATA_DIR:-/data/} \