From 478cee9e4727a27c39f5bd51617de3a1efd532f4 Mon Sep 17 00:00:00 2001 From: Leopere Date: Mon, 15 Oct 2018 02:59:17 -0400 Subject: [PATCH 1/8] Starting in on some multistage --- reflector.go/Dockerfile | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/reflector.go/Dockerfile b/reflector.go/Dockerfile index 977507b..3e04002 100644 --- a/reflector.go/Dockerfile +++ b/reflector.go/Dockerfile @@ -1,10 +1,28 @@ ## base image for github.com/lbryio/reflector.go release binaries -FROM ubuntu:18:04 +FROM golang:1.11.1 as builder MAINTAINER chamunks [at] gmail [dot] com +RUN export GOROOT=$GOPATH/bin && \ + go get -v -u github.com/lbryio/reflector.go && \ + cd "/go/src/github.com/lbryio/reflector.go" && \ + make && \ + mv ./bin/prism-bin /usr/bin/prism-bin && \ + chmod +x /usr/bin/prism-bin + +## base image for github.com/lbryio/reflector.go release binaries +FROM ubuntu:18:04 as app +MAINTAINER chamunks [at] gmail [dot] com + +COPY /go/src/github.com/lbryio/reflector.go/bin/prism-bin /usr/bin/prism-bin ADD start.sh /usr/local/bin/start ADD docker-entrypoint.sh /usr/local/bin/docker-entrypoint RUN adduser reflector --gecos GECOS --shell /bin/bash --disabled-password --home /data/ && \ apt-get update && \ - apt-get -y install + apt-get -y install && \ + wget -O /data/config.tmpl https://raw.githubusercontent.com/lbryio/reflector.go/master/config.tmpl && \ + chown -R 1000:1000 /data/config.tmpl + +USER reflector + +CMD ["start"] From 02b356fecbfec33a380fac5f0d276bfd947ebf30 Mon Sep 17 00:00:00 2001 From: Leopere Date: Mon, 15 Oct 2018 02:59:35 -0400 Subject: [PATCH 2/8] Adding start.sh --- reflector.go/start.sh | 120 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 120 insertions(+) diff --git a/reflector.go/start.sh b/reflector.go/start.sh index e69de29..2d0749e 100644 --- a/reflector.go/start.sh +++ b/reflector.go/start.sh @@ -0,0 +1,120 @@ +#!/bin/bash + +## Launch service will tell prism-bin what mode to run in. +LAUNCHMODE="${MODE:-$1}" + +## This variable will be what can override default launch args. I may modify this as I learn more about prism-bin +LAUNCHARGS="${CUSTOM_ARGS:-$2}" + +## This is setup this way to handle any situations that might arise from the +## config being JSON and bash not being any good at JSON. +# ## Strings to replace. +AWS_ID_STR="YOUR-AWS-ID" +AWS_SECRET_STR="YOUR-AWS-SECRET" +BUCKET_REGION_STR="YOUR-BUCKET-REGION" +BUCKET_NAME_STR="YOUR-BUCKET-NAME" +DB_USER_STR="USER" +DB_PASSWORD_STR="PASSWORD" +DB_HOSTIP_STR="localhost" +DB_PORT_STR="3306" +DB_NAME_STR="DBNAME" + +## For the most part this section is disabled +# ## Keys to re-insert +# AWS_ID_KEY='' +# AWS_SECRET_KEY='' +# BUCKET_REGION_KEY='' +# BUCKET_NAME_KEY='' +# DB_USER_KEY='' +# DB_PASSWORD_KEY='' +# DB_HOSTIP_KEY='' +# DB_PORT_KEY='' +# DB_NAME_KEY='' + +# Environment Variables/Defaults +## Json sucks in BASH/Shell so you need to add trailing commas intermittently. +## Just pay attention to this. Also at some point I'll need to make a fringe +## case for handling key/values that aren't included in the default config. +AWS_ID="${AWS_ID:-potato}" +AWS_SECRET="${AWS_SECRET:-potato}" +BUCKET_REGION="${BUCKET_REGION:-potato}" +BUCKET_NAME="${BUCKET_NAME:-potato}" +DB_USER="${DB_USER:-potato}" +DB_PASSWORD="${DB_PASSWORD:-potato}" +DB_HOSTIP="${DB_HOSTIP:-potato}" +DB_PORT="${DB_PORT:-potato}" +DB_NAME="${DB_NAME:-potato}" + +## Environment Variables +## Missing Vars off the hop SLACK_HOOK_URL +CONFIG_SETTINGS=( + AWS_ID + AWS_SECRET + BUCKET_REGION + BUCKET_NAME + DB_USER + DB_PASSWORD + DB_HOSTIP + DB_PORT + DB_NAME +) +CONFIG_SECRETS=( + AWS_ID + AWS_ID_STR + AWS_SECRET + AWS_SECRET_STR + BUCKET_NAME + BUCKET_NAME_STR + DB_USER + DB_USER_STR + DB_PASSWORD + DB_PASSWORD_STR + DB_HOSTIP + DB_HOSTIP_STR + DB_PORT + DB_PORT_STR + DB_NAME + DB_NAME_STR +) + +## This function might be a bit overkill as all key/value pairs are unique in this config. +for i in "${!CONFIG_SETTINGS[@]}"; do + echo ${CONFIG_SETTINGS[$i]}"_KEY" + ## Indirect references http://tldp.org/LDP/abs/html/ivr.html + eval FROM_STRING=\$"${CONFIG_SETTINGS[$i]}_STR" + eval VALUE_STRING=\$${CONFIG_SETTINGS[$i]} + eval KEY_STRING=\$"${CONFIG_SETTINGS[$i]}_KEY" + TO_STRING="$KEY_STRING$VALUE_STRING" + ## DEBUG + # echo DEBUG FROM_STRING: "$FROM_STRING" + # echo DEBUG VALUE_STRING: $VALUE_STRING + # echo DEBUG KEY_STRING: $KEY_STRING + # echo DEBUG TO_STRING: "$TO_STRING" + sed -i '' "s/$FROM_STRING/$TO_STRING/g" ./config.tmpl +done + +## Sanitization section +# Awaiting someone smarter than me to suggest a method for this. +# https://unix.stackexchange.com/questions/474097/i-want-to-unset-a-list-of-bash-variables-that-have-their-variable-strings-stored +for i in "${CONFIG_SECRETS[@]}"; do + unset $i +done + +# Actual launch invoked here +case $MODE in + cluster ) + prism-bin cluster ${LAUNCHARGS:-'-v --conf /data/config.tmpl'} + ;; + dht ) + ## Env vars NODEID DHTPORT + ## Figure out what port we want to run --rpcPort on by default + ## Figure out if we need seed strings and set default(s) + prism-bin dht ${LAUNCHARGS:-'-v --conf /data/config.tmpl --nodeID $NODEID --port "${DHTPORT:-4567}"'} + ;; + peer ) + prism-bin peer ${LAUNCHARGS:-'-v --conf /data/config.tmpl'} + ;; + reflector ) + prism-bin reflector ${LAUNCHARGS:-'-v --conf /data/config.tmpl'} + ;; +esac From 01e14f0029041244101eb3d8f5454c32f5bbf337 Mon Sep 17 00:00:00 2001 From: Leopere Date: Mon, 15 Oct 2018 03:00:30 -0400 Subject: [PATCH 3/8] Tossing this here for now --- reflector.go/docker-compose.yml | 56 +++++++++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) diff --git a/reflector.go/docker-compose.yml b/reflector.go/docker-compose.yml index e69de29..6b88d4e 100644 --- a/reflector.go/docker-compose.yml +++ b/reflector.go/docker-compose.yml @@ -0,0 +1,56 @@ +version: '3.4' + +networks: + traefik: + external: true + +services: +########### +## MYSQL ## +########### +## MariaDB is currently not supported and neither is later versions of MySQL this may change. +## https://hub.docker.com/r/_/mariadb/ + mysql: + image: mysql.5.7.23 + restart: always + networks: + traefik: + ipv4_address: 10.5.1.11 + aliases: + - mysql + environment: + ## These variables are stored in the .env file next to this docker-compose.yml file. + ## I will include a default .env file and .gitignore the ".env" pattern so you should be able to just git pull in the future if you need to. + MYSQL_SERVER: ${MYSQL_SERVER:-10.5.1.11} + MYSQL_USER: ${MYSQL_USER:-changeme} + MYSQL_PASSWORD: ${MYSQL_PASSWORD:-changeme} + MYSQL_DATABASE: ${MYSQL_DATABASE:-reflector} + MYSQL_ROOT_PASSWORD: ${MYSQL_ROOT_PASSWORD:-changeme} + expose: + - 3306 + volumes: + - ./data/db:/var/lib/mysql + +################## +## reflector.go ## +################## + reflector-go: + build: . + restart: always + networks: + traefik: + ipv4_address: 10.5.1.20 + environment: + AWS_ID: ${AWS_ID:-default} + AWS_SECRET: ${AWS_SECRET:-default} + BUCKET_REGION: ${BUCKET_REGION:-default} + BUCKET_NAME: ${BUCKET_NAME:-default} + DB_USER: ${MYSQL_USER:-changeme} + DB_PASSWORD: ${MYSQL_PASSWORD:-changeme} + DB_HOSTIP: ${MYSQL_SERVER:-10.5.1.11} + DB_PORT: ${DB_PORT:-3306} + DB_NAME: ${MYSQL_DATABASE:-reflector} + labels: + - "traefik.expose=false" + depends_on: + - mysql From 6a86e295570daf22256872e1dcb0924b2753a2ac Mon Sep 17 00:00:00 2001 From: Leopere Date: Mon, 15 Oct 2018 03:03:40 -0400 Subject: [PATCH 4/8] Need a copy of this here too. --- reflector.go/config.tmpl | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 reflector.go/config.tmpl diff --git a/reflector.go/config.tmpl b/reflector.go/config.tmpl new file mode 100644 index 0000000..4ada19c --- /dev/null +++ b/reflector.go/config.tmpl @@ -0,0 +1,8 @@ +{ + "aws_id": "YOUR-AWS-ID", + "aws_secret": "YOUR-AWS-SECRET", + "bucket_region": "YOUR-BUCKET-REGION", + "bucket_name": "YOUR-BUCKET-NAME", + "db_conn": "USER:PASSWORD@tcp(localhost:3306)/DBNAME", + "slack_hook_url": "" +} From 3f08eef7587c12fcf77ec9ad94df5bbb228a5cfb Mon Sep 17 00:00:00 2001 From: Leopere Date: Mon, 15 Oct 2018 03:07:06 -0400 Subject: [PATCH 5/8] Start pulling envvars from .env This is unnecessery but if you have to declare the same thing in more then one place it makes sense to pull these from a file. In certain environments it would make more sense to declare them explicitly under a environment: directive --- reflector.go/docker-compose.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/reflector.go/docker-compose.yml b/reflector.go/docker-compose.yml index 6b88d4e..b41ffb3 100644 --- a/reflector.go/docker-compose.yml +++ b/reflector.go/docker-compose.yml @@ -18,6 +18,8 @@ services: ipv4_address: 10.5.1.11 aliases: - mysql + env_file: + - .env environment: ## These variables are stored in the .env file next to this docker-compose.yml file. ## I will include a default .env file and .gitignore the ".env" pattern so you should be able to just git pull in the future if you need to. @@ -40,6 +42,8 @@ services: networks: traefik: ipv4_address: 10.5.1.20 + env_file: + - .env environment: AWS_ID: ${AWS_ID:-default} AWS_SECRET: ${AWS_SECRET:-default} From c4e521fb9fd15ea15885f4d19ecae6947dad8c9b Mon Sep 17 00:00:00 2001 From: Leopere Date: Mon, 15 Oct 2018 03:08:58 -0400 Subject: [PATCH 6/8] A lot of these will probably change Ideally a neighboring script might run the user through doing this. --- reflector.go/.env | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/reflector.go/.env b/reflector.go/.env index e69de29..3979cb3 100644 --- a/reflector.go/.env +++ b/reflector.go/.env @@ -0,0 +1,14 @@ +#MYSQL_SERVER=${MYSQL_SERVER:-10.5.1.11} +#MYSQL_USER=${MYSQL_USER:-changeme} +#MYSQL_PASSWORD=${MYSQL_PASSWORD:-changeme} +#MYSQL_DATABASE=${MYSQL_DATABASE:-reflector} +#MYSQL_ROOT_PASSWORD=${MYSQL_ROOT_PASSWORD:-changeme} +#AWS_ID=${AWS_ID:-default} +#AWS_SECRET=${AWS_SECRET:-default} +#BUCKET_REGION=${BUCKET_REGION:-default} +#BUCKET_NAME=${BUCKET_NAME:-default} +#DB_USER=${MYSQL_USER:-changeme} +#DB_PASSWORD=${MYSQL_PASSWORD:-changeme} +#DB_HOSTIP=${MYSQL_SERVER:-10.5.1.11} +#DB_PORT=${DB_PORT:-3306} +#DB_NAME=${MYSQL_DATABASE:-reflector} From 9d5e7756c043144d1e8d563c10cd38db39626cac Mon Sep 17 00:00:00 2001 From: Leopere Date: Mon, 15 Oct 2018 03:14:02 -0400 Subject: [PATCH 7/8] Leaving note to refactor this --- www.spee.ch/docker-entrypoint.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/www.spee.ch/docker-entrypoint.sh b/www.spee.ch/docker-entrypoint.sh index 0e4075a..54b2ce2 100755 --- a/www.spee.ch/docker-entrypoint.sh +++ b/www.spee.ch/docker-entrypoint.sh @@ -34,6 +34,8 @@ ENVVARS=("MYSQL_ENV_MYSQL_USER" # SITE_ADDRESS=alpha.address.com SITE_DESCRIPTION=alpha.description + +## There might be a better way to do this now I'm working on something for configuring things more magically. function set_conf() { case $1 in MYSQL_ENV_MYSQL_USER ) From a2ebd69f449388d8b5d0d07e1eb351b3009cfbae Mon Sep 17 00:00:00 2001 From: Leopere Date: Mon, 15 Oct 2018 03:15:35 -0400 Subject: [PATCH 8/8] Updated against namechange --- www.spee.ch/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/www.spee.ch/Dockerfile b/www.spee.ch/Dockerfile index fc8297c..54f7678 100644 --- a/www.spee.ch/Dockerfile +++ b/www.spee.ch/Dockerfile @@ -19,7 +19,7 @@ RUN wget -quiet -O /usr/bin/debugpaste https://github.com/nixc-us/debugpaste-it/ chmod +x /usr/bin/debugpaste ## Install container support files -RUN curl -s https://raw.githubusercontent.com/chamunks/docker-support/master/install | /bin/sh +RUN curl -s https://raw.githubusercontent.com/leopere/docker-support/master/install | /bin/sh ADD docker-entrypoint.sh /usr/local/bin/docker-entrypoint ADD healthcheck.sh /usr/local/bin/healthcheck ADD debugpaste-it.sh /usr/local/bin/debugpaste-it