diff --git a/.gitignore b/.gitignore index 6dd29b7..d4c0eca 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,4 @@ -bin/ \ No newline at end of file +bin/ +e2e/persist +.env +blobsfiles \ No newline at end of file diff --git a/.travis.yml b/.travis.yml index 387cf75..d410bc7 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,15 +1,42 @@ os: linux -dist: xenial +dist: bionic language: go -go_import_path: github.com/lbryio/ytsync go: -- 1.11.5 -- master + - 1.12.7 + - master + +install: true + +cache: + directories: + - $HOME/.cache/go-build + - $HOME/gopath/pkg/mod + +services: + - docker + +addons: + apt: + update: true + packages: + - ffmpeg + - tree + - python-pip + +before_script: + - sudo pip install -U youtube-dl + - sudo add-apt-repository -y ppa:jonathonf/ffmpeg-4 + env: global: - GO111MODULE=on #GITHUB_TOKEN - secure: "Ps3KocRP5xnM3/uA99CeYhDTVxRIuW7fGyrtqBeRWZW0cXzeA4XCTKxqcFbrPUPw67XkrBVgE58JDdWoQEJ7tm67PjMm/ltp5Evhx/QAJDh+YSofXyGDVpG1mrTZFI66R3NVVJLkSGALMkuWWXvfYZeU//AworJbyRoaIK/CVt5OP23i5N4tdd5UXc5dfLuYqnKRynyMmCkz9c3yEIQMXoPhG2hx7l7L2BeMJvcKmVhkSN7nQayjnrbUXGm/IRqrb88lvkyBevN5E3IB2V5IKEieIPZjbD/N0IfcnAt89Z96tgDhtIbx3ZvXm92lsvHA8buqQpG9d2AmSi6GKs64lQcnGeM5o0wER2JHWl1OSa1Nr/UAo5Xb/PM65Yt3yZE8AuMKHBmbfDSBzdkTXx58AeDzFUd3kMXD/fFjeQQWyXFlOss3ygH9SObl827Txmz9OJqZaxabs5Q3AP6m3EjKjz7zfLfrgpcxJM2WBiU1bN0ZxUgZkImy/CHk5gCZ7vhcnaLiDO4HZnzY/aRJwKYQPE5i0O2nHpIfovqkc0DFBA7U/7Cjin7e1E0UZvF3meLOxMqkfc6X7QTxqQpt2Tej6jlpdxw4CTLwGUhGkAw9IAPkUB3L0EbZ1/ksGhNvGDvUeSTq8hYdMAPmA+k9jS6653V4SQ+qBMy5++tbr5AeZQI=" + +script: + #- ./e2e/e2e.sh # Hold until we can resolve the /var/tmp issue - talk to beamer/niko + - make + deploy: provider: script skip_cleanup: true @@ -17,4 +44,4 @@ deploy: file: bin/ytsync on: repo: lbryio/ytsync - tags: true \ No newline at end of file + tags: true diff --git a/blobs_reflector/reflect.go b/blobs_reflector/reflect.go index 744a901..b816beb 100644 --- a/blobs_reflector/reflect.go +++ b/blobs_reflector/reflect.go @@ -2,16 +2,17 @@ package blobs_reflector import ( "encoding/json" + "io/ioutil" + "os" + "os/user" + "path/filepath" + "github.com/lbryio/lbry.go/extras/errors" "github.com/lbryio/reflector.go/cmd" "github.com/lbryio/reflector.go/db" "github.com/lbryio/reflector.go/reflector" "github.com/lbryio/reflector.go/store" - "github.com/mitchellh/go-ps" - "io/ioutil" - "os" - "os/user" - "path/filepath" + "github.com/lbryio/ytsync/util" log "github.com/sirupsen/logrus" ) @@ -21,7 +22,7 @@ func ReflectAndClean() error { if err != nil { return err } - return cleanupLbrynet() + return util.CleanupLbrynet() } func loadConfig(path string) (cmd.Config, error) { @@ -40,8 +41,11 @@ func loadConfig(path string) (cmd.Config, error) { } func reflectBlobs() error { + if util.IsBlobReflectionOff() { + return nil + } //make sure lbrynet is off - running, err := isLbrynetRunning() + running, err := util.IsLbrynetRunning() if err != nil { return err } @@ -90,57 +94,3 @@ func reflectBlobs() error { } return nil } - -func cleanupLbrynet() error { - //make sure lbrynet is off - running, err := isLbrynetRunning() - if err != nil { - return err - } - if running { - return errors.Prefix("cannot cleanup lbrynet as the daemon is running", err) - } - usr, err := user.Current() - if err != nil { - log.Errorln(err.Error()) - return errors.Err(err) - } - lbrynetDir := usr.HomeDir + "/.lbrynet/" - files, err := filepath.Glob(lbrynetDir + "lbrynet.sqlite*") - if err != nil { - return errors.Err(err) - } - for _, f := range files { - err = os.Remove(f) - if err != nil { - return errors.Err(err) - } - } - blobsDir := lbrynetDir + "/blobfiles/" - err = os.RemoveAll(blobsDir) - if err != nil { - return errors.Err(err) - } - err = os.Mkdir(blobsDir, 0755) - if err != nil { - return errors.Err(err) - } - return nil -} - -func isLbrynetRunning() (bool, error) { - processes, err := ps.Processes() - if err != nil { - return true, errors.Err(err) - } - var daemonProcessId = -1 - for _, p := range processes { - if p.Executable() == "lbrynet" { - daemonProcessId = p.Pid() - break - } - } - - running := daemonProcessId != -1 - return running, nil -} diff --git a/e2e/daemon_settings.yml b/e2e/daemon_settings.yml new file mode 100644 index 0000000..63b444e --- /dev/null +++ b/e2e/daemon_settings.yml @@ -0,0 +1,12 @@ +#blockchain_name: lbrycrd_main +#blockchain_name: lbrycrd_testnet +blockchain_name: lbrycrd_regtest +lbryum_servers: +# - spv1.lbry.com:50001 #Production Wallet Server + - walletserver:50001 +save_blobs: true +save_files: false +share_usage_data: false +tcp_port: 3333 +udp_port: 4444 +use_upnp: false \ No newline at end of file diff --git a/e2e/data_setup.sh b/e2e/data_setup.sh new file mode 100755 index 0000000..2d46529 --- /dev/null +++ b/e2e/data_setup.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash + +set -e + +#Add a ytsync user +ADDYTSYNCUSER='INSERT INTO user (given_name) VALUE("ytsync user")' +mysql -u lbry -plbry -D lbry -h "127.0.0.1" -P 15500 -e "$ADDYTSYNCUSER" +#Insert an auth token for the user to be used by ytsync +ADDYTSYNCAUTHTOKEN='INSERT INTO auth_token (user_id, value) VALUE(1,"ytsyntoken")' +mysql -u lbry -plbry -D lbry -h "127.0.0.1" -P 15500 -e "$ADDYTSYNCAUTHTOKEN" +#Give priveledges to ytsync user +ASSIGNGROOP='INSERT INTO user_groop (user_id, groop_id) VALUE( 1,3)' +mysql -u lbry -plbry -D lbry -h "127.0.0.1" -P 15500 -e "$ASSIGNGROOP" + +#Add youtuber to sync +ADDYTSYNCER='INSERT INTO user (given_name) VALUE("youtuber")' +mysql -u lbry -plbry -D lbry -h "127.0.0.1" -P 15500 -e "$ADDYTSYNCER" +#Add their youtube channel to be synced +ADDYTCHANNEL="INSERT INTO youtube_data (user_id, status_token,desired_lbry_channel,channel_id,channel_name,status,created_at,source,total_videos,total_subscribers) +VALUE(2,'3qzGyuVjQaf7t4pKKu2Er1NRW2LJkeWw','@beamertest','UCCyr5j8akeu9j4Q7urV0Lqw','BeamerAtLBRY','queued','2019-08-01 00:00:00','sync',1,0)" +mysql -u lbry -plbry -D lbry -h "127.0.0.1" -P 15500 -e "$ADDYTCHANNEL" diff --git a/e2e/docker-compose.yml b/e2e/docker-compose.yml new file mode 100644 index 0000000..217c5d9 --- /dev/null +++ b/e2e/docker-compose.yml @@ -0,0 +1,103 @@ +version: "3.5" +services: + ############# + ## Lbrycrd ## + ############# + lbrycrd: + image: lbry/lbrycrd:v0.12.4.1 + restart: always + ports: + - "15201:29246" + - "15200:29245" + expose: + - "29246" + - "29245" + ## host volumes for persistent data such as wallet private keys. + volumes: + - "./persist:/data" + environment: + - RUN_MODE=regtest + ################### + ## Wallet Server ## + ################### + walletserver: + image: lbry/wallet-server:v0.38.5 + restart: always + environment: + - DB_DIRECTORY=/database + - MAX_SEND=1000000000000000000000 + - DAEMON_URL=http://lbry:lbry@lbrycrd:29245 + - MAX_SUBS=1000000000000 + - BANDWIDTH_LIMIT=80000000000 + - SESSION_TIMEOUT=10000000000000000000000000 + - TCP_PORT=50001 + ports: + - "15300:50001" + expose: + - "50001" + depends_on: + - lbrycrd + ulimits: + nofile: + soft: 90000 + hard: 90000 + #command: lbry.wallet.server.coin.LBC + command: lbry.wallet.server.coin.LBCRegTest + ############# + ## Lbrynet ## + ############# + lbrynet: + image: lbry/lbrynet:v0.38.5 + restart: always + ports: + - "15100:5279" + - "15101:5280" + expose: + - "5279" + - "5280" + depends_on: + - walletserver + environment: + - LBRY_STREAMING_SERVER=0.0.0.0:5280 + volumes: + - "./persist/.lbrynet:/home/lbrynet" + - ".:/etc/lbry" #Put your daemon_settings.yml here + # /private/var/tmp for OSX and /var/tmp for Linux + - "${LOCAL_TMP_DIR}" + ########### + ## MySQL ## + ########### + mysql: + image: mysql/mysql-server:5.7.27 + restart: "no" + ports: + - "15500:3306" + expose: + - "3306" + environment: + - MYSQL_ALLOW_EMPTY_PASSWORD=true + - MYSQL_DATABASE=lbry + - MYSQL_USER=lbry + - MYSQL_PASSWORD=lbry + - MYSQL_LOG_CONSOLE=true + ################### + ## Internal APIs ## + ################### + internalapis: + image: lbry/internal-apis:master + restart: "no" + ports: + - "15400:8080" + expose: + - "8080" + depends_on: + - mysql + - lbrycrd + - lbrynet + environment: + - MYSQL_DSN=lbry:lbry@tcp(mysql:3306)/lbry + - LBRYCRD_CONNECT=rpc://lbry:lbry@lbrycrd:29245 + - MYSQL_USER=lbry + - MYSQL_PASS=lbry + - MYSQL_DATABASE=lbry + entrypoint: wait-for-it mysql:3306 -- wait-for-it lbrynet:5279 -- ./latest serve \ No newline at end of file diff --git a/e2e/e2e.sh b/e2e/e2e.sh new file mode 100755 index 0000000..027b5d0 --- /dev/null +++ b/e2e/e2e.sh @@ -0,0 +1,65 @@ +#!/usr/bin/env bash + +set -e + +#Always compile ytsync +make + +#OVERRIDE this in your .env file if running from mac. Check docker-compose.yml for details +export LOCAL_TMP_DIR="/var/tmp:/var/tmp" + +#Private Variables Set in local installations: SLACK_TOKEN,YOUTUBE_API_KEY,AWS_S3_ID,AWS_S3_SECRET,AWS_S3_REGION,AWS_S3_BUCKET +touch -a .env && set -o allexport; source ./.env; set +o allexport +echo "LOCAL_TMP_DIR=$LOCAL_TMP_DIR" +# Compose settings - docker only +export SLACK_CHANNEL="ytsync-travis" +export LBRY_API_TOKEN="ytsyntoken" +export LBRY_WEB_API="http://localhost:15400" +export LBRYNET_ADDRESS="http://localhost:15100" +export LBRYCRD_STRING="tcp://lbry:lbry@localhost:15200" +export LBRYNET_USE_DOCKER=true +export REFLECT_BLOBS=false +export CLEAN_ON_STARTUP=true +export REGTEST=true +# Local settings +export BLOBS_DIRECTORY="$(pwd)/e2e/blobsfiles" +export LBRYNET_DIR="$(pwd)/e2e/persist/.lbrynet/.local/share/lbry/lbrynet/" +export LBRYNET_WALLETS_DIR="$(pwd)/e2e/persist/.lbrynet/.local/share/lbry/lbryum" +export TMP_DIR="/var/tmp" +export UID + +cd ./e2e +docker-compose stop +docker-compose rm -f +echo "$DOCKER_PASSWORD" | docker login --username "$DOCKER_USERNAME" --password-stdin +docker-compose pull +if [[ -d persist ]]; then rm -rf persist; fi +mkdir -m 0777 -p ./persist +mkdir -m 777 -p ./persist/.walletserver +mkdir -m 777 -p ./persist/.lbrynet +#sudo chown -Rv 999:999 ./persist/.walletserver +#sudo chown -Rv 1000:1000 ./persist/.lbrynet +docker-compose up -d +printf 'waiting for internal apis' +until curl --output /dev/null --silent --head --fail http://localhost:15400; do + printf '.' + sleep 1 +done +echo "successfully started..." + +#Data Setup for test +./data_setup.sh + +# Execute the test! +./../bin/ytsync --channelID UCCyr5j8akeu9j4Q7urV0Lqw #Force channel intended...just in case. This channel lines up with the api container +# Assert the status +status=$(mysql -u lbry -plbry -ss -D lbry -h "127.0.0.1" -P 15500 -e 'SELECT status FROM youtube_data WHERE id=1') +videoStatus=$(mysql -u lbry -plbry -ss -D lbry -h "127.0.0.1" -P 15500 -e 'SELECT status FROM synced_video WHERE id=1') +if [[ $status != "synced" || $videoStatus != "published" ]]; then +docker-compose logs --tail="all" lbrycrd +docker-compose logs --tail="all" walletserver +docker-compose logs --tail="all" lbrynet +docker-compose logs --tail="all" internalapis +echo "List local /var/tmp" +find /var/tmp + exit 1; fi; \ No newline at end of file diff --git a/e2e/lbrycrd/docker-compose.yml b/e2e/lbrycrd/docker-compose.yml new file mode 100644 index 0000000..0254de3 --- /dev/null +++ b/e2e/lbrycrd/docker-compose.yml @@ -0,0 +1,26 @@ +version: "3" +networks: + lbry-network: + external: true + +services: + ############# + ## Lbrycrd ## + ############# + lbrycrd: + image: lbry/lbrycrd:v0.12.4.1 + restart: always + networks: + lbry-network: + ipv4_address: 10.6.1.1 + ports: + - "15201:29246" + - "15200:29245" + expose: + - "29246" + - "29245" + ## host volumes for persistent data such as wallet private keys. + volumes: + - "../persist/data:/data" + environment: + - RUN_MODE=regtest diff --git a/e2e/lbrycrd/docker/Dockerfile b/e2e/lbrycrd/docker/Dockerfile new file mode 100644 index 0000000..0d19eca --- /dev/null +++ b/e2e/lbrycrd/docker/Dockerfile @@ -0,0 +1,37 @@ +FROM ubuntu:18.04 as prep +LABEL MAINTAINER="leopere [at] nixc [dot] us" +## TODO: Implement version pinning. `apt-get install curl=` +RUN apt-get update && \ + apt-get -y install unzip curl build-essential && \ + apt-get autoclean -y && \ + rm -rf /var/lib/apt/lists/* +WORKDIR / +SHELL ["/bin/bash", "-o", "pipefail", "-c"] +COPY ./start.sh start +COPY ./healthcheck.sh healthcheck +COPY ./advance_blocks.sh advance +COPY ./fix-permissions.c fix-permissions.c + +## Add lbrycrd - Change the version below to create an image for a different tag/version +ARG VERSION="v0.12.4.1" +RUN URL=$(curl -s https://api.github.com/repos/lbryio/lbrycrd/releases/$(if [ "${VERSION}" = 'latest' ]; then echo "latest"; else echo "tags/${VERSION}"; fi) | grep browser_download_url | grep lbrycrd-linux.zip | cut -d'"' -f4) && echo $URL && curl -L -o /lbrycrd-linux.zip $URL + +RUN unzip ./lbrycrd-linux.zip && \ + gcc fix-permissions.c -o fix-permissions && \ + chmod +x ./lbrycrdd ./lbrycrd-cli ./lbrycrd-tx ./start ./healthcheck ./fix-permissions ./advance + +FROM ubuntu:18.04 as app +COPY --from=prep /lbrycrdd /lbrycrd-cli /lbrycrd-tx /start /healthcheck /fix-permissions /advance /usr/bin/ +RUN addgroup --gid 1000 lbrycrd && \ + adduser lbrycrd --uid 1000 --gid 1000 --gecos GECOS --shell /bin/bash --disabled-password --home /data && \ + mkdir /etc/lbry && \ + chown lbrycrd /etc/lbry && \ + chmod a+s /usr/bin/fix-permissions +VOLUME ["/data"] +WORKDIR /data +## TODO: Implement healthcheck. +# HEALTHCHECK ["healthcheck"] +EXPOSE 9246 9245 11337 29245 + +USER lbrycrd +CMD ["start"] \ No newline at end of file diff --git a/e2e/lbrycrd/docker/advance_blocks.sh b/e2e/lbrycrd/docker/advance_blocks.sh new file mode 100644 index 0000000..2a6c2e2 --- /dev/null +++ b/e2e/lbrycrd/docker/advance_blocks.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash +while true; do + lbrycrd-cli -conf=/etc/lbry/lbrycrd.conf generate 100 >> /tmp/output.log + sleep 2 +done \ No newline at end of file diff --git a/e2e/lbrycrd/docker/build.sh b/e2e/lbrycrd/docker/build.sh new file mode 100755 index 0000000..0487e72 --- /dev/null +++ b/e2e/lbrycrd/docker/build.sh @@ -0,0 +1,8 @@ +#!/bin/bash +if [ $# -eq 0 ] + then + echo "No docker tag argument supplied. Use './build.sh '" + exit 1 +fi +docker build --tag lbry/lbrycrd:$1 . +docker push lbry/lbrycrd:$1 \ No newline at end of file diff --git a/e2e/lbrycrd/docker/fix-permissions.c b/e2e/lbrycrd/docker/fix-permissions.c new file mode 100644 index 0000000..2f247e8 --- /dev/null +++ b/e2e/lbrycrd/docker/fix-permissions.c @@ -0,0 +1,9 @@ +#include +int main() { + // This program needs to run with setuid == root + // This needs to be in a compiled language because you cannot setuid bash scripts + setuid(0); + execle("/bin/bash", "bash", "-c", + "/bin/chown -R lbrycrd:lbrycrd /data && /bin/chmod -R 755 /data/", + (char*) NULL, (char*) NULL); +} \ No newline at end of file diff --git a/e2e/lbrycrd/docker/healthcheck.sh b/e2e/lbrycrd/docker/healthcheck.sh new file mode 100644 index 0000000..b9ee099 --- /dev/null +++ b/e2e/lbrycrd/docker/healthcheck.sh @@ -0,0 +1,4 @@ +## TODO: Implement a healthcheck for lbrycrd. +curl --data-binary '{"jsonrpc":"1.0","id":"curltext","method":"getinfo","params":[]}' -H 'content-type:text/plain;' http://$RPC_USER:$RPC_PASSWORD@127.0.0.1:9246 +## OR +lbrycrd-cli getinfo \ No newline at end of file diff --git a/e2e/lbrycrd/docker/start.sh b/e2e/lbrycrd/docker/start.sh new file mode 100644 index 0000000..89bbb76 --- /dev/null +++ b/e2e/lbrycrd/docker/start.sh @@ -0,0 +1,114 @@ +#!/usr/bin/env bash +CONFIG_PATH=/etc/lbry/lbrycrd.conf + +function override_config_option() { + # Remove existing config line from a config file + # and replace with environment fed value. + # Does nothing if the variable does not exist. + # var Name of ENV variable + # option Name of config option + # config Path of config file + local var=$1 option=$2 config=$3 + if [[ -v $var ]]; then + # Remove the existing config option: + sed -i "/^$option\W*=/d" $config + # Add the value from the environment: + echo "$option=${!var}" >> $config + fi +} + +function set_config() { + if [ -d "$CONFIG_PATH" ]; then + echo "$CONFIG_PATH is a directory when it should be a file." + exit 1 + elif [ -f "$CONFIG_PATH" ]; then + echo "Merging the mounted config file with environment variables." + local MERGED_CONFIG=/tmp/lbrycrd_merged.conf + cat $CONFIG_PATH > $MERGED_CONFIG + echo "" >> $MERGED_CONFIG + override_config_option PORT port $MERGED_CONFIG + override_config_option RPC_USER rpcuser $MERGED_CONFIG + override_config_option RPC_PASSWORD rpcpassword $MERGED_CONFIG + override_config_option RPC_ALLOW_IP rpcallowip $MERGED_CONFIG + override_config_option RPC_PORT rpcport $MERGED_CONFIG + override_config_option RPC_BIND rpcbind $MERGED_CONFIG + # Make the new merged config file the new CONFIG_PATH + # This ensures that the original file the user mounted remains unmodified + CONFIG_PATH=$MERGED_CONFIG + else + echo "Creating a fresh config file from environment variables." + ## Set config params + echo "port=${PORT=9246}" > $CONFIG_PATH + echo "rpcuser=${RPC_USER=lbry}" >> $CONFIG_PATH + echo "rpcpassword=${RPC_PASSWORD=lbry}" >> $CONFIG_PATH + echo "rpcallowip=${RPC_ALLOW_IP=127.0.0.1/24}" >> $CONFIG_PATH + echo "rpcport=${RPC_PORT=9245}" >> $CONFIG_PATH + echo "rpcbind=${RPC_BIND=0.0.0.0}" >> $CONFIG_PATH + fi + echo "Config: " + cat $CONFIG_PATH +} + +## Ensure perms are correct prior to running main binary +/usr/bin/fix-permissions + +## You can optionally specify a run mode if you want to use lbry defined presets for compatibility. +case $RUN_MODE in + default ) + set_config + lbrycrdd -server -conf=$CONFIG_PATH -printtoconsole + ;; + ## If it's a first run you need to do a full index including all transactions + ## tx index creates an index of every single transaction in the block history if + ## not specified it will only create an index for transactions that are related to the wallet or have unspent outputs. + ## This is generally specific to chainquery. + reindex ) + ## Apply this RUN_MODE in the case you need to update a dataset. NOTE: you do not need to use `RUN_MODE reindex` for more than one complete run. + set_config + lbrycrdd -server -txindex -reindex -conf=$CONFIG_PATH -printtoconsole + ;; + chainquery ) + ## If your only goal is to run Chainquery against this instance of lbrycrd and you're starting a + ## fresh local dataset use this run mode. + set_config + lbrycrdd -server -txindex -conf=$CONFIG_PATH -printtoconsole + ;; + regtest ) + ## Set config params + ## TODO: Make this more automagic in the future. + mkdir -p `dirname $CONFIG_PATH` + echo "rpcuser=lbry" > $CONFIG_PATH + echo "rpcpassword=lbry" >> $CONFIG_PATH + echo "rpcport=29245" >> $CONFIG_PATH + echo "rpcbind=0.0.0.0" >> $CONFIG_PATH + echo "rpcallowip=0.0.0.0/0" >> $CONFIG_PATH + echo "regtest=1" >> $CONFIG_PATH + echo "txindex=1" >> $CONFIG_PATH + echo "server=1" >> $CONFIG_PATH + echo "printtoconsole=1" >> $CONFIG_PATH + + #nohup advance &>/dev/null & + lbrycrdd -conf=$CONFIG_PATH $1 + ;; + testnet ) + ## Set config params + ## TODO: Make this more automagic in the future. + mkdir -p `dirname $CONFIG_PATH` + echo "rpcuser=lbry" > $CONFIG_PATH + echo "rpcpassword=lbry" >> $CONFIG_PATH + echo "rpcport=29245" >> $CONFIG_PATH + echo "rpcbind=0.0.0.0" >> $CONFIG_PATH + echo "rpcallowip=0.0.0.0/0" >> $CONFIG_PATH + echo "testnet=1" >> $CONFIG_PATH + echo "txindex=1" >> $CONFIG_PATH + echo "server=1" >> $CONFIG_PATH + echo "printtoconsole=1" >> $CONFIG_PATH + + #nohup advance &>/dev/null & + lbrycrdd -conf=$CONFIG_PATH $1 + ;; + * ) + echo "Error, you must define a RUN_MODE environment variable." + echo "Available options are testnet, regtest, chainquery, default, and reindex" + ;; +esac \ No newline at end of file diff --git a/e2e/lbrynet/docker-compose.yml b/e2e/lbrynet/docker-compose.yml new file mode 100644 index 0000000..d49c0f7 --- /dev/null +++ b/e2e/lbrynet/docker-compose.yml @@ -0,0 +1,23 @@ +version: "3" +networks: + lbry-network: + external: true + +services: + ############# + ## Lbrynet ## + ############# + lbrynet: + image: lbry/lbrynet:v0.38.5 + restart: "no" + networks: + lbry-network: + ipv4_address: 10.6.1.3 + ports: + - "15100:5279" + - "15101:5280" + environment: + - LBRY_STREAMING_SERVER=0.0.0.0:5280 + volumes: + - "../persist/data/.lbrynet:/home/lbrynet" + - "./settings:/etc/lbry" #Put your daemon_settings.yml here diff --git a/e2e/lbrynet/docker/Dockerfile b/e2e/lbrynet/docker/Dockerfile new file mode 100644 index 0000000..d7ae70a --- /dev/null +++ b/e2e/lbrynet/docker/Dockerfile @@ -0,0 +1,27 @@ +## This base image is for running the latest lbrynet-daemon release. +FROM ubuntu:16.04 as prep +LABEL MAINTAINER="leopere [at] nixc [dot] us" +RUN apt-get update && apt-get -y install unzip curl telnet wait-for-it + +## Add lbrynet +ARG VERSION="v0.38.5" +RUN URL=$(curl -s https://api.github.com/repos/lbryio/lbry-sdk/releases/$(if [ "${VERSION}" = 'latest' ]; then echo "latest"; else echo "tags/${VERSION}"; fi) | grep browser_download_url | grep lbrynet-linux.zip | cut -d'"' -f4) && echo $URL && curl -L -o /lbrynet.linux.zip $URL + +COPY start.sh /usr/bin/start +COPY checkmount.sh /usr/bin/checkmount +RUN unzip /lbrynet.linux.zip -d /lbrynet/ && \ + mv /lbrynet/lbrynet /usr/bin && \ + chmod a+x /usr/bin/checkmount /usr/bin/start /usr/bin/lbrynet + +FROM ubuntu:16.04 as app +COPY --from=prep /usr/bin/start /usr/bin/checkmount /usr/bin/lbrynet /usr/bin/ +RUN adduser lbrynet --gecos GECOS --shell /bin/bash --disabled-password --home /home/lbrynet +## Daemon port [Intended for internal use] +## LBRYNET talks to peers on port 3333 [Intended for external use] this port is used to discover other lbrynet daemons with blobs. +## Expose 5566 Reflector port to listen on +## Expose 5279 Port the daemon API will listen on +## the lbryumx aka Wallet port [Intended for internal use] +#EXPOSE 4444 3333 5566 5279 50001 +USER lbrynet +ENTRYPOINT ["/usr/bin/checkmount"] +CMD ["start"] \ No newline at end of file diff --git a/e2e/lbrynet/docker/build.sh b/e2e/lbrynet/docker/build.sh new file mode 100755 index 0000000..8cb5ebf --- /dev/null +++ b/e2e/lbrynet/docker/build.sh @@ -0,0 +1,8 @@ +#!/bin/bash +if [ $# -eq 0 ] + then + echo "No docker tag argument supplied. Use './build.sh '" + exit 1 +fi +docker build --tag lbry/lbrynet:$1 . +docker push lbry/lbrynet:$1 \ No newline at end of file diff --git a/e2e/lbrynet/docker/checkmount.sh b/e2e/lbrynet/docker/checkmount.sh new file mode 100644 index 0000000..4f77c11 --- /dev/null +++ b/e2e/lbrynet/docker/checkmount.sh @@ -0,0 +1,12 @@ +#!/bin/bash + +## TODO: Make a bit more aware of the run mode of this appliance in case there is ever a test mode enabled in the start.sh +mountpoint=/home/lbrynet + +if ! grep -qs ".* $mountpoint " /proc/mounts; then + echo "$mountpoint not mounted, refusing to run." + ## TODO: We should have documentation that this error references directly with a URL as to why it won't run without a volume. + exit 1 +else + bash -c "$*" +fi diff --git a/e2e/lbrynet/docker/start.sh b/e2e/lbrynet/docker/start.sh new file mode 100644 index 0000000..69beadc --- /dev/null +++ b/e2e/lbrynet/docker/start.sh @@ -0,0 +1,4 @@ +#!/bin/bash +lbrynet start \ + --api="${API_BIND_IP:-0.0.0.0}":"${API_PORT:-5279}" \ + --config="${CONFIG_PATH:-/etc/lbry/daemon_settings.yml}" \ No newline at end of file diff --git a/e2e/lbrynet/settings/daemon_settings.yml b/e2e/lbrynet/settings/daemon_settings.yml new file mode 100644 index 0000000..8a6bcb4 --- /dev/null +++ b/e2e/lbrynet/settings/daemon_settings.yml @@ -0,0 +1,12 @@ +#blockchain_name: lbrycrd_main +#blockchain_name: lbrycrd_testnet +blockchain_name: lbrycrd_regtest +lbryum_servers: +# - spv1.lbry.com:50001 #Production Wallet Server + - walletserver:50001 +save_blobs: true +save_files: false +share_usage_data: false +tcp_port: 3333 +udp_port: 4444 +use_upnp: true \ No newline at end of file diff --git a/e2e/walletserver/docker-compose.yml b/e2e/walletserver/docker-compose.yml new file mode 100644 index 0000000..a3104ba --- /dev/null +++ b/e2e/walletserver/docker-compose.yml @@ -0,0 +1,35 @@ +version: "3" +networks: + lbry-network: + external: true + +services: + ################### + ## Wallet Server ## + ################### + walletserver: + image: lbry/wallet-server:v0.38.5 + restart: always + networks: + lbry-network: + ipv4_address: 10.6.1.2 + volumes: + - "../persist/data/.walletserver/database:/database" + environment: + - DB_DIRECTORY=/database + - MAX_SEND=1000000000000000000000 + - DAEMON_URL=http://lbry:lbry@lbrycrd:29245/ + - MAX_SUBS=1000000000000 + - BANDWIDTH_LIMIT=80000000000 + - SESSION_TIMEOUT=10000000000000000000000000 + - TCP_PORT=50001 + #network_mode: host + #network_mode: bridge + ports: + - "50001:50001" + expose: + - "50001" + ulimits: + nofile: 90000 +# command: lbry.wallet.server.coin.LBC + command: lbry.wallet.server.coin.LBCRegTest \ No newline at end of file diff --git a/go.mod b/go.mod index 5498de8..1bdb2ee 100644 --- a/go.mod +++ b/go.mod @@ -1,46 +1,34 @@ module github.com/lbryio/ytsync require ( - cloud.google.com/go v0.37.4 // indirect github.com/ChannelMeter/iso8601duration v0.0.0-20150204201828-8da3af7a2a61 + github.com/Microsoft/go-winio v0.4.13 // indirect github.com/armon/go-metrics v0.0.0-20190430140413-ec5e00d3c878 // indirect github.com/asaskevich/govalidator v0.0.0-20180720115003-f9ffefc3facf github.com/aws/aws-sdk-go v1.17.3 github.com/channelmeter/iso8601duration v0.0.0-20150204201828-8da3af7a2a61 // indirect - github.com/go-ini/ini v1.42.0 // indirect - github.com/go-sql-driver/mysql v1.4.1 // indirect - github.com/gopherjs/gopherjs v0.0.0-20190411002643-bd77b112433e // indirect + github.com/docker/distribution v2.7.1+incompatible // indirect + github.com/docker/docker v1.13.1 + github.com/docker/go-connections v0.4.0 // indirect + github.com/docker/go-units v0.4.0 // indirect github.com/hashicorp/go-msgpack v0.5.5 // indirect - github.com/hashicorp/golang-lru v0.5.1 // indirect github.com/hashicorp/memberlist v0.1.4 // indirect github.com/hashicorp/serf v0.8.2 // indirect - github.com/konsorten/go-windows-terminal-sequences v1.0.2 // indirect github.com/kr/pretty v0.1.0 // indirect github.com/lbryio/errors.go v0.0.0-20180223142025-ad03d3cc6a5c - github.com/lbryio/lbry.go v1.0.15 + github.com/lbryio/lbry.go v1.0.17 github.com/lbryio/reflector.go v1.0.6-0.20190806185326-2e4f235489f4 - github.com/lusis/slack-test v0.0.0-20190408224659-6cf59653add2 // indirect github.com/mitchellh/go-ps v0.0.0-20170309133038-4fdf99ab2936 - github.com/onsi/ginkgo v1.8.0 // indirect - github.com/onsi/gomega v1.5.0 // indirect - github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2 // indirect - github.com/prometheus/common v0.3.0 + github.com/opencontainers/go-digest v1.0.0-rc1 // indirect github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24 github.com/sirupsen/logrus v1.4.1 - github.com/smartystreets/assertions v0.0.0-20190401211740-f487f9de1cd3 // indirect - github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a // indirect github.com/spf13/cobra v0.0.0-20190109003409-7547e83b2d85 github.com/spf13/pflag v1.0.3 // indirect - github.com/ybbus/jsonrpc v2.1.2+incompatible // indirect - go.opencensus.io v0.20.2 // indirect - golang.org/x/crypto v0.0.0-20190418165655-df01cb2cc480 // indirect + golang.org/x/crypto v0.0.0-20190325154230-a5d413f7728c // indirect golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c // indirect - golang.org/x/oauth2 v0.0.0-20190402181905-9f3314589c9a // indirect golang.org/x/sync v0.0.0-20190423024810-112230192c58 // indirect + golang.org/x/sys v0.0.0-20190523142557-0e01d883c5c5 // indirect golang.org/x/text v0.3.2 // indirect google.golang.org/api v0.3.2 - google.golang.org/appengine v1.5.0 // indirect - google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7 // indirect google.golang.org/grpc v1.20.0 // indirect - gopkg.in/ini.v1 v1.42.0 // indirect ) diff --git a/go.sum b/go.sum index 5d33bb0..72e4dbe 100644 --- a/go.sum +++ b/go.sum @@ -1,21 +1,19 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.34.0 h1:eOI3/cP2VTU6uZLDYAoic+eyzzB9YyGmJ7eIjl8rOPg= cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -cloud.google.com/go v0.37.4 h1:glPeL3BQJsbF6aIIYfZizMwc5LTYz250bDMjttbBGAU= -cloud.google.com/go v0.37.4/go.mod h1:NHPJ89PdicEuT9hdPXMROBD91xc5uRDxsMtSB16k7hw= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/ChannelMeter/iso8601duration v0.0.0-20150204201828-8da3af7a2a61 h1:N5Vqww5QISEHsWHOWDEx4PzdIay3Cg0Jp7zItq2ZAro= github.com/ChannelMeter/iso8601duration v0.0.0-20150204201828-8da3af7a2a61/go.mod h1:GnKXcK+7DYNy/8w2Ex//Uql4IgfaU82Cd5rWKb7ah00= github.com/DataDog/datadog-go v2.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ= +github.com/Microsoft/go-winio v0.4.13 h1:Hmi80lzZuI/CaYmlJp/b+FjZdRZhKu9c2mDVqKlLWVs= +github.com/Microsoft/go-winio v0.4.13/go.mod h1:qXqCSQ3Xa7+6tgxaGTIe4Kpcdsi+P8jBhyzoq1bpyYA= github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo= github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI= github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII= -github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc h1:cAKDfWh5VpdgMhJosfJnn5/FoN2SRZ4p7fJNX58YPaU= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= -github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf h1:qet1QNfXsQxTZqLG4oE62mJzwPIB8+Tee4RNCL9ulrY= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= -github.com/armon/go-metrics v0.0.0-20180713145231-3c58d8115a78 h1:mdRSArcFLfW0VoL34LZAKSz6LkkK4jFxVx2xYavACMg= github.com/armon/go-metrics v0.0.0-20180713145231-3c58d8115a78/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= github.com/armon/go-metrics v0.0.0-20190430140413-ec5e00d3c878 h1:EFSB7Zo9Eg91v7MJPVsifUysc/wPdN+NOnVe6bWbdBM= @@ -56,6 +54,14 @@ github.com/davecgh/go-spew v0.0.0-20171005155431-ecdeabc65495/go.mod h1:J7Y8YcW2 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/docker/distribution v2.7.1+incompatible h1:a5mlkVzth6W5A4fOsS3D2EO5BUmsJpcB+cRlLU7cSug= +github.com/docker/distribution v2.7.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= +github.com/docker/docker v1.13.1 h1:IkZjBSIc8hBjLpqeAbeE5mca5mNgeatLHBy3GO78BWo= +github.com/docker/docker v1.13.1/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ= +github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec= +github.com/docker/go-units v0.4.0 h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw= +github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs= github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU= github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= @@ -68,15 +74,13 @@ github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMo github.com/go-errors/errors v1.0.1 h1:LUHzmkK3GUKUrL/1gfBUxAHzcev3apQlezX/+O7ma6w= github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q= github.com/go-ini/ini v1.38.2/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= +github.com/go-ini/ini v1.41.0 h1:526aoxDtxRHFQKMZfcX2OG9oOI8TJ5yPLM0Mkno/uTY= github.com/go-ini/ini v1.41.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= -github.com/go-ini/ini v1.42.0 h1:TWr1wGj35+UiWHlBA8er89seFXxzwFn11spilrrj+38= -github.com/go-ini/ini v1.42.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-ozzo/ozzo-validation v3.5.0+incompatible/go.mod h1:gsEKFIVnabGBt6mXmxK0MoFy+cZoTJY6mu5Ll3LVLBU= +github.com/go-sql-driver/mysql v0.0.0-20180719071942-99ff426eb706 h1:P3NPKb7qq581SeMCB+dU1SuCX1kQh8VoQ/4HmT2ftQY= github.com/go-sql-driver/mysql v0.0.0-20180719071942-99ff426eb706/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= -github.com/go-sql-driver/mysql v1.4.1 h1:g24URVg0OFbNUTx9qqY1IRZ9D9z3iPyi5zKhQZpNwpA= -github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/gogo/googleapis v1.1.0/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFGgqEef3s= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= @@ -95,13 +99,8 @@ github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c h1:964Od4U6p2jUkFxvCy github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0 h1:+dTQ8DZQJz0Mb/HjFlkptS1FeQ4cWSnN941F8aEG4SQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= -github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= -github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= -github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= -github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= +github.com/gopherjs/gopherjs v0.0.0-20181103185306-d547d1d9531e h1:JKmoR8x90Iww1ks85zJ1lfDGgIiMDuIptTOhJq+zKyg= github.com/gopherjs/gopherjs v0.0.0-20181103185306-d547d1d9531e/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= -github.com/gopherjs/gopherjs v0.0.0-20190411002643-bd77b112433e h1:XWcjeEtTFTOVA9Fs1w7n2XBftk5ib4oZrhzWk0B+3eA= -github.com/gopherjs/gopherjs v0.0.0-20190411002643-bd77b112433e/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gorilla/context v1.1.1 h1:AWwleXJkX/nhcU9bZSnZoi3h/qGYqQAGhq6zZe/aQW8= github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg= github.com/gorilla/mux v1.6.2 h1:Pgr17XVTNXAk3q/r4CpKzC5xBM/qW1uVLV+IhRZpIIk= @@ -111,26 +110,21 @@ github.com/gorilla/rpc v1.1.0/go.mod h1:V4h9r+4sF5HnzqbwIez0fKSpANP0zlYd3qR7p36j github.com/gorilla/websocket v1.2.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gorilla/websocket v1.4.0 h1:WDFjx/TMzVgy9VdMMQi2K2Emtwi2QcUQsztZ/zLaH/Q= github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= -github.com/hashicorp/errwrap v0.0.0-20180715044906-d6c0cd880357 h1:Rem2+U35z1QtPQc6r+WolF7yXiefXqDKyk+lN2pE164= github.com/hashicorp/errwrap v0.0.0-20180715044906-d6c0cd880357/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/go-cleanhttp v0.5.0/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= -github.com/hashicorp/go-immutable-radix v0.0.0-20180129170900-7f3cd4390caa h1:0nA8i+6Rwqaq9xlpmVxxTwk6rxiEhX+E6Wh4vPNHiS8= github.com/hashicorp/go-immutable-radix v0.0.0-20180129170900-7f3cd4390caa/go.mod h1:6ij3Z20p+OhOkCSrA0gImAWoHYQRGbnlcuk6XYTiaRw= github.com/hashicorp/go-immutable-radix v1.0.0 h1:AKDB1HM5PWEA7i4nhcpwOrO2byshxBjXVn/J/3+z5/0= github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= -github.com/hashicorp/go-msgpack v0.0.0-20150518234257-fa3f63826f7c h1:BTAbnbegUIMB6xmQCwWE8yRzbA4XSpnZY5hvRJC188I= github.com/hashicorp/go-msgpack v0.0.0-20150518234257-fa3f63826f7c/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= github.com/hashicorp/go-msgpack v0.5.5 h1:i9R9JSrqIz0QVLz3sz+i3YJdT7TTSLcfLLzJi9aZTuI= github.com/hashicorp/go-msgpack v0.5.5/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= -github.com/hashicorp/go-multierror v0.0.0-20180717150148-3d5d8f294aa0 h1:j30noezaCfvNLcdMYSvHLv81DxYRSt1grlpseG67vhU= github.com/hashicorp/go-multierror v0.0.0-20180717150148-3d5d8f294aa0/go.mod h1:JMRHfdO9jKNzS/+BTlxCjKNQHg/jZAft8U7LloJvN7I= github.com/hashicorp/go-multierror v1.0.0 h1:iVjPR7a6H0tWELX5NxNe7bYopibicUzc7uPribsnS6o= github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= github.com/hashicorp/go-retryablehttp v0.5.3/go.mod h1:9B5zBasrRhHXnJnui7y6sL7es7NDiJgTc6Er0maI1Xs= -github.com/hashicorp/go-sockaddr v0.0.0-20180320115054-6d291a969b86 h1:7YOlAIO2YWnJZkQp7B5eFykaIY7C9JndqAFQyVV5BhM= github.com/hashicorp/go-sockaddr v0.0.0-20180320115054-6d291a969b86/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= github.com/hashicorp/go-sockaddr v1.0.0 h1:GeH6tui99pF4NJgfnhp+L6+FfobzVW3Ah46sLo0ICXs= github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= @@ -140,17 +134,14 @@ github.com/hashicorp/go-uuid v1.0.1 h1:fv1ep09latC32wFoVwnqcnKJGnMSdBanPczbHAYm1 github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= github.com/hashicorp/golang-lru v0.0.0-20180201235237-0fb14efe8c47/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru v0.5.0 h1:CL2msUPvZTLb5O648aiLNJw3hnBxN2+1Jq8rCOH9wdo= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/hashicorp/golang-lru v0.5.1 h1:0hERBMJE1eitiLkihrMvRVBYAkpHzc/J3QdDN+dAcgU= -github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ= -github.com/hashicorp/memberlist v0.1.0 h1:qSsCiC0WYD39lbSitKNt40e30uorm2Ss/d4JGU1hzH8= github.com/hashicorp/memberlist v0.1.0/go.mod h1:ncdBp14cuox2iFOq3kDiquKU6fqsTBc3W6JvZwjxxsE= github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= github.com/hashicorp/memberlist v0.1.4 h1:gkyML/r71w3FL8gUi74Vk76avkj/9lYAY9lvg0OcoGs= github.com/hashicorp/memberlist v0.1.4/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= -github.com/hashicorp/serf v0.0.0-20180530155958-984a73625de3 h1:NUr1hG6WO9sI1x8ofSimmpqfJ+rEHiHP/PLEA33rcfQ= github.com/hashicorp/serf v0.0.0-20180530155958-984a73625de3/go.mod h1:h/Ru6tmZazX7WO/GDmwdpS975F019L4t5ng5IgwbNrE= github.com/hashicorp/serf v0.8.2 h1:YZ7UKsJv+hKjqGVUUbtE3HNj79Eln2oQ75tniF6iPt0= github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= @@ -169,17 +160,14 @@ github.com/johntdyer/slack-go v0.0.0-20180213144715-95fac1160b22/go.mod h1:u0Jo4 github.com/johntdyer/slackrus v0.0.0-20180518184837-f7aae3243a07 h1:+kBG/8rjCa6vxJZbUjAiE4MQmBEBYc8nLEb51frnvBY= github.com/johntdyer/slackrus v0.0.0-20180518184837-f7aae3243a07/go.mod h1:j1kV/8f3jowErEq4XyeypkCdvg5EeHkf0YCKCcq5Ybo= github.com/jrick/logrotate v1.0.0/go.mod h1:LNinyqDIJnpAur+b8yyulnQw/wDuN1+BYKlTRt3OuAQ= -github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= +github.com/jtolds/gls v4.2.1+incompatible h1:fSuqC+Gmlu6l/ZYAoZzx2pyucC8Xza35fpRVWLVmUEE= github.com/jtolds/gls v4.2.1+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= -github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= -github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4= github.com/kkdai/bstream v0.0.0-20181106074824-b3251f7901ec/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4= +github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/konsorten/go-windows-terminal-sequences v1.0.2 h1:DB17ag19krx9CFsz4o3enTrPXyIXCl+2iCXH/aMAp9s= -github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= @@ -189,10 +177,11 @@ github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/lbryio/errors.go v0.0.0-20180223142025-ad03d3cc6a5c h1:BhdcWGsuKif/XoSZnqVGNqJ1iEmH0czWR5upj+AuR8M= github.com/lbryio/errors.go v0.0.0-20180223142025-ad03d3cc6a5c/go.mod h1:muH7wpUqE8hRA3OrYYosw9+Sl681BF9cwcjzE+OCNK8= github.com/lbryio/lbry.go v0.0.0-20190109223729-30c312501602/go.mod h1:YEuFJD/oHNra6BFy+NfuvS84Wg6RMWJFGtiCCCc6MmQ= -github.com/lbryio/lbry.go v1.0.15 h1:g4g9cDDUsobmWgTA+1jEIb5k2fRCP0/NvPOMXduP8xY= -github.com/lbryio/lbry.go v1.0.15/go.mod h1:JtyI30bU51rm0LZ/po3mQuzf++14OWb6kR/6mMRAmKU= +github.com/lbryio/lbry.go v1.0.17 h1:Lt7xIguw6Q5LMKcEMfxD7IB7kUdP7xwrc0OxvBUl39E= +github.com/lbryio/lbry.go v1.0.17/go.mod h1:JtyI30bU51rm0LZ/po3mQuzf++14OWb6kR/6mMRAmKU= github.com/lbryio/lbryschema.go v0.0.0-20190428231007-c54836bca002 h1:urfYK5ElpUrAv90auPLldoVC60LwiGAcY0OE6HJB9KI= github.com/lbryio/lbryschema.go v0.0.0-20190428231007-c54836bca002/go.mod h1:dAzPCBj3CKKWBGYBZxK6tKBP5SCgY2tqd9SnQd/OyKo= +github.com/lbryio/ozzo-validation v0.0.0-20170323141101-d1008ad1fd04 h1:Nze+C2HbeKvhjI/kVn+9Poj/UuEW5sOQxcsxqO7L3GI= github.com/lbryio/ozzo-validation v0.0.0-20170323141101-d1008ad1fd04/go.mod h1:fbG/dzobG8r95KzMwckXiLMHfFjZaBRQqC9hPs2XAQ4= github.com/lbryio/reflector.go v1.0.6-0.20190806185326-2e4f235489f4 h1:SpUbq2YBg3ncetkw8APUgn8nFF8dscKzzhyiWMM7XCc= github.com/lbryio/reflector.go v1.0.6-0.20190806185326-2e4f235489f4/go.mod h1:7Y3YYeAKS6egH2WzwfU8f6+uNGjVHfLzKvwn+Nv3VMY= @@ -201,16 +190,14 @@ github.com/lbryio/types v0.0.0-20190422033210-321fb2abda9c h1:m3O7561xBQ00lfUVay github.com/lbryio/types v0.0.0-20190422033210-321fb2abda9c/go.mod h1:CG3wsDv5BiVYQd5i1Jp7wGsaVyjZTJshqXeWMVKsISE= github.com/lusis/go-slackbot v0.0.0-20180109053408-401027ccfef5 h1:AsEBgzv3DhuYHI/GiQh2HxvTP71HCCE9E/tzGUzGdtU= github.com/lusis/go-slackbot v0.0.0-20180109053408-401027ccfef5/go.mod h1:c2mYKRyMb1BPkO5St0c/ps62L4S0W2NAkaTXj9qEI+0= +github.com/lusis/slack-test v0.0.0-20180109053238-3c758769bfa6 h1:iOAVXzZyXtW408TMYejlUPo6BIn92HmOacWtIfNyYns= github.com/lusis/slack-test v0.0.0-20180109053238-3c758769bfa6/go.mod h1:sFlOUpQL1YcjhFVXhg1CG8ZASEs/Mf1oVb6H75JL/zg= -github.com/lusis/slack-test v0.0.0-20190408224659-6cf59653add2 h1:CTzpAplpuXay3gDDqbErv/OSV6SoKZ9UTsGxLKfxB8M= -github.com/lusis/slack-test v0.0.0-20190408224659-6cf59653add2/go.mod h1:sFlOUpQL1YcjhFVXhg1CG8ZASEs/Mf1oVb6H75JL/zg= github.com/lyft/protoc-gen-validate v0.0.13/go.mod h1:XbGvPuh87YZc5TdIa2/I4pLk0QoUACkjt2znoq26NVQ= github.com/lyoshenka/bencode v0.0.0-20180323155644-b7abd7672df5 h1:mG83tLXWSRdcXMWfkoumVwhcCbf3jHF9QKv/m37BkM0= github.com/lyoshenka/bencode v0.0.0-20180323155644-b7abd7672df5/go.mod h1:H0aPCWffGOaDcjkw1iB7W9DVLp6GXmfcJY/7YZCWPA4= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= -github.com/miekg/dns v1.0.8 h1:Zi8HNpze3NeRWH1PQV6O71YcvJRQ6j0lORO6DAEmAAI= github.com/miekg/dns v1.0.8/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/miekg/dns v1.0.14 h1:9jZdLNd/P4+SfEJ0TNyxYpsK8N4GtfylBLqtbYN1sbA= github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= @@ -229,20 +216,19 @@ github.com/nlopes/slack v0.4.0/go.mod h1:jVI4BBK3lSktibKahxBF74txcK2vyvkza1z/+rR github.com/nlopes/slack v0.5.0 h1:NbIae8Kd0NpqaEI3iUrsuS0KbcEDhzhc939jLW5fNm0= github.com/nlopes/slack v0.5.0/go.mod h1:jVI4BBK3lSktibKahxBF74txcK2vyvkza1z/+rRnVAM= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.7.0 h1:WSHQ+IS43OoUrWtD1/bbclrwK8TTH5hzp+umCiuxHgs= github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.8.0 h1:VkHVNpR4iVnU8XQR6DBm8BqYjN7CRzw+xKUbVVbbW9w= -github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/gomega v1.4.1/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= +github.com/onsi/gomega v1.4.3 h1:RE1xgDvH7imwFD45h+u2SgIfERHlS2yNG4DObb5BSKU= github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= -github.com/onsi/gomega v1.5.0 h1:izbySO9zDPmjJ8rDjLvkA2zJHIo+HkYXHnf7eN7SSyo= -github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= +github.com/opencontainers/go-digest v1.0.0-rc1 h1:WzifXhOVOEOuFYOJAW6aQqW0TooG2iki3E3Ii+WN7gQ= +github.com/opencontainers/go-digest v1.0.0-rc1/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= github.com/openzipkin/zipkin-go v0.1.6/go.mod h1:QgAqvLzwWbR/WpD4A3cGpPtJrZXNIiJc5AZX7/PBEpw= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/pascaldekloe/goe v0.1.0 h1:cBOtyMzM9HTpWjXfbbunk26uA6nG3a8n06Wieeh0MwY= github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= +github.com/phayes/freeport v0.0.0-20171002185219-e27662a4a9d6 h1:2bae6N0SZjgzk+Zg8mzTsfmpwHXY9VBNp9UdjhaElA0= github.com/phayes/freeport v0.0.0-20171002185219-e27662a4a9d6/go.mod h1:iIss55rKnNBTvrwdmkUpLnDpZoAHvWaiq5+iMmen4AE= -github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2 h1:JhzVVoYvbOACxoUmOs6V/G4D5nPVUW73rKvXxP4XUJc= -github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2/go.mod h1:iIss55rKnNBTvrwdmkUpLnDpZoAHvWaiq5+iMmen4AE= github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= @@ -257,8 +243,6 @@ github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1: github.com/prometheus/client_model v0.0.0-20190115171406-56726106282f/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/common v0.0.0-20181126121408-4724e9255275/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= github.com/prometheus/common v0.2.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= -github.com/prometheus/common v0.3.0 h1:taZ4h8Tkxv2kNyoSctBvfXEHmBmxrwmIidZTIaHons4= -github.com/prometheus/common v0.3.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= @@ -278,13 +262,10 @@ github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPx github.com/sirupsen/logrus v1.3.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.1 h1:GL2rEmy6nsikmW0r8opw9JIRScdMF5hA8cOYLH7In1k= github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= +github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= -github.com/smartystreets/assertions v0.0.0-20190401211740-f487f9de1cd3 h1:hBSHahWMEgzwRyS6dRpxY0XyjZsHyQ61s084wo5PJe0= -github.com/smartystreets/assertions v0.0.0-20190401211740-f487f9de1cd3/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= +github.com/smartystreets/goconvey v0.0.0-20181108003508-044398e4856c h1:Ho+uVpkel/udgjbwB5Lktg9BtvJSh2DT0Hi6LPSyI2w= github.com/smartystreets/goconvey v0.0.0-20181108003508-044398e4856c/go.mod h1:XDJAKZRPZ1CvBcN2aX5YOUTYGHki24fSF0Iv48Ibg0s= -github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a h1:pa8hGb/2YqsZKovtsgrwcDH1RZhVbTKCjLp47XpqCDs= -github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= -github.com/spf13/cast v1.2.0 h1:HHl1DSRbEQN2i8tJmtS6ViPyHx35+p51amrdsiTCrkg= github.com/spf13/cast v1.2.0/go.mod h1:r2rcYCSwa1IExKTDiTfzaxqT2FNHs8hODu4LnUfgKEg= github.com/spf13/cast v1.3.0 h1:oget//CVOEoFewqQxwr0Ej5yjygnqGkvggSE/gB35Q8= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= @@ -302,12 +283,10 @@ github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UV github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM= github.com/uber-go/atomic v1.3.2 h1:Azu9lPBWRNKzYXSIwRfgRuDuS0YKsK4NFhiQv98gkxo= github.com/uber-go/atomic v1.3.2/go.mod h1:/Ct5t2lcmbJ4OSe/waGBoaVvVqtO0bmtfVNex1PFV8g= +github.com/ybbus/jsonrpc v0.0.0-20180411222309-2a548b7d822d h1:tQo6hjclyv3RHUgZOl6iWb2Y44A/sN9bf9LAYfuioEg= github.com/ybbus/jsonrpc v0.0.0-20180411222309-2a548b7d822d/go.mod h1:XJrh1eMSzdIYFbM08flv0wp5G35eRniyeGut1z+LSiE= -github.com/ybbus/jsonrpc v2.1.2+incompatible h1:V4mkE9qhbDQ92/MLMIhlhMSbz8jNXdagC3xBR5NDwaQ= -github.com/ybbus/jsonrpc v2.1.2+incompatible/go.mod h1:XJrh1eMSzdIYFbM08flv0wp5G35eRniyeGut1z+LSiE= +go.opencensus.io v0.20.1 h1:pMEjRZ1M4ebWGikflH7nQpV6+Zr88KBMA2XJD3sbijw= go.opencensus.io v0.20.1/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= -go.opencensus.io v0.20.2 h1:NAfh7zF0/3/HqtMvJNZ/RFrSlCE6ZTlHmKfhL/Dm1Jk= -go.opencensus.io v0.20.2/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= go.uber.org/atomic v1.3.2 h1:2Oa65PReHzfn29GpvgsYwloV9AVFHPDk8tYxt2c2tr4= go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= golang.org/x/crypto v0.0.0-20170930174604-9419663f5a44/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= @@ -316,8 +295,8 @@ golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnf golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190103213133-ff983b9c42bc/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20190418165655-df01cb2cc480 h1:O5YqonU5IWby+w98jVUG9h7zlCWCcH4RHyPVReBmhzk= -golang.org/x/crypto v0.0.0-20190418165655-df01cb2cc480/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE= +golang.org/x/crypto v0.0.0-20190325154230-a5d413f7728c h1:Vj5n4GlwjmQteupaxJ9+0FNOmBrHfq7vN4btdGoDZgI= +golang.org/x/crypto v0.0.0-20190325154230-a5d413f7728c/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= @@ -341,13 +320,11 @@ golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c h1:uOCk1iQW6Vc18bnC13MfzScl+ golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421 h1:Wo7BWFiOk0QRFMLYMqJGFMd9CgUAcGx7V+qEg/h5IBI= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20190402181905-9f3314589c9a h1:tImsplftrFpALCYumobsd0K86vlAs/eXGFms2txfJfA= -golang.org/x/oauth2 v0.0.0-20190402181905-9f3314589c9a/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6 h1:bjcUS9ztw9kFmmIxJInhon/0Is3p+EHBKNgquIzo1OI= golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58 h1:8gQV6CLnAEikrhgkHFbMAEhagSSnXWGV915qUMm9mrU= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -360,12 +337,11 @@ golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20181122145206-62eef0e2fa9b/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190109145017-48ac38b7c8cb/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190520201301-c432e742b0af h1:NXfmMfXz6JqGfG3ikSxcz2N93j6DgScr19Oo2uwFu88= +golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190520201301-c432e742b0af/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190523142557-0e01d883c5c5 h1:sM3evRHxE/1RuMe1FYAL3j7C7fUfIjkbE+NiDAYUF8U= +golang.org/x/sys v0.0.0-20190523142557-0e01d883c5c5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2 h1:z99zHgr7hKfrUcX/KsoJk5FJfjTceCKIp96+biqP4To= -golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c h1:fqgJT0MGcGpPgpWU7VRdRjuArfcOvC4AoJmILihzhDg= @@ -377,29 +353,24 @@ golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGm golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= google.golang.org/api v0.3.1/go.mod h1:6wY9I6uQWHQ8EM57III9mq/AjF+i8G65rmVagqKMtkk= google.golang.org/api v0.3.2 h1:iTp+3yyl/KOtxa/d1/JUE0GGSoR6FuW5udver22iwpw= google.golang.org/api v0.3.2/go.mod h1:6wY9I6uQWHQ8EM57III9mq/AjF+i8G65rmVagqKMtkk= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.4.0 h1:/wp5JvzpHIxhs/dumFmF7BXTf3Z+dd4uXta4kVyO508= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/appengine v1.5.0 h1:KxkO13IPW4Lslp2bz+KHP2E3gtFlrIGNThxkZQ3g+4c= -google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20180831171423-11092d34479b/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20181004005441-af9cb2a35e7f/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190108161440-ae2f86662275/go.mod h1:7Ep/1NZk928CDR8SjdVbjWNpdIf6nzjE3BTgJDr2Atg= +google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19 h1:Lj2SnHtxkRGJDqnGaSjo+CCdIieEnwVazbOXILwQemk= google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190404172233-64821d5d2107/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7 h1:ZUjXAXmrAyrmmCPHgCA/vChHcpsX27MZ3yBonD/z1KE= -google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/grpc v1.16.0/go.mod h1:0JHn/cJsOMiMfNA9+DeHDlAU7KAAB5GDlYFpa9MZMio= google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.0 h1:DlsSIrgEBuZAUFJcta2B5i/lzeHHbnfkNFAfFXLVFYQ= google.golang.org/grpc v1.20.0/go.mod h1:chYK+tFQF0nDUGJgXMSgLCQk3phJEuONr2DCgLDdAQM= gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U= -gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= @@ -407,9 +378,8 @@ gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2/go.mod h1:Xk6kEKp8OKb+X14hQBKWaSkCsqBpgog8nAV2xsGOxlo= +gopkg.in/ini.v1 v1.41.0 h1:Ka3ViY6gNYSKiVy71zXBEqKplnV35ImDLVG+8uoIklE= gopkg.in/ini.v1 v1.41.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= -gopkg.in/ini.v1 v1.42.0 h1:7N3gPTt50s8GuLortA00n8AqRTk75qOP98+mTPpgzRk= -gopkg.in/ini.v1 v1.42.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/nullbio/null.v6 v6.0.0-20161116030900-40264a2e6b79 h1:FpCr9V8wuOei4BAen+93HtVJ+XSi+KPbaPKm0Vj5R64= gopkg.in/nullbio/null.v6 v6.0.0-20161116030900-40264a2e6b79/go.mod h1:gWkaRU7CoXpezCBWfWjm3999QqS+1pYPXGbqQCTMzo8= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= @@ -421,5 +391,4 @@ gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190109154334-5bcec433c8ea/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/main.go b/main.go index dfc5019..51fc505 100644 --- a/main.go +++ b/main.go @@ -4,16 +4,16 @@ import ( "fmt" "math/rand" "os" - "os/user" "time" + "github.com/lbryio/lbry.go/extras/errors" "github.com/lbryio/lbry.go/extras/util" + "github.com/lbryio/ytsync/manager" "github.com/lbryio/ytsync/sdk" + ytUtils "github.com/lbryio/ytsync/util" + log "github.com/sirupsen/logrus" "github.com/spf13/cobra" - - "github.com/lbryio/ytsync/manager" - logUtils "github.com/lbryio/ytsync/util" ) var Version string @@ -88,6 +88,10 @@ func ytSync(cmd *cobra.Command, args []string) { log.Error("could not detect system hostname") hostname = "ytsync-unknown" } + if len(hostname) > 30 { + hostname = hostname[0:30] + } + util.InitSlack(os.Getenv("SLACK_TOKEN"), os.Getenv("SLACK_CHANNEL"), hostname) } @@ -113,7 +117,6 @@ func ytSync(cmd *cobra.Command, args []string) { apiURL := os.Getenv("LBRY_WEB_API") apiToken := os.Getenv("LBRY_API_TOKEN") youtubeAPIKey := os.Getenv("YOUTUBE_API_KEY") - blobsDir := os.Getenv("BLOBS_DIRECTORY") lbrycrdString := os.Getenv("LBRYCRD_STRING") awsS3ID := os.Getenv("AWS_S3_ID") awsS3Secret := os.Getenv("AWS_S3_SECRET") @@ -150,14 +153,8 @@ func ytSync(cmd *cobra.Command, args []string) { if lbrycrdString == "" { log.Infoln("Using default (local) lbrycrd instance. Set LBRYCRD_STRING if you want to use something else") } - if blobsDir == "" { - usr, err := user.Current() - if err != nil { - log.Errorln(err.Error()) - return - } - blobsDir = usr.HomeDir + "/.lbrynet/blobfiles/" - } + + blobsDir := ytUtils.GetBlobsDir() syncProperties := &sdk.SyncProperties{ SyncFrom: syncFrom, @@ -198,7 +195,7 @@ func ytSync(cmd *cobra.Command, args []string) { ) err := sm.Start() if err != nil { - logUtils.SendErrorToSlack(err.Error()) + ytUtils.SendErrorToSlack(errors.FullTrace(err)) } - logUtils.SendInfoToSlack("Syncing process terminated!") + ytUtils.SendInfoToSlack("Syncing process terminated!") } diff --git a/manager/manager.go b/manager/manager.go index 4f44c8f..7108a71 100644 --- a/manager/manager.go +++ b/manager/manager.go @@ -101,11 +101,18 @@ const ( func (s *SyncManager) Start() error { + if logUtils.ShouldCleanOnStartup() { + err := logUtils.CleanForStartup() + if err != nil { + return err + } + } + syncCount := 0 for { err := s.checkUsedSpace() if err != nil { - return err + return errors.Err(err) } var syncs []Sync @@ -115,7 +122,7 @@ func (s *SyncManager) Start() error { if isSingleChannelSync { channels, err := s.apiConfig.FetchChannels("", s.syncProperties) if err != nil { - return err + return errors.Err(err) } if len(channels) != 1 { return errors.Err("Expected 1 channel, %d returned", len(channels)) @@ -239,9 +246,10 @@ func (s *SyncManager) GetS3AWSConfig() aws.Config { } } func (s *SyncManager) checkUsedSpace() error { - usedPctile, err := GetUsedSpace(s.blobsDir) + logUtils.SendInfoToSlack(logUtils.GetBlobsDir()) + usedPctile, err := GetUsedSpace(logUtils.GetBlobsDir()) if err != nil { - return err + return errors.Err(err) } if usedPctile >= 0.90 && !s.skipSpaceCheck { return errors.Err(fmt.Sprintf("more than 90%% of the space has been used. use --skip-space-check to ignore. Used: %.1f%%", usedPctile*100)) diff --git a/manager/setup.go b/manager/setup.go index 5cf5c01..90f171c 100644 --- a/manager/setup.go +++ b/manager/setup.go @@ -4,15 +4,12 @@ import ( "fmt" "math" "net/http" - "os" "strconv" "time" "github.com/lbryio/lbry.go/extras/errors" "github.com/lbryio/lbry.go/extras/jsonrpc" "github.com/lbryio/lbry.go/extras/util" - "github.com/lbryio/lbry.go/lbrycrd" - "github.com/lbryio/ytsync/tagsManager" "github.com/lbryio/ytsync/thumbs" logUtils "github.com/lbryio/ytsync/util" @@ -29,7 +26,7 @@ func (s *Sync) enableAddressReuse() error { return errors.Err(err) } accounts := accountsResponse.LBCMainnet - if os.Getenv("REGTEST") == "true" { + if logUtils.IsRegTest() { accounts = accountsResponse.LBCRegtest } for _, a := range accounts { @@ -143,7 +140,7 @@ func (s *Sync) ensureEnoughUTXOs() error { return errors.Err(err) } accountsNet := (*accounts).LBCMainnet - if os.Getenv("REGTEST") == "true" { + if logUtils.IsRegTest() { accountsNet = (*accounts).LBCRegtest } defaultAccount := "" @@ -224,6 +221,16 @@ func (s *Sync) ensureEnoughUTXOs() error { } func (s *Sync) waitForNewBlock() error { + if logUtils.IsRegTest() && logUtils.IsUsingDocker() { + lbrycrd, err := logUtils.GetLbrycrdClient(s.LbrycrdString) + if err != nil { + return errors.Prefix("error getting lbrycrd client: ", err) + } + txs, err := lbrycrd.Generate(1) + for _, tx := range txs { + log.Info("Generated tx: ", tx.String()) + } + } status, err := s.daemon.Status() if err != nil { return err @@ -417,18 +424,9 @@ func allUTXOsConfirmed(utxolist *jsonrpc.UTXOListResponse) bool { func (s *Sync) addCredits(amountToAdd float64) error { log.Printf("Adding %f credits", amountToAdd) - var lbrycrdd *lbrycrd.Client - var err error - if s.LbrycrdString == "" { - lbrycrdd, err = lbrycrd.NewWithDefaultURL() - if err != nil { - return err - } - } else { - lbrycrdd, err = lbrycrd.New(s.LbrycrdString) - if err != nil { - return err - } + lbrycrdd, err := logUtils.GetLbrycrdClient(s.LbrycrdString) + if err != nil { + return err } addressResp, err := s.daemon.AddressUnused(nil) diff --git a/manager/ytsync.go b/manager/ytsync.go index 54e482c..39a93d8 100644 --- a/manager/ytsync.go +++ b/manager/ytsync.go @@ -5,7 +5,6 @@ import ( "io/ioutil" "net/http" "os" - "os/exec" "os/signal" "runtime/debug" "sort" @@ -116,28 +115,20 @@ func (s *Sync) IsInterrupted() bool { } func (s *Sync) downloadWallet() error { - defaultWalletDir := os.Getenv("HOME") + "/.lbryum/wallets/default_wallet" - defaultTempWalletDir := os.Getenv("HOME") + "/.lbryum/wallets/tmp_wallet" - key := aws.String("/wallets/" + s.YoutubeChannelID) - if os.Getenv("REGTEST") == "true" { - defaultWalletDir = os.Getenv("HOME") + "/.lbryum_regtest/wallets/default_wallet" - defaultTempWalletDir = os.Getenv("HOME") + "/.lbryum_regtest/wallets/tmp_wallet" - key = aws.String("/regtest/" + s.YoutubeChannelID) - } - - if _, err := os.Stat(defaultWalletDir); !os.IsNotExist(err) { - return errors.Err("default_wallet already exists") + defaultWalletDir, defaultTempWalletDir, key, err := s.getWalletPaths() + if err != nil { + return errors.Err(err) } creds := credentials.NewStaticCredentials(s.AwsS3ID, s.AwsS3Secret, "") s3Session, err := session.NewSession(&aws.Config{Region: aws.String(s.AwsS3Region), Credentials: creds}) if err != nil { - return err + return errors.Prefix("error starting session: ", err) } downloader := s3manager.NewDownloader(s3Session) out, err := os.Create(defaultTempWalletDir) if err != nil { - return err + return errors.Prefix("error creating temp wallet: ", err) } defer out.Close() @@ -165,14 +156,41 @@ func (s *Sync) downloadWallet() error { return errors.Err("zero bytes written") } - return os.Rename(defaultTempWalletDir, defaultWalletDir) + err = os.Rename(defaultTempWalletDir, defaultWalletDir) + if err != nil { + return errors.Prefix("error replacing temp wallet for default wallet: ", err) + } + + return nil +} + +func (s *Sync) getWalletPaths() (defaultWallet, tempWallet string, key *string, err error) { + + defaultWallet = os.Getenv("HOME") + "/.lbryum/wallets/default_wallet" + tempWallet = os.Getenv("HOME") + "/.lbryum/wallets/tmp_wallet" + key = aws.String("/wallets/" + s.YoutubeChannelID) + if logUtils.IsRegTest() { + defaultWallet = os.Getenv("HOME") + "/.lbryum_regtest/wallets/default_wallet" + tempWallet = os.Getenv("HOME") + "/.lbryum_regtest/wallets/tmp_wallet" + key = aws.String("/regtest/" + s.YoutubeChannelID) + } + + walletPath := os.Getenv("LBRYNET_WALLETS_DIR") + if walletPath != "" { + defaultWallet = walletPath + "/wallets/default_wallet" + tempWallet = walletPath + "/wallets/tmp_wallet" + } + + if _, err := os.Stat(defaultWallet); !os.IsNotExist(err) { + return "", "", nil, errors.Err("default_wallet already exists") + } + return } func (s *Sync) uploadWallet() error { - defaultWalletDir := os.Getenv("HOME") + "/.lbryum/wallets/default_wallet" + defaultWalletDir := logUtils.GetDefaultWalletPath() key := aws.String("/wallets/" + s.YoutubeChannelID) - if os.Getenv("REGTEST") == "true" { - defaultWalletDir = os.Getenv("HOME") + "/.lbryum_regtest/wallets/default_wallet" + if logUtils.IsRegTest() { key = aws.String("/regtest/" + s.YoutubeChannelID) } @@ -265,20 +283,24 @@ func (s *Sync) FullCycle() (e error) { defer s.stopAndUploadWallet(&e) - s.videoDirectory, err = ioutil.TempDir("", "ytsync") + s.videoDirectory, err = ioutil.TempDir(os.Getenv("TMP_DIR"), "ytsync") if err != nil { return errors.Wrap(err, 0) } + err = os.Chmod(s.videoDirectory, 0766) + if err != nil { + return errors.Err(err) + } defer deleteSyncFolder(s.videoDirectory) log.Printf("Starting daemon") - err = startDaemonViaSystemd() + err = logUtils.StartDaemon() if err != nil { return err } log.Infoln("Waiting for daemon to finish starting...") - s.daemon = jsonrpc.NewClient("") + s.daemon = jsonrpc.NewClient(os.Getenv("LBRYNET_ADDRESS")) s.daemon.SetRPCTimeout(40 * time.Minute) err = s.waitForDaemonStart() @@ -345,7 +367,7 @@ func (s *Sync) waitForDaemonStart() error { func (s *Sync) stopAndUploadWallet(e *error) { log.Printf("Stopping daemon") - shutdownErr := stopDaemonViaSystemd() + shutdownErr := logUtils.StopDaemon() if shutdownErr != nil { logShutdownError(shutdownErr) } else { @@ -369,7 +391,7 @@ func (s *Sync) stopAndUploadWallet(e *error) { } } func logShutdownError(shutdownErr error) { - logUtils.SendErrorToSlack("error shutting down daemon: %v", shutdownErr) + logUtils.SendErrorToSlack("error shutting down daemon: %s", errors.FullTrace(shutdownErr)) logUtils.SendErrorToSlack("WALLET HAS NOT BEEN MOVED TO THE WALLET BACKUP DIR") } @@ -548,6 +570,18 @@ func (s *Sync) doSync() error { if err != nil { return errors.Prefix("error updating remote database", err) } + + cert, err := s.daemon.ChannelExport(s.lbryChannelID, nil, nil) + if err != nil { + return errors.Prefix("error getting channel cert", err) + } + if cert != nil { + err = s.APIConfig.SetChannelCert(string(*cert), s.lbryChannelID) + if err != nil { + return errors.Prefix("error setting channel cert", err) + } + } + if nFixed > 0 || nRemoved > 0 { err := s.setStatusSyncing() if err != nil { @@ -673,7 +707,7 @@ func (s *Sync) startWorker(workerNum int) { err := s.waitForNewBlock() if err != nil { s.grp.Stop() - logUtils.SendErrorToSlack("something went wrong while waiting for a block: %v", err) + logUtils.SendErrorToSlack("something went wrong while waiting for a block: %s", errors.FullTrace(err)) break } } else if util.SubstringInSlice(err.Error(), []string{ @@ -685,7 +719,7 @@ func (s *Sync) startWorker(workerNum int) { err := s.walletSetup() if err != nil { s.grp.Stop() - logUtils.SendErrorToSlack("failed to setup the wallet for a refill: %v", err) + logUtils.SendErrorToSlack("failed to setup the wallet for a refill: %s", errors.FullTrace(err)) break } } else if strings.Contains(err.Error(), "Error in daemon: 'str' object has no attribute 'get'") { @@ -720,7 +754,7 @@ func (s *Sync) startWorker(workerNum int) { } err = s.Manager.apiConfig.MarkVideoStatus(s.YoutubeChannelID, v.ID(), videoStatus, existingClaimID, existingClaimName, err.Error(), &existingClaimSize, 0) if err != nil { - logUtils.SendErrorToSlack("Failed to mark video on the database: %s", err.Error()) + logUtils.SendErrorToSlack("Failed to mark video on the database: %s", errors.FullTrace(err)) } } break @@ -911,28 +945,12 @@ func (s *Sync) processVideo(v video) (err error) { s.AppendSyncedVideo(v.ID(), true, "", summary.ClaimName, summary.ClaimID, newMetadataVersion, *v.Size()) err = s.Manager.apiConfig.MarkVideoStatus(s.YoutubeChannelID, v.ID(), VideoStatusPublished, summary.ClaimID, summary.ClaimName, "", v.Size(), 2) if err != nil { - logUtils.SendErrorToSlack("Failed to mark video on the database: %s", err.Error()) + logUtils.SendErrorToSlack("Failed to mark video on the database: %s", errors.FullTrace(err)) } return nil } -func startDaemonViaSystemd() error { - err := exec.Command("/usr/bin/sudo", "/bin/systemctl", "start", "lbrynet.service").Run() - if err != nil { - return errors.Err(err) - } - return nil -} - -func stopDaemonViaSystemd() error { - err := exec.Command("/usr/bin/sudo", "/bin/systemctl", "stop", "lbrynet.service").Run() - if err != nil { - return errors.Err(err) - } - return nil -} - // waitForDaemonProcess observes the running processes and returns when the process is no longer running or when the timeout is up func waitForDaemonProcess(timeout time.Duration) error { processes, err := ps.Processes() diff --git a/sdk/api.go b/sdk/api.go index 821f8a8..952200b 100644 --- a/sdk/api.go +++ b/sdk/api.go @@ -71,7 +71,7 @@ func (a *APIConfig) FetchChannels(status string, cp *SyncProperties) ([]YoutubeC var response apiJobsResponse err = json.Unmarshal(body, &response) if err != nil { - return nil, err + return nil, errors.Err(err) } if response.Data == nil { return nil, errors.Err(response.Error) @@ -98,6 +98,39 @@ func sanitizeFailureReason(s *string) { *s = (*s)[:MaxReasonLength] } } + +func (a *APIConfig) SetChannelCert(certHex string, channelID string) error { + + type apiSetChannelCertResponse struct { + Success bool `json:"success"` + Error null.String `json:"error"` + Data string `json:"data"` + } + + endpoint := a.ApiURL + "/yt/channel_cert" + + res, _ := http.PostForm(endpoint, url.Values{ + "channel_claim_id": {channelID}, + "channel_cert": {certHex}, + "auth_token": {a.ApiToken}, + }) + + defer res.Body.Close() + + body, _ := ioutil.ReadAll(res.Body) + var response apiSetChannelCertResponse + err := json.Unmarshal(body, &response) + if err != nil { + return errors.Err(err) + } + if !response.Error.IsNull() { + return errors.Err(response.Error.String) + } + + return nil + +} + func (a *APIConfig) SetChannelStatus(channelID string, status string, failureReason string) (map[string]SyncedVideo, map[string]bool, error) { type apiChannelStatusResponse struct { Success bool `json:"success"` @@ -119,7 +152,7 @@ func (a *APIConfig) SetChannelStatus(channelID string, status string, failureRea var response apiChannelStatusResponse err := json.Unmarshal(body, &response) if err != nil { - return nil, nil, err + return nil, nil, errors.Err(err) } if !response.Error.IsNull() { return nil, nil, errors.Err(response.Error.String) diff --git a/sources/youtubeVideo.go b/sources/youtubeVideo.go index 8ede581..09551b3 100644 --- a/sources/youtubeVideo.go +++ b/sources/youtubeVideo.go @@ -24,7 +24,7 @@ import ( "github.com/lbryio/ytsync/tagsManager" "github.com/lbryio/ytsync/thumbs" - "github.com/ChannelMeter/iso8601duration" + duration "github.com/ChannelMeter/iso8601duration" "github.com/aws/aws-sdk-go/aws" "github.com/shopspring/decimal" log "github.com/sirupsen/logrus" @@ -178,7 +178,7 @@ func (v *YoutubeVideo) getAbbrevDescription() string { func (v *YoutubeVideo) download(useIPv6 bool) error { videoPath := v.getFullPath() - err := os.Mkdir(v.videoDir(), 0750) + err := os.Mkdir(v.videoDir(), 0777) if err != nil && !strings.Contains(err.Error(), "file exists") { return errors.Wrap(err, 0) } @@ -294,22 +294,26 @@ runcmd: log.Debugln(string(outLog)) if strings.Contains(string(outLog), "does not pass filter duration") { - _ = v.delete() + _ = v.delete("does not pass filter duration") return errors.Err("video is too long to process") } if strings.Contains(string(outLog), "File is larger than max-filesize") { - _ = v.delete() + _ = v.delete("File is larger than max-filesize") return errors.Err("the video is too big to sync, skipping for now") } if string(errorLog) != "" { log.Printf("Command finished with error: %v", errors.Err(string(errorLog))) - _ = v.delete() + _ = v.delete("due to error") return errors.Err(string(errorLog)) } fi, err := os.Stat(v.getFullPath()) if err != nil { return errors.Err(err) } + err = os.Chmod(v.getFullPath(), 0777) + if err != nil { + return errors.Err(err) + } videoSize := fi.Size() v.size = &videoSize return nil @@ -338,14 +342,14 @@ func (v *YoutubeVideo) getDownloadedPath() (string, error) { return "", errors.Err("could not find any downloaded videos") } -func (v *YoutubeVideo) delete() error { +func (v *YoutubeVideo) delete(reason string) error { videoPath, err := v.getDownloadedPath() if err != nil { log.Errorln(err) return err } err = os.Remove(videoPath) - log.Debugf("%s deleted from disk (%s)", v.id, videoPath) + log.Debugf("%s deleted from disk for '%s' (%s)", v.id, reason, videoPath) if err != nil { err = errors.Prefix("delete error", err) @@ -451,7 +455,7 @@ func (v *YoutubeVideo) downloadAndPublish(daemon *jsonrpc.Client, params SyncPar summary, err := v.publish(daemon, params) //delete the video in all cases (and ignore the error) - _ = v.delete() + _ = v.delete("finished download and publish") return summary, errors.Prefix("publish error", err) } diff --git a/thumbs/uploader.go b/thumbs/uploader.go index b6ee684..4beff61 100644 --- a/thumbs/uploader.go +++ b/thumbs/uploader.go @@ -1,7 +1,6 @@ package thumbs import ( - "google.golang.org/api/youtube/v3" "io" "net/http" "os" @@ -11,7 +10,8 @@ import ( "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/session" "github.com/aws/aws-sdk-go/service/s3/s3manager" - "github.com/prometheus/common/log" + log "github.com/sirupsen/logrus" + "google.golang.org/api/youtube/v3" ) type thumbnailUploader struct { @@ -25,7 +25,7 @@ const thumbnailPath = "/tmp/ytsync_thumbnails/" const ThumbnailEndpoint = "https://thumbnails.lbry.com/" func (u *thumbnailUploader) downloadThumbnail() error { - _ = os.Mkdir(thumbnailPath, 0750) + _ = os.Mkdir(thumbnailPath, 0777) img, err := os.Create("/tmp/ytsync_thumbnails/" + u.name) if err != nil { return errors.Err(err) diff --git a/util/util.go b/util/util.go new file mode 100644 index 0000000..3b5a081 --- /dev/null +++ b/util/util.go @@ -0,0 +1,305 @@ +package util + +import ( + "context" + "os" + "os/exec" + "os/user" + "path/filepath" + "strconv" + + "github.com/lbryio/lbry.go/extras/errors" + "github.com/lbryio/lbry.go/lbrycrd" + + "github.com/docker/docker/api/types" + "github.com/docker/docker/api/types/filters" + "github.com/docker/docker/client" + "github.com/mitchellh/go-ps" + log "github.com/sirupsen/logrus" +) + +func GetBlobsDir() string { + blobsDir := os.Getenv("BLOBS_DIRECTORY") + if blobsDir == "" { + usr, err := user.Current() + if err != nil { + log.Error(err.Error()) + return "" + } + blobsDir = usr.HomeDir + "/.lbrynet/blobfiles/" + } + + return blobsDir +} + +func IsBlobReflectionOff() bool { + return os.Getenv("REFLECT_BLOBS") == "false" +} + +func GetLBRYNetDir() string { + lbrynetDir := os.Getenv("LBRYNET_DIR") + if lbrynetDir == "" { + usr, err := user.Current() + if err != nil { + log.Errorln(err.Error()) + return "" + } + return usr.HomeDir + "/.lbrynet/" + } + return lbrynetDir +} + +const ALL = true +const ONLINE = false + +func GetLBRYNetContainer(all bool) (*types.Container, error) { + return getDockerContainer("lbrynet", all) +} + +func getDockerContainer(name string, all bool) (*types.Container, error) { + cli, err := client.NewEnvClient() + if err != nil { + panic(err) + } + filters := filters.NewArgs() + filters.Add("name", name) + containers, err := cli.ContainerList(context.Background(), types.ContainerListOptions{All: all, Filters: filters}) + if err != nil { + panic(err) + } + if len(containers) == 0 { + return nil, nil + } + if len(containers) > 1 { + return nil, errors.Err("more than one %s container found", name) + } + + return &containers[0], nil +} + +func IsUsingDocker() bool { + useDocker, err := strconv.ParseBool(os.Getenv("LBRYNET_USE_DOCKER")) + if err != nil { + return false + } + return useDocker +} + +func IsRegTest() bool { + usesRegtest, err := strconv.ParseBool(os.Getenv("REGTEST")) + if err != nil { + return false + } + return usesRegtest +} + +func GetLbrycrdClient(lbrycrdString string) (*lbrycrd.Client, error) { + var lbrycrdd *lbrycrd.Client + var err error + if lbrycrdString == "" { + lbrycrdd, err = lbrycrd.NewWithDefaultURL() + if err != nil { + return nil, err + } + } else { + lbrycrdd, err = lbrycrd.New(lbrycrdString) + if err != nil { + return nil, err + } + } + + return lbrycrdd, nil +} + +func ShouldCleanOnStartup() bool { + shouldClean, err := strconv.ParseBool(os.Getenv("CLEAN_ON_STARTUP")) + if err != nil { + return false + } + return shouldClean +} + +func IsLbrynetRunning() (bool, error) { + if IsUsingDocker() { + container, err := GetLBRYNetContainer(ONLINE) + if err != nil { + return false, err + } + return container != nil, nil + } + + processes, err := ps.Processes() + if err != nil { + return true, errors.Err(err) + } + var daemonProcessId = -1 + for _, p := range processes { + if p.Executable() == "lbrynet" { + daemonProcessId = p.Pid() + break + } + } + + running := daemonProcessId != -1 + return running, nil +} + +func CleanForStartup() error { + if !IsRegTest() { + return errors.Err("never cleanup wallet outside of regtest and with caution. this should only be done in local testing and requires regtest to be on") + } + + running, err := IsLbrynetRunning() + if err != nil { + return err + } + if running { + err := StopDaemon() + if err != nil { + return err + } + } + + err = CleanupLbrynet() + if err != nil { + return errors.Err(err) + } + + lbrycrd, err := GetLbrycrdClient(os.Getenv("LBRYCRD_STRING")) + if err != nil { + return errors.Prefix("error getting lbrycrd client: ", err) + } + height, err := lbrycrd.GetBlockCount() + if err != nil { + return errors.Err(err) + } + const minBlocksForUTXO = 110 + if height < minBlocksForUTXO { + //Start reg test will some credits + txs, err := lbrycrd.Generate(uint32(110) - uint32(height)) + if err != nil { + return errors.Err(err) + } + log.Debugf("REGTEST: Generated %d transactions to get some LBC!", len(txs)) + } + + defaultWalletDir := GetDefaultWalletPath() + _, err = os.Stat(defaultWalletDir) + if os.IsNotExist(err) { + return nil + } + return errors.Err(os.Remove(defaultWalletDir)) +} + +func CleanupLbrynet() error { + //make sure lbrynet is off + running, err := IsLbrynetRunning() + if err != nil { + return err + } + if running { + return errors.Prefix("cannot cleanup lbrynet as the daemon is running", err) + } + lbrynetDir := GetLBRYNetDir() + files, err := filepath.Glob(lbrynetDir + "lbrynet.sqlite*") + if err != nil { + return errors.Err(err) + } + for _, f := range files { + err = os.Remove(f) + if err != nil { + return errors.Err(err) + } + } + blobsDir := GetBlobsDir() + err = os.RemoveAll(blobsDir) + if err != nil { + return errors.Err(err) + } + err = os.Mkdir(blobsDir, 0777) + if err != nil { + return errors.Err(err) + } + return nil +} + +func StartDaemon() error { + if IsUsingDocker() { + return startDaemonViaDocker() + } + return startDaemonViaSystemd() +} + +func StopDaemon() error { + if IsUsingDocker() { + return stopDaemonViaDocker() + } + return stopDaemonViaSystemd() +} + +func startDaemonViaDocker() error { + container, err := GetLBRYNetContainer(true) + if err != nil { + return err + } + + cli, err := client.NewEnvClient() + if err != nil { + panic(err) + } + + err = cli.ContainerStart(context.Background(), container.ID, types.ContainerStartOptions{}) + if err != nil { + return errors.Err(err) + } + + return nil +} + +func stopDaemonViaDocker() error { + container, err := GetLBRYNetContainer(ONLINE) + if err != nil { + return err + } + + cli, err := client.NewEnvClient() + if err != nil { + panic(err) + } + + err = cli.ContainerStop(context.Background(), container.ID, nil) + if err != nil { + return errors.Err(err) + } + + return nil +} + +func startDaemonViaSystemd() error { + err := exec.Command("/usr/bin/sudo", "/bin/systemctl", "start", "lbrynet.service").Run() + if err != nil { + return errors.Err(err) + } + return nil +} + +func stopDaemonViaSystemd() error { + err := exec.Command("/usr/bin/sudo", "/bin/systemctl", "stop", "lbrynet.service").Run() + if err != nil { + return errors.Err(err) + } + return nil +} + +func GetDefaultWalletPath() string { + defaultWalletDir := os.Getenv("HOME") + "/.lbryum/wallets/default_wallet" + if IsRegTest() { + defaultWalletDir = os.Getenv("HOME") + "/.lbryum_regtest/wallets/default_wallet" + } + + walletPath := os.Getenv("LBRYNET_WALLETS_DIR") + if walletPath != "" { + defaultWalletDir = walletPath + "/wallets/default_wallet" + } + return defaultWalletDir +}