Merge branch 'setchannelcerts'

.gitignore

@@ -1 +1,4 @@
 bin/
+e2e/persist
+.env
+blobsfiles

.travis.yml

@@ -1,15 +1,42 @@
 os: linux
-dist: xenial
+dist: bionic
 language: go
-go_import_path: github.com/lbryio/ytsync
 go:
-- 1.11.5
+  - 1.12.7
-- master
+  - master
+
+install: true
+
+cache:
+  directories:
+    - $HOME/.cache/go-build
+    - $HOME/gopath/pkg/mod
+
+services:
+  - docker
+
+addons:
+  apt:
+    update: true
+    packages:
+      - ffmpeg
+      - tree
+      - python-pip
+
+before_script:
+  - sudo pip install -U youtube-dl
+  - sudo add-apt-repository -y ppa:jonathonf/ffmpeg-4
+
 env:
   global:
     - GO111MODULE=on
     #GITHUB_TOKEN
     - secure: "Ps3KocRP5xnM3/uA99CeYhDTVxRIuW7fGyrtqBeRWZW0cXzeA4XCTKxqcFbrPUPw67XkrBVgE58JDdWoQEJ7tm67PjMm/ltp5Evhx/QAJDh+YSofXyGDVpG1mrTZFI66R3NVVJLkSGALMkuWWXvfYZeU//AworJbyRoaIK/CVt5OP23i5N4tdd5UXc5dfLuYqnKRynyMmCkz9c3yEIQMXoPhG2hx7l7L2BeMJvcKmVhkSN7nQayjnrbUXGm/IRqrb88lvkyBevN5E3IB2V5IKEieIPZjbD/N0IfcnAt89Z96tgDhtIbx3ZvXm92lsvHA8buqQpG9d2AmSi6GKs64lQcnGeM5o0wER2JHWl1OSa1Nr/UAo5Xb/PM65Yt3yZE8AuMKHBmbfDSBzdkTXx58AeDzFUd3kMXD/fFjeQQWyXFlOss3ygH9SObl827Txmz9OJqZaxabs5Q3AP6m3EjKjz7zfLfrgpcxJM2WBiU1bN0ZxUgZkImy/CHk5gCZ7vhcnaLiDO4HZnzY/aRJwKYQPE5i0O2nHpIfovqkc0DFBA7U/7Cjin7e1E0UZvF3meLOxMqkfc6X7QTxqQpt2Tej6jlpdxw4CTLwGUhGkAw9IAPkUB3L0EbZ1/ksGhNvGDvUeSTq8hYdMAPmA+k9jS6653V4SQ+qBMy5++tbr5AeZQI="
+
+script:
+  #- ./e2e/e2e.sh # Hold until we can resolve the /var/tmp issue - talk to beamer/niko
+  - make
+
 deploy:
   provider: script
   skip_cleanup: true

blobs_reflector/reflect.go

@@ -2,16 +2,17 @@ package blobs_reflector
 
 import (
 	"encoding/json"
+	"io/ioutil"
+	"os"
+	"os/user"
+	"path/filepath"
+
 	"github.com/lbryio/lbry.go/extras/errors"
 	"github.com/lbryio/reflector.go/cmd"
 	"github.com/lbryio/reflector.go/db"
 	"github.com/lbryio/reflector.go/reflector"
 	"github.com/lbryio/reflector.go/store"
-	"github.com/mitchellh/go-ps"
+	"github.com/lbryio/ytsync/util"
-	"io/ioutil"
-	"os"
-	"os/user"
-	"path/filepath"
 
 	log "github.com/sirupsen/logrus"
 )
@@ -21,7 +22,7 @@ func ReflectAndClean() error {
 	if err != nil {
 		return err
 	}
-	return cleanupLbrynet()
+	return util.CleanupLbrynet()
 }
 
 func loadConfig(path string) (cmd.Config, error) {
@@ -40,8 +41,11 @@ func loadConfig(path string) (cmd.Config, error) {
 }
 
 func reflectBlobs() error {
+	if util.IsBlobReflectionOff() {
+		return nil
+	}
 	//make sure lbrynet is off
-	running, err := isLbrynetRunning()
+	running, err := util.IsLbrynetRunning()
 	if err != nil {
 		return err
 	}
@@ -90,57 +94,3 @@ func reflectBlobs() error {
 	}
 	return nil
 }
-
-func cleanupLbrynet() error {
-	//make sure lbrynet is off
-	running, err := isLbrynetRunning()
-	if err != nil {
-		return err
-	}
-	if running {
-		return errors.Prefix("cannot cleanup lbrynet as the daemon is running", err)
-	}
-	usr, err := user.Current()
-	if err != nil {
-		log.Errorln(err.Error())
-		return errors.Err(err)
-	}
-	lbrynetDir := usr.HomeDir + "/.lbrynet/"
-	files, err := filepath.Glob(lbrynetDir + "lbrynet.sqlite*")
-	if err != nil {
-		return errors.Err(err)
-	}
-	for _, f := range files {
-		err = os.Remove(f)
-		if err != nil {
-			return errors.Err(err)
-		}
-	}
-	blobsDir := lbrynetDir + "/blobfiles/"
-	err = os.RemoveAll(blobsDir)
-	if err != nil {
-		return errors.Err(err)
-	}
-	err = os.Mkdir(blobsDir, 0755)
-	if err != nil {
-		return errors.Err(err)
-	}
-	return nil
-}
-
-func isLbrynetRunning() (bool, error) {
-	processes, err := ps.Processes()
-	if err != nil {
-		return true, errors.Err(err)
-	}
-	var daemonProcessId = -1
-	for _, p := range processes {
-		if p.Executable() == "lbrynet" {
-			daemonProcessId = p.Pid()
-			break
-		}
-	}
-
-	running := daemonProcessId != -1
-	return running, nil
-}

e2e/daemon_settings.yml

@@ -0,0 +1,12 @@
+#blockchain_name: lbrycrd_main
+#blockchain_name: lbrycrd_testnet
+blockchain_name: lbrycrd_regtest
+lbryum_servers:
+#  - spv1.lbry.com:50001 #Production Wallet Server
+  - walletserver:50001
+save_blobs: true
+save_files: false
+share_usage_data: false
+tcp_port: 3333
+udp_port: 4444
+use_upnp: false

e2e/data_setup.sh

@@ -0,0 +1,21 @@
+#!/usr/bin/env bash
+
+set -e
+
+#Add a ytsync user
+ADDYTSYNCUSER='INSERT INTO user (given_name) VALUE("ytsync user")'
+mysql -u lbry -plbry -D lbry -h "127.0.0.1" -P 15500 -e "$ADDYTSYNCUSER"
+#Insert an auth token for the user to be used by ytsync
+ADDYTSYNCAUTHTOKEN='INSERT INTO auth_token (user_id, value) VALUE(1,"ytsyntoken")'
+mysql -u lbry -plbry -D lbry -h "127.0.0.1" -P 15500 -e "$ADDYTSYNCAUTHTOKEN"
+#Give priveledges to ytsync user
+ASSIGNGROOP='INSERT INTO user_groop (user_id, groop_id) VALUE( 1,3)'
+mysql -u lbry -plbry -D lbry -h "127.0.0.1" -P 15500 -e "$ASSIGNGROOP"
+
+#Add youtuber to sync
+ADDYTSYNCER='INSERT INTO user (given_name) VALUE("youtuber")'
+mysql -u lbry -plbry -D lbry -h "127.0.0.1" -P 15500 -e "$ADDYTSYNCER"
+#Add their youtube channel to be synced
+ADDYTCHANNEL="INSERT INTO youtube_data (user_id, status_token,desired_lbry_channel,channel_id,channel_name,status,created_at,source,total_videos,total_subscribers)
+VALUE(2,'3qzGyuVjQaf7t4pKKu2Er1NRW2LJkeWw','@beamertest','UCCyr5j8akeu9j4Q7urV0Lqw','BeamerAtLBRY','queued','2019-08-01 00:00:00','sync',1,0)"
+mysql -u lbry -plbry -D lbry -h "127.0.0.1" -P 15500 -e "$ADDYTCHANNEL"

e2e/docker-compose.yml

@@ -0,0 +1,103 @@
+version: "3.5"
+services:
+  #############
+  ## Lbrycrd ##
+  #############
+  lbrycrd:
+    image: lbry/lbrycrd:v0.12.4.1
+    restart: always
+    ports:
+      - "15201:29246"
+      - "15200:29245"
+    expose:
+      - "29246"
+      - "29245"
+    ## host volumes for persistent data such as wallet private keys.
+    volumes:
+      - "./persist:/data"
+    environment:
+      - RUN_MODE=regtest
+  ###################
+  ## Wallet Server ##
+  ###################
+  walletserver:
+    image: lbry/wallet-server:v0.38.5
+    restart: always
+    environment:
+      - DB_DIRECTORY=/database
+      - MAX_SEND=1000000000000000000000
+      - DAEMON_URL=http://lbry:lbry@lbrycrd:29245
+      - MAX_SUBS=1000000000000
+      - BANDWIDTH_LIMIT=80000000000
+      - SESSION_TIMEOUT=10000000000000000000000000
+      - TCP_PORT=50001
+    ports:
+      - "15300:50001"
+    expose:
+      - "50001"
+    depends_on:
+      - lbrycrd
+    ulimits:
+      nofile:
+        soft: 90000
+        hard: 90000
+    #command: lbry.wallet.server.coin.LBC
+    command: lbry.wallet.server.coin.LBCRegTest
+  #############
+  ## Lbrynet ##
+  #############
+  lbrynet:
+    image: lbry/lbrynet:v0.38.5
+    restart: always
+    ports:
+      - "15100:5279"
+      - "15101:5280"
+    expose:
+      - "5279"
+      - "5280"
+    depends_on:
+      - walletserver
+    environment:
+      - LBRY_STREAMING_SERVER=0.0.0.0:5280
+    volumes:
+      - "./persist/.lbrynet:/home/lbrynet"
+      - ".:/etc/lbry" #Put your daemon_settings.yml here
+      # /private/var/tmp for OSX and /var/tmp for Linux
+      - "${LOCAL_TMP_DIR}"
+  ###########
+  ## MySQL ##
+  ###########
+  mysql:
+    image: mysql/mysql-server:5.7.27
+    restart: "no"
+    ports:
+      - "15500:3306"
+    expose:
+      - "3306"
+    environment:
+      - MYSQL_ALLOW_EMPTY_PASSWORD=true
+      - MYSQL_DATABASE=lbry
+      - MYSQL_USER=lbry
+      - MYSQL_PASSWORD=lbry
+      - MYSQL_LOG_CONSOLE=true
+  ###################
+  ## Internal APIs ##
+  ###################
+  internalapis:
+    image: lbry/internal-apis:master
+    restart: "no"
+    ports:
+      - "15400:8080"
+    expose:
+      - "8080"
+    depends_on:
+      - mysql
+      - lbrycrd
+      - lbrynet
+    environment:
+      - MYSQL_DSN=lbry:lbry@tcp(mysql:3306)/lbry
+      - LBRYCRD_CONNECT=rpc://lbry:lbry@lbrycrd:29245
+      - MYSQL_USER=lbry
+      - MYSQL_PASS=lbry
+      - MYSQL_DATABASE=lbry
+    entrypoint: wait-for-it mysql:3306 -- wait-for-it lbrynet:5279 -- ./latest serve

e2e/e2e.sh

@@ -0,0 +1,65 @@
+#!/usr/bin/env bash
+
+set -e
+
+#Always compile ytsync
+make
+
+#OVERRIDE this in your .env file if running from mac. Check docker-compose.yml for details
+export LOCAL_TMP_DIR="/var/tmp:/var/tmp"
+
+#Private Variables Set in local installations: SLACK_TOKEN,YOUTUBE_API_KEY,AWS_S3_ID,AWS_S3_SECRET,AWS_S3_REGION,AWS_S3_BUCKET
+touch -a .env && set -o allexport; source ./.env; set +o allexport
+echo "LOCAL_TMP_DIR=$LOCAL_TMP_DIR"
+# Compose settings - docker only
+export SLACK_CHANNEL="ytsync-travis"
+export LBRY_API_TOKEN="ytsyntoken"
+export LBRY_WEB_API="http://localhost:15400"
+export LBRYNET_ADDRESS="http://localhost:15100"
+export LBRYCRD_STRING="tcp://lbry:lbry@localhost:15200"
+export LBRYNET_USE_DOCKER=true
+export REFLECT_BLOBS=false
+export CLEAN_ON_STARTUP=true
+export REGTEST=true
+# Local settings
+export BLOBS_DIRECTORY="$(pwd)/e2e/blobsfiles"
+export LBRYNET_DIR="$(pwd)/e2e/persist/.lbrynet/.local/share/lbry/lbrynet/"
+export LBRYNET_WALLETS_DIR="$(pwd)/e2e/persist/.lbrynet/.local/share/lbry/lbryum"
+export TMP_DIR="/var/tmp"
+export UID
+
+cd ./e2e
+docker-compose stop
+docker-compose rm -f
+echo "$DOCKER_PASSWORD" | docker login --username "$DOCKER_USERNAME" --password-stdin
+docker-compose pull
+if [[ -d persist ]]; then rm -rf persist; fi
+mkdir  -m 0777 -p ./persist
+mkdir -m 777 -p ./persist/.walletserver
+mkdir -m 777 -p ./persist/.lbrynet
+#sudo chown -Rv 999:999 ./persist/.walletserver
+#sudo chown -Rv 1000:1000 ./persist/.lbrynet
+docker-compose up -d
+printf 'waiting for internal apis'
+until curl --output /dev/null --silent --head --fail http://localhost:15400; do
+    printf '.'
+    sleep 1
+done
+echo "successfully started..."
+
+#Data Setup for test
+./data_setup.sh
+
+# Execute the test!
+./../bin/ytsync --channelID UCCyr5j8akeu9j4Q7urV0Lqw #Force channel intended...just in case. This channel lines up with the api container
+# Assert the status
+status=$(mysql -u lbry -plbry -ss -D lbry -h "127.0.0.1" -P 15500 -e 'SELECT status FROM youtube_data WHERE id=1')
+videoStatus=$(mysql -u lbry -plbry -ss -D lbry -h "127.0.0.1" -P 15500 -e 'SELECT status FROM synced_video WHERE id=1')
+if [[ $status != "synced" || $videoStatus != "published" ]]; then
+docker-compose logs --tail="all" lbrycrd
+docker-compose logs --tail="all" walletserver
+docker-compose logs --tail="all" lbrynet
+docker-compose logs --tail="all" internalapis
+echo "List local /var/tmp"
+find /var/tmp
+ exit 1; fi;

e2e/lbrycrd/docker-compose.yml

@@ -0,0 +1,26 @@
+version: "3"
+networks:
+  lbry-network:
+    external: true
+
+services:
+  #############
+  ## Lbrycrd ##
+  #############
+  lbrycrd:
+    image: lbry/lbrycrd:v0.12.4.1
+    restart: always
+    networks:
+      lbry-network:
+        ipv4_address: 10.6.1.1
+    ports:
+      - "15201:29246"
+      - "15200:29245"
+    expose:
+      - "29246"
+      - "29245"
+    ## host volumes for persistent data such as wallet private keys.
+    volumes:
+      - "../persist/data:/data"
+    environment:
+      - RUN_MODE=regtest

e2e/lbrycrd/docker/Dockerfile

@@ -0,0 +1,37 @@
+FROM ubuntu:18.04 as prep
+LABEL MAINTAINER="leopere [at] nixc [dot] us"
+## TODO: Implement version pinning. `apt-get install curl=<version>`
+RUN apt-get update && \
+  apt-get -y install unzip curl build-essential && \
+  apt-get autoclean -y && \
+  rm -rf /var/lib/apt/lists/*
+WORKDIR /
+SHELL ["/bin/bash", "-o", "pipefail", "-c"]
+COPY ./start.sh start
+COPY ./healthcheck.sh healthcheck
+COPY ./advance_blocks.sh advance
+COPY ./fix-permissions.c fix-permissions.c
+
+## Add lbrycrd - Change the version below to create an image for a different tag/version
+ARG VERSION="v0.12.4.1"
+RUN URL=$(curl -s https://api.github.com/repos/lbryio/lbrycrd/releases/$(if [ "${VERSION}" = 'latest' ]; then echo "latest"; else echo "tags/${VERSION}"; fi) | grep browser_download_url | grep lbrycrd-linux.zip | cut -d'"' -f4) && echo $URL && curl -L -o /lbrycrd-linux.zip $URL
+
+RUN unzip ./lbrycrd-linux.zip && \
+  gcc fix-permissions.c -o fix-permissions && \
+  chmod +x ./lbrycrdd ./lbrycrd-cli ./lbrycrd-tx ./start ./healthcheck ./fix-permissions ./advance
+
+FROM ubuntu:18.04 as app
+COPY --from=prep /lbrycrdd /lbrycrd-cli /lbrycrd-tx /start /healthcheck /fix-permissions /advance /usr/bin/
+RUN addgroup --gid 1000 lbrycrd && \
+  adduser lbrycrd --uid 1000 --gid 1000 --gecos GECOS --shell /bin/bash --disabled-password --home /data && \
+  mkdir /etc/lbry && \
+  chown lbrycrd /etc/lbry && \
+  chmod a+s /usr/bin/fix-permissions
+VOLUME ["/data"]
+WORKDIR /data
+## TODO: Implement healthcheck.
+# HEALTHCHECK ["healthcheck"]
+EXPOSE 9246 9245 11337 29245
+
+USER lbrycrd
+CMD ["start"]

e2e/lbrycrd/docker/advance_blocks.sh

@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+while true; do
+        lbrycrd-cli -conf=/etc/lbry/lbrycrd.conf generate 100 >> /tmp/output.log
+        sleep 2
+done

e2e/lbrycrd/docker/build.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+if [ $# -eq 0 ]
+  then
+    echo "No docker tag argument supplied. Use './build.sh <tag>'"
+    exit 1
+fi
+docker build --tag lbry/lbrycrd:$1 .
+docker push lbry/lbrycrd:$1

e2e/lbrycrd/docker/fix-permissions.c

@@ -0,0 +1,9 @@
+#include <unistd.h>
+int main() {
+  // This program needs to run with setuid == root
+  // This needs to be in a compiled language because you cannot setuid bash scripts
+  setuid(0);
+  execle("/bin/bash", "bash", "-c",
+         "/bin/chown -R lbrycrd:lbrycrd /data && /bin/chmod -R 755 /data/",
+         (char*) NULL, (char*) NULL);
+}

e2e/lbrycrd/docker/healthcheck.sh

@@ -0,0 +1,4 @@
+## TODO: Implement a healthcheck for lbrycrd.
+curl --data-binary '{"jsonrpc":"1.0","id":"curltext","method":"getinfo","params":[]}' -H 'content-type:text/plain;'  http://$RPC_USER:$RPC_PASSWORD@127.0.0.1:9246
+## OR
+lbrycrd-cli getinfo

e2e/lbrycrd/docker/start.sh

@@ -0,0 +1,114 @@
+#!/usr/bin/env bash
+CONFIG_PATH=/etc/lbry/lbrycrd.conf
+
+function override_config_option() {
+    # Remove existing config line from a config file
+    #  and replace with environment fed value.
+    # Does nothing if the variable does not exist.
+    #  var     Name of ENV variable
+    #  option  Name of config option
+    #  config  Path of config file
+    local var=$1 option=$2 config=$3
+    if [[ -v $var ]]; then
+        # Remove the existing config option:
+        sed -i "/^$option\W*=/d" $config
+        # Add the value from the environment:
+        echo "$option=${!var}" >> $config
+    fi
+}
+
+function set_config() {
+  if [ -d "$CONFIG_PATH" ]; then
+      echo "$CONFIG_PATH is a directory when it should be a file."
+      exit 1
+  elif [ -f "$CONFIG_PATH" ]; then
+      echo "Merging the mounted config file with environment variables."
+      local MERGED_CONFIG=/tmp/lbrycrd_merged.conf
+      cat $CONFIG_PATH > $MERGED_CONFIG
+      echo "" >> $MERGED_CONFIG
+      override_config_option PORT port $MERGED_CONFIG
+      override_config_option RPC_USER rpcuser $MERGED_CONFIG
+      override_config_option RPC_PASSWORD rpcpassword $MERGED_CONFIG
+      override_config_option RPC_ALLOW_IP rpcallowip $MERGED_CONFIG
+      override_config_option RPC_PORT rpcport $MERGED_CONFIG
+      override_config_option RPC_BIND rpcbind $MERGED_CONFIG
+      # Make the new merged config file the new CONFIG_PATH
+      # This ensures that the original file the user mounted remains unmodified
+      CONFIG_PATH=$MERGED_CONFIG
+  else
+      echo "Creating a fresh config file from environment variables."
+      ## Set config params
+      echo "port=${PORT=9246}" > $CONFIG_PATH
+      echo "rpcuser=${RPC_USER=lbry}" >> $CONFIG_PATH
+      echo "rpcpassword=${RPC_PASSWORD=lbry}" >> $CONFIG_PATH
+      echo "rpcallowip=${RPC_ALLOW_IP=127.0.0.1/24}" >> $CONFIG_PATH
+      echo "rpcport=${RPC_PORT=9245}" >> $CONFIG_PATH
+      echo "rpcbind=${RPC_BIND=0.0.0.0}" >> $CONFIG_PATH
+  fi
+  echo "Config: "
+  cat $CONFIG_PATH
+}
+
+## Ensure perms are correct prior to running main binary
+/usr/bin/fix-permissions
+
+## You can optionally specify a run mode if you want to use lbry defined presets for compatibility.
+case $RUN_MODE in
+  default )
+    set_config
+    lbrycrdd -server -conf=$CONFIG_PATH -printtoconsole
+    ;;
+  ## If it's a first run you need to do a full index including all transactions
+  ## tx index creates an index of every single transaction in the block history if
+  ## not specified it will only create an index for transactions that are related to the wallet or have unspent outputs.
+  ## This is generally specific to chainquery.
+  reindex )
+    ## Apply this RUN_MODE in the case you need to update a dataset.  NOTE: you do not need to use `RUN_MODE reindex` for more than one complete run.
+    set_config
+    lbrycrdd -server -txindex -reindex -conf=$CONFIG_PATH -printtoconsole
+    ;;
+  chainquery )
+    ## If your only goal is to run Chainquery against this instance of lbrycrd and you're starting a
+    ## fresh local dataset use this run mode.
+    set_config
+    lbrycrdd -server -txindex -conf=$CONFIG_PATH -printtoconsole
+    ;;
+  regtest )
+    ## Set config params
+    ## TODO: Make this more automagic in the future.
+    mkdir -p `dirname $CONFIG_PATH`
+    echo "rpcuser=lbry" >           $CONFIG_PATH
+    echo "rpcpassword=lbry" >>      $CONFIG_PATH
+    echo "rpcport=29245" >>         $CONFIG_PATH
+    echo "rpcbind=0.0.0.0" >>       $CONFIG_PATH
+    echo "rpcallowip=0.0.0.0/0" >>  $CONFIG_PATH
+    echo "regtest=1" >>             $CONFIG_PATH
+    echo "txindex=1" >>             $CONFIG_PATH
+    echo "server=1" >>              $CONFIG_PATH
+    echo "printtoconsole=1" >>      $CONFIG_PATH
+
+    #nohup advance &>/dev/null &
+    lbrycrdd -conf=$CONFIG_PATH $1
+    ;;
+  testnet )
+    ## Set config params
+    ## TODO: Make this more automagic in the future.
+    mkdir -p `dirname $CONFIG_PATH`
+    echo "rpcuser=lbry" >           $CONFIG_PATH
+    echo "rpcpassword=lbry" >>      $CONFIG_PATH
+    echo "rpcport=29245" >>         $CONFIG_PATH
+    echo "rpcbind=0.0.0.0" >>       $CONFIG_PATH
+    echo "rpcallowip=0.0.0.0/0" >>  $CONFIG_PATH
+    echo "testnet=1" >>             $CONFIG_PATH
+    echo "txindex=1" >>             $CONFIG_PATH
+    echo "server=1" >>              $CONFIG_PATH
+    echo "printtoconsole=1" >>      $CONFIG_PATH
+
+    #nohup advance &>/dev/null &
+    lbrycrdd -conf=$CONFIG_PATH $1
+    ;;
+  * )
+    echo "Error, you must define a RUN_MODE environment variable."
+    echo "Available options are testnet, regtest, chainquery, default, and reindex"
+    ;;
+esac

e2e/lbrynet/docker-compose.yml

@@ -0,0 +1,23 @@
+version: "3"
+networks:
+  lbry-network:
+    external: true
+
+services:
+  #############
+  ## Lbrynet ##
+  #############
+  lbrynet:
+    image: lbry/lbrynet:v0.38.5
+    restart: "no"
+    networks:
+      lbry-network:
+        ipv4_address: 10.6.1.3
+    ports:
+      - "15100:5279"
+      - "15101:5280"
+    environment:
+      - LBRY_STREAMING_SERVER=0.0.0.0:5280
+    volumes:
+      - "../persist/data/.lbrynet:/home/lbrynet"
+      - "./settings:/etc/lbry" #Put your daemon_settings.yml here

e2e/lbrynet/docker/Dockerfile

@@ -0,0 +1,27 @@
+## This base image is for running the latest lbrynet-daemon release.
+FROM ubuntu:16.04 as prep
+LABEL MAINTAINER="leopere [at] nixc [dot] us"
+RUN apt-get update && apt-get -y install unzip curl telnet wait-for-it
+
+## Add lbrynet
+ARG VERSION="v0.38.5"
+RUN URL=$(curl -s https://api.github.com/repos/lbryio/lbry-sdk/releases/$(if [ "${VERSION}" = 'latest' ]; then echo "latest"; else echo "tags/${VERSION}"; fi) | grep browser_download_url | grep lbrynet-linux.zip | cut -d'"' -f4) && echo $URL && curl -L -o /lbrynet.linux.zip $URL
+
+COPY start.sh /usr/bin/start
+COPY checkmount.sh /usr/bin/checkmount
+RUN unzip /lbrynet.linux.zip -d /lbrynet/ && \
+    mv /lbrynet/lbrynet /usr/bin && \
+    chmod a+x /usr/bin/checkmount /usr/bin/start /usr/bin/lbrynet
+
+FROM ubuntu:16.04 as app
+COPY --from=prep /usr/bin/start /usr/bin/checkmount /usr/bin/lbrynet /usr/bin/
+RUN adduser lbrynet --gecos GECOS --shell /bin/bash --disabled-password --home /home/lbrynet
+## Daemon port [Intended for internal use]
+## LBRYNET talks to peers on port 3333 [Intended for external use] this port is used to discover other lbrynet daemons with blobs.
+## Expose 5566 Reflector port to listen on
+## Expose 5279 Port the daemon API will listen on
+## the lbryumx aka Wallet port [Intended for internal use]
+#EXPOSE 4444 3333 5566 5279 50001
+USER lbrynet
+ENTRYPOINT ["/usr/bin/checkmount"]
+CMD ["start"]

e2e/lbrynet/docker/build.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+if [ $# -eq 0 ]
+  then
+    echo "No docker tag argument supplied. Use './build.sh <tag>'"
+    exit 1
+fi
+docker build --tag lbry/lbrynet:$1 .
+docker push lbry/lbrynet:$1

e2e/lbrynet/docker/checkmount.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+
+## TODO: Make a bit more aware of the run mode of this appliance in case there is ever a test mode enabled in the start.sh
+mountpoint=/home/lbrynet
+
+if ! grep -qs ".* $mountpoint " /proc/mounts; then
+    echo "$mountpoint not mounted, refusing to run."
+    ## TODO: We should have documentation that this error references directly with a URL as to why it won't run without a volume.
+    exit 1
+else
+    bash -c "$*"
+fi

e2e/lbrynet/docker/start.sh

@@ -0,0 +1,4 @@
+#!/bin/bash
+lbrynet start \
+  --api="${API_BIND_IP:-0.0.0.0}":"${API_PORT:-5279}" \
+  --config="${CONFIG_PATH:-/etc/lbry/daemon_settings.yml}"

e2e/lbrynet/settings/daemon_settings.yml

@@ -0,0 +1,12 @@
+#blockchain_name: lbrycrd_main
+#blockchain_name: lbrycrd_testnet
+blockchain_name: lbrycrd_regtest
+lbryum_servers:
+#  - spv1.lbry.com:50001 #Production Wallet Server
+  - walletserver:50001
+save_blobs: true
+save_files: false
+share_usage_data: false
+tcp_port: 3333
+udp_port: 4444
+use_upnp: true

e2e/walletserver/docker-compose.yml

@@ -0,0 +1,35 @@
+version: "3"
+networks:
+  lbry-network:
+    external: true
+
+services:
+  ###################
+  ## Wallet Server ##
+  ###################
+  walletserver:
+    image: lbry/wallet-server:v0.38.5
+    restart: always
+    networks:
+      lbry-network:
+        ipv4_address: 10.6.1.2
+    volumes:
+      - "../persist/data/.walletserver/database:/database"
+    environment:
+      - DB_DIRECTORY=/database
+      - MAX_SEND=1000000000000000000000
+      - DAEMON_URL=http://lbry:lbry@lbrycrd:29245/
+      - MAX_SUBS=1000000000000
+      - BANDWIDTH_LIMIT=80000000000
+      - SESSION_TIMEOUT=10000000000000000000000000
+      - TCP_PORT=50001
+    #network_mode: host
+    #network_mode: bridge
+    ports:
+      - "50001:50001"
+    expose:
+      - "50001"
+    ulimits:
+      nofile: 90000
+#    command: lbry.wallet.server.coin.LBC
+    command: lbry.wallet.server.coin.LBCRegTest

go.mod

@@ -1,46 +1,34 @@
 module github.com/lbryio/ytsync
 
 require (
-	cloud.google.com/go v0.37.4 // indirect
 	github.com/ChannelMeter/iso8601duration v0.0.0-20150204201828-8da3af7a2a61
+	github.com/Microsoft/go-winio v0.4.13 // indirect
 	github.com/armon/go-metrics v0.0.0-20190430140413-ec5e00d3c878 // indirect
 	github.com/asaskevich/govalidator v0.0.0-20180720115003-f9ffefc3facf
 	github.com/aws/aws-sdk-go v1.17.3
 	github.com/channelmeter/iso8601duration v0.0.0-20150204201828-8da3af7a2a61 // indirect
-	github.com/go-ini/ini v1.42.0 // indirect
+	github.com/docker/distribution v2.7.1+incompatible // indirect
-	github.com/go-sql-driver/mysql v1.4.1 // indirect
+	github.com/docker/docker v1.13.1
-	github.com/gopherjs/gopherjs v0.0.0-20190411002643-bd77b112433e // indirect
+	github.com/docker/go-connections v0.4.0 // indirect
+	github.com/docker/go-units v0.4.0 // indirect
 	github.com/hashicorp/go-msgpack v0.5.5 // indirect
-	github.com/hashicorp/golang-lru v0.5.1 // indirect
 	github.com/hashicorp/memberlist v0.1.4 // indirect
 	github.com/hashicorp/serf v0.8.2 // indirect
-	github.com/konsorten/go-windows-terminal-sequences v1.0.2 // indirect
 	github.com/kr/pretty v0.1.0 // indirect
 	github.com/lbryio/errors.go v0.0.0-20180223142025-ad03d3cc6a5c
-	github.com/lbryio/lbry.go v1.0.15
+	github.com/lbryio/lbry.go v1.0.17
 	github.com/lbryio/reflector.go v1.0.6-0.20190806185326-2e4f235489f4
-	github.com/lusis/slack-test v0.0.0-20190408224659-6cf59653add2 // indirect
 	github.com/mitchellh/go-ps v0.0.0-20170309133038-4fdf99ab2936
-	github.com/onsi/ginkgo v1.8.0 // indirect
+	github.com/opencontainers/go-digest v1.0.0-rc1 // indirect
-	github.com/onsi/gomega v1.5.0 // indirect
-	github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2 // indirect
-	github.com/prometheus/common v0.3.0
 	github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24
 	github.com/sirupsen/logrus v1.4.1
-	github.com/smartystreets/assertions v0.0.0-20190401211740-f487f9de1cd3 // indirect
-	github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a // indirect
 	github.com/spf13/cobra v0.0.0-20190109003409-7547e83b2d85
 	github.com/spf13/pflag v1.0.3 // indirect
-	github.com/ybbus/jsonrpc v2.1.2+incompatible // indirect
+	golang.org/x/crypto v0.0.0-20190325154230-a5d413f7728c // indirect
-	go.opencensus.io v0.20.2 // indirect
-	golang.org/x/crypto v0.0.0-20190418165655-df01cb2cc480 // indirect
 	golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c // indirect
-	golang.org/x/oauth2 v0.0.0-20190402181905-9f3314589c9a // indirect
 	golang.org/x/sync v0.0.0-20190423024810-112230192c58 // indirect
+	golang.org/x/sys v0.0.0-20190523142557-0e01d883c5c5 // indirect
 	golang.org/x/text v0.3.2 // indirect
 	google.golang.org/api v0.3.2
-	google.golang.org/appengine v1.5.0 // indirect
-	google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7 // indirect
 	google.golang.org/grpc v1.20.0 // indirect
-	gopkg.in/ini.v1 v1.42.0 // indirect
 )

go.sum

@@ -1,21 +1,19 @@
 cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
+cloud.google.com/go v0.34.0 h1:eOI3/cP2VTU6uZLDYAoic+eyzzB9YyGmJ7eIjl8rOPg=
 cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
-cloud.google.com/go v0.37.4 h1:glPeL3BQJsbF6aIIYfZizMwc5LTYz250bDMjttbBGAU=
-cloud.google.com/go v0.37.4/go.mod h1:NHPJ89PdicEuT9hdPXMROBD91xc5uRDxsMtSB16k7hw=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/ChannelMeter/iso8601duration v0.0.0-20150204201828-8da3af7a2a61 h1:N5Vqww5QISEHsWHOWDEx4PzdIay3Cg0Jp7zItq2ZAro=
 github.com/ChannelMeter/iso8601duration v0.0.0-20150204201828-8da3af7a2a61/go.mod h1:GnKXcK+7DYNy/8w2Ex//Uql4IgfaU82Cd5rWKb7ah00=
 github.com/DataDog/datadog-go v2.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ=
+github.com/Microsoft/go-winio v0.4.13 h1:Hmi80lzZuI/CaYmlJp/b+FjZdRZhKu9c2mDVqKlLWVs=
+github.com/Microsoft/go-winio v0.4.13/go.mod h1:qXqCSQ3Xa7+6tgxaGTIe4Kpcdsi+P8jBhyzoq1bpyYA=
 github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo=
 github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI=
 github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII=
-github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc h1:cAKDfWh5VpdgMhJosfJnn5/FoN2SRZ4p7fJNX58YPaU=
 github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
-github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf h1:qet1QNfXsQxTZqLG4oE62mJzwPIB8+Tee4RNCL9ulrY=
 github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
 github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ=
 github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o=
-github.com/armon/go-metrics v0.0.0-20180713145231-3c58d8115a78 h1:mdRSArcFLfW0VoL34LZAKSz6LkkK4jFxVx2xYavACMg=
 github.com/armon/go-metrics v0.0.0-20180713145231-3c58d8115a78/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY=
 github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY=
 github.com/armon/go-metrics v0.0.0-20190430140413-ec5e00d3c878 h1:EFSB7Zo9Eg91v7MJPVsifUysc/wPdN+NOnVe6bWbdBM=
@@ -56,6 +54,14 @@ github.com/davecgh/go-spew v0.0.0-20171005155431-ecdeabc65495/go.mod h1:J7Y8YcW2
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/docker/distribution v2.7.1+incompatible h1:a5mlkVzth6W5A4fOsS3D2EO5BUmsJpcB+cRlLU7cSug=
+github.com/docker/distribution v2.7.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=
+github.com/docker/docker v1.13.1 h1:IkZjBSIc8hBjLpqeAbeE5mca5mNgeatLHBy3GO78BWo=
+github.com/docker/docker v1.13.1/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
+github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ=
+github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec=
+github.com/docker/go-units v0.4.0 h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw=
+github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
 github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs=
 github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU=
 github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I=
@@ -68,15 +74,13 @@ github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMo
 github.com/go-errors/errors v1.0.1 h1:LUHzmkK3GUKUrL/1gfBUxAHzcev3apQlezX/+O7ma6w=
 github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q=
 github.com/go-ini/ini v1.38.2/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8=
+github.com/go-ini/ini v1.41.0 h1:526aoxDtxRHFQKMZfcX2OG9oOI8TJ5yPLM0Mkno/uTY=
 github.com/go-ini/ini v1.41.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8=
-github.com/go-ini/ini v1.42.0 h1:TWr1wGj35+UiWHlBA8er89seFXxzwFn11spilrrj+38=
-github.com/go-ini/ini v1.42.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8=
 github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
 github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
 github.com/go-ozzo/ozzo-validation v3.5.0+incompatible/go.mod h1:gsEKFIVnabGBt6mXmxK0MoFy+cZoTJY6mu5Ll3LVLBU=
+github.com/go-sql-driver/mysql v0.0.0-20180719071942-99ff426eb706 h1:P3NPKb7qq581SeMCB+dU1SuCX1kQh8VoQ/4HmT2ftQY=
 github.com/go-sql-driver/mysql v0.0.0-20180719071942-99ff426eb706/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
-github.com/go-sql-driver/mysql v1.4.1 h1:g24URVg0OFbNUTx9qqY1IRZ9D9z3iPyi5zKhQZpNwpA=
-github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
 github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
 github.com/gogo/googleapis v1.1.0/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFGgqEef3s=
 github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
@@ -95,13 +99,8 @@ github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c h1:964Od4U6p2jUkFxvCy
 github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/go-cmp v0.2.0 h1:+dTQ8DZQJz0Mb/HjFlkptS1FeQ4cWSnN941F8aEG4SQ=
 github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
-github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
+github.com/gopherjs/gopherjs v0.0.0-20181103185306-d547d1d9531e h1:JKmoR8x90Iww1ks85zJ1lfDGgIiMDuIptTOhJq+zKyg=
-github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
-github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
-github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
 github.com/gopherjs/gopherjs v0.0.0-20181103185306-d547d1d9531e/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
-github.com/gopherjs/gopherjs v0.0.0-20190411002643-bd77b112433e h1:XWcjeEtTFTOVA9Fs1w7n2XBftk5ib4oZrhzWk0B+3eA=
-github.com/gopherjs/gopherjs v0.0.0-20190411002643-bd77b112433e/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
 github.com/gorilla/context v1.1.1 h1:AWwleXJkX/nhcU9bZSnZoi3h/qGYqQAGhq6zZe/aQW8=
 github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg=
 github.com/gorilla/mux v1.6.2 h1:Pgr17XVTNXAk3q/r4CpKzC5xBM/qW1uVLV+IhRZpIIk=
@@ -111,26 +110,21 @@ github.com/gorilla/rpc v1.1.0/go.mod h1:V4h9r+4sF5HnzqbwIez0fKSpANP0zlYd3qR7p36j
 github.com/gorilla/websocket v1.2.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
 github.com/gorilla/websocket v1.4.0 h1:WDFjx/TMzVgy9VdMMQi2K2Emtwi2QcUQsztZ/zLaH/Q=
 github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
-github.com/hashicorp/errwrap v0.0.0-20180715044906-d6c0cd880357 h1:Rem2+U35z1QtPQc6r+WolF7yXiefXqDKyk+lN2pE164=
 github.com/hashicorp/errwrap v0.0.0-20180715044906-d6c0cd880357/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
 github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA=
 github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
 github.com/hashicorp/go-cleanhttp v0.5.0/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80=
-github.com/hashicorp/go-immutable-radix v0.0.0-20180129170900-7f3cd4390caa h1:0nA8i+6Rwqaq9xlpmVxxTwk6rxiEhX+E6Wh4vPNHiS8=
 github.com/hashicorp/go-immutable-radix v0.0.0-20180129170900-7f3cd4390caa/go.mod h1:6ij3Z20p+OhOkCSrA0gImAWoHYQRGbnlcuk6XYTiaRw=
 github.com/hashicorp/go-immutable-radix v1.0.0 h1:AKDB1HM5PWEA7i4nhcpwOrO2byshxBjXVn/J/3+z5/0=
 github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60=
-github.com/hashicorp/go-msgpack v0.0.0-20150518234257-fa3f63826f7c h1:BTAbnbegUIMB6xmQCwWE8yRzbA4XSpnZY5hvRJC188I=
 github.com/hashicorp/go-msgpack v0.0.0-20150518234257-fa3f63826f7c/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM=
 github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM=
 github.com/hashicorp/go-msgpack v0.5.5 h1:i9R9JSrqIz0QVLz3sz+i3YJdT7TTSLcfLLzJi9aZTuI=
 github.com/hashicorp/go-msgpack v0.5.5/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM=
-github.com/hashicorp/go-multierror v0.0.0-20180717150148-3d5d8f294aa0 h1:j30noezaCfvNLcdMYSvHLv81DxYRSt1grlpseG67vhU=
 github.com/hashicorp/go-multierror v0.0.0-20180717150148-3d5d8f294aa0/go.mod h1:JMRHfdO9jKNzS/+BTlxCjKNQHg/jZAft8U7LloJvN7I=
 github.com/hashicorp/go-multierror v1.0.0 h1:iVjPR7a6H0tWELX5NxNe7bYopibicUzc7uPribsnS6o=
 github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk=
 github.com/hashicorp/go-retryablehttp v0.5.3/go.mod h1:9B5zBasrRhHXnJnui7y6sL7es7NDiJgTc6Er0maI1Xs=
-github.com/hashicorp/go-sockaddr v0.0.0-20180320115054-6d291a969b86 h1:7YOlAIO2YWnJZkQp7B5eFykaIY7C9JndqAFQyVV5BhM=
 github.com/hashicorp/go-sockaddr v0.0.0-20180320115054-6d291a969b86/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU=
 github.com/hashicorp/go-sockaddr v1.0.0 h1:GeH6tui99pF4NJgfnhp+L6+FfobzVW3Ah46sLo0ICXs=
 github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU=
@@ -140,17 +134,14 @@ github.com/hashicorp/go-uuid v1.0.1 h1:fv1ep09latC32wFoVwnqcnKJGnMSdBanPczbHAYm1
 github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
 github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90=
 github.com/hashicorp/golang-lru v0.0.0-20180201235237-0fb14efe8c47/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
+github.com/hashicorp/golang-lru v0.5.0 h1:CL2msUPvZTLb5O648aiLNJw3hnBxN2+1Jq8rCOH9wdo=
 github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
-github.com/hashicorp/golang-lru v0.5.1 h1:0hERBMJE1eitiLkihrMvRVBYAkpHzc/J3QdDN+dAcgU=
-github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
 github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64=
 github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ=
-github.com/hashicorp/memberlist v0.1.0 h1:qSsCiC0WYD39lbSitKNt40e30uorm2Ss/d4JGU1hzH8=
 github.com/hashicorp/memberlist v0.1.0/go.mod h1:ncdBp14cuox2iFOq3kDiquKU6fqsTBc3W6JvZwjxxsE=
 github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I=
 github.com/hashicorp/memberlist v0.1.4 h1:gkyML/r71w3FL8gUi74Vk76avkj/9lYAY9lvg0OcoGs=
 github.com/hashicorp/memberlist v0.1.4/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I=
-github.com/hashicorp/serf v0.0.0-20180530155958-984a73625de3 h1:NUr1hG6WO9sI1x8ofSimmpqfJ+rEHiHP/PLEA33rcfQ=
 github.com/hashicorp/serf v0.0.0-20180530155958-984a73625de3/go.mod h1:h/Ru6tmZazX7WO/GDmwdpS975F019L4t5ng5IgwbNrE=
 github.com/hashicorp/serf v0.8.2 h1:YZ7UKsJv+hKjqGVUUbtE3HNj79Eln2oQ75tniF6iPt0=
 github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc=
@@ -169,17 +160,14 @@ github.com/johntdyer/slack-go v0.0.0-20180213144715-95fac1160b22/go.mod h1:u0Jo4
 github.com/johntdyer/slackrus v0.0.0-20180518184837-f7aae3243a07 h1:+kBG/8rjCa6vxJZbUjAiE4MQmBEBYc8nLEb51frnvBY=
 github.com/johntdyer/slackrus v0.0.0-20180518184837-f7aae3243a07/go.mod h1:j1kV/8f3jowErEq4XyeypkCdvg5EeHkf0YCKCcq5Ybo=
 github.com/jrick/logrotate v1.0.0/go.mod h1:LNinyqDIJnpAur+b8yyulnQw/wDuN1+BYKlTRt3OuAQ=
-github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
+github.com/jtolds/gls v4.2.1+incompatible h1:fSuqC+Gmlu6l/ZYAoZzx2pyucC8Xza35fpRVWLVmUEE=
 github.com/jtolds/gls v4.2.1+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
-github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo=
-github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
 github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
 github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4=
 github.com/kkdai/bstream v0.0.0-20181106074824-b3251f7901ec/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4=
+github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk=
 github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
-github.com/konsorten/go-windows-terminal-sequences v1.0.2 h1:DB17ag19krx9CFsz4o3enTrPXyIXCl+2iCXH/aMAp9s=
-github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
 github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
@@ -189,10 +177,11 @@ github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/lbryio/errors.go v0.0.0-20180223142025-ad03d3cc6a5c h1:BhdcWGsuKif/XoSZnqVGNqJ1iEmH0czWR5upj+AuR8M=
 github.com/lbryio/errors.go v0.0.0-20180223142025-ad03d3cc6a5c/go.mod h1:muH7wpUqE8hRA3OrYYosw9+Sl681BF9cwcjzE+OCNK8=
 github.com/lbryio/lbry.go v0.0.0-20190109223729-30c312501602/go.mod h1:YEuFJD/oHNra6BFy+NfuvS84Wg6RMWJFGtiCCCc6MmQ=
-github.com/lbryio/lbry.go v1.0.15 h1:g4g9cDDUsobmWgTA+1jEIb5k2fRCP0/NvPOMXduP8xY=
+github.com/lbryio/lbry.go v1.0.17 h1:Lt7xIguw6Q5LMKcEMfxD7IB7kUdP7xwrc0OxvBUl39E=
-github.com/lbryio/lbry.go v1.0.15/go.mod h1:JtyI30bU51rm0LZ/po3mQuzf++14OWb6kR/6mMRAmKU=
+github.com/lbryio/lbry.go v1.0.17/go.mod h1:JtyI30bU51rm0LZ/po3mQuzf++14OWb6kR/6mMRAmKU=
 github.com/lbryio/lbryschema.go v0.0.0-20190428231007-c54836bca002 h1:urfYK5ElpUrAv90auPLldoVC60LwiGAcY0OE6HJB9KI=
 github.com/lbryio/lbryschema.go v0.0.0-20190428231007-c54836bca002/go.mod h1:dAzPCBj3CKKWBGYBZxK6tKBP5SCgY2tqd9SnQd/OyKo=
+github.com/lbryio/ozzo-validation v0.0.0-20170323141101-d1008ad1fd04 h1:Nze+C2HbeKvhjI/kVn+9Poj/UuEW5sOQxcsxqO7L3GI=
 github.com/lbryio/ozzo-validation v0.0.0-20170323141101-d1008ad1fd04/go.mod h1:fbG/dzobG8r95KzMwckXiLMHfFjZaBRQqC9hPs2XAQ4=
 github.com/lbryio/reflector.go v1.0.6-0.20190806185326-2e4f235489f4 h1:SpUbq2YBg3ncetkw8APUgn8nFF8dscKzzhyiWMM7XCc=
 github.com/lbryio/reflector.go v1.0.6-0.20190806185326-2e4f235489f4/go.mod h1:7Y3YYeAKS6egH2WzwfU8f6+uNGjVHfLzKvwn+Nv3VMY=
@@ -201,16 +190,14 @@ github.com/lbryio/types v0.0.0-20190422033210-321fb2abda9c h1:m3O7561xBQ00lfUVay
 github.com/lbryio/types v0.0.0-20190422033210-321fb2abda9c/go.mod h1:CG3wsDv5BiVYQd5i1Jp7wGsaVyjZTJshqXeWMVKsISE=
 github.com/lusis/go-slackbot v0.0.0-20180109053408-401027ccfef5 h1:AsEBgzv3DhuYHI/GiQh2HxvTP71HCCE9E/tzGUzGdtU=
 github.com/lusis/go-slackbot v0.0.0-20180109053408-401027ccfef5/go.mod h1:c2mYKRyMb1BPkO5St0c/ps62L4S0W2NAkaTXj9qEI+0=
+github.com/lusis/slack-test v0.0.0-20180109053238-3c758769bfa6 h1:iOAVXzZyXtW408TMYejlUPo6BIn92HmOacWtIfNyYns=
 github.com/lusis/slack-test v0.0.0-20180109053238-3c758769bfa6/go.mod h1:sFlOUpQL1YcjhFVXhg1CG8ZASEs/Mf1oVb6H75JL/zg=
-github.com/lusis/slack-test v0.0.0-20190408224659-6cf59653add2 h1:CTzpAplpuXay3gDDqbErv/OSV6SoKZ9UTsGxLKfxB8M=
-github.com/lusis/slack-test v0.0.0-20190408224659-6cf59653add2/go.mod h1:sFlOUpQL1YcjhFVXhg1CG8ZASEs/Mf1oVb6H75JL/zg=
 github.com/lyft/protoc-gen-validate v0.0.13/go.mod h1:XbGvPuh87YZc5TdIa2/I4pLk0QoUACkjt2znoq26NVQ=
 github.com/lyoshenka/bencode v0.0.0-20180323155644-b7abd7672df5 h1:mG83tLXWSRdcXMWfkoumVwhcCbf3jHF9QKv/m37BkM0=
 github.com/lyoshenka/bencode v0.0.0-20180323155644-b7abd7672df5/go.mod h1:H0aPCWffGOaDcjkw1iB7W9DVLp6GXmfcJY/7YZCWPA4=
 github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
 github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
 github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
-github.com/miekg/dns v1.0.8 h1:Zi8HNpze3NeRWH1PQV6O71YcvJRQ6j0lORO6DAEmAAI=
 github.com/miekg/dns v1.0.8/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
 github.com/miekg/dns v1.0.14 h1:9jZdLNd/P4+SfEJ0TNyxYpsK8N4GtfylBLqtbYN1sbA=
 github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
@@ -229,20 +216,19 @@ github.com/nlopes/slack v0.4.0/go.mod h1:jVI4BBK3lSktibKahxBF74txcK2vyvkza1z/+rR
 github.com/nlopes/slack v0.5.0 h1:NbIae8Kd0NpqaEI3iUrsuS0KbcEDhzhc939jLW5fNm0=
 github.com/nlopes/slack v0.5.0/go.mod h1:jVI4BBK3lSktibKahxBF74txcK2vyvkza1z/+rRnVAM=
 github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
+github.com/onsi/ginkgo v1.7.0 h1:WSHQ+IS43OoUrWtD1/bbclrwK8TTH5hzp+umCiuxHgs=
 github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
-github.com/onsi/ginkgo v1.8.0 h1:VkHVNpR4iVnU8XQR6DBm8BqYjN7CRzw+xKUbVVbbW9w=
-github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/gomega v1.4.1/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA=
+github.com/onsi/gomega v1.4.3 h1:RE1xgDvH7imwFD45h+u2SgIfERHlS2yNG4DObb5BSKU=
 github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
-github.com/onsi/gomega v1.5.0 h1:izbySO9zDPmjJ8rDjLvkA2zJHIo+HkYXHnf7eN7SSyo=
+github.com/opencontainers/go-digest v1.0.0-rc1 h1:WzifXhOVOEOuFYOJAW6aQqW0TooG2iki3E3Ii+WN7gQ=
-github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
+github.com/opencontainers/go-digest v1.0.0-rc1/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s=
 github.com/openzipkin/zipkin-go v0.1.6/go.mod h1:QgAqvLzwWbR/WpD4A3cGpPtJrZXNIiJc5AZX7/PBEpw=
 github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
 github.com/pascaldekloe/goe v0.1.0 h1:cBOtyMzM9HTpWjXfbbunk26uA6nG3a8n06Wieeh0MwY=
 github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
+github.com/phayes/freeport v0.0.0-20171002185219-e27662a4a9d6 h1:2bae6N0SZjgzk+Zg8mzTsfmpwHXY9VBNp9UdjhaElA0=
 github.com/phayes/freeport v0.0.0-20171002185219-e27662a4a9d6/go.mod h1:iIss55rKnNBTvrwdmkUpLnDpZoAHvWaiq5+iMmen4AE=
-github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2 h1:JhzVVoYvbOACxoUmOs6V/G4D5nPVUW73rKvXxP4XUJc=
-github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2/go.mod h1:iIss55rKnNBTvrwdmkUpLnDpZoAHvWaiq5+iMmen4AE=
 github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY=
 github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
@@ -257,8 +243,6 @@ github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:
 github.com/prometheus/client_model v0.0.0-20190115171406-56726106282f/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
 github.com/prometheus/common v0.0.0-20181126121408-4724e9255275/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=
 github.com/prometheus/common v0.2.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
-github.com/prometheus/common v0.3.0 h1:taZ4h8Tkxv2kNyoSctBvfXEHmBmxrwmIidZTIaHons4=
-github.com/prometheus/common v0.3.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
 github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
 github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
 github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
@@ -278,13 +262,10 @@ github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPx
 github.com/sirupsen/logrus v1.3.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
 github.com/sirupsen/logrus v1.4.1 h1:GL2rEmy6nsikmW0r8opw9JIRScdMF5hA8cOYLH7In1k=
 github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q=
+github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM=
 github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
-github.com/smartystreets/assertions v0.0.0-20190401211740-f487f9de1cd3 h1:hBSHahWMEgzwRyS6dRpxY0XyjZsHyQ61s084wo5PJe0=
+github.com/smartystreets/goconvey v0.0.0-20181108003508-044398e4856c h1:Ho+uVpkel/udgjbwB5Lktg9BtvJSh2DT0Hi6LPSyI2w=
-github.com/smartystreets/assertions v0.0.0-20190401211740-f487f9de1cd3/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
 github.com/smartystreets/goconvey v0.0.0-20181108003508-044398e4856c/go.mod h1:XDJAKZRPZ1CvBcN2aX5YOUTYGHki24fSF0Iv48Ibg0s=
-github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a h1:pa8hGb/2YqsZKovtsgrwcDH1RZhVbTKCjLp47XpqCDs=
-github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
-github.com/spf13/cast v1.2.0 h1:HHl1DSRbEQN2i8tJmtS6ViPyHx35+p51amrdsiTCrkg=
 github.com/spf13/cast v1.2.0/go.mod h1:r2rcYCSwa1IExKTDiTfzaxqT2FNHs8hODu4LnUfgKEg=
 github.com/spf13/cast v1.3.0 h1:oget//CVOEoFewqQxwr0Ej5yjygnqGkvggSE/gB35Q8=
 github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
@@ -302,12 +283,10 @@ github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UV
 github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM=
 github.com/uber-go/atomic v1.3.2 h1:Azu9lPBWRNKzYXSIwRfgRuDuS0YKsK4NFhiQv98gkxo=
 github.com/uber-go/atomic v1.3.2/go.mod h1:/Ct5t2lcmbJ4OSe/waGBoaVvVqtO0bmtfVNex1PFV8g=
+github.com/ybbus/jsonrpc v0.0.0-20180411222309-2a548b7d822d h1:tQo6hjclyv3RHUgZOl6iWb2Y44A/sN9bf9LAYfuioEg=
 github.com/ybbus/jsonrpc v0.0.0-20180411222309-2a548b7d822d/go.mod h1:XJrh1eMSzdIYFbM08flv0wp5G35eRniyeGut1z+LSiE=
-github.com/ybbus/jsonrpc v2.1.2+incompatible h1:V4mkE9qhbDQ92/MLMIhlhMSbz8jNXdagC3xBR5NDwaQ=
+go.opencensus.io v0.20.1 h1:pMEjRZ1M4ebWGikflH7nQpV6+Zr88KBMA2XJD3sbijw=
-github.com/ybbus/jsonrpc v2.1.2+incompatible/go.mod h1:XJrh1eMSzdIYFbM08flv0wp5G35eRniyeGut1z+LSiE=
 go.opencensus.io v0.20.1/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk=
-go.opencensus.io v0.20.2 h1:NAfh7zF0/3/HqtMvJNZ/RFrSlCE6ZTlHmKfhL/Dm1Jk=
-go.opencensus.io v0.20.2/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk=
 go.uber.org/atomic v1.3.2 h1:2Oa65PReHzfn29GpvgsYwloV9AVFHPDk8tYxt2c2tr4=
 go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
 golang.org/x/crypto v0.0.0-20170930174604-9419663f5a44/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
@@ -316,8 +295,8 @@ golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnf
 golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20190103213133-ff983b9c42bc/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
-golang.org/x/crypto v0.0.0-20190418165655-df01cb2cc480 h1:O5YqonU5IWby+w98jVUG9h7zlCWCcH4RHyPVReBmhzk=
+golang.org/x/crypto v0.0.0-20190325154230-a5d413f7728c h1:Vj5n4GlwjmQteupaxJ9+0FNOmBrHfq7vN4btdGoDZgI=
-golang.org/x/crypto v0.0.0-20190418165655-df01cb2cc480/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE=
+golang.org/x/crypto v0.0.0-20190325154230-a5d413f7728c/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
 golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
@@ -341,13 +320,11 @@ golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c h1:uOCk1iQW6Vc18bnC13MfzScl+
 golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
+golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421 h1:Wo7BWFiOk0QRFMLYMqJGFMd9CgUAcGx7V+qEg/h5IBI=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
-golang.org/x/oauth2 v0.0.0-20190402181905-9f3314589c9a h1:tImsplftrFpALCYumobsd0K86vlAs/eXGFms2txfJfA=
-golang.org/x/oauth2 v0.0.0-20190402181905-9f3314589c9a/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6 h1:bjcUS9ztw9kFmmIxJInhon/0Is3p+EHBKNgquIzo1OI=
 golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58 h1:8gQV6CLnAEikrhgkHFbMAEhagSSnXWGV915qUMm9mrU=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -360,12 +337,11 @@ golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5h
 golang.org/x/sys v0.0.0-20181122145206-62eef0e2fa9b/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190109145017-48ac38b7c8cb/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190520201301-c432e742b0af h1:NXfmMfXz6JqGfG3ikSxcz2N93j6DgScr19Oo2uwFu88=
 golang.org/x/sys v0.0.0-20190520201301-c432e742b0af/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190523142557-0e01d883c5c5 h1:sM3evRHxE/1RuMe1FYAL3j7C7fUfIjkbE+NiDAYUF8U=
+golang.org/x/sys v0.0.0-20190523142557-0e01d883c5c5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
-golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2 h1:z99zHgr7hKfrUcX/KsoJk5FJfjTceCKIp96+biqP4To=
-golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
 golang.org/x/time v0.0.0-20181108054448-85acf8d2951c h1:fqgJT0MGcGpPgpWU7VRdRjuArfcOvC4AoJmILihzhDg=
@@ -377,29 +353,24 @@ golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGm
 golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
 golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
 golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
-golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
 google.golang.org/api v0.3.1/go.mod h1:6wY9I6uQWHQ8EM57III9mq/AjF+i8G65rmVagqKMtkk=
 google.golang.org/api v0.3.2 h1:iTp+3yyl/KOtxa/d1/JUE0GGSoR6FuW5udver22iwpw=
 google.golang.org/api v0.3.2/go.mod h1:6wY9I6uQWHQ8EM57III9mq/AjF+i8G65rmVagqKMtkk=
 google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
+google.golang.org/appengine v1.4.0 h1:/wp5JvzpHIxhs/dumFmF7BXTf3Z+dd4uXta4kVyO508=
 google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
-google.golang.org/appengine v1.5.0 h1:KxkO13IPW4Lslp2bz+KHP2E3gtFlrIGNThxkZQ3g+4c=
-google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
 google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
 google.golang.org/genproto v0.0.0-20180831171423-11092d34479b/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
 google.golang.org/genproto v0.0.0-20181004005441-af9cb2a35e7f/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
 google.golang.org/genproto v0.0.0-20190108161440-ae2f86662275/go.mod h1:7Ep/1NZk928CDR8SjdVbjWNpdIf6nzjE3BTgJDr2Atg=
+google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19 h1:Lj2SnHtxkRGJDqnGaSjo+CCdIieEnwVazbOXILwQemk=
 google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
-google.golang.org/genproto v0.0.0-20190404172233-64821d5d2107/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
-google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7 h1:ZUjXAXmrAyrmmCPHgCA/vChHcpsX27MZ3yBonD/z1KE=
-google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
 google.golang.org/grpc v1.16.0/go.mod h1:0JHn/cJsOMiMfNA9+DeHDlAU7KAAB5GDlYFpa9MZMio=
 google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs=
 google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
 google.golang.org/grpc v1.20.0 h1:DlsSIrgEBuZAUFJcta2B5i/lzeHHbnfkNFAfFXLVFYQ=
 google.golang.org/grpc v1.20.0/go.mod h1:chYK+tFQF0nDUGJgXMSgLCQk3phJEuONr2DCgLDdAQM=
 gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U=
-gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc=
 gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
@@ -407,9 +378,8 @@ gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8
 gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4=
 gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
 gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2/go.mod h1:Xk6kEKp8OKb+X14hQBKWaSkCsqBpgog8nAV2xsGOxlo=
+gopkg.in/ini.v1 v1.41.0 h1:Ka3ViY6gNYSKiVy71zXBEqKplnV35ImDLVG+8uoIklE=
 gopkg.in/ini.v1 v1.41.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
-gopkg.in/ini.v1 v1.42.0 h1:7N3gPTt50s8GuLortA00n8AqRTk75qOP98+mTPpgzRk=
-gopkg.in/ini.v1 v1.42.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
 gopkg.in/nullbio/null.v6 v6.0.0-20161116030900-40264a2e6b79 h1:FpCr9V8wuOei4BAen+93HtVJ+XSi+KPbaPKm0Vj5R64=
 gopkg.in/nullbio/null.v6 v6.0.0-20161116030900-40264a2e6b79/go.mod h1:gWkaRU7CoXpezCBWfWjm3999QqS+1pYPXGbqQCTMzo8=
 gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
@@ -421,5 +391,4 @@ gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo=
 gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw=
 honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
-honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190109154334-5bcec433c8ea/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=

main.go

@@ -4,16 +4,16 @@ import (
 	"fmt"
 	"math/rand"
 	"os"
-	"os/user"
 	"time"
 
+	"github.com/lbryio/lbry.go/extras/errors"
 	"github.com/lbryio/lbry.go/extras/util"
+	"github.com/lbryio/ytsync/manager"
 	"github.com/lbryio/ytsync/sdk"
+	ytUtils "github.com/lbryio/ytsync/util"
+
 	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
-
-	"github.com/lbryio/ytsync/manager"
-	logUtils "github.com/lbryio/ytsync/util"
 )
 
 var Version string
@@ -88,6 +88,10 @@ func ytSync(cmd *cobra.Command, args []string) {
 			log.Error("could not detect system hostname")
 			hostname = "ytsync-unknown"
 		}
+		if len(hostname) > 30 {
+			hostname = hostname[0:30]
+		}
+
 		util.InitSlack(os.Getenv("SLACK_TOKEN"), os.Getenv("SLACK_CHANNEL"), hostname)
 	}
 
@@ -113,7 +117,6 @@ func ytSync(cmd *cobra.Command, args []string) {
 	apiURL := os.Getenv("LBRY_WEB_API")
 	apiToken := os.Getenv("LBRY_API_TOKEN")
 	youtubeAPIKey := os.Getenv("YOUTUBE_API_KEY")
-	blobsDir := os.Getenv("BLOBS_DIRECTORY")
 	lbrycrdString := os.Getenv("LBRYCRD_STRING")
 	awsS3ID := os.Getenv("AWS_S3_ID")
 	awsS3Secret := os.Getenv("AWS_S3_SECRET")
@@ -150,14 +153,8 @@ func ytSync(cmd *cobra.Command, args []string) {
 	if lbrycrdString == "" {
 		log.Infoln("Using default (local) lbrycrd instance. Set LBRYCRD_STRING if you want to use something else")
 	}
-	if blobsDir == "" {
+
-		usr, err := user.Current()
+	blobsDir := ytUtils.GetBlobsDir()
-		if err != nil {
-			log.Errorln(err.Error())
-			return
-		}
-		blobsDir = usr.HomeDir + "/.lbrynet/blobfiles/"
-	}
 
 	syncProperties := &sdk.SyncProperties{
 		SyncFrom:         syncFrom,
@@ -198,7 +195,7 @@ func ytSync(cmd *cobra.Command, args []string) {
 	)
 	err := sm.Start()
 	if err != nil {
-		logUtils.SendErrorToSlack(err.Error())
+		ytUtils.SendErrorToSlack(errors.FullTrace(err))
 	}
-	logUtils.SendInfoToSlack("Syncing process terminated!")
+	ytUtils.SendInfoToSlack("Syncing process terminated!")
 }

manager/manager.go

@@ -101,11 +101,18 @@ const (
 
 func (s *SyncManager) Start() error {
 
+	if logUtils.ShouldCleanOnStartup() {
+		err := logUtils.CleanForStartup()
+		if err != nil {
+			return err
+		}
+	}
+
 	syncCount := 0
 	for {
 		err := s.checkUsedSpace()
 		if err != nil {
-			return err
+			return errors.Err(err)
 		}
 
 		var syncs []Sync
@@ -115,7 +122,7 @@ func (s *SyncManager) Start() error {
 		if isSingleChannelSync {
 			channels, err := s.apiConfig.FetchChannels("", s.syncProperties)
 			if err != nil {
-				return err
+				return errors.Err(err)
 			}
 			if len(channels) != 1 {
 				return errors.Err("Expected 1 channel, %d returned", len(channels))
@@ -239,9 +246,10 @@ func (s *SyncManager) GetS3AWSConfig() aws.Config {
 	}
 }
 func (s *SyncManager) checkUsedSpace() error {
-	usedPctile, err := GetUsedSpace(s.blobsDir)
+	logUtils.SendInfoToSlack(logUtils.GetBlobsDir())
+	usedPctile, err := GetUsedSpace(logUtils.GetBlobsDir())
 	if err != nil {
-		return err
+		return errors.Err(err)
 	}
 	if usedPctile >= 0.90 && !s.skipSpaceCheck {
 		return errors.Err(fmt.Sprintf("more than 90%% of the space has been used. use --skip-space-check to ignore. Used: %.1f%%", usedPctile*100))

manager/setup.go

@@ -4,15 +4,12 @@ import (
 	"fmt"
 	"math"
 	"net/http"
-	"os"
 	"strconv"
 	"time"
 
 	"github.com/lbryio/lbry.go/extras/errors"
 	"github.com/lbryio/lbry.go/extras/jsonrpc"
 	"github.com/lbryio/lbry.go/extras/util"
-	"github.com/lbryio/lbry.go/lbrycrd"
-
 	"github.com/lbryio/ytsync/tagsManager"
 	"github.com/lbryio/ytsync/thumbs"
 	logUtils "github.com/lbryio/ytsync/util"
@@ -29,7 +26,7 @@ func (s *Sync) enableAddressReuse() error {
 		return errors.Err(err)
 	}
 	accounts := accountsResponse.LBCMainnet
-	if os.Getenv("REGTEST") == "true" {
+	if logUtils.IsRegTest() {
 		accounts = accountsResponse.LBCRegtest
 	}
 	for _, a := range accounts {
@@ -143,7 +140,7 @@ func (s *Sync) ensureEnoughUTXOs() error {
 		return errors.Err(err)
 	}
 	accountsNet := (*accounts).LBCMainnet
-	if os.Getenv("REGTEST") == "true" {
+	if logUtils.IsRegTest() {
 		accountsNet = (*accounts).LBCRegtest
 	}
 	defaultAccount := ""
@@ -224,6 +221,16 @@ func (s *Sync) ensureEnoughUTXOs() error {
 }
 
 func (s *Sync) waitForNewBlock() error {
+	if logUtils.IsRegTest() && logUtils.IsUsingDocker() {
+		lbrycrd, err := logUtils.GetLbrycrdClient(s.LbrycrdString)
+		if err != nil {
+			return errors.Prefix("error getting lbrycrd client: ", err)
+		}
+		txs, err := lbrycrd.Generate(1)
+		for _, tx := range txs {
+			log.Info("Generated tx: ", tx.String())
+		}
+	}
 	status, err := s.daemon.Status()
 	if err != nil {
 		return err
@@ -417,19 +424,10 @@ func allUTXOsConfirmed(utxolist *jsonrpc.UTXOListResponse) bool {
 
 func (s *Sync) addCredits(amountToAdd float64) error {
 	log.Printf("Adding %f credits", amountToAdd)
-	var lbrycrdd *lbrycrd.Client
+	lbrycrdd, err := logUtils.GetLbrycrdClient(s.LbrycrdString)
-	var err error
-	if s.LbrycrdString == "" {
-		lbrycrdd, err = lbrycrd.NewWithDefaultURL()
 	if err != nil {
 		return err
 	}
-	} else {
-		lbrycrdd, err = lbrycrd.New(s.LbrycrdString)
-		if err != nil {
-			return err
-		}
-	}
 
 	addressResp, err := s.daemon.AddressUnused(nil)
 	if err != nil {

manager/ytsync.go

@@ -5,7 +5,6 @@ import (
 	"io/ioutil"
 	"net/http"
 	"os"
-	"os/exec"
 	"os/signal"
 	"runtime/debug"
 	"sort"
@@ -116,28 +115,20 @@ func (s *Sync) IsInterrupted() bool {
 }
 
 func (s *Sync) downloadWallet() error {
-	defaultWalletDir := os.Getenv("HOME") + "/.lbryum/wallets/default_wallet"
+	defaultWalletDir, defaultTempWalletDir, key, err := s.getWalletPaths()
-	defaultTempWalletDir := os.Getenv("HOME") + "/.lbryum/wallets/tmp_wallet"
+	if err != nil {
-	key := aws.String("/wallets/" + s.YoutubeChannelID)
+		return errors.Err(err)
-	if os.Getenv("REGTEST") == "true" {
-		defaultWalletDir = os.Getenv("HOME") + "/.lbryum_regtest/wallets/default_wallet"
-		defaultTempWalletDir = os.Getenv("HOME") + "/.lbryum_regtest/wallets/tmp_wallet"
-		key = aws.String("/regtest/" + s.YoutubeChannelID)
-	}
-
-	if _, err := os.Stat(defaultWalletDir); !os.IsNotExist(err) {
-		return errors.Err("default_wallet already exists")
 	}
 
 	creds := credentials.NewStaticCredentials(s.AwsS3ID, s.AwsS3Secret, "")
 	s3Session, err := session.NewSession(&aws.Config{Region: aws.String(s.AwsS3Region), Credentials: creds})
 	if err != nil {
-		return err
+		return errors.Prefix("error starting session: ", err)
 	}
 	downloader := s3manager.NewDownloader(s3Session)
 	out, err := os.Create(defaultTempWalletDir)
 	if err != nil {
-		return err
+		return errors.Prefix("error creating temp wallet: ", err)
 	}
 	defer out.Close()
 
@@ -165,14 +156,41 @@ func (s *Sync) downloadWallet() error {
 		return errors.Err("zero bytes written")
 	}
 
-	return os.Rename(defaultTempWalletDir, defaultWalletDir)
+	err = os.Rename(defaultTempWalletDir, defaultWalletDir)
+	if err != nil {
+		return errors.Prefix("error replacing temp wallet for default wallet: ", err)
+	}
+
+	return nil
+}
+
+func (s *Sync) getWalletPaths() (defaultWallet, tempWallet string, key *string, err error) {
+
+	defaultWallet = os.Getenv("HOME") + "/.lbryum/wallets/default_wallet"
+	tempWallet = os.Getenv("HOME") + "/.lbryum/wallets/tmp_wallet"
+	key = aws.String("/wallets/" + s.YoutubeChannelID)
+	if logUtils.IsRegTest() {
+		defaultWallet = os.Getenv("HOME") + "/.lbryum_regtest/wallets/default_wallet"
+		tempWallet = os.Getenv("HOME") + "/.lbryum_regtest/wallets/tmp_wallet"
+		key = aws.String("/regtest/" + s.YoutubeChannelID)
+	}
+
+	walletPath := os.Getenv("LBRYNET_WALLETS_DIR")
+	if walletPath != "" {
+		defaultWallet = walletPath + "/wallets/default_wallet"
+		tempWallet = walletPath + "/wallets/tmp_wallet"
+	}
+
+	if _, err := os.Stat(defaultWallet); !os.IsNotExist(err) {
+		return "", "", nil, errors.Err("default_wallet already exists")
+	}
+	return
 }
 
 func (s *Sync) uploadWallet() error {
-	defaultWalletDir := os.Getenv("HOME") + "/.lbryum/wallets/default_wallet"
+	defaultWalletDir := logUtils.GetDefaultWalletPath()
 	key := aws.String("/wallets/" + s.YoutubeChannelID)
-	if os.Getenv("REGTEST") == "true" {
+	if logUtils.IsRegTest() {
-		defaultWalletDir = os.Getenv("HOME") + "/.lbryum_regtest/wallets/default_wallet"
 		key = aws.String("/regtest/" + s.YoutubeChannelID)
 	}
 
@@ -265,20 +283,24 @@ func (s *Sync) FullCycle() (e error) {
 
 	defer s.stopAndUploadWallet(&e)
 
-	s.videoDirectory, err = ioutil.TempDir("", "ytsync")
+	s.videoDirectory, err = ioutil.TempDir(os.Getenv("TMP_DIR"), "ytsync")
 	if err != nil {
 		return errors.Wrap(err, 0)
 	}
+	err = os.Chmod(s.videoDirectory, 0766)
+	if err != nil {
+		return errors.Err(err)
+	}
 
 	defer deleteSyncFolder(s.videoDirectory)
 	log.Printf("Starting daemon")
-	err = startDaemonViaSystemd()
+	err = logUtils.StartDaemon()
 	if err != nil {
 		return err
 	}
 
 	log.Infoln("Waiting for daemon to finish starting...")
-	s.daemon = jsonrpc.NewClient("")
+	s.daemon = jsonrpc.NewClient(os.Getenv("LBRYNET_ADDRESS"))
 	s.daemon.SetRPCTimeout(40 * time.Minute)
 
 	err = s.waitForDaemonStart()
@@ -345,7 +367,7 @@ func (s *Sync) waitForDaemonStart() error {
 
 func (s *Sync) stopAndUploadWallet(e *error) {
 	log.Printf("Stopping daemon")
-	shutdownErr := stopDaemonViaSystemd()
+	shutdownErr := logUtils.StopDaemon()
 	if shutdownErr != nil {
 		logShutdownError(shutdownErr)
 	} else {
@@ -369,7 +391,7 @@ func (s *Sync) stopAndUploadWallet(e *error) {
 	}
 }
 func logShutdownError(shutdownErr error) {
-	logUtils.SendErrorToSlack("error shutting down daemon: %v", shutdownErr)
+	logUtils.SendErrorToSlack("error shutting down daemon: %s", errors.FullTrace(shutdownErr))
 	logUtils.SendErrorToSlack("WALLET HAS NOT BEEN MOVED TO THE WALLET BACKUP DIR")
 }
 
@@ -548,6 +570,18 @@ func (s *Sync) doSync() error {
 	if err != nil {
 		return errors.Prefix("error updating remote database", err)
 	}
+
+	cert, err := s.daemon.ChannelExport(s.lbryChannelID, nil, nil)
+	if err != nil {
+		return errors.Prefix("error getting channel cert", err)
+	}
+	if cert != nil {
+		err = s.APIConfig.SetChannelCert(string(*cert), s.lbryChannelID)
+		if err != nil {
+			return errors.Prefix("error setting channel cert", err)
+		}
+	}
+
 	if nFixed > 0 || nRemoved > 0 {
 		err := s.setStatusSyncing()
 		if err != nil {
@@ -673,7 +707,7 @@ func (s *Sync) startWorker(workerNum int) {
 							err := s.waitForNewBlock()
 							if err != nil {
 								s.grp.Stop()
-								logUtils.SendErrorToSlack("something went wrong while waiting for a block: %v", err)
+								logUtils.SendErrorToSlack("something went wrong while waiting for a block: %s", errors.FullTrace(err))
 								break
 							}
 						} else if util.SubstringInSlice(err.Error(), []string{
@@ -685,7 +719,7 @@ func (s *Sync) startWorker(workerNum int) {
 							err := s.walletSetup()
 							if err != nil {
 								s.grp.Stop()
-								logUtils.SendErrorToSlack("failed to setup the wallet for a refill: %v", err)
+								logUtils.SendErrorToSlack("failed to setup the wallet for a refill: %s", errors.FullTrace(err))
 								break
 							}
 						} else if strings.Contains(err.Error(), "Error in daemon: 'str' object has no attribute 'get'") {
@@ -720,7 +754,7 @@ func (s *Sync) startWorker(workerNum int) {
 				}
 				err = s.Manager.apiConfig.MarkVideoStatus(s.YoutubeChannelID, v.ID(), videoStatus, existingClaimID, existingClaimName, err.Error(), &existingClaimSize, 0)
 				if err != nil {
-					logUtils.SendErrorToSlack("Failed to mark video on the database: %s", err.Error())
+					logUtils.SendErrorToSlack("Failed to mark video on the database: %s", errors.FullTrace(err))
 				}
 			}
 			break
@@ -911,28 +945,12 @@ func (s *Sync) processVideo(v video) (err error) {
 	s.AppendSyncedVideo(v.ID(), true, "", summary.ClaimName, summary.ClaimID, newMetadataVersion, *v.Size())
 	err = s.Manager.apiConfig.MarkVideoStatus(s.YoutubeChannelID, v.ID(), VideoStatusPublished, summary.ClaimID, summary.ClaimName, "", v.Size(), 2)
 	if err != nil {
-		logUtils.SendErrorToSlack("Failed to mark video on the database: %s", err.Error())
+		logUtils.SendErrorToSlack("Failed to mark video on the database: %s", errors.FullTrace(err))
 	}
 
 	return nil
 }
 
-func startDaemonViaSystemd() error {
-	err := exec.Command("/usr/bin/sudo", "/bin/systemctl", "start", "lbrynet.service").Run()
-	if err != nil {
-		return errors.Err(err)
-	}
-	return nil
-}
-
-func stopDaemonViaSystemd() error {
-	err := exec.Command("/usr/bin/sudo", "/bin/systemctl", "stop", "lbrynet.service").Run()
-	if err != nil {
-		return errors.Err(err)
-	}
-	return nil
-}
-
 // waitForDaemonProcess observes the running processes and returns when the process is no longer running or when the timeout is up
 func waitForDaemonProcess(timeout time.Duration) error {
 	processes, err := ps.Processes()

sdk/api.go

@@ -71,7 +71,7 @@ func (a *APIConfig) FetchChannels(status string, cp *SyncProperties) ([]YoutubeC
 	var response apiJobsResponse
 	err = json.Unmarshal(body, &response)
 	if err != nil {
-		return nil, err
+		return nil, errors.Err(err)
 	}
 	if response.Data == nil {
 		return nil, errors.Err(response.Error)
@@ -98,6 +98,39 @@ func sanitizeFailureReason(s *string) {
 		*s = (*s)[:MaxReasonLength]
 	}
 }
+
+func (a *APIConfig) SetChannelCert(certHex string, channelID string) error {
+
+	type apiSetChannelCertResponse struct {
+		Success bool        `json:"success"`
+		Error   null.String `json:"error"`
+		Data    string      `json:"data"`
+	}
+
+	endpoint := a.ApiURL + "/yt/channel_cert"
+
+	res, _ := http.PostForm(endpoint, url.Values{
+		"channel_claim_id": {channelID},
+		"channel_cert":     {certHex},
+		"auth_token":       {a.ApiToken},
+	})
+
+	defer res.Body.Close()
+
+	body, _ := ioutil.ReadAll(res.Body)
+	var response apiSetChannelCertResponse
+	err := json.Unmarshal(body, &response)
+	if err != nil {
+		return errors.Err(err)
+	}
+	if !response.Error.IsNull() {
+		return errors.Err(response.Error.String)
+	}
+
+	return nil
+
+}
+
 func (a *APIConfig) SetChannelStatus(channelID string, status string, failureReason string) (map[string]SyncedVideo, map[string]bool, error) {
 	type apiChannelStatusResponse struct {
 		Success bool          `json:"success"`
@@ -119,7 +152,7 @@ func (a *APIConfig) SetChannelStatus(channelID string, status string, failureRea
 	var response apiChannelStatusResponse
 	err := json.Unmarshal(body, &response)
 	if err != nil {
-		return nil, nil, err
+		return nil, nil, errors.Err(err)
 	}
 	if !response.Error.IsNull() {
 		return nil, nil, errors.Err(response.Error.String)

sources/youtubeVideo.go

@@ -24,7 +24,7 @@ import (
 	"github.com/lbryio/ytsync/tagsManager"
 	"github.com/lbryio/ytsync/thumbs"
 
-	"github.com/ChannelMeter/iso8601duration"
+	duration "github.com/ChannelMeter/iso8601duration"
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/shopspring/decimal"
 	log "github.com/sirupsen/logrus"
@@ -178,7 +178,7 @@ func (v *YoutubeVideo) getAbbrevDescription() string {
 func (v *YoutubeVideo) download(useIPv6 bool) error {
 	videoPath := v.getFullPath()
 
-	err := os.Mkdir(v.videoDir(), 0750)
+	err := os.Mkdir(v.videoDir(), 0777)
 	if err != nil && !strings.Contains(err.Error(), "file exists") {
 		return errors.Wrap(err, 0)
 	}
@@ -294,22 +294,26 @@ runcmd:
 	log.Debugln(string(outLog))
 
 	if strings.Contains(string(outLog), "does not pass filter duration") {
-		_ = v.delete()
+		_ = v.delete("does not pass filter duration")
 		return errors.Err("video is too long to process")
 	}
 	if strings.Contains(string(outLog), "File is larger than max-filesize") {
-		_ = v.delete()
+		_ = v.delete("File is larger than max-filesize")
 		return errors.Err("the video is too big to sync, skipping for now")
 	}
 	if string(errorLog) != "" {
 		log.Printf("Command finished with error: %v", errors.Err(string(errorLog)))
-		_ = v.delete()
+		_ = v.delete("due to error")
 		return errors.Err(string(errorLog))
 	}
 	fi, err := os.Stat(v.getFullPath())
 	if err != nil {
 		return errors.Err(err)
 	}
+	err = os.Chmod(v.getFullPath(), 0777)
+	if err != nil {
+		return errors.Err(err)
+	}
 	videoSize := fi.Size()
 	v.size = &videoSize
 	return nil
@@ -338,14 +342,14 @@ func (v *YoutubeVideo) getDownloadedPath() (string, error) {
 	return "", errors.Err("could not find any downloaded videos")
 
 }
-func (v *YoutubeVideo) delete() error {
+func (v *YoutubeVideo) delete(reason string) error {
 	videoPath, err := v.getDownloadedPath()
 	if err != nil {
 		log.Errorln(err)
 		return err
 	}
 	err = os.Remove(videoPath)
-	log.Debugf("%s deleted from disk (%s)", v.id, videoPath)
+	log.Debugf("%s deleted from disk for '%s' (%s)", v.id, reason, videoPath)
 
 	if err != nil {
 		err = errors.Prefix("delete error", err)
@@ -451,7 +455,7 @@ func (v *YoutubeVideo) downloadAndPublish(daemon *jsonrpc.Client, params SyncPar
 
 	summary, err := v.publish(daemon, params)
 	//delete the video in all cases (and ignore the error)
-	_ = v.delete()
+	_ = v.delete("finished download and publish")
 
 	return summary, errors.Prefix("publish error", err)
 }

thumbs/uploader.go

@@ -1,7 +1,6 @@
 package thumbs
 
 import (
-	"google.golang.org/api/youtube/v3"
 	"io"
 	"net/http"
 	"os"
@@ -11,7 +10,8 @@ import (
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/aws/session"
 	"github.com/aws/aws-sdk-go/service/s3/s3manager"
-	"github.com/prometheus/common/log"
+	log "github.com/sirupsen/logrus"
+	"google.golang.org/api/youtube/v3"
 )
 
 type thumbnailUploader struct {
@@ -25,7 +25,7 @@ const thumbnailPath = "/tmp/ytsync_thumbnails/"
 const ThumbnailEndpoint = "https://thumbnails.lbry.com/"
 
 func (u *thumbnailUploader) downloadThumbnail() error {
-	_ = os.Mkdir(thumbnailPath, 0750)
+	_ = os.Mkdir(thumbnailPath, 0777)
 	img, err := os.Create("/tmp/ytsync_thumbnails/" + u.name)
 	if err != nil {
 		return errors.Err(err)

util/util.go

@@ -0,0 +1,305 @@
+package util
+
+import (
+	"context"
+	"os"
+	"os/exec"
+	"os/user"
+	"path/filepath"
+	"strconv"
+
+	"github.com/lbryio/lbry.go/extras/errors"
+	"github.com/lbryio/lbry.go/lbrycrd"
+
+	"github.com/docker/docker/api/types"
+	"github.com/docker/docker/api/types/filters"
+	"github.com/docker/docker/client"
+	"github.com/mitchellh/go-ps"
+	log "github.com/sirupsen/logrus"
+)
+
+func GetBlobsDir() string {
+	blobsDir := os.Getenv("BLOBS_DIRECTORY")
+	if blobsDir == "" {
+		usr, err := user.Current()
+		if err != nil {
+			log.Error(err.Error())
+			return ""
+		}
+		blobsDir = usr.HomeDir + "/.lbrynet/blobfiles/"
+	}
+
+	return blobsDir
+}
+
+func IsBlobReflectionOff() bool {
+	return os.Getenv("REFLECT_BLOBS") == "false"
+}
+
+func GetLBRYNetDir() string {
+	lbrynetDir := os.Getenv("LBRYNET_DIR")
+	if lbrynetDir == "" {
+		usr, err := user.Current()
+		if err != nil {
+			log.Errorln(err.Error())
+			return ""
+		}
+		return usr.HomeDir + "/.lbrynet/"
+	}
+	return lbrynetDir
+}
+
+const ALL = true
+const ONLINE = false
+
+func GetLBRYNetContainer(all bool) (*types.Container, error) {
+	return getDockerContainer("lbrynet", all)
+}
+
+func getDockerContainer(name string, all bool) (*types.Container, error) {
+	cli, err := client.NewEnvClient()
+	if err != nil {
+		panic(err)
+	}
+	filters := filters.NewArgs()
+	filters.Add("name", name)
+	containers, err := cli.ContainerList(context.Background(), types.ContainerListOptions{All: all, Filters: filters})
+	if err != nil {
+		panic(err)
+	}
+	if len(containers) == 0 {
+		return nil, nil
+	}
+	if len(containers) > 1 {
+		return nil, errors.Err("more than one %s container found", name)
+	}
+
+	return &containers[0], nil
+}
+
+func IsUsingDocker() bool {
+	useDocker, err := strconv.ParseBool(os.Getenv("LBRYNET_USE_DOCKER"))
+	if err != nil {
+		return false
+	}
+	return useDocker
+}
+
+func IsRegTest() bool {
+	usesRegtest, err := strconv.ParseBool(os.Getenv("REGTEST"))
+	if err != nil {
+		return false
+	}
+	return usesRegtest
+}
+
+func GetLbrycrdClient(lbrycrdString string) (*lbrycrd.Client, error) {
+	var lbrycrdd *lbrycrd.Client
+	var err error
+	if lbrycrdString == "" {
+		lbrycrdd, err = lbrycrd.NewWithDefaultURL()
+		if err != nil {
+			return nil, err
+		}
+	} else {
+		lbrycrdd, err = lbrycrd.New(lbrycrdString)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	return lbrycrdd, nil
+}
+
+func ShouldCleanOnStartup() bool {
+	shouldClean, err := strconv.ParseBool(os.Getenv("CLEAN_ON_STARTUP"))
+	if err != nil {
+		return false
+	}
+	return shouldClean
+}
+
+func IsLbrynetRunning() (bool, error) {
+	if IsUsingDocker() {
+		container, err := GetLBRYNetContainer(ONLINE)
+		if err != nil {
+			return false, err
+		}
+		return container != nil, nil
+	}
+
+	processes, err := ps.Processes()
+	if err != nil {
+		return true, errors.Err(err)
+	}
+	var daemonProcessId = -1
+	for _, p := range processes {
+		if p.Executable() == "lbrynet" {
+			daemonProcessId = p.Pid()
+			break
+		}
+	}
+
+	running := daemonProcessId != -1
+	return running, nil
+}
+
+func CleanForStartup() error {
+	if !IsRegTest() {
+		return errors.Err("never cleanup wallet outside of regtest and with caution. this should only be done in local testing and requires regtest to be on")
+	}
+
+	running, err := IsLbrynetRunning()
+	if err != nil {
+		return err
+	}
+	if running {
+		err := StopDaemon()
+		if err != nil {
+			return err
+		}
+	}
+
+	err = CleanupLbrynet()
+	if err != nil {
+		return errors.Err(err)
+	}
+
+	lbrycrd, err := GetLbrycrdClient(os.Getenv("LBRYCRD_STRING"))
+	if err != nil {
+		return errors.Prefix("error getting lbrycrd client: ", err)
+	}
+	height, err := lbrycrd.GetBlockCount()
+	if err != nil {
+		return errors.Err(err)
+	}
+	const minBlocksForUTXO = 110
+	if height < minBlocksForUTXO {
+		//Start reg test will some credits
+		txs, err := lbrycrd.Generate(uint32(110) - uint32(height))
+		if err != nil {
+			return errors.Err(err)
+		}
+		log.Debugf("REGTEST: Generated %d transactions to get some LBC!", len(txs))
+	}
+
+	defaultWalletDir := GetDefaultWalletPath()
+	_, err = os.Stat(defaultWalletDir)
+	if os.IsNotExist(err) {
+		return nil
+	}
+	return errors.Err(os.Remove(defaultWalletDir))
+}
+
+func CleanupLbrynet() error {
+	//make sure lbrynet is off
+	running, err := IsLbrynetRunning()
+	if err != nil {
+		return err
+	}
+	if running {
+		return errors.Prefix("cannot cleanup lbrynet as the daemon is running", err)
+	}
+	lbrynetDir := GetLBRYNetDir()
+	files, err := filepath.Glob(lbrynetDir + "lbrynet.sqlite*")
+	if err != nil {
+		return errors.Err(err)
+	}
+	for _, f := range files {
+		err = os.Remove(f)
+		if err != nil {
+			return errors.Err(err)
+		}
+	}
+	blobsDir := GetBlobsDir()
+	err = os.RemoveAll(blobsDir)
+	if err != nil {
+		return errors.Err(err)
+	}
+	err = os.Mkdir(blobsDir, 0777)
+	if err != nil {
+		return errors.Err(err)
+	}
+	return nil
+}
+
+func StartDaemon() error {
+	if IsUsingDocker() {
+		return startDaemonViaDocker()
+	}
+	return startDaemonViaSystemd()
+}
+
+func StopDaemon() error {
+	if IsUsingDocker() {
+		return stopDaemonViaDocker()
+	}
+	return stopDaemonViaSystemd()
+}
+
+func startDaemonViaDocker() error {
+	container, err := GetLBRYNetContainer(true)
+	if err != nil {
+		return err
+	}
+
+	cli, err := client.NewEnvClient()
+	if err != nil {
+		panic(err)
+	}
+
+	err = cli.ContainerStart(context.Background(), container.ID, types.ContainerStartOptions{})
+	if err != nil {
+		return errors.Err(err)
+	}
+
+	return nil
+}
+
+func stopDaemonViaDocker() error {
+	container, err := GetLBRYNetContainer(ONLINE)
+	if err != nil {
+		return err
+	}
+
+	cli, err := client.NewEnvClient()
+	if err != nil {
+		panic(err)
+	}
+
+	err = cli.ContainerStop(context.Background(), container.ID, nil)
+	if err != nil {
+		return errors.Err(err)
+	}
+
+	return nil
+}
+
+func startDaemonViaSystemd() error {
+	err := exec.Command("/usr/bin/sudo", "/bin/systemctl", "start", "lbrynet.service").Run()
+	if err != nil {
+		return errors.Err(err)
+	}
+	return nil
+}
+
+func stopDaemonViaSystemd() error {
+	err := exec.Command("/usr/bin/sudo", "/bin/systemctl", "stop", "lbrynet.service").Run()
+	if err != nil {
+		return errors.Err(err)
+	}
+	return nil
+}
+
+func GetDefaultWalletPath() string {
+	defaultWalletDir := os.Getenv("HOME") + "/.lbryum/wallets/default_wallet"
+	if IsRegTest() {
+		defaultWalletDir = os.Getenv("HOME") + "/.lbryum_regtest/wallets/default_wallet"
+	}
+
+	walletPath := os.Getenv("LBRYNET_WALLETS_DIR")
+	if walletPath != "" {
+		defaultWalletDir = walletPath + "/wallets/default_wallet"
+	}
+	return defaultWalletDir
+}