From 5e56ac0516ff59c7504cb8c57ba89ce85a858b63 Mon Sep 17 00:00:00 2001 From: Mark Beamer Jr Date: Thu, 25 Jul 2019 23:46:35 -0400 Subject: [PATCH 01/11] Add set channel cert on new channels. --- go.mod | 4 ++-- go.sum | 3 +++ manager/ytsync.go | 10 ++++++++++ sdk/api.go | 29 +++++++++++++++++++++++++++++ 4 files changed, 44 insertions(+), 2 deletions(-) diff --git a/go.mod b/go.mod index 5498de8..1263e35 100644 --- a/go.mod +++ b/go.mod @@ -17,8 +17,8 @@ require ( github.com/konsorten/go-windows-terminal-sequences v1.0.2 // indirect github.com/kr/pretty v0.1.0 // indirect github.com/lbryio/errors.go v0.0.0-20180223142025-ad03d3cc6a5c - github.com/lbryio/lbry.go v1.0.15 - github.com/lbryio/reflector.go v1.0.6-0.20190806185326-2e4f235489f4 + github.com/lbryio/lbry.go v1.0.16-0.20190726033352-f69bba9977f8 + github.com/lbryio/reflector.go v1.0.6-0.20190710201919-7c7ed9da72ce github.com/lusis/slack-test v0.0.0-20190408224659-6cf59653add2 // indirect github.com/mitchellh/go-ps v0.0.0-20170309133038-4fdf99ab2936 github.com/onsi/ginkgo v1.8.0 // indirect diff --git a/go.sum b/go.sum index 5d33bb0..6e37f14 100644 --- a/go.sum +++ b/go.sum @@ -191,8 +191,11 @@ github.com/lbryio/errors.go v0.0.0-20180223142025-ad03d3cc6a5c/go.mod h1:muH7wpU github.com/lbryio/lbry.go v0.0.0-20190109223729-30c312501602/go.mod h1:YEuFJD/oHNra6BFy+NfuvS84Wg6RMWJFGtiCCCc6MmQ= github.com/lbryio/lbry.go v1.0.15 h1:g4g9cDDUsobmWgTA+1jEIb5k2fRCP0/NvPOMXduP8xY= github.com/lbryio/lbry.go v1.0.15/go.mod h1:JtyI30bU51rm0LZ/po3mQuzf++14OWb6kR/6mMRAmKU= +github.com/lbryio/lbry.go v1.0.16-0.20190726033352-f69bba9977f8 h1:oNQh5rAVygkVLLRYxLd098/smLHcMoKrn7ozfwFhx2Y= +github.com/lbryio/lbry.go v1.0.16-0.20190726033352-f69bba9977f8/go.mod h1:JtyI30bU51rm0LZ/po3mQuzf++14OWb6kR/6mMRAmKU= github.com/lbryio/lbryschema.go v0.0.0-20190428231007-c54836bca002 h1:urfYK5ElpUrAv90auPLldoVC60LwiGAcY0OE6HJB9KI= github.com/lbryio/lbryschema.go v0.0.0-20190428231007-c54836bca002/go.mod h1:dAzPCBj3CKKWBGYBZxK6tKBP5SCgY2tqd9SnQd/OyKo= +github.com/lbryio/ozzo-validation v0.0.0-20170323141101-d1008ad1fd04 h1:Nze+C2HbeKvhjI/kVn+9Poj/UuEW5sOQxcsxqO7L3GI= github.com/lbryio/ozzo-validation v0.0.0-20170323141101-d1008ad1fd04/go.mod h1:fbG/dzobG8r95KzMwckXiLMHfFjZaBRQqC9hPs2XAQ4= github.com/lbryio/reflector.go v1.0.6-0.20190806185326-2e4f235489f4 h1:SpUbq2YBg3ncetkw8APUgn8nFF8dscKzzhyiWMM7XCc= github.com/lbryio/reflector.go v1.0.6-0.20190806185326-2e4f235489f4/go.mod h1:7Y3YYeAKS6egH2WzwfU8f6+uNGjVHfLzKvwn+Nv3VMY= diff --git a/manager/ytsync.go b/manager/ytsync.go index 54e482c..e7c46df 100644 --- a/manager/ytsync.go +++ b/manager/ytsync.go @@ -548,6 +548,16 @@ func (s *Sync) doSync() error { if err != nil { return errors.Prefix("error updating remote database", err) } + if pubsOnWallet == 0 { + cert, err := s.daemon.ChannelExport(s.lbryChannelID, nil, nil) + if err != nil { + return errors.Prefix("error getting channel cert", err) + } + err = s.APIConfig.SetChannelCert(string(*cert), s.lbryChannelID) + if err != nil { + return errors.Prefix("error setting channel cert", err) + } + } if nFixed > 0 || nRemoved > 0 { err := s.setStatusSyncing() if err != nil { diff --git a/sdk/api.go b/sdk/api.go index 821f8a8..6f3de9b 100644 --- a/sdk/api.go +++ b/sdk/api.go @@ -11,6 +11,8 @@ import ( "strings" "time" + "github.com/lbryio/lbry.go/extras/api" + "github.com/lbryio/lbry.go/extras/errors" "github.com/lbryio/lbry.go/extras/null" @@ -98,6 +100,33 @@ func sanitizeFailureReason(s *string) { *s = (*s)[:MaxReasonLength] } } + +func (a *APIConfig) SetChannelCert(certHex string, channelID string) error { + + endpoint := a.ApiURL + "/yt/channel_cert" + + res, _ := http.PostForm(endpoint, url.Values{ + "channel_id": {channelID}, + "channel_cert": {certHex}, + "auth_token": {a.ApiToken}, + }) + + defer res.Body.Close() + + body, _ := ioutil.ReadAll(res.Body) + var response api.Response + err := json.Unmarshal(body, &response) + if err != nil { + return errors.Err(err) + } + if response.Error != nil { + return errors.Err(response.Error) + } + + return nil + +} + func (a *APIConfig) SetChannelStatus(channelID string, status string, failureReason string) (map[string]SyncedVideo, map[string]bool, error) { type apiChannelStatusResponse struct { Success bool `json:"success"` From f26712b0220b935247c7ed2dc8599a4ea624dc97 Mon Sep 17 00:00:00 2001 From: Mark Beamer Jr Date: Sun, 28 Jul 2019 23:58:05 -0400 Subject: [PATCH 02/11] Added e2e test docker-compose and supporting work so that ytsync can run independently of the mainnet for testing/CI --- .gitignore | 3 +- e2e/daemon_settings.yml | 12 +++ e2e/docker-compose.yml | 53 +++++++++++ e2e/lbrycrd/docker-compose.yml | 26 ++++++ e2e/lbrycrd/docker/Dockerfile | 37 ++++++++ e2e/lbrycrd/docker/advance_blocks.sh | 5 + e2e/lbrycrd/docker/fix-permissions.c | 9 ++ e2e/lbrycrd/docker/healthcheck.sh | 4 + e2e/lbrycrd/docker/start.sh | 114 +++++++++++++++++++++++ e2e/lbrynet/docker-compose.yml | 23 +++++ e2e/lbrynet/docker/Dockerfile | 27 ++++++ e2e/lbrynet/docker/checkmount.sh | 12 +++ e2e/lbrynet/docker/start.sh | 4 + e2e/lbrynet/settings/daemon_settings.yml | 12 +++ e2e/walletserver/docker-compose.yml | 35 +++++++ 15 files changed, 375 insertions(+), 1 deletion(-) create mode 100644 e2e/daemon_settings.yml create mode 100644 e2e/docker-compose.yml create mode 100644 e2e/lbrycrd/docker-compose.yml create mode 100644 e2e/lbrycrd/docker/Dockerfile create mode 100644 e2e/lbrycrd/docker/advance_blocks.sh create mode 100644 e2e/lbrycrd/docker/fix-permissions.c create mode 100644 e2e/lbrycrd/docker/healthcheck.sh create mode 100644 e2e/lbrycrd/docker/start.sh create mode 100644 e2e/lbrynet/docker-compose.yml create mode 100644 e2e/lbrynet/docker/Dockerfile create mode 100644 e2e/lbrynet/docker/checkmount.sh create mode 100644 e2e/lbrynet/docker/start.sh create mode 100644 e2e/lbrynet/settings/daemon_settings.yml create mode 100644 e2e/walletserver/docker-compose.yml diff --git a/.gitignore b/.gitignore index 6dd29b7..6ba26e6 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,2 @@ -bin/ \ No newline at end of file +bin/ +e2e/persist \ No newline at end of file diff --git a/e2e/daemon_settings.yml b/e2e/daemon_settings.yml new file mode 100644 index 0000000..8a6bcb4 --- /dev/null +++ b/e2e/daemon_settings.yml @@ -0,0 +1,12 @@ +#blockchain_name: lbrycrd_main +#blockchain_name: lbrycrd_testnet +blockchain_name: lbrycrd_regtest +lbryum_servers: +# - spv1.lbry.com:50001 #Production Wallet Server + - walletserver:50001 +save_blobs: true +save_files: false +share_usage_data: false +tcp_port: 3333 +udp_port: 4444 +use_upnp: true \ No newline at end of file diff --git a/e2e/docker-compose.yml b/e2e/docker-compose.yml new file mode 100644 index 0000000..5c36dc9 --- /dev/null +++ b/e2e/docker-compose.yml @@ -0,0 +1,53 @@ +version: "3" + +services: + ############# + ## Lbrycrd ## + ############# + lbrycrd: + image: lbry/lbrycrd:v0.12.4.1 + restart: always + ports: + - "15201:29246" + - "15200:29245" + ## host volumes for persistent data such as wallet private keys. + volumes: + - "./persist:/data" + environment: + - RUN_MODE=regtest + ################### + ## Wallet Server ## + ################### + walletserver: + image: lbry/wallet-server:v0.38.5 + restart: always + volumes: + - "./persist/.walletserver/database:/database" + environment: + - DB_DIRECTORY=/database + - MAX_SEND=1000000000000000000000 + - DAEMON_URL=http://lbry:lbry@lbrycrd:29245/ + - MAX_SUBS=1000000000000 + - BANDWIDTH_LIMIT=80000000000 + - SESSION_TIMEOUT=10000000000000000000000000 + - TCP_PORT=50001 + ports: + - "15300:50001" + ulimits: + nofile: 90000 + # command: lbry.wallet.server.coin.LBC + command: lbry.wallet.server.coin.LBCRegTest + ############# + ## Lbrynet ## + ############# + lbrynet: + image: lbry/lbrynet:v0.38.5 + restart: "no" + ports: + - "15100:5279" + - "15101:5280" + environment: + - LBRY_STREAMING_SERVER=0.0.0.0:5280 + volumes: + - "./persist/.lbrynet:/home/lbrynet" + - ".:/etc/lbry" #Put your daemon_settings.yml here diff --git a/e2e/lbrycrd/docker-compose.yml b/e2e/lbrycrd/docker-compose.yml new file mode 100644 index 0000000..0254de3 --- /dev/null +++ b/e2e/lbrycrd/docker-compose.yml @@ -0,0 +1,26 @@ +version: "3" +networks: + lbry-network: + external: true + +services: + ############# + ## Lbrycrd ## + ############# + lbrycrd: + image: lbry/lbrycrd:v0.12.4.1 + restart: always + networks: + lbry-network: + ipv4_address: 10.6.1.1 + ports: + - "15201:29246" + - "15200:29245" + expose: + - "29246" + - "29245" + ## host volumes for persistent data such as wallet private keys. + volumes: + - "../persist/data:/data" + environment: + - RUN_MODE=regtest diff --git a/e2e/lbrycrd/docker/Dockerfile b/e2e/lbrycrd/docker/Dockerfile new file mode 100644 index 0000000..0d19eca --- /dev/null +++ b/e2e/lbrycrd/docker/Dockerfile @@ -0,0 +1,37 @@ +FROM ubuntu:18.04 as prep +LABEL MAINTAINER="leopere [at] nixc [dot] us" +## TODO: Implement version pinning. `apt-get install curl=` +RUN apt-get update && \ + apt-get -y install unzip curl build-essential && \ + apt-get autoclean -y && \ + rm -rf /var/lib/apt/lists/* +WORKDIR / +SHELL ["/bin/bash", "-o", "pipefail", "-c"] +COPY ./start.sh start +COPY ./healthcheck.sh healthcheck +COPY ./advance_blocks.sh advance +COPY ./fix-permissions.c fix-permissions.c + +## Add lbrycrd - Change the version below to create an image for a different tag/version +ARG VERSION="v0.12.4.1" +RUN URL=$(curl -s https://api.github.com/repos/lbryio/lbrycrd/releases/$(if [ "${VERSION}" = 'latest' ]; then echo "latest"; else echo "tags/${VERSION}"; fi) | grep browser_download_url | grep lbrycrd-linux.zip | cut -d'"' -f4) && echo $URL && curl -L -o /lbrycrd-linux.zip $URL + +RUN unzip ./lbrycrd-linux.zip && \ + gcc fix-permissions.c -o fix-permissions && \ + chmod +x ./lbrycrdd ./lbrycrd-cli ./lbrycrd-tx ./start ./healthcheck ./fix-permissions ./advance + +FROM ubuntu:18.04 as app +COPY --from=prep /lbrycrdd /lbrycrd-cli /lbrycrd-tx /start /healthcheck /fix-permissions /advance /usr/bin/ +RUN addgroup --gid 1000 lbrycrd && \ + adduser lbrycrd --uid 1000 --gid 1000 --gecos GECOS --shell /bin/bash --disabled-password --home /data && \ + mkdir /etc/lbry && \ + chown lbrycrd /etc/lbry && \ + chmod a+s /usr/bin/fix-permissions +VOLUME ["/data"] +WORKDIR /data +## TODO: Implement healthcheck. +# HEALTHCHECK ["healthcheck"] +EXPOSE 9246 9245 11337 29245 + +USER lbrycrd +CMD ["start"] \ No newline at end of file diff --git a/e2e/lbrycrd/docker/advance_blocks.sh b/e2e/lbrycrd/docker/advance_blocks.sh new file mode 100644 index 0000000..922d38a --- /dev/null +++ b/e2e/lbrycrd/docker/advance_blocks.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash +while true; do + lbrycrd-cli -conf=/data/.lbrycrd/lbrycrd.conf generate 1 >> /tmp/output.log + sleep 2 +done \ No newline at end of file diff --git a/e2e/lbrycrd/docker/fix-permissions.c b/e2e/lbrycrd/docker/fix-permissions.c new file mode 100644 index 0000000..2f247e8 --- /dev/null +++ b/e2e/lbrycrd/docker/fix-permissions.c @@ -0,0 +1,9 @@ +#include +int main() { + // This program needs to run with setuid == root + // This needs to be in a compiled language because you cannot setuid bash scripts + setuid(0); + execle("/bin/bash", "bash", "-c", + "/bin/chown -R lbrycrd:lbrycrd /data && /bin/chmod -R 755 /data/", + (char*) NULL, (char*) NULL); +} \ No newline at end of file diff --git a/e2e/lbrycrd/docker/healthcheck.sh b/e2e/lbrycrd/docker/healthcheck.sh new file mode 100644 index 0000000..b9ee099 --- /dev/null +++ b/e2e/lbrycrd/docker/healthcheck.sh @@ -0,0 +1,4 @@ +## TODO: Implement a healthcheck for lbrycrd. +curl --data-binary '{"jsonrpc":"1.0","id":"curltext","method":"getinfo","params":[]}' -H 'content-type:text/plain;' http://$RPC_USER:$RPC_PASSWORD@127.0.0.1:9246 +## OR +lbrycrd-cli getinfo \ No newline at end of file diff --git a/e2e/lbrycrd/docker/start.sh b/e2e/lbrycrd/docker/start.sh new file mode 100644 index 0000000..89bbb76 --- /dev/null +++ b/e2e/lbrycrd/docker/start.sh @@ -0,0 +1,114 @@ +#!/usr/bin/env bash +CONFIG_PATH=/etc/lbry/lbrycrd.conf + +function override_config_option() { + # Remove existing config line from a config file + # and replace with environment fed value. + # Does nothing if the variable does not exist. + # var Name of ENV variable + # option Name of config option + # config Path of config file + local var=$1 option=$2 config=$3 + if [[ -v $var ]]; then + # Remove the existing config option: + sed -i "/^$option\W*=/d" $config + # Add the value from the environment: + echo "$option=${!var}" >> $config + fi +} + +function set_config() { + if [ -d "$CONFIG_PATH" ]; then + echo "$CONFIG_PATH is a directory when it should be a file." + exit 1 + elif [ -f "$CONFIG_PATH" ]; then + echo "Merging the mounted config file with environment variables." + local MERGED_CONFIG=/tmp/lbrycrd_merged.conf + cat $CONFIG_PATH > $MERGED_CONFIG + echo "" >> $MERGED_CONFIG + override_config_option PORT port $MERGED_CONFIG + override_config_option RPC_USER rpcuser $MERGED_CONFIG + override_config_option RPC_PASSWORD rpcpassword $MERGED_CONFIG + override_config_option RPC_ALLOW_IP rpcallowip $MERGED_CONFIG + override_config_option RPC_PORT rpcport $MERGED_CONFIG + override_config_option RPC_BIND rpcbind $MERGED_CONFIG + # Make the new merged config file the new CONFIG_PATH + # This ensures that the original file the user mounted remains unmodified + CONFIG_PATH=$MERGED_CONFIG + else + echo "Creating a fresh config file from environment variables." + ## Set config params + echo "port=${PORT=9246}" > $CONFIG_PATH + echo "rpcuser=${RPC_USER=lbry}" >> $CONFIG_PATH + echo "rpcpassword=${RPC_PASSWORD=lbry}" >> $CONFIG_PATH + echo "rpcallowip=${RPC_ALLOW_IP=127.0.0.1/24}" >> $CONFIG_PATH + echo "rpcport=${RPC_PORT=9245}" >> $CONFIG_PATH + echo "rpcbind=${RPC_BIND=0.0.0.0}" >> $CONFIG_PATH + fi + echo "Config: " + cat $CONFIG_PATH +} + +## Ensure perms are correct prior to running main binary +/usr/bin/fix-permissions + +## You can optionally specify a run mode if you want to use lbry defined presets for compatibility. +case $RUN_MODE in + default ) + set_config + lbrycrdd -server -conf=$CONFIG_PATH -printtoconsole + ;; + ## If it's a first run you need to do a full index including all transactions + ## tx index creates an index of every single transaction in the block history if + ## not specified it will only create an index for transactions that are related to the wallet or have unspent outputs. + ## This is generally specific to chainquery. + reindex ) + ## Apply this RUN_MODE in the case you need to update a dataset. NOTE: you do not need to use `RUN_MODE reindex` for more than one complete run. + set_config + lbrycrdd -server -txindex -reindex -conf=$CONFIG_PATH -printtoconsole + ;; + chainquery ) + ## If your only goal is to run Chainquery against this instance of lbrycrd and you're starting a + ## fresh local dataset use this run mode. + set_config + lbrycrdd -server -txindex -conf=$CONFIG_PATH -printtoconsole + ;; + regtest ) + ## Set config params + ## TODO: Make this more automagic in the future. + mkdir -p `dirname $CONFIG_PATH` + echo "rpcuser=lbry" > $CONFIG_PATH + echo "rpcpassword=lbry" >> $CONFIG_PATH + echo "rpcport=29245" >> $CONFIG_PATH + echo "rpcbind=0.0.0.0" >> $CONFIG_PATH + echo "rpcallowip=0.0.0.0/0" >> $CONFIG_PATH + echo "regtest=1" >> $CONFIG_PATH + echo "txindex=1" >> $CONFIG_PATH + echo "server=1" >> $CONFIG_PATH + echo "printtoconsole=1" >> $CONFIG_PATH + + #nohup advance &>/dev/null & + lbrycrdd -conf=$CONFIG_PATH $1 + ;; + testnet ) + ## Set config params + ## TODO: Make this more automagic in the future. + mkdir -p `dirname $CONFIG_PATH` + echo "rpcuser=lbry" > $CONFIG_PATH + echo "rpcpassword=lbry" >> $CONFIG_PATH + echo "rpcport=29245" >> $CONFIG_PATH + echo "rpcbind=0.0.0.0" >> $CONFIG_PATH + echo "rpcallowip=0.0.0.0/0" >> $CONFIG_PATH + echo "testnet=1" >> $CONFIG_PATH + echo "txindex=1" >> $CONFIG_PATH + echo "server=1" >> $CONFIG_PATH + echo "printtoconsole=1" >> $CONFIG_PATH + + #nohup advance &>/dev/null & + lbrycrdd -conf=$CONFIG_PATH $1 + ;; + * ) + echo "Error, you must define a RUN_MODE environment variable." + echo "Available options are testnet, regtest, chainquery, default, and reindex" + ;; +esac \ No newline at end of file diff --git a/e2e/lbrynet/docker-compose.yml b/e2e/lbrynet/docker-compose.yml new file mode 100644 index 0000000..d49c0f7 --- /dev/null +++ b/e2e/lbrynet/docker-compose.yml @@ -0,0 +1,23 @@ +version: "3" +networks: + lbry-network: + external: true + +services: + ############# + ## Lbrynet ## + ############# + lbrynet: + image: lbry/lbrynet:v0.38.5 + restart: "no" + networks: + lbry-network: + ipv4_address: 10.6.1.3 + ports: + - "15100:5279" + - "15101:5280" + environment: + - LBRY_STREAMING_SERVER=0.0.0.0:5280 + volumes: + - "../persist/data/.lbrynet:/home/lbrynet" + - "./settings:/etc/lbry" #Put your daemon_settings.yml here diff --git a/e2e/lbrynet/docker/Dockerfile b/e2e/lbrynet/docker/Dockerfile new file mode 100644 index 0000000..4184b30 --- /dev/null +++ b/e2e/lbrynet/docker/Dockerfile @@ -0,0 +1,27 @@ +## This base image is for running the latest lbrynet-daemon release. +FROM ubuntu:18.04 as prep +LABEL MAINTAINER="leopere [at] nixc [dot] us" +RUN apt-get update && apt-get -y install unzip curl telnet + +## Add lbrynet +ARG VERSION="v0.38.5" +RUN URL=$(curl -s https://api.github.com/repos/lbryio/lbry-sdk/releases/$(if [ "${VERSION}" = 'latest' ]; then echo "latest"; else echo "tags/${VERSION}"; fi) | grep browser_download_url | grep lbrynet-linux.zip | cut -d'"' -f4) && echo $URL && curl -L -o /lbrynet.linux.zip $URL + +COPY start.sh /usr/bin/start +COPY checkmount.sh /usr/bin/checkmount +RUN unzip /lbrynet.linux.zip -d /lbrynet/ && \ + mv /lbrynet/lbrynet /usr/bin && \ + chmod a+x /usr/bin/checkmount /usr/bin/start /usr/bin/lbrynet + +FROM ubuntu:18.04 as app +COPY --from=prep /usr/bin/start /usr/bin/checkmount /usr/bin/lbrynet /usr/bin/ +RUN adduser lbrynet --gecos GECOS --shell /bin/bash --disabled-password --home /home/lbrynet +## Daemon port [Intended for internal use] +## LBRYNET talks to peers on port 3333 [Intended for external use] this port is used to discover other lbrynet daemons with blobs. +## Expose 5566 Reflector port to listen on +## Expose 5279 Port the daemon API will listen on +## the lbryumx aka Wallet port [Intended for internal use] +#EXPOSE 4444 3333 5566 5279 50001 +USER lbrynet +ENTRYPOINT ["/usr/bin/checkmount"] +CMD ["start"] \ No newline at end of file diff --git a/e2e/lbrynet/docker/checkmount.sh b/e2e/lbrynet/docker/checkmount.sh new file mode 100644 index 0000000..4f77c11 --- /dev/null +++ b/e2e/lbrynet/docker/checkmount.sh @@ -0,0 +1,12 @@ +#!/bin/bash + +## TODO: Make a bit more aware of the run mode of this appliance in case there is ever a test mode enabled in the start.sh +mountpoint=/home/lbrynet + +if ! grep -qs ".* $mountpoint " /proc/mounts; then + echo "$mountpoint not mounted, refusing to run." + ## TODO: We should have documentation that this error references directly with a URL as to why it won't run without a volume. + exit 1 +else + bash -c "$*" +fi diff --git a/e2e/lbrynet/docker/start.sh b/e2e/lbrynet/docker/start.sh new file mode 100644 index 0000000..69beadc --- /dev/null +++ b/e2e/lbrynet/docker/start.sh @@ -0,0 +1,4 @@ +#!/bin/bash +lbrynet start \ + --api="${API_BIND_IP:-0.0.0.0}":"${API_PORT:-5279}" \ + --config="${CONFIG_PATH:-/etc/lbry/daemon_settings.yml}" \ No newline at end of file diff --git a/e2e/lbrynet/settings/daemon_settings.yml b/e2e/lbrynet/settings/daemon_settings.yml new file mode 100644 index 0000000..8a6bcb4 --- /dev/null +++ b/e2e/lbrynet/settings/daemon_settings.yml @@ -0,0 +1,12 @@ +#blockchain_name: lbrycrd_main +#blockchain_name: lbrycrd_testnet +blockchain_name: lbrycrd_regtest +lbryum_servers: +# - spv1.lbry.com:50001 #Production Wallet Server + - walletserver:50001 +save_blobs: true +save_files: false +share_usage_data: false +tcp_port: 3333 +udp_port: 4444 +use_upnp: true \ No newline at end of file diff --git a/e2e/walletserver/docker-compose.yml b/e2e/walletserver/docker-compose.yml new file mode 100644 index 0000000..a3104ba --- /dev/null +++ b/e2e/walletserver/docker-compose.yml @@ -0,0 +1,35 @@ +version: "3" +networks: + lbry-network: + external: true + +services: + ################### + ## Wallet Server ## + ################### + walletserver: + image: lbry/wallet-server:v0.38.5 + restart: always + networks: + lbry-network: + ipv4_address: 10.6.1.2 + volumes: + - "../persist/data/.walletserver/database:/database" + environment: + - DB_DIRECTORY=/database + - MAX_SEND=1000000000000000000000 + - DAEMON_URL=http://lbry:lbry@lbrycrd:29245/ + - MAX_SUBS=1000000000000 + - BANDWIDTH_LIMIT=80000000000 + - SESSION_TIMEOUT=10000000000000000000000000 + - TCP_PORT=50001 + #network_mode: host + #network_mode: bridge + ports: + - "50001:50001" + expose: + - "50001" + ulimits: + nofile: 90000 +# command: lbry.wallet.server.coin.LBC + command: lbry.wallet.server.coin.LBCRegTest \ No newline at end of file From c84e1e6fb9a554bfeaffd144609486eebe639869 Mon Sep 17 00:00:00 2001 From: Mark Beamer Jr Date: Tue, 30 Jul 2019 23:32:02 -0400 Subject: [PATCH 03/11] Fixed bugs around sending channel cert and enabling local testing with docker. --- .gitignore | 4 +- blobs_reflector/reflect.go | 27 ++++++----- go.mod | 6 +++ go.sum | 16 +++++- main.go | 18 ++----- manager/ytsync.go | 99 ++++++++++++++++++++++++++++++++------ sdk/api.go | 20 +++++--- util/util.go | 70 +++++++++++++++++++++++++++ 8 files changed, 211 insertions(+), 49 deletions(-) create mode 100644 util/util.go diff --git a/.gitignore b/.gitignore index 6ba26e6..d4c0eca 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,4 @@ bin/ -e2e/persist \ No newline at end of file +e2e/persist +.env +blobsfiles \ No newline at end of file diff --git a/blobs_reflector/reflect.go b/blobs_reflector/reflect.go index 744a901..49abcb0 100644 --- a/blobs_reflector/reflect.go +++ b/blobs_reflector/reflect.go @@ -2,16 +2,18 @@ package blobs_reflector import ( "encoding/json" + "io/ioutil" + "os" + "os/user" + "path/filepath" + "github.com/lbryio/lbry.go/extras/errors" "github.com/lbryio/reflector.go/cmd" "github.com/lbryio/reflector.go/db" "github.com/lbryio/reflector.go/reflector" "github.com/lbryio/reflector.go/store" + "github.com/lbryio/ytsync/util" "github.com/mitchellh/go-ps" - "io/ioutil" - "os" - "os/user" - "path/filepath" log "github.com/sirupsen/logrus" ) @@ -100,12 +102,7 @@ func cleanupLbrynet() error { if running { return errors.Prefix("cannot cleanup lbrynet as the daemon is running", err) } - usr, err := user.Current() - if err != nil { - log.Errorln(err.Error()) - return errors.Err(err) - } - lbrynetDir := usr.HomeDir + "/.lbrynet/" + lbrynetDir := util.GetLBRYNetDir() files, err := filepath.Glob(lbrynetDir + "lbrynet.sqlite*") if err != nil { return errors.Err(err) @@ -116,7 +113,7 @@ func cleanupLbrynet() error { return errors.Err(err) } } - blobsDir := lbrynetDir + "/blobfiles/" + blobsDir := util.GetBlobsDir() err = os.RemoveAll(blobsDir) if err != nil { return errors.Err(err) @@ -129,6 +126,14 @@ func cleanupLbrynet() error { } func isLbrynetRunning() (bool, error) { + if util.IsUsingDocker() { + container, err := util.GetLBRYNetContainer(false) + if err != nil { + return false, err + } + return container != nil, nil + } + processes, err := ps.Processes() if err != nil { return true, errors.Err(err) diff --git a/go.mod b/go.mod index 1263e35..21725a2 100644 --- a/go.mod +++ b/go.mod @@ -3,10 +3,15 @@ module github.com/lbryio/ytsync require ( cloud.google.com/go v0.37.4 // indirect github.com/ChannelMeter/iso8601duration v0.0.0-20150204201828-8da3af7a2a61 + github.com/Microsoft/go-winio v0.4.13 // indirect github.com/armon/go-metrics v0.0.0-20190430140413-ec5e00d3c878 // indirect github.com/asaskevich/govalidator v0.0.0-20180720115003-f9ffefc3facf github.com/aws/aws-sdk-go v1.17.3 github.com/channelmeter/iso8601duration v0.0.0-20150204201828-8da3af7a2a61 // indirect + github.com/docker/distribution v2.7.1+incompatible // indirect + github.com/docker/docker v1.13.1 + github.com/docker/go-connections v0.4.0 // indirect + github.com/docker/go-units v0.4.0 // indirect github.com/go-ini/ini v1.42.0 // indirect github.com/go-sql-driver/mysql v1.4.1 // indirect github.com/gopherjs/gopherjs v0.0.0-20190411002643-bd77b112433e // indirect @@ -23,6 +28,7 @@ require ( github.com/mitchellh/go-ps v0.0.0-20170309133038-4fdf99ab2936 github.com/onsi/ginkgo v1.8.0 // indirect github.com/onsi/gomega v1.5.0 // indirect + github.com/opencontainers/go-digest v1.0.0-rc1 // indirect github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2 // indirect github.com/prometheus/common v0.3.0 github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24 diff --git a/go.sum b/go.sum index 6e37f14..3342537 100644 --- a/go.sum +++ b/go.sum @@ -6,6 +6,8 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03 github.com/ChannelMeter/iso8601duration v0.0.0-20150204201828-8da3af7a2a61 h1:N5Vqww5QISEHsWHOWDEx4PzdIay3Cg0Jp7zItq2ZAro= github.com/ChannelMeter/iso8601duration v0.0.0-20150204201828-8da3af7a2a61/go.mod h1:GnKXcK+7DYNy/8w2Ex//Uql4IgfaU82Cd5rWKb7ah00= github.com/DataDog/datadog-go v2.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ= +github.com/Microsoft/go-winio v0.4.13 h1:Hmi80lzZuI/CaYmlJp/b+FjZdRZhKu9c2mDVqKlLWVs= +github.com/Microsoft/go-winio v0.4.13/go.mod h1:qXqCSQ3Xa7+6tgxaGTIe4Kpcdsi+P8jBhyzoq1bpyYA= github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo= github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI= github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII= @@ -56,6 +58,14 @@ github.com/davecgh/go-spew v0.0.0-20171005155431-ecdeabc65495/go.mod h1:J7Y8YcW2 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/docker/distribution v2.7.1+incompatible h1:a5mlkVzth6W5A4fOsS3D2EO5BUmsJpcB+cRlLU7cSug= +github.com/docker/distribution v2.7.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= +github.com/docker/docker v1.13.1 h1:IkZjBSIc8hBjLpqeAbeE5mca5mNgeatLHBy3GO78BWo= +github.com/docker/docker v1.13.1/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ= +github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec= +github.com/docker/go-units v0.4.0 h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw= +github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs= github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU= github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= @@ -73,6 +83,7 @@ github.com/go-ini/ini v1.42.0 h1:TWr1wGj35+UiWHlBA8er89seFXxzwFn11spilrrj+38= github.com/go-ini/ini v1.42.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= +github.com/go-ozzo/ozzo-validation v3.5.0+incompatible h1:sUy/in/P6askYr16XJgTKq/0SZhiWsdg4WZGaLsGQkM= github.com/go-ozzo/ozzo-validation v3.5.0+incompatible/go.mod h1:gsEKFIVnabGBt6mXmxK0MoFy+cZoTJY6mu5Ll3LVLBU= github.com/go-sql-driver/mysql v0.0.0-20180719071942-99ff426eb706/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= github.com/go-sql-driver/mysql v1.4.1 h1:g24URVg0OFbNUTx9qqY1IRZ9D9z3iPyi5zKhQZpNwpA= @@ -189,8 +200,6 @@ github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/lbryio/errors.go v0.0.0-20180223142025-ad03d3cc6a5c h1:BhdcWGsuKif/XoSZnqVGNqJ1iEmH0czWR5upj+AuR8M= github.com/lbryio/errors.go v0.0.0-20180223142025-ad03d3cc6a5c/go.mod h1:muH7wpUqE8hRA3OrYYosw9+Sl681BF9cwcjzE+OCNK8= github.com/lbryio/lbry.go v0.0.0-20190109223729-30c312501602/go.mod h1:YEuFJD/oHNra6BFy+NfuvS84Wg6RMWJFGtiCCCc6MmQ= -github.com/lbryio/lbry.go v1.0.15 h1:g4g9cDDUsobmWgTA+1jEIb5k2fRCP0/NvPOMXduP8xY= -github.com/lbryio/lbry.go v1.0.15/go.mod h1:JtyI30bU51rm0LZ/po3mQuzf++14OWb6kR/6mMRAmKU= github.com/lbryio/lbry.go v1.0.16-0.20190726033352-f69bba9977f8 h1:oNQh5rAVygkVLLRYxLd098/smLHcMoKrn7ozfwFhx2Y= github.com/lbryio/lbry.go v1.0.16-0.20190726033352-f69bba9977f8/go.mod h1:JtyI30bU51rm0LZ/po3mQuzf++14OWb6kR/6mMRAmKU= github.com/lbryio/lbryschema.go v0.0.0-20190428231007-c54836bca002 h1:urfYK5ElpUrAv90auPLldoVC60LwiGAcY0OE6HJB9KI= @@ -239,6 +248,8 @@ github.com/onsi/gomega v1.4.1/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5 github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.5.0 h1:izbySO9zDPmjJ8rDjLvkA2zJHIo+HkYXHnf7eN7SSyo= github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= +github.com/opencontainers/go-digest v1.0.0-rc1 h1:WzifXhOVOEOuFYOJAW6aQqW0TooG2iki3E3Ii+WN7gQ= +github.com/opencontainers/go-digest v1.0.0-rc1/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= github.com/openzipkin/zipkin-go v0.1.6/go.mod h1:QgAqvLzwWbR/WpD4A3cGpPtJrZXNIiJc5AZX7/PBEpw= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/pascaldekloe/goe v0.1.0 h1:cBOtyMzM9HTpWjXfbbunk26uA6nG3a8n06Wieeh0MwY= @@ -364,6 +375,7 @@ golang.org/x/sys v0.0.0-20181122145206-62eef0e2fa9b/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20190109145017-48ac38b7c8cb/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190520201301-c432e742b0af h1:NXfmMfXz6JqGfG3ikSxcz2N93j6DgScr19Oo2uwFu88= golang.org/x/sys v0.0.0-20190520201301-c432e742b0af/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= diff --git a/main.go b/main.go index dfc5019..8f27f77 100644 --- a/main.go +++ b/main.go @@ -4,7 +4,6 @@ import ( "fmt" "math/rand" "os" - "os/user" "time" "github.com/lbryio/lbry.go/extras/util" @@ -13,7 +12,7 @@ import ( "github.com/spf13/cobra" "github.com/lbryio/ytsync/manager" - logUtils "github.com/lbryio/ytsync/util" + ytUtils "github.com/lbryio/ytsync/util" ) var Version string @@ -113,7 +112,6 @@ func ytSync(cmd *cobra.Command, args []string) { apiURL := os.Getenv("LBRY_WEB_API") apiToken := os.Getenv("LBRY_API_TOKEN") youtubeAPIKey := os.Getenv("YOUTUBE_API_KEY") - blobsDir := os.Getenv("BLOBS_DIRECTORY") lbrycrdString := os.Getenv("LBRYCRD_STRING") awsS3ID := os.Getenv("AWS_S3_ID") awsS3Secret := os.Getenv("AWS_S3_SECRET") @@ -150,14 +148,8 @@ func ytSync(cmd *cobra.Command, args []string) { if lbrycrdString == "" { log.Infoln("Using default (local) lbrycrd instance. Set LBRYCRD_STRING if you want to use something else") } - if blobsDir == "" { - usr, err := user.Current() - if err != nil { - log.Errorln(err.Error()) - return - } - blobsDir = usr.HomeDir + "/.lbrynet/blobfiles/" - } + + blobsDir := ytUtils.GetBlobsDir() syncProperties := &sdk.SyncProperties{ SyncFrom: syncFrom, @@ -198,7 +190,7 @@ func ytSync(cmd *cobra.Command, args []string) { ) err := sm.Start() if err != nil { - logUtils.SendErrorToSlack(err.Error()) + ytUtils.SendErrorToSlack(err.Error()) } - logUtils.SendInfoToSlack("Syncing process terminated!") + ytUtils.SendInfoToSlack("Syncing process terminated!") } diff --git a/manager/ytsync.go b/manager/ytsync.go index e7c46df..2391eb0 100644 --- a/manager/ytsync.go +++ b/manager/ytsync.go @@ -1,6 +1,7 @@ package manager import ( + "context" "fmt" "io/ioutil" "net/http" @@ -31,6 +32,8 @@ import ( "github.com/aws/aws-sdk-go/aws/session" "github.com/aws/aws-sdk-go/service/s3" "github.com/aws/aws-sdk-go/service/s3/s3manager" + "github.com/docker/docker/api/types" + "github.com/docker/docker/client" "github.com/mitchellh/go-ps" log "github.com/sirupsen/logrus" "google.golang.org/api/googleapi/transport" @@ -116,17 +119,9 @@ func (s *Sync) IsInterrupted() bool { } func (s *Sync) downloadWallet() error { - defaultWalletDir := os.Getenv("HOME") + "/.lbryum/wallets/default_wallet" - defaultTempWalletDir := os.Getenv("HOME") + "/.lbryum/wallets/tmp_wallet" - key := aws.String("/wallets/" + s.YoutubeChannelID) - if os.Getenv("REGTEST") == "true" { - defaultWalletDir = os.Getenv("HOME") + "/.lbryum_regtest/wallets/default_wallet" - defaultTempWalletDir = os.Getenv("HOME") + "/.lbryum_regtest/wallets/tmp_wallet" - key = aws.String("/regtest/" + s.YoutubeChannelID) - } - - if _, err := os.Stat(defaultWalletDir); !os.IsNotExist(err) { - return errors.Err("default_wallet already exists") + defaultWalletDir, defaultTempWalletDir, key, err := s.getWalletPaths() + if err != nil { + return errors.Err(err) } creds := credentials.NewStaticCredentials(s.AwsS3ID, s.AwsS3Secret, "") @@ -168,6 +163,29 @@ func (s *Sync) downloadWallet() error { return os.Rename(defaultTempWalletDir, defaultWalletDir) } +func (s *Sync) getWalletPaths() (defaultWallet, tempWallet string, key *string, err error) { + + defaultWallet = os.Getenv("HOME") + "/.lbryum/wallets/default_wallet" + tempWallet = os.Getenv("HOME") + "/.lbryum/wallets/tmp_wallet" + key = aws.String("/wallets/" + s.YoutubeChannelID) + if os.Getenv("REGTEST") == "true" { + defaultWallet = os.Getenv("HOME") + "/.lbryum_regtest/wallets/default_wallet" + tempWallet = os.Getenv("HOME") + "/.lbryum_regtest/wallets/tmp_wallet" + key = aws.String("/regtest/" + s.YoutubeChannelID) + } + + walletPath := os.Getenv("LBRYNET_WALLETS_DIR") + if walletPath != "" { + defaultWallet = walletPath + "/wallets/default_wallet" + tempWallet = walletPath + "/wallets/tmp_wallet" + } + + if _, err := os.Stat(defaultWallet); !os.IsNotExist(err) { + return "", "", nil, errors.Err("default_wallet already exists") + } + return +} + func (s *Sync) uploadWallet() error { defaultWalletDir := os.Getenv("HOME") + "/.lbryum/wallets/default_wallet" key := aws.String("/wallets/" + s.YoutubeChannelID) @@ -272,13 +290,13 @@ func (s *Sync) FullCycle() (e error) { defer deleteSyncFolder(s.videoDirectory) log.Printf("Starting daemon") - err = startDaemonViaSystemd() + err = startDaemon() if err != nil { return err } log.Infoln("Waiting for daemon to finish starting...") - s.daemon = jsonrpc.NewClient("") + s.daemon = jsonrpc.NewClient(os.Getenv("LBRYNET_ADDRESS")) s.daemon.SetRPCTimeout(40 * time.Minute) err = s.waitForDaemonStart() @@ -345,7 +363,7 @@ func (s *Sync) waitForDaemonStart() error { func (s *Sync) stopAndUploadWallet(e *error) { log.Printf("Stopping daemon") - shutdownErr := stopDaemonViaSystemd() + shutdownErr := stopDaemon() if shutdownErr != nil { logShutdownError(shutdownErr) } else { @@ -553,6 +571,7 @@ func (s *Sync) doSync() error { if err != nil { return errors.Prefix("error getting channel cert", err) } + println("lbryChannelID:", s.lbryChannelID) err = s.APIConfig.SetChannelCert(string(*cert), s.lbryChannelID) if err != nil { return errors.Prefix("error setting channel cert", err) @@ -927,6 +946,58 @@ func (s *Sync) processVideo(v video) (err error) { return nil } +func startDaemon() error { + if logUtils.IsUsingDocker() { + return startDaemonViaDocker() + } + return startDaemonViaSystemd() +} + +func stopDaemon() error { + if logUtils.IsUsingDocker() { + return stopDaemonViaDocker() + } + return stopDaemonViaSystemd() +} + +func startDaemonViaDocker() error { + container, err := logUtils.GetLBRYNetContainer(true) + if err != nil { + return err + } + + cli, err := client.NewEnvClient() + if err != nil { + panic(err) + } + + err = cli.ContainerStart(context.Background(), container.ID, types.ContainerStartOptions{}) + if err != nil { + return errors.Err(err) + } + + return nil +} + +func stopDaemonViaDocker() error { + container, err := logUtils.GetLBRYNetContainer(false) + if err != nil { + return err + } + + cli, err := client.NewEnvClient() + if err != nil { + panic(err) + } + + err = cli.ContainerStop(context.Background(), container.ID, nil) + if err != nil { + return errors.Err(err) + } + + return nil +} + func startDaemonViaSystemd() error { err := exec.Command("/usr/bin/sudo", "/bin/systemctl", "start", "lbrynet.service").Run() if err != nil { diff --git a/sdk/api.go b/sdk/api.go index 6f3de9b..9fbdd3d 100644 --- a/sdk/api.go +++ b/sdk/api.go @@ -11,8 +11,6 @@ import ( "strings" "time" - "github.com/lbryio/lbry.go/extras/api" - "github.com/lbryio/lbry.go/extras/errors" "github.com/lbryio/lbry.go/extras/null" @@ -103,24 +101,30 @@ func sanitizeFailureReason(s *string) { func (a *APIConfig) SetChannelCert(certHex string, channelID string) error { + type apiSetChannelCertResponse struct { + Success bool `json:"success"` + Error null.String `json:"error"` + Data string `json:"data"` + } + endpoint := a.ApiURL + "/yt/channel_cert" res, _ := http.PostForm(endpoint, url.Values{ - "channel_id": {channelID}, - "channel_cert": {certHex}, - "auth_token": {a.ApiToken}, + "channel_claim_id": {channelID}, + "channel_cert": {certHex}, + "auth_token": {a.ApiToken}, }) defer res.Body.Close() body, _ := ioutil.ReadAll(res.Body) - var response api.Response + var response apiSetChannelCertResponse err := json.Unmarshal(body, &response) if err != nil { return errors.Err(err) } - if response.Error != nil { - return errors.Err(response.Error) + if !response.Error.IsNull() { + return errors.Err(response.Error.String) } return nil diff --git a/util/util.go b/util/util.go new file mode 100644 index 0000000..b5f164d --- /dev/null +++ b/util/util.go @@ -0,0 +1,70 @@ +package util + +import ( + "context" + "fmt" + "os" + "os/user" + + "github.com/docker/docker/api/types" + "github.com/docker/docker/api/types/filters" + "github.com/docker/docker/client" + "github.com/lbryio/lbry.go/extras/errors" + + "github.com/prometheus/common/log" +) + +func GetBlobsDir() string { + blobsDir := os.Getenv("BLOBS_DIRECTORY") + if blobsDir == "" { + usr, err := user.Current() + if err != nil { + log.Errorln(err.Error()) + return "" + } + blobsDir = usr.HomeDir + "/.lbrynet/blobfiles/" + } + + return blobsDir +} + +func GetLBRYNetDir() string { + lbrynetDir := os.Getenv("LBRYNET_DIR") + if lbrynetDir == "" { + usr, err := user.Current() + if err != nil { + log.Errorln(err.Error()) + return "" + } + return usr.HomeDir + "/.lbrynet/" + } + return lbrynetDir +} + +func GetLBRYNetContainer(all bool) (*types.Container, error) { + cli, err := client.NewEnvClient() + if err != nil { + panic(err) + } + filters := filters.NewArgs() + filters.Add("name", "lbrynet") + containers, err := cli.ContainerList(context.Background(), types.ContainerListOptions{All: all, Filters: filters}) + if err != nil { + panic(err) + } + for _, container := range containers { + fmt.Printf("%s %s\n", container.ID[:10], container.Image) + } + if len(containers) == 0 { + return nil, nil + } + if len(containers) > 1 { + return nil, errors.Err("more than one lbrynet container found") + } + + return &containers[0], nil +} + +func IsUsingDocker() bool { + return os.Getenv("LBRYNET_USE_DOCKER") == "true" +} From 517c2c56a3f49c7311aee7f3d72c1f42088d0714 Mon Sep 17 00:00:00 2001 From: Mark Beamer Jr Date: Wed, 31 Jul 2019 22:42:22 -0400 Subject: [PATCH 04/11] Add disabling of blob uploading for local testing until reflector docker image can be created( not urgent now ). Add ability to generate blocks when in regtest mode and using docker. Add refactor of environment variable accessing. --- blobs_reflector/reflect.go | 5 ++++- manager/setup.go | 27 +++++++++++----------- util/util.go | 46 +++++++++++++++++++++++++++++++++----- 3 files changed, 57 insertions(+), 21 deletions(-) diff --git a/blobs_reflector/reflect.go b/blobs_reflector/reflect.go index 49abcb0..52f1edc 100644 --- a/blobs_reflector/reflect.go +++ b/blobs_reflector/reflect.go @@ -42,6 +42,9 @@ func loadConfig(path string) (cmd.Config, error) { } func reflectBlobs() error { + if util.IsBlobReflectionOff() { + return nil + } //make sure lbrynet is off running, err := isLbrynetRunning() if err != nil { @@ -127,7 +130,7 @@ func cleanupLbrynet() error { func isLbrynetRunning() (bool, error) { if util.IsUsingDocker() { - container, err := util.GetLBRYNetContainer(false) + container, err := util.GetLBRYNetContainer(util.ONLINE) if err != nil { return false, err } diff --git a/manager/setup.go b/manager/setup.go index 5cf5c01..5c26ffc 100644 --- a/manager/setup.go +++ b/manager/setup.go @@ -11,8 +11,6 @@ import ( "github.com/lbryio/lbry.go/extras/errors" "github.com/lbryio/lbry.go/extras/jsonrpc" "github.com/lbryio/lbry.go/extras/util" - "github.com/lbryio/lbry.go/lbrycrd" - "github.com/lbryio/ytsync/tagsManager" "github.com/lbryio/ytsync/thumbs" logUtils "github.com/lbryio/ytsync/util" @@ -224,6 +222,16 @@ func (s *Sync) ensureEnoughUTXOs() error { } func (s *Sync) waitForNewBlock() error { + if logUtils.IsRegTest() && logUtils.IsUsingDocker() { + lbrycrd, err := logUtils.GetLbrycrdClient(s.LbrycrdString) + if err != nil { + return errors.Prefix("error getting lbrycrd client: ", err) + } + txs, err := lbrycrd.Generate(1) + for _, tx := range txs { + log.Info("Generated tx: ", tx.String()) + } + } status, err := s.daemon.Status() if err != nil { return err @@ -417,18 +425,9 @@ func allUTXOsConfirmed(utxolist *jsonrpc.UTXOListResponse) bool { func (s *Sync) addCredits(amountToAdd float64) error { log.Printf("Adding %f credits", amountToAdd) - var lbrycrdd *lbrycrd.Client - var err error - if s.LbrycrdString == "" { - lbrycrdd, err = lbrycrd.NewWithDefaultURL() - if err != nil { - return err - } - } else { - lbrycrdd, err = lbrycrd.New(s.LbrycrdString) - if err != nil { - return err - } + lbrycrdd, err := logUtils.GetLbrycrdClient(s.LbrycrdString) + if err != nil { + return err } addressResp, err := s.daemon.AddressUnused(nil) diff --git a/util/util.go b/util/util.go index b5f164d..353d165 100644 --- a/util/util.go +++ b/util/util.go @@ -6,12 +6,13 @@ import ( "os" "os/user" + "github.com/lbryio/lbry.go/extras/errors" + "github.com/lbryio/lbry.go/lbrycrd" + "github.com/docker/docker/api/types" "github.com/docker/docker/api/types/filters" "github.com/docker/docker/client" - "github.com/lbryio/lbry.go/extras/errors" - - "github.com/prometheus/common/log" + log "github.com/sirupsen/logrus" ) func GetBlobsDir() string { @@ -19,7 +20,7 @@ func GetBlobsDir() string { if blobsDir == "" { usr, err := user.Current() if err != nil { - log.Errorln(err.Error()) + log.Error(err.Error()) return "" } blobsDir = usr.HomeDir + "/.lbrynet/blobfiles/" @@ -28,6 +29,10 @@ func GetBlobsDir() string { return blobsDir } +func IsBlobReflectionOff() bool { + return os.Getenv("REFLECT_BLOBS") == "false" +} + func GetLBRYNetDir() string { lbrynetDir := os.Getenv("LBRYNET_DIR") if lbrynetDir == "" { @@ -41,13 +46,20 @@ func GetLBRYNetDir() string { return lbrynetDir } +const ALL = true +const ONLINE = false + func GetLBRYNetContainer(all bool) (*types.Container, error) { + return getDockerContainer("lbrynet", all) +} + +func getDockerContainer(name string, all bool) (*types.Container, error) { cli, err := client.NewEnvClient() if err != nil { panic(err) } filters := filters.NewArgs() - filters.Add("name", "lbrynet") + filters.Add("name", name) containers, err := cli.ContainerList(context.Background(), types.ContainerListOptions{All: all, Filters: filters}) if err != nil { panic(err) @@ -59,7 +71,7 @@ func GetLBRYNetContainer(all bool) (*types.Container, error) { return nil, nil } if len(containers) > 1 { - return nil, errors.Err("more than one lbrynet container found") + return nil, errors.Err("more than one %s container found", name) } return &containers[0], nil @@ -68,3 +80,25 @@ func GetLBRYNetContainer(all bool) (*types.Container, error) { func IsUsingDocker() bool { return os.Getenv("LBRYNET_USE_DOCKER") == "true" } + +func IsRegTest() bool { + return os.Getenv("REGTEST") == "true" +} + +func GetLbrycrdClient(lbrycrdString string) (*lbrycrd.Client, error) { + var lbrycrdd *lbrycrd.Client + var err error + if lbrycrdString == "" { + lbrycrdd, err = lbrycrd.NewWithDefaultURL() + if err != nil { + return nil, err + } + } else { + lbrycrdd, err = lbrycrd.New(lbrycrdString) + if err != nil { + return nil, err + } + } + + return lbrycrdd, nil +} From 73937a541bae65cbf06f865d88b49edff2506c4e Mon Sep 17 00:00:00 2001 From: Mark Beamer Jr Date: Thu, 1 Aug 2019 21:43:41 -0400 Subject: [PATCH 05/11] Add var/tmp volume for ytsync tmp files communicated between lbrynet and ytsync Add reference to wallet dir env var for when the wallet file is uploaded so it is cleared properly when testing locally --- e2e/docker-compose.yml | 2 ++ manager/ytsync.go | 7 ++++++- sources/youtubeVideo.go | 14 +++++++------- 3 files changed, 15 insertions(+), 8 deletions(-) diff --git a/e2e/docker-compose.yml b/e2e/docker-compose.yml index 5c36dc9..bf07474 100644 --- a/e2e/docker-compose.yml +++ b/e2e/docker-compose.yml @@ -51,3 +51,5 @@ services: volumes: - "./persist/.lbrynet:/home/lbrynet" - ".:/etc/lbry" #Put your daemon_settings.yml here + # /private/var/tmp for OSX and /var/tmp for Linux + - "/private/var/tmp:/var/tmp" diff --git a/manager/ytsync.go b/manager/ytsync.go index 2391eb0..14e8611 100644 --- a/manager/ytsync.go +++ b/manager/ytsync.go @@ -194,6 +194,11 @@ func (s *Sync) uploadWallet() error { key = aws.String("/regtest/" + s.YoutubeChannelID) } + walletPath := os.Getenv("LBRYNET_WALLETS_DIR") + if walletPath != "" { + defaultWalletDir = walletPath + "/wallets/default_wallet" + } + if _, err := os.Stat(defaultWalletDir); os.IsNotExist(err) { return errors.Err("default_wallet does not exist") } @@ -283,7 +288,7 @@ func (s *Sync) FullCycle() (e error) { defer s.stopAndUploadWallet(&e) - s.videoDirectory, err = ioutil.TempDir("", "ytsync") + s.videoDirectory, err = ioutil.TempDir(os.Getenv("TMP_DIR"), "ytsync") if err != nil { return errors.Wrap(err, 0) } diff --git a/sources/youtubeVideo.go b/sources/youtubeVideo.go index 8ede581..aba0a6e 100644 --- a/sources/youtubeVideo.go +++ b/sources/youtubeVideo.go @@ -24,7 +24,7 @@ import ( "github.com/lbryio/ytsync/tagsManager" "github.com/lbryio/ytsync/thumbs" - "github.com/ChannelMeter/iso8601duration" + duration "github.com/ChannelMeter/iso8601duration" "github.com/aws/aws-sdk-go/aws" "github.com/shopspring/decimal" log "github.com/sirupsen/logrus" @@ -294,16 +294,16 @@ runcmd: log.Debugln(string(outLog)) if strings.Contains(string(outLog), "does not pass filter duration") { - _ = v.delete() + _ = v.delete("does not pass filter duration") return errors.Err("video is too long to process") } if strings.Contains(string(outLog), "File is larger than max-filesize") { - _ = v.delete() + _ = v.delete("File is larger than max-filesize") return errors.Err("the video is too big to sync, skipping for now") } if string(errorLog) != "" { log.Printf("Command finished with error: %v", errors.Err(string(errorLog))) - _ = v.delete() + _ = v.delete("due to error") return errors.Err(string(errorLog)) } fi, err := os.Stat(v.getFullPath()) @@ -338,14 +338,14 @@ func (v *YoutubeVideo) getDownloadedPath() (string, error) { return "", errors.Err("could not find any downloaded videos") } -func (v *YoutubeVideo) delete() error { +func (v *YoutubeVideo) delete(reason string) error { videoPath, err := v.getDownloadedPath() if err != nil { log.Errorln(err) return err } err = os.Remove(videoPath) - log.Debugf("%s deleted from disk (%s)", v.id, videoPath) + log.Debugf("%s deleted from disk for '%s' (%s)", v.id, reason, videoPath) if err != nil { err = errors.Prefix("delete error", err) @@ -451,7 +451,7 @@ func (v *YoutubeVideo) downloadAndPublish(daemon *jsonrpc.Client, params SyncPar summary, err := v.publish(daemon, params) //delete the video in all cases (and ignore the error) - _ = v.delete() + _ = v.delete("finished download and publish") return summary, errors.Prefix("publish error", err) } From 6438048f40cd7b2ee55589616e11872db94b5289 Mon Sep 17 00:00:00 2001 From: Mark Beamer Jr Date: Sat, 3 Aug 2019 18:34:48 -0400 Subject: [PATCH 06/11] Add CleanOnStartup for local testing and e2e testing Update methods to be refactored into util package to be used in many locations. Add end to end test of ytsync for CI --- .gitlab-ci.yml | 29 ++++ .travis.yml | 30 +++- blobs_reflector/reflect.go | 62 +------- e2e/docker-compose.yml | 80 +++++++++-- e2e/e2e.sh | 88 ++++++++++++ e2e/lbrycrd/docker/advance_blocks.sh | 2 +- e2e/lbrycrd/docker/build.sh | 8 ++ e2e/lbrynet/docker/Dockerfile | 6 +- e2e/lbrynet/docker/build.sh | 8 ++ go.mod | 26 +--- go.sum | 99 ++++--------- main.go | 7 +- manager/manager.go | 12 +- manager/setup.go | 5 +- manager/ytsync.go | 131 ++++++----------- thumbs/uploader.go | 4 +- util/util.go | 204 ++++++++++++++++++++++++++- 17 files changed, 528 insertions(+), 273 deletions(-) create mode 100644 .gitlab-ci.yml create mode 100755 e2e/e2e.sh create mode 100755 e2e/lbrycrd/docker/build.sh create mode 100755 e2e/lbrynet/docker/build.sh diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml new file mode 100644 index 0000000..abafa89 --- /dev/null +++ b/.gitlab-ci.yml @@ -0,0 +1,29 @@ +job: + image: golang:1.12.7-stretch + + variables: + GO111MODULE: "on" + + script: + #Fetch master to be used in deploy condition to prevent deploying on old commits. + - whoami && pwd + - USER=$(whoami) + - ./e2e/e2e.sh + + cache: + key: "$CI_COMMIT_REF_SLUG" + paths: + - $HOME/.cache/go-build + - $HOME/gopath/pkg/mod + + before_script: + - whoami && pwd + - USER=$(whoami) + - apt update -q && apt install -qqy --no-install-recommends locales-all sudo youtube-dl ffmpeg tree curl + - apt install -qqy --no-install-recommends python-pip python-dev libffi-dev libssl-dev gcc libc-dev make + - curl -fsSL https://get.docker.com -o get-docker.sh + - sudo sh get-docker.sh + - sudo usermod -aG docker $USER + - sudo curl -L "https://github.com/docker/compose/releases/download/1.24.1/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose + - sudo chmod +x /usr/local/bin/docker-compose + - sudo ln -s /usr/local/bin/docker-compose /usr/bin/docker-compose diff --git a/.travis.yml b/.travis.yml index 387cf75..38fc146 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,15 +1,35 @@ os: linux -dist: xenial +dist: bionic language: go -go_import_path: github.com/lbryio/ytsync go: -- 1.11.5 -- master + - 1.12.1 + - master + +install: true + +cache: + directories: + - $HOME/.cache/go-build + - $HOME/gopath/pkg/mod + +services: + - docker + +addons: + apt: + update: true + packages: + - youtube-dl + - ffmpeg + - tree + env: global: - GO111MODULE=on #GITHUB_TOKEN - secure: "Ps3KocRP5xnM3/uA99CeYhDTVxRIuW7fGyrtqBeRWZW0cXzeA4XCTKxqcFbrPUPw67XkrBVgE58JDdWoQEJ7tm67PjMm/ltp5Evhx/QAJDh+YSofXyGDVpG1mrTZFI66R3NVVJLkSGALMkuWWXvfYZeU//AworJbyRoaIK/CVt5OP23i5N4tdd5UXc5dfLuYqnKRynyMmCkz9c3yEIQMXoPhG2hx7l7L2BeMJvcKmVhkSN7nQayjnrbUXGm/IRqrb88lvkyBevN5E3IB2V5IKEieIPZjbD/N0IfcnAt89Z96tgDhtIbx3ZvXm92lsvHA8buqQpG9d2AmSi6GKs64lQcnGeM5o0wER2JHWl1OSa1Nr/UAo5Xb/PM65Yt3yZE8AuMKHBmbfDSBzdkTXx58AeDzFUd3kMXD/fFjeQQWyXFlOss3ygH9SObl827Txmz9OJqZaxabs5Q3AP6m3EjKjz7zfLfrgpcxJM2WBiU1bN0ZxUgZkImy/CHk5gCZ7vhcnaLiDO4HZnzY/aRJwKYQPE5i0O2nHpIfovqkc0DFBA7U/7Cjin7e1E0UZvF3meLOxMqkfc6X7QTxqQpt2Tej6jlpdxw4CTLwGUhGkAw9IAPkUB3L0EbZ1/ksGhNvGDvUeSTq8hYdMAPmA+k9jS6653V4SQ+qBMy5++tbr5AeZQI=" +script: + - ./e2e/e2e.sh deploy: provider: script skip_cleanup: true @@ -17,4 +37,4 @@ deploy: file: bin/ytsync on: repo: lbryio/ytsync - tags: true \ No newline at end of file + tags: true diff --git a/blobs_reflector/reflect.go b/blobs_reflector/reflect.go index 52f1edc..b816beb 100644 --- a/blobs_reflector/reflect.go +++ b/blobs_reflector/reflect.go @@ -13,7 +13,6 @@ import ( "github.com/lbryio/reflector.go/reflector" "github.com/lbryio/reflector.go/store" "github.com/lbryio/ytsync/util" - "github.com/mitchellh/go-ps" log "github.com/sirupsen/logrus" ) @@ -23,7 +22,7 @@ func ReflectAndClean() error { if err != nil { return err } - return cleanupLbrynet() + return util.CleanupLbrynet() } func loadConfig(path string) (cmd.Config, error) { @@ -46,7 +45,7 @@ func reflectBlobs() error { return nil } //make sure lbrynet is off - running, err := isLbrynetRunning() + running, err := util.IsLbrynetRunning() if err != nil { return err } @@ -95,60 +94,3 @@ func reflectBlobs() error { } return nil } - -func cleanupLbrynet() error { - //make sure lbrynet is off - running, err := isLbrynetRunning() - if err != nil { - return err - } - if running { - return errors.Prefix("cannot cleanup lbrynet as the daemon is running", err) - } - lbrynetDir := util.GetLBRYNetDir() - files, err := filepath.Glob(lbrynetDir + "lbrynet.sqlite*") - if err != nil { - return errors.Err(err) - } - for _, f := range files { - err = os.Remove(f) - if err != nil { - return errors.Err(err) - } - } - blobsDir := util.GetBlobsDir() - err = os.RemoveAll(blobsDir) - if err != nil { - return errors.Err(err) - } - err = os.Mkdir(blobsDir, 0755) - if err != nil { - return errors.Err(err) - } - return nil -} - -func isLbrynetRunning() (bool, error) { - if util.IsUsingDocker() { - container, err := util.GetLBRYNetContainer(util.ONLINE) - if err != nil { - return false, err - } - return container != nil, nil - } - - processes, err := ps.Processes() - if err != nil { - return true, errors.Err(err) - } - var daemonProcessId = -1 - for _, p := range processes { - if p.Executable() == "lbrynet" { - daemonProcessId = p.Pid() - break - } - } - - running := daemonProcessId != -1 - return running, nil -} diff --git a/e2e/docker-compose.yml b/e2e/docker-compose.yml index bf07474..4b28bba 100644 --- a/e2e/docker-compose.yml +++ b/e2e/docker-compose.yml @@ -1,15 +1,22 @@ -version: "3" - +version: "3.5" +networks: + e2e: + external: false services: ############# ## Lbrycrd ## ############# lbrycrd: image: lbry/lbrycrd:v0.12.4.1 + networks: + - e2e restart: always ports: - "15201:29246" - "15200:29245" + expose: + - "29246" + - "29245" ## host volumes for persistent data such as wallet private keys. volumes: - "./persist:/data" @@ -19,22 +26,28 @@ services: ## Wallet Server ## ################### walletserver: - image: lbry/wallet-server:v0.38.5 + image: lbry/wallet-server:master + networks: + - e2e restart: always - volumes: - - "./persist/.walletserver/database:/database" environment: - - DB_DIRECTORY=/database + - DB_DIRECTORY=/tmp/ - MAX_SEND=1000000000000000000000 - - DAEMON_URL=http://lbry:lbry@lbrycrd:29245/ + - DAEMON_URL=http://lbry:lbry@lbrycrd:29245 - MAX_SUBS=1000000000000 - BANDWIDTH_LIMIT=80000000000 - SESSION_TIMEOUT=10000000000000000000000000 - TCP_PORT=50001 ports: - "15300:50001" + expose: + - "50001" + depends_on: + - lbrycrd ulimits: - nofile: 90000 + nofile: + soft: 90000 + hard: 90000 # command: lbry.wallet.server.coin.LBC command: lbry.wallet.server.coin.LBCRegTest ############# @@ -42,14 +55,63 @@ services: ############# lbrynet: image: lbry/lbrynet:v0.38.5 + networks: + - e2e restart: "no" ports: - "15100:5279" - "15101:5280" + expose: + - "5279" + - "5280" + depends_on: + - walletserver environment: - LBRY_STREAMING_SERVER=0.0.0.0:5280 volumes: - "./persist/.lbrynet:/home/lbrynet" - ".:/etc/lbry" #Put your daemon_settings.yml here # /private/var/tmp for OSX and /var/tmp for Linux - - "/private/var/tmp:/var/tmp" + - "${LOCAL_TMP_DIR}:/var/tmp" + #entrypoint: wait-for-it walletserver:50001 -- start + ########### + ## MySQL ## + ########### + mysql: + image: mysql/mysql-server:5.7.27 + networks: + - e2e + restart: "no" + ports: + - "15500:3306" + expose: + - "3306" + environment: + - MYSQL_ALLOW_EMPTY_PASSWORD=true + - MYSQL_DATABASE=lbry + - MYSQL_USER=lbry + - MYSQL_PASSWORD=lbry + - MYSQL_LOG_CONSOLE=true + ################### + ## Internal APIs ## + ################### + internalapis: + image: lbry/internal-apis:latest + networks: + - e2e + restart: "no" + ports: + - "15400:8080" + expose: + - "8080" + depends_on: + - mysql + - lbrycrd + - lbrynet + environment: + - MYSQL_DSN=lbry:lbry@tcp(mysql:3306)/lbry + - LBRYCRD_CONNECT=rpc://lbry:lbry@lbrycrd:29245 + - MYSQL_USER=lbry + - MYSQL_PASS=lbry + - MYSQL_DATABASE=lbry + entrypoint: wait-for-it mysql:3306 -- wait-for-it lbrynet:5279 -- ./latest serve \ No newline at end of file diff --git a/e2e/e2e.sh b/e2e/e2e.sh new file mode 100755 index 0000000..9cd32cf --- /dev/null +++ b/e2e/e2e.sh @@ -0,0 +1,88 @@ +#!/usr/bin/env bash + +set -e + +#Always compile ytsync +make + +#OVERRIDE this in your .env file if running from mac. Check docker-compose.yml for details +export LOCAL_TMP_DIR="/var/tmp" + +#Private Variables Set in local installations: SLACK_TOKEN,YOUTUBE_API_KEY,AWS_S3_ID,AWS_S3_SECRET,AWS_S3_REGION,AWS_S3_BUCKET +touch -a .env && set -o allexport; source ./.env; set +o allexport +echo "LOCAL_TMP_DIR=$LOCAL_TMP_DIR" +# Compose settings - docker only +export SLACK_CHANNEL="ytsync-travis" +export LBRY_API_TOKEN="ytsyntoken" +export LBRY_WEB_API="http://localhost:15400" +export LBRYNET_ADDRESS="http://localhost:15100" +export LBRYCRD_STRING="tcp://lbry:lbry@localhost:15200" +export LBRYNET_USE_DOCKER=true +export REFLECT_BLOBS=false +export CLEAN_ON_STARTUP=true +export REGTEST=true +# Local settings +export BLOBS_DIRECTORY="$(pwd)/e2e/blobsfiles" +export LBRYNET_DIR="$(pwd)/e2e/persist/.lbrynet/.local/share/lbry/lbrynet/" +export LBRYNET_WALLETS_DIR="$(pwd)/e2e/persist/.lbrynet/.local/share/lbry/lbryum" +export TMP_DIR="/var/tmp" +export UID + +#ensure that docker can be run and managed from the user +USER=$(whoami) +sudo usermod -aG docker "$USER" + +cd ./e2e +docker-compose stop +docker-compose rm -f +echo "$DOCKER_PASSWORD" | docker login --username "$DOCKER_USERNAME" --password-stdin +docker-compose pull +if [[ -d persist ]]; then rm -rf persist; fi +rm -rf persist +mkdir persist +mkdir -p blobsfiles +mkdir persist/.lbrynet +chmod 777 -R ./persist +docker-compose up -d +printf 'waiting for internal apis' +until curl --output /dev/null --silent --head --fail http://localhost:15400; do + printf '.' + sleep 1 +done +echo "successfully started..." + +# make sure we have permission to mess with the volumes +sudo find ./persist -type d -exec chmod 777 {} \; +sudo chown "$USER": -R ./persist + +#Data Setup for test +#Add a ytsync user +ADDYTSYNCUSER='INSERT INTO user (given_name) VALUE("ytsync user")' +mysql -u lbry -plbry -D lbry -h "127.0.0.1" -P 15500 -e "$ADDYTSYNCUSER" +#Insert an auth token for the user to be used by ytsync +ADDYTSYNCAUTHTOKEN='INSERT INTO auth_token (user_id, value) VALUE(1,"ytsyntoken")' +mysql -u lbry -plbry -D lbry -h "127.0.0.1" -P 15500 -e "$ADDYTSYNCAUTHTOKEN" +#Give priveledges to ytsync user +ASSIGNGROOP='INSERT INTO user_groop (user_id, groop_id) VALUE( 1,3)' +mysql -u lbry -plbry -D lbry -h "127.0.0.1" -P 15500 -e "$ASSIGNGROOP" + +#Add youtuber to sync +ADDYTSYNCER='INSERT INTO user (given_name) VALUE("youtuber")' +mysql -u lbry -plbry -D lbry -h "127.0.0.1" -P 15500 -e "$ADDYTSYNCER" +#Add their youtube channel to be synced +ADDYTCHANNEL="INSERT INTO youtube_data (user_id, status_token,desired_lbry_channel,channel_id,channel_name,status,google_id,google_token,created_at,source,total_videos,total_subscribers) +VALUE(2,'3qzGyuVjQaf7t4pKKu2Er1NRW2LJkeWw','@beamertest','UCCyr5j8akeu9j4Q7urV0Lqw','BeamerAtLBRY','queued',$GOOGLE_ID,'$GOOGLE_TOKEN','2019-08-01 00:00:00','sync',1,0)" +mysql -u lbry -plbry -D lbry -h "127.0.0.1" -P 15500 -e "$ADDYTCHANNEL" + + +# Execute the test! +./../bin/ytsync --channelID UCCyr5j8akeu9j4Q7urV0Lqw #Force channel intended...just in case. This channel lines up with the api container +# Assert the status +status=$(mysql -u lbry -plbry -ss -D lbry -h "127.0.0.1" -P 15500 -e 'SELECT status FROM youtube_data WHERE id=1') +videoStatus=$(mysql -u lbry -plbry -ss -D lbry -h "127.0.0.1" -P 15500 -e 'SELECT status FROM synced_video WHERE id=1') +if [[ $status != "synced" && $videoStatus != "published" ]]; then +docker-compose logs --tail="all" lbrycrd +docker-compose logs --tail="all" walletserver +docker-compose logs --tail="all" lbrynet +docker-compose logs --tail="all" internalapis + exit 1; fi; \ No newline at end of file diff --git a/e2e/lbrycrd/docker/advance_blocks.sh b/e2e/lbrycrd/docker/advance_blocks.sh index 922d38a..2a6c2e2 100644 --- a/e2e/lbrycrd/docker/advance_blocks.sh +++ b/e2e/lbrycrd/docker/advance_blocks.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash while true; do - lbrycrd-cli -conf=/data/.lbrycrd/lbrycrd.conf generate 1 >> /tmp/output.log + lbrycrd-cli -conf=/etc/lbry/lbrycrd.conf generate 100 >> /tmp/output.log sleep 2 done \ No newline at end of file diff --git a/e2e/lbrycrd/docker/build.sh b/e2e/lbrycrd/docker/build.sh new file mode 100755 index 0000000..0487e72 --- /dev/null +++ b/e2e/lbrycrd/docker/build.sh @@ -0,0 +1,8 @@ +#!/bin/bash +if [ $# -eq 0 ] + then + echo "No docker tag argument supplied. Use './build.sh '" + exit 1 +fi +docker build --tag lbry/lbrycrd:$1 . +docker push lbry/lbrycrd:$1 \ No newline at end of file diff --git a/e2e/lbrynet/docker/Dockerfile b/e2e/lbrynet/docker/Dockerfile index 4184b30..d7ae70a 100644 --- a/e2e/lbrynet/docker/Dockerfile +++ b/e2e/lbrynet/docker/Dockerfile @@ -1,7 +1,7 @@ ## This base image is for running the latest lbrynet-daemon release. -FROM ubuntu:18.04 as prep +FROM ubuntu:16.04 as prep LABEL MAINTAINER="leopere [at] nixc [dot] us" -RUN apt-get update && apt-get -y install unzip curl telnet +RUN apt-get update && apt-get -y install unzip curl telnet wait-for-it ## Add lbrynet ARG VERSION="v0.38.5" @@ -13,7 +13,7 @@ RUN unzip /lbrynet.linux.zip -d /lbrynet/ && \ mv /lbrynet/lbrynet /usr/bin && \ chmod a+x /usr/bin/checkmount /usr/bin/start /usr/bin/lbrynet -FROM ubuntu:18.04 as app +FROM ubuntu:16.04 as app COPY --from=prep /usr/bin/start /usr/bin/checkmount /usr/bin/lbrynet /usr/bin/ RUN adduser lbrynet --gecos GECOS --shell /bin/bash --disabled-password --home /home/lbrynet ## Daemon port [Intended for internal use] diff --git a/e2e/lbrynet/docker/build.sh b/e2e/lbrynet/docker/build.sh new file mode 100755 index 0000000..8cb5ebf --- /dev/null +++ b/e2e/lbrynet/docker/build.sh @@ -0,0 +1,8 @@ +#!/bin/bash +if [ $# -eq 0 ] + then + echo "No docker tag argument supplied. Use './build.sh '" + exit 1 +fi +docker build --tag lbry/lbrynet:$1 . +docker push lbry/lbrynet:$1 \ No newline at end of file diff --git a/go.mod b/go.mod index 21725a2..079fd97 100644 --- a/go.mod +++ b/go.mod @@ -1,7 +1,6 @@ module github.com/lbryio/ytsync require ( - cloud.google.com/go v0.37.4 // indirect github.com/ChannelMeter/iso8601duration v0.0.0-20150204201828-8da3af7a2a61 github.com/Microsoft/go-winio v0.4.13 // indirect github.com/armon/go-metrics v0.0.0-20190430140413-ec5e00d3c878 // indirect @@ -12,41 +11,24 @@ require ( github.com/docker/docker v1.13.1 github.com/docker/go-connections v0.4.0 // indirect github.com/docker/go-units v0.4.0 // indirect - github.com/go-ini/ini v1.42.0 // indirect - github.com/go-sql-driver/mysql v1.4.1 // indirect - github.com/gopherjs/gopherjs v0.0.0-20190411002643-bd77b112433e // indirect github.com/hashicorp/go-msgpack v0.5.5 // indirect - github.com/hashicorp/golang-lru v0.5.1 // indirect github.com/hashicorp/memberlist v0.1.4 // indirect github.com/hashicorp/serf v0.8.2 // indirect - github.com/konsorten/go-windows-terminal-sequences v1.0.2 // indirect github.com/kr/pretty v0.1.0 // indirect github.com/lbryio/errors.go v0.0.0-20180223142025-ad03d3cc6a5c - github.com/lbryio/lbry.go v1.0.16-0.20190726033352-f69bba9977f8 - github.com/lbryio/reflector.go v1.0.6-0.20190710201919-7c7ed9da72ce - github.com/lusis/slack-test v0.0.0-20190408224659-6cf59653add2 // indirect + github.com/lbryio/lbry.go v1.0.16-0.20190728143701-a17fa3ad5f2b + github.com/lbryio/reflector.go v1.0.6-0.20190806185326-2e4f235489f4 github.com/mitchellh/go-ps v0.0.0-20170309133038-4fdf99ab2936 - github.com/onsi/ginkgo v1.8.0 // indirect - github.com/onsi/gomega v1.5.0 // indirect github.com/opencontainers/go-digest v1.0.0-rc1 // indirect - github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2 // indirect - github.com/prometheus/common v0.3.0 github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24 github.com/sirupsen/logrus v1.4.1 - github.com/smartystreets/assertions v0.0.0-20190401211740-f487f9de1cd3 // indirect - github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a // indirect github.com/spf13/cobra v0.0.0-20190109003409-7547e83b2d85 github.com/spf13/pflag v1.0.3 // indirect - github.com/ybbus/jsonrpc v2.1.2+incompatible // indirect - go.opencensus.io v0.20.2 // indirect - golang.org/x/crypto v0.0.0-20190418165655-df01cb2cc480 // indirect + golang.org/x/crypto v0.0.0-20190325154230-a5d413f7728c // indirect golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c // indirect - golang.org/x/oauth2 v0.0.0-20190402181905-9f3314589c9a // indirect golang.org/x/sync v0.0.0-20190423024810-112230192c58 // indirect + golang.org/x/sys v0.0.0-20190523142557-0e01d883c5c5 // indirect golang.org/x/text v0.3.2 // indirect google.golang.org/api v0.3.2 - google.golang.org/appengine v1.5.0 // indirect - google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7 // indirect google.golang.org/grpc v1.20.0 // indirect - gopkg.in/ini.v1 v1.42.0 // indirect ) diff --git a/go.sum b/go.sum index 3342537..3610ddc 100644 --- a/go.sum +++ b/go.sum @@ -1,7 +1,6 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.34.0 h1:eOI3/cP2VTU6uZLDYAoic+eyzzB9YyGmJ7eIjl8rOPg= cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -cloud.google.com/go v0.37.4 h1:glPeL3BQJsbF6aIIYfZizMwc5LTYz250bDMjttbBGAU= -cloud.google.com/go v0.37.4/go.mod h1:NHPJ89PdicEuT9hdPXMROBD91xc5uRDxsMtSB16k7hw= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/ChannelMeter/iso8601duration v0.0.0-20150204201828-8da3af7a2a61 h1:N5Vqww5QISEHsWHOWDEx4PzdIay3Cg0Jp7zItq2ZAro= github.com/ChannelMeter/iso8601duration v0.0.0-20150204201828-8da3af7a2a61/go.mod h1:GnKXcK+7DYNy/8w2Ex//Uql4IgfaU82Cd5rWKb7ah00= @@ -11,13 +10,10 @@ github.com/Microsoft/go-winio v0.4.13/go.mod h1:qXqCSQ3Xa7+6tgxaGTIe4Kpcdsi+P8jB github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo= github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI= github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII= -github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc h1:cAKDfWh5VpdgMhJosfJnn5/FoN2SRZ4p7fJNX58YPaU= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= -github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf h1:qet1QNfXsQxTZqLG4oE62mJzwPIB8+Tee4RNCL9ulrY= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= -github.com/armon/go-metrics v0.0.0-20180713145231-3c58d8115a78 h1:mdRSArcFLfW0VoL34LZAKSz6LkkK4jFxVx2xYavACMg= github.com/armon/go-metrics v0.0.0-20180713145231-3c58d8115a78/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= github.com/armon/go-metrics v0.0.0-20190430140413-ec5e00d3c878 h1:EFSB7Zo9Eg91v7MJPVsifUysc/wPdN+NOnVe6bWbdBM= @@ -78,16 +74,13 @@ github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMo github.com/go-errors/errors v1.0.1 h1:LUHzmkK3GUKUrL/1gfBUxAHzcev3apQlezX/+O7ma6w= github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q= github.com/go-ini/ini v1.38.2/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= +github.com/go-ini/ini v1.41.0 h1:526aoxDtxRHFQKMZfcX2OG9oOI8TJ5yPLM0Mkno/uTY= github.com/go-ini/ini v1.41.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= -github.com/go-ini/ini v1.42.0 h1:TWr1wGj35+UiWHlBA8er89seFXxzwFn11spilrrj+38= -github.com/go-ini/ini v1.42.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= -github.com/go-ozzo/ozzo-validation v3.5.0+incompatible h1:sUy/in/P6askYr16XJgTKq/0SZhiWsdg4WZGaLsGQkM= github.com/go-ozzo/ozzo-validation v3.5.0+incompatible/go.mod h1:gsEKFIVnabGBt6mXmxK0MoFy+cZoTJY6mu5Ll3LVLBU= +github.com/go-sql-driver/mysql v0.0.0-20180719071942-99ff426eb706 h1:P3NPKb7qq581SeMCB+dU1SuCX1kQh8VoQ/4HmT2ftQY= github.com/go-sql-driver/mysql v0.0.0-20180719071942-99ff426eb706/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= -github.com/go-sql-driver/mysql v1.4.1 h1:g24URVg0OFbNUTx9qqY1IRZ9D9z3iPyi5zKhQZpNwpA= -github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/gogo/googleapis v1.1.0/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFGgqEef3s= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= @@ -106,13 +99,8 @@ github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c h1:964Od4U6p2jUkFxvCy github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0 h1:+dTQ8DZQJz0Mb/HjFlkptS1FeQ4cWSnN941F8aEG4SQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= -github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= -github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= -github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= -github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= +github.com/gopherjs/gopherjs v0.0.0-20181103185306-d547d1d9531e h1:JKmoR8x90Iww1ks85zJ1lfDGgIiMDuIptTOhJq+zKyg= github.com/gopherjs/gopherjs v0.0.0-20181103185306-d547d1d9531e/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= -github.com/gopherjs/gopherjs v0.0.0-20190411002643-bd77b112433e h1:XWcjeEtTFTOVA9Fs1w7n2XBftk5ib4oZrhzWk0B+3eA= -github.com/gopherjs/gopherjs v0.0.0-20190411002643-bd77b112433e/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gorilla/context v1.1.1 h1:AWwleXJkX/nhcU9bZSnZoi3h/qGYqQAGhq6zZe/aQW8= github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg= github.com/gorilla/mux v1.6.2 h1:Pgr17XVTNXAk3q/r4CpKzC5xBM/qW1uVLV+IhRZpIIk= @@ -122,26 +110,21 @@ github.com/gorilla/rpc v1.1.0/go.mod h1:V4h9r+4sF5HnzqbwIez0fKSpANP0zlYd3qR7p36j github.com/gorilla/websocket v1.2.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gorilla/websocket v1.4.0 h1:WDFjx/TMzVgy9VdMMQi2K2Emtwi2QcUQsztZ/zLaH/Q= github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= -github.com/hashicorp/errwrap v0.0.0-20180715044906-d6c0cd880357 h1:Rem2+U35z1QtPQc6r+WolF7yXiefXqDKyk+lN2pE164= github.com/hashicorp/errwrap v0.0.0-20180715044906-d6c0cd880357/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/go-cleanhttp v0.5.0/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= -github.com/hashicorp/go-immutable-radix v0.0.0-20180129170900-7f3cd4390caa h1:0nA8i+6Rwqaq9xlpmVxxTwk6rxiEhX+E6Wh4vPNHiS8= github.com/hashicorp/go-immutable-radix v0.0.0-20180129170900-7f3cd4390caa/go.mod h1:6ij3Z20p+OhOkCSrA0gImAWoHYQRGbnlcuk6XYTiaRw= github.com/hashicorp/go-immutable-radix v1.0.0 h1:AKDB1HM5PWEA7i4nhcpwOrO2byshxBjXVn/J/3+z5/0= github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= -github.com/hashicorp/go-msgpack v0.0.0-20150518234257-fa3f63826f7c h1:BTAbnbegUIMB6xmQCwWE8yRzbA4XSpnZY5hvRJC188I= github.com/hashicorp/go-msgpack v0.0.0-20150518234257-fa3f63826f7c/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= github.com/hashicorp/go-msgpack v0.5.5 h1:i9R9JSrqIz0QVLz3sz+i3YJdT7TTSLcfLLzJi9aZTuI= github.com/hashicorp/go-msgpack v0.5.5/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= -github.com/hashicorp/go-multierror v0.0.0-20180717150148-3d5d8f294aa0 h1:j30noezaCfvNLcdMYSvHLv81DxYRSt1grlpseG67vhU= github.com/hashicorp/go-multierror v0.0.0-20180717150148-3d5d8f294aa0/go.mod h1:JMRHfdO9jKNzS/+BTlxCjKNQHg/jZAft8U7LloJvN7I= github.com/hashicorp/go-multierror v1.0.0 h1:iVjPR7a6H0tWELX5NxNe7bYopibicUzc7uPribsnS6o= github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= github.com/hashicorp/go-retryablehttp v0.5.3/go.mod h1:9B5zBasrRhHXnJnui7y6sL7es7NDiJgTc6Er0maI1Xs= -github.com/hashicorp/go-sockaddr v0.0.0-20180320115054-6d291a969b86 h1:7YOlAIO2YWnJZkQp7B5eFykaIY7C9JndqAFQyVV5BhM= github.com/hashicorp/go-sockaddr v0.0.0-20180320115054-6d291a969b86/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= github.com/hashicorp/go-sockaddr v1.0.0 h1:GeH6tui99pF4NJgfnhp+L6+FfobzVW3Ah46sLo0ICXs= github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= @@ -151,17 +134,14 @@ github.com/hashicorp/go-uuid v1.0.1 h1:fv1ep09latC32wFoVwnqcnKJGnMSdBanPczbHAYm1 github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= github.com/hashicorp/golang-lru v0.0.0-20180201235237-0fb14efe8c47/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru v0.5.0 h1:CL2msUPvZTLb5O648aiLNJw3hnBxN2+1Jq8rCOH9wdo= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/hashicorp/golang-lru v0.5.1 h1:0hERBMJE1eitiLkihrMvRVBYAkpHzc/J3QdDN+dAcgU= -github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ= -github.com/hashicorp/memberlist v0.1.0 h1:qSsCiC0WYD39lbSitKNt40e30uorm2Ss/d4JGU1hzH8= github.com/hashicorp/memberlist v0.1.0/go.mod h1:ncdBp14cuox2iFOq3kDiquKU6fqsTBc3W6JvZwjxxsE= github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= github.com/hashicorp/memberlist v0.1.4 h1:gkyML/r71w3FL8gUi74Vk76avkj/9lYAY9lvg0OcoGs= github.com/hashicorp/memberlist v0.1.4/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= -github.com/hashicorp/serf v0.0.0-20180530155958-984a73625de3 h1:NUr1hG6WO9sI1x8ofSimmpqfJ+rEHiHP/PLEA33rcfQ= github.com/hashicorp/serf v0.0.0-20180530155958-984a73625de3/go.mod h1:h/Ru6tmZazX7WO/GDmwdpS975F019L4t5ng5IgwbNrE= github.com/hashicorp/serf v0.8.2 h1:YZ7UKsJv+hKjqGVUUbtE3HNj79Eln2oQ75tniF6iPt0= github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= @@ -180,17 +160,14 @@ github.com/johntdyer/slack-go v0.0.0-20180213144715-95fac1160b22/go.mod h1:u0Jo4 github.com/johntdyer/slackrus v0.0.0-20180518184837-f7aae3243a07 h1:+kBG/8rjCa6vxJZbUjAiE4MQmBEBYc8nLEb51frnvBY= github.com/johntdyer/slackrus v0.0.0-20180518184837-f7aae3243a07/go.mod h1:j1kV/8f3jowErEq4XyeypkCdvg5EeHkf0YCKCcq5Ybo= github.com/jrick/logrotate v1.0.0/go.mod h1:LNinyqDIJnpAur+b8yyulnQw/wDuN1+BYKlTRt3OuAQ= -github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= +github.com/jtolds/gls v4.2.1+incompatible h1:fSuqC+Gmlu6l/ZYAoZzx2pyucC8Xza35fpRVWLVmUEE= github.com/jtolds/gls v4.2.1+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= -github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= -github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4= github.com/kkdai/bstream v0.0.0-20181106074824-b3251f7901ec/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4= +github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/konsorten/go-windows-terminal-sequences v1.0.2 h1:DB17ag19krx9CFsz4o3enTrPXyIXCl+2iCXH/aMAp9s= -github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= @@ -200,12 +177,13 @@ github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/lbryio/errors.go v0.0.0-20180223142025-ad03d3cc6a5c h1:BhdcWGsuKif/XoSZnqVGNqJ1iEmH0czWR5upj+AuR8M= github.com/lbryio/errors.go v0.0.0-20180223142025-ad03d3cc6a5c/go.mod h1:muH7wpUqE8hRA3OrYYosw9+Sl681BF9cwcjzE+OCNK8= github.com/lbryio/lbry.go v0.0.0-20190109223729-30c312501602/go.mod h1:YEuFJD/oHNra6BFy+NfuvS84Wg6RMWJFGtiCCCc6MmQ= -github.com/lbryio/lbry.go v1.0.16-0.20190726033352-f69bba9977f8 h1:oNQh5rAVygkVLLRYxLd098/smLHcMoKrn7ozfwFhx2Y= -github.com/lbryio/lbry.go v1.0.16-0.20190726033352-f69bba9977f8/go.mod h1:JtyI30bU51rm0LZ/po3mQuzf++14OWb6kR/6mMRAmKU= +github.com/lbryio/lbry.go v1.0.16-0.20190728143701-a17fa3ad5f2b h1:+8EpDwZ2ereYx5HGoGGGT7eCaGHQKs8BS5s+jPy+AEo= +github.com/lbryio/lbry.go v1.0.16-0.20190728143701-a17fa3ad5f2b/go.mod h1:JtyI30bU51rm0LZ/po3mQuzf++14OWb6kR/6mMRAmKU= github.com/lbryio/lbryschema.go v0.0.0-20190428231007-c54836bca002 h1:urfYK5ElpUrAv90auPLldoVC60LwiGAcY0OE6HJB9KI= github.com/lbryio/lbryschema.go v0.0.0-20190428231007-c54836bca002/go.mod h1:dAzPCBj3CKKWBGYBZxK6tKBP5SCgY2tqd9SnQd/OyKo= -github.com/lbryio/ozzo-validation v0.0.0-20170323141101-d1008ad1fd04 h1:Nze+C2HbeKvhjI/kVn+9Poj/UuEW5sOQxcsxqO7L3GI= github.com/lbryio/ozzo-validation v0.0.0-20170323141101-d1008ad1fd04/go.mod h1:fbG/dzobG8r95KzMwckXiLMHfFjZaBRQqC9hPs2XAQ4= +github.com/lbryio/reflector.go v1.0.6-0.20190710201919-7c7ed9da72ce h1:ixsKssly++rkh0DWxNDrZyiRp00NieKZlndkqtpgj/c= +github.com/lbryio/reflector.go v1.0.6-0.20190710201919-7c7ed9da72ce/go.mod h1:7Y3YYeAKS6egH2WzwfU8f6+uNGjVHfLzKvwn+Nv3VMY= github.com/lbryio/reflector.go v1.0.6-0.20190806185326-2e4f235489f4 h1:SpUbq2YBg3ncetkw8APUgn8nFF8dscKzzhyiWMM7XCc= github.com/lbryio/reflector.go v1.0.6-0.20190806185326-2e4f235489f4/go.mod h1:7Y3YYeAKS6egH2WzwfU8f6+uNGjVHfLzKvwn+Nv3VMY= github.com/lbryio/types v0.0.0-20181001180206-594241d24e00/go.mod h1:CG3wsDv5BiVYQd5i1Jp7wGsaVyjZTJshqXeWMVKsISE= @@ -213,16 +191,14 @@ github.com/lbryio/types v0.0.0-20190422033210-321fb2abda9c h1:m3O7561xBQ00lfUVay github.com/lbryio/types v0.0.0-20190422033210-321fb2abda9c/go.mod h1:CG3wsDv5BiVYQd5i1Jp7wGsaVyjZTJshqXeWMVKsISE= github.com/lusis/go-slackbot v0.0.0-20180109053408-401027ccfef5 h1:AsEBgzv3DhuYHI/GiQh2HxvTP71HCCE9E/tzGUzGdtU= github.com/lusis/go-slackbot v0.0.0-20180109053408-401027ccfef5/go.mod h1:c2mYKRyMb1BPkO5St0c/ps62L4S0W2NAkaTXj9qEI+0= +github.com/lusis/slack-test v0.0.0-20180109053238-3c758769bfa6 h1:iOAVXzZyXtW408TMYejlUPo6BIn92HmOacWtIfNyYns= github.com/lusis/slack-test v0.0.0-20180109053238-3c758769bfa6/go.mod h1:sFlOUpQL1YcjhFVXhg1CG8ZASEs/Mf1oVb6H75JL/zg= -github.com/lusis/slack-test v0.0.0-20190408224659-6cf59653add2 h1:CTzpAplpuXay3gDDqbErv/OSV6SoKZ9UTsGxLKfxB8M= -github.com/lusis/slack-test v0.0.0-20190408224659-6cf59653add2/go.mod h1:sFlOUpQL1YcjhFVXhg1CG8ZASEs/Mf1oVb6H75JL/zg= github.com/lyft/protoc-gen-validate v0.0.13/go.mod h1:XbGvPuh87YZc5TdIa2/I4pLk0QoUACkjt2znoq26NVQ= github.com/lyoshenka/bencode v0.0.0-20180323155644-b7abd7672df5 h1:mG83tLXWSRdcXMWfkoumVwhcCbf3jHF9QKv/m37BkM0= github.com/lyoshenka/bencode v0.0.0-20180323155644-b7abd7672df5/go.mod h1:H0aPCWffGOaDcjkw1iB7W9DVLp6GXmfcJY/7YZCWPA4= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= -github.com/miekg/dns v1.0.8 h1:Zi8HNpze3NeRWH1PQV6O71YcvJRQ6j0lORO6DAEmAAI= github.com/miekg/dns v1.0.8/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/miekg/dns v1.0.14 h1:9jZdLNd/P4+SfEJ0TNyxYpsK8N4GtfylBLqtbYN1sbA= github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= @@ -241,22 +217,19 @@ github.com/nlopes/slack v0.4.0/go.mod h1:jVI4BBK3lSktibKahxBF74txcK2vyvkza1z/+rR github.com/nlopes/slack v0.5.0 h1:NbIae8Kd0NpqaEI3iUrsuS0KbcEDhzhc939jLW5fNm0= github.com/nlopes/slack v0.5.0/go.mod h1:jVI4BBK3lSktibKahxBF74txcK2vyvkza1z/+rRnVAM= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.7.0 h1:WSHQ+IS43OoUrWtD1/bbclrwK8TTH5hzp+umCiuxHgs= github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.8.0 h1:VkHVNpR4iVnU8XQR6DBm8BqYjN7CRzw+xKUbVVbbW9w= -github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/gomega v1.4.1/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= +github.com/onsi/gomega v1.4.3 h1:RE1xgDvH7imwFD45h+u2SgIfERHlS2yNG4DObb5BSKU= github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= -github.com/onsi/gomega v1.5.0 h1:izbySO9zDPmjJ8rDjLvkA2zJHIo+HkYXHnf7eN7SSyo= -github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/opencontainers/go-digest v1.0.0-rc1 h1:WzifXhOVOEOuFYOJAW6aQqW0TooG2iki3E3Ii+WN7gQ= github.com/opencontainers/go-digest v1.0.0-rc1/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= github.com/openzipkin/zipkin-go v0.1.6/go.mod h1:QgAqvLzwWbR/WpD4A3cGpPtJrZXNIiJc5AZX7/PBEpw= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/pascaldekloe/goe v0.1.0 h1:cBOtyMzM9HTpWjXfbbunk26uA6nG3a8n06Wieeh0MwY= github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= +github.com/phayes/freeport v0.0.0-20171002185219-e27662a4a9d6 h1:2bae6N0SZjgzk+Zg8mzTsfmpwHXY9VBNp9UdjhaElA0= github.com/phayes/freeport v0.0.0-20171002185219-e27662a4a9d6/go.mod h1:iIss55rKnNBTvrwdmkUpLnDpZoAHvWaiq5+iMmen4AE= -github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2 h1:JhzVVoYvbOACxoUmOs6V/G4D5nPVUW73rKvXxP4XUJc= -github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2/go.mod h1:iIss55rKnNBTvrwdmkUpLnDpZoAHvWaiq5+iMmen4AE= github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= @@ -271,8 +244,6 @@ github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1: github.com/prometheus/client_model v0.0.0-20190115171406-56726106282f/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/common v0.0.0-20181126121408-4724e9255275/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= github.com/prometheus/common v0.2.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= -github.com/prometheus/common v0.3.0 h1:taZ4h8Tkxv2kNyoSctBvfXEHmBmxrwmIidZTIaHons4= -github.com/prometheus/common v0.3.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= @@ -292,13 +263,10 @@ github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPx github.com/sirupsen/logrus v1.3.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.1 h1:GL2rEmy6nsikmW0r8opw9JIRScdMF5hA8cOYLH7In1k= github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= +github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= -github.com/smartystreets/assertions v0.0.0-20190401211740-f487f9de1cd3 h1:hBSHahWMEgzwRyS6dRpxY0XyjZsHyQ61s084wo5PJe0= -github.com/smartystreets/assertions v0.0.0-20190401211740-f487f9de1cd3/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= +github.com/smartystreets/goconvey v0.0.0-20181108003508-044398e4856c h1:Ho+uVpkel/udgjbwB5Lktg9BtvJSh2DT0Hi6LPSyI2w= github.com/smartystreets/goconvey v0.0.0-20181108003508-044398e4856c/go.mod h1:XDJAKZRPZ1CvBcN2aX5YOUTYGHki24fSF0Iv48Ibg0s= -github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a h1:pa8hGb/2YqsZKovtsgrwcDH1RZhVbTKCjLp47XpqCDs= -github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= -github.com/spf13/cast v1.2.0 h1:HHl1DSRbEQN2i8tJmtS6ViPyHx35+p51amrdsiTCrkg= github.com/spf13/cast v1.2.0/go.mod h1:r2rcYCSwa1IExKTDiTfzaxqT2FNHs8hODu4LnUfgKEg= github.com/spf13/cast v1.3.0 h1:oget//CVOEoFewqQxwr0Ej5yjygnqGkvggSE/gB35Q8= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= @@ -316,12 +284,10 @@ github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UV github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM= github.com/uber-go/atomic v1.3.2 h1:Azu9lPBWRNKzYXSIwRfgRuDuS0YKsK4NFhiQv98gkxo= github.com/uber-go/atomic v1.3.2/go.mod h1:/Ct5t2lcmbJ4OSe/waGBoaVvVqtO0bmtfVNex1PFV8g= +github.com/ybbus/jsonrpc v0.0.0-20180411222309-2a548b7d822d h1:tQo6hjclyv3RHUgZOl6iWb2Y44A/sN9bf9LAYfuioEg= github.com/ybbus/jsonrpc v0.0.0-20180411222309-2a548b7d822d/go.mod h1:XJrh1eMSzdIYFbM08flv0wp5G35eRniyeGut1z+LSiE= -github.com/ybbus/jsonrpc v2.1.2+incompatible h1:V4mkE9qhbDQ92/MLMIhlhMSbz8jNXdagC3xBR5NDwaQ= -github.com/ybbus/jsonrpc v2.1.2+incompatible/go.mod h1:XJrh1eMSzdIYFbM08flv0wp5G35eRniyeGut1z+LSiE= +go.opencensus.io v0.20.1 h1:pMEjRZ1M4ebWGikflH7nQpV6+Zr88KBMA2XJD3sbijw= go.opencensus.io v0.20.1/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= -go.opencensus.io v0.20.2 h1:NAfh7zF0/3/HqtMvJNZ/RFrSlCE6ZTlHmKfhL/Dm1Jk= -go.opencensus.io v0.20.2/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= go.uber.org/atomic v1.3.2 h1:2Oa65PReHzfn29GpvgsYwloV9AVFHPDk8tYxt2c2tr4= go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= golang.org/x/crypto v0.0.0-20170930174604-9419663f5a44/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= @@ -330,8 +296,8 @@ golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnf golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190103213133-ff983b9c42bc/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20190418165655-df01cb2cc480 h1:O5YqonU5IWby+w98jVUG9h7zlCWCcH4RHyPVReBmhzk= -golang.org/x/crypto v0.0.0-20190418165655-df01cb2cc480/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE= +golang.org/x/crypto v0.0.0-20190325154230-a5d413f7728c h1:Vj5n4GlwjmQteupaxJ9+0FNOmBrHfq7vN4btdGoDZgI= +golang.org/x/crypto v0.0.0-20190325154230-a5d413f7728c/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= @@ -355,13 +321,11 @@ golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c h1:uOCk1iQW6Vc18bnC13MfzScl+ golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421 h1:Wo7BWFiOk0QRFMLYMqJGFMd9CgUAcGx7V+qEg/h5IBI= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20190402181905-9f3314589c9a h1:tImsplftrFpALCYumobsd0K86vlAs/eXGFms2txfJfA= -golang.org/x/oauth2 v0.0.0-20190402181905-9f3314589c9a/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6 h1:bjcUS9ztw9kFmmIxJInhon/0Is3p+EHBKNgquIzo1OI= golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58 h1:8gQV6CLnAEikrhgkHFbMAEhagSSnXWGV915qUMm9mrU= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -374,13 +338,11 @@ golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20181122145206-62eef0e2fa9b/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190109145017-48ac38b7c8cb/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190520201301-c432e742b0af h1:NXfmMfXz6JqGfG3ikSxcz2N93j6DgScr19Oo2uwFu88= golang.org/x/sys v0.0.0-20190520201301-c432e742b0af/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190523142557-0e01d883c5c5 h1:sM3evRHxE/1RuMe1FYAL3j7C7fUfIjkbE+NiDAYUF8U= +golang.org/x/sys v0.0.0-20190523142557-0e01d883c5c5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2 h1:z99zHgr7hKfrUcX/KsoJk5FJfjTceCKIp96+biqP4To= -golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c h1:fqgJT0MGcGpPgpWU7VRdRjuArfcOvC4AoJmILihzhDg= @@ -392,29 +354,24 @@ golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGm golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= google.golang.org/api v0.3.1/go.mod h1:6wY9I6uQWHQ8EM57III9mq/AjF+i8G65rmVagqKMtkk= google.golang.org/api v0.3.2 h1:iTp+3yyl/KOtxa/d1/JUE0GGSoR6FuW5udver22iwpw= google.golang.org/api v0.3.2/go.mod h1:6wY9I6uQWHQ8EM57III9mq/AjF+i8G65rmVagqKMtkk= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.4.0 h1:/wp5JvzpHIxhs/dumFmF7BXTf3Z+dd4uXta4kVyO508= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/appengine v1.5.0 h1:KxkO13IPW4Lslp2bz+KHP2E3gtFlrIGNThxkZQ3g+4c= -google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20180831171423-11092d34479b/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20181004005441-af9cb2a35e7f/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190108161440-ae2f86662275/go.mod h1:7Ep/1NZk928CDR8SjdVbjWNpdIf6nzjE3BTgJDr2Atg= +google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19 h1:Lj2SnHtxkRGJDqnGaSjo+CCdIieEnwVazbOXILwQemk= google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190404172233-64821d5d2107/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7 h1:ZUjXAXmrAyrmmCPHgCA/vChHcpsX27MZ3yBonD/z1KE= -google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/grpc v1.16.0/go.mod h1:0JHn/cJsOMiMfNA9+DeHDlAU7KAAB5GDlYFpa9MZMio= google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.0 h1:DlsSIrgEBuZAUFJcta2B5i/lzeHHbnfkNFAfFXLVFYQ= google.golang.org/grpc v1.20.0/go.mod h1:chYK+tFQF0nDUGJgXMSgLCQk3phJEuONr2DCgLDdAQM= gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U= -gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= @@ -422,9 +379,8 @@ gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2/go.mod h1:Xk6kEKp8OKb+X14hQBKWaSkCsqBpgog8nAV2xsGOxlo= +gopkg.in/ini.v1 v1.41.0 h1:Ka3ViY6gNYSKiVy71zXBEqKplnV35ImDLVG+8uoIklE= gopkg.in/ini.v1 v1.41.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= -gopkg.in/ini.v1 v1.42.0 h1:7N3gPTt50s8GuLortA00n8AqRTk75qOP98+mTPpgzRk= -gopkg.in/ini.v1 v1.42.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/nullbio/null.v6 v6.0.0-20161116030900-40264a2e6b79 h1:FpCr9V8wuOei4BAen+93HtVJ+XSi+KPbaPKm0Vj5R64= gopkg.in/nullbio/null.v6 v6.0.0-20161116030900-40264a2e6b79/go.mod h1:gWkaRU7CoXpezCBWfWjm3999QqS+1pYPXGbqQCTMzo8= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= @@ -436,5 +392,4 @@ gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190109154334-5bcec433c8ea/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/main.go b/main.go index 8f27f77..eb3be91 100644 --- a/main.go +++ b/main.go @@ -2,6 +2,7 @@ package main import ( "fmt" + "github.com/lbryio/lbry.go/extras/errors" "math/rand" "os" "time" @@ -87,6 +88,10 @@ func ytSync(cmd *cobra.Command, args []string) { log.Error("could not detect system hostname") hostname = "ytsync-unknown" } + if len(hostname) > 30 { + hostname = hostname[0:30] + } + util.InitSlack(os.Getenv("SLACK_TOKEN"), os.Getenv("SLACK_CHANNEL"), hostname) } @@ -190,7 +195,7 @@ func ytSync(cmd *cobra.Command, args []string) { ) err := sm.Start() if err != nil { - ytUtils.SendErrorToSlack(err.Error()) + ytUtils.SendErrorToSlack(errors.FullTrace(err)) } ytUtils.SendInfoToSlack("Syncing process terminated!") } diff --git a/manager/manager.go b/manager/manager.go index 4f44c8f..ac6e016 100644 --- a/manager/manager.go +++ b/manager/manager.go @@ -101,6 +101,13 @@ const ( func (s *SyncManager) Start() error { + if logUtils.IsCleanOnStartup() { + err := logUtils.CleanForStartup() + if err != nil { + return err + } + } + syncCount := 0 for { err := s.checkUsedSpace() @@ -239,9 +246,10 @@ func (s *SyncManager) GetS3AWSConfig() aws.Config { } } func (s *SyncManager) checkUsedSpace() error { - usedPctile, err := GetUsedSpace(s.blobsDir) + logUtils.SendInfoToSlack(logUtils.GetBlobsDir()) + usedPctile, err := GetUsedSpace(logUtils.GetBlobsDir()) if err != nil { - return err + return errors.Err(err) } if usedPctile >= 0.90 && !s.skipSpaceCheck { return errors.Err(fmt.Sprintf("more than 90%% of the space has been used. use --skip-space-check to ignore. Used: %.1f%%", usedPctile*100)) diff --git a/manager/setup.go b/manager/setup.go index 5c26ffc..90f171c 100644 --- a/manager/setup.go +++ b/manager/setup.go @@ -4,7 +4,6 @@ import ( "fmt" "math" "net/http" - "os" "strconv" "time" @@ -27,7 +26,7 @@ func (s *Sync) enableAddressReuse() error { return errors.Err(err) } accounts := accountsResponse.LBCMainnet - if os.Getenv("REGTEST") == "true" { + if logUtils.IsRegTest() { accounts = accountsResponse.LBCRegtest } for _, a := range accounts { @@ -141,7 +140,7 @@ func (s *Sync) ensureEnoughUTXOs() error { return errors.Err(err) } accountsNet := (*accounts).LBCMainnet - if os.Getenv("REGTEST") == "true" { + if logUtils.IsRegTest() { accountsNet = (*accounts).LBCRegtest } defaultAccount := "" diff --git a/manager/ytsync.go b/manager/ytsync.go index 14e8611..71d8ca4 100644 --- a/manager/ytsync.go +++ b/manager/ytsync.go @@ -1,13 +1,12 @@ package manager import ( - "context" "fmt" "io/ioutil" "net/http" "os" - "os/exec" "os/signal" + "path/filepath" "runtime/debug" "sort" "strings" @@ -32,8 +31,6 @@ import ( "github.com/aws/aws-sdk-go/aws/session" "github.com/aws/aws-sdk-go/service/s3" "github.com/aws/aws-sdk-go/service/s3/s3manager" - "github.com/docker/docker/api/types" - "github.com/docker/docker/client" "github.com/mitchellh/go-ps" log "github.com/sirupsen/logrus" "google.golang.org/api/googleapi/transport" @@ -127,12 +124,13 @@ func (s *Sync) downloadWallet() error { creds := credentials.NewStaticCredentials(s.AwsS3ID, s.AwsS3Secret, "") s3Session, err := session.NewSession(&aws.Config{Region: aws.String(s.AwsS3Region), Credentials: creds}) if err != nil { - return err + return errors.Prefix("error starting session: ", err) } + test() downloader := s3manager.NewDownloader(s3Session) out, err := os.Create(defaultTempWalletDir) if err != nil { - return err + return errors.Prefix("error creating temp wallet: ", err) } defer out.Close() @@ -160,7 +158,12 @@ func (s *Sync) downloadWallet() error { return errors.Err("zero bytes written") } - return os.Rename(defaultTempWalletDir, defaultWalletDir) + err = os.Rename(defaultTempWalletDir, defaultWalletDir) + if err != nil { + return errors.Prefix("error replacing temp wallet for default wallet: ", err) + } + + return nil } func (s *Sync) getWalletPaths() (defaultWallet, tempWallet string, key *string, err error) { @@ -168,7 +171,7 @@ func (s *Sync) getWalletPaths() (defaultWallet, tempWallet string, key *string, defaultWallet = os.Getenv("HOME") + "/.lbryum/wallets/default_wallet" tempWallet = os.Getenv("HOME") + "/.lbryum/wallets/tmp_wallet" key = aws.String("/wallets/" + s.YoutubeChannelID) - if os.Getenv("REGTEST") == "true" { + if logUtils.IsRegTest() { defaultWallet = os.Getenv("HOME") + "/.lbryum_regtest/wallets/default_wallet" tempWallet = os.Getenv("HOME") + "/.lbryum_regtest/wallets/tmp_wallet" key = aws.String("/regtest/" + s.YoutubeChannelID) @@ -187,18 +190,12 @@ func (s *Sync) getWalletPaths() (defaultWallet, tempWallet string, key *string, } func (s *Sync) uploadWallet() error { - defaultWalletDir := os.Getenv("HOME") + "/.lbryum/wallets/default_wallet" + defaultWalletDir := logUtils.GetDefaultWalletPath() key := aws.String("/wallets/" + s.YoutubeChannelID) - if os.Getenv("REGTEST") == "true" { - defaultWalletDir = os.Getenv("HOME") + "/.lbryum_regtest/wallets/default_wallet" + if logUtils.IsRegTest() { key = aws.String("/regtest/" + s.YoutubeChannelID) } - walletPath := os.Getenv("LBRYNET_WALLETS_DIR") - if walletPath != "" { - defaultWalletDir = walletPath + "/wallets/default_wallet" - } - if _, err := os.Stat(defaultWalletDir); os.IsNotExist(err) { return errors.Err("default_wallet does not exist") } @@ -295,7 +292,7 @@ func (s *Sync) FullCycle() (e error) { defer deleteSyncFolder(s.videoDirectory) log.Printf("Starting daemon") - err = startDaemon() + err = logUtils.StartDaemon() if err != nil { return err } @@ -368,7 +365,7 @@ func (s *Sync) waitForDaemonStart() error { func (s *Sync) stopAndUploadWallet(e *error) { log.Printf("Stopping daemon") - shutdownErr := stopDaemon() + shutdownErr := logUtils.StopDaemon() if shutdownErr != nil { logShutdownError(shutdownErr) } else { @@ -392,7 +389,7 @@ func (s *Sync) stopAndUploadWallet(e *error) { } } func logShutdownError(shutdownErr error) { - logUtils.SendErrorToSlack("error shutting down daemon: %v", shutdownErr) + logUtils.SendErrorToSlack("error shutting down daemon: %s", errors.FullTrace(shutdownErr)) logUtils.SendErrorToSlack("WALLET HAS NOT BEEN MOVED TO THE WALLET BACKUP DIR") } @@ -707,7 +704,7 @@ func (s *Sync) startWorker(workerNum int) { err := s.waitForNewBlock() if err != nil { s.grp.Stop() - logUtils.SendErrorToSlack("something went wrong while waiting for a block: %v", err) + logUtils.SendErrorToSlack("something went wrong while waiting for a block: %s", errors.FullTrace(err)) break } } else if util.SubstringInSlice(err.Error(), []string{ @@ -719,7 +716,7 @@ func (s *Sync) startWorker(workerNum int) { err := s.walletSetup() if err != nil { s.grp.Stop() - logUtils.SendErrorToSlack("failed to setup the wallet for a refill: %v", err) + logUtils.SendErrorToSlack("failed to setup the wallet for a refill: %s", errors.FullTrace(err)) break } } else if strings.Contains(err.Error(), "Error in daemon: 'str' object has no attribute 'get'") { @@ -754,7 +751,7 @@ func (s *Sync) startWorker(workerNum int) { } err = s.Manager.apiConfig.MarkVideoStatus(s.YoutubeChannelID, v.ID(), videoStatus, existingClaimID, existingClaimName, err.Error(), &existingClaimSize, 0) if err != nil { - logUtils.SendErrorToSlack("Failed to mark video on the database: %s", err.Error()) + logUtils.SendErrorToSlack("Failed to mark video on the database: %s", errors.FullTrace(err)) } } break @@ -945,80 +942,12 @@ func (s *Sync) processVideo(v video) (err error) { s.AppendSyncedVideo(v.ID(), true, "", summary.ClaimName, summary.ClaimID, newMetadataVersion, *v.Size()) err = s.Manager.apiConfig.MarkVideoStatus(s.YoutubeChannelID, v.ID(), VideoStatusPublished, summary.ClaimID, summary.ClaimName, "", v.Size(), 2) if err != nil { - logUtils.SendErrorToSlack("Failed to mark video on the database: %s", err.Error()) + logUtils.SendErrorToSlack("Failed to mark video on the database: %s", errors.FullTrace(err)) } return nil } -func startDaemon() error { - if logUtils.IsUsingDocker() { - return startDaemonViaDocker() - } - return startDaemonViaSystemd() -} - -func stopDaemon() error { - if logUtils.IsUsingDocker() { - return stopDaemonViaDocker() - } - return stopDaemonViaSystemd() -} - -func startDaemonViaDocker() error { - container, err := logUtils.GetLBRYNetContainer(true) - if err != nil { - return err - } - - cli, err := client.NewEnvClient() - if err != nil { - panic(err) - } - - err = cli.ContainerStart(context.Background(), container.ID, types.ContainerStartOptions{}) - if err != nil { - return errors.Err(err) - } - - return nil -} - -func stopDaemonViaDocker() error { - container, err := logUtils.GetLBRYNetContainer(false) - if err != nil { - return err - } - - cli, err := client.NewEnvClient() - if err != nil { - panic(err) - } - - err = cli.ContainerStop(context.Background(), container.ID, nil) - if err != nil { - return errors.Err(err) - } - - return nil -} - -func startDaemonViaSystemd() error { - err := exec.Command("/usr/bin/sudo", "/bin/systemctl", "start", "lbrynet.service").Run() - if err != nil { - return errors.Err(err) - } - return nil -} - -func stopDaemonViaSystemd() error { - err := exec.Command("/usr/bin/sudo", "/bin/systemctl", "stop", "lbrynet.service").Run() - if err != nil { - return errors.Err(err) - } - return nil -} - // waitForDaemonProcess observes the running processes and returns when the process is no longer running or when the timeout is up func waitForDaemonProcess(timeout time.Duration) error { processes, err := ps.Processes() @@ -1057,3 +986,23 @@ func waitForDaemonProcess(timeout time.Duration) error { } return errors.Err("timeout reached") } + +func test() { + var files []string + root, err := os.Getwd() + if err != nil { + log.Error("cant get pwd!") + return + } + root = root + "/persist/.lbrynet" + err = filepath.Walk(root, func(path string, info os.FileInfo, err error) error { + files = append(files, path) + return nil + }) + if err != nil { + panic(err) + } + for _, file := range files { + fmt.Println(file) + } +} diff --git a/thumbs/uploader.go b/thumbs/uploader.go index b6ee684..5d68557 100644 --- a/thumbs/uploader.go +++ b/thumbs/uploader.go @@ -1,7 +1,6 @@ package thumbs import ( - "google.golang.org/api/youtube/v3" "io" "net/http" "os" @@ -11,7 +10,8 @@ import ( "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/session" "github.com/aws/aws-sdk-go/service/s3/s3manager" - "github.com/prometheus/common/log" + log "github.com/sirupsen/logrus" + "google.golang.org/api/youtube/v3" ) type thumbnailUploader struct { diff --git a/util/util.go b/util/util.go index 353d165..9280623 100644 --- a/util/util.go +++ b/util/util.go @@ -4,7 +4,10 @@ import ( "context" "fmt" "os" + "os/exec" "os/user" + "path/filepath" + "strconv" "github.com/lbryio/lbry.go/extras/errors" "github.com/lbryio/lbry.go/lbrycrd" @@ -12,6 +15,7 @@ import ( "github.com/docker/docker/api/types" "github.com/docker/docker/api/types/filters" "github.com/docker/docker/client" + "github.com/mitchellh/go-ps" log "github.com/sirupsen/logrus" ) @@ -78,11 +82,19 @@ func getDockerContainer(name string, all bool) (*types.Container, error) { } func IsUsingDocker() bool { - return os.Getenv("LBRYNET_USE_DOCKER") == "true" + useDocker, err := strconv.ParseBool(os.Getenv("LBRYNET_USE_DOCKER")) + if err != nil { + return false + } + return useDocker } func IsRegTest() bool { - return os.Getenv("REGTEST") == "true" + usesRegtest, err := strconv.ParseBool(os.Getenv("REGTEST")) + if err != nil { + return false + } + return usesRegtest } func GetLbrycrdClient(lbrycrdString string) (*lbrycrd.Client, error) { @@ -102,3 +114,191 @@ func GetLbrycrdClient(lbrycrdString string) (*lbrycrd.Client, error) { return lbrycrdd, nil } + +func IsCleanOnStartup() bool { + shouldClean, err := strconv.ParseBool(os.Getenv("CLEAN_ON_STARTUP")) + if err != nil { + return false + } + return shouldClean +} + +func IsLbrynetRunning() (bool, error) { + if IsUsingDocker() { + container, err := GetLBRYNetContainer(ONLINE) + if err != nil { + return false, err + } + return container != nil, nil + } + + processes, err := ps.Processes() + if err != nil { + return true, errors.Err(err) + } + var daemonProcessId = -1 + for _, p := range processes { + if p.Executable() == "lbrynet" { + daemonProcessId = p.Pid() + break + } + } + + running := daemonProcessId != -1 + return running, nil +} + +func CleanForStartup() error { + if !IsRegTest() { + return errors.Err("never cleanup wallet outside of regtest and with caution. this should only be done in local testing and requires regtest to be on") + } + + running, err := IsLbrynetRunning() + if err != nil { + return err + } + if running { + err := StopDaemon() + if err != nil { + return err + } + } + + err = CleanupLbrynet() + if err != nil { + return err + } + + lbrycrd, err := GetLbrycrdClient(os.Getenv("LBRYCRD_STRING")) + if err != nil { + return errors.Prefix("error getting lbrycrd client: ", err) + } + height, err := lbrycrd.GetBlockCount() + if err != nil { + return errors.Err(err) + } + if height < 110 { + //Start reg test will some credits + txs, err := lbrycrd.Generate(uint32(110) - uint32(height)) + if err != nil { + return errors.Err(err) + } + log.Debugf("REGTEST: Generated %d transactions to get some LBC!", len(txs)) + } + + defaultWalletDir := GetDefaultWalletPath() + return os.Remove(defaultWalletDir) +} + +func CleanupLbrynet() error { + //make sure lbrynet is off + running, err := IsLbrynetRunning() + if err != nil { + return err + } + if running { + return errors.Prefix("cannot cleanup lbrynet as the daemon is running", err) + } + lbrynetDir := GetLBRYNetDir() + files, err := filepath.Glob(lbrynetDir + "lbrynet.sqlite*") + if err != nil { + return errors.Err(err) + } + for _, f := range files { + err = os.Remove(f) + if err != nil { + return errors.Err(err) + } + } + blobsDir := GetBlobsDir() + err = os.RemoveAll(blobsDir) + if err != nil { + return errors.Err(err) + } + err = os.Mkdir(blobsDir, 0755) + if err != nil { + return errors.Err(err) + } + return nil +} + +func StartDaemon() error { + if IsUsingDocker() { + return startDaemonViaDocker() + } + return startDaemonViaSystemd() +} + +func StopDaemon() error { + if IsUsingDocker() { + return stopDaemonViaDocker() + } + return stopDaemonViaSystemd() +} + +func startDaemonViaDocker() error { + container, err := GetLBRYNetContainer(true) + if err != nil { + return err + } + + cli, err := client.NewEnvClient() + if err != nil { + panic(err) + } + + err = cli.ContainerStart(context.Background(), container.ID, types.ContainerStartOptions{}) + if err != nil { + return errors.Err(err) + } + + return nil +} + +func stopDaemonViaDocker() error { + container, err := GetLBRYNetContainer(ONLINE) + if err != nil { + return err + } + + cli, err := client.NewEnvClient() + if err != nil { + panic(err) + } + + err = cli.ContainerStop(context.Background(), container.ID, nil) + if err != nil { + return errors.Err(err) + } + + return nil +} + +func startDaemonViaSystemd() error { + err := exec.Command("/usr/bin/sudo", "/bin/systemctl", "start", "lbrynet.service").Run() + if err != nil { + return errors.Err(err) + } + return nil +} + +func stopDaemonViaSystemd() error { + err := exec.Command("/usr/bin/sudo", "/bin/systemctl", "stop", "lbrynet.service").Run() + if err != nil { + return errors.Err(err) + } + return nil +} + +func GetDefaultWalletPath() string { + defaultWalletDir := os.Getenv("HOME") + "/.lbryum/wallets/default_wallet" + if IsRegTest() { + defaultWalletDir = os.Getenv("HOME") + "/.lbryum_regtest/wallets/default_wallet" + } + + walletPath := os.Getenv("LBRYNET_WALLETS_DIR") + if walletPath != "" { + defaultWalletDir = walletPath + "/wallets/default_wallet" + } + return defaultWalletDir +} From e47775939510babe18a4815513b30b40949b339b Mon Sep 17 00:00:00 2001 From: Mark Beamer Jr Date: Sat, 10 Aug 2019 22:50:43 -0400 Subject: [PATCH 07/11] change to master tag of internal-apis docker image --- .travis.yml | 10 ++++++++-- e2e/data_setup.sh | 21 ++++++++++++++++++++ e2e/docker-compose.yml | 17 +++++++++++----- e2e/e2e.sh | 43 ++++++++++------------------------------- manager/manager.go | 4 ++-- manager/ytsync.go | 4 ++++ sdk/api.go | 4 ++-- sources/youtubeVideo.go | 6 +++++- thumbs/uploader.go | 2 +- util/util.go | 10 +++++++--- 10 files changed, 72 insertions(+), 49 deletions(-) create mode 100755 e2e/data_setup.sh diff --git a/.travis.yml b/.travis.yml index 38fc146..7af129e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -19,17 +19,23 @@ addons: apt: update: true packages: - - youtube-dl - ffmpeg - tree + - python-pip + +before_script: + - sudo pip install -U youtube-dl env: global: - GO111MODULE=on #GITHUB_TOKEN - secure: "Ps3KocRP5xnM3/uA99CeYhDTVxRIuW7fGyrtqBeRWZW0cXzeA4XCTKxqcFbrPUPw67XkrBVgE58JDdWoQEJ7tm67PjMm/ltp5Evhx/QAJDh+YSofXyGDVpG1mrTZFI66R3NVVJLkSGALMkuWWXvfYZeU//AworJbyRoaIK/CVt5OP23i5N4tdd5UXc5dfLuYqnKRynyMmCkz9c3yEIQMXoPhG2hx7l7L2BeMJvcKmVhkSN7nQayjnrbUXGm/IRqrb88lvkyBevN5E3IB2V5IKEieIPZjbD/N0IfcnAt89Z96tgDhtIbx3ZvXm92lsvHA8buqQpG9d2AmSi6GKs64lQcnGeM5o0wER2JHWl1OSa1Nr/UAo5Xb/PM65Yt3yZE8AuMKHBmbfDSBzdkTXx58AeDzFUd3kMXD/fFjeQQWyXFlOss3ygH9SObl827Txmz9OJqZaxabs5Q3AP6m3EjKjz7zfLfrgpcxJM2WBiU1bN0ZxUgZkImy/CHk5gCZ7vhcnaLiDO4HZnzY/aRJwKYQPE5i0O2nHpIfovqkc0DFBA7U/7Cjin7e1E0UZvF3meLOxMqkfc6X7QTxqQpt2Tej6jlpdxw4CTLwGUhGkAw9IAPkUB3L0EbZ1/ksGhNvGDvUeSTq8hYdMAPmA+k9jS6653V4SQ+qBMy5++tbr5AeZQI=" + script: - - ./e2e/e2e.sh + #- ./e2e/e2e.sh # Hold until we can resolve the /var/tmp issue - talk to beamer/niko + - make + deploy: provider: script skip_cleanup: true diff --git a/e2e/data_setup.sh b/e2e/data_setup.sh new file mode 100755 index 0000000..2d46529 --- /dev/null +++ b/e2e/data_setup.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash + +set -e + +#Add a ytsync user +ADDYTSYNCUSER='INSERT INTO user (given_name) VALUE("ytsync user")' +mysql -u lbry -plbry -D lbry -h "127.0.0.1" -P 15500 -e "$ADDYTSYNCUSER" +#Insert an auth token for the user to be used by ytsync +ADDYTSYNCAUTHTOKEN='INSERT INTO auth_token (user_id, value) VALUE(1,"ytsyntoken")' +mysql -u lbry -plbry -D lbry -h "127.0.0.1" -P 15500 -e "$ADDYTSYNCAUTHTOKEN" +#Give priveledges to ytsync user +ASSIGNGROOP='INSERT INTO user_groop (user_id, groop_id) VALUE( 1,3)' +mysql -u lbry -plbry -D lbry -h "127.0.0.1" -P 15500 -e "$ASSIGNGROOP" + +#Add youtuber to sync +ADDYTSYNCER='INSERT INTO user (given_name) VALUE("youtuber")' +mysql -u lbry -plbry -D lbry -h "127.0.0.1" -P 15500 -e "$ADDYTSYNCER" +#Add their youtube channel to be synced +ADDYTCHANNEL="INSERT INTO youtube_data (user_id, status_token,desired_lbry_channel,channel_id,channel_name,status,created_at,source,total_videos,total_subscribers) +VALUE(2,'3qzGyuVjQaf7t4pKKu2Er1NRW2LJkeWw','@beamertest','UCCyr5j8akeu9j4Q7urV0Lqw','BeamerAtLBRY','queued','2019-08-01 00:00:00','sync',1,0)" +mysql -u lbry -plbry -D lbry -h "127.0.0.1" -P 15500 -e "$ADDYTCHANNEL" diff --git a/e2e/docker-compose.yml b/e2e/docker-compose.yml index 4b28bba..6244f75 100644 --- a/e2e/docker-compose.yml +++ b/e2e/docker-compose.yml @@ -26,12 +26,14 @@ services: ## Wallet Server ## ################### walletserver: - image: lbry/wallet-server:master + image: lbry/wallet-server:v0.38.5 networks: - e2e restart: always + #volumes: + # - "./persist/.walletserver:/database" environment: - - DB_DIRECTORY=/tmp/ + - DB_DIRECTORY=/database - MAX_SEND=1000000000000000000000 - DAEMON_URL=http://lbry:lbry@lbrycrd:29245 - MAX_SUBS=1000000000000 @@ -57,7 +59,7 @@ services: image: lbry/lbrynet:v0.38.5 networks: - e2e - restart: "no" + restart: always ports: - "15100:5279" - "15101:5280" @@ -72,7 +74,12 @@ services: - "./persist/.lbrynet:/home/lbrynet" - ".:/etc/lbry" #Put your daemon_settings.yml here # /private/var/tmp for OSX and /var/tmp for Linux - - "${LOCAL_TMP_DIR}:/var/tmp" + - "${LOCAL_TMP_DIR}" +# - type: bind +# source: "${LOCAL_TMP_DIR}" +# target: "/tmp" +# bind: +# propagation: rshared #entrypoint: wait-for-it walletserver:50001 -- start ########### ## MySQL ## @@ -96,7 +103,7 @@ services: ## Internal APIs ## ################### internalapis: - image: lbry/internal-apis:latest + image: lbry/internal-apis:master networks: - e2e restart: "no" diff --git a/e2e/e2e.sh b/e2e/e2e.sh index 9cd32cf..027b5d0 100755 --- a/e2e/e2e.sh +++ b/e2e/e2e.sh @@ -6,7 +6,7 @@ set -e make #OVERRIDE this in your .env file if running from mac. Check docker-compose.yml for details -export LOCAL_TMP_DIR="/var/tmp" +export LOCAL_TMP_DIR="/var/tmp:/var/tmp" #Private Variables Set in local installations: SLACK_TOKEN,YOUTUBE_API_KEY,AWS_S3_ID,AWS_S3_SECRET,AWS_S3_REGION,AWS_S3_BUCKET touch -a .env && set -o allexport; source ./.env; set +o allexport @@ -28,21 +28,17 @@ export LBRYNET_WALLETS_DIR="$(pwd)/e2e/persist/.lbrynet/.local/share/lbry/lbryum export TMP_DIR="/var/tmp" export UID -#ensure that docker can be run and managed from the user -USER=$(whoami) -sudo usermod -aG docker "$USER" - cd ./e2e docker-compose stop docker-compose rm -f echo "$DOCKER_PASSWORD" | docker login --username "$DOCKER_USERNAME" --password-stdin docker-compose pull if [[ -d persist ]]; then rm -rf persist; fi -rm -rf persist -mkdir persist -mkdir -p blobsfiles -mkdir persist/.lbrynet -chmod 777 -R ./persist +mkdir -m 0777 -p ./persist +mkdir -m 777 -p ./persist/.walletserver +mkdir -m 777 -p ./persist/.lbrynet +#sudo chown -Rv 999:999 ./persist/.walletserver +#sudo chown -Rv 1000:1000 ./persist/.lbrynet docker-compose up -d printf 'waiting for internal apis' until curl --output /dev/null --silent --head --fail http://localhost:15400; do @@ -51,38 +47,19 @@ until curl --output /dev/null --silent --head --fail http://localhost:15400; do done echo "successfully started..." -# make sure we have permission to mess with the volumes -sudo find ./persist -type d -exec chmod 777 {} \; -sudo chown "$USER": -R ./persist - #Data Setup for test -#Add a ytsync user -ADDYTSYNCUSER='INSERT INTO user (given_name) VALUE("ytsync user")' -mysql -u lbry -plbry -D lbry -h "127.0.0.1" -P 15500 -e "$ADDYTSYNCUSER" -#Insert an auth token for the user to be used by ytsync -ADDYTSYNCAUTHTOKEN='INSERT INTO auth_token (user_id, value) VALUE(1,"ytsyntoken")' -mysql -u lbry -plbry -D lbry -h "127.0.0.1" -P 15500 -e "$ADDYTSYNCAUTHTOKEN" -#Give priveledges to ytsync user -ASSIGNGROOP='INSERT INTO user_groop (user_id, groop_id) VALUE( 1,3)' -mysql -u lbry -plbry -D lbry -h "127.0.0.1" -P 15500 -e "$ASSIGNGROOP" - -#Add youtuber to sync -ADDYTSYNCER='INSERT INTO user (given_name) VALUE("youtuber")' -mysql -u lbry -plbry -D lbry -h "127.0.0.1" -P 15500 -e "$ADDYTSYNCER" -#Add their youtube channel to be synced -ADDYTCHANNEL="INSERT INTO youtube_data (user_id, status_token,desired_lbry_channel,channel_id,channel_name,status,google_id,google_token,created_at,source,total_videos,total_subscribers) -VALUE(2,'3qzGyuVjQaf7t4pKKu2Er1NRW2LJkeWw','@beamertest','UCCyr5j8akeu9j4Q7urV0Lqw','BeamerAtLBRY','queued',$GOOGLE_ID,'$GOOGLE_TOKEN','2019-08-01 00:00:00','sync',1,0)" -mysql -u lbry -plbry -D lbry -h "127.0.0.1" -P 15500 -e "$ADDYTCHANNEL" - +./data_setup.sh # Execute the test! ./../bin/ytsync --channelID UCCyr5j8akeu9j4Q7urV0Lqw #Force channel intended...just in case. This channel lines up with the api container # Assert the status status=$(mysql -u lbry -plbry -ss -D lbry -h "127.0.0.1" -P 15500 -e 'SELECT status FROM youtube_data WHERE id=1') videoStatus=$(mysql -u lbry -plbry -ss -D lbry -h "127.0.0.1" -P 15500 -e 'SELECT status FROM synced_video WHERE id=1') -if [[ $status != "synced" && $videoStatus != "published" ]]; then +if [[ $status != "synced" || $videoStatus != "published" ]]; then docker-compose logs --tail="all" lbrycrd docker-compose logs --tail="all" walletserver docker-compose logs --tail="all" lbrynet docker-compose logs --tail="all" internalapis +echo "List local /var/tmp" +find /var/tmp exit 1; fi; \ No newline at end of file diff --git a/manager/manager.go b/manager/manager.go index ac6e016..39572ef 100644 --- a/manager/manager.go +++ b/manager/manager.go @@ -112,7 +112,7 @@ func (s *SyncManager) Start() error { for { err := s.checkUsedSpace() if err != nil { - return err + return errors.Err(err) } var syncs []Sync @@ -122,7 +122,7 @@ func (s *SyncManager) Start() error { if isSingleChannelSync { channels, err := s.apiConfig.FetchChannels("", s.syncProperties) if err != nil { - return err + return errors.Err(err) } if len(channels) != 1 { return errors.Err("Expected 1 channel, %d returned", len(channels)) diff --git a/manager/ytsync.go b/manager/ytsync.go index 71d8ca4..4ab4b3b 100644 --- a/manager/ytsync.go +++ b/manager/ytsync.go @@ -289,6 +289,10 @@ func (s *Sync) FullCycle() (e error) { if err != nil { return errors.Wrap(err, 0) } + err = os.Chmod(s.videoDirectory, 0766) + if err != nil { + return errors.Err(err) + } defer deleteSyncFolder(s.videoDirectory) log.Printf("Starting daemon") diff --git a/sdk/api.go b/sdk/api.go index 9fbdd3d..952200b 100644 --- a/sdk/api.go +++ b/sdk/api.go @@ -71,7 +71,7 @@ func (a *APIConfig) FetchChannels(status string, cp *SyncProperties) ([]YoutubeC var response apiJobsResponse err = json.Unmarshal(body, &response) if err != nil { - return nil, err + return nil, errors.Err(err) } if response.Data == nil { return nil, errors.Err(response.Error) @@ -152,7 +152,7 @@ func (a *APIConfig) SetChannelStatus(channelID string, status string, failureRea var response apiChannelStatusResponse err := json.Unmarshal(body, &response) if err != nil { - return nil, nil, err + return nil, nil, errors.Err(err) } if !response.Error.IsNull() { return nil, nil, errors.Err(response.Error.String) diff --git a/sources/youtubeVideo.go b/sources/youtubeVideo.go index aba0a6e..09551b3 100644 --- a/sources/youtubeVideo.go +++ b/sources/youtubeVideo.go @@ -178,7 +178,7 @@ func (v *YoutubeVideo) getAbbrevDescription() string { func (v *YoutubeVideo) download(useIPv6 bool) error { videoPath := v.getFullPath() - err := os.Mkdir(v.videoDir(), 0750) + err := os.Mkdir(v.videoDir(), 0777) if err != nil && !strings.Contains(err.Error(), "file exists") { return errors.Wrap(err, 0) } @@ -310,6 +310,10 @@ runcmd: if err != nil { return errors.Err(err) } + err = os.Chmod(v.getFullPath(), 0777) + if err != nil { + return errors.Err(err) + } videoSize := fi.Size() v.size = &videoSize return nil diff --git a/thumbs/uploader.go b/thumbs/uploader.go index 5d68557..4beff61 100644 --- a/thumbs/uploader.go +++ b/thumbs/uploader.go @@ -25,7 +25,7 @@ const thumbnailPath = "/tmp/ytsync_thumbnails/" const ThumbnailEndpoint = "https://thumbnails.lbry.com/" func (u *thumbnailUploader) downloadThumbnail() error { - _ = os.Mkdir(thumbnailPath, 0750) + _ = os.Mkdir(thumbnailPath, 0777) img, err := os.Create("/tmp/ytsync_thumbnails/" + u.name) if err != nil { return errors.Err(err) diff --git a/util/util.go b/util/util.go index 9280623..a546caf 100644 --- a/util/util.go +++ b/util/util.go @@ -166,7 +166,7 @@ func CleanForStartup() error { err = CleanupLbrynet() if err != nil { - return err + return errors.Err(err) } lbrycrd, err := GetLbrycrdClient(os.Getenv("LBRYCRD_STRING")) @@ -187,7 +187,11 @@ func CleanForStartup() error { } defaultWalletDir := GetDefaultWalletPath() - return os.Remove(defaultWalletDir) + _, err = os.Stat(defaultWalletDir) + if os.IsNotExist(err) { + return nil + } + return errors.Err(os.Remove(defaultWalletDir)) } func CleanupLbrynet() error { @@ -215,7 +219,7 @@ func CleanupLbrynet() error { if err != nil { return errors.Err(err) } - err = os.Mkdir(blobsDir, 0755) + err = os.Mkdir(blobsDir, 0777) if err != nil { return errors.Err(err) } From b90e367e823dcc7436496823bb326e2adb339fd4 Mon Sep 17 00:00:00 2001 From: Mark Beamer Jr Date: Mon, 12 Aug 2019 23:20:09 -0400 Subject: [PATCH 08/11] Addressed comments --- .gitlab-ci.yml | 29 ----------------------------- .travis.yml | 3 ++- e2e/daemon_settings.yml | 2 +- e2e/docker-compose.yml | 23 +---------------------- main.go | 8 ++++---- manager/manager.go | 2 +- manager/ytsync.go | 25 +------------------------ util/util.go | 9 +++------ 8 files changed, 13 insertions(+), 88 deletions(-) delete mode 100644 .gitlab-ci.yml diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml deleted file mode 100644 index abafa89..0000000 --- a/.gitlab-ci.yml +++ /dev/null @@ -1,29 +0,0 @@ -job: - image: golang:1.12.7-stretch - - variables: - GO111MODULE: "on" - - script: - #Fetch master to be used in deploy condition to prevent deploying on old commits. - - whoami && pwd - - USER=$(whoami) - - ./e2e/e2e.sh - - cache: - key: "$CI_COMMIT_REF_SLUG" - paths: - - $HOME/.cache/go-build - - $HOME/gopath/pkg/mod - - before_script: - - whoami && pwd - - USER=$(whoami) - - apt update -q && apt install -qqy --no-install-recommends locales-all sudo youtube-dl ffmpeg tree curl - - apt install -qqy --no-install-recommends python-pip python-dev libffi-dev libssl-dev gcc libc-dev make - - curl -fsSL https://get.docker.com -o get-docker.sh - - sudo sh get-docker.sh - - sudo usermod -aG docker $USER - - sudo curl -L "https://github.com/docker/compose/releases/download/1.24.1/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose - - sudo chmod +x /usr/local/bin/docker-compose - - sudo ln -s /usr/local/bin/docker-compose /usr/bin/docker-compose diff --git a/.travis.yml b/.travis.yml index 7af129e..9a67845 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,7 +2,7 @@ os: linux dist: bionic language: go go: - - 1.12.1 + - 1.12.7 - master install: true @@ -25,6 +25,7 @@ addons: before_script: - sudo pip install -U youtube-dl + - sudo add-apt-repository ppa:jonathonf/ffmpeg-4 env: global: diff --git a/e2e/daemon_settings.yml b/e2e/daemon_settings.yml index 8a6bcb4..63b444e 100644 --- a/e2e/daemon_settings.yml +++ b/e2e/daemon_settings.yml @@ -9,4 +9,4 @@ save_files: false share_usage_data: false tcp_port: 3333 udp_port: 4444 -use_upnp: true \ No newline at end of file +use_upnp: false \ No newline at end of file diff --git a/e2e/docker-compose.yml b/e2e/docker-compose.yml index 6244f75..217c5d9 100644 --- a/e2e/docker-compose.yml +++ b/e2e/docker-compose.yml @@ -1,15 +1,10 @@ version: "3.5" -networks: - e2e: - external: false services: ############# ## Lbrycrd ## ############# lbrycrd: image: lbry/lbrycrd:v0.12.4.1 - networks: - - e2e restart: always ports: - "15201:29246" @@ -27,11 +22,7 @@ services: ################### walletserver: image: lbry/wallet-server:v0.38.5 - networks: - - e2e restart: always - #volumes: - # - "./persist/.walletserver:/database" environment: - DB_DIRECTORY=/database - MAX_SEND=1000000000000000000000 @@ -50,15 +41,13 @@ services: nofile: soft: 90000 hard: 90000 - # command: lbry.wallet.server.coin.LBC + #command: lbry.wallet.server.coin.LBC command: lbry.wallet.server.coin.LBCRegTest ############# ## Lbrynet ## ############# lbrynet: image: lbry/lbrynet:v0.38.5 - networks: - - e2e restart: always ports: - "15100:5279" @@ -75,19 +64,11 @@ services: - ".:/etc/lbry" #Put your daemon_settings.yml here # /private/var/tmp for OSX and /var/tmp for Linux - "${LOCAL_TMP_DIR}" -# - type: bind -# source: "${LOCAL_TMP_DIR}" -# target: "/tmp" -# bind: -# propagation: rshared - #entrypoint: wait-for-it walletserver:50001 -- start ########### ## MySQL ## ########### mysql: image: mysql/mysql-server:5.7.27 - networks: - - e2e restart: "no" ports: - "15500:3306" @@ -104,8 +85,6 @@ services: ################### internalapis: image: lbry/internal-apis:master - networks: - - e2e restart: "no" ports: - "15400:8080" diff --git a/main.go b/main.go index eb3be91..51fc505 100644 --- a/main.go +++ b/main.go @@ -2,18 +2,18 @@ package main import ( "fmt" - "github.com/lbryio/lbry.go/extras/errors" "math/rand" "os" "time" + "github.com/lbryio/lbry.go/extras/errors" "github.com/lbryio/lbry.go/extras/util" + "github.com/lbryio/ytsync/manager" "github.com/lbryio/ytsync/sdk" + ytUtils "github.com/lbryio/ytsync/util" + log "github.com/sirupsen/logrus" "github.com/spf13/cobra" - - "github.com/lbryio/ytsync/manager" - ytUtils "github.com/lbryio/ytsync/util" ) var Version string diff --git a/manager/manager.go b/manager/manager.go index 39572ef..7108a71 100644 --- a/manager/manager.go +++ b/manager/manager.go @@ -101,7 +101,7 @@ const ( func (s *SyncManager) Start() error { - if logUtils.IsCleanOnStartup() { + if logUtils.ShouldCleanOnStartup() { err := logUtils.CleanForStartup() if err != nil { return err diff --git a/manager/ytsync.go b/manager/ytsync.go index 4ab4b3b..3cf0ca2 100644 --- a/manager/ytsync.go +++ b/manager/ytsync.go @@ -6,7 +6,6 @@ import ( "net/http" "os" "os/signal" - "path/filepath" "runtime/debug" "sort" "strings" @@ -126,7 +125,6 @@ func (s *Sync) downloadWallet() error { if err != nil { return errors.Prefix("error starting session: ", err) } - test() downloader := s3manager.NewDownloader(s3Session) out, err := os.Create(defaultTempWalletDir) if err != nil { @@ -572,12 +570,11 @@ func (s *Sync) doSync() error { if err != nil { return errors.Prefix("error updating remote database", err) } - if pubsOnWallet == 0 { + if pubsOnWallet == 0 { //Todo - This needs to be done if the cert is nil in internal-apis cert, err := s.daemon.ChannelExport(s.lbryChannelID, nil, nil) if err != nil { return errors.Prefix("error getting channel cert", err) } - println("lbryChannelID:", s.lbryChannelID) err = s.APIConfig.SetChannelCert(string(*cert), s.lbryChannelID) if err != nil { return errors.Prefix("error setting channel cert", err) @@ -990,23 +987,3 @@ func waitForDaemonProcess(timeout time.Duration) error { } return errors.Err("timeout reached") } - -func test() { - var files []string - root, err := os.Getwd() - if err != nil { - log.Error("cant get pwd!") - return - } - root = root + "/persist/.lbrynet" - err = filepath.Walk(root, func(path string, info os.FileInfo, err error) error { - files = append(files, path) - return nil - }) - if err != nil { - panic(err) - } - for _, file := range files { - fmt.Println(file) - } -} diff --git a/util/util.go b/util/util.go index a546caf..3b5a081 100644 --- a/util/util.go +++ b/util/util.go @@ -2,7 +2,6 @@ package util import ( "context" - "fmt" "os" "os/exec" "os/user" @@ -68,9 +67,6 @@ func getDockerContainer(name string, all bool) (*types.Container, error) { if err != nil { panic(err) } - for _, container := range containers { - fmt.Printf("%s %s\n", container.ID[:10], container.Image) - } if len(containers) == 0 { return nil, nil } @@ -115,7 +111,7 @@ func GetLbrycrdClient(lbrycrdString string) (*lbrycrd.Client, error) { return lbrycrdd, nil } -func IsCleanOnStartup() bool { +func ShouldCleanOnStartup() bool { shouldClean, err := strconv.ParseBool(os.Getenv("CLEAN_ON_STARTUP")) if err != nil { return false @@ -177,7 +173,8 @@ func CleanForStartup() error { if err != nil { return errors.Err(err) } - if height < 110 { + const minBlocksForUTXO = 110 + if height < minBlocksForUTXO { //Start reg test will some credits txs, err := lbrycrd.Generate(uint32(110) - uint32(height)) if err != nil { From cd57e1ab0804d5d0d61cf3744ed661aeb64bf898 Mon Sep 17 00:00:00 2001 From: Mark Beamer Jr Date: Tue, 13 Aug 2019 21:31:41 -0400 Subject: [PATCH 09/11] Always send channel export --- manager/ytsync.go | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/manager/ytsync.go b/manager/ytsync.go index 3cf0ca2..39a93d8 100644 --- a/manager/ytsync.go +++ b/manager/ytsync.go @@ -570,16 +570,18 @@ func (s *Sync) doSync() error { if err != nil { return errors.Prefix("error updating remote database", err) } - if pubsOnWallet == 0 { //Todo - This needs to be done if the cert is nil in internal-apis - cert, err := s.daemon.ChannelExport(s.lbryChannelID, nil, nil) - if err != nil { - return errors.Prefix("error getting channel cert", err) - } + + cert, err := s.daemon.ChannelExport(s.lbryChannelID, nil, nil) + if err != nil { + return errors.Prefix("error getting channel cert", err) + } + if cert != nil { err = s.APIConfig.SetChannelCert(string(*cert), s.lbryChannelID) if err != nil { return errors.Prefix("error setting channel cert", err) } } + if nFixed > 0 || nRemoved > 0 { err := s.setStatusSyncing() if err != nil { From 7a76a4b79d486e431433866baf0959006d443bba Mon Sep 17 00:00:00 2001 From: Mark Beamer Jr Date: Tue, 13 Aug 2019 22:12:06 -0400 Subject: [PATCH 10/11] Upgrade to 1.0.17 of lbry.go --- go.mod | 2 +- go.sum | 7 +++---- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/go.mod b/go.mod index 079fd97..1bdb2ee 100644 --- a/go.mod +++ b/go.mod @@ -16,7 +16,7 @@ require ( github.com/hashicorp/serf v0.8.2 // indirect github.com/kr/pretty v0.1.0 // indirect github.com/lbryio/errors.go v0.0.0-20180223142025-ad03d3cc6a5c - github.com/lbryio/lbry.go v1.0.16-0.20190728143701-a17fa3ad5f2b + github.com/lbryio/lbry.go v1.0.17 github.com/lbryio/reflector.go v1.0.6-0.20190806185326-2e4f235489f4 github.com/mitchellh/go-ps v0.0.0-20170309133038-4fdf99ab2936 github.com/opencontainers/go-digest v1.0.0-rc1 // indirect diff --git a/go.sum b/go.sum index 3610ddc..72e4dbe 100644 --- a/go.sum +++ b/go.sum @@ -177,13 +177,12 @@ github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/lbryio/errors.go v0.0.0-20180223142025-ad03d3cc6a5c h1:BhdcWGsuKif/XoSZnqVGNqJ1iEmH0czWR5upj+AuR8M= github.com/lbryio/errors.go v0.0.0-20180223142025-ad03d3cc6a5c/go.mod h1:muH7wpUqE8hRA3OrYYosw9+Sl681BF9cwcjzE+OCNK8= github.com/lbryio/lbry.go v0.0.0-20190109223729-30c312501602/go.mod h1:YEuFJD/oHNra6BFy+NfuvS84Wg6RMWJFGtiCCCc6MmQ= -github.com/lbryio/lbry.go v1.0.16-0.20190728143701-a17fa3ad5f2b h1:+8EpDwZ2ereYx5HGoGGGT7eCaGHQKs8BS5s+jPy+AEo= -github.com/lbryio/lbry.go v1.0.16-0.20190728143701-a17fa3ad5f2b/go.mod h1:JtyI30bU51rm0LZ/po3mQuzf++14OWb6kR/6mMRAmKU= +github.com/lbryio/lbry.go v1.0.17 h1:Lt7xIguw6Q5LMKcEMfxD7IB7kUdP7xwrc0OxvBUl39E= +github.com/lbryio/lbry.go v1.0.17/go.mod h1:JtyI30bU51rm0LZ/po3mQuzf++14OWb6kR/6mMRAmKU= github.com/lbryio/lbryschema.go v0.0.0-20190428231007-c54836bca002 h1:urfYK5ElpUrAv90auPLldoVC60LwiGAcY0OE6HJB9KI= github.com/lbryio/lbryschema.go v0.0.0-20190428231007-c54836bca002/go.mod h1:dAzPCBj3CKKWBGYBZxK6tKBP5SCgY2tqd9SnQd/OyKo= +github.com/lbryio/ozzo-validation v0.0.0-20170323141101-d1008ad1fd04 h1:Nze+C2HbeKvhjI/kVn+9Poj/UuEW5sOQxcsxqO7L3GI= github.com/lbryio/ozzo-validation v0.0.0-20170323141101-d1008ad1fd04/go.mod h1:fbG/dzobG8r95KzMwckXiLMHfFjZaBRQqC9hPs2XAQ4= -github.com/lbryio/reflector.go v1.0.6-0.20190710201919-7c7ed9da72ce h1:ixsKssly++rkh0DWxNDrZyiRp00NieKZlndkqtpgj/c= -github.com/lbryio/reflector.go v1.0.6-0.20190710201919-7c7ed9da72ce/go.mod h1:7Y3YYeAKS6egH2WzwfU8f6+uNGjVHfLzKvwn+Nv3VMY= github.com/lbryio/reflector.go v1.0.6-0.20190806185326-2e4f235489f4 h1:SpUbq2YBg3ncetkw8APUgn8nFF8dscKzzhyiWMM7XCc= github.com/lbryio/reflector.go v1.0.6-0.20190806185326-2e4f235489f4/go.mod h1:7Y3YYeAKS6egH2WzwfU8f6+uNGjVHfLzKvwn+Nv3VMY= github.com/lbryio/types v0.0.0-20181001180206-594241d24e00/go.mod h1:CG3wsDv5BiVYQd5i1Jp7wGsaVyjZTJshqXeWMVKsISE= From d73c6945dd7bfb395b9b65b2e83c69101f21fd4d Mon Sep 17 00:00:00 2001 From: Mark Beamer Jr Date: Tue, 13 Aug 2019 22:15:06 -0400 Subject: [PATCH 11/11] Fix travis.yml non interactive --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 9a67845..d410bc7 100644 --- a/.travis.yml +++ b/.travis.yml @@ -25,7 +25,7 @@ addons: before_script: - sudo pip install -U youtube-dl - - sudo add-apt-repository ppa:jonathonf/ffmpeg-4 + - sudo add-apt-repository -y ppa:jonathonf/ffmpeg-4 env: global: