From 751bc84ce59b65dd4b00bb3b411026a2f8b5e432 Mon Sep 17 00:00:00 2001 From: Niko Storni Date: Tue, 4 Aug 2020 00:55:26 +0200 Subject: [PATCH] grab channel info without APIs fix new channel syncs fix e2e tests --- e2e/data_setup.sh | 4 +- e2e/e2e.sh | 2 +- go.mod | 6 +- go.sum | 84 +-------------- manager/s3_storage.go | 240 ++++++++++++++++++++++++++++++++++++++++++ manager/setup.go | 31 +++--- manager/ytsync.go | 129 +++-------------------- util/util.go | 26 ++++- ytapi/ytapi.go | 186 +++++++++++++++++++++++++++++--- ytapi/ytapi_test.go | 12 +++ 10 files changed, 481 insertions(+), 239 deletions(-) create mode 100644 manager/s3_storage.go create mode 100644 ytapi/ytapi_test.go diff --git a/e2e/data_setup.sh b/e2e/data_setup.sh index e15c1db..a6dfae4 100755 --- a/e2e/data_setup.sh +++ b/e2e/data_setup.sh @@ -19,6 +19,6 @@ mysql -u lbry -plbry -D lbry -h "127.0.0.1" -P 15500 -e "$ADDYTSYNCER" ADDYTSYNCAUTHTOKEN='INSERT INTO auth_token (user_id, value) VALUE(2,"youtubertoken")' mysql -u lbry -plbry -D lbry -h "127.0.0.1" -P 15500 -e "$ADDYTSYNCAUTHTOKEN" #Add their youtube channel to be synced -ADDYTCHANNEL="INSERT INTO youtube_data (user_id, status_token,desired_lbry_channel,channel_id,channel_name,status,created_at,source,total_videos,total_subscribers,should_sync,redeemable) -VALUE(2,'3qzGyuVjQaf7t4pKKu2Er1NRW2LJkeWw','@beamertest','UCNQfQvFMPnInwsU_iGYArJQ','BeamerAtLBRY','queued','2019-08-01 00:00:00','sync',10,10,1,1)" +ADDYTCHANNEL="INSERT INTO youtube_data (user_id, status_token,desired_lbry_channel,channel_id,channel_name,status,created_at,source,total_videos,total_subscribers,should_sync,redeemable,total_views,reviewed,last_uploaded_video) +VALUE(2,'3qzGyuVjQaf7t4pKKu2Er1NRW2LJkeWw','@beamertest','UCNQfQvFMPnInwsU_iGYArJQ','BeamerAtLBRY','queued','2019-08-01 00:00:00','sync',10,10,1,1,10000,1,'7bBV2Z-9wpo')" mysql -u lbry -plbry -D lbry -h "127.0.0.1" -P 15500 -e "$ADDYTCHANNEL" diff --git a/e2e/e2e.sh b/e2e/e2e.sh index e051e27..f6fbc94 100755 --- a/e2e/e2e.sh +++ b/e2e/e2e.sh @@ -26,7 +26,7 @@ export REGTEST=true # Local settings export BLOBS_DIRECTORY="$(pwd)/e2e/blobsfiles" export LBRYNET_DIR="$(pwd)/e2e/persist/.lbrynet/.local/share/lbry/lbrynet/" -export LBRYNET_WALLETS_DIR="$(pwd)/e2e/persist/.lbrynet/.local/share/lbry/lbryum" +export LBRYUM_DIR="$(pwd)/e2e/persist/.lbrynet/.local/share/lbry/lbryum" export TMP_DIR="/var/tmp" export CHAINNAME="lbrycrd_regtest" export UID diff --git a/go.mod b/go.mod index c915950..914602c 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,6 @@ module github.com/lbryio/ytsync/v5 replace github.com/btcsuite/btcd => github.com/lbryio/lbrycrd.go v0.0.0-20200203050410-e1076f12bf19 require ( - cloud.google.com/go v0.46.3 // indirect github.com/Microsoft/go-winio v0.4.14 // indirect github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a github.com/aws/aws-sdk-go v1.25.9 @@ -13,7 +12,7 @@ require ( github.com/docker/go-connections v0.4.0 // indirect github.com/docker/go-units v0.4.0 // indirect github.com/go-sql-driver/mysql v1.4.1 // indirect - github.com/golang/groupcache v0.0.0-20191002201903-404acd9df4cc // indirect + github.com/google/btree v1.0.0 // indirect github.com/hashicorp/go-immutable-radix v1.1.0 // indirect github.com/hashicorp/go-sockaddr v1.0.2 // indirect github.com/hashicorp/golang-lru v0.5.3 // indirect @@ -30,9 +29,8 @@ require ( github.com/sirupsen/logrus v1.4.2 github.com/spf13/cobra v0.0.5 github.com/spf13/pflag v1.0.5 // indirect - go.opencensus.io v0.22.1 // indirect + github.com/stretchr/testify v1.4.0 golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e // indirect - google.golang.org/api v0.11.0 google.golang.org/appengine v1.6.5 // indirect ) diff --git a/go.sum b/go.sum index 05ca74d..367f184 100644 --- a/go.sum +++ b/go.sum @@ -1,16 +1,5 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= -cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= -cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= -cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= -cloud.google.com/go v0.46.3 h1:AVXDdKsrtX33oR9fbCMu/+c1o8Ofjq6Ku/MInaLVg5Y= -cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= -cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= -cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= -cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/DataDog/datadog-go v2.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ= github.com/Microsoft/go-winio v0.4.14 h1:+hMXMk01us9KgxGb7ftKQt2Xpf5hH/yky+TDA+qxleU= github.com/Microsoft/go-winio v0.4.14/go.mod h1:qXqCSQ3Xa7+6tgxaGTIe4Kpcdsi+P8jBhyzoq1bpyYA= @@ -78,12 +67,7 @@ github.com/go-sql-driver/mysql v1.4.1 h1:g24URVg0OFbNUTx9qqY1IRZ9D9z3iPyi5zKhQZp github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= -github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20191002201903-404acd9df4cc h1:55rEp52jU6bkyslZ1+C/7NGfpQsEc6pxGLAGDOctqbw= -github.com/golang/groupcache v0.0.0-20191002201903-404acd9df4cc/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= -github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= -github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.0/go.mod h1:Qd/q+1AKNOZr9uGQzbzCmRO6sUih6GTPZv6a1/R87v0= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -93,16 +77,8 @@ github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Z github.com/google/btree v1.0.0 h1:0udJVsspx3VBr5FwtLhQQtuAsVc79tTq0ocGIPAU6qo= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= -github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1 h1:Xye71clBPdm5HgqGwUkwhbynsUJZhDbS20FvLhQ2izg= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= -github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= -github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= -github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= -github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= -github.com/googleapis/gax-go/v2 v2.0.5 h1:sjZBwGj9Jlw33ImPtvFviGYvseOtDM7hkSKB7+Tv3SM= -github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gopherjs/gopherjs v0.0.0-20181103185306-d547d1d9531e/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gopherjs/gopherjs v0.0.0-20190915194858-d3ddacdb130f h1:TyqzGm2z1h3AGhjOoRYyeLcW4WlW81MDQkWa+rx/000= @@ -139,7 +115,6 @@ github.com/hashicorp/go-uuid v1.0.1 h1:fv1ep09latC32wFoVwnqcnKJGnMSdBanPczbHAYm1 github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.3 h1:YPkqC67at8FYaadspW/6uE0COsBxS2656RLEr8Bppgk= github.com/hashicorp/golang-lru v0.5.3/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= @@ -166,7 +141,6 @@ github.com/johntdyer/slack-go v0.0.0-20180213144715-95fac1160b22/go.mod h1:u0Jo4 github.com/johntdyer/slackrus v0.0.0-20180518184837-f7aae3243a07 h1:+kBG/8rjCa6vxJZbUjAiE4MQmBEBYc8nLEb51frnvBY= github.com/johntdyer/slackrus v0.0.0-20180518184837-f7aae3243a07/go.mod h1:j1kV/8f3jowErEq4XyeypkCdvg5EeHkf0YCKCcq5Ybo= github.com/jrick/logrotate v1.0.0/go.mod h1:LNinyqDIJnpAur+b8yyulnQw/wDuN1+BYKlTRt3OuAQ= -github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jtolds/gls v4.2.1+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= @@ -254,7 +228,6 @@ github.com/prometheus/common v0.0.0-20181126121408-4724e9255275 h1:PnBWHBf+6L0jO github.com/prometheus/common v0.0.0-20181126121408-4724e9255275/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a h1:9a8MnZMP0X2nLJdBg+pBmGgkJlSaKC2KaQmTCk1XDtE= github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= -github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/ryanuber/columnize v2.1.0+incompatible/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= @@ -305,10 +278,6 @@ github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljT github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= github.com/ybbus/jsonrpc v0.0.0-20180411222309-2a548b7d822d h1:tQo6hjclyv3RHUgZOl6iWb2Y44A/sN9bf9LAYfuioEg= github.com/ybbus/jsonrpc v0.0.0-20180411222309-2a548b7d822d/go.mod h1:XJrh1eMSzdIYFbM08flv0wp5G35eRniyeGut1z+LSiE= -go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= -go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= -go.opencensus.io v0.22.1 h1:8dP3SGL7MPB94crU3bEPplMPe83FI4EouesJUeFHv50= -go.opencensus.io v0.22.1/go.mod h1:Ap50jQcDJrx6rB6VgeeFPtuPIf3wMRvRfrfYDO6+BmA= go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.4.0 h1:cxzIVoETapQEqDhQu3QfnvXAV4AlzcvUCxkVUFw3+EU= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= @@ -318,20 +287,12 @@ golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnf golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190325154230-a5d413f7728c/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190923035154-9ee001bba392/go.mod h1:/lpIB1dKB+9EgE3H3cr1v9wB50oz8l4C4h62xy7jSTY= golang.org/x/crypto v0.0.0-20191002192127-34f69633bfdc h1:c0o/qxkaO2LF5t6fQrT4b5hzyggAkLLlCUjqfRxd8Q4= golang.org/x/crypto v0.0.0-20191002192127-34f69633bfdc/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= -golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= -golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= -golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= -golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= @@ -339,17 +300,16 @@ golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= +golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -357,13 +317,9 @@ golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20191009170851-d66e71096ffb h1:TR699M2v0qoKTOHxeLgp6zPqaQNs74f01a/ob9W0qko= golang.org/x/net v0.0.0-20191009170851-d66e71096ffb/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= -golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45 h1:SVwTIAaPC2U/AvvLNZ2a7OVsmBpC8L5BlwK1whH3hm0= -golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e h1:vcxGaoTs7kV8m5Np9uUNQin4BrLOthgV7252N8V+FwY= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -374,25 +330,19 @@ golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190520201301-c432e742b0af/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190523142557-0e01d883c5c5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190922100055-0a153f010e69/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191009170203-06d7bd2c5f4f h1:hjzMYz/7Ea1mNKfOnFOfktR0mlA5jqhvywClCMHM/qw= golang.org/x/sys v0.0.0-20191009170203-06d7bd2c5f4f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190921001708-c4c64cad1fd0 h1:xQwXv67TxFo9nC1GJFyab5eq/5B590r6RlnL/G8Sz7w= golang.org/x/time v0.0.0-20190921001708-c4c64cad1fd0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -400,54 +350,26 @@ golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGm golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190907020128-2ca718005c18/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= -google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= -google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= -google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= -google.golang.org/api v0.11.0 h1:n/qM3q0/rV2F0pox7o0CvNhlPvZAo7pLbef122cbLJ0= -google.golang.org/api v0.11.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= google.golang.org/appengine v1.6.5 h1:tycE03LOZYQNhDpS27tcQdAzLCVMaj7QT2SXxebnpCM= google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20180831171423-11092d34479b/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20181004005441-af9cb2a35e7f/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= -google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= -google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= -google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= google.golang.org/genproto v0.0.0-20191009194640-548a555dbc03 h1:4HYDjxeNXAOTv3o1N2tjo8UUSlhQgAD52FVkwxnWgM8= google.golang.org/genproto v0.0.0-20191009194640-548a555dbc03/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= -google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= -google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= google.golang.org/grpc v1.24.0 h1:vb/1TCsVn3DcJlQ0Gs1yB1pKI6Do2/QNwxdKqmc/b0s= google.golang.org/grpc v1.24.0/go.mod h1:XDChyiUovWa60DnaeDeZmSW86xtLtjtZbwvSiRnRtcA= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/ini.v1 v1.41.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= @@ -465,8 +387,4 @@ gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= -rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= diff --git a/manager/s3_storage.go b/manager/s3_storage.go new file mode 100644 index 0000000..cccb5a1 --- /dev/null +++ b/manager/s3_storage.go @@ -0,0 +1,240 @@ +package manager + +import ( + "os" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/awserr" + "github.com/aws/aws-sdk-go/aws/credentials" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/s3" + "github.com/aws/aws-sdk-go/service/s3/s3manager" + + "github.com/lbryio/lbry.go/v2/extras/errors" + + logUtils "github.com/lbryio/ytsync/v5/util" +) + +func (s *Sync) getS3Downloader() (*s3manager.Downloader, error) { + creds := credentials.NewStaticCredentials(s.AwsS3ID, s.AwsS3Secret, "") + s3Session, err := session.NewSession(&aws.Config{Region: aws.String(s.AwsS3Region), Credentials: creds}) + if err != nil { + return nil, errors.Prefix("error starting session: ", err) + } + downloader := s3manager.NewDownloader(s3Session) + return downloader, nil +} +func (s *Sync) getS3Uploader() (*s3manager.Uploader, error) { + creds := credentials.NewStaticCredentials(s.AwsS3ID, s.AwsS3Secret, "") + s3Session, err := session.NewSession(&aws.Config{Region: aws.String(s.AwsS3Region), Credentials: creds}) + if err != nil { + return nil, errors.Prefix("error starting session: ", err) + } + uploader := s3manager.NewUploader(s3Session) + return uploader, nil +} + +func (s *Sync) downloadWallet() error { + defaultWalletDir, defaultTempWalletDir, key, err := s.getWalletPaths() + if err != nil { + return errors.Err(err) + } + downloader, err := s.getS3Downloader() + if err != nil { + return err + } + out, err := os.Create(defaultTempWalletDir) + if err != nil { + return errors.Prefix("error creating temp wallet: ", err) + } + defer out.Close() + + bytesWritten, err := downloader.Download(out, &s3.GetObjectInput{ + Bucket: aws.String(s.AwsS3Bucket), + Key: key, + }) + if err != nil { + // Casting to the awserr.Error type will allow you to inspect the error + // code returned by the service in code. The error code can be used + // to switch on context specific functionality. In this case a context + // specific error message is printed to the user based on the bucket + // and key existing. + // + // For information on other S3 API error codes see: + // http://docs.aws.amazon.com/AmazonS3/latest/API/ErrorResponses.html + if aerr, ok := err.(awserr.Error); ok { + code := aerr.Code() + if code == s3.ErrCodeNoSuchKey { + return errors.Err("wallet not on S3") + } + } + return err + } else if bytesWritten == 0 { + return errors.Err("zero bytes written") + } + + err = os.Rename(defaultTempWalletDir, defaultWalletDir) + if err != nil { + return errors.Prefix("error replacing temp wallet for default wallet: ", err) + } + + return nil +} + +func (s *Sync) downloadBlockchainDB() error { + defaultBDBDir, defaultTempBDBDir, key, err := s.getBlockchainDBPaths() + if err != nil { + return errors.Err(err) + } + _ = os.Remove(defaultBDBDir) + + downloader, err := s.getS3Downloader() + if err != nil { + return errors.Err(err) + } + out, err := os.Create(defaultTempBDBDir) + if err != nil { + return errors.Prefix("error creating temp wallet: ", err) + } + defer out.Close() + + bytesWritten, err := downloader.Download(out, &s3.GetObjectInput{ + Bucket: aws.String(s.AwsS3Bucket), + Key: key, + }) + if err != nil { + // Casting to the awserr.Error type will allow you to inspect the error + // code returned by the service in code. The error code can be used + // to switch on context specific functionality. In this case a context + // specific error message is printed to the user based on the bucket + // and key existing. + // + // For information on other S3 API error codes see: + // http://docs.aws.amazon.com/AmazonS3/latest/API/ErrorResponses.html + if aerr, ok := err.(awserr.Error); ok { + code := aerr.Code() + if code == s3.ErrCodeNoSuchKey { + return nil // let ytsync sync the database by itself + } + } + return errors.Err(err) + } else if bytesWritten == 0 { + return errors.Err("zero bytes written") + } + + err = os.Rename(defaultTempBDBDir, defaultBDBDir) + if err != nil { + return errors.Prefix("error replacing temp blockchain.db for default blockchain.db: ", err) + } + + return nil +} + +func (s *Sync) getWalletPaths() (defaultWallet, tempWallet string, key *string, err error) { + defaultWallet = os.Getenv("HOME") + "/.lbryum/wallets/default_wallet" + tempWallet = os.Getenv("HOME") + "/.lbryum/wallets/tmp_wallet" + key = aws.String("/wallets/" + s.YoutubeChannelID) + if logUtils.IsRegTest() { + defaultWallet = os.Getenv("HOME") + "/.lbryum_regtest/wallets/default_wallet" + tempWallet = os.Getenv("HOME") + "/.lbryum_regtest/wallets/tmp_wallet" + key = aws.String("/regtest/" + s.YoutubeChannelID) + } + + lbryumDir := os.Getenv("LBRYUM_DIR") + if lbryumDir != "" { + defaultWallet = lbryumDir + "/wallets/default_wallet" + tempWallet = lbryumDir + "/wallets/tmp_wallet" + } + + if _, err := os.Stat(defaultWallet); !os.IsNotExist(err) { + return "", "", nil, errors.Err("default_wallet already exists") + } + return +} + +func (s *Sync) getBlockchainDBPaths() (defaultDB, tempDB string, key *string, err error) { + lbryumDir := os.Getenv("LBRYUM_DIR") + if lbryumDir == "" { + if logUtils.IsRegTest() { + lbryumDir = os.Getenv("HOME") + "/.lbryum_regtest" + } else { + lbryumDir = os.Getenv("HOME") + "/.lbryum" + } + } + defaultDB = lbryumDir + "/lbc_mainnet/blockchain.db" + tempDB = lbryumDir + "/lbc_mainnet/tmp_blockchain.db" + key = aws.String("/blockchain_dbs/" + s.YoutubeChannelID) + if logUtils.IsRegTest() { + defaultDB = lbryumDir + "/lbc_regtest/blockchain.db" + tempDB = lbryumDir + "/lbc_regtest/tmp_blockchain.db" + key = aws.String("/regtest_dbs/" + s.YoutubeChannelID) + } + return +} + +func (s *Sync) uploadWallet() error { + defaultWalletDir := logUtils.GetDefaultWalletPath() + key := aws.String("/wallets/" + s.YoutubeChannelID) + if logUtils.IsRegTest() { + key = aws.String("/regtest/" + s.YoutubeChannelID) + } + + if _, err := os.Stat(defaultWalletDir); os.IsNotExist(err) { + return errors.Err("default_wallet does not exist") + } + + uploader, err := s.getS3Uploader() + if err != nil { + return err + } + + file, err := os.Open(defaultWalletDir) + if err != nil { + return err + } + defer file.Close() + + _, err = uploader.Upload(&s3manager.UploadInput{ + Bucket: aws.String(s.AwsS3Bucket), + Key: key, + Body: file, + }) + if err != nil { + return err + } + + return os.Remove(defaultWalletDir) +} + +func (s *Sync) uploadBlockchainDB() error { + defaultBDBDir, _, key, err := s.getBlockchainDBPaths() + if err != nil { + return errors.Err(err) + } + + if _, err := os.Stat(defaultBDBDir); os.IsNotExist(err) { + return errors.Err("blockchain.db does not exist") + } + + uploader, err := s.getS3Uploader() + if err != nil { + return err + } + + file, err := os.Open(defaultBDBDir) + if err != nil { + return err + } + defer file.Close() + + _, err = uploader.Upload(&s3manager.UploadInput{ + Bucket: aws.String(s.AwsS3Bucket), + Key: key, + Body: file, + }) + if err != nil { + return err + } + + return os.Remove(defaultBDBDir) +} diff --git a/manager/setup.go b/manager/setup.go index f37c682..af2eaad 100644 --- a/manager/setup.go +++ b/manager/setup.go @@ -383,20 +383,20 @@ func (s *Sync) ensureChannelOwnership() error { } } - channelInfo, channelBranding, err := ytapi.ChannelInfo(s.APIConfig.YoutubeAPIKey, s.YoutubeChannelID) + channelInfo, err := ytapi.ChannelInfo(s.YoutubeChannelID) if err != nil { return err } - thumbnail := thumbs.GetBestThumbnail(nil) //Ignore this for now ( ChannelInfo Panics) - thumbnailURL, err := thumbs.MirrorThumbnail(thumbnail.URL, s.YoutubeChannelID, s.Manager.GetS3AWSConfig()) + thumbnail := channelInfo.Header.C4TabbedHeaderRenderer.Avatar.Thumbnails[len(channelInfo.Header.C4TabbedHeaderRenderer.Avatar.Thumbnails)-1].URL + thumbnailURL, err := thumbs.MirrorThumbnail(thumbnail, s.YoutubeChannelID, s.Manager.GetS3AWSConfig()) if err != nil { return err } var bannerURL *string - if channelBranding.Image != nil && channelBranding.Image.BannerImageUrl != "" { - bURL, err := thumbs.MirrorThumbnail(channelBranding.Image.BannerImageUrl, "banner-"+s.YoutubeChannelID, s.Manager.GetS3AWSConfig()) + if channelInfo.Header.C4TabbedHeaderRenderer.Banner.Thumbnails != nil { + bURL, err := thumbs.MirrorThumbnail(channelInfo.Header.C4TabbedHeaderRenderer.Banner.Thumbnails[len(channelInfo.Header.C4TabbedHeaderRenderer.Banner.Thumbnails)-1].URL, "banner-"+s.YoutubeChannelID, s.Manager.GetS3AWSConfig()) if err != nil { return err } @@ -404,20 +404,21 @@ func (s *Sync) ensureChannelOwnership() error { } var languages []string = nil - if channelInfo.DefaultLanguage != "" { - if channelInfo.DefaultLanguage == "iw" { - channelInfo.DefaultLanguage = "he" - } - languages = []string{channelInfo.DefaultLanguage} - } + //we don't have this data without the API + //if channelInfo.DefaultLanguage != "" { + // if channelInfo.DefaultLanguage == "iw" { + // channelInfo.DefaultLanguage = "he" + // } + // languages = []string{channelInfo.DefaultLanguage} + //} var locations []jsonrpc.Location = nil - if channelInfo.Country != "" { - locations = []jsonrpc.Location{{Country: util.PtrToString(channelInfo.Country)}} + if channelInfo.Topbar.DesktopTopbarRenderer.CountryCode != "" { + locations = []jsonrpc.Location{{Country: util.PtrToString(channelInfo.Topbar.DesktopTopbarRenderer.CountryCode)}} } var c *jsonrpc.TransactionSummary claimCreateOptions := jsonrpc.ClaimCreateOptions{ - Title: &channelInfo.Title, - Description: &channelInfo.Description, + Title: &channelInfo.Microformat.MicroformatDataRenderer.Title, + Description: &channelInfo.Microformat.MicroformatDataRenderer.Description, Tags: tags_manager.GetTagsForChannel(s.YoutubeChannelID), Languages: languages, Locations: locations, diff --git a/manager/ytsync.go b/manager/ytsync.go index e32dae5..4339656 100644 --- a/manager/ytsync.go +++ b/manager/ytsync.go @@ -26,12 +26,6 @@ import ( "github.com/lbryio/lbry.go/v2/extras/stop" "github.com/lbryio/lbry.go/v2/extras/util" - "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/aws/awserr" - "github.com/aws/aws-sdk-go/aws/credentials" - "github.com/aws/aws-sdk-go/aws/session" - "github.com/aws/aws-sdk-go/service/s3" - "github.com/aws/aws-sdk-go/service/s3/s3manager" log "github.com/sirupsen/logrus" ) @@ -100,116 +94,6 @@ func (s *Sync) IsInterrupted() bool { } } -func (s *Sync) downloadWallet() error { - defaultWalletDir, defaultTempWalletDir, key, err := s.getWalletPaths() - if err != nil { - return errors.Err(err) - } - - creds := credentials.NewStaticCredentials(s.AwsS3ID, s.AwsS3Secret, "") - s3Session, err := session.NewSession(&aws.Config{Region: aws.String(s.AwsS3Region), Credentials: creds}) - if err != nil { - return errors.Prefix("error starting session: ", err) - } - downloader := s3manager.NewDownloader(s3Session) - out, err := os.Create(defaultTempWalletDir) - if err != nil { - return errors.Prefix("error creating temp wallet: ", err) - } - defer out.Close() - - bytesWritten, err := downloader.Download(out, &s3.GetObjectInput{ - Bucket: aws.String(s.AwsS3Bucket), - Key: key, - }) - if err != nil { - // Casting to the awserr.Error type will allow you to inspect the error - // code returned by the service in code. The error code can be used - // to switch on context specific functionality. In this case a context - // specific error message is printed to the user based on the bucket - // and key existing. - // - // For information on other S3 API error codes see: - // http://docs.aws.amazon.com/AmazonS3/latest/API/ErrorResponses.html - if aerr, ok := err.(awserr.Error); ok { - code := aerr.Code() - if code == s3.ErrCodeNoSuchKey { - return errors.Err("wallet not on S3") - } - } - return err - } else if bytesWritten == 0 { - return errors.Err("zero bytes written") - } - - err = os.Rename(defaultTempWalletDir, defaultWalletDir) - if err != nil { - return errors.Prefix("error replacing temp wallet for default wallet: ", err) - } - - return nil -} - -func (s *Sync) getWalletPaths() (defaultWallet, tempWallet string, key *string, err error) { - - defaultWallet = os.Getenv("HOME") + "/.lbryum/wallets/default_wallet" - tempWallet = os.Getenv("HOME") + "/.lbryum/wallets/tmp_wallet" - key = aws.String("/wallets/" + s.YoutubeChannelID) - if logUtils.IsRegTest() { - defaultWallet = os.Getenv("HOME") + "/.lbryum_regtest/wallets/default_wallet" - tempWallet = os.Getenv("HOME") + "/.lbryum_regtest/wallets/tmp_wallet" - key = aws.String("/regtest/" + s.YoutubeChannelID) - } - - walletPath := os.Getenv("LBRYNET_WALLETS_DIR") - if walletPath != "" { - defaultWallet = walletPath + "/wallets/default_wallet" - tempWallet = walletPath + "/wallets/tmp_wallet" - } - - if _, err := os.Stat(defaultWallet); !os.IsNotExist(err) { - return "", "", nil, errors.Err("default_wallet already exists") - } - return -} - -func (s *Sync) uploadWallet() error { - defaultWalletDir := logUtils.GetDefaultWalletPath() - key := aws.String("/wallets/" + s.YoutubeChannelID) - if logUtils.IsRegTest() { - key = aws.String("/regtest/" + s.YoutubeChannelID) - } - - if _, err := os.Stat(defaultWalletDir); os.IsNotExist(err) { - return errors.Err("default_wallet does not exist") - } - - creds := credentials.NewStaticCredentials(s.AwsS3ID, s.AwsS3Secret, "") - s3Session, err := session.NewSession(&aws.Config{Region: aws.String(s.AwsS3Region), Credentials: creds}) - if err != nil { - return err - } - - uploader := s3manager.NewUploader(s3Session) - - file, err := os.Open(defaultWalletDir) - if err != nil { - return err - } - defer file.Close() - - _, err = uploader.Upload(&s3manager.UploadInput{ - Bucket: aws.String(s.AwsS3Bucket), - Key: key, - Body: file, - }) - if err != nil { - return err - } - - return os.Remove(defaultWalletDir) -} - func (s *Sync) setStatusSyncing() error { syncedVideos, claimNames, err := s.Manager.apiConfig.SetChannelStatus(s.YoutubeChannelID, StatusSyncing, "", nil) if err != nil { @@ -270,6 +154,10 @@ func (s *Sync) FullCycle() (e error) { } else { log.Println("Starting new wallet") } + err = s.downloadBlockchainDB() + if err != nil { + return errors.Prefix("failure in downloading blockchain.db", err) + } defer s.stopAndUploadWallet(&e) @@ -450,7 +338,14 @@ func (s *Sync) stopAndUploadWallet(e *error) { if err != nil { if *e == nil { e = &err - return + } else { + *e = errors.Prefix("failure uploading wallet", *e) + } + } + err = s.uploadBlockchainDB() + if err != nil { + if *e == nil { + e = &err } else { *e = errors.Prefix("failure uploading wallet", *e) } diff --git a/util/util.go b/util/util.go index cfe6fb6..d3b1ab1 100644 --- a/util/util.go +++ b/util/util.go @@ -52,7 +52,7 @@ func GetLBRYNetDir() string { } func GetLbryumDir() string { - lbryumDir := os.Getenv("LBRYNET_WALLETS_DIR") + lbryumDir := os.Getenv("LBRYUM_DIR") if lbryumDir == "" { usr, err := user.Current() if err != nil { @@ -366,9 +366,31 @@ func GetDefaultWalletPath() string { defaultWalletDir = os.Getenv("HOME") + "/.lbryum_regtest/wallets/default_wallet" } - walletPath := os.Getenv("LBRYNET_WALLETS_DIR") + walletPath := os.Getenv("LBRYUM_DIR") if walletPath != "" { defaultWalletDir = walletPath + "/wallets/default_wallet" } return defaultWalletDir } +func GetBlockchainDBPath() string { + lbryumDir := os.Getenv("LBRYUM_DIR") + if lbryumDir == "" { + if IsRegTest() { + lbryumDir = os.Getenv("HOME") + "/.lbryum_regtest" + } else { + lbryumDir = os.Getenv("HOME") + "/.lbryum" + } + } + defaultDB := lbryumDir + "/lbc_mainnet/blockchain.db" + if IsRegTest() { + defaultDB = lbryumDir + "/lbc_regtest/blockchain.db" + } + return defaultDB +} +func GetBlockchainDirectoryName() string { + ledger := "lbc_mainnet" + if IsRegTest() { + ledger = "lbc_regtest" + } + return ledger +} diff --git a/ytapi/ytapi.go b/ytapi/ytapi.go index 6b98406..5b07d37 100644 --- a/ytapi/ytapi.go +++ b/ytapi/ytapi.go @@ -2,6 +2,8 @@ package ytapi import ( "bufio" + "encoding/json" + "io/ioutil" "net/http" "regexp" "sort" @@ -25,8 +27,6 @@ import ( "github.com/aws/aws-sdk-go/aws" log "github.com/sirupsen/logrus" - "google.golang.org/api/googleapi/transport" - ytlib "google.golang.org/api/youtube/v3" ) type Video interface { @@ -56,7 +56,7 @@ var mostRecentlyFailedChannel string // TODO: fix this hack! func GetVideosToSync(config *sdk.APIConfig, channelID string, syncedVideos map[string]sdk.SyncedVideo, quickSync bool, maxVideos int, videoParams VideoParams) ([]Video, error) { var videos []Video - if quickSync { + if quickSync && maxVideos > 50 { maxVideos = 50 } videoIDs, err := downloader.GetPlaylistVideoIDs(channelID, maxVideos, videoParams.Stopper.Ch(), videoParams.IPPool) @@ -143,24 +143,36 @@ func CountVideosInChannel(channelID string) (int, error) { return num, nil } -func ChannelInfo(apiKey, channelID string) (*ytlib.ChannelSnippet, *ytlib.ChannelBrandingSettings, error) { - return nil, nil, errors.Err("ChannelInfo doesn't work yet because we're focused on existing channels") +func ChannelInfo(channelID string) (*YoutubeStatsResponse, error) { + //return nil, nil, errors.Err("ChannelInfo doesn't work yet because we're focused on existing channels") + url := "https://www.youtube.com/channel/" + channelID + "/about" - service, err := ytlib.New(&http.Client{Transport: &transport.APIKey{Key: apiKey}}) + req, _ := http.NewRequest("GET", url, nil) + + req.Header.Add("User-Agent", "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.116 Safari/537.36") + req.Header.Add("Accept", "*/*") + + res, err := http.DefaultClient.Do(req) if err != nil { - return nil, nil, errors.Prefix("error creating YouTube service", err) + return nil, errors.Err(err) } - - response, err := service.Channels.List("snippet,brandingSettings").Id(channelID).Do() + defer res.Body.Close() + body, err := ioutil.ReadAll(res.Body) if err != nil { - return nil, nil, errors.Prefix("error getting channel details", err) + return nil, errors.Err(err) + } + pageBody := string(body) + dataStartIndex := strings.Index(pageBody, "window[\"ytInitialData\"] = ") + 26 + dataEndIndex := strings.Index(pageBody, "]}}};") + 4 + + data := pageBody[dataStartIndex:dataEndIndex] + var decodedResponse YoutubeStatsResponse + err = json.Unmarshal([]byte(data), &decodedResponse) + if err != nil { + return nil, errors.Err(err) } - if len(response.Items) < 1 { - return nil, nil, errors.Err("youtube channel not found") - } - - return response.Items[0].Snippet, response.Items[0].BrandingSettings, nil + return &decodedResponse, nil } func getVideos(config *sdk.APIConfig, channelID string, videoIDs []string, stopChan stop.Chan, ipPool *ip_manager.IPPool) ([]*ytdl.YtdlVideo, error) { @@ -205,3 +217,147 @@ func getVideos(config *sdk.APIConfig, channelID string, videoIDs []string, stopC } return videos, nil } + +type YoutubeStatsResponse struct { + Contents struct { + TwoColumnBrowseResultsRenderer struct { + Tabs []struct { + TabRenderer struct { + Title string `json:"title"` + Selected bool `json:"selected"` + Content struct { + SectionListRenderer struct { + Contents []struct { + ItemSectionRenderer struct { + Contents []struct { + ChannelAboutFullMetadataRenderer struct { + Description struct { + SimpleText string `json:"simpleText"` + } `json:"description"` + ViewCountText struct { + SimpleText string `json:"simpleText"` + } `json:"viewCountText"` + JoinedDateText struct { + Runs []struct { + Text string `json:"text"` + } `json:"runs"` + } `json:"joinedDateText"` + CanonicalChannelURL string `json:"canonicalChannelUrl"` + BypassBusinessEmailCaptcha bool `json:"bypassBusinessEmailCaptcha"` + Title struct { + SimpleText string `json:"simpleText"` + } `json:"title"` + Avatar struct { + Thumbnails []struct { + URL string `json:"url"` + Width int `json:"width"` + Height int `json:"height"` + } `json:"thumbnails"` + } `json:"avatar"` + ShowDescription bool `json:"showDescription"` + DescriptionLabel struct { + Runs []struct { + Text string `json:"text"` + } `json:"runs"` + } `json:"descriptionLabel"` + DetailsLabel struct { + Runs []struct { + Text string `json:"text"` + } `json:"runs"` + } `json:"detailsLabel"` + ChannelID string `json:"channelId"` + } `json:"channelAboutFullMetadataRenderer"` + } `json:"contents"` + } `json:"itemSectionRenderer"` + } `json:"contents"` + } `json:"sectionListRenderer"` + } `json:"content"` + } `json:"tabRenderer"` + } `json:"tabs"` + } `json:"twoColumnBrowseResultsRenderer"` + } `json:"contents"` + Header struct { + C4TabbedHeaderRenderer struct { + ChannelID string `json:"channelId"` + Title string `json:"title"` + Avatar struct { + Thumbnails []struct { + URL string `json:"url"` + Width int `json:"width"` + Height int `json:"height"` + } `json:"thumbnails"` + } `json:"avatar"` + Banner struct { + Thumbnails []struct { + URL string `json:"url"` + Width int `json:"width"` + Height int `json:"height"` + } `json:"thumbnails"` + } `json:"banner"` + VisitTracking struct { + RemarketingPing string `json:"remarketingPing"` + } `json:"visitTracking"` + SubscriberCountText struct { + SimpleText string `json:"simpleText"` + } `json:"subscriberCountText"` + } `json:"c4TabbedHeaderRenderer"` + } `json:"header"` + Metadata struct { + ChannelMetadataRenderer struct { + Title string `json:"title"` + Description string `json:"description"` + RssURL string `json:"rssUrl"` + ChannelConversionURL string `json:"channelConversionUrl"` + ExternalID string `json:"externalId"` + Keywords string `json:"keywords"` + OwnerUrls []string `json:"ownerUrls"` + Avatar struct { + Thumbnails []struct { + URL string `json:"url"` + Width int `json:"width"` + Height int `json:"height"` + } `json:"thumbnails"` + } `json:"avatar"` + ChannelURL string `json:"channelUrl"` + IsFamilySafe bool `json:"isFamilySafe"` + VanityChannelURL string `json:"vanityChannelUrl"` + } `json:"channelMetadataRenderer"` + } `json:"metadata"` + Topbar struct { + DesktopTopbarRenderer struct { + CountryCode string `json:"countryCode"` + } `json:"desktopTopbarRenderer"` + } `json:"topbar"` + Microformat struct { + MicroformatDataRenderer struct { + URLCanonical string `json:"urlCanonical"` + Title string `json:"title"` + Description string `json:"description"` + Thumbnail struct { + Thumbnails []struct { + URL string `json:"url"` + Width int `json:"width"` + Height int `json:"height"` + } `json:"thumbnails"` + } `json:"thumbnail"` + SiteName string `json:"siteName"` + AppName string `json:"appName"` + AndroidPackage string `json:"androidPackage"` + IosAppStoreID string `json:"iosAppStoreId"` + IosAppArguments string `json:"iosAppArguments"` + OgType string `json:"ogType"` + URLApplinksWeb string `json:"urlApplinksWeb"` + URLApplinksIos string `json:"urlApplinksIos"` + URLApplinksAndroid string `json:"urlApplinksAndroid"` + URLTwitterIos string `json:"urlTwitterIos"` + URLTwitterAndroid string `json:"urlTwitterAndroid"` + TwitterCardType string `json:"twitterCardType"` + TwitterSiteHandle string `json:"twitterSiteHandle"` + SchemaDotOrgType string `json:"schemaDotOrgType"` + Noindex bool `json:"noindex"` + Unlisted bool `json:"unlisted"` + FamilySafe bool `json:"familySafe"` + Tags []string `json:"tags"` + } `json:"microformatDataRenderer"` + } `json:"microformat"` +} diff --git a/ytapi/ytapi_test.go b/ytapi/ytapi_test.go new file mode 100644 index 0000000..93a612c --- /dev/null +++ b/ytapi/ytapi_test.go @@ -0,0 +1,12 @@ +package ytapi + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestChannelInfo(t *testing.T) { + _, _, err := ChannelInfo("", "UCNQfQvFMPnInwsU_iGYArJQ") + assert.NoError(t, err) +}