grab channel info without APIs
fix new channel syncs fix e2e tests
This commit is contained in:
Niko Storni
Alex Grintsvayg
parent
41fd9f6844
commit
751bc84ce5
10 changed files with 481 additions and 239 deletions
|
|
@ -19,6 +19,6 @@ mysql -u lbry -plbry -D lbry -h "127.0.0.1" -P 15500 -e "$ADDYTSYNCER"
|
|||
ADDYTSYNCAUTHTOKEN='INSERT INTO auth_token (user_id, value) VALUE(2,"youtubertoken")'
|
||||
mysql -u lbry -plbry -D lbry -h "127.0.0.1" -P 15500 -e "$ADDYTSYNCAUTHTOKEN"
|
||||
#Add their youtube channel to be synced
|
||||
ADDYTCHANNEL="INSERT INTO youtube_data (user_id, status_token,desired_lbry_channel,channel_id,channel_name,status,created_at,source,total_videos,total_subscribers,should_sync,redeemable)
|
||||
VALUE(2,'3qzGyuVjQaf7t4pKKu2Er1NRW2LJkeWw','@beamertest','UCNQfQvFMPnInwsU_iGYArJQ','BeamerAtLBRY','queued','2019-08-01 00:00:00','sync',10,10,1,1)"
|
||||
ADDYTCHANNEL="INSERT INTO youtube_data (user_id, status_token,desired_lbry_channel,channel_id,channel_name,status,created_at,source,total_videos,total_subscribers,should_sync,redeemable,total_views,reviewed,last_uploaded_video)
|
||||
VALUE(2,'3qzGyuVjQaf7t4pKKu2Er1NRW2LJkeWw','@beamertest','UCNQfQvFMPnInwsU_iGYArJQ','BeamerAtLBRY','queued','2019-08-01 00:00:00','sync',10,10,1,1,10000,1,'7bBV2Z-9wpo')"
|
||||
mysql -u lbry -plbry -D lbry -h "127.0.0.1" -P 15500 -e "$ADDYTCHANNEL"
|
||||
|
|
|
|||
|
|
@ -26,7 +26,7 @@ export REGTEST=true
|
|||
# Local settings
|
||||
export BLOBS_DIRECTORY="$(pwd)/e2e/blobsfiles"
|
||||
export LBRYNET_DIR="$(pwd)/e2e/persist/.lbrynet/.local/share/lbry/lbrynet/"
|
||||
export LBRYNET_WALLETS_DIR="$(pwd)/e2e/persist/.lbrynet/.local/share/lbry/lbryum"
|
||||
export LBRYUM_DIR="$(pwd)/e2e/persist/.lbrynet/.local/share/lbry/lbryum"
|
||||
export TMP_DIR="/var/tmp"
|
||||
export CHAINNAME="lbrycrd_regtest"
|
||||
export UID
|
||||
|
|
|
|||
6
go.mod
6
go.mod
|
|
@ -3,7 +3,6 @@ module github.com/lbryio/ytsync/v5
|
|||
replace github.com/btcsuite/btcd => github.com/lbryio/lbrycrd.go v0.0.0-20200203050410-e1076f12bf19
|
||||
|
||||
require (
|
||||
cloud.google.com/go v0.46.3 // indirect
|
||||
github.com/Microsoft/go-winio v0.4.14 // indirect
|
||||
github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a
|
||||
github.com/aws/aws-sdk-go v1.25.9
|
||||
|
|
@ -13,7 +12,7 @@ require (
|
|||
github.com/docker/go-connections v0.4.0 // indirect
|
||||
github.com/docker/go-units v0.4.0 // indirect
|
||||
github.com/go-sql-driver/mysql v1.4.1 // indirect
|
||||
github.com/golang/groupcache v0.0.0-20191002201903-404acd9df4cc // indirect
|
||||
github.com/google/btree v1.0.0 // indirect
|
||||
github.com/hashicorp/go-immutable-radix v1.1.0 // indirect
|
||||
github.com/hashicorp/go-sockaddr v1.0.2 // indirect
|
||||
github.com/hashicorp/golang-lru v0.5.3 // indirect
|
||||
|
|
@ -30,9 +29,8 @@ require (
|
|||
github.com/sirupsen/logrus v1.4.2
|
||||
github.com/spf13/cobra v0.0.5
|
||||
github.com/spf13/pflag v1.0.5 // indirect
|
||||
go.opencensus.io v0.22.1 // indirect
|
||||
github.com/stretchr/testify v1.4.0
|
||||
golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e // indirect
|
||||
google.golang.org/api v0.11.0
|
||||
google.golang.org/appengine v1.6.5 // indirect
|
||||
)
|
||||
|
||||
|
|
|
|||
84
go.sum
84
go.sum
|
|
@ -1,16 +1,5 @@
|
|||
cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
|
||||
cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
|
||||
cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU=
|
||||
cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU=
|
||||
cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY=
|
||||
cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc=
|
||||
cloud.google.com/go v0.46.3 h1:AVXDdKsrtX33oR9fbCMu/+c1o8Ofjq6Ku/MInaLVg5Y=
|
||||
cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0=
|
||||
cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o=
|
||||
cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE=
|
||||
cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I=
|
||||
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
|
||||
github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
|
||||
github.com/DataDog/datadog-go v2.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ=
|
||||
github.com/Microsoft/go-winio v0.4.14 h1:+hMXMk01us9KgxGb7ftKQt2Xpf5hH/yky+TDA+qxleU=
|
||||
github.com/Microsoft/go-winio v0.4.14/go.mod h1:qXqCSQ3Xa7+6tgxaGTIe4Kpcdsi+P8jBhyzoq1bpyYA=
|
||||
|
|
@ -78,12 +67,7 @@ github.com/go-sql-driver/mysql v1.4.1 h1:g24URVg0OFbNUTx9qqY1IRZ9D9z3iPyi5zKhQZp
|
|||
github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
|
||||
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58=
|
||||
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
|
||||
github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
|
||||
github.com/golang/groupcache v0.0.0-20191002201903-404acd9df4cc h1:55rEp52jU6bkyslZ1+C/7NGfpQsEc6pxGLAGDOctqbw=
|
||||
github.com/golang/groupcache v0.0.0-20191002201903-404acd9df4cc/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
|
||||
github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
|
||||
github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
|
||||
github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y=
|
||||
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
|
||||
github.com/golang/protobuf v1.3.0/go.mod h1:Qd/q+1AKNOZr9uGQzbzCmRO6sUih6GTPZv6a1/R87v0=
|
||||
github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
|
||||
|
|
@ -93,16 +77,8 @@ github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Z
|
|||
github.com/google/btree v1.0.0 h1:0udJVsspx3VBr5FwtLhQQtuAsVc79tTq0ocGIPAU6qo=
|
||||
github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
|
||||
github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
|
||||
github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
|
||||
github.com/google/go-cmp v0.3.1 h1:Xye71clBPdm5HgqGwUkwhbynsUJZhDbS20FvLhQ2izg=
|
||||
github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
|
||||
github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
|
||||
github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
|
||||
github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
|
||||
github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
|
||||
github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
|
||||
github.com/googleapis/gax-go/v2 v2.0.5 h1:sjZBwGj9Jlw33ImPtvFviGYvseOtDM7hkSKB7+Tv3SM=
|
||||
github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
|
||||
github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
|
||||
github.com/gopherjs/gopherjs v0.0.0-20181103185306-d547d1d9531e/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
|
||||
github.com/gopherjs/gopherjs v0.0.0-20190915194858-d3ddacdb130f h1:TyqzGm2z1h3AGhjOoRYyeLcW4WlW81MDQkWa+rx/000=
|
||||
|
|
@ -139,7 +115,6 @@ github.com/hashicorp/go-uuid v1.0.1 h1:fv1ep09latC32wFoVwnqcnKJGnMSdBanPczbHAYm1
|
|||
github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
|
||||
github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90=
|
||||
github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
|
||||
github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
|
||||
github.com/hashicorp/golang-lru v0.5.3 h1:YPkqC67at8FYaadspW/6uE0COsBxS2656RLEr8Bppgk=
|
||||
github.com/hashicorp/golang-lru v0.5.3/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4=
|
||||
github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
|
||||
|
|
@ -166,7 +141,6 @@ github.com/johntdyer/slack-go v0.0.0-20180213144715-95fac1160b22/go.mod h1:u0Jo4
|
|||
github.com/johntdyer/slackrus v0.0.0-20180518184837-f7aae3243a07 h1:+kBG/8rjCa6vxJZbUjAiE4MQmBEBYc8nLEb51frnvBY=
|
||||
github.com/johntdyer/slackrus v0.0.0-20180518184837-f7aae3243a07/go.mod h1:j1kV/8f3jowErEq4XyeypkCdvg5EeHkf0YCKCcq5Ybo=
|
||||
github.com/jrick/logrotate v1.0.0/go.mod h1:LNinyqDIJnpAur+b8yyulnQw/wDuN1+BYKlTRt3OuAQ=
|
||||
github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
|
||||
github.com/jtolds/gls v4.2.1+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
|
||||
github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo=
|
||||
github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
|
||||
|
|
@ -254,7 +228,6 @@ github.com/prometheus/common v0.0.0-20181126121408-4724e9255275 h1:PnBWHBf+6L0jO
|
|||
github.com/prometheus/common v0.0.0-20181126121408-4724e9255275/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=
|
||||
github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a h1:9a8MnZMP0X2nLJdBg+pBmGgkJlSaKC2KaQmTCk1XDtE=
|
||||
github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
|
||||
github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
|
||||
github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=
|
||||
github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
|
||||
github.com/ryanuber/columnize v2.1.0+incompatible/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
|
||||
|
|
@ -305,10 +278,6 @@ github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljT
|
|||
github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
|
||||
github.com/ybbus/jsonrpc v0.0.0-20180411222309-2a548b7d822d h1:tQo6hjclyv3RHUgZOl6iWb2Y44A/sN9bf9LAYfuioEg=
|
||||
github.com/ybbus/jsonrpc v0.0.0-20180411222309-2a548b7d822d/go.mod h1:XJrh1eMSzdIYFbM08flv0wp5G35eRniyeGut1z+LSiE=
|
||||
go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
|
||||
go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
|
||||
go.opencensus.io v0.22.1 h1:8dP3SGL7MPB94crU3bEPplMPe83FI4EouesJUeFHv50=
|
||||
go.opencensus.io v0.22.1/go.mod h1:Ap50jQcDJrx6rB6VgeeFPtuPIf3wMRvRfrfYDO6+BmA=
|
||||
go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
|
||||
go.uber.org/atomic v1.4.0 h1:cxzIVoETapQEqDhQu3QfnvXAV4AlzcvUCxkVUFw3+EU=
|
||||
go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
|
||||
|
|
@ -318,20 +287,12 @@ golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnf
|
|||
golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
|
||||
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
||||
golang.org/x/crypto v0.0.0-20190325154230-a5d413f7728c/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
||||
golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
|
||||
golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
|
||||
golang.org/x/crypto v0.0.0-20190923035154-9ee001bba392/go.mod h1:/lpIB1dKB+9EgE3H3cr1v9wB50oz8l4C4h62xy7jSTY=
|
||||
golang.org/x/crypto v0.0.0-20191002192127-34f69633bfdc h1:c0o/qxkaO2LF5t6fQrT4b5hzyggAkLLlCUjqfRxd8Q4=
|
||||
golang.org/x/crypto v0.0.0-20191002192127-34f69633bfdc/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
|
||||
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
|
||||
golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
|
||||
golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
|
||||
golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek=
|
||||
golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
|
||||
golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
|
||||
golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
|
||||
golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
|
||||
golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
|
||||
golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
|
||||
golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
|
||||
golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
|
||||
|
|
@ -339,17 +300,16 @@ golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU
|
|||
golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o=
|
||||
golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc=
|
||||
golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY=
|
||||
golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
||||
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
||||
golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
||||
golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
||||
golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
|
||||
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
|
|
@ -357,13 +317,9 @@ golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLL
|
|||
golang.org/x/net v0.0.0-20191009170851-d66e71096ffb h1:TR699M2v0qoKTOHxeLgp6zPqaQNs74f01a/ob9W0qko=
|
||||
golang.org/x/net v0.0.0-20191009170851-d66e71096ffb/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
|
||||
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
|
||||
golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45 h1:SVwTIAaPC2U/AvvLNZ2a7OVsmBpC8L5BlwK1whH3hm0=
|
||||
golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
|
||||
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e h1:vcxGaoTs7kV8m5Np9uUNQin4BrLOthgV7252N8V+FwY=
|
||||
golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
|
|
@ -374,25 +330,19 @@ golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5h
|
|||
golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20190520201301-c432e742b0af/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20190523142557-0e01d883c5c5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20190922100055-0a153f010e69/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20191009170203-06d7bd2c5f4f h1:hjzMYz/7Ea1mNKfOnFOfktR0mlA5jqhvywClCMHM/qw=
|
||||
golang.org/x/sys v0.0.0-20191009170203-06d7bd2c5f4f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
|
||||
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
|
||||
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||
golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||
golang.org/x/time v0.0.0-20190921001708-c4c64cad1fd0 h1:xQwXv67TxFo9nC1GJFyab5eq/5B590r6RlnL/G8Sz7w=
|
||||
golang.org/x/time v0.0.0-20190921001708-c4c64cad1fd0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||
golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
|
|
@ -400,54 +350,26 @@ golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGm
|
|||
golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
|
||||
golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
|
||||
golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
|
||||
golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
|
||||
golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
|
||||
golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
|
||||
golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
|
||||
golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
|
||||
golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
|
||||
golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
|
||||
golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
|
||||
golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
||||
golang.org/x/tools v0.0.0-20190907020128-2ca718005c18/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
||||
golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
||||
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=
|
||||
google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M=
|
||||
google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
|
||||
google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
|
||||
google.golang.org/api v0.11.0 h1:n/qM3q0/rV2F0pox7o0CvNhlPvZAo7pLbef122cbLJ0=
|
||||
google.golang.org/api v0.11.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
|
||||
google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
|
||||
google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
|
||||
google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
|
||||
google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0=
|
||||
google.golang.org/appengine v1.6.5 h1:tycE03LOZYQNhDpS27tcQdAzLCVMaj7QT2SXxebnpCM=
|
||||
google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
|
||||
google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
|
||||
google.golang.org/genproto v0.0.0-20180831171423-11092d34479b/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
|
||||
google.golang.org/genproto v0.0.0-20181004005441-af9cb2a35e7f/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
|
||||
google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
|
||||
google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
|
||||
google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
|
||||
google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
|
||||
google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
|
||||
google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
|
||||
google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8=
|
||||
google.golang.org/genproto v0.0.0-20191009194640-548a555dbc03 h1:4HYDjxeNXAOTv3o1N2tjo8UUSlhQgAD52FVkwxnWgM8=
|
||||
google.golang.org/genproto v0.0.0-20191009194640-548a555dbc03/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
|
||||
google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs=
|
||||
google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
|
||||
google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
|
||||
google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
|
||||
google.golang.org/grpc v1.24.0 h1:vb/1TCsVn3DcJlQ0Gs1yB1pKI6Do2/QNwxdKqmc/b0s=
|
||||
google.golang.org/grpc v1.24.0/go.mod h1:XDChyiUovWa60DnaeDeZmSW86xtLtjtZbwvSiRnRtcA=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=
|
||||
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
|
||||
gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4=
|
||||
gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
|
||||
gopkg.in/ini.v1 v1.41.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
|
||||
|
|
@ -465,8 +387,4 @@ gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo=
|
|||
gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw=
|
||||
honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
|
||||
honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
|
||||
honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
|
||||
honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
|
||||
honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
|
||||
honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
|
||||
rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
|
||||
|
|
|
|||
240
manager/s3_storage.go
Normal file
240
manager/s3_storage.go
Normal file
|
|
@ -0,0 +1,240 @@
|
|||
package manager
|
||||
|
||||
import (
|
||||
"os"
|
||||
|
||||
"github.com/aws/aws-sdk-go/aws"
|
||||
"github.com/aws/aws-sdk-go/aws/awserr"
|
||||
"github.com/aws/aws-sdk-go/aws/credentials"
|
||||
"github.com/aws/aws-sdk-go/aws/session"
|
||||
"github.com/aws/aws-sdk-go/service/s3"
|
||||
"github.com/aws/aws-sdk-go/service/s3/s3manager"
|
||||
|
||||
"github.com/lbryio/lbry.go/v2/extras/errors"
|
||||
|
||||
logUtils "github.com/lbryio/ytsync/v5/util"
|
||||
)
|
||||
|
||||
func (s *Sync) getS3Downloader() (*s3manager.Downloader, error) {
|
||||
creds := credentials.NewStaticCredentials(s.AwsS3ID, s.AwsS3Secret, "")
|
||||
s3Session, err := session.NewSession(&aws.Config{Region: aws.String(s.AwsS3Region), Credentials: creds})
|
||||
if err != nil {
|
||||
return nil, errors.Prefix("error starting session: ", err)
|
||||
}
|
||||
downloader := s3manager.NewDownloader(s3Session)
|
||||
return downloader, nil
|
||||
}
|
||||
func (s *Sync) getS3Uploader() (*s3manager.Uploader, error) {
|
||||
creds := credentials.NewStaticCredentials(s.AwsS3ID, s.AwsS3Secret, "")
|
||||
s3Session, err := session.NewSession(&aws.Config{Region: aws.String(s.AwsS3Region), Credentials: creds})
|
||||
if err != nil {
|
||||
return nil, errors.Prefix("error starting session: ", err)
|
||||
}
|
||||
uploader := s3manager.NewUploader(s3Session)
|
||||
return uploader, nil
|
||||
}
|
||||
|
||||
func (s *Sync) downloadWallet() error {
|
||||
defaultWalletDir, defaultTempWalletDir, key, err := s.getWalletPaths()
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
downloader, err := s.getS3Downloader()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
out, err := os.Create(defaultTempWalletDir)
|
||||
if err != nil {
|
||||
return errors.Prefix("error creating temp wallet: ", err)
|
||||
}
|
||||
defer out.Close()
|
||||
|
||||
bytesWritten, err := downloader.Download(out, &s3.GetObjectInput{
|
||||
Bucket: aws.String(s.AwsS3Bucket),
|
||||
Key: key,
|
||||
})
|
||||
if err != nil {
|
||||
// Casting to the awserr.Error type will allow you to inspect the error
|
||||
// code returned by the service in code. The error code can be used
|
||||
// to switch on context specific functionality. In this case a context
|
||||
// specific error message is printed to the user based on the bucket
|
||||
// and key existing.
|
||||
//
|
||||
// For information on other S3 API error codes see:
|
||||
// http://docs.aws.amazon.com/AmazonS3/latest/API/ErrorResponses.html
|
||||
if aerr, ok := err.(awserr.Error); ok {
|
||||
code := aerr.Code()
|
||||
if code == s3.ErrCodeNoSuchKey {
|
||||
return errors.Err("wallet not on S3")
|
||||
}
|
||||
}
|
||||
return err
|
||||
} else if bytesWritten == 0 {
|
||||
return errors.Err("zero bytes written")
|
||||
}
|
||||
|
||||
err = os.Rename(defaultTempWalletDir, defaultWalletDir)
|
||||
if err != nil {
|
||||
return errors.Prefix("error replacing temp wallet for default wallet: ", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Sync) downloadBlockchainDB() error {
|
||||
defaultBDBDir, defaultTempBDBDir, key, err := s.getBlockchainDBPaths()
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
_ = os.Remove(defaultBDBDir)
|
||||
|
||||
downloader, err := s.getS3Downloader()
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
out, err := os.Create(defaultTempBDBDir)
|
||||
if err != nil {
|
||||
return errors.Prefix("error creating temp wallet: ", err)
|
||||
}
|
||||
defer out.Close()
|
||||
|
||||
bytesWritten, err := downloader.Download(out, &s3.GetObjectInput{
|
||||
Bucket: aws.String(s.AwsS3Bucket),
|
||||
Key: key,
|
||||
})
|
||||
if err != nil {
|
||||
// Casting to the awserr.Error type will allow you to inspect the error
|
||||
// code returned by the service in code. The error code can be used
|
||||
// to switch on context specific functionality. In this case a context
|
||||
// specific error message is printed to the user based on the bucket
|
||||
// and key existing.
|
||||
//
|
||||
// For information on other S3 API error codes see:
|
||||
// http://docs.aws.amazon.com/AmazonS3/latest/API/ErrorResponses.html
|
||||
if aerr, ok := err.(awserr.Error); ok {
|
||||
code := aerr.Code()
|
||||
if code == s3.ErrCodeNoSuchKey {
|
||||
return nil // let ytsync sync the database by itself
|
||||
}
|
||||
}
|
||||
return errors.Err(err)
|
||||
} else if bytesWritten == 0 {
|
||||
return errors.Err("zero bytes written")
|
||||
}
|
||||
|
||||
err = os.Rename(defaultTempBDBDir, defaultBDBDir)
|
||||
if err != nil {
|
||||
return errors.Prefix("error replacing temp blockchain.db for default blockchain.db: ", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Sync) getWalletPaths() (defaultWallet, tempWallet string, key *string, err error) {
|
||||
defaultWallet = os.Getenv("HOME") + "/.lbryum/wallets/default_wallet"
|
||||
tempWallet = os.Getenv("HOME") + "/.lbryum/wallets/tmp_wallet"
|
||||
key = aws.String("/wallets/" + s.YoutubeChannelID)
|
||||
if logUtils.IsRegTest() {
|
||||
defaultWallet = os.Getenv("HOME") + "/.lbryum_regtest/wallets/default_wallet"
|
||||
tempWallet = os.Getenv("HOME") + "/.lbryum_regtest/wallets/tmp_wallet"
|
||||
key = aws.String("/regtest/" + s.YoutubeChannelID)
|
||||
}
|
||||
|
||||
lbryumDir := os.Getenv("LBRYUM_DIR")
|
||||
if lbryumDir != "" {
|
||||
defaultWallet = lbryumDir + "/wallets/default_wallet"
|
||||
tempWallet = lbryumDir + "/wallets/tmp_wallet"
|
||||
}
|
||||
|
||||
if _, err := os.Stat(defaultWallet); !os.IsNotExist(err) {
|
||||
return "", "", nil, errors.Err("default_wallet already exists")
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
func (s *Sync) getBlockchainDBPaths() (defaultDB, tempDB string, key *string, err error) {
|
||||
lbryumDir := os.Getenv("LBRYUM_DIR")
|
||||
if lbryumDir == "" {
|
||||
if logUtils.IsRegTest() {
|
||||
lbryumDir = os.Getenv("HOME") + "/.lbryum_regtest"
|
||||
} else {
|
||||
lbryumDir = os.Getenv("HOME") + "/.lbryum"
|
||||
}
|
||||
}
|
||||
defaultDB = lbryumDir + "/lbc_mainnet/blockchain.db"
|
||||
tempDB = lbryumDir + "/lbc_mainnet/tmp_blockchain.db"
|
||||
key = aws.String("/blockchain_dbs/" + s.YoutubeChannelID)
|
||||
if logUtils.IsRegTest() {
|
||||
defaultDB = lbryumDir + "/lbc_regtest/blockchain.db"
|
||||
tempDB = lbryumDir + "/lbc_regtest/tmp_blockchain.db"
|
||||
key = aws.String("/regtest_dbs/" + s.YoutubeChannelID)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
func (s *Sync) uploadWallet() error {
|
||||
defaultWalletDir := logUtils.GetDefaultWalletPath()
|
||||
key := aws.String("/wallets/" + s.YoutubeChannelID)
|
||||
if logUtils.IsRegTest() {
|
||||
key = aws.String("/regtest/" + s.YoutubeChannelID)
|
||||
}
|
||||
|
||||
if _, err := os.Stat(defaultWalletDir); os.IsNotExist(err) {
|
||||
return errors.Err("default_wallet does not exist")
|
||||
}
|
||||
|
||||
uploader, err := s.getS3Uploader()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
file, err := os.Open(defaultWalletDir)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer file.Close()
|
||||
|
||||
_, err = uploader.Upload(&s3manager.UploadInput{
|
||||
Bucket: aws.String(s.AwsS3Bucket),
|
||||
Key: key,
|
||||
Body: file,
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return os.Remove(defaultWalletDir)
|
||||
}
|
||||
|
||||
func (s *Sync) uploadBlockchainDB() error {
|
||||
defaultBDBDir, _, key, err := s.getBlockchainDBPaths()
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
|
||||
if _, err := os.Stat(defaultBDBDir); os.IsNotExist(err) {
|
||||
return errors.Err("blockchain.db does not exist")
|
||||
}
|
||||
|
||||
uploader, err := s.getS3Uploader()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
file, err := os.Open(defaultBDBDir)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer file.Close()
|
||||
|
||||
_, err = uploader.Upload(&s3manager.UploadInput{
|
||||
Bucket: aws.String(s.AwsS3Bucket),
|
||||
Key: key,
|
||||
Body: file,
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return os.Remove(defaultBDBDir)
|
||||
}
|
||||
|
|
@ -383,20 +383,20 @@ func (s *Sync) ensureChannelOwnership() error {
|
|||
}
|
||||
}
|
||||
|
||||
channelInfo, channelBranding, err := ytapi.ChannelInfo(s.APIConfig.YoutubeAPIKey, s.YoutubeChannelID)
|
||||
channelInfo, err := ytapi.ChannelInfo(s.YoutubeChannelID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
thumbnail := thumbs.GetBestThumbnail(nil) //Ignore this for now ( ChannelInfo Panics)
|
||||
thumbnailURL, err := thumbs.MirrorThumbnail(thumbnail.URL, s.YoutubeChannelID, s.Manager.GetS3AWSConfig())
|
||||
thumbnail := channelInfo.Header.C4TabbedHeaderRenderer.Avatar.Thumbnails[len(channelInfo.Header.C4TabbedHeaderRenderer.Avatar.Thumbnails)-1].URL
|
||||
thumbnailURL, err := thumbs.MirrorThumbnail(thumbnail, s.YoutubeChannelID, s.Manager.GetS3AWSConfig())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
var bannerURL *string
|
||||
if channelBranding.Image != nil && channelBranding.Image.BannerImageUrl != "" {
|
||||
bURL, err := thumbs.MirrorThumbnail(channelBranding.Image.BannerImageUrl, "banner-"+s.YoutubeChannelID, s.Manager.GetS3AWSConfig())
|
||||
if channelInfo.Header.C4TabbedHeaderRenderer.Banner.Thumbnails != nil {
|
||||
bURL, err := thumbs.MirrorThumbnail(channelInfo.Header.C4TabbedHeaderRenderer.Banner.Thumbnails[len(channelInfo.Header.C4TabbedHeaderRenderer.Banner.Thumbnails)-1].URL, "banner-"+s.YoutubeChannelID, s.Manager.GetS3AWSConfig())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
|
@ -404,20 +404,21 @@ func (s *Sync) ensureChannelOwnership() error {
|
|||
}
|
||||
|
||||
var languages []string = nil
|
||||
if channelInfo.DefaultLanguage != "" {
|
||||
if channelInfo.DefaultLanguage == "iw" {
|
||||
channelInfo.DefaultLanguage = "he"
|
||||
}
|
||||
languages = []string{channelInfo.DefaultLanguage}
|
||||
}
|
||||
//we don't have this data without the API
|
||||
//if channelInfo.DefaultLanguage != "" {
|
||||
// if channelInfo.DefaultLanguage == "iw" {
|
||||
// channelInfo.DefaultLanguage = "he"
|
||||
// }
|
||||
// languages = []string{channelInfo.DefaultLanguage}
|
||||
//}
|
||||
var locations []jsonrpc.Location = nil
|
||||
if channelInfo.Country != "" {
|
||||
locations = []jsonrpc.Location{{Country: util.PtrToString(channelInfo.Country)}}
|
||||
if channelInfo.Topbar.DesktopTopbarRenderer.CountryCode != "" {
|
||||
locations = []jsonrpc.Location{{Country: util.PtrToString(channelInfo.Topbar.DesktopTopbarRenderer.CountryCode)}}
|
||||
}
|
||||
var c *jsonrpc.TransactionSummary
|
||||
claimCreateOptions := jsonrpc.ClaimCreateOptions{
|
||||
Title: &channelInfo.Title,
|
||||
Description: &channelInfo.Description,
|
||||
Title: &channelInfo.Microformat.MicroformatDataRenderer.Title,
|
||||
Description: &channelInfo.Microformat.MicroformatDataRenderer.Description,
|
||||
Tags: tags_manager.GetTagsForChannel(s.YoutubeChannelID),
|
||||
Languages: languages,
|
||||
Locations: locations,
|
||||
|
|
|
|||
|
|
@ -26,12 +26,6 @@ import (
|
|||
"github.com/lbryio/lbry.go/v2/extras/stop"
|
||||
"github.com/lbryio/lbry.go/v2/extras/util"
|
||||
|
||||
"github.com/aws/aws-sdk-go/aws"
|
||||
"github.com/aws/aws-sdk-go/aws/awserr"
|
||||
"github.com/aws/aws-sdk-go/aws/credentials"
|
||||
"github.com/aws/aws-sdk-go/aws/session"
|
||||
"github.com/aws/aws-sdk-go/service/s3"
|
||||
"github.com/aws/aws-sdk-go/service/s3/s3manager"
|
||||
log "github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
|
|
@ -100,116 +94,6 @@ func (s *Sync) IsInterrupted() bool {
|
|||
}
|
||||
}
|
||||
|
||||
func (s *Sync) downloadWallet() error {
|
||||
defaultWalletDir, defaultTempWalletDir, key, err := s.getWalletPaths()
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
|
||||
creds := credentials.NewStaticCredentials(s.AwsS3ID, s.AwsS3Secret, "")
|
||||
s3Session, err := session.NewSession(&aws.Config{Region: aws.String(s.AwsS3Region), Credentials: creds})
|
||||
if err != nil {
|
||||
return errors.Prefix("error starting session: ", err)
|
||||
}
|
||||
downloader := s3manager.NewDownloader(s3Session)
|
||||
out, err := os.Create(defaultTempWalletDir)
|
||||
if err != nil {
|
||||
return errors.Prefix("error creating temp wallet: ", err)
|
||||
}
|
||||
defer out.Close()
|
||||
|
||||
bytesWritten, err := downloader.Download(out, &s3.GetObjectInput{
|
||||
Bucket: aws.String(s.AwsS3Bucket),
|
||||
Key: key,
|
||||
})
|
||||
if err != nil {
|
||||
// Casting to the awserr.Error type will allow you to inspect the error
|
||||
// code returned by the service in code. The error code can be used
|
||||
// to switch on context specific functionality. In this case a context
|
||||
// specific error message is printed to the user based on the bucket
|
||||
// and key existing.
|
||||
//
|
||||
// For information on other S3 API error codes see:
|
||||
// http://docs.aws.amazon.com/AmazonS3/latest/API/ErrorResponses.html
|
||||
if aerr, ok := err.(awserr.Error); ok {
|
||||
code := aerr.Code()
|
||||
if code == s3.ErrCodeNoSuchKey {
|
||||
return errors.Err("wallet not on S3")
|
||||
}
|
||||
}
|
||||
return err
|
||||
} else if bytesWritten == 0 {
|
||||
return errors.Err("zero bytes written")
|
||||
}
|
||||
|
||||
err = os.Rename(defaultTempWalletDir, defaultWalletDir)
|
||||
if err != nil {
|
||||
return errors.Prefix("error replacing temp wallet for default wallet: ", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Sync) getWalletPaths() (defaultWallet, tempWallet string, key *string, err error) {
|
||||
|
||||
defaultWallet = os.Getenv("HOME") + "/.lbryum/wallets/default_wallet"
|
||||
tempWallet = os.Getenv("HOME") + "/.lbryum/wallets/tmp_wallet"
|
||||
key = aws.String("/wallets/" + s.YoutubeChannelID)
|
||||
if logUtils.IsRegTest() {
|
||||
defaultWallet = os.Getenv("HOME") + "/.lbryum_regtest/wallets/default_wallet"
|
||||
tempWallet = os.Getenv("HOME") + "/.lbryum_regtest/wallets/tmp_wallet"
|
||||
key = aws.String("/regtest/" + s.YoutubeChannelID)
|
||||
}
|
||||
|
||||
walletPath := os.Getenv("LBRYNET_WALLETS_DIR")
|
||||
if walletPath != "" {
|
||||
defaultWallet = walletPath + "/wallets/default_wallet"
|
||||
tempWallet = walletPath + "/wallets/tmp_wallet"
|
||||
}
|
||||
|
||||
if _, err := os.Stat(defaultWallet); !os.IsNotExist(err) {
|
||||
return "", "", nil, errors.Err("default_wallet already exists")
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
func (s *Sync) uploadWallet() error {
|
||||
defaultWalletDir := logUtils.GetDefaultWalletPath()
|
||||
key := aws.String("/wallets/" + s.YoutubeChannelID)
|
||||
if logUtils.IsRegTest() {
|
||||
key = aws.String("/regtest/" + s.YoutubeChannelID)
|
||||
}
|
||||
|
||||
if _, err := os.Stat(defaultWalletDir); os.IsNotExist(err) {
|
||||
return errors.Err("default_wallet does not exist")
|
||||
}
|
||||
|
||||
creds := credentials.NewStaticCredentials(s.AwsS3ID, s.AwsS3Secret, "")
|
||||
s3Session, err := session.NewSession(&aws.Config{Region: aws.String(s.AwsS3Region), Credentials: creds})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
uploader := s3manager.NewUploader(s3Session)
|
||||
|
||||
file, err := os.Open(defaultWalletDir)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer file.Close()
|
||||
|
||||
_, err = uploader.Upload(&s3manager.UploadInput{
|
||||
Bucket: aws.String(s.AwsS3Bucket),
|
||||
Key: key,
|
||||
Body: file,
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return os.Remove(defaultWalletDir)
|
||||
}
|
||||
|
||||
func (s *Sync) setStatusSyncing() error {
|
||||
syncedVideos, claimNames, err := s.Manager.apiConfig.SetChannelStatus(s.YoutubeChannelID, StatusSyncing, "", nil)
|
||||
if err != nil {
|
||||
|
|
@ -270,6 +154,10 @@ func (s *Sync) FullCycle() (e error) {
|
|||
} else {
|
||||
log.Println("Starting new wallet")
|
||||
}
|
||||
err = s.downloadBlockchainDB()
|
||||
if err != nil {
|
||||
return errors.Prefix("failure in downloading blockchain.db", err)
|
||||
}
|
||||
|
||||
defer s.stopAndUploadWallet(&e)
|
||||
|
||||
|
|
@ -450,7 +338,14 @@ func (s *Sync) stopAndUploadWallet(e *error) {
|
|||
if err != nil {
|
||||
if *e == nil {
|
||||
e = &err
|
||||
return
|
||||
} else {
|
||||
*e = errors.Prefix("failure uploading wallet", *e)
|
||||
}
|
||||
}
|
||||
err = s.uploadBlockchainDB()
|
||||
if err != nil {
|
||||
if *e == nil {
|
||||
e = &err
|
||||
} else {
|
||||
*e = errors.Prefix("failure uploading wallet", *e)
|
||||
}
|
||||
|
|
|
|||
26
util/util.go
26
util/util.go
|
|
@ -52,7 +52,7 @@ func GetLBRYNetDir() string {
|
|||
}
|
||||
|
||||
func GetLbryumDir() string {
|
||||
lbryumDir := os.Getenv("LBRYNET_WALLETS_DIR")
|
||||
lbryumDir := os.Getenv("LBRYUM_DIR")
|
||||
if lbryumDir == "" {
|
||||
usr, err := user.Current()
|
||||
if err != nil {
|
||||
|
|
@ -366,9 +366,31 @@ func GetDefaultWalletPath() string {
|
|||
defaultWalletDir = os.Getenv("HOME") + "/.lbryum_regtest/wallets/default_wallet"
|
||||
}
|
||||
|
||||
walletPath := os.Getenv("LBRYNET_WALLETS_DIR")
|
||||
walletPath := os.Getenv("LBRYUM_DIR")
|
||||
if walletPath != "" {
|
||||
defaultWalletDir = walletPath + "/wallets/default_wallet"
|
||||
}
|
||||
return defaultWalletDir
|
||||
}
|
||||
func GetBlockchainDBPath() string {
|
||||
lbryumDir := os.Getenv("LBRYUM_DIR")
|
||||
if lbryumDir == "" {
|
||||
if IsRegTest() {
|
||||
lbryumDir = os.Getenv("HOME") + "/.lbryum_regtest"
|
||||
} else {
|
||||
lbryumDir = os.Getenv("HOME") + "/.lbryum"
|
||||
}
|
||||
}
|
||||
defaultDB := lbryumDir + "/lbc_mainnet/blockchain.db"
|
||||
if IsRegTest() {
|
||||
defaultDB = lbryumDir + "/lbc_regtest/blockchain.db"
|
||||
}
|
||||
return defaultDB
|
||||
}
|
||||
func GetBlockchainDirectoryName() string {
|
||||
ledger := "lbc_mainnet"
|
||||
if IsRegTest() {
|
||||
ledger = "lbc_regtest"
|
||||
}
|
||||
return ledger
|
||||
}
|
||||
|
|
|
|||
186
ytapi/ytapi.go
186
ytapi/ytapi.go
|
|
@ -2,6 +2,8 @@ package ytapi
|
|||
|
||||
import (
|
||||
"bufio"
|
||||
"encoding/json"
|
||||
"io/ioutil"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"sort"
|
||||
|
|
@ -25,8 +27,6 @@ import (
|
|||
|
||||
"github.com/aws/aws-sdk-go/aws"
|
||||
log "github.com/sirupsen/logrus"
|
||||
"google.golang.org/api/googleapi/transport"
|
||||
ytlib "google.golang.org/api/youtube/v3"
|
||||
)
|
||||
|
||||
type Video interface {
|
||||
|
|
@ -56,7 +56,7 @@ var mostRecentlyFailedChannel string // TODO: fix this hack!
|
|||
func GetVideosToSync(config *sdk.APIConfig, channelID string, syncedVideos map[string]sdk.SyncedVideo, quickSync bool, maxVideos int, videoParams VideoParams) ([]Video, error) {
|
||||
|
||||
var videos []Video
|
||||
if quickSync {
|
||||
if quickSync && maxVideos > 50 {
|
||||
maxVideos = 50
|
||||
}
|
||||
videoIDs, err := downloader.GetPlaylistVideoIDs(channelID, maxVideos, videoParams.Stopper.Ch(), videoParams.IPPool)
|
||||
|
|
@ -143,24 +143,36 @@ func CountVideosInChannel(channelID string) (int, error) {
|
|||
return num, nil
|
||||
}
|
||||
|
||||
func ChannelInfo(apiKey, channelID string) (*ytlib.ChannelSnippet, *ytlib.ChannelBrandingSettings, error) {
|
||||
return nil, nil, errors.Err("ChannelInfo doesn't work yet because we're focused on existing channels")
|
||||
func ChannelInfo(channelID string) (*YoutubeStatsResponse, error) {
|
||||
//return nil, nil, errors.Err("ChannelInfo doesn't work yet because we're focused on existing channels")
|
||||
url := "https://www.youtube.com/channel/" + channelID + "/about"
|
||||
|
||||
service, err := ytlib.New(&http.Client{Transport: &transport.APIKey{Key: apiKey}})
|
||||
req, _ := http.NewRequest("GET", url, nil)
|
||||
|
||||
req.Header.Add("User-Agent", "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.116 Safari/537.36")
|
||||
req.Header.Add("Accept", "*/*")
|
||||
|
||||
res, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
return nil, nil, errors.Prefix("error creating YouTube service", err)
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
|
||||
response, err := service.Channels.List("snippet,brandingSettings").Id(channelID).Do()
|
||||
defer res.Body.Close()
|
||||
body, err := ioutil.ReadAll(res.Body)
|
||||
if err != nil {
|
||||
return nil, nil, errors.Prefix("error getting channel details", err)
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
pageBody := string(body)
|
||||
dataStartIndex := strings.Index(pageBody, "window[\"ytInitialData\"] = ") + 26
|
||||
dataEndIndex := strings.Index(pageBody, "]}}};") + 4
|
||||
|
||||
data := pageBody[dataStartIndex:dataEndIndex]
|
||||
var decodedResponse YoutubeStatsResponse
|
||||
err = json.Unmarshal([]byte(data), &decodedResponse)
|
||||
if err != nil {
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
|
||||
if len(response.Items) < 1 {
|
||||
return nil, nil, errors.Err("youtube channel not found")
|
||||
}
|
||||
|
||||
return response.Items[0].Snippet, response.Items[0].BrandingSettings, nil
|
||||
return &decodedResponse, nil
|
||||
}
|
||||
|
||||
func getVideos(config *sdk.APIConfig, channelID string, videoIDs []string, stopChan stop.Chan, ipPool *ip_manager.IPPool) ([]*ytdl.YtdlVideo, error) {
|
||||
|
|
@ -205,3 +217,147 @@ func getVideos(config *sdk.APIConfig, channelID string, videoIDs []string, stopC
|
|||
}
|
||||
return videos, nil
|
||||
}
|
||||
|
||||
type YoutubeStatsResponse struct {
|
||||
Contents struct {
|
||||
TwoColumnBrowseResultsRenderer struct {
|
||||
Tabs []struct {
|
||||
TabRenderer struct {
|
||||
Title string `json:"title"`
|
||||
Selected bool `json:"selected"`
|
||||
Content struct {
|
||||
SectionListRenderer struct {
|
||||
Contents []struct {
|
||||
ItemSectionRenderer struct {
|
||||
Contents []struct {
|
||||
ChannelAboutFullMetadataRenderer struct {
|
||||
Description struct {
|
||||
SimpleText string `json:"simpleText"`
|
||||
} `json:"description"`
|
||||
ViewCountText struct {
|
||||
SimpleText string `json:"simpleText"`
|
||||
} `json:"viewCountText"`
|
||||
JoinedDateText struct {
|
||||
Runs []struct {
|
||||
Text string `json:"text"`
|
||||
} `json:"runs"`
|
||||
} `json:"joinedDateText"`
|
||||
CanonicalChannelURL string `json:"canonicalChannelUrl"`
|
||||
BypassBusinessEmailCaptcha bool `json:"bypassBusinessEmailCaptcha"`
|
||||
Title struct {
|
||||
SimpleText string `json:"simpleText"`
|
||||
} `json:"title"`
|
||||
Avatar struct {
|
||||
Thumbnails []struct {
|
||||
URL string `json:"url"`
|
||||
Width int `json:"width"`
|
||||
Height int `json:"height"`
|
||||
} `json:"thumbnails"`
|
||||
} `json:"avatar"`
|
||||
ShowDescription bool `json:"showDescription"`
|
||||
DescriptionLabel struct {
|
||||
Runs []struct {
|
||||
Text string `json:"text"`
|
||||
} `json:"runs"`
|
||||
} `json:"descriptionLabel"`
|
||||
DetailsLabel struct {
|
||||
Runs []struct {
|
||||
Text string `json:"text"`
|
||||
} `json:"runs"`
|
||||
} `json:"detailsLabel"`
|
||||
ChannelID string `json:"channelId"`
|
||||
} `json:"channelAboutFullMetadataRenderer"`
|
||||
} `json:"contents"`
|
||||
} `json:"itemSectionRenderer"`
|
||||
} `json:"contents"`
|
||||
} `json:"sectionListRenderer"`
|
||||
} `json:"content"`
|
||||
} `json:"tabRenderer"`
|
||||
} `json:"tabs"`
|
||||
} `json:"twoColumnBrowseResultsRenderer"`
|
||||
} `json:"contents"`
|
||||
Header struct {
|
||||
C4TabbedHeaderRenderer struct {
|
||||
ChannelID string `json:"channelId"`
|
||||
Title string `json:"title"`
|
||||
Avatar struct {
|
||||
Thumbnails []struct {
|
||||
URL string `json:"url"`
|
||||
Width int `json:"width"`
|
||||
Height int `json:"height"`
|
||||
} `json:"thumbnails"`
|
||||
} `json:"avatar"`
|
||||
Banner struct {
|
||||
Thumbnails []struct {
|
||||
URL string `json:"url"`
|
||||
Width int `json:"width"`
|
||||
Height int `json:"height"`
|
||||
} `json:"thumbnails"`
|
||||
} `json:"banner"`
|
||||
VisitTracking struct {
|
||||
RemarketingPing string `json:"remarketingPing"`
|
||||
} `json:"visitTracking"`
|
||||
SubscriberCountText struct {
|
||||
SimpleText string `json:"simpleText"`
|
||||
} `json:"subscriberCountText"`
|
||||
} `json:"c4TabbedHeaderRenderer"`
|
||||
} `json:"header"`
|
||||
Metadata struct {
|
||||
ChannelMetadataRenderer struct {
|
||||
Title string `json:"title"`
|
||||
Description string `json:"description"`
|
||||
RssURL string `json:"rssUrl"`
|
||||
ChannelConversionURL string `json:"channelConversionUrl"`
|
||||
ExternalID string `json:"externalId"`
|
||||
Keywords string `json:"keywords"`
|
||||
OwnerUrls []string `json:"ownerUrls"`
|
||||
Avatar struct {
|
||||
Thumbnails []struct {
|
||||
URL string `json:"url"`
|
||||
Width int `json:"width"`
|
||||
Height int `json:"height"`
|
||||
} `json:"thumbnails"`
|
||||
} `json:"avatar"`
|
||||
ChannelURL string `json:"channelUrl"`
|
||||
IsFamilySafe bool `json:"isFamilySafe"`
|
||||
VanityChannelURL string `json:"vanityChannelUrl"`
|
||||
} `json:"channelMetadataRenderer"`
|
||||
} `json:"metadata"`
|
||||
Topbar struct {
|
||||
DesktopTopbarRenderer struct {
|
||||
CountryCode string `json:"countryCode"`
|
||||
} `json:"desktopTopbarRenderer"`
|
||||
} `json:"topbar"`
|
||||
Microformat struct {
|
||||
MicroformatDataRenderer struct {
|
||||
URLCanonical string `json:"urlCanonical"`
|
||||
Title string `json:"title"`
|
||||
Description string `json:"description"`
|
||||
Thumbnail struct {
|
||||
Thumbnails []struct {
|
||||
URL string `json:"url"`
|
||||
Width int `json:"width"`
|
||||
Height int `json:"height"`
|
||||
} `json:"thumbnails"`
|
||||
} `json:"thumbnail"`
|
||||
SiteName string `json:"siteName"`
|
||||
AppName string `json:"appName"`
|
||||
AndroidPackage string `json:"androidPackage"`
|
||||
IosAppStoreID string `json:"iosAppStoreId"`
|
||||
IosAppArguments string `json:"iosAppArguments"`
|
||||
OgType string `json:"ogType"`
|
||||
URLApplinksWeb string `json:"urlApplinksWeb"`
|
||||
URLApplinksIos string `json:"urlApplinksIos"`
|
||||
URLApplinksAndroid string `json:"urlApplinksAndroid"`
|
||||
URLTwitterIos string `json:"urlTwitterIos"`
|
||||
URLTwitterAndroid string `json:"urlTwitterAndroid"`
|
||||
TwitterCardType string `json:"twitterCardType"`
|
||||
TwitterSiteHandle string `json:"twitterSiteHandle"`
|
||||
SchemaDotOrgType string `json:"schemaDotOrgType"`
|
||||
Noindex bool `json:"noindex"`
|
||||
Unlisted bool `json:"unlisted"`
|
||||
FamilySafe bool `json:"familySafe"`
|
||||
Tags []string `json:"tags"`
|
||||
} `json:"microformatDataRenderer"`
|
||||
} `json:"microformat"`
|
||||
}
|
||||
|
|
|
|||
12
ytapi/ytapi_test.go
Normal file
12
ytapi/ytapi_test.go
Normal file
|
|
@ -0,0 +1,12 @@
|
|||
package ytapi
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
func TestChannelInfo(t *testing.T) {
|
||||
_, _, err := ChannelInfo("", "UCNQfQvFMPnInwsU_iGYArJQ")
|
||||
assert.NoError(t, err)
|
||||
}
|
||||
Loading…
Reference in a new issue