fix api failures

update dependencies
fix e2e

e2e/docker-compose.yml

@@ -110,7 +110,7 @@ services:
   ## Internal APIs ##
   ###################
   internalapis:
-    image: lbry/internal-apis:master
+    image: odyseeteam/internal-apis:master
     restart: "no"
     ports:
       - "15400:8080"
@@ -128,7 +128,7 @@ services:
   ## Chainquery ##
   ################
   chainquery:
-    image: lbry/chainquery:master
+    image: odyseeteam/chainquery:master
     restart: "no"
     ports:
       - 6300:6300

go.mod

@@ -1,3 +1,5 @@
+go 1.17
+
 module github.com/lbryio/ytsync/v5
 
 replace github.com/btcsuite/btcd => github.com/lbryio/lbrycrd.go v0.0.0-20200203050410-e1076f12bf19
@@ -6,39 +8,146 @@ replace github.com/btcsuite/btcd => github.com/lbryio/lbrycrd.go v0.0.0-20200203
 //replace github.com/lbryio/reflector.go => /home/niko/go/src/github.com/lbryio/reflector.go/
 
 require (
-	github.com/Microsoft/go-winio v0.4.14 // indirect
 	github.com/abadojack/whatlanggo v1.0.1
-	github.com/armon/go-metrics v0.0.0-20190430140413-ec5e00d3c878 // indirect
+	github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d
-	github.com/asaskevich/govalidator v0.0.0-20200819183940-29e1ff8eb0bb
+	github.com/aws/aws-sdk-go v1.44.6
-	github.com/aws/aws-sdk-go v1.27.0
 	github.com/davecgh/go-spew v1.1.1
-	github.com/docker/distribution v2.7.1+incompatible // indirect
+	github.com/docker/docker v20.10.14+incompatible
-	github.com/docker/docker v1.13.1
+	github.com/lbryio/lbry.go/v2 v2.7.2-0.20220321182539-d0aeb0c22b22
-	github.com/docker/go-connections v0.4.0 // indirect
+	github.com/lbryio/reflector.go v1.1.3-0.20220502213236-c211f83ba732
-	github.com/docker/go-units v0.4.0 // indirect
+	github.com/mitchellh/go-ps v1.0.0
-	github.com/hashicorp/go-immutable-radix v1.1.0 // indirect
+	github.com/prometheus/client_golang v1.12.1
-	github.com/hashicorp/go-msgpack v0.5.5 // indirect
+	github.com/shopspring/decimal v1.3.1
-	github.com/hashicorp/go-sockaddr v1.0.2 // indirect
-	github.com/hashicorp/golang-lru v0.5.4 // indirect
-	github.com/kr/pretty v0.2.1 // indirect
-	github.com/lbryio/lbry.go v1.1.2 // indirect
-	github.com/lbryio/lbry.go/v2 v2.7.2-0.20210824154606-3e18b74da08b
-	github.com/lbryio/reflector.go v1.1.3-0.20211214213601-4d8e7739d704
-	github.com/mitchellh/go-ps v0.0.0-20190716172923-621e5597135b
-	github.com/opencontainers/go-digest v1.0.0-rc1 // indirect
-	github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2 // indirect
-	github.com/prometheus/client_golang v1.10.0
-	github.com/shopspring/decimal v0.0.0-20191009025716-f1972eb1d1f5
 	github.com/sirupsen/logrus v1.8.1
-	github.com/spf13/afero v1.4.1 // indirect
+	github.com/spf13/cobra v1.4.0
-	github.com/spf13/cobra v1.1.3
+	github.com/stretchr/testify v1.7.1
-	github.com/spf13/viper v1.7.1 // indirect
-	github.com/stretchr/testify v1.7.0
 	github.com/tkanos/gonfig v0.0.0-20210106201359-53e13348de2f
-	github.com/vbauerster/mpb/v7 v7.0.2
+	github.com/vbauerster/mpb/v7 v7.4.1
-	gopkg.in/ini.v1 v1.60.2 // indirect
+	gopkg.in/vansante/go-ffprobe.v2 v2.0.3
-	gopkg.in/vansante/go-ffprobe.v2 v2.0.2
 	gotest.tools v2.2.0+incompatible
 )
 
-go 1.13
+require (
+	github.com/Microsoft/go-winio v0.5.1 // indirect
+	github.com/VividCortex/ewma v1.2.0 // indirect
+	github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d // indirect
+	github.com/armon/go-metrics v0.0.0-20190430140413-ec5e00d3c878 // indirect
+	github.com/beorn7/perks v1.0.1 // indirect
+	github.com/bluele/gcache v0.0.2 // indirect
+	github.com/brk0v/directio v0.0.0-20190225130936-69406e757cf7 // indirect
+	github.com/btcsuite/btcd v0.0.0-20190824003749-130ea5bddde3 // indirect
+	github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f // indirect
+	github.com/btcsuite/btcutil v0.0.0-20190425235716-9e5f4b9a998d // indirect
+	github.com/btcsuite/go-socks v0.0.0-20170105172521-4720035b7bfd // indirect
+	github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792 // indirect
+	github.com/c2h5oh/datasize v0.0.0-20200825124411-48ed595a09d2 // indirect
+	github.com/cespare/xxhash/v2 v2.1.2 // indirect
+	github.com/cheekybits/genny v1.0.0 // indirect
+	github.com/containerd/containerd v1.6.4 // indirect
+	github.com/docker/distribution v2.7.1+incompatible // indirect
+	github.com/docker/go-connections v0.4.0 // indirect
+	github.com/docker/go-units v0.4.0 // indirect
+	github.com/ekyoung/gin-nice-recovery v0.0.0-20160510022553-1654dca486db // indirect
+	github.com/fatih/structs v1.1.0 // indirect
+	github.com/fsnotify/fsnotify v1.4.9 // indirect
+	github.com/ghodss/yaml v1.0.0 // indirect
+	github.com/gin-contrib/sse v0.1.0 // indirect
+	github.com/gin-gonic/gin v1.7.7 // indirect
+	github.com/go-errors/errors v1.1.1 // indirect
+	github.com/go-ini/ini v1.48.0 // indirect
+	github.com/go-playground/locales v0.13.0 // indirect
+	github.com/go-playground/universal-translator v0.17.0 // indirect
+	github.com/go-playground/validator/v10 v10.4.1 // indirect
+	github.com/go-sql-driver/mysql v1.6.0 // indirect
+	github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0 // indirect
+	github.com/gofrs/uuid v3.2.0+incompatible // indirect
+	github.com/gogo/protobuf v1.3.2 // indirect
+	github.com/golang/protobuf v1.5.2 // indirect
+	github.com/google/btree v1.0.1 // indirect
+	github.com/google/go-cmp v0.5.7 // indirect
+	github.com/gorilla/mux v1.8.0 // indirect
+	github.com/gorilla/rpc v1.2.0 // indirect
+	github.com/gorilla/websocket v1.4.2 // indirect
+	github.com/hashicorp/errwrap v1.1.0 // indirect
+	github.com/hashicorp/go-immutable-radix v1.1.0 // indirect
+	github.com/hashicorp/go-msgpack v0.5.5 // indirect
+	github.com/hashicorp/go-multierror v1.1.1 // indirect
+	github.com/hashicorp/go-sockaddr v1.0.2 // indirect
+	github.com/hashicorp/golang-lru v0.5.4 // indirect
+	github.com/hashicorp/hcl v1.0.0 // indirect
+	github.com/hashicorp/memberlist v0.3.0 // indirect
+	github.com/hashicorp/serf v0.9.7 // indirect
+	github.com/inconshreveable/go-update v0.0.0-20160112193335-8152e7eb6ccf // indirect
+	github.com/inconshreveable/mousetrap v1.0.0 // indirect
+	github.com/jmespath/go-jmespath v0.4.0 // indirect
+	github.com/johntdyer/slack-go v0.0.0-20180213144715-95fac1160b22 // indirect
+	github.com/johntdyer/slackrus v0.0.0-20211215141436-33e4a270affb // indirect
+	github.com/json-iterator/go v1.1.12 // indirect
+	github.com/karrick/godirwalk v1.16.1 // indirect
+	github.com/kr/text v0.2.0 // indirect
+	github.com/lbryio/chainquery v1.9.0 // indirect
+	github.com/lbryio/lbry.go v1.1.2 // indirect
+	github.com/lbryio/types v0.0.0-20220224142228-73610f6654a6 // indirect
+	github.com/leodido/go-urn v1.2.0 // indirect
+	github.com/lucas-clemente/quic-go v0.27.0 // indirect
+	github.com/lyoshenka/bencode v0.0.0-20180323155644-b7abd7672df5 // indirect
+	github.com/magiconair/properties v1.8.1 // indirect
+	github.com/marten-seemann/qpack v0.2.1 // indirect
+	github.com/marten-seemann/qtls-go1-16 v0.1.5 // indirect
+	github.com/marten-seemann/qtls-go1-17 v0.1.1 // indirect
+	github.com/marten-seemann/qtls-go1-18 v0.1.1 // indirect
+	github.com/mattn/go-isatty v0.0.12 // indirect
+	github.com/mattn/go-runewidth v0.0.13 // indirect
+	github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect
+	github.com/miekg/dns v1.1.41 // indirect
+	github.com/mitchellh/mapstructure v1.5.0 // indirect
+	github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6 // indirect
+	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
+	github.com/modern-go/reflect2 v1.0.2 // indirect
+	github.com/morikuni/aec v1.0.0 // indirect
+	github.com/nlopes/slack v0.6.0 // indirect
+	github.com/nxadm/tail v1.4.8 // indirect
+	github.com/onsi/ginkgo v1.16.4 // indirect
+	github.com/onsi/gomega v1.17.0 // indirect
+	github.com/opencontainers/go-digest v1.0.0 // indirect
+	github.com/opencontainers/image-spec v1.0.3-0.20211202183452-c5a74bcca799 // indirect
+	github.com/pelletier/go-toml v1.9.3 // indirect
+	github.com/pkg/errors v0.9.1 // indirect
+	github.com/pmezard/go-difflib v1.0.0 // indirect
+	github.com/prometheus/client_model v0.2.0 // indirect
+	github.com/prometheus/common v0.32.1 // indirect
+	github.com/prometheus/procfs v0.7.3 // indirect
+	github.com/rivo/uniseg v0.2.0 // indirect
+	github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529 // indirect
+	github.com/spf13/afero v1.4.1 // indirect
+	github.com/spf13/cast v1.4.1 // indirect
+	github.com/spf13/jwalterweatherman v1.0.0 // indirect
+	github.com/spf13/pflag v1.0.5 // indirect
+	github.com/spf13/viper v1.7.1 // indirect
+	github.com/subosito/gotenv v1.2.0 // indirect
+	github.com/ugorji/go/codec v1.1.7 // indirect
+	github.com/volatiletech/inflect v0.0.0-20170731032912-e7201282ae8d // indirect
+	github.com/volatiletech/null v8.0.0+incompatible // indirect
+	github.com/volatiletech/sqlboiler v3.4.0+incompatible // indirect
+	github.com/ybbus/jsonrpc v2.1.2+incompatible // indirect
+	go.uber.org/atomic v1.9.0 // indirect
+	golang.org/x/crypto v0.0.0-20210817164053-32db794688a5 // indirect
+	golang.org/x/mod v0.4.2 // indirect
+	golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd // indirect
+	golang.org/x/sync v0.0.0-20210220032951-036812b2e83c // indirect
+	golang.org/x/sys v0.0.0-20220209214540-3681064d5158 // indirect
+	golang.org/x/text v0.3.7 // indirect
+	golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac // indirect
+	golang.org/x/tools v0.1.5 // indirect
+	golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
+	google.golang.org/genproto v0.0.0-20211208223120-3a66f561d7aa // indirect
+	google.golang.org/grpc v1.43.0 // indirect
+	google.golang.org/protobuf v1.27.1 // indirect
+	gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect
+	gopkg.in/ini.v1 v1.60.2 // indirect
+	gopkg.in/nullbio/null.v6 v6.0.0-20161116030900-40264a2e6b79 // indirect
+	gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect
+	gopkg.in/yaml.v2 v2.4.0 // indirect
+	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
+	gotest.tools/v3 v3.2.0 // indirect
+)

manager/ytsync.go

@@ -562,7 +562,11 @@ func (s *Sync) updateRemoteDB(claims []jsonrpc.Claim, ownClaims []jsonrpc.Claim)
 		if sv.Transferred || sv.IsLbryFirst {
 			_, ok := allClaimsInfo[vID]
 			if !ok && sv.Published {
-				searchResponse, err := s.daemon.ClaimSearch(nil, &sv.ClaimID, nil, nil, 1, 20)
+				searchResponse, err := s.daemon.ClaimSearch(jsonrpc.ClaimSearchArgs{
+					ClaimID:  &sv.ClaimID,
+					Page:     1,
+					PageSize: 20,
+				})
 				if err != nil {
 					log.Error(err.Error())
 					continue

sdk/api.go

@@ -61,13 +61,10 @@ func (a *APIConfig) FetchChannels(status string, cliFlags *shared.SyncFlags) ([]
 		"channel_id":  {cliFlags.ChannelID},
 	})
 	if err != nil {
-		if strings.Contains(err.Error(), "EOF") {
+		util.SendErrorToSlack("error while trying to call %s. Waiting to retry: %s", endpoint, err.Error())
-			util.SendErrorToSlack("EOF error while trying to call %s. Waiting to retry", endpoint)
 		time.Sleep(30 * time.Second)
 		return a.FetchChannels(status, cliFlags)
 	}
-		return nil, errors.Err(err)
-	}
 	defer res.Body.Close()
 	body, _ := ioutil.ReadAll(res.Body)
 	if res.StatusCode != http.StatusOK {
@@ -124,13 +121,10 @@ func (a *APIConfig) SetChannelCert(certHex string, channelID string) error {
 		"auth_token":       {a.ApiToken},
 	})
 	if err != nil {
-		if strings.Contains(err.Error(), "EOF") {
+		util.SendErrorToSlack("error while trying to call %s. Waiting to retry: %s", endpoint, err.Error())
-			util.SendErrorToSlack("EOF error while trying to call %s. Waiting to retry", endpoint)
 		time.Sleep(30 * time.Second)
 		return a.SetChannelCert(certHex, channelID)
 	}
-		return errors.Err(err)
-	}
 	defer res.Body.Close()
 	body, _ := ioutil.ReadAll(res.Body)
 	if res.StatusCode != http.StatusOK {
@@ -172,13 +166,10 @@ func (a *APIConfig) SetChannelStatus(channelID string, status string, failureRea
 	}
 	res, err := http.PostForm(endpoint, params)
 	if err != nil {
-		if strings.Contains(err.Error(), "EOF") {
+		util.SendErrorToSlack("error while trying to call %s. Waiting to retry: %s", endpoint, err.Error())
-			util.SendErrorToSlack("EOF error while trying to call %s. Waiting to retry", endpoint)
 		time.Sleep(30 * time.Second)
 		return a.SetChannelStatus(channelID, status, failureReason, transferState)
 	}
-		return nil, nil, errors.Err(err)
-	}
 	defer res.Body.Close()
 	body, _ := ioutil.ReadAll(res.Body)
 	if res.StatusCode >= http.StatusInternalServerError {
@@ -222,13 +213,10 @@ func (a *APIConfig) SetChannelClaimID(channelID string, channelClaimID string) e
 		"channel_claim_id": {channelClaimID},
 	})
 	if err != nil {
-		if strings.Contains(err.Error(), "EOF") {
+		util.SendErrorToSlack("error while trying to call %s. Waiting to retry: %s", endpoint, err.Error())
-			util.SendErrorToSlack("EOF error while trying to call %s. Waiting to retry", endpoint)
 		time.Sleep(30 * time.Second)
 		return a.SetChannelClaimID(channelID, channelClaimID)
 	}
-		return errors.Err(err)
-	}
 	defer res.Body.Close()
 	body, _ := ioutil.ReadAll(res.Body)
 	if res.StatusCode != http.StatusOK {
@@ -266,13 +254,10 @@ func (a *APIConfig) DeleteVideos(videos []string) error {
 	}
 	res, err := http.PostForm(endpoint, vals)
 	if err != nil {
-		if strings.Contains(err.Error(), "EOF") {
+		util.SendErrorToSlack("error while trying to call %s. Waiting to retry: %s", endpoint, err.Error())
-			util.SendErrorToSlack("EOF error while trying to call %s. Waiting to retry", endpoint)
 		time.Sleep(30 * time.Second)
 		return a.DeleteVideos(videos)
 	}
-		return errors.Err(err)
-	}
 	defer res.Body.Close()
 	body, _ := ioutil.ReadAll(res.Body)
 	if res.StatusCode != http.StatusOK {
@@ -332,13 +317,10 @@ func (a *APIConfig) MarkVideoStatus(status shared.VideoStatus) error {
 	}
 	res, err := http.PostForm(endpoint, vals)
 	if err != nil {
-		if strings.Contains(err.Error(), "EOF") {
+		util.SendErrorToSlack("error while trying to call %s. Waiting to retry: %s", endpoint, err.Error())
-			util.SendErrorToSlack("EOF error while trying to call %s for %s. Waiting to retry", endpoint, status.ClaimName)
 		time.Sleep(30 * time.Second)
 		return a.MarkVideoStatus(status)
 	}
-		return errors.Err(err)
-	}
 	defer res.Body.Close()
 	body, _ := ioutil.ReadAll(res.Body)
 	if res.StatusCode != http.StatusOK {
@@ -374,13 +356,10 @@ func (a *APIConfig) VideoState(videoID string) (string, error) {
 
 	res, err := http.PostForm(endpoint, vals)
 	if err != nil {
-		if strings.Contains(err.Error(), "EOF") {
+		util.SendErrorToSlack("error while trying to call %s. Waiting to retry: %s", endpoint, err.Error())
-			util.SendErrorToSlack("EOF error while trying to call %s. Waiting to retry", endpoint)
 		time.Sleep(30 * time.Second)
 		return a.VideoState(videoID)
 	}
-		return "", errors.Err(err)
-	}
 	defer res.Body.Close()
 	body, _ := ioutil.ReadAll(res.Body)
 	if res.StatusCode == http.StatusNotFound {
@@ -428,13 +407,10 @@ func (a *APIConfig) GetReleasedDate(videoID string) (*VideoRelease, error) {
 
 	res, err := http.PostForm(endpoint, vals)
 	if err != nil {
-		if strings.Contains(err.Error(), "EOF") {
+		util.SendErrorToSlack("error while trying to call %s. Waiting to retry: %s", endpoint, err.Error())
-			util.SendErrorToSlack("EOF error while trying to call %s. Waiting to retry", endpoint)
 		time.Sleep(30 * time.Second)
 		return a.GetReleasedDate(videoID)
 	}
-		return nil, errors.Err(err)
-	}
 	defer res.Body.Close()
 	body, _ := ioutil.ReadAll(res.Body)
 	if res.StatusCode == http.StatusNotFound {

sources/youtubeVideo.go

@@ -515,21 +515,28 @@ func (v *YoutubeVideo) trackProgressBar(argsWithFilters []string, ticker *time.T
 			bar.Completed()
 			bar.Abort(true)
 		}()
+		origSize := int64(0)
+		lastUpdate := time.Now()
 		for {
 			select {
 			case <-done.Ch():
 				return
 			case <-ticker.C:
+				var err error
 				size, err := logUtils.DirSize(v.videoDir())
 				if err != nil {
 					log.Errorf("error while getting size of download directory: %s", errors.FullTrace(err))
 					return
 				}
+				if size > origSize {
+					origSize = size
 					bar.SetCurrent(size)
 					if size > int64(videoSize+audioSize) {
 						bar.SetTotal(size+2048, false)
 					}
-				bar.DecoratorEwmaUpdate(400 * time.Millisecond)
+					bar.DecoratorEwmaUpdate(time.Since(lastUpdate))
+					lastUpdate = time.Now()
+				}
 			}
 		}
 	}()
@@ -862,7 +869,11 @@ func (v *YoutubeVideo) getMetadata() (languages []string, locations []jsonrpc.Lo
 }
 
 func (v *YoutubeVideo) reprocess(daemon *jsonrpc.Client, params SyncParams, existingVideoData *sdk.SyncedVideo) (*SyncSummary, error) {
-	c, err := daemon.ClaimSearch(nil, &existingVideoData.ClaimID, nil, nil, 1, 20)
+	c, err := daemon.ClaimSearch(jsonrpc.ClaimSearchArgs{
+		ClaimID:  &existingVideoData.ClaimID,
+		Page:     1,
+		PageSize: 20,
+	})
 	if err != nil {
 		return nil, errors.Err(err)
 	}