LBRYCommunity/ytsync
1
0
Fork 0
Code Issues 39 Pull requests 8 Projects Releases 197 Packages Wiki Activity

fixes to throttling

Browse source
This commit is contained in:
Niko Storni 2019-12-14 14:58:04 +01:00
parent 5df6db6e96
commit 2bde06e4b9
3 changed files with 12 additions and 14 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
ip_manager/throttle.go
View file

@ -13,7 +13,7 @@ import (
log "github.com/sirupsen/logrus" log "github.com/sirupsen/logrus"
) )
const IPCooldownPeriod = 35 * time.Second const IPCooldownPeriod = 20 * time.Second
const unbanTimeout = 3 * time.Hour const unbanTimeout = 3 * time.Hour
var stopper = stop.New() var stopper = stop.New()

1
manager/ytsync.go
View file

@ -872,6 +872,7 @@ func (s *Sync) startWorker(workerNum int) {
"giving up after 0 fragment retries", "giving up after 0 fragment retries",
"Sorry about that", "Sorry about that",
"This video is not available", "This video is not available",
"requested format not available",
} }
if util.SubstringInSlice(err.Error(), errorsNoRetry) { if util.SubstringInSlice(err.Error(), errorsNoRetry) {
log.Println("This error should not be retried at all") log.Println("This error should not be retried at all")

23
sources/youtubeVideo.go
View file

@ -178,7 +178,7 @@ func (v *YoutubeVideo) getAbbrevDescription() string {
return description + "\n..." + additionalDescription return description + "\n..." + additionalDescription
} }
func (v *YoutubeVideo) download(useIPv6 bool) error { func (v *YoutubeVideo) download() error {
videoPath := v.getFullPath() videoPath := v.getFullPath()
err := os.Mkdir(v.videoDir(), 0777) err := os.Mkdir(v.videoDir(), 0777)
@ -281,7 +281,7 @@ runcmd:
v.pool.SetThrottled(sourceAddress, v.stopGroup) v.pool.SetThrottled(sourceAddress, v.stopGroup)
} else if strings.Contains(string(errorLog), "giving up after 0 fragment retries") && qualityIndex < len(qualities)-1 { } else if strings.Contains(string(errorLog), "giving up after 0 fragment retries") && qualityIndex < len(qualities)-1 {
qualityIndex++ qualityIndex++
goto runcmd goto runcmd //this bypasses the yt throttling IP redistribution... TODO: don't
} }
return errors.Err(string(errorLog)) return errors.Err(string(errorLog))
} }
@ -426,21 +426,18 @@ func (v *YoutubeVideo) Sync(daemon *jsonrpc.Client, params SyncParams, existingV
return v.downloadAndPublish(daemon, params) return v.downloadAndPublish(daemon, params)
} }
var isThrottled bool
func (v *YoutubeVideo) downloadAndPublish(daemon *jsonrpc.Client, params SyncParams) (*SyncSummary, error) { func (v *YoutubeVideo) downloadAndPublish(daemon *jsonrpc.Client, params SyncParams) (*SyncSummary, error) {
err := v.download(isThrottled) var err error
if err != nil { for {
if strings.Contains(err.Error(), "HTTP Error 429") && !isThrottled { err = v.download()
isThrottled = true if err != nil && strings.Contains(err.Error(), "HTTP Error 429") {
err = v.download(isThrottled) continue
if err != nil { } else if err != nil {
return nil, errors.Prefix("download error", err)
}
} else {
return nil, errors.Prefix("download error", err) return nil, errors.Prefix("download error", err)
} }
break
} }
log.Debugln("Downloaded " + v.id) log.Debugln("Downloaded " + v.id)
err = v.triggerThumbnailSave() err = v.triggerThumbnailSave()