LBRYCommunity/ytsync
1
0
Fork 0
Code Issues 39 Pull requests 8 Projects Releases 197 Packages Wiki Activity

don't waste loops

Browse source 
fix a deadlock?
This commit is contained in:
Niko Storni 2020-08-12 03:37:18 +02:00
parent 67da4142d5
commit eb8900c66a
1 changed files with 13 additions and 10 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

23
downloader/downloader.go
View file

@ -245,8 +245,8 @@ func getClient(ip *net.TCPAddr) *http.Client {
} }
func run(use string, args []string, withStdErr, withStdOut bool, stopChan stop.Chan, pool *ip_manager.IPPool) ([]string, error) { func run(use string, args []string, withStdErr, withStdOut bool, stopChan stop.Chan, pool *ip_manager.IPPool) ([]string, error) {
var maxtries = 10 var maxTries = 10
var attemps int var attempts int
var useragent []string var useragent []string
for { for {
sourceAddress, err := getIPFromPool(use, stopChan, pool) sourceAddress, err := getIPFromPool(use, stopChan, pool)
@ -295,7 +295,7 @@ func run(use string, args []string, withStdErr, withStdOut bool, stopChan stop.C
done := make(chan error, 1) done := make(chan error, 1)
go func() { go func() {
attemps++ attempts++
done <- cmd.Wait() done <- cmd.Wait()
}() }()
select { select {
@ -309,23 +309,26 @@ func run(use string, args []string, withStdErr, withStdOut bool, stopChan stop.C
if strings.Contains(err.Error(), "exit status 1") { if strings.Contains(err.Error(), "exit status 1") {
if strings.Contains(string(errorLog), "HTTP Error 429") || strings.Contains(string(errorLog), "returned non-zero exit status 8") { if strings.Contains(string(errorLog), "HTTP Error 429") || strings.Contains(string(errorLog), "returned non-zero exit status 8") {
pool.SetThrottled(sourceAddress) pool.SetThrottled(sourceAddress)
logrus.Debugf("known throttling error...try again (%d)", attemps) logrus.Debugf("known throttling error...try again (%d)", attempts)
continue
} }
if strings.Contains(string(errorLog), "YouTube said: Unable to extract video data") { if strings.Contains(string(errorLog), "YouTube said: Unable to extract video data") {
useragent = []string{"--user-agent", "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.88 Safari/537.36"} useragent = []string{"--user-agent", "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.88 Safari/537.36"}
if attemps == 1 { if attempts == 1 {
useragent = []string{"--user-agent", "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)"} useragent = []string{"--user-agent", "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)"}
} }
logrus.Debugf("known extraction issue, maybe user agent specification will work...try again (%d)", attemps) if attempts > 3 {
continue logrus.Debugf("It's pointless to keep trying here... skipping (%d)", attempts)
break
}
logrus.Debugf("known extraction issue, maybe user agent specification will work...try again (%d)", attempts)
} }
if attemps > maxtries { if attempts > maxTries {
logrus.Debug("too many tries returning failure") logrus.Debug("too many tries returning failure")
break break
} }
continue
} }
logrus.Debugf("Unkown error, returning failure: %s", err.Error()) logrus.Debugf("Unknown error, returning failure: %s", err.Error())
return nil, errors.Prefix("youtube-dl "+strings.Join(argsForCommand, " ")+" ["+string(errorLog)+"] ", err) return nil, errors.Prefix("youtube-dl "+strings.Join(argsForCommand, " ")+" ["+string(errorLog)+"] ", err)
} }
return strings.Split(strings.Replace(string(outLog), "\r\n", "\n", -1), "\n"), nil return strings.Split(strings.Replace(string(outLog), "\r\n", "\n", -1), "\n"), nil