LBRYCommunity/ytsync
1
0
Fork 0
Code Issues 39 Pull requests 8 Projects Releases 197 Packages Wiki Activity

Add retry on IP failures

Browse source
This commit is contained in:
Mark Beamer Jr 2020-07-30 12:14:06 -04:00 committed by Alex Grintsvayg
parent ad27425471
commit b1e10e7b09
No known key found for this signature in database
GPG key ID: AEB3F089F86A22B5
2 changed files with 79 additions and 65 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
Makefile
View file

@ -13,7 +13,7 @@ LDFLAGS = -ldflags "-X ${IMPORT_PATH}/meta.Version=${VERSION} -X ${IMPORT_PATH}/
build: build:
mkdir -p ${BIN_DIR} && CGO_ENABLED=0 go build ${LDFLAGS} -asmflags -trimpath=${DIR} -o ${BIN_DIR}/${BINARY} main.go mkdir -p ${BIN_DIR} && CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build ${LDFLAGS} -asmflags -trimpath=${DIR} -o ${BIN_DIR}/${BINARY} main.go
clean: clean:
if [ -f ${BIN_DIR}/${BINARY} ]; then rm ${BIN_DIR}/${BINARY}; fi if [ -f ${BIN_DIR}/${BINARY} ]; then rm ${BIN_DIR}/${BINARY}; fi

142
downloader/downloader.go
View file

@ -232,80 +232,94 @@ func getClient(ip *net.TCPAddr) *http.Client {
} }
func run(use string, args []string, withStdErr, withStdOut bool, stopChan stop.Chan, v ytapi.VideoParams) ([]string, error) { func run(use string, args []string, withStdErr, withStdOut bool, stopChan stop.Chan, v ytapi.VideoParams) ([]string, error) {
var sourceAddress string var maxtries = 10
var err error var attemps int
for { for {
sourceAddress, err = v.IPPool.GetIP(use) var sourceAddress string
if err != nil { var err error
if errors.Is(err, ip_manager.ErrAllThrottled) { for {
select { sourceAddress, err = v.IPPool.GetIP(use)
case <-stopChan: if err != nil {
return nil, errors.Err("interrupted by user") if errors.Is(err, ip_manager.ErrAllThrottled) {
default: select {
time.Sleep(ip_manager.IPCooldownPeriod) case <-stopChan:
continue return nil, errors.Err("interrupted by user")
default:
time.Sleep(ip_manager.IPCooldownPeriod)
continue
}
} else {
return nil, err
} }
} else { }
return nil, err break
}
defer v.IPPool.ReleaseIP(sourceAddress)
args = append(args, "--source-address", sourceAddress)
cmd := exec.Command("youtube-dl", args...)
logrus.Printf("Running command youtube-dl %s", strings.Join(args, " "))
var stderr io.ReadCloser
var errorLog []byte
if withStdErr {
var err error
stderr, err = cmd.StderrPipe()
if err != nil {
return nil, errors.Err(err)
}
errorLog, err = ioutil.ReadAll(stderr)
if err != nil {
return nil, errors.Err(err)
} }
} }
break
}
defer v.IPPool.ReleaseIP(sourceAddress)
args = append(args, "--source-address", sourceAddress)
cmd := exec.Command("youtube-dl", args...)
logrus.Printf("Running command youtube-dl %s", strings.Join(args, " "))
var stderr io.ReadCloser var stdout io.ReadCloser
var errorLog []byte var outLog []byte
if withStdErr { if withStdOut {
var err error var err error
stderr, err = cmd.StderrPipe() stdout, err = cmd.StdoutPipe()
if err != nil { if err != nil {
return nil, errors.Err(err) return nil, errors.Err(err)
} }
errorLog, err = ioutil.ReadAll(stderr)
if err != nil {
return nil, errors.Err(err)
}
}
var stdout io.ReadCloser if err := cmd.Start(); err != nil {
var outLog []byte return nil, errors.Err(err)
if withStdOut { }
var err error outLog, err = ioutil.ReadAll(stdout)
stdout, err = cmd.StdoutPipe() if err != nil {
if err != nil { return nil, errors.Err(err)
return nil, errors.Err(err) }
} }
if err := cmd.Start(); err != nil { done := make(chan error, 1)
return nil, errors.Err(err) go func() {
attemps++
done <- cmd.Wait()
}()
select {
case <-stopChan:
if err := cmd.Process.Kill(); err != nil {
return nil, errors.Prefix("failed to kill command after stopper cancellation", err)
}
return nil, errors.Err("canceled by stopper")
case err := <-done:
if err != nil {
if strings.Contains(err.Error(), "exit status 1") {
if strings.Contains(string(errorLog), "HTTP Error 429") || strings.Contains(string(errorLog), "returned non-zero exit status 8") {
v.IPPool.SetThrottled(sourceAddress)
}
if attemps > maxtries {
break
}
continue
}
return nil, errors.Prefix("youtube-dl "+strings.Join(args, " "), err)
}
return strings.Split(strings.Replace(string(outLog), "\r\n", "\n", -1), "\n"), nil
} }
outLog, err = ioutil.ReadAll(stdout)
if err != nil {
return nil, errors.Err(err)
}
}
done := make(chan error, 1) if len(errorLog) > 0 {
go func() { return nil, errors.Err(string(errorLog))
done <- cmd.Wait()
}()
select {
case <-stopChan:
if err := cmd.Process.Kill(); err != nil {
return nil, errors.Prefix("failed to kill command after stopper cancellation", err)
}
return nil, errors.Err("canceled by stopper")
case err := <-done:
if err != nil {
return nil, errors.Prefix("youtube-dl "+strings.Join(args, " "), err)
} }
} }
if len(errorLog) > 0 {
return nil, errors.Err(string(errorLog))
}
return strings.Split(strings.Replace(string(outLog), "\r\n", "\n", -1), "\n"), nil
} }