LBRYCommunity/ytsync
1
0
Fork 0
Code Issues 39 Pull requests 8 Projects Releases 197 Packages Wiki Activity

refactor download

Browse source 
get rid of goto
This commit is contained in:
Niko Storni 2019-12-20 18:49:33 +01:00
parent b6861dae9b
commit a5657aaf46
2 changed files with 67 additions and 68 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
e2e/docker-compose.yml
View file

@ -21,7 +21,7 @@ services:
## Wallet Server ## ## Wallet Server ##
################### ###################
walletserver: walletserver:
image: lbry/wallet-server:v0.48.1 image: lbry/wallet-server:v0.50.1
restart: always restart: always
environment: environment:
- DB_DIRECTORY=/database - DB_DIRECTORY=/database

133
sources/youtubeVideo.go
View file

@ -199,7 +199,6 @@ func (v *YoutubeVideo) download() error {
"480", "480",
"320", "320",
} }
qualityIndex := 0
ytdlArgs := []string{ ytdlArgs := []string{
"--no-progress", "--no-progress",
"-o" + strings.TrimSuffix(v.getFullPath(), ".mp4"), "-o" + strings.TrimSuffix(v.getFullPath(), ".mp4"),
@ -224,94 +223,94 @@ func (v *YoutubeVideo) download() error {
) )
} }
sourceAddress, err := v.pool.GetIP(v.id) var sourceAddress string
if err != nil { for {
if sourceAddress == "throttled" { sourceAddress, err = v.pool.GetIP(v.id)
for { if err != nil {
if errors.Is(err, ip_manager.ErrAllThrottled) {
select { select {
case <-v.stopGroup.Ch(): case <-v.stopGroup.Ch():
return errors.Err("interrupted by user") return errors.Err("interrupted by user")
default: default:
time.Sleep(ip_manager.IPCooldownPeriod)
continue
} }
} else {
time.Sleep(ip_manager.IPCooldownPeriod) return err
sourceAddress, err = v.pool.GetIP(v.id)
if err == nil {
break
} else if !errors.Is(err, ip_manager.ErrAllThrottled) {
return err
}
} }
} else {
return errors.Err(err)
} }
break
} }
defer v.pool.ReleaseIP(sourceAddress) defer v.pool.ReleaseIP(sourceAddress)
ytdlArgs = append(ytdlArgs, ytdlArgs = append(ytdlArgs,
"--source-address", "--source-address",
sourceAddress, sourceAddress,
"https://www.youtube.com/watch?v="+v.ID(), "https://www.youtube.com/watch?v="+v.ID(),
) )
runcmd: for i, quality := range qualities {
argsWithFilters := append(ytdlArgs, "-fbestvideo[ext=mp4][height<="+qualities[qualityIndex]+"]+bestaudio[ext!=webm]") argsWithFilters := append(ytdlArgs, "-fbestvideo[ext=mp4][height<="+quality+"]+bestaudio[ext!=webm]")
cmd := exec.Command("youtube-dl", argsWithFilters...) cmd := exec.Command("youtube-dl", argsWithFilters...)
log.Printf("Running command youtube-dl %s", strings.Join(argsWithFilters, " "))
log.Printf("Running command youtube-dl %s", strings.Join(argsWithFilters, " ")) stderr, err := cmd.StderrPipe()
if err != nil {
return errors.Err(err)
}
stdout, err := cmd.StdoutPipe()
if err != nil {
return errors.Err(err)
}
stderr, err := cmd.StderrPipe() if err := cmd.Start(); err != nil {
if err != nil { return errors.Err(err)
return errors.Err(err) }
}
stdout, err := cmd.StdoutPipe()
if err != nil {
return errors.Err(err)
}
if err := cmd.Start(); err != nil { errorLog, _ := ioutil.ReadAll(stderr)
return errors.Err(err) outLog, _ := ioutil.ReadAll(stdout)
} err = cmd.Wait()
if err != nil {
errorLog, _ := ioutil.ReadAll(stderr) if strings.Contains(err.Error(), "exit status 1") {
outLog, _ := ioutil.ReadAll(stdout) if strings.Contains(string(errorLog), "HTTP Error 429") || strings.Contains(string(errorLog), "returned non-zero exit status 8") {
v.pool.SetThrottled(sourceAddress)
if err = cmd.Wait(); err != nil { } else if strings.Contains(string(errorLog), "giving up after 0 fragment retries") {
if strings.Contains(err.Error(), "exit status 1") { if i == (len(qualities) - 1) {
if strings.Contains(string(errorLog), "HTTP Error 429") || strings.Contains(string(errorLog), "returned non-zero exit status 8") { return errors.Err(string(errorLog))
v.pool.SetThrottled(sourceAddress) }
} else if strings.Contains(string(errorLog), "giving up after 0 fragment retries") && qualityIndex < len(qualities)-1 { continue //this bypasses the yt throttling IP redistribution... TODO: don't
qualityIndex++ }
goto runcmd //this bypasses the yt throttling IP redistribution... TODO: don't return errors.Err(string(errorLog))
} }
return errors.Err(err)
}
log.Debugln(string(outLog))
if strings.Contains(string(outLog), "does not pass filter duration") {
_ = v.delete("does not pass filter duration")
return errors.Err("video is too long to process")
}
if strings.Contains(string(outLog), "File is larger than max-filesize") {
_ = v.delete("File is larger than max-filesize")
return errors.Err("the video is too big to sync, skipping for now")
}
if string(errorLog) != "" {
log.Printf("Command finished with error: %v", errors.Err(string(errorLog)))
_ = v.delete("due to error")
return errors.Err(string(errorLog)) return errors.Err(string(errorLog))
} }
return errors.Err(err) fi, err := os.Stat(v.getFullPath())
if err != nil {
return errors.Err(err)
}
err = os.Chmod(v.getFullPath(), 0777)
if err != nil {
return errors.Err(err)
}
videoSize := fi.Size()
v.size = &videoSize
break
} }
log.Debugln(string(outLog))
if strings.Contains(string(outLog), "does not pass filter duration") {
_ = v.delete("does not pass filter duration")
return errors.Err("video is too long to process")
}
if strings.Contains(string(outLog), "File is larger than max-filesize") {
_ = v.delete("File is larger than max-filesize")
return errors.Err("the video is too big to sync, skipping for now")
}
if string(errorLog) != "" {
log.Printf("Command finished with error: %v", errors.Err(string(errorLog)))
_ = v.delete("due to error")
return errors.Err(string(errorLog))
}
fi, err := os.Stat(v.getFullPath())
if err != nil {
return errors.Err(err)
}
err = os.Chmod(v.getFullPath(), 0777)
if err != nil {
return errors.Err(err)
}
videoSize := fi.Size()
v.size = &videoSize
return nil return nil
} }