From a5657aaf46b7fe8f5d2405548e0929ffbbdc10c5 Mon Sep 17 00:00:00 2001 From: Niko Storni Date: Fri, 20 Dec 2019 18:49:33 +0100 Subject: [PATCH] refactor download get rid of goto --- e2e/docker-compose.yml | 2 +- sources/youtubeVideo.go | 133 ++++++++++++++++++++-------------------- 2 files changed, 67 insertions(+), 68 deletions(-) diff --git a/e2e/docker-compose.yml b/e2e/docker-compose.yml index 1d7b382..97d6e6f 100644 --- a/e2e/docker-compose.yml +++ b/e2e/docker-compose.yml @@ -21,7 +21,7 @@ services: ## Wallet Server ## ################### walletserver: - image: lbry/wallet-server:v0.48.1 + image: lbry/wallet-server:v0.50.1 restart: always environment: - DB_DIRECTORY=/database diff --git a/sources/youtubeVideo.go b/sources/youtubeVideo.go index 0fc78d2..702604f 100644 --- a/sources/youtubeVideo.go +++ b/sources/youtubeVideo.go @@ -199,7 +199,6 @@ func (v *YoutubeVideo) download() error { "480", "320", } - qualityIndex := 0 ytdlArgs := []string{ "--no-progress", "-o" + strings.TrimSuffix(v.getFullPath(), ".mp4"), @@ -224,94 +223,94 @@ func (v *YoutubeVideo) download() error { ) } - sourceAddress, err := v.pool.GetIP(v.id) - if err != nil { - if sourceAddress == "throttled" { - for { + var sourceAddress string + for { + sourceAddress, err = v.pool.GetIP(v.id) + if err != nil { + if errors.Is(err, ip_manager.ErrAllThrottled) { select { case <-v.stopGroup.Ch(): return errors.Err("interrupted by user") default: + time.Sleep(ip_manager.IPCooldownPeriod) + continue } - - time.Sleep(ip_manager.IPCooldownPeriod) - sourceAddress, err = v.pool.GetIP(v.id) - if err == nil { - break - } else if !errors.Is(err, ip_manager.ErrAllThrottled) { - return err - } + } else { + return err } - } else { - return errors.Err(err) } + break } defer v.pool.ReleaseIP(sourceAddress) + ytdlArgs = append(ytdlArgs, "--source-address", sourceAddress, "https://www.youtube.com/watch?v="+v.ID(), ) -runcmd: - argsWithFilters := append(ytdlArgs, "-fbestvideo[ext=mp4][height<="+qualities[qualityIndex]+"]+bestaudio[ext!=webm]") - cmd := exec.Command("youtube-dl", argsWithFilters...) + for i, quality := range qualities { + argsWithFilters := append(ytdlArgs, "-fbestvideo[ext=mp4][height<="+quality+"]+bestaudio[ext!=webm]") + cmd := exec.Command("youtube-dl", argsWithFilters...) + log.Printf("Running command youtube-dl %s", strings.Join(argsWithFilters, " ")) - log.Printf("Running command youtube-dl %s", strings.Join(argsWithFilters, " ")) + stderr, err := cmd.StderrPipe() + if err != nil { + return errors.Err(err) + } + stdout, err := cmd.StdoutPipe() + if err != nil { + return errors.Err(err) + } - stderr, err := cmd.StderrPipe() - if err != nil { - return errors.Err(err) - } - stdout, err := cmd.StdoutPipe() - if err != nil { - return errors.Err(err) - } + if err := cmd.Start(); err != nil { + return errors.Err(err) + } - if err := cmd.Start(); err != nil { - return errors.Err(err) - } - - errorLog, _ := ioutil.ReadAll(stderr) - outLog, _ := ioutil.ReadAll(stdout) - - if err = cmd.Wait(); err != nil { - if strings.Contains(err.Error(), "exit status 1") { - if strings.Contains(string(errorLog), "HTTP Error 429") || strings.Contains(string(errorLog), "returned non-zero exit status 8") { - v.pool.SetThrottled(sourceAddress) - } else if strings.Contains(string(errorLog), "giving up after 0 fragment retries") && qualityIndex < len(qualities)-1 { - qualityIndex++ - goto runcmd //this bypasses the yt throttling IP redistribution... TODO: don't + errorLog, _ := ioutil.ReadAll(stderr) + outLog, _ := ioutil.ReadAll(stdout) + err = cmd.Wait() + if err != nil { + if strings.Contains(err.Error(), "exit status 1") { + if strings.Contains(string(errorLog), "HTTP Error 429") || strings.Contains(string(errorLog), "returned non-zero exit status 8") { + v.pool.SetThrottled(sourceAddress) + } else if strings.Contains(string(errorLog), "giving up after 0 fragment retries") { + if i == (len(qualities) - 1) { + return errors.Err(string(errorLog)) + } + continue //this bypasses the yt throttling IP redistribution... TODO: don't + } + return errors.Err(string(errorLog)) } + return errors.Err(err) + } + log.Debugln(string(outLog)) + + if strings.Contains(string(outLog), "does not pass filter duration") { + _ = v.delete("does not pass filter duration") + return errors.Err("video is too long to process") + } + if strings.Contains(string(outLog), "File is larger than max-filesize") { + _ = v.delete("File is larger than max-filesize") + return errors.Err("the video is too big to sync, skipping for now") + } + if string(errorLog) != "" { + log.Printf("Command finished with error: %v", errors.Err(string(errorLog))) + _ = v.delete("due to error") return errors.Err(string(errorLog)) } - return errors.Err(err) + fi, err := os.Stat(v.getFullPath()) + if err != nil { + return errors.Err(err) + } + err = os.Chmod(v.getFullPath(), 0777) + if err != nil { + return errors.Err(err) + } + videoSize := fi.Size() + v.size = &videoSize + break } - log.Debugln(string(outLog)) - - if strings.Contains(string(outLog), "does not pass filter duration") { - _ = v.delete("does not pass filter duration") - return errors.Err("video is too long to process") - } - if strings.Contains(string(outLog), "File is larger than max-filesize") { - _ = v.delete("File is larger than max-filesize") - return errors.Err("the video is too big to sync, skipping for now") - } - if string(errorLog) != "" { - log.Printf("Command finished with error: %v", errors.Err(string(errorLog))) - _ = v.delete("due to error") - return errors.Err(string(errorLog)) - } - fi, err := os.Stat(v.getFullPath()) - if err != nil { - return errors.Err(err) - } - err = os.Chmod(v.getFullPath(), 0777) - if err != nil { - return errors.Err(err) - } - videoSize := fi.Size() - v.size = &videoSize return nil }