LBRYCommunity/ytsync
1
0
Fork 0
Code Issues 39 Pull requests 8 Projects Releases 197 Packages Wiki Activity

improve youtube antithrottle

Browse source 
fix logging
This commit is contained in:
Niko Storni 2019-07-11 19:14:15 +02:00
parent 8ad2acc936
commit 51cb7f5e34
3 changed files with 79 additions and 23 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
manager/manager.go
View file

@ -199,7 +199,6 @@ func (s *SyncManager) Start() error {
"failure uploading wallet",
"the channel in the wallet is different than the channel in the database",
"this channel does not belong to this wallet!",
"Invalid status code: 429",
"HTTP Error 429",
}
if util.SubstringInSlice(err.Error(), fatalErrors) {

3
manager/ytsync.go
View file

@ -656,7 +656,6 @@ func (s *Sync) startWorker(workerNum int) {
"Cannot publish using channel",
"cannot concatenate 'str' and 'NoneType' objects",
"more than 90% of the space has been used.",
"Invalid status code: 429",
"HTTP Error 429",
}
if util.SubstringInSlice(err.Error(), fatalErrors) || s.StopOnError {
@ -679,7 +678,7 @@ func (s *Sync) startWorker(workerNum int) {
"Watch this video on YouTube.",
"have blocked it on copyright grounds",
"the video must be republished as we can't get the right size",
"Invalid status code: 403",
"HTTP Error 403",
}
if util.SubstringInSlice(err.Error(), errorsNoRetry) {
log.Println("This error should not be retried at all")

84
sources/youtubeVideo.go
View file

@ -204,30 +204,81 @@ func getNextIP() (string, error) {
return nextIP, nil
}
func (v *YoutubeVideo) download() error {
func (v *YoutubeVideo) download(youtubeAntiThrottle bool) error {
videoPath := v.getFullPath()
err := os.Mkdir(v.videoDir(), 0750)
if err != nil && !strings.Contains(err.Error(), "file exists") {
return errors.Wrap(err, 0)
}
_, err = os.Stat(videoPath)
if err != nil && !os.IsNotExist(err) {
return errors.Err(err)
} else if err == nil {
log.Debugln(v.id + " already exists at " + videoPath)
return nil
}
useIPv4Env := os.Getenv("USE_IPV4")
ytdlArgs := []string{
"--no-progress",
"-fbestvideo[ext=mp4,height<=1080,filesize<2000M]+best[ext=mp4,height<=1080,filesize<2000M]",
"-o" + strings.TrimRight(v.getFullPath(), ".mp4"),
"--merge-output-format",
"mp4",
}
useIPv4 := useIPv4Env != ""
if !useIPv4 || youtubeAntiThrottle {
sourceAddress, err := getNextIP()
if err != nil {
return errors.Err(err)
}
cmd := exec.Command("youtube-dl",
"--no-progress",
"-fbestvideo[ext=mp4,height<=1080,filesize<2000M]+best[ext=mp4,height<=1080,filesize<2000M]",
"-o"+strings.TrimRight(v.getFullPath(), ".mp4"),
"--merge-output-format",
"mp4",
log.Infof("using IPv6: %s", sourceAddress)
ytdlArgs = append(ytdlArgs,
"-6",
"--source-address",
sourceAddress,
"https://www.youtube.com/watch?v="+v.ID())
)
} else {
log.Infoln("using IPv4")
ytdlArgs = append(ytdlArgs, "-4")
}
ytdlArgs = append(ytdlArgs, "https://www.youtube.com/watch?v="+v.ID())
cmd := exec.Command("youtube-dl", ytdlArgs...)
log.Printf("Running command and waiting for it to finish...")
output, err := cmd.CombinedOutput()
log.Debugln(string(output))
stderr, err := cmd.StderrPipe()
if err != nil {
log.Printf("Command finished with error: %v", errors.Err(string(output)))
_ = v.delete()
return errors.Err(err)
}
stdout, err := cmd.StdoutPipe()
if err != nil {
return errors.Err(err)
}
if err := cmd.Start(); err != nil {
return errors.Err(err)
}
errorLog, _ := ioutil.ReadAll(stderr)
outLog, _ := ioutil.ReadAll(stdout)
if err = cmd.Wait(); err != nil {
if strings.Contains(err.Error(), "exit status 1") {
return errors.Err(string(errorLog))
}
return errors.Err(err)
}
log.Debugln(string(outLog))
if string(errorLog) != "" {
log.Printf("Command finished with error: %v", errors.Err(string(errorLog)))
_ = v.delete()
return errors.Err(string(errorLog))
}
fi, err := os.Stat(v.getFullPath())
if err != nil {
return errors.Err(err)
@ -349,10 +400,17 @@ func (v *YoutubeVideo) Sync(daemon *jsonrpc.Client, params SyncParams, existingV
}
func (v *YoutubeVideo) downloadAndPublish(daemon *jsonrpc.Client, params SyncParams) (*SyncSummary, error) {
err := v.download()
err := v.download(false)
if err != nil {
if strings.Contains(err.Error(), "HTTP Error 429") {
err = v.download(true)
if err != nil {
return nil, errors.Prefix("download error", err)
}
} else {
return nil, errors.Prefix("download error", err)
}
}
log.Debugln("Downloaded " + v.id)
err = v.triggerThumbnailSave()