improve youtube antithrottle

fix logging

manager/manager.go

@@ -199,7 +199,6 @@ func (s *SyncManager) Start() error {
 					"failure uploading wallet",
 					"the channel in the wallet is different than the channel in the database",
 					"this channel does not belong to this wallet!",
-					"Invalid status code: 429",
 					"HTTP Error 429",
 				}
 				if util.SubstringInSlice(err.Error(), fatalErrors) {

manager/ytsync.go

@@ -656,7 +656,6 @@ func (s *Sync) startWorker(workerNum int) {
 					"Cannot publish using channel",
 					"cannot concatenate 'str' and 'NoneType' objects",
 					"more than 90% of the space has been used.",
-					"Invalid status code: 429",
 					"HTTP Error 429",
 				}
 				if util.SubstringInSlice(err.Error(), fatalErrors) || s.StopOnError {
@@ -679,7 +678,7 @@ func (s *Sync) startWorker(workerNum int) {
 						"Watch this video on YouTube.",
 						"have blocked it on copyright grounds",
 						"the video must be republished as we can't get the right size",
-						"Invalid status code: 403",
+						"HTTP Error 403",
 					}
 					if util.SubstringInSlice(err.Error(), errorsNoRetry) {
 						log.Println("This error should not be retried at all")

sources/youtubeVideo.go

@@ -204,30 +204,81 @@ func getNextIP() (string, error) {
 	return nextIP, nil
 }
 
-func (v *YoutubeVideo) download() error {
+func (v *YoutubeVideo) download(youtubeAntiThrottle bool) error {
+	videoPath := v.getFullPath()
+
+	err := os.Mkdir(v.videoDir(), 0750)
+	if err != nil && !strings.Contains(err.Error(), "file exists") {
+		return errors.Wrap(err, 0)
+	}
+
+	_, err = os.Stat(videoPath)
+	if err != nil && !os.IsNotExist(err) {
+		return errors.Err(err)
+	} else if err == nil {
+		log.Debugln(v.id + " already exists at " + videoPath)
+		return nil
+	}
+
+	useIPv4Env := os.Getenv("USE_IPV4")
+	ytdlArgs := []string{
+		"--no-progress",
+		"-fbestvideo[ext=mp4,height<=1080,filesize<2000M]+best[ext=mp4,height<=1080,filesize<2000M]",
+		"-o" + strings.TrimRight(v.getFullPath(), ".mp4"),
+		"--merge-output-format",
+		"mp4",
+	}
+	useIPv4 := useIPv4Env != ""
+	if !useIPv4 || youtubeAntiThrottle {
 		sourceAddress, err := getNextIP()
 		if err != nil {
 			return errors.Err(err)
 		}
-	cmd := exec.Command("youtube-dl",
+		log.Infof("using IPv6: %s", sourceAddress)
-		"--no-progress",
+		ytdlArgs = append(ytdlArgs,
-		"-fbestvideo[ext=mp4,height<=1080,filesize<2000M]+best[ext=mp4,height<=1080,filesize<2000M]",
-		"-o"+strings.TrimRight(v.getFullPath(), ".mp4"),
-		"--merge-output-format",
-		"mp4",
 			"-6",
 			"--source-address",
 			sourceAddress,
-		"https://www.youtube.com/watch?v="+v.ID())
+		)
+	} else {
+		log.Infoln("using IPv4")
+		ytdlArgs = append(ytdlArgs, "-4")
+	}
+	ytdlArgs = append(ytdlArgs, "https://www.youtube.com/watch?v="+v.ID())
+	cmd := exec.Command("youtube-dl", ytdlArgs...)
 
 	log.Printf("Running command and waiting for it to finish...")
-	output, err := cmd.CombinedOutput()
+
-	log.Debugln(string(output))
+	stderr, err := cmd.StderrPipe()
 	if err != nil {
-		log.Printf("Command finished with error: %v", errors.Err(string(output)))
-		_ = v.delete()
 		return errors.Err(err)
 	}
+	stdout, err := cmd.StdoutPipe()
+	if err != nil {
+		return errors.Err(err)
+	}
+
+	if err := cmd.Start(); err != nil {
+		return errors.Err(err)
+	}
+
+	errorLog, _ := ioutil.ReadAll(stderr)
+	outLog, _ := ioutil.ReadAll(stdout)
+
+	if err = cmd.Wait(); err != nil {
+		if strings.Contains(err.Error(), "exit status 1") {
+			return errors.Err(string(errorLog))
+		}
+		return errors.Err(err)
+	}
+
+	log.Debugln(string(outLog))
+
+	if string(errorLog) != "" {
+		log.Printf("Command finished with error: %v", errors.Err(string(errorLog)))
+		_ = v.delete()
+		return errors.Err(string(errorLog))
+	}
 	fi, err := os.Stat(v.getFullPath())
 	if err != nil {
 		return errors.Err(err)
@@ -349,10 +400,17 @@ func (v *YoutubeVideo) Sync(daemon *jsonrpc.Client, params SyncParams, existingV
 }
 
 func (v *YoutubeVideo) downloadAndPublish(daemon *jsonrpc.Client, params SyncParams) (*SyncSummary, error) {
-	err := v.download()
+	err := v.download(false)
+	if err != nil {
+		if strings.Contains(err.Error(), "HTTP Error 429") {
+			err = v.download(true)
 			if err != nil {
 				return nil, errors.Prefix("download error", err)
 			}
+		} else {
+			return nil, errors.Prefix("download error", err)
+		}
+	}
 	log.Debugln("Downloaded " + v.id)
 
 	err = v.triggerThumbnailSave()