LBRYCommunity/ytsync
1
0
Fork 0
Code Issues 39 Pull requests 8 Projects Releases 197 Packages Wiki Activity

added logging

Browse source
This commit is contained in:
Mark Beamer Jr 2020-07-30 12:48:05 -04:00 committed by Alex Grintsvayg
parent b1e10e7b09
commit a1caea4a28
No known key found for this signature in database
GPG key ID: AEB3F089F86A22B5
2 changed files with 18 additions and 14 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

25
downloader/downloader.go
View file

@ -12,12 +12,10 @@ import (
"strings" "strings"
"time" "time"
"github.com/lbryio/ytsync/v5/ip_manager"
"github.com/lbryio/ytsync/v5/sdk"
"github.com/lbryio/ytsync/v5/ytapi"
"github.com/davecgh/go-spew/spew" "github.com/davecgh/go-spew/spew"
"github.com/lbryio/ytsync/v5/downloader/ytdl" "github.com/lbryio/ytsync/v5/downloader/ytdl"
"github.com/lbryio/ytsync/v5/ip_manager"
"github.com/lbryio/ytsync/v5/sdk"
"github.com/lbryio/lbry.go/v2/extras/errors" "github.com/lbryio/lbry.go/v2/extras/errors"
"github.com/lbryio/lbry.go/v2/extras/stop" "github.com/lbryio/lbry.go/v2/extras/stop"
@ -26,9 +24,9 @@ import (
"github.com/sirupsen/logrus" "github.com/sirupsen/logrus"
) )
func GetPlaylistVideoIDs(channelName string, maxVideos int, stopChan stop.Chan, videoParams ytapi.VideoParams) ([]string, error) { func GetPlaylistVideoIDs(channelName string, maxVideos int, stopChan stop.Chan, pool *ip_manager.IPPool) ([]string, error) {
args := []string{"--skip-download", "https://www.youtube.com/channel/" + channelName, "--get-id", "--flat-playlist"} args := []string{"--skip-download", "https://www.youtube.com/channel/" + channelName, "--get-id", "--flat-playlist"}
ids, err := run(channelName, args, false, true, stopChan, videoParams) ids, err := run(channelName, args, false, true, stopChan, pool)
if err != nil { if err != nil {
return nil, errors.Err(err) return nil, errors.Err(err)
} }
@ -44,9 +42,9 @@ func GetPlaylistVideoIDs(channelName string, maxVideos int, stopChan stop.Chan,
const releaseTimeFormat = "2006-01-02, 15:04:05 (MST)" const releaseTimeFormat = "2006-01-02, 15:04:05 (MST)"
func GetVideoInformation(config *sdk.APIConfig, videoID string, stopChan stop.Chan, ip *net.TCPAddr, videoParams ytapi.VideoParams) (*ytdl.YtdlVideo, error) { func GetVideoInformation(config *sdk.APIConfig, videoID string, stopChan stop.Chan, ip *net.TCPAddr, pool *ip_manager.IPPool) (*ytdl.YtdlVideo, error) {
args := []string{"--skip-download", "--print-json", "https://www.youtube.com/watch?v=" + videoID} args := []string{"--skip-download", "--print-json", "https://www.youtube.com/watch?v=" + videoID}
results, err := run(videoID, args, false, true, stopChan, videoParams) results, err := run(videoID, args, false, true, stopChan, pool)
if err != nil { if err != nil {
return nil, errors.Err(err) return nil, errors.Err(err)
} }
@ -231,14 +229,14 @@ func getClient(ip *net.TCPAddr) *http.Client {
} }
} }
func run(use string, args []string, withStdErr, withStdOut bool, stopChan stop.Chan, v ytapi.VideoParams) ([]string, error) { func run(use string, args []string, withStdErr, withStdOut bool, stopChan stop.Chan, pool *ip_manager.IPPool) ([]string, error) {
var maxtries = 10 var maxtries = 10
var attemps int var attemps int
for { for {
var sourceAddress string var sourceAddress string
var err error var err error
for { for {
sourceAddress, err = v.IPPool.GetIP(use) sourceAddress, err = pool.GetIP(use)
if err != nil { if err != nil {
if errors.Is(err, ip_manager.ErrAllThrottled) { if errors.Is(err, ip_manager.ErrAllThrottled) {
select { select {
@ -254,7 +252,7 @@ func run(use string, args []string, withStdErr, withStdOut bool, stopChan stop.C
} }
break break
} }
defer v.IPPool.ReleaseIP(sourceAddress) defer pool.ReleaseIP(sourceAddress)
args = append(args, "--source-address", sourceAddress) args = append(args, "--source-address", sourceAddress)
cmd := exec.Command("youtube-dl", args...) cmd := exec.Command("youtube-dl", args...)
logrus.Printf("Running command youtube-dl %s", strings.Join(args, " ")) logrus.Printf("Running command youtube-dl %s", strings.Join(args, " "))
@ -306,13 +304,16 @@ func run(use string, args []string, withStdErr, withStdOut bool, stopChan stop.C
if err != nil { if err != nil {
if strings.Contains(err.Error(), "exit status 1") { if strings.Contains(err.Error(), "exit status 1") {
if strings.Contains(string(errorLog), "HTTP Error 429") || strings.Contains(string(errorLog), "returned non-zero exit status 8") { if strings.Contains(string(errorLog), "HTTP Error 429") || strings.Contains(string(errorLog), "returned non-zero exit status 8") {
v.IPPool.SetThrottled(sourceAddress) pool.SetThrottled(sourceAddress)
} }
if attemps > maxtries { if attemps > maxtries {
logrus.Debug("too many tries returning failure")
break break
} }
logrus.Debugf("known throttling error...try again (%d)", attemps)
continue continue
} }
logrus.Debug("Unkown error, returning failure")
return nil, errors.Prefix("youtube-dl "+strings.Join(args, " "), err) return nil, errors.Prefix("youtube-dl "+strings.Join(args, " "), err)
} }
return strings.Split(strings.Replace(string(outLog), "\r\n", "\n", -1), "\n"), nil return strings.Split(strings.Replace(string(outLog), "\r\n", "\n", -1), "\n"), nil

7
ytapi/ytapi.go
View file

@ -57,7 +57,7 @@ func GetVideosToSync(config *sdk.APIConfig, channelID string, syncedVideos map[s
if quickSync { if quickSync {
maxVideos = 50 maxVideos = 50
} }
videoIDs, err := downloader.GetPlaylistVideoIDs(channelID, maxVideos, videoParams.Stopper.Ch()) videoIDs, err := downloader.GetPlaylistVideoIDs(channelID, maxVideos, videoParams.Stopper.Ch(), videoParams.IPPool)
if err != nil { if err != nil {
return nil, errors.Err(err) return nil, errors.Err(err)
} }
@ -164,6 +164,9 @@ func ChannelInfo(apiKey, channelID string) (*ytlib.ChannelSnippet, *ytlib.Channe
func getVideos(config *sdk.APIConfig, videoIDs []string, stopChan stop.Chan, ipPool *ip_manager.IPPool) ([]*ytdl.YtdlVideo, error) { func getVideos(config *sdk.APIConfig, videoIDs []string, stopChan stop.Chan, ipPool *ip_manager.IPPool) ([]*ytdl.YtdlVideo, error) {
var videos []*ytdl.YtdlVideo var videos []*ytdl.YtdlVideo
for _, videoID := range videoIDs { for _, videoID := range videoIDs {
if len(videoID) < 5 {
continue
}
select { select {
case <-stopChan: case <-stopChan:
return videos, errors.Err("canceled by stopper") return videos, errors.Err("canceled by stopper")
@ -182,7 +185,7 @@ func getVideos(config *sdk.APIConfig, videoIDs []string, stopChan stop.Chan, ipP
if state == "published" { if state == "published" {
continue continue
} }
video, err := downloader.GetVideoInformation(config, videoID, stopChan, nil) video, err := downloader.GetVideoInformation(config, videoID, stopChan, nil, ipPool)
if err != nil { if err != nil {
//ipPool.ReleaseIP(ip) //ipPool.ReleaseIP(ip)
return nil, errors.Err(err) return nil, errors.Err(err)