From f39fc11697a8c52ea371ab4736a674461d48cf1d Mon Sep 17 00:00:00 2001 From: Mark Beamer Jr Date: Mon, 27 Jul 2020 15:42:45 -0400 Subject: [PATCH] Add downloader to create api adjust GetVideosToSync --- downloader/downloader.go | 44 +++++++++++++ downloader/downloader_test.go | 20 ++++++ ytapi/ytapi.go | 119 ++++++++-------------------------- 3 files changed, 91 insertions(+), 92 deletions(-) create mode 100644 downloader/downloader.go create mode 100644 downloader/downloader_test.go diff --git a/downloader/downloader.go b/downloader/downloader.go new file mode 100644 index 0000000..43a7b06 --- /dev/null +++ b/downloader/downloader.go @@ -0,0 +1,44 @@ +package downloader + +import ( + "io/ioutil" + "os/exec" + "strings" + + "github.com/lbryio/lbry.go/v2/extras/errors" + "github.com/sirupsen/logrus" +) + +func GetPlaylistVideoIDs(channelName string) ([]string, error) { + args := []string{"--skip-download", "https://www.youtube.com/channel/" + channelName, "--get-id", "--flat-playlist"} + return run(args) +} + +func run(args []string) ([]string, error) { + cmd := exec.Command("youtube-dl", args...) + logrus.Printf("Running command youtube-dl %s", strings.Join(args, " ")) + + stderr, err := cmd.StderrPipe() + if err != nil { + return nil, errors.Err(err) + } + stdout, err := cmd.StdoutPipe() + if err != nil { + return nil, errors.Err(err) + } + + if err := cmd.Start(); err != nil { + return nil, errors.Err(err) + } + + errorLog, _ := ioutil.ReadAll(stderr) + outLog, _ := ioutil.ReadAll(stdout) + err = cmd.Wait() + if len(errorLog) > 0 { + return nil, errors.Err(err) + } + if len(errorLog) > 0 { + return nil, errors.Err(string(errorLog)) + } + return strings.Split(strings.Replace(string(outLog), "\r\n", "\n", -1), "\n"), nil +} diff --git a/downloader/downloader_test.go b/downloader/downloader_test.go new file mode 100644 index 0000000..6a5b5de --- /dev/null +++ b/downloader/downloader_test.go @@ -0,0 +1,20 @@ +package downloader + +import ( + "testing" + + "github.com/sirupsen/logrus" +) + +func TestRun(t *testing.T) { + //args := []string{"--skip-download", "https://www.youtube.com/c/Electroboom", "--get-id", "--flat-playlist"} + //videoIDs, err := GetPlaylistVideoIDs("Electroboom") + videoIDs, err := GetPlaylistVideoIDs("UCJ0-OtVpF0wOKEqT2Z1HEtA") + if err != nil { + logrus.Error(err) + } + for _, id := range videoIDs { + println(id) + } + t.Error("just stop") +} diff --git a/ytapi/ytapi.go b/ytapi/ytapi.go index bc28360..b8eeb06 100644 --- a/ytapi/ytapi.go +++ b/ytapi/ytapi.go @@ -7,6 +7,8 @@ import ( "sync" "time" + "github.com/lbryio/ytsync/v5/downloader" + "github.com/lbryio/ytsync/v5/ip_manager" "github.com/lbryio/ytsync/v5/sdk" "github.com/lbryio/ytsync/v5/sources" @@ -46,55 +48,35 @@ type VideoParams struct { var mostRecentlyFailedChannel string // TODO: fix this hack! func GetVideosToSync(apiKey, channelID string, syncedVideos map[string]sdk.SyncedVideo, quickSync bool, maxVideos int, videoParams VideoParams) ([]Video, error) { - playlistID, err := getPlaylistID(apiKey, channelID) + + var videos []Video + videoIDs, err := downloader.GetPlaylistVideoIDs(channelID) + if err != nil { + return nil, errors.Err(err) + } + + log.Infof("Got info for %d videos from youtube downloader", len(videoIDs)) + + playlistMap := make(map[string]int64) + for i, videoID := range videoIDs { + playlistMap[videoID] = int64(i) + } + + if len(videoIDs) < 1 { + if channelID == mostRecentlyFailedChannel { + return nil, errors.Err("playlist items not found") + } + mostRecentlyFailedChannel = channelID + } + + vids, err := getVideos(apiKey, videoIDs) if err != nil { return nil, err } - playlistMap := make(map[string]*ytlib.PlaylistItemSnippet, 50) - var playlistItems []*ytlib.PlaylistItem - var nextPageToken string - var videos []Video - - for { - playlistItems, nextPageToken, err = getPlaylistItems(apiKey, playlistID, nextPageToken) - - if len(playlistItems) < 1 { - // If there are 50+ videos in a playlist but less than 50 are actually returned by the API, youtube will still redirect - // clients to a next page. Such next page will however be empty. This logic prevents ytsync from failing. - youtubeIsLying := len(videos) > 0 - if youtubeIsLying { - break - } - if channelID == mostRecentlyFailedChannel { - return nil, errors.Err("playlist items not found") - } - mostRecentlyFailedChannel = channelID - break //return errors.Err("playlist items not found") //TODO: will this work? - } - - videoIDs := make([]string, len(playlistItems)) - for i, item := range playlistItems { - // normally we'd send the video into the channel here, but youtube api doesn't have sorting - // so we have to get ALL the videos, then sort them, then send them in - playlistMap[item.Snippet.ResourceId.VideoId] = item.Snippet - videoIDs[i] = item.Snippet.ResourceId.VideoId - } - - vids, err := getVideos(apiKey, videoIDs) - if err != nil { - return nil, err - } - - for _, item := range vids { - videos = append(videos, sources.NewYoutubeVideo(videoParams.VideoDir, item, playlistMap[item.Id].Position, videoParams.S3Config, videoParams.Grp, videoParams.IPPool)) - } - - log.Infof("Got info for %d videos from youtube API", len(videos)) - - if nextPageToken == "" || quickSync || len(videos) >= maxVideos { - break - } + for _, item := range vids { + positionInList := playlistMap[item.Id] + videos = append(videos, sources.NewYoutubeVideo(videoParams.VideoDir, item, positionInList, videoParams.S3Config, videoParams.Grp, videoParams.IPPool)) } for k, v := range syncedVideos { @@ -104,7 +86,6 @@ func GetVideosToSync(apiKey, channelID string, syncedVideos map[string]sdk.Synce if _, ok := playlistMap[k]; !ok { videos = append(videos, sources.NewMockedVideo(videoParams.VideoDir, k, channelID, videoParams.S3Config, videoParams.Grp, videoParams.IPPool)) } - } sort.Sort(byPublishedAt(videos)) @@ -152,52 +133,6 @@ func ChannelInfo(apiKey, channelID string) (*ytlib.ChannelSnippet, *ytlib.Channe return response.Items[0].Snippet, response.Items[0].BrandingSettings, nil } -func getPlaylistID(apiKey, channelID string) (string, error) { - service, err := ytlib.New(&http.Client{Transport: &transport.APIKey{Key: apiKey}}) - if err != nil { - return "", errors.Prefix("error creating YouTube service", err) - } - - response, err := service.Channels.List("contentDetails").Id(channelID).Do() - if err != nil { - return "", errors.Prefix("error getting channel details", err) - } - - if len(response.Items) < 1 { - return "", errors.Err("youtube channel not found") - } - - if response.Items[0].ContentDetails.RelatedPlaylists == nil { - return "", errors.Err("no related playlists") - } - - playlistID := response.Items[0].ContentDetails.RelatedPlaylists.Uploads - if playlistID == "" { - return "", errors.Err("no channel playlist") - } - - return playlistID, nil -} - -func getPlaylistItems(apiKey, playlistID, nextPageToken string) ([]*ytlib.PlaylistItem, string, error) { - service, err := ytlib.New(&http.Client{Transport: &transport.APIKey{Key: apiKey}}) - if err != nil { - return nil, "", errors.Prefix("error creating YouTube service", err) - } - - response, err := service.PlaylistItems.List("snippet"). - PlaylistId(playlistID). - MaxResults(50). - PageToken(nextPageToken). - Do() - - if err != nil { - return nil, "", errors.Prefix("error getting playlist items", err) - } - - return response.Items, response.NextPageToken, nil -} - func getVideos(apiKey string, videoIDs []string) ([]*ytlib.Video, error) { service, err := ytlib.New(&http.Client{Transport: &transport.APIKey{Key: apiKey}}) if err != nil {