Add downloader to create api

adjust GetVideosToSync

downloader/downloader.go

@@ -0,0 +1,44 @@
+package downloader
+
+import (
+	"io/ioutil"
+	"os/exec"
+	"strings"
+
+	"github.com/lbryio/lbry.go/v2/extras/errors"
+	"github.com/sirupsen/logrus"
+)
+
+func GetPlaylistVideoIDs(channelName string) ([]string, error) {
+	args := []string{"--skip-download", "https://www.youtube.com/channel/" + channelName, "--get-id", "--flat-playlist"}
+	return run(args)
+}
+
+func run(args []string) ([]string, error) {
+	cmd := exec.Command("youtube-dl", args...)
+	logrus.Printf("Running command youtube-dl %s", strings.Join(args, " "))
+
+	stderr, err := cmd.StderrPipe()
+	if err != nil {
+		return nil, errors.Err(err)
+	}
+	stdout, err := cmd.StdoutPipe()
+	if err != nil {
+		return nil, errors.Err(err)
+	}
+
+	if err := cmd.Start(); err != nil {
+		return nil, errors.Err(err)
+	}
+
+	errorLog, _ := ioutil.ReadAll(stderr)
+	outLog, _ := ioutil.ReadAll(stdout)
+	err = cmd.Wait()
+	if len(errorLog) > 0 {
+		return nil, errors.Err(err)
+	}
+	if len(errorLog) > 0 {
+		return nil, errors.Err(string(errorLog))
+	}
+	return strings.Split(strings.Replace(string(outLog), "\r\n", "\n", -1), "\n"), nil
+}

downloader/downloader_test.go

@@ -0,0 +1,20 @@
+package downloader
+
+import (
+	"testing"
+
+	"github.com/sirupsen/logrus"
+)
+
+func TestRun(t *testing.T) {
+	//args := []string{"--skip-download", "https://www.youtube.com/c/Electroboom", "--get-id", "--flat-playlist"}
+	//videoIDs, err := GetPlaylistVideoIDs("Electroboom")
+	videoIDs, err := GetPlaylistVideoIDs("UCJ0-OtVpF0wOKEqT2Z1HEtA")
+	if err != nil {
+		logrus.Error(err)
+	}
+	for _, id := range videoIDs {
+		println(id)
+	}
+	t.Error("just stop")
+}

ytapi/ytapi.go

@@ -7,6 +7,8 @@ import (
 	"sync"
 	"time"
 
+	"github.com/lbryio/ytsync/v5/downloader"
+
 	"github.com/lbryio/ytsync/v5/ip_manager"
 	"github.com/lbryio/ytsync/v5/sdk"
 	"github.com/lbryio/ytsync/v5/sources"
@@ -46,55 +48,35 @@ type VideoParams struct {
 var mostRecentlyFailedChannel string // TODO: fix this hack!
 
 func GetVideosToSync(apiKey, channelID string, syncedVideos map[string]sdk.SyncedVideo, quickSync bool, maxVideos int, videoParams VideoParams) ([]Video, error) {
-	playlistID, err := getPlaylistID(apiKey, channelID)
+
+	var videos []Video
+	videoIDs, err := downloader.GetPlaylistVideoIDs(channelID)
+	if err != nil {
+		return nil, errors.Err(err)
+	}
+
+	log.Infof("Got info for %d videos from youtube downloader", len(videoIDs))
+
+	playlistMap := make(map[string]int64)
+	for i, videoID := range videoIDs {
+		playlistMap[videoID] = int64(i)
+	}
+
+	if len(videoIDs) < 1 {
+		if channelID == mostRecentlyFailedChannel {
+			return nil, errors.Err("playlist items not found")
+		}
+		mostRecentlyFailedChannel = channelID
+	}
+
+	vids, err := getVideos(apiKey, videoIDs)
 	if err != nil {
 		return nil, err
 	}
 
-	playlistMap := make(map[string]*ytlib.PlaylistItemSnippet, 50)
+	for _, item := range vids {
-	var playlistItems []*ytlib.PlaylistItem
+		positionInList := playlistMap[item.Id]
-	var nextPageToken string
+		videos = append(videos, sources.NewYoutubeVideo(videoParams.VideoDir, item, positionInList, videoParams.S3Config, videoParams.Grp, videoParams.IPPool))
-	var videos []Video
-
-	for {
-		playlistItems, nextPageToken, err = getPlaylistItems(apiKey, playlistID, nextPageToken)
-
-		if len(playlistItems) < 1 {
-			// If there are 50+ videos in a playlist but less than 50 are actually returned by the API, youtube will still redirect
-			// clients to a next page. Such next page will however be empty. This logic prevents ytsync from failing.
-			youtubeIsLying := len(videos) > 0
-			if youtubeIsLying {
-				break
-			}
-			if channelID == mostRecentlyFailedChannel {
-				return nil, errors.Err("playlist items not found")
-			}
-			mostRecentlyFailedChannel = channelID
-			break //return errors.Err("playlist items not found") //TODO: will this work?
-		}
-
-		videoIDs := make([]string, len(playlistItems))
-		for i, item := range playlistItems {
-			// normally we'd send the video into the channel here, but youtube api doesn't have sorting
-			// so we have to get ALL the videos, then sort them, then send them in
-			playlistMap[item.Snippet.ResourceId.VideoId] = item.Snippet
-			videoIDs[i] = item.Snippet.ResourceId.VideoId
-		}
-
-		vids, err := getVideos(apiKey, videoIDs)
-		if err != nil {
-			return nil, err
-		}
-
-		for _, item := range vids {
-			videos = append(videos, sources.NewYoutubeVideo(videoParams.VideoDir, item, playlistMap[item.Id].Position, videoParams.S3Config, videoParams.Grp, videoParams.IPPool))
-		}
-
-		log.Infof("Got info for %d videos from youtube API", len(videos))
-
-		if nextPageToken == "" || quickSync || len(videos) >= maxVideos {
-			break
-		}
 	}
 
 	for k, v := range syncedVideos {
@@ -104,7 +86,6 @@ func GetVideosToSync(apiKey, channelID string, syncedVideos map[string]sdk.Synce
 		if _, ok := playlistMap[k]; !ok {
 			videos = append(videos, sources.NewMockedVideo(videoParams.VideoDir, k, channelID, videoParams.S3Config, videoParams.Grp, videoParams.IPPool))
 		}
-
 	}
 
 	sort.Sort(byPublishedAt(videos))
@@ -152,52 +133,6 @@ func ChannelInfo(apiKey, channelID string) (*ytlib.ChannelSnippet, *ytlib.Channe
 	return response.Items[0].Snippet, response.Items[0].BrandingSettings, nil
 }
 
-func getPlaylistID(apiKey, channelID string) (string, error) {
-	service, err := ytlib.New(&http.Client{Transport: &transport.APIKey{Key: apiKey}})
-	if err != nil {
-		return "", errors.Prefix("error creating YouTube service", err)
-	}
-
-	response, err := service.Channels.List("contentDetails").Id(channelID).Do()
-	if err != nil {
-		return "", errors.Prefix("error getting channel details", err)
-	}
-
-	if len(response.Items) < 1 {
-		return "", errors.Err("youtube channel not found")
-	}
-
-	if response.Items[0].ContentDetails.RelatedPlaylists == nil {
-		return "", errors.Err("no related playlists")
-	}
-
-	playlistID := response.Items[0].ContentDetails.RelatedPlaylists.Uploads
-	if playlistID == "" {
-		return "", errors.Err("no channel playlist")
-	}
-
-	return playlistID, nil
-}
-
-func getPlaylistItems(apiKey, playlistID, nextPageToken string) ([]*ytlib.PlaylistItem, string, error) {
-	service, err := ytlib.New(&http.Client{Transport: &transport.APIKey{Key: apiKey}})
-	if err != nil {
-		return nil, "", errors.Prefix("error creating YouTube service", err)
-	}
-
-	response, err := service.PlaylistItems.List("snippet").
-		PlaylistId(playlistID).
-		MaxResults(50).
-		PageToken(nextPageToken).
-		Do()
-
-	if err != nil {
-		return nil, "", errors.Prefix("error getting playlist items", err)
-	}
-
-	return response.Items, response.NextPageToken, nil
-}
-
 func getVideos(apiKey string, videoIDs []string) ([]*ytlib.Video, error) {
 	service, err := ytlib.New(&http.Client{Transport: &transport.APIKey{Key: apiKey}})
 	if err != nil {