Add downloader to create api
adjust GetVideosToSync
This commit is contained in:
Mark Beamer Jr
parent
bffc0823be
commit
f39fc11697
3 changed files with 91 additions and 92 deletions
44
downloader/downloader.go
Normal file
44
downloader/downloader.go
Normal file
|
|
@ -0,0 +1,44 @@
|
|||
package downloader
|
||||
|
||||
import (
|
||||
"io/ioutil"
|
||||
"os/exec"
|
||||
"strings"
|
||||
|
||||
"github.com/lbryio/lbry.go/v2/extras/errors"
|
||||
"github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
func GetPlaylistVideoIDs(channelName string) ([]string, error) {
|
||||
args := []string{"--skip-download", "https://www.youtube.com/channel/" + channelName, "--get-id", "--flat-playlist"}
|
||||
return run(args)
|
||||
}
|
||||
|
||||
func run(args []string) ([]string, error) {
|
||||
cmd := exec.Command("youtube-dl", args...)
|
||||
logrus.Printf("Running command youtube-dl %s", strings.Join(args, " "))
|
||||
|
||||
stderr, err := cmd.StderrPipe()
|
||||
if err != nil {
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
stdout, err := cmd.StdoutPipe()
|
||||
if err != nil {
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
|
||||
if err := cmd.Start(); err != nil {
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
|
||||
errorLog, _ := ioutil.ReadAll(stderr)
|
||||
outLog, _ := ioutil.ReadAll(stdout)
|
||||
err = cmd.Wait()
|
||||
if len(errorLog) > 0 {
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
if len(errorLog) > 0 {
|
||||
return nil, errors.Err(string(errorLog))
|
||||
}
|
||||
return strings.Split(strings.Replace(string(outLog), "\r\n", "\n", -1), "\n"), nil
|
||||
}
|
||||
20
downloader/downloader_test.go
Normal file
20
downloader/downloader_test.go
Normal file
|
|
@ -0,0 +1,20 @@
|
|||
package downloader
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
func TestRun(t *testing.T) {
|
||||
//args := []string{"--skip-download", "https://www.youtube.com/c/Electroboom", "--get-id", "--flat-playlist"}
|
||||
//videoIDs, err := GetPlaylistVideoIDs("Electroboom")
|
||||
videoIDs, err := GetPlaylistVideoIDs("UCJ0-OtVpF0wOKEqT2Z1HEtA")
|
||||
if err != nil {
|
||||
logrus.Error(err)
|
||||
}
|
||||
for _, id := range videoIDs {
|
||||
println(id)
|
||||
}
|
||||
t.Error("just stop")
|
||||
}
|
||||
|
|
@ -7,6 +7,8 @@ import (
|
|||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/lbryio/ytsync/v5/downloader"
|
||||
|
||||
"github.com/lbryio/ytsync/v5/ip_manager"
|
||||
"github.com/lbryio/ytsync/v5/sdk"
|
||||
"github.com/lbryio/ytsync/v5/sources"
|
||||
|
|
@ -46,39 +48,25 @@ type VideoParams struct {
|
|||
var mostRecentlyFailedChannel string // TODO: fix this hack!
|
||||
|
||||
func GetVideosToSync(apiKey, channelID string, syncedVideos map[string]sdk.SyncedVideo, quickSync bool, maxVideos int, videoParams VideoParams) ([]Video, error) {
|
||||
playlistID, err := getPlaylistID(apiKey, channelID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
playlistMap := make(map[string]*ytlib.PlaylistItemSnippet, 50)
|
||||
var playlistItems []*ytlib.PlaylistItem
|
||||
var nextPageToken string
|
||||
var videos []Video
|
||||
|
||||
for {
|
||||
playlistItems, nextPageToken, err = getPlaylistItems(apiKey, playlistID, nextPageToken)
|
||||
|
||||
if len(playlistItems) < 1 {
|
||||
// If there are 50+ videos in a playlist but less than 50 are actually returned by the API, youtube will still redirect
|
||||
// clients to a next page. Such next page will however be empty. This logic prevents ytsync from failing.
|
||||
youtubeIsLying := len(videos) > 0
|
||||
if youtubeIsLying {
|
||||
break
|
||||
videoIDs, err := downloader.GetPlaylistVideoIDs(channelID)
|
||||
if err != nil {
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
|
||||
log.Infof("Got info for %d videos from youtube downloader", len(videoIDs))
|
||||
|
||||
playlistMap := make(map[string]int64)
|
||||
for i, videoID := range videoIDs {
|
||||
playlistMap[videoID] = int64(i)
|
||||
}
|
||||
|
||||
if len(videoIDs) < 1 {
|
||||
if channelID == mostRecentlyFailedChannel {
|
||||
return nil, errors.Err("playlist items not found")
|
||||
}
|
||||
mostRecentlyFailedChannel = channelID
|
||||
break //return errors.Err("playlist items not found") //TODO: will this work?
|
||||
}
|
||||
|
||||
videoIDs := make([]string, len(playlistItems))
|
||||
for i, item := range playlistItems {
|
||||
// normally we'd send the video into the channel here, but youtube api doesn't have sorting
|
||||
// so we have to get ALL the videos, then sort them, then send them in
|
||||
playlistMap[item.Snippet.ResourceId.VideoId] = item.Snippet
|
||||
videoIDs[i] = item.Snippet.ResourceId.VideoId
|
||||
}
|
||||
|
||||
vids, err := getVideos(apiKey, videoIDs)
|
||||
|
|
@ -87,14 +75,8 @@ func GetVideosToSync(apiKey, channelID string, syncedVideos map[string]sdk.Synce
|
|||
}
|
||||
|
||||
for _, item := range vids {
|
||||
videos = append(videos, sources.NewYoutubeVideo(videoParams.VideoDir, item, playlistMap[item.Id].Position, videoParams.S3Config, videoParams.Grp, videoParams.IPPool))
|
||||
}
|
||||
|
||||
log.Infof("Got info for %d videos from youtube API", len(videos))
|
||||
|
||||
if nextPageToken == "" || quickSync || len(videos) >= maxVideos {
|
||||
break
|
||||
}
|
||||
positionInList := playlistMap[item.Id]
|
||||
videos = append(videos, sources.NewYoutubeVideo(videoParams.VideoDir, item, positionInList, videoParams.S3Config, videoParams.Grp, videoParams.IPPool))
|
||||
}
|
||||
|
||||
for k, v := range syncedVideos {
|
||||
|
|
@ -104,7 +86,6 @@ func GetVideosToSync(apiKey, channelID string, syncedVideos map[string]sdk.Synce
|
|||
if _, ok := playlistMap[k]; !ok {
|
||||
videos = append(videos, sources.NewMockedVideo(videoParams.VideoDir, k, channelID, videoParams.S3Config, videoParams.Grp, videoParams.IPPool))
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
sort.Sort(byPublishedAt(videos))
|
||||
|
|
@ -152,52 +133,6 @@ func ChannelInfo(apiKey, channelID string) (*ytlib.ChannelSnippet, *ytlib.Channe
|
|||
return response.Items[0].Snippet, response.Items[0].BrandingSettings, nil
|
||||
}
|
||||
|
||||
func getPlaylistID(apiKey, channelID string) (string, error) {
|
||||
service, err := ytlib.New(&http.Client{Transport: &transport.APIKey{Key: apiKey}})
|
||||
if err != nil {
|
||||
return "", errors.Prefix("error creating YouTube service", err)
|
||||
}
|
||||
|
||||
response, err := service.Channels.List("contentDetails").Id(channelID).Do()
|
||||
if err != nil {
|
||||
return "", errors.Prefix("error getting channel details", err)
|
||||
}
|
||||
|
||||
if len(response.Items) < 1 {
|
||||
return "", errors.Err("youtube channel not found")
|
||||
}
|
||||
|
||||
if response.Items[0].ContentDetails.RelatedPlaylists == nil {
|
||||
return "", errors.Err("no related playlists")
|
||||
}
|
||||
|
||||
playlistID := response.Items[0].ContentDetails.RelatedPlaylists.Uploads
|
||||
if playlistID == "" {
|
||||
return "", errors.Err("no channel playlist")
|
||||
}
|
||||
|
||||
return playlistID, nil
|
||||
}
|
||||
|
||||
func getPlaylistItems(apiKey, playlistID, nextPageToken string) ([]*ytlib.PlaylistItem, string, error) {
|
||||
service, err := ytlib.New(&http.Client{Transport: &transport.APIKey{Key: apiKey}})
|
||||
if err != nil {
|
||||
return nil, "", errors.Prefix("error creating YouTube service", err)
|
||||
}
|
||||
|
||||
response, err := service.PlaylistItems.List("snippet").
|
||||
PlaylistId(playlistID).
|
||||
MaxResults(50).
|
||||
PageToken(nextPageToken).
|
||||
Do()
|
||||
|
||||
if err != nil {
|
||||
return nil, "", errors.Prefix("error getting playlist items", err)
|
||||
}
|
||||
|
||||
return response.Items, response.NextPageToken, nil
|
||||
}
|
||||
|
||||
func getVideos(apiKey string, videoIDs []string) ([]*ytlib.Video, error) {
|
||||
service, err := ytlib.New(&http.Client{Transport: &transport.APIKey{Key: apiKey}})
|
||||
if err != nil {
|
||||
|
|
|
|||
Loading…
Reference in a new issue