diff --git a/downloader/downloader.go b/downloader/downloader.go index f63ba4b..b49d6e5 100644 --- a/downloader/downloader.go +++ b/downloader/downloader.go @@ -1,17 +1,21 @@ package downloader import ( + "encoding/json" + "io" "io/ioutil" "os/exec" "strings" + "github.com/lbryio/ytsync/v5/downloader/ytdl" + "github.com/lbryio/lbry.go/v2/extras/errors" "github.com/sirupsen/logrus" ) func GetPlaylistVideoIDs(channelName string, maxVideos int) ([]string, error) { args := []string{"--skip-download", "https://www.youtube.com/channel/" + channelName, "--get-id", "--flat-playlist"} - ids, err := run(args) + ids, err := run(args, true, true) if err != nil { return nil, errors.Err(err) } @@ -25,26 +29,56 @@ func GetPlaylistVideoIDs(channelName string, maxVideos int) ([]string, error) { return videoIDs, nil } -func run(args []string) ([]string, error) { +func GetVideoInformation(videoID string) (*ytdl.YtdlVideo, error) { + args := []string{"--skip-download", "--print-json", "https://www.youtube.com/watch?v=" + videoID} + results, err := run(args, false, true) + if err != nil { + return nil, errors.Err(err) + } + var video *ytdl.YtdlVideo + err = json.Unmarshal([]byte(results[0]), &video) + if err != nil { + return nil, errors.Err(err) + } + return video, nil +} + +func run(args []string, withStdErr, withStdOut bool) ([]string, error) { cmd := exec.Command("youtube-dl", args...) logrus.Printf("Running command youtube-dl %s", strings.Join(args, " ")) - stderr, err := cmd.StderrPipe() - if err != nil { - return nil, errors.Err(err) - } - stdout, err := cmd.StdoutPipe() - if err != nil { - return nil, errors.Err(err) + var stderr io.ReadCloser + var errorLog []byte + if withStdErr { + var err error + stderr, err = cmd.StderrPipe() + if err != nil { + return nil, errors.Err(err) + } + errorLog, err = ioutil.ReadAll(stderr) + if err != nil { + return nil, errors.Err(err) + } } - if err := cmd.Start(); err != nil { - return nil, errors.Err(err) - } + var stdout io.ReadCloser + var outLog []byte + if withStdOut { + var err error + stdout, err = cmd.StdoutPipe() + if err != nil { + return nil, errors.Err(err) + } - errorLog, _ := ioutil.ReadAll(stderr) - outLog, _ := ioutil.ReadAll(stdout) - err = cmd.Wait() + if err := cmd.Start(); err != nil { + return nil, errors.Err(err) + } + outLog, err = ioutil.ReadAll(stdout) + if err != nil { + return nil, errors.Err(err) + } + } + err := cmd.Wait() if len(errorLog) > 0 { return nil, errors.Err(err) } diff --git a/downloader/downloader_test.go b/downloader/downloader_test.go index 26e2aaf..1a80d6d 100644 --- a/downloader/downloader_test.go +++ b/downloader/downloader_test.go @@ -6,9 +6,7 @@ import ( "github.com/sirupsen/logrus" ) -func TestRun(t *testing.T) { - //args := []string{"--skip-download", "https://www.youtube.com/c/Electroboom", "--get-id", "--flat-playlist"} - //videoIDs, err := GetPlaylistVideoIDs("Electroboom") +func TestGetPlaylistVideoIDs(t *testing.T) { videoIDs, err := GetPlaylistVideoIDs("UCJ0-OtVpF0wOKEqT2Z1HEtA", 50) if err != nil { logrus.Error(err) @@ -17,3 +15,13 @@ func TestRun(t *testing.T) { println(id) } } + +func TestGetVideoInformation(t *testing.T) { + video, err := GetVideoInformation("zj7pXM9gE5M") + if err != nil { + logrus.Error(err) + } + if video != nil { + logrus.Info(video.ID) + } +} diff --git a/downloader/ytdl/Video.go b/downloader/ytdl/Video.go new file mode 100644 index 0000000..4dff358 --- /dev/null +++ b/downloader/ytdl/Video.go @@ -0,0 +1,140 @@ +package ytdl + +type YtdlVideo struct { + UploadDate string `json:"upload_date"` + Extractor string `json:"extractor"` + Series interface{} `json:"series"` + Format string `json:"format"` + Vbr interface{} `json:"vbr"` + Chapters interface{} `json:"chapters"` + Height int `json:"height"` + LikeCount interface{} `json:"like_count"` + Duration int `json:"duration"` + Fulltitle string `json:"fulltitle"` + PlaylistIndex interface{} `json:"playlist_index"` + Album interface{} `json:"album"` + ViewCount int `json:"view_count"` + Playlist interface{} `json:"playlist"` + Title string `json:"title"` + Filename string `json:"_filename"` + Creator interface{} `json:"creator"` + Ext string `json:"ext"` + ID string `json:"id"` + DislikeCount interface{} `json:"dislike_count"` + AverageRating float64 `json:"average_rating"` + Abr int `json:"abr"` + UploaderURL string `json:"uploader_url"` + Categories []string `json:"categories"` + Fps int `json:"fps"` + StretchedRatio interface{} `json:"stretched_ratio"` + SeasonNumber interface{} `json:"season_number"` + Annotations interface{} `json:"annotations"` + WebpageURLBasename string `json:"webpage_url_basename"` + Acodec string `json:"acodec"` + DisplayID string `json:"display_id"` + RequestedFormats []struct { + Asr interface{} `json:"asr"` + Tbr float64 `json:"tbr"` + Container string `json:"container"` + Language interface{} `json:"language"` + Format string `json:"format"` + URL string `json:"url"` + Vcodec string `json:"vcodec"` + FormatNote string `json:"format_note"` + Height int `json:"height"` + Width int `json:"width"` + Ext string `json:"ext"` + FragmentBaseURL string `json:"fragment_base_url"` + Filesize interface{} `json:"filesize"` + Fps int `json:"fps"` + ManifestURL string `json:"manifest_url"` + Protocol string `json:"protocol"` + FormatID string `json:"format_id"` + HTTPHeaders struct { + AcceptCharset string `json:"Accept-Charset"` + AcceptLanguage string `json:"Accept-Language"` + AcceptEncoding string `json:"Accept-Encoding"` + Accept string `json:"Accept"` + UserAgent string `json:"User-Agent"` + } `json:"http_headers"` + Fragments []struct { + Path string `json:"path"` + Duration float64 `json:"duration,omitempty"` + } `json:"fragments"` + Acodec string `json:"acodec"` + Abr int `json:"abr,omitempty"` + } `json:"requested_formats"` + AutomaticCaptions struct { + } `json:"automatic_captions"` + Description string `json:"description"` + Tags []string `json:"tags"` + Track interface{} `json:"track"` + RequestedSubtitles interface{} `json:"requested_subtitles"` + StartTime interface{} `json:"start_time"` + Uploader string `json:"uploader"` + ExtractorKey string `json:"extractor_key"` + FormatID string `json:"format_id"` + EpisodeNumber interface{} `json:"episode_number"` + UploaderID string `json:"uploader_id"` + Subtitles struct { + } `json:"subtitles"` + ReleaseYear interface{} `json:"release_year"` + Thumbnails []Thumbnail `json:"thumbnails"` + License interface{} `json:"license"` + Artist interface{} `json:"artist"` + AgeLimit int `json:"age_limit"` + ReleaseDate interface{} `json:"release_date"` + AltTitle interface{} `json:"alt_title"` + Thumbnail string `json:"thumbnail"` + ChannelID string `json:"channel_id"` + IsLive interface{} `json:"is_live"` + Width int `json:"width"` + EndTime interface{} `json:"end_time"` + WebpageURL string `json:"webpage_url"` + Formats []struct { + Asr int `json:"asr"` + Tbr float64 `json:"tbr"` + Protocol string `json:"protocol"` + Format string `json:"format"` + FormatNote string `json:"format_note"` + Height interface{} `json:"height"` + ManifestURL string `json:"manifest_url,omitempty"` + FormatID string `json:"format_id"` + Container string `json:"container,omitempty"` + Language interface{} `json:"language,omitempty"` + HTTPHeaders HTTPHeaders `json:"http_headers"` + URL string `json:"url"` + Vcodec string `json:"vcodec"` + Abr int `json:"abr,omitempty"` + Width interface{} `json:"width"` + Ext string `json:"ext"` + FragmentBaseURL string `json:"fragment_base_url,omitempty"` + Filesize interface{} `json:"filesize"` + Fps interface{} `json:"fps"` + Fragments []struct { + Path string `json:"path"` + Duration float64 `json:"duration,omitempty"` + } `json:"fragments,omitempty"` + Acodec string `json:"acodec"` + PlayerURL interface{} `json:"player_url,omitempty"` + } `json:"formats"` + ChannelURL string `json:"channel_url"` + Resolution interface{} `json:"resolution"` + Vcodec string `json:"vcodec"` +} + +type Thumbnail struct { + URL string `json:"url"` + Width int `json:"width"` + Resolution string `json:"resolution"` + ID string `json:"id"` + Height int `json:"height"` +} + +type HTTPHeaders struct { + AcceptCharset string `json:"Accept-Charset"` + AcceptLanguage string `json:"Accept-Language"` + AcceptEncoding string `json:"Accept-Encoding"` + Accept string `json:"Accept"` + UserAgent string `json:"User-Agent"` +} diff --git a/sources/youtubeVideo.go b/sources/youtubeVideo.go index 2953240..9d11e8a 100644 --- a/sources/youtubeVideo.go +++ b/sources/youtubeVideo.go @@ -13,6 +13,8 @@ import ( "sync" "time" + "github.com/lbryio/ytsync/v5/downloader/ytdl" + "github.com/lbryio/ytsync/v5/ip_manager" "github.com/lbryio/ytsync/v5/namer" "github.com/lbryio/ytsync/v5/sdk" @@ -26,11 +28,9 @@ import ( "github.com/lbryio/lbry.go/v2/extras/stop" "github.com/lbryio/lbry.go/v2/extras/util" - duration "github.com/ChannelMeter/iso8601duration" "github.com/aws/aws-sdk-go/aws" "github.com/shopspring/decimal" log "github.com/sirupsen/logrus" - ytlib "google.golang.org/api/youtube/v3" ) type YoutubeVideo struct { @@ -43,7 +43,7 @@ type YoutubeVideo struct { maxVideoLength float64 publishedAt time.Time dir string - youtubeInfo *ytlib.Video + youtubeInfo *ytdl.YtdlVideo youtubeChannelID string tags []string awsConfig aws.Config @@ -90,22 +90,25 @@ var youtubeCategories = map[string]string{ "44": "trailers", } -func NewYoutubeVideo(directory string, videoData *ytlib.Video, playlistPosition int64, awsConfig aws.Config, stopGroup *stop.Group, pool *ip_manager.IPPool) *YoutubeVideo { - publishedAt, _ := time.Parse(time.RFC3339Nano, videoData.Snippet.PublishedAt) // ignore parse errors +func NewYoutubeVideo(directory string, videoData *ytdl.YtdlVideo, playlistPosition int64, awsConfig aws.Config, stopGroup *stop.Group, pool *ip_manager.IPPool) (*YoutubeVideo, error) { + publishedAt, err := time.Parse("20060102", videoData.UploadDate) + if err != nil { + return nil, errors.Err(err) + } return &YoutubeVideo{ - id: videoData.Id, - title: videoData.Snippet.Title, - description: videoData.Snippet.Description, + id: videoData.ID, + title: videoData.Title, + description: videoData.Description, playlistPosition: playlistPosition, publishedAt: publishedAt, dir: directory, youtubeInfo: videoData, awsConfig: awsConfig, mocked: false, - youtubeChannelID: videoData.Snippet.ChannelId, + youtubeChannelID: videoData.ChannelID, stopGroup: stopGroup, pool: pool, - } + }, nil } func NewMockedVideo(directory string, videoID string, youtubeChannelID string, awsConfig aws.Config, stopGroup *stop.Group, pool *ip_manager.IPPool) *YoutubeVideo { return &YoutubeVideo{ @@ -185,7 +188,7 @@ func (v *YoutubeVideo) download() error { defer func(start time.Time) { timing.TimedComponent("download").Add(time.Since(start)) }(start) - if v.youtubeInfo.Snippet.LiveBroadcastContent != "none" { + if v.youtubeInfo.IsLive != nil { return errors.Err("video is a live stream and hasn't completed yet") } videoPath := v.getFullPath() @@ -370,8 +373,8 @@ func (v *YoutubeVideo) delete(reason string) error { } func (v *YoutubeVideo) triggerThumbnailSave() (err error) { - thumbnail := thumbs.GetBestThumbnail(v.youtubeInfo.Snippet.Thumbnails) - v.thumbnailURL, err = thumbs.MirrorThumbnail(thumbnail.Url, v.ID(), v.awsConfig) + thumbnail := thumbs.GetBestThumbnail(v.youtubeInfo.Thumbnails) + v.thumbnailURL, err = thumbs.MirrorThumbnail(thumbnail.URL, v.ID(), v.awsConfig) return err } @@ -448,13 +451,10 @@ func (v *YoutubeVideo) Sync(daemon *jsonrpc.Client, params SyncParams, existingV func (v *YoutubeVideo) downloadAndPublish(daemon *jsonrpc.Client, params SyncParams) (*SyncSummary, error) { var err error - videoDuration, err := duration.FromString(v.youtubeInfo.ContentDetails.Duration) - if err != nil { - return nil, errors.Err(err) - } - if videoDuration.ToDuration() > time.Duration(v.maxVideoLength*60)*time.Minute { - log.Infof("%s is %s long and the limit is %s", v.id, videoDuration.ToDuration().String(), (time.Duration(v.maxVideoLength*60) * time.Minute).String()) - logUtils.SendErrorToSlack("%s is %s long and the limit is %s", v.id, videoDuration.ToDuration().String(), (time.Duration(v.maxVideoLength*60) * time.Minute).String()) + + if float64(v.youtubeInfo.Duration) > v.maxVideoLength { + log.Infof("%s is %d long and the limit is %s", v.id, v.youtubeInfo.Duration, (time.Duration(v.maxVideoLength*60) * time.Minute).String()) + logUtils.SendErrorToSlack("%s is %d long and the limit is %s", v.id, v.youtubeInfo.Duration, (time.Duration(v.maxVideoLength*60) * time.Minute).String()) return nil, errors.Err("video is too long to process") } for { @@ -487,27 +487,30 @@ func (v *YoutubeVideo) getMetadata() (languages []string, locations []jsonrpc.Lo locations = nil tags = nil if !v.mocked { - if v.youtubeInfo.Snippet.DefaultLanguage != "" { - if v.youtubeInfo.Snippet.DefaultLanguage == "iw" { - v.youtubeInfo.Snippet.DefaultLanguage = "he" - } - languages = []string{v.youtubeInfo.Snippet.DefaultLanguage} - } + /* + if v.youtubeInfo.Snippet.DefaultLanguage != "" { + if v.youtubeInfo.Snippet.DefaultLanguage == "iw" { + v.youtubeInfo.Snippet.DefaultLanguage = "he" + } + languages = []string{v.youtubeInfo.Snippet.DefaultLanguage} + }*/ - if v.youtubeInfo.RecordingDetails != nil && v.youtubeInfo.RecordingDetails.Location != nil { + /*if v.youtubeInfo.!= nil && v.youtubeInfo.RecordingDetails.Location != nil { locations = []jsonrpc.Location{{ Latitude: util.PtrToString(fmt.Sprintf("%.7f", v.youtubeInfo.RecordingDetails.Location.Latitude)), Longitude: util.PtrToString(fmt.Sprintf("%.7f", v.youtubeInfo.RecordingDetails.Location.Longitude)), }} - } - tags = v.youtubeInfo.Snippet.Tags + }*/ + tags = v.youtubeInfo.Tags } tags, err := tags_manager.SanitizeTags(tags, v.youtubeChannelID) if err != nil { log.Errorln(err.Error()) } if !v.mocked { - tags = append(tags, youtubeCategories[v.youtubeInfo.Snippet.CategoryId]) + for _, category := range v.youtubeInfo.Categories { + tags = append(tags, youtubeCategories[category]) + } } return languages, locations, tags @@ -532,8 +535,8 @@ func (v *YoutubeVideo) reprocess(daemon *jsonrpc.Client, params SyncParams, exis if v.mocked { return nil, errors.Err("could not find thumbnail for mocked video") } - thumbnail := thumbs.GetBestThumbnail(v.youtubeInfo.Snippet.Thumbnails) - thumbnailURL, err = thumbs.MirrorThumbnail(thumbnail.Url, v.ID(), v.awsConfig) + thumbnail := thumbs.GetBestThumbnail(v.youtubeInfo.Thumbnails) + thumbnailURL, err = thumbs.MirrorThumbnail(thumbnail.URL, v.ID(), v.awsConfig) } else { thumbnailURL = thumbs.ThumbnailEndpoint + v.ID() } @@ -605,14 +608,9 @@ func (v *YoutubeVideo) reprocess(daemon *jsonrpc.Client, params SyncParams, exis }, nil } - videoDuration, err := duration.FromString(v.youtubeInfo.ContentDetails.Duration) - if err != nil { - return nil, errors.Err(err) - } - streamCreateOptions.ClaimCreateOptions.Title = &v.title streamCreateOptions.ClaimCreateOptions.Description = util.PtrToString(v.getAbbrevDescription()) - streamCreateOptions.Duration = util.PtrToUint64(uint64(math.Ceil(videoDuration.ToDuration().Seconds()))) + streamCreateOptions.Duration = util.PtrToUint64(uint64(v.youtubeInfo.Duration)) streamCreateOptions.ReleaseTime = util.PtrToInt64(v.publishedAt.Unix()) start := time.Now() pr, err := daemon.StreamUpdate(existingVideoData.ClaimID, jsonrpc.StreamUpdateOptions{ diff --git a/thumbs/uploader.go b/thumbs/uploader.go index 19e9f84..e1ff62e 100644 --- a/thumbs/uploader.go +++ b/thumbs/uploader.go @@ -5,13 +5,14 @@ import ( "net/http" "os" + "github.com/lbryio/ytsync/v5/downloader/ytdl" + "github.com/lbryio/lbry.go/v2/extras/errors" "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/session" "github.com/aws/aws-sdk-go/service/s3/s3manager" log "github.com/sirupsen/logrus" - ytlib "google.golang.org/api/youtube/v3" ) type thumbnailUploader struct { @@ -98,15 +99,14 @@ func MirrorThumbnail(url string, name string, s3Config aws.Config) (string, erro return tu.mirroredUrl, nil } -func GetBestThumbnail(thumbnails *ytlib.ThumbnailDetails) *ytlib.Thumbnail { - if thumbnails.Maxres != nil { - return thumbnails.Maxres - } else if thumbnails.High != nil { - return thumbnails.High - } else if thumbnails.Medium != nil { - return thumbnails.Medium - } else if thumbnails.Standard != nil { - return thumbnails.Standard +func GetBestThumbnail(thumbnails []ytdl.Thumbnail) *ytdl.Thumbnail { + var bestWidth *ytdl.Thumbnail + for _, thumbnail := range thumbnails { + if bestWidth == nil { + bestWidth = &thumbnail + } else if bestWidth.Width < thumbnail.Width { + bestWidth = &thumbnail + } } - return thumbnails.Default + return bestWidth } diff --git a/ytapi/ytapi.go b/ytapi/ytapi.go index 9f6cca0..a290b9e 100644 --- a/ytapi/ytapi.go +++ b/ytapi/ytapi.go @@ -10,6 +10,8 @@ import ( "sync" "time" + "github.com/lbryio/ytsync/v5/downloader/ytdl" + "github.com/lbryio/ytsync/v5/downloader" "github.com/lbryio/ytsync/v5/ip_manager" "github.com/lbryio/ytsync/v5/sdk" @@ -74,14 +76,18 @@ func GetVideosToSync(apiKey, channelID string, syncedVideos map[string]sdk.Synce mostRecentlyFailedChannel = channelID } - vids, err := getVideos(apiKey, videoIDs) + vids, err := getVideos(videoIDs) if err != nil { return nil, err } for _, item := range vids { - positionInList := playlistMap[item.Id] - videos = append(videos, sources.NewYoutubeVideo(videoParams.VideoDir, item, positionInList, videoParams.S3Config, videoParams.Grp, videoParams.IPPool)) + positionInList := playlistMap[item.ID] + videoToAdd, err := sources.NewYoutubeVideo(videoParams.VideoDir, item, positionInList, videoParams.S3Config, videoParams.Grp, videoParams.IPPool) + if err != nil { + return nil, errors.Err(err) + } + videos = append(videos, videoToAdd) } for k, v := range syncedVideos { @@ -154,16 +160,14 @@ func ChannelInfo(apiKey, channelID string) (*ytlib.ChannelSnippet, *ytlib.Channe return response.Items[0].Snippet, response.Items[0].BrandingSettings, nil } -func getVideos(apiKey string, videoIDs []string) ([]*ytlib.Video, error) { - service, err := ytlib.New(&http.Client{Transport: &transport.APIKey{Key: apiKey}}) - if err != nil { - return nil, errors.Prefix("error creating YouTube service", err) +func getVideos(videoIDs []string) ([]*ytdl.YtdlVideo, error) { + var videos []*ytdl.YtdlVideo + for _, videoID := range videoIDs { + video, err := downloader.GetVideoInformation(videoID) + if err != nil { + return nil, errors.Err(err) + } + videos = append(videos, video) } - - response, err := service.Videos.List("snippet,contentDetails,recordingDetails").Id(strings.Join(videoIDs[:], ",")).Do() - if err != nil { - return nil, errors.Prefix("error getting videos info", err) - } - - return response.Items, nil + return videos, nil }