commit my shit

downloader/downloader.go

@@ -1,17 +1,21 @@
 package downloader
 
 import (
+	"encoding/json"
+	"io"
 	"io/ioutil"
 	"os/exec"
 	"strings"
 
+	"github.com/lbryio/ytsync/v5/downloader/ytdl"
+
 	"github.com/lbryio/lbry.go/v2/extras/errors"
 	"github.com/sirupsen/logrus"
 )
 
 func GetPlaylistVideoIDs(channelName string, maxVideos int) ([]string, error) {
 	args := []string{"--skip-download", "https://www.youtube.com/channel/" + channelName, "--get-id", "--flat-playlist"}
-	ids, err := run(args)
+	ids, err := run(args, true, true)
 	if err != nil {
 		return nil, errors.Err(err)
 	}
@@ -25,15 +29,43 @@ func GetPlaylistVideoIDs(channelName string, maxVideos int) ([]string, error) {
 	return videoIDs, nil
 }
 
-func run(args []string) ([]string, error) {
-	cmd := exec.Command("youtube-dl", args...)
-	logrus.Printf("Running command youtube-dl %s", strings.Join(args, " "))
-
-	stderr, err := cmd.StderrPipe()
+func GetVideoInformation(videoID string) (*ytdl.YtdlVideo, error) {
+	args := []string{"--skip-download", "--print-json", "https://www.youtube.com/watch?v=" + videoID}
+	results, err := run(args, false, true)
 	if err != nil {
 		return nil, errors.Err(err)
 	}
-	stdout, err := cmd.StdoutPipe()
+	var video *ytdl.YtdlVideo
+	err = json.Unmarshal([]byte(results[0]), &video)
+	if err != nil {
+		return nil, errors.Err(err)
+	}
+	return video, nil
+}
+
+func run(args []string, withStdErr, withStdOut bool) ([]string, error) {
+	cmd := exec.Command("youtube-dl", args...)
+	logrus.Printf("Running command youtube-dl %s", strings.Join(args, " "))
+
+	var stderr io.ReadCloser
+	var errorLog []byte
+	if withStdErr {
+		var err error
+		stderr, err = cmd.StderrPipe()
+		if err != nil {
+			return nil, errors.Err(err)
+		}
+		errorLog, err = ioutil.ReadAll(stderr)
+		if err != nil {
+			return nil, errors.Err(err)
+		}
+	}
+
+	var stdout io.ReadCloser
+	var outLog []byte
+	if withStdOut {
+		var err error
+		stdout, err = cmd.StdoutPipe()
 		if err != nil {
 			return nil, errors.Err(err)
 		}
@@ -41,10 +73,12 @@ func run(args []string) ([]string, error) {
 		if err := cmd.Start(); err != nil {
 			return nil, errors.Err(err)
 		}
-
-	errorLog, _ := ioutil.ReadAll(stderr)
-	outLog, _ := ioutil.ReadAll(stdout)
-	err = cmd.Wait()
+		outLog, err = ioutil.ReadAll(stdout)
+		if err != nil {
+			return nil, errors.Err(err)
+		}
+	}
+	err := cmd.Wait()
 	if len(errorLog) > 0 {
 		return nil, errors.Err(err)
 	}

downloader/downloader_test.go

@@ -6,9 +6,7 @@ import (
 	"github.com/sirupsen/logrus"
 )
 
-func TestRun(t *testing.T) {
-	//args := []string{"--skip-download", "https://www.youtube.com/c/Electroboom", "--get-id", "--flat-playlist"}
-	//videoIDs, err := GetPlaylistVideoIDs("Electroboom")
+func TestGetPlaylistVideoIDs(t *testing.T) {
 	videoIDs, err := GetPlaylistVideoIDs("UCJ0-OtVpF0wOKEqT2Z1HEtA", 50)
 	if err != nil {
 		logrus.Error(err)
@@ -17,3 +15,13 @@ func TestRun(t *testing.T) {
 		println(id)
 	}
 }
+
+func TestGetVideoInformation(t *testing.T) {
+	video, err := GetVideoInformation("zj7pXM9gE5M")
+	if err != nil {
+		logrus.Error(err)
+	}
+	if video != nil {
+		logrus.Info(video.ID)
+	}
+}

downloader/ytdl/Video.go

@@ -0,0 +1,140 @@
+package ytdl
+
+type YtdlVideo struct {
+	UploadDate         string      `json:"upload_date"`
+	Extractor          string      `json:"extractor"`
+	Series             interface{} `json:"series"`
+	Format             string      `json:"format"`
+	Vbr                interface{} `json:"vbr"`
+	Chapters           interface{} `json:"chapters"`
+	Height             int         `json:"height"`
+	LikeCount          interface{} `json:"like_count"`
+	Duration           int         `json:"duration"`
+	Fulltitle          string      `json:"fulltitle"`
+	PlaylistIndex      interface{} `json:"playlist_index"`
+	Album              interface{} `json:"album"`
+	ViewCount          int         `json:"view_count"`
+	Playlist           interface{} `json:"playlist"`
+	Title              string      `json:"title"`
+	Filename           string      `json:"_filename"`
+	Creator            interface{} `json:"creator"`
+	Ext                string      `json:"ext"`
+	ID                 string      `json:"id"`
+	DislikeCount       interface{} `json:"dislike_count"`
+	AverageRating      float64     `json:"average_rating"`
+	Abr                int         `json:"abr"`
+	UploaderURL        string      `json:"uploader_url"`
+	Categories         []string    `json:"categories"`
+	Fps                int         `json:"fps"`
+	StretchedRatio     interface{} `json:"stretched_ratio"`
+	SeasonNumber       interface{} `json:"season_number"`
+	Annotations        interface{} `json:"annotations"`
+	WebpageURLBasename string      `json:"webpage_url_basename"`
+	Acodec             string      `json:"acodec"`
+	DisplayID          string      `json:"display_id"`
+	RequestedFormats   []struct {
+		Asr             interface{} `json:"asr"`
+		Tbr             float64     `json:"tbr"`
+		Container       string      `json:"container"`
+		Language        interface{} `json:"language"`
+		Format          string      `json:"format"`
+		URL             string      `json:"url"`
+		Vcodec          string      `json:"vcodec"`
+		FormatNote      string      `json:"format_note"`
+		Height          int         `json:"height"`
+		Width           int         `json:"width"`
+		Ext             string      `json:"ext"`
+		FragmentBaseURL string      `json:"fragment_base_url"`
+		Filesize        interface{} `json:"filesize"`
+		Fps             int         `json:"fps"`
+		ManifestURL     string      `json:"manifest_url"`
+		Protocol        string      `json:"protocol"`
+		FormatID        string      `json:"format_id"`
+		HTTPHeaders     struct {
+			AcceptCharset  string `json:"Accept-Charset"`
+			AcceptLanguage string `json:"Accept-Language"`
+			AcceptEncoding string `json:"Accept-Encoding"`
+			Accept         string `json:"Accept"`
+			UserAgent      string `json:"User-Agent"`
+		} `json:"http_headers"`
+		Fragments []struct {
+			Path     string  `json:"path"`
+			Duration float64 `json:"duration,omitempty"`
+		} `json:"fragments"`
+		Acodec string `json:"acodec"`
+		Abr    int    `json:"abr,omitempty"`
+	} `json:"requested_formats"`
+	AutomaticCaptions struct {
+	} `json:"automatic_captions"`
+	Description        string      `json:"description"`
+	Tags               []string    `json:"tags"`
+	Track              interface{} `json:"track"`
+	RequestedSubtitles interface{} `json:"requested_subtitles"`
+	StartTime          interface{} `json:"start_time"`
+	Uploader           string      `json:"uploader"`
+	ExtractorKey       string      `json:"extractor_key"`
+	FormatID           string      `json:"format_id"`
+	EpisodeNumber      interface{} `json:"episode_number"`
+	UploaderID         string      `json:"uploader_id"`
+	Subtitles          struct {
+	} `json:"subtitles"`
+	ReleaseYear interface{} `json:"release_year"`
+	Thumbnails  []Thumbnail `json:"thumbnails"`
+	License     interface{} `json:"license"`
+	Artist      interface{} `json:"artist"`
+	AgeLimit    int         `json:"age_limit"`
+	ReleaseDate interface{} `json:"release_date"`
+	AltTitle    interface{} `json:"alt_title"`
+	Thumbnail   string      `json:"thumbnail"`
+	ChannelID   string      `json:"channel_id"`
+	IsLive      interface{} `json:"is_live"`
+	Width       int         `json:"width"`
+	EndTime     interface{} `json:"end_time"`
+	WebpageURL  string      `json:"webpage_url"`
+	Formats     []struct {
+		Asr             int         `json:"asr"`
+		Tbr             float64     `json:"tbr"`
+		Protocol        string      `json:"protocol"`
+		Format          string      `json:"format"`
+		FormatNote      string      `json:"format_note"`
+		Height          interface{} `json:"height"`
+		ManifestURL     string      `json:"manifest_url,omitempty"`
+		FormatID        string      `json:"format_id"`
+		Container       string      `json:"container,omitempty"`
+		Language        interface{} `json:"language,omitempty"`
+		HTTPHeaders     HTTPHeaders `json:"http_headers"`
+		URL             string      `json:"url"`
+		Vcodec          string      `json:"vcodec"`
+		Abr             int         `json:"abr,omitempty"`
+		Width           interface{} `json:"width"`
+		Ext             string      `json:"ext"`
+		FragmentBaseURL string      `json:"fragment_base_url,omitempty"`
+		Filesize        interface{} `json:"filesize"`
+		Fps             interface{} `json:"fps"`
+		Fragments       []struct {
+			Path     string  `json:"path"`
+			Duration float64 `json:"duration,omitempty"`
+		} `json:"fragments,omitempty"`
+		Acodec    string      `json:"acodec"`
+		PlayerURL interface{} `json:"player_url,omitempty"`
+	} `json:"formats"`
+	ChannelURL string      `json:"channel_url"`
+	Resolution interface{} `json:"resolution"`
+	Vcodec     string      `json:"vcodec"`
+}
+
+type Thumbnail struct {
+	URL        string `json:"url"`
+	Width      int    `json:"width"`
+	Resolution string `json:"resolution"`
+	ID         string `json:"id"`
+	Height     int    `json:"height"`
+}
+
+type HTTPHeaders struct {
+	AcceptCharset  string `json:"Accept-Charset"`
+	AcceptLanguage string `json:"Accept-Language"`
+	AcceptEncoding string `json:"Accept-Encoding"`
+	Accept         string `json:"Accept"`
+	UserAgent      string `json:"User-Agent"`
+}

sources/youtubeVideo.go

@@ -13,6 +13,8 @@ import (
 	"sync"
 	"time"
 
+	"github.com/lbryio/ytsync/v5/downloader/ytdl"
+
 	"github.com/lbryio/ytsync/v5/ip_manager"
 	"github.com/lbryio/ytsync/v5/namer"
 	"github.com/lbryio/ytsync/v5/sdk"
@@ -26,11 +28,9 @@ import (
 	"github.com/lbryio/lbry.go/v2/extras/stop"
 	"github.com/lbryio/lbry.go/v2/extras/util"
 
-	duration "github.com/ChannelMeter/iso8601duration"
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/shopspring/decimal"
 	log "github.com/sirupsen/logrus"
-	ytlib "google.golang.org/api/youtube/v3"
 )
 
 type YoutubeVideo struct {
@@ -43,7 +43,7 @@ type YoutubeVideo struct {
 	maxVideoLength   float64
 	publishedAt      time.Time
 	dir              string
-	youtubeInfo      *ytlib.Video
+	youtubeInfo      *ytdl.YtdlVideo
 	youtubeChannelID string
 	tags             []string
 	awsConfig        aws.Config
@@ -90,22 +90,25 @@ var youtubeCategories = map[string]string{
 	"44": "trailers",
 }
 
-func NewYoutubeVideo(directory string, videoData *ytlib.Video, playlistPosition int64, awsConfig aws.Config, stopGroup *stop.Group, pool *ip_manager.IPPool) *YoutubeVideo {
-	publishedAt, _ := time.Parse(time.RFC3339Nano, videoData.Snippet.PublishedAt) // ignore parse errors
+func NewYoutubeVideo(directory string, videoData *ytdl.YtdlVideo, playlistPosition int64, awsConfig aws.Config, stopGroup *stop.Group, pool *ip_manager.IPPool) (*YoutubeVideo, error) {
+	publishedAt, err := time.Parse("20060102", videoData.UploadDate)
+	if err != nil {
+		return nil, errors.Err(err)
+	}
 	return &YoutubeVideo{
-		id:               videoData.Id,
-		title:            videoData.Snippet.Title,
-		description:      videoData.Snippet.Description,
+		id:               videoData.ID,
+		title:            videoData.Title,
+		description:      videoData.Description,
 		playlistPosition: playlistPosition,
 		publishedAt:      publishedAt,
 		dir:              directory,
 		youtubeInfo:      videoData,
 		awsConfig:        awsConfig,
 		mocked:           false,
-		youtubeChannelID: videoData.Snippet.ChannelId,
+		youtubeChannelID: videoData.ChannelID,
 		stopGroup:        stopGroup,
 		pool:             pool,
-	}
+	}, nil
 }
 func NewMockedVideo(directory string, videoID string, youtubeChannelID string, awsConfig aws.Config, stopGroup *stop.Group, pool *ip_manager.IPPool) *YoutubeVideo {
 	return &YoutubeVideo{
@@ -185,7 +188,7 @@ func (v *YoutubeVideo) download() error {
 	defer func(start time.Time) {
 		timing.TimedComponent("download").Add(time.Since(start))
 	}(start)
-	if v.youtubeInfo.Snippet.LiveBroadcastContent != "none" {
+	if v.youtubeInfo.IsLive != nil {
 		return errors.Err("video is a live stream and hasn't completed yet")
 	}
 	videoPath := v.getFullPath()
@@ -370,8 +373,8 @@ func (v *YoutubeVideo) delete(reason string) error {
 }
 
 func (v *YoutubeVideo) triggerThumbnailSave() (err error) {
-	thumbnail := thumbs.GetBestThumbnail(v.youtubeInfo.Snippet.Thumbnails)
-	v.thumbnailURL, err = thumbs.MirrorThumbnail(thumbnail.Url, v.ID(), v.awsConfig)
+	thumbnail := thumbs.GetBestThumbnail(v.youtubeInfo.Thumbnails)
+	v.thumbnailURL, err = thumbs.MirrorThumbnail(thumbnail.URL, v.ID(), v.awsConfig)
 	return err
 }
 
@@ -448,13 +451,10 @@ func (v *YoutubeVideo) Sync(daemon *jsonrpc.Client, params SyncParams, existingV
 
 func (v *YoutubeVideo) downloadAndPublish(daemon *jsonrpc.Client, params SyncParams) (*SyncSummary, error) {
 	var err error
-	videoDuration, err := duration.FromString(v.youtubeInfo.ContentDetails.Duration)
-	if err != nil {
-		return nil, errors.Err(err)
-	}
-	if videoDuration.ToDuration() > time.Duration(v.maxVideoLength*60)*time.Minute {
-		log.Infof("%s is %s long and the limit is %s", v.id, videoDuration.ToDuration().String(), (time.Duration(v.maxVideoLength*60) * time.Minute).String())
-		logUtils.SendErrorToSlack("%s is %s long and the limit is %s", v.id, videoDuration.ToDuration().String(), (time.Duration(v.maxVideoLength*60) * time.Minute).String())
+
+	if float64(v.youtubeInfo.Duration) > v.maxVideoLength {
+		log.Infof("%s is %d long and the limit is %s", v.id, v.youtubeInfo.Duration, (time.Duration(v.maxVideoLength*60) * time.Minute).String())
+		logUtils.SendErrorToSlack("%s is %d long and the limit is %s", v.id, v.youtubeInfo.Duration, (time.Duration(v.maxVideoLength*60) * time.Minute).String())
 		return nil, errors.Err("video is too long to process")
 	}
 	for {
@@ -487,27 +487,30 @@ func (v *YoutubeVideo) getMetadata() (languages []string, locations []jsonrpc.Lo
 	locations = nil
 	tags = nil
 	if !v.mocked {
+		/*
 			if v.youtubeInfo.Snippet.DefaultLanguage != "" {
 				if v.youtubeInfo.Snippet.DefaultLanguage == "iw" {
 					v.youtubeInfo.Snippet.DefaultLanguage = "he"
 				}
 				languages = []string{v.youtubeInfo.Snippet.DefaultLanguage}
-		}
+			}*/
 
-		if v.youtubeInfo.RecordingDetails != nil && v.youtubeInfo.RecordingDetails.Location != nil {
+		/*if v.youtubeInfo.!= nil && v.youtubeInfo.RecordingDetails.Location != nil {
 			locations = []jsonrpc.Location{{
 				Latitude:  util.PtrToString(fmt.Sprintf("%.7f", v.youtubeInfo.RecordingDetails.Location.Latitude)),
 				Longitude: util.PtrToString(fmt.Sprintf("%.7f", v.youtubeInfo.RecordingDetails.Location.Longitude)),
 			}}
-		}
-		tags = v.youtubeInfo.Snippet.Tags
+		}*/
+		tags = v.youtubeInfo.Tags
 	}
 	tags, err := tags_manager.SanitizeTags(tags, v.youtubeChannelID)
 	if err != nil {
 		log.Errorln(err.Error())
 	}
 	if !v.mocked {
-		tags = append(tags, youtubeCategories[v.youtubeInfo.Snippet.CategoryId])
+		for _, category := range v.youtubeInfo.Categories {
+			tags = append(tags, youtubeCategories[category])
+		}
 	}
 
 	return languages, locations, tags
@@ -532,8 +535,8 @@ func (v *YoutubeVideo) reprocess(daemon *jsonrpc.Client, params SyncParams, exis
 		if v.mocked {
 			return nil, errors.Err("could not find thumbnail for mocked video")
 		}
-		thumbnail := thumbs.GetBestThumbnail(v.youtubeInfo.Snippet.Thumbnails)
-		thumbnailURL, err = thumbs.MirrorThumbnail(thumbnail.Url, v.ID(), v.awsConfig)
+		thumbnail := thumbs.GetBestThumbnail(v.youtubeInfo.Thumbnails)
+		thumbnailURL, err = thumbs.MirrorThumbnail(thumbnail.URL, v.ID(), v.awsConfig)
 	} else {
 		thumbnailURL = thumbs.ThumbnailEndpoint + v.ID()
 	}
@@ -605,14 +608,9 @@ func (v *YoutubeVideo) reprocess(daemon *jsonrpc.Client, params SyncParams, exis
 		}, nil
 	}
 
-	videoDuration, err := duration.FromString(v.youtubeInfo.ContentDetails.Duration)
-	if err != nil {
-		return nil, errors.Err(err)
-	}
-
 	streamCreateOptions.ClaimCreateOptions.Title = &v.title
 	streamCreateOptions.ClaimCreateOptions.Description = util.PtrToString(v.getAbbrevDescription())
-	streamCreateOptions.Duration = util.PtrToUint64(uint64(math.Ceil(videoDuration.ToDuration().Seconds())))
+	streamCreateOptions.Duration = util.PtrToUint64(uint64(v.youtubeInfo.Duration))
 	streamCreateOptions.ReleaseTime = util.PtrToInt64(v.publishedAt.Unix())
 	start := time.Now()
 	pr, err := daemon.StreamUpdate(existingVideoData.ClaimID, jsonrpc.StreamUpdateOptions{

thumbs/uploader.go

@@ -5,13 +5,14 @@ import (
 	"net/http"
 	"os"
 
+	"github.com/lbryio/ytsync/v5/downloader/ytdl"
+
 	"github.com/lbryio/lbry.go/v2/extras/errors"
 
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/aws/session"
 	"github.com/aws/aws-sdk-go/service/s3/s3manager"
 	log "github.com/sirupsen/logrus"
-	ytlib "google.golang.org/api/youtube/v3"
 )
 
 type thumbnailUploader struct {
@@ -98,15 +99,14 @@ func MirrorThumbnail(url string, name string, s3Config aws.Config) (string, erro
 	return tu.mirroredUrl, nil
 }
 
-func GetBestThumbnail(thumbnails *ytlib.ThumbnailDetails) *ytlib.Thumbnail {
-	if thumbnails.Maxres != nil {
-		return thumbnails.Maxres
-	} else if thumbnails.High != nil {
-		return thumbnails.High
-	} else if thumbnails.Medium != nil {
-		return thumbnails.Medium
-	} else if thumbnails.Standard != nil {
-		return thumbnails.Standard
+func GetBestThumbnail(thumbnails []ytdl.Thumbnail) *ytdl.Thumbnail {
+	var bestWidth *ytdl.Thumbnail
+	for _, thumbnail := range thumbnails {
+		if bestWidth == nil {
+			bestWidth = &thumbnail
+		} else if bestWidth.Width < thumbnail.Width {
+			bestWidth = &thumbnail
 		}
-	return thumbnails.Default
+	}
+	return bestWidth
 }

ytapi/ytapi.go

@@ -10,6 +10,8 @@ import (
 	"sync"
 	"time"
 
+	"github.com/lbryio/ytsync/v5/downloader/ytdl"
+
 	"github.com/lbryio/ytsync/v5/downloader"
 	"github.com/lbryio/ytsync/v5/ip_manager"
 	"github.com/lbryio/ytsync/v5/sdk"
@@ -74,14 +76,18 @@ func GetVideosToSync(apiKey, channelID string, syncedVideos map[string]sdk.Synce
 		mostRecentlyFailedChannel = channelID
 	}
 
-	vids, err := getVideos(apiKey, videoIDs)
+	vids, err := getVideos(videoIDs)
 	if err != nil {
 		return nil, err
 	}
 
 	for _, item := range vids {
-		positionInList := playlistMap[item.Id]
-		videos = append(videos, sources.NewYoutubeVideo(videoParams.VideoDir, item, positionInList, videoParams.S3Config, videoParams.Grp, videoParams.IPPool))
+		positionInList := playlistMap[item.ID]
+		videoToAdd, err := sources.NewYoutubeVideo(videoParams.VideoDir, item, positionInList, videoParams.S3Config, videoParams.Grp, videoParams.IPPool)
+		if err != nil {
+			return nil, errors.Err(err)
+		}
+		videos = append(videos, videoToAdd)
 	}
 
 	for k, v := range syncedVideos {
@@ -154,16 +160,14 @@ func ChannelInfo(apiKey, channelID string) (*ytlib.ChannelSnippet, *ytlib.Channe
 	return response.Items[0].Snippet, response.Items[0].BrandingSettings, nil
 }
 
-func getVideos(apiKey string, videoIDs []string) ([]*ytlib.Video, error) {
-	service, err := ytlib.New(&http.Client{Transport: &transport.APIKey{Key: apiKey}})
+func getVideos(videoIDs []string) ([]*ytdl.YtdlVideo, error) {
+	var videos []*ytdl.YtdlVideo
+	for _, videoID := range videoIDs {
+		video, err := downloader.GetVideoInformation(videoID)
 		if err != nil {
-		return nil, errors.Prefix("error creating YouTube service", err)
+			return nil, errors.Err(err)
 		}
-
-	response, err := service.Videos.List("snippet,contentDetails,recordingDetails").Id(strings.Join(videoIDs[:], ",")).Do()
-	if err != nil {
-		return nil, errors.Prefix("error getting videos info", err)
+		videos = append(videos, video)
 	}
-
-	return response.Items, nil
+	return videos, nil
 }