Determine release time via YouTube API

local/local.go

@@ -1,20 +1,17 @@
 package local
 
 import (
-	"encoding/json"
 	"errors"
-	"fmt"
-	"io/ioutil"
 	"os"
-	"path"
 	"regexp"
 	"strings"
+	"time"
 
 	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
 	"github.com/abadojack/whatlanggo"
 
-	"github.com/lbryio/ytsync/v5/downloader/ytdl"
+	"github.com/lbryio/lbry.go/v2/extras/util"
 	"github.com/lbryio/ytsync/v5/namer"
 	"github.com/lbryio/ytsync/v5/tags_manager"
 )
@@ -24,6 +21,7 @@ type SyncContext struct {
 	LbrynetAddr     string
 	ChannelID       string
 	PublishBid      float64
+	YouTubeSourceConfig *YouTubeSourceConfig
 }
 
 func (c *SyncContext) Validate() error {
@@ -42,6 +40,10 @@ func (c *SyncContext) Validate() error {
 	return nil
 }
 
+type YouTubeSourceConfig struct {
+	YouTubeAPIKey string
+}
+
 var syncContext SyncContext
 
 func AddCommand(rootCmd *cobra.Command) {
@@ -55,6 +57,10 @@ func AddCommand(rootCmd *cobra.Command) {
 	cmd.Flags().Float64Var(&syncContext.PublishBid, "publish-bid", 0.01, "Bid amount for the stream claim")
 	cmd.Flags().StringVar(&syncContext.LbrynetAddr, "lbrynet-address", getEnvDefault("LBRYNET_ADDRESS", ""), "JSONRPC address of the local LBRYNet daemon")
 	cmd.Flags().StringVar(&syncContext.ChannelID, "channel-id", "", "LBRY channel ID to publish to")
+
+	// For now, assume source is always YouTube
+	syncContext.YouTubeSourceConfig = &YouTubeSourceConfig{}
+	cmd.Flags().StringVar(&syncContext.YouTubeSourceConfig.YouTubeAPIKey, "youtube-api-key", getEnvDefault("YOUTUBE_API_KEY", ""), "YouTube API Key")
 	rootCmd.AddCommand(cmd)
 }
 
@@ -71,11 +77,9 @@ func localCmd(cmd *cobra.Command, args []string) {
 		log.Error(err)
 		return
 	}
-	fmt.Println(syncContext.LbrynetAddr)
-
 	videoID := args[0]
 
-	log.Debugf("Running sync for YouTube video ID %s", videoID)
+	log.Debugf("Running sync for video ID %s", videoID)
 
 	var publisher VideoPublisher
 	publisher, err = NewLocalSDKPublisher(syncContext.LbrynetAddr, syncContext.ChannelID, syncContext.PublishBid)
@@ -85,10 +89,12 @@ func localCmd(cmd *cobra.Command, args []string) {
 	}
 
 	var videoSource VideoSource
-	videoSource, err = NewYtdlVideoSource(syncContext.TempDir)
+	if syncContext.YouTubeSourceConfig != nil {
-	if err != nil {
+		videoSource, err = NewYtdlVideoSource(syncContext.TempDir, syncContext.YouTubeSourceConfig)
-		log.Errorf("Error setting up video source: %v", err)
+		if err != nil {
-		return
+			log.Errorf("Error setting up video source: %v", err)
+			return
+		}
 	}
 
 	sourceVideo, err := videoSource.GetVideo(videoID)
@@ -116,78 +122,6 @@ func localCmd(cmd *cobra.Command, args []string) {
 	log.Info("Done")
 }
 
-func getVideoMetadata(basePath, videoID string) (*ytdl.YtdlVideo, string, error) {
-	metadataPath := basePath + ".info.json"
-
-	_, err := os.Stat(metadataPath)
-	if err != nil && !os.IsNotExist(err) {
-		log.Errorf("Error determining if video metadata already exists: %v", err)
-		return nil, "", err
-	} else if err == nil {
-		log.Debugf("Video metadata file %s already exists. Attempting to load existing file.", metadataPath)
-		videoMetadata, err := loadVideoMetadata(metadataPath)
-		if err != nil {
-			log.Debugf("Error loading pre-existing video metadata: %v. Deleting file and attempting re-download.", err)
-		} else {
-			return videoMetadata, metadataPath, nil
-		}
-	}
-
-	if err := downloadVideoMetadata(basePath, videoID); err != nil {
-		return nil, "", err
-	}
-
-	videoMetadata, err := loadVideoMetadata(metadataPath)
-	return videoMetadata, metadataPath, err
-}
-
-func loadVideoMetadata(path string) (*ytdl.YtdlVideo, error) {
-	metadataBytes, err := os.ReadFile(path)
-	if err != nil {
-		return nil, err
-	}
-
-	var videoMetadata *ytdl.YtdlVideo
-	err = json.Unmarshal(metadataBytes, &videoMetadata)
-	if err != nil {
-		return nil, err
-	}
-
-	return videoMetadata, nil
-}
-
-func getVideoDownloadedPath(videoDir, videoID string) (string, error) {
-	files, err := ioutil.ReadDir(videoDir)
-	if err != nil {
-		return "", err
-	}
-
-	for _, f := range files {
-		if f.IsDir() {
-			continue
-		}
-		if path.Ext(f.Name()) == ".mp4" && strings.Contains(f.Name(), videoID) {
-			return path.Join(videoDir, f.Name()), nil
-		}
-	}
-	return "", errors.New("could not find any downloaded videos")
-
-}
-
-func getAbbrevDescription(v SourceVideo) string {
-	if v.Description == nil {
-		return v.SourceURL
-	}
-
-	maxLength := 2800
-	description := strings.TrimSpace(*v.Description)
-	additionalDescription := "\n" + v.SourceURL
-	if len(description) > maxLength {
-		description = description[:maxLength]
-	}
-	return description + "\n..." + additionalDescription
-}
-
 type SourceVideo struct {
 	ID string
 	Title *string
@@ -243,9 +177,14 @@ func processVideoForPublishing(source SourceVideo, channelID string) (*Publishab
 
 	claimName := namer.NewNamer().GetNextName(title)
 
-	thumbnailURL := ""
+	thumbnailURL := source.ThumbnailURL
-	if source.ThumbnailURL != nil {
+	if thumbnailURL == nil {
-		thumbnailURL = *source.ThumbnailURL
+		thumbnailURL = util.PtrToString("")
+	}
+
+	releaseTime := source.ReleaseTime
+	if releaseTime == nil {
+		releaseTime = util.PtrToInt64(time.Now().Unix())
 	}
 
 	processed := PublishableVideo {
@@ -254,8 +193,8 @@ func processVideoForPublishing(source SourceVideo, channelID string) (*Publishab
 		Description:  getAbbrevDescription(source),
 		Languages: languages,
 		Tags: tags,
-		ReleaseTime: *source.ReleaseTime,
+		ReleaseTime: *releaseTime,
-		ThumbnailURL: thumbnailURL,
+		ThumbnailURL: *thumbnailURL,
 		FullLocalPath: source.FullLocalPath,
 	}
 
@@ -264,6 +203,20 @@ func processVideoForPublishing(source SourceVideo, channelID string) (*Publishab
 	return &processed, nil
 }
 
+func getAbbrevDescription(v SourceVideo) string {
+	if v.Description == nil {
+		return v.SourceURL
+	}
+
+	maxLength := 2800
+	description := strings.TrimSpace(*v.Description)
+	additionalDescription := "\n" + v.SourceURL
+	if len(description) > maxLength {
+		description = description[:maxLength]
+	}
+	return description + "\n..." + additionalDescription
+}
+
 type VideoSource interface {
 	GetVideo(id string) (*SourceVideo, error)
 }

local/youtubeEnricher.go

@@ -0,0 +1,45 @@
+package local
+
+import (
+	"time"
+
+	log "github.com/sirupsen/logrus"
+
+	"github.com/lbryio/lbry.go/v2/extras/util"
+)
+
+type YouTubeVideoEnricher interface {
+	EnrichMissing(source *SourceVideo) error
+}
+
+type YouTubeAPIVideoEnricher struct {
+	api *YouTubeAPI
+}
+
+func NewYouTubeAPIVideoEnricher(apiKey string) (*YouTubeAPIVideoEnricher) {
+	enricher := YouTubeAPIVideoEnricher{
+		api: NewYouTubeAPI(apiKey),
+	}
+	return &enricher
+}
+
+func (e *YouTubeAPIVideoEnricher) EnrichMissing(source *SourceVideo) error {
+	if source.ReleaseTime != nil {
+		log.Debugf("Video %s does not need enrichment. YouTubeAPIVideoEnricher is skipping.", source.ID)
+		return nil
+	}
+
+	snippet, err := e.api.GetVideoSnippet(source.ID)
+	if err != nil {
+		log.Errorf("Error snippet data for video %s: %v", err)
+		return err
+	}
+
+	publishedAt, err := time.Parse(time.RFC3339, snippet.PublishedAt)
+	if err != nil {
+		log.Errorf("Error converting publishedAt to timestamp: %v", err)
+	} else {
+		source.ReleaseTime = util.PtrToInt64(publishedAt.Unix())
+	}
+	return nil
+}

local/ytapi.go

@@ -0,0 +1,83 @@
+package local
+
+import (
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"time"
+
+	log "github.com/sirupsen/logrus"
+)
+
+type YouTubeAPI struct {
+	apiKey string
+	client *http.Client
+}
+
+func NewYouTubeAPI(apiKey string) (*YouTubeAPI) {
+	client := &http.Client {
+		Transport: &http.Transport{
+			MaxIdleConns:       10,
+			IdleConnTimeout:    30 * time.Second,
+			DisableCompression: true,
+		},
+	}
+
+	api := YouTubeAPI {
+		apiKey: apiKey,
+		client: client,
+	}
+
+	return &api
+}
+
+func (a *YouTubeAPI) GetVideoSnippet(videoID string) (*VideoSnippet, error) {
+	req, err := http.NewRequest("GET", "https://youtube.googleapis.com/youtube/v3/videos", nil)
+	if err != nil {
+		log.Errorf("Error creating http client for YouTube API: %v", err)
+		return nil, err
+	}
+
+	query := req.URL.Query()
+	query.Add("part", "snippet")
+	query.Add("id", videoID)
+	query.Add("key", a.apiKey)
+	req.URL.RawQuery = query.Encode()
+
+	req.Header.Add("Accept", "application/json")
+
+	resp, err := a.client.Do(req)
+	defer resp.Body.Close()
+	if err != nil {
+		log.Errorf("Error from YouTube API: %v", err)
+		return nil, err
+	}
+
+	body, err := io.ReadAll(resp.Body)
+	log.Tracef("Response from YouTube API: %s", string(body[:]))
+
+	var result videoListResponse
+	err = json.Unmarshal(body, &result)
+	if err != nil {
+		log.Errorf("Error deserializing video list response from YouTube API: %v", err)
+		return nil, err
+	}
+
+	if len(result.Items) != 1 {
+		err = fmt.Errorf("YouTube API responded with incorrect number of snippets (%d) while attempting to get snippet data for video %s", len(result.Items), videoID)
+		return nil, err
+	}
+
+	return &result.Items[0].Snippet, nil
+}
+
+type videoListResponse struct {
+	Items []struct {
+		Snippet VideoSnippet `json:"snippet"`
+	} `json:"items"`
+}
+
+type VideoSnippet struct {
+	PublishedAt string `json:"publishedAt"`
+}

local/ytdlVideoSource.go

@@ -1,18 +1,17 @@
 package local
 
 import (
-	"time"
-
 	log "github.com/sirupsen/logrus"
 
-	"github.com/lbryio/lbry.go/v2/extras/util"
+	"github.com/lbryio/ytsync/v5/downloader/ytdl"
 )
 
 type YtdlVideoSource struct {
 	downloader Ytdl
+	enrichers []YouTubeVideoEnricher
 }
 
-func NewYtdlVideoSource(downloadDir string) (*YtdlVideoSource, error) {
+func NewYtdlVideoSource(downloadDir string, config *YouTubeSourceConfig) (*YtdlVideoSource, error) {
 	ytdl, err := NewYtdl(downloadDir)
 	if err != nil {
 		return nil, err
@@ -22,6 +21,11 @@ func NewYtdlVideoSource(downloadDir string) (*YtdlVideoSource, error) {
 		downloader: *ytdl,
 	}
 
+	if config.YouTubeAPIKey != "" {
+		ytapiEnricher := NewYouTubeAPIVideoEnricher(config.YouTubeAPIKey)
+		source.enrichers = append(source.enrichers, ytapiEnricher)
+	}
+
 	return &source, nil
 }
 
@@ -36,6 +40,13 @@ func (s *YtdlVideoSource) GetVideo(id string) (*SourceVideo, error) {
 		return nil, err
 	}
 
+	var bestThumbnail *ytdl.Thumbnail = nil
+	for i, thumbnail := range metadata.Thumbnails {
+		if i == 0 || bestThumbnail.Width < thumbnail.Width {
+			bestThumbnail = &thumbnail
+		}
+	}
+
 	sourceVideo := SourceVideo {
 		ID: id,
 		Title: &metadata.Title,
@@ -43,11 +54,18 @@ func (s *YtdlVideoSource) GetVideo(id string) (*SourceVideo, error) {
 		SourceURL: "\nhttps://www.youtube.com/watch?v=" + id,
 		Languages: []string{},
 		Tags: metadata.Tags,
-		ReleaseTime: util.PtrToInt64(time.Now().Unix()),
+		ReleaseTime: nil,
-		ThumbnailURL: nil,
+		ThumbnailURL: &bestThumbnail.URL,
 		FullLocalPath: videoPath,
 	}
 
+	for _, enricher := range s.enrichers {
+		err = enricher.EnrichMissing(&sourceVideo)
+		if err != nil {
+			log.Warnf("Error enriching video %s, continuing enrichment: %v", id, err)
+		}
+	}
+
 	log.Debugf("Source video retrieved via ytdl: %v", sourceVideo)
 
 	return &sourceVideo, nil