LBRYCommunity/ytsync
1
0
Fork 0
Code Issues 39 Pull requests 8 Projects Releases 197 Packages Wiki Activity

Determine release time via YouTube API

Browse source
This commit is contained in:
pseudoscalar 2021-11-04 15:28:28 -05:00
parent 2ba960ae01
commit eb30fa4299
4 changed files with 194 additions and 95 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

131
local/local.go
View file

@ -1,20 +1,17 @@
package local
import (
"encoding/json"
"errors"
"fmt"
"io/ioutil"
"os"
"path"
"regexp"
"strings"
"time"
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
"github.com/abadojack/whatlanggo"
"github.com/lbryio/ytsync/v5/downloader/ytdl"
"github.com/lbryio/lbry.go/v2/extras/util"
"github.com/lbryio/ytsync/v5/namer"
"github.com/lbryio/ytsync/v5/tags_manager"
)
@ -24,6 +21,7 @@ type SyncContext struct {
LbrynetAddr string
ChannelID string
PublishBid float64
YouTubeSourceConfig *YouTubeSourceConfig
}
func (c *SyncContext) Validate() error {
@ -42,6 +40,10 @@ func (c *SyncContext) Validate() error {
return nil
}
type YouTubeSourceConfig struct {
YouTubeAPIKey string
}
var syncContext SyncContext
func AddCommand(rootCmd *cobra.Command) {
@ -55,6 +57,10 @@ func AddCommand(rootCmd *cobra.Command) {
cmd.Flags().Float64Var(&syncContext.PublishBid, "publish-bid", 0.01, "Bid amount for the stream claim")
cmd.Flags().StringVar(&syncContext.LbrynetAddr, "lbrynet-address", getEnvDefault("LBRYNET_ADDRESS", ""), "JSONRPC address of the local LBRYNet daemon")
cmd.Flags().StringVar(&syncContext.ChannelID, "channel-id", "", "LBRY channel ID to publish to")
// For now, assume source is always YouTube
syncContext.YouTubeSourceConfig = &YouTubeSourceConfig{}
cmd.Flags().StringVar(&syncContext.YouTubeSourceConfig.YouTubeAPIKey, "youtube-api-key", getEnvDefault("YOUTUBE_API_KEY", ""), "YouTube API Key")
rootCmd.AddCommand(cmd)
}
@ -71,11 +77,9 @@ func localCmd(cmd *cobra.Command, args []string) {
log.Error(err)
return
}
fmt.Println(syncContext.LbrynetAddr)
videoID := args[0]
log.Debugf("Running sync for YouTube video ID %s", videoID)
log.Debugf("Running sync for video ID %s", videoID)
var publisher VideoPublisher
publisher, err = NewLocalSDKPublisher(syncContext.LbrynetAddr, syncContext.ChannelID, syncContext.PublishBid)
@ -85,10 +89,12 @@ func localCmd(cmd *cobra.Command, args []string) {
}
var videoSource VideoSource
videoSource, err = NewYtdlVideoSource(syncContext.TempDir)
if err != nil {
log.Errorf("Error setting up video source: %v", err)
return
if syncContext.YouTubeSourceConfig != nil {
videoSource, err = NewYtdlVideoSource(syncContext.TempDir, syncContext.YouTubeSourceConfig)
if err != nil {
log.Errorf("Error setting up video source: %v", err)
return
}
}
sourceVideo, err := videoSource.GetVideo(videoID)
@ -116,78 +122,6 @@ func localCmd(cmd *cobra.Command, args []string) {
log.Info("Done")
}
func getVideoMetadata(basePath, videoID string) (*ytdl.YtdlVideo, string, error) {
metadataPath := basePath + ".info.json"
_, err := os.Stat(metadataPath)
if err != nil && !os.IsNotExist(err) {
log.Errorf("Error determining if video metadata already exists: %v", err)
return nil, "", err
} else if err == nil {
log.Debugf("Video metadata file %s already exists. Attempting to load existing file.", metadataPath)
videoMetadata, err := loadVideoMetadata(metadataPath)
if err != nil {
log.Debugf("Error loading pre-existing video metadata: %v. Deleting file and attempting re-download.", err)
} else {
return videoMetadata, metadataPath, nil
}
}
if err := downloadVideoMetadata(basePath, videoID); err != nil {
return nil, "", err
}
videoMetadata, err := loadVideoMetadata(metadataPath)
return videoMetadata, metadataPath, err
}
func loadVideoMetadata(path string) (*ytdl.YtdlVideo, error) {
metadataBytes, err := os.ReadFile(path)
if err != nil {
return nil, err
}
var videoMetadata *ytdl.YtdlVideo
err = json.Unmarshal(metadataBytes, &videoMetadata)
if err != nil {
return nil, err
}
return videoMetadata, nil
}
func getVideoDownloadedPath(videoDir, videoID string) (string, error) {
files, err := ioutil.ReadDir(videoDir)
if err != nil {
return "", err
}
for _, f := range files {
if f.IsDir() {
continue
}
if path.Ext(f.Name()) == ".mp4" && strings.Contains(f.Name(), videoID) {
return path.Join(videoDir, f.Name()), nil
}
}
return "", errors.New("could not find any downloaded videos")
}
func getAbbrevDescription(v SourceVideo) string {
if v.Description == nil {
return v.SourceURL
}
maxLength := 2800
description := strings.TrimSpace(*v.Description)
additionalDescription := "\n" + v.SourceURL
if len(description) > maxLength {
description = description[:maxLength]
}
return description + "\n..." + additionalDescription
}
type SourceVideo struct {
ID string
Title *string
@ -243,9 +177,14 @@ func processVideoForPublishing(source SourceVideo, channelID string) (*Publishab
claimName := namer.NewNamer().GetNextName(title)
thumbnailURL := ""
if source.ThumbnailURL != nil {
thumbnailURL = *source.ThumbnailURL
thumbnailURL := source.ThumbnailURL
if thumbnailURL == nil {
thumbnailURL = util.PtrToString("")
}
releaseTime := source.ReleaseTime
if releaseTime == nil {
releaseTime = util.PtrToInt64(time.Now().Unix())
}
processed := PublishableVideo {
@ -254,8 +193,8 @@ func processVideoForPublishing(source SourceVideo, channelID string) (*Publishab
Description: getAbbrevDescription(source),
Languages: languages,
Tags: tags,
ReleaseTime: *source.ReleaseTime,
ThumbnailURL: thumbnailURL,
ReleaseTime: *releaseTime,
ThumbnailURL: *thumbnailURL,
FullLocalPath: source.FullLocalPath,
}
@ -264,6 +203,20 @@ func processVideoForPublishing(source SourceVideo, channelID string) (*Publishab
return &processed, nil
}
func getAbbrevDescription(v SourceVideo) string {
if v.Description == nil {
return v.SourceURL
}
maxLength := 2800
description := strings.TrimSpace(*v.Description)
additionalDescription := "\n" + v.SourceURL
if len(description) > maxLength {
description = description[:maxLength]
}
return description + "\n..." + additionalDescription
}
type VideoSource interface {
GetVideo(id string) (*SourceVideo, error)
}

45
local/youtubeEnricher.go Normal file
View file

@ -0,0 +1,45 @@
package local
import (
"time"
log "github.com/sirupsen/logrus"
"github.com/lbryio/lbry.go/v2/extras/util"
)
type YouTubeVideoEnricher interface {
EnrichMissing(source *SourceVideo) error
}
type YouTubeAPIVideoEnricher struct {
api *YouTubeAPI
}
func NewYouTubeAPIVideoEnricher(apiKey string) (*YouTubeAPIVideoEnricher) {
enricher := YouTubeAPIVideoEnricher{
api: NewYouTubeAPI(apiKey),
}
return &enricher
}
func (e *YouTubeAPIVideoEnricher) EnrichMissing(source *SourceVideo) error {
if source.ReleaseTime != nil {
log.Debugf("Video %s does not need enrichment. YouTubeAPIVideoEnricher is skipping.", source.ID)
return nil
}
snippet, err := e.api.GetVideoSnippet(source.ID)
if err != nil {
log.Errorf("Error snippet data for video %s: %v", err)
return err
}
publishedAt, err := time.Parse(time.RFC3339, snippet.PublishedAt)
if err != nil {
log.Errorf("Error converting publishedAt to timestamp: %v", err)
} else {
source.ReleaseTime = util.PtrToInt64(publishedAt.Unix())
}
return nil
}

83
local/ytapi.go Normal file
View file

@ -0,0 +1,83 @@
package local
import (
"encoding/json"
"fmt"
"io"
"net/http"
"time"
log "github.com/sirupsen/logrus"
)
type YouTubeAPI struct {
apiKey string
client *http.Client
}
func NewYouTubeAPI(apiKey string) (*YouTubeAPI) {
client := &http.Client {
Transport: &http.Transport{
MaxIdleConns: 10,
IdleConnTimeout: 30 * time.Second,
DisableCompression: true,
},
}
api := YouTubeAPI {
apiKey: apiKey,
client: client,
}
return &api
}
func (a *YouTubeAPI) GetVideoSnippet(videoID string) (*VideoSnippet, error) {
req, err := http.NewRequest("GET", "https://youtube.googleapis.com/youtube/v3/videos", nil)
if err != nil {
log.Errorf("Error creating http client for YouTube API: %v", err)
return nil, err
}
query := req.URL.Query()
query.Add("part", "snippet")
query.Add("id", videoID)
query.Add("key", a.apiKey)
req.URL.RawQuery = query.Encode()
req.Header.Add("Accept", "application/json")
resp, err := a.client.Do(req)
defer resp.Body.Close()
if err != nil {
log.Errorf("Error from YouTube API: %v", err)
return nil, err
}
body, err := io.ReadAll(resp.Body)
log.Tracef("Response from YouTube API: %s", string(body[:]))
var result videoListResponse
err = json.Unmarshal(body, &result)
if err != nil {
log.Errorf("Error deserializing video list response from YouTube API: %v", err)
return nil, err
}
if len(result.Items) != 1 {
err = fmt.Errorf("YouTube API responded with incorrect number of snippets (%d) while attempting to get snippet data for video %s", len(result.Items), videoID)
return nil, err
}
return &result.Items[0].Snippet, nil
}
type videoListResponse struct {
Items []struct {
Snippet VideoSnippet `json:"snippet"`
} `json:"items"`
}
type VideoSnippet struct {
PublishedAt string `json:"publishedAt"`
}

30
local/ytdlVideoSource.go
View file

@ -1,18 +1,17 @@
package local
import (
"time"
log "github.com/sirupsen/logrus"
"github.com/lbryio/lbry.go/v2/extras/util"
"github.com/lbryio/ytsync/v5/downloader/ytdl"
)
type YtdlVideoSource struct {
downloader Ytdl
enrichers []YouTubeVideoEnricher
}
func NewYtdlVideoSource(downloadDir string) (*YtdlVideoSource, error) {
func NewYtdlVideoSource(downloadDir string, config *YouTubeSourceConfig) (*YtdlVideoSource, error) {
ytdl, err := NewYtdl(downloadDir)
if err != nil {
return nil, err
@ -22,6 +21,11 @@ func NewYtdlVideoSource(downloadDir string) (*YtdlVideoSource, error) {
downloader: *ytdl,
}
if config.YouTubeAPIKey != "" {
ytapiEnricher := NewYouTubeAPIVideoEnricher(config.YouTubeAPIKey)
source.enrichers = append(source.enrichers, ytapiEnricher)
}
return &source, nil
}
@ -36,6 +40,13 @@ func (s *YtdlVideoSource) GetVideo(id string) (*SourceVideo, error) {
return nil, err
}
var bestThumbnail *ytdl.Thumbnail = nil
for i, thumbnail := range metadata.Thumbnails {
if i == 0 || bestThumbnail.Width < thumbnail.Width {
bestThumbnail = &thumbnail
}
}
sourceVideo := SourceVideo {
ID: id,
Title: &metadata.Title,
@ -43,11 +54,18 @@ func (s *YtdlVideoSource) GetVideo(id string) (*SourceVideo, error) {
SourceURL: "\nhttps://www.youtube.com/watch?v=" + id,
Languages: []string{},
Tags: metadata.Tags,
ReleaseTime: util.PtrToInt64(time.Now().Unix()),
ThumbnailURL: nil,
ReleaseTime: nil,
ThumbnailURL: &bestThumbnail.URL,
FullLocalPath: videoPath,
}
for _, enricher := range s.enrichers {
err = enricher.EnrichMissing(&sourceVideo)
if err != nil {
log.Warnf("Error enriching video %s, continuing enrichment: %v", id, err)
}
}
log.Debugf("Source video retrieved via ytdl: %v", sourceVideo)
return &sourceVideo, nil