move all youtube api calls into a single place
This commit is contained in:
parent
be7fd7ddd8
commit
843303301a
6 changed files with 253 additions and 184 deletions
|
@ -1,32 +1,9 @@
|
||||||
package manager
|
package manager
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"net/http"
|
"github.com/lbryio/ytsync/v5/ytapi"
|
||||||
|
|
||||||
"github.com/lbryio/lbry.go/v2/extras/errors"
|
|
||||||
|
|
||||||
"google.golang.org/api/googleapi/transport"
|
|
||||||
"google.golang.org/api/youtube/v3"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
func (s *Sync) CountVideos() (uint64, error) {
|
func (s *Sync) CountVideos() (uint64, error) {
|
||||||
client := &http.Client{
|
return ytapi.VideosInChannel(s.APIConfig.YoutubeAPIKey, s.YoutubeChannelID)
|
||||||
Transport: &transport.APIKey{Key: s.APIConfig.YoutubeAPIKey},
|
|
||||||
}
|
|
||||||
|
|
||||||
service, err := youtube.New(client)
|
|
||||||
if err != nil {
|
|
||||||
return 0, errors.Prefix("error creating YouTube service", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
response, err := service.Channels.List("statistics").Id(s.YoutubeChannelID).Do()
|
|
||||||
if err != nil {
|
|
||||||
return 0, errors.Prefix("error getting channels", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if len(response.Items) < 1 {
|
|
||||||
return 0, errors.Err("youtube channel not found")
|
|
||||||
}
|
|
||||||
|
|
||||||
return response.Items[0].Statistics.VideoCount, nil
|
|
||||||
}
|
}
|
||||||
|
|
|
@ -3,7 +3,6 @@ package manager
|
||||||
import (
|
import (
|
||||||
"fmt"
|
"fmt"
|
||||||
"math"
|
"math"
|
||||||
"net/http"
|
|
||||||
"strconv"
|
"strconv"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
@ -12,14 +11,13 @@ import (
|
||||||
"github.com/lbryio/lbry.go/v2/extras/util"
|
"github.com/lbryio/lbry.go/v2/extras/util"
|
||||||
"github.com/lbryio/ytsync/v5/timing"
|
"github.com/lbryio/ytsync/v5/timing"
|
||||||
logUtils "github.com/lbryio/ytsync/v5/util"
|
logUtils "github.com/lbryio/ytsync/v5/util"
|
||||||
|
"github.com/lbryio/ytsync/v5/ytapi"
|
||||||
|
|
||||||
"github.com/lbryio/ytsync/v5/tags_manager"
|
"github.com/lbryio/ytsync/v5/tags_manager"
|
||||||
"github.com/lbryio/ytsync/v5/thumbs"
|
"github.com/lbryio/ytsync/v5/thumbs"
|
||||||
|
|
||||||
"github.com/shopspring/decimal"
|
"github.com/shopspring/decimal"
|
||||||
log "github.com/sirupsen/logrus"
|
log "github.com/sirupsen/logrus"
|
||||||
"google.golang.org/api/googleapi/transport"
|
|
||||||
"google.golang.org/api/youtube/v3"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
func (s *Sync) enableAddressReuse() error {
|
func (s *Sync) enableAddressReuse() error {
|
||||||
|
@ -266,15 +264,14 @@ func (s *Sync) ensureEnoughUTXOs() error {
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *Sync) waitForNewBlock() error {
|
func (s *Sync) waitForNewBlock() error {
|
||||||
start := time.Now()
|
defer func(start time.Time) { timing.TimedComponent("waitForNewBlock").Add(time.Since(start)) }(time.Now())
|
||||||
defer func(start time.Time) {
|
|
||||||
timing.TimedComponent("waitForNewBlock").Add(time.Since(start))
|
|
||||||
}(start)
|
|
||||||
log.Printf("regtest: %t, docker: %t", logUtils.IsRegTest(), logUtils.IsUsingDocker())
|
log.Printf("regtest: %t, docker: %t", logUtils.IsRegTest(), logUtils.IsUsingDocker())
|
||||||
status, err := s.daemon.Status()
|
status, err := s.daemon.Status()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
for status.Wallet.Blocks == 0 || status.Wallet.BlocksBehind != 0 {
|
for status.Wallet.Blocks == 0 || status.Wallet.BlocksBehind != 0 {
|
||||||
time.Sleep(5 * time.Second)
|
time.Sleep(5 * time.Second)
|
||||||
status, err = s.daemon.Status()
|
status, err = s.daemon.Status()
|
||||||
|
@ -308,10 +305,12 @@ func (s *Sync) GenerateRegtestBlock() error {
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return errors.Prefix("error getting lbrycrd client: ", err)
|
return errors.Prefix("error getting lbrycrd client: ", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
txs, err := lbrycrd.Generate(1)
|
txs, err := lbrycrd.Generate(1)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return errors.Prefix("error generating new block: ", err)
|
return errors.Prefix("error generating new block: ", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, tx := range txs {
|
for _, tx := range txs {
|
||||||
log.Info("Generated tx: ", tx.String())
|
log.Info("Generated tx: ", tx.String())
|
||||||
}
|
}
|
||||||
|
@ -319,10 +318,8 @@ func (s *Sync) GenerateRegtestBlock() error {
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *Sync) ensureChannelOwnership() error {
|
func (s *Sync) ensureChannelOwnership() error {
|
||||||
start := time.Now()
|
defer func(start time.Time) { timing.TimedComponent("ensureChannelOwnership").Add(time.Since(start)) }(time.Now())
|
||||||
defer func(start time.Time) {
|
|
||||||
timing.TimedComponent("ensureChannelOwnership").Add(time.Since(start))
|
|
||||||
}(start)
|
|
||||||
if s.LbryChannelName == "" {
|
if s.LbryChannelName == "" {
|
||||||
return errors.Err("no channel name set")
|
return errors.Err("no channel name set")
|
||||||
}
|
}
|
||||||
|
@ -386,27 +383,12 @@ func (s *Sync) ensureChannelOwnership() error {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
client := &http.Client{
|
|
||||||
Transport: &transport.APIKey{Key: s.APIConfig.YoutubeAPIKey},
|
|
||||||
}
|
|
||||||
|
|
||||||
service, err := youtube.New(client)
|
channelInfo, channelBranding, err := ytapi.ChannelInfo(s.APIConfig.YoutubeAPIKey, s.YoutubeChannelID)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return errors.Prefix("error creating YouTube service", err)
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
response, err := service.Channels.List("snippet,brandingSettings").Id(s.YoutubeChannelID).Do()
|
|
||||||
if err != nil {
|
|
||||||
return errors.Prefix("error getting channel details", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if len(response.Items) < 1 {
|
|
||||||
return errors.Err("youtube channel not found")
|
|
||||||
}
|
|
||||||
|
|
||||||
channelInfo := response.Items[0].Snippet
|
|
||||||
channelBranding := response.Items[0].BrandingSettings
|
|
||||||
|
|
||||||
thumbnail := thumbs.GetBestThumbnail(channelInfo.Thumbnails)
|
thumbnail := thumbs.GetBestThumbnail(channelInfo.Thumbnails)
|
||||||
thumbnailURL, err := thumbs.MirrorThumbnail(thumbnail.Url, s.YoutubeChannelID, s.Manager.GetS3AWSConfig())
|
thumbnailURL, err := thumbs.MirrorThumbnail(thumbnail.Url, s.YoutubeChannelID, s.Manager.GetS3AWSConfig())
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
@ -467,6 +449,7 @@ func (s *Sync) ensureChannelOwnership() error {
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
s.lbryChannelID = c.Outputs[0].ClaimID
|
s.lbryChannelID = c.Outputs[0].ClaimID
|
||||||
return s.Manager.apiConfig.SetChannelClaimID(s.YoutubeChannelID, s.lbryChannelID)
|
return s.Manager.apiConfig.SetChannelClaimID(s.YoutubeChannelID, s.lbryChannelID)
|
||||||
}
|
}
|
||||||
|
|
|
@ -3,11 +3,9 @@ package manager
|
||||||
import (
|
import (
|
||||||
"fmt"
|
"fmt"
|
||||||
"io/ioutil"
|
"io/ioutil"
|
||||||
"net/http"
|
|
||||||
"os"
|
"os"
|
||||||
"os/signal"
|
"os/signal"
|
||||||
"runtime/debug"
|
"runtime/debug"
|
||||||
"sort"
|
|
||||||
"strconv"
|
"strconv"
|
||||||
"strings"
|
"strings"
|
||||||
"sync"
|
"sync"
|
||||||
|
@ -21,6 +19,7 @@ import (
|
||||||
"github.com/lbryio/ytsync/v5/thumbs"
|
"github.com/lbryio/ytsync/v5/thumbs"
|
||||||
"github.com/lbryio/ytsync/v5/timing"
|
"github.com/lbryio/ytsync/v5/timing"
|
||||||
logUtils "github.com/lbryio/ytsync/v5/util"
|
logUtils "github.com/lbryio/ytsync/v5/util"
|
||||||
|
"github.com/lbryio/ytsync/v5/ytapi"
|
||||||
|
|
||||||
"github.com/lbryio/lbry.go/v2/extras/errors"
|
"github.com/lbryio/lbry.go/v2/extras/errors"
|
||||||
"github.com/lbryio/lbry.go/v2/extras/jsonrpc"
|
"github.com/lbryio/lbry.go/v2/extras/jsonrpc"
|
||||||
|
@ -34,8 +33,6 @@ import (
|
||||||
"github.com/aws/aws-sdk-go/service/s3"
|
"github.com/aws/aws-sdk-go/service/s3"
|
||||||
"github.com/aws/aws-sdk-go/service/s3/s3manager"
|
"github.com/aws/aws-sdk-go/service/s3/s3manager"
|
||||||
log "github.com/sirupsen/logrus"
|
log "github.com/sirupsen/logrus"
|
||||||
"google.golang.org/api/googleapi/transport"
|
|
||||||
"google.golang.org/api/youtube/v3"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
const (
|
const (
|
||||||
|
@ -47,22 +44,6 @@ const (
|
||||||
maxReasonLength = 500
|
maxReasonLength = 500
|
||||||
)
|
)
|
||||||
|
|
||||||
type video interface {
|
|
||||||
Size() *int64
|
|
||||||
ID() string
|
|
||||||
IDAndNum() string
|
|
||||||
PlaylistPosition() int
|
|
||||||
PublishedAt() time.Time
|
|
||||||
Sync(*jsonrpc.Client, sources.SyncParams, *sdk.SyncedVideo, bool, *sync.RWMutex) (*sources.SyncSummary, error)
|
|
||||||
}
|
|
||||||
|
|
||||||
// sorting videos
|
|
||||||
type byPublishedAt []video
|
|
||||||
|
|
||||||
func (a byPublishedAt) Len() int { return len(a) }
|
|
||||||
func (a byPublishedAt) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
|
|
||||||
func (a byPublishedAt) Less(i, j int) bool { return a[i].PublishedAt().Before(a[j].PublishedAt()) }
|
|
||||||
|
|
||||||
// Sync stores the options that control how syncing happens
|
// Sync stores the options that control how syncing happens
|
||||||
type Sync struct {
|
type Sync struct {
|
||||||
APIConfig *sdk.APIConfig
|
APIConfig *sdk.APIConfig
|
||||||
|
@ -87,7 +68,7 @@ type Sync struct {
|
||||||
lbryChannelID string
|
lbryChannelID string
|
||||||
namer *namer.Namer
|
namer *namer.Namer
|
||||||
walletMux *sync.RWMutex
|
walletMux *sync.RWMutex
|
||||||
queue chan video
|
queue chan ytapi.Video
|
||||||
transferState int
|
transferState int
|
||||||
clientPublishAddress string
|
clientPublishAddress string
|
||||||
publicKey string
|
publicKey string
|
||||||
|
@ -263,7 +244,7 @@ func (s *Sync) FullCycle() (e error) {
|
||||||
s.syncedVideosMux = &sync.RWMutex{}
|
s.syncedVideosMux = &sync.RWMutex{}
|
||||||
s.walletMux = &sync.RWMutex{}
|
s.walletMux = &sync.RWMutex{}
|
||||||
s.grp = stopGroup
|
s.grp = stopGroup
|
||||||
s.queue = make(chan video)
|
s.queue = make(chan ytapi.Video)
|
||||||
interruptChan := make(chan os.Signal, 1)
|
interruptChan := make(chan os.Signal, 1)
|
||||||
signal.Notify(interruptChan, os.Interrupt, syscall.SIGTERM)
|
signal.Notify(interruptChan, os.Interrupt, syscall.SIGTERM)
|
||||||
defer signal.Stop(interruptChan)
|
defer signal.Stop(interruptChan)
|
||||||
|
@ -847,7 +828,7 @@ func (s *Sync) doSync() error {
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *Sync) startWorker(workerNum int) {
|
func (s *Sync) startWorker(workerNum int) {
|
||||||
var v video
|
var v ytapi.Video
|
||||||
var more bool
|
var more bool
|
||||||
|
|
||||||
for {
|
for {
|
||||||
|
@ -996,107 +977,23 @@ func (s *Sync) startWorker(workerNum int) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
var mostRecentlyFailedChannel string
|
|
||||||
|
|
||||||
func (s *Sync) enqueueYoutubeVideos() error {
|
func (s *Sync) enqueueYoutubeVideos() error {
|
||||||
start := time.Now()
|
defer func(start time.Time) { timing.TimedComponent("enqueueYoutubeVideos").Add(time.Since(start)) }(time.Now())
|
||||||
defer func(start time.Time) {
|
|
||||||
timing.TimedComponent("enqueueYoutubeVideos").Add(time.Since(start))
|
|
||||||
}(start)
|
|
||||||
client := &http.Client{
|
|
||||||
Transport: &transport.APIKey{Key: s.APIConfig.YoutubeAPIKey},
|
|
||||||
}
|
|
||||||
|
|
||||||
service, err := youtube.New(client)
|
|
||||||
if err != nil {
|
|
||||||
return errors.Prefix("error creating YouTube service", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
response, err := service.Channels.List("contentDetails").Id(s.YoutubeChannelID).Do()
|
|
||||||
if err != nil {
|
|
||||||
return errors.Prefix("error getting channels", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if len(response.Items) < 1 {
|
|
||||||
return errors.Err("youtube channel not found")
|
|
||||||
}
|
|
||||||
|
|
||||||
if response.Items[0].ContentDetails.RelatedPlaylists == nil {
|
|
||||||
return errors.Err("no related playlists")
|
|
||||||
}
|
|
||||||
|
|
||||||
playlistID := response.Items[0].ContentDetails.RelatedPlaylists.Uploads
|
|
||||||
if playlistID == "" {
|
|
||||||
return errors.Err("no channel playlist")
|
|
||||||
}
|
|
||||||
|
|
||||||
var videos []video
|
|
||||||
ipPool, err := ip_manager.GetIPPool(s.grp)
|
ipPool, err := ip_manager.GetIPPool(s.grp)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
playlistMap := make(map[string]*youtube.PlaylistItemSnippet, 50)
|
videos, err := ytapi.Enqueue(s.APIConfig.YoutubeAPIKey, s.YoutubeChannelID, s.syncedVideos, s.Manager.SyncFlags.QuickSync, s.Manager.videosLimit, ytapi.VideoParams{
|
||||||
nextPageToken := ""
|
VideoDir: s.videoDirectory,
|
||||||
for {
|
S3Config: s.Manager.GetS3AWSConfig(),
|
||||||
req := service.PlaylistItems.List("snippet").
|
Grp: s.grp,
|
||||||
PlaylistId(playlistID).
|
IPPool: ipPool,
|
||||||
MaxResults(50).
|
})
|
||||||
PageToken(nextPageToken)
|
if err != nil {
|
||||||
|
return err
|
||||||
playlistResponse, err := req.Do()
|
|
||||||
if err != nil {
|
|
||||||
return errors.Prefix("error getting playlist items", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if len(playlistResponse.Items) < 1 {
|
|
||||||
// If there are 50+ videos in a playlist but less than 50 are actually returned by the API, youtube will still redirect
|
|
||||||
// clients to a next page. Such next page will however be empty. This logic prevents ytsync from failing.
|
|
||||||
youtubeIsLying := len(videos) > 0
|
|
||||||
if youtubeIsLying {
|
|
||||||
break
|
|
||||||
}
|
|
||||||
if s.YoutubeChannelID == mostRecentlyFailedChannel {
|
|
||||||
return errors.Err("playlist items not found")
|
|
||||||
}
|
|
||||||
mostRecentlyFailedChannel = s.YoutubeChannelID
|
|
||||||
break //return errors.Err("playlist items not found") //TODO: will this work?
|
|
||||||
}
|
|
||||||
videoIDs := make([]string, 50)
|
|
||||||
for i, item := range playlistResponse.Items {
|
|
||||||
// normally we'd send the video into the channel here, but youtube api doesn't have sorting
|
|
||||||
// so we have to get ALL the videos, then sort them, then send them in
|
|
||||||
playlistMap[item.Snippet.ResourceId.VideoId] = item.Snippet
|
|
||||||
videoIDs[i] = item.Snippet.ResourceId.VideoId
|
|
||||||
}
|
|
||||||
req2 := service.Videos.List("snippet,contentDetails,recordingDetails").Id(strings.Join(videoIDs[:], ","))
|
|
||||||
|
|
||||||
videosListResponse, err := req2.Do()
|
|
||||||
if err != nil {
|
|
||||||
return errors.Prefix("error getting videos info", err)
|
|
||||||
}
|
|
||||||
for _, item := range videosListResponse.Items {
|
|
||||||
videos = append(videos, sources.NewYoutubeVideo(s.videoDirectory, item, playlistMap[item.Id].Position, s.Manager.GetS3AWSConfig(), s.grp, ipPool))
|
|
||||||
}
|
|
||||||
|
|
||||||
log.Infof("Got info for %d videos from youtube API", len(videos))
|
|
||||||
|
|
||||||
nextPageToken = playlistResponse.NextPageToken
|
|
||||||
if nextPageToken == "" || s.Manager.SyncFlags.QuickSync || len(videos) >= s.Manager.videosLimit {
|
|
||||||
break
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
for k, v := range s.syncedVideos {
|
|
||||||
if !v.Published {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
_, ok := playlistMap[k]
|
|
||||||
if !ok {
|
|
||||||
videos = append(videos, sources.NewMockedVideo(s.videoDirectory, k, s.YoutubeChannelID, s.Manager.GetS3AWSConfig(), s.grp, ipPool))
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
||||||
sort.Sort(byPublishedAt(videos))
|
|
||||||
|
|
||||||
Enqueue:
|
Enqueue:
|
||||||
for _, v := range videos {
|
for _, v := range videos {
|
||||||
|
@ -1116,7 +1013,7 @@ Enqueue:
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *Sync) processVideo(v video) (err error) {
|
func (s *Sync) processVideo(v ytapi.Video) (err error) {
|
||||||
defer func() {
|
defer func() {
|
||||||
if p := recover(); p != nil {
|
if p := recover(); p != nil {
|
||||||
logUtils.SendErrorToSlack("Video processing panic! %s", debug.Stack())
|
logUtils.SendErrorToSlack("Video processing panic! %s", debug.Stack())
|
||||||
|
@ -1284,8 +1181,7 @@ func (s *Sync) getUnsentSupports() (float64, error) {
|
||||||
|
|
||||||
// waitForDaemonProcess observes the running processes and returns when the process is no longer running or when the timeout is up
|
// waitForDaemonProcess observes the running processes and returns when the process is no longer running or when the timeout is up
|
||||||
func waitForDaemonProcess(timeout time.Duration) error {
|
func waitForDaemonProcess(timeout time.Duration) error {
|
||||||
then := time.Now()
|
stopTime := time.Now().Add(timeout * time.Second)
|
||||||
stopTime := then.Add(time.Duration(timeout * time.Second))
|
|
||||||
for !time.Now().After(stopTime) {
|
for !time.Now().After(stopTime) {
|
||||||
wait := 10 * time.Second
|
wait := 10 * time.Second
|
||||||
log.Println("the daemon is still running, waiting for it to exit")
|
log.Println("the daemon is still running, waiting for it to exit")
|
||||||
|
|
|
@ -13,24 +13,24 @@ import (
|
||||||
"sync"
|
"sync"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/lbryio/lbry.go/v2/extras/errors"
|
|
||||||
"github.com/lbryio/lbry.go/v2/extras/jsonrpc"
|
|
||||||
"github.com/lbryio/lbry.go/v2/extras/stop"
|
|
||||||
"github.com/lbryio/lbry.go/v2/extras/util"
|
|
||||||
"github.com/lbryio/ytsync/v5/timing"
|
|
||||||
logUtils "github.com/lbryio/ytsync/v5/util"
|
|
||||||
|
|
||||||
"github.com/lbryio/ytsync/v5/ip_manager"
|
"github.com/lbryio/ytsync/v5/ip_manager"
|
||||||
"github.com/lbryio/ytsync/v5/namer"
|
"github.com/lbryio/ytsync/v5/namer"
|
||||||
"github.com/lbryio/ytsync/v5/sdk"
|
"github.com/lbryio/ytsync/v5/sdk"
|
||||||
"github.com/lbryio/ytsync/v5/tags_manager"
|
"github.com/lbryio/ytsync/v5/tags_manager"
|
||||||
"github.com/lbryio/ytsync/v5/thumbs"
|
"github.com/lbryio/ytsync/v5/thumbs"
|
||||||
|
"github.com/lbryio/ytsync/v5/timing"
|
||||||
|
logUtils "github.com/lbryio/ytsync/v5/util"
|
||||||
|
|
||||||
|
"github.com/lbryio/lbry.go/v2/extras/errors"
|
||||||
|
"github.com/lbryio/lbry.go/v2/extras/jsonrpc"
|
||||||
|
"github.com/lbryio/lbry.go/v2/extras/stop"
|
||||||
|
"github.com/lbryio/lbry.go/v2/extras/util"
|
||||||
|
|
||||||
duration "github.com/ChannelMeter/iso8601duration"
|
duration "github.com/ChannelMeter/iso8601duration"
|
||||||
"github.com/aws/aws-sdk-go/aws"
|
"github.com/aws/aws-sdk-go/aws"
|
||||||
"github.com/shopspring/decimal"
|
"github.com/shopspring/decimal"
|
||||||
log "github.com/sirupsen/logrus"
|
log "github.com/sirupsen/logrus"
|
||||||
"google.golang.org/api/youtube/v3"
|
ytlib "google.golang.org/api/youtube/v3"
|
||||||
)
|
)
|
||||||
|
|
||||||
type YoutubeVideo struct {
|
type YoutubeVideo struct {
|
||||||
|
@ -43,7 +43,7 @@ type YoutubeVideo struct {
|
||||||
maxVideoLength float64
|
maxVideoLength float64
|
||||||
publishedAt time.Time
|
publishedAt time.Time
|
||||||
dir string
|
dir string
|
||||||
youtubeInfo *youtube.Video
|
youtubeInfo *ytlib.Video
|
||||||
youtubeChannelID string
|
youtubeChannelID string
|
||||||
tags []string
|
tags []string
|
||||||
awsConfig aws.Config
|
awsConfig aws.Config
|
||||||
|
@ -90,7 +90,7 @@ var youtubeCategories = map[string]string{
|
||||||
"44": "trailers",
|
"44": "trailers",
|
||||||
}
|
}
|
||||||
|
|
||||||
func NewYoutubeVideo(directory string, videoData *youtube.Video, playlistPosition int64, awsConfig aws.Config, stopGroup *stop.Group, pool *ip_manager.IPPool) *YoutubeVideo {
|
func NewYoutubeVideo(directory string, videoData *ytlib.Video, playlistPosition int64, awsConfig aws.Config, stopGroup *stop.Group, pool *ip_manager.IPPool) *YoutubeVideo {
|
||||||
publishedAt, _ := time.Parse(time.RFC3339Nano, videoData.Snippet.PublishedAt) // ignore parse errors
|
publishedAt, _ := time.Parse(time.RFC3339Nano, videoData.Snippet.PublishedAt) // ignore parse errors
|
||||||
return &YoutubeVideo{
|
return &YoutubeVideo{
|
||||||
id: videoData.Id,
|
id: videoData.Id,
|
||||||
|
|
|
@ -11,7 +11,7 @@ import (
|
||||||
"github.com/aws/aws-sdk-go/aws/session"
|
"github.com/aws/aws-sdk-go/aws/session"
|
||||||
"github.com/aws/aws-sdk-go/service/s3/s3manager"
|
"github.com/aws/aws-sdk-go/service/s3/s3manager"
|
||||||
log "github.com/sirupsen/logrus"
|
log "github.com/sirupsen/logrus"
|
||||||
"google.golang.org/api/youtube/v3"
|
ytlib "google.golang.org/api/youtube/v3"
|
||||||
)
|
)
|
||||||
|
|
||||||
type thumbnailUploader struct {
|
type thumbnailUploader struct {
|
||||||
|
@ -98,7 +98,7 @@ func MirrorThumbnail(url string, name string, s3Config aws.Config) (string, erro
|
||||||
return tu.mirroredUrl, nil
|
return tu.mirroredUrl, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func GetBestThumbnail(thumbnails *youtube.ThumbnailDetails) *youtube.Thumbnail {
|
func GetBestThumbnail(thumbnails *ytlib.ThumbnailDetails) *ytlib.Thumbnail {
|
||||||
if thumbnails.Maxres != nil {
|
if thumbnails.Maxres != nil {
|
||||||
return thumbnails.Maxres
|
return thumbnails.Maxres
|
||||||
} else if thumbnails.High != nil {
|
} else if thumbnails.High != nil {
|
||||||
|
|
213
ytapi/ytapi.go
Normal file
213
ytapi/ytapi.go
Normal file
|
@ -0,0 +1,213 @@
|
||||||
|
package ytapi
|
||||||
|
|
||||||
|
import (
|
||||||
|
"net/http"
|
||||||
|
"sort"
|
||||||
|
"strings"
|
||||||
|
"sync"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/lbryio/ytsync/v5/ip_manager"
|
||||||
|
"github.com/lbryio/ytsync/v5/sdk"
|
||||||
|
"github.com/lbryio/ytsync/v5/sources"
|
||||||
|
|
||||||
|
"github.com/lbryio/lbry.go/v2/extras/errors"
|
||||||
|
"github.com/lbryio/lbry.go/v2/extras/jsonrpc"
|
||||||
|
"github.com/lbryio/lbry.go/v2/extras/stop"
|
||||||
|
|
||||||
|
"github.com/aws/aws-sdk-go/aws"
|
||||||
|
log "github.com/sirupsen/logrus"
|
||||||
|
"google.golang.org/api/googleapi/transport"
|
||||||
|
ytlib "google.golang.org/api/youtube/v3"
|
||||||
|
)
|
||||||
|
|
||||||
|
type Video interface {
|
||||||
|
Size() *int64
|
||||||
|
ID() string
|
||||||
|
IDAndNum() string
|
||||||
|
PlaylistPosition() int
|
||||||
|
PublishedAt() time.Time
|
||||||
|
Sync(*jsonrpc.Client, sources.SyncParams, *sdk.SyncedVideo, bool, *sync.RWMutex) (*sources.SyncSummary, error)
|
||||||
|
}
|
||||||
|
|
||||||
|
type byPublishedAt []Video
|
||||||
|
|
||||||
|
func (a byPublishedAt) Len() int { return len(a) }
|
||||||
|
func (a byPublishedAt) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
|
||||||
|
func (a byPublishedAt) Less(i, j int) bool { return a[i].PublishedAt().Before(a[j].PublishedAt()) }
|
||||||
|
|
||||||
|
type VideoParams struct {
|
||||||
|
VideoDir string
|
||||||
|
S3Config aws.Config
|
||||||
|
Grp *stop.Group
|
||||||
|
IPPool *ip_manager.IPPool
|
||||||
|
}
|
||||||
|
|
||||||
|
var mostRecentlyFailedChannel string // TODO: fix this hack!
|
||||||
|
|
||||||
|
func Enqueue(apiKey, channelID string, syncedVideos map[string]sdk.SyncedVideo, quickSync bool, maxVideos int, videoParams VideoParams) ([]Video, error) {
|
||||||
|
playlistID, err := PlaylistID(apiKey, channelID)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
playlistMap := make(map[string]*ytlib.PlaylistItemSnippet, 50)
|
||||||
|
var playlistItems []*ytlib.PlaylistItem
|
||||||
|
var nextPageToken string
|
||||||
|
var videos []Video
|
||||||
|
|
||||||
|
for {
|
||||||
|
playlistItems, nextPageToken, err = PlaylistItems(apiKey, playlistID, nextPageToken)
|
||||||
|
|
||||||
|
if len(playlistItems) < 1 {
|
||||||
|
// If there are 50+ videos in a playlist but less than 50 are actually returned by the API, youtube will still redirect
|
||||||
|
// clients to a next page. Such next page will however be empty. This logic prevents ytsync from failing.
|
||||||
|
youtubeIsLying := len(videos) > 0
|
||||||
|
if youtubeIsLying {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
if channelID == mostRecentlyFailedChannel {
|
||||||
|
return nil, errors.Err("playlist items not found")
|
||||||
|
}
|
||||||
|
mostRecentlyFailedChannel = channelID
|
||||||
|
break //return errors.Err("playlist items not found") //TODO: will this work?
|
||||||
|
}
|
||||||
|
|
||||||
|
videoIDs := make([]string, len(playlistItems))
|
||||||
|
for i, item := range playlistItems {
|
||||||
|
// normally we'd send the video into the channel here, but youtube api doesn't have sorting
|
||||||
|
// so we have to get ALL the videos, then sort them, then send them in
|
||||||
|
playlistMap[item.Snippet.ResourceId.VideoId] = item.Snippet
|
||||||
|
videoIDs[i] = item.Snippet.ResourceId.VideoId
|
||||||
|
}
|
||||||
|
|
||||||
|
vids, err := Videos(apiKey, videoIDs)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, item := range vids {
|
||||||
|
videos = append(videos, sources.NewYoutubeVideo(videoParams.VideoDir, item, playlistMap[item.Id].Position, videoParams.S3Config, videoParams.Grp, videoParams.IPPool))
|
||||||
|
}
|
||||||
|
|
||||||
|
log.Infof("Got info for %d videos from youtube API", len(videos))
|
||||||
|
|
||||||
|
if nextPageToken == "" || quickSync || len(videos) >= maxVideos {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
for k, v := range syncedVideos {
|
||||||
|
if !v.Published {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if _, ok := playlistMap[k]; !ok {
|
||||||
|
videos = append(videos, sources.NewMockedVideo(videoParams.VideoDir, k, channelID, videoParams.S3Config, videoParams.Grp, videoParams.IPPool))
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
sort.Sort(byPublishedAt(videos))
|
||||||
|
|
||||||
|
return videos, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func VideosInChannel(apiKey, channelID string) (uint64, error) {
|
||||||
|
client := &http.Client{
|
||||||
|
Transport: &transport.APIKey{Key: apiKey},
|
||||||
|
}
|
||||||
|
|
||||||
|
service, err := ytlib.New(client)
|
||||||
|
if err != nil {
|
||||||
|
return 0, errors.Prefix("error creating YouTube service", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
response, err := service.Channels.List("statistics").Id(channelID).Do()
|
||||||
|
if err != nil {
|
||||||
|
return 0, errors.Prefix("error getting channels", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if len(response.Items) < 1 {
|
||||||
|
return 0, errors.Err("youtube channel not found")
|
||||||
|
}
|
||||||
|
|
||||||
|
return response.Items[0].Statistics.VideoCount, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func ChannelInfo(apiKey, channelID string) (*ytlib.ChannelSnippet, *ytlib.ChannelBrandingSettings, error) {
|
||||||
|
service, err := ytlib.New(&http.Client{Transport: &transport.APIKey{Key: apiKey}})
|
||||||
|
if err != nil {
|
||||||
|
return nil, nil, errors.Prefix("error creating YouTube service", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
response, err := service.Channels.List("snippet,brandingSettings").Id(channelID).Do()
|
||||||
|
if err != nil {
|
||||||
|
return nil, nil, errors.Prefix("error getting channel details", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if len(response.Items) < 1 {
|
||||||
|
return nil, nil, errors.Err("youtube channel not found")
|
||||||
|
}
|
||||||
|
|
||||||
|
return response.Items[0].Snippet, response.Items[0].BrandingSettings, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func PlaylistID(apiKey, channelID string) (string, error) {
|
||||||
|
service, err := ytlib.New(&http.Client{Transport: &transport.APIKey{Key: apiKey}})
|
||||||
|
if err != nil {
|
||||||
|
return "", errors.Prefix("error creating YouTube service", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
response, err := service.Channels.List("contentDetails").Id(channelID).Do()
|
||||||
|
if err != nil {
|
||||||
|
return "", errors.Prefix("error getting channel details", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if len(response.Items) < 1 {
|
||||||
|
return "", errors.Err("youtube channel not found")
|
||||||
|
}
|
||||||
|
|
||||||
|
if response.Items[0].ContentDetails.RelatedPlaylists == nil {
|
||||||
|
return "", errors.Err("no related playlists")
|
||||||
|
}
|
||||||
|
|
||||||
|
playlistID := response.Items[0].ContentDetails.RelatedPlaylists.Uploads
|
||||||
|
if playlistID == "" {
|
||||||
|
return "", errors.Err("no channel playlist")
|
||||||
|
}
|
||||||
|
|
||||||
|
return playlistID, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func PlaylistItems(apiKey, playlistID, nextPageToken string) ([]*ytlib.PlaylistItem, string, error) {
|
||||||
|
service, err := ytlib.New(&http.Client{Transport: &transport.APIKey{Key: apiKey}})
|
||||||
|
if err != nil {
|
||||||
|
return nil, "", errors.Prefix("error creating YouTube service", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
response, err := service.PlaylistItems.List("snippet").
|
||||||
|
PlaylistId(playlistID).
|
||||||
|
MaxResults(50).
|
||||||
|
PageToken(nextPageToken).
|
||||||
|
Do()
|
||||||
|
|
||||||
|
if err != nil {
|
||||||
|
return nil, "", errors.Prefix("error getting playlist items", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return response.Items, response.NextPageToken, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func Videos(apiKey string, videoIDs []string) ([]*ytlib.Video, error) {
|
||||||
|
service, err := ytlib.New(&http.Client{Transport: &transport.APIKey{Key: apiKey}})
|
||||||
|
if err != nil {
|
||||||
|
return nil, errors.Prefix("error creating YouTube service", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
response, err := service.Videos.List("snippet,contentDetails,recordingDetails").Id(strings.Join(videoIDs[:], ",")).Do()
|
||||||
|
if err != nil {
|
||||||
|
return nil, errors.Prefix("error getting videos info", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return response.Items, nil
|
||||||
|
}
|
Loading…
Reference in a new issue