move all youtube api calls into a single place
This commit is contained in:
Alex Grintsvayg
parent
be7fd7ddd8
commit
843303301a
6 changed files with 253 additions and 184 deletions
|
|
@ -1,32 +1,9 @@
|
|||
package manager
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
|
||||
"github.com/lbryio/lbry.go/v2/extras/errors"
|
||||
|
||||
"google.golang.org/api/googleapi/transport"
|
||||
"google.golang.org/api/youtube/v3"
|
||||
"github.com/lbryio/ytsync/v5/ytapi"
|
||||
)
|
||||
|
||||
func (s *Sync) CountVideos() (uint64, error) {
|
||||
client := &http.Client{
|
||||
Transport: &transport.APIKey{Key: s.APIConfig.YoutubeAPIKey},
|
||||
}
|
||||
|
||||
service, err := youtube.New(client)
|
||||
if err != nil {
|
||||
return 0, errors.Prefix("error creating YouTube service", err)
|
||||
}
|
||||
|
||||
response, err := service.Channels.List("statistics").Id(s.YoutubeChannelID).Do()
|
||||
if err != nil {
|
||||
return 0, errors.Prefix("error getting channels", err)
|
||||
}
|
||||
|
||||
if len(response.Items) < 1 {
|
||||
return 0, errors.Err("youtube channel not found")
|
||||
}
|
||||
|
||||
return response.Items[0].Statistics.VideoCount, nil
|
||||
return ytapi.VideosInChannel(s.APIConfig.YoutubeAPIKey, s.YoutubeChannelID)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -3,7 +3,6 @@ package manager
|
|||
import (
|
||||
"fmt"
|
||||
"math"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
|
|
@ -12,14 +11,13 @@ import (
|
|||
"github.com/lbryio/lbry.go/v2/extras/util"
|
||||
"github.com/lbryio/ytsync/v5/timing"
|
||||
logUtils "github.com/lbryio/ytsync/v5/util"
|
||||
"github.com/lbryio/ytsync/v5/ytapi"
|
||||
|
||||
"github.com/lbryio/ytsync/v5/tags_manager"
|
||||
"github.com/lbryio/ytsync/v5/thumbs"
|
||||
|
||||
"github.com/shopspring/decimal"
|
||||
log "github.com/sirupsen/logrus"
|
||||
"google.golang.org/api/googleapi/transport"
|
||||
"google.golang.org/api/youtube/v3"
|
||||
)
|
||||
|
||||
func (s *Sync) enableAddressReuse() error {
|
||||
|
|
@ -266,15 +264,14 @@ func (s *Sync) ensureEnoughUTXOs() error {
|
|||
}
|
||||
|
||||
func (s *Sync) waitForNewBlock() error {
|
||||
start := time.Now()
|
||||
defer func(start time.Time) {
|
||||
timing.TimedComponent("waitForNewBlock").Add(time.Since(start))
|
||||
}(start)
|
||||
defer func(start time.Time) { timing.TimedComponent("waitForNewBlock").Add(time.Since(start)) }(time.Now())
|
||||
|
||||
log.Printf("regtest: %t, docker: %t", logUtils.IsRegTest(), logUtils.IsUsingDocker())
|
||||
status, err := s.daemon.Status()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for status.Wallet.Blocks == 0 || status.Wallet.BlocksBehind != 0 {
|
||||
time.Sleep(5 * time.Second)
|
||||
status, err = s.daemon.Status()
|
||||
|
|
@ -308,10 +305,12 @@ func (s *Sync) GenerateRegtestBlock() error {
|
|||
if err != nil {
|
||||
return errors.Prefix("error getting lbrycrd client: ", err)
|
||||
}
|
||||
|
||||
txs, err := lbrycrd.Generate(1)
|
||||
if err != nil {
|
||||
return errors.Prefix("error generating new block: ", err)
|
||||
}
|
||||
|
||||
for _, tx := range txs {
|
||||
log.Info("Generated tx: ", tx.String())
|
||||
}
|
||||
|
|
@ -319,10 +318,8 @@ func (s *Sync) GenerateRegtestBlock() error {
|
|||
}
|
||||
|
||||
func (s *Sync) ensureChannelOwnership() error {
|
||||
start := time.Now()
|
||||
defer func(start time.Time) {
|
||||
timing.TimedComponent("ensureChannelOwnership").Add(time.Since(start))
|
||||
}(start)
|
||||
defer func(start time.Time) { timing.TimedComponent("ensureChannelOwnership").Add(time.Since(start)) }(time.Now())
|
||||
|
||||
if s.LbryChannelName == "" {
|
||||
return errors.Err("no channel name set")
|
||||
}
|
||||
|
|
@ -386,27 +383,12 @@ func (s *Sync) ensureChannelOwnership() error {
|
|||
return err
|
||||
}
|
||||
}
|
||||
client := &http.Client{
|
||||
Transport: &transport.APIKey{Key: s.APIConfig.YoutubeAPIKey},
|
||||
}
|
||||
|
||||
service, err := youtube.New(client)
|
||||
channelInfo, channelBranding, err := ytapi.ChannelInfo(s.APIConfig.YoutubeAPIKey, s.YoutubeChannelID)
|
||||
if err != nil {
|
||||
return errors.Prefix("error creating YouTube service", err)
|
||||
return err
|
||||
}
|
||||
|
||||
response, err := service.Channels.List("snippet,brandingSettings").Id(s.YoutubeChannelID).Do()
|
||||
if err != nil {
|
||||
return errors.Prefix("error getting channel details", err)
|
||||
}
|
||||
|
||||
if len(response.Items) < 1 {
|
||||
return errors.Err("youtube channel not found")
|
||||
}
|
||||
|
||||
channelInfo := response.Items[0].Snippet
|
||||
channelBranding := response.Items[0].BrandingSettings
|
||||
|
||||
thumbnail := thumbs.GetBestThumbnail(channelInfo.Thumbnails)
|
||||
thumbnailURL, err := thumbs.MirrorThumbnail(thumbnail.Url, s.YoutubeChannelID, s.Manager.GetS3AWSConfig())
|
||||
if err != nil {
|
||||
|
|
@ -467,6 +449,7 @@ func (s *Sync) ensureChannelOwnership() error {
|
|||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
s.lbryChannelID = c.Outputs[0].ClaimID
|
||||
return s.Manager.apiConfig.SetChannelClaimID(s.YoutubeChannelID, s.lbryChannelID)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -3,11 +3,9 @@ package manager
|
|||
import (
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"net/http"
|
||||
"os"
|
||||
"os/signal"
|
||||
"runtime/debug"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
|
|
@ -21,6 +19,7 @@ import (
|
|||
"github.com/lbryio/ytsync/v5/thumbs"
|
||||
"github.com/lbryio/ytsync/v5/timing"
|
||||
logUtils "github.com/lbryio/ytsync/v5/util"
|
||||
"github.com/lbryio/ytsync/v5/ytapi"
|
||||
|
||||
"github.com/lbryio/lbry.go/v2/extras/errors"
|
||||
"github.com/lbryio/lbry.go/v2/extras/jsonrpc"
|
||||
|
|
@ -34,8 +33,6 @@ import (
|
|||
"github.com/aws/aws-sdk-go/service/s3"
|
||||
"github.com/aws/aws-sdk-go/service/s3/s3manager"
|
||||
log "github.com/sirupsen/logrus"
|
||||
"google.golang.org/api/googleapi/transport"
|
||||
"google.golang.org/api/youtube/v3"
|
||||
)
|
||||
|
||||
const (
|
||||
|
|
@ -47,22 +44,6 @@ const (
|
|||
maxReasonLength = 500
|
||||
)
|
||||
|
||||
type video interface {
|
||||
Size() *int64
|
||||
ID() string
|
||||
IDAndNum() string
|
||||
PlaylistPosition() int
|
||||
PublishedAt() time.Time
|
||||
Sync(*jsonrpc.Client, sources.SyncParams, *sdk.SyncedVideo, bool, *sync.RWMutex) (*sources.SyncSummary, error)
|
||||
}
|
||||
|
||||
// sorting videos
|
||||
type byPublishedAt []video
|
||||
|
||||
func (a byPublishedAt) Len() int { return len(a) }
|
||||
func (a byPublishedAt) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
|
||||
func (a byPublishedAt) Less(i, j int) bool { return a[i].PublishedAt().Before(a[j].PublishedAt()) }
|
||||
|
||||
// Sync stores the options that control how syncing happens
|
||||
type Sync struct {
|
||||
APIConfig *sdk.APIConfig
|
||||
|
|
@ -87,7 +68,7 @@ type Sync struct {
|
|||
lbryChannelID string
|
||||
namer *namer.Namer
|
||||
walletMux *sync.RWMutex
|
||||
queue chan video
|
||||
queue chan ytapi.Video
|
||||
transferState int
|
||||
clientPublishAddress string
|
||||
publicKey string
|
||||
|
|
@ -263,7 +244,7 @@ func (s *Sync) FullCycle() (e error) {
|
|||
s.syncedVideosMux = &sync.RWMutex{}
|
||||
s.walletMux = &sync.RWMutex{}
|
||||
s.grp = stopGroup
|
||||
s.queue = make(chan video)
|
||||
s.queue = make(chan ytapi.Video)
|
||||
interruptChan := make(chan os.Signal, 1)
|
||||
signal.Notify(interruptChan, os.Interrupt, syscall.SIGTERM)
|
||||
defer signal.Stop(interruptChan)
|
||||
|
|
@ -847,7 +828,7 @@ func (s *Sync) doSync() error {
|
|||
}
|
||||
|
||||
func (s *Sync) startWorker(workerNum int) {
|
||||
var v video
|
||||
var v ytapi.Video
|
||||
var more bool
|
||||
|
||||
for {
|
||||
|
|
@ -996,107 +977,23 @@ func (s *Sync) startWorker(workerNum int) {
|
|||
}
|
||||
}
|
||||
|
||||
var mostRecentlyFailedChannel string
|
||||
|
||||
func (s *Sync) enqueueYoutubeVideos() error {
|
||||
start := time.Now()
|
||||
defer func(start time.Time) {
|
||||
timing.TimedComponent("enqueueYoutubeVideos").Add(time.Since(start))
|
||||
}(start)
|
||||
client := &http.Client{
|
||||
Transport: &transport.APIKey{Key: s.APIConfig.YoutubeAPIKey},
|
||||
}
|
||||
defer func(start time.Time) { timing.TimedComponent("enqueueYoutubeVideos").Add(time.Since(start)) }(time.Now())
|
||||
|
||||
service, err := youtube.New(client)
|
||||
if err != nil {
|
||||
return errors.Prefix("error creating YouTube service", err)
|
||||
}
|
||||
|
||||
response, err := service.Channels.List("contentDetails").Id(s.YoutubeChannelID).Do()
|
||||
if err != nil {
|
||||
return errors.Prefix("error getting channels", err)
|
||||
}
|
||||
|
||||
if len(response.Items) < 1 {
|
||||
return errors.Err("youtube channel not found")
|
||||
}
|
||||
|
||||
if response.Items[0].ContentDetails.RelatedPlaylists == nil {
|
||||
return errors.Err("no related playlists")
|
||||
}
|
||||
|
||||
playlistID := response.Items[0].ContentDetails.RelatedPlaylists.Uploads
|
||||
if playlistID == "" {
|
||||
return errors.Err("no channel playlist")
|
||||
}
|
||||
|
||||
var videos []video
|
||||
ipPool, err := ip_manager.GetIPPool(s.grp)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
playlistMap := make(map[string]*youtube.PlaylistItemSnippet, 50)
|
||||
nextPageToken := ""
|
||||
for {
|
||||
req := service.PlaylistItems.List("snippet").
|
||||
PlaylistId(playlistID).
|
||||
MaxResults(50).
|
||||
PageToken(nextPageToken)
|
||||
|
||||
playlistResponse, err := req.Do()
|
||||
if err != nil {
|
||||
return errors.Prefix("error getting playlist items", err)
|
||||
}
|
||||
|
||||
if len(playlistResponse.Items) < 1 {
|
||||
// If there are 50+ videos in a playlist but less than 50 are actually returned by the API, youtube will still redirect
|
||||
// clients to a next page. Such next page will however be empty. This logic prevents ytsync from failing.
|
||||
youtubeIsLying := len(videos) > 0
|
||||
if youtubeIsLying {
|
||||
break
|
||||
}
|
||||
if s.YoutubeChannelID == mostRecentlyFailedChannel {
|
||||
return errors.Err("playlist items not found")
|
||||
}
|
||||
mostRecentlyFailedChannel = s.YoutubeChannelID
|
||||
break //return errors.Err("playlist items not found") //TODO: will this work?
|
||||
}
|
||||
videoIDs := make([]string, 50)
|
||||
for i, item := range playlistResponse.Items {
|
||||
// normally we'd send the video into the channel here, but youtube api doesn't have sorting
|
||||
// so we have to get ALL the videos, then sort them, then send them in
|
||||
playlistMap[item.Snippet.ResourceId.VideoId] = item.Snippet
|
||||
videoIDs[i] = item.Snippet.ResourceId.VideoId
|
||||
}
|
||||
req2 := service.Videos.List("snippet,contentDetails,recordingDetails").Id(strings.Join(videoIDs[:], ","))
|
||||
|
||||
videosListResponse, err := req2.Do()
|
||||
if err != nil {
|
||||
return errors.Prefix("error getting videos info", err)
|
||||
}
|
||||
for _, item := range videosListResponse.Items {
|
||||
videos = append(videos, sources.NewYoutubeVideo(s.videoDirectory, item, playlistMap[item.Id].Position, s.Manager.GetS3AWSConfig(), s.grp, ipPool))
|
||||
}
|
||||
|
||||
log.Infof("Got info for %d videos from youtube API", len(videos))
|
||||
|
||||
nextPageToken = playlistResponse.NextPageToken
|
||||
if nextPageToken == "" || s.Manager.SyncFlags.QuickSync || len(videos) >= s.Manager.videosLimit {
|
||||
break
|
||||
}
|
||||
videos, err := ytapi.Enqueue(s.APIConfig.YoutubeAPIKey, s.YoutubeChannelID, s.syncedVideos, s.Manager.SyncFlags.QuickSync, s.Manager.videosLimit, ytapi.VideoParams{
|
||||
VideoDir: s.videoDirectory,
|
||||
S3Config: s.Manager.GetS3AWSConfig(),
|
||||
Grp: s.grp,
|
||||
IPPool: ipPool,
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for k, v := range s.syncedVideos {
|
||||
if !v.Published {
|
||||
continue
|
||||
}
|
||||
_, ok := playlistMap[k]
|
||||
if !ok {
|
||||
videos = append(videos, sources.NewMockedVideo(s.videoDirectory, k, s.YoutubeChannelID, s.Manager.GetS3AWSConfig(), s.grp, ipPool))
|
||||
}
|
||||
|
||||
}
|
||||
sort.Sort(byPublishedAt(videos))
|
||||
|
||||
Enqueue:
|
||||
for _, v := range videos {
|
||||
|
|
@ -1116,7 +1013,7 @@ Enqueue:
|
|||
return nil
|
||||
}
|
||||
|
||||
func (s *Sync) processVideo(v video) (err error) {
|
||||
func (s *Sync) processVideo(v ytapi.Video) (err error) {
|
||||
defer func() {
|
||||
if p := recover(); p != nil {
|
||||
logUtils.SendErrorToSlack("Video processing panic! %s", debug.Stack())
|
||||
|
|
@ -1284,8 +1181,7 @@ func (s *Sync) getUnsentSupports() (float64, error) {
|
|||
|
||||
// waitForDaemonProcess observes the running processes and returns when the process is no longer running or when the timeout is up
|
||||
func waitForDaemonProcess(timeout time.Duration) error {
|
||||
then := time.Now()
|
||||
stopTime := then.Add(time.Duration(timeout * time.Second))
|
||||
stopTime := time.Now().Add(timeout * time.Second)
|
||||
for !time.Now().After(stopTime) {
|
||||
wait := 10 * time.Second
|
||||
log.Println("the daemon is still running, waiting for it to exit")
|
||||
|
|
|
|||
|
|
@ -13,24 +13,24 @@ import (
|
|||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/lbryio/lbry.go/v2/extras/errors"
|
||||
"github.com/lbryio/lbry.go/v2/extras/jsonrpc"
|
||||
"github.com/lbryio/lbry.go/v2/extras/stop"
|
||||
"github.com/lbryio/lbry.go/v2/extras/util"
|
||||
"github.com/lbryio/ytsync/v5/timing"
|
||||
logUtils "github.com/lbryio/ytsync/v5/util"
|
||||
|
||||
"github.com/lbryio/ytsync/v5/ip_manager"
|
||||
"github.com/lbryio/ytsync/v5/namer"
|
||||
"github.com/lbryio/ytsync/v5/sdk"
|
||||
"github.com/lbryio/ytsync/v5/tags_manager"
|
||||
"github.com/lbryio/ytsync/v5/thumbs"
|
||||
"github.com/lbryio/ytsync/v5/timing"
|
||||
logUtils "github.com/lbryio/ytsync/v5/util"
|
||||
|
||||
"github.com/lbryio/lbry.go/v2/extras/errors"
|
||||
"github.com/lbryio/lbry.go/v2/extras/jsonrpc"
|
||||
"github.com/lbryio/lbry.go/v2/extras/stop"
|
||||
"github.com/lbryio/lbry.go/v2/extras/util"
|
||||
|
||||
duration "github.com/ChannelMeter/iso8601duration"
|
||||
"github.com/aws/aws-sdk-go/aws"
|
||||
"github.com/shopspring/decimal"
|
||||
log "github.com/sirupsen/logrus"
|
||||
"google.golang.org/api/youtube/v3"
|
||||
ytlib "google.golang.org/api/youtube/v3"
|
||||
)
|
||||
|
||||
type YoutubeVideo struct {
|
||||
|
|
@ -43,7 +43,7 @@ type YoutubeVideo struct {
|
|||
maxVideoLength float64
|
||||
publishedAt time.Time
|
||||
dir string
|
||||
youtubeInfo *youtube.Video
|
||||
youtubeInfo *ytlib.Video
|
||||
youtubeChannelID string
|
||||
tags []string
|
||||
awsConfig aws.Config
|
||||
|
|
@ -90,7 +90,7 @@ var youtubeCategories = map[string]string{
|
|||
"44": "trailers",
|
||||
}
|
||||
|
||||
func NewYoutubeVideo(directory string, videoData *youtube.Video, playlistPosition int64, awsConfig aws.Config, stopGroup *stop.Group, pool *ip_manager.IPPool) *YoutubeVideo {
|
||||
func NewYoutubeVideo(directory string, videoData *ytlib.Video, playlistPosition int64, awsConfig aws.Config, stopGroup *stop.Group, pool *ip_manager.IPPool) *YoutubeVideo {
|
||||
publishedAt, _ := time.Parse(time.RFC3339Nano, videoData.Snippet.PublishedAt) // ignore parse errors
|
||||
return &YoutubeVideo{
|
||||
id: videoData.Id,
|
||||
|
|
|
|||
|
|
@ -11,7 +11,7 @@ import (
|
|||
"github.com/aws/aws-sdk-go/aws/session"
|
||||
"github.com/aws/aws-sdk-go/service/s3/s3manager"
|
||||
log "github.com/sirupsen/logrus"
|
||||
"google.golang.org/api/youtube/v3"
|
||||
ytlib "google.golang.org/api/youtube/v3"
|
||||
)
|
||||
|
||||
type thumbnailUploader struct {
|
||||
|
|
@ -98,7 +98,7 @@ func MirrorThumbnail(url string, name string, s3Config aws.Config) (string, erro
|
|||
return tu.mirroredUrl, nil
|
||||
}
|
||||
|
||||
func GetBestThumbnail(thumbnails *youtube.ThumbnailDetails) *youtube.Thumbnail {
|
||||
func GetBestThumbnail(thumbnails *ytlib.ThumbnailDetails) *ytlib.Thumbnail {
|
||||
if thumbnails.Maxres != nil {
|
||||
return thumbnails.Maxres
|
||||
} else if thumbnails.High != nil {
|
||||
|
|
|
|||
213
ytapi/ytapi.go
Normal file
213
ytapi/ytapi.go
Normal file
|
|
@ -0,0 +1,213 @@
|
|||
package ytapi
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/lbryio/ytsync/v5/ip_manager"
|
||||
"github.com/lbryio/ytsync/v5/sdk"
|
||||
"github.com/lbryio/ytsync/v5/sources"
|
||||
|
||||
"github.com/lbryio/lbry.go/v2/extras/errors"
|
||||
"github.com/lbryio/lbry.go/v2/extras/jsonrpc"
|
||||
"github.com/lbryio/lbry.go/v2/extras/stop"
|
||||
|
||||
"github.com/aws/aws-sdk-go/aws"
|
||||
log "github.com/sirupsen/logrus"
|
||||
"google.golang.org/api/googleapi/transport"
|
||||
ytlib "google.golang.org/api/youtube/v3"
|
||||
)
|
||||
|
||||
type Video interface {
|
||||
Size() *int64
|
||||
ID() string
|
||||
IDAndNum() string
|
||||
PlaylistPosition() int
|
||||
PublishedAt() time.Time
|
||||
Sync(*jsonrpc.Client, sources.SyncParams, *sdk.SyncedVideo, bool, *sync.RWMutex) (*sources.SyncSummary, error)
|
||||
}
|
||||
|
||||
type byPublishedAt []Video
|
||||
|
||||
func (a byPublishedAt) Len() int { return len(a) }
|
||||
func (a byPublishedAt) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
|
||||
func (a byPublishedAt) Less(i, j int) bool { return a[i].PublishedAt().Before(a[j].PublishedAt()) }
|
||||
|
||||
type VideoParams struct {
|
||||
VideoDir string
|
||||
S3Config aws.Config
|
||||
Grp *stop.Group
|
||||
IPPool *ip_manager.IPPool
|
||||
}
|
||||
|
||||
var mostRecentlyFailedChannel string // TODO: fix this hack!
|
||||
|
||||
func Enqueue(apiKey, channelID string, syncedVideos map[string]sdk.SyncedVideo, quickSync bool, maxVideos int, videoParams VideoParams) ([]Video, error) {
|
||||
playlistID, err := PlaylistID(apiKey, channelID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
playlistMap := make(map[string]*ytlib.PlaylistItemSnippet, 50)
|
||||
var playlistItems []*ytlib.PlaylistItem
|
||||
var nextPageToken string
|
||||
var videos []Video
|
||||
|
||||
for {
|
||||
playlistItems, nextPageToken, err = PlaylistItems(apiKey, playlistID, nextPageToken)
|
||||
|
||||
if len(playlistItems) < 1 {
|
||||
// If there are 50+ videos in a playlist but less than 50 are actually returned by the API, youtube will still redirect
|
||||
// clients to a next page. Such next page will however be empty. This logic prevents ytsync from failing.
|
||||
youtubeIsLying := len(videos) > 0
|
||||
if youtubeIsLying {
|
||||
break
|
||||
}
|
||||
if channelID == mostRecentlyFailedChannel {
|
||||
return nil, errors.Err("playlist items not found")
|
||||
}
|
||||
mostRecentlyFailedChannel = channelID
|
||||
break //return errors.Err("playlist items not found") //TODO: will this work?
|
||||
}
|
||||
|
||||
videoIDs := make([]string, len(playlistItems))
|
||||
for i, item := range playlistItems {
|
||||
// normally we'd send the video into the channel here, but youtube api doesn't have sorting
|
||||
// so we have to get ALL the videos, then sort them, then send them in
|
||||
playlistMap[item.Snippet.ResourceId.VideoId] = item.Snippet
|
||||
videoIDs[i] = item.Snippet.ResourceId.VideoId
|
||||
}
|
||||
|
||||
vids, err := Videos(apiKey, videoIDs)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
for _, item := range vids {
|
||||
videos = append(videos, sources.NewYoutubeVideo(videoParams.VideoDir, item, playlistMap[item.Id].Position, videoParams.S3Config, videoParams.Grp, videoParams.IPPool))
|
||||
}
|
||||
|
||||
log.Infof("Got info for %d videos from youtube API", len(videos))
|
||||
|
||||
if nextPageToken == "" || quickSync || len(videos) >= maxVideos {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
for k, v := range syncedVideos {
|
||||
if !v.Published {
|
||||
continue
|
||||
}
|
||||
if _, ok := playlistMap[k]; !ok {
|
||||
videos = append(videos, sources.NewMockedVideo(videoParams.VideoDir, k, channelID, videoParams.S3Config, videoParams.Grp, videoParams.IPPool))
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
sort.Sort(byPublishedAt(videos))
|
||||
|
||||
return videos, nil
|
||||
}
|
||||
|
||||
func VideosInChannel(apiKey, channelID string) (uint64, error) {
|
||||
client := &http.Client{
|
||||
Transport: &transport.APIKey{Key: apiKey},
|
||||
}
|
||||
|
||||
service, err := ytlib.New(client)
|
||||
if err != nil {
|
||||
return 0, errors.Prefix("error creating YouTube service", err)
|
||||
}
|
||||
|
||||
response, err := service.Channels.List("statistics").Id(channelID).Do()
|
||||
if err != nil {
|
||||
return 0, errors.Prefix("error getting channels", err)
|
||||
}
|
||||
|
||||
if len(response.Items) < 1 {
|
||||
return 0, errors.Err("youtube channel not found")
|
||||
}
|
||||
|
||||
return response.Items[0].Statistics.VideoCount, nil
|
||||
}
|
||||
|
||||
func ChannelInfo(apiKey, channelID string) (*ytlib.ChannelSnippet, *ytlib.ChannelBrandingSettings, error) {
|
||||
service, err := ytlib.New(&http.Client{Transport: &transport.APIKey{Key: apiKey}})
|
||||
if err != nil {
|
||||
return nil, nil, errors.Prefix("error creating YouTube service", err)
|
||||
}
|
||||
|
||||
response, err := service.Channels.List("snippet,brandingSettings").Id(channelID).Do()
|
||||
if err != nil {
|
||||
return nil, nil, errors.Prefix("error getting channel details", err)
|
||||
}
|
||||
|
||||
if len(response.Items) < 1 {
|
||||
return nil, nil, errors.Err("youtube channel not found")
|
||||
}
|
||||
|
||||
return response.Items[0].Snippet, response.Items[0].BrandingSettings, nil
|
||||
}
|
||||
|
||||
func PlaylistID(apiKey, channelID string) (string, error) {
|
||||
service, err := ytlib.New(&http.Client{Transport: &transport.APIKey{Key: apiKey}})
|
||||
if err != nil {
|
||||
return "", errors.Prefix("error creating YouTube service", err)
|
||||
}
|
||||
|
||||
response, err := service.Channels.List("contentDetails").Id(channelID).Do()
|
||||
if err != nil {
|
||||
return "", errors.Prefix("error getting channel details", err)
|
||||
}
|
||||
|
||||
if len(response.Items) < 1 {
|
||||
return "", errors.Err("youtube channel not found")
|
||||
}
|
||||
|
||||
if response.Items[0].ContentDetails.RelatedPlaylists == nil {
|
||||
return "", errors.Err("no related playlists")
|
||||
}
|
||||
|
||||
playlistID := response.Items[0].ContentDetails.RelatedPlaylists.Uploads
|
||||
if playlistID == "" {
|
||||
return "", errors.Err("no channel playlist")
|
||||
}
|
||||
|
||||
return playlistID, nil
|
||||
}
|
||||
|
||||
func PlaylistItems(apiKey, playlistID, nextPageToken string) ([]*ytlib.PlaylistItem, string, error) {
|
||||
service, err := ytlib.New(&http.Client{Transport: &transport.APIKey{Key: apiKey}})
|
||||
if err != nil {
|
||||
return nil, "", errors.Prefix("error creating YouTube service", err)
|
||||
}
|
||||
|
||||
response, err := service.PlaylistItems.List("snippet").
|
||||
PlaylistId(playlistID).
|
||||
MaxResults(50).
|
||||
PageToken(nextPageToken).
|
||||
Do()
|
||||
|
||||
if err != nil {
|
||||
return nil, "", errors.Prefix("error getting playlist items", err)
|
||||
}
|
||||
|
||||
return response.Items, response.NextPageToken, nil
|
||||
}
|
||||
|
||||
func Videos(apiKey string, videoIDs []string) ([]*ytlib.Video, error) {
|
||||
service, err := ytlib.New(&http.Client{Transport: &transport.APIKey{Key: apiKey}})
|
||||
if err != nil {
|
||||
return nil, errors.Prefix("error creating YouTube service", err)
|
||||
}
|
||||
|
||||
response, err := service.Videos.List("snippet,contentDetails,recordingDetails").Id(strings.Join(videoIDs[:], ",")).Do()
|
||||
if err != nil {
|
||||
return nil, errors.Prefix("error getting videos info", err)
|
||||
}
|
||||
|
||||
return response.Items, nil
|
||||
}
|
||||
Loading…
Reference in a new issue