diff --git a/go.mod b/go.mod index f86808c..77b5e8f 100644 --- a/go.mod +++ b/go.mod @@ -17,7 +17,7 @@ require ( github.com/konsorten/go-windows-terminal-sequences v1.0.2 // indirect github.com/kr/pretty v0.1.0 // indirect github.com/lbryio/errors.go v0.0.0-20180223142025-ad03d3cc6a5c - github.com/lbryio/lbry.go v0.0.0-20190422142237-ad33acfc936f + github.com/lbryio/lbry.go v0.0.0-20190503025608-3a22a0af0a45 github.com/lusis/slack-test v0.0.0-20190408224659-6cf59653add2 // indirect github.com/mitchellh/go-ps v0.0.0-20170309133038-4fdf99ab2936 github.com/mitchellh/mapstructure v1.1.2 // indirect diff --git a/go.sum b/go.sum index e7c62b6..2260b70 100644 --- a/go.sum +++ b/go.sum @@ -125,17 +125,11 @@ github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/lbryio/errors.go v0.0.0-20180223142025-ad03d3cc6a5c h1:BhdcWGsuKif/XoSZnqVGNqJ1iEmH0czWR5upj+AuR8M= github.com/lbryio/errors.go v0.0.0-20180223142025-ad03d3cc6a5c/go.mod h1:muH7wpUqE8hRA3OrYYosw9+Sl681BF9cwcjzE+OCNK8= -github.com/lbryio/lbry.go v0.0.0-20190419005332-80b25b225e18 h1:ZhWjtvaq5r7julhcF9OSgx4bLv9UsdIx27zH1/fbrEc= -github.com/lbryio/lbry.go v0.0.0-20190419005332-80b25b225e18/go.mod h1:kd08aOMCuBVYJ3EafY4Kx3dRAWWQYhobJ9beREgsaRI= -github.com/lbryio/lbry.go v0.0.0-20190422142237-ad33acfc936f h1:o6EZ7bAdYrS6pKp85SEr6Ywyy2JDJS1CY3ChkVsvSM4= -github.com/lbryio/lbry.go v0.0.0-20190422142237-ad33acfc936f/go.mod h1:FubnMAYvLt2jGasG7BrQsokYHZ2wpNtWethPHUVauMc= -github.com/lbryio/lbryschema.go v0.0.0-20190422030648-322c658307e0 h1:/YWLlbbDefRGLs/ozyuRpvpwpFISYehwR4AAVlPthA8= -github.com/lbryio/lbryschema.go v0.0.0-20190422030648-322c658307e0/go.mod h1:dAzPCBj3CKKWBGYBZxK6tKBP5SCgY2tqd9SnQd/OyKo= +github.com/lbryio/lbry.go v0.0.0-20190503025608-3a22a0af0a45 h1:6fpRUui1G8xKiQqL51fHJwrpqYSd4G3RIYWsRd+R8x8= +github.com/lbryio/lbry.go v0.0.0-20190503025608-3a22a0af0a45/go.mod h1:rAREtjHq/Wkkqy4+yt89Er/7JAVu6uMo3v8nevCEEIA= +github.com/lbryio/lbryschema.go v0.0.0-20190428231007-c54836bca002 h1:urfYK5ElpUrAv90auPLldoVC60LwiGAcY0OE6HJB9KI= +github.com/lbryio/lbryschema.go v0.0.0-20190428231007-c54836bca002/go.mod h1:dAzPCBj3CKKWBGYBZxK6tKBP5SCgY2tqd9SnQd/OyKo= github.com/lbryio/ozzo-validation v0.0.0-20170323141101-d1008ad1fd04/go.mod h1:fbG/dzobG8r95KzMwckXiLMHfFjZaBRQqC9hPs2XAQ4= -github.com/lbryio/types v0.0.0-20190405005919-54c3c28f676a h1:twWvrsBDvSb+qnmpSq3nvFrodgC5PpXUipyo4T/W790= -github.com/lbryio/types v0.0.0-20190405005919-54c3c28f676a/go.mod h1:CG3wsDv5BiVYQd5i1Jp7wGsaVyjZTJshqXeWMVKsISE= -github.com/lbryio/types v0.0.0-20190415181811-35ddf1afe731 h1:iERWR8Dkng30eRInI7gzolTEJBW9nBSK/sT+Z9aSipI= -github.com/lbryio/types v0.0.0-20190415181811-35ddf1afe731/go.mod h1:CG3wsDv5BiVYQd5i1Jp7wGsaVyjZTJshqXeWMVKsISE= github.com/lbryio/types v0.0.0-20190422033210-321fb2abda9c h1:m3O7561xBQ00lfUVayW4c6SnpVbUDQtPUwGcGYSUYQA= github.com/lbryio/types v0.0.0-20190422033210-321fb2abda9c/go.mod h1:CG3wsDv5BiVYQd5i1Jp7wGsaVyjZTJshqXeWMVKsISE= github.com/lusis/go-slackbot v0.0.0-20180109053408-401027ccfef5 h1:AsEBgzv3DhuYHI/GiQh2HxvTP71HCCE9E/tzGUzGdtU= @@ -304,6 +298,7 @@ gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWD gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/manager/manager.go b/manager/manager.go index 9f4fe1f..b56c4f8 100644 --- a/manager/manager.go +++ b/manager/manager.go @@ -2,6 +2,8 @@ package manager import ( "fmt" + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/credentials" "strings" "syscall" "time" @@ -70,17 +72,18 @@ func NewSyncManager(stopOnError bool, maxTries int, takeOverExistingChannel bool } const ( - StatusPending = "pending" // waiting for permission to sync - StatusPendingEmail = "pendingemail" // permission granted but missing email - StatusQueued = "queued" // in sync queue. will be synced soon - StatusSyncing = "syncing" // syncing now - StatusSynced = "synced" // done - StatusFailed = "failed" - StatusFinalized = "finalized" // no more changes allowed - StatusAbandoned = "abandoned" // deleted on youtube or banned + StatusPending = "pending" // waiting for permission to sync + StatusPendingEmail = "pendingemail" // permission granted but missing email + StatusQueued = "queued" // in sync queue. will be synced soon + StatusPendingUpgrade = "pendingupgrade" // in sync queue. will be synced soon + StatusSyncing = "syncing" // syncing now + StatusSynced = "synced" // done + StatusFailed = "failed" + StatusFinalized = "finalized" // no more changes allowed + StatusAbandoned = "abandoned" // deleted on youtube or banned ) -var SyncStatuses = []string{StatusPending, StatusPendingEmail, StatusQueued, StatusSyncing, StatusSynced, StatusFailed, StatusFinalized, StatusAbandoned} +var SyncStatuses = []string{StatusPending, StatusPendingEmail, StatusPendingUpgrade, StatusQueued, StatusSyncing, StatusSynced, StatusFailed, StatusFinalized, StatusAbandoned} const ( VideoStatusPublished = "published" @@ -207,7 +210,12 @@ func (s *SyncManager) Start() error { } return nil } - +func (s *SyncManager) GetS3AWSConfig() aws.Config { + return aws.Config{ + Credentials: credentials.NewStaticCredentials(s.awsS3ID, s.awsS3Secret, ""), + Region: &s.awsS3Region, + } +} func (s *SyncManager) checkUsedSpace() error { usedPctile, err := GetUsedSpace(s.blobsDir) if err != nil { diff --git a/manager/setup.go b/manager/setup.go index b24282b..393c2c6 100644 --- a/manager/setup.go +++ b/manager/setup.go @@ -7,8 +7,6 @@ import ( "strconv" "time" - "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/aws/credentials" "github.com/lbryio/lbry.go/extras/errors" "github.com/lbryio/lbry.go/extras/jsonrpc" "github.com/lbryio/lbry.go/extras/util" @@ -301,20 +299,8 @@ func (s *Sync) ensureChannelOwnership() error { channelInfo := response.Items[0].Snippet - thumbnail := channelInfo.Thumbnails.Default - if channelInfo.Thumbnails.Maxres != nil { - thumbnail = channelInfo.Thumbnails.Maxres - } else if channelInfo.Thumbnails.High != nil { - thumbnail = channelInfo.Thumbnails.High - } else if channelInfo.Thumbnails.Medium != nil { - thumbnail = channelInfo.Thumbnails.Medium - } else if channelInfo.Thumbnails.Standard != nil { - thumbnail = channelInfo.Thumbnails.Standard - } - thumbnailURL, err := thumbs.MirrorThumbnail(thumbnail.Url, s.YoutubeChannelID, aws.Config{ - Credentials: credentials.NewStaticCredentials(s.AwsS3ID, s.AwsS3Secret, ""), - Region: &s.AwsS3Region, - }) + thumbnail := thumbs.GetBestThumbnail(channelInfo.Thumbnails) + thumbnailURL, err := thumbs.MirrorThumbnail(thumbnail.Url, s.YoutubeChannelID, s.Manager.GetS3AWSConfig()) if err != nil { return err } diff --git a/manager/ytsync.go b/manager/ytsync.go index 7b9a511..3a5295c 100644 --- a/manager/ytsync.go +++ b/manager/ytsync.go @@ -1,11 +1,7 @@ package manager import ( - "bufio" - "encoding/csv" - "encoding/json" "fmt" - "io" "io/ioutil" "net/http" "os" @@ -21,6 +17,7 @@ import ( "github.com/lbryio/ytsync/namer" "github.com/lbryio/ytsync/sdk" "github.com/lbryio/ytsync/sources" + "github.com/lbryio/ytsync/thumbs" "github.com/lbryio/lbry.go/extras/errors" "github.com/lbryio/lbry.go/extras/jsonrpc" @@ -51,7 +48,7 @@ type video interface { IDAndNum() string PlaylistPosition() int PublishedAt() time.Time - Sync(*jsonrpc.Client, string, float64, string, int, *namer.Namer, float64) (*sources.SyncSummary, error) + Sync(*jsonrpc.Client, sources.SyncParams, *sdk.SyncedVideo, bool) (*sources.SyncSummary, error) } // sorting videos @@ -319,6 +316,7 @@ func (s *Sync) FullCycle() (e error) { return nil } + func (s *Sync) setChannelTerminationStatus(e *error) { if *e != nil { //conditions for which a channel shouldn't be marked as failed @@ -390,7 +388,7 @@ func logShutdownError(shutdownErr error) { var thumbnailHosts = []string{ "berk.ninja/thumbnails/", - "https://thumbnails.lbry.com/", + thumbs.ThumbnailEndpoint, } func isYtsyncClaim(c jsonrpc.Claim) bool { @@ -545,7 +543,7 @@ func (s *Sync) doSync() error { } if s.LbryChannelName == "@UCBerkeley" { - err = s.enqueueUCBVideos() + err = errors.Err("UCB is not supported on this version of YTSYNC") } else { err = s.enqueueYoutubeVideos() } @@ -713,14 +711,14 @@ func (s *Sync) enqueueYoutubeVideos() error { playlistMap[item.Snippet.ResourceId.VideoId] = item.Snippet videoIDs[i] = item.Snippet.ResourceId.VideoId } - req2 := service.Videos.List("snippet,contentDetails").Id(strings.Join(videoIDs[:], ",")) + req2 := service.Videos.List("snippet,contentDetails,recordingDetails").Id(strings.Join(videoIDs[:], ",")) videosListResponse, err := req2.Do() if err != nil { return errors.Prefix("error getting videos info", err) } for _, item := range videosListResponse.Items { - videos = append(videos, sources.NewYoutubeVideo(s.videoDirectory, item, playlistMap[item.Id].Position)) + videos = append(videos, sources.NewYoutubeVideo(s.videoDirectory, item, playlistMap[item.Id].Position, s.Manager.GetS3AWSConfig())) } log.Infof("Got info for %d videos from youtube API", len(videos)) @@ -752,55 +750,6 @@ Enqueue: return nil } -func (s *Sync) enqueueUCBVideos() error { - var videos []video - - csvFile, err := os.Open("ucb.csv") - if err != nil { - return err - } - - reader := csv.NewReader(bufio.NewReader(csvFile)) - for { - line, err := reader.Read() - if err == io.EOF { - break - } else if err != nil { - return err - } - data := struct { - PublishedAt string `json:"publishedAt"` - }{} - err = json.Unmarshal([]byte(line[4]), &data) - if err != nil { - return err - } - - videos = append(videos, sources.NewUCBVideo(line[0], line[2], line[1], line[3], data.PublishedAt, s.videoDirectory)) - } - - log.Printf("Publishing %d videos\n", len(videos)) - - sort.Sort(byPublishedAt(videos)) - -Enqueue: - for _, v := range videos { - select { - case <-s.grp.Ch(): - break Enqueue - default: - } - - select { - case s.queue <- v: - case <-s.grp.Ch(): - break Enqueue - } - } - - return nil -} - func (s *Sync) processVideo(v video) (err error) { defer func() { if p := recover(); p != nil { @@ -851,8 +800,17 @@ func (s *Sync) processVideo(v video) (err error) { if err != nil { return err } + sp := sources.SyncParams{ + ClaimAddress: s.claimAddress, + Amount: publishAmount, + ChannelID: s.lbryChannelID, + MaxVideoSize: s.Manager.maxVideoSize, + Namer: s.namer, + MaxVideoLength: s.Manager.maxVideoLength, + } - summary, err := v.Sync(s.daemon, s.claimAddress, publishAmount, s.lbryChannelID, s.Manager.maxVideoSize, s.namer, s.Manager.maxVideoLength) + isUpgradeSync := s.Manager.syncStatus == StatusPendingUpgrade + summary, err := v.Sync(s.daemon, sp, &sv, isUpgradeSync) if err != nil { return err } diff --git a/sdk/api.go b/sdk/api.go index fd74908..4263124 100644 --- a/sdk/api.go +++ b/sdk/api.go @@ -76,10 +76,13 @@ func (a *APIConfig) FetchChannels(status string, cp *SyncProperties) ([]YoutubeC } type SyncedVideo struct { - VideoID string `json:"video_id"` - Published bool `json:"published"` - FailureReason string `json:"failure_reason"` - ClaimName string `json:"claim_name"` + VideoID string `json:"video_id"` + Published bool `json:"published"` + FailureReason string `json:"failure_reason"` + ClaimName string `json:"claim_name"` + ClaimID string `json:"claim_id"` + Size int64 `json:"size"` + MetadataVersion int8 `json:"metadata_version"` } func sanitizeFailureReason(s *string) { @@ -121,7 +124,9 @@ func (a *APIConfig) SetChannelStatus(channelID string, status string, failureRea claimNames := make(map[string]bool) for _, v := range response.Data { svs[v.VideoID] = v - claimNames[v.ClaimName] = v.Published + if v.ClaimName != "" { + claimNames[v.ClaimName] = v.Published + } } return svs, claimNames, nil } diff --git a/sources/ucbVideo.go b/sources/ucbVideo.go deleted file mode 100644 index 3e771d2..0000000 --- a/sources/ucbVideo.go +++ /dev/null @@ -1,219 +0,0 @@ -package sources - -import ( - "net/http" - "os" - "regexp" - "strconv" - "strings" - "sync" - "time" - - "github.com/aws/aws-sdk-go/aws/credentials" - "github.com/aws/aws-sdk-go/service/s3" - "github.com/aws/aws-sdk-go/service/s3/s3manager" - "github.com/lbryio/lbry.go/extras/errors" - "github.com/lbryio/lbry.go/extras/jsonrpc" - "github.com/lbryio/ytsync/namer" - - "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/aws/session" - log "github.com/sirupsen/logrus" -) - -type ucbVideo struct { - id string - title string - channel string - description string - publishedAt time.Time - dir string - claimNames map[string]bool - syncedVideosMux *sync.RWMutex -} - -func NewUCBVideo(id, title, channel, description, publishedAt, dir string) *ucbVideo { - p, _ := time.Parse(time.RFC3339Nano, publishedAt) // ignore parse errors - return &ucbVideo{ - id: id, - title: title, - description: description, - channel: channel, - dir: dir, - publishedAt: p, - } -} - -func (v *ucbVideo) ID() string { - return v.id -} - -func (v *ucbVideo) PlaylistPosition() int { - return 0 -} - -func (v *ucbVideo) IDAndNum() string { - return v.ID() + " (?)" -} - -func (v *ucbVideo) PublishedAt() time.Time { - return v.publishedAt - //r := regexp.MustCompile(`(\d\d\d\d)-(\d\d)-(\d\d)`) - //matches := r.FindStringSubmatch(v.title) - //if len(matches) > 0 { - // year, _ := strconv.Atoi(matches[1]) - // month, _ := strconv.Atoi(matches[2]) - // day, _ := strconv.Atoi(matches[3]) - // return time.Date(year, time.Month(month), day, 0, 0, 0, 0, time.UTC) - //} - //return time.Now() -} - -func (v *ucbVideo) getFilename() string { - return v.dir + "/" + v.id + ".mp4" -} - -func (v *ucbVideo) getClaimName(attempt int) string { - reg := regexp.MustCompile(`[^a-zA-Z0-9]+`) - suffix := "" - if attempt > 1 { - suffix = "-" + strconv.Itoa(attempt) - } - maxLen := 40 - len(suffix) - - chunks := strings.Split(strings.ToLower(strings.Trim(reg.ReplaceAllString(v.title, "-"), "-")), "-") - - name := chunks[0] - if len(name) > maxLen { - return name[:maxLen] - } - - for _, chunk := range chunks[1:] { - tmpName := name + "-" + chunk - if len(tmpName) > maxLen { - if len(name) < 20 { - name = tmpName[:maxLen] - } - break - } - name = tmpName - } - - return name + suffix -} - -func (v *ucbVideo) getAbbrevDescription() string { - maxLines := 10 - description := strings.TrimSpace(v.description) - if strings.Count(description, "\n") < maxLines { - return description - } - return strings.Join(strings.Split(description, "\n")[:maxLines], "\n") + "\n..." -} - -func (v *ucbVideo) download() error { - videoPath := v.getFilename() - - _, err := os.Stat(videoPath) - if err != nil && !os.IsNotExist(err) { - return err - } else if err == nil { - log.Debugln(v.id + " already exists at " + videoPath) - return nil - } - - creds := credentials.NewStaticCredentials("ID-GOES-HERE", "SECRET-GOES-HERE", "") - s, err := session.NewSession(&aws.Config{Region: aws.String("us-east-2"), Credentials: creds}) - if err != nil { - return err - } - downloader := s3manager.NewDownloader(s) - - out, err := os.Create(videoPath) - if err != nil { - return err - } - defer out.Close() - - log.Println("lbry-niko2/videos/" + v.channel + "/" + v.id) - - bytesWritten, err := downloader.Download(out, &s3.GetObjectInput{ - Bucket: aws.String("lbry-niko2"), - Key: aws.String("/videos/" + v.channel + "/" + v.id + ".mp4"), - }) - if err != nil { - return err - } else if bytesWritten == 0 { - return errors.Err("zero bytes written") - } - - return nil -} - -func (v *ucbVideo) saveThumbnail() error { - resp, err := http.Get("https://s3.us-east-2.amazonaws.com/lbry-niko2/thumbnails/" + v.id) - if err != nil { - return err - } - defer resp.Body.Close() - - creds := credentials.NewStaticCredentials("ID-GOES-HERE", "SECRET-GOES-HERE", "") - s, err := session.NewSession(&aws.Config{Region: aws.String("us-east-2"), Credentials: creds}) - if err != nil { - return err - } - uploader := s3manager.NewUploader(s) - - _, err = uploader.Upload(&s3manager.UploadInput{ - Bucket: aws.String("berk.ninja"), - Key: aws.String("thumbnails/" + v.id), - ContentType: aws.String("image/jpeg"), - Body: resp.Body, - }) - - return err -} - -func (v *ucbVideo) publish(daemon *jsonrpc.Client, claimAddress string, amount float64, channelID string, namer *namer.Namer) (*SyncSummary, error) { - options := jsonrpc.StreamCreateOptions{ - ClaimCreateOptions: jsonrpc.ClaimCreateOptions{ - Title: v.title, - Description: v.getAbbrevDescription(), - ClaimAddress: &claimAddress, - Languages: []string{"en"}, - ThumbnailURL: strPtr("https://berk.ninja/thumbnails/" + v.id), - Tags: []string{}, - }, - Author: strPtr("UC Berkeley"), - License: strPtr("see description"), - StreamType: &jsonrpc.StreamTypeVideo, - ChannelID: &channelID, - } - return publishAndRetryExistingNames(daemon, v.title, v.getFilename(), amount, options, namer) -} - -func (v *ucbVideo) Size() *int64 { - return nil -} - -func (v *ucbVideo) Sync(daemon *jsonrpc.Client, claimAddress string, amount float64, channelID string, maxVideoSize int, namer *namer.Namer, maxVideoLength float64) (*SyncSummary, error) { - //download and thumbnail can be done in parallel - err := v.download() - if err != nil { - return nil, errors.Prefix("download error", err) - } - log.Debugln("Downloaded " + v.id) - - //err = v.SaveThumbnail() - //if err != nil { - // return errors.WrapPrefix(err, "thumbnail error", 0) - //} - //log.Debugln("Created thumbnail for " + v.id) - - summary, err := v.publish(daemon, claimAddress, amount, channelID, namer) - if err != nil { - return nil, errors.Prefix("publish error", err) - } - - return summary, nil -} diff --git a/sources/youtubeVideo.go b/sources/youtubeVideo.go index e2a8781..51628dc 100644 --- a/sources/youtubeVideo.go +++ b/sources/youtubeVideo.go @@ -3,6 +3,7 @@ package sources import ( "bytes" "encoding/json" + "fmt" "io/ioutil" "math" "net/http" @@ -12,12 +13,16 @@ import ( "strings" "time" + "github.com/lbryio/ytsync/sdk" + "github.com/lbryio/ytsync/thumbs" + "github.com/lbryio/lbry.go/extras/errors" "github.com/lbryio/lbry.go/extras/jsonrpc" "github.com/lbryio/lbry.go/extras/util" "github.com/lbryio/ytsync/namer" "github.com/ChannelMeter/iso8601duration" + "github.com/aws/aws-sdk-go/aws" "github.com/nikooo777/ytdl" log "github.com/sirupsen/logrus" "google.golang.org/api/youtube/v3" @@ -36,9 +41,45 @@ type YoutubeVideo struct { dir string youtubeInfo *youtube.Video tags []string + awsConfig aws.Config } -func NewYoutubeVideo(directory string, videoData *youtube.Video, playlistPosition int64) *YoutubeVideo { +var youtubeCategories = map[string]string{ + "1": "Film & Animation", + "2": "Autos & Vehicles", + "10": "Music", + "15": "Pets & Animals", + "17": "Sports", + "18": "Short Movies", + "19": "Travel & Events", + "20": "Gaming", + "21": "Videoblogging", + "22": "People & Blogs", + "23": "Comedy", + "24": "Entertainment", + "25": "News & Politics", + "26": "Howto & Style", + "27": "Education", + "28": "Science & Technology", + "29": "Nonprofits & Activism", + "30": "Movies", + "31": "Anime/Animation", + "32": "Action/Adventure", + "33": "Classics", + "34": "Comedy", + "35": "Documentary", + "36": "Drama", + "37": "Family", + "38": "Foreign", + "39": "Horror", + "40": "Sci-Fi/Fantasy", + "41": "Thriller", + "42": "Shorts", + "43": "Shows", + "44": "Trailers", +} + +func NewYoutubeVideo(directory string, videoData *youtube.Video, playlistPosition int64, awsConfig aws.Config) *YoutubeVideo { publishedAt, _ := time.Parse(time.RFC3339Nano, videoData.Snippet.PublishedAt) // ignore parse errors return &YoutubeVideo{ id: videoData.Id, @@ -49,6 +90,7 @@ func NewYoutubeVideo(directory string, videoData *youtube.Video, playlistPositio publishedAt: publishedAt, dir: directory, youtubeInfo: videoData, + awsConfig: awsConfig, } } @@ -267,7 +309,7 @@ func (v *YoutubeVideo) publish(daemon *jsonrpc.Client, claimAddress string, amou Description: v.getAbbrevDescription() + additionalDescription, ClaimAddress: &claimAddress, Languages: languages, - ThumbnailURL: strPtr("https://thumbnails.lbry.com/" + v.id), + ThumbnailURL: strPtr(thumbs.ThumbnailEndpoint + v.id), Tags: v.youtubeInfo.Snippet.Tags, }, Author: strPtr(v.channelTitle), @@ -285,9 +327,24 @@ func (v *YoutubeVideo) Size() *int64 { return v.size } -func (v *YoutubeVideo) Sync(daemon *jsonrpc.Client, claimAddress string, amount float64, channelID string, maxVideoSize int, namer *namer.Namer, maxVideoLength float64) (*SyncSummary, error) { - v.maxVideoSize = int64(maxVideoSize) * 1024 * 1024 - v.maxVideoLength = maxVideoLength +type SyncParams struct { + ClaimAddress string + Amount float64 + ChannelID string + MaxVideoSize int + Namer *namer.Namer + MaxVideoLength float64 +} + +func (v *YoutubeVideo) Sync(daemon *jsonrpc.Client, params SyncParams, existingVideoData *sdk.SyncedVideo, reprocess bool) (*SyncSummary, error) { + v.maxVideoSize = int64(params.MaxVideoSize) * 1024 * 1024 + v.maxVideoLength = params.MaxVideoLength + + if reprocess { + summary, err := v.reprocess(daemon, params.ChannelID, existingVideoData) + + return summary, err + } //download and thumbnail can be done in parallel err := v.download() if err != nil { @@ -301,9 +358,66 @@ func (v *YoutubeVideo) Sync(daemon *jsonrpc.Client, claimAddress string, amount } log.Debugln("Created thumbnail for " + v.id) - summary, err := v.publish(daemon, claimAddress, amount, channelID, namer) + summary, err := v.publish(daemon, params.ClaimAddress, params.Amount, params.ChannelID, params.Namer) //delete the video in all cases (and ignore the error) _ = v.delete() return summary, errors.Prefix("publish error", err) } + +func (v *YoutubeVideo) reprocess(daemon *jsonrpc.Client, channelID string, existingVideoData *sdk.SyncedVideo) (*SyncSummary, error) { + c, err := daemon.ClaimSearch(nil, &existingVideoData.ClaimID, nil, nil) + if err != nil { + return nil, errors.Err(err) + } + if len(c.Claims) == 0 { + return nil, errors.Err("cannot reprocess: no claim found for this video") + } else if len(c.Claims) > 1 { + return nil, errors.Err("cannot reprocess: too many claims. claimID: %s", existingVideoData.ClaimID) + } + + currentClaim := c.Claims[0] + var languages []string = nil + if v.youtubeInfo.Snippet.DefaultLanguage != "" { + languages = []string{v.youtubeInfo.Snippet.DefaultLanguage} + } + + var locations []jsonrpc.Location = nil + if v.youtubeInfo.RecordingDetails.Location != nil { + locations = []jsonrpc.Location{{ + Latitude: util.PtrToString(fmt.Sprintf("%.7f", v.youtubeInfo.RecordingDetails.Location.Latitude)), + Longitude: util.PtrToString(fmt.Sprintf("%.7f", v.youtubeInfo.RecordingDetails.Location.Longitude)), + }} + } + + thumbnailURL := "" + if currentClaim.Value.GetThumbnail() == nil { + thumbnail := thumbs.GetBestThumbnail(v.youtubeInfo.Snippet.Thumbnails) + thumbnailURL, err = thumbs.MirrorThumbnail(thumbnail.Url, v.ID(), v.awsConfig) + } else { + thumbnailURL = thumbs.ThumbnailEndpoint + v.ID() + } + + tags := append(v.youtubeInfo.Snippet.Tags, youtubeCategories[v.youtubeInfo.Snippet.CategoryId]) + + videoDuration, err := duration.FromString(v.youtubeInfo.ContentDetails.Duration) + _, err = daemon.StreamUpdate(existingVideoData.ClaimID, jsonrpc.StreamUpdateOptions{ + StreamCreateOptions: &jsonrpc.StreamCreateOptions{ + ClaimCreateOptions: jsonrpc.ClaimCreateOptions{ + Title: v.title, + Description: v.getAbbrevDescription(), + Tags: tags, + Languages: languages, + Locations: locations, + ThumbnailURL: &thumbnailURL, + }, + Author: util.PtrToString(""), + License: strPtr("Copyrighted (contact author)"), + ReleaseTime: util.PtrToInt64(v.publishedAt.Unix()), + VideoDuration: util.PtrToUint64(uint64(math.Ceil(videoDuration.ToDuration().Seconds()))), + + ChannelID: &channelID, + }, + }) + return &SyncSummary{}, nil +} diff --git a/thumbs/uploader.go b/thumbs/uploader.go index 619d98f..89580f7 100644 --- a/thumbs/uploader.go +++ b/thumbs/uploader.go @@ -1,6 +1,7 @@ package thumbs import ( + "google.golang.org/api/youtube/v3" "io" "net/http" "os" @@ -21,6 +22,7 @@ type thumbnailUploader struct { } const thumbnailPath = "/tmp/ytsync_thumbnails/" +const ThumbnailEndpoint = "https://thumbnails.lbry.com/" func (u *thumbnailUploader) downloadThumbnail() error { _ = os.Mkdir(thumbnailPath, 0750) @@ -63,7 +65,7 @@ func (u *thumbnailUploader) uploadThumbnail() error { Key: key, Body: thumb, }) - u.mirroredUrl = "https://thumbnails.lbry.com/" + u.name + u.mirroredUrl = ThumbnailEndpoint + u.name return errors.Err(err) } @@ -92,3 +94,16 @@ func MirrorThumbnail(url string, name string, s3Config aws.Config) (string, erro return tu.mirroredUrl, nil } + +func GetBestThumbnail(thumbnails *youtube.ThumbnailDetails) *youtube.Thumbnail { + if thumbnails.Maxres != nil { + return thumbnails.Maxres + } else if thumbnails.High != nil { + return thumbnails.High + } else if thumbnails.Medium != nil { + return thumbnails.Medium + } else if thumbnails.Standard != nil { + return thumbnails.Standard + } + return thumbnails.Default +}