diff --git a/manager/setup.go b/manager/setup.go index 546e0af..4f4adc8 100644 --- a/manager/setup.go +++ b/manager/setup.go @@ -11,6 +11,8 @@ import ( "github.com/lbryio/lbry.go/extras/jsonrpc" "github.com/lbryio/lbry.go/extras/util" "github.com/lbryio/lbry.go/lbrycrd" + + "github.com/lbryio/ytsync/tagsManager" "github.com/lbryio/ytsync/thumbs" "github.com/shopspring/decimal" @@ -355,13 +357,14 @@ func (s *Sync) ensureChannelOwnership() error { var c *jsonrpc.TransactionSummary if channelUsesOldMetadata { c, err = s.daemon.ChannelUpdate(s.lbryChannelID, jsonrpc.ChannelUpdateOptions{ + ClearTags: util.PtrToBool(true), ClearLocations: util.PtrToBool(true), ClearLanguages: util.PtrToBool(true), ChannelCreateOptions: jsonrpc.ChannelCreateOptions{ ClaimCreateOptions: jsonrpc.ClaimCreateOptions{ Title: channelInfo.Title, Description: channelInfo.Description, - Tags: nil, + Tags: tagsManager.GetTagsForChannel(s.YoutubeChannelID), Languages: languages, Locations: locations, ThumbnailURL: &thumbnailURL, @@ -374,7 +377,7 @@ func (s *Sync) ensureChannelOwnership() error { ClaimCreateOptions: jsonrpc.ClaimCreateOptions{ Title: channelInfo.Title, Description: channelInfo.Description, - Tags: nil, + Tags: tagsManager.GetTagsForChannel(s.YoutubeChannelID), Languages: languages, Locations: locations, ThumbnailURL: &thumbnailURL, diff --git a/manager/ytsync.go b/manager/ytsync.go index 88c197a..fa88307 100644 --- a/manager/ytsync.go +++ b/manager/ytsync.go @@ -459,8 +459,12 @@ func (s *Sync) updateRemoteDB(claims []jsonrpc.Claim) (total int, fixed int, err if !ok || pv.ClaimName != c.Name { fixed++ log.Debugf("adding %s to the database", c.Name) - - err = s.Manager.apiConfig.MarkVideoStatus(s.YoutubeChannelID, videoID, VideoStatusPublished, c.ClaimID, c.Name, "", nil) + size, err := c.GetStreamSizeByMagic() + if err != nil { + size = 0 + } + metadataVersion := uint(1) + err = s.Manager.apiConfig.MarkVideoStatus(s.YoutubeChannelID, videoID, VideoStatusPublished, c.ClaimID, c.Name, "", util.PtrToInt64(int64(size)), metadataVersion) if err != nil { return count, fixed, err } @@ -663,7 +667,7 @@ func (s *Sync) startWorker(workerNum int) { } s.AppendSyncedVideo(v.ID(), false, err.Error(), "") - err = s.Manager.apiConfig.MarkVideoStatus(s.YoutubeChannelID, v.ID(), VideoStatusFailed, existingClaimID, existingClaimName, err.Error(), existingClaimSize) + err = s.Manager.apiConfig.MarkVideoStatus(s.YoutubeChannelID, v.ID(), VideoStatusFailed, existingClaimID, existingClaimName, err.Error(), existingClaimSize, 1) if err != nil { SendErrorToSlack("Failed to mark video on the database: %s", err.Error()) } @@ -749,14 +753,13 @@ func (s *Sync) enqueueYoutubeVideos() error { break } } - notOnYoutube := make([]video, 0, len(s.syncedVideos)) for k, v := range s.syncedVideos { if !v.Published { continue } _, ok := playlistMap[k] if !ok { - notOnYoutube = append(notOnYoutube, sources.NewMockedVideo(s.videoDirectory, k, s.YoutubeChannelID, s.Manager.GetS3AWSConfig())) + videos = append(videos, sources.NewMockedVideo(s.videoDirectory, k, s.YoutubeChannelID, s.Manager.GetS3AWSConfig())) } } @@ -851,7 +854,7 @@ func (s *Sync) processVideo(v video) (err error) { return err } - err = s.Manager.apiConfig.MarkVideoStatus(s.YoutubeChannelID, v.ID(), VideoStatusPublished, summary.ClaimID, summary.ClaimName, "", v.Size()) + err = s.Manager.apiConfig.MarkVideoStatus(s.YoutubeChannelID, v.ID(), VideoStatusPublished, summary.ClaimID, summary.ClaimName, "", v.Size(), 2) if err != nil { SendErrorToSlack("Failed to mark video on the database: %s", err.Error()) } diff --git a/sdk/api.go b/sdk/api.go index 44d2b72..603a1f9 100644 --- a/sdk/api.go +++ b/sdk/api.go @@ -2,6 +2,7 @@ package sdk import ( "encoding/json" + "fmt" "io/ioutil" "net/http" "net/url" @@ -166,7 +167,7 @@ const ( VideoStatusFailed = "failed" ) -func (a *APIConfig) MarkVideoStatus(channelID string, videoID string, status string, claimID string, claimName string, failureReason string, size *int64) error { +func (a *APIConfig) MarkVideoStatus(channelID string, videoID string, status string, claimID string, claimName string, failureReason string, size *int64, metadataVersion uint) error { endpoint := a.ApiURL + "/yt/video_status" sanitizeFailureReason(&failureReason) @@ -183,7 +184,7 @@ func (a *APIConfig) MarkVideoStatus(channelID string, videoID string, status str vals.Add("published_at", strconv.FormatInt(time.Now().Unix(), 10)) vals.Add("claim_id", claimID) vals.Add("claim_name", claimName) - vals.Add("metadata_version", "2") + vals.Add("metadata_version", fmt.Sprintf("%d", metadataVersion)) if size != nil { vals.Add("size", strconv.FormatInt(*size, 10)) } diff --git a/sources/youtubeVideo.go b/sources/youtubeVideo.go index 26ffeb2..5be0a5d 100644 --- a/sources/youtubeVideo.go +++ b/sources/youtubeVideo.go @@ -453,11 +453,6 @@ func (v *YoutubeVideo) reprocess(daemon *jsonrpc.Client, params SyncParams, exis } } - videoDuration, err := duration.FromString(v.youtubeInfo.ContentDetails.Duration) - if err != nil { - return nil, errors.Err(err) - } - if v.mocked { pr, err := daemon.StreamUpdate(existingVideoData.ClaimID, jsonrpc.StreamUpdateOptions{ StreamCreateOptions: &jsonrpc.StreamCreateOptions{ @@ -466,7 +461,7 @@ func (v *YoutubeVideo) reprocess(daemon *jsonrpc.Client, params SyncParams, exis ThumbnailURL: &thumbnailURL, }, Author: util.PtrToString(""), - License: util.PtrToString("Copyrighted (contact author)"), + License: util.PtrToString("Copyrighted (contact publisher)"), ChannelID: &v.lbryChannelID, }, FileSize: &videoSize, @@ -480,6 +475,12 @@ func (v *YoutubeVideo) reprocess(daemon *jsonrpc.Client, params SyncParams, exis ClaimName: pr.Outputs[0].Name, }, nil } + + videoDuration, err := duration.FromString(v.youtubeInfo.ContentDetails.Duration) + if err != nil { + return nil, errors.Err(err) + } + pr, err := daemon.StreamUpdate(existingVideoData.ClaimID, jsonrpc.StreamUpdateOptions{ ClearLanguages: util.PtrToBool(true), ClearLocations: util.PtrToBool(true), @@ -495,6 +496,8 @@ func (v *YoutubeVideo) reprocess(daemon *jsonrpc.Client, params SyncParams, exis }, Author: util.PtrToString(""), License: util.PtrToString("Copyrighted (contact publisher)"), + VideoHeight: util.PtrToUint(720), + VideoWidth: util.PtrToUint(1280), ReleaseTime: util.PtrToInt64(v.publishedAt.Unix()), Duration: util.PtrToUint64(uint64(math.Ceil(videoDuration.ToDuration().Seconds()))), ChannelID: &v.lbryChannelID, diff --git a/tagsManager/tags_mapping.go b/tagsManager/tags_mapping.go index 871dd4a..79f45d5 100644 --- a/tagsManager/tags_mapping.go +++ b/tagsManager/tags_mapping.go @@ -24,6 +24,11 @@ const ( Weapons = "weapons" ) +func GetTagsForChannel(channelID string) []string { + tags, _ := channelWideTags[channelID] + return tags +} + func SanitizeTags(tags []string, youtubeChannelID string) ([]string, error) { unsanitized := make([]string, 0, len(tags)) for _, t := range tags { @@ -54,7 +59,7 @@ func SanitizeTags(tags []string, youtubeChannelID string) ([]string, error) { originalTags = append(originalTags, t) } } - sanitizedTags := make([]string, len(originalTags)+len(curatedTags)) + sanitizedTags := make([]string, 0, len(originalTags)+len(curatedTags)) sanitizedTags = append(sanitizedTags, curatedTags...) sanitizedTags = append(sanitizedTags, originalTags...) return sanitizedTags, nil @@ -117,17 +122,19 @@ func (ts *tagsSanitizer) add() { extraTags, ok := channelWideTags[ts.ChannelID] if ok { for _, t := range extraTags { - ts.Sanitized[t] = false + ts.Sanitized[t] = true } } } const ( - Lunduke = "UCkK9UDm_ZNrq_rIXCz3xCGA" + Lunduke = "UCkK9UDm_ZNrq_rIXCz3xCGA" + SwissExperiments = "UCNQfQvFMPnInwsU_iGYArJQ" ) var channelWideTags = map[string][]string{ - Lunduke: {"linux", "technology"}, + Lunduke: {"linux", "technology"}, + SwissExperiments: {"science & technology", "experiments", "switzerland"}, } var tagsToSkip = map[string]*struct{}{ "#hangoutsonair": nil, diff --git a/tagsManager/tags_mapping_test.go b/tagsManager/tags_mapping_test.go new file mode 100644 index 0000000..156281f --- /dev/null +++ b/tagsManager/tags_mapping_test.go @@ -0,0 +1,46 @@ +package tagsManager + +import ( + "testing" +) + +func TestSanitizeTags(t *testing.T) { + got, err := SanitizeTags([]string{"this", "super", "expensive", "test", "has", "a lot of", "crypto", "currency", "in it", "trump", "will build the", "wall"}, "UCNQfQvFMPnInwsU_iGYArJQ") + if err != nil { + t.Error(err) + return + } + expectedTags := []string{ + "blockchain", + "switzerland", + "news", + "science & technology", + "economics", + "experiments", + "this", + "in it", + "will build the", + "has", + "crypto", + "trump", + "wall", + "expensive", + "currency", + "a lot of", + } + if len(expectedTags) != len(got) { + t.Error("number of tags differ") + return + } +outer: + for _, et := range expectedTags { + for _, t := range got { + if et == t { + continue outer + } + } + t.Error("tag not found") + return + } + +}