From 3389a8be933224dce2d708c68a13eb75624faa58 Mon Sep 17 00:00:00 2001 From: Niko Storni Date: Sat, 1 Jun 2019 01:46:16 +0200 Subject: [PATCH] fix many bugs calculate size for claims missing in the database fix reprocessing for videos no longer on youtube pass metadata instead of hardcoding it add video height when updating streams add tags to channels add test for tags --- manager/setup.go | 7 +++-- manager/ytsync.go | 15 ++++++----- sdk/api.go | 5 ++-- sources/youtubeVideo.go | 15 ++++++----- tagsManager/tags_mapping.go | 15 ++++++++--- tagsManager/tags_mapping_test.go | 46 ++++++++++++++++++++++++++++++++ 6 files changed, 83 insertions(+), 20 deletions(-) create mode 100644 tagsManager/tags_mapping_test.go diff --git a/manager/setup.go b/manager/setup.go index 546e0af..4f4adc8 100644 --- a/manager/setup.go +++ b/manager/setup.go @@ -11,6 +11,8 @@ import ( "github.com/lbryio/lbry.go/extras/jsonrpc" "github.com/lbryio/lbry.go/extras/util" "github.com/lbryio/lbry.go/lbrycrd" + + "github.com/lbryio/ytsync/tagsManager" "github.com/lbryio/ytsync/thumbs" "github.com/shopspring/decimal" @@ -355,13 +357,14 @@ func (s *Sync) ensureChannelOwnership() error { var c *jsonrpc.TransactionSummary if channelUsesOldMetadata { c, err = s.daemon.ChannelUpdate(s.lbryChannelID, jsonrpc.ChannelUpdateOptions{ + ClearTags: util.PtrToBool(true), ClearLocations: util.PtrToBool(true), ClearLanguages: util.PtrToBool(true), ChannelCreateOptions: jsonrpc.ChannelCreateOptions{ ClaimCreateOptions: jsonrpc.ClaimCreateOptions{ Title: channelInfo.Title, Description: channelInfo.Description, - Tags: nil, + Tags: tagsManager.GetTagsForChannel(s.YoutubeChannelID), Languages: languages, Locations: locations, ThumbnailURL: &thumbnailURL, @@ -374,7 +377,7 @@ func (s *Sync) ensureChannelOwnership() error { ClaimCreateOptions: jsonrpc.ClaimCreateOptions{ Title: channelInfo.Title, Description: channelInfo.Description, - Tags: nil, + Tags: tagsManager.GetTagsForChannel(s.YoutubeChannelID), Languages: languages, Locations: locations, ThumbnailURL: &thumbnailURL, diff --git a/manager/ytsync.go b/manager/ytsync.go index 88c197a..fa88307 100644 --- a/manager/ytsync.go +++ b/manager/ytsync.go @@ -459,8 +459,12 @@ func (s *Sync) updateRemoteDB(claims []jsonrpc.Claim) (total int, fixed int, err if !ok || pv.ClaimName != c.Name { fixed++ log.Debugf("adding %s to the database", c.Name) - - err = s.Manager.apiConfig.MarkVideoStatus(s.YoutubeChannelID, videoID, VideoStatusPublished, c.ClaimID, c.Name, "", nil) + size, err := c.GetStreamSizeByMagic() + if err != nil { + size = 0 + } + metadataVersion := uint(1) + err = s.Manager.apiConfig.MarkVideoStatus(s.YoutubeChannelID, videoID, VideoStatusPublished, c.ClaimID, c.Name, "", util.PtrToInt64(int64(size)), metadataVersion) if err != nil { return count, fixed, err } @@ -663,7 +667,7 @@ func (s *Sync) startWorker(workerNum int) { } s.AppendSyncedVideo(v.ID(), false, err.Error(), "") - err = s.Manager.apiConfig.MarkVideoStatus(s.YoutubeChannelID, v.ID(), VideoStatusFailed, existingClaimID, existingClaimName, err.Error(), existingClaimSize) + err = s.Manager.apiConfig.MarkVideoStatus(s.YoutubeChannelID, v.ID(), VideoStatusFailed, existingClaimID, existingClaimName, err.Error(), existingClaimSize, 1) if err != nil { SendErrorToSlack("Failed to mark video on the database: %s", err.Error()) } @@ -749,14 +753,13 @@ func (s *Sync) enqueueYoutubeVideos() error { break } } - notOnYoutube := make([]video, 0, len(s.syncedVideos)) for k, v := range s.syncedVideos { if !v.Published { continue } _, ok := playlistMap[k] if !ok { - notOnYoutube = append(notOnYoutube, sources.NewMockedVideo(s.videoDirectory, k, s.YoutubeChannelID, s.Manager.GetS3AWSConfig())) + videos = append(videos, sources.NewMockedVideo(s.videoDirectory, k, s.YoutubeChannelID, s.Manager.GetS3AWSConfig())) } } @@ -851,7 +854,7 @@ func (s *Sync) processVideo(v video) (err error) { return err } - err = s.Manager.apiConfig.MarkVideoStatus(s.YoutubeChannelID, v.ID(), VideoStatusPublished, summary.ClaimID, summary.ClaimName, "", v.Size()) + err = s.Manager.apiConfig.MarkVideoStatus(s.YoutubeChannelID, v.ID(), VideoStatusPublished, summary.ClaimID, summary.ClaimName, "", v.Size(), 2) if err != nil { SendErrorToSlack("Failed to mark video on the database: %s", err.Error()) } diff --git a/sdk/api.go b/sdk/api.go index 44d2b72..603a1f9 100644 --- a/sdk/api.go +++ b/sdk/api.go @@ -2,6 +2,7 @@ package sdk import ( "encoding/json" + "fmt" "io/ioutil" "net/http" "net/url" @@ -166,7 +167,7 @@ const ( VideoStatusFailed = "failed" ) -func (a *APIConfig) MarkVideoStatus(channelID string, videoID string, status string, claimID string, claimName string, failureReason string, size *int64) error { +func (a *APIConfig) MarkVideoStatus(channelID string, videoID string, status string, claimID string, claimName string, failureReason string, size *int64, metadataVersion uint) error { endpoint := a.ApiURL + "/yt/video_status" sanitizeFailureReason(&failureReason) @@ -183,7 +184,7 @@ func (a *APIConfig) MarkVideoStatus(channelID string, videoID string, status str vals.Add("published_at", strconv.FormatInt(time.Now().Unix(), 10)) vals.Add("claim_id", claimID) vals.Add("claim_name", claimName) - vals.Add("metadata_version", "2") + vals.Add("metadata_version", fmt.Sprintf("%d", metadataVersion)) if size != nil { vals.Add("size", strconv.FormatInt(*size, 10)) } diff --git a/sources/youtubeVideo.go b/sources/youtubeVideo.go index 26ffeb2..5be0a5d 100644 --- a/sources/youtubeVideo.go +++ b/sources/youtubeVideo.go @@ -453,11 +453,6 @@ func (v *YoutubeVideo) reprocess(daemon *jsonrpc.Client, params SyncParams, exis } } - videoDuration, err := duration.FromString(v.youtubeInfo.ContentDetails.Duration) - if err != nil { - return nil, errors.Err(err) - } - if v.mocked { pr, err := daemon.StreamUpdate(existingVideoData.ClaimID, jsonrpc.StreamUpdateOptions{ StreamCreateOptions: &jsonrpc.StreamCreateOptions{ @@ -466,7 +461,7 @@ func (v *YoutubeVideo) reprocess(daemon *jsonrpc.Client, params SyncParams, exis ThumbnailURL: &thumbnailURL, }, Author: util.PtrToString(""), - License: util.PtrToString("Copyrighted (contact author)"), + License: util.PtrToString("Copyrighted (contact publisher)"), ChannelID: &v.lbryChannelID, }, FileSize: &videoSize, @@ -480,6 +475,12 @@ func (v *YoutubeVideo) reprocess(daemon *jsonrpc.Client, params SyncParams, exis ClaimName: pr.Outputs[0].Name, }, nil } + + videoDuration, err := duration.FromString(v.youtubeInfo.ContentDetails.Duration) + if err != nil { + return nil, errors.Err(err) + } + pr, err := daemon.StreamUpdate(existingVideoData.ClaimID, jsonrpc.StreamUpdateOptions{ ClearLanguages: util.PtrToBool(true), ClearLocations: util.PtrToBool(true), @@ -495,6 +496,8 @@ func (v *YoutubeVideo) reprocess(daemon *jsonrpc.Client, params SyncParams, exis }, Author: util.PtrToString(""), License: util.PtrToString("Copyrighted (contact publisher)"), + VideoHeight: util.PtrToUint(720), + VideoWidth: util.PtrToUint(1280), ReleaseTime: util.PtrToInt64(v.publishedAt.Unix()), Duration: util.PtrToUint64(uint64(math.Ceil(videoDuration.ToDuration().Seconds()))), ChannelID: &v.lbryChannelID, diff --git a/tagsManager/tags_mapping.go b/tagsManager/tags_mapping.go index 871dd4a..79f45d5 100644 --- a/tagsManager/tags_mapping.go +++ b/tagsManager/tags_mapping.go @@ -24,6 +24,11 @@ const ( Weapons = "weapons" ) +func GetTagsForChannel(channelID string) []string { + tags, _ := channelWideTags[channelID] + return tags +} + func SanitizeTags(tags []string, youtubeChannelID string) ([]string, error) { unsanitized := make([]string, 0, len(tags)) for _, t := range tags { @@ -54,7 +59,7 @@ func SanitizeTags(tags []string, youtubeChannelID string) ([]string, error) { originalTags = append(originalTags, t) } } - sanitizedTags := make([]string, len(originalTags)+len(curatedTags)) + sanitizedTags := make([]string, 0, len(originalTags)+len(curatedTags)) sanitizedTags = append(sanitizedTags, curatedTags...) sanitizedTags = append(sanitizedTags, originalTags...) return sanitizedTags, nil @@ -117,17 +122,19 @@ func (ts *tagsSanitizer) add() { extraTags, ok := channelWideTags[ts.ChannelID] if ok { for _, t := range extraTags { - ts.Sanitized[t] = false + ts.Sanitized[t] = true } } } const ( - Lunduke = "UCkK9UDm_ZNrq_rIXCz3xCGA" + Lunduke = "UCkK9UDm_ZNrq_rIXCz3xCGA" + SwissExperiments = "UCNQfQvFMPnInwsU_iGYArJQ" ) var channelWideTags = map[string][]string{ - Lunduke: {"linux", "technology"}, + Lunduke: {"linux", "technology"}, + SwissExperiments: {"science & technology", "experiments", "switzerland"}, } var tagsToSkip = map[string]*struct{}{ "#hangoutsonair": nil, diff --git a/tagsManager/tags_mapping_test.go b/tagsManager/tags_mapping_test.go new file mode 100644 index 0000000..156281f --- /dev/null +++ b/tagsManager/tags_mapping_test.go @@ -0,0 +1,46 @@ +package tagsManager + +import ( + "testing" +) + +func TestSanitizeTags(t *testing.T) { + got, err := SanitizeTags([]string{"this", "super", "expensive", "test", "has", "a lot of", "crypto", "currency", "in it", "trump", "will build the", "wall"}, "UCNQfQvFMPnInwsU_iGYArJQ") + if err != nil { + t.Error(err) + return + } + expectedTags := []string{ + "blockchain", + "switzerland", + "news", + "science & technology", + "economics", + "experiments", + "this", + "in it", + "will build the", + "has", + "crypto", + "trump", + "wall", + "expensive", + "currency", + "a lot of", + } + if len(expectedTags) != len(got) { + t.Error("number of tags differ") + return + } +outer: + for _, et := range expectedTags { + for _, t := range got { + if et == t { + continue outer + } + } + t.Error("tag not found") + return + } + +}