From 76e653fb9be801bd665005a26791a5dcc2274f61 Mon Sep 17 00:00:00 2001 From: Niko Storni Date: Thu, 6 Jun 2019 02:16:07 +0200 Subject: [PATCH] add Fee support lowercase all tags refactor code implement and test regexps for tags --- go.sum | 2 + manager/manager.go | 2 + manager/ytsync.go | 24 ++--- sdk/api.go | 13 +-- sources/youtubeVideo.go | 158 +++++++++++++++++-------------- tagsManager/tags_mapping.go | 29 ++++-- tagsManager/tags_mapping_test.go | 98 +++++++++++++++++++ 7 files changed, 229 insertions(+), 97 deletions(-) diff --git a/go.sum b/go.sum index 6cb066f..49063ec 100644 --- a/go.sum +++ b/go.sum @@ -18,6 +18,7 @@ github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRF github.com/andybalholm/cascadia v1.0.0 h1:hOCXnnZ5A+3eVDX8pvgl4kofXv2ELss0bKcqRySc45o= github.com/andybalholm/cascadia v1.0.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y= github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= +github.com/asaskevich/govalidator v0.0.0-20180720115003-f9ffefc3facf h1:eg0MeVzsP1G42dRafH3vf+al2vQIJU0YHX+1Tw87oco= github.com/asaskevich/govalidator v0.0.0-20180720115003-f9ffefc3facf/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= github.com/aws/aws-sdk-go v1.17.3 h1:KBXxg7Jh0TxE5zmpNB2DwKmJeDUqh0O6jhy25TuYOmc= github.com/aws/aws-sdk-go v1.17.3/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= @@ -59,6 +60,7 @@ github.com/go-ini/ini v1.42.0 h1:TWr1wGj35+UiWHlBA8er89seFXxzwFn11spilrrj+38= github.com/go-ini/ini v1.42.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= +github.com/go-ozzo/ozzo-validation v3.5.0+incompatible h1:sUy/in/P6askYr16XJgTKq/0SZhiWsdg4WZGaLsGQkM= github.com/go-ozzo/ozzo-validation v3.5.0+incompatible/go.mod h1:gsEKFIVnabGBt6mXmxK0MoFy+cZoTJY6mu5Ll3LVLBU= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/gogo/googleapis v1.1.0/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFGgqEef3s= diff --git a/manager/manager.go b/manager/manager.go index ed8c3cf..ff533d8 100644 --- a/manager/manager.go +++ b/manager/manager.go @@ -133,6 +133,7 @@ func (s *SyncManager) Start() error { AwsS3Region: s.awsS3Region, AwsS3Bucket: s.awsS3Bucket, namer: namer.NewNamer(), + Fee: channels[0].Fee, } shouldInterruptLoop = true } else { @@ -167,6 +168,7 @@ func (s *SyncManager) Start() error { AwsS3Region: s.awsS3Region, AwsS3Bucket: s.awsS3Bucket, namer: namer.NewNamer(), + Fee: c.Fee, }) } } diff --git a/manager/ytsync.go b/manager/ytsync.go index 1fe04d7..aad209a 100644 --- a/manager/ytsync.go +++ b/manager/ytsync.go @@ -74,18 +74,17 @@ type Sync struct { AwsS3Secret string AwsS3Region string AwsS3Bucket string - - daemon *jsonrpc.Client - claimAddress string - videoDirectory string - syncedVideosMux *sync.RWMutex - syncedVideos map[string]sdk.SyncedVideo - grp *stop.Group - lbryChannelID string - namer *namer.Namer - - walletMux *sync.Mutex - queue chan video + Fee *sdk.Fee + daemon *jsonrpc.Client + claimAddress string + videoDirectory string + syncedVideosMux *sync.RWMutex + syncedVideos map[string]sdk.SyncedVideo + grp *stop.Group + lbryChannelID string + namer *namer.Namer + walletMux *sync.Mutex + queue chan video } func (s *Sync) AppendSyncedVideo(videoID string, published bool, failureReason string, claimName string) { @@ -876,6 +875,7 @@ func (s *Sync) processVideo(v video) (err error) { MaxVideoSize: s.Manager.maxVideoSize, Namer: s.namer, MaxVideoLength: s.Manager.maxVideoLength, + Fee: s.Fee, } summary, err := v.Sync(s.daemon, sp, &sv, isUpgradeSync) diff --git a/sdk/api.go b/sdk/api.go index 1c19974..b243d2f 100644 --- a/sdk/api.go +++ b/sdk/api.go @@ -35,16 +35,17 @@ type SyncProperties struct { YoutubeChannelID string } +type Fee struct { + Amount string `json:"amount"` + Address string `json:"address"` + Currency string `json:"currency"` +} type YoutubeChannel struct { ChannelId string `json:"channel_id"` TotalVideos uint `json:"total_videos"` DesiredChannelName string `json:"desired_channel_name"` - Fee *struct { - Amount string `json:"amount"` - Address string `json:"address"` - Currency string `json:"currency"` - } `json:"fee"` - ChannelClaimID string `json:"channel_claim_id"` + Fee *Fee `json:"fee"` + ChannelClaimID string `json:"channel_claim_id"` } func (a *APIConfig) FetchChannels(status string, cp *SyncProperties) ([]YoutubeChannel, error) { diff --git a/sources/youtubeVideo.go b/sources/youtubeVideo.go index bb259d6..5d61cc7 100644 --- a/sources/youtubeVideo.go +++ b/sources/youtubeVideo.go @@ -24,6 +24,7 @@ import ( "github.com/ChannelMeter/iso8601duration" "github.com/aws/aws-sdk-go/aws" "github.com/nikooo777/ytdl" + "github.com/shopspring/decimal" log "github.com/sirupsen/logrus" "google.golang.org/api/youtube/v3" ) @@ -50,38 +51,38 @@ type YoutubeVideo struct { const reflectorURL = "http://blobs.lbry.io/" var youtubeCategories = map[string]string{ - "1": "Film & Animation", - "2": "Autos & Vehicles", - "10": "Music", - "15": "Pets & Animals", - "17": "Sports", - "18": "Short Movies", - "19": "Travel & Events", - "20": "Gaming", - "21": "Videoblogging", - "22": "People & Blogs", - "23": "Comedy", - "24": "Entertainment", - "25": "News & Politics", - "26": "Howto & Style", - "27": "Education", - "28": "Science & Technology", - "29": "Nonprofits & Activism", - "30": "Movies", - "31": "Anime/Animation", - "32": "Action/Adventure", - "33": "Classics", - "34": "Comedy", - "35": "Documentary", - "36": "Drama", - "37": "Family", - "38": "Foreign", - "39": "Horror", - "40": "Sci-Fi/Fantasy", - "41": "Thriller", - "42": "Shorts", - "43": "Shows", - "44": "Trailers", + "1": "film & animation", + "2": "autos & vehicles", + "10": "music", + "15": "pets & animals", + "17": "sports", + "18": "short movies", + "19": "travel & events", + "20": "gaming", + "21": "videoblogging", + "22": "people & blogs", + "23": "comedy", + "24": "entertainment", + "25": "news & politics", + "26": "howto & style", + "27": "education", + "28": "science & technology", + "29": "nonprofits & activism", + "30": "movies", + "31": "anime/animation", + "32": "action/adventure", + "33": "classics", + "34": "comedy", + "35": "documentary", + "36": "drama", + "37": "family", + "38": "foreign", + "39": "horror", + "40": "sci-fi/fantasy", + "41": "thriller", + "42": "shorts", + "43": "shows", + "44": "trailers", } func NewYoutubeVideo(directory string, videoData *youtube.Video, playlistPosition int64, awsConfig aws.Config) *YoutubeVideo { @@ -319,19 +320,33 @@ func (v *YoutubeVideo) triggerThumbnailSave() (err error) { return err } -func (v *YoutubeVideo) publish(daemon *jsonrpc.Client, claimAddress string, amount float64, namer *namer.Namer) (*SyncSummary, error) { +func (v *YoutubeVideo) publish(daemon *jsonrpc.Client, params SyncParams) (*SyncSummary, error) { languages, locations, tags := v.getMetadata() + var fee *jsonrpc.Fee + if params.Fee != nil { + feeAmount, err := decimal.NewFromString(params.Fee.Amount) + if err != nil { + return nil, errors.Err(err) + } + fee = &jsonrpc.Fee{ + FeeAddress: ¶ms.Fee.Address, + FeeAmount: feeAmount, + FeeCurrency: jsonrpc.Currency(params.Fee.Currency), + } + } + options := jsonrpc.StreamCreateOptions{ ClaimCreateOptions: jsonrpc.ClaimCreateOptions{ Title: &v.title, Description: util.PtrToString(v.getAbbrevDescription()), - ClaimAddress: &claimAddress, + ClaimAddress: ¶ms.ClaimAddress, Languages: languages, ThumbnailURL: &v.thumbnailURL, Tags: tags, Locations: locations, }, + Fee: fee, License: util.PtrToString("Copyrighted (contact publisher)"), ReleaseTime: util.PtrToInt64(v.publishedAt.Unix()), ChannelID: &v.lbryChannelID, @@ -340,7 +355,7 @@ func (v *YoutubeVideo) publish(daemon *jsonrpc.Client, claimAddress string, amou if err != nil { return nil, err } - return publishAndRetryExistingNames(daemon, v.title, downloadPath, amount, options, namer) + return publishAndRetryExistingNames(daemon, v.title, downloadPath, params.Amount, options, params.Namer) } func (v *YoutubeVideo) Size() *int64 { @@ -354,6 +369,7 @@ type SyncParams struct { MaxVideoSize int Namer *namer.Namer MaxVideoLength float64 + Fee *sdk.Fee } func (v *YoutubeVideo) Sync(daemon *jsonrpc.Client, params SyncParams, existingVideoData *sdk.SyncedVideo, reprocess bool) (*SyncSummary, error) { @@ -385,7 +401,7 @@ func (v *YoutubeVideo) downloadAndPublish(daemon *jsonrpc.Client, params SyncPar } log.Debugln("Created thumbnail for " + v.id) - summary, err := v.publish(daemon, params.ClaimAddress, params.Amount, params.Namer) + summary, err := v.publish(daemon, params) //delete the video in all cases (and ignore the error) _ = v.delete() @@ -455,21 +471,37 @@ func (v *YoutubeVideo) reprocess(daemon *jsonrpc.Client, params SyncParams, exis return nil, errors.Err("the video must be republished as we can't get the right size") } } + var fee *jsonrpc.Fee + if params.Fee != nil { + feeAmount, err := decimal.NewFromString(params.Fee.Amount) + if err != nil { + return nil, errors.Err(err) + } + fee = &jsonrpc.Fee{ + FeeAddress: ¶ms.Fee.Address, + FeeAmount: feeAmount, + FeeCurrency: jsonrpc.Currency(params.Fee.Currency), + } + } + streamCreateOptions := &jsonrpc.StreamCreateOptions{ + ClaimCreateOptions: jsonrpc.ClaimCreateOptions{ + Tags: tags, + ThumbnailURL: &thumbnailURL, + Languages: languages, + Locations: locations, + }, + Author: util.PtrToString(""), + License: util.PtrToString("Copyrighted (contact publisher)"), + ChannelID: &v.lbryChannelID, + Height: util.PtrToUint(720), + Width: util.PtrToUint(1280), + Fee: fee, + } if v.mocked { pr, err := daemon.StreamUpdate(existingVideoData.ClaimID, jsonrpc.StreamUpdateOptions{ - StreamCreateOptions: &jsonrpc.StreamCreateOptions{ - ClaimCreateOptions: jsonrpc.ClaimCreateOptions{ - Tags: tags, - ThumbnailURL: &thumbnailURL, - }, - Author: util.PtrToString(""), - License: util.PtrToString("Copyrighted (contact publisher)"), - ChannelID: &v.lbryChannelID, - Height: util.PtrToUint(720), - Width: util.PtrToUint(1280), - }, - FileSize: &videoSize, + StreamCreateOptions: streamCreateOptions, + FileSize: &videoSize, }) if err != nil { return nil, err @@ -486,28 +518,16 @@ func (v *YoutubeVideo) reprocess(daemon *jsonrpc.Client, params SyncParams, exis return nil, errors.Err(err) } + streamCreateOptions.ClaimCreateOptions.Title = &v.title + streamCreateOptions.ClaimCreateOptions.Description = util.PtrToString(v.getAbbrevDescription()) + streamCreateOptions.Duration = util.PtrToUint64(uint64(math.Ceil(videoDuration.ToDuration().Seconds()))) + streamCreateOptions.ReleaseTime = util.PtrToInt64(v.publishedAt.Unix()) pr, err := daemon.StreamUpdate(existingVideoData.ClaimID, jsonrpc.StreamUpdateOptions{ - ClearLanguages: util.PtrToBool(true), - ClearLocations: util.PtrToBool(true), - ClearTags: util.PtrToBool(true), - StreamCreateOptions: &jsonrpc.StreamCreateOptions{ - ClaimCreateOptions: jsonrpc.ClaimCreateOptions{ - Title: &v.title, - Description: util.PtrToString(v.getAbbrevDescription()), - Tags: tags, - Languages: languages, - Locations: locations, - ThumbnailURL: &thumbnailURL, - }, - Author: util.PtrToString(""), - License: util.PtrToString("Copyrighted (contact publisher)"), - Height: util.PtrToUint(720), - Width: util.PtrToUint(1280), - ReleaseTime: util.PtrToInt64(v.publishedAt.Unix()), - Duration: util.PtrToUint64(uint64(math.Ceil(videoDuration.ToDuration().Seconds()))), - ChannelID: &v.lbryChannelID, - }, - FileSize: &videoSize, + ClearLanguages: util.PtrToBool(true), + ClearLocations: util.PtrToBool(true), + ClearTags: util.PtrToBool(true), + StreamCreateOptions: streamCreateOptions, + FileSize: &videoSize, }) if err != nil { return nil, err diff --git a/tagsManager/tags_mapping.go b/tagsManager/tags_mapping.go index adb1024..c164cc0 100644 --- a/tagsManager/tags_mapping.go +++ b/tagsManager/tags_mapping.go @@ -1,8 +1,11 @@ package tagsManager import ( + "regexp" "sort" "strings" + + log "github.com/sirupsen/logrus" ) const ( @@ -70,17 +73,23 @@ func SanitizeTags(tags []string, youtubeChannelID string) ([]string, error) { func normalizeTag(t string) (string, error) { t = strings.ToLower(t) + multipleSpaces := regexp.MustCompile(`\s{2,}`) + leadingAndTrailingSpaces := regexp.MustCompile(`^\s+|\s$`) + hashTags := regexp.MustCompile(`(#\d+\s)|#+`) + inParenthesis := regexp.MustCompile(`\([^\)]+\)`) + weirdChars := regexp.MustCompile(`[^-\w'& +\/A-Za-zÀ-ÖØ-öø-ÿ]`) + startsOrEndsInWeirdChars := regexp.MustCompile(`^[^A-Za-zÀ-ÖØ-öø-ÿ0-9]+|[^A-Za-zÀ-ÖØ-öø-ÿ0-9]+$`) + t = hashTags.ReplaceAllString(t, "") + t = inParenthesis.ReplaceAllString(t, " ") + t = startsOrEndsInWeirdChars.ReplaceAllString(t, "") + t = multipleSpaces.ReplaceAllString(t, " ") + t = leadingAndTrailingSpaces.ReplaceAllString(t, "") + if weirdChars.MatchString(t) { + log.Debugf("tag '%s' has weird stuff in it, skipping\n", t) + return "", nil + + } return t, nil - //r, err := regexp.Compile("/\\([^\\)]+\\)/g") - //if err != nil { - // return "", errors.Err(err) - //} - //r2, err := regexp.Compile("/[^\\w-'& \+]/g") - //if err != nil { - // return "", errors.Err(err) - //} - //t = r.ReplaceAllString(t, "") - //return r2.ReplaceAllString(t, ""), nil } type tagsSanitizer struct { diff --git a/tagsManager/tags_mapping_test.go b/tagsManager/tags_mapping_test.go index 156281f..be7fe76 100644 --- a/tagsManager/tags_mapping_test.go +++ b/tagsManager/tags_mapping_test.go @@ -1,6 +1,7 @@ package tagsManager import ( + "fmt" "testing" ) @@ -44,3 +45,100 @@ outer: } } +func TestNormalizeTag(t *testing.T) { + tags := []string{ + "blockchain", + "Switzerland", + "news ", + " science & Technology ", + "economics", + "experiments", + "this", + "in it", + "will build the (WOOPS)", + "~has", + "crypto", + "trump", + "wall", + "expensive", + "!currency", + " a lot of ", + "#", + "#whatever", + "#123", + "#123 Something else", + "#123aaa", + "!asdasd", + "CASA BLANCA", + "wwe 2k18 Elimination chamber!", + "pero'", + "però", + "è proprio", + "Ep 29", + "sctest29 Keddr", + "mortal kombat 11 shang tsung", + "!asdasd!", + } + normalizedTags := make([]string, 0, len(tags)) + for _, tag := range tags { + got, err := normalizeTag(tag) + if err != nil { + t.Error(err) + return + } + if got != "" { + normalizedTags = append(normalizedTags, got) + } + fmt.Printf("Got tag: '%s'\n", got) + } + expected := []string{ + "blockchain", + "switzerland", + "news", + "science & technology", + "economics", + "experiments", + "this", + "in it", + "will build the", + "has", + "crypto", + "trump", + "wall", + "expensive", + "currency", + "a lot of", + "whatever", + "123", + "something else", + "123aaa", + "asdasd", + "casa blanca", + "wwe 2k18 elimination chamber", + "pero", + "però", + "è proprio", + "ep 29", + "sctest29 keddr", + "mortal kombat 11 shang tsung", + "asdasd", + } + if !Equal(normalizedTags, expected) { + t.Error("result not as expected") + return + } + +} +func Equal(a, b []string) bool { + if len(a) != len(b) { + fmt.Printf("expected length %d but got %d", len(b), len(a)) + return false + } + for i, v := range a { + if v != b[i] { + fmt.Printf("expected %s but bot %s\n", b[i], v) + return false + } + } + return true +}