add Fee support

lowercase all tags
refactor code
implement and test regexps for tags

go.sum

@@ -18,6 +18,7 @@ github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRF
 github.com/andybalholm/cascadia v1.0.0 h1:hOCXnnZ5A+3eVDX8pvgl4kofXv2ELss0bKcqRySc45o=
 github.com/andybalholm/cascadia v1.0.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y=
 github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ=
+github.com/asaskevich/govalidator v0.0.0-20180720115003-f9ffefc3facf h1:eg0MeVzsP1G42dRafH3vf+al2vQIJU0YHX+1Tw87oco=
 github.com/asaskevich/govalidator v0.0.0-20180720115003-f9ffefc3facf/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY=
 github.com/aws/aws-sdk-go v1.17.3 h1:KBXxg7Jh0TxE5zmpNB2DwKmJeDUqh0O6jhy25TuYOmc=
 github.com/aws/aws-sdk-go v1.17.3/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
@@ -59,6 +60,7 @@ github.com/go-ini/ini v1.42.0 h1:TWr1wGj35+UiWHlBA8er89seFXxzwFn11spilrrj+38=
 github.com/go-ini/ini v1.42.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8=
 github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
 github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
+github.com/go-ozzo/ozzo-validation v3.5.0+incompatible h1:sUy/in/P6askYr16XJgTKq/0SZhiWsdg4WZGaLsGQkM=
 github.com/go-ozzo/ozzo-validation v3.5.0+incompatible/go.mod h1:gsEKFIVnabGBt6mXmxK0MoFy+cZoTJY6mu5Ll3LVLBU=
 github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
 github.com/gogo/googleapis v1.1.0/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFGgqEef3s=

manager/manager.go

@@ -133,6 +133,7 @@ func (s *SyncManager) Start() error {
 				AwsS3Region:             s.awsS3Region,
 				AwsS3Bucket:             s.awsS3Bucket,
 				namer:                   namer.NewNamer(),
+				Fee:                     channels[0].Fee,
 			}
 			shouldInterruptLoop = true
 		} else {
@@ -167,6 +168,7 @@ func (s *SyncManager) Start() error {
 						AwsS3Region:             s.awsS3Region,
 						AwsS3Bucket:             s.awsS3Bucket,
 						namer:                   namer.NewNamer(),
+						Fee:                     c.Fee,
 					})
 				}
 			}

manager/ytsync.go

@@ -74,18 +74,17 @@ type Sync struct {
 	AwsS3Secret             string
 	AwsS3Region             string
 	AwsS3Bucket             string
-
+	Fee                     *sdk.Fee
-	daemon          *jsonrpc.Client
+	daemon                  *jsonrpc.Client
-	claimAddress    string
+	claimAddress            string
-	videoDirectory  string
+	videoDirectory          string
-	syncedVideosMux *sync.RWMutex
+	syncedVideosMux         *sync.RWMutex
-	syncedVideos    map[string]sdk.SyncedVideo
+	syncedVideos            map[string]sdk.SyncedVideo
-	grp             *stop.Group
+	grp                     *stop.Group
-	lbryChannelID   string
+	lbryChannelID           string
-	namer           *namer.Namer
+	namer                   *namer.Namer
-
+	walletMux               *sync.Mutex
-	walletMux *sync.Mutex
+	queue                   chan video
-	queue     chan video
 }
 
 func (s *Sync) AppendSyncedVideo(videoID string, published bool, failureReason string, claimName string) {
@@ -876,6 +875,7 @@ func (s *Sync) processVideo(v video) (err error) {
 		MaxVideoSize:   s.Manager.maxVideoSize,
 		Namer:          s.namer,
 		MaxVideoLength: s.Manager.maxVideoLength,
+		Fee:            s.Fee,
 	}
 
 	summary, err := v.Sync(s.daemon, sp, &sv, isUpgradeSync)

sdk/api.go

@@ -35,16 +35,17 @@ type SyncProperties struct {
 	YoutubeChannelID string
 }
 
+type Fee struct {
+	Amount   string `json:"amount"`
+	Address  string `json:"address"`
+	Currency string `json:"currency"`
+}
 type YoutubeChannel struct {
 	ChannelId          string `json:"channel_id"`
 	TotalVideos        uint   `json:"total_videos"`
 	DesiredChannelName string `json:"desired_channel_name"`
-	Fee                *struct {
+	Fee                *Fee   `json:"fee"`
-		Amount   string `json:"amount"`
+	ChannelClaimID     string `json:"channel_claim_id"`
-		Address  string `json:"address"`
-		Currency string `json:"currency"`
-	} `json:"fee"`
-	ChannelClaimID string `json:"channel_claim_id"`
 }
 
 func (a *APIConfig) FetchChannels(status string, cp *SyncProperties) ([]YoutubeChannel, error) {

sources/youtubeVideo.go

@@ -24,6 +24,7 @@ import (
 	"github.com/ChannelMeter/iso8601duration"
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/nikooo777/ytdl"
+	"github.com/shopspring/decimal"
 	log "github.com/sirupsen/logrus"
 	"google.golang.org/api/youtube/v3"
 )
@@ -50,38 +51,38 @@ type YoutubeVideo struct {
 const reflectorURL = "http://blobs.lbry.io/"
 
 var youtubeCategories = map[string]string{
-	"1":  "Film & Animation",
+	"1":  "film & animation",
-	"2":  "Autos & Vehicles",
+	"2":  "autos & vehicles",
-	"10": "Music",
+	"10": "music",
-	"15": "Pets & Animals",
+	"15": "pets & animals",
-	"17": "Sports",
+	"17": "sports",
-	"18": "Short Movies",
+	"18": "short movies",
-	"19": "Travel & Events",
+	"19": "travel & events",
-	"20": "Gaming",
+	"20": "gaming",
-	"21": "Videoblogging",
+	"21": "videoblogging",
-	"22": "People & Blogs",
+	"22": "people & blogs",
-	"23": "Comedy",
+	"23": "comedy",
-	"24": "Entertainment",
+	"24": "entertainment",
-	"25": "News & Politics",
+	"25": "news & politics",
-	"26": "Howto & Style",
+	"26": "howto & style",
-	"27": "Education",
+	"27": "education",
-	"28": "Science & Technology",
+	"28": "science & technology",
-	"29": "Nonprofits & Activism",
+	"29": "nonprofits & activism",
-	"30": "Movies",
+	"30": "movies",
-	"31": "Anime/Animation",
+	"31": "anime/animation",
-	"32": "Action/Adventure",
+	"32": "action/adventure",
-	"33": "Classics",
+	"33": "classics",
-	"34": "Comedy",
+	"34": "comedy",
-	"35": "Documentary",
+	"35": "documentary",
-	"36": "Drama",
+	"36": "drama",
-	"37": "Family",
+	"37": "family",
-	"38": "Foreign",
+	"38": "foreign",
-	"39": "Horror",
+	"39": "horror",
-	"40": "Sci-Fi/Fantasy",
+	"40": "sci-fi/fantasy",
-	"41": "Thriller",
+	"41": "thriller",
-	"42": "Shorts",
+	"42": "shorts",
-	"43": "Shows",
+	"43": "shows",
-	"44": "Trailers",
+	"44": "trailers",
 }
 
 func NewYoutubeVideo(directory string, videoData *youtube.Video, playlistPosition int64, awsConfig aws.Config) *YoutubeVideo {
@@ -319,19 +320,33 @@ func (v *YoutubeVideo) triggerThumbnailSave() (err error) {
 	return err
 }
 
-func (v *YoutubeVideo) publish(daemon *jsonrpc.Client, claimAddress string, amount float64, namer *namer.Namer) (*SyncSummary, error) {
+func (v *YoutubeVideo) publish(daemon *jsonrpc.Client, params SyncParams) (*SyncSummary, error) {
 	languages, locations, tags := v.getMetadata()
 
+	var fee *jsonrpc.Fee
+	if params.Fee != nil {
+		feeAmount, err := decimal.NewFromString(params.Fee.Amount)
+		if err != nil {
+			return nil, errors.Err(err)
+		}
+		fee = &jsonrpc.Fee{
+			FeeAddress:  &params.Fee.Address,
+			FeeAmount:   feeAmount,
+			FeeCurrency: jsonrpc.Currency(params.Fee.Currency),
+		}
+	}
+
 	options := jsonrpc.StreamCreateOptions{
 		ClaimCreateOptions: jsonrpc.ClaimCreateOptions{
 			Title:        &v.title,
 			Description:  util.PtrToString(v.getAbbrevDescription()),
-			ClaimAddress: &claimAddress,
+			ClaimAddress: &params.ClaimAddress,
 			Languages:    languages,
 			ThumbnailURL: &v.thumbnailURL,
 			Tags:         tags,
 			Locations:    locations,
 		},
+		Fee:         fee,
 		License:     util.PtrToString("Copyrighted (contact publisher)"),
 		ReleaseTime: util.PtrToInt64(v.publishedAt.Unix()),
 		ChannelID:   &v.lbryChannelID,
@@ -340,7 +355,7 @@ func (v *YoutubeVideo) publish(daemon *jsonrpc.Client, claimAddress string, amou
 	if err != nil {
 		return nil, err
 	}
-	return publishAndRetryExistingNames(daemon, v.title, downloadPath, amount, options, namer)
+	return publishAndRetryExistingNames(daemon, v.title, downloadPath, params.Amount, options, params.Namer)
 }
 
 func (v *YoutubeVideo) Size() *int64 {
@@ -354,6 +369,7 @@ type SyncParams struct {
 	MaxVideoSize   int
 	Namer          *namer.Namer
 	MaxVideoLength float64
+	Fee            *sdk.Fee
 }
 
 func (v *YoutubeVideo) Sync(daemon *jsonrpc.Client, params SyncParams, existingVideoData *sdk.SyncedVideo, reprocess bool) (*SyncSummary, error) {
@@ -385,7 +401,7 @@ func (v *YoutubeVideo) downloadAndPublish(daemon *jsonrpc.Client, params SyncPar
 	}
 	log.Debugln("Created thumbnail for " + v.id)
 
-	summary, err := v.publish(daemon, params.ClaimAddress, params.Amount, params.Namer)
+	summary, err := v.publish(daemon, params)
 	//delete the video in all cases (and ignore the error)
 	_ = v.delete()
 
@@ -455,21 +471,37 @@ func (v *YoutubeVideo) reprocess(daemon *jsonrpc.Client, params SyncParams, exis
 			return nil, errors.Err("the video must be republished as we can't get the right size")
 		}
 	}
+	var fee *jsonrpc.Fee
+	if params.Fee != nil {
+		feeAmount, err := decimal.NewFromString(params.Fee.Amount)
+		if err != nil {
+			return nil, errors.Err(err)
+		}
+		fee = &jsonrpc.Fee{
+			FeeAddress:  &params.Fee.Address,
+			FeeAmount:   feeAmount,
+			FeeCurrency: jsonrpc.Currency(params.Fee.Currency),
+		}
+	}
+	streamCreateOptions := &jsonrpc.StreamCreateOptions{
+		ClaimCreateOptions: jsonrpc.ClaimCreateOptions{
+			Tags:         tags,
+			ThumbnailURL: &thumbnailURL,
+			Languages:    languages,
+			Locations:    locations,
+		},
+		Author:    util.PtrToString(""),
+		License:   util.PtrToString("Copyrighted (contact publisher)"),
+		ChannelID: &v.lbryChannelID,
+		Height:    util.PtrToUint(720),
+		Width:     util.PtrToUint(1280),
+		Fee:       fee,
+	}
 
 	if v.mocked {
 		pr, err := daemon.StreamUpdate(existingVideoData.ClaimID, jsonrpc.StreamUpdateOptions{
-			StreamCreateOptions: &jsonrpc.StreamCreateOptions{
+			StreamCreateOptions: streamCreateOptions,
-				ClaimCreateOptions: jsonrpc.ClaimCreateOptions{
+			FileSize:            &videoSize,
-					Tags:         tags,
-					ThumbnailURL: &thumbnailURL,
-				},
-				Author:    util.PtrToString(""),
-				License:   util.PtrToString("Copyrighted (contact publisher)"),
-				ChannelID: &v.lbryChannelID,
-				Height:    util.PtrToUint(720),
-				Width:     util.PtrToUint(1280),
-			},
-			FileSize: &videoSize,
 		})
 		if err != nil {
 			return nil, err
@@ -486,28 +518,16 @@ func (v *YoutubeVideo) reprocess(daemon *jsonrpc.Client, params SyncParams, exis
 		return nil, errors.Err(err)
 	}
 
+	streamCreateOptions.ClaimCreateOptions.Title = &v.title
+	streamCreateOptions.ClaimCreateOptions.Description = util.PtrToString(v.getAbbrevDescription())
+	streamCreateOptions.Duration = util.PtrToUint64(uint64(math.Ceil(videoDuration.ToDuration().Seconds())))
+	streamCreateOptions.ReleaseTime = util.PtrToInt64(v.publishedAt.Unix())
 	pr, err := daemon.StreamUpdate(existingVideoData.ClaimID, jsonrpc.StreamUpdateOptions{
-		ClearLanguages: util.PtrToBool(true),
+		ClearLanguages:      util.PtrToBool(true),
-		ClearLocations: util.PtrToBool(true),
+		ClearLocations:      util.PtrToBool(true),
-		ClearTags:      util.PtrToBool(true),
+		ClearTags:           util.PtrToBool(true),
-		StreamCreateOptions: &jsonrpc.StreamCreateOptions{
+		StreamCreateOptions: streamCreateOptions,
-			ClaimCreateOptions: jsonrpc.ClaimCreateOptions{
+		FileSize:            &videoSize,
-				Title:        &v.title,
-				Description:  util.PtrToString(v.getAbbrevDescription()),
-				Tags:         tags,
-				Languages:    languages,
-				Locations:    locations,
-				ThumbnailURL: &thumbnailURL,
-			},
-			Author:      util.PtrToString(""),
-			License:     util.PtrToString("Copyrighted (contact publisher)"),
-			Height:      util.PtrToUint(720),
-			Width:       util.PtrToUint(1280),
-			ReleaseTime: util.PtrToInt64(v.publishedAt.Unix()),
-			Duration:    util.PtrToUint64(uint64(math.Ceil(videoDuration.ToDuration().Seconds()))),
-			ChannelID:   &v.lbryChannelID,
-		},
-		FileSize: &videoSize,
 	})
 	if err != nil {
 		return nil, err

tagsManager/tags_mapping.go

@@ -1,8 +1,11 @@
 package tagsManager
 
 import (
+	"regexp"
 	"sort"
 	"strings"
+
+	log "github.com/sirupsen/logrus"
 )
 
 const (
@@ -70,17 +73,23 @@ func SanitizeTags(tags []string, youtubeChannelID string) ([]string, error) {
 
 func normalizeTag(t string) (string, error) {
 	t = strings.ToLower(t)
+	multipleSpaces := regexp.MustCompile(`\s{2,}`)
+	leadingAndTrailingSpaces := regexp.MustCompile(`^\s+|\s$`)
+	hashTags := regexp.MustCompile(`(#\d+\s)|#+`)
+	inParenthesis := regexp.MustCompile(`\([^\)]+\)`)
+	weirdChars := regexp.MustCompile(`[^-\w'& +\/A-Za-zÀ-ÖØ-öø-ÿ]`)
+	startsOrEndsInWeirdChars := regexp.MustCompile(`^[^A-Za-zÀ-ÖØ-öø-ÿ0-9]+|[^A-Za-zÀ-ÖØ-öø-ÿ0-9]+$`)
+	t = hashTags.ReplaceAllString(t, "")
+	t = inParenthesis.ReplaceAllString(t, " ")
+	t = startsOrEndsInWeirdChars.ReplaceAllString(t, "")
+	t = multipleSpaces.ReplaceAllString(t, " ")
+	t = leadingAndTrailingSpaces.ReplaceAllString(t, "")
+	if weirdChars.MatchString(t) {
+		log.Debugf("tag '%s' has weird stuff in it, skipping\n", t)
+		return "", nil
+
+	}
 	return t, nil
-	//r, err := regexp.Compile("/\\([^\\)]+\\)/g")
-	//if err != nil {
-	//	return "", errors.Err(err)
-	//}
-	//r2, err := regexp.Compile("/[^\\w-'& \+]/g")
-	//if err != nil {
-	//	return "", errors.Err(err)
-	//}
-	//t = r.ReplaceAllString(t, "")
-	//return r2.ReplaceAllString(t, ""), nil
 }
 
 type tagsSanitizer struct {

tagsManager/tags_mapping_test.go

@@ -1,6 +1,7 @@
 package tagsManager
 
 import (
+	"fmt"
 	"testing"
 )
 
@@ -44,3 +45,100 @@ outer:
 	}
 
 }
+func TestNormalizeTag(t *testing.T) {
+	tags := []string{
+		"blockchain",
+		"Switzerland",
+		"news         ",
+		"  science  &  Technology  ",
+		"economics",
+		"experiments",
+		"this",
+		"in it",
+		"will build the (WOOPS)",
+		"~has",
+		"crypto",
+		"trump",
+		"wall",
+		"expensive",
+		"!currency",
+		" a lot of  ",
+		"#",
+		"#whatever",
+		"#123",
+		"#123 Something else",
+		"#123aaa",
+		"!asdasd",
+		"CASA BLANCA",
+		"wwe 2k18 Elimination chamber!",
+		"pero'",
+		"però",
+		"è proprio",
+		"Ep 29",
+		"sctest29 Keddr",
+		"mortal kombat 11 shang tsung",
+		"!asdasd!",
+	}
+	normalizedTags := make([]string, 0, len(tags))
+	for _, tag := range tags {
+		got, err := normalizeTag(tag)
+		if err != nil {
+			t.Error(err)
+			return
+		}
+		if got != "" {
+			normalizedTags = append(normalizedTags, got)
+		}
+		fmt.Printf("Got tag: '%s'\n", got)
+	}
+	expected := []string{
+		"blockchain",
+		"switzerland",
+		"news",
+		"science & technology",
+		"economics",
+		"experiments",
+		"this",
+		"in it",
+		"will build the",
+		"has",
+		"crypto",
+		"trump",
+		"wall",
+		"expensive",
+		"currency",
+		"a lot of",
+		"whatever",
+		"123",
+		"something else",
+		"123aaa",
+		"asdasd",
+		"casa blanca",
+		"wwe 2k18 elimination chamber",
+		"pero",
+		"però",
+		"è proprio",
+		"ep 29",
+		"sctest29 keddr",
+		"mortal kombat 11 shang tsung",
+		"asdasd",
+	}
+	if !Equal(normalizedTags, expected) {
+		t.Error("result not as expected")
+		return
+	}
+
+}
+func Equal(a, b []string) bool {
+	if len(a) != len(b) {
+		fmt.Printf("expected length %d but got %d", len(b), len(a))
+		return false
+	}
+	for i, v := range a {
+		if v != b[i] {
+			fmt.Printf("expected %s but bot %s\n", b[i], v)
+			return false
+		}
+	}
+	return true
+}