manager/setup.go

@@ -11,6 +11,8 @@ import (
 	"github.com/lbryio/lbry.go/extras/jsonrpc"
 	"github.com/lbryio/lbry.go/extras/util"
 	"github.com/lbryio/lbry.go/lbrycrd"
+
+	"github.com/lbryio/ytsync/tagsManager"
 	"github.com/lbryio/ytsync/thumbs"
 
 	"github.com/shopspring/decimal"
@@ -355,13 +357,14 @@ func (s *Sync) ensureChannelOwnership() error {
 	var c *jsonrpc.TransactionSummary
 	if channelUsesOldMetadata {
 		c, err = s.daemon.ChannelUpdate(s.lbryChannelID, jsonrpc.ChannelUpdateOptions{
+			ClearTags:      util.PtrToBool(true),
 			ClearLocations: util.PtrToBool(true),
 			ClearLanguages: util.PtrToBool(true),
 			ChannelCreateOptions: jsonrpc.ChannelCreateOptions{
 				ClaimCreateOptions: jsonrpc.ClaimCreateOptions{
 					Title:        channelInfo.Title,
 					Description:  channelInfo.Description,
-					Tags:         nil,
+					Tags:         tagsManager.GetTagsForChannel(s.YoutubeChannelID),
 					Languages:    languages,
 					Locations:    locations,
 					ThumbnailURL: &thumbnailURL,
@@ -374,7 +377,7 @@ func (s *Sync) ensureChannelOwnership() error {
 			ClaimCreateOptions: jsonrpc.ClaimCreateOptions{
 				Title:        channelInfo.Title,
 				Description:  channelInfo.Description,
-				Tags:         nil,
+				Tags:         tagsManager.GetTagsForChannel(s.YoutubeChannelID),
 				Languages:    languages,
 				Locations:    locations,
 				ThumbnailURL: &thumbnailURL,

manager/ytsync.go

@@ -459,8 +459,12 @@ func (s *Sync) updateRemoteDB(claims []jsonrpc.Claim) (total int, fixed int, err
 		if !ok || pv.ClaimName != c.Name {
 			fixed++
 			log.Debugf("adding %s to the database", c.Name)
-
+			size, err := c.GetStreamSizeByMagic()
-			err = s.Manager.apiConfig.MarkVideoStatus(s.YoutubeChannelID, videoID, VideoStatusPublished, c.ClaimID, c.Name, "", nil)
+			if err != nil {
+				size = 0
+			}
+			metadataVersion := uint(1)
+			err = s.Manager.apiConfig.MarkVideoStatus(s.YoutubeChannelID, videoID, VideoStatusPublished, c.ClaimID, c.Name, "", util.PtrToInt64(int64(size)), metadataVersion)
 			if err != nil {
 				return count, fixed, err
 			}
@@ -663,7 +667,7 @@ func (s *Sync) startWorker(workerNum int) {
 				}
 
 				s.AppendSyncedVideo(v.ID(), false, err.Error(), "")
-				err = s.Manager.apiConfig.MarkVideoStatus(s.YoutubeChannelID, v.ID(), VideoStatusFailed, existingClaimID, existingClaimName, err.Error(), existingClaimSize)
+				err = s.Manager.apiConfig.MarkVideoStatus(s.YoutubeChannelID, v.ID(), VideoStatusFailed, existingClaimID, existingClaimName, err.Error(), existingClaimSize, 1)
 				if err != nil {
 					SendErrorToSlack("Failed to mark video on the database: %s", err.Error())
 				}
@@ -749,14 +753,13 @@ func (s *Sync) enqueueYoutubeVideos() error {
 			break
 		}
 	}
-	notOnYoutube := make([]video, 0, len(s.syncedVideos))
 	for k, v := range s.syncedVideos {
 		if !v.Published {
 			continue
 		}
 		_, ok := playlistMap[k]
 		if !ok {
-			notOnYoutube = append(notOnYoutube, sources.NewMockedVideo(s.videoDirectory, k, s.YoutubeChannelID, s.Manager.GetS3AWSConfig()))
+			videos = append(videos, sources.NewMockedVideo(s.videoDirectory, k, s.YoutubeChannelID, s.Manager.GetS3AWSConfig()))
 		}
 
 	}
@@ -851,7 +854,7 @@ func (s *Sync) processVideo(v video) (err error) {
 		return err
 	}
 
-	err = s.Manager.apiConfig.MarkVideoStatus(s.YoutubeChannelID, v.ID(), VideoStatusPublished, summary.ClaimID, summary.ClaimName, "", v.Size())
+	err = s.Manager.apiConfig.MarkVideoStatus(s.YoutubeChannelID, v.ID(), VideoStatusPublished, summary.ClaimID, summary.ClaimName, "", v.Size(), 2)
 	if err != nil {
 		SendErrorToSlack("Failed to mark video on the database: %s", err.Error())
 	}

sdk/api.go

@@ -2,6 +2,7 @@ package sdk
 
 import (
 	"encoding/json"
+	"fmt"
 	"io/ioutil"
 	"net/http"
 	"net/url"
@@ -166,7 +167,7 @@ const (
 	VideoStatusFailed    = "failed"
 )
 
-func (a *APIConfig) MarkVideoStatus(channelID string, videoID string, status string, claimID string, claimName string, failureReason string, size *int64) error {
+func (a *APIConfig) MarkVideoStatus(channelID string, videoID string, status string, claimID string, claimName string, failureReason string, size *int64, metadataVersion uint) error {
 	endpoint := a.ApiURL + "/yt/video_status"
 
 	sanitizeFailureReason(&failureReason)
@@ -183,7 +184,7 @@ func (a *APIConfig) MarkVideoStatus(channelID string, videoID string, status str
 		vals.Add("published_at", strconv.FormatInt(time.Now().Unix(), 10))
 		vals.Add("claim_id", claimID)
 		vals.Add("claim_name", claimName)
-		vals.Add("metadata_version", "2")
+		vals.Add("metadata_version", fmt.Sprintf("%d", metadataVersion))
 		if size != nil {
 			vals.Add("size", strconv.FormatInt(*size, 10))
 		}

sources/youtubeVideo.go

@@ -453,11 +453,6 @@ func (v *YoutubeVideo) reprocess(daemon *jsonrpc.Client, params SyncParams, exis
 		}
 	}
 
-	videoDuration, err := duration.FromString(v.youtubeInfo.ContentDetails.Duration)
-	if err != nil {
-		return nil, errors.Err(err)
-	}
-
 	if v.mocked {
 		pr, err := daemon.StreamUpdate(existingVideoData.ClaimID, jsonrpc.StreamUpdateOptions{
 			StreamCreateOptions: &jsonrpc.StreamCreateOptions{
@@ -466,7 +461,7 @@ func (v *YoutubeVideo) reprocess(daemon *jsonrpc.Client, params SyncParams, exis
 					ThumbnailURL: &thumbnailURL,
 				},
 				Author:    util.PtrToString(""),
-				License:   util.PtrToString("Copyrighted (contact author)"),
+				License:   util.PtrToString("Copyrighted (contact publisher)"),
 				ChannelID: &v.lbryChannelID,
 			},
 			FileSize: &videoSize,
@@ -480,6 +475,12 @@ func (v *YoutubeVideo) reprocess(daemon *jsonrpc.Client, params SyncParams, exis
 			ClaimName: pr.Outputs[0].Name,
 		}, nil
 	}
+
+	videoDuration, err := duration.FromString(v.youtubeInfo.ContentDetails.Duration)
+	if err != nil {
+		return nil, errors.Err(err)
+	}
+
 	pr, err := daemon.StreamUpdate(existingVideoData.ClaimID, jsonrpc.StreamUpdateOptions{
 		ClearLanguages: util.PtrToBool(true),
 		ClearLocations: util.PtrToBool(true),
@@ -495,6 +496,8 @@ func (v *YoutubeVideo) reprocess(daemon *jsonrpc.Client, params SyncParams, exis
 			},
 			Author:      util.PtrToString(""),
 			License:     util.PtrToString("Copyrighted (contact publisher)"),
+			VideoHeight: util.PtrToUint(720),
+			VideoWidth:  util.PtrToUint(1280),
 			ReleaseTime: util.PtrToInt64(v.publishedAt.Unix()),
 			Duration:    util.PtrToUint64(uint64(math.Ceil(videoDuration.ToDuration().Seconds()))),
 			ChannelID:   &v.lbryChannelID,

tagsManager/tags_mapping.go

@@ -24,6 +24,11 @@ const (
 	Weapons    = "weapons"
 )
 
+func GetTagsForChannel(channelID string) []string {
+	tags, _ := channelWideTags[channelID]
+	return tags
+}
+
 func SanitizeTags(tags []string, youtubeChannelID string) ([]string, error) {
 	unsanitized := make([]string, 0, len(tags))
 	for _, t := range tags {
@@ -54,7 +59,7 @@ func SanitizeTags(tags []string, youtubeChannelID string) ([]string, error) {
 			originalTags = append(originalTags, t)
 		}
 	}
-	sanitizedTags := make([]string, len(originalTags)+len(curatedTags))
+	sanitizedTags := make([]string, 0, len(originalTags)+len(curatedTags))
 	sanitizedTags = append(sanitizedTags, curatedTags...)
 	sanitizedTags = append(sanitizedTags, originalTags...)
 	return sanitizedTags, nil
@@ -117,17 +122,19 @@ func (ts *tagsSanitizer) add() {
 	extraTags, ok := channelWideTags[ts.ChannelID]
 	if ok {
 		for _, t := range extraTags {
-			ts.Sanitized[t] = false
+			ts.Sanitized[t] = true
 		}
 	}
 }
 
 const (
-	Lunduke = "UCkK9UDm_ZNrq_rIXCz3xCGA"
+	Lunduke          = "UCkK9UDm_ZNrq_rIXCz3xCGA"
+	SwissExperiments = "UCNQfQvFMPnInwsU_iGYArJQ"
 )
 
 var channelWideTags = map[string][]string{
-	Lunduke: {"linux", "technology"},
+	Lunduke:          {"linux", "technology"},
+	SwissExperiments: {"science & technology", "experiments", "switzerland"},
 }
 var tagsToSkip = map[string]*struct{}{
 	"#hangoutsonair":                         nil,

tagsManager/tags_mapping_test.go

@@ -0,0 +1,46 @@
+package tagsManager
+
+import (
+	"testing"
+)
+
+func TestSanitizeTags(t *testing.T) {
+	got, err := SanitizeTags([]string{"this", "super", "expensive", "test", "has", "a lot of", "crypto", "currency", "in it", "trump", "will build the", "wall"}, "UCNQfQvFMPnInwsU_iGYArJQ")
+	if err != nil {
+		t.Error(err)
+		return
+	}
+	expectedTags := []string{
+		"blockchain",
+		"switzerland",
+		"news",
+		"science & technology",
+		"economics",
+		"experiments",
+		"this",
+		"in it",
+		"will build the",
+		"has",
+		"crypto",
+		"trump",
+		"wall",
+		"expensive",
+		"currency",
+		"a lot of",
+	}
+	if len(expectedTags) != len(got) {
+		t.Error("number of tags differ")
+		return
+	}
+outer:
+	for _, et := range expectedTags {
+		for _, t := range got {
+			if et == t {
+				continue outer
+			}
+		}
+		t.Error("tag not found")
+		return
+	}
+
+}