fix many bugs

calculate size for claims missing in the database
fix reprocessing for videos no longer on youtube
pass metadata instead of hardcoding it
add video height when updating streams
add tags to channels
add test for tags

manager/setup.go

@@ -11,6 +11,8 @@ import (
 	"github.com/lbryio/lbry.go/extras/jsonrpc"
 	"github.com/lbryio/lbry.go/extras/util"
 	"github.com/lbryio/lbry.go/lbrycrd"
+
+	"github.com/lbryio/ytsync/tagsManager"
 	"github.com/lbryio/ytsync/thumbs"
 
 	"github.com/shopspring/decimal"
@@ -355,13 +357,14 @@ func (s *Sync) ensureChannelOwnership() error {
 	var c *jsonrpc.TransactionSummary
 	if channelUsesOldMetadata {
 		c, err = s.daemon.ChannelUpdate(s.lbryChannelID, jsonrpc.ChannelUpdateOptions{
+			ClearTags:      util.PtrToBool(true),
 			ClearLocations: util.PtrToBool(true),
 			ClearLanguages: util.PtrToBool(true),
 			ChannelCreateOptions: jsonrpc.ChannelCreateOptions{
 				ClaimCreateOptions: jsonrpc.ClaimCreateOptions{
 					Title:        channelInfo.Title,
 					Description:  channelInfo.Description,
-					Tags:         nil,
+					Tags:         tagsManager.GetTagsForChannel(s.YoutubeChannelID),
 					Languages:    languages,
 					Locations:    locations,
 					ThumbnailURL: &thumbnailURL,
@@ -374,7 +377,7 @@ func (s *Sync) ensureChannelOwnership() error {
 			ClaimCreateOptions: jsonrpc.ClaimCreateOptions{
 				Title:        channelInfo.Title,
 				Description:  channelInfo.Description,
-				Tags:         nil,
+				Tags:         tagsManager.GetTagsForChannel(s.YoutubeChannelID),
 				Languages:    languages,
 				Locations:    locations,
 				ThumbnailURL: &thumbnailURL,

manager/ytsync.go

@@ -459,8 +459,12 @@ func (s *Sync) updateRemoteDB(claims []jsonrpc.Claim) (total int, fixed int, err
 		if !ok || pv.ClaimName != c.Name {
 			fixed++
 			log.Debugf("adding %s to the database", c.Name)
-
+			size, err := c.GetStreamSizeByMagic()
-			err = s.Manager.apiConfig.MarkVideoStatus(s.YoutubeChannelID, videoID, VideoStatusPublished, c.ClaimID, c.Name, "", nil)
+			if err != nil {
+				size = 0
+			}
+			metadataVersion := uint(1)
+			err = s.Manager.apiConfig.MarkVideoStatus(s.YoutubeChannelID, videoID, VideoStatusPublished, c.ClaimID, c.Name, "", util.PtrToInt64(int64(size)), metadataVersion)
 			if err != nil {
 				return count, fixed, err
 			}
@@ -663,7 +667,7 @@ func (s *Sync) startWorker(workerNum int) {
 				}
 
 				s.AppendSyncedVideo(v.ID(), false, err.Error(), "")
-				err = s.Manager.apiConfig.MarkVideoStatus(s.YoutubeChannelID, v.ID(), VideoStatusFailed, existingClaimID, existingClaimName, err.Error(), existingClaimSize)
+				err = s.Manager.apiConfig.MarkVideoStatus(s.YoutubeChannelID, v.ID(), VideoStatusFailed, existingClaimID, existingClaimName, err.Error(), existingClaimSize, 1)
 				if err != nil {
 					SendErrorToSlack("Failed to mark video on the database: %s", err.Error())
 				}
@@ -749,14 +753,13 @@ func (s *Sync) enqueueYoutubeVideos() error {
 			break
 		}
 	}
-	notOnYoutube := make([]video, 0, len(s.syncedVideos))
 	for k, v := range s.syncedVideos {
 		if !v.Published {
 			continue
 		}
 		_, ok := playlistMap[k]
 		if !ok {
-			notOnYoutube = append(notOnYoutube, sources.NewMockedVideo(s.videoDirectory, k, s.YoutubeChannelID, s.Manager.GetS3AWSConfig()))
+			videos = append(videos, sources.NewMockedVideo(s.videoDirectory, k, s.YoutubeChannelID, s.Manager.GetS3AWSConfig()))
 		}
 
 	}
@@ -851,7 +854,7 @@ func (s *Sync) processVideo(v video) (err error) {
 		return err
 	}
 
-	err = s.Manager.apiConfig.MarkVideoStatus(s.YoutubeChannelID, v.ID(), VideoStatusPublished, summary.ClaimID, summary.ClaimName, "", v.Size())
+	err = s.Manager.apiConfig.MarkVideoStatus(s.YoutubeChannelID, v.ID(), VideoStatusPublished, summary.ClaimID, summary.ClaimName, "", v.Size(), 2)
 	if err != nil {
 		SendErrorToSlack("Failed to mark video on the database: %s", err.Error())
 	}

sdk/api.go

@@ -2,6 +2,7 @@ package sdk
 
 import (
 	"encoding/json"
+	"fmt"
 	"io/ioutil"
 	"net/http"
 	"net/url"
@@ -166,7 +167,7 @@ const (
 	VideoStatusFailed    = "failed"
 )
 
-func (a *APIConfig) MarkVideoStatus(channelID string, videoID string, status string, claimID string, claimName string, failureReason string, size *int64) error {
+func (a *APIConfig) MarkVideoStatus(channelID string, videoID string, status string, claimID string, claimName string, failureReason string, size *int64, metadataVersion uint) error {
 	endpoint := a.ApiURL + "/yt/video_status"
 
 	sanitizeFailureReason(&failureReason)
@@ -183,7 +184,7 @@ func (a *APIConfig) MarkVideoStatus(channelID string, videoID string, status str
 		vals.Add("published_at", strconv.FormatInt(time.Now().Unix(), 10))
 		vals.Add("claim_id", claimID)
 		vals.Add("claim_name", claimName)
-		vals.Add("metadata_version", "2")
+		vals.Add("metadata_version", fmt.Sprintf("%d", metadataVersion))
 		if size != nil {
 			vals.Add("size", strconv.FormatInt(*size, 10))
 		}

sources/youtubeVideo.go

@@ -453,11 +453,6 @@ func (v *YoutubeVideo) reprocess(daemon *jsonrpc.Client, params SyncParams, exis
 		}
 	}
 
-	videoDuration, err := duration.FromString(v.youtubeInfo.ContentDetails.Duration)
-	if err != nil {
-		return nil, errors.Err(err)
-	}
-
 	if v.mocked {
 		pr, err := daemon.StreamUpdate(existingVideoData.ClaimID, jsonrpc.StreamUpdateOptions{
 			StreamCreateOptions: &jsonrpc.StreamCreateOptions{
@@ -466,7 +461,7 @@ func (v *YoutubeVideo) reprocess(daemon *jsonrpc.Client, params SyncParams, exis
 					ThumbnailURL: &thumbnailURL,
 				},
 				Author:    util.PtrToString(""),
-				License:   util.PtrToString("Copyrighted (contact author)"),
+				License:   util.PtrToString("Copyrighted (contact publisher)"),
 				ChannelID: &v.lbryChannelID,
 			},
 			FileSize: &videoSize,
@@ -480,6 +475,12 @@ func (v *YoutubeVideo) reprocess(daemon *jsonrpc.Client, params SyncParams, exis
 			ClaimName: pr.Outputs[0].Name,
 		}, nil
 	}
+
+	videoDuration, err := duration.FromString(v.youtubeInfo.ContentDetails.Duration)
+	if err != nil {
+		return nil, errors.Err(err)
+	}
+
 	pr, err := daemon.StreamUpdate(existingVideoData.ClaimID, jsonrpc.StreamUpdateOptions{
 		ClearLanguages: util.PtrToBool(true),
 		ClearLocations: util.PtrToBool(true),
@@ -495,6 +496,8 @@ func (v *YoutubeVideo) reprocess(daemon *jsonrpc.Client, params SyncParams, exis
 			},
 			Author:      util.PtrToString(""),
 			License:     util.PtrToString("Copyrighted (contact publisher)"),
+			VideoHeight: util.PtrToUint(720),
+			VideoWidth:  util.PtrToUint(1280),
 			ReleaseTime: util.PtrToInt64(v.publishedAt.Unix()),
 			Duration:    util.PtrToUint64(uint64(math.Ceil(videoDuration.ToDuration().Seconds()))),
 			ChannelID:   &v.lbryChannelID,

tagsManager/tags_mapping.go

@@ -24,6 +24,11 @@ const (
 	Weapons    = "weapons"
 )
 
+func GetTagsForChannel(channelID string) []string {
+	tags, _ := channelWideTags[channelID]
+	return tags
+}
+
 func SanitizeTags(tags []string, youtubeChannelID string) ([]string, error) {
 	unsanitized := make([]string, 0, len(tags))
 	for _, t := range tags {
@@ -54,7 +59,7 @@ func SanitizeTags(tags []string, youtubeChannelID string) ([]string, error) {
 			originalTags = append(originalTags, t)
 		}
 	}
-	sanitizedTags := make([]string, len(originalTags)+len(curatedTags))
+	sanitizedTags := make([]string, 0, len(originalTags)+len(curatedTags))
 	sanitizedTags = append(sanitizedTags, curatedTags...)
 	sanitizedTags = append(sanitizedTags, originalTags...)
 	return sanitizedTags, nil
@@ -117,17 +122,19 @@ func (ts *tagsSanitizer) add() {
 	extraTags, ok := channelWideTags[ts.ChannelID]
 	if ok {
 		for _, t := range extraTags {
-			ts.Sanitized[t] = false
+			ts.Sanitized[t] = true
 		}
 	}
 }
 
 const (
-	Lunduke = "UCkK9UDm_ZNrq_rIXCz3xCGA"
+	Lunduke          = "UCkK9UDm_ZNrq_rIXCz3xCGA"
+	SwissExperiments = "UCNQfQvFMPnInwsU_iGYArJQ"
 )
 
 var channelWideTags = map[string][]string{
-	Lunduke: {"linux", "technology"},
+	Lunduke:          {"linux", "technology"},
+	SwissExperiments: {"science & technology", "experiments", "switzerland"},
 }
 var tagsToSkip = map[string]*struct{}{
 	"#hangoutsonair":                         nil,

tagsManager/tags_mapping_test.go

@@ -0,0 +1,46 @@
+package tagsManager
+
+import (
+	"testing"
+)
+
+func TestSanitizeTags(t *testing.T) {
+	got, err := SanitizeTags([]string{"this", "super", "expensive", "test", "has", "a lot of", "crypto", "currency", "in it", "trump", "will build the", "wall"}, "UCNQfQvFMPnInwsU_iGYArJQ")
+	if err != nil {
+		t.Error(err)
+		return
+	}
+	expectedTags := []string{
+		"blockchain",
+		"switzerland",
+		"news",
+		"science & technology",
+		"economics",
+		"experiments",
+		"this",
+		"in it",
+		"will build the",
+		"has",
+		"crypto",
+		"trump",
+		"wall",
+		"expensive",
+		"currency",
+		"a lot of",
+	}
+	if len(expectedTags) != len(got) {
+		t.Error("number of tags differ")
+		return
+	}
+outer:
+	for _, et := range expectedTags {
+		for _, t := range got {
+			if et == t {
+				continue outer
+			}
+		}
+		t.Error("tag not found")
+		return
+	}
+
+}