Add support for new SDK (0.37.*) and support for upgrading channels and claims to new metadata #28
6 changed files with 83 additions and 20 deletions
|
@ -11,6 +11,8 @@ import (
|
|||
"github.com/lbryio/lbry.go/extras/jsonrpc"
|
||||
"github.com/lbryio/lbry.go/extras/util"
|
||||
"github.com/lbryio/lbry.go/lbrycrd"
|
||||
|
||||
"github.com/lbryio/ytsync/tagsManager"
|
||||
"github.com/lbryio/ytsync/thumbs"
|
||||
|
||||
"github.com/shopspring/decimal"
|
||||
|
@ -355,13 +357,14 @@ func (s *Sync) ensureChannelOwnership() error {
|
|||
var c *jsonrpc.TransactionSummary
|
||||
if channelUsesOldMetadata {
|
||||
c, err = s.daemon.ChannelUpdate(s.lbryChannelID, jsonrpc.ChannelUpdateOptions{
|
||||
ClearTags: util.PtrToBool(true),
|
||||
ClearLocations: util.PtrToBool(true),
|
||||
ClearLanguages: util.PtrToBool(true),
|
||||
ChannelCreateOptions: jsonrpc.ChannelCreateOptions{
|
||||
ClaimCreateOptions: jsonrpc.ClaimCreateOptions{
|
||||
Title: channelInfo.Title,
|
||||
Description: channelInfo.Description,
|
||||
Tags: nil,
|
||||
Tags: tagsManager.GetTagsForChannel(s.YoutubeChannelID),
|
||||
Languages: languages,
|
||||
Locations: locations,
|
||||
ThumbnailURL: &thumbnailURL,
|
||||
|
@ -374,7 +377,7 @@ func (s *Sync) ensureChannelOwnership() error {
|
|||
ClaimCreateOptions: jsonrpc.ClaimCreateOptions{
|
||||
Title: channelInfo.Title,
|
||||
Description: channelInfo.Description,
|
||||
Tags: nil,
|
||||
Tags: tagsManager.GetTagsForChannel(s.YoutubeChannelID),
|
||||
Languages: languages,
|
||||
Locations: locations,
|
||||
ThumbnailURL: &thumbnailURL,
|
||||
|
|
|
@ -459,8 +459,12 @@ func (s *Sync) updateRemoteDB(claims []jsonrpc.Claim) (total int, fixed int, err
|
|||
if !ok || pv.ClaimName != c.Name {
|
||||
fixed++
|
||||
log.Debugf("adding %s to the database", c.Name)
|
||||
|
||||
err = s.Manager.apiConfig.MarkVideoStatus(s.YoutubeChannelID, videoID, VideoStatusPublished, c.ClaimID, c.Name, "", nil)
|
||||
size, err := c.GetStreamSizeByMagic()
|
||||
if err != nil {
|
||||
size = 0
|
||||
}
|
||||
metadataVersion := uint(1)
|
||||
err = s.Manager.apiConfig.MarkVideoStatus(s.YoutubeChannelID, videoID, VideoStatusPublished, c.ClaimID, c.Name, "", util.PtrToInt64(int64(size)), metadataVersion)
|
||||
if err != nil {
|
||||
return count, fixed, err
|
||||
}
|
||||
|
@ -663,7 +667,7 @@ func (s *Sync) startWorker(workerNum int) {
|
|||
}
|
||||
|
||||
s.AppendSyncedVideo(v.ID(), false, err.Error(), "")
|
||||
err = s.Manager.apiConfig.MarkVideoStatus(s.YoutubeChannelID, v.ID(), VideoStatusFailed, existingClaimID, existingClaimName, err.Error(), existingClaimSize)
|
||||
err = s.Manager.apiConfig.MarkVideoStatus(s.YoutubeChannelID, v.ID(), VideoStatusFailed, existingClaimID, existingClaimName, err.Error(), existingClaimSize, 1)
|
||||
if err != nil {
|
||||
SendErrorToSlack("Failed to mark video on the database: %s", err.Error())
|
||||
}
|
||||
|
@ -749,14 +753,13 @@ func (s *Sync) enqueueYoutubeVideos() error {
|
|||
break
|
||||
}
|
||||
}
|
||||
notOnYoutube := make([]video, 0, len(s.syncedVideos))
|
||||
for k, v := range s.syncedVideos {
|
||||
if !v.Published {
|
||||
continue
|
||||
}
|
||||
_, ok := playlistMap[k]
|
||||
if !ok {
|
||||
notOnYoutube = append(notOnYoutube, sources.NewMockedVideo(s.videoDirectory, k, s.YoutubeChannelID, s.Manager.GetS3AWSConfig()))
|
||||
videos = append(videos, sources.NewMockedVideo(s.videoDirectory, k, s.YoutubeChannelID, s.Manager.GetS3AWSConfig()))
|
||||
}
|
||||
|
||||
}
|
||||
|
@ -851,7 +854,7 @@ func (s *Sync) processVideo(v video) (err error) {
|
|||
return err
|
||||
}
|
||||
|
||||
err = s.Manager.apiConfig.MarkVideoStatus(s.YoutubeChannelID, v.ID(), VideoStatusPublished, summary.ClaimID, summary.ClaimName, "", v.Size())
|
||||
err = s.Manager.apiConfig.MarkVideoStatus(s.YoutubeChannelID, v.ID(), VideoStatusPublished, summary.ClaimID, summary.ClaimName, "", v.Size(), 2)
|
||||
if err != nil {
|
||||
SendErrorToSlack("Failed to mark video on the database: %s", err.Error())
|
||||
}
|
||||
|
|
|
@ -2,6 +2,7 @@ package sdk
|
|||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"net/http"
|
||||
"net/url"
|
||||
|
@ -166,7 +167,7 @@ const (
|
|||
VideoStatusFailed = "failed"
|
||||
)
|
||||
|
||||
func (a *APIConfig) MarkVideoStatus(channelID string, videoID string, status string, claimID string, claimName string, failureReason string, size *int64) error {
|
||||
func (a *APIConfig) MarkVideoStatus(channelID string, videoID string, status string, claimID string, claimName string, failureReason string, size *int64, metadataVersion uint) error {
|
||||
endpoint := a.ApiURL + "/yt/video_status"
|
||||
|
||||
sanitizeFailureReason(&failureReason)
|
||||
|
@ -183,7 +184,7 @@ func (a *APIConfig) MarkVideoStatus(channelID string, videoID string, status str
|
|||
vals.Add("published_at", strconv.FormatInt(time.Now().Unix(), 10))
|
||||
vals.Add("claim_id", claimID)
|
||||
vals.Add("claim_name", claimName)
|
||||
vals.Add("metadata_version", "2")
|
||||
vals.Add("metadata_version", fmt.Sprintf("%d", metadataVersion))
|
||||
if size != nil {
|
||||
vals.Add("size", strconv.FormatInt(*size, 10))
|
||||
}
|
||||
|
|
|
@ -453,11 +453,6 @@ func (v *YoutubeVideo) reprocess(daemon *jsonrpc.Client, params SyncParams, exis
|
|||
}
|
||||
}
|
||||
|
||||
videoDuration, err := duration.FromString(v.youtubeInfo.ContentDetails.Duration)
|
||||
if err != nil {
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
|
||||
if v.mocked {
|
||||
pr, err := daemon.StreamUpdate(existingVideoData.ClaimID, jsonrpc.StreamUpdateOptions{
|
||||
StreamCreateOptions: &jsonrpc.StreamCreateOptions{
|
||||
|
@ -466,7 +461,7 @@ func (v *YoutubeVideo) reprocess(daemon *jsonrpc.Client, params SyncParams, exis
|
|||
ThumbnailURL: &thumbnailURL,
|
||||
},
|
||||
Author: util.PtrToString(""),
|
||||
License: util.PtrToString("Copyrighted (contact author)"),
|
||||
License: util.PtrToString("Copyrighted (contact publisher)"),
|
||||
ChannelID: &v.lbryChannelID,
|
||||
},
|
||||
FileSize: &videoSize,
|
||||
|
@ -480,6 +475,12 @@ func (v *YoutubeVideo) reprocess(daemon *jsonrpc.Client, params SyncParams, exis
|
|||
ClaimName: pr.Outputs[0].Name,
|
||||
}, nil
|
||||
}
|
||||
|
||||
videoDuration, err := duration.FromString(v.youtubeInfo.ContentDetails.Duration)
|
||||
if err != nil {
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
|
||||
pr, err := daemon.StreamUpdate(existingVideoData.ClaimID, jsonrpc.StreamUpdateOptions{
|
||||
ClearLanguages: util.PtrToBool(true),
|
||||
ClearLocations: util.PtrToBool(true),
|
||||
|
@ -495,6 +496,8 @@ func (v *YoutubeVideo) reprocess(daemon *jsonrpc.Client, params SyncParams, exis
|
|||
},
|
||||
Author: util.PtrToString(""),
|
||||
License: util.PtrToString("Copyrighted (contact publisher)"),
|
||||
VideoHeight: util.PtrToUint(720),
|
||||
VideoWidth: util.PtrToUint(1280),
|
||||
ReleaseTime: util.PtrToInt64(v.publishedAt.Unix()),
|
||||
Duration: util.PtrToUint64(uint64(math.Ceil(videoDuration.ToDuration().Seconds()))),
|
||||
ChannelID: &v.lbryChannelID,
|
||||
|
|
|
@ -24,6 +24,11 @@ const (
|
|||
Weapons = "weapons"
|
||||
)
|
||||
|
||||
func GetTagsForChannel(channelID string) []string {
|
||||
tags, _ := channelWideTags[channelID]
|
||||
return tags
|
||||
}
|
||||
|
||||
func SanitizeTags(tags []string, youtubeChannelID string) ([]string, error) {
|
||||
unsanitized := make([]string, 0, len(tags))
|
||||
for _, t := range tags {
|
||||
|
@ -54,7 +59,7 @@ func SanitizeTags(tags []string, youtubeChannelID string) ([]string, error) {
|
|||
originalTags = append(originalTags, t)
|
||||
}
|
||||
}
|
||||
sanitizedTags := make([]string, len(originalTags)+len(curatedTags))
|
||||
sanitizedTags := make([]string, 0, len(originalTags)+len(curatedTags))
|
||||
sanitizedTags = append(sanitizedTags, curatedTags...)
|
||||
sanitizedTags = append(sanitizedTags, originalTags...)
|
||||
return sanitizedTags, nil
|
||||
|
@ -117,17 +122,19 @@ func (ts *tagsSanitizer) add() {
|
|||
extraTags, ok := channelWideTags[ts.ChannelID]
|
||||
if ok {
|
||||
for _, t := range extraTags {
|
||||
ts.Sanitized[t] = false
|
||||
ts.Sanitized[t] = true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const (
|
||||
Lunduke = "UCkK9UDm_ZNrq_rIXCz3xCGA"
|
||||
Lunduke = "UCkK9UDm_ZNrq_rIXCz3xCGA"
|
||||
SwissExperiments = "UCNQfQvFMPnInwsU_iGYArJQ"
|
||||
)
|
||||
|
||||
var channelWideTags = map[string][]string{
|
||||
Lunduke: {"linux", "technology"},
|
||||
Lunduke: {"linux", "technology"},
|
||||
SwissExperiments: {"science & technology", "experiments", "switzerland"},
|
||||
}
|
||||
var tagsToSkip = map[string]*struct{}{
|
||||
"#hangoutsonair": nil,
|
||||
|
|
46
tagsManager/tags_mapping_test.go
Normal file
46
tagsManager/tags_mapping_test.go
Normal file
|
@ -0,0 +1,46 @@
|
|||
package tagsManager
|
||||
|
||||
import (
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestSanitizeTags(t *testing.T) {
|
||||
got, err := SanitizeTags([]string{"this", "super", "expensive", "test", "has", "a lot of", "crypto", "currency", "in it", "trump", "will build the", "wall"}, "UCNQfQvFMPnInwsU_iGYArJQ")
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
return
|
||||
}
|
||||
expectedTags := []string{
|
||||
"blockchain",
|
||||
"switzerland",
|
||||
"news",
|
||||
"science & technology",
|
||||
"economics",
|
||||
"experiments",
|
||||
"this",
|
||||
"in it",
|
||||
"will build the",
|
||||
"has",
|
||||
"crypto",
|
||||
"trump",
|
||||
"wall",
|
||||
"expensive",
|
||||
"currency",
|
||||
"a lot of",
|
||||
}
|
||||
if len(expectedTags) != len(got) {
|
||||
t.Error("number of tags differ")
|
||||
return
|
||||
}
|
||||
outer:
|
||||
for _, et := range expectedTags {
|
||||
for _, t := range got {
|
||||
if et == t {
|
||||
continue outer
|
||||
}
|
||||
}
|
||||
t.Error("tag not found")
|
||||
return
|
||||
}
|
||||
|
||||
}
|
Loading…
Reference in a new issue