LBRYCommunity/ytsync
1
0
Fork 0
Code Issues 39 Pull requests 8 Packages Projects Releases 197 Wiki Activity

Add support for new SDK (0.37.*) and support for upgrading channels and claims to new metadata #28

Merged
nikooo777 merged 37 commits from metadata_upgrade into master 2019-06-13 20:14:14 +02:00
Conversation 0 Commits 37 Files changed 16 +83 -20
6 changed files with 83 additions and 20 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files
Showing only changes of commit 3389a8be93 - Show all commits

7
manager/setup.go
View file

@ -11,6 +11,8 @@ import (
"github.com/lbryio/lbry.go/extras/jsonrpc"
"github.com/lbryio/lbry.go/extras/util"
"github.com/lbryio/lbry.go/lbrycrd"
"github.com/lbryio/ytsync/tagsManager"
"github.com/lbryio/ytsync/thumbs"
"github.com/shopspring/decimal"
@ -355,13 +357,14 @@ func (s *Sync) ensureChannelOwnership() error {
var c *jsonrpc.TransactionSummary
if channelUsesOldMetadata {
c, err = s.daemon.ChannelUpdate(s.lbryChannelID, jsonrpc.ChannelUpdateOptions{
ClearTags: util.PtrToBool(true),
ClearLocations: util.PtrToBool(true),
ClearLanguages: util.PtrToBool(true),
ChannelCreateOptions: jsonrpc.ChannelCreateOptions{
ClaimCreateOptions: jsonrpc.ClaimCreateOptions{
Title: channelInfo.Title,
Description: channelInfo.Description,
Tags: nil,
Tags: tagsManager.GetTagsForChannel(s.YoutubeChannelID),
Languages: languages,
Locations: locations,
ThumbnailURL: &thumbnailURL,
@ -374,7 +377,7 @@ func (s *Sync) ensureChannelOwnership() error {
ClaimCreateOptions: jsonrpc.ClaimCreateOptions{
Title: channelInfo.Title,
Description: channelInfo.Description,
Tags: nil,
Tags: tagsManager.GetTagsForChannel(s.YoutubeChannelID),
Languages: languages,
Locations: locations,
ThumbnailURL: &thumbnailURL,

15
manager/ytsync.go
View file

@ -459,8 +459,12 @@ func (s *Sync) updateRemoteDB(claims []jsonrpc.Claim) (total int, fixed int, err
if !ok || pv.ClaimName != c.Name {
fixed++
log.Debugf("adding %s to the database", c.Name)
err = s.Manager.apiConfig.MarkVideoStatus(s.YoutubeChannelID, videoID, VideoStatusPublished, c.ClaimID, c.Name, "", nil)
size, err := c.GetStreamSizeByMagic()
if err != nil {
size = 0
}
metadataVersion := uint(1)
err = s.Manager.apiConfig.MarkVideoStatus(s.YoutubeChannelID, videoID, VideoStatusPublished, c.ClaimID, c.Name, "", util.PtrToInt64(int64(size)), metadataVersion)
if err != nil {
return count, fixed, err
}
@ -663,7 +667,7 @@ func (s *Sync) startWorker(workerNum int) {
}
s.AppendSyncedVideo(v.ID(), false, err.Error(), "")
err = s.Manager.apiConfig.MarkVideoStatus(s.YoutubeChannelID, v.ID(), VideoStatusFailed, existingClaimID, existingClaimName, err.Error(), existingClaimSize)
err = s.Manager.apiConfig.MarkVideoStatus(s.YoutubeChannelID, v.ID(), VideoStatusFailed, existingClaimID, existingClaimName, err.Error(), existingClaimSize, 1)
if err != nil {
SendErrorToSlack("Failed to mark video on the database: %s", err.Error())
}
@ -749,14 +753,13 @@ func (s *Sync) enqueueYoutubeVideos() error {
break
}
}
notOnYoutube := make([]video, 0, len(s.syncedVideos))
for k, v := range s.syncedVideos {
if !v.Published {
continue
}
_, ok := playlistMap[k]
if !ok {
notOnYoutube = append(notOnYoutube, sources.NewMockedVideo(s.videoDirectory, k, s.YoutubeChannelID, s.Manager.GetS3AWSConfig()))
videos = append(videos, sources.NewMockedVideo(s.videoDirectory, k, s.YoutubeChannelID, s.Manager.GetS3AWSConfig()))
}
}
@ -851,7 +854,7 @@ func (s *Sync) processVideo(v video) (err error) {
return err
}
err = s.Manager.apiConfig.MarkVideoStatus(s.YoutubeChannelID, v.ID(), VideoStatusPublished, summary.ClaimID, summary.ClaimName, "", v.Size())
err = s.Manager.apiConfig.MarkVideoStatus(s.YoutubeChannelID, v.ID(), VideoStatusPublished, summary.ClaimID, summary.ClaimName, "", v.Size(), 2)
if err != nil {
SendErrorToSlack("Failed to mark video on the database: %s", err.Error())
}

5
sdk/api.go
View file

@ -2,6 +2,7 @@ package sdk
import (
"encoding/json"
"fmt"
"io/ioutil"
"net/http"
"net/url"
@ -166,7 +167,7 @@ const (
VideoStatusFailed = "failed"
)
func (a *APIConfig) MarkVideoStatus(channelID string, videoID string, status string, claimID string, claimName string, failureReason string, size *int64) error {
func (a *APIConfig) MarkVideoStatus(channelID string, videoID string, status string, claimID string, claimName string, failureReason string, size *int64, metadataVersion uint) error {
endpoint := a.ApiURL + "/yt/video_status"
sanitizeFailureReason(&failureReason)
@ -183,7 +184,7 @@ func (a *APIConfig) MarkVideoStatus(channelID string, videoID string, status str
vals.Add("published_at", strconv.FormatInt(time.Now().Unix(), 10))
vals.Add("claim_id", claimID)
vals.Add("claim_name", claimName)
vals.Add("metadata_version", "2")
vals.Add("metadata_version", fmt.Sprintf("%d", metadataVersion))
if size != nil {
vals.Add("size", strconv.FormatInt(*size, 10))
}

15
sources/youtubeVideo.go
View file

@ -453,11 +453,6 @@ func (v *YoutubeVideo) reprocess(daemon *jsonrpc.Client, params SyncParams, exis
}
}
videoDuration, err := duration.FromString(v.youtubeInfo.ContentDetails.Duration)
if err != nil {
return nil, errors.Err(err)
}
if v.mocked {
pr, err := daemon.StreamUpdate(existingVideoData.ClaimID, jsonrpc.StreamUpdateOptions{
StreamCreateOptions: &jsonrpc.StreamCreateOptions{
@ -466,7 +461,7 @@ func (v *YoutubeVideo) reprocess(daemon *jsonrpc.Client, params SyncParams, exis
ThumbnailURL: &thumbnailURL,
},
Author: util.PtrToString(""),
License: util.PtrToString("Copyrighted (contact author)"),
License: util.PtrToString("Copyrighted (contact publisher)"),
ChannelID: &v.lbryChannelID,
},
FileSize: &videoSize,
@ -480,6 +475,12 @@ func (v *YoutubeVideo) reprocess(daemon *jsonrpc.Client, params SyncParams, exis
ClaimName: pr.Outputs[0].Name,
}, nil
}
videoDuration, err := duration.FromString(v.youtubeInfo.ContentDetails.Duration)
if err != nil {
return nil, errors.Err(err)
}
pr, err := daemon.StreamUpdate(existingVideoData.ClaimID, jsonrpc.StreamUpdateOptions{
ClearLanguages: util.PtrToBool(true),
ClearLocations: util.PtrToBool(true),
@ -495,6 +496,8 @@ func (v *YoutubeVideo) reprocess(daemon *jsonrpc.Client, params SyncParams, exis
},
Author: util.PtrToString(""),
License: util.PtrToString("Copyrighted (contact publisher)"),
VideoHeight: util.PtrToUint(720),
VideoWidth: util.PtrToUint(1280),
ReleaseTime: util.PtrToInt64(v.publishedAt.Unix()),
Duration: util.PtrToUint64(uint64(math.Ceil(videoDuration.ToDuration().Seconds()))),
ChannelID: &v.lbryChannelID,

15
tagsManager/tags_mapping.go
View file

@ -24,6 +24,11 @@ const (
Weapons = "weapons"
)
func GetTagsForChannel(channelID string) []string {
tags, _ := channelWideTags[channelID]
return tags
}
func SanitizeTags(tags []string, youtubeChannelID string) ([]string, error) {
unsanitized := make([]string, 0, len(tags))
for _, t := range tags {
@ -54,7 +59,7 @@ func SanitizeTags(tags []string, youtubeChannelID string) ([]string, error) {
originalTags = append(originalTags, t)
}
}
sanitizedTags := make([]string, len(originalTags)+len(curatedTags))
sanitizedTags := make([]string, 0, len(originalTags)+len(curatedTags))
sanitizedTags = append(sanitizedTags, curatedTags...)
sanitizedTags = append(sanitizedTags, originalTags...)
return sanitizedTags, nil
@ -117,17 +122,19 @@ func (ts *tagsSanitizer) add() {
extraTags, ok := channelWideTags[ts.ChannelID]
if ok {
for _, t := range extraTags {
ts.Sanitized[t] = false
ts.Sanitized[t] = true
}
}
}
const (
Lunduke = "UCkK9UDm_ZNrq_rIXCz3xCGA"
Lunduke = "UCkK9UDm_ZNrq_rIXCz3xCGA"
SwissExperiments = "UCNQfQvFMPnInwsU_iGYArJQ"
)
var channelWideTags = map[string][]string{
Lunduke: {"linux", "technology"},
Lunduke: {"linux", "technology"},
SwissExperiments: {"science & technology", "experiments", "switzerland"},
}
var tagsToSkip = map[string]*struct{}{
"#hangoutsonair": nil,

46
tagsManager/tags_mapping_test.go Normal file
View file

@ -0,0 +1,46 @@
package tagsManager
import (
"testing"
)
func TestSanitizeTags(t *testing.T) {
got, err := SanitizeTags([]string{"this", "super", "expensive", "test", "has", "a lot of", "crypto", "currency", "in it", "trump", "will build the", "wall"}, "UCNQfQvFMPnInwsU_iGYArJQ")
if err != nil {
t.Error(err)
return
}
expectedTags := []string{
"blockchain",
"switzerland",
"news",
"science & technology",
"economics",
"experiments",
"this",
"in it",
"will build the",
"has",
"crypto",
"trump",
"wall",
"expensive",
"currency",
"a lot of",
}
if len(expectedTags) != len(got) {
t.Error("number of tags differ")
return
}
outer:
for _, et := range expectedTags {
for _, t := range got {
if et == t {
continue outer
}
}
t.Error("tag not found")
return
}
}