LBRYCommunity/ytsync
1
0
Fork 0
Code Issues 39 Pull requests 8 Projects Releases 197 Packages Wiki Activity

fix many bugs

Browse source 
calculate size for claims missing in the database
fix reprocessing for videos no longer on youtube
pass metadata instead of hardcoding it
add video height when updating streams
add tags to channels
add test for tags
This commit is contained in:
Niko Storni 2019-06-01 01:46:16 +02:00
parent 68c164ed5d
commit 3389a8be93
6 changed files with 83 additions and 20 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

7
manager/setup.go
View file

@ -11,6 +11,8 @@ import (
"github.com/lbryio/lbry.go/extras/jsonrpc"
"github.com/lbryio/lbry.go/extras/util"
"github.com/lbryio/lbry.go/lbrycrd"
"github.com/lbryio/ytsync/tagsManager"
"github.com/lbryio/ytsync/thumbs"
"github.com/shopspring/decimal"
@ -355,13 +357,14 @@ func (s *Sync) ensureChannelOwnership() error {
var c *jsonrpc.TransactionSummary
if channelUsesOldMetadata {
c, err = s.daemon.ChannelUpdate(s.lbryChannelID, jsonrpc.ChannelUpdateOptions{
ClearTags: util.PtrToBool(true),
ClearLocations: util.PtrToBool(true),
ClearLanguages: util.PtrToBool(true),
ChannelCreateOptions: jsonrpc.ChannelCreateOptions{
ClaimCreateOptions: jsonrpc.ClaimCreateOptions{
Title: channelInfo.Title,
Description: channelInfo.Description,
Tags: nil,
Tags: tagsManager.GetTagsForChannel(s.YoutubeChannelID),
Languages: languages,
Locations: locations,
ThumbnailURL: &thumbnailURL,
@ -374,7 +377,7 @@ func (s *Sync) ensureChannelOwnership() error {
ClaimCreateOptions: jsonrpc.ClaimCreateOptions{
Title: channelInfo.Title,
Description: channelInfo.Description,
Tags: nil,
Tags: tagsManager.GetTagsForChannel(s.YoutubeChannelID),
Languages: languages,
Locations: locations,
ThumbnailURL: &thumbnailURL,

15
manager/ytsync.go
View file

@ -459,8 +459,12 @@ func (s *Sync) updateRemoteDB(claims []jsonrpc.Claim) (total int, fixed int, err
if !ok || pv.ClaimName != c.Name {
fixed++
log.Debugf("adding %s to the database", c.Name)
err = s.Manager.apiConfig.MarkVideoStatus(s.YoutubeChannelID, videoID, VideoStatusPublished, c.ClaimID, c.Name, "", nil)
size, err := c.GetStreamSizeByMagic()
if err != nil {
size = 0
}
metadataVersion := uint(1)
err = s.Manager.apiConfig.MarkVideoStatus(s.YoutubeChannelID, videoID, VideoStatusPublished, c.ClaimID, c.Name, "", util.PtrToInt64(int64(size)), metadataVersion)
if err != nil {
return count, fixed, err
}
@ -663,7 +667,7 @@ func (s *Sync) startWorker(workerNum int) {
}
s.AppendSyncedVideo(v.ID(), false, err.Error(), "")
err = s.Manager.apiConfig.MarkVideoStatus(s.YoutubeChannelID, v.ID(), VideoStatusFailed, existingClaimID, existingClaimName, err.Error(), existingClaimSize)
err = s.Manager.apiConfig.MarkVideoStatus(s.YoutubeChannelID, v.ID(), VideoStatusFailed, existingClaimID, existingClaimName, err.Error(), existingClaimSize, 1)
if err != nil {
SendErrorToSlack("Failed to mark video on the database: %s", err.Error())
}
@ -749,14 +753,13 @@ func (s *Sync) enqueueYoutubeVideos() error {
break
}
}
notOnYoutube := make([]video, 0, len(s.syncedVideos))
for k, v := range s.syncedVideos {
if !v.Published {
continue
}
_, ok := playlistMap[k]
if !ok {
notOnYoutube = append(notOnYoutube, sources.NewMockedVideo(s.videoDirectory, k, s.YoutubeChannelID, s.Manager.GetS3AWSConfig()))
videos = append(videos, sources.NewMockedVideo(s.videoDirectory, k, s.YoutubeChannelID, s.Manager.GetS3AWSConfig()))
}
}
@ -851,7 +854,7 @@ func (s *Sync) processVideo(v video) (err error) {
return err
}
err = s.Manager.apiConfig.MarkVideoStatus(s.YoutubeChannelID, v.ID(), VideoStatusPublished, summary.ClaimID, summary.ClaimName, "", v.Size())
err = s.Manager.apiConfig.MarkVideoStatus(s.YoutubeChannelID, v.ID(), VideoStatusPublished, summary.ClaimID, summary.ClaimName, "", v.Size(), 2)
if err != nil {
SendErrorToSlack("Failed to mark video on the database: %s", err.Error())
}

5
sdk/api.go
View file

@ -2,6 +2,7 @@ package sdk
import (
"encoding/json"
"fmt"
"io/ioutil"
"net/http"
"net/url"
@ -166,7 +167,7 @@ const (
VideoStatusFailed = "failed"
)
func (a *APIConfig) MarkVideoStatus(channelID string, videoID string, status string, claimID string, claimName string, failureReason string, size *int64) error {
func (a *APIConfig) MarkVideoStatus(channelID string, videoID string, status string, claimID string, claimName string, failureReason string, size *int64, metadataVersion uint) error {
endpoint := a.ApiURL + "/yt/video_status"
sanitizeFailureReason(&failureReason)
@ -183,7 +184,7 @@ func (a *APIConfig) MarkVideoStatus(channelID string, videoID string, status str
vals.Add("published_at", strconv.FormatInt(time.Now().Unix(), 10))
vals.Add("claim_id", claimID)
vals.Add("claim_name", claimName)
vals.Add("metadata_version", "2")
vals.Add("metadata_version", fmt.Sprintf("%d", metadataVersion))
if size != nil {
vals.Add("size", strconv.FormatInt(*size, 10))
}

15
sources/youtubeVideo.go
View file

@ -453,11 +453,6 @@ func (v *YoutubeVideo) reprocess(daemon *jsonrpc.Client, params SyncParams, exis
}
}
videoDuration, err := duration.FromString(v.youtubeInfo.ContentDetails.Duration)
if err != nil {
return nil, errors.Err(err)
}
if v.mocked {
pr, err := daemon.StreamUpdate(existingVideoData.ClaimID, jsonrpc.StreamUpdateOptions{
StreamCreateOptions: &jsonrpc.StreamCreateOptions{
@ -466,7 +461,7 @@ func (v *YoutubeVideo) reprocess(daemon *jsonrpc.Client, params SyncParams, exis
ThumbnailURL: &thumbnailURL,
},
Author: util.PtrToString(""),
License: util.PtrToString("Copyrighted (contact author)"),
License: util.PtrToString("Copyrighted (contact publisher)"),
ChannelID: &v.lbryChannelID,
},
FileSize: &videoSize,
@ -480,6 +475,12 @@ func (v *YoutubeVideo) reprocess(daemon *jsonrpc.Client, params SyncParams, exis
ClaimName: pr.Outputs[0].Name,
}, nil
}
videoDuration, err := duration.FromString(v.youtubeInfo.ContentDetails.Duration)
if err != nil {
return nil, errors.Err(err)
}
pr, err := daemon.StreamUpdate(existingVideoData.ClaimID, jsonrpc.StreamUpdateOptions{
ClearLanguages: util.PtrToBool(true),
ClearLocations: util.PtrToBool(true),
@ -495,6 +496,8 @@ func (v *YoutubeVideo) reprocess(daemon *jsonrpc.Client, params SyncParams, exis
},
Author: util.PtrToString(""),
License: util.PtrToString("Copyrighted (contact publisher)"),
VideoHeight: util.PtrToUint(720),
VideoWidth: util.PtrToUint(1280),
ReleaseTime: util.PtrToInt64(v.publishedAt.Unix()),
Duration: util.PtrToUint64(uint64(math.Ceil(videoDuration.ToDuration().Seconds()))),
ChannelID: &v.lbryChannelID,

15
tagsManager/tags_mapping.go
View file

@ -24,6 +24,11 @@ const (
Weapons = "weapons"
)
func GetTagsForChannel(channelID string) []string {
tags, _ := channelWideTags[channelID]
return tags
}
func SanitizeTags(tags []string, youtubeChannelID string) ([]string, error) {
unsanitized := make([]string, 0, len(tags))
for _, t := range tags {
@ -54,7 +59,7 @@ func SanitizeTags(tags []string, youtubeChannelID string) ([]string, error) {
originalTags = append(originalTags, t)
}
}
sanitizedTags := make([]string, len(originalTags)+len(curatedTags))
sanitizedTags := make([]string, 0, len(originalTags)+len(curatedTags))
sanitizedTags = append(sanitizedTags, curatedTags...)
sanitizedTags = append(sanitizedTags, originalTags...)
return sanitizedTags, nil
@ -117,17 +122,19 @@ func (ts *tagsSanitizer) add() {
extraTags, ok := channelWideTags[ts.ChannelID]
if ok {
for _, t := range extraTags {
ts.Sanitized[t] = false
ts.Sanitized[t] = true
}
}
}
const (
Lunduke = "UCkK9UDm_ZNrq_rIXCz3xCGA"
Lunduke = "UCkK9UDm_ZNrq_rIXCz3xCGA"
SwissExperiments = "UCNQfQvFMPnInwsU_iGYArJQ"
)
var channelWideTags = map[string][]string{
Lunduke: {"linux", "technology"},
Lunduke: {"linux", "technology"},
SwissExperiments: {"science & technology", "experiments", "switzerland"},
}
var tagsToSkip = map[string]*struct{}{
"#hangoutsonair": nil,

46
tagsManager/tags_mapping_test.go Normal file
View file

@ -0,0 +1,46 @@
package tagsManager
import (
"testing"
)
func TestSanitizeTags(t *testing.T) {
got, err := SanitizeTags([]string{"this", "super", "expensive", "test", "has", "a lot of", "crypto", "currency", "in it", "trump", "will build the", "wall"}, "UCNQfQvFMPnInwsU_iGYArJQ")
if err != nil {
t.Error(err)
return
}
expectedTags := []string{
"blockchain",
"switzerland",
"news",
"science & technology",
"economics",
"experiments",
"this",
"in it",
"will build the",
"has",
"crypto",
"trump",
"wall",
"expensive",
"currency",
"a lot of",
}
if len(expectedTags) != len(got) {
t.Error("number of tags differ")
return
}
outer:
for _, et := range expectedTags {
for _, t := range got {
if et == t {
continue outer
}
}
t.Error("tag not found")
return
}
}