LBRYCommunity/ytsync
1
0
Fork 0
Code Issues 39 Pull requests 8 Projects Releases 197 Packages Wiki Activity

Add support for new SDK (0.37.*) and support for upgrading channels and claims to new metadata #28

Merged
nikooo777 merged 37 commits from metadata_upgrade into master 2019-06-13 20:14:14 +02:00
Conversation 0 Commits 37 Files changed 16 +72 -38
5 changed files with 72 additions and 38 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files
Showing only changes of commit 502bfa924b - Show all commits

2
go.mod
View file

@ -17,7 +17,7 @@ require (
github.com/konsorten/go-windows-terminal-sequences v1.0.2 // indirect
github.com/kr/pretty v0.1.0 // indirect
github.com/lbryio/errors.go v0.0.0-20180223142025-ad03d3cc6a5c
github.com/lbryio/lbry.go v0.0.0-20190503025608-3a22a0af0a45
github.com/lbryio/lbry.go v0.0.0-20190506195255-f01bba8d4336
github.com/lusis/slack-test v0.0.0-20190408224659-6cf59653add2 // indirect
github.com/mitchellh/go-ps v0.0.0-20170309133038-4fdf99ab2936
github.com/mitchellh/mapstructure v1.1.2 // indirect

7
go.sum
View file

@ -125,8 +125,10 @@ github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/lbryio/errors.go v0.0.0-20180223142025-ad03d3cc6a5c h1:BhdcWGsuKif/XoSZnqVGNqJ1iEmH0czWR5upj+AuR8M=
github.com/lbryio/errors.go v0.0.0-20180223142025-ad03d3cc6a5c/go.mod h1:muH7wpUqE8hRA3OrYYosw9+Sl681BF9cwcjzE+OCNK8=
github.com/lbryio/lbry.go v0.0.0-20190503025608-3a22a0af0a45 h1:6fpRUui1G8xKiQqL51fHJwrpqYSd4G3RIYWsRd+R8x8=
github.com/lbryio/lbry.go v0.0.0-20190503025608-3a22a0af0a45/go.mod h1:rAREtjHq/Wkkqy4+yt89Er/7JAVu6uMo3v8nevCEEIA=
github.com/lbryio/lbry.go v0.0.0-20190506193558-20f2cbad6754 h1:Kim8dIsKTrv8L3GCtL91atvVC3CD+t1TlPwmRJ9EmCQ=
github.com/lbryio/lbry.go v0.0.0-20190506193558-20f2cbad6754/go.mod h1:rAREtjHq/Wkkqy4+yt89Er/7JAVu6uMo3v8nevCEEIA=
github.com/lbryio/lbry.go v0.0.0-20190506195255-f01bba8d4336 h1:4jNCUftcPnkCwkyHBVaJXmtUIGN2FLsxpr+ZO+NLSy0=
github.com/lbryio/lbry.go v0.0.0-20190506195255-f01bba8d4336/go.mod h1:rAREtjHq/Wkkqy4+yt89Er/7JAVu6uMo3v8nevCEEIA=
github.com/lbryio/lbryschema.go v0.0.0-20190428231007-c54836bca002 h1:urfYK5ElpUrAv90auPLldoVC60LwiGAcY0OE6HJB9KI=
github.com/lbryio/lbryschema.go v0.0.0-20190428231007-c54836bca002/go.mod h1:dAzPCBj3CKKWBGYBZxK6tKBP5SCgY2tqd9SnQd/OyKo=
github.com/lbryio/ozzo-validation v0.0.0-20170323141101-d1008ad1fd04/go.mod h1:fbG/dzobG8r95KzMwckXiLMHfFjZaBRQqC9hPs2XAQ4=
@ -178,6 +180,7 @@ github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R
github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
github.com/sebdah/goldie v0.0.0-20180424091453-8784dd1ab561/go.mod h1:lvjGftC8oe7XPtyrOidaMi0rp5B9+XY/ZRUynGnuaxQ=
github.com/sergi/go-diff v1.0.0 h1:Kpca3qRNrduNnOQeazBd0ysaKrUJiIuISHxogkT9RPQ=
github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
github.com/shopspring/decimal v0.0.0-20180607144847-19e3cb6c2930/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4=
github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24 h1:pntxY8Ary0t43dCZ5dqY4YTJCObLY1kIXl0uzMv+7DE=

42
manager/setup.go
View file

@ -19,6 +19,27 @@ import (
"google.golang.org/api/youtube/v3"
)
const minimumRefillAmount = 3
func (s *Sync) enableAddressReuse() error {
accountsResponse, err := s.daemon.AccountList()
if err != nil {
return errors.Err(err)
}
accounts := accountsResponse.LBCMainnet
if os.Getenv("REGTEST") == "true" {
accounts = accountsResponse.LBCRegtest
}
for _, a := range accounts {
_, err = s.daemon.AccountSet(a.ID, jsonrpc.AccountSettings{
ChangeMaxUses: 1000,
})
if err != nil {
return errors.Err(err)
}
}
return nil
}
func (s *Sync) walletSetup() error {
//prevent unnecessary concurrent execution
s.walletMux.Lock()
@ -40,16 +61,11 @@ func (s *Sync) walletSetup() error {
}
log.Debugf("Starting balance is %.4f", balance)
var numOnSource int
if s.LbryChannelName == "@UCBerkeley" {
numOnSource = 10104
} else {
n, err := s.CountVideos()
if err != nil {
return err
}
numOnSource = int(n)
n, err := s.CountVideos()
if err != nil {
return err
}
numOnSource := int(n)
log.Debugf("Source channel has %d videos", numOnSource)
if numOnSource == 0 {
@ -66,9 +82,9 @@ func (s *Sync) walletSetup() error {
}
minBalance := (float64(numOnSource)-float64(numPublished))*(publishAmount+0.1) + channelClaimAmount
if numPublished > numOnSource && balance < 1 {
if numPublished > numOnSource && balance < minimumRefillAmount {
SendErrorToSlack("something is going on as we published more videos than those available on source: %d/%d", numPublished, numOnSource)
minBalance = 1 //since we ended up in this function it means some juice is still needed
minBalance = minimumRefillAmount
}
amountToAdd := minBalance - balance
@ -81,8 +97,8 @@ func (s *Sync) walletSetup() error {
}
if amountToAdd > 0 {
if amountToAdd < 1 {
amountToAdd = 1 // no reason to bother adding less than 1 credit
if amountToAdd < minimumRefillAmount {
amountToAdd = minimumRefillAmount
}
err := s.addCredits(amountToAdd)
if err != nil {

5
manager/ytsync.go
View file

@ -479,7 +479,10 @@ func (s *Sync) getClaims() ([]jsonrpc.Claim, error) {
}
func (s *Sync) doSync() error {
var err error
err := s.enableAddressReuse()
if err != nil {
return errors.Prefix("could not set address reuse policy", err)
}
err = s.walletSetup()
if err != nil {
return errors.Prefix("Initial wallet setup failed! Manual Intervention is required.", err)

54
sources/youtubeVideo.go
View file

@ -13,13 +13,13 @@ import (
"strings"
"time"
"github.com/lbryio/ytsync/sdk"
"github.com/lbryio/ytsync/thumbs"
"github.com/lbryio/lbry.go/extras/errors"
"github.com/lbryio/lbry.go/extras/jsonrpc"
"github.com/lbryio/lbry.go/extras/util"
"github.com/lbryio/ytsync/namer"
"github.com/lbryio/ytsync/sdk"
"github.com/lbryio/ytsync/thumbs"
"github.com/ChannelMeter/iso8601duration"
"github.com/aws/aws-sdk-go/aws"
@ -44,6 +44,8 @@ type YoutubeVideo struct {
awsConfig aws.Config
}
const reflectorURL = "http://blobs.lbry.io/"
var youtubeCategories = map[string]string{
"1": "Film & Animation",
"2": "Autos & Vehicles",
@ -285,8 +287,6 @@ func (v *YoutubeVideo) triggerThumbnailSave() error {
return nil
}
func strPtr(s string) *string { return &s }
func (v *YoutubeVideo) publish(daemon *jsonrpc.Client, claimAddress string, amount float64, channelID string, namer *namer.Namer) (*SyncSummary, error) {
additionalDescription := "\nhttps://www.youtube.com/watch?v=" + v.id
khanAcademyClaimID := "5fc52291980268b82413ca4c0ace1b8d749f3ffb"
@ -309,15 +309,15 @@ func (v *YoutubeVideo) publish(daemon *jsonrpc.Client, claimAddress string, amou
Description: v.getAbbrevDescription() + additionalDescription,
ClaimAddress: &claimAddress,
Languages: languages,
ThumbnailURL: strPtr(thumbs.ThumbnailEndpoint + v.id),
ThumbnailURL: util.PtrToString(thumbs.ThumbnailEndpoint + v.id),
Tags: v.youtubeInfo.Snippet.Tags,
},
Author: strPtr(v.channelTitle),
License: strPtr("Copyrighted (contact author)"),
StreamType: &jsonrpc.StreamTypeVideo,
ReleaseTime: util.PtrToInt64(v.publishedAt.Unix()),
VideoDuration: util.PtrToUint64(uint64(math.Ceil(videoDuration.ToDuration().Seconds()))),
ChannelID: &channelID,
Author: util.PtrToString(v.channelTitle),
License: util.PtrToString("Copyrighted (contact author)"),
StreamType: &jsonrpc.StreamTypeVideo,
ReleaseTime: util.PtrToInt64(v.publishedAt.Unix()),
Duration: util.PtrToUint64(uint64(math.Ceil(videoDuration.ToDuration().Seconds()))),
ChannelID: &channelID,
}
return publishAndRetryExistingNames(daemon, v.title, v.getFullPath(), amount, options, namer)
@ -341,10 +341,13 @@ func (v *YoutubeVideo) Sync(daemon *jsonrpc.Client, params SyncParams, existingV
v.maxVideoLength = params.MaxVideoLength
if reprocess {
summary, err := v.reprocess(daemon, params.ChannelID, existingVideoData)
summary, err := v.reprocess(daemon, params, existingVideoData)
return summary, err
}
return v.downloadAndPublish(daemon, params)
}
func (v *YoutubeVideo) downloadAndPublish(daemon *jsonrpc.Client, params SyncParams) (*SyncSummary, error) {
//download and thumbnail can be done in parallel
err := v.download()
if err != nil {
@ -365,7 +368,7 @@ func (v *YoutubeVideo) Sync(daemon *jsonrpc.Client, params SyncParams, existingV
return summary, errors.Prefix("publish error", err)
}
func (v *YoutubeVideo) reprocess(daemon *jsonrpc.Client, channelID string, existingVideoData *sdk.SyncedVideo) (*SyncSummary, error) {
func (v *YoutubeVideo) reprocess(daemon *jsonrpc.Client, params SyncParams, existingVideoData *sdk.SyncedVideo) (*SyncSummary, error) {
c, err := daemon.ClaimSearch(nil, &existingVideoData.ClaimID, nil, nil)
if err != nil {
return nil, errors.Err(err)
@ -400,6 +403,15 @@ func (v *YoutubeVideo) reprocess(daemon *jsonrpc.Client, channelID string, exist
tags := append(v.youtubeInfo.Snippet.Tags, youtubeCategories[v.youtubeInfo.Snippet.CategoryId])
videoSize, err := currentClaim.GetStreamSizeByMagic()
if err != nil {
if existingVideoData.Size > 0 {
videoSize = uint64(existingVideoData.Size)
} else {
log.Infof("%s: the video must be republished as we can't get the right size", v.ID())
return v.downloadAndPublish(daemon, params)
}
}
videoDuration, err := duration.FromString(v.youtubeInfo.ContentDetails.Duration)
_, err = daemon.StreamUpdate(existingVideoData.ClaimID, jsonrpc.StreamUpdateOptions{
StreamCreateOptions: &jsonrpc.StreamCreateOptions{
@ -411,13 +423,13 @@ func (v *YoutubeVideo) reprocess(daemon *jsonrpc.Client, channelID string, exist
Locations: locations,
ThumbnailURL: &thumbnailURL,
},
Author: util.PtrToString(""),
License: strPtr("Copyrighted (contact author)"),
ReleaseTime: util.PtrToInt64(v.publishedAt.Unix()),
VideoDuration: util.PtrToUint64(uint64(math.Ceil(videoDuration.ToDuration().Seconds()))),
ChannelID: &channelID,
Author: util.PtrToString(""),
License: util.PtrToString("Copyrighted (contact author)"),
ReleaseTime: util.PtrToInt64(v.publishedAt.Unix()),
Duration: util.PtrToUint64(uint64(math.Ceil(videoDuration.ToDuration().Seconds()))),
ChannelID: &params.ChannelID,
},
FileSize: util.PtrToString(fmt.Sprintf("%d", videoSize)),
})
return &SyncSummary{}, nil
}