LBRYCommunity/ytsync
1
0
Fork 0
Code Issues 39 Pull requests 8 Projects Releases 197 Packages Wiki Activity

Add support for new SDK (0.37.*) and support for upgrading channels and claims to new metadata #28

Merged
nikooo777 merged 37 commits from metadata_upgrade into master 2019-06-13 20:14:14 +02:00
Conversation 0 Commits 37 Files changed 16 +38 -21
6 changed files with 38 additions and 21 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files
Showing only changes of commit dce71d2451 - Show all commits

2
go.mod
View file

@ -15,7 +15,7 @@ require (
github.com/konsorten/go-windows-terminal-sequences v1.0.2 // indirect
github.com/kr/pretty v0.1.0 // indirect
github.com/lbryio/errors.go v0.0.0-20180223142025-ad03d3cc6a5c
github.com/lbryio/lbry.go v1.0.11
github.com/lbryio/lbry.go v1.0.12
github.com/lusis/slack-test v0.0.0-20190408224659-6cf59653add2 // indirect
github.com/mitchellh/go-ps v0.0.0-20170309133038-4fdf99ab2936
github.com/mitchellh/mapstructure v1.1.2 // indirect

4
go.sum
View file

@ -129,8 +129,8 @@ github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/lbryio/errors.go v0.0.0-20180223142025-ad03d3cc6a5c h1:BhdcWGsuKif/XoSZnqVGNqJ1iEmH0czWR5upj+AuR8M=
github.com/lbryio/errors.go v0.0.0-20180223142025-ad03d3cc6a5c/go.mod h1:muH7wpUqE8hRA3OrYYosw9+Sl681BF9cwcjzE+OCNK8=
github.com/lbryio/lbry.go v1.0.11 h1:dTaTNWF5wyWX9WQrgpIolfDEfGLl/ay2c2f2WOMayY4=
github.com/lbryio/lbry.go v1.0.11/go.mod h1:JtyI30bU51rm0LZ/po3mQuzf++14OWb6kR/6mMRAmKU=
github.com/lbryio/lbry.go v1.0.12 h1:hS0d3uBdohz3zvr5tCWpxiMkjkV3Kkq1Q+0HrfJBXlQ=
github.com/lbryio/lbry.go v1.0.12/go.mod h1:JtyI30bU51rm0LZ/po3mQuzf++14OWb6kR/6mMRAmKU=
github.com/lbryio/lbryschema.go v0.0.0-20190428231007-c54836bca002 h1:urfYK5ElpUrAv90auPLldoVC60LwiGAcY0OE6HJB9KI=
github.com/lbryio/lbryschema.go v0.0.0-20190428231007-c54836bca002/go.mod h1:dAzPCBj3CKKWBGYBZxK6tKBP5SCgY2tqd9SnQd/OyKo=
github.com/lbryio/ozzo-validation v0.0.0-20170323141101-d1008ad1fd04 h1:Nze+C2HbeKvhjI/kVn+9Poj/UuEW5sOQxcsxqO7L3GI=

4
manager/setup.go
View file

@ -1,6 +1,7 @@
package manager
import (
"fmt"
"math"
"net/http"
"os"
@ -187,7 +188,8 @@ func (s *Sync) ensureEnoughUTXOs() error {
desiredUTXOCount := uint64(math.Floor((balanceAmount) / 0.1))
log.Infof("Splitting balance of %s evenly between %d UTXOs", *balance, desiredUTXOCount)
prefillTx, err := s.daemon.AccountFund(defaultAccount, defaultAccount, "0.0", desiredUTXOCount, true)
bradcastFee := 0.01
prefillTx, err := s.daemon.AccountFund(defaultAccount, defaultAccount, fmt.Sprintf("%.4f", balanceAmount-bradcastFee), desiredUTXOCount, false)
if err != nil {
return err
} else if prefillTx == nil {

29
manager/ytsync.go
View file

@ -40,7 +40,7 @@ const (
channelClaimAmount = 0.01
estimatedMaxTxFee = 0.1
minimumAccountBalance = 4.0
minimumRefillAmount = 4
minimumRefillAmount = 1
publishAmount = 0.01
maxReasonLength = 500
)
@ -90,13 +90,17 @@ type Sync struct {
queue chan video
}
func (s *Sync) AppendSyncedVideo(videoID string, published bool, failureReason string, claimName string) {
func (s *Sync) AppendSyncedVideo(videoID string, published bool, failureReason string, claimName string, claimID string, metadataVersion int8, size int64) {
s.syncedVideosMux.Lock()
defer s.syncedVideosMux.Unlock()
s.syncedVideos[videoID] = sdk.SyncedVideo{
VideoID: videoID,
Published: published,
FailureReason: failureReason,
VideoID: videoID,
Published: published,
FailureReason: failureReason,
ClaimID: claimID,
ClaimName: claimName,
MetadataVersion: metadataVersion,
Size: size,
}
}
@ -706,20 +710,24 @@ func (s *Sync) startWorker(workerNum int) {
existingClaim, ok := s.syncedVideos[v.ID()]
existingClaimID := ""
existingClaimName := ""
existingClaimSize := v.Size()
existingClaimSize := int64(0)
if v.Size() != nil {
existingClaimSize = *v.Size()
}
if ok {
existingClaimID = existingClaim.ClaimID
existingClaimName = existingClaim.ClaimName
if existingClaim.Size > 0 {
existingClaimSize = &existingClaim.Size
existingClaimSize = existingClaim.Size
}
}
videoStatus := VideoStatusFailed
if strings.Contains(err.Error(), "upgrade failed") {
videoStatus = VideoStatusUpgradeFailed
} else {
s.AppendSyncedVideo(v.ID(), false, err.Error(), existingClaimName, existingClaimID, 0, existingClaimSize)
}
s.AppendSyncedVideo(v.ID(), false, err.Error(), "")
err = s.Manager.apiConfig.MarkVideoStatus(s.YoutubeChannelID, v.ID(), videoStatus, existingClaimID, existingClaimName, err.Error(), existingClaimSize, 0)
err = s.Manager.apiConfig.MarkVideoStatus(s.YoutubeChannelID, v.ID(), videoStatus, existingClaimID, existingClaimName, err.Error(), &existingClaimSize, 0)
if err != nil {
SendErrorToSlack("Failed to mark video on the database: %s", err.Error())
}
@ -907,13 +915,12 @@ func (s *Sync) processVideo(v video) (err error) {
return err
}
s.AppendSyncedVideo(v.ID(), true, "", summary.ClaimName, summary.ClaimID, newMetadataVersion, *v.Size())
err = s.Manager.apiConfig.MarkVideoStatus(s.YoutubeChannelID, v.ID(), VideoStatusPublished, summary.ClaimID, summary.ClaimName, "", v.Size(), 2)
if err != nil {
SendErrorToSlack("Failed to mark video on the database: %s", err.Error())
}
s.AppendSyncedVideo(v.ID(), true, "", summary.ClaimName)
return nil
}

9
sdk/api.go
View file

@ -164,8 +164,9 @@ func (a *APIConfig) SetChannelClaimID(channelID string, channelClaimID string) e
}
const (
VideoStatusPublished = "published"
VideoStatusFailed = "failed"
VideoStatusPublished = "published"
VideoStatusUpgradeFailed = "upgradefailed"
VideoStatusFailed = "failed"
)
func (a *APIConfig) DeleteVideos(videos []string) error {
@ -207,9 +208,9 @@ func (a *APIConfig) MarkVideoStatus(channelID string, videoID string, status str
"status": {status},
"auth_token": {a.ApiToken},
}
if status == VideoStatusPublished {
if status == VideoStatusPublished || status == VideoStatusUpgradeFailed {
if claimID == "" || claimName == "" {
return errors.Err("claimID or claimName missing")
return errors.Err("claimID (%s) or claimName (%s) missing", claimID, claimName)
}
vals.Add("published_at", strconv.FormatInt(time.Now().Unix(), 10))
vals.Add("claim_id", claimID)

11
sources/youtubeVideo.go
View file

@ -175,12 +175,12 @@ func (v *YoutubeVideo) getAbbrevDescription() string {
func (v *YoutubeVideo) fallbackDownload() error {
cmd := exec.Command("youtube-dl",
v.ID(),
"--no-progress",
"-fbestvideo[ext=mp4,height<=1080,filesize<2000M]+best[ext=mp4,height<=1080,filesize<2000M]",
"-o"+strings.TrimRight(v.getFullPath(), ".mp4"),
"--merge-output-format",
"mp4")
"mp4",
"https://www.youtube.com/watch?v="+v.ID())
log.Printf("Running command and waiting for it to finish...")
output, err := cmd.CombinedOutput()
@ -189,6 +189,12 @@ func (v *YoutubeVideo) fallbackDownload() error {
log.Printf("Command finished with error: %v", errors.Err(string(output)))
return errors.Err(err)
}
fi, err := os.Stat(v.getFullPath())
if err != nil {
return errors.Err(err)
}
videoSize := fi.Size()
v.size = &videoSize
return nil
}
@ -378,6 +384,7 @@ func (v *YoutubeVideo) Sync(daemon *jsonrpc.Client, params SyncParams, existingV
v.maxVideoSize = int64(params.MaxVideoSize) * 1024 * 1024
v.maxVideoLength = params.MaxVideoLength
v.lbryChannelID = params.ChannelID
v.walletLock = walletLock
if reprocess && existingVideoData != nil && existingVideoData.Published {
summary, err := v.reprocess(daemon, params, existingVideoData)
return summary, errors.Prefix("upgrade failed", err)