go.mod

@@ -17,7 +17,7 @@ require (
 	github.com/konsorten/go-windows-terminal-sequences v1.0.2 // indirect
 	github.com/kr/pretty v0.1.0 // indirect
 	github.com/lbryio/errors.go v0.0.0-20180223142025-ad03d3cc6a5c
-	github.com/lbryio/lbry.go v0.0.0-20190503025608-3a22a0af0a45
+	github.com/lbryio/lbry.go v0.0.0-20190506195255-f01bba8d4336
 	github.com/lusis/slack-test v0.0.0-20190408224659-6cf59653add2 // indirect
 	github.com/mitchellh/go-ps v0.0.0-20170309133038-4fdf99ab2936
 	github.com/mitchellh/mapstructure v1.1.2 // indirect

go.sum

@@ -125,8 +125,10 @@ github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/lbryio/errors.go v0.0.0-20180223142025-ad03d3cc6a5c h1:BhdcWGsuKif/XoSZnqVGNqJ1iEmH0czWR5upj+AuR8M=
 github.com/lbryio/errors.go v0.0.0-20180223142025-ad03d3cc6a5c/go.mod h1:muH7wpUqE8hRA3OrYYosw9+Sl681BF9cwcjzE+OCNK8=
-github.com/lbryio/lbry.go v0.0.0-20190503025608-3a22a0af0a45 h1:6fpRUui1G8xKiQqL51fHJwrpqYSd4G3RIYWsRd+R8x8=
+github.com/lbryio/lbry.go v0.0.0-20190506193558-20f2cbad6754 h1:Kim8dIsKTrv8L3GCtL91atvVC3CD+t1TlPwmRJ9EmCQ=
-github.com/lbryio/lbry.go v0.0.0-20190503025608-3a22a0af0a45/go.mod h1:rAREtjHq/Wkkqy4+yt89Er/7JAVu6uMo3v8nevCEEIA=
+github.com/lbryio/lbry.go v0.0.0-20190506193558-20f2cbad6754/go.mod h1:rAREtjHq/Wkkqy4+yt89Er/7JAVu6uMo3v8nevCEEIA=
+github.com/lbryio/lbry.go v0.0.0-20190506195255-f01bba8d4336 h1:4jNCUftcPnkCwkyHBVaJXmtUIGN2FLsxpr+ZO+NLSy0=
+github.com/lbryio/lbry.go v0.0.0-20190506195255-f01bba8d4336/go.mod h1:rAREtjHq/Wkkqy4+yt89Er/7JAVu6uMo3v8nevCEEIA=
 github.com/lbryio/lbryschema.go v0.0.0-20190428231007-c54836bca002 h1:urfYK5ElpUrAv90auPLldoVC60LwiGAcY0OE6HJB9KI=
 github.com/lbryio/lbryschema.go v0.0.0-20190428231007-c54836bca002/go.mod h1:dAzPCBj3CKKWBGYBZxK6tKBP5SCgY2tqd9SnQd/OyKo=
 github.com/lbryio/ozzo-validation v0.0.0-20170323141101-d1008ad1fd04/go.mod h1:fbG/dzobG8r95KzMwckXiLMHfFjZaBRQqC9hPs2XAQ4=
@@ -178,6 +180,7 @@ github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R
 github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
 github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
 github.com/sebdah/goldie v0.0.0-20180424091453-8784dd1ab561/go.mod h1:lvjGftC8oe7XPtyrOidaMi0rp5B9+XY/ZRUynGnuaxQ=
+github.com/sergi/go-diff v1.0.0 h1:Kpca3qRNrduNnOQeazBd0ysaKrUJiIuISHxogkT9RPQ=
 github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
 github.com/shopspring/decimal v0.0.0-20180607144847-19e3cb6c2930/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4=
 github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24 h1:pntxY8Ary0t43dCZ5dqY4YTJCObLY1kIXl0uzMv+7DE=

manager/setup.go

@@ -19,6 +19,27 @@ import (
 	"google.golang.org/api/youtube/v3"
 )
 
+const minimumRefillAmount = 3
+
+func (s *Sync) enableAddressReuse() error {
+	accountsResponse, err := s.daemon.AccountList()
+	if err != nil {
+		return errors.Err(err)
+	}
+	accounts := accountsResponse.LBCMainnet
+	if os.Getenv("REGTEST") == "true" {
+		accounts = accountsResponse.LBCRegtest
+	}
+	for _, a := range accounts {
+		_, err = s.daemon.AccountSet(a.ID, jsonrpc.AccountSettings{
+			ChangeMaxUses: 1000,
+		})
+		if err != nil {
+			return errors.Err(err)
+		}
+	}
+	return nil
+}
 func (s *Sync) walletSetup() error {
 	//prevent unnecessary concurrent execution
 	s.walletMux.Lock()
@@ -40,16 +61,11 @@ func (s *Sync) walletSetup() error {
 	}
 	log.Debugf("Starting balance is %.4f", balance)
 
-	var numOnSource int
+	n, err := s.CountVideos()
-	if s.LbryChannelName == "@UCBerkeley" {
+	if err != nil {
-		numOnSource = 10104
+		return err
-	} else {
-		n, err := s.CountVideos()
-		if err != nil {
-			return err
-		}
-		numOnSource = int(n)
 	}
+	numOnSource := int(n)
 
 	log.Debugf("Source channel has %d videos", numOnSource)
 	if numOnSource == 0 {
@@ -66,9 +82,9 @@ func (s *Sync) walletSetup() error {
 	}
 
 	minBalance := (float64(numOnSource)-float64(numPublished))*(publishAmount+0.1) + channelClaimAmount
-	if numPublished > numOnSource && balance < 1 {
+	if numPublished > numOnSource && balance < minimumRefillAmount {
 		SendErrorToSlack("something is going on as we published more videos than those available on source: %d/%d", numPublished, numOnSource)
-		minBalance = 1 //since we ended up in this function it means some juice is still needed
+		minBalance = minimumRefillAmount
 	}
 	amountToAdd := minBalance - balance
 
@@ -81,8 +97,8 @@ func (s *Sync) walletSetup() error {
 	}
 
 	if amountToAdd > 0 {
-		if amountToAdd < 1 {
+		if amountToAdd < minimumRefillAmount {
-			amountToAdd = 1 // no reason to bother adding less than 1 credit
+			amountToAdd = minimumRefillAmount
 		}
 		err := s.addCredits(amountToAdd)
 		if err != nil {

manager/ytsync.go

@@ -479,7 +479,10 @@ func (s *Sync) getClaims() ([]jsonrpc.Claim, error) {
 }
 
 func (s *Sync) doSync() error {
-	var err error
+	err := s.enableAddressReuse()
+	if err != nil {
+		return errors.Prefix("could not set address reuse policy", err)
+	}
 	err = s.walletSetup()
 	if err != nil {
 		return errors.Prefix("Initial wallet setup failed! Manual Intervention is required.", err)

sources/youtubeVideo.go

@@ -13,13 +13,13 @@ import (
 	"strings"
 	"time"
 
-	"github.com/lbryio/ytsync/sdk"
-	"github.com/lbryio/ytsync/thumbs"
-
 	"github.com/lbryio/lbry.go/extras/errors"
 	"github.com/lbryio/lbry.go/extras/jsonrpc"
 	"github.com/lbryio/lbry.go/extras/util"
+
 	"github.com/lbryio/ytsync/namer"
+	"github.com/lbryio/ytsync/sdk"
+	"github.com/lbryio/ytsync/thumbs"
 
 	"github.com/ChannelMeter/iso8601duration"
 	"github.com/aws/aws-sdk-go/aws"
@@ -44,6 +44,8 @@ type YoutubeVideo struct {
 	awsConfig        aws.Config
 }
 
+const reflectorURL = "http://blobs.lbry.io/"
+
 var youtubeCategories = map[string]string{
 	"1":  "Film & Animation",
 	"2":  "Autos & Vehicles",
@@ -285,8 +287,6 @@ func (v *YoutubeVideo) triggerThumbnailSave() error {
 	return nil
 }
 
-func strPtr(s string) *string { return &s }
-
 func (v *YoutubeVideo) publish(daemon *jsonrpc.Client, claimAddress string, amount float64, channelID string, namer *namer.Namer) (*SyncSummary, error) {
 	additionalDescription := "\nhttps://www.youtube.com/watch?v=" + v.id
 	khanAcademyClaimID := "5fc52291980268b82413ca4c0ace1b8d749f3ffb"
@@ -309,15 +309,15 @@ func (v *YoutubeVideo) publish(daemon *jsonrpc.Client, claimAddress string, amou
 			Description:  v.getAbbrevDescription() + additionalDescription,
 			ClaimAddress: &claimAddress,
 			Languages:    languages,
-			ThumbnailURL: strPtr(thumbs.ThumbnailEndpoint + v.id),
+			ThumbnailURL: util.PtrToString(thumbs.ThumbnailEndpoint + v.id),
 			Tags:         v.youtubeInfo.Snippet.Tags,
 		},
-		Author:        strPtr(v.channelTitle),
+		Author:      util.PtrToString(v.channelTitle),
-		License:       strPtr("Copyrighted (contact author)"),
+		License:     util.PtrToString("Copyrighted (contact author)"),
-		StreamType:    &jsonrpc.StreamTypeVideo,
+		StreamType:  &jsonrpc.StreamTypeVideo,
-		ReleaseTime:   util.PtrToInt64(v.publishedAt.Unix()),
+		ReleaseTime: util.PtrToInt64(v.publishedAt.Unix()),
-		VideoDuration: util.PtrToUint64(uint64(math.Ceil(videoDuration.ToDuration().Seconds()))),
+		Duration:    util.PtrToUint64(uint64(math.Ceil(videoDuration.ToDuration().Seconds()))),
-		ChannelID:     &channelID,
+		ChannelID:   &channelID,
 	}
 
 	return publishAndRetryExistingNames(daemon, v.title, v.getFullPath(), amount, options, namer)
@@ -341,10 +341,13 @@ func (v *YoutubeVideo) Sync(daemon *jsonrpc.Client, params SyncParams, existingV
 	v.maxVideoLength = params.MaxVideoLength
 
 	if reprocess {
-		summary, err := v.reprocess(daemon, params.ChannelID, existingVideoData)
+		summary, err := v.reprocess(daemon, params, existingVideoData)
-
 		return summary, err
 	}
+	return v.downloadAndPublish(daemon, params)
+}
+
+func (v *YoutubeVideo) downloadAndPublish(daemon *jsonrpc.Client, params SyncParams) (*SyncSummary, error) {
 	//download and thumbnail can be done in parallel
 	err := v.download()
 	if err != nil {
@@ -365,7 +368,7 @@ func (v *YoutubeVideo) Sync(daemon *jsonrpc.Client, params SyncParams, existingV
 	return summary, errors.Prefix("publish error", err)
 }
 
-func (v *YoutubeVideo) reprocess(daemon *jsonrpc.Client, channelID string, existingVideoData *sdk.SyncedVideo) (*SyncSummary, error) {
+func (v *YoutubeVideo) reprocess(daemon *jsonrpc.Client, params SyncParams, existingVideoData *sdk.SyncedVideo) (*SyncSummary, error) {
 	c, err := daemon.ClaimSearch(nil, &existingVideoData.ClaimID, nil, nil)
 	if err != nil {
 		return nil, errors.Err(err)
@@ -400,6 +403,15 @@ func (v *YoutubeVideo) reprocess(daemon *jsonrpc.Client, channelID string, exist
 
 	tags := append(v.youtubeInfo.Snippet.Tags, youtubeCategories[v.youtubeInfo.Snippet.CategoryId])
 
+	videoSize, err := currentClaim.GetStreamSizeByMagic()
+	if err != nil {
+		if existingVideoData.Size > 0 {
+			videoSize = uint64(existingVideoData.Size)
+		} else {
+			log.Infof("%s: the video must be republished as we can't get the right size", v.ID())
+			return v.downloadAndPublish(daemon, params)
+		}
+	}
 	videoDuration, err := duration.FromString(v.youtubeInfo.ContentDetails.Duration)
 	_, err = daemon.StreamUpdate(existingVideoData.ClaimID, jsonrpc.StreamUpdateOptions{
 		StreamCreateOptions: &jsonrpc.StreamCreateOptions{
@@ -411,13 +423,13 @@ func (v *YoutubeVideo) reprocess(daemon *jsonrpc.Client, channelID string, exist
 				Locations:    locations,
 				ThumbnailURL: &thumbnailURL,
 			},
-			Author:        util.PtrToString(""),
+			Author:      util.PtrToString(""),
-			License:       strPtr("Copyrighted (contact author)"),
+			License:     util.PtrToString("Copyrighted (contact author)"),
-			ReleaseTime:   util.PtrToInt64(v.publishedAt.Unix()),
+			ReleaseTime: util.PtrToInt64(v.publishedAt.Unix()),
-			VideoDuration: util.PtrToUint64(uint64(math.Ceil(videoDuration.ToDuration().Seconds()))),
+			Duration:    util.PtrToUint64(uint64(math.Ceil(videoDuration.ToDuration().Seconds()))),
-
+			ChannelID:   &params.ChannelID,
-			ChannelID: &channelID,
 		},
+		FileSize: util.PtrToString(fmt.Sprintf("%d", videoSize)),
 	})
 	return &SyncSummary{}, nil
 }