upgrade ytsync to latest SDK

Gopkg.lock

@@ -104,6 +104,14 @@
   pruneopts = ""
   revision = "31079b6807923eb23992c421b114992b95131b55"
 
+[[projects]]
+  digest = "1:2defd14040b1ca18840524e9d0f9cfab0ea8d2f2d396b343d221f8bc8e21ab9d"
+  name = "github.com/fatih/structs"
+  packages = ["."]
+  pruneopts = ""
+  revision = "4966fc68f5b7593aafa6cbbba2d65ec6e1416047"
+  version = "v1.1.0"
+
 [[projects]]
   digest = "1:968d8903d598e3fae738325d3410f33f07ea6a2b9ee5591e9c262ee37df6845a"
   name = "github.com/go-errors/errors"
@@ -113,12 +121,12 @@
   version = "v1.0.1"
 
 [[projects]]
-  branch = "master"
+  digest = "1:ed30e45cd59986b6f98b4d191157a54050f5dd550e2cec47ad032b6230e4af08"
-  digest = "1:a6f43a35242db0a1387907e6fd0c7e1dccfbed95c7176f166d31d0b5a96c65fc"
   name = "github.com/go-ini/ini"
   packages = ["."]
   pruneopts = ""
-  revision = "7b294651033cd7d9e7f0d9ffa1b75ed1e198e737"
+  revision = "6ed8d5f64cd79a498d1f3fab5880cc376ce41bbe"
+  version = "v1.41.0"
 
 [[projects]]
   digest = "1:3dd078fda7500c341bc26cfbc6c6a34614f295a2457149fc1045cab767cbcf18"
@@ -128,14 +136,6 @@
   revision = "aa810b61a9c79d51363740d207bb46cf8e620ed5"
   version = "v1.2.0"
 
-[[projects]]
-  digest = "1:09aa5dd1332b93c96bde671bafb053249dc813febf7d5ca84e8f382ba255d67d"
-  name = "github.com/gorilla/websocket"
-  packages = ["."]
-  pruneopts = ""
-  revision = "66b9c49e59c6c48f0ffce28c2d8b8a5678502c6d"
-  version = "v1.4.0"
-
 [[projects]]
   digest = "1:870d441fe217b8e689d7949fef6e43efbc787e50f200cb1e70dbca9204a1d6be"
   name = "github.com/inconshreveable/mousetrap"
@@ -161,8 +161,8 @@
   version = "v1.0.1"
 
 [[projects]]
-  branch = "master"
+  branch = "lbrynet-updates"
-  digest = "1:39ddb14ec4559ea9fe8bc574654a50e96659ccbc75845dbdf7a1f267debedd3b"
+  digest = "1:d98f9853a5d737be2a133ee35f91acd07ef2bb9dfa1b7edbb24893f5d106850f"
   name = "github.com/lbryio/lbry.go"
   packages = [
     "extras/errors",
@@ -173,7 +173,7 @@
     "lbrycrd",
   ]
   pruneopts = ""
-  revision = "51d03937c259085e271bdd4c611b8100213c2dd6"
+  revision = "d7286d45411366b24f0c453fa6876349a944a1e7"
 
 [[projects]]
   branch = "master"
@@ -192,31 +192,19 @@
   revision = "4fdf99ab29366514c69ccccddab5dc58b8d84062"
 
 [[projects]]
-  branch = "master"
   digest = "1:bcc46a0fbd9e933087bef394871256b5c60269575bb661935874729c65bbbf60"
   name = "github.com/mitchellh/mapstructure"
   packages = ["."]
   pruneopts = ""
   revision = "3536a929edddb9a5b34bd6861dc4a9647cb459fe"
+  version = "v1.1.2"
 
 [[projects]]
-  digest = "1:9407d3c57389f400ef8c1039e352ce00e2ab97c41bf1babba97c2a6922b7111e"
+  digest = "1:a52e6788022ede52706a9607f2a53331cf7ef2cf846377bc3ded8d523c2bc1af"
   name = "github.com/nlopes/slack"
-  packages = [
-    ".",
-    "slackutilsx",
-  ]
-  pruneopts = ""
-  revision = "b9033a72a20bf84563485e86a2adbea4bf265804"
-  version = "v0.4.0"
-
-[[projects]]
-  digest = "1:7365acd48986e205ccb8652cc746f09c8b7876030d53710ea6ef7d0bd0dcd7ca"
-  name = "github.com/pkg/errors"
   packages = ["."]
   pruneopts = ""
-  revision = "645ef00459ed84a119197bfb8d8205042c6df63d"
+  revision = "f243c7602fdf906248fc1f165284daa4e4d2d09e"
-  version = "v0.8.0"
 
 [[projects]]
   branch = "master"
@@ -227,20 +215,20 @@
   revision = "44fe64bba886b85c00fc7045e092d796484d6f90"
 
 [[projects]]
-  branch = "master"
   digest = "1:615c827f6a892973a587c754ae5fad7acfc4352657aff23d0238fe0ba2a154df"
   name = "github.com/shopspring/decimal"
   packages = ["."]
   pruneopts = ""
   revision = "cd690d0c9e2447b1ef2a129a6b7b49077da89b8e"
+  version = "1.1.0"
 
 [[projects]]
-  branch = "master"
+  digest = "1:9a3c631555e0351fdc4e696577bb63afd90c399d782a8462dba9d100d7021db3"
-  digest = "1:c0959eb8fc519d62edc717e2997032859688c0e1ddd9d4463512ead83576d247"
   name = "github.com/sirupsen/logrus"
   packages = ["."]
   pruneopts = ""
-  revision = "3f90cee1e41a38ba27c831844c002952512997c0"
+  revision = "e1e72e9de974bd926e5c56f83753fba2df402ce5"
+  version = "v1.3.0"
 
 [[projects]]
   branch = "master"
@@ -259,12 +247,12 @@
   version = "v1.0.3"
 
 [[projects]]
-  branch = "master"
+  digest = "1:1a7da329bbaa93d90fad3ee12d8d0fc7dff08bcb3e59d4002c827b91f35182b4"
-  digest = "1:22d3674d44ee93f52a9c0b6a22d1f736a0ad9ac3f9d2c1ca8648f3c9ce9910bd"
   name = "github.com/ybbus/jsonrpc"
   packages = ["."]
   pruneopts = ""
-  revision = "2a548b7d822dd62717337a6b1e817fae1b14660a"
+  revision = "dd866631e904a5df2067d934985c5def68f391ac"
+  version = "v2.1.2"
 
 [[projects]]
   branch = "master"
@@ -286,6 +274,7 @@
     "context/ctxhttp",
     "html",
     "html/atom",
+    "websocket",
   ]
   pruneopts = ""
   revision = "146acd28ed5894421fb5aac80ca93bc1b1f46f87"

Gopkg.toml

@@ -3,12 +3,12 @@
   name = "github.com/rylio/ytdl"
 
 [[constraint]]
-  branch = "master"
+  version = "^1.1.0"
   name = "github.com/shopspring/decimal"
 
 [[constraint]]
   name = "github.com/sirupsen/logrus"
-  branch = "master"
+  version = "^1.3.0"
 
 [[constraint]]
   branch = "master"
@@ -24,4 +24,4 @@
 
 [[constraint]]
   name = "github.com/lbryio/lbry.go"
-  branch = "master"
+  branch = "lbrynet-updates"

manager/setup.go

@@ -1,6 +1,9 @@
 package manager
 
 import (
+	"fmt"
+	"os"
+	"strconv"
 	"time"
 
 	"github.com/lbryio/lbry.go/extras/errors"
@@ -20,14 +23,17 @@ func (s *Sync) walletSetup() error {
 		return err
 	}
 
-	balanceResp, err := s.daemon.WalletBalance()
+	balanceResp, err := s.daemon.AccountBalance(nil)
 	if err != nil {
 		return err
 	} else if balanceResp == nil {
 		return errors.Err("no response")
 	}
-	balance := decimal.Decimal(*balanceResp)
+	balance, err := strconv.ParseFloat((string)(*balanceResp), 64)
-	log.Debugf("Starting balance is %s", balance.String())
+	if err != nil {
+		return errors.Err(err)
+	}
+	log.Debugf("Starting balance is %.4f", balance)
 
 	var numOnSource int
 	if s.LbryChannelName == "@UCBerkeley" {
@@ -54,11 +60,11 @@ func (s *Sync) walletSetup() error {
 	}
 
 	minBalance := (float64(numOnSource)-float64(numPublished))*(publishAmount+0.1) + channelClaimAmount
-	if numPublished > numOnSource && balance.LessThan(decimal.NewFromFloat(1)) {
+	if numPublished > numOnSource && balance < 1 {
 		SendErrorToSlack("something is going on as we published more videos than those available on source: %d/%d", numPublished, numOnSource)
 		minBalance = 1 //since we ended up in this function it means some juice is still needed
 	}
-	amountToAdd, _ := decimal.NewFromFloat(minBalance).Sub(balance).Float64()
+	amountToAdd := minBalance - balance
 
 	if s.Refill > 0 {
 		if amountToAdd < 0 {
@@ -72,16 +78,19 @@ func (s *Sync) walletSetup() error {
 		if amountToAdd < 1 {
 			amountToAdd = 1 // no reason to bother adding less than 1 credit
 		}
-		s.addCredits(amountToAdd)
+		err := s.addCredits(amountToAdd)
+		if err != nil {
+			return errors.Err(err)
+		}
 	}
 
-	claimAddress, err := s.daemon.WalletUnusedAddress()
+	claimAddress, err := s.daemon.AddressList(nil)
 	if err != nil {
 		return err
 	} else if claimAddress == nil {
 		return errors.Err("could not get unused address")
 	}
-	s.claimAddress = string(*claimAddress)
+	s.claimAddress = string((*claimAddress)[0])
 	if s.claimAddress == "" {
 		return errors.Err("found blank claim address")
 	}
@@ -95,7 +104,26 @@ func (s *Sync) walletSetup() error {
 }
 
 func (s *Sync) ensureEnoughUTXOs() error {
-	utxolist, err := s.daemon.UTXOList()
+	accounts, err := s.daemon.AccountList()
+	if err != nil {
+		return errors.Err(err)
+	}
+	accountsNet := (*accounts).LBCMainnet
+	if os.Getenv("REGTEST") == "true" {
+		accountsNet = (*accounts).LBCRegtest
+	}
+	defaultAccount := ""
+	for _, account := range accountsNet {
+		if account.IsDefaultAccount {
+			defaultAccount = account.ID
+			break
+		}
+	}
+	if defaultAccount == "" {
+		return errors.Err("No default account found")
+	}
+
+	utxolist, err := s.daemon.UTXOList(&defaultAccount)
 	if err != nil {
 		return err
 	} else if utxolist == nil {
@@ -107,26 +135,30 @@ func (s *Sync) ensureEnoughUTXOs() error {
 	count := 0
 
 	for _, utxo := range *utxolist {
-		if !utxo.IsClaim && !utxo.IsSupport && !utxo.IsUpdate && utxo.Amount.Cmp(decimal.New(0, 0)) == 1 {
+		amount, _ := strconv.ParseFloat(utxo.Amount, 64)
+		if !utxo.IsClaim && !utxo.IsSupport && !utxo.IsUpdate && amount != 0.0 {
 			count++
 		}
 	}
-
+	log.Infof("utxo count: %d", count)
 	if count < target-slack {
-		newAddresses := target - count
+		balance, err := s.daemon.AccountBalance(&defaultAccount)
-
-		balance, err := s.daemon.WalletBalance()
 		if err != nil {
 			return err
 		} else if balance == nil {
 			return errors.Err("no response")
 		}
 
-		log.Println("balance is " + decimal.Decimal(*balance).String())
+		balanceAmount, err := strconv.ParseFloat((string)(*balance), 64)
+		if err != nil {
+			return errors.Err(err)
+		}
+		broadcastFee := 0.001
+		amountToSplit := fmt.Sprintf("%.6f", balanceAmount-broadcastFee)
 
-		amountPerAddress := decimal.Decimal(*balance).Div(decimal.NewFromFloat(float64(target)))
+		log.Infof("Splitting balance of %s evenly between 40 UTXOs", *balance)
-		log.Infof("Putting %s credits into each of %d new addresses", amountPerAddress.String(), newAddresses)
+
-		prefillTx, err := s.daemon.WalletPrefillAddresses(newAddresses, amountPerAddress, true)
+		prefillTx, err := s.daemon.AccountFund(defaultAccount, defaultAccount, amountToSplit, uint64(target))
 		if err != nil {
 			return err
 		} else if prefillTx == nil {
@@ -180,21 +212,21 @@ func (s *Sync) ensureChannelOwnership() error {
 		return errors.Err("no channel name set")
 	}
 
-	channels, err := s.daemon.ChannelList()
+	channels, err := s.daemon.ChannelList(nil, 1, 50)
 	if err != nil {
 		return err
 	} else if channels == nil {
 		return errors.Err("no channel response")
 	}
 	//special case for wallets we don't retain full control anymore
-	if len(*channels) > 1 {
+	if len((*channels).Items) > 1 {
 		// This wallet is probably not under our control anymore but we still want to publish to it
 		// here we shall check if within all the channels there is one that was created by ytsync
 		SendInfoToSlack("we are dealing with a wallet that has multiple channels. This indicates that the wallet was probably transferred but we still want to sync their content. YoutubeID: %s", s.YoutubeChannelID)
 		if s.lbryChannelID == "" {
 			return errors.Err("this channel does not have a recorded claimID in the database. To prevent failures, updates are not supported until an entry is manually added in the database")
 		}
-		for _, c := range *channels {
+		for _, c := range (*channels).Items {
 			if c.ClaimID != s.lbryChannelID {
 				if c.Name != s.LbryChannelName {
 					return errors.Err("the channel in the wallet is different than the channel in the database")
@@ -203,8 +235,8 @@ func (s *Sync) ensureChannelOwnership() error {
 			}
 		}
 	}
-	if len(*channels) == 1 {
+	if len((*channels).Items) == 1 {
-		channel := (*channels)[0]
+		channel := ((*channels).Items)[0]
 		if channel.Name == s.LbryChannelName {
 			//TODO: eventually get rid of this when the whole db is filled
 			if s.lbryChannelID == "" {
@@ -221,13 +253,16 @@ func (s *Sync) ensureChannelOwnership() error {
 
 	channelBidAmount := channelClaimAmount
 
-	balanceResp, err := s.daemon.WalletBalance()
+	balanceResp, err := s.daemon.AccountBalance(nil)
 	if err != nil {
 		return err
 	} else if balanceResp == nil {
 		return errors.Err("no response")
 	}
-	balance := decimal.Decimal(*balanceResp)
+	balance, err := decimal.NewFromString((string)(*balanceResp))
+	if err != nil {
+		return errors.Err(err)
+	}
 
 	if balance.LessThan(decimal.NewFromFloat(channelBidAmount)) {
 		err = s.addCredits(channelBidAmount + 0.1)
@@ -236,7 +271,7 @@ func (s *Sync) ensureChannelOwnership() error {
 		}
 	}
 
-	c, err := s.daemon.ChannelNew(s.LbryChannelName, channelBidAmount)
+	c, err := s.daemon.ChannelNew(s.LbryChannelName, channelBidAmount, nil)
 	if err != nil {
 		return err
 	}
@@ -278,7 +313,7 @@ func (s *Sync) addCredits(amountToAdd float64) error {
 		}
 	}
 
-	addressResp, err := s.daemon.WalletUnusedAddress()
+	addressResp, err := s.daemon.AddressUnused(nil)
 	if err != nil {
 		return err
 	} else if addressResp == nil {

manager/ytsync.go

@@ -388,7 +388,7 @@ func logShutdownError(shutdownErr error) {
 }
 
 func isYtsyncClaim(c jsonrpc.Claim) bool {
-	if !util.InSlice(c.Category, []string{"claim", "update"}) || c.Value.Stream == nil {
+	if !util.InSlice(c.Type, []string{"claim", "update"}) || c.Value.Stream == nil {
 		return false
 	}
 	if c.Value.Stream.Metadata == nil || c.Value.Stream.Metadata.Thumbnail == nil {
@@ -424,7 +424,7 @@ func (s *Sync) fixDupes(claims []jsonrpc.Claim) (bool, error) {
 			videoIDs[videoID] = c
 		}
 		log.Debugf("abandoning %+v", claimToAbandon)
-		_, err := s.daemon.ClaimAbandon(claimToAbandon.Txid, claimToAbandon.Nout)
+		_, err := s.daemon.ClaimAbandon(claimToAbandon.Txid, claimToAbandon.Nout, nil, false)
 		if err != nil {
 			return true, err
 		}
@@ -459,18 +459,31 @@ func (s *Sync) updateRemoteDB(claims []jsonrpc.Claim) (total int, fixed int, err
 	return count, fixed, nil
 }
 
+func (s *Sync) getClaims() ([]jsonrpc.Claim, error) {
+	totalPages := uint64(1)
+	var allClaims []jsonrpc.Claim
+	for page := uint64(1); page <= totalPages; page++ {
+		claims, err := s.daemon.ClaimListMine(nil, page, 50)
+		if err != nil {
+			return nil, errors.Prefix("cannot list claims", err)
+		}
+		allClaims = append(allClaims, (*claims).Claims...)
+		totalPages = (*claims).TotalPages
+	}
+	return allClaims, nil
+}
+
 func (s *Sync) doSync() error {
 	var err error
 	err = s.walletSetup()
 	if err != nil {
 		return errors.Prefix("Initial wallet setup failed! Manual Intervention is required.", err)
 	}
-
+	allClaims, err := s.getClaims()
-	claims, err := s.daemon.ClaimListMine()
 	if err != nil {
-		return errors.Prefix("cannot list claims", err)
+		return err
 	}
-	hasDupes, err := s.fixDupes(*claims)
+	hasDupes, err := s.fixDupes(allClaims)
 	if err != nil {
 		return errors.Prefix("error checking for duplicates", err)
 	}
@@ -480,13 +493,13 @@ func (s *Sync) doSync() error {
 		if err != nil {
 			return err
 		}
-		claims, err = s.daemon.ClaimListMine()
+		allClaims, err = s.getClaims()
 		if err != nil {
-			return errors.Prefix("cannot list claims", err)
+			return err
 		}
 	}
 
-	pubsOnWallet, nFixed, err := s.updateRemoteDB(*claims)
+	pubsOnWallet, nFixed, err := s.updateRemoteDB(allClaims)
 	if err != nil {
 		return errors.Prefix("error counting claims", err)
 	}
@@ -852,7 +865,7 @@ func waitForDaemonProcess(timeout time.Duration) error {
 	}
 	var daemonProcessId = -1
 	for _, p := range processes {
-		if p.Executable() == "lbrynet-daemon" {
+		if p.Executable() == "lbrynet" {
 			daemonProcessId = p.Pid()
 			break
 		}

sources/ucbVideo.go

@@ -176,17 +176,19 @@ func (v *ucbVideo) saveThumbnail() error {
 
 func (v *ucbVideo) publish(daemon *jsonrpc.Client, claimAddress string, amount float64, channelID string, namer *namer.Namer) (*SyncSummary, error) {
 	options := jsonrpc.PublishOptions{
-		Title:         &v.title,
+		Metadata: &jsonrpc.Metadata{
-		Author:        strPtr("UC Berkeley"),
+			Title:       v.title,
-		Description:   strPtr(v.getAbbrevDescription()),
+			Description: v.getAbbrevDescription(),
-		Language:      strPtr("en"),
+			Author:      "UC Berkeley",
-		ClaimAddress:  &claimAddress,
+			Language:    "en",
-		Thumbnail:     strPtr("https://berk.ninja/thumbnails/" + v.id),
+			License:     "see description",
-		License:       strPtr("see description"),
+			Thumbnail:   strPtr("https://berk.ninja/thumbnails/" + v.id),
+			NSFW:        false,
+		},
 		ChannelID:     &channelID,
+		ClaimAddress:  &claimAddress,
 		ChangeAddress: &claimAddress,
 	}
-
 	return publishAndRetryExistingNames(daemon, v.title, v.getFilename(), amount, options, namer)
 }
 

sources/youtubeVideo.go

@@ -244,17 +244,19 @@ func (v *YoutubeVideo) publish(daemon *jsonrpc.Client, claimAddress string, amou
 		return nil, errors.Err("a claim_id for the channel wasn't provided") //TODO: this is probably not needed?
 	}
 	options := jsonrpc.PublishOptions{
-		Title:         &v.title,
+		Metadata: &jsonrpc.Metadata{
-		Author:        &v.channelTitle,
+			Title:       v.title,
-		Description:   strPtr(v.getAbbrevDescription() + "\nhttps://www.youtube.com/watch?v=" + v.id),
+			Description: v.getAbbrevDescription() + "\nhttps://www.youtube.com/watch?v=" + v.id,
-		Language:      strPtr("en"),
+			Author:      v.channelTitle,
-		ClaimAddress:  &claimAddress,
+			Language:    "en",
-		Thumbnail:     strPtr("https://berk.ninja/thumbnails/" + v.id),
+			License:     "Copyrighted (contact author)",
-		License:       strPtr("Copyrighted (contact author)"),
+			Thumbnail:   strPtr("https://berk.ninja/thumbnails/" + v.id),
-		ChangeAddress: &claimAddress,
+			NSFW:        false,
+		},
 		ChannelID:     &channelID,
+		ClaimAddress:  &claimAddress,
+		ChangeAddress: &claimAddress,
 	}
-
 	return publishAndRetryExistingNames(daemon, v.title, v.getFullPath(), amount, options, namer)
 }