From 5e125fa64143fa19f79c2bd2cdf615d75b51928f Mon Sep 17 00:00:00 2001 From: Niko Storni Date: Wed, 30 Jan 2019 13:42:23 +0100 Subject: [PATCH] upgrade ytsync to latest SDK --- Gopkg.lock | 61 ++++++++++++---------------- Gopkg.toml | 6 +-- manager/setup.go | 89 ++++++++++++++++++++++++++++------------- manager/ytsync.go | 33 ++++++++++----- sources/ucbVideo.go | 18 +++++---- sources/youtubeVideo.go | 20 ++++----- 6 files changed, 134 insertions(+), 93 deletions(-) diff --git a/Gopkg.lock b/Gopkg.lock index 9362e58..0b40a78 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -104,6 +104,14 @@ pruneopts = "" revision = "31079b6807923eb23992c421b114992b95131b55" +[[projects]] + digest = "1:2defd14040b1ca18840524e9d0f9cfab0ea8d2f2d396b343d221f8bc8e21ab9d" + name = "github.com/fatih/structs" + packages = ["."] + pruneopts = "" + revision = "4966fc68f5b7593aafa6cbbba2d65ec6e1416047" + version = "v1.1.0" + [[projects]] digest = "1:968d8903d598e3fae738325d3410f33f07ea6a2b9ee5591e9c262ee37df6845a" name = "github.com/go-errors/errors" @@ -113,12 +121,12 @@ version = "v1.0.1" [[projects]] - branch = "master" - digest = "1:a6f43a35242db0a1387907e6fd0c7e1dccfbed95c7176f166d31d0b5a96c65fc" + digest = "1:ed30e45cd59986b6f98b4d191157a54050f5dd550e2cec47ad032b6230e4af08" name = "github.com/go-ini/ini" packages = ["."] pruneopts = "" - revision = "7b294651033cd7d9e7f0d9ffa1b75ed1e198e737" + revision = "6ed8d5f64cd79a498d1f3fab5880cc376ce41bbe" + version = "v1.41.0" [[projects]] digest = "1:3dd078fda7500c341bc26cfbc6c6a34614f295a2457149fc1045cab767cbcf18" @@ -128,14 +136,6 @@ revision = "aa810b61a9c79d51363740d207bb46cf8e620ed5" version = "v1.2.0" -[[projects]] - digest = "1:09aa5dd1332b93c96bde671bafb053249dc813febf7d5ca84e8f382ba255d67d" - name = "github.com/gorilla/websocket" - packages = ["."] - pruneopts = "" - revision = "66b9c49e59c6c48f0ffce28c2d8b8a5678502c6d" - version = "v1.4.0" - [[projects]] digest = "1:870d441fe217b8e689d7949fef6e43efbc787e50f200cb1e70dbca9204a1d6be" name = "github.com/inconshreveable/mousetrap" @@ -161,8 +161,8 @@ version = "v1.0.1" [[projects]] - branch = "master" - digest = "1:39ddb14ec4559ea9fe8bc574654a50e96659ccbc75845dbdf7a1f267debedd3b" + branch = "lbrynet-updates" + digest = "1:d98f9853a5d737be2a133ee35f91acd07ef2bb9dfa1b7edbb24893f5d106850f" name = "github.com/lbryio/lbry.go" packages = [ "extras/errors", @@ -173,7 +173,7 @@ "lbrycrd", ] pruneopts = "" - revision = "51d03937c259085e271bdd4c611b8100213c2dd6" + revision = "d7286d45411366b24f0c453fa6876349a944a1e7" [[projects]] branch = "master" @@ -192,31 +192,19 @@ revision = "4fdf99ab29366514c69ccccddab5dc58b8d84062" [[projects]] - branch = "master" digest = "1:bcc46a0fbd9e933087bef394871256b5c60269575bb661935874729c65bbbf60" name = "github.com/mitchellh/mapstructure" packages = ["."] pruneopts = "" revision = "3536a929edddb9a5b34bd6861dc4a9647cb459fe" + version = "v1.1.2" [[projects]] - digest = "1:9407d3c57389f400ef8c1039e352ce00e2ab97c41bf1babba97c2a6922b7111e" + digest = "1:a52e6788022ede52706a9607f2a53331cf7ef2cf846377bc3ded8d523c2bc1af" name = "github.com/nlopes/slack" - packages = [ - ".", - "slackutilsx", - ] - pruneopts = "" - revision = "b9033a72a20bf84563485e86a2adbea4bf265804" - version = "v0.4.0" - -[[projects]] - digest = "1:7365acd48986e205ccb8652cc746f09c8b7876030d53710ea6ef7d0bd0dcd7ca" - name = "github.com/pkg/errors" packages = ["."] pruneopts = "" - revision = "645ef00459ed84a119197bfb8d8205042c6df63d" - version = "v0.8.0" + revision = "f243c7602fdf906248fc1f165284daa4e4d2d09e" [[projects]] branch = "master" @@ -227,20 +215,20 @@ revision = "44fe64bba886b85c00fc7045e092d796484d6f90" [[projects]] - branch = "master" digest = "1:615c827f6a892973a587c754ae5fad7acfc4352657aff23d0238fe0ba2a154df" name = "github.com/shopspring/decimal" packages = ["."] pruneopts = "" revision = "cd690d0c9e2447b1ef2a129a6b7b49077da89b8e" + version = "1.1.0" [[projects]] - branch = "master" - digest = "1:c0959eb8fc519d62edc717e2997032859688c0e1ddd9d4463512ead83576d247" + digest = "1:9a3c631555e0351fdc4e696577bb63afd90c399d782a8462dba9d100d7021db3" name = "github.com/sirupsen/logrus" packages = ["."] pruneopts = "" - revision = "3f90cee1e41a38ba27c831844c002952512997c0" + revision = "e1e72e9de974bd926e5c56f83753fba2df402ce5" + version = "v1.3.0" [[projects]] branch = "master" @@ -259,12 +247,12 @@ version = "v1.0.3" [[projects]] - branch = "master" - digest = "1:22d3674d44ee93f52a9c0b6a22d1f736a0ad9ac3f9d2c1ca8648f3c9ce9910bd" + digest = "1:1a7da329bbaa93d90fad3ee12d8d0fc7dff08bcb3e59d4002c827b91f35182b4" name = "github.com/ybbus/jsonrpc" packages = ["."] pruneopts = "" - revision = "2a548b7d822dd62717337a6b1e817fae1b14660a" + revision = "dd866631e904a5df2067d934985c5def68f391ac" + version = "v2.1.2" [[projects]] branch = "master" @@ -286,6 +274,7 @@ "context/ctxhttp", "html", "html/atom", + "websocket", ] pruneopts = "" revision = "146acd28ed5894421fb5aac80ca93bc1b1f46f87" diff --git a/Gopkg.toml b/Gopkg.toml index bc68084..f74caf2 100644 --- a/Gopkg.toml +++ b/Gopkg.toml @@ -3,12 +3,12 @@ name = "github.com/rylio/ytdl" [[constraint]] - branch = "master" + version = "^1.1.0" name = "github.com/shopspring/decimal" [[constraint]] name = "github.com/sirupsen/logrus" - branch = "master" + version = "^1.3.0" [[constraint]] branch = "master" @@ -24,4 +24,4 @@ [[constraint]] name = "github.com/lbryio/lbry.go" - branch = "master" + branch = "lbrynet-updates" diff --git a/manager/setup.go b/manager/setup.go index d8720a3..599d6c1 100644 --- a/manager/setup.go +++ b/manager/setup.go @@ -1,6 +1,9 @@ package manager import ( + "fmt" + "os" + "strconv" "time" "github.com/lbryio/lbry.go/extras/errors" @@ -20,14 +23,17 @@ func (s *Sync) walletSetup() error { return err } - balanceResp, err := s.daemon.WalletBalance() + balanceResp, err := s.daemon.AccountBalance(nil) if err != nil { return err } else if balanceResp == nil { return errors.Err("no response") } - balance := decimal.Decimal(*balanceResp) - log.Debugf("Starting balance is %s", balance.String()) + balance, err := strconv.ParseFloat((string)(*balanceResp), 64) + if err != nil { + return errors.Err(err) + } + log.Debugf("Starting balance is %.4f", balance) var numOnSource int if s.LbryChannelName == "@UCBerkeley" { @@ -54,11 +60,11 @@ func (s *Sync) walletSetup() error { } minBalance := (float64(numOnSource)-float64(numPublished))*(publishAmount+0.1) + channelClaimAmount - if numPublished > numOnSource && balance.LessThan(decimal.NewFromFloat(1)) { + if numPublished > numOnSource && balance < 1 { SendErrorToSlack("something is going on as we published more videos than those available on source: %d/%d", numPublished, numOnSource) minBalance = 1 //since we ended up in this function it means some juice is still needed } - amountToAdd, _ := decimal.NewFromFloat(minBalance).Sub(balance).Float64() + amountToAdd := minBalance - balance if s.Refill > 0 { if amountToAdd < 0 { @@ -72,16 +78,19 @@ func (s *Sync) walletSetup() error { if amountToAdd < 1 { amountToAdd = 1 // no reason to bother adding less than 1 credit } - s.addCredits(amountToAdd) + err := s.addCredits(amountToAdd) + if err != nil { + return errors.Err(err) + } } - claimAddress, err := s.daemon.WalletUnusedAddress() + claimAddress, err := s.daemon.AddressList(nil) if err != nil { return err } else if claimAddress == nil { return errors.Err("could not get unused address") } - s.claimAddress = string(*claimAddress) + s.claimAddress = string((*claimAddress)[0]) if s.claimAddress == "" { return errors.Err("found blank claim address") } @@ -95,7 +104,26 @@ func (s *Sync) walletSetup() error { } func (s *Sync) ensureEnoughUTXOs() error { - utxolist, err := s.daemon.UTXOList() + accounts, err := s.daemon.AccountList() + if err != nil { + return errors.Err(err) + } + accountsNet := (*accounts).LBCMainnet + if os.Getenv("REGTEST") == "true" { + accountsNet = (*accounts).LBCRegtest + } + defaultAccount := "" + for _, account := range accountsNet { + if account.IsDefaultAccount { + defaultAccount = account.ID + break + } + } + if defaultAccount == "" { + return errors.Err("No default account found") + } + + utxolist, err := s.daemon.UTXOList(&defaultAccount) if err != nil { return err } else if utxolist == nil { @@ -107,26 +135,30 @@ func (s *Sync) ensureEnoughUTXOs() error { count := 0 for _, utxo := range *utxolist { - if !utxo.IsClaim && !utxo.IsSupport && !utxo.IsUpdate && utxo.Amount.Cmp(decimal.New(0, 0)) == 1 { + amount, _ := strconv.ParseFloat(utxo.Amount, 64) + if !utxo.IsClaim && !utxo.IsSupport && !utxo.IsUpdate && amount != 0.0 { count++ } } - + log.Infof("utxo count: %d", count) if count < target-slack { - newAddresses := target - count - - balance, err := s.daemon.WalletBalance() + balance, err := s.daemon.AccountBalance(&defaultAccount) if err != nil { return err } else if balance == nil { return errors.Err("no response") } - log.Println("balance is " + decimal.Decimal(*balance).String()) + balanceAmount, err := strconv.ParseFloat((string)(*balance), 64) + if err != nil { + return errors.Err(err) + } + broadcastFee := 0.001 + amountToSplit := fmt.Sprintf("%.6f", balanceAmount-broadcastFee) - amountPerAddress := decimal.Decimal(*balance).Div(decimal.NewFromFloat(float64(target))) - log.Infof("Putting %s credits into each of %d new addresses", amountPerAddress.String(), newAddresses) - prefillTx, err := s.daemon.WalletPrefillAddresses(newAddresses, amountPerAddress, true) + log.Infof("Splitting balance of %s evenly between 40 UTXOs", *balance) + + prefillTx, err := s.daemon.AccountFund(defaultAccount, defaultAccount, amountToSplit, uint64(target)) if err != nil { return err } else if prefillTx == nil { @@ -180,21 +212,21 @@ func (s *Sync) ensureChannelOwnership() error { return errors.Err("no channel name set") } - channels, err := s.daemon.ChannelList() + channels, err := s.daemon.ChannelList(nil, 1, 50) if err != nil { return err } else if channels == nil { return errors.Err("no channel response") } //special case for wallets we don't retain full control anymore - if len(*channels) > 1 { + if len((*channels).Items) > 1 { // This wallet is probably not under our control anymore but we still want to publish to it // here we shall check if within all the channels there is one that was created by ytsync SendInfoToSlack("we are dealing with a wallet that has multiple channels. This indicates that the wallet was probably transferred but we still want to sync their content. YoutubeID: %s", s.YoutubeChannelID) if s.lbryChannelID == "" { return errors.Err("this channel does not have a recorded claimID in the database. To prevent failures, updates are not supported until an entry is manually added in the database") } - for _, c := range *channels { + for _, c := range (*channels).Items { if c.ClaimID != s.lbryChannelID { if c.Name != s.LbryChannelName { return errors.Err("the channel in the wallet is different than the channel in the database") @@ -203,8 +235,8 @@ func (s *Sync) ensureChannelOwnership() error { } } } - if len(*channels) == 1 { - channel := (*channels)[0] + if len((*channels).Items) == 1 { + channel := ((*channels).Items)[0] if channel.Name == s.LbryChannelName { //TODO: eventually get rid of this when the whole db is filled if s.lbryChannelID == "" { @@ -221,13 +253,16 @@ func (s *Sync) ensureChannelOwnership() error { channelBidAmount := channelClaimAmount - balanceResp, err := s.daemon.WalletBalance() + balanceResp, err := s.daemon.AccountBalance(nil) if err != nil { return err } else if balanceResp == nil { return errors.Err("no response") } - balance := decimal.Decimal(*balanceResp) + balance, err := decimal.NewFromString((string)(*balanceResp)) + if err != nil { + return errors.Err(err) + } if balance.LessThan(decimal.NewFromFloat(channelBidAmount)) { err = s.addCredits(channelBidAmount + 0.1) @@ -236,7 +271,7 @@ func (s *Sync) ensureChannelOwnership() error { } } - c, err := s.daemon.ChannelNew(s.LbryChannelName, channelBidAmount) + c, err := s.daemon.ChannelNew(s.LbryChannelName, channelBidAmount, nil) if err != nil { return err } @@ -278,7 +313,7 @@ func (s *Sync) addCredits(amountToAdd float64) error { } } - addressResp, err := s.daemon.WalletUnusedAddress() + addressResp, err := s.daemon.AddressUnused(nil) if err != nil { return err } else if addressResp == nil { diff --git a/manager/ytsync.go b/manager/ytsync.go index 0d79bbb..08c407a 100644 --- a/manager/ytsync.go +++ b/manager/ytsync.go @@ -388,7 +388,7 @@ func logShutdownError(shutdownErr error) { } func isYtsyncClaim(c jsonrpc.Claim) bool { - if !util.InSlice(c.Category, []string{"claim", "update"}) || c.Value.Stream == nil { + if !util.InSlice(c.Type, []string{"claim", "update"}) || c.Value.Stream == nil { return false } if c.Value.Stream.Metadata == nil || c.Value.Stream.Metadata.Thumbnail == nil { @@ -424,7 +424,7 @@ func (s *Sync) fixDupes(claims []jsonrpc.Claim) (bool, error) { videoIDs[videoID] = c } log.Debugf("abandoning %+v", claimToAbandon) - _, err := s.daemon.ClaimAbandon(claimToAbandon.Txid, claimToAbandon.Nout) + _, err := s.daemon.ClaimAbandon(claimToAbandon.Txid, claimToAbandon.Nout, nil, false) if err != nil { return true, err } @@ -459,18 +459,31 @@ func (s *Sync) updateRemoteDB(claims []jsonrpc.Claim) (total int, fixed int, err return count, fixed, nil } +func (s *Sync) getClaims() ([]jsonrpc.Claim, error) { + totalPages := uint64(1) + var allClaims []jsonrpc.Claim + for page := uint64(1); page <= totalPages; page++ { + claims, err := s.daemon.ClaimListMine(nil, page, 50) + if err != nil { + return nil, errors.Prefix("cannot list claims", err) + } + allClaims = append(allClaims, (*claims).Claims...) + totalPages = (*claims).TotalPages + } + return allClaims, nil +} + func (s *Sync) doSync() error { var err error err = s.walletSetup() if err != nil { return errors.Prefix("Initial wallet setup failed! Manual Intervention is required.", err) } - - claims, err := s.daemon.ClaimListMine() + allClaims, err := s.getClaims() if err != nil { - return errors.Prefix("cannot list claims", err) + return err } - hasDupes, err := s.fixDupes(*claims) + hasDupes, err := s.fixDupes(allClaims) if err != nil { return errors.Prefix("error checking for duplicates", err) } @@ -480,13 +493,13 @@ func (s *Sync) doSync() error { if err != nil { return err } - claims, err = s.daemon.ClaimListMine() + allClaims, err = s.getClaims() if err != nil { - return errors.Prefix("cannot list claims", err) + return err } } - pubsOnWallet, nFixed, err := s.updateRemoteDB(*claims) + pubsOnWallet, nFixed, err := s.updateRemoteDB(allClaims) if err != nil { return errors.Prefix("error counting claims", err) } @@ -852,7 +865,7 @@ func waitForDaemonProcess(timeout time.Duration) error { } var daemonProcessId = -1 for _, p := range processes { - if p.Executable() == "lbrynet-daemon" { + if p.Executable() == "lbrynet" { daemonProcessId = p.Pid() break } diff --git a/sources/ucbVideo.go b/sources/ucbVideo.go index c4d7540..94de1ac 100644 --- a/sources/ucbVideo.go +++ b/sources/ucbVideo.go @@ -176,17 +176,19 @@ func (v *ucbVideo) saveThumbnail() error { func (v *ucbVideo) publish(daemon *jsonrpc.Client, claimAddress string, amount float64, channelID string, namer *namer.Namer) (*SyncSummary, error) { options := jsonrpc.PublishOptions{ - Title: &v.title, - Author: strPtr("UC Berkeley"), - Description: strPtr(v.getAbbrevDescription()), - Language: strPtr("en"), - ClaimAddress: &claimAddress, - Thumbnail: strPtr("https://berk.ninja/thumbnails/" + v.id), - License: strPtr("see description"), + Metadata: &jsonrpc.Metadata{ + Title: v.title, + Description: v.getAbbrevDescription(), + Author: "UC Berkeley", + Language: "en", + License: "see description", + Thumbnail: strPtr("https://berk.ninja/thumbnails/" + v.id), + NSFW: false, + }, ChannelID: &channelID, + ClaimAddress: &claimAddress, ChangeAddress: &claimAddress, } - return publishAndRetryExistingNames(daemon, v.title, v.getFilename(), amount, options, namer) } diff --git a/sources/youtubeVideo.go b/sources/youtubeVideo.go index b29f152..08bc009 100644 --- a/sources/youtubeVideo.go +++ b/sources/youtubeVideo.go @@ -244,17 +244,19 @@ func (v *YoutubeVideo) publish(daemon *jsonrpc.Client, claimAddress string, amou return nil, errors.Err("a claim_id for the channel wasn't provided") //TODO: this is probably not needed? } options := jsonrpc.PublishOptions{ - Title: &v.title, - Author: &v.channelTitle, - Description: strPtr(v.getAbbrevDescription() + "\nhttps://www.youtube.com/watch?v=" + v.id), - Language: strPtr("en"), - ClaimAddress: &claimAddress, - Thumbnail: strPtr("https://berk.ninja/thumbnails/" + v.id), - License: strPtr("Copyrighted (contact author)"), - ChangeAddress: &claimAddress, + Metadata: &jsonrpc.Metadata{ + Title: v.title, + Description: v.getAbbrevDescription() + "\nhttps://www.youtube.com/watch?v=" + v.id, + Author: v.channelTitle, + Language: "en", + License: "Copyrighted (contact author)", + Thumbnail: strPtr("https://berk.ninja/thumbnails/" + v.id), + NSFW: false, + }, ChannelID: &channelID, + ClaimAddress: &claimAddress, + ChangeAddress: &claimAddress, } - return publishAndRetryExistingNames(daemon, v.title, v.getFullPath(), amount, options, namer) }