LBRYCommunity/ytsync
1
0
Fork 0
Code Issues 39 Pull requests 8 Packages Projects Releases 197 Wiki Activity

upgrade ytsync to latest SDK

Browse source
This commit is contained in:
Niko Storni 2019-01-30 13:42:23 +01:00
parent 6c3338cad3
commit 5e125fa641
6 changed files with 134 additions and 93 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

61
Gopkg.lock generated
View file

@ -104,6 +104,14 @@
pruneopts = ""
revision = "31079b6807923eb23992c421b114992b95131b55"
[[projects]]
digest = "1:2defd14040b1ca18840524e9d0f9cfab0ea8d2f2d396b343d221f8bc8e21ab9d"
name = "github.com/fatih/structs"
packages = ["."]
pruneopts = ""
revision = "4966fc68f5b7593aafa6cbbba2d65ec6e1416047"
version = "v1.1.0"
[[projects]]
digest = "1:968d8903d598e3fae738325d3410f33f07ea6a2b9ee5591e9c262ee37df6845a"
name = "github.com/go-errors/errors"
@ -113,12 +121,12 @@
version = "v1.0.1"
[[projects]]
branch = "master"
digest = "1:a6f43a35242db0a1387907e6fd0c7e1dccfbed95c7176f166d31d0b5a96c65fc"
digest = "1:ed30e45cd59986b6f98b4d191157a54050f5dd550e2cec47ad032b6230e4af08"
name = "github.com/go-ini/ini"
packages = ["."]
pruneopts = ""
revision = "7b294651033cd7d9e7f0d9ffa1b75ed1e198e737"
revision = "6ed8d5f64cd79a498d1f3fab5880cc376ce41bbe"
version = "v1.41.0"
[[projects]]
digest = "1:3dd078fda7500c341bc26cfbc6c6a34614f295a2457149fc1045cab767cbcf18"
@ -128,14 +136,6 @@
revision = "aa810b61a9c79d51363740d207bb46cf8e620ed5"
version = "v1.2.0"
[[projects]]
digest = "1:09aa5dd1332b93c96bde671bafb053249dc813febf7d5ca84e8f382ba255d67d"
name = "github.com/gorilla/websocket"
packages = ["."]
pruneopts = ""
revision = "66b9c49e59c6c48f0ffce28c2d8b8a5678502c6d"
version = "v1.4.0"
[[projects]]
digest = "1:870d441fe217b8e689d7949fef6e43efbc787e50f200cb1e70dbca9204a1d6be"
name = "github.com/inconshreveable/mousetrap"
@ -161,8 +161,8 @@
version = "v1.0.1"
[[projects]]
branch = "master"
digest = "1:39ddb14ec4559ea9fe8bc574654a50e96659ccbc75845dbdf7a1f267debedd3b"
branch = "lbrynet-updates"
digest = "1:d98f9853a5d737be2a133ee35f91acd07ef2bb9dfa1b7edbb24893f5d106850f"
name = "github.com/lbryio/lbry.go"
packages = [
"extras/errors",
@ -173,7 +173,7 @@
"lbrycrd",
]
pruneopts = ""
revision = "51d03937c259085e271bdd4c611b8100213c2dd6"
revision = "d7286d45411366b24f0c453fa6876349a944a1e7"
[[projects]]
branch = "master"
@ -192,31 +192,19 @@
revision = "4fdf99ab29366514c69ccccddab5dc58b8d84062"
[[projects]]
branch = "master"
digest = "1:bcc46a0fbd9e933087bef394871256b5c60269575bb661935874729c65bbbf60"
name = "github.com/mitchellh/mapstructure"
packages = ["."]
pruneopts = ""
revision = "3536a929edddb9a5b34bd6861dc4a9647cb459fe"
version = "v1.1.2"
[[projects]]
digest = "1:9407d3c57389f400ef8c1039e352ce00e2ab97c41bf1babba97c2a6922b7111e"
digest = "1:a52e6788022ede52706a9607f2a53331cf7ef2cf846377bc3ded8d523c2bc1af"
name = "github.com/nlopes/slack"
packages = [
".",
"slackutilsx",
]
pruneopts = ""
revision = "b9033a72a20bf84563485e86a2adbea4bf265804"
version = "v0.4.0"
[[projects]]
digest = "1:7365acd48986e205ccb8652cc746f09c8b7876030d53710ea6ef7d0bd0dcd7ca"
name = "github.com/pkg/errors"
packages = ["."]
pruneopts = ""
revision = "645ef00459ed84a119197bfb8d8205042c6df63d"
version = "v0.8.0"
revision = "f243c7602fdf906248fc1f165284daa4e4d2d09e"
[[projects]]
branch = "master"
@ -227,20 +215,20 @@
revision = "44fe64bba886b85c00fc7045e092d796484d6f90"
[[projects]]
branch = "master"
digest = "1:615c827f6a892973a587c754ae5fad7acfc4352657aff23d0238fe0ba2a154df"
name = "github.com/shopspring/decimal"
packages = ["."]
pruneopts = ""
revision = "cd690d0c9e2447b1ef2a129a6b7b49077da89b8e"
version = "1.1.0"
[[projects]]
branch = "master"
digest = "1:c0959eb8fc519d62edc717e2997032859688c0e1ddd9d4463512ead83576d247"
digest = "1:9a3c631555e0351fdc4e696577bb63afd90c399d782a8462dba9d100d7021db3"
name = "github.com/sirupsen/logrus"
packages = ["."]
pruneopts = ""
revision = "3f90cee1e41a38ba27c831844c002952512997c0"
revision = "e1e72e9de974bd926e5c56f83753fba2df402ce5"
version = "v1.3.0"
[[projects]]
branch = "master"
@ -259,12 +247,12 @@
version = "v1.0.3"
[[projects]]
branch = "master"
digest = "1:22d3674d44ee93f52a9c0b6a22d1f736a0ad9ac3f9d2c1ca8648f3c9ce9910bd"
digest = "1:1a7da329bbaa93d90fad3ee12d8d0fc7dff08bcb3e59d4002c827b91f35182b4"
name = "github.com/ybbus/jsonrpc"
packages = ["."]
pruneopts = ""
revision = "2a548b7d822dd62717337a6b1e817fae1b14660a"
revision = "dd866631e904a5df2067d934985c5def68f391ac"
version = "v2.1.2"
[[projects]]
branch = "master"
@ -286,6 +274,7 @@
"context/ctxhttp",
"html",
"html/atom",
"websocket",
]
pruneopts = ""
revision = "146acd28ed5894421fb5aac80ca93bc1b1f46f87"

6
Gopkg.toml
View file

@ -3,12 +3,12 @@
name = "github.com/rylio/ytdl"
[[constraint]]
branch = "master"
version = "^1.1.0"
name = "github.com/shopspring/decimal"
[[constraint]]
name = "github.com/sirupsen/logrus"
branch = "master"
version = "^1.3.0"
[[constraint]]
branch = "master"
@ -24,4 +24,4 @@
[[constraint]]
name = "github.com/lbryio/lbry.go"
branch = "master"
branch = "lbrynet-updates"

89
manager/setup.go
View file

@ -1,6 +1,9 @@
package manager
import (
"fmt"
"os"
"strconv"
"time"
"github.com/lbryio/lbry.go/extras/errors"
@ -20,14 +23,17 @@ func (s *Sync) walletSetup() error {
return err
}
balanceResp, err := s.daemon.WalletBalance()
balanceResp, err := s.daemon.AccountBalance(nil)
if err != nil {
return err
} else if balanceResp == nil {
return errors.Err("no response")
}
balance := decimal.Decimal(*balanceResp)
log.Debugf("Starting balance is %s", balance.String())
balance, err := strconv.ParseFloat((string)(*balanceResp), 64)
if err != nil {
return errors.Err(err)
}
log.Debugf("Starting balance is %.4f", balance)
var numOnSource int
if s.LbryChannelName == "@UCBerkeley" {
@ -54,11 +60,11 @@ func (s *Sync) walletSetup() error {
}
minBalance := (float64(numOnSource)-float64(numPublished))*(publishAmount+0.1) + channelClaimAmount
if numPublished > numOnSource && balance.LessThan(decimal.NewFromFloat(1)) {
if numPublished > numOnSource && balance < 1 {
SendErrorToSlack("something is going on as we published more videos than those available on source: %d/%d", numPublished, numOnSource)
minBalance = 1 //since we ended up in this function it means some juice is still needed
}
amountToAdd, _ := decimal.NewFromFloat(minBalance).Sub(balance).Float64()
amountToAdd := minBalance - balance
if s.Refill > 0 {
if amountToAdd < 0 {
@ -72,16 +78,19 @@ func (s *Sync) walletSetup() error {
if amountToAdd < 1 {
amountToAdd = 1 // no reason to bother adding less than 1 credit
}
s.addCredits(amountToAdd)
err := s.addCredits(amountToAdd)
if err != nil {
return errors.Err(err)
}
}
claimAddress, err := s.daemon.WalletUnusedAddress()
claimAddress, err := s.daemon.AddressList(nil)
if err != nil {
return err
} else if claimAddress == nil {
return errors.Err("could not get unused address")
}
s.claimAddress = string(*claimAddress)
s.claimAddress = string((*claimAddress)[0])
if s.claimAddress == "" {
return errors.Err("found blank claim address")
}
@ -95,7 +104,26 @@ func (s *Sync) walletSetup() error {
}
func (s *Sync) ensureEnoughUTXOs() error {
utxolist, err := s.daemon.UTXOList()
accounts, err := s.daemon.AccountList()
if err != nil {
return errors.Err(err)
}
accountsNet := (*accounts).LBCMainnet
if os.Getenv("REGTEST") == "true" {
accountsNet = (*accounts).LBCRegtest
}
defaultAccount := ""
for _, account := range accountsNet {
if account.IsDefaultAccount {
defaultAccount = account.ID
break
}
}
if defaultAccount == "" {
return errors.Err("No default account found")
}
utxolist, err := s.daemon.UTXOList(&defaultAccount)
if err != nil {
return err
} else if utxolist == nil {
@ -107,26 +135,30 @@ func (s *Sync) ensureEnoughUTXOs() error {
count := 0
for _, utxo := range *utxolist {
if !utxo.IsClaim && !utxo.IsSupport && !utxo.IsUpdate && utxo.Amount.Cmp(decimal.New(0, 0)) == 1 {
amount, _ := strconv.ParseFloat(utxo.Amount, 64)
if !utxo.IsClaim && !utxo.IsSupport && !utxo.IsUpdate && amount != 0.0 {
count++
}
}
log.Infof("utxo count: %d", count)
if count < target-slack {
newAddresses := target - count
balance, err := s.daemon.WalletBalance()
balance, err := s.daemon.AccountBalance(&defaultAccount)
if err != nil {
return err
} else if balance == nil {
return errors.Err("no response")
}
log.Println("balance is " + decimal.Decimal(*balance).String())
balanceAmount, err := strconv.ParseFloat((string)(*balance), 64)
if err != nil {
return errors.Err(err)
}
broadcastFee := 0.001
amountToSplit := fmt.Sprintf("%.6f", balanceAmount-broadcastFee)
amountPerAddress := decimal.Decimal(*balance).Div(decimal.NewFromFloat(float64(target)))
log.Infof("Putting %s credits into each of %d new addresses", amountPerAddress.String(), newAddresses)
prefillTx, err := s.daemon.WalletPrefillAddresses(newAddresses, amountPerAddress, true)
log.Infof("Splitting balance of %s evenly between 40 UTXOs", *balance)
prefillTx, err := s.daemon.AccountFund(defaultAccount, defaultAccount, amountToSplit, uint64(target))
if err != nil {
return err
} else if prefillTx == nil {
@ -180,21 +212,21 @@ func (s *Sync) ensureChannelOwnership() error {
return errors.Err("no channel name set")
}
channels, err := s.daemon.ChannelList()
channels, err := s.daemon.ChannelList(nil, 1, 50)
if err != nil {
return err
} else if channels == nil {
return errors.Err("no channel response")
}
//special case for wallets we don't retain full control anymore
if len(*channels) > 1 {
if len((*channels).Items) > 1 {
// This wallet is probably not under our control anymore but we still want to publish to it
// here we shall check if within all the channels there is one that was created by ytsync
SendInfoToSlack("we are dealing with a wallet that has multiple channels. This indicates that the wallet was probably transferred but we still want to sync their content. YoutubeID: %s", s.YoutubeChannelID)
if s.lbryChannelID == "" {
return errors.Err("this channel does not have a recorded claimID in the database. To prevent failures, updates are not supported until an entry is manually added in the database")
}
for _, c := range *channels {
for _, c := range (*channels).Items {
if c.ClaimID != s.lbryChannelID {
if c.Name != s.LbryChannelName {
return errors.Err("the channel in the wallet is different than the channel in the database")
@ -203,8 +235,8 @@ func (s *Sync) ensureChannelOwnership() error {
}
}
}
if len(*channels) == 1 {
channel := (*channels)[0]
if len((*channels).Items) == 1 {
channel := ((*channels).Items)[0]
if channel.Name == s.LbryChannelName {
//TODO: eventually get rid of this when the whole db is filled
if s.lbryChannelID == "" {
@ -221,13 +253,16 @@ func (s *Sync) ensureChannelOwnership() error {
channelBidAmount := channelClaimAmount
balanceResp, err := s.daemon.WalletBalance()
balanceResp, err := s.daemon.AccountBalance(nil)
if err != nil {
return err
} else if balanceResp == nil {
return errors.Err("no response")
}
balance := decimal.Decimal(*balanceResp)
balance, err := decimal.NewFromString((string)(*balanceResp))
if err != nil {
return errors.Err(err)
}
if balance.LessThan(decimal.NewFromFloat(channelBidAmount)) {
err = s.addCredits(channelBidAmount + 0.1)
@ -236,7 +271,7 @@ func (s *Sync) ensureChannelOwnership() error {
}
}
c, err := s.daemon.ChannelNew(s.LbryChannelName, channelBidAmount)
c, err := s.daemon.ChannelNew(s.LbryChannelName, channelBidAmount, nil)
if err != nil {
return err
}
@ -278,7 +313,7 @@ func (s *Sync) addCredits(amountToAdd float64) error {
}
}
addressResp, err := s.daemon.WalletUnusedAddress()
addressResp, err := s.daemon.AddressUnused(nil)
if err != nil {
return err
} else if addressResp == nil {

33
manager/ytsync.go
View file

@ -388,7 +388,7 @@ func logShutdownError(shutdownErr error) {
}
func isYtsyncClaim(c jsonrpc.Claim) bool {
if !util.InSlice(c.Category, []string{"claim", "update"}) || c.Value.Stream == nil {
if !util.InSlice(c.Type, []string{"claim", "update"}) || c.Value.Stream == nil {
return false
}
if c.Value.Stream.Metadata == nil || c.Value.Stream.Metadata.Thumbnail == nil {
@ -424,7 +424,7 @@ func (s *Sync) fixDupes(claims []jsonrpc.Claim) (bool, error) {
videoIDs[videoID] = c
}
log.Debugf("abandoning %+v", claimToAbandon)
_, err := s.daemon.ClaimAbandon(claimToAbandon.Txid, claimToAbandon.Nout)
_, err := s.daemon.ClaimAbandon(claimToAbandon.Txid, claimToAbandon.Nout, nil, false)
if err != nil {
return true, err
}
@ -459,18 +459,31 @@ func (s *Sync) updateRemoteDB(claims []jsonrpc.Claim) (total int, fixed int, err
return count, fixed, nil
}
func (s *Sync) getClaims() ([]jsonrpc.Claim, error) {
totalPages := uint64(1)
var allClaims []jsonrpc.Claim
for page := uint64(1); page <= totalPages; page++ {
claims, err := s.daemon.ClaimListMine(nil, page, 50)
if err != nil {
return nil, errors.Prefix("cannot list claims", err)
}
allClaims = append(allClaims, (*claims).Claims...)
totalPages = (*claims).TotalPages
}
return allClaims, nil
}
func (s *Sync) doSync() error {
var err error
err = s.walletSetup()
if err != nil {
return errors.Prefix("Initial wallet setup failed! Manual Intervention is required.", err)
}
claims, err := s.daemon.ClaimListMine()
allClaims, err := s.getClaims()
if err != nil {
return errors.Prefix("cannot list claims", err)
return err
}
hasDupes, err := s.fixDupes(*claims)
hasDupes, err := s.fixDupes(allClaims)
if err != nil {
return errors.Prefix("error checking for duplicates", err)
}
@ -480,13 +493,13 @@ func (s *Sync) doSync() error {
if err != nil {
return err
}
claims, err = s.daemon.ClaimListMine()
allClaims, err = s.getClaims()
if err != nil {
return errors.Prefix("cannot list claims", err)
return err
}
}
pubsOnWallet, nFixed, err := s.updateRemoteDB(*claims)
pubsOnWallet, nFixed, err := s.updateRemoteDB(allClaims)
if err != nil {
return errors.Prefix("error counting claims", err)
}
@ -852,7 +865,7 @@ func waitForDaemonProcess(timeout time.Duration) error {
}
var daemonProcessId = -1
for _, p := range processes {
if p.Executable() == "lbrynet-daemon" {
if p.Executable() == "lbrynet" {
daemonProcessId = p.Pid()
break
}

16
sources/ucbVideo.go
View file

@ -176,17 +176,19 @@ func (v *ucbVideo) saveThumbnail() error {
func (v *ucbVideo) publish(daemon *jsonrpc.Client, claimAddress string, amount float64, channelID string, namer *namer.Namer) (*SyncSummary, error) {
options := jsonrpc.PublishOptions{
Title: &v.title,
Author: strPtr("UC Berkeley"),
Description: strPtr(v.getAbbrevDescription()),
Language: strPtr("en"),
ClaimAddress: &claimAddress,
Metadata: &jsonrpc.Metadata{
Title: v.title,
Description: v.getAbbrevDescription(),
Author: "UC Berkeley",
Language: "en",
License: "see description",
Thumbnail: strPtr("https://berk.ninja/thumbnails/" + v.id),
License: strPtr("see description"),
NSFW: false,
},
ChannelID: &channelID,
ClaimAddress: &claimAddress,
ChangeAddress: &claimAddress,
}
return publishAndRetryExistingNames(daemon, v.title, v.getFilename(), amount, options, namer)
}

18
sources/youtubeVideo.go
View file

@ -244,17 +244,19 @@ func (v *YoutubeVideo) publish(daemon *jsonrpc.Client, claimAddress string, amou
return nil, errors.Err("a claim_id for the channel wasn't provided") //TODO: this is probably not needed?
}
options := jsonrpc.PublishOptions{
Title: &v.title,
Author: &v.channelTitle,
Description: strPtr(v.getAbbrevDescription() + "\nhttps://www.youtube.com/watch?v=" + v.id),
Language: strPtr("en"),
ClaimAddress: &claimAddress,
Metadata: &jsonrpc.Metadata{
Title: v.title,
Description: v.getAbbrevDescription() + "\nhttps://www.youtube.com/watch?v=" + v.id,
Author: v.channelTitle,
Language: "en",
License: "Copyrighted (contact author)",
Thumbnail: strPtr("https://berk.ninja/thumbnails/" + v.id),
License: strPtr("Copyrighted (contact author)"),
ChangeAddress: &claimAddress,
NSFW: false,
},
ChannelID: &channelID,
ClaimAddress: &claimAddress,
ChangeAddress: &claimAddress,
}
return publishAndRetryExistingNames(daemon, v.title, v.getFullPath(), amount, options, namer)
}