upgrade ytsync to latest SDK

upgrade e2e tests

e2e/daemon_settings.yml

@@ -1,12 +1,9 @@
-#blockchain_name: lbrycrd_main
-#blockchain_name: lbrycrd_testnet
 blockchain_name: lbrycrd_regtest
 lbryum_servers:
-#  - spv1.lbry.com:50001 #Production Wallet Server
+- walletserver:50001
-  - walletserver:50001
+reflect_streams: false
 save_blobs: true
 save_files: false
-reflect_streams: false #for the love of god, don't upload regtest streams to reflector!
 share_usage_data: false
 tcp_port: 3333
 udp_port: 4444

e2e/docker-compose.yml

@@ -4,7 +4,7 @@ services:
   ## Lbrycrd ##
   #############
   lbrycrd:
-    image: lbry/lbrycrd:v0.17.2.1
+    image: lbry/lbrycrd:v0.17.3.2-deprecatedrpc
     restart: "no"
     ports:
       - "15201:29246"
@@ -21,7 +21,7 @@ services:
   ## Wallet Server ##
   ###################
   walletserver:
-    image: lbry/wallet-server:v0.42.2
+    image: lbry/wallet-server:v0.48.1
     restart: always
     environment:
       - DB_DIRECTORY=/database
@@ -47,7 +47,7 @@ services:
   ## Lbrynet ##
   #############
   lbrynet:
-    image: lbry/lbrynet:v0.42.2
+    image: lbry/lbrynet:v0.48.1
     restart: always
     ports:
       - "15100:5279"

e2e/lbrycrd/docker/start.sh

@@ -11,9 +11,9 @@ function override_config_option() {
     local var=$1 option=$2 config=$3
     if [[ -v $var ]]; then
         # Remove the existing config option:
-        sed -i "/^$option\W*=/d" $config
+        sed -i "/^$option\W*=/d" "$config"
         # Add the value from the environment:
-        echo "$option=${!var}" >> $config
+        echo "$option=${!var}" >> "$config"
     fi
 }
 
@@ -38,15 +38,17 @@ function set_config() {
   else
       echo "Creating a fresh config file from environment variables."
       ## Set config params
-      echo "port=${PORT=9246}" >                        $CONFIG_PATH
+      {
-      echo "rpcuser=${RPC_USER=lbry}" >>                $CONFIG_PATH
+        echo "port=${PORT=9246}"
-      echo "rpcpassword=${RPC_PASSWORD=lbry}" >>        $CONFIG_PATH
+        echo "rpcuser=${RPC_USER=lbry}"
-      echo "rpcallowip=${RPC_ALLOW_IP=127.0.0.1/24}" >> $CONFIG_PATH
+        echo "rpcpassword=${RPC_PASSWORD=lbry}"
-      echo "rpcport=${RPC_PORT=9245}" >>                $CONFIG_PATH
+        echo "rpcallowip=${RPC_ALLOW_IP=127.0.0.1/24}"
-      echo "rpcbind=${RPC_BIND=0.0.0.0}" >>             $CONFIG_PATH
+        echo "rpcport=${RPC_PORT=9245}"
-      echo "deprecatedrpc=accounts" >>                  $CONFIG_PATH
+        echo "rpcbind=${RPC_BIND=0.0.0.0}"
-      echo "deprecatedrpc=validateaddress" >>           $CONFIG_PATH
+        echo "deprecatedrpc=accounts"
-      echo "deprecatedrpc=signrawtransaction" >>        $CONFIG_PATH
+        echo "deprecatedrpc=validateaddress"
+        echo "deprecatedrpc=signrawtransaction"
+      } >>  $CONFIG_PATH
   fi
   echo "Config: "
   cat $CONFIG_PATH
@@ -79,7 +81,7 @@ case $RUN_MODE in
   regtest )
     ## Set config params
     ## TODO: Make this more automagic in the future.
-    mkdir -p `dirname $CONFIG_PATH`
+    mkdir -p "$(dirname $CONFIG_PATH)"
     echo "rpcuser=lbry" >                       $CONFIG_PATH
     echo "rpcpassword=lbry" >>                  $CONFIG_PATH
     echo "rpcport=29245" >>                     $CONFIG_PATH
@@ -101,7 +103,7 @@ case $RUN_MODE in
   testnet )
     ## Set config params
     ## TODO: Make this more automagic in the future.
-    mkdir -p `dirname $CONFIG_PATH`
+    mkdir -p "$(dirname $CONFIG_PATH)"
     echo "rpcuser=lbry" >                       $CONFIG_PATH
     echo "rpcpassword=lbry" >>                  $CONFIG_PATH
     echo "rpcport=29245" >>                     $CONFIG_PATH

e2e/lbrynet/docker-compose.yml

@@ -8,7 +8,7 @@ services:
   ## Lbrynet ##
   #############
   lbrynet:
-    image: lbry/lbrynet:v0.38.6
+    image: lbry/lbrynet:v0.48.1
     restart: "no"
     networks:
       lbry-network:

go.mod

@@ -13,13 +13,12 @@ require (
 	github.com/docker/go-units v0.4.0 // indirect
 	github.com/go-sql-driver/mysql v1.4.1 // indirect
 	github.com/golang/groupcache v0.0.0-20191002201903-404acd9df4cc // indirect
-	github.com/google/btree v1.0.0 // indirect
 	github.com/hashicorp/go-immutable-radix v1.1.0 // indirect
 	github.com/hashicorp/go-sockaddr v1.0.2 // indirect
 	github.com/hashicorp/golang-lru v0.5.3 // indirect
 	github.com/hashicorp/memberlist v0.1.5 // indirect
 	github.com/hashicorp/serf v0.8.5 // indirect
-	github.com/lbryio/lbry.go/v2 v2.3.3-0.20191010144133-fa9bab156a4e
+	github.com/lbryio/lbry.go/v2 v2.4.1-0.20191206194319-06764c3d007e
 	github.com/lbryio/reflector.go v1.0.6-0.20190828131602-ce3d4403dbc6
 	github.com/miekg/dns v1.1.22 // indirect
 	github.com/mitchellh/go-ps v0.0.0-20190716172923-621e5597135b
@@ -34,3 +33,5 @@ require (
 	google.golang.org/api v0.11.0
 	google.golang.org/appengine v1.6.5 // indirect
 )
+
+go 1.13

go.sum

@@ -213,8 +213,8 @@ github.com/lbryio/errors.go v0.0.0-20180223142025-ad03d3cc6a5c/go.mod h1:muH7wpU
 github.com/lbryio/lbry.go v0.0.0-20190828131228-f3a1fbdd5303/go.mod h1:qR+Ui0hYhemIU4fXqM3d1P9eiaRFlof777VJgV7KJ8w=
 github.com/lbryio/lbry.go v1.1.2 h1:Dyxc+glT/rVWJwHfIf7vjlPYYbjzrQz5ARmJd5Hp69c=
 github.com/lbryio/lbry.go v1.1.2/go.mod h1:JtyI30bU51rm0LZ/po3mQuzf++14OWb6kR/6mMRAmKU=
-github.com/lbryio/lbry.go/v2 v2.3.3-0.20191010144133-fa9bab156a4e h1:B1ekb7lXFuMol/tUdwTajOe56ab38drBJmr0X8IoZYE=
+github.com/lbryio/lbry.go/v2 v2.4.1-0.20191206194319-06764c3d007e h1:wN6r46L8EjgC2JyOImagQXaQvyF8Bd2QPjfXqv7oDAg=
-github.com/lbryio/lbry.go/v2 v2.3.3-0.20191010144133-fa9bab156a4e/go.mod h1:Fm4jBlDyDauHIBC5OABuqcBWKBpcbRgPk5K0eG2mhSs=
+github.com/lbryio/lbry.go/v2 v2.4.1-0.20191206194319-06764c3d007e/go.mod h1:gbs/+PVII96AqsRgf+hY/F7sJkb/ciiX2yy3j+NApd0=
 github.com/lbryio/lbryschema.go v0.0.0-20190428231007-c54836bca002/go.mod h1:dAzPCBj3CKKWBGYBZxK6tKBP5SCgY2tqd9SnQd/OyKo=
 github.com/lbryio/lbryschema.go v0.0.0-20190602173230-6d2f69a36f46 h1:LemfR+rMxhf7nnOrzy2HqS7Me7SZ5gEwOcNFzKC8ySQ=
 github.com/lbryio/lbryschema.go v0.0.0-20190602173230-6d2f69a36f46/go.mod h1:dAzPCBj3CKKWBGYBZxK6tKBP5SCgY2tqd9SnQd/OyKo=

manager/setup.go

@@ -22,14 +22,21 @@ import (
 )
 
 func (s *Sync) enableAddressReuse() error {
-	accountsResponse, err := s.daemon.AccountList()
+	accountsResponse, err := s.daemon.AccountList(1, 50)
 	if err != nil {
 		return errors.Err(err)
 	}
-	accounts := accountsResponse.LBCMainnet
+	accounts := make([]jsonrpc.Account, 0, len(accountsResponse.Items))
+	ledger := "lbc_mainnet"
 	if logUtils.IsRegTest() {
-		accounts = accountsResponse.LBCRegtest
+		ledger = "lbc_regtest"
 	}
+	for _, a := range accountsResponse.Items {
+		if *a.Ledger == ledger {
+			accounts = append(accounts, a)
+		}
+	}
+
 	for _, a := range accounts {
 		_, err = s.daemon.AccountSet(a.ID, jsonrpc.AccountSettings{
 			ChangeMaxUses:    util.PtrToInt(1000),
@@ -121,13 +128,13 @@ func (s *Sync) walletSetup() error {
 		}
 	}
 
-	claimAddress, err := s.daemon.AddressList(nil, nil)
+	claimAddress, err := s.daemon.AddressList(nil, nil, 1, 20)
 	if err != nil {
 		return err
 	} else if claimAddress == nil {
-		return errors.Err("could not get unused address")
+		return errors.Err("could not get an address")
 	}
-	s.claimAddress = string((*claimAddress)[0].Address)
+	s.claimAddress = string(claimAddress.Items[0].Address)
 	if s.claimAddress == "" {
 		return errors.Err("found blank claim address")
 	}
@@ -145,20 +152,23 @@ func (s *Sync) walletSetup() error {
 
 func (s *Sync) getDefaultAccount() (string, error) {
 	if s.defaultAccountID == "" {
-		accounts, err := s.daemon.AccountList()
+		accountsResponse, err := s.daemon.AccountList(1, 50)
 		if err != nil {
 			return "", errors.Err(err)
 		}
-		accountsNet := (*accounts).LBCMainnet
+		ledger := "lbc_mainnet"
 		if logUtils.IsRegTest() {
-			accountsNet = (*accounts).LBCRegtest
+			ledger = "lbc_regtest"
 		}
-		for _, account := range accountsNet {
+		for _, a := range accountsResponse.Items {
-			if account.IsDefault {
+			if *a.Ledger == ledger {
-				s.defaultAccountID = account.ID
+				if a.IsDefault {
-				break
+					s.defaultAccountID = a.ID
+					break
+				}
 			}
 		}
+
 		if s.defaultAccountID == "" {
 			return "", errors.Err("No default account found")
 		}
@@ -172,7 +182,7 @@ func (s *Sync) ensureEnoughUTXOs() error {
 		return err
 	}
 
-	utxolist, err := s.daemon.UTXOList(&defaultAccount)
+	utxolist, err := s.daemon.UTXOList(&defaultAccount, 1, 10000)
 	if err != nil {
 		return err
 	} else if utxolist == nil {
@@ -184,7 +194,7 @@ func (s *Sync) ensureEnoughUTXOs() error {
 	count := 0
 	confirmedCount := 0
 
-	for _, utxo := range *utxolist {
+	for _, utxo := range utxolist.Items {
 		amount, _ := strconv.ParseFloat(utxo.Amount, 64)
 		if utxo.IsMine && utxo.Type == "payment" && amount > 0.001 {
 			if utxo.Confirmations > 0 {
@@ -241,24 +251,10 @@ func (s *Sync) ensureEnoughUTXOs() error {
 
 func (s *Sync) waitForNewBlock() error {
 	log.Printf("regtest: %t, docker: %t", logUtils.IsRegTest(), logUtils.IsUsingDocker())
-	if logUtils.IsRegTest() && logUtils.IsUsingDocker() {
-		lbrycrd, err := logUtils.GetLbrycrdClient(s.LbrycrdString)
-		if err != nil {
-			return errors.Prefix("error getting lbrycrd client: ", err)
-		}
-		txs, err := lbrycrd.Generate(1)
-		if err != nil {
-			return errors.Prefix("error generating new block: ", err)
-		}
-		for _, tx := range txs {
-			log.Info("Generated tx: ", tx.String())
-		}
-	}
 	status, err := s.daemon.Status()
 	if err != nil {
 		return err
 	}
-
 	for status.Wallet.Blocks == 0 || status.Wallet.BlocksBehind != 0 {
 		time.Sleep(5 * time.Second)
 		status, err = s.daemon.Status()
@@ -266,11 +262,18 @@ func (s *Sync) waitForNewBlock() error {
 			return err
 		}
 	}
+
 	currentBlock := status.Wallet.Blocks
 	for i := 0; status.Wallet.Blocks <= currentBlock; i++ {
 		if i%3 == 0 {
 			log.Printf("Waiting for new block (%d)...", currentBlock+1)
 		}
+		if logUtils.IsRegTest() && logUtils.IsUsingDocker() {
+			err = s.GenerateRegtestBlock()
+			if err != nil {
+				return err
+			}
+		}
 		time.Sleep(10 * time.Second)
 		status, err = s.daemon.Status()
 		if err != nil {
@@ -280,6 +283,21 @@ func (s *Sync) waitForNewBlock() error {
 	return nil
 }
 
+func (s *Sync) GenerateRegtestBlock() error {
+	lbrycrd, err := logUtils.GetLbrycrdClient(s.LbrycrdString)
+	if err != nil {
+		return errors.Prefix("error getting lbrycrd client: ", err)
+	}
+	txs, err := lbrycrd.Generate(1)
+	if err != nil {
+		return errors.Prefix("error generating new block: ", err)
+	}
+	for _, tx := range txs {
+		log.Info("Generated tx: ", tx.String())
+	}
+	return nil
+}
+
 func (s *Sync) ensureChannelOwnership() error {
 	if s.LbryChannelName == "" {
 		return errors.Err("no channel name set")

manager/transfer.go

@@ -24,14 +24,14 @@ func waitConfirmations(s *Sync) error {
 	allConfirmed := false
 waiting:
 	for !allConfirmed {
-		utxolist, err := s.daemon.UTXOList(&defaultAccount)
+		utxolist, err := s.daemon.UTXOList(&defaultAccount, 1, 10000)
 		if err != nil {
 			return err
 		} else if utxolist == nil {
 			return errors.Err("no response")
 		}
 
-		for _, utxo := range *utxolist {
+		for _, utxo := range utxolist.Items {
 			if utxo.Confirmations <= 0 {
 				err = s.waitForNewBlock()
 				if err != nil {
@@ -177,7 +177,7 @@ func transferVideos(s *Sync) error {
 	if err != nil {
 		return err
 	}
-	streams, err := s.daemon.StreamList(&account)
+	streams, err := s.daemon.StreamList(&account, 1, 30000)
 	if err != nil {
 		return errors.Err(err)
 	}
@@ -191,7 +191,7 @@ func transferVideos(s *Sync) error {
 			}
 
 			var stream *jsonrpc.Claim = nil
-			for _, c := range *streams {
+			for _, c := range streams.Items {
 				if c.ClaimID != video.ClaimID {
 					continue
 				}

manager/ytsync.go

@@ -354,7 +354,7 @@ func (s *Sync) processTransfers() (e error) {
 			return err
 		}
 		isTip := true
-		summary, err := s.daemon.SupportCreate(s.lbryChannelID, fmt.Sprintf("%.6f", supportAmount), &isTip, nil, []string{defaultAccount})
+		summary, err := s.daemon.SupportCreate(s.lbryChannelID, fmt.Sprintf("%.6f", supportAmount), &isTip, nil, []string{defaultAccount}, nil)
 		if err != nil {
 			return errors.Err(err)
 		}
@@ -670,12 +670,12 @@ func (s *Sync) getClaims(defaultOnly bool) ([]jsonrpc.Claim, error) {
 		}
 		account = &a
 	}
-	claims, err := s.daemon.StreamList(account)
+	claims, err := s.daemon.StreamList(account, 1, 30000)
 	if err != nil {
 		return nil, errors.Prefix("cannot list claims", err)
 	}
 
-	return *claims, nil
+	return claims.Items, nil
 }
 
 func (s *Sync) checkIntegrity() error {
@@ -1134,17 +1134,19 @@ func (s *Sync) processVideo(v video) (err error) {
 
 func (s *Sync) importPublicKey() error {
 	if s.publicKey != "" {
-		accountsResponse, err := s.daemon.AccountList()
+		accountsResponse, err := s.daemon.AccountList(1, 50)
 		if err != nil {
 			return errors.Err(err)
 		}
-		accounts := accountsResponse.LBCMainnet
+		ledger := "lbc_mainnet"
 		if logUtils.IsRegTest() {
-			accounts = accountsResponse.LBCRegtest
+			ledger = "lbc_regtest"
 		}
-		for _, a := range accounts {
+		for _, a := range accountsResponse.Items {
-			if a.PublicKey == s.publicKey {
+			if *a.Ledger == ledger {
-				return nil
+				if a.PublicKey == s.publicKey {
+					return nil
+				}
 			}
 		}
 		log.Infof("Could not find public key %s in the wallet. Importing it...", s.publicKey)
@@ -1172,12 +1174,12 @@ func (s *Sync) getUnsentSupports() (float64, error) {
 		if err != nil {
 			return 0, errors.Err(err)
 		}
-		transactionList, err := s.daemon.TransactionList(&defaultAccount)
+		transactionList, err := s.daemon.TransactionList(&defaultAccount, 1, 90000)
 		if err != nil {
 			return 0, errors.Err(err)
 		}
 		sentSupports := 0.0
-		for _, t := range *transactionList {
+		for _, t := range transactionList.Items {
 			if len(t.SupportInfo) == 0 {
 				continue
 			}

sources/youtubeVideo.go

@@ -489,7 +489,7 @@ func (v *YoutubeVideo) getMetadata() (languages []string, locations []jsonrpc.Lo
 }
 
 func (v *YoutubeVideo) reprocess(daemon *jsonrpc.Client, params SyncParams, existingVideoData *sdk.SyncedVideo) (*SyncSummary, error) {
-	c, err := daemon.ClaimSearch(nil, &existingVideoData.ClaimID, nil, nil)
+	c, err := daemon.ClaimSearch(nil, &existingVideoData.ClaimID, nil, nil, 1, 20)
 	if err != nil {
 		return nil, errors.Err(err)
 	}