fix e2e tests

update wallet server
cleanup go.sum
Fix broken balance logging
Prevent failures once channel is transferred
Wait for all UTXOs to confirm before transferring
Fix various bugs

e2e/docker-compose.yml

@@ -21,7 +21,7 @@ services:
   ## Wallet Server ##
   ###################
   walletserver:
-    image: lbry/wallet-server:v0.39.1
+    image: lbry/wallet-server:v0.39.2
     restart: always
     environment:
       - DB_DIRECTORY=/database
@@ -84,7 +84,7 @@ services:
   ## Internal APIs ##
   ###################
   internalapis:
-    image: lbry/internal-apis:master
+    image: lbry/internal-apis:transfers
     restart: "no"
     ports:
       - "15400:8080"

e2e/e2e.sh

@@ -61,14 +61,19 @@ curl -i -H 'Accept: application/json' -H 'Content-Type: application/json' 'http:
 # Execute the transfer test!
 ./../bin/ytsync --channelID UCCyr5j8akeu9j4Q7urV0Lqw #Force channel intended...just in case. This channel lines up with the api container
 # ALSO CHECK THAT VIDEO IS MARKED TRANSFERRED
-transferStatus=$(mysql -u lbry -plbry -ss -D lbry -h "127.0.0.1" -P 15500 -e 'SELECT transferred FROM youtube_data WHERE id=1')
+channelTransferStatus=$(mysql -u lbry -plbry -ss -D lbry -h "127.0.0.1" -P 15500 -e 'SELECT transfer_state FROM youtube_data WHERE id=1')
-if [[ $status != "synced" || $videoStatus != "published" || transferStatus != "1" ]]; then
+videoTransferStatus=$(mysql -u lbry -plbry -ss -D lbry -h "127.0.0.1" -P 15500 -e 'SELECT transferred FROM synced_video WHERE id=1')
-echo "~~!!!~~~FAILED~~~!!!~~"
+if [[ $status != "synced" || $videoStatus != "published" || $channelTransferStatus != "2" || $videoTransferStatus != "1" ]]; then
-echo "Channel Status: $status"
+    echo "~~!!!~~~FAILED~~~!!!~~"
-echo "Video Status: $videoStatus"
+    echo "Channel Status: $status"
-echo "Transfer Status: $transferStatus"
+    echo "Video Status: $videoStatus"
-#docker-compose logs --tail="all" lbrycrd
+    echo "Channel Transfer Status: $channelTransferStatus"
-#docker-compose logs --tail="all" walletserver
+    echo "Video Transfer Status: $videoTransferStatus"
-#docker-compose logs --tail="all" lbrynet
+    #docker-compose logs --tail="all" lbrycrd
-#docker-compose logs --tail="all" internalapis
+    #docker-compose logs --tail="all" walletserver
- exit 1; fi;
+    #docker-compose logs --tail="all" lbrynet
+    #docker-compose logs --tail="all" internalapis
+    exit 1;
+fi;
+
+#perhaps query lbrynet again (should be restarted) to see if the claim and the channel are actually on the right address

go.sum

@@ -177,16 +177,6 @@ github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/lbryio/errors.go v0.0.0-20180223142025-ad03d3cc6a5c h1:BhdcWGsuKif/XoSZnqVGNqJ1iEmH0czWR5upj+AuR8M=
 github.com/lbryio/errors.go v0.0.0-20180223142025-ad03d3cc6a5c/go.mod h1:muH7wpUqE8hRA3OrYYosw9+Sl681BF9cwcjzE+OCNK8=
 github.com/lbryio/lbry.go v0.0.0-20190109223729-30c312501602/go.mod h1:YEuFJD/oHNra6BFy+NfuvS84Wg6RMWJFGtiCCCc6MmQ=
-github.com/lbryio/lbry.go v1.0.17-0.20190818212926-db6034b41035 h1:eGjqH+JB8+PR98NrgCOfhFXNe4jM9TWGeqXpAVtvaR4=
-github.com/lbryio/lbry.go v1.0.17-0.20190818212926-db6034b41035/go.mod h1:JtyI30bU51rm0LZ/po3mQuzf++14OWb6kR/6mMRAmKU=
-github.com/lbryio/lbry.go v1.0.17-0.20190818213826-4eae32ab60ea h1:vFNLqaOFnFe5x0yOhGJJUmCCMfbSKjfh/KzILwJj5+Q=
-github.com/lbryio/lbry.go v1.0.17-0.20190818213826-4eae32ab60ea/go.mod h1:JtyI30bU51rm0LZ/po3mQuzf++14OWb6kR/6mMRAmKU=
-github.com/lbryio/lbry.go v1.0.17 h1:Lt7xIguw6Q5LMKcEMfxD7IB7kUdP7xwrc0OxvBUl39E=
-github.com/lbryio/lbry.go v1.0.17/go.mod h1:JtyI30bU51rm0LZ/po3mQuzf++14OWb6kR/6mMRAmKU=
-github.com/lbryio/lbry.go v1.0.18 h1:88ot2tNyMFM0g8xCxyruCra2c+e0huazcwihT3pX/Lc=
-github.com/lbryio/lbry.go v1.0.18/go.mod h1:JtyI30bU51rm0LZ/po3mQuzf++14OWb6kR/6mMRAmKU=
-github.com/lbryio/lbry.go v1.1.0 h1:LDOjMROscAVbmjLYBh988E/UFqzunPik35kcRd/PUR4=
-github.com/lbryio/lbry.go v1.1.0/go.mod h1:JtyI30bU51rm0LZ/po3mQuzf++14OWb6kR/6mMRAmKU=
 github.com/lbryio/lbry.go v1.1.1-0.20190820035946-9ac18d083579 h1:3FKVv1m/J8mVQJTwIcF5BFv2E87y+nYqBqofJs6F720=
 github.com/lbryio/lbry.go v1.1.1-0.20190820035946-9ac18d083579/go.mod h1:JtyI30bU51rm0LZ/po3mQuzf++14OWb6kR/6mMRAmKU=
 github.com/lbryio/lbryschema.go v0.0.0-20190428231007-c54836bca002 h1:urfYK5ElpUrAv90auPLldoVC60LwiGAcY0OE6HJB9KI=

manager/setup.go

@@ -197,7 +197,8 @@ func (s *Sync) ensureEnoughUTXOs() error {
 		if desiredUTXOCount > maxUTXOs {
 			desiredUTXOCount = maxUTXOs
 		}
-		log.Infof("Splitting balance of %s evenly between %d UTXOs", *balance, desiredUTXOCount)
+		availableBalance, _ := balance.Available.Float64()
+		log.Infof("Splitting balance of %.3f evenly between %d UTXOs", availableBalance, desiredUTXOCount)
 
 		broadcastFee := 0.1
 		prefillTx, err := s.daemon.AccountFund(defaultAccount, defaultAccount, fmt.Sprintf("%.4f", balanceAmount-broadcastFee), desiredUTXOCount, false)
@@ -268,6 +269,9 @@ func (s *Sync) ensureChannelOwnership() error {
 	if s.YoutubeChannelID == "UCW-thz5HxE-goYq8yPds1Gw" {
 		return nil
 	}
+	if s.transferState == TransferStateComplete {
+		return nil
+	}
 	channels, err := s.daemon.ChannelList(nil, 1, 50)
 	if err != nil {
 		return err

manager/transfer.go

@@ -5,12 +5,43 @@ import (
 	"github.com/lbryio/lbry.go/extras/jsonrpc"
 	"github.com/lbryio/lbry.go/extras/util"
 	"github.com/lbryio/ytsync/sdk"
+	log "github.com/sirupsen/logrus"
 )
 
+func waitConfirmations(s *Sync) error {
+	allConfirmed := false
+waiting:
+	for !allConfirmed {
+		utxolist, err := s.daemon.UTXOList(nil)
+		if err != nil {
+			return err
+		} else if utxolist == nil {
+			return errors.Err("no response")
+		}
+
+		for _, utxo := range *utxolist {
+			if utxo.Confirmations <= 0 {
+				err = s.waitForNewBlock()
+				if err != nil {
+					return err
+				}
+				continue waiting
+			}
+		}
+		allConfirmed = true
+	}
+	return nil
+}
+
 func TransferChannelAndVideos(channel *Sync) error {
+	err := waitConfirmations(channel)
+	if err != nil {
+		return err
+	}
 	cleanTransfer := true
 	for _, video := range channel.syncedVideos {
-		if !channel.syncedVideos[video.ClaimID].Published || channel.syncedVideos[video.ClaimID].Transferred || channel.syncedVideos[video.ClaimID].MetadataVersion != LatestMetadataVersion {
+		if !video.Published || video.Transferred || video.MetadataVersion != LatestMetadataVersion {
+			log.Debugf("skipping video: %s", video.VideoID)
 			continue
 		}
 
@@ -41,18 +72,19 @@ func TransferChannelAndVideos(channel *Sync) error {
 			IsTransferred: util.PtrToBool(true),
 		}
 
-		_, err = channel.daemon.StreamUpdate(video.ClaimID, streamUpdateOptions)
+		result, updateError := channel.daemon.StreamUpdate(video.ClaimID, streamUpdateOptions)
-		if err != nil {
+		if updateError != nil {
 			cleanTransfer = false
-			videoStatus.FailureReason = err.Error()
+			videoStatus.FailureReason = updateError.Error()
 			videoStatus.Status = VideoStatusTranferFailed
 			videoStatus.IsTransferred = util.PtrToBool(false)
 		}
+		log.Printf("TRANSFER RESULT %+v", *result) //TODO: actually check the results to be sure it worked
 		statusErr := channel.APIConfig.MarkVideoStatus(videoStatus)
 		if statusErr != nil {
-			return errors.Err(err)
+			return errors.Err(statusErr)
 		}
-		if err != nil {
+		if updateError != nil {
 			return errors.Err(err)
 		}
 	}
@@ -65,7 +97,7 @@ func TransferChannelAndVideos(channel *Sync) error {
 	if err != nil {
 		return errors.Err(err)
 	}
-	if channelClaim == nil {
+	if channelClaim == nil || len(channelClaim.Claims) == 0 {
 		return errors.Err("There is no channel claim for channel %s", channel.LbryChannelName)
 	}
 	updateOptions := jsonrpc.ChannelUpdateOptions{
@@ -75,6 +107,8 @@ func TransferChannelAndVideos(channel *Sync) error {
 			},
 		},
 	}
-	_, err = channel.daemon.ChannelUpdate(channel.lbryChannelID, updateOptions)
+	result, err := channel.daemon.ChannelUpdate(channel.lbryChannelID, updateOptions)
+	log.Printf("TRANSFER RESULT %+v", *result) //TODO: actually check the results to be sure it worked
+
 	return errors.Err(err)
 }

manager/ytsync.go

@@ -339,9 +339,9 @@ func (s *Sync) setChannelTerminationStatus(e *error) {
 
 	if s.shouldTransfer() {
 		if *e != nil {
-			transferState = util.PtrToInt(TransferStateComplete)
-		} else {
 			transferState = util.PtrToInt(TransferStateFailed)
+		} else {
+			transferState = util.PtrToInt(TransferStateComplete)
 		}
 	}
 	if *e != nil {
@@ -538,6 +538,10 @@ func (s *Sync) updateRemoteDB(claims []jsonrpc.Claim) (total, fixed, removed int
 	}
 	idsToRemove := make([]string, 0, len(videoIDMap))
 	for vID, sv := range s.syncedVideos {
+		if sv.Transferred {
+			log.Infof("%s: claim was transferred, ignoring")
+			continue
+		}
 		_, ok := videoIDMap[vID]
 		if !ok && sv.Published {
 			log.Debugf("%s: claims to be published but wasn't found in the list of claims and will be removed if --remove-db-unpublished was specified", vID)
@@ -547,10 +551,10 @@ func (s *Sync) updateRemoteDB(claims []jsonrpc.Claim) (total, fixed, removed int
 	if s.Manager.removeDBUnpublished && len(idsToRemove) > 0 {
 		err := s.Manager.apiConfig.DeleteVideos(idsToRemove)
 		if err != nil {
-			return count, fixed, 0, err
+			return count, fixed, len(idsToRemove), err
 		}
 	}
-	return count, fixed, len(idsToRemove), nil
+	return count, fixed, 0, nil
 }
 
 func (s *Sync) getClaims() ([]jsonrpc.Claim, error) {
@@ -567,15 +571,7 @@ func (s *Sync) getClaims() ([]jsonrpc.Claim, error) {
 	return allClaims, nil
 }
 
-func (s *Sync) doSync() error {
+func (s *Sync) checkIntegrity() error {
-	err := s.enableAddressReuse()
-	if err != nil {
-		return errors.Prefix("could not set address reuse policy", err)
-	}
-	err = s.walletSetup()
-	if err != nil {
-		return errors.Prefix("Initial wallet setup failed! Manual Intervention is required.", err)
-	}
 	allClaims, err := s.getClaims()
 	if err != nil {
 		return err
@@ -601,17 +597,6 @@ func (s *Sync) doSync() error {
 		return errors.Prefix("error updating remote database", err)
 	}
 
-	cert, err := s.daemon.ChannelExport(s.lbryChannelID, nil, nil)
-	if err != nil {
-		return errors.Prefix("error getting channel cert", err)
-	}
-	if cert != nil {
-		err = s.APIConfig.SetChannelCert(string(*cert), s.lbryChannelID)
-		if err != nil {
-			return errors.Prefix("error setting channel cert", err)
-		}
-	}
-
 	if nFixed > 0 || nRemoved > 0 {
 		err := s.setStatusSyncing()
 		if err != nil {
@@ -638,6 +623,36 @@ func (s *Sync) doSync() error {
 	if pubsOnWallet < pubsOnDB {
 		logUtils.SendInfoToSlack("we're claiming to have published %d videos but we only published %d (%s)", pubsOnDB, pubsOnWallet, s.YoutubeChannelID)
 	}
+	return nil
+}
+
+func (s *Sync) doSync() error {
+	err := s.enableAddressReuse()
+	if err != nil {
+		return errors.Prefix("could not set address reuse policy", err)
+	}
+	err = s.walletSetup()
+	if err != nil {
+		return errors.Prefix("Initial wallet setup failed! Manual Intervention is required.", err)
+	}
+
+	err = s.checkIntegrity()
+	if err != nil {
+		return err
+	}
+
+	if s.transferState != TransferStateComplete {
+		cert, err := s.daemon.ChannelExport(s.lbryChannelID, nil, nil)
+		if err != nil {
+			return errors.Prefix("error getting channel cert", err)
+		}
+		if cert != nil {
+			err = s.APIConfig.SetChannelCert(string(*cert), s.lbryChannelID)
+			if err != nil {
+				return errors.Prefix("error setting channel cert", err)
+			}
+		}
+	}
 
 	if s.StopOnError {
 		log.Println("Will stop publishing if an error is detected")
@@ -989,6 +1004,7 @@ func (s *Sync) processVideo(v video) (err error) {
 		ClaimName:       summary.ClaimName,
 		Size:            v.Size(),
 		MetaDataVersion: LatestMetadataVersion,
+		IsTransferred:   util.PtrToBool(s.shouldTransfer()),
 	})
 	if err != nil {
 		logUtils.SendErrorToSlack("Failed to mark video on the database: %s", errors.FullTrace(err))