LBRYCommunity/ytsync
1
0
Fork 0
Code Issues 39 Pull requests 8 Packages Projects Releases 197 Wiki Activity

fix e2e tests

Browse source 
update wallet server
cleanup go.sum
Fix broken balance logging
Prevent failures once channel is transferred
Wait for all UTXOs to confirm before transferring
Fix various bugs
This commit is contained in:
Niko Storni 2019-08-23 01:28:51 +02:00
parent 1fb1077a94
commit 4d16a1d6e8
6 changed files with 105 additions and 56 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
e2e/docker-compose.yml
View file

@ -21,7 +21,7 @@ services:
## Wallet Server ##
###################
walletserver:
image: lbry/wallet-server:v0.39.1
image: lbry/wallet-server:v0.39.2
restart: always
environment:
- DB_DIRECTORY=/database
@ -84,7 +84,7 @@ services:
## Internal APIs ##
###################
internalapis:
image: lbry/internal-apis:master
image: lbry/internal-apis:transfers
restart: "no"
ports:
- "15400:8080"

13
e2e/e2e.sh
View file

@ -61,14 +61,19 @@ curl -i -H 'Accept: application/json' -H 'Content-Type: application/json' 'http:
# Execute the transfer test!
./../bin/ytsync --channelID UCCyr5j8akeu9j4Q7urV0Lqw #Force channel intended...just in case. This channel lines up with the api container
# ALSO CHECK THAT VIDEO IS MARKED TRANSFERRED
transferStatus=$(mysql -u lbry -plbry -ss -D lbry -h "127.0.0.1" -P 15500 -e 'SELECT transferred FROM youtube_data WHERE id=1')
if [[ $status != "synced" || $videoStatus != "published" || transferStatus != "1" ]]; then
channelTransferStatus=$(mysql -u lbry -plbry -ss -D lbry -h "127.0.0.1" -P 15500 -e 'SELECT transfer_state FROM youtube_data WHERE id=1')
videoTransferStatus=$(mysql -u lbry -plbry -ss -D lbry -h "127.0.0.1" -P 15500 -e 'SELECT transferred FROM synced_video WHERE id=1')
if [[ $status != "synced" || $videoStatus != "published" || $channelTransferStatus != "2" || $videoTransferStatus != "1" ]]; then
echo "~~!!!~~~FAILED~~~!!!~~"
echo "Channel Status: $status"
echo "Video Status: $videoStatus"
echo "Transfer Status: $transferStatus"
echo "Channel Transfer Status: $channelTransferStatus"
echo "Video Transfer Status: $videoTransferStatus"
#docker-compose logs --tail="all" lbrycrd
#docker-compose logs --tail="all" walletserver
#docker-compose logs --tail="all" lbrynet
#docker-compose logs --tail="all" internalapis
exit 1; fi;
exit 1;
fi;
#perhaps query lbrynet again (should be restarted) to see if the claim and the channel are actually on the right address

10
go.sum
View file

@ -177,16 +177,6 @@ github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/lbryio/errors.go v0.0.0-20180223142025-ad03d3cc6a5c h1:BhdcWGsuKif/XoSZnqVGNqJ1iEmH0czWR5upj+AuR8M=
github.com/lbryio/errors.go v0.0.0-20180223142025-ad03d3cc6a5c/go.mod h1:muH7wpUqE8hRA3OrYYosw9+Sl681BF9cwcjzE+OCNK8=
github.com/lbryio/lbry.go v0.0.0-20190109223729-30c312501602/go.mod h1:YEuFJD/oHNra6BFy+NfuvS84Wg6RMWJFGtiCCCc6MmQ=
github.com/lbryio/lbry.go v1.0.17-0.20190818212926-db6034b41035 h1:eGjqH+JB8+PR98NrgCOfhFXNe4jM9TWGeqXpAVtvaR4=
github.com/lbryio/lbry.go v1.0.17-0.20190818212926-db6034b41035/go.mod h1:JtyI30bU51rm0LZ/po3mQuzf++14OWb6kR/6mMRAmKU=
github.com/lbryio/lbry.go v1.0.17-0.20190818213826-4eae32ab60ea h1:vFNLqaOFnFe5x0yOhGJJUmCCMfbSKjfh/KzILwJj5+Q=
github.com/lbryio/lbry.go v1.0.17-0.20190818213826-4eae32ab60ea/go.mod h1:JtyI30bU51rm0LZ/po3mQuzf++14OWb6kR/6mMRAmKU=
github.com/lbryio/lbry.go v1.0.17 h1:Lt7xIguw6Q5LMKcEMfxD7IB7kUdP7xwrc0OxvBUl39E=
github.com/lbryio/lbry.go v1.0.17/go.mod h1:JtyI30bU51rm0LZ/po3mQuzf++14OWb6kR/6mMRAmKU=
github.com/lbryio/lbry.go v1.0.18 h1:88ot2tNyMFM0g8xCxyruCra2c+e0huazcwihT3pX/Lc=
github.com/lbryio/lbry.go v1.0.18/go.mod h1:JtyI30bU51rm0LZ/po3mQuzf++14OWb6kR/6mMRAmKU=
github.com/lbryio/lbry.go v1.1.0 h1:LDOjMROscAVbmjLYBh988E/UFqzunPik35kcRd/PUR4=
github.com/lbryio/lbry.go v1.1.0/go.mod h1:JtyI30bU51rm0LZ/po3mQuzf++14OWb6kR/6mMRAmKU=
github.com/lbryio/lbry.go v1.1.1-0.20190820035946-9ac18d083579 h1:3FKVv1m/J8mVQJTwIcF5BFv2E87y+nYqBqofJs6F720=
github.com/lbryio/lbry.go v1.1.1-0.20190820035946-9ac18d083579/go.mod h1:JtyI30bU51rm0LZ/po3mQuzf++14OWb6kR/6mMRAmKU=
github.com/lbryio/lbryschema.go v0.0.0-20190428231007-c54836bca002 h1:urfYK5ElpUrAv90auPLldoVC60LwiGAcY0OE6HJB9KI=

6
manager/setup.go
View file

@ -197,7 +197,8 @@ func (s *Sync) ensureEnoughUTXOs() error {
if desiredUTXOCount > maxUTXOs {
desiredUTXOCount = maxUTXOs
}
log.Infof("Splitting balance of %s evenly between %d UTXOs", *balance, desiredUTXOCount)
availableBalance, _ := balance.Available.Float64()
log.Infof("Splitting balance of %.3f evenly between %d UTXOs", availableBalance, desiredUTXOCount)
broadcastFee := 0.1
prefillTx, err := s.daemon.AccountFund(defaultAccount, defaultAccount, fmt.Sprintf("%.4f", balanceAmount-broadcastFee), desiredUTXOCount, false)
@ -268,6 +269,9 @@ func (s *Sync) ensureChannelOwnership() error {
if s.YoutubeChannelID == "UCW-thz5HxE-goYq8yPds1Gw" {
return nil
}
if s.transferState == TransferStateComplete {
return nil
}
channels, err := s.daemon.ChannelList(nil, 1, 50)
if err != nil {
return err

50
manager/transfer.go
View file

@ -5,12 +5,43 @@ import (
"github.com/lbryio/lbry.go/extras/jsonrpc"
"github.com/lbryio/lbry.go/extras/util"
"github.com/lbryio/ytsync/sdk"
log "github.com/sirupsen/logrus"
)
func waitConfirmations(s *Sync) error {
allConfirmed := false
waiting:
for !allConfirmed {
utxolist, err := s.daemon.UTXOList(nil)
if err != nil {
return err
} else if utxolist == nil {
return errors.Err("no response")
}
for _, utxo := range *utxolist {
if utxo.Confirmations <= 0 {
err = s.waitForNewBlock()
if err != nil {
return err
}
continue waiting
}
}
allConfirmed = true
}
return nil
}
func TransferChannelAndVideos(channel *Sync) error {
err := waitConfirmations(channel)
if err != nil {
return err
}
cleanTransfer := true
for _, video := range channel.syncedVideos {
if !channel.syncedVideos[video.ClaimID].Published || channel.syncedVideos[video.ClaimID].Transferred || channel.syncedVideos[video.ClaimID].MetadataVersion != LatestMetadataVersion {
if !video.Published || video.Transferred || video.MetadataVersion != LatestMetadataVersion {
log.Debugf("skipping video: %s", video.VideoID)
continue
}
@ -41,18 +72,19 @@ func TransferChannelAndVideos(channel *Sync) error {
IsTransferred: util.PtrToBool(true),
}
_, err = channel.daemon.StreamUpdate(video.ClaimID, streamUpdateOptions)
if err != nil {
result, updateError := channel.daemon.StreamUpdate(video.ClaimID, streamUpdateOptions)
if updateError != nil {
cleanTransfer = false
videoStatus.FailureReason = err.Error()
videoStatus.FailureReason = updateError.Error()
videoStatus.Status = VideoStatusTranferFailed
videoStatus.IsTransferred = util.PtrToBool(false)
}
log.Printf("TRANSFER RESULT %+v", *result) //TODO: actually check the results to be sure it worked
statusErr := channel.APIConfig.MarkVideoStatus(videoStatus)
if statusErr != nil {
return errors.Err(err)
return errors.Err(statusErr)
}
if err != nil {
if updateError != nil {
return errors.Err(err)
}
}
@ -65,7 +97,7 @@ func TransferChannelAndVideos(channel *Sync) error {
if err != nil {
return errors.Err(err)
}
if channelClaim == nil {
if channelClaim == nil || len(channelClaim.Claims) == 0 {
return errors.Err("There is no channel claim for channel %s", channel.LbryChannelName)
}
updateOptions := jsonrpc.ChannelUpdateOptions{
@ -75,6 +107,8 @@ func TransferChannelAndVideos(channel *Sync) error {
},
},
}
_, err = channel.daemon.ChannelUpdate(channel.lbryChannelID, updateOptions)
result, err := channel.daemon.ChannelUpdate(channel.lbryChannelID, updateOptions)
log.Printf("TRANSFER RESULT %+v", *result) //TODO: actually check the results to be sure it worked
return errors.Err(err)
}

64
manager/ytsync.go
View file

@ -339,9 +339,9 @@ func (s *Sync) setChannelTerminationStatus(e *error) {
if s.shouldTransfer() {
if *e != nil {
transferState = util.PtrToInt(TransferStateComplete)
} else {
transferState = util.PtrToInt(TransferStateFailed)
} else {
transferState = util.PtrToInt(TransferStateComplete)
}
}
if *e != nil {
@ -538,6 +538,10 @@ func (s *Sync) updateRemoteDB(claims []jsonrpc.Claim) (total, fixed, removed int
}
idsToRemove := make([]string, 0, len(videoIDMap))
for vID, sv := range s.syncedVideos {
if sv.Transferred {
log.Infof("%s: claim was transferred, ignoring")
continue
}
_, ok := videoIDMap[vID]
if !ok && sv.Published {
log.Debugf("%s: claims to be published but wasn't found in the list of claims and will be removed if --remove-db-unpublished was specified", vID)
@ -547,10 +551,10 @@ func (s *Sync) updateRemoteDB(claims []jsonrpc.Claim) (total, fixed, removed int
if s.Manager.removeDBUnpublished && len(idsToRemove) > 0 {
err := s.Manager.apiConfig.DeleteVideos(idsToRemove)
if err != nil {
return count, fixed, 0, err
return count, fixed, len(idsToRemove), err
}
}
return count, fixed, len(idsToRemove), nil
return count, fixed, 0, nil
}
func (s *Sync) getClaims() ([]jsonrpc.Claim, error) {
@ -567,15 +571,7 @@ func (s *Sync) getClaims() ([]jsonrpc.Claim, error) {
return allClaims, nil
}
func (s *Sync) doSync() error {
err := s.enableAddressReuse()
if err != nil {
return errors.Prefix("could not set address reuse policy", err)
}
err = s.walletSetup()
if err != nil {
return errors.Prefix("Initial wallet setup failed! Manual Intervention is required.", err)
}
func (s *Sync) checkIntegrity() error {
allClaims, err := s.getClaims()
if err != nil {
return err
@ -601,17 +597,6 @@ func (s *Sync) doSync() error {
return errors.Prefix("error updating remote database", err)
}
cert, err := s.daemon.ChannelExport(s.lbryChannelID, nil, nil)
if err != nil {
return errors.Prefix("error getting channel cert", err)
}
if cert != nil {
err = s.APIConfig.SetChannelCert(string(*cert), s.lbryChannelID)
if err != nil {
return errors.Prefix("error setting channel cert", err)
}
}
if nFixed > 0 || nRemoved > 0 {
err := s.setStatusSyncing()
if err != nil {
@ -638,6 +623,36 @@ func (s *Sync) doSync() error {
if pubsOnWallet < pubsOnDB {
logUtils.SendInfoToSlack("we're claiming to have published %d videos but we only published %d (%s)", pubsOnDB, pubsOnWallet, s.YoutubeChannelID)
}
return nil
}
func (s *Sync) doSync() error {
err := s.enableAddressReuse()
if err != nil {
return errors.Prefix("could not set address reuse policy", err)
}
err = s.walletSetup()
if err != nil {
return errors.Prefix("Initial wallet setup failed! Manual Intervention is required.", err)
}
err = s.checkIntegrity()
if err != nil {
return err
}
if s.transferState != TransferStateComplete {
cert, err := s.daemon.ChannelExport(s.lbryChannelID, nil, nil)
if err != nil {
return errors.Prefix("error getting channel cert", err)
}
if cert != nil {
err = s.APIConfig.SetChannelCert(string(*cert), s.lbryChannelID)
if err != nil {
return errors.Prefix("error setting channel cert", err)
}
}
}
if s.StopOnError {
log.Println("Will stop publishing if an error is detected")
@ -989,6 +1004,7 @@ func (s *Sync) processVideo(v video) (err error) {
ClaimName: summary.ClaimName,
Size: v.Size(),
MetaDataVersion: LatestMetadataVersion,
IsTransferred: util.PtrToBool(s.shouldTransfer()),
})
if err != nil {
logUtils.SendErrorToSlack("Failed to mark video on the database: %s", errors.FullTrace(err))