fix e2e tests
update wallet server cleanup go.sum Fix broken balance logging Prevent failures once channel is transferred Wait for all UTXOs to confirm before transferring Fix various bugs
This commit is contained in:
Niko Storni
parent
1fb1077a94
commit
4d16a1d6e8
6 changed files with 105 additions and 56 deletions
|
|
@ -21,7 +21,7 @@ services:
|
|||
## Wallet Server ##
|
||||
###################
|
||||
walletserver:
|
||||
image: lbry/wallet-server:v0.39.1
|
||||
image: lbry/wallet-server:v0.39.2
|
||||
restart: always
|
||||
environment:
|
||||
- DB_DIRECTORY=/database
|
||||
|
|
@ -84,7 +84,7 @@ services:
|
|||
## Internal APIs ##
|
||||
###################
|
||||
internalapis:
|
||||
image: lbry/internal-apis:master
|
||||
image: lbry/internal-apis:transfers
|
||||
restart: "no"
|
||||
ports:
|
||||
- "15400:8080"
|
||||
|
|
|
|||
27
e2e/e2e.sh
27
e2e/e2e.sh
|
|
@ -61,14 +61,19 @@ curl -i -H 'Accept: application/json' -H 'Content-Type: application/json' 'http:
|
|||
# Execute the transfer test!
|
||||
./../bin/ytsync --channelID UCCyr5j8akeu9j4Q7urV0Lqw #Force channel intended...just in case. This channel lines up with the api container
|
||||
# ALSO CHECK THAT VIDEO IS MARKED TRANSFERRED
|
||||
transferStatus=$(mysql -u lbry -plbry -ss -D lbry -h "127.0.0.1" -P 15500 -e 'SELECT transferred FROM youtube_data WHERE id=1')
|
||||
if [[ $status != "synced" || $videoStatus != "published" || transferStatus != "1" ]]; then
|
||||
echo "~~!!!~~~FAILED~~~!!!~~"
|
||||
echo "Channel Status: $status"
|
||||
echo "Video Status: $videoStatus"
|
||||
echo "Transfer Status: $transferStatus"
|
||||
#docker-compose logs --tail="all" lbrycrd
|
||||
#docker-compose logs --tail="all" walletserver
|
||||
#docker-compose logs --tail="all" lbrynet
|
||||
#docker-compose logs --tail="all" internalapis
|
||||
exit 1; fi;
|
||||
channelTransferStatus=$(mysql -u lbry -plbry -ss -D lbry -h "127.0.0.1" -P 15500 -e 'SELECT transfer_state FROM youtube_data WHERE id=1')
|
||||
videoTransferStatus=$(mysql -u lbry -plbry -ss -D lbry -h "127.0.0.1" -P 15500 -e 'SELECT transferred FROM synced_video WHERE id=1')
|
||||
if [[ $status != "synced" || $videoStatus != "published" || $channelTransferStatus != "2" || $videoTransferStatus != "1" ]]; then
|
||||
echo "~~!!!~~~FAILED~~~!!!~~"
|
||||
echo "Channel Status: $status"
|
||||
echo "Video Status: $videoStatus"
|
||||
echo "Channel Transfer Status: $channelTransferStatus"
|
||||
echo "Video Transfer Status: $videoTransferStatus"
|
||||
#docker-compose logs --tail="all" lbrycrd
|
||||
#docker-compose logs --tail="all" walletserver
|
||||
#docker-compose logs --tail="all" lbrynet
|
||||
#docker-compose logs --tail="all" internalapis
|
||||
exit 1;
|
||||
fi;
|
||||
|
||||
#perhaps query lbrynet again (should be restarted) to see if the claim and the channel are actually on the right address
|
||||
10
go.sum
10
go.sum
|
|
@ -177,16 +177,6 @@ github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
|
|||
github.com/lbryio/errors.go v0.0.0-20180223142025-ad03d3cc6a5c h1:BhdcWGsuKif/XoSZnqVGNqJ1iEmH0czWR5upj+AuR8M=
|
||||
github.com/lbryio/errors.go v0.0.0-20180223142025-ad03d3cc6a5c/go.mod h1:muH7wpUqE8hRA3OrYYosw9+Sl681BF9cwcjzE+OCNK8=
|
||||
github.com/lbryio/lbry.go v0.0.0-20190109223729-30c312501602/go.mod h1:YEuFJD/oHNra6BFy+NfuvS84Wg6RMWJFGtiCCCc6MmQ=
|
||||
github.com/lbryio/lbry.go v1.0.17-0.20190818212926-db6034b41035 h1:eGjqH+JB8+PR98NrgCOfhFXNe4jM9TWGeqXpAVtvaR4=
|
||||
github.com/lbryio/lbry.go v1.0.17-0.20190818212926-db6034b41035/go.mod h1:JtyI30bU51rm0LZ/po3mQuzf++14OWb6kR/6mMRAmKU=
|
||||
github.com/lbryio/lbry.go v1.0.17-0.20190818213826-4eae32ab60ea h1:vFNLqaOFnFe5x0yOhGJJUmCCMfbSKjfh/KzILwJj5+Q=
|
||||
github.com/lbryio/lbry.go v1.0.17-0.20190818213826-4eae32ab60ea/go.mod h1:JtyI30bU51rm0LZ/po3mQuzf++14OWb6kR/6mMRAmKU=
|
||||
github.com/lbryio/lbry.go v1.0.17 h1:Lt7xIguw6Q5LMKcEMfxD7IB7kUdP7xwrc0OxvBUl39E=
|
||||
github.com/lbryio/lbry.go v1.0.17/go.mod h1:JtyI30bU51rm0LZ/po3mQuzf++14OWb6kR/6mMRAmKU=
|
||||
github.com/lbryio/lbry.go v1.0.18 h1:88ot2tNyMFM0g8xCxyruCra2c+e0huazcwihT3pX/Lc=
|
||||
github.com/lbryio/lbry.go v1.0.18/go.mod h1:JtyI30bU51rm0LZ/po3mQuzf++14OWb6kR/6mMRAmKU=
|
||||
github.com/lbryio/lbry.go v1.1.0 h1:LDOjMROscAVbmjLYBh988E/UFqzunPik35kcRd/PUR4=
|
||||
github.com/lbryio/lbry.go v1.1.0/go.mod h1:JtyI30bU51rm0LZ/po3mQuzf++14OWb6kR/6mMRAmKU=
|
||||
github.com/lbryio/lbry.go v1.1.1-0.20190820035946-9ac18d083579 h1:3FKVv1m/J8mVQJTwIcF5BFv2E87y+nYqBqofJs6F720=
|
||||
github.com/lbryio/lbry.go v1.1.1-0.20190820035946-9ac18d083579/go.mod h1:JtyI30bU51rm0LZ/po3mQuzf++14OWb6kR/6mMRAmKU=
|
||||
github.com/lbryio/lbryschema.go v0.0.0-20190428231007-c54836bca002 h1:urfYK5ElpUrAv90auPLldoVC60LwiGAcY0OE6HJB9KI=
|
||||
|
|
|
|||
|
|
@ -197,7 +197,8 @@ func (s *Sync) ensureEnoughUTXOs() error {
|
|||
if desiredUTXOCount > maxUTXOs {
|
||||
desiredUTXOCount = maxUTXOs
|
||||
}
|
||||
log.Infof("Splitting balance of %s evenly between %d UTXOs", *balance, desiredUTXOCount)
|
||||
availableBalance, _ := balance.Available.Float64()
|
||||
log.Infof("Splitting balance of %.3f evenly between %d UTXOs", availableBalance, desiredUTXOCount)
|
||||
|
||||
broadcastFee := 0.1
|
||||
prefillTx, err := s.daemon.AccountFund(defaultAccount, defaultAccount, fmt.Sprintf("%.4f", balanceAmount-broadcastFee), desiredUTXOCount, false)
|
||||
|
|
@ -268,6 +269,9 @@ func (s *Sync) ensureChannelOwnership() error {
|
|||
if s.YoutubeChannelID == "UCW-thz5HxE-goYq8yPds1Gw" {
|
||||
return nil
|
||||
}
|
||||
if s.transferState == TransferStateComplete {
|
||||
return nil
|
||||
}
|
||||
channels, err := s.daemon.ChannelList(nil, 1, 50)
|
||||
if err != nil {
|
||||
return err
|
||||
|
|
|
|||
|
|
@ -5,12 +5,43 @@ import (
|
|||
"github.com/lbryio/lbry.go/extras/jsonrpc"
|
||||
"github.com/lbryio/lbry.go/extras/util"
|
||||
"github.com/lbryio/ytsync/sdk"
|
||||
log "github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
func waitConfirmations(s *Sync) error {
|
||||
allConfirmed := false
|
||||
waiting:
|
||||
for !allConfirmed {
|
||||
utxolist, err := s.daemon.UTXOList(nil)
|
||||
if err != nil {
|
||||
return err
|
||||
} else if utxolist == nil {
|
||||
return errors.Err("no response")
|
||||
}
|
||||
|
||||
for _, utxo := range *utxolist {
|
||||
if utxo.Confirmations <= 0 {
|
||||
err = s.waitForNewBlock()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
continue waiting
|
||||
}
|
||||
}
|
||||
allConfirmed = true
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func TransferChannelAndVideos(channel *Sync) error {
|
||||
err := waitConfirmations(channel)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
cleanTransfer := true
|
||||
for _, video := range channel.syncedVideos {
|
||||
if !channel.syncedVideos[video.ClaimID].Published || channel.syncedVideos[video.ClaimID].Transferred || channel.syncedVideos[video.ClaimID].MetadataVersion != LatestMetadataVersion {
|
||||
if !video.Published || video.Transferred || video.MetadataVersion != LatestMetadataVersion {
|
||||
log.Debugf("skipping video: %s", video.VideoID)
|
||||
continue
|
||||
}
|
||||
|
||||
|
|
@ -41,18 +72,19 @@ func TransferChannelAndVideos(channel *Sync) error {
|
|||
IsTransferred: util.PtrToBool(true),
|
||||
}
|
||||
|
||||
_, err = channel.daemon.StreamUpdate(video.ClaimID, streamUpdateOptions)
|
||||
if err != nil {
|
||||
result, updateError := channel.daemon.StreamUpdate(video.ClaimID, streamUpdateOptions)
|
||||
if updateError != nil {
|
||||
cleanTransfer = false
|
||||
videoStatus.FailureReason = err.Error()
|
||||
videoStatus.FailureReason = updateError.Error()
|
||||
videoStatus.Status = VideoStatusTranferFailed
|
||||
videoStatus.IsTransferred = util.PtrToBool(false)
|
||||
}
|
||||
log.Printf("TRANSFER RESULT %+v", *result) //TODO: actually check the results to be sure it worked
|
||||
statusErr := channel.APIConfig.MarkVideoStatus(videoStatus)
|
||||
if statusErr != nil {
|
||||
return errors.Err(err)
|
||||
return errors.Err(statusErr)
|
||||
}
|
||||
if err != nil {
|
||||
if updateError != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
}
|
||||
|
|
@ -65,7 +97,7 @@ func TransferChannelAndVideos(channel *Sync) error {
|
|||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
if channelClaim == nil {
|
||||
if channelClaim == nil || len(channelClaim.Claims) == 0 {
|
||||
return errors.Err("There is no channel claim for channel %s", channel.LbryChannelName)
|
||||
}
|
||||
updateOptions := jsonrpc.ChannelUpdateOptions{
|
||||
|
|
@ -75,6 +107,8 @@ func TransferChannelAndVideos(channel *Sync) error {
|
|||
},
|
||||
},
|
||||
}
|
||||
_, err = channel.daemon.ChannelUpdate(channel.lbryChannelID, updateOptions)
|
||||
result, err := channel.daemon.ChannelUpdate(channel.lbryChannelID, updateOptions)
|
||||
log.Printf("TRANSFER RESULT %+v", *result) //TODO: actually check the results to be sure it worked
|
||||
|
||||
return errors.Err(err)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -339,9 +339,9 @@ func (s *Sync) setChannelTerminationStatus(e *error) {
|
|||
|
||||
if s.shouldTransfer() {
|
||||
if *e != nil {
|
||||
transferState = util.PtrToInt(TransferStateComplete)
|
||||
} else {
|
||||
transferState = util.PtrToInt(TransferStateFailed)
|
||||
} else {
|
||||
transferState = util.PtrToInt(TransferStateComplete)
|
||||
}
|
||||
}
|
||||
if *e != nil {
|
||||
|
|
@ -538,6 +538,10 @@ func (s *Sync) updateRemoteDB(claims []jsonrpc.Claim) (total, fixed, removed int
|
|||
}
|
||||
idsToRemove := make([]string, 0, len(videoIDMap))
|
||||
for vID, sv := range s.syncedVideos {
|
||||
if sv.Transferred {
|
||||
log.Infof("%s: claim was transferred, ignoring")
|
||||
continue
|
||||
}
|
||||
_, ok := videoIDMap[vID]
|
||||
if !ok && sv.Published {
|
||||
log.Debugf("%s: claims to be published but wasn't found in the list of claims and will be removed if --remove-db-unpublished was specified", vID)
|
||||
|
|
@ -547,10 +551,10 @@ func (s *Sync) updateRemoteDB(claims []jsonrpc.Claim) (total, fixed, removed int
|
|||
if s.Manager.removeDBUnpublished && len(idsToRemove) > 0 {
|
||||
err := s.Manager.apiConfig.DeleteVideos(idsToRemove)
|
||||
if err != nil {
|
||||
return count, fixed, 0, err
|
||||
return count, fixed, len(idsToRemove), err
|
||||
}
|
||||
}
|
||||
return count, fixed, len(idsToRemove), nil
|
||||
return count, fixed, 0, nil
|
||||
}
|
||||
|
||||
func (s *Sync) getClaims() ([]jsonrpc.Claim, error) {
|
||||
|
|
@ -567,15 +571,7 @@ func (s *Sync) getClaims() ([]jsonrpc.Claim, error) {
|
|||
return allClaims, nil
|
||||
}
|
||||
|
||||
func (s *Sync) doSync() error {
|
||||
err := s.enableAddressReuse()
|
||||
if err != nil {
|
||||
return errors.Prefix("could not set address reuse policy", err)
|
||||
}
|
||||
err = s.walletSetup()
|
||||
if err != nil {
|
||||
return errors.Prefix("Initial wallet setup failed! Manual Intervention is required.", err)
|
||||
}
|
||||
func (s *Sync) checkIntegrity() error {
|
||||
allClaims, err := s.getClaims()
|
||||
if err != nil {
|
||||
return err
|
||||
|
|
@ -601,17 +597,6 @@ func (s *Sync) doSync() error {
|
|||
return errors.Prefix("error updating remote database", err)
|
||||
}
|
||||
|
||||
cert, err := s.daemon.ChannelExport(s.lbryChannelID, nil, nil)
|
||||
if err != nil {
|
||||
return errors.Prefix("error getting channel cert", err)
|
||||
}
|
||||
if cert != nil {
|
||||
err = s.APIConfig.SetChannelCert(string(*cert), s.lbryChannelID)
|
||||
if err != nil {
|
||||
return errors.Prefix("error setting channel cert", err)
|
||||
}
|
||||
}
|
||||
|
||||
if nFixed > 0 || nRemoved > 0 {
|
||||
err := s.setStatusSyncing()
|
||||
if err != nil {
|
||||
|
|
@ -638,6 +623,36 @@ func (s *Sync) doSync() error {
|
|||
if pubsOnWallet < pubsOnDB {
|
||||
logUtils.SendInfoToSlack("we're claiming to have published %d videos but we only published %d (%s)", pubsOnDB, pubsOnWallet, s.YoutubeChannelID)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Sync) doSync() error {
|
||||
err := s.enableAddressReuse()
|
||||
if err != nil {
|
||||
return errors.Prefix("could not set address reuse policy", err)
|
||||
}
|
||||
err = s.walletSetup()
|
||||
if err != nil {
|
||||
return errors.Prefix("Initial wallet setup failed! Manual Intervention is required.", err)
|
||||
}
|
||||
|
||||
err = s.checkIntegrity()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if s.transferState != TransferStateComplete {
|
||||
cert, err := s.daemon.ChannelExport(s.lbryChannelID, nil, nil)
|
||||
if err != nil {
|
||||
return errors.Prefix("error getting channel cert", err)
|
||||
}
|
||||
if cert != nil {
|
||||
err = s.APIConfig.SetChannelCert(string(*cert), s.lbryChannelID)
|
||||
if err != nil {
|
||||
return errors.Prefix("error setting channel cert", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if s.StopOnError {
|
||||
log.Println("Will stop publishing if an error is detected")
|
||||
|
|
@ -989,6 +1004,7 @@ func (s *Sync) processVideo(v video) (err error) {
|
|||
ClaimName: summary.ClaimName,
|
||||
Size: v.Size(),
|
||||
MetaDataVersion: LatestMetadataVersion,
|
||||
IsTransferred: util.PtrToBool(s.shouldTransfer()),
|
||||
})
|
||||
if err != nil {
|
||||
logUtils.SendErrorToSlack("Failed to mark video on the database: %s", errors.FullTrace(err))
|
||||
|
|
|
|||
Loading…
Reference in a new issue