add language to channels
improve logging (timestamps) retry wallet uploads for 30 minutes don't fail if the db isn't tracking all publishes
This commit is contained in:
parent
ee8eb83d07
commit
5a01983203
7 changed files with 34 additions and 22 deletions
|
@ -19,7 +19,7 @@ func TestGetPlaylistVideoIDs(t *testing.T) {
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestGetVideoInformation(t *testing.T) {
|
func TestGetVideoInformation(t *testing.T) {
|
||||||
video, err := GetVideoInformation(nil, "zj7pXM9gE5M", nil, nil, nil)
|
video, err := GetVideoInformation("zj7pXM9gE5M", nil, nil)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
logrus.Error(err)
|
logrus.Error(err)
|
||||||
}
|
}
|
||||||
|
|
4
main.go
4
main.go
|
@ -32,6 +32,10 @@ var (
|
||||||
func main() {
|
func main() {
|
||||||
rand.Seed(time.Now().UnixNano())
|
rand.Seed(time.Now().UnixNano())
|
||||||
log.SetLevel(log.DebugLevel)
|
log.SetLevel(log.DebugLevel)
|
||||||
|
customFormatter := new(log.TextFormatter)
|
||||||
|
customFormatter.TimestampFormat = "2006-01-02 15:04:05"
|
||||||
|
customFormatter.FullTimestamp = true
|
||||||
|
log.SetFormatter(customFormatter)
|
||||||
http.Handle("/metrics", promhttp.Handler())
|
http.Handle("/metrics", promhttp.Handler())
|
||||||
go func() {
|
go func() {
|
||||||
log.Error(http.ListenAndServe(":2112", nil))
|
log.Error(http.ListenAndServe(":2112", nil))
|
||||||
|
|
|
@ -138,7 +138,7 @@ func (s *SyncManager) Start() error {
|
||||||
"WALLET HAS NOT BEEN MOVED TO THE WALLET BACKUP DIR",
|
"WALLET HAS NOT BEEN MOVED TO THE WALLET BACKUP DIR",
|
||||||
"NotEnoughFunds",
|
"NotEnoughFunds",
|
||||||
"no space left on device",
|
"no space left on device",
|
||||||
"failure uploading wallet",
|
"there was a problem uploading the wallet",
|
||||||
"the channel in the wallet is different than the channel in the database",
|
"the channel in the wallet is different than the channel in the database",
|
||||||
"this channel does not belong to this wallet!",
|
"this channel does not belong to this wallet!",
|
||||||
"You already have a stream claim published under the name",
|
"You already have a stream claim published under the name",
|
||||||
|
|
|
@ -4,6 +4,7 @@ import (
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"strings"
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
"github.com/lbryio/ytsync/v5/configs"
|
"github.com/lbryio/ytsync/v5/configs"
|
||||||
"github.com/lbryio/ytsync/v5/util"
|
"github.com/lbryio/ytsync/v5/util"
|
||||||
|
@ -215,13 +216,22 @@ func (s *Sync) uploadWallet() error {
|
||||||
}
|
}
|
||||||
defer file.Close()
|
defer file.Close()
|
||||||
|
|
||||||
_, err = uploader.Upload(&s3manager.UploadInput{
|
start := time.Now()
|
||||||
Bucket: aws.String(configs.Configuration.WalletS3Config.Bucket),
|
|
||||||
Key: key,
|
for time.Since(start) < 30*time.Minute {
|
||||||
Body: file,
|
_, err = uploader.Upload(&s3manager.UploadInput{
|
||||||
})
|
Bucket: aws.String(configs.Configuration.WalletS3Config.Bucket),
|
||||||
|
Key: key,
|
||||||
|
Body: file,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
time.Sleep(30 * time.Second)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
break
|
||||||
|
}
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return errors.Prefix("there was a problem uploading the wallet to S3", errors.Err(err))
|
||||||
}
|
}
|
||||||
log.Println("wallet uploaded to S3")
|
log.Println("wallet uploaded to S3")
|
||||||
|
|
||||||
|
|
|
@ -375,14 +375,12 @@ func (s *Sync) ensureChannelOwnership() error {
|
||||||
|
|
||||||
channelUsesOldMetadata := false
|
channelUsesOldMetadata := false
|
||||||
if channelToUse != nil {
|
if channelToUse != nil {
|
||||||
channelUsesOldMetadata = channelToUse.Value.GetThumbnail() == nil
|
channelUsesOldMetadata = channelToUse.Value.GetThumbnail() == nil || (len(channelToUse.Value.Languages) == 0 && s.DbChannelData.Language == "")
|
||||||
if !channelUsesOldMetadata {
|
if !channelUsesOldMetadata {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
channelBidAmount := channelClaimAmount
|
|
||||||
|
|
||||||
balanceResp, err := s.daemon.AccountBalance(nil)
|
balanceResp, err := s.daemon.AccountBalance(nil)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
|
@ -394,8 +392,8 @@ func (s *Sync) ensureChannelOwnership() error {
|
||||||
return errors.Err(err)
|
return errors.Err(err)
|
||||||
}
|
}
|
||||||
|
|
||||||
if balance.LessThan(decimal.NewFromFloat(channelBidAmount)) {
|
if balance.LessThan(decimal.NewFromFloat(channelClaimAmount)) {
|
||||||
err = s.addCredits(channelBidAmount + 0.3)
|
err = s.addCredits(channelClaimAmount + estimatedMaxTxFee*3)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
@ -433,6 +431,9 @@ func (s *Sync) ensureChannelOwnership() error {
|
||||||
}
|
}
|
||||||
|
|
||||||
var languages []string = nil
|
var languages []string = nil
|
||||||
|
if s.DbChannelData.Language != "" {
|
||||||
|
languages = []string{s.DbChannelData.Language}
|
||||||
|
}
|
||||||
//we don't have this data without the API
|
//we don't have this data without the API
|
||||||
//if channelInfo.DefaultLanguage != "" {
|
//if channelInfo.DefaultLanguage != "" {
|
||||||
// if channelInfo.DefaultLanguage == "iw" {
|
// if channelInfo.DefaultLanguage == "iw" {
|
||||||
|
|
|
@ -336,7 +336,7 @@ func (s *Sync) waitForDaemonStart() error {
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *Sync) stopAndUploadWallet(e *error) {
|
func (s *Sync) stopAndUploadWallet(e *error) {
|
||||||
log.Printf("Stopping daemon")
|
log.Println("Stopping daemon")
|
||||||
shutdownErr := logUtils.StopDaemon()
|
shutdownErr := logUtils.StopDaemon()
|
||||||
if shutdownErr != nil {
|
if shutdownErr != nil {
|
||||||
logShutdownError(shutdownErr)
|
logShutdownError(shutdownErr)
|
||||||
|
@ -349,16 +349,11 @@ func (s *Sync) stopAndUploadWallet(e *error) {
|
||||||
logShutdownError(processDeathError)
|
logShutdownError(processDeathError)
|
||||||
} else {
|
} else {
|
||||||
err := s.uploadWallet()
|
err := s.uploadWallet()
|
||||||
if err != nil {
|
|
||||||
time.Sleep(10 * time.Second)
|
|
||||||
logUtils.SendErrorToSlack("there was a problem uploading the wallet to S3, waiting 10 seconds and retrying: %s", err.Error())
|
|
||||||
err = s.uploadWallet()
|
|
||||||
}
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
if *e == nil {
|
if *e == nil {
|
||||||
*e = err
|
*e = err
|
||||||
} else {
|
} else {
|
||||||
*e = errors.Prefix(fmt.Sprintf("failure uploading wallet: %s + original error", errors.FullTrace(err)), *e)
|
*e = errors.Prefix(fmt.Sprintf("%s + original error", errors.FullTrace(err)), *e)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
err = s.uploadBlockchainDB()
|
err = s.uploadBlockchainDB()
|
||||||
|
@ -501,7 +496,7 @@ func (s *Sync) updateRemoteDB(claims []jsonrpc.Claim, ownClaims []jsonrpc.Claim)
|
||||||
claimMarkedUnpublished := claimInDatabase && !sv.Published
|
claimMarkedUnpublished := claimInDatabase && !sv.Published
|
||||||
_, isOwnClaim := ownClaimsInfo[videoID]
|
_, isOwnClaim := ownClaimsInfo[videoID]
|
||||||
transferred := !isOwnClaim || s.DbChannelData.TransferState == 3
|
transferred := !isOwnClaim || s.DbChannelData.TransferState == 3
|
||||||
transferStatusMismatch := sv.Transferred != transferred
|
transferStatusMismatch := claimInDatabase && sv.Transferred != transferred
|
||||||
|
|
||||||
if metadataDiffers {
|
if metadataDiffers {
|
||||||
log.Debugf("%s: Mismatch in database for metadata. DB: %d - Blockchain: %d", videoID, sv.MetadataVersion, chainInfo.MetadataVersion)
|
log.Debugf("%s: Mismatch in database for metadata. DB: %d - Blockchain: %d", videoID, sv.MetadataVersion, chainInfo.MetadataVersion)
|
||||||
|
@ -682,7 +677,8 @@ func (s *Sync) checkIntegrity() error {
|
||||||
|
|
||||||
if pubsOnWallet > pubsOnDB { //This case should never happen
|
if pubsOnWallet > pubsOnDB { //This case should never happen
|
||||||
logUtils.SendInfoToSlack("We're claiming to have published %d videos but in reality we published %d (%s)", pubsOnDB, pubsOnWallet, s.DbChannelData.ChannelId)
|
logUtils.SendInfoToSlack("We're claiming to have published %d videos but in reality we published %d (%s)", pubsOnDB, pubsOnWallet, s.DbChannelData.ChannelId)
|
||||||
return errors.Err("not all published videos are in the database")
|
//we never really done anything about those. it happens when a user updates the channel for a publish to another ytsync channel
|
||||||
|
//return errors.Err("not all published videos are in the database")
|
||||||
}
|
}
|
||||||
if pubsOnWallet < pubsOnDB {
|
if pubsOnWallet < pubsOnDB {
|
||||||
logUtils.SendInfoToSlack("we're claiming to have published %d videos but we only published %d (%s)", pubsOnDB, pubsOnWallet, s.DbChannelData.ChannelId)
|
logUtils.SendInfoToSlack("we're claiming to have published %d videos but we only published %d (%s)", pubsOnDB, pubsOnWallet, s.DbChannelData.ChannelId)
|
||||||
|
|
|
@ -26,6 +26,7 @@ type YoutubeChannel struct {
|
||||||
SizeLimit int `json:"size_limit"`
|
SizeLimit int `json:"size_limit"`
|
||||||
LastUploadedVideo string `json:"last_uploaded_video"`
|
LastUploadedVideo string `json:"last_uploaded_video"`
|
||||||
WipeDB bool `json:"wipe_db"`
|
WipeDB bool `json:"wipe_db"`
|
||||||
|
Language string `json:"language"`
|
||||||
}
|
}
|
||||||
|
|
||||||
type PublishAddress struct {
|
type PublishAddress struct {
|
||||||
|
|
Loading…
Reference in a new issue