add language to channels

improve logging (timestamps)
retry wallet uploads for 30 minutes
don't fail if the db isn't tracking all publishes
This commit is contained in:
Niko Storni 2022-08-09 22:11:42 +02:00
parent ee8eb83d07
commit 5a01983203
7 changed files with 34 additions and 22 deletions

View file

@ -19,7 +19,7 @@ func TestGetPlaylistVideoIDs(t *testing.T) {
} }
func TestGetVideoInformation(t *testing.T) { func TestGetVideoInformation(t *testing.T) {
video, err := GetVideoInformation(nil, "zj7pXM9gE5M", nil, nil, nil) video, err := GetVideoInformation("zj7pXM9gE5M", nil, nil)
if err != nil { if err != nil {
logrus.Error(err) logrus.Error(err)
} }

View file

@ -32,6 +32,10 @@ var (
func main() { func main() {
rand.Seed(time.Now().UnixNano()) rand.Seed(time.Now().UnixNano())
log.SetLevel(log.DebugLevel) log.SetLevel(log.DebugLevel)
customFormatter := new(log.TextFormatter)
customFormatter.TimestampFormat = "2006-01-02 15:04:05"
customFormatter.FullTimestamp = true
log.SetFormatter(customFormatter)
http.Handle("/metrics", promhttp.Handler()) http.Handle("/metrics", promhttp.Handler())
go func() { go func() {
log.Error(http.ListenAndServe(":2112", nil)) log.Error(http.ListenAndServe(":2112", nil))

View file

@ -138,7 +138,7 @@ func (s *SyncManager) Start() error {
"WALLET HAS NOT BEEN MOVED TO THE WALLET BACKUP DIR", "WALLET HAS NOT BEEN MOVED TO THE WALLET BACKUP DIR",
"NotEnoughFunds", "NotEnoughFunds",
"no space left on device", "no space left on device",
"failure uploading wallet", "there was a problem uploading the wallet",
"the channel in the wallet is different than the channel in the database", "the channel in the wallet is different than the channel in the database",
"this channel does not belong to this wallet!", "this channel does not belong to this wallet!",
"You already have a stream claim published under the name", "You already have a stream claim published under the name",

View file

@ -4,6 +4,7 @@ import (
"os" "os"
"path/filepath" "path/filepath"
"strings" "strings"
"time"
"github.com/lbryio/ytsync/v5/configs" "github.com/lbryio/ytsync/v5/configs"
"github.com/lbryio/ytsync/v5/util" "github.com/lbryio/ytsync/v5/util"
@ -215,13 +216,22 @@ func (s *Sync) uploadWallet() error {
} }
defer file.Close() defer file.Close()
_, err = uploader.Upload(&s3manager.UploadInput{ start := time.Now()
Bucket: aws.String(configs.Configuration.WalletS3Config.Bucket),
Key: key, for time.Since(start) < 30*time.Minute {
Body: file, _, err = uploader.Upload(&s3manager.UploadInput{
}) Bucket: aws.String(configs.Configuration.WalletS3Config.Bucket),
Key: key,
Body: file,
})
if err != nil {
time.Sleep(30 * time.Second)
continue
}
break
}
if err != nil { if err != nil {
return err return errors.Prefix("there was a problem uploading the wallet to S3", errors.Err(err))
} }
log.Println("wallet uploaded to S3") log.Println("wallet uploaded to S3")

View file

@ -375,14 +375,12 @@ func (s *Sync) ensureChannelOwnership() error {
channelUsesOldMetadata := false channelUsesOldMetadata := false
if channelToUse != nil { if channelToUse != nil {
channelUsesOldMetadata = channelToUse.Value.GetThumbnail() == nil channelUsesOldMetadata = channelToUse.Value.GetThumbnail() == nil || (len(channelToUse.Value.Languages) == 0 && s.DbChannelData.Language == "")
if !channelUsesOldMetadata { if !channelUsesOldMetadata {
return nil return nil
} }
} }
channelBidAmount := channelClaimAmount
balanceResp, err := s.daemon.AccountBalance(nil) balanceResp, err := s.daemon.AccountBalance(nil)
if err != nil { if err != nil {
return err return err
@ -394,8 +392,8 @@ func (s *Sync) ensureChannelOwnership() error {
return errors.Err(err) return errors.Err(err)
} }
if balance.LessThan(decimal.NewFromFloat(channelBidAmount)) { if balance.LessThan(decimal.NewFromFloat(channelClaimAmount)) {
err = s.addCredits(channelBidAmount + 0.3) err = s.addCredits(channelClaimAmount + estimatedMaxTxFee*3)
if err != nil { if err != nil {
return err return err
} }
@ -433,6 +431,9 @@ func (s *Sync) ensureChannelOwnership() error {
} }
var languages []string = nil var languages []string = nil
if s.DbChannelData.Language != "" {
languages = []string{s.DbChannelData.Language}
}
//we don't have this data without the API //we don't have this data without the API
//if channelInfo.DefaultLanguage != "" { //if channelInfo.DefaultLanguage != "" {
// if channelInfo.DefaultLanguage == "iw" { // if channelInfo.DefaultLanguage == "iw" {

View file

@ -336,7 +336,7 @@ func (s *Sync) waitForDaemonStart() error {
} }
func (s *Sync) stopAndUploadWallet(e *error) { func (s *Sync) stopAndUploadWallet(e *error) {
log.Printf("Stopping daemon") log.Println("Stopping daemon")
shutdownErr := logUtils.StopDaemon() shutdownErr := logUtils.StopDaemon()
if shutdownErr != nil { if shutdownErr != nil {
logShutdownError(shutdownErr) logShutdownError(shutdownErr)
@ -349,16 +349,11 @@ func (s *Sync) stopAndUploadWallet(e *error) {
logShutdownError(processDeathError) logShutdownError(processDeathError)
} else { } else {
err := s.uploadWallet() err := s.uploadWallet()
if err != nil {
time.Sleep(10 * time.Second)
logUtils.SendErrorToSlack("there was a problem uploading the wallet to S3, waiting 10 seconds and retrying: %s", err.Error())
err = s.uploadWallet()
}
if err != nil { if err != nil {
if *e == nil { if *e == nil {
*e = err *e = err
} else { } else {
*e = errors.Prefix(fmt.Sprintf("failure uploading wallet: %s + original error", errors.FullTrace(err)), *e) *e = errors.Prefix(fmt.Sprintf("%s + original error", errors.FullTrace(err)), *e)
} }
} }
err = s.uploadBlockchainDB() err = s.uploadBlockchainDB()
@ -501,7 +496,7 @@ func (s *Sync) updateRemoteDB(claims []jsonrpc.Claim, ownClaims []jsonrpc.Claim)
claimMarkedUnpublished := claimInDatabase && !sv.Published claimMarkedUnpublished := claimInDatabase && !sv.Published
_, isOwnClaim := ownClaimsInfo[videoID] _, isOwnClaim := ownClaimsInfo[videoID]
transferred := !isOwnClaim || s.DbChannelData.TransferState == 3 transferred := !isOwnClaim || s.DbChannelData.TransferState == 3
transferStatusMismatch := sv.Transferred != transferred transferStatusMismatch := claimInDatabase && sv.Transferred != transferred
if metadataDiffers { if metadataDiffers {
log.Debugf("%s: Mismatch in database for metadata. DB: %d - Blockchain: %d", videoID, sv.MetadataVersion, chainInfo.MetadataVersion) log.Debugf("%s: Mismatch in database for metadata. DB: %d - Blockchain: %d", videoID, sv.MetadataVersion, chainInfo.MetadataVersion)
@ -682,7 +677,8 @@ func (s *Sync) checkIntegrity() error {
if pubsOnWallet > pubsOnDB { //This case should never happen if pubsOnWallet > pubsOnDB { //This case should never happen
logUtils.SendInfoToSlack("We're claiming to have published %d videos but in reality we published %d (%s)", pubsOnDB, pubsOnWallet, s.DbChannelData.ChannelId) logUtils.SendInfoToSlack("We're claiming to have published %d videos but in reality we published %d (%s)", pubsOnDB, pubsOnWallet, s.DbChannelData.ChannelId)
return errors.Err("not all published videos are in the database") //we never really done anything about those. it happens when a user updates the channel for a publish to another ytsync channel
//return errors.Err("not all published videos are in the database")
} }
if pubsOnWallet < pubsOnDB { if pubsOnWallet < pubsOnDB {
logUtils.SendInfoToSlack("we're claiming to have published %d videos but we only published %d (%s)", pubsOnDB, pubsOnWallet, s.DbChannelData.ChannelId) logUtils.SendInfoToSlack("we're claiming to have published %d videos but we only published %d (%s)", pubsOnDB, pubsOnWallet, s.DbChannelData.ChannelId)

View file

@ -26,6 +26,7 @@ type YoutubeChannel struct {
SizeLimit int `json:"size_limit"` SizeLimit int `json:"size_limit"`
LastUploadedVideo string `json:"last_uploaded_video"` LastUploadedVideo string `json:"last_uploaded_video"`
WipeDB bool `json:"wipe_db"` WipeDB bool `json:"wipe_db"`
Language string `json:"language"`
} }
type PublishAddress struct { type PublishAddress struct {