add language to channels

improve logging (timestamps)
retry wallet uploads for 30 minutes
don't fail if the db isn't tracking all publishes

downloader/downloader_test.go

@@ -19,7 +19,7 @@ func TestGetPlaylistVideoIDs(t *testing.T) {
 }
 
 func TestGetVideoInformation(t *testing.T) {
-	video, err := GetVideoInformation(nil, "zj7pXM9gE5M", nil, nil, nil)
+	video, err := GetVideoInformation("zj7pXM9gE5M", nil, nil)
 	if err != nil {
 		logrus.Error(err)
 	}

main.go

@@ -32,6 +32,10 @@ var (
 func main() {
 	rand.Seed(time.Now().UnixNano())
 	log.SetLevel(log.DebugLevel)
+	customFormatter := new(log.TextFormatter)
+	customFormatter.TimestampFormat = "2006-01-02 15:04:05"
+	customFormatter.FullTimestamp = true
+	log.SetFormatter(customFormatter)
 	http.Handle("/metrics", promhttp.Handler())
 	go func() {
 		log.Error(http.ListenAndServe(":2112", nil))

manager/manager.go

@@ -138,7 +138,7 @@ func (s *SyncManager) Start() error {
 					"WALLET HAS NOT BEEN MOVED TO THE WALLET BACKUP DIR",
 					"NotEnoughFunds",
 					"no space left on device",
-					"failure uploading wallet",
+					"there was a problem uploading the wallet",
 					"the channel in the wallet is different than the channel in the database",
 					"this channel does not belong to this wallet!",
 					"You already have a stream claim published under the name",

manager/s3_storage.go

@@ -4,6 +4,7 @@ import (
 	"os"
 	"path/filepath"
 	"strings"
+	"time"
 
 	"github.com/lbryio/ytsync/v5/configs"
 	"github.com/lbryio/ytsync/v5/util"
@@ -215,13 +216,22 @@ func (s *Sync) uploadWallet() error {
 	}
 	defer file.Close()
 
-	_, err = uploader.Upload(&s3manager.UploadInput{
-		Bucket: aws.String(configs.Configuration.WalletS3Config.Bucket),
-		Key:    key,
-		Body:   file,
-	})
+	start := time.Now()
+
+	for time.Since(start) < 30*time.Minute {
+		_, err = uploader.Upload(&s3manager.UploadInput{
+			Bucket: aws.String(configs.Configuration.WalletS3Config.Bucket),
+			Key:    key,
+			Body:   file,
+		})
+		if err != nil {
+			time.Sleep(30 * time.Second)
+			continue
+		}
+		break
+	}
 	if err != nil {
-		return err
+		return errors.Prefix("there was a problem uploading the wallet to S3", errors.Err(err))
 	}
 	log.Println("wallet uploaded to S3")
 

manager/setup.go

@@ -375,14 +375,12 @@ func (s *Sync) ensureChannelOwnership() error {
 
 	channelUsesOldMetadata := false
 	if channelToUse != nil {
-		channelUsesOldMetadata = channelToUse.Value.GetThumbnail() == nil
+		channelUsesOldMetadata = channelToUse.Value.GetThumbnail() == nil || (len(channelToUse.Value.Languages) == 0 && s.DbChannelData.Language == "")
 		if !channelUsesOldMetadata {
 			return nil
 		}
 	}
 
-	channelBidAmount := channelClaimAmount
-
 	balanceResp, err := s.daemon.AccountBalance(nil)
 	if err != nil {
 		return err
@@ -394,8 +392,8 @@ func (s *Sync) ensureChannelOwnership() error {
 		return errors.Err(err)
 	}
 
-	if balance.LessThan(decimal.NewFromFloat(channelBidAmount)) {
-		err = s.addCredits(channelBidAmount + 0.3)
+	if balance.LessThan(decimal.NewFromFloat(channelClaimAmount)) {
+		err = s.addCredits(channelClaimAmount + estimatedMaxTxFee*3)
 		if err != nil {
 			return err
 		}
@@ -433,6 +431,9 @@ func (s *Sync) ensureChannelOwnership() error {
 	}
 
 	var languages []string = nil
+	if s.DbChannelData.Language != "" {
+		languages = []string{s.DbChannelData.Language}
+	}
 	//we don't have this data without the API
 	//if channelInfo.DefaultLanguage != "" {
 	//	if channelInfo.DefaultLanguage == "iw" {

manager/ytsync.go

@@ -336,7 +336,7 @@ func (s *Sync) waitForDaemonStart() error {
 }
 
 func (s *Sync) stopAndUploadWallet(e *error) {
-	log.Printf("Stopping daemon")
+	log.Println("Stopping daemon")
 	shutdownErr := logUtils.StopDaemon()
 	if shutdownErr != nil {
 		logShutdownError(shutdownErr)
@@ -349,16 +349,11 @@ func (s *Sync) stopAndUploadWallet(e *error) {
 			logShutdownError(processDeathError)
 		} else {
 			err := s.uploadWallet()
-			if err != nil {
-				time.Sleep(10 * time.Second)
-				logUtils.SendErrorToSlack("there was a problem uploading the wallet to S3, waiting 10 seconds and retrying: %s", err.Error())
-				err = s.uploadWallet()
-			}
 			if err != nil {
 				if *e == nil {
 					*e = err
 				} else {
-					*e = errors.Prefix(fmt.Sprintf("failure uploading wallet: %s + original error", errors.FullTrace(err)), *e)
+					*e = errors.Prefix(fmt.Sprintf("%s + original error", errors.FullTrace(err)), *e)
 				}
 			}
 			err = s.uploadBlockchainDB()
@@ -501,7 +496,7 @@ func (s *Sync) updateRemoteDB(claims []jsonrpc.Claim, ownClaims []jsonrpc.Claim)
 		claimMarkedUnpublished := claimInDatabase && !sv.Published
 		_, isOwnClaim := ownClaimsInfo[videoID]
 		transferred := !isOwnClaim || s.DbChannelData.TransferState == 3
-		transferStatusMismatch := sv.Transferred != transferred
+		transferStatusMismatch := claimInDatabase && sv.Transferred != transferred
 
 		if metadataDiffers {
 			log.Debugf("%s: Mismatch in database for metadata. DB: %d - Blockchain: %d", videoID, sv.MetadataVersion, chainInfo.MetadataVersion)
@@ -682,7 +677,8 @@ func (s *Sync) checkIntegrity() error {
 
 	if pubsOnWallet > pubsOnDB { //This case should never happen
 		logUtils.SendInfoToSlack("We're claiming to have published %d videos but in reality we published %d (%s)", pubsOnDB, pubsOnWallet, s.DbChannelData.ChannelId)
-		return errors.Err("not all published videos are in the database")
+		//we never really done anything about those. it happens when a user updates the channel for a publish to another ytsync channel
+		//return errors.Err("not all published videos are in the database")
 	}
 	if pubsOnWallet < pubsOnDB {
 		logUtils.SendInfoToSlack("we're claiming to have published %d videos but we only published %d (%s)", pubsOnDB, pubsOnWallet, s.DbChannelData.ChannelId)

shared/shared.go

@@ -26,6 +26,7 @@ type YoutubeChannel struct {
 	SizeLimit          int            `json:"size_limit"`
 	LastUploadedVideo  string         `json:"last_uploaded_video"`
 	WipeDB             bool           `json:"wipe_db"`
+	Language           string         `json:"language"`
 }
 
 type PublishAddress struct {