fix selfsync issues with the queue

remove margin for fees
handle non latin names
adjusted error handling

cmd/selfsync.go

@@ -71,7 +71,8 @@ func fetchChannels(status string) ([]APIYoutubeChannel, error) {
 	res, _ := http.PostForm(url, url2.Values{
 		"auth_token":  {APIToken},
 		"sync_status": {status},
-		"after":       {strconv.Itoa(int(time.Now().AddDate(0, 0, -1).Unix()))},
+		"after":       {strconv.Itoa(int(time.Date(2018, 5, 25, 0, 0, 0, 0, time.UTC).Unix()))},
+		"min_videos":  {strconv.Itoa(1)},
 	})
 	defer res.Body.Close()
 	body, _ := ioutil.ReadAll(res.Body)
@@ -83,6 +84,7 @@ func fetchChannels(status string) ([]APIYoutubeChannel, error) {
 	if response.Data == nil {
 		return nil, errors.Err(response.Error)
 	}
+	log.Printf("Fetched channels: %d", len(response.Data))
 	return response.Data, nil
 }
 
@@ -212,8 +214,21 @@ func processQueue(queueStatus string, ytAPIKey string, syncCount *int) (bool, er
 	if err != nil {
 		return false, errors.Prefix("failed to fetch channels", err)
 	}
+	filteredChannelsToSync := make([]APIYoutubeChannel, len(channelsToSync))
+	host, err := os.Hostname()
+	if err != nil {
+		return false, errors.Err("could not detect system hostname")
+	}
+	index := 0
+	for _, v := range channelsToSync {
+		if !v.SyncServer.IsNull() && v.SyncServer.String != host {
+			filteredChannelsToSync[index] = v
+			index++
+		}
+	}
+	interrupted, err := syncChannels(channelsToSync, ytAPIKey, syncCount)
 	util.SendToSlackInfo("Finished syncing %s channels", queueStatus)
-	return syncChannels(channelsToSync, ytAPIKey, syncCount)
+	return interrupted, err
 }
 
 // syncChannels processes a slice of youtube channels (channelsToSync) and returns a bool that indicates whether
@@ -223,13 +238,14 @@ func syncChannels(channelsToSync []APIYoutubeChannel, ytAPIKey string, syncCount
 	if err != nil {
 		host = ""
 	}
-	for ; *syncCount < len(channelsToSync) && (limit == 0 || *syncCount < limit); *syncCount++ {
+	for loop := 0; loop < len(channelsToSync) && (limit == 0 || *syncCount < limit); loop++ {
+		//log.Printf("inside loop: %d", loop)
 		err = spaceCheck()
 		if err != nil {
 			return false, err
 		}
 		//avoid dereferencing
-		channel := channelsToSync[*syncCount]
+		channel := channelsToSync[loop]
 		channelID := channel.ChannelId
 		lbryChannelName := channel.DesiredChannelName
 		if channel.TotalVideos < 1 {
@@ -247,6 +263,8 @@ func syncChannels(channelsToSync []APIYoutubeChannel, ytAPIKey string, syncCount
 			util.SendToSlackError("Failed acquiring sync rights for channel %s: %v", lbryChannelName, err)
 			continue
 		}
+		//increment only if successfully acquired lock
+		*syncCount++
 		util.SendToSlackInfo("Syncing %s to LBRY! (iteration %d)", lbryChannelName, *syncCount)
 
 		s := sync.Sync{
@@ -289,5 +307,8 @@ func syncChannels(channelsToSync []APIYoutubeChannel, ytAPIKey string, syncCount
 			return false, errors.Prefix(msg, err)
 		}
 	}
+	if limit != 0 && *syncCount >= limit {
+		return true, nil
+	}
 	return false, nil
 }

util/slice.go

@@ -11,9 +11,9 @@ func InSlice(str string, values []string) bool {
 	return false
 }
 
-func InSliceContains(str string, values []string) bool {
+func InSliceContains(subStr string, values []string) bool {
 	for _, v := range values {
-		if strings.Contains(v, str) {
+		if strings.Contains(v, subStr) {
 			return true
 		}
 	}

ytsync/setup.go

@@ -51,7 +51,6 @@ func (s *Sync) walletSetup() error {
 		minBalance = 1 //since we ended up in this function it means some juice is still needed
 	}
 	amountToAdd, _ := decimal.NewFromFloat(minBalance).Sub(balance).Float64()
-	amountToAdd *= 6 // add 600% margin for fees, future publishes, etc (insane i know)
 
 	if s.Refill > 0 {
 		if amountToAdd < 0 {

ytsync/sources/shared.go

@@ -1,11 +1,15 @@
 package sources
 
 import (
+	"fmt"
 	"regexp"
 	"strconv"
 	"strings"
 	"sync"
 
+	"crypto/md5"
+	"encoding/hex"
+
 	"github.com/lbryio/lbry.go/jsonrpc"
 	log "github.com/sirupsen/logrus"
 )
@@ -56,6 +60,12 @@ func publishAndRetryExistingNames(daemon *jsonrpc.Client, title, filename string
 			log.Printf("name exists, retrying (%d attempts so far)\n", attempt)
 			continue
 		}
+		//if for some reasons the title can't be converted in a valid claim name (too short or not latin) then we use a hash
+		if len(name) < 2 {
+			hasher := md5.New()
+			hasher.Write([]byte(title))
+			name = fmt.Sprintf("%s-%d", hex.EncodeToString(hasher.Sum(nil))[:15], attempt)
+		}
 
 		_, err := daemon.Publish(name, filename, amount, options)
 		if err == nil || strings.Contains(err.Error(), "failed: Multiple claims (") {

ytsync/sources/youtubeVideo.go

@@ -59,10 +59,6 @@ func (v YoutubeVideo) PublishedAt() time.Time {
 }
 
 func (v YoutubeVideo) getFilename() string {
-	return v.dir + "/" + v.getClaimName() + ".mp4"
-}
-
-func (v YoutubeVideo) getClaimName() string {
 	maxLen := 30
 	reg := regexp.MustCompile(`[^a-zA-Z0-9]+`)
 
@@ -83,8 +79,10 @@ func (v YoutubeVideo) getClaimName() string {
 		}
 		name = tmpName
 	}
-
-	return name
+	if len(name) < 1 {
+		name = v.id
+	}
+	return v.dir + "/" + name + ".mp4"
 }
 
 func (v YoutubeVideo) getAbbrevDescription() string {

ytsync/ytsync.go

@@ -272,10 +272,10 @@ func (s *Sync) startWorker(workerNum int) {
 					if util.InSliceContains(err.Error(), errorsNoRetry) {
 						log.Println("This error should not be retried at all")
 					} else if tryCount < s.MaxTries {
-						if strings.Contains(err.Error(), "The transaction was rejected by network rules.(258: txn-mempool-conflict)") ||
+						if strings.Contains(err.Error(), "258: txn-mempool-conflict") ||
 							strings.Contains(err.Error(), "failed: Not enough funds") ||
 							strings.Contains(err.Error(), "Error in daemon: Insufficient funds, please deposit additional LBC") ||
-							strings.Contains(err.Error(), "The transaction was rejected by network rules.(64: too-long-mempool-chain)") {
+							strings.Contains(err.Error(), "64: too-long-mempool-chain") {
 							log.Println("waiting for a block and refilling addresses before retrying")
 							err = s.walletSetup()
 							if err != nil {