add concurrency support

fix error handling (FINALLY)
fix slack message formatting
add locking for wallet setup to support concurrency
remove UTXO bug workaround (lbryumx should fix it)
add limits to filesize

cmd/common.go

@@ -13,6 +13,7 @@ var (
 	syncStatus              string
 	syncFrom                int64
 	syncUntil               int64
+	concurrentJobs          int
 )
 
 const (

cmd/selfsync.go

@@ -41,6 +41,7 @@ func init() {
 	selfSyncCmd.Flags().StringVar(&syncStatus, "status", StatusQueued, "Specify which queue to pull from. Overrides --update (Default: queued)")
 	selfSyncCmd.Flags().Int64Var(&syncFrom, "after", time.Unix(0, 0).Unix(), "Specify from when to pull jobs [Unix time](Default: 0)")
 	selfSyncCmd.Flags().Int64Var(&syncUntil, "before", time.Now().Unix(), "Specify until when to pull jobs [Unix time](Default: current Unix time)")
+	selfSyncCmd.Flags().IntVar(&concurrentJobs, "concurrent-jobs", 1, "how many jobs to process concurrently (Default: 1)")
 
 	RootCmd.AddCommand(selfSyncCmd)
 	APIURL = os.Getenv("LBRY_API")
@@ -136,7 +137,7 @@ func spaceCheck() error {
 		return err
 	}
 	if usedPctile >= 0.90 && !skipSpaceCheck {
-		return errors.Err("more than 90%% of the space has been used. use --skip-space-check to ignore. Used: %.1f%%", usedPctile*100)
+		return errors.Err(fmt.Sprintf("more than 90%% of the space has been used. use --skip-space-check to ignore. Used: %.1f%%", usedPctile*100))
 	}
 	util.SendToSlackInfo("disk usage: %.1f%%", usedPctile*100)
 	return nil
@@ -276,7 +277,7 @@ func syncChannels(channelsToSync []APIYoutubeChannel, ytAPIKey string, syncCount
 			LbryChannelName:         lbryChannelName,
 			StopOnError:             stopOnError,
 			MaxTries:                maxTries,
-			ConcurrentVideos:        1,
+			ConcurrentVideos:        concurrentJobs,
 			TakeOverExistingChannel: takeOverExistingChannel,
 			Refill:                  refill,
 		}

util/slack.go

@@ -19,14 +19,20 @@ func InitSlack(token string, channel string, username string) {
 	defaultUsername = username
 }
 func SendToSlackInfo(format string, a ...interface{}) {
-	message := fmt.Sprintf(format, a...)
+	message := format
+	if len(a) > 0 {
+		message = fmt.Sprintf(format, a...)
+	}
 	if slackApi != nil {
 		sendToSlack(":information_source: " + message)
 	}
 	log.Debugln(message)
 }
 func SendToSlackError(format string, a ...interface{}) {
-	message := fmt.Sprintf(format, a...)
+	message := format
+	if len(a) > 0 {
+		message = fmt.Sprintf(format, a...)
+	}
 	if slackApi != nil {
 		sendToSlack(":sos: " + message)
 	}

util/slice.go

@@ -11,9 +11,9 @@ func InSlice(str string, values []string) bool {
 	return false
 }
 
-func InSliceContains(subStr string, values []string) bool {
-	for _, v := range values {
-		if strings.Contains(v, subStr) {
+func InSliceContains(fullString string, candidates []string) bool {
+	for _, v := range candidates {
+		if strings.Contains(fullString, v) {
 			return true
 		}
 	}

ytsync/setup.go

@@ -14,6 +14,9 @@ import (
 )
 
 func (s *Sync) walletSetup() error {
+	//prevent unnecessary concurrent execution
+	s.mux.Lock()
+	defer s.mux.Unlock()
 	err := s.ensureChannelOwnership()
 	if err != nil {
 		return err
@@ -87,6 +90,7 @@ func (s *Sync) walletSetup() error {
 }
 
 func (s *Sync) ensureEnoughUTXOs() error {
+
 	utxolist, err := s.daemon.UTXOList()
 	if err != nil {
 		return err
@@ -162,32 +166,7 @@ func (s *Sync) waitUntilUTXOsConfirmed() error {
 		if time.Now().After(origin.Add(15 * time.Minute)) {
 			//lbryum is messing with us or something. restart the daemon
 			//this could also be a very long block
-			err := stopDaemonViaSystemd()
-			if err != nil {
-				logShutdownError(err)
-				return err
-			}
-			var waitTimeout time.Duration = 60 * 8
-			err = waitForDaemonProcess(waitTimeout)
-			if err != nil {
-				logShutdownError(err)
-				return err
-			}
-			err = startDaemonViaSystemd()
-			if err != nil {
-				return err
-			}
-			log.Infoln("Waiting for daemon to finish starting...")
-			s.daemon = jsonrpc.NewClient("")
-			s.daemon.SetRPCTimeout(5 * time.Minute)
-
-			for {
-				_, err := s.daemon.WalletBalance()
-				if err == nil {
-					break
-				}
-				time.Sleep(5 * time.Second)
-			}
+			util.SendToSlackError("We've been waiting UTXOs confirmation for %s... and this isn't normal", time.Now().Sub(origin).String())
 		}
 		wait := 30 * time.Second
 		log.Println("Waiting " + wait.String() + "...")

ytsync/sources/youtubeVideo.go

@@ -126,6 +126,7 @@ func (v YoutubeVideo) delete() error {
 	videoPath := v.getFilename()
 	err := os.Remove(videoPath)
 	if err != nil {
+		log.Debugln(errors.Prefix("delete error", err))
 		return err
 	}
 	log.Debugln(v.id + " deleted from disk (" + videoPath + ")")
@@ -201,6 +202,16 @@ func (v YoutubeVideo) Sync(daemon *jsonrpc.Client, claimAddress string, amount f
 	}
 	log.Debugln("Downloaded " + v.id)
 
+	fi, err := os.Stat(v.getFilename())
+	if err != nil {
+		return err
+	}
+	if fi.Size() > 1*1024*1024*1024 {
+		//delete the video and ignore the error
+		_ = v.delete()
+		return errors.Err("video is bigger than 1GB, skipping for now")
+	}
+
 	err = v.triggerThumbnailSave()
 	if err != nil {
 		return errors.Prefix("thumbnail error", err)
@@ -208,13 +219,8 @@ func (v YoutubeVideo) Sync(daemon *jsonrpc.Client, claimAddress string, amount f
 	log.Debugln("Created thumbnail for " + v.id)
 
 	err = v.publish(daemon, claimAddress, amount, channelName)
-	//delete the video in all cases
-	dErr := v.delete()
-	if dErr != nil {
-		// the video was published anyway so it should be marked as published
-		// for that to happen, no errors should be returned any further than here
-		log.Debugln(errors.Prefix("delete error", dErr))
-	}
+	//delete the video in all cases (and ignore the error)
+	_ = v.delete()
 	if err != nil {
 		return errors.Prefix("publish error", err)
 	}

ytsync/ytsync.go

@@ -67,6 +67,7 @@ type Sync struct {
 
 	grp *stop.Group
 
+	mux   sync.Mutex
 	wg    sync.WaitGroup
 	queue chan video
 }
@@ -255,6 +256,7 @@ func (s *Sync) startWorker(workerNum int) {
 				fatalErrors := []string{
 					":5279: read: connection reset by peer",
 					"net/http: request canceled (Client.Timeout exceeded while awaiting headers)",
+					"no space left on device",
 				}
 				if util.InSliceContains(err.Error(), fatalErrors) || s.StopOnError {
 					s.grp.Stop()
@@ -268,6 +270,7 @@ func (s *Sync) startWorker(workerNum int) {
 						"Playback on other websites has been disabled by the video owner",
 						"Error in daemon: Cannot publish empty file",
 						"Error extracting sts from embedded url response",
+						"Client.Timeout exceeded while awaiting headers)",
 					}
 					if util.InSliceContains(err.Error(), errorsNoRetry) {
 						log.Println("This error should not be retried at all")