add disk space check

remove unneeded panics
improve error handling
add tests
fix error checking that was comparing for exact values
sort imports

cmd/common.go

@@ -8,6 +8,7 @@ var (
 	takeOverExistingChannel bool
 	refill                  int
 	limit                   int
+	skipSpaceCheck          bool
 )
 
 const (

cmd/selfsync.go

@@ -1,18 +1,18 @@
 package cmd
 
 import (
-	"github.com/lbryio/lbry.go/errors"
-	sync "github.com/lbryio/lbry.go/ytsync"
-
 	"encoding/json"
 	"fmt"
 	"io/ioutil"
 	"net/http"
 	"os"
+	"os/user"
 	"strings"
 
+	"github.com/lbryio/lbry.go/errors"
 	"github.com/lbryio/lbry.go/null"
 	"github.com/lbryio/lbry.go/util"
+	sync "github.com/lbryio/lbry.go/ytsync"
 	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
 )
@@ -28,6 +28,8 @@ func init() {
 	selfSyncCmd.Flags().IntVar(&maxTries, "max-tries", defaultMaxTries, "Number of times to try a publish that fails")
 	selfSyncCmd.Flags().BoolVar(&takeOverExistingChannel, "takeover-existing-channel", false, "If channel exists and we don't own it, take over the channel")
 	selfSyncCmd.Flags().IntVar(&limit, "limit", 0, "limit the amount of channels to sync")
+	selfSyncCmd.Flags().BoolVar(&skipSpaceCheck, "skip-space-check", false, "Do not perform free space check on startup")
+
 	RootCmd.AddCommand(selfSyncCmd)
 }
 
@@ -103,6 +105,21 @@ func setChannelSyncStatus(authToken string, channelID string, status string) err
 }
 
 func selfSync(cmd *cobra.Command, args []string) {
+	usr, err := user.Current()
+	if err != nil {
+		util.SendToSlackError(err.Error())
+		return
+	}
+	usedPctile, err := util.GetUsedSpace(usr.HomeDir + "/.lbrynet/blobfiles/")
+	if err != nil {
+		util.SendToSlackError(err.Error())
+		return
+	}
+	if usedPctile > 0.9 && !skipSpaceCheck {
+		util.SendToSlackError("more than 90% of the space has been used. use --skip-space-check to ignore. %.1f", usedPctile*100)
+		return
+	}
+	util.SendToSlackInfo("disk usage: %.1f", usedPctile*100)
 	slackToken := os.Getenv("SLACK_TOKEN")
 	if slackToken == "" {
 		log.Error("A slack token was not present in env vars! Slack messages disabled!")
@@ -172,26 +189,32 @@ func selfSync(cmd *cobra.Command, args []string) {
 		util.SendToSlackInfo("Syncing " + lbryChannelName + " reached an end.")
 		if err != nil {
 			util.SendToSlackError(errors.FullTrace(err))
+			fatalErrors := []string{
+				"default_wallet already exists",
+				"WALLET HAS NOT BEEN MOVED TO THE WALLET BACKUP DIR",
+			}
+			if util.InSliceContains(err.Error(), fatalErrors) {
+				util.SendToSlackError("@Nikooo777 this requires manual intervention! Exiting...")
+				break
+			}
 			//mark video as failed
 			err := setChannelSyncStatus(authToken, channelID, StatusFailed)
 			if err != nil {
 				msg := fmt.Sprintf("Failed setting failed state for channel %s: %v", lbryChannelName, err)
 				util.SendToSlackError(msg)
-				util.SendToSlackError("@Nikooo777 this requires manual intervention! Panicing...")
-				panic(msg)
+				util.SendToSlackError("@Nikooo777 this requires manual intervention! Exiting...")
+				break
 			}
-			break
+			continue
 		}
 		//mark video as synced
 		err = setChannelSyncStatus(authToken, channelID, StatusSynced)
 		if err != nil {
 			msg := fmt.Sprintf("Failed setting synced state for channel %s: %v", lbryChannelName, err)
 			util.SendToSlackError(msg)
-			util.SendToSlackError("@Nikooo777 this requires manual intervention! Panicing...")
-			log.Error(msg)
+			util.SendToSlackError("@Nikooo777 this requires manual intervention! Exiting...")
 			//this error is very bad. it requires manual intervention
-			panic(msg)
-			continue
+			break
 		}
 
 		if limit != 0 && loops >= limit {

cmd/ytsync.go

@@ -1,12 +1,12 @@
 package cmd
 
 import (
-	"github.com/lbryio/lbry.go/errors"
-	sync "github.com/lbryio/lbry.go/ytsync"
-
 	"os"
+	"os/user"
 
+	"github.com/lbryio/lbry.go/errors"
 	"github.com/lbryio/lbry.go/util"
+	sync "github.com/lbryio/lbry.go/ytsync"
 	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
 )
@@ -22,10 +22,26 @@ func init() {
 	ytSyncCmd.Flags().IntVar(&maxTries, "max-tries", defaultMaxTries, "Number of times to try a publish that fails")
 	ytSyncCmd.Flags().BoolVar(&takeOverExistingChannel, "takeover-existing-channel", false, "If channel exists and we don't own it, take over the channel")
 	ytSyncCmd.Flags().IntVar(&refill, "refill", 0, "Also add this many credits to the wallet")
+	ytSyncCmd.Flags().BoolVar(&skipSpaceCheck, "skip-space-check", false, "Do not perform free space check on startup")
 	RootCmd.AddCommand(ytSyncCmd)
 }
 
 func ytsync(cmd *cobra.Command, args []string) {
+	usr, err := user.Current()
+	if err != nil {
+		util.SendToSlackError(err.Error())
+		return
+	}
+	usedPctile, err := util.GetUsedSpace(usr.HomeDir + "/.lbrynet/blobfiles/")
+	if err != nil {
+		util.SendToSlackError(err.Error())
+		return
+	}
+	if usedPctile > 0.9 && !skipSpaceCheck {
+		util.SendToSlackError("more than 90% of the space has been used. use --skip-space-check to ignore. %.1f", usedPctile*100)
+		return
+	}
+	util.SendToSlackInfo("disk usage: %.1f", usedPctile*100)
 	slackToken := os.Getenv("SLACK_TOKEN")
 	if slackToken == "" {
 		log.Error("A slack token was not present in env vars! Slack messages disabled!")
@@ -65,7 +81,7 @@ func ytsync(cmd *cobra.Command, args []string) {
 		Refill:                  refill,
 	}
 
-	err := s.FullCycle()
+	err = s.FullCycle()
 
 	if err != nil {
 		util.SendToSlackError(errors.FullTrace(err))

util/disk_space.go

@@ -0,0 +1,18 @@
+package util
+
+import "syscall"
+
+// GetUsedSpace returns a value between 0 and 1, with 0 being completely empty and 1 being full, for the disk that holds the provided path
+func GetUsedSpace(path string) (float32, error) {
+	var stat syscall.Statfs_t
+	err := syscall.Statfs(path, &stat)
+	if err != nil {
+		return 0, err
+	}
+	// Available blocks * size per block = available space in bytes
+	all := stat.Blocks * uint64(stat.Bsize)
+	free := stat.Bfree * uint64(stat.Bsize)
+	used := all - free
+
+	return float32(used) / float32(all), nil
+}

util/disk_space_test.go

@@ -0,0 +1,21 @@
+package util
+
+import (
+	"os/user"
+	"testing"
+)
+
+func TestGetUsedSpace(t *testing.T) {
+	usr, err := user.Current()
+	if err != nil {
+		t.Error(err)
+	}
+	usedPctile, err := GetUsedSpace(usr.HomeDir + "/.lbrynet/blobfiles/")
+	if err != nil {
+		t.Error(err)
+	}
+	if usedPctile > 1 {
+		t.Errorf("over 1: %.2f", usedPctile)
+	}
+	t.Logf("used space: %.2f", usedPctile)
+}

util/slack_test.go

@@ -11,5 +11,5 @@ func TestSendToSlack(t *testing.T) {
 		t.Error("A slack token was not provided")
 	}
 	InitSlack(slackToken)
-	SendToSlack("This is a test :)")
+	SendToSlackInfo("This is a test :)")
 }

util/slice.go

@@ -1,5 +1,7 @@
 package util
 
+import "strings"
+
 func InSlice(str string, values []string) bool {
 	for _, v := range values {
 		if str == v {
@@ -8,3 +10,12 @@ func InSlice(str string, values []string) bool {
 	}
 	return false
 }
+
+func InSliceContains(str string, values []string) bool {
+	for _, v := range values {
+		if strings.Contains(v, str) {
+			return true
+		}
+	}
+	return false
+}

ytsync/ytsync.go

@@ -4,6 +4,7 @@ import (
 	"bufio"
 	"encoding/csv"
 	"encoding/json"
+	"fmt"
 	"io"
 	"io/ioutil"
 	"net/http"
@@ -22,13 +23,11 @@ import (
 	"github.com/lbryio/lbry.go/ytsync/redisdb"
 	"github.com/lbryio/lbry.go/ytsync/sources"
 
-	"fmt"
-
 	"github.com/lbryio/lbry.go/util"
 	"github.com/mitchellh/go-ps"
 	log "github.com/sirupsen/logrus"
 	"google.golang.org/api/googleapi/transport"
-	"google.golang.org/api/youtube/v3"
+	youtube "google.golang.org/api/youtube/v3"
 )
 
 const (
@@ -246,7 +245,7 @@ func (s *Sync) startWorker(workerNum int) {
 					":5279: read: connection reset by peer",
 					"net/http: request canceled (Client.Timeout exceeded while awaiting headers)",
 				}
-				if util.InSlice(err.Error(), fatalErrors) || s.StopOnError {
+				if util.InSliceContains(err.Error(), fatalErrors) || s.StopOnError {
 					s.grp.Stop()
 				} else if s.MaxTries > 1 {
 					errorsNoRetry := []string{
@@ -259,12 +258,13 @@ func (s *Sync) startWorker(workerNum int) {
 						"Error in daemon: Cannot publish empty file",
 						"Error extracting sts from embedded url response",
 					}
-					if util.InSlice(err.Error(), errorsNoRetry) {
+					if util.InSliceContains(err.Error(), errorsNoRetry) {
 						log.Println("This error should not be retried at all")
 					} else if tryCount < s.MaxTries {
 						if strings.Contains(err.Error(), "The transaction was rejected by network rules.(258: txn-mempool-conflict)") ||
 							strings.Contains(err.Error(), "failed: Not enough funds") ||
-							strings.Contains(err.Error(), "Error in daemon: Insufficient funds, please deposit additional LBC") {
+							strings.Contains(err.Error(), "Error in daemon: Insufficient funds, please deposit additional LBC") ||
+							strings.Contains(err.Error(), "The transaction was rejected by network rules.(64: too-long-mempool-chain)") {
 							log.Println("waiting for a block and refilling addresses before retrying")
 							err = s.walletSetup()
 							if err != nil {