improve error handling

retry wallet uploads on failure

manager/ytsync.go

@@ -286,6 +286,8 @@ func (s *Sync) setChannelTerminationStatus(e *error) {
 			"interrupted during daemon startup",
 			"interrupted by user",
 			"use --skip-space-check to ignore",
+			"failure uploading blockchain DB",
+			"default_wallet already exists",
 		}
 		dbWipeConditions := []string{
 			"Missing inputs",
@@ -348,11 +350,16 @@ func (s *Sync) stopAndUploadWallet(e *error) {
 			logShutdownError(processDeathError)
 		} else {
 			err := s.uploadWallet()
+			if err != nil {
+				time.Sleep(10 * time.Second)
+				logUtils.SendErrorToSlack("there was a problem uploading the wallet to S3, waiting 10 seconds and retrying: %s", err.Error())
+				err = s.uploadWallet()
+			}
 			if err != nil {
 				if *e == nil {
 					*e = err
 				} else {
-					*e = errors.Prefix("failure uploading wallet", *e)
+					*e = errors.Prefix(fmt.Sprintf("failure uploading wallet: %s + original error", errors.FullTrace(err)), *e)
 				}
 			}
 			err = s.uploadBlockchainDB()

util/log_wrapper.go

@@ -14,7 +14,9 @@ func SendErrorToSlack(format string, a ...interface{}) {
 		message = fmt.Sprintf(format, a...)
 	}
 	log.Errorln(message)
+	log.SetLevel(log.InfoLevel) //I don't want to change the underlying lib so this will do...
 	err := util.SendToSlack(":sos: ```" + message + "```")
+	log.SetLevel(log.DebugLevel)
 	if err != nil {
 		log.Errorln(err)
 	}
@@ -27,7 +29,9 @@ func SendInfoToSlack(format string, a ...interface{}) {
 		message = fmt.Sprintf(format, a...)
 	}
 	log.Infoln(message)
+	log.SetLevel(log.InfoLevel) //I don't want to change the underlying lib so this will do...
 	err := util.SendToSlack(":information_source: " + message)
+	log.SetLevel(log.DebugLevel)
 	if err != nil {
 		log.Errorln(err)
 	}