add slack logging

reduce claim name length (fees are too high)
delete videos in case of failure too
reduce synced videos to 1000 most recent

cmd/selfsync.go

@@ -46,7 +46,7 @@ type APIYoutubeChannel struct {
 
 //PoC
 func fetchChannels(authToken string) ([]APIYoutubeChannel, error) {
-	url := "http://localhost:8080/yt/jobs"
+	url := "https://api.lbry.io/yt/jobs"
 	payload := strings.NewReader("------WebKitFormBoundary7MA4YWxkTrZu0gW\r\nContent-Disposition: form-data; name=\"auth_token\"\r\n\r\n" + authToken + "\r\n------WebKitFormBoundary7MA4YWxkTrZu0gW--")
 	req, _ := http.NewRequest("POST", url, payload)
 	req.Header.Add("content-type", "multipart/form-data; boundary=----WebKitFormBoundary7MA4YWxkTrZu0gW")
@@ -74,7 +74,7 @@ func setChannelSyncStatus(authToken string, channelID string, status string) err
 	if err != nil {
 		return errors.Err("could not detect system hostname")
 	}
-	url := "http://localhost:8080/yt/sync_update"
+	url := "https://api.lbry.io/yt/sync_update"
 	payload := strings.NewReader("------WebKitFormBoundary7MA4YWxkTrZu0gW\r\nContent-Disposition: form-data;" +
 		" name=\"channel_id\"\r\n\r\n" + channelID + "\r\n------WebKitFormBoundary7MA4YWxkTrZu0gW\r\n" +
 		"Content-Disposition: form-data; name=\"sync_server\"\r\n\r\n" + host + "\r\n------WebKitFormBoundary7MA4YWxkTrZu0gW\r\n" +
@@ -128,41 +128,34 @@ func selfSync(cmd *cobra.Command, args []string) {
 	}
 	channelsToSync, err := fetchChannels(authToken)
 	if err != nil {
-		msg := fmt.Sprintf("failed to fetch channels: %v", err)
-		log.Errorln(msg)
-		util.SendToSlack(msg)
+		util.SendToSlackError("failed to fetch channels: %v", err)
 		return
 	}
-
-	for loops := 0; loops < len(channelsToSync); loops++ {
+	host, err := os.Hostname()
+	if err != nil {
+		host = ""
+	}
+	for loops := 0; loops < len(channelsToSync) && (limit != 0 && loops < limit); loops++ {
 		//avoid dereferencing
 		channel := channelsToSync[loops]
 		channelID := channel.ChannelId
 		lbryChannelName := channel.DesiredChannelName
 		if channel.TotalVideos < 1 {
-			msg := fmt.Sprintf("Channnel %s has no videos. Skipping", lbryChannelName)
-			util.SendToSlack(msg)
-			log.Debugln(msg)
+			util.SendToSlackInfo("Channnel %s has no videos. Skipping", lbryChannelName)
 			continue
 		}
-		if !channel.SyncServer.IsNull() {
-			msg := fmt.Sprintf("Channnel %s is being synced by another server: %s", lbryChannelName, channel.SyncServer.String)
-			util.SendToSlack(msg)
-			log.Debugln(msg)
+		if !channel.SyncServer.IsNull() && channel.SyncServer.String != host {
+			util.SendToSlackInfo("Channnel %s is being synced by another server: %s", lbryChannelName, channel.SyncServer.String)
 			continue
 		}
 
 		//acquire the lock on the channel
 		err := setChannelSyncStatus(authToken, channelID, StatusSyncing)
 		if err != nil {
-			msg := fmt.Sprintf("Failed aquiring sync rights for channel %s: %v", lbryChannelName, err)
-			util.SendToSlack(msg)
-			log.Error(msg)
+			util.SendToSlackError("Failed aquiring sync rights for channel %s: %v", lbryChannelName, err)
 			continue
 		}
-		msg := fmt.Sprintf("Syncing %s to LBRY! (iteration %d)", lbryChannelName, loops)
-		util.SendToSlack(msg)
-		log.Debugln(msg)
+		util.SendToSlackInfo("Syncing %s to LBRY! (iteration %d)", lbryChannelName, loops)
 
 		s := sync.Sync{
 			YoutubeAPIKey:           ytAPIKey,
@@ -176,17 +169,15 @@ func selfSync(cmd *cobra.Command, args []string) {
 		}
 
 		err = s.FullCycle()
-		util.SendToSlack("Syncing " + lbryChannelName + " reached an end.")
+		util.SendToSlackInfo("Syncing " + lbryChannelName + " reached an end.")
 		if err != nil {
-			log.Error(errors.FullTrace(err))
-			util.SendToSlack(errors.FullTrace(err))
+			util.SendToSlackError(errors.FullTrace(err))
 			//mark video as failed
 			err := setChannelSyncStatus(authToken, channelID, StatusFailed)
 			if err != nil {
 				msg := fmt.Sprintf("Failed setting failed state for channel %s: %v", lbryChannelName, err)
-				util.SendToSlack(msg)
-				util.SendToSlack("@Nikooo777 this requires manual intervention! Panicing...")
-				log.Error(msg)
+				util.SendToSlackError(msg)
+				util.SendToSlackError("@Nikooo777 this requires manual intervention! Panicing...")
 				panic(msg)
 			}
 			break
@@ -195,8 +186,8 @@ func selfSync(cmd *cobra.Command, args []string) {
 		err = setChannelSyncStatus(authToken, channelID, StatusSynced)
 		if err != nil {
 			msg := fmt.Sprintf("Failed setting synced state for channel %s: %v", lbryChannelName, err)
-			util.SendToSlack(msg)
-			util.SendToSlack("@Nikooo777 this requires manual intervention! Panicing...")
+			util.SendToSlackError(msg)
+			util.SendToSlackError("@Nikooo777 this requires manual intervention! Panicing...")
 			log.Error(msg)
 			//this error is very bad. it requires manual intervention
 			panic(msg)
@@ -204,12 +195,9 @@ func selfSync(cmd *cobra.Command, args []string) {
 		}
 
 		if limit != 0 && loops >= limit {
-			msg := fmt.Sprintf("limit of %d reached! Stopping", limit)
-			util.SendToSlack(msg)
-			log.Debugln(msg)
+			util.SendToSlackInfo("limit of %d reached! Stopping", limit)
 			break
 		}
 	}
-	util.SendToSlack("Syncing process terminated!")
-	log.Debugln("Syncing process terminated!")
+	util.SendToSlackInfo("Syncing process terminated!")
 }

cmd/ytsync.go

@@ -52,7 +52,7 @@ func ytsync(cmd *cobra.Command, args []string) {
 		log.Errorln("setting --max-tries less than 1 doesn't make sense")
 		return
 	}
-	util.SendToSlack("Syncing " + lbryChannelName + " to LBRY!")
+	util.SendToSlackInfo("Syncing " + lbryChannelName + " to LBRY!")
 
 	s := sync.Sync{
 		YoutubeAPIKey:           ytAPIKey,
@@ -68,9 +68,8 @@ func ytsync(cmd *cobra.Command, args []string) {
 	err := s.FullCycle()
 
 	if err != nil {
-		log.Error(errors.FullTrace(err))
-		util.SendToSlack(errors.FullTrace(err))
+		util.SendToSlackError(errors.FullTrace(err))
 
 	}
-	util.SendToSlack("Syncing " + lbryChannelName + " reached an end.")
+	util.SendToSlackInfo("Syncing " + lbryChannelName + " reached an end.")
 }

util/slack.go

@@ -4,7 +4,6 @@ import (
 	"strings"
 
 	"github.com/lbryio/lbry.go/errors"
-
 	"github.com/nlopes/slack"
 	log "github.com/sirupsen/logrus"
 )
@@ -19,6 +18,20 @@ func InitSlack(token string, channel string, username string) {
 	defaultChannel = channel
 	defaultUsername = username
 }
+func SendToSlackInfo(format string, a ...interface{}) {
+	message := fmt.Sprintf(format, a...)
+	if slackApi != nil {
+		sendToSlack(":information_source: " + message)
+	}
+	log.Debugln(message)
+}
+func SendToSlackError(format string, a ...interface{}) {
+	message := fmt.Sprintf(format, a...)
+	if slackApi != nil {
+		sendToSlack(":sos: " + message)
+	}
+	log.Errorln(message)
+}
 
 // SendToSlackUser Sends message to a specific user.
 func SendToSlackUser(user, username, message string) error {
@@ -48,7 +61,6 @@ func SendToSlack(message string) error {
 
 func sendToSlack(channel, username, message string) error {
 	var err error
-
 	if slackApi == nil {
 		err = errors.Err("no slack token provided")
 	} else {

ytsync/sources/youtubeVideo.go

@@ -63,7 +63,7 @@ func (v YoutubeVideo) getFilename() string {
 }
 
 func (v YoutubeVideo) getClaimName() string {
-	maxLen := 40
+	maxLen := 30
 	reg := regexp.MustCompile(`[^a-zA-Z0-9]+`)
 
 	chunks := strings.Split(strings.ToLower(strings.Trim(reg.ReplaceAllString(v.title, "-"), "-")), "-")
@@ -209,16 +209,15 @@ func (v YoutubeVideo) Sync(daemon *jsonrpc.Client, claimAddress string, amount f
 	log.Debugln("Created thumbnail for " + v.id)
 
 	err = v.publish(daemon, claimAddress, amount, channelName)
-	if err != nil {
-		return errors.Prefix("publish error", err)
-	}
-
-	err = v.delete()
-	if err != nil {
+	//delete the video in all cases
+	if v.delete() != nil {
 		// the video was published anyway so it should be marked as published
 		// for that to happen, no errors should be returned any further than here
 		log.Debugln(errors.Prefix("delete error", err))
 	}
+	if err != nil {
+		return errors.Prefix("publish error", err)
+	}
 
 	return nil
 }

ytsync/ytsync.go

@@ -177,11 +177,8 @@ WaitForDaemonStart:
 	return nil
 }
 func logShutdownError(shutdownErr error) {
-	errMsg := fmt.Sprintf("error shutting down daemon: %v", shutdownErr)
-	log.Errorf(errMsg)
-	util.SendToSlack(errMsg)
-	log.Errorf("WALLET HAS NOT BEEN MOVED TO THE WALLET BACKUP DIR")
-	util.SendToSlack("WALLET HAS NOT BEEN MOVED TO THE WALLET BACKUP DIR")
+	util.SendToSlackError("error shutting down daemon: %v", shutdownErr)
+	util.SendToSlackError("WALLET HAS NOT BEEN MOVED TO THE WALLET BACKUP DIR")
 }
 
 func (s *Sync) doSync() error {
@@ -276,9 +273,7 @@ func (s *Sync) startWorker(workerNum int) {
 						log.Println("Retrying")
 						continue
 					}
-					logMsg = fmt.Sprintf("Video failed after %d retries, skipping. Stack: %s", s.MaxTries, logMsg)
-					log.Printf(logMsg)
-					util.SendToSlack(logMsg)
+					util.SendToSlackInfo("Video failed after %d retries, skipping. Stack: %s", s.MaxTries, logMsg)
 				}
 			}
 			break
@@ -334,6 +329,7 @@ func (s *Sync) enqueueYoutubeVideos() error {
 
 		for _, item := range playlistResponse.Items {
 			// todo: there's thumbnail info here. why did we need lambda???
+			// because when videos are taken down from youtube, the thumb is gone too
 
 			// normally we'd send the video into the channel here, but youtube api doesn't have sorting
 			// so we have to get ALL the videos, then sort them, then send them in
@@ -445,7 +441,7 @@ func (s *Sync) processVideo(v video) (err error) {
 		return nil
 	}
 
-	if v.PlaylistPosition() > 3000 {
+	if v.PlaylistPosition() > 1000 {
 		log.Println(v.ID() + " is old: skipping")
 		return nil
 	}