LBRYCommunity/lbry.go
1
0
Fork 0
Code Issues 5 Pull requests Packages Projects Releases 46 Wiki Activity

added limit to error retrying, excluded common unfixable errors

Browse source
This commit is contained in:
Alex Grintsvayg 2017-10-10 20:53:11 -04:00
parent 4dc52a2a94
commit bb0583dfb9
No known key found for this signature in database
GPG key ID: AEB3F089F86A22B5
1 changed files with 25 additions and 11 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

36
cmd/ytsync.go
View file

@ -32,8 +32,8 @@ func init() {
Short: "Publish youtube channel into LBRY network.", Short: "Publish youtube channel into LBRY network.",
Run: ytsync, Run: ytsync,
} }
ytSyncCmd.Flags().BoolVar(&stopOnError, "stop-on-error", false, "If a video fails, stop publishing") ytSyncCmd.Flags().BoolVar(&stopOnError, "stop-on-error", false, "If a publish fails, stop all publishing and exit")
ytSyncCmd.Flags().BoolVar(&retryErrors, "retry-errors", false, "Retry failed publishes") ytSyncCmd.Flags().IntVar(&maxTries, "max-tries", defaultMaxTries, "Number of times to try a publish that fails")
RootCmd.AddCommand(ytSyncCmd) RootCmd.AddCommand(ytSyncCmd)
} }
@ -41,6 +41,7 @@ const (
concurrentVideos = 1 concurrentVideos = 1
redisHashKey = "ytsync" redisHashKey = "ytsync"
redisSyncedVal = "t" redisSyncedVal = "t"
defaultMaxTries = 1
) )
type video struct { type video struct {
@ -75,7 +76,7 @@ var (
channelID string channelID string
lbryChannelName string lbryChannelName string
stopOnError bool stopOnError bool
retryErrors bool maxTries int
daemon *jsonrpc.Client daemon *jsonrpc.Client
claimAddress string claimAddress string
@ -92,8 +93,12 @@ func ytsync(cmd *cobra.Command, args []string) {
lbryChannelName = args[2] lbryChannelName = args[2]
} }
if stopOnError && retryErrors { if stopOnError && maxTries != defaultMaxTries {
log.Errorln("--stop-on-error and --retry-errors are mutually exclusive") log.Errorln("--stop-on-error and --max-tries are mutually exclusive")
return
}
if maxTries < 1 {
log.Errorln("setting --max-tries less than 1 doesn't make sense")
return return
} }
@ -161,8 +166,11 @@ func ytsync(cmd *cobra.Command, args []string) {
return return
} }
tryCount := 0
for { for {
tryCount++
err := processVideo(v) err := processVideo(v)
if err != nil { if err != nil {
log.Errorln("error processing video: " + err.Error()) log.Errorln("error processing video: " + err.Error())
if stopOnError { if stopOnError {
@ -170,14 +178,19 @@ func ytsync(cmd *cobra.Command, args []string) {
sendStopEnqueuing.Do(func() { sendStopEnqueuing.Do(func() {
stopEnqueuing <- struct{}{} stopEnqueuing <- struct{}{}
}) })
} } else if maxTries != defaultMaxTries {
} if strings.Contains(err.Error(), "non 200 status code received") ||
if err != nil && retryErrors { strings.Contains(err.Error(), " reason: 'This video contains content from") {
log.Println("Retrying") log.Println("This error should not be retried at all")
} else if tryCount >= maxTries {
log.Println("Video failed after " + strconv.Itoa(maxTries) + " retries, moving on")
} else { } else {
break log.Println("Retrying")
continue
} }
}
}
break
} }
} }
}() }()
@ -319,6 +332,7 @@ func enqueueVideosFromChannel(channelID string, videoChan *chan video, stopEnque
} }
func processVideo(v video) error { func processVideo(v video) error {
log.Println("========================================")
log.Println("Processing " + v.id + " (" + strconv.Itoa(int(v.playlistPosition)) + " in channel)") log.Println("Processing " + v.id + " (" + strconv.Itoa(int(v.playlistPosition)) + " in channel)")
conn := redisPool.Get() conn := redisPool.Get()