remove unused flag
always go through syncing first don't sync videos shorter than 7 seconds refactor code in video error handling add interface to handle hard video failures (incomplete)
This commit is contained in:
Niko Storni
parent
beade71aa6
commit
fecf67118c
5 changed files with 108 additions and 95 deletions
5
main.go
5
main.go
|
|
@ -42,7 +42,6 @@ func main() {
|
|||
Args: cobra.RangeArgs(0, 0),
|
||||
}
|
||||
|
||||
cmd.Flags().BoolVar(&cliFlags.StopOnError, "stop-on-error", false, "If a publish fails, stop all publishing and exit")
|
||||
cmd.Flags().IntVar(&cliFlags.MaxTries, "max-tries", defaultMaxTries, "Number of times to try a publish that fails")
|
||||
cmd.Flags().BoolVar(&cliFlags.TakeOverExistingChannel, "takeover-existing-channel", false, "If channel exists and we don't own it, take over the channel")
|
||||
cmd.Flags().IntVar(&cliFlags.Limit, "limit", 0, "limit the amount of channels to sync")
|
||||
|
|
@ -93,10 +92,6 @@ func ytSync(cmd *cobra.Command, args []string) {
|
|||
return
|
||||
}
|
||||
|
||||
if cliFlags.StopOnError && cliFlags.MaxTries != defaultMaxTries {
|
||||
log.Errorln("--stop-on-error and --max-tries are mutually exclusive")
|
||||
return
|
||||
}
|
||||
if cliFlags.MaxTries < 1 {
|
||||
log.Errorln("setting --max-tries less than 1 doesn't make sense")
|
||||
return
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@ package manager
|
|||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
"sync"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
|
|
@ -43,6 +44,9 @@ func (s *SyncManager) enqueueChannel(channel *shared.YoutubeChannel) {
|
|||
DbChannelData: channel,
|
||||
Manager: s,
|
||||
namer: namer.NewNamer(),
|
||||
hardVideoFailure: hardVideoFailure{
|
||||
lock: &sync.Mutex{},
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
|
|
@ -78,7 +82,7 @@ func (s *SyncManager) Start() error {
|
|||
} else {
|
||||
var queuesToSync []string
|
||||
if s.CliFlags.Status != "" {
|
||||
queuesToSync = append(queuesToSync, s.CliFlags.Status)
|
||||
queuesToSync = append(queuesToSync, shared.StatusSyncing, s.CliFlags.Status)
|
||||
} else if s.CliFlags.SyncUpdate {
|
||||
queuesToSync = append(queuesToSync, shared.StatusSyncing, shared.StatusSynced)
|
||||
} else {
|
||||
|
|
|
|||
|
|
@ -53,6 +53,23 @@ type Sync struct {
|
|||
walletMux *sync.RWMutex
|
||||
queue chan ytapi.Video
|
||||
defaultAccountID string
|
||||
hardVideoFailure hardVideoFailure
|
||||
}
|
||||
|
||||
type hardVideoFailure struct {
|
||||
lock *sync.Mutex
|
||||
failed bool
|
||||
failureReason string
|
||||
}
|
||||
|
||||
func (hv *hardVideoFailure) flagFailure(reason string) {
|
||||
hv.lock.Lock()
|
||||
defer hv.lock.Unlock()
|
||||
if hv.failed {
|
||||
return
|
||||
}
|
||||
hv.failed = true
|
||||
hv.failureReason = reason
|
||||
}
|
||||
|
||||
func (s *Sync) AppendSyncedVideo(videoID string, published bool, failureReason string, claimName string, claimID string, metadataVersion int8, size int64) {
|
||||
|
|
@ -690,10 +707,6 @@ func (s *Sync) doSync() error {
|
|||
}
|
||||
}
|
||||
|
||||
if s.Manager.CliFlags.StopOnError {
|
||||
log.Println("Will stop publishing if an error is detected")
|
||||
}
|
||||
|
||||
for i := 0; i < s.Manager.CliFlags.ConcurrentJobs; i++ {
|
||||
s.grp.Add(1)
|
||||
go func(i int) {
|
||||
|
|
@ -712,26 +725,74 @@ func (s *Sync) doSync() error {
|
|||
return err
|
||||
}
|
||||
|
||||
func (s *Sync) startWorker(workerNum int) {
|
||||
func (s *Sync) startWorker(workerNum int) error {
|
||||
var v ytapi.Video
|
||||
var more bool
|
||||
|
||||
fatalErrors := []string{
|
||||
":5279: read: connection reset by peer",
|
||||
"no space left on device",
|
||||
"NotEnoughFunds",
|
||||
"Cannot publish using channel",
|
||||
"cannot concatenate 'str' and 'NoneType' objects",
|
||||
"more than 90% of the space has been used.",
|
||||
"Couldn't find private key for id",
|
||||
"You already have a stream claim published under the name",
|
||||
"Missing inputs",
|
||||
}
|
||||
errorsNoRetry := []string{
|
||||
"non 200 status code received",
|
||||
"This video contains content from",
|
||||
"dont know which claim to update",
|
||||
"uploader has not made this video available in your country",
|
||||
"download error: AccessDenied: Access Denied",
|
||||
"Playback on other websites has been disabled by the video owner",
|
||||
"Error in daemon: Cannot publish empty file",
|
||||
"Error extracting sts from embedded url response",
|
||||
"Unable to extract signature tokens",
|
||||
"Client.Timeout exceeded while awaiting headers",
|
||||
"the video is too big to sync, skipping for now",
|
||||
"video is too long to process",
|
||||
"video is too short to process",
|
||||
"no compatible format available for this video",
|
||||
"Watch this video on YouTube.",
|
||||
"have blocked it on copyright grounds",
|
||||
"the video must be republished as we can't get the right size",
|
||||
"HTTP Error 403",
|
||||
"giving up after 0 fragment retries",
|
||||
"Sorry about that",
|
||||
"This video is not available",
|
||||
"requested format not available",
|
||||
"interrupted by user",
|
||||
"Sign in to confirm your age",
|
||||
"This video is unavailable",
|
||||
"video is a live stream and hasn't completed yet",
|
||||
}
|
||||
walletErrors := []string{
|
||||
"Not enough funds to cover this transaction",
|
||||
"failed: Not enough funds",
|
||||
"Error in daemon: Insufficient funds, please deposit additional LBC",
|
||||
//"Missing inputs",
|
||||
}
|
||||
blockchainErrors := []string{
|
||||
"txn-mempool-conflict",
|
||||
"too-long-mempool-chain",
|
||||
}
|
||||
for {
|
||||
select {
|
||||
case <-s.grp.Ch():
|
||||
log.Printf("Stopping worker %d", workerNum)
|
||||
return
|
||||
return nil
|
||||
default:
|
||||
}
|
||||
|
||||
select {
|
||||
case v, more = <-s.queue:
|
||||
if !more {
|
||||
return
|
||||
return nil
|
||||
}
|
||||
case <-s.grp.Ch():
|
||||
log.Printf("Stopping worker %d", workerNum)
|
||||
return
|
||||
return nil
|
||||
}
|
||||
|
||||
log.Println("================================================================================")
|
||||
|
|
@ -741,95 +802,43 @@ func (s *Sync) startWorker(workerNum int) {
|
|||
select { // check again inside the loop so this dies faster
|
||||
case <-s.grp.Ch():
|
||||
log.Printf("Stopping worker %d", workerNum)
|
||||
return
|
||||
return nil
|
||||
default:
|
||||
}
|
||||
tryCount++
|
||||
err := s.processVideo(v)
|
||||
|
||||
if err != nil {
|
||||
util.SendToSlack("Tried to process %s. Error: %v", v.ID(), err)
|
||||
logMsg := fmt.Sprintf("error processing video %s: %s", v.ID(), err.Error())
|
||||
log.Errorln(logMsg)
|
||||
logUtils.SendErrorToSlack("error processing video %s: %s", v.ID(), err.Error())
|
||||
shouldRetry := s.Manager.CliFlags.MaxTries > 1 && !util.SubstringInSlice(err.Error(), errorsNoRetry) && tryCount < s.Manager.CliFlags.MaxTries
|
||||
if strings.Contains(strings.ToLower(err.Error()), "interrupted by user") {
|
||||
return
|
||||
}
|
||||
fatalErrors := []string{
|
||||
":5279: read: connection reset by peer",
|
||||
"no space left on device",
|
||||
"NotEnoughFunds",
|
||||
"Cannot publish using channel",
|
||||
"cannot concatenate 'str' and 'NoneType' objects",
|
||||
"more than 90% of the space has been used.",
|
||||
"Couldn't find private key for id",
|
||||
"You already have a stream claim published under the name",
|
||||
"Missing inputs",
|
||||
}
|
||||
if util.SubstringInSlice(err.Error(), fatalErrors) || s.Manager.CliFlags.StopOnError {
|
||||
s.grp.Stop()
|
||||
} else if s.Manager.CliFlags.MaxTries > 1 {
|
||||
errorsNoRetry := []string{
|
||||
"non 200 status code received",
|
||||
"This video contains content from",
|
||||
"dont know which claim to update",
|
||||
"uploader has not made this video available in your country",
|
||||
"download error: AccessDenied: Access Denied",
|
||||
"Playback on other websites has been disabled by the video owner",
|
||||
"Error in daemon: Cannot publish empty file",
|
||||
"Error extracting sts from embedded url response",
|
||||
"Unable to extract signature tokens",
|
||||
"Client.Timeout exceeded while awaiting headers",
|
||||
"the video is too big to sync, skipping for now",
|
||||
"video is too long to process",
|
||||
"no compatible format available for this video",
|
||||
"Watch this video on YouTube.",
|
||||
"have blocked it on copyright grounds",
|
||||
"the video must be republished as we can't get the right size",
|
||||
"HTTP Error 403",
|
||||
"giving up after 0 fragment retries",
|
||||
"Sorry about that",
|
||||
"This video is not available",
|
||||
"requested format not available",
|
||||
"interrupted by user",
|
||||
"Sign in to confirm your age",
|
||||
"This video is unavailable",
|
||||
"video is a live stream and hasn't completed yet",
|
||||
}
|
||||
if util.SubstringInSlice(err.Error(), errorsNoRetry) {
|
||||
log.Println("This error should not be retried at all")
|
||||
} else if tryCount < s.Manager.CliFlags.MaxTries {
|
||||
if util.SubstringInSlice(err.Error(), []string{
|
||||
"txn-mempool-conflict",
|
||||
"too-long-mempool-chain",
|
||||
}) {
|
||||
log.Println("waiting for a block before retrying")
|
||||
err := s.waitForNewBlock()
|
||||
if err != nil {
|
||||
s.grp.Stop()
|
||||
logUtils.SendErrorToSlack("something went wrong while waiting for a block: %s", errors.FullTrace(err))
|
||||
break
|
||||
}
|
||||
} else if util.SubstringInSlice(err.Error(), []string{
|
||||
"Not enough funds to cover this transaction",
|
||||
"failed: Not enough funds",
|
||||
"Error in daemon: Insufficient funds, please deposit additional LBC",
|
||||
//"Missing inputs",
|
||||
}) {
|
||||
log.Println("checking funds and UTXOs before retrying...")
|
||||
err := s.walletSetup()
|
||||
if err != nil {
|
||||
s.grp.Stop()
|
||||
logUtils.SendErrorToSlack("failed to setup the wallet for a refill: %s", errors.FullTrace(err))
|
||||
break
|
||||
}
|
||||
} else if strings.Contains(err.Error(), "Error in daemon: 'str' object has no attribute 'get'") {
|
||||
time.Sleep(5 * time.Second)
|
||||
} else if util.SubstringInSlice(err.Error(), fatalErrors) {
|
||||
s.grp.Stop()
|
||||
} else if shouldRetry {
|
||||
if util.SubstringInSlice(err.Error(), blockchainErrors) {
|
||||
log.Println("waiting for a block before retrying")
|
||||
err := s.waitForNewBlock()
|
||||
if err != nil {
|
||||
s.grp.Stop()
|
||||
logUtils.SendErrorToSlack("something went wrong while waiting for a block: %s", errors.FullTrace(err))
|
||||
break
|
||||
}
|
||||
log.Println("Retrying")
|
||||
continue
|
||||
} else if util.SubstringInSlice(err.Error(), walletErrors) {
|
||||
log.Println("checking funds and UTXOs before retrying...")
|
||||
err := s.walletSetup()
|
||||
if err != nil {
|
||||
s.grp.Stop()
|
||||
logUtils.SendErrorToSlack("failed to setup the wallet for a refill: %s", errors.FullTrace(err))
|
||||
break
|
||||
}
|
||||
} else if strings.Contains(err.Error(), "Error in daemon: 'str' object has no attribute 'get'") {
|
||||
time.Sleep(5 * time.Second)
|
||||
}
|
||||
logUtils.SendErrorToSlack("Video failed after %d retries, skipping. Stack: %s", tryCount, logMsg)
|
||||
log.Println("Retrying")
|
||||
continue
|
||||
}
|
||||
logUtils.SendErrorToSlack("Video %s failed after %d retries, skipping. Stack: %s", tryCount, v.ID(), errors.FullTrace(err))
|
||||
|
||||
s.syncedVideosMux.RLock()
|
||||
existingClaim, ok := s.syncedVideos[v.ID()]
|
||||
s.syncedVideosMux.RUnlock()
|
||||
|
|
|
|||
|
|
@ -33,6 +33,7 @@ var NeverRetryFailures = []string{
|
|||
"Unable to extract signature tokens",
|
||||
"the video is too big to sync, skipping for now",
|
||||
"video is too long to process",
|
||||
"video is too short to process",
|
||||
"This video contains content from",
|
||||
"no compatible format available for this video",
|
||||
"Watch this video on YouTube.",
|
||||
|
|
@ -42,7 +43,6 @@ var NeverRetryFailures = []string{
|
|||
}
|
||||
|
||||
type SyncFlags struct {
|
||||
StopOnError bool
|
||||
TakeOverExistingChannel bool
|
||||
SkipSpaceCheck bool
|
||||
SyncUpdate bool
|
||||
|
|
|
|||
|
|
@ -471,11 +471,16 @@ func (v *YoutubeVideo) downloadAndPublish(daemon *jsonrpc.Client, params SyncPar
|
|||
var err error
|
||||
|
||||
dur := time.Duration(v.youtubeInfo.Duration) * time.Second
|
||||
minDuration := 7 * time.Second
|
||||
|
||||
if dur > v.maxVideoLength {
|
||||
logUtils.SendErrorToSlack("%s is %s long and the limit is %s", v.id, dur.String(), v.maxVideoLength.String())
|
||||
return nil, errors.Err("video is too long to process")
|
||||
}
|
||||
if dur < minDuration {
|
||||
logUtils.SendErrorToSlack("%s is %s long and the minimum is %s", v.id, dur.String(), minDuration.String())
|
||||
return nil, errors.Err("video is too short to process")
|
||||
}
|
||||
for {
|
||||
err = v.download()
|
||||
if err != nil && strings.Contains(err.Error(), "HTTP Error 429") {
|
||||
|
|
|
|||
Loading…
Reference in a new issue