LBRYCommunity/ytsync
1
0
Fork 0
Code Issues 39 Pull requests 8 Projects Releases 197 Packages Wiki Activity

fix update not picking up fresh accounts

Browse source 
fix huge videos clogging up the queue
This commit is contained in:
Niko Storni 2018-10-11 17:21:05 -04:00
parent ea06ed54a6
commit a7b2bc816d
No known key found for this signature in database
GPG key ID: F37FE63398800368
4 changed files with 8 additions and 2 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
main.go
View file

@ -57,7 +57,7 @@ func main() {
cmd.Flags().StringVar(&syncStatus, "status", "", "Specify which queue to pull from. Overrides --update")
cmd.Flags().StringVar(&channelID, "channelID", "", "If specified, only this channel will be synced.")
cmd.Flags().Int64Var(&syncFrom, "after", time.Unix(0, 0).Unix(), "Specify from when to pull jobs [Unix time](Default: 0)")
cmd.Flags().Int64Var(&syncUntil, "before", time.Now().Unix(), "Specify until when to pull jobs [Unix time](Default: current Unix time)")
cmd.Flags().Int64Var(&syncUntil, "before", time.Now().AddDate(1, 0, 0).Unix(), "Specify until when to pull jobs [Unix time](Default: current Unix time)")
cmd.Flags().IntVar(&concurrentJobs, "concurrent-jobs", 1, "how many jobs to process concurrently")
cmd.Flags().IntVar(&videosLimit, "videos-limit", 1000, "how many videos to process per channel")
cmd.Flags().IntVar(&maxVideoSize, "max-size", 2048, "Maximum video size to process (in MB)")

2
sdk/api.go
View file

@ -14,7 +14,7 @@ import (
)
const (
MaxReasonLength = 500
MaxReasonLength = 490
)
type APIConfig struct {

4
sources/youtubeVideo.go
View file

@ -142,6 +142,10 @@ func (v *YoutubeVideo) download() error {
return errors.Err("no compatible format available for this video")
}
maxRetryAttempts := 5
if videoInfo.Duration.Hours() > 2 {
return errors.Err("video is too long to process")
}
for i := 0; i < len(formats) && i < maxRetryAttempts; i++ {
formatIndex := i
if i == maxRetryAttempts-1 {

2
ytsync.go
View file

@ -564,6 +564,7 @@ func (s *Sync) startWorker(workerNum int) {
"Unable to extract signature tokens",
"Client.Timeout exceeded while awaiting headers)",
"the video is too big to sync, skipping for now",
"video is too long to process",
}
if util.SubstringInSlice(err.Error(), errorsNoRetry) {
log.Println("This error should not be retried at all")
@ -760,6 +761,7 @@ func (s *Sync) processVideo(v video) (err error) {
"Error extracting sts from embedded url response",
"Unable to extract signature tokens",
"the video is too big to sync, skipping for now",
"video is too long to process",
}
if ok && !sv.Published && util.SubstringInSlice(sv.FailureReason, neverRetryFailures) {
log.Println(v.ID() + " can't ever be published")