use durations

This commit is contained in:
Alex Grintsvayg 2020-07-27 19:35:07 -04:00
parent 8e61cde0a0
commit 70ad891dfa
No known key found for this signature in database
GPG key ID: AEB3F089F86A22B5
4 changed files with 16 additions and 15 deletions

View file

@ -34,7 +34,7 @@ var (
concurrentJobs int concurrentJobs int
videosLimit int videosLimit int
maxVideoSize int maxVideoSize int
maxVideoLength float64 maxVideoLength int
) )
func main() { func main() {
@ -69,7 +69,7 @@ func main() {
cmd.Flags().IntVar(&concurrentJobs, "concurrent-jobs", 1, "how many jobs to process concurrently") cmd.Flags().IntVar(&concurrentJobs, "concurrent-jobs", 1, "how many jobs to process concurrently")
cmd.Flags().IntVar(&videosLimit, "videos-limit", 1000, "how many videos to process per channel") cmd.Flags().IntVar(&videosLimit, "videos-limit", 1000, "how many videos to process per channel")
cmd.Flags().IntVar(&maxVideoSize, "max-size", 2048, "Maximum video size to process (in MB)") cmd.Flags().IntVar(&maxVideoSize, "max-size", 2048, "Maximum video size to process (in MB)")
cmd.Flags().Float64Var(&maxVideoLength, "max-length", 2.0, "Maximum video length to process (in hours)") cmd.Flags().IntVar(&maxVideoLength, "max-length", 2, "Maximum video length to process (in hours)")
if err := cmd.Execute(); err != nil { if err := cmd.Execute(); err != nil {
fmt.Println(err) fmt.Println(err)
@ -186,7 +186,7 @@ func ytSync(cmd *cobra.Command, args []string) {
syncStatus, syncStatus,
syncProperties, syncProperties,
apiConfig, apiConfig,
maxVideoLength, time.Duration(maxVideoLength)*time.Hour,
) )
err := sm.Start() err := sm.Start()
if err != nil { if err != nil {

View file

@ -30,7 +30,7 @@ type SyncManager struct {
blobsDir string blobsDir string
videosLimit int videosLimit int
maxVideoSize int maxVideoSize int
maxVideoLength float64 maxVideoLength time.Duration
lbrycrdString string lbrycrdString string
awsS3ID string awsS3ID string
awsS3Secret string awsS3Secret string
@ -43,7 +43,7 @@ type SyncManager struct {
func NewSyncManager(syncFlags sdk.SyncFlags, maxTries int, refill int, limit int, concurrentJobs int, concurrentVideos int, blobsDir string, videosLimit int, func NewSyncManager(syncFlags sdk.SyncFlags, maxTries int, refill int, limit int, concurrentJobs int, concurrentVideos int, blobsDir string, videosLimit int,
maxVideoSize int, lbrycrdString string, awsS3ID string, awsS3Secret string, awsS3Region string, awsS3Bucket string, maxVideoSize int, lbrycrdString string, awsS3ID string, awsS3Secret string, awsS3Region string, awsS3Bucket string,
syncStatus string, syncProperties *sdk.SyncProperties, apiConfig *sdk.APIConfig, maxVideoLength float64) *SyncManager { syncStatus string, syncProperties *sdk.SyncProperties, apiConfig *sdk.APIConfig, maxVideoLength time.Duration) *SyncManager {
return &SyncManager{ return &SyncManager{
SyncFlags: syncFlags, SyncFlags: syncFlags,
maxTries: maxTries, maxTries: maxTries,
@ -171,7 +171,7 @@ func (s *SyncManager) Start() error {
log.Infof("There are %d channels in the \"%s\" queue", len(channels)-i, q) log.Infof("There are %d channels in the \"%s\" queue", len(channels)-i, q)
maxVideoLength := s.maxVideoLength maxVideoLength := s.maxVideoLength
if c.TotalSubscribers < 1000 { if c.TotalSubscribers < 1000 {
maxVideoLength = 1.0 maxVideoLength = 1 * time.Hour
} }
syncs = append(syncs, Sync{ syncs = append(syncs, Sync{
APIConfig: s.apiConfig, APIConfig: s.apiConfig,

View file

@ -73,7 +73,7 @@ type Sync struct {
clientPublishAddress string clientPublishAddress string
publicKey string publicKey string
defaultAccountID string defaultAccountID string
MaxVideoLength float64 MaxVideoLength time.Duration
} }
func (s *Sync) AppendSyncedVideo(videoID string, published bool, failureReason string, claimName string, claimID string, metadataVersion int8, size int64) { func (s *Sync) AppendSyncedVideo(videoID string, published bool, failureReason string, claimName string, claimID string, metadataVersion int8, size int64) {
@ -224,7 +224,7 @@ func (s *Sync) setStatusSyncing() error {
func (s *Sync) setExceptions() { func (s *Sync) setExceptions() {
if s.YoutubeChannelID == "UCwjQfNRW6sGYb__pd7d4nUg" { //@FreeTalkLive if s.YoutubeChannelID == "UCwjQfNRW6sGYb__pd7d4nUg" { //@FreeTalkLive
s.MaxVideoLength = 9999.0 // skips max length checks s.MaxVideoLength = 9999 * time.Hour // skips max length checks
s.Manager.maxVideoSize = 0 s.Manager.maxVideoSize = 0
} }
} }

View file

@ -3,7 +3,6 @@ package sources
import ( import (
"fmt" "fmt"
"io/ioutil" "io/ioutil"
"math"
"os" "os"
"os/exec" "os/exec"
"path/filepath" "path/filepath"
@ -40,7 +39,7 @@ type YoutubeVideo struct {
playlistPosition int64 playlistPosition int64
size *int64 size *int64
maxVideoSize int64 maxVideoSize int64
maxVideoLength float64 maxVideoLength time.Duration
publishedAt time.Time publishedAt time.Time
dir string dir string
youtubeInfo *ytdl.YtdlVideo youtubeInfo *ytdl.YtdlVideo
@ -236,7 +235,7 @@ func (v *YoutubeVideo) download() error {
if v.maxVideoLength > 0 { if v.maxVideoLength > 0 {
ytdlArgs = append(ytdlArgs, ytdlArgs = append(ytdlArgs,
"--match-filter", "--match-filter",
fmt.Sprintf("duration <= %d", int(math.Round(v.maxVideoLength*3600))), fmt.Sprintf("duration <= %d", int(v.maxVideoLength.Seconds())),
) )
} }
@ -432,7 +431,7 @@ type SyncParams struct {
ChannelID string ChannelID string
MaxVideoSize int MaxVideoSize int
Namer *namer.Namer Namer *namer.Namer
MaxVideoLength float64 MaxVideoLength time.Duration
Fee *sdk.Fee Fee *sdk.Fee
DefaultAccount string DefaultAccount string
} }
@ -452,9 +451,11 @@ func (v *YoutubeVideo) Sync(daemon *jsonrpc.Client, params SyncParams, existingV
func (v *YoutubeVideo) downloadAndPublish(daemon *jsonrpc.Client, params SyncParams) (*SyncSummary, error) { func (v *YoutubeVideo) downloadAndPublish(daemon *jsonrpc.Client, params SyncParams) (*SyncSummary, error) {
var err error var err error
if float64(v.youtubeInfo.Duration) > v.maxVideoLength { dur := time.Duration(v.youtubeInfo.Duration) * time.Second
log.Infof("%s is %d long and the limit is %s", v.id, v.youtubeInfo.Duration, (time.Duration(v.maxVideoLength*60) * time.Minute).String())
logUtils.SendErrorToSlack("%s is %d long and the limit is %s", v.id, v.youtubeInfo.Duration, (time.Duration(v.maxVideoLength*60) * time.Minute).String()) if dur > v.maxVideoLength {
log.Infof("%s is %d long and the limit is %s", v.id, dur.String(), v.maxVideoLength.String())
logUtils.SendErrorToSlack("%s is %d long and the limit is %s", v.id, dur.String(), v.maxVideoLength.String())
return nil, errors.Err("video is too long to process") return nil, errors.Err("video is too long to process")
} }
for { for {