From 7a3d2bf4bc18963d03a31a407e9b2443c0a8d4f7 Mon Sep 17 00:00:00 2001 From: Niko Storni Date: Wed, 1 Aug 2018 08:56:04 -0400 Subject: [PATCH] add more startup parameters --- cmd/ytsync.go | 10 ++++++++-- ytsync/manager.go | 2 ++ ytsync/setup.go | 4 ++-- ytsync/sources/ucbVideo.go | 2 +- ytsync/sources/youtubeVideo.go | 6 +++--- ytsync/ytsync.go | 13 ++++++------- 6 files changed, 22 insertions(+), 15 deletions(-) diff --git a/cmd/ytsync.go b/cmd/ytsync.go index 05c2947..d5284bb 100644 --- a/cmd/ytsync.go +++ b/cmd/ytsync.go @@ -28,6 +28,8 @@ var ( syncFrom int64 syncUntil int64 concurrentJobs int + videosLimit int + maxVideoSize int ) func init() { @@ -42,12 +44,14 @@ func init() { ytSyncCmd.Flags().BoolVar(&takeOverExistingChannel, "takeover-existing-channel", false, "If channel exists and we don't own it, take over the channel") ytSyncCmd.Flags().IntVar(&limit, "limit", 0, "limit the amount of channels to sync") ytSyncCmd.Flags().BoolVar(&skipSpaceCheck, "skip-space-check", false, "Do not perform free space check on startup") - ytSyncCmd.Flags().BoolVar(&syncUpdate, "update", false, "Update previously synced channels instead of syncing new ones (short for --status synced)") + ytSyncCmd.Flags().BoolVar(&syncUpdate, "update", false, "Update previously synced channels instead of syncing new ones") ytSyncCmd.Flags().StringVar(&syncStatus, "status", "", "Specify which queue to pull from. Overrides --update") ytSyncCmd.Flags().StringVar(&channelID, "channelID", "", "If specified, only this channel will be synced.") ytSyncCmd.Flags().Int64Var(&syncFrom, "after", time.Unix(0, 0).Unix(), "Specify from when to pull jobs [Unix time](Default: 0)") ytSyncCmd.Flags().Int64Var(&syncUntil, "before", time.Now().Unix(), "Specify until when to pull jobs [Unix time](Default: current Unix time)") - ytSyncCmd.Flags().IntVar(&concurrentJobs, "concurrent-jobs", 1, "how many jobs to process concurrently (Default: 1)") + ytSyncCmd.Flags().IntVar(&concurrentJobs, "concurrent-jobs", 1, "how many jobs to process concurrently") + ytSyncCmd.Flags().IntVar(&videosLimit, "videos-limit", 1000, "how many videos to process per channel") + ytSyncCmd.Flags().IntVar(&maxVideoSize, "max-size", 2048, "Maximum video size to process (in MB)") RootCmd.AddCommand(ytSyncCmd) } @@ -130,6 +134,8 @@ func ytSync(cmd *cobra.Command, args []string) { ApiURL: apiURL, ApiToken: apiToken, BlobsDir: blobsDir, + VideosLimit: videosLimit, + MaxVideoSize: maxVideoSize, } err := sm.Start() diff --git a/ytsync/manager.go b/ytsync/manager.go index a9bc274..d64958c 100644 --- a/ytsync/manager.go +++ b/ytsync/manager.go @@ -37,6 +37,8 @@ type SyncManager struct { ApiURL string ApiToken string BlobsDir string + VideosLimit int + MaxVideoSize int } const ( diff --git a/ytsync/setup.go b/ytsync/setup.go index 8ff85af..0046d8a 100644 --- a/ytsync/setup.go +++ b/ytsync/setup.go @@ -50,8 +50,8 @@ func (s *Sync) walletSetup() error { } log.Debugf("We already published %d videos", numPublished) - if float64(numOnSource)-float64(numPublished) > maximumVideosToPublish { - numOnSource = maximumVideosToPublish + if float64(numOnSource)-float64(numPublished) > float64(s.Manager.VideosLimit) { + numOnSource = uint64(s.Manager.VideosLimit) } minBalance := (float64(numOnSource)-float64(numPublished))*(publishAmount+0.1) + channelClaimAmount if numPublished > numOnSource { diff --git a/ytsync/sources/ucbVideo.go b/ytsync/sources/ucbVideo.go index ea220a5..514b676 100644 --- a/ytsync/sources/ucbVideo.go +++ b/ytsync/sources/ucbVideo.go @@ -188,7 +188,7 @@ func (v ucbVideo) publish(daemon *jsonrpc.Client, claimAddress string, amount fl return publishAndRetryExistingNames(daemon, v.title, v.getFilename(), amount, options) } -func (v ucbVideo) Sync(daemon *jsonrpc.Client, claimAddress string, amount float64, channelName string) (*SyncSummary, error) { +func (v ucbVideo) Sync(daemon *jsonrpc.Client, claimAddress string, amount float64, channelName string, maxVideoSize int) (*SyncSummary, error) { //download and thumbnail can be done in parallel err := v.download() if err != nil { diff --git a/ytsync/sources/youtubeVideo.go b/ytsync/sources/youtubeVideo.go index 9e28f14..0f24419 100644 --- a/ytsync/sources/youtubeVideo.go +++ b/ytsync/sources/youtubeVideo.go @@ -203,7 +203,7 @@ func (v YoutubeVideo) publish(daemon *jsonrpc.Client, claimAddress string, amoun return publishAndRetryExistingNames(daemon, v.title, v.getFilename(), amount, options) } -func (v YoutubeVideo) Sync(daemon *jsonrpc.Client, claimAddress string, amount float64, channelName string) (*SyncSummary, error) { +func (v YoutubeVideo) Sync(daemon *jsonrpc.Client, claimAddress string, amount float64, channelName string, maxVideoSize int) (*SyncSummary, error) { //download and thumbnail can be done in parallel err := v.download() if err != nil { @@ -215,10 +215,10 @@ func (v YoutubeVideo) Sync(daemon *jsonrpc.Client, claimAddress string, amount f if err != nil { return nil, err } - if fi.Size() > 2*1024*1024*1024 { + if fi.Size() > int64(maxVideoSize)*1024*1024 { //delete the video and ignore the error _ = v.delete() - return nil, errors.Err("video is bigger than 2GB, skipping for now") + return nil, errors.Err("the video is too big to sync, skipping for now") } err = v.triggerThumbnailSave() diff --git a/ytsync/ytsync.go b/ytsync/ytsync.go index 4ad2d14..af08d0e 100644 --- a/ytsync/ytsync.go +++ b/ytsync/ytsync.go @@ -30,9 +30,8 @@ import ( ) const ( - channelClaimAmount = 0.01 - publishAmount = 0.01 - maximumVideosToPublish = 1000 + channelClaimAmount = 0.01 + publishAmount = 0.01 ) type video interface { @@ -40,7 +39,7 @@ type video interface { IDAndNum() string PlaylistPosition() int PublishedAt() time.Time - Sync(*jsonrpc.Client, string, float64, string) (*sources.SyncSummary, error) + Sync(*jsonrpc.Client, string, float64, string, int) (*sources.SyncSummary, error) } // sorting videos @@ -315,7 +314,7 @@ func (s *Sync) startWorker(workerNum int) { "Error in daemon: Cannot publish empty file", "Error extracting sts from embedded url response", "Client.Timeout exceeded while awaiting headers)", - "video is bigger than 2GB, skipping for now", + "the video is too big to sync, skipping for now", } if util.SubstringInSlice(err.Error(), errorsNoRetry) { log.Println("This error should not be retried at all") @@ -504,11 +503,11 @@ func (s *Sync) processVideo(v video) (err error) { return nil } - if v.PlaylistPosition() > maximumVideosToPublish { + if v.PlaylistPosition() > s.Manager.VideosLimit { log.Println(v.ID() + " is old: skipping") return nil } - summary, err := v.Sync(s.daemon, s.claimAddress, publishAmount, s.LbryChannelName) + summary, err := v.Sync(s.daemon, s.claimAddress, publishAmount, s.LbryChannelName, s.Manager.MaxVideoSize) if err != nil { return err }