From daffded4f03d6ba755f2975b02003c17c92a6d59 Mon Sep 17 00:00:00 2001 From: Alex Grintsvayg Date: Fri, 9 Mar 2018 11:47:38 -0500 Subject: [PATCH] recover ytsync panics when processing videos, switch to our errors everywhere --- count.go | 9 +++++---- redisdb/redisdb.go | 7 ++++--- setup.go | 30 +++++++++++++++--------------- sources/ucbVideo.go | 8 ++++---- sources/youtubeVideo.go | 10 +++++----- ytsync.go | 39 +++++++++++++++++++++++++-------------- 6 files changed, 58 insertions(+), 45 deletions(-) diff --git a/count.go b/count.go index 83a5000..7900921 100644 --- a/count.go +++ b/count.go @@ -3,7 +3,8 @@ package ytsync import ( "net/http" - "github.com/go-errors/errors" + "github.com/lbryio/lbry.go/errors" + "google.golang.org/api/googleapi/transport" "google.golang.org/api/youtube/v3" ) @@ -15,16 +16,16 @@ func (s *Sync) CountVideos() (uint64, error) { service, err := youtube.New(client) if err != nil { - return 0, errors.WrapPrefix(err, "error creating YouTube service", 0) + return 0, errors.Prefix("error creating YouTube service", err) } response, err := service.Channels.List("statistics").Id(s.YoutubeChannelID).Do() if err != nil { - return 0, errors.WrapPrefix(err, "error getting channels", 0) + return 0, errors.Prefix("error getting channels", err) } if len(response.Items) < 1 { - return 0, errors.New("youtube channel not found") + return 0, errors.Err("youtube channel not found") } return response.Items[0].Statistics.VideoCount, nil diff --git a/redisdb/redisdb.go b/redisdb/redisdb.go index d9187b6..207d7c2 100644 --- a/redisdb/redisdb.go +++ b/redisdb/redisdb.go @@ -3,8 +3,9 @@ package redisdb import ( "time" + "github.com/lbryio/lbry.go/errors" + "github.com/garyburd/redigo/redis" - "github.com/go-errors/errors" ) const ( @@ -39,7 +40,7 @@ func (r DB) IsPublished(id string) (bool, error) { alreadyPublished, err := redis.String(conn.Do("HGET", redisHashKey, id)) if err != nil && err != redis.ErrNil { - return false, errors.WrapPrefix(err, "redis error", 0) + return false, errors.Prefix("redis error", err) } @@ -56,7 +57,7 @@ func (r DB) SetPublished(id string) error { _, err := redis.Bool(conn.Do("HSET", redisHashKey, id, redisSyncedVal)) if err != nil { - return errors.New("redis error: " + err.Error()) + return errors.Prefix("redis error", err) } return nil } diff --git a/setup.go b/setup.go index 847dc52..fe4de03 100644 --- a/setup.go +++ b/setup.go @@ -4,10 +4,10 @@ import ( "strings" "time" + "github.com/lbryio/lbry.go/errors" "github.com/lbryio/lbry.go/jsonrpc" "github.com/lbryio/lbry.go/lbrycrd" - "github.com/go-errors/errors" "github.com/shopspring/decimal" log "github.com/sirupsen/logrus" ) @@ -22,7 +22,7 @@ func (s *Sync) walletSetup() error { if err != nil { return err } else if balanceResp == nil { - return errors.New("no response") + return errors.Err("no response") } balance := decimal.Decimal(*balanceResp) log.Debugf("Starting balance is %s", balance.String()) @@ -59,11 +59,11 @@ func (s *Sync) walletSetup() error { if err != nil { return err } else if claimAddress == nil { - return errors.New("could not get unused address") + return errors.Err("could not get unused address") } s.claimAddress = string(*claimAddress) if s.claimAddress == "" { - return errors.New("found blank claim address") + return errors.Err("found blank claim address") } err = s.ensureEnoughUTXOs() @@ -79,7 +79,7 @@ func (s *Sync) ensureEnoughUTXOs() error { if err != nil { return err } else if utxolist == nil { - return errors.New("no response") + return errors.Err("no response") } if !allUTXOsConfirmed(utxolist) { @@ -103,7 +103,7 @@ func (s *Sync) ensureEnoughUTXOs() error { if err != nil { return err } else if balance == nil { - return errors.New("no response") + return errors.Err("no response") } log.Println("balance is " + decimal.Decimal(*balance).String()) @@ -114,9 +114,9 @@ func (s *Sync) ensureEnoughUTXOs() error { if err != nil { return err } else if prefillTx == nil { - return errors.New("no response") + return errors.Err("no response") } else if !prefillTx.Complete || !prefillTx.Broadcast { - return errors.New("failed to prefill addresses") + return errors.Err("failed to prefill addresses") } wait := 15 * time.Second @@ -139,7 +139,7 @@ func (s *Sync) waitUntilUTXOsConfirmed() error { if err != nil { return err } else if r == nil { - return errors.New("no response") + return errors.Err("no response") } if allUTXOsConfirmed(r) { @@ -154,14 +154,14 @@ func (s *Sync) waitUntilUTXOsConfirmed() error { func (s *Sync) ensureChannelOwnership() error { if s.LbryChannelName == "" { - return errors.New("no channel name set") + return errors.Err("no channel name set") } channels, err := s.daemon.ChannelListMine() if err != nil { return err } else if channels == nil { - return errors.New("no channel response") + return errors.Err("no channel response") } isChannelMine := false @@ -169,7 +169,7 @@ func (s *Sync) ensureChannelOwnership() error { if channel.Name == s.LbryChannelName { isChannelMine = true } else { - return errors.New("this wallet has multiple channels. maybe something went wrong during setup?") + return errors.Err("this wallet has multiple channels. maybe something went wrong during setup?") } } if isChannelMine { @@ -187,7 +187,7 @@ func (s *Sync) ensureChannelOwnership() error { channelNotFound := channel.Error != nil && strings.Contains(*(channel.Error), "cannot be resolved") if !channelNotFound { if !s.TakeOverExistingChannel { - return errors.New("Channel exists and we don't own it. Pick another channel.") + return errors.Err("Channel exists and we don't own it. Pick another channel.") } log.Println("Channel exists and we don't own it. Outbidding existing claim.") channelBidAmount, _ = channel.Certificate.Amount.Add(decimal.NewFromFloat(channelClaimAmount)).Float64() @@ -197,7 +197,7 @@ func (s *Sync) ensureChannelOwnership() error { if err != nil { return err } else if balanceResp == nil { - return errors.New("no response") + return errors.Err("no response") } balance := decimal.Decimal(*balanceResp) @@ -247,7 +247,7 @@ func (s *Sync) addCredits(amountToAdd float64) error { if err != nil { return err } else if addressResp == nil { - return errors.New("no response") + return errors.Err("no response") } address := string(*addressResp) diff --git a/sources/ucbVideo.go b/sources/ucbVideo.go index 366d39c..23adfb8 100644 --- a/sources/ucbVideo.go +++ b/sources/ucbVideo.go @@ -11,11 +11,11 @@ import ( "github.com/aws/aws-sdk-go/aws/credentials" "github.com/aws/aws-sdk-go/service/s3" "github.com/aws/aws-sdk-go/service/s3/s3manager" + "github.com/lbryio/lbry.go/errors" "github.com/lbryio/lbry.go/jsonrpc" "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/session" - "github.com/go-errors/errors" log "github.com/sirupsen/logrus" ) @@ -136,7 +136,7 @@ func (v ucbVideo) download() error { if err != nil { return err } else if bytesWritten == 0 { - return errors.New("zero bytes written") + return errors.Err("zero bytes written") } return nil @@ -188,7 +188,7 @@ func (v ucbVideo) Sync(daemon *jsonrpc.Client, claimAddress string, amount float //download and thumbnail can be done in parallel err := v.download() if err != nil { - return errors.WrapPrefix(err, "download error", 0) + return errors.Prefix("download error", err) } log.Debugln("Downloaded " + v.id) @@ -200,7 +200,7 @@ func (v ucbVideo) Sync(daemon *jsonrpc.Client, claimAddress string, amount float err = v.publish(daemon, claimAddress, amount, channelName) if err != nil { - return errors.WrapPrefix(err, "publish error", 0) + return errors.Prefix("publish error", err) } return nil diff --git a/sources/youtubeVideo.go b/sources/youtubeVideo.go index b7240e8..b70a510 100644 --- a/sources/youtubeVideo.go +++ b/sources/youtubeVideo.go @@ -11,9 +11,9 @@ import ( "strings" "time" + "github.com/lbryio/lbry.go/errors" "github.com/lbryio/lbry.go/jsonrpc" - "github.com/go-errors/errors" ytdl "github.com/kkdai/youtube" log "github.com/sirupsen/logrus" "google.golang.org/api/youtube/v3" @@ -150,7 +150,7 @@ func (v YoutubeVideo) triggerThumbnailSave() error { } if decoded.error != 0 { - return errors.New("error creating thumbnail: " + decoded.message) + return errors.Err("error creating thumbnail: " + decoded.message) } return nil @@ -179,19 +179,19 @@ func (v YoutubeVideo) Sync(daemon *jsonrpc.Client, claimAddress string, amount f //download and thumbnail can be done in parallel err := v.download() if err != nil { - return errors.WrapPrefix(err, "download error", 0) + return errors.Prefix("download error", err) } log.Debugln("Downloaded " + v.id) err = v.triggerThumbnailSave() if err != nil { - return errors.WrapPrefix(err, "thumbnail error", 0) + return errors.Prefix("thumbnail error", err) } log.Debugln("Created thumbnail for " + v.id) err = v.publish(daemon, claimAddress, amount, channelName) if err != nil { - return errors.WrapPrefix(err, "publish error", 0) + return errors.Prefix("publish error", err) } return nil diff --git a/ytsync.go b/ytsync.go index d9f4d28..d56d04a 100644 --- a/ytsync.go +++ b/ytsync.go @@ -16,12 +16,12 @@ import ( "syscall" "time" + "github.com/lbryio/lbry.go/errors" "github.com/lbryio/lbry.go/jsonrpc" "github.com/lbryio/lbry.go/stopOnce" "github.com/lbryio/lbry.go/ytsync/redisdb" "github.com/lbryio/lbry.go/ytsync/sources" - "github.com/go-errors/errors" log "github.com/sirupsen/logrus" "google.golang.org/api/googleapi/transport" "google.golang.org/api/youtube/v3" @@ -70,7 +70,7 @@ type Sync struct { func (s *Sync) FullCycle() error { var err error if os.Getenv("HOME") == "" { - return errors.New("no $HOME env var found") + return errors.Err("no $HOME env var found") } if s.YoutubeChannelID == "" { @@ -85,7 +85,7 @@ func (s *Sync) FullCycle() error { walletBackupDir := os.Getenv("HOME") + "/wallets/" + strings.Replace(s.LbryChannelName, "@", "", 1) if _, err := os.Stat(defaultWalletDir); !os.IsNotExist(err) { - return errors.New("default_wallet already exists") + return errors.Err("default_wallet already exists") } if _, err = os.Stat(walletBackupDir); !os.IsNotExist(err) { @@ -255,25 +255,25 @@ func (s *Sync) enqueueYoutubeVideos() error { service, err := youtube.New(client) if err != nil { - return errors.WrapPrefix(err, "error creating YouTube service", 0) + return errors.Prefix("error creating YouTube service", err) } response, err := service.Channels.List("contentDetails").Id(s.YoutubeChannelID).Do() if err != nil { - return errors.WrapPrefix(err, "error getting channels", 0) + return errors.Prefix("error getting channels", err) } if len(response.Items) < 1 { - return errors.New("youtube channel not found") + return errors.Err("youtube channel not found") } if response.Items[0].ContentDetails.RelatedPlaylists == nil { - return errors.New("no related playlists") + return errors.Err("no related playlists") } playlistID := response.Items[0].ContentDetails.RelatedPlaylists.Uploads if playlistID == "" { - return errors.New("no channel playlist") + return errors.Err("no channel playlist") } var videos []video @@ -287,11 +287,11 @@ func (s *Sync) enqueueYoutubeVideos() error { playlistResponse, err := req.Do() if err != nil { - return errors.WrapPrefix(err, "error getting playlist items", 0) + return errors.Prefix("error getting playlist items", err) } if len(playlistResponse.Items) < 1 { - return errors.New("playlist items not found") + return errors.Err("playlist items not found") } for _, item := range playlistResponse.Items { @@ -380,7 +380,18 @@ Enqueue: return nil } -func (s *Sync) processVideo(v video) error { +func (s *Sync) processVideo(v video) (err error) { + defer func() { + if p := recover(); p != nil { + var ok bool + err, ok = p.(error) + if !ok { + err = errors.Err("%v", p) + } + err = errors.Wrap(p, 2) + } + }() + log.Println("Processing " + v.IDAndNum()) defer func(start time.Time) { log.Println(v.ID() + " took " + time.Since(start).String()) @@ -412,7 +423,7 @@ func (s *Sync) processVideo(v video) error { func startDaemonViaSystemd() error { err := exec.Command("/usr/bin/sudo", "/bin/systemctl", "start", "lbrynet.service").Run() if err != nil { - return errors.New(err) + return errors.Err(err) } return nil } @@ -420,7 +431,7 @@ func startDaemonViaSystemd() error { func stopDaemonViaSystemd() error { err := exec.Command("/usr/bin/sudo", "/bin/systemctl", "stop", "lbrynet.service").Run() if err != nil { - return errors.New(err) + return errors.Err(err) } return nil } @@ -439,7 +450,7 @@ func getChannelIDFromFile(channelName string) (string, error) { channelID, ok := channels[channelName] if !ok { - return "", errors.New("channel not in list") + return "", errors.Err("channel not in list") } return channelID, nil