From 6eca7bfb7682430a4cd175526e304ca8040ce32f Mon Sep 17 00:00:00 2001 From: Niko Storni Date: Wed, 19 Sep 2018 19:05:47 -0400 Subject: [PATCH] add dupes patcher and database patcher add claim_abandon function refactor logic --- setup.go | 2 +- ytsync.go | 102 +++++++++++++++++++++++++++++++++++++++--------------- 2 files changed, 76 insertions(+), 28 deletions(-) diff --git a/setup.go b/setup.go index 6d72a03..113bbb5 100644 --- a/setup.go +++ b/setup.go @@ -48,7 +48,7 @@ func (s *Sync) walletSetup() error { s.syncedVideosMux.RLock() numPublished := len(s.syncedVideos) //should we only count published videos? Credits are allocated even for failed ones... s.syncedVideosMux.RUnlock() - log.Debugf("We already published %d videos", numPublished) + log.Debugf("We already allocated credits for %d videos", numPublished) if numOnSource-numPublished > s.Manager.VideosLimit { numOnSource = s.Manager.VideosLimit diff --git a/ytsync.go b/ytsync.go index 17c645e..e726efd 100644 --- a/ytsync.go +++ b/ytsync.go @@ -220,6 +220,18 @@ func (s *Sync) uploadWallet() error { return os.Remove(defaultWalletDir) } +func (s *Sync) setStatusSyncing() error { + syncedVideos, claimNames, err := s.Manager.setChannelStatus(s.YoutubeChannelID, StatusSyncing, "") + if err != nil { + return err + } + s.syncedVideosMux.Lock() + s.syncedVideos = syncedVideos + s.claimNames = claimNames + s.syncedVideosMux.Unlock() + return nil +} + func (s *Sync) FullCycle() (e error) { if os.Getenv("HOME") == "" { return errors.Err("no $HOME env var found") @@ -239,16 +251,12 @@ func (s *Sync) FullCycle() (e error) { log.Println("Got interrupt signal, shutting down (if publishing, will shut down after current publish)") s.grp.Stop() }() - syncedVideos, claimNames, err := s.Manager.setChannelStatus(s.YoutubeChannelID, StatusSyncing, "") + err := s.setStatusSyncing() if err != nil { return err } - s.syncedVideosMux.Lock() - s.syncedVideos = syncedVideos - s.claimNames = claimNames - s.syncedVideosMux.Unlock() - defer s.updateChannelStatus(&e) + defer s.setChannelTerminationStatus(&e) err = s.downloadWallet() if err != nil && err.Error() != "wallet not on S3" { @@ -293,7 +301,7 @@ func (s *Sync) FullCycle() (e error) { return nil } -func (s *Sync) updateChannelStatus(e *error) { +func (s *Sync) setChannelTerminationStatus(e *error) { if *e != nil { //conditions for which a channel shouldn't be marked as failed noFailConditions := []string{ @@ -362,8 +370,10 @@ func logShutdownError(shutdownErr error) { SendErrorToSlack("WALLET HAS NOT BEEN MOVED TO THE WALLET BACKUP DIR") } -func hasDupes(claims []jsonrpc.Claim) (bool, error) { - videoIDs := make(map[string]string) +// fixDupes abandons duplicate claims +func (s *Sync) fixDupes(claims []jsonrpc.Claim) (bool, error) { + abandonedClaims := false + videoIDs := make(map[string]jsonrpc.Claim) for _, c := range claims { if !util.InSlice(c.Category, []string{"claim", "update"}) || c.Value.Stream == nil { continue @@ -375,29 +385,53 @@ func hasDupes(claims []jsonrpc.Claim) (bool, error) { videoID := tn[strings.LastIndex(tn, "/")+1:] log.Infof("claimid: %s, claimName: %s, videoID: %s", c.ClaimID, c.Name, videoID) - cID, ok := videoIDs[videoID] - if !ok || cID == c.ClaimID { - videoIDs[videoID] = c.ClaimID + cl, ok := videoIDs[videoID] + if !ok || cl.ClaimID == c.ClaimID { + videoIDs[videoID] = c continue } - return true, nil + // only keep the most recent one + claimToAbandon := c + videoIDs[videoID] = cl + if c.Height > cl.Height { + claimToAbandon = cl + videoIDs[videoID] = c + } + _, err := s.daemon.ClaimAbandon(claimToAbandon.Txid, claimToAbandon.Nout) + if err != nil { + return true, err + } + log.Debugf("abandoning %+v", claimToAbandon) + abandonedClaims = true + //return true, nil } - return false, nil + return abandonedClaims, nil } -//publishesCount counts the amount of videos published so far -func publishesCount(claims []jsonrpc.Claim) (int, error) { +//updateRemoteDB counts the amount of videos published so far and updates the remote db if some videos weren't marked as published +func (s *Sync) updateRemoteDB(claims []jsonrpc.Claim) (total int, fixed int, err error) { count := 0 for _, c := range claims { if !util.InSlice(c.Category, []string{"claim", "update"}) || c.Value.Stream == nil { continue } if c.Value.Stream.Metadata == nil || c.Value.Stream.Metadata.Thumbnail == nil { - return count, errors.Err("something is wrong with the this claim: %s", c.ClaimID) + return count, fixed, errors.Err("something is wrong with the this claim: %s", c.ClaimID) } - count++ + //check if claimID is in remote db + tn := *c.Value.Stream.Metadata.Thumbnail + videoID := tn[strings.LastIndex(tn, "/")+1:] + pv, ok := s.syncedVideos[videoID] + if !ok || pv.ClaimName != c.Name { + fixed++ + err = s.Manager.MarkVideoStatus(s.YoutubeChannelID, videoID, VideoStatusPublished, c.ClaimID, c.Name, "", nil) + if err != nil { + return total, fixed, err + } + } + total++ } - return count, nil + return total, fixed, nil } func (s *Sync) doSync() error { @@ -406,29 +440,43 @@ func (s *Sync) doSync() error { if err != nil { return errors.Prefix("cannot list claims: ", err) } - hasDupes, err := hasDupes(*claims) + hasDupes, err := s.fixDupes(*claims) if err != nil { return errors.Prefix("error checking for duplicates: ", err) } if hasDupes { - return errors.Err("channel has duplicates! Manual fix required") + SendInfoToSlack("Channel had dupes and was fixed!") + claims, err = s.daemon.ClaimListMine() + if err != nil { + return errors.Prefix("cannot list claims: ", err) + } } - pubsOnWallet, err := publishesCount(*claims) + + pubsOnWallet, nFixed, err := s.updateRemoteDB(*claims) if err != nil { return errors.Prefix("error counting claims: ", err) } + if nFixed > 0 { + err := s.setStatusSyncing() + if err != nil { + return err + } + SendInfoToSlack("%d claims were not on the remote database and were fixed", nFixed) + + } pubsOnDB := 0 for _, sv := range s.syncedVideos { if sv.Published { pubsOnDB++ } } - //if pubsOnWallet > pubsOnDB { - // SendInfoToSlack("We're claiming to have published %d videos but in reality we published %d (%s)", pubsOnDB, pubsOnWallet, s.YoutubeChannelID) - // return errors.Err("not all published videos are in the database") - // } + + if pubsOnWallet > pubsOnDB { //This case should never happen + SendInfoToSlack("We're claiming to have published %d videos but in reality we published %d (%s)", pubsOnDB, pubsOnWallet, s.YoutubeChannelID) + return errors.Err("not all published videos are in the database") + } if pubsOnWallet < pubsOnDB { - SendInfoToSlack("We're claiming to have published %d videos but we only published %d (%s)", pubsOnDB, pubsOnWallet, s.YoutubeChannelID) + SendInfoToSlack("we're claiming to have published %d videos but we only published %d (%s)", pubsOnDB, pubsOnWallet, s.YoutubeChannelID) } err = s.walletSetup() if err != nil {