From f918f9085372a78f647fb6fd4af09046c012973e Mon Sep 17 00:00:00 2001 From: Niko Storni Date: Fri, 31 Aug 2018 11:42:15 -0400 Subject: [PATCH] update response signature add dupes check add de-sync checks --- jsonrpc/daemon.go | 12 ++++++++ jsonrpc/daemon_types.go | 49 +++++++++++++++++------------ ytsync/ytsync.go | 68 +++++++++++++++++++++++++++++++++++++++-- 3 files changed, 106 insertions(+), 23 deletions(-) diff --git a/jsonrpc/daemon.go b/jsonrpc/daemon.go index 27bdfba..07d7374 100644 --- a/jsonrpc/daemon.go +++ b/jsonrpc/daemon.go @@ -456,3 +456,15 @@ func (d *Client) NumClaimsInChannel(url string) (uint64, error) { } return channel.ClaimsInChannel, nil } + +func (d *Client) ClaimListMine() (*ClaimListMineResponse, error) { + response := new(ClaimListMineResponse) + err := d.call(response, "claim_list_mine", map[string]interface{}{}) + if err != nil { + return nil, err + } else if response == nil { + return nil, errors.Err("no response") + } + + return response, nil +} diff --git a/jsonrpc/daemon_types.go b/jsonrpc/daemon_types.go index 6b03286..d1177d3 100644 --- a/jsonrpc/daemon_types.go +++ b/jsonrpc/daemon_types.go @@ -31,25 +31,32 @@ type Support struct { } type Claim struct { - Address string `json:"address"` - Amount decimal.Decimal `json:"amount"` - ClaimID string `json:"claim_id"` - ClaimSequence int `json:"claim_sequence"` - DecodedClaim bool `json:"decoded_claim"` - Depth int `json:"depth"` - EffectiveAmount decimal.Decimal `json:"effective_amount"` - Height int `json:"height"` - Hex string `json:"hex"` - Name string `json:"name"` - Nout int `json:"nout"` - Supports []Support `json:"supports"` - Txid string `json:"txid"` - ValidAtHeight int `json:"valid_at_height"` - Value lbryschema.Claim `json:"value"` - Error *string `json:"error,omitempty"` - ChannelName *string `json:"channel_name,omitempty"` - HasSignature *bool `json:"has_signature,omitempty"` - SignatureIsValid *bool `json:"signature_is_valid,omitempty"` + Address string `json:"address"` + Amount decimal.Decimal `json:"amount"` + BlocksToExpiration int `json:"blocks_to_expiration"` + Category string `json:"category"` + ClaimID string `json:"claim_id"` + ClaimSequence int `json:"claim_sequence"` + Confirmations int `json:"confirmations"` + DecodedClaim bool `json:"decoded_claim"` + Depth int `json:"depth"` + EffectiveAmount decimal.Decimal `json:"effective_amount"` + ExpirationHeight int `json:"expiration_height"` + Expired bool `json:"expired"` + Height int `json:"height"` + Hex string `json:"hex"` + IsSpent bool `json:"is_spent"` + Name string `json:"name"` + Nout int `json:"nout"` + PermanentUrl string `json:"permanent_url"` + Supports []Support `json:"supports"` + Txid string `json:"txid"` + ValidAtHeight int `json:"valid_at_height"` + Value lbryschema.Claim `json:"value"` + Error *string `json:"error,omitempty"` + ChannelName *string `json:"channel_name,omitempty"` + HasSignature *bool `json:"has_signature,omitempty"` + SignatureIsValid *bool `json:"signature_is_valid,omitempty"` } type File struct { @@ -234,7 +241,9 @@ type ClaimListResponse struct { LastTakeoverHeight int `json:"last_takeover_height"` SupportsWithoutClaims []Support `json:"supports_without_claims"` } - +type ClaimListMineResponse struct { + Claims []Claim `json:"claims"` +} type ClaimShowResponse Claim type PeerListResponsePeer struct { diff --git a/ytsync/ytsync.go b/ytsync/ytsync.go index d44185b..75ee593 100644 --- a/ytsync/ytsync.go +++ b/ytsync/ytsync.go @@ -365,9 +365,71 @@ func logShutdownError(shutdownErr error) { SendErrorToSlack("WALLET HAS NOT BEEN MOVED TO THE WALLET BACKUP DIR") } +func hasDupes(claims []jsonrpc.Claim) (bool, error) { + videoIDs := make(map[string]interface{}) + for _, c := range claims { + if !util.InSlice(c.Category, []string{"claim", "update"}) || c.Value.Stream == nil { + continue + } + if c.Value.Stream.Metadata == nil || c.Value.Stream.Metadata.Thumbnail == nil { + return false, errors.Err("something is wrong with the this claim: %s", c.ClaimID) + } + tn := *c.Value.Stream.Metadata.Thumbnail + videoID := tn[:strings.LastIndex(tn, "/")+1] + _, ok := videoIDs[videoID] + if !ok { + videoIDs[videoID] = nil + continue + } + return true, nil + } + return false, nil +} + +//publishesCount counts the amount of videos published so far +func publishesCount(claims []jsonrpc.Claim) (int, error) { + count := 0 + for _, c := range claims { + if !util.InSlice(c.Category, []string{"claim", "update"}) || c.Value.Stream == nil { + continue + } + if c.Value.Stream.Metadata == nil || c.Value.Stream.Metadata.Thumbnail == nil { + return count, errors.Err("something is wrong with the this claim: %s", c.ClaimID) + } + count++ + } + return count, nil +} + func (s *Sync) doSync() error { var err error - + claims, err := s.daemon.ClaimListMine() + if err != nil { + return errors.Prefix("cannot list claims: ", err) + } + hasDupes, err := hasDupes(claims.Claims) + if err != nil { + return errors.Prefix("error checking for duplicates: ", err) + } + if hasDupes { + return errors.Err("channel has duplicates! Manual fix required") + } + pubsOnWallet, err := publishesCount(claims.Claims) + if err != nil { + return errors.Prefix("error counting claims: ", err) + } + pubsOnDB := 0 + for _, sv := range s.syncedVideos { + if sv.Published { + pubsOnDB++ + } + } + if pubsOnWallet > pubsOnDB { + return errors.Err("not all published videos are in the database") + } + if pubsOnWallet < pubsOnDB { + SendInfoToSlack("We're claiming to have published %d videos but we only published %d (%s)", pubsOnDB, pubsOnWallet, s.lbryChannelID) + } err = s.walletSetup() if err != nil { return errors.Prefix("Initial wallet setup failed! Manual Intervention is required.", err) @@ -379,10 +441,10 @@ func (s *Sync) doSync() error { for i := 0; i < s.ConcurrentVideos; i++ { s.grp.Add(1) - go func() { + go func(i int) { defer s.grp.Done() s.startWorker(i) - }() + }(i) } if s.LbryChannelName == "@UCBerkeley" {