update response signature

add dupes check
add de-sync checks

jsonrpc/daemon.go

@@ -456,3 +456,15 @@ func (d *Client) NumClaimsInChannel(url string) (uint64, error) {
 	}
 	return channel.ClaimsInChannel, nil
 }
+
+func (d *Client) ClaimListMine() (*ClaimListMineResponse, error) {
+	response := new(ClaimListMineResponse)
+	err := d.call(response, "claim_list_mine", map[string]interface{}{})
+	if err != nil {
+		return nil, err
+	} else if response == nil {
+		return nil, errors.Err("no response")
+	}
+
+	return response, nil
+}

jsonrpc/daemon_types.go

@@ -31,25 +31,32 @@ type Support struct {
 }
 
 type Claim struct {
-	Address          string           `json:"address"`
+	Address            string           `json:"address"`
-	Amount           decimal.Decimal  `json:"amount"`
+	Amount             decimal.Decimal  `json:"amount"`
-	ClaimID          string           `json:"claim_id"`
+	BlocksToExpiration int              `json:"blocks_to_expiration"`
-	ClaimSequence    int              `json:"claim_sequence"`
+	Category           string           `json:"category"`
-	DecodedClaim     bool             `json:"decoded_claim"`
+	ClaimID            string           `json:"claim_id"`
-	Depth            int              `json:"depth"`
+	ClaimSequence      int              `json:"claim_sequence"`
-	EffectiveAmount  decimal.Decimal  `json:"effective_amount"`
+	Confirmations      int              `json:"confirmations"`
-	Height           int              `json:"height"`
+	DecodedClaim       bool             `json:"decoded_claim"`
-	Hex              string           `json:"hex"`
+	Depth              int              `json:"depth"`
-	Name             string           `json:"name"`
+	EffectiveAmount    decimal.Decimal  `json:"effective_amount"`
-	Nout             int              `json:"nout"`
+	ExpirationHeight   int              `json:"expiration_height"`
-	Supports         []Support        `json:"supports"`
+	Expired            bool             `json:"expired"`
-	Txid             string           `json:"txid"`
+	Height             int              `json:"height"`
-	ValidAtHeight    int              `json:"valid_at_height"`
+	Hex                string           `json:"hex"`
-	Value            lbryschema.Claim `json:"value"`
+	IsSpent            bool             `json:"is_spent"`
-	Error            *string          `json:"error,omitempty"`
+	Name               string           `json:"name"`
-	ChannelName      *string          `json:"channel_name,omitempty"`
+	Nout               int              `json:"nout"`
-	HasSignature     *bool            `json:"has_signature,omitempty"`
+	PermanentUrl       string           `json:"permanent_url"`
-	SignatureIsValid *bool            `json:"signature_is_valid,omitempty"`
+	Supports           []Support        `json:"supports"`
+	Txid               string           `json:"txid"`
+	ValidAtHeight      int              `json:"valid_at_height"`
+	Value              lbryschema.Claim `json:"value"`
+	Error              *string          `json:"error,omitempty"`
+	ChannelName        *string          `json:"channel_name,omitempty"`
+	HasSignature       *bool            `json:"has_signature,omitempty"`
+	SignatureIsValid   *bool            `json:"signature_is_valid,omitempty"`
 }
 
 type File struct {
@@ -234,7 +241,9 @@ type ClaimListResponse struct {
 	LastTakeoverHeight    int       `json:"last_takeover_height"`
 	SupportsWithoutClaims []Support `json:"supports_without_claims"`
 }
-
+type ClaimListMineResponse struct {
+	Claims []Claim `json:"claims"`
+}
 type ClaimShowResponse Claim
 
 type PeerListResponsePeer struct {

ytsync/ytsync.go

@@ -365,9 +365,71 @@ func logShutdownError(shutdownErr error) {
 	SendErrorToSlack("WALLET HAS NOT BEEN MOVED TO THE WALLET BACKUP DIR")
 }
 
+func hasDupes(claims []jsonrpc.Claim) (bool, error) {
+	videoIDs := make(map[string]interface{})
+	for _, c := range claims {
+		if !util.InSlice(c.Category, []string{"claim", "update"}) || c.Value.Stream == nil {
+			continue
+		}
+		if c.Value.Stream.Metadata == nil || c.Value.Stream.Metadata.Thumbnail == nil {
+			return false, errors.Err("something is wrong with the this claim: %s", c.ClaimID)
+		}
+		tn := *c.Value.Stream.Metadata.Thumbnail
+		videoID := tn[:strings.LastIndex(tn, "/")+1]
+		_, ok := videoIDs[videoID]
+		if !ok {
+			videoIDs[videoID] = nil
+			continue
+		}
+		return true, nil
+	}
+	return false, nil
+}
+
+//publishesCount counts the amount of videos published so far
+func publishesCount(claims []jsonrpc.Claim) (int, error) {
+	count := 0
+	for _, c := range claims {
+		if !util.InSlice(c.Category, []string{"claim", "update"}) || c.Value.Stream == nil {
+			continue
+		}
+		if c.Value.Stream.Metadata == nil || c.Value.Stream.Metadata.Thumbnail == nil {
+			return count, errors.Err("something is wrong with the this claim: %s", c.ClaimID)
+		}
+		count++
+	}
+	return count, nil
+}
+
 func (s *Sync) doSync() error {
 	var err error
-
+	claims, err := s.daemon.ClaimListMine()
+	if err != nil {
+		return errors.Prefix("cannot list claims: ", err)
+	}
+	hasDupes, err := hasDupes(claims.Claims)
+	if err != nil {
+		return errors.Prefix("error checking for duplicates: ", err)
+	}
+	if hasDupes {
+		return errors.Err("channel has duplicates! Manual fix required")
+	}
+	pubsOnWallet, err := publishesCount(claims.Claims)
+	if err != nil {
+		return errors.Prefix("error counting claims: ", err)
+	}
+	pubsOnDB := 0
+	for _, sv := range s.syncedVideos {
+		if sv.Published {
+			pubsOnDB++
+		}
+	}
+	if pubsOnWallet > pubsOnDB {
+		return errors.Err("not all published videos are in the database")
+	}
+	if pubsOnWallet < pubsOnDB {
+		SendInfoToSlack("We're claiming to have published %d videos but we only published %d (%s)", pubsOnDB, pubsOnWallet, s.lbryChannelID)
+	}
 	err = s.walletSetup()
 	if err != nil {
 		return errors.Prefix("Initial wallet setup failed! Manual Intervention is required.", err)
@@ -379,10 +441,10 @@ func (s *Sync) doSync() error {
 
 	for i := 0; i < s.ConcurrentVideos; i++ {
 		s.grp.Add(1)
-		go func() {
+		go func(i int) {
 			defer s.grp.Done()
 			s.startWorker(i)
-		}()
+		}(i)
 	}
 
 	if s.LbryChannelName == "@UCBerkeley" {