LBRYCommunity/herald.go
1
0
Fork 0
Code Issues 16 Pull requests Packages Projects Releases 14 Wiki Activity

field changes

Browse source
This commit is contained in:
Victor Shyba 2021-08-12 04:19:25 -03:00
parent fb161ee6f5
commit d45f215789
2 changed files with 73 additions and 96 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

10
main.go
View file

@ -108,22 +108,22 @@ func parseArgs(searchRequest *pb.SearchRequest) *server.Args {
searchRequest.Text = *text searchRequest.Text = *text
} }
if *name != "" { if *name != "" {
searchRequest.Name = []string{*name} searchRequest.ClaimName = *name
} }
if *claimType != "" { if *claimType != "" {
searchRequest.ClaimType = []string{*claimType} searchRequest.ClaimType = []string{*claimType}
} }
if *id != "" { if *id != "" {
searchRequest.XId = [][]byte{[]byte(*id)} searchRequest.ClaimId = &pb.InvertibleField{Invert: false, Value: []string{*id}}
} }
if *author != "" { if *author != "" {
searchRequest.Author = []string{*author} searchRequest.Author = *author
} }
if *title != "" { if *title != "" {
searchRequest.Title = []string{*title} searchRequest.Title = *title
} }
if *description != "" { if *description != "" {
searchRequest.Description = []string{*description} searchRequest.Description = *description
} }
if *channelId != "" { if *channelId != "" {
searchRequest.ChannelId = &pb.InvertibleField{Invert: false, Value: []string{*channelId}} searchRequest.ChannelId = &pb.InvertibleField{Invert: false, Value: []string{*channelId}}

159
server/search.go
View file

@ -13,7 +13,6 @@ import (
//"github.com/lbryio/hub/schema" //"github.com/lbryio/hub/schema"
"github.com/btcsuite/btcutil/base58" "github.com/btcsuite/btcutil/base58"
"github.com/golang/protobuf/ptypes/wrappers"
pb "github.com/lbryio/hub/protobuf/go" pb "github.com/lbryio/hub/protobuf/go"
"github.com/lbryio/lbry.go/v2/extras/util" "github.com/lbryio/lbry.go/v2/extras/util"
"github.com/olivere/elastic/v7" "github.com/olivere/elastic/v7"
@ -25,30 +24,30 @@ import (
const DefaultSearchSize = 1000 const DefaultSearchSize = 1000
type record struct { type record struct {
Txid string `json:"tx_id"` Txid string `json:"tx_id"`
Nout uint32 `json:"tx_nout"` Nout uint32 `json:"tx_nout"`
Height uint32 `json:"height"` Height uint32 `json:"height"`
ClaimId string `json:"claim_id"` ClaimId string `json:"claim_id"`
ChannelId string `json:"channel_id"` ChannelId string `json:"channel_id"`
RepostedClaimId string `json:"reposted_claim_id"` RepostedClaimId string `json:"reposted_claim_id"`
CensorType uint32 `json:"censor_type"` CensorType uint32 `json:"censor_type"`
CensoringChannelHash string `json:"censoring_channel_id"` CensoringChannelId string `json:"censoring_channel_id"`
ShortUrl string `json:"short_url"` ShortUrl string `json:"short_url"`
CanonicalUrl string `json:"canonical_url"` CanonicalUrl string `json:"canonical_url"`
IsControlling bool `json:"is_controlling"` IsControlling bool `json:"is_controlling"`
TakeOverHeight uint32 `json:"last_take_over_height"` TakeOverHeight uint32 `json:"last_take_over_height"`
CreationHeight uint32 `json:"creation_height"` CreationHeight uint32 `json:"creation_height"`
ActivationHeight uint32 `json:"activation_height"` ActivationHeight uint32 `json:"activation_height"`
ExpirationHeight uint32 `json:"expiration_height"` ExpirationHeight uint32 `json:"expiration_height"`
ClaimsInChannel uint32 `json:"claims_in_channel"` ClaimsInChannel uint32 `json:"claims_in_channel"`
Reposted uint32 `json:"repost_count"` Reposted uint32 `json:"repost_count"`
EffectiveAmount uint64 `json:"effective_amount"` EffectiveAmount uint64 `json:"effective_amount"`
SupportAmount uint64 `json:"support_amount"` SupportAmount uint64 `json:"support_amount"`
TrendingGroup uint32 `json:"trending_group"` TrendingGroup uint32 `json:"trending_group"`
TrendingMixed float32 `json:"trending_mixed"` TrendingMixed float32 `json:"trending_mixed"`
TrendingLocal float32 `json:"trending_local"` TrendingLocal float32 `json:"trending_local"`
TrendingGlobal float32 `json:"trending_global"` TrendingGlobal float32 `json:"trending_global"`
Name string `json:"name"` Name string `json:"name"`
} }
type orderField struct { type orderField struct {
@ -72,6 +71,13 @@ func AddTermsField(q *elastic.BoolQuery, arr []string, name string) *elastic.Boo
return q.Must(elastic.NewTermsQuery(name, searchVals...)) return q.Must(elastic.NewTermsQuery(name, searchVals...))
} }
func AddTermField(q *elastic.BoolQuery, value string, name string) *elastic.BoolQuery {
if value != "" {
return q.Must(elastic.NewTermQuery(name, value))
}
return q
}
func AddIndividualTermFields(q *elastic.BoolQuery, arr []string, name string, invert bool) *elastic.BoolQuery { func AddIndividualTermFields(q *elastic.BoolQuery, arr []string, name string, invert bool) *elastic.BoolQuery {
for _, x := range arr { for _, x := range arr {
if invert { if invert {
@ -160,10 +166,6 @@ func (s *Server) Search(ctx context.Context, in *pb.SearchRequest) (*pb.Outputs,
q = s.setupEsQuery(q, in, &pageSize, &from, &orderBy) q = s.setupEsQuery(q, in, &pageSize, &from, &orderBy)
if len(in.SearchIndices) > 0 {
searchIndices = in.SearchIndices
}
fsc := elastic.NewFetchSourceContext(true).Exclude("description", "title") fsc := elastic.NewFetchSourceContext(true).Exclude("description", "title")
search := client.Search(). search := client.Search().
Index(searchIndices...). Index(searchIndices...).
@ -185,7 +187,7 @@ func (s *Server) Search(ctx context.Context, in *pb.SearchRequest) (*pb.Outputs,
txos, extraTxos, blocked := s.postProcessResults(ctx, client, searchResult, in, pageSize, from, searchIndices) txos, extraTxos, blocked := s.postProcessResults(ctx, client, searchResult, in, pageSize, from, searchIndices)
if in.NoTotals != nil && !in.NoTotals.Value { if in.NoTotals {
return &pb.Outputs{ return &pb.Outputs{
Txos: txos, Txos: txos,
ExtraTxos: extraTxos, ExtraTxos: extraTxos,
@ -253,12 +255,12 @@ func (s *Server) postProcessResults(
//printJsonFullResults(searchResult) //printJsonFullResults(searchResult)
records, blockedRecords, blockedMap = removeBlocked(records) records, blockedRecords, blockedMap = removeBlocked(records)
if in.RemoveDuplicates != nil { if in.RemoveDuplicates {
records = removeDuplicates(records) records = removeDuplicates(records)
} }
if in.LimitClaimsPerChannel != nil && in.LimitClaimsPerChannel.Value > 0 { if in.LimitClaimsPerChannel > 0 {
records = searchAhead(records, pageSize, int(in.LimitClaimsPerChannel.Value)) records = searchAhead(records, pageSize, int(in.LimitClaimsPerChannel))
} }
finalLength := int(math.Min(float64(len(records)), float64(pageSize))) finalLength := int(math.Min(float64(len(records)), float64(pageSize)))
@ -353,30 +355,20 @@ func (s *Server) setupEsQuery(
"tags": true, "tags": true,
} }
if in.IsControlling != nil { if in.IsControlling {
q = q.Must(elastic.NewTermQuery("is_controlling", in.IsControlling.Value)) q = q.Must(elastic.NewTermQuery("is_controlling", in.IsControlling))
} }
if in.AmountOrder != nil { if in.Limit > 0 {
in.Limit.Value = 1 *pageSize = int(in.Limit)
in.OrderBy = []string{"effective_amount"}
in.Offset = &wrappers.Int32Value{Value: in.AmountOrder.Value - 1}
} }
if in.Limit != nil { if in.Offset > 0 {
*pageSize = int(in.Limit.Value) *from = int(in.Offset)
} }
if in.Offset != nil { if len(in.ClaimName) > 0 {
*from = int(in.Offset.Value) in.NormalizedName = util.NormalizeName(in.ClaimName)
}
if len(in.Name) > 0 {
normalized := make([]string, len(in.Name))
for i := 0; i < len(in.Name); i++ {
normalized[i] = util.NormalizeName(in.Name[i])
}
in.Normalized = normalized
} }
if len(in.OrderBy) > 0 { if len(in.OrderBy) > 0 {
@ -416,19 +408,6 @@ func (s *Server) setupEsQuery(
q = q.Must(elastic.NewTermsQuery("stream_type", searchVals...)) q = q.Must(elastic.NewTermsQuery("stream_type", searchVals...))
} }
if len(in.XId) > 0 {
searchVals := make([]interface{}, len(in.XId))
for i := 0; i < len(in.XId); i++ {
util.ReverseBytesInPlace(in.XId[i])
searchVals[i] = hex.Dump(in.XId[i])
}
if len(in.XId) == 1 && len(in.XId[0]) < 20 {
q = q.Must(elastic.NewPrefixQuery("_id", string(in.XId[0])))
} else {
q = q.Must(elastic.NewTermsQuery("_id", searchVals...))
}
}
if in.ClaimId != nil { if in.ClaimId != nil {
searchVals := StrArrToInterface(in.ClaimId.Value) searchVals := StrArrToInterface(in.ClaimId.Value)
if len(in.ClaimId.Value) == 1 && len(in.ClaimId.Value[0]) < 20 { if len(in.ClaimId.Value) == 1 && len(in.ClaimId.Value[0]) < 20 {
@ -451,15 +430,15 @@ func (s *Server) setupEsQuery(
q = q.Must(elastic.NewTermQuery("public_key_id.keyword", value)) q = q.Must(elastic.NewTermQuery("public_key_id.keyword", value))
} }
if in.HasChannelSignature != nil && in.HasChannelSignature.Value { if in.HasChannelSignature {
q = q.Must(elastic.NewExistsQuery("signature_digest")) q = q.Must(elastic.NewExistsQuery("signature_digest"))
if in.SignatureValid != nil { if in.IsSignatureValid != nil {
q = q.Must(elastic.NewTermQuery("is_signature_valid", in.SignatureValid.Value)) q = q.Must(elastic.NewTermQuery("is_signature_valid", in.IsSignatureValid.Value))
} }
} else if in.SignatureValid != nil { } else if in.IsSignatureValid != nil {
q = q.MinimumNumberShouldMatch(1) q = q.MinimumNumberShouldMatch(1)
q = q.Should(elastic.NewBoolQuery().MustNot(elastic.NewExistsQuery("signature_digest"))) q = q.Should(elastic.NewBoolQuery().MustNot(elastic.NewExistsQuery("signature_digest")))
q = q.Should(elastic.NewTermQuery("is_signature_valid", in.SignatureValid.Value)) q = q.Should(elastic.NewTermQuery("is_signature_valid", in.IsSignatureValid.Value))
} }
if in.HasSource != nil { if in.HasSource != nil {
@ -474,21 +453,20 @@ func (s *Server) setupEsQuery(
q = q.Must(elastic.NewTermQuery("tx_nout", in.TxNout.Value)) q = q.Must(elastic.NewTermQuery("tx_nout", in.TxNout.Value))
} }
q = AddTermsField(q, in.PublicKeyHash, "public_key_id.keyword") q = AddTermField(q, in.Author, "author.keyword")
q = AddTermsField(q, in.Author, "author.keyword") q = AddTermField(q, in.Title, "title.keyword")
q = AddTermsField(q, in.Title, "title.keyword") q = AddTermField(q, in.CanonicalUrl, "canonical_url.keyword")
q = AddTermsField(q, in.CanonicalUrl, "canonical_url.keyword") q = AddTermField(q, in.ClaimName, "claim_name.keyword")
q = AddTermsField(q, in.ClaimName, "claim_name.keyword") q = AddTermField(q, in.Description, "description.keyword")
q = AddTermsField(q, in.Description, "description.keyword")
q = AddTermsField(q, in.MediaType, "media_type.keyword") q = AddTermsField(q, in.MediaType, "media_type.keyword")
q = AddTermsField(q, in.Normalized, "normalized_name.keyword") q = AddTermField(q, in.NormalizedName, "normalized_name.keyword")
q = AddTermsField(q, in.PublicKeyBytes, "public_key_bytes.keyword") q = AddTermField(q, in.PublicKeyBytes, "public_key_bytes.keyword")
q = AddTermsField(q, in.ShortUrl, "short_url.keyword") q = AddTermField(q, in.ShortUrl, "short_url.keyword")
q = AddTermsField(q, in.Signature, "signature.keyword") q = AddTermField(q, in.Signature, "signature.keyword")
q = AddTermsField(q, in.SignatureDigest, "signature_digest.keyword") q = AddTermField(q, in.SignatureDigest, "signature_digest.keyword")
q = AddTermsField(q, in.TxId, "tx_id.keyword") q = AddTermField(q, in.TxId, "tx_id.keyword")
q = AddTermsField(q, in.FeeCurrency, "fee_currency.keyword") q = AddTermField(q, in.FeeCurrency, "fee_currency.keyword")
q = AddTermsField(q, in.RepostedClaimId, "reposted_claim_id.keyword") q = AddTermField(q, in.RepostedClaimId, "reposted_claim_id.keyword")
q = AddTermsField(q, s.cleanTags(in.AnyTags), "tags.keyword") q = AddTermsField(q, s.cleanTags(in.AnyTags), "tags.keyword")
q = AddIndividualTermFields(q, s.cleanTags(in.AllTags), "tags.keyword", false) q = AddIndividualTermFields(q, s.cleanTags(in.AllTags), "tags.keyword", false)
@ -497,7 +475,6 @@ func (s *Server) setupEsQuery(
q = AddIndividualTermFields(q, in.AllLanguages, "languages", false) q = AddIndividualTermFields(q, in.AllLanguages, "languages", false)
q = AddInvertibleField(q, in.ChannelId, "channel_id.keyword") q = AddInvertibleField(q, in.ChannelId, "channel_id.keyword")
q = AddInvertibleField(q, in.ChannelIds, "channel_id.keyword")
q = AddRangeField(q, in.TxPosition, "tx_position") q = AddRangeField(q, in.TxPosition, "tx_position")
q = AddRangeField(q, in.Amount, "amount") q = AddRangeField(q, in.Amount, "amount")
@ -508,7 +485,7 @@ func (s *Server) setupEsQuery(
q = AddRangeField(q, in.ActivationHeight, "activation_height") q = AddRangeField(q, in.ActivationHeight, "activation_height")
q = AddRangeField(q, in.ExpirationHeight, "expiration_height") q = AddRangeField(q, in.ExpirationHeight, "expiration_height")
q = AddRangeField(q, in.ReleaseTime, "release_time") q = AddRangeField(q, in.ReleaseTime, "release_time")
q = AddRangeField(q, in.Reposted, "repost_count") q = AddRangeField(q, in.RepostCount, "repost_count")
q = AddRangeField(q, in.FeeAmount, "fee_amount") q = AddRangeField(q, in.FeeAmount, "fee_amount")
q = AddRangeField(q, in.Duration, "duration") q = AddRangeField(q, in.Duration, "duration")
q = AddRangeField(q, in.CensorType, "censor_type") q = AddRangeField(q, in.CensorType, "censor_type")
@ -548,9 +525,9 @@ func getUniqueChannels(records []*record, client *elastic.Client, ctx context.Co
mget = mget.Add(nmget) mget = mget.Add(nmget)
totalChannels++ totalChannels++
} }
if r.CensorType != 0 && !channelsSet[r.CensoringChannelHash] { if r.CensorType != 0 && !channelsSet[r.CensoringChannelId] {
channelsSet[r.CensoringChannelHash] = true channelsSet[r.CensoringChannelId] = true
nmget := elastic.NewMultiGetItem().Id(r.CensoringChannelHash).Index(searchIndex) nmget := elastic.NewMultiGetItem().Id(r.CensoringChannelId).Index(searchIndex)
mget = mget.Add(nmget) mget = mget.Add(nmget)
totalChannels++ totalChannels++
} }
@ -764,15 +741,15 @@ func removeBlocked(searchHits []*record) ([]*record, []*record, map[string]*pb.B
blockedChannels := make(map[string]*pb.Blocked) blockedChannels := make(map[string]*pb.Blocked)
for _, r := range searchHits { for _, r := range searchHits {
if r.CensorType != 0 { if r.CensorType != 0 {
if blockedChannels[r.CensoringChannelHash] == nil { if blockedChannels[r.CensoringChannelId] == nil {
blockedObj := &pb.Blocked{ blockedObj := &pb.Blocked{
Count: 1, Count: 1,
Channel: nil, Channel: nil,
} }
blockedChannels[r.CensoringChannelHash] = blockedObj blockedChannels[r.CensoringChannelId] = blockedObj
blockedHits = append(blockedHits, r) blockedHits = append(blockedHits, r)
} else { } else {
blockedChannels[r.CensoringChannelHash].Count += 1 blockedChannels[r.CensoringChannelId].Count += 1
} }
} else { } else {
newHits = append(newHits, r) newHits = append(newHits, r)