Fix search indices issue
This commit is contained in:
Jeffrey Picard
parent
dc71fb1a99
commit
f04af2b12e
3 changed files with 49 additions and 34 deletions
|
|
@ -92,4 +92,5 @@ message SearchRequest {
|
|||
repeated string all_languages = 75;
|
||||
.google.protobuf.BoolValue remove_duplicates = 76;
|
||||
.google.protobuf.BoolValue no_totals = 77;
|
||||
repeated string search_indices = 78;
|
||||
}
|
||||
|
|
@ -255,6 +255,7 @@ type SearchRequest struct {
|
|||
AllLanguages []string `protobuf:"bytes,75,rep,name=all_languages,json=allLanguages,proto3" json:"all_languages"`
|
||||
RemoveDuplicates *wrapperspb.BoolValue `protobuf:"bytes,76,opt,name=remove_duplicates,json=removeDuplicates,proto3" json:"remove_duplicates"`
|
||||
NoTotals *wrapperspb.BoolValue `protobuf:"bytes,77,opt,name=no_totals,json=noTotals,proto3" json:"no_totals"`
|
||||
SearchIndices []string `protobuf:"bytes,78,rep,name=search_indices,json=searchIndices,proto3" json:"search_indices"`
|
||||
}
|
||||
|
||||
func (x *SearchRequest) Reset() {
|
||||
|
|
@ -737,6 +738,13 @@ func (x *SearchRequest) GetNoTotals() *wrapperspb.BoolValue {
|
|||
return nil
|
||||
}
|
||||
|
||||
func (x *SearchRequest) GetSearchIndices() []string {
|
||||
if x != nil {
|
||||
return x.SearchIndices
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
var File_hub_proto protoreflect.FileDescriptor
|
||||
|
||||
var file_hub_proto_rawDesc = []byte{
|
||||
|
|
@ -755,7 +763,7 @@ var file_hub_proto_rawDesc = []byte{
|
|||
0x76, 0x61, 0x6c, 0x75, 0x65, 0x22, 0x2e, 0x0a, 0x02, 0x4f, 0x70, 0x12, 0x06, 0x0a, 0x02, 0x45,
|
||||
0x51, 0x10, 0x00, 0x12, 0x07, 0x0a, 0x03, 0x4c, 0x54, 0x45, 0x10, 0x01, 0x12, 0x07, 0x0a, 0x03,
|
||||
0x47, 0x54, 0x45, 0x10, 0x02, 0x12, 0x06, 0x0a, 0x02, 0x4c, 0x54, 0x10, 0x03, 0x12, 0x06, 0x0a,
|
||||
0x02, 0x47, 0x54, 0x10, 0x04, 0x22, 0xd2, 0x16, 0x0a, 0x0d, 0x53, 0x65, 0x61, 0x72, 0x63, 0x68,
|
||||
0x02, 0x47, 0x54, 0x10, 0x04, 0x22, 0xf9, 0x16, 0x0a, 0x0d, 0x53, 0x65, 0x61, 0x72, 0x63, 0x68,
|
||||
0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x74, 0x65, 0x78, 0x74, 0x18,
|
||||
0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x74, 0x65, 0x78, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x6e,
|
||||
0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x03, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12,
|
||||
|
|
@ -936,13 +944,16 @@ var file_hub_proto_rawDesc = []byte{
|
|||
0x73, 0x12, 0x37, 0x0a, 0x09, 0x6e, 0x6f, 0x5f, 0x74, 0x6f, 0x74, 0x61, 0x6c, 0x73, 0x18, 0x4d,
|
||||
0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72,
|
||||
0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x42, 0x6f, 0x6f, 0x6c, 0x56, 0x61, 0x6c, 0x75, 0x65,
|
||||
0x52, 0x08, 0x6e, 0x6f, 0x54, 0x6f, 0x74, 0x61, 0x6c, 0x73, 0x32, 0x31, 0x0a, 0x03, 0x48, 0x75,
|
||||
0x62, 0x12, 0x2a, 0x0a, 0x06, 0x53, 0x65, 0x61, 0x72, 0x63, 0x68, 0x12, 0x11, 0x2e, 0x70, 0x62,
|
||||
0x2e, 0x53, 0x65, 0x61, 0x72, 0x63, 0x68, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x0b,
|
||||
0x2e, 0x70, 0x62, 0x2e, 0x4f, 0x75, 0x74, 0x70, 0x75, 0x74, 0x73, 0x22, 0x00, 0x42, 0x26, 0x5a,
|
||||
0x24, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x6c, 0x62, 0x72, 0x79,
|
||||
0x69, 0x6f, 0x2f, 0x68, 0x75, 0x62, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2f,
|
||||
0x67, 0x6f, 0x2f, 0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
|
||||
0x52, 0x08, 0x6e, 0x6f, 0x54, 0x6f, 0x74, 0x61, 0x6c, 0x73, 0x12, 0x25, 0x0a, 0x0e, 0x73, 0x65,
|
||||
0x61, 0x72, 0x63, 0x68, 0x5f, 0x69, 0x6e, 0x64, 0x69, 0x63, 0x65, 0x73, 0x18, 0x4e, 0x20, 0x03,
|
||||
0x28, 0x09, 0x52, 0x0d, 0x73, 0x65, 0x61, 0x72, 0x63, 0x68, 0x49, 0x6e, 0x64, 0x69, 0x63, 0x65,
|
||||
0x73, 0x32, 0x31, 0x0a, 0x03, 0x48, 0x75, 0x62, 0x12, 0x2a, 0x0a, 0x06, 0x53, 0x65, 0x61, 0x72,
|
||||
0x63, 0x68, 0x12, 0x11, 0x2e, 0x70, 0x62, 0x2e, 0x53, 0x65, 0x61, 0x72, 0x63, 0x68, 0x52, 0x65,
|
||||
0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x0b, 0x2e, 0x70, 0x62, 0x2e, 0x4f, 0x75, 0x74, 0x70, 0x75,
|
||||
0x74, 0x73, 0x22, 0x00, 0x42, 0x26, 0x5a, 0x24, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63,
|
||||
0x6f, 0x6d, 0x2f, 0x6c, 0x62, 0x72, 0x79, 0x69, 0x6f, 0x2f, 0x68, 0x75, 0x62, 0x2f, 0x70, 0x72,
|
||||
0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2f, 0x67, 0x6f, 0x2f, 0x70, 0x62, 0x62, 0x06, 0x70, 0x72,
|
||||
0x6f, 0x74, 0x6f, 0x33,
|
||||
}
|
||||
|
||||
var (
|
||||
|
|
|
|||
|
|
@ -410,7 +410,7 @@ func (s *Server) Search(ctx context.Context, in *pb.SearchRequest) (*pb.Outputs,
|
|||
}
|
||||
|
||||
var searchIndices = []string{}
|
||||
if s.Args.Dev {
|
||||
if s.Args.Dev && len(in.SearchIndices) == 0 {
|
||||
// If we're running in dev mode ignore the mainnet claims index
|
||||
indices, err := client.IndexNames()
|
||||
if err != nil {
|
||||
|
|
@ -431,6 +431,10 @@ func (s *Server) Search(ctx context.Context, in *pb.SearchRequest) (*pb.Outputs,
|
|||
}
|
||||
}
|
||||
|
||||
if len(in.SearchIndices) > 0 {
|
||||
searchIndices = in.SearchIndices
|
||||
}
|
||||
|
||||
fsc := elastic.NewFetchSourceContext(true).Exclude("description", "title")//.Include("_id")
|
||||
search := client.Search().
|
||||
Index(searchIndices...).
|
||||
|
|
@ -489,15 +493,10 @@ func (s *Server) Search(ctx context.Context, in *pb.SearchRequest) (*pb.Outputs,
|
|||
|
||||
//printJsonFullRecords(blockedRecords)
|
||||
|
||||
var searchIndex = "claims" //default
|
||||
if len(searchIndices) > 0 {
|
||||
searchIndex = searchIndices[0]
|
||||
}
|
||||
|
||||
//Get claims for reposts
|
||||
repostClaims, repostRecords, repostedMap := getClaimsForReposts(records, client, ctx, searchIndex)
|
||||
repostClaims, repostRecords, repostedMap := getClaimsForReposts(records, client, ctx, searchIndices)
|
||||
//get all unique channels
|
||||
channels, channelMap := getUniqueChannels(append(append(records, repostRecords...), blockedRecords...), client, ctx, searchIndex)
|
||||
channels, channelMap := getUniqueChannels(append(append(records, repostRecords...), blockedRecords...), client, ctx, searchIndices)
|
||||
//add these to extra txos
|
||||
extraTxos := append(repostClaims, channels...)
|
||||
|
||||
|
|
@ -544,23 +543,25 @@ func (s *Server) Search(ctx context.Context, in *pb.SearchRequest) (*pb.Outputs,
|
|||
}, nil
|
||||
}
|
||||
|
||||
func getUniqueChannels(records []*record, client *elastic.Client, ctx context.Context, searchIndex string) ([]*pb.Output, map[string]*pb.Output) {
|
||||
func getUniqueChannels(records []*record, client *elastic.Client, ctx context.Context, searchIndices []string) ([]*pb.Output, map[string]*pb.Output) {
|
||||
channels := make(map[string]*pb.Output)
|
||||
channelsSet := make(map[string]bool)
|
||||
var mget = client.Mget()
|
||||
var totalChannels = 0
|
||||
for _, r := range records {
|
||||
if r.ChannelId != "" && !channelsSet[r.ChannelId] {
|
||||
channelsSet[r.ChannelId] = true
|
||||
nmget := elastic.NewMultiGetItem().Id(r.ChannelId).Index(searchIndex)
|
||||
mget = mget.Add(nmget)
|
||||
totalChannels++
|
||||
}
|
||||
if r.CensorType != 0 && !channelsSet[r.CensoringChannelHash] {
|
||||
channelsSet[r.CensoringChannelHash] = true
|
||||
nmget := elastic.NewMultiGetItem().Id(r.CensoringChannelHash).Index(searchIndex)
|
||||
mget = mget.Add(nmget)
|
||||
totalChannels++
|
||||
for _, searchIndex := range searchIndices {
|
||||
if r.ChannelId != "" && !channelsSet[r.ChannelId] {
|
||||
channelsSet[r.ChannelId] = true
|
||||
nmget := elastic.NewMultiGetItem().Id(r.ChannelId).Index(searchIndex)
|
||||
mget = mget.Add(nmget)
|
||||
totalChannels++
|
||||
}
|
||||
if r.CensorType != 0 && !channelsSet[r.CensoringChannelHash] {
|
||||
channelsSet[r.CensoringChannelHash] = true
|
||||
nmget := elastic.NewMultiGetItem().Id(r.CensoringChannelHash).Index(searchIndex)
|
||||
mget = mget.Add(nmget)
|
||||
totalChannels++
|
||||
}
|
||||
}
|
||||
}
|
||||
if totalChannels == 0 {
|
||||
|
|
@ -592,7 +593,7 @@ func getUniqueChannels(records []*record, client *elastic.Client, ctx context.Co
|
|||
return channelTxos, channels
|
||||
}
|
||||
|
||||
func getClaimsForReposts(records []*record, client *elastic.Client, ctx context.Context, searchIndex string) ([]*pb.Output, []*record, map[string]*pb.Output) {
|
||||
func getClaimsForReposts(records []*record, client *elastic.Client, ctx context.Context, searchIndices []string) ([]*pb.Output, []*record, map[string]*pb.Output) {
|
||||
|
||||
var totalReposted = 0
|
||||
var mget = client.Mget()//.StoredFields("_id")
|
||||
|
|
@ -603,11 +604,13 @@ func getClaimsForReposts(records []*record, client *elastic.Client, ctx context.
|
|||
}
|
||||
*/
|
||||
for _, r := range records {
|
||||
if r.RepostedClaimId != "" {
|
||||
var nmget = elastic.NewMultiGetItem().Id(r.RepostedClaimId).Index(searchIndex)
|
||||
//nmget = nmget.Id(r.RepostedClaimId)
|
||||
mget = mget.Add(nmget)
|
||||
totalReposted++
|
||||
for _, searchIndex := range searchIndices {
|
||||
if r.RepostedClaimId != "" {
|
||||
var nmget = elastic.NewMultiGetItem().Id(r.RepostedClaimId).Index(searchIndex)
|
||||
//nmget = nmget.Id(r.RepostedClaimId)
|
||||
mget = mget.Add(nmget)
|
||||
totalReposted++
|
||||
}
|
||||
}
|
||||
}
|
||||
//mget = mget.Add(nmget)
|
||||
|
|
|
|||
Loading…
Reference in a new issue