LBRYCommunity/herald.go
1
0
Fork 0
Code Issues 16 Pull requests Projects Releases 14 Packages Wiki Activity

Fix search indices issue

Browse source
This commit is contained in:
Jeffrey Picard 2021-06-17 12:15:05 -04:00
parent dc71fb1a99
commit f04af2b12e
3 changed files with 49 additions and 34 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
protobuf/definitions/hub.proto
View file

@ -92,4 +92,5 @@ message SearchRequest {
repeated string all_languages = 75; repeated string all_languages = 75;
.google.protobuf.BoolValue remove_duplicates = 76; .google.protobuf.BoolValue remove_duplicates = 76;
.google.protobuf.BoolValue no_totals = 77; .google.protobuf.BoolValue no_totals = 77;
repeated string search_indices = 78;
} }

27
protobuf/go/hub.pb.go
View file

@ -255,6 +255,7 @@ type SearchRequest struct {
AllLanguages []string `protobuf:"bytes,75,rep,name=all_languages,json=allLanguages,proto3" json:"all_languages"` AllLanguages []string `protobuf:"bytes,75,rep,name=all_languages,json=allLanguages,proto3" json:"all_languages"`
RemoveDuplicates *wrapperspb.BoolValue `protobuf:"bytes,76,opt,name=remove_duplicates,json=removeDuplicates,proto3" json:"remove_duplicates"` RemoveDuplicates *wrapperspb.BoolValue `protobuf:"bytes,76,opt,name=remove_duplicates,json=removeDuplicates,proto3" json:"remove_duplicates"`
NoTotals *wrapperspb.BoolValue `protobuf:"bytes,77,opt,name=no_totals,json=noTotals,proto3" json:"no_totals"` NoTotals *wrapperspb.BoolValue `protobuf:"bytes,77,opt,name=no_totals,json=noTotals,proto3" json:"no_totals"`
SearchIndices []string `protobuf:"bytes,78,rep,name=search_indices,json=searchIndices,proto3" json:"search_indices"`
} }
func (x *SearchRequest) Reset() { func (x *SearchRequest) Reset() {
@ -737,6 +738,13 @@ func (x *SearchRequest) GetNoTotals() *wrapperspb.BoolValue {
return nil return nil
} }
func (x *SearchRequest) GetSearchIndices() []string {
if x != nil {
return x.SearchIndices
}
return nil
}
var File_hub_proto protoreflect.FileDescriptor var File_hub_proto protoreflect.FileDescriptor
var file_hub_proto_rawDesc = []byte{ var file_hub_proto_rawDesc = []byte{
@ -755,7 +763,7 @@ var file_hub_proto_rawDesc = []byte{
0x76, 0x61, 0x6c, 0x75, 0x65, 0x22, 0x2e, 0x0a, 0x02, 0x4f, 0x70, 0x12, 0x06, 0x0a, 0x02, 0x45, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x22, 0x2e, 0x0a, 0x02, 0x4f, 0x70, 0x12, 0x06, 0x0a, 0x02, 0x45,
0x51, 0x10, 0x00, 0x12, 0x07, 0x0a, 0x03, 0x4c, 0x54, 0x45, 0x10, 0x01, 0x12, 0x07, 0x0a, 0x03, 0x51, 0x10, 0x00, 0x12, 0x07, 0x0a, 0x03, 0x4c, 0x54, 0x45, 0x10, 0x01, 0x12, 0x07, 0x0a, 0x03,
0x47, 0x54, 0x45, 0x10, 0x02, 0x12, 0x06, 0x0a, 0x02, 0x4c, 0x54, 0x10, 0x03, 0x12, 0x06, 0x0a, 0x47, 0x54, 0x45, 0x10, 0x02, 0x12, 0x06, 0x0a, 0x02, 0x4c, 0x54, 0x10, 0x03, 0x12, 0x06, 0x0a,
0x02, 0x47, 0x54, 0x10, 0x04, 0x22, 0xd2, 0x16, 0x0a, 0x0d, 0x53, 0x65, 0x61, 0x72, 0x63, 0x68, 0x02, 0x47, 0x54, 0x10, 0x04, 0x22, 0xf9, 0x16, 0x0a, 0x0d, 0x53, 0x65, 0x61, 0x72, 0x63, 0x68,
0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x74, 0x65, 0x78, 0x74, 0x18, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x74, 0x65, 0x78, 0x74, 0x18,
0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x74, 0x65, 0x78, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x74, 0x65, 0x78, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x6e,
0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x03, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x03, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12,
@ -936,13 +944,16 @@ var file_hub_proto_rawDesc = []byte{
0x73, 0x12, 0x37, 0x0a, 0x09, 0x6e, 0x6f, 0x5f, 0x74, 0x6f, 0x74, 0x61, 0x6c, 0x73, 0x18, 0x4d, 0x73, 0x12, 0x37, 0x0a, 0x09, 0x6e, 0x6f, 0x5f, 0x74, 0x6f, 0x74, 0x61, 0x6c, 0x73, 0x18, 0x4d,
0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72,
0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x42, 0x6f, 0x6f, 0x6c, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x42, 0x6f, 0x6f, 0x6c, 0x56, 0x61, 0x6c, 0x75, 0x65,
0x52, 0x08, 0x6e, 0x6f, 0x54, 0x6f, 0x74, 0x61, 0x6c, 0x73, 0x32, 0x31, 0x0a, 0x03, 0x48, 0x75, 0x52, 0x08, 0x6e, 0x6f, 0x54, 0x6f, 0x74, 0x61, 0x6c, 0x73, 0x12, 0x25, 0x0a, 0x0e, 0x73, 0x65,
0x62, 0x12, 0x2a, 0x0a, 0x06, 0x53, 0x65, 0x61, 0x72, 0x63, 0x68, 0x12, 0x11, 0x2e, 0x70, 0x62, 0x61, 0x72, 0x63, 0x68, 0x5f, 0x69, 0x6e, 0x64, 0x69, 0x63, 0x65, 0x73, 0x18, 0x4e, 0x20, 0x03,
0x2e, 0x53, 0x65, 0x61, 0x72, 0x63, 0x68, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x0b, 0x28, 0x09, 0x52, 0x0d, 0x73, 0x65, 0x61, 0x72, 0x63, 0x68, 0x49, 0x6e, 0x64, 0x69, 0x63, 0x65,
0x2e, 0x70, 0x62, 0x2e, 0x4f, 0x75, 0x74, 0x70, 0x75, 0x74, 0x73, 0x22, 0x00, 0x42, 0x26, 0x5a, 0x73, 0x32, 0x31, 0x0a, 0x03, 0x48, 0x75, 0x62, 0x12, 0x2a, 0x0a, 0x06, 0x53, 0x65, 0x61, 0x72,
0x24, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x6c, 0x62, 0x72, 0x79, 0x63, 0x68, 0x12, 0x11, 0x2e, 0x70, 0x62, 0x2e, 0x53, 0x65, 0x61, 0x72, 0x63, 0x68, 0x52, 0x65,
0x69, 0x6f, 0x2f, 0x68, 0x75, 0x62, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2f, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x0b, 0x2e, 0x70, 0x62, 0x2e, 0x4f, 0x75, 0x74, 0x70, 0x75,
0x67, 0x6f, 0x2f, 0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, 0x74, 0x73, 0x22, 0x00, 0x42, 0x26, 0x5a, 0x24, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63,
0x6f, 0x6d, 0x2f, 0x6c, 0x62, 0x72, 0x79, 0x69, 0x6f, 0x2f, 0x68, 0x75, 0x62, 0x2f, 0x70, 0x72,
0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2f, 0x67, 0x6f, 0x2f, 0x70, 0x62, 0x62, 0x06, 0x70, 0x72,
0x6f, 0x74, 0x6f, 0x33,
} }
var ( var (

55
server/search.go
View file

@ -410,7 +410,7 @@ func (s *Server) Search(ctx context.Context, in *pb.SearchRequest) (*pb.Outputs,
} }
var searchIndices = []string{} var searchIndices = []string{}
if s.Args.Dev { if s.Args.Dev && len(in.SearchIndices) == 0 {
// If we're running in dev mode ignore the mainnet claims index // If we're running in dev mode ignore the mainnet claims index
indices, err := client.IndexNames() indices, err := client.IndexNames()
if err != nil { if err != nil {
@ -431,6 +431,10 @@ func (s *Server) Search(ctx context.Context, in *pb.SearchRequest) (*pb.Outputs,
} }
} }
if len(in.SearchIndices) > 0 {
searchIndices = in.SearchIndices
}
fsc := elastic.NewFetchSourceContext(true).Exclude("description", "title")//.Include("_id") fsc := elastic.NewFetchSourceContext(true).Exclude("description", "title")//.Include("_id")
search := client.Search(). search := client.Search().
Index(searchIndices...). Index(searchIndices...).
@ -489,15 +493,10 @@ func (s *Server) Search(ctx context.Context, in *pb.SearchRequest) (*pb.Outputs,
//printJsonFullRecords(blockedRecords) //printJsonFullRecords(blockedRecords)
var searchIndex = "claims" //default
if len(searchIndices) > 0 {
searchIndex = searchIndices[0]
}
//Get claims for reposts //Get claims for reposts
repostClaims, repostRecords, repostedMap := getClaimsForReposts(records, client, ctx, searchIndex) repostClaims, repostRecords, repostedMap := getClaimsForReposts(records, client, ctx, searchIndices)
//get all unique channels //get all unique channels
channels, channelMap := getUniqueChannels(append(append(records, repostRecords...), blockedRecords...), client, ctx, searchIndex) channels, channelMap := getUniqueChannels(append(append(records, repostRecords...), blockedRecords...), client, ctx, searchIndices)
//add these to extra txos //add these to extra txos
extraTxos := append(repostClaims, channels...) extraTxos := append(repostClaims, channels...)
@ -544,23 +543,25 @@ func (s *Server) Search(ctx context.Context, in *pb.SearchRequest) (*pb.Outputs,
}, nil }, nil
} }
func getUniqueChannels(records []*record, client *elastic.Client, ctx context.Context, searchIndex string) ([]*pb.Output, map[string]*pb.Output) { func getUniqueChannels(records []*record, client *elastic.Client, ctx context.Context, searchIndices []string) ([]*pb.Output, map[string]*pb.Output) {
channels := make(map[string]*pb.Output) channels := make(map[string]*pb.Output)
channelsSet := make(map[string]bool) channelsSet := make(map[string]bool)
var mget = client.Mget() var mget = client.Mget()
var totalChannels = 0 var totalChannels = 0
for _, r := range records { for _, r := range records {
if r.ChannelId != "" && !channelsSet[r.ChannelId] { for _, searchIndex := range searchIndices {
channelsSet[r.ChannelId] = true if r.ChannelId != "" && !channelsSet[r.ChannelId] {
nmget := elastic.NewMultiGetItem().Id(r.ChannelId).Index(searchIndex) channelsSet[r.ChannelId] = true
mget = mget.Add(nmget) nmget := elastic.NewMultiGetItem().Id(r.ChannelId).Index(searchIndex)
totalChannels++ mget = mget.Add(nmget)
} totalChannels++
if r.CensorType != 0 && !channelsSet[r.CensoringChannelHash] { }
channelsSet[r.CensoringChannelHash] = true if r.CensorType != 0 && !channelsSet[r.CensoringChannelHash] {
nmget := elastic.NewMultiGetItem().Id(r.CensoringChannelHash).Index(searchIndex) channelsSet[r.CensoringChannelHash] = true
mget = mget.Add(nmget) nmget := elastic.NewMultiGetItem().Id(r.CensoringChannelHash).Index(searchIndex)
totalChannels++ mget = mget.Add(nmget)
totalChannels++
}
} }
} }
if totalChannels == 0 { if totalChannels == 0 {
@ -592,7 +593,7 @@ func getUniqueChannels(records []*record, client *elastic.Client, ctx context.Co
return channelTxos, channels return channelTxos, channels
} }
func getClaimsForReposts(records []*record, client *elastic.Client, ctx context.Context, searchIndex string) ([]*pb.Output, []*record, map[string]*pb.Output) { func getClaimsForReposts(records []*record, client *elastic.Client, ctx context.Context, searchIndices []string) ([]*pb.Output, []*record, map[string]*pb.Output) {
var totalReposted = 0 var totalReposted = 0
var mget = client.Mget()//.StoredFields("_id") var mget = client.Mget()//.StoredFields("_id")
@ -603,11 +604,13 @@ func getClaimsForReposts(records []*record, client *elastic.Client, ctx context.
} }
*/ */
for _, r := range records { for _, r := range records {
if r.RepostedClaimId != "" { for _, searchIndex := range searchIndices {
var nmget = elastic.NewMultiGetItem().Id(r.RepostedClaimId).Index(searchIndex) if r.RepostedClaimId != "" {
//nmget = nmget.Id(r.RepostedClaimId) var nmget = elastic.NewMultiGetItem().Id(r.RepostedClaimId).Index(searchIndex)
mget = mget.Add(nmget) //nmget = nmget.Id(r.RepostedClaimId)
totalReposted++ mget = mget.Add(nmget)
totalReposted++
}
} }
} }
//mget = mget.Add(nmget) //mget = mget.Add(nmget)