Fix search indices issue

protobuf/definitions/hub.proto

@@ -92,4 +92,5 @@ message SearchRequest {
   repeated string all_languages = 75;
   .google.protobuf.BoolValue remove_duplicates = 76;
   .google.protobuf.BoolValue no_totals = 77;
+  repeated string search_indices = 78;
 }

protobuf/go/hub.pb.go

@@ -255,6 +255,7 @@ type SearchRequest struct {
 	AllLanguages          []string               `protobuf:"bytes,75,rep,name=all_languages,json=allLanguages,proto3" json:"all_languages"`
 	RemoveDuplicates      *wrapperspb.BoolValue  `protobuf:"bytes,76,opt,name=remove_duplicates,json=removeDuplicates,proto3" json:"remove_duplicates"`
 	NoTotals              *wrapperspb.BoolValue  `protobuf:"bytes,77,opt,name=no_totals,json=noTotals,proto3" json:"no_totals"`
+	SearchIndices         []string               `protobuf:"bytes,78,rep,name=search_indices,json=searchIndices,proto3" json:"search_indices"`
 }
 
 func (x *SearchRequest) Reset() {
@@ -737,6 +738,13 @@ func (x *SearchRequest) GetNoTotals() *wrapperspb.BoolValue {
 	return nil
 }
 
+func (x *SearchRequest) GetSearchIndices() []string {
+	if x != nil {
+		return x.SearchIndices
+	}
+	return nil
+}
+
 var File_hub_proto protoreflect.FileDescriptor
 
 var file_hub_proto_rawDesc = []byte{
@@ -755,7 +763,7 @@ var file_hub_proto_rawDesc = []byte{
 	0x76, 0x61, 0x6c, 0x75, 0x65, 0x22, 0x2e, 0x0a, 0x02, 0x4f, 0x70, 0x12, 0x06, 0x0a, 0x02, 0x45,
 	0x51, 0x10, 0x00, 0x12, 0x07, 0x0a, 0x03, 0x4c, 0x54, 0x45, 0x10, 0x01, 0x12, 0x07, 0x0a, 0x03,
 	0x47, 0x54, 0x45, 0x10, 0x02, 0x12, 0x06, 0x0a, 0x02, 0x4c, 0x54, 0x10, 0x03, 0x12, 0x06, 0x0a,
-	0x02, 0x47, 0x54, 0x10, 0x04, 0x22, 0xd2, 0x16, 0x0a, 0x0d, 0x53, 0x65, 0x61, 0x72, 0x63, 0x68,
+	0x02, 0x47, 0x54, 0x10, 0x04, 0x22, 0xf9, 0x16, 0x0a, 0x0d, 0x53, 0x65, 0x61, 0x72, 0x63, 0x68,
 	0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x74, 0x65, 0x78, 0x74, 0x18,
 	0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x74, 0x65, 0x78, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x6e,
 	0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x03, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12,
@@ -936,13 +944,16 @@ var file_hub_proto_rawDesc = []byte{
 	0x73, 0x12, 0x37, 0x0a, 0x09, 0x6e, 0x6f, 0x5f, 0x74, 0x6f, 0x74, 0x61, 0x6c, 0x73, 0x18, 0x4d,
 	0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72,
 	0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x42, 0x6f, 0x6f, 0x6c, 0x56, 0x61, 0x6c, 0x75, 0x65,
-	0x52, 0x08, 0x6e, 0x6f, 0x54, 0x6f, 0x74, 0x61, 0x6c, 0x73, 0x32, 0x31, 0x0a, 0x03, 0x48, 0x75,
+	0x52, 0x08, 0x6e, 0x6f, 0x54, 0x6f, 0x74, 0x61, 0x6c, 0x73, 0x12, 0x25, 0x0a, 0x0e, 0x73, 0x65,
-	0x62, 0x12, 0x2a, 0x0a, 0x06, 0x53, 0x65, 0x61, 0x72, 0x63, 0x68, 0x12, 0x11, 0x2e, 0x70, 0x62,
+	0x61, 0x72, 0x63, 0x68, 0x5f, 0x69, 0x6e, 0x64, 0x69, 0x63, 0x65, 0x73, 0x18, 0x4e, 0x20, 0x03,
-	0x2e, 0x53, 0x65, 0x61, 0x72, 0x63, 0x68, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x0b,
+	0x28, 0x09, 0x52, 0x0d, 0x73, 0x65, 0x61, 0x72, 0x63, 0x68, 0x49, 0x6e, 0x64, 0x69, 0x63, 0x65,
-	0x2e, 0x70, 0x62, 0x2e, 0x4f, 0x75, 0x74, 0x70, 0x75, 0x74, 0x73, 0x22, 0x00, 0x42, 0x26, 0x5a,
+	0x73, 0x32, 0x31, 0x0a, 0x03, 0x48, 0x75, 0x62, 0x12, 0x2a, 0x0a, 0x06, 0x53, 0x65, 0x61, 0x72,
-	0x24, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x6c, 0x62, 0x72, 0x79,
+	0x63, 0x68, 0x12, 0x11, 0x2e, 0x70, 0x62, 0x2e, 0x53, 0x65, 0x61, 0x72, 0x63, 0x68, 0x52, 0x65,
-	0x69, 0x6f, 0x2f, 0x68, 0x75, 0x62, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2f,
+	0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x0b, 0x2e, 0x70, 0x62, 0x2e, 0x4f, 0x75, 0x74, 0x70, 0x75,
-	0x67, 0x6f, 0x2f, 0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
+	0x74, 0x73, 0x22, 0x00, 0x42, 0x26, 0x5a, 0x24, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63,
+	0x6f, 0x6d, 0x2f, 0x6c, 0x62, 0x72, 0x79, 0x69, 0x6f, 0x2f, 0x68, 0x75, 0x62, 0x2f, 0x70, 0x72,
+	0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2f, 0x67, 0x6f, 0x2f, 0x70, 0x62, 0x62, 0x06, 0x70, 0x72,
+	0x6f, 0x74, 0x6f, 0x33,
 }
 
 var (

server/search.go

@@ -410,7 +410,7 @@ func (s *Server) Search(ctx context.Context, in *pb.SearchRequest) (*pb.Outputs,
 	}
 
 	var searchIndices = []string{}
-	if s.Args.Dev {
+	if s.Args.Dev && len(in.SearchIndices) == 0 {
 		// If we're running in dev mode ignore the mainnet claims index
 		indices, err := client.IndexNames()
 		if err != nil {
@@ -431,6 +431,10 @@ func (s *Server) Search(ctx context.Context, in *pb.SearchRequest) (*pb.Outputs,
 		}
 	}
 
+	if len(in.SearchIndices) > 0 {
+		searchIndices = in.SearchIndices
+	}
+
 	fsc := elastic.NewFetchSourceContext(true).Exclude("description", "title")//.Include("_id")
 	search := client.Search().
 		Index(searchIndices...).
@@ -489,15 +493,10 @@ func (s *Server) Search(ctx context.Context, in *pb.SearchRequest) (*pb.Outputs,
 
 	//printJsonFullRecords(blockedRecords)
 
-	var searchIndex = "claims" //default
-	if len(searchIndices) > 0 {
-		searchIndex = searchIndices[0]
-	}
-
 	//Get claims for reposts
-	repostClaims, repostRecords, repostedMap := getClaimsForReposts(records, client, ctx, searchIndex)
+	repostClaims, repostRecords, repostedMap := getClaimsForReposts(records, client, ctx, searchIndices)
 	//get all unique channels
-	channels, channelMap := getUniqueChannels(append(append(records, repostRecords...), blockedRecords...), client, ctx, searchIndex)
+	channels, channelMap := getUniqueChannels(append(append(records, repostRecords...), blockedRecords...), client, ctx, searchIndices)
 	//add these to extra txos
 	extraTxos := append(repostClaims, channels...)
 
@@ -544,12 +543,13 @@ func (s *Server) Search(ctx context.Context, in *pb.SearchRequest) (*pb.Outputs,
 	}, nil
 }
 
-func getUniqueChannels(records []*record, client *elastic.Client, ctx context.Context, searchIndex string) ([]*pb.Output, map[string]*pb.Output) {
+func getUniqueChannels(records []*record, client *elastic.Client, ctx context.Context, searchIndices []string) ([]*pb.Output, map[string]*pb.Output) {
 	channels := make(map[string]*pb.Output)
 	channelsSet := make(map[string]bool)
 	var mget = client.Mget()
 	var totalChannels = 0
 	for _, r := range records {
+		for _, searchIndex := range searchIndices {
 			if r.ChannelId != "" && !channelsSet[r.ChannelId] {
 				channelsSet[r.ChannelId] = true
 				nmget := elastic.NewMultiGetItem().Id(r.ChannelId).Index(searchIndex)
@@ -563,6 +563,7 @@ func getUniqueChannels(records []*record, client *elastic.Client, ctx context.Co
 				totalChannels++
 			}
 		}
+	}
 	if totalChannels == 0 {
 		return []*pb.Output{}, make(map[string]*pb.Output)
 	}
@@ -592,7 +593,7 @@ func getUniqueChannels(records []*record, client *elastic.Client, ctx context.Co
 	return channelTxos, channels
 }
 
-func getClaimsForReposts(records []*record, client *elastic.Client, ctx context.Context, searchIndex string) ([]*pb.Output, []*record, map[string]*pb.Output) {
+func getClaimsForReposts(records []*record, client *elastic.Client, ctx context.Context, searchIndices []string) ([]*pb.Output, []*record, map[string]*pb.Output) {
 
 	var totalReposted = 0
 	var mget = client.Mget()//.StoredFields("_id")
@@ -603,6 +604,7 @@ func getClaimsForReposts(records []*record, client *elastic.Client, ctx context.
 	}
 	 */
 	for _, r := range records {
+		for _, searchIndex := range searchIndices {
 			if r.RepostedClaimId != "" {
 				var nmget = elastic.NewMultiGetItem().Id(r.RepostedClaimId).Index(searchIndex)
 				//nmget = nmget.Id(r.RepostedClaimId)
@@ -610,6 +612,7 @@ func getClaimsForReposts(records []*record, client *elastic.Client, ctx context.
 				totalReposted++
 			}
 		}
+	}
 	//mget = mget.Add(nmget)
 	if totalReposted == 0 {
 		return []*pb.Output{}, []*record{}, make(map[string]*pb.Output)