LBRYCommunity/herald.go
1
0
Fork 0
Code Issues 16 Pull requests Projects Releases 14 Packages Wiki Activity

Query caching #15

Merged
jeffreypicard merged 6 commits from query-caching into master 2021-10-05 00:05:44 +02:00
Conversation 1 Commits 6 Files changed 8 +58 -35
1 changed files with 58 additions and 35 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files
Showing only changes of commit 076adcca39 - Show all commits

31
server/search.go
View file

@ -179,10 +179,22 @@ func (s *Server) Search(ctx context.Context, in *pb.SearchRequest) (*pb.Outputs,
var pageSize = 10 var pageSize = 10
lyoshenka commented 2021-10-04 20:53:46 +02:00 (Migrated from github.com)
Review
Copy link

does this need fixing?

does this need fixing?
lyoshenka commented 2021-10-04 20:55:16 +02:00 (Migrated from github.com)
Review
Copy link

i generally don't like swallowing errors. at the very least you should log it

i generally don't like swallowing errors. at the very least you should log it
jeffreypicard commented 2021-10-04 21:59:06 +02:00 (Migrated from github.com)
Review
Copy link

No, on seconds though, I don't think so. We accept a single ES index as an argument, and in all our use cases I believe we only ever have to deal with one.

No, on seconds though, I don't think so. We accept a single ES index as an argument, and in all our use cases I believe we only ever have to deal with one.
jeffreypicard commented 2021-10-05 00:02:25 +02:00 (Migrated from github.com)
Review
Copy link

Done.

Done.
var orderBy []orderField var orderBy []orderField
var searchIndices []string var searchIndices []string
var searchResult *elastic.SearchResult = nil
client := s.EsClient client := s.EsClient
searchIndices = make([]string, 0, 1) searchIndices = make([]string, 0, 1)
searchIndices = append(searchIndices, s.Args.EsIndex) searchIndices = append(searchIndices, s.Args.EsIndex)
cacheHit := false
var cachedRecords []*record
/*
TODO: cache based on search request params
include from value and number of results.
When another search request comes in with same search params
and same or increased offset (which we currently don't even use?)
that will be a cache hit.
*/
if !cacheHit {
q := elastic.NewBoolQuery() q := elastic.NewBoolQuery()
err := s.checkQuery(in) err := s.checkQuery(in)
@ -202,7 +214,7 @@ func (s *Server) Search(ctx context.Context, in *pb.SearchRequest) (*pb.Outputs,
search = search.Sort(x.Field, x.IsAsc) search = search.Sort(x.Field, x.IsAsc)
} }
searchResult, err := search.Do(ctx) // execute searchResult, err = search.Do(ctx) // execute
if err != nil && elastic.IsNotFound(err) { if err != nil && elastic.IsNotFound(err) {
log.Println("Index returned 404! Check writer. Index: ", searchIndices) log.Println("Index returned 404! Check writer. Index: ", searchIndices)
return &pb.Outputs{}, nil return &pb.Outputs{}, nil
@ -215,7 +227,13 @@ func (s *Server) Search(ctx context.Context, in *pb.SearchRequest) (*pb.Outputs,
log.Printf("%s: found %d results in %dms\n", in.Text, len(searchResult.Hits.Hits), searchResult.TookInMillis) log.Printf("%s: found %d results in %dms\n", in.Text, len(searchResult.Hits.Hits), searchResult.TookInMillis)
txos, extraTxos, blocked := s.postProcessResults(ctx, client, searchResult, in, pageSize, from, searchIndices) cachedRecords = make([]*record, 0, 0)
} else {
//TODO fill cached records here
cachedRecords = make([]*record, 0, 0)
}
txos, extraTxos, blocked := s.postProcessResults(ctx, client, searchResult, in, pageSize, from, searchIndices, cachedRecords)
t1 := time.Now() t1 := time.Now()
@ -272,13 +290,15 @@ func (s *Server) postProcessResults(
in *pb.SearchRequest, in *pb.SearchRequest,
pageSize int, pageSize int,
from int, from int,
searchIndices []string) ([]*pb.Output, []*pb.Output, []*pb.Blocked) { searchIndices []string,
cachedRecords []*record) ([]*pb.Output, []*pb.Output, []*pb.Blocked) {
var txos []*pb.Output var txos []*pb.Output
var records []*record var records []*record
var blockedRecords []*record var blockedRecords []*record
var blocked []*pb.Blocked var blocked []*pb.Blocked
var blockedMap map[string]*pb.Blocked var blockedMap map[string]*pb.Blocked
if len(cachedRecords) < 0 {
records = make([]*record, 0, searchResult.TotalHits()) records = make([]*record, 0, searchResult.TotalHits())
var r record var r record
@ -287,6 +307,9 @@ func (s *Server) postProcessResults(
records = append(records, &t) records = append(records, &t)
} }
} }
} else {
records = cachedRecords
}
//printJsonFullResults(searchResult) //printJsonFullResults(searchResult)
records, blockedRecords, blockedMap = removeBlocked(records) records, blockedRecords, blockedMap = removeBlocked(records)
@ -617,7 +640,7 @@ func (s *Server) getUniqueChannels(records []*record, client *elastic.Client, ct
return channelTxos, channels return channelTxos, channels
} }
func (s * Server) getClaimsForReposts(ctx context.Context, client *elastic.Client, records []*record, searchIndices []string) ([]*pb.Output, []*record, map[string]*pb.Output) { func (s *Server) getClaimsForReposts(ctx context.Context, client *elastic.Client, records []*record, searchIndices []string) ([]*pb.Output, []*record, map[string]*pb.Output) {
var totalReposted = 0 var totalReposted = 0
var mget = client.Mget() //.StoredFields("_id") var mget = client.Mget() //.StoredFields("_id")