From e043ca89c9255531b36c60c787038c7f0f0b9c8e Mon Sep 17 00:00:00 2001 From: Jeffrey Picard Date: Fri, 15 Oct 2021 16:26:33 -0400 Subject: [PATCH 1/2] Record metrics There are a couple metrics that try to mirror the ones from python, many of the python ones don't apply, and vice-versa. --- go.mod | 1 - go.sum | 4 ---- internal/metrics/metrics.go | 36 +++++++++++++++-------------- server/search.go | 46 ++++++++++++++++++++----------------- server/server.go | 2 ++ 5 files changed, 46 insertions(+), 43 deletions(-) diff --git a/go.mod b/go.mod index c58e386..98831c8 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,6 @@ go 1.16 require ( github.com/ReneKroon/ttlcache/v2 v2.8.1 github.com/akamensky/argparse v1.2.2 - github.com/btcsuite/btcutil v1.0.2 github.com/lbryio/lbry.go/v2 v2.7.2-0.20210625145058-2b155597bf57 github.com/olivere/elastic/v7 v7.0.24 github.com/prometheus/client_golang v1.11.0 diff --git a/go.sum b/go.sum index 6c0e5c7..980e6d7 100644 --- a/go.sum +++ b/go.sum @@ -18,12 +18,9 @@ github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+Ce github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/btcsuite/btcd v0.0.0-20190213025234-306aecffea32/go.mod h1:DrZx5ec/dmnfpw9KyYoQyYo7d0KEvTkk/5M/vbZjAr8= -github.com/btcsuite/btcd v0.20.1-beta/go.mod h1:wVuoA8VJLEcwgqHBwHmzLRazpKxTv13Px/pDuV7OomQ= github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f/go.mod h1:TdznJufoqS23FtqVCzL0ZqgP5MqXbb4fg/WgDys70nA= github.com/btcsuite/btcutil v0.0.0-20190207003914-4c204d697803/go.mod h1:+5NJ2+qvTyV9exUAL/rxXi3DcLg2Ts+ymUAY5y4NvMg= github.com/btcsuite/btcutil v0.0.0-20190425235716-9e5f4b9a998d/go.mod h1:+5NJ2+qvTyV9exUAL/rxXi3DcLg2Ts+ymUAY5y4NvMg= -github.com/btcsuite/btcutil v1.0.2 h1:9iZ1Terx9fMIOtq1VrwdqfsATL9MC2l8ZrUY6YZ2uts= -github.com/btcsuite/btcutil v1.0.2/go.mod h1:j9HUFwoQRsZL3V4n+qG+CUnEGHOarIxfC3Le2Yhbcts= github.com/btcsuite/go-socks v0.0.0-20170105172521-4720035b7bfd/go.mod h1:HHNXQzUsZCxOoE+CPiyCTO6x34Zs86zZUiwtpXoGdtg= github.com/btcsuite/goleveldb v0.0.0-20160330041536-7834afc9e8cd/go.mod h1:F+uVaaLLH7j4eDXPRvw78tMflu7Ie2bzYOH4Y8rRKBY= github.com/btcsuite/snappy-go v0.0.0-20151229074030-0bdef8d06723/go.mod h1:8woku9dyThutzjeg+3xrA5iCpBRH8XEEg3lh6TiUghc= @@ -205,7 +202,6 @@ golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnf golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191002192127-34f69633bfdc/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20200115085410-6d4e4cb37c7d/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= diff --git a/internal/metrics/metrics.go b/internal/metrics/metrics.go index f095f4d..f8a36ef 100644 --- a/internal/metrics/metrics.go +++ b/internal/metrics/metrics.go @@ -1,12 +1,26 @@ package metrics import ( + "github.com/lbryio/hub/meta" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promauto" - "time" ) var ( + HistogramBuckets = []float64{0.005, 0.025, 0.05, 0.1, 0.25, 0.4, 1, 2, 5, 10, 20, 60, 120, 300} + // These mirror counters from the python code + RequestsCount = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "requests_count", + Help: "Total number of searches", + }, []string{"method"}) + SessionCount = promauto.NewGauge(prometheus.GaugeOpts{ + Name: "session_count", + Help: "Number of client sessions", + ConstLabels: map[string]string{ + "version": meta.Version, + }, + }) + // These are unique to the go code PingsCounter = promauto.NewCounter(prometheus.CounterOpts{ Name: "pings", Help: "Number of pings", @@ -31,14 +45,6 @@ var ( Name: "mget_error_counter", Help: "Mget errors", }) - SearchCounter = promauto.NewCounter(prometheus.CounterOpts{ - Name: "searche_counter", - Help: "Total number of searches", - }) - ClientCreationErrorCounter = promauto.NewCounter(prometheus.CounterOpts{ - Name: "client_creation_error_counter", - Help: "Number of errors", - }) SearchErrorCounter = promauto.NewCounter(prometheus.CounterOpts{ Name: "search_error_counter", Help: "Number of errors", @@ -47,14 +53,10 @@ var ( Name: "fatal_error_counter", Help: "Number of errors", }) - ErrorCounter = promauto.NewCounter(prometheus.CounterOpts{ - Name: "error_counter", - Help: "Number of errors", - }) - QueryTime = promauto.NewSummary(prometheus.SummaryOpts{ - MaxAge: time.Hour, + QueryTime = promauto.NewHistogramVec(prometheus.HistogramOpts{ Name: "query_time", - Help: "hourly summary of query time", - }) + Help: "Histogram of query times", + Buckets: HistogramBuckets, + }, []string{"method"}) ) diff --git a/server/search.go b/server/search.go index 69e4620..545958a 100644 --- a/server/search.go +++ b/server/search.go @@ -13,9 +13,11 @@ import ( //"github.com/lbryio/hub/schema" + "github.com/lbryio/hub/internal/metrics" pb "github.com/lbryio/hub/protobuf/go" "github.com/lbryio/lbry.go/v2/extras/util" "github.com/olivere/elastic/v7" + "github.com/prometheus/client_golang/prometheus" "golang.org/x/text/cases" "golang.org/x/text/language" "google.golang.org/protobuf/encoding/protojson" @@ -44,7 +46,7 @@ type record struct { RepostCount uint32 `json:"repost_count"` EffectiveAmount uint64 `json:"effective_amount"` SupportAmount uint64 `json:"support_amount"` - TrendingScore float64 `json:"TrendingScore"` + TrendingScore float64 `json:"trending_score"` ClaimName string `json:"claim_name"` } @@ -125,14 +127,10 @@ func AddInvertibleField(q *elastic.BoolQuery, field *pb.InvertibleField, name st return q.Must(elastic.NewTermsQuery(name, searchVals...)) } } -func (s *Server) recordErrorAndReturn(err error, typ string) (interface{}, error) { - // TODO record metric - log.Println(err) - return nil, err -} func (s *Server) recordErrorAndDie(err error) { // TODO record metric fatal_error_counter + metrics.FatalErrorCounter.Inc() log.Fatalln(err) } @@ -173,8 +171,16 @@ func RoundUpReleaseTime(q *elastic.BoolQuery, rq *pb.RangeField, name string) *e // 8) return streams referenced by repost and all channel referenced in extra_txos //*/ func (s *Server) Search(ctx context.Context, in *pb.SearchRequest) (*pb.Outputs, error) { - // TODO record metric search_counter - t0 := time.Now() + metrics.RequestsCount.With(prometheus.Labels{"method": "search"}).Inc() + metrics.SessionCount.Inc() + defer func() {metrics.SessionCount.Dec()}() + defer func(t time.Time) { + delta := time.Since(t).Seconds() + metrics. + QueryTime. + With(prometheus.Labels{"method": "search"}). + Observe(delta) + }(time.Now()) var from = 0 var pageSize = 10 @@ -256,7 +262,7 @@ func (s *Server) Search(ctx context.Context, in *pb.SearchRequest) (*pb.Outputs, return &pb.Outputs{}, nil } else if err != nil { - s.recordErrorAndReturn(err, "search_errors") + metrics.SearchErrorCounter.Inc() log.Println("Error executing query: ", err) return nil, err } @@ -275,12 +281,6 @@ func (s *Server) Search(ctx context.Context, in *pb.SearchRequest) (*pb.Outputs, txos, extraTxos, blocked := s.postProcessResults(ctx, client, records, in, pageSize, from, searchIndices) - t1 := time.Now() - - delta := t1.Unix() - t0.Unix() - log.Printf("delta: %d\n", delta) - // TODO record metric query_time - if in.NoTotals { return &pb.Outputs{ Txos: txos, @@ -650,13 +650,14 @@ func (s *Server) getUniqueChannels(records []*record, client *elastic.Client, ct } } if totalChannels == 0 { - s.recordErrorAndReturn(nil, "zero_channels_counter") + metrics.ZeroChannelsCounter.Inc() return []*pb.Output{}, make(map[string]*pb.Output) } res, err := mget.Do(ctx) if err != nil { - s.recordErrorAndReturn(err, "get_unique_channels_errors") + metrics.GetUniqueChannelsErrorCounter.Inc() + log.Println(err) return []*pb.Output{}, make(map[string]*pb.Output) } @@ -668,7 +669,8 @@ func (s *Server) getUniqueChannels(records []*record, client *elastic.Client, ct var r record err := json.Unmarshal(doc.Source, &r) if err != nil { - s.recordErrorAndReturn(err, "json_errors") + metrics.JsonErrorCounter.Inc() + log.Println(err) return []*pb.Output{}, make(map[string]*pb.Output) } channelTxos[i] = r.recordToOutput() @@ -702,13 +704,14 @@ func (s *Server) getClaimsForReposts(ctx context.Context, client *elastic.Client } //mget = mget.Add(nmget) if totalReposted == 0 { - // TODO record metric no_reposted_counter + metrics.NoRepostedCounter.Inc() return []*pb.Output{}, []*record{}, make(map[string]*pb.Output) } res, err := mget.Do(ctx) if err != nil { - s.recordErrorAndReturn(err, "mget_error_counter") + metrics.MgetErrorCounter.Inc() + log.Println(err) return []*pb.Output{}, []*record{}, make(map[string]*pb.Output) } @@ -721,7 +724,8 @@ func (s *Server) getClaimsForReposts(ctx context.Context, client *elastic.Client var r record err := json.Unmarshal(doc.Source, &r) if err != nil { - s.recordErrorAndReturn(err, "json_error_counter") + metrics.JsonErrorCounter.Inc() + log.Println(err) return []*pb.Output{}, []*record{}, make(map[string]*pb.Output) } claims[i] = r.recordToOutput() diff --git a/server/server.go b/server/server.go index 66a88b6..c70a102 100644 --- a/server/server.go +++ b/server/server.go @@ -12,6 +12,7 @@ import ( "time" "github.com/ReneKroon/ttlcache/v2" + "github.com/lbryio/hub/internal/metrics" "github.com/lbryio/hub/meta" pb "github.com/lbryio/hub/protobuf/go" "github.com/olivere/elastic/v7" @@ -178,6 +179,7 @@ func (s *Server) Hello(context context.Context, args *FederatedServer) (*Federat } func (s *Server) Ping(context context.Context, args *pb.EmptyMessage) (*pb.StringValue, error) { + metrics.PingsCounter.Inc() return &pb.StringValue{Value: "Hello, world!"}, nil } From f0369e12e7138f9778ad33c8e09893b2ceaf7ea2 Mon Sep 17 00:00:00 2001 From: Jeffrey Picard Date: Thu, 21 Oct 2021 02:52:03 -0400 Subject: [PATCH 2/2] Cleanup metrics Consolidated error metrics into a vector counter with a label, removed other uneeded counters. --- internal/metrics/metrics.go | 44 ++++--------------------------------- server/search.go | 17 +++++--------- server/server.go | 4 +++- 3 files changed, 13 insertions(+), 52 deletions(-) diff --git a/internal/metrics/metrics.go b/internal/metrics/metrics.go index f8a36ef..c78f03e 100644 --- a/internal/metrics/metrics.go +++ b/internal/metrics/metrics.go @@ -1,7 +1,6 @@ package metrics import ( - "github.com/lbryio/hub/meta" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promauto" ) @@ -13,46 +12,11 @@ var ( Name: "requests_count", Help: "Total number of searches", }, []string{"method"}) - SessionCount = promauto.NewGauge(prometheus.GaugeOpts{ - Name: "session_count", - Help: "Number of client sessions", - ConstLabels: map[string]string{ - "version": meta.Version, - }, - }) // These are unique to the go code - PingsCounter = promauto.NewCounter(prometheus.CounterOpts{ - Name: "pings", - Help: "Number of pings", - }) - ZeroChannelsCounter = promauto.NewCounter(prometheus.CounterOpts{ - Name: "zero_channels_counter", - Help: "Number of times zero channels were returned in getUniqueChannels", - }) - NoRepostedCounter = promauto.NewCounter(prometheus.CounterOpts{ - Name: "no_reposted_counter", - Help: "Number of times zero reposted were returned in getClaimsForRepost", - }) - GetUniqueChannelsErrorCounter = promauto.NewCounter(prometheus.CounterOpts{ - Name: "get_unique_channels_error_counter", - Help: "Number of errors", - }) - JsonErrorCounter = promauto.NewCounter(prometheus.CounterOpts{ - Name: "json_error_counter", - Help: "JSON parsing errors", - }) - MgetErrorCounter = promauto.NewCounter(prometheus.CounterOpts{ - Name: "mget_error_counter", - Help: "Mget errors", - }) - SearchErrorCounter = promauto.NewCounter(prometheus.CounterOpts{ - Name: "search_error_counter", - Help: "Number of errors", - }) - FatalErrorCounter = promauto.NewCounter(prometheus.CounterOpts{ - Name: "fatal_error_counter", - Help: "Number of errors", - }) + ErrorsCounter = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "errors", + Help: "Number of errors by type", + }, []string{"error_type"}) QueryTime = promauto.NewHistogramVec(prometheus.HistogramOpts{ Name: "query_time", Help: "Histogram of query times", diff --git a/server/search.go b/server/search.go index 545958a..44b754d 100644 --- a/server/search.go +++ b/server/search.go @@ -129,8 +129,7 @@ func AddInvertibleField(q *elastic.BoolQuery, field *pb.InvertibleField, name st } func (s *Server) recordErrorAndDie(err error) { - // TODO record metric fatal_error_counter - metrics.FatalErrorCounter.Inc() + metrics.ErrorsCounter.With(prometheus.Labels{"error_type": "fatal"}).Inc() log.Fatalln(err) } @@ -172,8 +171,6 @@ func RoundUpReleaseTime(q *elastic.BoolQuery, rq *pb.RangeField, name string) *e //*/ func (s *Server) Search(ctx context.Context, in *pb.SearchRequest) (*pb.Outputs, error) { metrics.RequestsCount.With(prometheus.Labels{"method": "search"}).Inc() - metrics.SessionCount.Inc() - defer func() {metrics.SessionCount.Dec()}() defer func(t time.Time) { delta := time.Since(t).Seconds() metrics. @@ -262,7 +259,7 @@ func (s *Server) Search(ctx context.Context, in *pb.SearchRequest) (*pb.Outputs, return &pb.Outputs{}, nil } else if err != nil { - metrics.SearchErrorCounter.Inc() + metrics.ErrorsCounter.With(prometheus.Labels{"error_type": "search"}).Inc() log.Println("Error executing query: ", err) return nil, err } @@ -650,13 +647,12 @@ func (s *Server) getUniqueChannels(records []*record, client *elastic.Client, ct } } if totalChannels == 0 { - metrics.ZeroChannelsCounter.Inc() return []*pb.Output{}, make(map[string]*pb.Output) } res, err := mget.Do(ctx) if err != nil { - metrics.GetUniqueChannelsErrorCounter.Inc() + metrics.ErrorsCounter.With(prometheus.Labels{"error_type": "get_unique_channels"}).Inc() log.Println(err) return []*pb.Output{}, make(map[string]*pb.Output) } @@ -669,7 +665,7 @@ func (s *Server) getUniqueChannels(records []*record, client *elastic.Client, ct var r record err := json.Unmarshal(doc.Source, &r) if err != nil { - metrics.JsonErrorCounter.Inc() + metrics.ErrorsCounter.With(prometheus.Labels{"error_type": "json"}).Inc() log.Println(err) return []*pb.Output{}, make(map[string]*pb.Output) } @@ -704,13 +700,12 @@ func (s *Server) getClaimsForReposts(ctx context.Context, client *elastic.Client } //mget = mget.Add(nmget) if totalReposted == 0 { - metrics.NoRepostedCounter.Inc() return []*pb.Output{}, []*record{}, make(map[string]*pb.Output) } res, err := mget.Do(ctx) if err != nil { - metrics.MgetErrorCounter.Inc() + metrics.ErrorsCounter.With(prometheus.Labels{"error_type": "mget"}).Inc() log.Println(err) return []*pb.Output{}, []*record{}, make(map[string]*pb.Output) } @@ -724,7 +719,7 @@ func (s *Server) getClaimsForReposts(ctx context.Context, client *elastic.Client var r record err := json.Unmarshal(doc.Source, &r) if err != nil { - metrics.JsonErrorCounter.Inc() + metrics.ErrorsCounter.With(prometheus.Labels{"error_type": "json"}).Inc() log.Println(err) return []*pb.Output{}, []*record{}, make(map[string]*pb.Output) } diff --git a/server/server.go b/server/server.go index c70a102..b73a6ec 100644 --- a/server/server.go +++ b/server/server.go @@ -16,6 +16,7 @@ import ( "github.com/lbryio/hub/meta" pb "github.com/lbryio/hub/protobuf/go" "github.com/olivere/elastic/v7" + "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promhttp" "google.golang.org/grpc" ) @@ -179,10 +180,11 @@ func (s *Server) Hello(context context.Context, args *FederatedServer) (*Federat } func (s *Server) Ping(context context.Context, args *pb.EmptyMessage) (*pb.StringValue, error) { - metrics.PingsCounter.Inc() + metrics.RequestsCount.With(prometheus.Labels{"method": "ping"}).Inc() return &pb.StringValue{Value: "Hello, world!"}, nil } func (s *Server) Version(context context.Context, args *pb.EmptyMessage) (*pb.StringValue, error) { + metrics.RequestsCount.With(prometheus.Labels{"method": "version"}).Inc() return &pb.StringValue{Value: getVersion()}, nil }