Merge pull request #17 from lbryio/jeffreypicard/8/add-prometheus-metrics

Record metrics
2021-10-21 08:32:43 -04:00 · 2021-10-21 08:32:43 -04:00 · e12c26fe20
commit e12c26fe20
parent d69af2db8a f0369e12e7
5 changed files with 39 additions and 75 deletions
--- a/go.mod
+++ b/go.mod
@ -5,7 +5,6 @@ go 1.16
 require (
 	github.com/ReneKroon/ttlcache/v2 v2.8.1
 	github.com/akamensky/argparse v1.2.2
-	github.com/btcsuite/btcutil v1.0.2
 	github.com/lbryio/lbry.go/v2 v2.7.2-0.20210625145058-2b155597bf57
 	github.com/olivere/elastic/v7 v7.0.24
 	github.com/prometheus/client_golang v1.11.0
--- a/go.sum
+++ b/go.sum
@ -18,12 +18,9 @@ github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+Ce
 github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
 github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
 github.com/btcsuite/btcd v0.0.0-20190213025234-306aecffea32/go.mod h1:DrZx5ec/dmnfpw9KyYoQyYo7d0KEvTkk/5M/vbZjAr8=
-github.com/btcsuite/btcd v0.20.1-beta/go.mod h1:wVuoA8VJLEcwgqHBwHmzLRazpKxTv13Px/pDuV7OomQ=
 github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f/go.mod h1:TdznJufoqS23FtqVCzL0ZqgP5MqXbb4fg/WgDys70nA=
 github.com/btcsuite/btcutil v0.0.0-20190207003914-4c204d697803/go.mod h1:+5NJ2+qvTyV9exUAL/rxXi3DcLg2Ts+ymUAY5y4NvMg=
 github.com/btcsuite/btcutil v0.0.0-20190425235716-9e5f4b9a998d/go.mod h1:+5NJ2+qvTyV9exUAL/rxXi3DcLg2Ts+ymUAY5y4NvMg=
-github.com/btcsuite/btcutil v1.0.2 h1:9iZ1Terx9fMIOtq1VrwdqfsATL9MC2l8ZrUY6YZ2uts=
-github.com/btcsuite/btcutil v1.0.2/go.mod h1:j9HUFwoQRsZL3V4n+qG+CUnEGHOarIxfC3Le2Yhbcts=
 github.com/btcsuite/go-socks v0.0.0-20170105172521-4720035b7bfd/go.mod h1:HHNXQzUsZCxOoE+CPiyCTO6x34Zs86zZUiwtpXoGdtg=
 github.com/btcsuite/goleveldb v0.0.0-20160330041536-7834afc9e8cd/go.mod h1:F+uVaaLLH7j4eDXPRvw78tMflu7Ie2bzYOH4Y8rRKBY=
 github.com/btcsuite/snappy-go v0.0.0-20151229074030-0bdef8d06723/go.mod h1:8woku9dyThutzjeg+3xrA5iCpBRH8XEEg3lh6TiUghc=
@ -205,7 +202,6 @@ golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnf
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20191002192127-34f69633bfdc/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
-golang.org/x/crypto v0.0.0-20200115085410-6d4e4cb37c7d/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
--- a/internal/metrics/metrics.go
+++ b/internal/metrics/metrics.go
@ -3,58 +3,24 @@ package metrics
 import (
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promauto"
-	"time"
 )

 var (
-	PingsCounter = promauto.NewCounter(prometheus.CounterOpts{
-		Name: "pings",
-		Help: "Number of pings",
-	})
-	ZeroChannelsCounter = promauto.NewCounter(prometheus.CounterOpts{
-		Name: "zero_channels_counter",
-		Help: "Number of times zero channels were returned in getUniqueChannels",
-	})
-	NoRepostedCounter = promauto.NewCounter(prometheus.CounterOpts{
-		Name: "no_reposted_counter",
-		Help: "Number of times zero reposted were returned in getClaimsForRepost",
-	})
-	GetUniqueChannelsErrorCounter = promauto.NewCounter(prometheus.CounterOpts{
-		Name: "get_unique_channels_error_counter",
-		Help: "Number of errors",
-	})
-	JsonErrorCounter = promauto.NewCounter(prometheus.CounterOpts{
-		Name: "json_error_counter",
-		Help: "JSON parsing errors",
-	})
-	MgetErrorCounter = promauto.NewCounter(prometheus.CounterOpts{
-		Name: "mget_error_counter",
-		Help: "Mget errors",
-	})
-	SearchCounter = promauto.NewCounter(prometheus.CounterOpts{
-		Name: "searche_counter",
+	HistogramBuckets = []float64{0.005, 0.025, 0.05, 0.1, 0.25, 0.4, 1, 2, 5, 10, 20, 60, 120, 300}
+	// These mirror counters from the python code
+	RequestsCount = promauto.NewCounterVec(prometheus.CounterOpts{
+		Name: "requests_count",
 		Help: "Total number of searches",
-	})
-	ClientCreationErrorCounter =  promauto.NewCounter(prometheus.CounterOpts{
-		Name: "client_creation_error_counter",
-		Help: "Number of errors",
-	})
-	SearchErrorCounter = promauto.NewCounter(prometheus.CounterOpts{
-		Name: "search_error_counter",
-		Help: "Number of errors",
-	})
-	FatalErrorCounter = promauto.NewCounter(prometheus.CounterOpts{
-		Name: "fatal_error_counter",
-		Help: "Number of errors",
-	})
-	ErrorCounter = promauto.NewCounter(prometheus.CounterOpts{
-		Name: "error_counter",
-		Help: "Number of errors",
-	})
-	QueryTime = promauto.NewSummary(prometheus.SummaryOpts{
-		MaxAge: time.Hour,
+	}, []string{"method"})
+	// These are unique to the go code
+	ErrorsCounter = promauto.NewCounterVec(prometheus.CounterOpts{
+		Name: "errors",
+		Help: "Number of errors by type",
+	}, []string{"error_type"})
+	QueryTime = promauto.NewHistogramVec(prometheus.HistogramOpts{
 		Name: "query_time",
-		Help: "hourly summary of query time",
-	})
+		Help: "Histogram of query times",
+		Buckets: HistogramBuckets,
+	}, []string{"method"})
 )

--- a/server/search.go
+++ b/server/search.go
@ -13,9 +13,11 @@ import (

 	//"github.com/lbryio/hub/schema"

+	"github.com/lbryio/hub/internal/metrics"
 	pb "github.com/lbryio/hub/protobuf/go"
 	"github.com/lbryio/lbry.go/v2/extras/util"
 	"github.com/olivere/elastic/v7"
+	"github.com/prometheus/client_golang/prometheus"
 	"golang.org/x/text/cases"
 	"golang.org/x/text/language"
 	"google.golang.org/protobuf/encoding/protojson"
@ -44,7 +46,7 @@ type record struct {
 	RepostCount        uint32  `json:"repost_count"`
 	EffectiveAmount    uint64  `json:"effective_amount"`
 	SupportAmount      uint64  `json:"support_amount"`
-	TrendingScore      float64 `json:"TrendingScore"`
+	TrendingScore      float64 `json:"trending_score"`
 	ClaimName          string  `json:"claim_name"`
 }

@ -125,14 +127,9 @@ func AddInvertibleField(q *elastic.BoolQuery, field *pb.InvertibleField, name st
 		return q.Must(elastic.NewTermsQuery(name, searchVals...))
 	}
 }
-func (s *Server) recordErrorAndReturn(err error, typ string) (interface{}, error) {
-	// TODO record metric
-	log.Println(err)
-	return nil, err
-}

 func (s *Server) recordErrorAndDie(err error) {
-	// TODO record metric fatal_error_counter
+	metrics.ErrorsCounter.With(prometheus.Labels{"error_type": "fatal"}).Inc()
 	log.Fatalln(err)
 }

@ -173,8 +170,14 @@ func RoundUpReleaseTime(q *elastic.BoolQuery, rq *pb.RangeField, name string) *e
 // 8) return streams referenced by repost and all channel referenced in extra_txos
 //*/
 func (s *Server) Search(ctx context.Context, in *pb.SearchRequest) (*pb.Outputs, error) {
-	// TODO record metric search_counter
-	t0 := time.Now()
+	metrics.RequestsCount.With(prometheus.Labels{"method": "search"}).Inc()
+	defer func(t time.Time) {
+		delta := time.Since(t).Seconds()
+		metrics.
+			QueryTime.
+			With(prometheus.Labels{"method": "search"}).
+			Observe(delta)
+	}(time.Now())

 	var from = 0
 	var pageSize = 10
@ -256,7 +259,7 @@ func (s *Server) Search(ctx context.Context, in *pb.SearchRequest) (*pb.Outputs,
 			return &pb.Outputs{}, nil

 		} else if err != nil {
-			s.recordErrorAndReturn(err, "search_errors")
+			metrics.ErrorsCounter.With(prometheus.Labels{"error_type": "search"}).Inc()
 			log.Println("Error executing query: ", err)
 			return nil, err
 		}
@ -275,12 +278,6 @@ func (s *Server) Search(ctx context.Context, in *pb.SearchRequest) (*pb.Outputs,

 	txos, extraTxos, blocked := s.postProcessResults(ctx, client, records, in, pageSize, from, searchIndices)

-	t1 := time.Now()
-
-	delta := t1.Unix() - t0.Unix()
-	log.Printf("delta: %d\n", delta)
-	// TODO record metric query_time
-
 	if in.NoTotals {
 		return &pb.Outputs{
 			Txos:      txos,
@ -650,13 +647,13 @@ func (s *Server) getUniqueChannels(records []*record, client *elastic.Client, ct
 		}
 	}
 	if totalChannels == 0 {
-		s.recordErrorAndReturn(nil, "zero_channels_counter")
 		return []*pb.Output{}, make(map[string]*pb.Output)
 	}

 	res, err := mget.Do(ctx)
 	if err != nil {
-		s.recordErrorAndReturn(err, "get_unique_channels_errors")
+		metrics.ErrorsCounter.With(prometheus.Labels{"error_type": "get_unique_channels"}).Inc()
+		log.Println(err)
 		return []*pb.Output{}, make(map[string]*pb.Output)
 	}

@ -668,7 +665,8 @@ func (s *Server) getUniqueChannels(records []*record, client *elastic.Client, ct
 		var r record
 		err := json.Unmarshal(doc.Source, &r)
 		if err != nil {
-			s.recordErrorAndReturn(err, "json_errors")
+			metrics.ErrorsCounter.With(prometheus.Labels{"error_type": "json"}).Inc()
+			log.Println(err)
 			return []*pb.Output{}, make(map[string]*pb.Output)
 		}
 		channelTxos[i] = r.recordToOutput()
@ -702,13 +700,13 @@ func (s *Server) getClaimsForReposts(ctx context.Context, client *elastic.Client
 	}
 	//mget = mget.Add(nmget)
 	if totalReposted == 0 {
-		// TODO record metric no_reposted_counter
 		return []*pb.Output{}, []*record{}, make(map[string]*pb.Output)
 	}

 	res, err := mget.Do(ctx)
 	if err != nil {
-		s.recordErrorAndReturn(err, "mget_error_counter")
+		metrics.ErrorsCounter.With(prometheus.Labels{"error_type": "mget"}).Inc()
+		log.Println(err)
 		return []*pb.Output{}, []*record{}, make(map[string]*pb.Output)
 	}

@ -721,7 +719,8 @@ func (s *Server) getClaimsForReposts(ctx context.Context, client *elastic.Client
 		var r record
 		err := json.Unmarshal(doc.Source, &r)
 		if err != nil {
-			s.recordErrorAndReturn(err, "json_error_counter")
+			metrics.ErrorsCounter.With(prometheus.Labels{"error_type": "json"}).Inc()
+			log.Println(err)
 			return []*pb.Output{}, []*record{}, make(map[string]*pb.Output)
 		}
 		claims[i] = r.recordToOutput()
--- a/server/server.go
+++ b/server/server.go
@ -12,9 +12,11 @@ import (
 	"time"

 	"github.com/ReneKroon/ttlcache/v2"
+	"github.com/lbryio/hub/internal/metrics"
 	"github.com/lbryio/hub/meta"
 	pb "github.com/lbryio/hub/protobuf/go"
 	"github.com/olivere/elastic/v7"
+	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
 	"google.golang.org/grpc"
 )
@ -178,9 +180,11 @@ func (s *Server) Hello(context context.Context, args *FederatedServer) (*Federat
 }

 func (s *Server) Ping(context context.Context, args *pb.EmptyMessage) (*pb.StringValue, error) {
+	metrics.RequestsCount.With(prometheus.Labels{"method": "ping"}).Inc()
 	return &pb.StringValue{Value: "Hello, world!"}, nil
 }

 func (s *Server) Version(context context.Context, args *pb.EmptyMessage) (*pb.StringValue, error) {
+	metrics.RequestsCount.With(prometheus.Labels{"method": "version"}).Inc()
 	return &pb.StringValue{Value: getVersion()}, nil
 }