Record metrics
There are a couple metrics that try to mirror the ones from python, many of the python ones don't apply, and vice-versa.
This commit is contained in:
Jeffrey Picard
parent
0d51cbfde4
commit
e043ca89c9
5 changed files with 46 additions and 43 deletions
1
go.mod
1
go.mod
|
|
@ -5,7 +5,6 @@ go 1.16
|
|||
require (
|
||||
github.com/ReneKroon/ttlcache/v2 v2.8.1
|
||||
github.com/akamensky/argparse v1.2.2
|
||||
github.com/btcsuite/btcutil v1.0.2
|
||||
github.com/lbryio/lbry.go/v2 v2.7.2-0.20210625145058-2b155597bf57
|
||||
github.com/olivere/elastic/v7 v7.0.24
|
||||
github.com/prometheus/client_golang v1.11.0
|
||||
|
|
|
|||
4
go.sum
4
go.sum
|
|
@ -18,12 +18,9 @@ github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+Ce
|
|||
github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
|
||||
github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
|
||||
github.com/btcsuite/btcd v0.0.0-20190213025234-306aecffea32/go.mod h1:DrZx5ec/dmnfpw9KyYoQyYo7d0KEvTkk/5M/vbZjAr8=
|
||||
github.com/btcsuite/btcd v0.20.1-beta/go.mod h1:wVuoA8VJLEcwgqHBwHmzLRazpKxTv13Px/pDuV7OomQ=
|
||||
github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f/go.mod h1:TdznJufoqS23FtqVCzL0ZqgP5MqXbb4fg/WgDys70nA=
|
||||
github.com/btcsuite/btcutil v0.0.0-20190207003914-4c204d697803/go.mod h1:+5NJ2+qvTyV9exUAL/rxXi3DcLg2Ts+ymUAY5y4NvMg=
|
||||
github.com/btcsuite/btcutil v0.0.0-20190425235716-9e5f4b9a998d/go.mod h1:+5NJ2+qvTyV9exUAL/rxXi3DcLg2Ts+ymUAY5y4NvMg=
|
||||
github.com/btcsuite/btcutil v1.0.2 h1:9iZ1Terx9fMIOtq1VrwdqfsATL9MC2l8ZrUY6YZ2uts=
|
||||
github.com/btcsuite/btcutil v1.0.2/go.mod h1:j9HUFwoQRsZL3V4n+qG+CUnEGHOarIxfC3Le2Yhbcts=
|
||||
github.com/btcsuite/go-socks v0.0.0-20170105172521-4720035b7bfd/go.mod h1:HHNXQzUsZCxOoE+CPiyCTO6x34Zs86zZUiwtpXoGdtg=
|
||||
github.com/btcsuite/goleveldb v0.0.0-20160330041536-7834afc9e8cd/go.mod h1:F+uVaaLLH7j4eDXPRvw78tMflu7Ie2bzYOH4Y8rRKBY=
|
||||
github.com/btcsuite/snappy-go v0.0.0-20151229074030-0bdef8d06723/go.mod h1:8woku9dyThutzjeg+3xrA5iCpBRH8XEEg3lh6TiUghc=
|
||||
|
|
@ -205,7 +202,6 @@ golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnf
|
|||
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
||||
golang.org/x/crypto v0.0.0-20191002192127-34f69633bfdc/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
|
||||
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
|
||||
golang.org/x/crypto v0.0.0-20200115085410-6d4e4cb37c7d/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
|
||||
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
|
||||
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
|
||||
golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
|
||||
|
|
|
|||
|
|
@ -1,12 +1,26 @@
|
|||
package metrics
|
||||
|
||||
import (
|
||||
"github.com/lbryio/hub/meta"
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
"github.com/prometheus/client_golang/prometheus/promauto"
|
||||
"time"
|
||||
)
|
||||
|
||||
var (
|
||||
HistogramBuckets = []float64{0.005, 0.025, 0.05, 0.1, 0.25, 0.4, 1, 2, 5, 10, 20, 60, 120, 300}
|
||||
// These mirror counters from the python code
|
||||
RequestsCount = promauto.NewCounterVec(prometheus.CounterOpts{
|
||||
Name: "requests_count",
|
||||
Help: "Total number of searches",
|
||||
}, []string{"method"})
|
||||
SessionCount = promauto.NewGauge(prometheus.GaugeOpts{
|
||||
Name: "session_count",
|
||||
Help: "Number of client sessions",
|
||||
ConstLabels: map[string]string{
|
||||
"version": meta.Version,
|
||||
},
|
||||
})
|
||||
// These are unique to the go code
|
||||
PingsCounter = promauto.NewCounter(prometheus.CounterOpts{
|
||||
Name: "pings",
|
||||
Help: "Number of pings",
|
||||
|
|
@ -31,14 +45,6 @@ var (
|
|||
Name: "mget_error_counter",
|
||||
Help: "Mget errors",
|
||||
})
|
||||
SearchCounter = promauto.NewCounter(prometheus.CounterOpts{
|
||||
Name: "searche_counter",
|
||||
Help: "Total number of searches",
|
||||
})
|
||||
ClientCreationErrorCounter = promauto.NewCounter(prometheus.CounterOpts{
|
||||
Name: "client_creation_error_counter",
|
||||
Help: "Number of errors",
|
||||
})
|
||||
SearchErrorCounter = promauto.NewCounter(prometheus.CounterOpts{
|
||||
Name: "search_error_counter",
|
||||
Help: "Number of errors",
|
||||
|
|
@ -47,14 +53,10 @@ var (
|
|||
Name: "fatal_error_counter",
|
||||
Help: "Number of errors",
|
||||
})
|
||||
ErrorCounter = promauto.NewCounter(prometheus.CounterOpts{
|
||||
Name: "error_counter",
|
||||
Help: "Number of errors",
|
||||
})
|
||||
QueryTime = promauto.NewSummary(prometheus.SummaryOpts{
|
||||
MaxAge: time.Hour,
|
||||
QueryTime = promauto.NewHistogramVec(prometheus.HistogramOpts{
|
||||
Name: "query_time",
|
||||
Help: "hourly summary of query time",
|
||||
})
|
||||
Help: "Histogram of query times",
|
||||
Buckets: HistogramBuckets,
|
||||
}, []string{"method"})
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -13,9 +13,11 @@ import (
|
|||
|
||||
//"github.com/lbryio/hub/schema"
|
||||
|
||||
"github.com/lbryio/hub/internal/metrics"
|
||||
pb "github.com/lbryio/hub/protobuf/go"
|
||||
"github.com/lbryio/lbry.go/v2/extras/util"
|
||||
"github.com/olivere/elastic/v7"
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
"golang.org/x/text/cases"
|
||||
"golang.org/x/text/language"
|
||||
"google.golang.org/protobuf/encoding/protojson"
|
||||
|
|
@ -44,7 +46,7 @@ type record struct {
|
|||
RepostCount uint32 `json:"repost_count"`
|
||||
EffectiveAmount uint64 `json:"effective_amount"`
|
||||
SupportAmount uint64 `json:"support_amount"`
|
||||
TrendingScore float64 `json:"TrendingScore"`
|
||||
TrendingScore float64 `json:"trending_score"`
|
||||
ClaimName string `json:"claim_name"`
|
||||
}
|
||||
|
||||
|
|
@ -125,14 +127,10 @@ func AddInvertibleField(q *elastic.BoolQuery, field *pb.InvertibleField, name st
|
|||
return q.Must(elastic.NewTermsQuery(name, searchVals...))
|
||||
}
|
||||
}
|
||||
func (s *Server) recordErrorAndReturn(err error, typ string) (interface{}, error) {
|
||||
// TODO record metric
|
||||
log.Println(err)
|
||||
return nil, err
|
||||
}
|
||||
|
||||
func (s *Server) recordErrorAndDie(err error) {
|
||||
// TODO record metric fatal_error_counter
|
||||
metrics.FatalErrorCounter.Inc()
|
||||
log.Fatalln(err)
|
||||
}
|
||||
|
||||
|
|
@ -173,8 +171,16 @@ func RoundUpReleaseTime(q *elastic.BoolQuery, rq *pb.RangeField, name string) *e
|
|||
// 8) return streams referenced by repost and all channel referenced in extra_txos
|
||||
//*/
|
||||
func (s *Server) Search(ctx context.Context, in *pb.SearchRequest) (*pb.Outputs, error) {
|
||||
// TODO record metric search_counter
|
||||
t0 := time.Now()
|
||||
metrics.RequestsCount.With(prometheus.Labels{"method": "search"}).Inc()
|
||||
metrics.SessionCount.Inc()
|
||||
defer func() {metrics.SessionCount.Dec()}()
|
||||
defer func(t time.Time) {
|
||||
delta := time.Since(t).Seconds()
|
||||
metrics.
|
||||
QueryTime.
|
||||
With(prometheus.Labels{"method": "search"}).
|
||||
Observe(delta)
|
||||
}(time.Now())
|
||||
|
||||
var from = 0
|
||||
var pageSize = 10
|
||||
|
|
@ -256,7 +262,7 @@ func (s *Server) Search(ctx context.Context, in *pb.SearchRequest) (*pb.Outputs,
|
|||
return &pb.Outputs{}, nil
|
||||
|
||||
} else if err != nil {
|
||||
s.recordErrorAndReturn(err, "search_errors")
|
||||
metrics.SearchErrorCounter.Inc()
|
||||
log.Println("Error executing query: ", err)
|
||||
return nil, err
|
||||
}
|
||||
|
|
@ -275,12 +281,6 @@ func (s *Server) Search(ctx context.Context, in *pb.SearchRequest) (*pb.Outputs,
|
|||
|
||||
txos, extraTxos, blocked := s.postProcessResults(ctx, client, records, in, pageSize, from, searchIndices)
|
||||
|
||||
t1 := time.Now()
|
||||
|
||||
delta := t1.Unix() - t0.Unix()
|
||||
log.Printf("delta: %d\n", delta)
|
||||
// TODO record metric query_time
|
||||
|
||||
if in.NoTotals {
|
||||
return &pb.Outputs{
|
||||
Txos: txos,
|
||||
|
|
@ -650,13 +650,14 @@ func (s *Server) getUniqueChannels(records []*record, client *elastic.Client, ct
|
|||
}
|
||||
}
|
||||
if totalChannels == 0 {
|
||||
s.recordErrorAndReturn(nil, "zero_channels_counter")
|
||||
metrics.ZeroChannelsCounter.Inc()
|
||||
return []*pb.Output{}, make(map[string]*pb.Output)
|
||||
}
|
||||
|
||||
res, err := mget.Do(ctx)
|
||||
if err != nil {
|
||||
s.recordErrorAndReturn(err, "get_unique_channels_errors")
|
||||
metrics.GetUniqueChannelsErrorCounter.Inc()
|
||||
log.Println(err)
|
||||
return []*pb.Output{}, make(map[string]*pb.Output)
|
||||
}
|
||||
|
||||
|
|
@ -668,7 +669,8 @@ func (s *Server) getUniqueChannels(records []*record, client *elastic.Client, ct
|
|||
var r record
|
||||
err := json.Unmarshal(doc.Source, &r)
|
||||
if err != nil {
|
||||
s.recordErrorAndReturn(err, "json_errors")
|
||||
metrics.JsonErrorCounter.Inc()
|
||||
log.Println(err)
|
||||
return []*pb.Output{}, make(map[string]*pb.Output)
|
||||
}
|
||||
channelTxos[i] = r.recordToOutput()
|
||||
|
|
@ -702,13 +704,14 @@ func (s *Server) getClaimsForReposts(ctx context.Context, client *elastic.Client
|
|||
}
|
||||
//mget = mget.Add(nmget)
|
||||
if totalReposted == 0 {
|
||||
// TODO record metric no_reposted_counter
|
||||
metrics.NoRepostedCounter.Inc()
|
||||
return []*pb.Output{}, []*record{}, make(map[string]*pb.Output)
|
||||
}
|
||||
|
||||
res, err := mget.Do(ctx)
|
||||
if err != nil {
|
||||
s.recordErrorAndReturn(err, "mget_error_counter")
|
||||
metrics.MgetErrorCounter.Inc()
|
||||
log.Println(err)
|
||||
return []*pb.Output{}, []*record{}, make(map[string]*pb.Output)
|
||||
}
|
||||
|
||||
|
|
@ -721,7 +724,8 @@ func (s *Server) getClaimsForReposts(ctx context.Context, client *elastic.Client
|
|||
var r record
|
||||
err := json.Unmarshal(doc.Source, &r)
|
||||
if err != nil {
|
||||
s.recordErrorAndReturn(err, "json_error_counter")
|
||||
metrics.JsonErrorCounter.Inc()
|
||||
log.Println(err)
|
||||
return []*pb.Output{}, []*record{}, make(map[string]*pb.Output)
|
||||
}
|
||||
claims[i] = r.recordToOutput()
|
||||
|
|
|
|||
|
|
@ -12,6 +12,7 @@ import (
|
|||
"time"
|
||||
|
||||
"github.com/ReneKroon/ttlcache/v2"
|
||||
"github.com/lbryio/hub/internal/metrics"
|
||||
"github.com/lbryio/hub/meta"
|
||||
pb "github.com/lbryio/hub/protobuf/go"
|
||||
"github.com/olivere/elastic/v7"
|
||||
|
|
@ -178,6 +179,7 @@ func (s *Server) Hello(context context.Context, args *FederatedServer) (*Federat
|
|||
}
|
||||
|
||||
func (s *Server) Ping(context context.Context, args *pb.EmptyMessage) (*pb.StringValue, error) {
|
||||
metrics.PingsCounter.Inc()
|
||||
return &pb.StringValue{Value: "Hello, world!"}, nil
|
||||
}
|
||||
|
||||
|
|
|
|||
Loading…
Reference in a new issue