Extract percentile implementation
This commit is contained in:
Justin Li
parent
dda56c4357
commit
dfeda26c16
3 changed files with 9 additions and 221 deletions
|
|
@ -1,116 +0,0 @@
|
||||||
// Copyright 2014 The Chihaya Authors. All rights reserved.
|
|
||||||
// Use of this source code is governed by the BSD 2-Clause license,
|
|
||||||
// which can be found in the LICENSE file.
|
|
||||||
|
|
||||||
package stats
|
|
||||||
|
|
||||||
import (
|
|
||||||
"encoding/json"
|
|
||||||
"math"
|
|
||||||
"sort"
|
|
||||||
"sync/atomic"
|
|
||||||
)
|
|
||||||
|
|
||||||
// Percentile implements an efficient percentile calculation of
|
|
||||||
// arbitrary float64 samples.
|
|
||||||
type Percentile struct {
|
|
||||||
percentile float64
|
|
||||||
|
|
||||||
samples int64
|
|
||||||
offset int64
|
|
||||||
|
|
||||||
values []float64
|
|
||||||
value uint64 // These bits are really a float64.
|
|
||||||
}
|
|
||||||
|
|
||||||
// NewPercentile returns a Percentile with a given threshold.
|
|
||||||
func NewPercentile(percentile float64) *Percentile {
|
|
||||||
return &Percentile{
|
|
||||||
percentile: percentile,
|
|
||||||
|
|
||||||
// 256 samples is fast, and accurate for most distributions.
|
|
||||||
values: make([]float64, 0, 256),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// NewPercentileWithWindow returns a Percentile with a given threshold
|
|
||||||
// and window size (accuracy).
|
|
||||||
func NewPercentileWithWindow(percentile float64, sampleWindow int) *Percentile {
|
|
||||||
return &Percentile{
|
|
||||||
percentile: percentile,
|
|
||||||
values: make([]float64, 0, sampleWindow),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Value returns the current value at the stored percentile.
|
|
||||||
// It is thread-safe, and may be called concurrently with AddSample.
|
|
||||||
func (p *Percentile) Value() float64 {
|
|
||||||
bits := atomic.LoadUint64(&p.value)
|
|
||||||
return math.Float64frombits(bits)
|
|
||||||
}
|
|
||||||
|
|
||||||
// AddSample adds a single float64 sample to the data set.
|
|
||||||
// It is not thread-safe, and not be called concurrently.
|
|
||||||
func (p *Percentile) AddSample(sample float64) {
|
|
||||||
p.samples++
|
|
||||||
|
|
||||||
if len(p.values) == cap(p.values) {
|
|
||||||
target := float64(p.samples)*p.percentile - float64(cap(p.values))/2
|
|
||||||
offset := round(math.Max(target, 0))
|
|
||||||
|
|
||||||
if sample > p.values[0] {
|
|
||||||
if offset > p.offset {
|
|
||||||
idx := sort.SearchFloat64s(p.values[1:], sample)
|
|
||||||
copy(p.values, p.values[1:idx+1])
|
|
||||||
|
|
||||||
p.values[idx] = sample
|
|
||||||
p.offset++
|
|
||||||
} else if sample < p.values[len(p.values)-1] {
|
|
||||||
idx := sort.SearchFloat64s(p.values, sample)
|
|
||||||
copy(p.values[idx+1:], p.values[idx:])
|
|
||||||
|
|
||||||
p.values[idx] = sample
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
if offset > p.offset {
|
|
||||||
p.offset++
|
|
||||||
} else {
|
|
||||||
copy(p.values[1:], p.values)
|
|
||||||
p.values[0] = sample
|
|
||||||
}
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
idx := sort.SearchFloat64s(p.values, sample)
|
|
||||||
p.values = p.values[:len(p.values)+1]
|
|
||||||
copy(p.values[idx+1:], p.values[idx:])
|
|
||||||
p.values[idx] = sample
|
|
||||||
}
|
|
||||||
|
|
||||||
bits := math.Float64bits(p.values[p.index()])
|
|
||||||
atomic.StoreUint64(&p.value, bits)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (p *Percentile) index() int64 {
|
|
||||||
idx := round(float64(p.samples)*p.percentile - float64(p.offset))
|
|
||||||
last := int64(len(p.values)) - 1
|
|
||||||
|
|
||||||
if idx > last {
|
|
||||||
return last
|
|
||||||
}
|
|
||||||
|
|
||||||
return idx
|
|
||||||
}
|
|
||||||
|
|
||||||
func (p *Percentile) MarshalJSON() ([]byte, error) {
|
|
||||||
return json.Marshal(p.Value())
|
|
||||||
}
|
|
||||||
|
|
||||||
func round(value float64) int64 {
|
|
||||||
if value < 0.0 {
|
|
||||||
value -= 0.5
|
|
||||||
} else {
|
|
||||||
value += 0.5
|
|
||||||
}
|
|
||||||
|
|
||||||
return int64(value)
|
|
||||||
}
|
|
||||||
|
|
@ -1,98 +0,0 @@
|
||||||
// Copyright 2014 The Chihaya Authors. All rights reserved.
|
|
||||||
// Use of this source code is governed by the BSD 2-Clause license,
|
|
||||||
// which can be found in the LICENSE file.
|
|
||||||
|
|
||||||
package stats
|
|
||||||
|
|
||||||
import (
|
|
||||||
"math"
|
|
||||||
"math/rand"
|
|
||||||
"sort"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestPercentiles(t *testing.T) {
|
|
||||||
rand.Seed(time.Now().Unix())
|
|
||||||
|
|
||||||
testSlice(t, uniform(10000, 1), 0.5)
|
|
||||||
testSlice(t, uniform(10000, 1), 0.9)
|
|
||||||
testSlice(t, uniform(10000, 10000), 0.5)
|
|
||||||
testSlice(t, uniform(10000, 10000), 0.9)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestLogNormPercentiles(t *testing.T) {
|
|
||||||
rand.Seed(time.Now().Unix())
|
|
||||||
|
|
||||||
testSlice(t, logNorm(10000, 1), 0.5)
|
|
||||||
testSlice(t, logNorm(10000, 1), 0.9)
|
|
||||||
}
|
|
||||||
|
|
||||||
func uniform(n int, scale float64) sort.Float64Slice {
|
|
||||||
numbers := make(sort.Float64Slice, n)
|
|
||||||
|
|
||||||
for i := 0; i < n; i++ {
|
|
||||||
numbers[i] = rand.Float64() * scale
|
|
||||||
}
|
|
||||||
|
|
||||||
return numbers
|
|
||||||
}
|
|
||||||
|
|
||||||
func logNorm(n int, scale float64) sort.Float64Slice {
|
|
||||||
numbers := make(sort.Float64Slice, n)
|
|
||||||
|
|
||||||
for i := 0; i < n; i++ {
|
|
||||||
numbers[i] = math.Exp(rand.NormFloat64()) * scale
|
|
||||||
}
|
|
||||||
|
|
||||||
return numbers
|
|
||||||
}
|
|
||||||
|
|
||||||
func testSlice(t *testing.T, numbers sort.Float64Slice, percentile float64) {
|
|
||||||
p := NewPercentile(percentile)
|
|
||||||
|
|
||||||
for i := 0; i < len(numbers); i++ {
|
|
||||||
p.AddSample(numbers[i])
|
|
||||||
}
|
|
||||||
|
|
||||||
sort.Sort(numbers)
|
|
||||||
got := p.Value()
|
|
||||||
index := round(float64(len(numbers)) * percentile)
|
|
||||||
|
|
||||||
if got != numbers[index] && got != numbers[index-1] && got != numbers[index+1] {
|
|
||||||
t.Errorf("Percentile incorrect\n actual: %f\nexpected: %f, %f, %f\n", got, numbers[index-1], numbers[index], numbers[index+1])
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func BenchmarkPercentiles64(b *testing.B) {
|
|
||||||
benchmarkSlice(b, uniform(b.N, 1), 64, 0.5)
|
|
||||||
}
|
|
||||||
|
|
||||||
func BenchmarkPercentiles128(b *testing.B) {
|
|
||||||
benchmarkSlice(b, uniform(b.N, 1), 128, 0.5)
|
|
||||||
}
|
|
||||||
|
|
||||||
func BenchmarkPercentiles256(b *testing.B) {
|
|
||||||
benchmarkSlice(b, uniform(b.N, 1), 256, 0.5)
|
|
||||||
}
|
|
||||||
|
|
||||||
func BenchmarkPercentiles512(b *testing.B) {
|
|
||||||
benchmarkSlice(b, uniform(b.N, 1), 512, 0.5)
|
|
||||||
}
|
|
||||||
|
|
||||||
func BenchmarkLNPercentiles128(b *testing.B) {
|
|
||||||
benchmarkSlice(b, logNorm(b.N, 1), 128, 0.5)
|
|
||||||
}
|
|
||||||
|
|
||||||
func BenchmarkLNPercentiles256(b *testing.B) {
|
|
||||||
benchmarkSlice(b, logNorm(b.N, 1), 258, 0.5)
|
|
||||||
}
|
|
||||||
|
|
||||||
func benchmarkSlice(b *testing.B, numbers sort.Float64Slice, window int, percentile float64) {
|
|
||||||
p := NewPercentileWithWindow(percentile, window)
|
|
||||||
|
|
||||||
b.ResetTimer()
|
|
||||||
for i := 0; i < b.N; i++ {
|
|
||||||
p.AddSample(numbers[i])
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
@ -9,8 +9,10 @@ package stats
|
||||||
import (
|
import (
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/chihaya/chihaya/config"
|
"github.com/pushrax/faststats"
|
||||||
"github.com/pushrax/flatjson"
|
"github.com/pushrax/flatjson"
|
||||||
|
|
||||||
|
"github.com/chihaya/chihaya/config"
|
||||||
)
|
)
|
||||||
|
|
||||||
const (
|
const (
|
||||||
|
|
@ -59,9 +61,9 @@ type PeerStats struct {
|
||||||
}
|
}
|
||||||
|
|
||||||
type PercentileTimes struct {
|
type PercentileTimes struct {
|
||||||
P50 *Percentile
|
P50 *faststats.Percentile
|
||||||
P90 *Percentile
|
P90 *faststats.Percentile
|
||||||
P95 *Percentile
|
P95 *faststats.Percentile
|
||||||
}
|
}
|
||||||
|
|
||||||
type Stats struct {
|
type Stats struct {
|
||||||
|
|
@ -107,9 +109,9 @@ func New(cfg config.StatsConfig) *Stats {
|
||||||
responseTimeEvents: make(chan time.Duration, cfg.BufferSize),
|
responseTimeEvents: make(chan time.Duration, cfg.BufferSize),
|
||||||
|
|
||||||
ResponseTime: PercentileTimes{
|
ResponseTime: PercentileTimes{
|
||||||
P50: NewPercentile(0.5),
|
P50: faststats.NewPercentile(0.5),
|
||||||
P90: NewPercentile(0.9),
|
P90: faststats.NewPercentile(0.9),
|
||||||
P95: NewPercentile(0.95),
|
P95: faststats.NewPercentile(0.95),
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
Loading…
Add table
Reference in a new issue