Add memory statistics

chihaya.go

@@ -72,6 +72,8 @@ func Boot() {
 	}
 
 	stats.DefaultBufferSize = cfg.StatsBufferSize
+	stats.DefaultIncludeMemStats = cfg.IncludeMemStats
+	stats.DefaultVerboseMemStats = cfg.VerboseMemStats
 
 	tkr, err := tracker.New(cfg)
 	if err != nil {

config/config.go

@@ -67,6 +67,9 @@ type Config struct {
 	RequestTimeout  Duration `json:"request_timeout"`
 	NumWantFallback int      `json:"default_num_want"`
 	StatsBufferSize int      `json:"stats_buffer_size"`
+	IncludeMemStats bool     `json:"include_mem_stats"`
+	VerboseMemStats bool     `json:"verbose_mem_stats"`
+	MemStatInterval Duration `json:"mem_stats_interval"`
 
 	NetConfig
 }
@@ -93,6 +96,9 @@ var DefaultConfig = Config{
 	RequestTimeout:  Duration{10 * time.Second},
 	NumWantFallback: 50,
 	StatsBufferSize: 0,
+	IncludeMemStats: true,
+	VerboseMemStats: false,
+	MemStatInterval: Duration{15 * time.Second},
 
 	NetConfig: NetConfig{
 		AllowIPSpoofing:  true,

stats/mem.go

@@ -0,0 +1,79 @@
+// Copyright 2014 The Chihaya Authors. All rights reserved.
+// Use of this source code is governed by the BSD 2-Clause license,
+// which can be found in the LICENSE file.
+
+package stats
+
+import (
+	"encoding/json"
+	"runtime"
+)
+
+// BasicMemStats includes a few of the fields from runtime.MemStats suitable for
+// general logging.
+type BasicMemStats struct {
+	// General statistics.
+	Alloc      uint64 // bytes allocated and still in use
+	TotalAlloc uint64 // bytes allocated (even if freed)
+	Sys        uint64 // bytes obtained from system (sum of XxxSys below)
+	Lookups    uint64 // number of pointer lookups
+	Mallocs    uint64 // number of mallocs
+	Frees      uint64 // number of frees
+
+	// Main allocation heap statistics.
+	HeapAlloc    uint64 // bytes allocated and still in use
+	HeapSys      uint64 // bytes obtained from system
+	HeapIdle     uint64 // bytes in idle spans
+	HeapInuse    uint64 // bytes in non-idle span
+	HeapReleased uint64 // bytes released to the OS
+	HeapObjects  uint64 // total number of allocated objects
+
+	// Garbage collector statistics.
+	PauseTotalNs uint64
+}
+
+type MemStatsWrapper struct {
+	basic   *BasicMemStats
+	full    *runtime.MemStats
+	verbose bool
+}
+
+func NewMemStatsWrapper(verbose bool) *MemStatsWrapper {
+	stats := &MemStatsWrapper{
+		verbose: verbose,
+		full:    &runtime.MemStats{},
+	}
+	if !verbose {
+		stats.basic = &BasicMemStats{}
+	}
+	return stats
+}
+
+func (s *MemStatsWrapper) MarshalJSON() ([]byte, error) {
+	if s.verbose {
+		return json.Marshal(s.full)
+	} else {
+		return json.Marshal(s.basic)
+	}
+}
+
+func (s *MemStatsWrapper) Update() {
+	runtime.ReadMemStats(s.full)
+
+	if !s.verbose {
+		// Gross, but any decent editor can generate this in a couple commands.
+		s.basic.Alloc = s.full.Alloc
+		s.basic.TotalAlloc = s.full.TotalAlloc
+		s.basic.Sys = s.full.Sys
+		s.basic.Lookups = s.full.Lookups
+		s.basic.Mallocs = s.full.Mallocs
+		s.basic.Frees = s.full.Frees
+		s.basic.HeapAlloc = s.full.HeapAlloc
+		s.basic.HeapSys = s.full.HeapSys
+		s.basic.HeapIdle = s.full.HeapIdle
+		s.basic.HeapInuse = s.full.HeapInuse
+		s.basic.HeapReleased = s.full.HeapReleased
+		s.basic.HeapObjects = s.full.HeapObjects
+		s.basic.PauseTotalNs = s.full.PauseTotalNs
+	}
+}

stats/stats.go

@@ -37,8 +37,10 @@ const (
 // channel for broadcasting events unless specified otherwise via a command
 // line flag.
 var (
-	DefaultStats      *Stats
-	DefaultBufferSize int
+	DefaultStats           *Stats
+	DefaultBufferSize      int
+	DefaultIncludeMemStats bool
+	DefaultVerboseMemStats bool
 )
 
 type PeerStats struct {
@@ -82,15 +84,17 @@ type Stats struct {
 	RequestsHandled uint64 `json:"requests_handled"`
 	RequestsErrored uint64 `json:"requests_errored"`
 
-	ResponseTime PercentileTimes `json:"response_time"`
+	ResponseTime PercentileTimes  `json:"response_time"`
+	MemStats     *MemStatsWrapper `json:"mem_stats,omitempty"`
 
 	events             chan int
 	ipv4PeerEvents     chan int
 	ipv6PeerEvents     chan int
 	responseTimeEvents chan time.Duration
+	recordMemStats     <-chan time.Time
 }
 
-func New(chanSize int) *Stats {
+func New(chanSize int, mem bool, verboseMem bool) *Stats {
 	s := &Stats{
 		Start:  time.Now(),
 		events: make(chan int, chanSize),
@@ -106,6 +110,11 @@ func New(chanSize int) *Stats {
 		},
 	}
 
+	if mem {
+		s.MemStats = NewMemStatsWrapper(verboseMem)
+		s.recordMemStats = time.NewTicker(time.Second * 10).C
+	}
+
 	go s.handleEvents()
 	return s
 }
@@ -156,6 +165,9 @@ func (s *Stats) handleEvents() {
 			s.ResponseTime.P50.AddSample(f)
 			s.ResponseTime.P90.AddSample(f)
 			s.ResponseTime.P95.AddSample(f)
+
+		case <-s.recordMemStats:
+			s.MemStats.Update()
 		}
 	}
 }
@@ -239,7 +251,7 @@ func (s *Stats) handlePeerEvent(ps *PeerStats, event int) {
 // RecordEvent broadcasts an event to the default stats queue.
 func RecordEvent(event int) {
 	if DefaultStats == nil {
-		DefaultStats = New(DefaultBufferSize)
+		DefaultStats = New(DefaultBufferSize, DefaultIncludeMemStats, DefaultVerboseMemStats)
 	}
 
 	DefaultStats.RecordEvent(event)
@@ -248,7 +260,7 @@ func RecordEvent(event int) {
 // RecordPeerEvent broadcasts a peer event to the default stats queue.
 func RecordPeerEvent(event int, ipv6 bool) {
 	if DefaultStats == nil {
-		DefaultStats = New(DefaultBufferSize)
+		DefaultStats = New(DefaultBufferSize, DefaultIncludeMemStats, DefaultVerboseMemStats)
 	}
 
 	DefaultStats.RecordPeerEvent(event, ipv6)
@@ -257,7 +269,7 @@ func RecordPeerEvent(event int, ipv6 bool) {
 // RecordTiming broadcasts a timing event to the default stats queue.
 func RecordTiming(event int, duration time.Duration) {
 	if DefaultStats == nil {
-		DefaultStats = New(DefaultBufferSize)
+		DefaultStats = New(DefaultBufferSize, DefaultIncludeMemStats, DefaultVerboseMemStats)
 	}
 
 	DefaultStats.RecordTiming(event, duration)