Merge pull request #515 from jzelinskie/http-profiles

Move profiling into the metrics server

.github/workflows/CI.yaml

@@ -93,3 +93,23 @@ jobs:
         sleep 2
         chihaya e2e --debug
         kill $pid
+
+  dist:
+    name: Helm Template
+    runs-on: ubuntu-latest
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v2
+    - name: Install Helm
+      uses: engineerd/configurator@v0.0.5
+      with:
+        name: helm
+        pathInArchive: linux-amd64/helm
+        fromGitHubReleases: true
+        repo: helm/helm
+        version: ^v3
+        urlTemplate: https://get.helm.sh/helm-{{version}}-linux-amd64.tar.gz
+        token: ${{ secrets.GITHUB_TOKEN }}
+    - name: Helm Template
+      working-directory: ./dist/helm/chihaya
+      run: helm template . --debug

cmd/chihaya/config.go

@@ -30,7 +30,7 @@ type storageConfig struct {
 // Config represents the configuration used for executing Chihaya.
 type Config struct {
 	middleware.ResponseConfig `yaml:",inline"`
-	PrometheusAddr            string                  `yaml:"prometheus_addr"`
+	MetricsAddr               string                  `yaml:"metrics_addr"`
 	HTTPConfig                http.Config             `yaml:"http"`
 	UDPConfig                 udp.Config              `yaml:"udp"`
 	Storage                   storageConfig           `yaml:"storage"`

cmd/chihaya/main.go

@@ -5,8 +5,6 @@ import (
 	"os"
 	"os/signal"
 	"runtime"
-	"runtime/pprof"
-	"runtime/trace"
 	"strings"
 	"syscall"
 	"time"
@@ -18,7 +16,7 @@ import (
 	"github.com/chihaya/chihaya/frontend/udp"
 	"github.com/chihaya/chihaya/middleware"
 	"github.com/chihaya/chihaya/pkg/log"
-	"github.com/chihaya/chihaya/pkg/prometheus"
+	"github.com/chihaya/chihaya/pkg/metrics"
 	"github.com/chihaya/chihaya/pkg/stop"
 	"github.com/chihaya/chihaya/storage"
 )
@@ -52,8 +50,8 @@ func (r *Run) Start(ps storage.PeerStore) error {
 
 	r.sg = stop.NewGroup()
 
-	log.Info("starting Prometheus server", log.Fields{"addr": cfg.PrometheusAddr})
-	r.sg.Add(prometheus.NewServer(cfg.PrometheusAddr))
+	log.Info("starting metrics server", log.Fields{"addr": cfg.MetricsAddr})
+	r.sg.Add(metrics.NewServer(cfg.MetricsAddr))
 
 	if ps == nil {
 		log.Info("starting storage", log.Fields{"name": cfg.Storage.Name})
@@ -112,7 +110,7 @@ func combineErrors(prefix string, errs []error) error {
 
 // Stop shuts down an instance of Chihaya.
 func (r *Run) Stop(keepPeerStore bool) (storage.PeerStore, error) {
-	log.Debug("stopping frontends and prometheus endpoint")
+	log.Debug("stopping frontends and metrics server")
 	if errs := r.sg.Stop().Wait(); len(errs) != 0 {
 		return nil, combineErrors("failed while shutting down frontends", errs)
 	}
@@ -202,42 +200,12 @@ func RootPreRunCmdFunc(cmd *cobra.Command, args []string) error {
 		log.Info("enabled debug logging")
 	}
 
-	tracePath, err := cmd.Flags().GetString("trace")
-	if err != nil {
-		return err
-	}
-	if tracePath != "" {
-		f, err := os.Create(tracePath)
-		if err != nil {
-			return err
-		}
-		trace.Start(f)
-		log.Info("enabled tracing", log.Fields{"path": tracePath})
-	}
-
-	cpuProfilePath, err := cmd.Flags().GetString("cpuprofile")
-	if err != nil {
-		return err
-	}
-	if cpuProfilePath != "" {
-		f, err := os.Create(cpuProfilePath)
-		if err != nil {
-			return err
-		}
-		pprof.StartCPUProfile(f)
-		log.Info("enabled CPU profiling", log.Fields{"path": cpuProfilePath})
-	}
-
 	return nil
 }
 
 // RootPostRunCmdFunc handles clean up of any state initialized by command line
 // flags.
 func RootPostRunCmdFunc(cmd *cobra.Command, args []string) error {
-	// These can be called regardless because it noops when not profiling.
-	pprof.StopCPUProfile()
-	trace.Stop()
-
 	return nil
 }
 
@@ -251,8 +219,6 @@ func main() {
 		PersistentPostRunE: RootPostRunCmdFunc,
 	}
 
-	rootCmd.PersistentFlags().String("cpuprofile", "", "location to save a CPU profile")
-	rootCmd.PersistentFlags().String("trace", "", "location to save a trace")
 	rootCmd.PersistentFlags().Bool("debug", false, "enable debug logging")
 	rootCmd.PersistentFlags().Bool("json", false, "enable json logging")
 	if runtime.GOOS == "windows" {

dist/example_config.yaml

@@ -8,9 +8,11 @@ chihaya:
   min_announce_interval: 15m
 
   # The network interface that will bind to an HTTP endpoint that can be
-  # scraped by an instance of the Prometheus time series database.
-  # For more info see: https://prometheus.io
-  prometheus_addr: "0.0.0.0:6880"
+  # scraped by programs collecting metrics.
+  # 
+  # /metrics serves metrics in the Prometheus format
+  # /debug/pprof/{cmdline,profile,symbol,trace} serves profiles in the pprof format
+  metrics_addr: "0.0.0.0:6880"
 
   # This block defines configuration for the tracker's HTTP interface.
   # If you do not wish to run this, delete this section.

dist/helm/chihaya/templates/NOTES.txt

@@ -1,6 +1,6 @@
 You can port forward a local port to Prometheus or the HTTP tracker by running:
   export POD_NAME=$(kubectl get pods --namespace {{ .Release.Namespace }} -l "app={{ template "fullname" . }}" -o jsonpath="{.items[0].metadata.name}")
-  # Prometheus port
-  kubectl port-forward $POD_NAME 8080:{{ $v := .Values.config.chihaya.prometheus_addr | split ":" }}{{ $v._1 }}
+  # Metrics port
+  kubectl port-forward $POD_NAME 8080:{{ $v := .Values.config.chihaya.metrics_addr | split ":" }}{{ $v._1 }}
   # HTTP tracker port
   kubectl port-forward $POD_NAME 8080:{{ $v := .Values.config.chihaya.http.addr | split ":" }}{{ $v._1 }}

dist/helm/chihaya/templates/deployment.yaml

@@ -31,11 +31,11 @@ spec:
           containerPort: {{ $v := .Values.config.chihaya.udp.addr | split ":" }}{{ $v._1 }}
           protocol: UDP
         - name: metrics
-          containerPort: {{ $v := .Values.config.chihaya.prometheus_addr | split ":" }}{{ $v._1 }}
+          containerPort: {{ $v := .Values.config.chihaya.metrics_addr | split ":" }}{{ $v._1 }}
         livenessProbe:
           httpGet:
             path: /
-            port: {{ $v := .Values.config.chihaya.prometheus_addr | split ":" }}{{ $v._1 }}
+            port: {{ $v := .Values.config.chihaya.metrics_addr | split ":" }}{{ $v._1 }}
         volumeMounts:
         - name: config
           mountPath: /etc/chihaya

dist/helm/chihaya/templates/service.yaml

@@ -6,8 +6,8 @@ metadata:
     chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
   annotations:
     prometheus.io/scrape: "true"
-    prometheus.io/path: "/"
-    prometheus.io/port: {{ $v := .Values.config.chihaya.prometheus_addr | split ":" }}{{ $v._1 | quote }}
+    prometheus.io/path: "/metrics"
+    prometheus.io/port: {{ $v := .Values.config.chihaya.metrics_addr | split ":" }}{{ $v._1 | quote }}
 spec:
   type: {{ .Values.service.type }}
   ports:
@@ -20,8 +20,8 @@ spec:
     targetPort: {{ $v := .Values.config.chihaya.udp.addr | split ":" }}{{ $v._1 }}
     protocol: UDP
   - name: metrics
-    port: {{ $v := .Values.config.chihaya.prometheus_addr | split ":" }}{{ $v._1 }}
-    targetPort: {{ $v := .Values.config.chihaya.prometheus_addr | split ":" }}{{ $v._1 }}
+    port: {{ $v := .Values.config.chihaya.metrics_addr | split ":" }}{{ $v._1 }}
+    targetPort: {{ $v := .Values.config.chihaya.metrics_addr | split ":" }}{{ $v._1 }}
     protocol: TCP
   selector:
     app: {{ template "fullname" . }}

dist/helm/chihaya/values.yaml

@@ -21,9 +21,11 @@ config:
     min_announce_interval: 15m
 
     # The network interface that will bind to an HTTP endpoint that can be
-    # scraped by an instance of the Prometheus time series database.
-    # For more info see: https://prometheus.io
-    prometheus_addr: "0.0.0.0:6880"
+    # scraped by programs collecting metrics.
+    #
+    # /metrics serves metrics in the Prometheus format
+    # /debug/pprof/{cmdline,profile,symbol,trace} serves profiles in the pprof format
+    metrics_addr: "0.0.0.0:6880"
 
     # The maximum number of peers returned in an announce.
     max_numwant: 50

go.mod

@@ -1,6 +1,6 @@
 module github.com/chihaya/chihaya
 
-go 1.14
+go 1.16
 
 require (
 	github.com/SermoDigital/jose v0.9.2-0.20180104203859-803625baeddc

go.sum

@@ -139,6 +139,7 @@ github.com/anacrolix/torrent v1.15.2 h1:qA+t1TyhFbRl42Lw1DiwtCiEIVZrAKJMCVSVllcb
 github.com/anacrolix/torrent v1.15.2/go.mod h1:sJtcAZtlGaZLo7wCXT/EZV+hATsq0Bg6pVhhzACY0E0=
 github.com/anacrolix/torrent v1.16.0 h1:EWQDsS3D0qX2jVFXUFhkt+pd7vQu1tJr0d9xHaV5uxQ=
 github.com/anacrolix/torrent v1.16.0/go.mod h1:XWo/fJN1oKgcjgxM+pUZpvalHfqHDs27BY5mBZjIQWo=
+github.com/anacrolix/torrent v1.18.1 h1:SUJKsFJSz+0kb6f0YuimIRPSz35JzNwvY7tCltjv3jk=
 github.com/anacrolix/torrent v1.18.1/go.mod h1:JnoMhFCq4Hq5Q/A1BmXljEnjHzKWD2auPqbqf/xwFHA=
 github.com/anacrolix/torrent v1.19.2/go.mod h1:WjA5XIOm/3qWM8DUEhd5ACfD771/wsddW90nCAhCUOQ=
 github.com/anacrolix/torrent v1.22.0/go.mod h1:GWTwQkOAilf0LR3C6A74XEkWPg0ejfFD9GcEIe57ess=
@@ -607,6 +608,7 @@ github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y8
 github.com/prometheus/common v0.7.0/go.mod h1:DjGbpBbp5NYNiECxcL/VnbXCCaQpKd3tt26CguLLsqA=
 github.com/prometheus/common v0.9.1 h1:KOMtN28tlbam3/7ZKEYKHhKoJZYYj3gMH4uc62x7X7U=
 github.com/prometheus/common v0.9.1/go.mod h1:yhUN8i9wzaXS3w1O07YhxHEBxD+W35wd8bs7vj7HSQ4=
+github.com/prometheus/common v0.10.0 h1:RyRA7RzGXQZiW+tGMr7sxa85G1z0yOpM1qq5c8lNawc=
 github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo=
 github.com/prometheus/common v0.15.0/go.mod h1:U+gB1OBLb1lF3O42bTCL+FK18tX9Oar16Clt/msog/s=
 github.com/prometheus/procfs v0.0.0-20180725123919-05ee40e3a273/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
@@ -617,6 +619,7 @@ github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsT
 github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A=
 github.com/prometheus/procfs v0.0.11 h1:DhHlBtkHWPYi8O2y31JkK0TF+DGM+51OopZjH/Ia5qI=
 github.com/prometheus/procfs v0.0.11/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU=
+github.com/prometheus/procfs v0.1.3 h1:F0+tqvhOksq22sc6iCHF5WGlWjdwj92p0udFh1VFBS8=
 github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU=
 github.com/prometheus/procfs v0.2.0/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU=
 github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU=

pkg/metrics/server.go

@@ -1,10 +1,11 @@
-// Package prometheus implements a standalone HTTP server for serving a
-// Prometheus metrics endpoint.
-package prometheus
+// Package metrics implements a standalone HTTP server for serving pprof
+// profiles and Prometheus metrics.
+package metrics
 
 import (
 	"context"
 	"net/http"
+	"net/http/pprof"
 
 	"github.com/prometheus/client_golang/prometheus/promhttp"
 
@@ -31,10 +32,19 @@ func (s *Server) Stop() stop.Result {
 // NewServer creates a new instance of a Prometheus server that asynchronously
 // serves requests.
 func NewServer(addr string) *Server {
+	mux := http.NewServeMux()
+
+	mux.Handle("/metrics", promhttp.Handler())
+	mux.HandleFunc("/debug/pprof/", pprof.Index)
+	mux.HandleFunc("/debug/pprof/cmdline", pprof.Cmdline)
+	mux.HandleFunc("/debug/pprof/profile", pprof.Profile)
+	mux.HandleFunc("/debug/pprof/symbol", pprof.Symbol)
+	mux.HandleFunc("/debug/pprof/trace", pprof.Trace)
+
 	s := &Server{
 		srv: &http.Server{
 			Addr:    addr,
-			Handler: promhttp.Handler(),
+			Handler: mux,
 		},
 	}