Fast and extensible multi-platform HTTP/1-2-3 web server with automatic HTTPS
Mirrored from https://github.com/Slazee/caddy.git
Synced Nov 12, 07:00 UTC
metrics: scope metrics to active config, add optional per-host metrics (#6531)
* Add per host config * Pass host label when option is enabled * Test per host enabled * metrics: scope metrics per loaded config * doc and linter Signed-off-by: Mohammed Al Sahaf <msaa1990@gmail.com> * inject the custom registry into the admin handler Co-Authored-By: Dave Henderson <dhenderson@gmail.com> * remove `TODO` comment * fixes Signed-off-by: Mohammed Al Sahaf <msaa1990@gmail.com> * refactor to delay metrics admin handler provision Signed-off-by: Mohammed Al Sahaf <msaa1990@gmail.com> --------- Signed-off-by: Mohammed Al Sahaf <msaa1990@gmail.com> Co-authored-by: Hussam Almarzooq <me@hussam.io> Co-authored-by: Dave Henderson <dhenderson@gmail.com>
Mohammed Al Sahaf committed
Oct 2, 2024 at 17:23 UTC
41f5dd56e1b93ec815daa98dd1f1caa7f2087312
12 files changed
+365
-66
admin.go
+6
-5
@@ -214,7 +214,7 @@ type AdminPermissions struct {
214
215
// newAdminHandler reads admin's config and returns an http.Handler suitable
216
// for use in an admin endpoint server, which will be listening on listenAddr.
217
-func (admin *AdminConfig) newAdminHandler(addr NetworkAddress, remote bool) adminHandler {
217
+func (admin *AdminConfig) newAdminHandler(addr NetworkAddress, remote bool, ctx Context) adminHandler {
218
muxWrap := adminHandler{mux: http.NewServeMux()}
219
220
// secure the local or remote endpoint respectively
@@ -270,7 +270,6 @@ func (admin *AdminConfig) newAdminHandler(addr NetworkAddress, remote bool) admi
270
// register third-party module endpoints
271
for _, m := range GetModules("admin.api") {
272
router := m.New().(AdminRouter)
273
- handlerLabel := m.ID.Name()
273
for _, route := range router.Routes() {
274
addRoute(route.Pattern, handlerLabel, route.Handler)
275
}
@@ -382,7 +381,9 @@ func (admin AdminConfig) allowedOrigins(addr NetworkAddress) []*url.URL {
381
// for the admin endpoint exists in cfg, a default one is used, so
382
// that there is always an admin server (unless it is explicitly
383
// configured to be disabled).
385
-func replaceLocalAdminServer(cfg *Config) error {
384
+// Critically note that some elements and functionality of the context
385
+// may not be ready, e.g. storage. Tread carefully.
386
+func replaceLocalAdminServer(cfg *Config, ctx Context) error {
387
// always* be sure to close down the old admin endpoint
388
// as gracefully as possible, even if the new one is
389
// disabled -- careful to use reference to the current
@@ -424,7 +425,7 @@ func replaceLocalAdminServer(cfg *Config) error {
425
return err
426
}
427
427
- handler := cfg.Admin.newAdminHandler(addr, false)
428
+ handler := cfg.Admin.newAdminHandler(addr, false, ctx)
429
430
ln, err := addr.Listen(context.TODO(), 0, net.ListenConfig{})
431
if err != nil {
@@ -545,7 +546,7 @@ func replaceRemoteAdminServer(ctx Context, cfg *Config) error {
546
547
// make the HTTP handler but disable Host/Origin enforcement
548
// because we are using TLS authentication instead
548
- handler := cfg.Admin.newAdminHandler(addr, true)
549
+ handler := cfg.Admin.newAdminHandler(addr, true, ctx)
550
551
// create client certificate pool for TLS mutual auth, and extract public keys
552
// so that we can enforce access controls at the application layer
caddy.go
+7
-2
@@ -399,6 +399,7 @@ func unsyncedDecodeAndRun(cfgJSON []byte, allowPersist bool) error {
399
func run(newCfg *Config, start bool) (Context, error) {
400
ctx, err := provisionContext(newCfg, start)
401
if err != nil {
402
+ globalMetrics.configSuccess.Set(0)
403
return ctx, err
404
}
405
@@ -410,6 +411,7 @@ func run(newCfg *Config, start bool) (Context, error) {
411
// some of the other apps at runtime
412
err = ctx.cfg.Admin.provisionAdminRouters(ctx)
413
if err != nil {
414
+ globalMetrics.configSuccess.Set(0)
415
return ctx, err
416
}
417
@@ -435,9 +437,11 @@ func run(newCfg *Config, start bool) (Context, error) {
437
return nil
438
}()
439
if err != nil {
440
+ globalMetrics.configSuccess.Set(0)
441
return ctx, err
442
}
440
-
443
+ globalMetrics.configSuccess.Set(1)
444
+ globalMetrics.configSuccessTime.SetToCurrentTime()
445
// now that the user's config is running, finish setting up anything else,
446
// such as remote admin endpoint, config loader, etc.
447
return ctx, finishSettingUp(ctx, ctx.cfg)
@@ -471,6 +475,7 @@ func provisionContext(newCfg *Config, replaceAdminServer bool) (Context, error)
475
ctx, cancel := NewContext(Context{Context: context.Background(), cfg: newCfg})
476
defer func() {
477
if err != nil {
478
+ globalMetrics.configSuccess.Set(0)
479
// if there were any errors during startup,
480
// we should cancel the new context we created
481
// since the associated config won't be used;
@@ -497,7 +502,7 @@ func provisionContext(newCfg *Config, replaceAdminServer bool) (Context, error)
502
503
// start the admin endpoint (and stop any prior one)
504
if replaceAdminServer {
500
- err = replaceLocalAdminServer(newCfg)
505
+ err = replaceLocalAdminServer(newCfg, ctx)
506
if err != nil {
507
return ctx, fmt.Errorf("starting caddy administration endpoint: %v", err)
508
}
caddyconfig/httpcaddyfile/serveroptions.go
+6
-6
@@ -240,13 +240,13 @@ func unmarshalCaddyfileServerOptions(d *caddyfile.Dispenser) (any, error) {
240
}
241
242
case "metrics":
243
- if d.NextArg() {
244
- return nil, d.ArgErr()
245
- }
246
- if nesting := d.Nesting(); d.NextBlock(nesting) {
247
- return nil, d.ArgErr()
248
- }
243
serverOpts.Metrics = new(caddyhttp.Metrics)
244
+ for nesting := d.Nesting(); d.NextBlock(nesting); {
245
+ switch d.Val() {
246
+ case "per_host":
247
+ serverOpts.Metrics.PerHost = true
248
+ }
249
+ }
250
251
case "trace":
252
if d.NextArg() {
caddytest/integration/caddyfile_adapt/metrics_perhost.caddyfiletest
new
+37
@@ -0,0 +1,37 @@
1
+{
2
+ servers :80 {
3
+ metrics {
4
+ per_host
5
+ }
6
+ }
7
+}
8
+:80 {
9
+ respond "Hello"
10
+}
11
+----------
12
+{
13
+ "apps": {
14
+ "http": {
15
+ "servers": {
16
+ "srv0": {
17
+ "listen": [
18
+ ":80"
19
+ ],
20
+ "routes": [
21
+ {
22
+ "handle": [
23
+ {
24
+ "body": "Hello",
25
+ "handler": "static_response"
26
+ }
27
+ ]
28
+ }
29
+ ],
30
+ "metrics": {
31
+ "per_host": true
32
+ }
33
+ }
34
+ }
35
+ }
36
+ }
37
+}
context.go
+21
-1
@@ -23,6 +23,8 @@ import (
23
"reflect"
24
25
"github.com/caddyserver/certmagic"
26
+ "github.com/prometheus/client_golang/prometheus"
27
+ "github.com/prometheus/client_golang/prometheus/collectors"
28
"go.uber.org/zap"
29
"go.uber.org/zap/exp/zapslog"
30
@@ -47,6 +49,7 @@ type Context struct {
49
ancestry []Module
50
cleanupFuncs []func() // invoked at every config unload
51
exitFuncs []func(context.Context) // invoked at config unload ONLY IF the process is exiting (EXPERIMENTAL)
52
+ metricsRegistry *prometheus.Registry
53
}
54
55
// NewContext provides a new context derived from the given
@@ -58,7 +61,7 @@ type Context struct {
61
// modules which are loaded will be properly unloaded.
62
// See standard library context package's documentation.
63
func NewContext(ctx Context) (Context, context.CancelFunc) {
61
- newCtx := Context{moduleInstances: make(map[string][]Module), cfg: ctx.cfg}
64
+ newCtx := Context{moduleInstances: make(map[string][]Module), cfg: ctx.cfg, metricsRegistry: prometheus.NewPedanticRegistry()}
65
c, cancel := context.WithCancel(ctx.Context)
66
wrappedCancel := func() {
67
cancel()
@@ -79,6 +82,7 @@ func NewContext(ctx Context) (Context, context.CancelFunc) {
82
}
83
}
84
newCtx.Context = c
85
+ newCtx.initMetrics()
86
return newCtx, wrappedCancel
87
}
88
@@ -97,6 +101,22 @@ func (ctx *Context) Filesystems() FileSystems {
101
return ctx.cfg.filesystems
102
}
103
104
+// Returns the active metrics registry for the context
105
+// EXPERIMENTAL: This API is subject to change.
106
+func (ctx *Context) GetMetricsRegistry() *prometheus.Registry {
107
+ return ctx.metricsRegistry
108
+}
109
+
110
+func (ctx *Context) initMetrics() {
111
+ ctx.metricsRegistry.MustRegister(
112
+ collectors.NewBuildInfoCollector(),
113
+ adminMetrics.requestCount,
114
+ adminMetrics.requestErrors,
115
+ globalMetrics.configSuccess,
116
+ globalMetrics.configSuccessTime,
117
+ )
118
+}
119
+
120
// OnExit executes f when the process exits gracefully.
121
// The function is only executed if the process is gracefully
122
// shut down while this context is active.
metrics.go
+16
-7
@@ -4,30 +4,33 @@ import (
4
"net/http"
5
6
"github.com/prometheus/client_golang/prometheus"
7
- "github.com/prometheus/client_golang/prometheus/collectors"
8
- "github.com/prometheus/client_golang/prometheus/promauto"
7
8
"github.com/caddyserver/caddy/v2/internal/metrics"
9
)
10
11
// define and register the metrics used in this package.
12
func init() {
15
- prometheus.MustRegister(collectors.NewBuildInfoCollector())
16
-
13
const ns, sub = "caddy", "admin"
18
-
19
- adminMetrics.requestCount = promauto.NewCounterVec(prometheus.CounterOpts{
14
+ adminMetrics.requestCount = prometheus.NewCounterVec(prometheus.CounterOpts{
15
Namespace: ns,
16
Subsystem: sub,
17
Name: "http_requests_total",
18
Help: "Counter of requests made to the Admin API's HTTP endpoints.",
19
}, []string{"handler", "path", "code", "method"})
25
- adminMetrics.requestErrors = promauto.NewCounterVec(prometheus.CounterOpts{
20
+ adminMetrics.requestErrors = prometheus.NewCounterVec(prometheus.CounterOpts{
21
Namespace: ns,
22
Subsystem: sub,
23
Name: "http_request_errors_total",
24
Help: "Number of requests resulting in middleware errors.",
25
}, []string{"handler", "path", "method"})
26
+ globalMetrics.configSuccess = prometheus.NewGauge(prometheus.GaugeOpts{
27
+ Name: "caddy_config_last_reload_successful",
28
+ Help: "Whether the last configuration reload attempt was successful.",
29
+ })
30
+ globalMetrics.configSuccessTime = prometheus.NewGauge(prometheus.GaugeOpts{
31
+ Name: "caddy_config_last_reload_success_timestamp_seconds",
32
+ Help: "Timestamp of the last successful configuration reload.",
33
+ })
34
}
35
36
// adminMetrics is a collection of metrics that can be tracked for the admin API.
@@ -36,6 +39,12 @@ var adminMetrics = struct {
39
requestErrors *prometheus.CounterVec
40
}{}
41
42
+// globalMetrics is a collection of metrics that can be tracked for Caddy global state
43
+var globalMetrics = struct {
44
+ configSuccess prometheus.Gauge
45
+ configSuccessTime prometheus.Gauge
46
+}{}
47
+
48
// Similar to promhttp.InstrumentHandlerCounter, but upper-cases method names
49
// instead of lower-casing them.
50
//
modules/caddyhttp/app.go
+4
@@ -347,6 +347,10 @@ func (app *App) Provision(ctx caddy.Context) error {
347
// route handler so that important security checks are done, etc.
348
primaryRoute := emptyHandler
349
if srv.Routes != nil {
350
+ if srv.Metrics != nil {
351
+ srv.Metrics.init = sync.Once{}
352
+ srv.Metrics.httpMetrics = &httpMetrics{}
353
+ }
354
err := srv.Routes.ProvisionHandlers(ctx, srv.Metrics)
355
if err != nil {
356
return fmt.Errorf("server %s: setting up route handlers: %v", srvName, err)
modules/caddyhttp/metrics.go
+43
-25
@@ -10,15 +10,23 @@ import (
10
"github.com/prometheus/client_golang/prometheus"
11
"github.com/prometheus/client_golang/prometheus/promauto"
12
13
+ "github.com/caddyserver/caddy/v2"
14
"github.com/caddyserver/caddy/v2/internal/metrics"
15
)
16
17
// Metrics configures metrics observations.
18
// EXPERIMENTAL and subject to change or removal.
18
-type Metrics struct{}
19
+type Metrics struct {
20
+ // Enable per-host metrics. Enabling this option may
21
+ // incur high-memory consumption, depending on the number of hosts
22
+ // managed by Caddy.
23
+ PerHost bool `json:"per_host,omitempty"`
24
+
25
+ init sync.Once
26
+ httpMetrics *httpMetrics `json:"-"`
27
+}
28
20
-var httpMetrics = struct {
21
- init sync.Once
29
+type httpMetrics struct {
30
requestInFlight *prometheus.GaugeVec
31
requestCount *prometheus.CounterVec
32
requestErrors *prometheus.CounterVec
@@ -26,27 +34,28 @@ var httpMetrics = struct {
34
requestSize *prometheus.HistogramVec
35
responseSize *prometheus.HistogramVec
36
responseDuration *prometheus.HistogramVec
29
-}{
30
- init: sync.Once{},
37
}
38
33
-func initHTTPMetrics() {
39
+func initHTTPMetrics(ctx caddy.Context, metrics *Metrics) {
40
const ns, sub = "caddy", "http"
35
-
41
+ registry := ctx.GetMetricsRegistry()
42
basicLabels := []string{"server", "handler"}
37
- httpMetrics.requestInFlight = promauto.NewGaugeVec(prometheus.GaugeOpts{
43
+ if metrics.PerHost {
44
+ basicLabels = append(basicLabels, "host")
45
+ }
46
+ metrics.httpMetrics.requestInFlight = promauto.With(registry).NewGaugeVec(prometheus.GaugeOpts{
47
Namespace: ns,
48
Subsystem: sub,
49
Name: "requests_in_flight",
50
Help: "Number of requests currently handled by this server.",
51
}, basicLabels)
43
- httpMetrics.requestErrors = promauto.NewCounterVec(prometheus.CounterOpts{
52
+ metrics.httpMetrics.requestErrors = promauto.With(registry).NewCounterVec(prometheus.CounterOpts{
53
Namespace: ns,
54
Subsystem: sub,
55
Name: "request_errors_total",
56
Help: "Number of requests resulting in middleware errors.",
57
}, basicLabels)
49
- httpMetrics.requestCount = promauto.NewCounterVec(prometheus.CounterOpts{
58
+ metrics.httpMetrics.requestCount = promauto.With(registry).NewCounterVec(prometheus.CounterOpts{
59
Namespace: ns,
60
Subsystem: sub,
61
Name: "requests_total",
@@ -58,28 +67,31 @@ func initHTTPMetrics() {
67
sizeBuckets := prometheus.ExponentialBuckets(256, 4, 8)
68
69
httpLabels := []string{"server", "handler", "code", "method"}
61
- httpMetrics.requestDuration = promauto.NewHistogramVec(prometheus.HistogramOpts{
70
+ if metrics.PerHost {
71
+ httpLabels = append(httpLabels, "host")
72
+ }
73
+ metrics.httpMetrics.requestDuration = promauto.With(registry).NewHistogramVec(prometheus.HistogramOpts{
74
Namespace: ns,
75
Subsystem: sub,
76
Name: "request_duration_seconds",
77
Help: "Histogram of round-trip request durations.",
78
Buckets: durationBuckets,
79
}, httpLabels)
68
- httpMetrics.requestSize = promauto.NewHistogramVec(prometheus.HistogramOpts{
80
+ metrics.httpMetrics.requestSize = promauto.With(registry).NewHistogramVec(prometheus.HistogramOpts{
81
Namespace: ns,
82
Subsystem: sub,
83
Name: "request_size_bytes",
84
Help: "Total size of the request. Includes body",
85
Buckets: sizeBuckets,
86
}, httpLabels)
75
- httpMetrics.responseSize = promauto.NewHistogramVec(prometheus.HistogramOpts{
87
+ metrics.httpMetrics.responseSize = promauto.With(registry).NewHistogramVec(prometheus.HistogramOpts{
88
Namespace: ns,
89
Subsystem: sub,
90
Name: "response_size_bytes",
91
Help: "Size of the returned response.",
92
Buckets: sizeBuckets,
93
}, httpLabels)
82
- httpMetrics.responseDuration = promauto.NewHistogramVec(prometheus.HistogramOpts{
94
+ metrics.httpMetrics.responseDuration = promauto.With(registry).NewHistogramVec(prometheus.HistogramOpts{
95
Namespace: ns,
96
Subsystem: sub,
97
Name: "response_duration_seconds",
@@ -101,14 +113,15 @@ func serverNameFromContext(ctx context.Context) string {
113
type metricsInstrumentedHandler struct {
114
handler string
115
mh MiddlewareHandler
116
+ metrics *Metrics
117
}
118
106
-func newMetricsInstrumentedHandler(handler string, mh MiddlewareHandler) *metricsInstrumentedHandler {
107
- httpMetrics.init.Do(func() {
108
- initHTTPMetrics()
119
+func newMetricsInstrumentedHandler(ctx caddy.Context, handler string, mh MiddlewareHandler, metrics *Metrics) *metricsInstrumentedHandler {
120
+ metrics.init.Do(func() {
121
+ initHTTPMetrics(ctx, metrics)
122
})
123
111
- return &metricsInstrumentedHandler{handler, mh}
124
+ return &metricsInstrumentedHandler{handler, mh, metrics}
125
}
126
127
func (h *metricsInstrumentedHandler) ServeHTTP(w http.ResponseWriter, r *http.Request, next Handler) error {
@@ -119,7 +132,12 @@ func (h *metricsInstrumentedHandler) ServeHTTP(w http.ResponseWriter, r *http.Re
132
// of a panic
133
statusLabels := prometheus.Labels{"server": server, "handler": h.handler, "method": method, "code": ""}
134
122
- inFlight := httpMetrics.requestInFlight.With(labels)
135
+ if h.metrics.PerHost {
136
+ labels["host"] = r.Host
137
+ statusLabels["host"] = r.Host
138
+ }
139
+
140
+ inFlight := h.metrics.httpMetrics.requestInFlight.With(labels)
141
inFlight.Inc()
142
defer inFlight.Dec()
143
@@ -131,13 +149,13 @@ func (h *metricsInstrumentedHandler) ServeHTTP(w http.ResponseWriter, r *http.Re
149
writeHeaderRecorder := ShouldBufferFunc(func(status int, header http.Header) bool {
150
statusLabels["code"] = metrics.SanitizeCode(status)
151
ttfb := time.Since(start).Seconds()
134
- httpMetrics.responseDuration.With(statusLabels).Observe(ttfb)
152
+ h.metrics.httpMetrics.responseDuration.With(statusLabels).Observe(ttfb)
153
return false
154
})
155
wrec := NewResponseRecorder(w, nil, writeHeaderRecorder)
156
err := h.mh.ServeHTTP(wrec, r, next)
157
dur := time.Since(start).Seconds()
140
- httpMetrics.requestCount.With(labels).Inc()
158
+ h.metrics.httpMetrics.requestCount.With(labels).Inc()
159
160
observeRequest := func(status int) {
161
// If the code hasn't been set yet, and we didn't encounter an error, we're
@@ -148,9 +166,9 @@ func (h *metricsInstrumentedHandler) ServeHTTP(w http.ResponseWriter, r *http.Re
166
statusLabels["code"] = metrics.SanitizeCode(status)
167
}
168
151
- httpMetrics.requestDuration.With(statusLabels).Observe(dur)
152
- httpMetrics.requestSize.With(statusLabels).Observe(float64(computeApproximateRequestSize(r)))
153
- httpMetrics.responseSize.With(statusLabels).Observe(float64(wrec.Size()))
169
+ h.metrics.httpMetrics.requestDuration.With(statusLabels).Observe(dur)
170
+ h.metrics.httpMetrics.requestSize.With(statusLabels).Observe(float64(computeApproximateRequestSize(r)))
171
+ h.metrics.httpMetrics.responseSize.With(statusLabels).Observe(float64(wrec.Size()))
172
}
173
174
if err != nil {
@@ -159,7 +177,7 @@ func (h *metricsInstrumentedHandler) ServeHTTP(w http.ResponseWriter, r *http.Re
177
observeRequest(handlerErr.StatusCode)
178
}
179
162
- httpMetrics.requestErrors.With(labels).Inc()
180
+ h.metrics.httpMetrics.requestErrors.With(labels).Inc()
181
182
return err
183
}
modules/caddyhttp/metrics_test.go
+189
-7
@@ -6,9 +6,10 @@ import (
6
"net/http"
7
"net/http/httptest"
8
"strings"
9
+ "sync"
10
"testing"
11
11
- "github.com/prometheus/client_golang/prometheus"
12
+ "github.com/caddyserver/caddy/v2"
13
"github.com/prometheus/client_golang/prometheus/testutil"
14
)
15
@@ -27,10 +28,15 @@ func TestServerNameFromContext(t *testing.T) {
28
}
29
30
func TestMetricsInstrumentedHandler(t *testing.T) {
31
+ ctx, _ := caddy.NewContext(caddy.Context{Context: context.Background()})
32
+ metrics := &Metrics{
33
+ init: sync.Once{},
34
+ httpMetrics: &httpMetrics{},
35
+ }
36
handlerErr := errors.New("oh noes")
37
response := []byte("hello world!")
38
h := HandlerFunc(func(w http.ResponseWriter, r *http.Request) error {
33
- if actual := testutil.ToFloat64(httpMetrics.requestInFlight); actual != 1.0 {
39
+ if actual := testutil.ToFloat64(metrics.httpMetrics.requestInFlight); actual != 1.0 {
40
t.Errorf("Not same: expected %#v, but got %#v", 1.0, actual)
41
}
42
if handlerErr == nil {
@@ -43,7 +49,7 @@ func TestMetricsInstrumentedHandler(t *testing.T) {
49
return h.ServeHTTP(w, r)
50
})
51
46
- ih := newMetricsInstrumentedHandler("bar", mh)
52
+ ih := newMetricsInstrumentedHandler(ctx, "bar", mh, metrics)
53
54
r := httptest.NewRequest("GET", "/", nil)
55
w := httptest.NewRecorder()
@@ -51,7 +57,7 @@ func TestMetricsInstrumentedHandler(t *testing.T) {
57
if actual := ih.ServeHTTP(w, r, h); actual != handlerErr {
58
t.Errorf("Not same: expected %#v, but got %#v", handlerErr, actual)
59
}
54
- if actual := testutil.ToFloat64(httpMetrics.requestInFlight); actual != 0.0 {
60
+ if actual := testutil.ToFloat64(metrics.httpMetrics.requestInFlight); actual != 0.0 {
61
t.Errorf("Not same: expected %#v, but got %#v", 0.0, actual)
62
}
63
@@ -64,7 +70,7 @@ func TestMetricsInstrumentedHandler(t *testing.T) {
70
mh = middlewareHandlerFunc(func(w http.ResponseWriter, r *http.Request, h Handler) error {
71
return nil
72
})
67
- ih = newMetricsInstrumentedHandler("empty", mh)
73
+ ih = newMetricsInstrumentedHandler(ctx, "empty", mh, metrics)
74
r = httptest.NewRequest("GET", "/", nil)
75
w = httptest.NewRecorder()
76
@@ -83,7 +89,7 @@ func TestMetricsInstrumentedHandler(t *testing.T) {
89
return Error(http.StatusTooManyRequests, nil)
90
})
91
86
- ih = newMetricsInstrumentedHandler("foo", mh)
92
+ ih = newMetricsInstrumentedHandler(ctx, "foo", mh, metrics)
93
94
r = httptest.NewRequest("GET", "/", nil)
95
w = httptest.NewRecorder()
@@ -183,7 +189,183 @@ func TestMetricsInstrumentedHandler(t *testing.T) {
189
caddy_http_request_errors_total{handler="bar",server="UNKNOWN"} 1
190
caddy_http_request_errors_total{handler="foo",server="UNKNOWN"} 1
191
`
186
- if err := testutil.GatherAndCompare(prometheus.DefaultGatherer, strings.NewReader(expected),
192
+ if err := testutil.GatherAndCompare(ctx.GetMetricsRegistry(), strings.NewReader(expected),
193
+ "caddy_http_request_size_bytes",
194
+ "caddy_http_response_size_bytes",
195
+ // caddy_http_request_duration_seconds_sum will vary based on how long the test took to run,
196
+ // so we check just the _bucket and _count metrics
197
+ "caddy_http_request_duration_seconds_bucket",
198
+ "caddy_http_request_duration_seconds_count",
199
+ "caddy_http_request_errors_total",
200
+ ); err != nil {
201
+ t.Errorf("received unexpected error: %s", err)
202
+ }
203
+}
204
+
205
+func TestMetricsInstrumentedHandlerPerHost(t *testing.T) {
206
+ ctx, _ := caddy.NewContext(caddy.Context{Context: context.Background()})
207
+ metrics := &Metrics{
208
+ PerHost: true,
209
+ init: sync.Once{},
210
+ httpMetrics: &httpMetrics{},
211
+ }
212
+ handlerErr := errors.New("oh noes")
213
+ response := []byte("hello world!")
214
+ h := HandlerFunc(func(w http.ResponseWriter, r *http.Request) error {
215
+ if actual := testutil.ToFloat64(metrics.httpMetrics.requestInFlight); actual != 1.0 {
216
+ t.Errorf("Not same: expected %#v, but got %#v", 1.0, actual)
217
+ }
218
+ if handlerErr == nil {
219
+ w.Write(response)
220
+ }
221
+ return handlerErr
222
+ })
223
+
224
+ mh := middlewareHandlerFunc(func(w http.ResponseWriter, r *http.Request, h Handler) error {
225
+ return h.ServeHTTP(w, r)
226
+ })
227
+
228
+ ih := newMetricsInstrumentedHandler(ctx, "bar", mh, metrics)
229
+
230
+ r := httptest.NewRequest("GET", "/", nil)
231
+ w := httptest.NewRecorder()
232
+
233
+ if actual := ih.ServeHTTP(w, r, h); actual != handlerErr {
234
+ t.Errorf("Not same: expected %#v, but got %#v", handlerErr, actual)
235
+ }
236
+ if actual := testutil.ToFloat64(metrics.httpMetrics.requestInFlight); actual != 0.0 {
237
+ t.Errorf("Not same: expected %#v, but got %#v", 0.0, actual)
238
+ }
239
+
240
+ handlerErr = nil
241
+ if err := ih.ServeHTTP(w, r, h); err != nil {
242
+ t.Errorf("Received unexpected error: %v", err)
243
+ }
244
+
245
+ // an empty handler - no errors, no header written
246
+ mh = middlewareHandlerFunc(func(w http.ResponseWriter, r *http.Request, h Handler) error {
247
+ return nil
248
+ })
249
+ ih = newMetricsInstrumentedHandler(ctx, "empty", mh, metrics)
250
+ r = httptest.NewRequest("GET", "/", nil)
251
+ w = httptest.NewRecorder()
252
+
253
+ if err := ih.ServeHTTP(w, r, h); err != nil {
254
+ t.Errorf("Received unexpected error: %v", err)
255
+ }
256
+ if actual := w.Result().StatusCode; actual != 200 {
257
+ t.Errorf("Not same: expected status code %#v, but got %#v", 200, actual)
258
+ }
259
+ if actual := w.Result().Header; len(actual) != 0 {
260
+ t.Errorf("Not empty: expected headers to be empty, but got %#v", actual)
261
+ }
262
+
263
+ // handler returning an error with an HTTP status
264
+ mh = middlewareHandlerFunc(func(w http.ResponseWriter, r *http.Request, h Handler) error {
265
+ return Error(http.StatusTooManyRequests, nil)
266
+ })
267
+
268
+ ih = newMetricsInstrumentedHandler(ctx, "foo", mh, metrics)
269
+
270
+ r = httptest.NewRequest("GET", "/", nil)
271
+ w = httptest.NewRecorder()
272
+
273
+ if err := ih.ServeHTTP(w, r, nil); err == nil {
274
+ t.Errorf("expected error to be propagated")
275
+ }
276
+
277
+ expected := `
278
+ # HELP caddy_http_request_duration_seconds Histogram of round-trip request durations.
279
+ # TYPE caddy_http_request_duration_seconds histogram
280
+ caddy_http_request_duration_seconds_bucket{code="429",handler="foo",host="example.com",method="GET",server="UNKNOWN",le="0.005"} 1
281
+ caddy_http_request_duration_seconds_bucket{code="429",handler="foo",host="example.com",method="GET",server="UNKNOWN",le="0.01"} 1
282
+ caddy_http_request_duration_seconds_bucket{code="429",handler="foo",host="example.com",method="GET",server="UNKNOWN",le="0.025"} 1
283
+ caddy_http_request_duration_seconds_bucket{code="429",handler="foo",host="example.com",method="GET",server="UNKNOWN",le="0.05"} 1
284
+ caddy_http_request_duration_seconds_bucket{code="429",handler="foo",host="example.com",method="GET",server="UNKNOWN",le="0.1"} 1
285
+ caddy_http_request_duration_seconds_bucket{code="429",handler="foo",host="example.com",method="GET",server="UNKNOWN",le="0.25"} 1
286
+ caddy_http_request_duration_seconds_bucket{code="429",handler="foo",host="example.com",method="GET",server="UNKNOWN",le="0.5"} 1
287
+ caddy_http_request_duration_seconds_bucket{code="429",handler="foo",host="example.com",method="GET",server="UNKNOWN",le="1"} 1
288
+ caddy_http_request_duration_seconds_bucket{code="429",handler="foo",host="example.com",method="GET",server="UNKNOWN",le="2.5"} 1
289
+ caddy_http_request_duration_seconds_bucket{code="429",handler="foo",host="example.com",method="GET",server="UNKNOWN",le="5"} 1
290
+ caddy_http_request_duration_seconds_bucket{code="429",handler="foo",host="example.com",method="GET",server="UNKNOWN",le="10"} 1
291
+ caddy_http_request_duration_seconds_bucket{code="429",handler="foo",host="example.com",method="GET",server="UNKNOWN",le="+Inf"} 1
292
+ caddy_http_request_duration_seconds_count{code="429",handler="foo",host="example.com",method="GET",server="UNKNOWN"} 1
293
+ # HELP caddy_http_request_size_bytes Total size of the request. Includes body
294
+ # TYPE caddy_http_request_size_bytes histogram
295
+ caddy_http_request_size_bytes_bucket{code="200",handler="bar",host="example.com",method="GET",server="UNKNOWN",le="256"} 1
296
+ caddy_http_request_size_bytes_bucket{code="200",handler="bar",host="example.com",method="GET",server="UNKNOWN",le="1024"} 1
297
+ caddy_http_request_size_bytes_bucket{code="200",handler="bar",host="example.com",method="GET",server="UNKNOWN",le="4096"} 1
298
+ caddy_http_request_size_bytes_bucket{code="200",handler="bar",host="example.com",method="GET",server="UNKNOWN",le="16384"} 1
299
+ caddy_http_request_size_bytes_bucket{code="200",handler="bar",host="example.com",method="GET",server="UNKNOWN",le="65536"} 1
300
+ caddy_http_request_size_bytes_bucket{code="200",handler="bar",host="example.com",method="GET",server="UNKNOWN",le="262144"} 1
301
+ caddy_http_request_size_bytes_bucket{code="200",handler="bar",host="example.com",method="GET",server="UNKNOWN",le="1.048576e+06"} 1
302
+ caddy_http_request_size_bytes_bucket{code="200",handler="bar",host="example.com",method="GET",server="UNKNOWN",le="4.194304e+06"} 1
303
+ caddy_http_request_size_bytes_bucket{code="200",handler="bar",host="example.com",method="GET",server="UNKNOWN",le="+Inf"} 1
304
+ caddy_http_request_size_bytes_sum{code="200",handler="bar",host="example.com",method="GET",server="UNKNOWN"} 23
305
+ caddy_http_request_size_bytes_count{code="200",handler="bar",host="example.com",method="GET",server="UNKNOWN"} 1
306
+ caddy_http_request_size_bytes_bucket{code="200",handler="empty",host="example.com",method="GET",server="UNKNOWN",le="256"} 1
307
+ caddy_http_request_size_bytes_bucket{code="200",handler="empty",host="example.com",method="GET",server="UNKNOWN",le="1024"} 1
308
+ caddy_http_request_size_bytes_bucket{code="200",handler="empty",host="example.com",method="GET",server="UNKNOWN",le="4096"} 1
309
+ caddy_http_request_size_bytes_bucket{code="200",handler="empty",host="example.com",method="GET",server="UNKNOWN",le="16384"} 1
310
+ caddy_http_request_size_bytes_bucket{code="200",handler="empty",host="example.com",method="GET",server="UNKNOWN",le="65536"} 1
311
+ caddy_http_request_size_bytes_bucket{code="200",handler="empty",host="example.com",method="GET",server="UNKNOWN",le="262144"} 1
312
+ caddy_http_request_size_bytes_bucket{code="200",handler="empty",host="example.com",method="GET",server="UNKNOWN",le="1.048576e+06"} 1
313
+ caddy_http_request_size_bytes_bucket{code="200",handler="empty",host="example.com",method="GET",server="UNKNOWN",le="4.194304e+06"} 1
314
+ caddy_http_request_size_bytes_bucket{code="200",handler="empty",host="example.com",method="GET",server="UNKNOWN",le="+Inf"} 1
315
+ caddy_http_request_size_bytes_sum{code="200",handler="empty",host="example.com",method="GET",server="UNKNOWN"} 23
316
+ caddy_http_request_size_bytes_count{code="200",handler="empty",host="example.com",method="GET",server="UNKNOWN"} 1
317
+ caddy_http_request_size_bytes_bucket{code="429",handler="foo",host="example.com",method="GET",server="UNKNOWN",le="256"} 1
318
+ caddy_http_request_size_bytes_bucket{code="429",handler="foo",host="example.com",method="GET",server="UNKNOWN",le="1024"} 1
319
+ caddy_http_request_size_bytes_bucket{code="429",handler="foo",host="example.com",method="GET",server="UNKNOWN",le="4096"} 1
320
+ caddy_http_request_size_bytes_bucket{code="429",handler="foo",host="example.com",method="GET",server="UNKNOWN",le="16384"} 1
321
+ caddy_http_request_size_bytes_bucket{code="429",handler="foo",host="example.com",method="GET",server="UNKNOWN",le="65536"} 1
322
+ caddy_http_request_size_bytes_bucket{code="429",handler="foo",host="example.com",method="GET",server="UNKNOWN",le="262144"} 1
323
+ caddy_http_request_size_bytes_bucket{code="429",handler="foo",host="example.com",method="GET",server="UNKNOWN",le="1.048576e+06"} 1
324
+ caddy_http_request_size_bytes_bucket{code="429",handler="foo",host="example.com",method="GET",server="UNKNOWN",le="4.194304e+06"} 1
325
+ caddy_http_request_size_bytes_bucket{code="429",handler="foo",host="example.com",method="GET",server="UNKNOWN",le="+Inf"} 1
326
+ caddy_http_request_size_bytes_sum{code="429",handler="foo",host="example.com",method="GET",server="UNKNOWN"} 23
327
+ caddy_http_request_size_bytes_count{code="429",handler="foo",host="example.com",method="GET",server="UNKNOWN"} 1
328
+ # HELP caddy_http_response_size_bytes Size of the returned response.
329
+ # TYPE caddy_http_response_size_bytes histogram
330
+ caddy_http_response_size_bytes_bucket{code="200",handler="bar",host="example.com",method="GET",server="UNKNOWN",le="256"} 1
331
+ caddy_http_response_size_bytes_bucket{code="200",handler="bar",host="example.com",method="GET",server="UNKNOWN",le="1024"} 1
332
+ caddy_http_response_size_bytes_bucket{code="200",handler="bar",host="example.com",method="GET",server="UNKNOWN",le="4096"} 1
333
+ caddy_http_response_size_bytes_bucket{code="200",handler="bar",host="example.com",method="GET",server="UNKNOWN",le="16384"} 1
334
+ caddy_http_response_size_bytes_bucket{code="200",handler="bar",host="example.com",method="GET",server="UNKNOWN",le="65536"} 1
335
+ caddy_http_response_size_bytes_bucket{code="200",handler="bar",host="example.com",method="GET",server="UNKNOWN",le="262144"} 1
336
+ caddy_http_response_size_bytes_bucket{code="200",handler="bar",host="example.com",method="GET",server="UNKNOWN",le="1.048576e+06"} 1
337
+ caddy_http_response_size_bytes_bucket{code="200",handler="bar",host="example.com",method="GET",server="UNKNOWN",le="4.194304e+06"} 1
338
+ caddy_http_response_size_bytes_bucket{code="200",handler="bar",host="example.com",method="GET",server="UNKNOWN",le="+Inf"} 1
339
+ caddy_http_response_size_bytes_sum{code="200",handler="bar",host="example.com",method="GET",server="UNKNOWN"} 12
340
+ caddy_http_response_size_bytes_count{code="200",handler="bar",host="example.com",method="GET",server="UNKNOWN"} 1
341
+ caddy_http_response_size_bytes_bucket{code="200",handler="empty",host="example.com",method="GET",server="UNKNOWN",le="256"} 1
342
+ caddy_http_response_size_bytes_bucket{code="200",handler="empty",host="example.com",method="GET",server="UNKNOWN",le="1024"} 1
343
+ caddy_http_response_size_bytes_bucket{code="200",handler="empty",host="example.com",method="GET",server="UNKNOWN",le="4096"} 1
344
+ caddy_http_response_size_bytes_bucket{code="200",handler="empty",host="example.com",method="GET",server="UNKNOWN",le="16384"} 1
345
+ caddy_http_response_size_bytes_bucket{code="200",handler="empty",host="example.com",method="GET",server="UNKNOWN",le="65536"} 1
346
+ caddy_http_response_size_bytes_bucket{code="200",handler="empty",host="example.com",method="GET",server="UNKNOWN",le="262144"} 1
347
+ caddy_http_response_size_bytes_bucket{code="200",handler="empty",host="example.com",method="GET",server="UNKNOWN",le="1.048576e+06"} 1
348
+ caddy_http_response_size_bytes_bucket{code="200",handler="empty",host="example.com",method="GET",server="UNKNOWN",le="4.194304e+06"} 1
349
+ caddy_http_response_size_bytes_bucket{code="200",handler="empty",host="example.com",method="GET",server="UNKNOWN",le="+Inf"} 1
350
+ caddy_http_response_size_bytes_sum{code="200",handler="empty",host="example.com",method="GET",server="UNKNOWN"} 0
351
+ caddy_http_response_size_bytes_count{code="200",handler="empty",host="example.com",method="GET",server="UNKNOWN"} 1
352
+ caddy_http_response_size_bytes_bucket{code="429",handler="foo",host="example.com",method="GET",server="UNKNOWN",le="256"} 1
353
+ caddy_http_response_size_bytes_bucket{code="429",handler="foo",host="example.com",method="GET",server="UNKNOWN",le="1024"} 1
354
+ caddy_http_response_size_bytes_bucket{code="429",handler="foo",host="example.com",method="GET",server="UNKNOWN",le="4096"} 1
355
+ caddy_http_response_size_bytes_bucket{code="429",handler="foo",host="example.com",method="GET",server="UNKNOWN",le="16384"} 1
356
+ caddy_http_response_size_bytes_bucket{code="429",handler="foo",host="example.com",method="GET",server="UNKNOWN",le="65536"} 1
357
+ caddy_http_response_size_bytes_bucket{code="429",handler="foo",host="example.com",method="GET",server="UNKNOWN",le="262144"} 1
358
+ caddy_http_response_size_bytes_bucket{code="429",handler="foo",host="example.com",method="GET",server="UNKNOWN",le="1.048576e+06"} 1
359
+ caddy_http_response_size_bytes_bucket{code="429",handler="foo",host="example.com",method="GET",server="UNKNOWN",le="4.194304e+06"} 1
360
+ caddy_http_response_size_bytes_bucket{code="429",handler="foo",host="example.com",method="GET",server="UNKNOWN",le="+Inf"} 1
361
+ caddy_http_response_size_bytes_sum{code="429",handler="foo",host="example.com",method="GET",server="UNKNOWN"} 0
362
+ caddy_http_response_size_bytes_count{code="429",handler="foo",host="example.com",method="GET",server="UNKNOWN"} 1
363
+ # HELP caddy_http_request_errors_total Number of requests resulting in middleware errors.
364
+ # TYPE caddy_http_request_errors_total counter
365
+ caddy_http_request_errors_total{handler="bar",host="example.com",server="UNKNOWN"} 1
366
+ caddy_http_request_errors_total{handler="foo",host="example.com",server="UNKNOWN"} 1
367
+ `
368
+ if err := testutil.GatherAndCompare(ctx.GetMetricsRegistry(), strings.NewReader(expected),
369
"caddy_http_request_size_bytes",
370
"caddy_http_response_size_bytes",
371
// caddy_http_request_duration_seconds_sum will vary based on how long the test took to run,
modules/caddyhttp/routes.go
+2
-2
@@ -314,11 +314,11 @@ func wrapRoute(route Route) Middleware {
314
// we need to pull this particular MiddlewareHandler
315
// pointer into its own stack frame to preserve it so it
316
// won't be overwritten in future loop iterations.
317
-func wrapMiddleware(_ caddy.Context, mh MiddlewareHandler, metrics *Metrics) Middleware {
317
+func wrapMiddleware(ctx caddy.Context, mh MiddlewareHandler, metrics *Metrics) Middleware {
318
handlerToUse := mh
319
if metrics != nil {
320
// wrap the middleware with metrics instrumentation
321
- handlerToUse = newMetricsInstrumentedHandler(caddy.GetModuleName(mh), mh)
321
+ handlerToUse = newMetricsInstrumentedHandler(ctx, caddy.GetModuleName(mh), mh, metrics)
322
}
323
324
return func(next Handler) Handler {
modules/metrics/adminmetrics.go
+25
-7
@@ -15,8 +15,11 @@
15
package metrics
16
17
import (
18
+ "errors"
19
"net/http"
20
21
+ "github.com/prometheus/client_golang/prometheus"
22
+
23
"github.com/caddyserver/caddy/v2"
24
)
25
@@ -29,7 +32,11 @@ func init() {
32
// is permanently mounted to the admin API endpoint at "/metrics".
33
// See the Metrics module for a configurable endpoint that is usable if the
34
// Admin API is disabled.
32
-type AdminMetrics struct{}
35
+type AdminMetrics struct {
36
+ registry *prometheus.Registry
37
+
38
+ metricsHandler http.Handler
39
+}
40
41
// CaddyModule returns the Caddy module information.
42
func (AdminMetrics) CaddyModule() caddy.ModuleInfo {
@@ -39,17 +46,28 @@ func (AdminMetrics) CaddyModule() caddy.ModuleInfo {
46
}
47
}
48
49
+// Provision -
50
+func (m *AdminMetrics) Provision(ctx caddy.Context) error {
51
+ m.registry = ctx.GetMetricsRegistry()
52
+ if m.registry == nil {
53
+ return errors.New("no metrics registry found")
54
+ }
55
+ m.metricsHandler = createMetricsHandler(nil, false, m.registry)
56
+ return nil
57
+}
58
+
59
// Routes returns a route for the /metrics endpoint.
60
func (m *AdminMetrics) Routes() []caddy.AdminRoute {
44
- metricsHandler := createMetricsHandler(nil, false)
45
- h := caddy.AdminHandlerFunc(func(w http.ResponseWriter, r *http.Request) error {
46
- metricsHandler.ServeHTTP(w, r)
47
- return nil
48
- })
49
- return []caddy.AdminRoute{{Pattern: "/metrics", Handler: h}}
61
+ return []caddy.AdminRoute{{Pattern: "/metrics", Handler: caddy.AdminHandlerFunc(m.serveHTTP)}}
62
+}
63
+
64
+func (m *AdminMetrics) serveHTTP(w http.ResponseWriter, r *http.Request) error {
65
+ m.metricsHandler.ServeHTTP(w, r)
66
+ return nil
67
}
68
69
// Interface guards
70
var (
71
+ _ caddy.Provisioner = (*AdminMetrics)(nil)
72
_ caddy.AdminRouter = (*AdminMetrics)(nil)
73
)
modules/metrics/metrics.go
+9
-4
@@ -15,6 +15,7 @@
15
package metrics
16
17
import (
18
+ "errors"
19
"net/http"
20
21
"github.com/prometheus/client_golang/prometheus"
@@ -62,7 +63,11 @@ func (l *zapLogger) Println(v ...any) {
63
// Provision sets up m.
64
func (m *Metrics) Provision(ctx caddy.Context) error {
65
log := ctx.Logger()
65
- m.metricsHandler = createMetricsHandler(&zapLogger{log}, !m.DisableOpenMetrics)
66
+ registry := ctx.GetMetricsRegistry()
67
+ if registry == nil {
68
+ return errors.New("no metrics registry found")
69
+ }
70
+ m.metricsHandler = createMetricsHandler(&zapLogger{log}, !m.DisableOpenMetrics, registry)
71
return nil
72
}
73
@@ -107,9 +112,9 @@ var (
112
_ caddyfile.Unmarshaler = (*Metrics)(nil)
113
)
114
110
-func createMetricsHandler(logger promhttp.Logger, enableOpenMetrics bool) http.Handler {
111
- return promhttp.InstrumentMetricHandler(prometheus.DefaultRegisterer,
112
- promhttp.HandlerFor(prometheus.DefaultGatherer, promhttp.HandlerOpts{
115
+func createMetricsHandler(logger promhttp.Logger, enableOpenMetrics bool, registry *prometheus.Registry) http.Handler {
116
+ return promhttp.InstrumentMetricHandler(registry,
117
+ promhttp.HandlerFor(registry, promhttp.HandlerOpts{
118
// will only log errors if logger is non-nil
119
ErrorLog: logger,
120