Fast and extensible multi-platform HTTP/1-2-3 web server with automatic HTTPS
Mirrored from https://github.com/Slazee/caddy.git
Synced Nov 12, 19:30 UTC
metrics: Record request metrics on HTTP errors (#5979)
Tim Geoghegan committed
Dec 15, 2023 at 12:14 UTC
387545a895ea71ae9f4f4e512dca136f92e45a74
2 files changed
+142
-11
modules/caddyhttp/metrics.go
+23
-11
@@ -2,6 +2,7 @@ package caddyhttp
2
3
import (
4
"context"
5
+ "errors"
6
"net/http"
7
"sync"
8
"time"
@@ -137,22 +138,33 @@ func (h *metricsInstrumentedHandler) ServeHTTP(w http.ResponseWriter, r *http.Re
138
err := h.mh.ServeHTTP(wrec, r, next)
139
dur := time.Since(start).Seconds()
140
httpMetrics.requestCount.With(labels).Inc()
141
+
142
+ observeRequest := func(status int) {
143
+ // If the code hasn't been set yet, and we didn't encounter an error, we're
144
+ // probably falling through with an empty handler.
145
+ if statusLabels["code"] == "" {
146
+ // we still sanitize it, even though it's likely to be 0. A 200 is
147
+ // returned on fallthrough so we want to reflect that.
148
+ statusLabels["code"] = metrics.SanitizeCode(status)
149
+ }
150
+
151
+ httpMetrics.requestDuration.With(statusLabels).Observe(dur)
152
+ httpMetrics.requestSize.With(statusLabels).Observe(float64(computeApproximateRequestSize(r)))
153
+ httpMetrics.responseSize.With(statusLabels).Observe(float64(wrec.Size()))
154
+ }
155
+
156
if err != nil {
157
+ var handlerErr HandlerError
158
+ if errors.As(err, &handlerErr) {
159
+ observeRequest(handlerErr.StatusCode)
160
+ }
161
+
162
httpMetrics.requestErrors.With(labels).Inc()
142
- return err
143
- }
163
145
- // If the code hasn't been set yet, and we didn't encounter an error, we're
146
- // probably falling through with an empty handler.
147
- if statusLabels["code"] == "" {
148
- // we still sanitize it, even though it's likely to be 0. A 200 is
149
- // returned on fallthrough so we want to reflect that.
150
- statusLabels["code"] = metrics.SanitizeCode(wrec.Status())
164
+ return err
165
}
166
153
- httpMetrics.requestDuration.With(statusLabels).Observe(dur)
154
- httpMetrics.requestSize.With(statusLabels).Observe(float64(computeApproximateRequestSize(r)))
155
- httpMetrics.responseSize.With(statusLabels).Observe(float64(wrec.Size()))
167
+ observeRequest(wrec.Status())
168
169
return nil
170
}
modules/caddyhttp/metrics_test.go
+119
@@ -5,8 +5,10 @@ import (
5
"errors"
6
"net/http"
7
"net/http/httptest"
8
+ "strings"
9
"testing"
10
11
+ "github.com/prometheus/client_golang/prometheus"
12
"github.com/prometheus/client_golang/prometheus/testutil"
13
)
14
@@ -75,6 +77,123 @@ func TestMetricsInstrumentedHandler(t *testing.T) {
77
if actual := w.Result().Header; len(actual) != 0 {
78
t.Errorf("Not empty: expected headers to be empty, but got %#v", actual)
79
}
80
+
81
+ // handler returning an error with an HTTP status
82
+ mh = middlewareHandlerFunc(func(w http.ResponseWriter, r *http.Request, h Handler) error {
83
+ return Error(http.StatusTooManyRequests, nil)
84
+ })
85
+
86
+ ih = newMetricsInstrumentedHandler("foo", mh)
87
+
88
+ r = httptest.NewRequest("GET", "/", nil)
89
+ w = httptest.NewRecorder()
90
+
91
+ if err := ih.ServeHTTP(w, r, nil); err == nil {
92
+ t.Errorf("expected error to be propagated")
93
+ }
94
+
95
+ expected := `
96
+ # HELP caddy_http_request_duration_seconds Histogram of round-trip request durations.
97
+ # TYPE caddy_http_request_duration_seconds histogram
98
+ caddy_http_request_duration_seconds_bucket{code="429",handler="foo",method="GET",server="UNKNOWN",le="0.005"} 1
99
+ caddy_http_request_duration_seconds_bucket{code="429",handler="foo",method="GET",server="UNKNOWN",le="0.01"} 1
100
+ caddy_http_request_duration_seconds_bucket{code="429",handler="foo",method="GET",server="UNKNOWN",le="0.025"} 1
101
+ caddy_http_request_duration_seconds_bucket{code="429",handler="foo",method="GET",server="UNKNOWN",le="0.05"} 1
102
+ caddy_http_request_duration_seconds_bucket{code="429",handler="foo",method="GET",server="UNKNOWN",le="0.1"} 1
103
+ caddy_http_request_duration_seconds_bucket{code="429",handler="foo",method="GET",server="UNKNOWN",le="0.25"} 1
104
+ caddy_http_request_duration_seconds_bucket{code="429",handler="foo",method="GET",server="UNKNOWN",le="0.5"} 1
105
+ caddy_http_request_duration_seconds_bucket{code="429",handler="foo",method="GET",server="UNKNOWN",le="1"} 1
106
+ caddy_http_request_duration_seconds_bucket{code="429",handler="foo",method="GET",server="UNKNOWN",le="2.5"} 1
107
+ caddy_http_request_duration_seconds_bucket{code="429",handler="foo",method="GET",server="UNKNOWN",le="5"} 1
108
+ caddy_http_request_duration_seconds_bucket{code="429",handler="foo",method="GET",server="UNKNOWN",le="10"} 1
109
+ caddy_http_request_duration_seconds_bucket{code="429",handler="foo",method="GET",server="UNKNOWN",le="+Inf"} 1
110
+ caddy_http_request_duration_seconds_count{code="429",handler="foo",method="GET",server="UNKNOWN"} 1
111
+ # HELP caddy_http_request_size_bytes Total size of the request. Includes body
112
+ # TYPE caddy_http_request_size_bytes histogram
113
+ caddy_http_request_size_bytes_bucket{code="200",handler="bar",method="GET",server="UNKNOWN",le="256"} 1
114
+ caddy_http_request_size_bytes_bucket{code="200",handler="bar",method="GET",server="UNKNOWN",le="1024"} 1
115
+ caddy_http_request_size_bytes_bucket{code="200",handler="bar",method="GET",server="UNKNOWN",le="4096"} 1
116
+ caddy_http_request_size_bytes_bucket{code="200",handler="bar",method="GET",server="UNKNOWN",le="16384"} 1
117
+ caddy_http_request_size_bytes_bucket{code="200",handler="bar",method="GET",server="UNKNOWN",le="65536"} 1
118
+ caddy_http_request_size_bytes_bucket{code="200",handler="bar",method="GET",server="UNKNOWN",le="262144"} 1
119
+ caddy_http_request_size_bytes_bucket{code="200",handler="bar",method="GET",server="UNKNOWN",le="1.048576e+06"} 1
120
+ caddy_http_request_size_bytes_bucket{code="200",handler="bar",method="GET",server="UNKNOWN",le="4.194304e+06"} 1
121
+ caddy_http_request_size_bytes_bucket{code="200",handler="bar",method="GET",server="UNKNOWN",le="+Inf"} 1
122
+ caddy_http_request_size_bytes_sum{code="200",handler="bar",method="GET",server="UNKNOWN"} 23
123
+ caddy_http_request_size_bytes_count{code="200",handler="bar",method="GET",server="UNKNOWN"} 1
124
+ caddy_http_request_size_bytes_bucket{code="200",handler="empty",method="GET",server="UNKNOWN",le="256"} 1
125
+ caddy_http_request_size_bytes_bucket{code="200",handler="empty",method="GET",server="UNKNOWN",le="1024"} 1
126
+ caddy_http_request_size_bytes_bucket{code="200",handler="empty",method="GET",server="UNKNOWN",le="4096"} 1
127
+ caddy_http_request_size_bytes_bucket{code="200",handler="empty",method="GET",server="UNKNOWN",le="16384"} 1
128
+ caddy_http_request_size_bytes_bucket{code="200",handler="empty",method="GET",server="UNKNOWN",le="65536"} 1
129
+ caddy_http_request_size_bytes_bucket{code="200",handler="empty",method="GET",server="UNKNOWN",le="262144"} 1
130
+ caddy_http_request_size_bytes_bucket{code="200",handler="empty",method="GET",server="UNKNOWN",le="1.048576e+06"} 1
131
+ caddy_http_request_size_bytes_bucket{code="200",handler="empty",method="GET",server="UNKNOWN",le="4.194304e+06"} 1
132
+ caddy_http_request_size_bytes_bucket{code="200",handler="empty",method="GET",server="UNKNOWN",le="+Inf"} 1
133
+ caddy_http_request_size_bytes_sum{code="200",handler="empty",method="GET",server="UNKNOWN"} 23
134
+ caddy_http_request_size_bytes_count{code="200",handler="empty",method="GET",server="UNKNOWN"} 1
135
+ caddy_http_request_size_bytes_bucket{code="429",handler="foo",method="GET",server="UNKNOWN",le="256"} 1
136
+ caddy_http_request_size_bytes_bucket{code="429",handler="foo",method="GET",server="UNKNOWN",le="1024"} 1
137
+ caddy_http_request_size_bytes_bucket{code="429",handler="foo",method="GET",server="UNKNOWN",le="4096"} 1
138
+ caddy_http_request_size_bytes_bucket{code="429",handler="foo",method="GET",server="UNKNOWN",le="16384"} 1
139
+ caddy_http_request_size_bytes_bucket{code="429",handler="foo",method="GET",server="UNKNOWN",le="65536"} 1
140
+ caddy_http_request_size_bytes_bucket{code="429",handler="foo",method="GET",server="UNKNOWN",le="262144"} 1
141
+ caddy_http_request_size_bytes_bucket{code="429",handler="foo",method="GET",server="UNKNOWN",le="1.048576e+06"} 1
142
+ caddy_http_request_size_bytes_bucket{code="429",handler="foo",method="GET",server="UNKNOWN",le="4.194304e+06"} 1
143
+ caddy_http_request_size_bytes_bucket{code="429",handler="foo",method="GET",server="UNKNOWN",le="+Inf"} 1
144
+ caddy_http_request_size_bytes_sum{code="429",handler="foo",method="GET",server="UNKNOWN"} 23
145
+ caddy_http_request_size_bytes_count{code="429",handler="foo",method="GET",server="UNKNOWN"} 1
146
+ # HELP caddy_http_response_size_bytes Size of the returned response.
147
+ # TYPE caddy_http_response_size_bytes histogram
148
+ caddy_http_response_size_bytes_bucket{code="200",handler="bar",method="GET",server="UNKNOWN",le="256"} 1
149
+ caddy_http_response_size_bytes_bucket{code="200",handler="bar",method="GET",server="UNKNOWN",le="1024"} 1
150
+ caddy_http_response_size_bytes_bucket{code="200",handler="bar",method="GET",server="UNKNOWN",le="4096"} 1
151
+ caddy_http_response_size_bytes_bucket{code="200",handler="bar",method="GET",server="UNKNOWN",le="16384"} 1
152
+ caddy_http_response_size_bytes_bucket{code="200",handler="bar",method="GET",server="UNKNOWN",le="65536"} 1
153
+ caddy_http_response_size_bytes_bucket{code="200",handler="bar",method="GET",server="UNKNOWN",le="262144"} 1
154
+ caddy_http_response_size_bytes_bucket{code="200",handler="bar",method="GET",server="UNKNOWN",le="1.048576e+06"} 1
155
+ caddy_http_response_size_bytes_bucket{code="200",handler="bar",method="GET",server="UNKNOWN",le="4.194304e+06"} 1
156
+ caddy_http_response_size_bytes_bucket{code="200",handler="bar",method="GET",server="UNKNOWN",le="+Inf"} 1
157
+ caddy_http_response_size_bytes_sum{code="200",handler="bar",method="GET",server="UNKNOWN"} 12
158
+ caddy_http_response_size_bytes_count{code="200",handler="bar",method="GET",server="UNKNOWN"} 1
159
+ caddy_http_response_size_bytes_bucket{code="200",handler="empty",method="GET",server="UNKNOWN",le="256"} 1
160
+ caddy_http_response_size_bytes_bucket{code="200",handler="empty",method="GET",server="UNKNOWN",le="1024"} 1
161
+ caddy_http_response_size_bytes_bucket{code="200",handler="empty",method="GET",server="UNKNOWN",le="4096"} 1
162
+ caddy_http_response_size_bytes_bucket{code="200",handler="empty",method="GET",server="UNKNOWN",le="16384"} 1
163
+ caddy_http_response_size_bytes_bucket{code="200",handler="empty",method="GET",server="UNKNOWN",le="65536"} 1
164
+ caddy_http_response_size_bytes_bucket{code="200",handler="empty",method="GET",server="UNKNOWN",le="262144"} 1
165
+ caddy_http_response_size_bytes_bucket{code="200",handler="empty",method="GET",server="UNKNOWN",le="1.048576e+06"} 1
166
+ caddy_http_response_size_bytes_bucket{code="200",handler="empty",method="GET",server="UNKNOWN",le="4.194304e+06"} 1
167
+ caddy_http_response_size_bytes_bucket{code="200",handler="empty",method="GET",server="UNKNOWN",le="+Inf"} 1
168
+ caddy_http_response_size_bytes_sum{code="200",handler="empty",method="GET",server="UNKNOWN"} 0
169
+ caddy_http_response_size_bytes_count{code="200",handler="empty",method="GET",server="UNKNOWN"} 1
170
+ caddy_http_response_size_bytes_bucket{code="429",handler="foo",method="GET",server="UNKNOWN",le="256"} 1
171
+ caddy_http_response_size_bytes_bucket{code="429",handler="foo",method="GET",server="UNKNOWN",le="1024"} 1
172
+ caddy_http_response_size_bytes_bucket{code="429",handler="foo",method="GET",server="UNKNOWN",le="4096"} 1
173
+ caddy_http_response_size_bytes_bucket{code="429",handler="foo",method="GET",server="UNKNOWN",le="16384"} 1
174
+ caddy_http_response_size_bytes_bucket{code="429",handler="foo",method="GET",server="UNKNOWN",le="65536"} 1
175
+ caddy_http_response_size_bytes_bucket{code="429",handler="foo",method="GET",server="UNKNOWN",le="262144"} 1
176
+ caddy_http_response_size_bytes_bucket{code="429",handler="foo",method="GET",server="UNKNOWN",le="1.048576e+06"} 1
177
+ caddy_http_response_size_bytes_bucket{code="429",handler="foo",method="GET",server="UNKNOWN",le="4.194304e+06"} 1
178
+ caddy_http_response_size_bytes_bucket{code="429",handler="foo",method="GET",server="UNKNOWN",le="+Inf"} 1
179
+ caddy_http_response_size_bytes_sum{code="429",handler="foo",method="GET",server="UNKNOWN"} 0
180
+ caddy_http_response_size_bytes_count{code="429",handler="foo",method="GET",server="UNKNOWN"} 1
181
+ # HELP caddy_http_request_errors_total Number of requests resulting in middleware errors.
182
+ # TYPE caddy_http_request_errors_total counter
183
+ caddy_http_request_errors_total{handler="bar",server="UNKNOWN"} 1
184
+ caddy_http_request_errors_total{handler="foo",server="UNKNOWN"} 1
185
+ `
186
+ if err := testutil.GatherAndCompare(prometheus.DefaultGatherer, strings.NewReader(expected),
187
+ "caddy_http_request_size_bytes",
188
+ "caddy_http_response_size_bytes",
189
+ // caddy_http_request_duration_seconds_sum will vary based on how long the test took to run,
190
+ // so we check just the _bucket and _count metrics
191
+ "caddy_http_request_duration_seconds_bucket",
192
+ "caddy_http_request_duration_seconds_count",
193
+ "caddy_http_request_errors_total",
194
+ ); err != nil {
195
+ t.Errorf("received unexpected error: %s", err)
196
+ }
197
}
198
199
type middlewareHandlerFunc func(http.ResponseWriter, *http.Request, Handler) error