Fast and extensible multi-platform HTTP/1-2-3 web server with automatic HTTPS
Mirrored from https://github.com/Slazee/caddy.git
Synced Nov 11, 12:15 UTC
metrics: Implement pushing via OLTP (#7664)
Kévin Dunglas committed
Apr 25, 2026 at 12:52 UTC
2a3ed96f8cbf0c72ce4621de84939e8828726d59
7 files changed
+183
-8
caddyconfig/httpcaddyfile/options.go
+2
@@ -484,6 +484,8 @@ func unmarshalCaddyfileMetricsOptions(d *caddyfile.Dispenser) (any, error) {
484
metrics.PerHost = true
485
case "observe_catchall_hosts":
486
metrics.ObserveCatchallHosts = true
487
+ case "otlp":
488
+ metrics.OTLP = true
489
default:
490
return nil, d.Errf("unrecognized servers option '%s'", d.Val())
491
}
caddytest/integration/caddyfile_adapt/metrics_otlp.caddyfiletest
new
+35
@@ -0,0 +1,35 @@
1
+{
2
+ metrics {
3
+ otlp
4
+ }
5
+}
6
+:80 {
7
+ respond "Hello"
8
+}
9
+----------
10
+{
11
+ "apps": {
12
+ "http": {
13
+ "servers": {
14
+ "srv0": {
15
+ "listen": [
16
+ ":80"
17
+ ],
18
+ "routes": [
19
+ {
20
+ "handle": [
21
+ {
22
+ "body": "Hello",
23
+ "handler": "static_response"
24
+ }
25
+ ]
26
+ }
27
+ ]
28
+ }
29
+ },
30
+ "metrics": {
31
+ "otlp": true
32
+ }
33
+ }
34
+ }
35
+}
go.mod
+2
-2
@@ -30,11 +30,13 @@ require (
30
github.com/tailscale/tscert v0.0.0-20251216020129-aea342f6d747
31
github.com/yuin/goldmark v1.8.2
32
github.com/yuin/goldmark-highlighting/v2 v2.0.0-20230729083705-37449abec8cc
33
+ go.opentelemetry.io/contrib/bridges/prometheus v0.68.0
34
go.opentelemetry.io/contrib/exporters/autoexport v0.65.0
35
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.65.0
36
go.opentelemetry.io/contrib/propagators/autoprop v0.65.0
37
go.opentelemetry.io/otel v1.43.0
38
go.opentelemetry.io/otel/sdk v1.43.0
39
+ go.opentelemetry.io/otel/sdk/metric v1.43.0
40
go.step.sm/crypto v0.77.1
41
go.uber.org/automaxprocs v1.6.0
42
go.uber.org/zap v1.27.1
@@ -87,7 +89,6 @@ require (
89
github.com/x448/float16 v0.8.4 // indirect
90
github.com/zeebo/blake3 v0.2.4 // indirect
91
go.opentelemetry.io/auto/sdk v1.2.1 // indirect
90
- go.opentelemetry.io/contrib/bridges/prometheus v0.68.0 // indirect
92
go.opentelemetry.io/contrib/propagators/aws v1.43.0 // indirect
93
go.opentelemetry.io/contrib/propagators/b3 v1.43.0 // indirect
94
go.opentelemetry.io/contrib/propagators/jaeger v1.43.0 // indirect
@@ -104,7 +105,6 @@ require (
105
go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.43.0 // indirect
106
go.opentelemetry.io/otel/log v0.19.0 // indirect
107
go.opentelemetry.io/otel/sdk/log v0.19.0 // indirect
107
- go.opentelemetry.io/otel/sdk/metric v1.43.0 // indirect
108
go.yaml.in/yaml/v2 v2.4.4 // indirect
109
go.yaml.in/yaml/v3 v3.0.4 // indirect
110
golang.org/x/exp v0.0.0-20251023183803-a4bb9ffd2546 // indirect
modules/caddyhttp/app.go
+9
@@ -208,6 +208,9 @@ func (app *App) Provision(ctx caddy.Context) error {
208
app.Metrics.httpMetrics = &httpMetrics{}
209
// Scan config for allowed hosts to prevent cardinality explosion
210
app.Metrics.scanConfigForHosts(app)
211
+ if err := app.Metrics.provisionOTLP(ctx); err != nil {
212
+ return err
213
+ }
214
}
215
// prepare each server
216
oldContext := ctx.Context
@@ -817,6 +820,12 @@ func (app *App) Stop() error {
820
}
821
}
822
823
+ // flush and shut down the OTLP metrics exporter (if configured) so any
824
+ // last data point reaches the collector before the process exits
825
+ if err := app.Metrics.shutdown(ctx); err != nil {
826
+ app.logger.Error("shutting down OTLP metrics", zap.Error(err))
827
+ }
828
+
829
app.stopped = true
830
return nil
831
}
modules/caddyhttp/metrics.go
+84
-4
@@ -3,6 +3,7 @@ package caddyhttp
3
import (
4
"context"
5
"errors"
6
+ "fmt"
7
"net/http"
8
"strings"
9
"sync"
@@ -10,9 +11,14 @@ import (
11
12
"github.com/prometheus/client_golang/prometheus"
13
"github.com/prometheus/client_golang/prometheus/promauto"
14
+ otelprom "go.opentelemetry.io/contrib/bridges/prometheus"
15
+ "go.opentelemetry.io/contrib/exporters/autoexport"
16
+ sdkmetric "go.opentelemetry.io/otel/sdk/metric"
17
+ "go.opentelemetry.io/otel/sdk/resource"
18
+ semconv "go.opentelemetry.io/otel/semconv/v1.26.0"
19
20
"github.com/caddyserver/caddy/v2"
15
- "github.com/caddyserver/caddy/v2/internal/metrics"
21
+ caddymetrics "github.com/caddyserver/caddy/v2/internal/metrics"
22
)
23
24
// Metrics configures metrics observations.
@@ -67,10 +73,20 @@ type Metrics struct {
73
// for production environments exposed to the internet).
74
ObserveCatchallHosts bool `json:"observe_catchall_hosts,omitempty"`
75
76
+ // Enable pushing metrics via OTLP in addition to the existing Prometheus
77
+ // scrape endpoints. When set, a PeriodicReader is attached to the shared
78
+ // Prometheus registry (via a Prometheus -> OpenTelemetry bridge), and the
79
+ // exporter is autoconfigured from the standard OTEL_* environment
80
+ // variables (OTEL_EXPORTER_OTLP_ENDPOINT, OTEL_EXPORTER_OTLP_PROTOCOL,
81
+ // OTEL_METRICS_EXPORTER, ...). Set OTEL_METRICS_EXPORTER=none or simply
82
+ // keep this field false to disable OTLP export.
83
+ OTLP bool `json:"otlp,omitempty"`
84
+
85
init sync.Once
86
httpMetrics *httpMetrics
87
allowedHosts map[string]struct{}
88
hasHTTPSServer bool
89
+ meterProvider *sdkmetric.MeterProvider
90
}
91
92
type httpMetrics struct {
@@ -147,6 +163,70 @@ func initHTTPMetrics(ctx caddy.Context, metrics *Metrics) {
163
}, httpLabels)
164
}
165
166
+// provisionOTLP wires a MeterProvider that periodically reads the process-wide
167
+// Prometheus registry and pushes the result via OTLP. The exporter and reader
168
+// are autoconfigured from the standard OTEL_* environment variables, matching
169
+// the ergonomics of the existing `tracing` directive. It is a no-op when
170
+// m.OTLP is false, and honors OTEL_METRICS_EXPORTER=none (autoexport
171
+// short-circuits to a no-op reader in that case).
172
+func (m *Metrics) provisionOTLP(ctx caddy.Context) error {
173
+ if !m.OTLP {
174
+ return nil
175
+ }
176
+
177
+ // Register a Prometheus -> OpenTelemetry bridge against the process-wide
178
+ // Prometheus registry as the *default* source the NewMetricReader below
179
+ // will read from.
180
+ //
181
+ // NB: despite the "With*" naming, autoexport.WithFallbackMetricProducer is
182
+ // a package-level setter (it returns nothing) — it mutates autoexport's
183
+ // internal producer registry and takes effect on the very next call to
184
+ // NewMetricReader. It is NOT a MetricOption and must not be passed as one.
185
+ // Users can still override the source by setting OTEL_METRICS_PRODUCERS.
186
+ reg := ctx.GetMetricsRegistry()
187
+ autoexport.WithFallbackMetricProducer(func(context.Context) (sdkmetric.Producer, error) {
188
+ return otelprom.NewMetricProducer(otelprom.WithGatherer(reg)), nil
189
+ })
190
+
191
+ reader, err := autoexport.NewMetricReader(ctx)
192
+ if err != nil {
193
+ return fmt.Errorf("creating OTLP metric reader: %w", err)
194
+ }
195
+
196
+ version, _ := caddy.Version()
197
+ res, err := resource.Merge(resource.Default(), resource.NewSchemaless(
198
+ semconv.WebEngineName(ServerHeader),
199
+ semconv.WebEngineVersion(version),
200
+ ))
201
+ if err != nil {
202
+ return fmt.Errorf("building OTLP metrics resource: %w", err)
203
+ }
204
+
205
+ m.meterProvider = sdkmetric.NewMeterProvider(
206
+ sdkmetric.WithResource(res),
207
+ sdkmetric.WithReader(reader),
208
+ )
209
+
210
+ return nil
211
+}
212
+
213
+// shutdown flushes and tears down the OTLP MeterProvider if one was provisioned.
214
+// Both ForceFlush and Shutdown are always attempted so that a flush failure
215
+// does not prevent the reader goroutines from being stopped; errors from both
216
+// are returned joined.
217
+func (m *Metrics) shutdown(ctx context.Context) error {
218
+ if m == nil || m.meterProvider == nil {
219
+ return nil
220
+ }
221
+
222
+ // ForceFlush gives the final collection a chance to reach the collector
223
+ // before the reader goroutine is stopped by Shutdown.
224
+ return errors.Join(
225
+ m.meterProvider.ForceFlush(ctx),
226
+ m.meterProvider.Shutdown(ctx),
227
+ )
228
+}
229
+
230
// scanConfigForHosts scans the HTTP app configuration to build a set of allowed hosts
231
// for metrics collection, similar to how auto-HTTPS scans for domain names.
232
func (m *Metrics) scanConfigForHosts(app *App) {
@@ -234,7 +314,7 @@ func newMetricsInstrumentedRoute(ctx caddy.Context, handler string, next Handler
314
func (h *metricsInstrumentedRoute) ServeHTTP(w http.ResponseWriter, r *http.Request) error {
315
server := serverNameFromContext(r.Context())
316
labels := prometheus.Labels{"server": server, "handler": h.handler}
237
- method := metrics.SanitizeMethod(r.Method)
317
+ method := caddymetrics.SanitizeMethod(r.Method)
318
// the "code" value is set later, but initialized here to eliminate the possibility
319
// of a panic
320
statusLabels := prometheus.Labels{"server": server, "handler": h.handler, "method": method, "code": ""}
@@ -264,7 +344,7 @@ func (h *metricsInstrumentedRoute) ServeHTTP(w http.ResponseWriter, r *http.Requ
344
// being called when the headers are written.
345
// Effectively the same behaviour as promhttp.InstrumentHandlerTimeToWriteHeader.
346
writeHeaderRecorder := ShouldBufferFunc(func(status int, header http.Header) bool {
267
- statusLabels["code"] = metrics.SanitizeCode(status)
347
+ statusLabels["code"] = caddymetrics.SanitizeCode(status)
348
ttfb := time.Since(start).Seconds()
349
h.metrics.httpMetrics.responseDuration.With(statusLabels).Observe(ttfb)
350
return false
@@ -280,7 +360,7 @@ func (h *metricsInstrumentedRoute) ServeHTTP(w http.ResponseWriter, r *http.Requ
360
if statusLabels["code"] == "" {
361
// we still sanitize it, even though it's likely to be 0. A 200 is
362
// returned on fallthrough so we want to reflect that.
283
- statusLabels["code"] = metrics.SanitizeCode(status)
363
+ statusLabels["code"] = caddymetrics.SanitizeCode(status)
364
}
365
366
h.metrics.httpMetrics.requestDuration.With(statusLabels).Observe(dur)
modules/caddyhttp/metrics_test.go
+50
@@ -523,6 +523,56 @@ func TestMetricsInstrumentedRoute(t *testing.T) {
523
}
524
}
525
526
+func TestMetricsProvisionOTLPDisabled(t *testing.T) {
527
+ ctx, _ := caddy.NewContext(caddy.Context{Context: context.Background()})
528
+
529
+ m := &Metrics{OTLP: false}
530
+
531
+ if err := m.provisionOTLP(ctx); err != nil {
532
+ t.Fatalf("provisionOTLP returned unexpected error: %v", err)
533
+ }
534
+ if m.meterProvider != nil {
535
+ t.Fatalf("meterProvider should remain nil when OTLP is disabled")
536
+ }
537
+
538
+ // shutdown must be safe on a never-provisioned Metrics.
539
+ if err := m.shutdown(context.Background()); err != nil {
540
+ t.Fatalf("shutdown returned unexpected error: %v", err)
541
+ }
542
+}
543
+
544
+func TestMetricsProvisionOTLPNoopExporter(t *testing.T) {
545
+ // OTEL_METRICS_EXPORTER=none makes autoexport return its built-in
546
+ // no-op reader, which avoids any network I/O while still exercising
547
+ // the full provisionOTLP -> shutdown lifecycle.
548
+ t.Setenv("OTEL_METRICS_EXPORTER", "none")
549
+
550
+ ctx, _ := caddy.NewContext(caddy.Context{Context: context.Background()})
551
+
552
+ m := &Metrics{OTLP: true}
553
+
554
+ if err := m.provisionOTLP(ctx); err != nil {
555
+ t.Fatalf("provisionOTLP returned unexpected error: %v", err)
556
+ }
557
+ if m.meterProvider == nil {
558
+ t.Fatalf("provisionOTLP did not create a MeterProvider")
559
+ }
560
+
561
+ if err := m.shutdown(context.Background()); err != nil {
562
+ t.Fatalf("shutdown returned unexpected error: %v", err)
563
+ }
564
+}
565
+
566
+// shutdown on a nil receiver is a convenience so App.Stop can call it
567
+// without guarding against app.Metrics being unset.
568
+func TestMetricsShutdownNilReceiver(t *testing.T) {
569
+ var m *Metrics
570
+
571
+ if err := m.shutdown(context.Background()); err != nil {
572
+ t.Fatalf("shutdown on nil Metrics returned unexpected error: %v", err)
573
+ }
574
+}
575
+
576
func BenchmarkMetricsInstrumentedRoute(b *testing.B) {
577
ctx, _ := caddy.NewContext(caddy.Context{Context: context.Background()})
578
m := &Metrics{
modules/caddyhttp/tracing/tracer.go
+1
-2
@@ -21,7 +21,6 @@ import (
21
)
22
23
const (
24
- webEngineName = "Caddy"
24
defaultSpanName = "handler"
25
nextCallCtxKey caddy.CtxKey = "nextCall"
26
)
@@ -58,7 +57,7 @@ func newOpenTelemetryWrapper(
57
}
58
59
version, _ := caddy.Version()
61
- res, err := ot.newResource(webEngineName, version)
60
+ res, err := ot.newResource(caddyhttp.ServerHeader, version)
61
if err != nil {
62
return ot, fmt.Errorf("creating resource error: %w", err)
63
}