fix: drop high-cardinality server.address from http_server metrics (#11208)
otelhttp derives server.address from the Host header, which creates a unique time series for every subdomain hostname on public gateways (e.g. each CID.ipfs.dweb.link). this caused multi-gigabyte prometheus responses and scrape timeouts. - cmd/ipfs/kubo/daemon.go: add OTel SDK View that drops server.address from all http.server.* metrics at the MeterProvider level - core/corehttp/gateway.go: add server.domain attribute to Gateway and HostnameGateway handlers, grouping by Gateway.PublicGateways suffix (e.g. "dweb.link"), with "localhost", "loopback", or "other" fallbacks - core/corehttp/commands.go: add server.domain="api" to RPC handler - core/corehttp/gateway.go: add server.domain="libp2p" to libp2p handler - docs/changelogs/v0.40.md: add changelog highlight - docs/metrics.md: document server_domain label and server_address drop
Marcin Rataj committed
Feb 25, 2026 at 00:32 UTC
6a082f60b1b2043765b33c214fa0a046c9aa50d6
5 files changed
+142
-4
cmd/ipfs/kubo/daemon.go
+15
@@ -44,6 +44,7 @@ import (
44
prometheus "github.com/prometheus/client_golang/prometheus"
45
promauto "github.com/prometheus/client_golang/prometheus/promauto"
46
"go.opentelemetry.io/otel"
47
+ "go.opentelemetry.io/otel/attribute"
48
promexporter "go.opentelemetry.io/otel/exporters/prometheus"
49
sdkmetric "go.opentelemetry.io/otel/sdk/metric"
50
)
@@ -224,6 +225,20 @@ func daemonFunc(req *cmds.Request, re cmds.ResponseEmitter, env cmds.Environment
225
log.Errorf("Creating prometheus exporter for OpenTelemetry failed: %s (some metrics will be missing from /debug/metrics/prometheus)\n", err.Error())
226
} else {
227
meterProvider := sdkmetric.NewMeterProvider(
228
+ // Drop high-cardinality server.address attribute from http.server.*
229
+ // metrics. otelhttp derives it from the Host header, which causes
230
+ // cardinality explosion on subdomain gateways where each
231
+ // CID.ipfs.example.com hostname is a unique label value.
232
+ // Per-domain visibility is provided by the lower-cardinality
233
+ // server.domain attribute added in core/corehttp/gateway.go.
234
+ sdkmetric.WithView(sdkmetric.NewView(
235
+ sdkmetric.Instrument{Name: "http.server.*"},
236
+ sdkmetric.Stream{
237
+ AttributeFilter: attribute.NewDenyKeysFilter(
238
+ attribute.Key("server.address"),
239
+ ),
240
+ },
241
+ )),
242
sdkmetric.WithReader(exporter),
243
)
244
otel.SetMeterProvider(meterProvider)
core/corehttp/commands.go
+3
-1
@@ -146,7 +146,9 @@ func commandsOption(cctx oldcmds.Context, command *cmds.Command) ServeOption {
146
cmdHandler = withAuthSecrets(authorizations, cmdHandler)
147
}
148
149
- cmdHandler = otelhttp.NewHandler(cmdHandler, "corehttp.cmdsHandler")
149
+ cmdHandler = otelhttp.NewHandler(cmdHandler, "corehttp.cmdsHandler",
150
+ otelhttp.WithMetricAttributesFn(staticServerDomainAttrFn("api")),
151
+ )
152
mux.Handle(APIPath+"/", cmdHandler)
153
return mux, nil
154
}
core/corehttp/gateway.go
+105
-3
@@ -8,6 +8,8 @@ import (
8
"maps"
9
"net"
10
"net/http"
11
+ "slices"
12
+ "strings"
13
"time"
14
15
"github.com/ipfs/boxo/blockservice"
@@ -25,6 +27,7 @@ import (
27
"github.com/ipfs/kubo/core/node"
28
"github.com/libp2p/go-libp2p/core/routing"
29
"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp"
30
+ "go.opentelemetry.io/otel/attribute"
31
)
32
33
func GatewayOption(paths ...string) ServeOption {
@@ -41,7 +44,11 @@ func GatewayOption(paths ...string) ServeOption {
44
45
handler := gateway.NewHandler(config, backend)
46
handler = gateway.NewHeaders(headers).ApplyCors().Wrap(handler)
44
- handler = otelhttp.NewHandler(handler, "Gateway")
47
+ var otelOpts []otelhttp.Option
48
+ if fn := newServerDomainAttrFn(n); fn != nil {
49
+ otelOpts = append(otelOpts, otelhttp.WithMetricAttributesFn(fn))
50
+ }
51
+ handler = otelhttp.NewHandler(handler, "Gateway", otelOpts...)
52
53
for _, p := range paths {
54
mux.Handle(p+"/", handler)
@@ -68,7 +75,11 @@ func HostnameOption() ServeOption {
75
var handler http.Handler
76
handler = gateway.NewHostnameHandler(config, backend, childMux)
77
handler = gateway.NewHeaders(headers).ApplyCors().Wrap(handler)
71
- handler = otelhttp.NewHandler(handler, "HostnameGateway")
78
+ var otelOpts []otelhttp.Option
79
+ if fn := newServerDomainAttrFn(n); fn != nil {
80
+ otelOpts = append(otelOpts, otelhttp.WithMetricAttributesFn(fn))
81
+ }
82
+ handler = otelhttp.NewHandler(handler, "HostnameGateway", otelOpts...)
83
84
mux.Handle("/", handler)
85
return childMux, nil
@@ -120,7 +131,9 @@ func Libp2pGatewayOption() ServeOption {
131
}
132
133
handler := gateway.NewHandler(gwConfig, &offlineGatewayErrWrapper{gwimpl: backend})
123
- handler = otelhttp.NewHandler(handler, "Libp2p-Gateway")
134
+ handler = otelhttp.NewHandler(handler, "Libp2p-Gateway",
135
+ otelhttp.WithMetricAttributesFn(staticServerDomainAttrFn("libp2p")),
136
+ )
137
138
mux.Handle("/ipfs/", handler)
139
@@ -252,6 +265,95 @@ var _ gateway.IPFSBackend = (*offlineGatewayErrWrapper)(nil)
265
266
var defaultPaths = []string{"/ipfs/", "/ipns/", "/p2p/"}
267
268
+// serverDomainAttrKey is the OTel attribute key for the logical server domain.
269
+// It replaces the high-cardinality server.address attribute (dropped by the
270
+// View in cmd/ipfs/kubo/daemon.go) with a bounded set of values: configured
271
+// Gateway.PublicGateways suffixes, "localhost", "loopback", "api", "libp2p",
272
+// or "other".
273
+var serverDomainAttrKey = attribute.Key("server.domain")
274
+
275
+// staticServerDomainAttrFn returns a MetricAttributesFn that always returns
276
+// a fixed server.domain value. Use for handlers where the domain is known
277
+// statically (e.g. "api", "libp2p") to keep the label set consistent across
278
+// all http_server_* metrics.
279
+func staticServerDomainAttrFn(domain string) func(*http.Request) []attribute.KeyValue {
280
+ attrs := []attribute.KeyValue{serverDomainAttrKey.String(domain)}
281
+ return func(*http.Request) []attribute.KeyValue { return attrs }
282
+}
283
+
284
+// newServerDomainAttrFn returns an otelhttp.WithMetricAttributesFn callback
285
+// that adds a server.domain attribute grouping requests by their matching
286
+// Gateway.PublicGateways hostname suffix (e.g. "dweb.link", "ipfs.io").
287
+// Requests that don't match any configured gateway get "other".
288
+//
289
+// All return values are pre-allocated at setup time so the per-request
290
+// closure is zero-allocation.
291
+func newServerDomainAttrFn(n *core.IpfsNode) func(*http.Request) []attribute.KeyValue {
292
+ cfg, err := n.Repo.Config()
293
+ if err != nil {
294
+ return nil
295
+ }
296
+
297
+ // Collect non-nil gateway domain suffixes, sorted longest-first
298
+ // so more-specific suffixes match before shorter ones.
299
+ // Strip ports from keys to match boxo's fallback behavior
300
+ // (boxo tries exact match with port, then strips port and retries).
301
+ seen := make(map[string]struct{}, len(cfg.Gateway.PublicGateways))
302
+ suffixes := make([]string, 0, len(cfg.Gateway.PublicGateways))
303
+ for hostname, gw := range cfg.Gateway.PublicGateways {
304
+ if gw == nil {
305
+ continue
306
+ }
307
+ if h, _, err := net.SplitHostPort(hostname); err == nil {
308
+ hostname = h
309
+ }
310
+ if _, ok := seen[hostname]; ok {
311
+ continue
312
+ }
313
+ seen[hostname] = struct{}{}
314
+ suffixes = append(suffixes, hostname)
315
+ }
316
+ slices.SortFunc(suffixes, func(a, b string) int {
317
+ return len(b) - len(a)
318
+ })
319
+
320
+ // Pre-allocate attribute slices so the per-request closure only returns
321
+ // existing slices and does not allocate.
322
+ suffixAttrs := make([][]attribute.KeyValue, len(suffixes))
323
+ for i, s := range suffixes {
324
+ suffixAttrs[i] = []attribute.KeyValue{serverDomainAttrKey.String(s)}
325
+ }
326
+ localhostAttr := []attribute.KeyValue{serverDomainAttrKey.String("localhost")}
327
+ loopbackAttr := []attribute.KeyValue{serverDomainAttrKey.String("loopback")}
328
+ otherAttr := []attribute.KeyValue{serverDomainAttrKey.String("other")}
329
+
330
+ return func(r *http.Request) []attribute.KeyValue {
331
+ host := r.Host
332
+ if h, _, err := net.SplitHostPort(host); err == nil {
333
+ host = h
334
+ }
335
+
336
+ // Check localhost/loopback before iterating suffixes.
337
+ // "localhost" is an implicit default gateway (defaultKnownGateways)
338
+ // not present in cfg.Gateway.PublicGateways, so it won't appear
339
+ // in suffixes.
340
+ if host == "localhost" || strings.HasSuffix(host, ".localhost") {
341
+ return localhostAttr
342
+ }
343
+ if strings.HasPrefix(host, "127.") || host == "::1" {
344
+ return loopbackAttr
345
+ }
346
+
347
+ for i, suffix := range suffixes {
348
+ if strings.HasSuffix(host, suffix) {
349
+ return suffixAttrs[i]
350
+ }
351
+ }
352
+
353
+ return otherAttr
354
+ }
355
+}
356
+
357
var subdomainGatewaySpec = &gateway.PublicGateway{
358
Paths: defaultPaths,
359
UseSubdomains: true,
docs/changelogs/v0.40.md
+12
@@ -30,6 +30,7 @@ This release was brought to you by the [Shipyard](https://ipshipyard.com/) team.
30
- [🔖 New `ipfs name get|put` commands](#-new-ipfs-name-getput-commands)
31
- [📋 Long listing format for `ipfs ls`](#-long-listing-format-for-ipfs-ls)
32
- [🖥️ WebUI Improvements](#-webui-improvements)
33
+ - [📉 Fixed Prometheus metrics bloat on popular subdomain gateways](#-fixed-prometheus-metrics-bloat-on-popular-subdomain-gateways)
34
- [📢 libp2p announces all interface addresses](#-libp2p-announces-all-interface-addresses)
35
- [🗑️ Badger v1 datastore slated for removal this year](#-badger-v1-datastore-slated-for-removal-this-year)
36
- [🐹 Go 1.26](#-go-126)
@@ -300,6 +301,17 @@ The Inspect button now resolves `/ipfs/` and `/ipns/` paths to their final CID b
301
302
> 
303
304
+#### 📉 Fixed Prometheus metrics bloat on popular subdomain gateways
305
+
306
+Most Kubo users are unaffected by this change. It matters if you run Kubo as a public subdomain gateway (with [`Gateway.PublicGateways`](https://github.com/ipfs/kubo/blob/master/docs/config.md#gatewaypublicgateways) and `UseSubdomains: true`), where the `otelhttp` instrumentation was including the raw `Host` header as the `server_address` metric label. Every unique hostname (e.g., each `CID.ipfs.dweb.link`) created a separate time series, resulting in millions of metric lines, multi-gigabyte `/debug/metrics/prometheus` responses, and Prometheus scrape timeouts.
307
+
308
+**What changed:**
309
+
310
+- The unbounded `server_address` label is now dropped from all `http_server_*` metrics via an OTel SDK View.
311
+- All handlers add a `server_domain` label instead. Gateway handlers group by matching `Gateway.PublicGateways` suffix (e.g., `dweb.link`, `ipfs.io`), with `localhost`, `loopback`, or `other` for unmatched hosts. The RPC API and Libp2p Gateway handlers use fixed values (`api`, `libp2p`).
312
+
313
+If you use [Rainbow](https://github.com/ipfs/rainbow) for your public gateway (recommended), this issue never applied to you -- Rainbow uses its own low-cardinality HTTP metrics.
314
+
315
#### 📢 libp2p announces all interface addresses
316
317
go-libp2p [v0.47.0](https://github.com/libp2p/go-libp2p/releases/tag/v0.47.0) includes a rewritten routing library ([`go-netroute`](https://github.com/libp2p/go-netroute/pull/64)) that fixes interop with VPN and WireGuard/Tailscale setups. A side effect: when listening on `0.0.0.0`, libp2p now returns addresses from all network interfaces instead of just the primary one ([go-libp2p#3460](https://github.com/libp2p/go-libp2p/issues/3460)).
docs/metrics.md
+7
@@ -110,6 +110,13 @@ Additional HTTP instrumentation for all handlers (Gateway, API commands, etc.):
110
111
These metrics are automatically added to Gateway handlers, Hostname Gateway, Libp2p Gateway, and API command handlers.
112
113
+> [!NOTE]
114
+> The `server_address` label from `otelhttp` is dropped via an OTel SDK View to prevent cardinality explosion on subdomain gateways (where each unique `Host` header creates a new time series). All handlers include a `server_domain` label instead:
115
+>
116
+> - Gateway and Hostname Gateway handlers group requests by their matching [`Gateway.PublicGateways`](config.md#gatewaypublicgateways) domain suffix (e.g., `dweb.link`, `ipfs.io`). Unmatched hosts are labeled `localhost`, `loopback`, or `other`.
117
+> - The RPC API handler uses `api`.
118
+> - The Libp2p Gateway handler uses `libp2p`.
119
+
120
## OpenTelemetry Metadata
121
122
Kubo uses Prometheus for metrics collection for historical reasons, but OpenTelemetry metrics are automatically exposed through the same Prometheus endpoint. These metadata metrics provide context about the instrumentation: