[Metrics] Add --custom-histogram-buckets to override histogram bucket families (#48867)
The core engine histograms ship with default bucket boundaries tuned for typical serving workloads. The `--custom-histogram-buckets` option replaces the boundaries of one or more bucket families. Use it, for example, to track sub-300ms latency SLOs with the request-phase histograms, whose smallest default boundary is 0.3s. Signed-off-by: Guy Stone <guys@spotify.com> Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
Guy Stone committed
Sep 30, 2026 at 21:03 UTC
37d6174096fd99a8d362c9f5b379f7ec59deedb8
8 files changed
+278
-11
docs/usage/metrics.md
+42
@@ -75,6 +75,48 @@ These metrics are available via `--enable-mfu-metrics`:
75
76
--8<-- "gen:metrics-mfu"
77
78
+## Custom Histogram Buckets
79
+
80
+The core engine histograms ship with default bucket boundaries tuned for
81
+typical serving workloads. The `--custom-histogram-buckets` option replaces
82
+the boundaries of one or more *bucket families* — exactly the histograms
83
+listed in the table below — with your own list; histograms owned by other
84
+subsystems (for example, the NIXL connector metrics) are not affected. Use it,
85
+for example, to track sub-300ms latency SLOs with the request-phase
86
+histograms, whose smallest default boundary is 0.3s:
87
+
88
+```bash
89
+vllm serve Qwen/Qwen3-0.6B \
90
+ --custom-histogram-buckets '{"request_latency": [0.01, 0.05, 0.1, 0.25, 0.5, 1.0, 5.0, 30.0]}'
91
+```
92
+
93
+Each family key overrides a group of related histograms:
94
+
95
+| Family key | Histograms |
96
+| --- | --- |
97
+| `request_latency` | `vllm:e2e_request_latency_seconds`, `vllm:request_queue_time_seconds`, `vllm:request_inference_time_seconds`, `vllm:request_prefill_time_seconds`, `vllm:request_decode_time_seconds` |
98
+| `time_to_first_token` | `vllm:time_to_first_token_seconds` |
99
+| `inter_token_latency` | `vllm:inter_token_latency_seconds`, `vllm:request_time_per_output_token_seconds` |
100
+| `iteration_tokens` | `vllm:iteration_tokens_total` |
101
+| `request_params_n` | `vllm:request_params_n` |
102
+| `request_num_preemptions` | `vllm:request_num_preemptions` |
103
+| `request_tokens` | `vllm:request_prompt_tokens`, `vllm:request_generation_tokens`, `vllm:request_max_num_generation_tokens`, `vllm:request_params_max_tokens`, `vllm:request_prefill_kv_computed_tokens` |
104
+| `kv_cache_residency` | `vllm:kv_block_lifetime_seconds`, `vllm:kv_block_idle_before_evict_seconds`, `vllm:kv_block_reuse_gap_seconds` |
105
+
106
+Bucket values must be positive, finite, and strictly increasing; unknown
107
+family keys are rejected at startup. Families you do not list keep their
108
+default boundaries. The `request_tokens` defaults normally scale with
109
+`--max-model-len`; an override replaces that computed list. The
110
+`kv_cache_residency` family only takes effect when `--kv-cache-metrics` is
111
+enabled.
112
+
113
+!!! warning "Bucket cardinality"
114
+ Every bucket boundary creates one extra time series per metric and per
115
+ label combination (model and engine index, multiplied under data-parallel
116
+ deployments). Long bucket lists inflate Prometheus storage, scrape sizes,
117
+ and query costs. Keep custom lists short, and only override the families
118
+ you actively monitor.
119
+
120
## Deprecation Policy
121
122
Note: when metrics are deprecated in version `X.Y`, they are hidden in version `X.Y+1`
tests/config/test_observability.py
new
+53
@@ -0,0 +1,53 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+"""Tests for ObservabilityConfig validation."""
4
+
5
+import pytest
6
+from pydantic import ValidationError
7
+
8
+from vllm.config.observability import ObservabilityConfig
9
+
10
+
11
+def test_custom_histogram_buckets_default_none():
12
+ assert ObservabilityConfig().custom_histogram_buckets is None
13
+
14
+
15
+def test_custom_histogram_buckets_valid():
16
+ config = ObservabilityConfig(
17
+ custom_histogram_buckets={
18
+ "request_latency": [0.01, 0.05, 0.1, 0.5],
19
+ "time_to_first_token": [1, 2, 5],
20
+ }
21
+ )
22
+ assert config.custom_histogram_buckets == {
23
+ "request_latency": [0.01, 0.05, 0.1, 0.5],
24
+ "time_to_first_token": [1.0, 2.0, 5.0],
25
+ }
26
+
27
+
28
+@pytest.mark.parametrize(
29
+ ("buckets", "match"),
30
+ [
31
+ ({"bogus": [1.0, 2.0]}, "unknown bucket family 'bogus'"),
32
+ ({"request_latency": []}, "must not be empty"),
33
+ ({"request_latency": [0.0, 1.0]}, "must be finite and greater than 0"),
34
+ ({"request_latency": [-1.0, 1.0]}, "must be finite and greater than 0"),
35
+ (
36
+ {"request_latency": [1.0, float("inf")]},
37
+ "must be finite and greater than 0",
38
+ ),
39
+ ({"request_latency": [float("nan")]}, "must be finite and greater than 0"),
40
+ ({"request_latency": [1.0, 1.0]}, "must be strictly increasing"),
41
+ ({"request_latency": [2.0, 1.0]}, "must be strictly increasing"),
42
+ ({"request_latency": [True, 2.0]}, "must be a number, not a boolean"),
43
+ ({"request_latency": [False]}, "must be a number, not a boolean"),
44
+ (
45
+ {"request_latency": [1.0, 2.0], "request_tokens": [16, 8]},
46
+ r"request_tokens.*must be strictly increasing",
47
+ ),
48
+ ],
49
+)
50
+def test_custom_histogram_buckets_rejects_invalid(buckets, match):
51
+ """Each validation rule must name the field, the value, and the rule."""
52
+ with pytest.raises(ValidationError, match=match):
53
+ ObservabilityConfig(custom_histogram_buckets=buckets)
tests/v1/engine/test_engine_args.py
+31
@@ -108,6 +108,37 @@ def test_mm_prefix_lm_raises_batched_tokens_floor():
108
assert vllm_config.scheduler_config.max_num_batched_tokens >= 2496
109
110
111
+def test_custom_histogram_buckets_from_cli():
112
+ parser = EngineArgs.add_cli_args(FlexibleArgumentParser())
113
+
114
+ # Unset: byte-identical default behavior.
115
+ args = parser.parse_args([])
116
+ vllm_config = EngineArgs.from_cli_args(args=args).create_engine_config()
117
+ assert vllm_config.observability_config.custom_histogram_buckets is None
118
+
119
+ # A JSON mapping round-trips into the config verbatim.
120
+ args = parser.parse_args(
121
+ [
122
+ "--custom-histogram-buckets",
123
+ '{"request_latency": [0.01, 0.05, 0.1, 0.5]}',
124
+ ]
125
+ )
126
+ vllm_config = EngineArgs.from_cli_args(args=args).create_engine_config()
127
+ assert vllm_config.observability_config.custom_histogram_buckets == {
128
+ "request_latency": [0.01, 0.05, 0.1, 0.5]
129
+ }
130
+
131
+ # Malformed JSON is rejected at argument-parsing time.
132
+ parser.exit_on_error = False
133
+ with pytest.raises(ArgumentError):
134
+ parser.parse_args(["--custom-histogram-buckets", "{not json"])
135
+
136
+ # An unknown family key parses as JSON but fails config validation.
137
+ args = parser.parse_args(["--custom-histogram-buckets", '{"bogus": [1.0, 2.0]}'])
138
+ with pytest.raises(ValueError, match="unknown bucket family"):
139
+ EngineArgs.from_cli_args(args=args).create_engine_config()
140
+
141
+
142
def test_data_parallel_start_rank_zero_infers_hybrid_lb():
143
"""An explicit --data-parallel-start-rank 0 must be treated the same as
144
any other explicit start rank when inferring hybrid LB mode, not as
tests/v1/metrics/test_histogram_buckets.py
+46
@@ -271,3 +271,49 @@ def test_prometheus_logger_default_buckets():
271
assert found[metric_name] == [float(b) for b in expected], metric_name
272
finally:
273
unregister_vllm_metrics()
274
+
275
+
276
+def test_histogram_buckets_override_precedence():
277
+ """A present family key wins verbatim; absent keys keep defaults."""
278
+ overrides = {
279
+ "request_latency": [0.5, 1.0],
280
+ "request_tokens": [16.0, 4096.0],
281
+ }
282
+ got = histogram_buckets("request_latency", overrides=overrides)
283
+ assert got == [0.5, 1.0]
284
+ assert got is not overrides["request_latency"]
285
+ # An override replaces the max_model_len-derived series entirely.
286
+ assert histogram_buckets("request_tokens", overrides=overrides) == [
287
+ 16.0,
288
+ 4096.0,
289
+ ]
290
+ assert (
291
+ histogram_buckets("time_to_first_token", overrides=overrides)
292
+ == DEFAULT_BUCKET_SNAPSHOTS["time_to_first_token"]
293
+ )
294
+
295
+
296
+def test_prometheus_logger_applies_overrides():
297
+ """Every family must honour an override, with a distinct list each.
298
+
299
+ Overriding all of them at once is what catches a histogram wired to the
300
+ defaults instead of the override, and a family wired to the wrong list.
301
+ """
302
+ overridden: dict[str, list[float]] = {
303
+ family: [index + 1.0, index + 2.0]
304
+ for index, family in enumerate(sorted(BUCKET_FAMILY_KEYS))
305
+ }
306
+ config = build_logger_config(
307
+ ObservabilityConfig(
308
+ kv_cache_metrics=True,
309
+ custom_histogram_buckets=overridden,
310
+ )
311
+ )
312
+ try:
313
+ PrometheusStatLogger(config)
314
+ found = collect_histogram_buckets()
315
+ assert set(found) == set(METRIC_FAMILIES)
316
+ for metric_name, family in METRIC_FAMILIES.items():
317
+ assert found[metric_name] == overridden[family], metric_name
318
+ finally:
319
+ unregister_vllm_metrics()
vllm/config/observability.py
+63
@@ -1,7 +1,9 @@
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
4
+import math
5
from functools import cached_property
6
+from itertools import pairwise
7
from typing import Any, Literal
8
9
from packaging.version import parse
@@ -65,6 +67,18 @@ class ObservabilityConfig:
67
kv_cache_metrics_sample: float = Field(default=0.01, gt=0, le=1)
68
"""Sampling rate for KV cache metrics (0.0, 1.0]. Default 0.01 = 1% of blocks."""
69
70
+ custom_histogram_buckets: dict[str, list[float]] | None = None
71
+ """Custom Prometheus histogram bucket boundaries, as a JSON mapping from
72
+ bucket-family key to a list of strictly increasing, positive, finite
73
+ upper bounds. When a family key is present, its list replaces the default
74
+ buckets for every histogram in that family; families not listed keep
75
+ their defaults. Known families: `request_latency`, `time_to_first_token`,
76
+ `inter_token_latency`, `iteration_tokens`, `request_params_n`,
77
+ `request_num_preemptions`, `request_tokens`, `kv_cache_residency`. Example:
78
+ `--custom-histogram-buckets '{"request_latency": [0.01, 0.05, 0.1, 0.5]}'`.
79
+ Note that every extra bucket adds one time series per metric and label
80
+ set."""
81
+
82
cudagraph_metrics: bool = False
83
"""Enable CUDA graph metrics (number of padded/unpadded tokens, runtime cudagraph
84
dispatch modes, and their observed frequencies at every logging interval)."""
@@ -150,6 +164,55 @@ class ObservabilityConfig:
164
)
165
return value
166
167
+ @field_validator("custom_histogram_buckets", mode="before")
168
+ @classmethod
169
+ def _reject_bool_histogram_bounds(cls, value: object) -> object:
170
+ """Reject booleans before pydantic silently coerces them to floats."""
171
+ if isinstance(value, dict):
172
+ for family, buckets in value.items():
173
+ if not isinstance(buckets, list):
174
+ continue
175
+ for bound in buckets:
176
+ if isinstance(bound, bool):
177
+ raise ValueError(
178
+ f"custom_histogram_buckets[{family!r}]: bound "
179
+ f"{bound!r} must be a number, not a boolean"
180
+ )
181
+ return value
182
+
183
+ @field_validator("custom_histogram_buckets")
184
+ @classmethod
185
+ def _validate_custom_histogram_buckets(
186
+ cls, value: dict[str, list[float]] | None
187
+ ) -> dict[str, list[float]] | None:
188
+ if value is None:
189
+ return value
190
+ from vllm.v1.metrics.buckets import BUCKET_FAMILY_KEYS
191
+
192
+ for family, buckets in value.items():
193
+ if family not in BUCKET_FAMILY_KEYS:
194
+ raise ValueError(
195
+ f"custom_histogram_buckets: unknown bucket family "
196
+ f"{family!r}; known families: {sorted(BUCKET_FAMILY_KEYS)}"
197
+ )
198
+ if not buckets:
199
+ raise ValueError(
200
+ f"custom_histogram_buckets[{family!r}]: bucket list "
201
+ "must not be empty"
202
+ )
203
+ for bound in buckets:
204
+ if not math.isfinite(bound) or bound <= 0:
205
+ raise ValueError(
206
+ f"custom_histogram_buckets[{family!r}]: bound "
207
+ f"{bound!r} must be finite and greater than 0"
208
+ )
209
+ if any(a >= b for a, b in pairwise(buckets)):
210
+ raise ValueError(
211
+ f"custom_histogram_buckets[{family!r}]: bounds {buckets} "
212
+ "must be strictly increasing"
213
+ )
214
+ return value
215
+
216
@model_validator(mode="after")
217
def _validate_tracing_config(self):
218
if self.collect_detailed_traces and not self.otlp_traces_endpoint:
vllm/engine/arg_utils.py
+8
@@ -723,6 +723,9 @@ class EngineArgs:
723
kv_cache_metrics_sample: float = get_field(
724
ObservabilityConfig, "kv_cache_metrics_sample"
725
)
726
+ custom_histogram_buckets: dict[str, list[float]] | None = (
727
+ ObservabilityConfig.custom_histogram_buckets
728
+ )
729
cudagraph_metrics: bool = ObservabilityConfig.cudagraph_metrics
730
enable_layerwise_nvtx_tracing: bool = (
731
ObservabilityConfig.enable_layerwise_nvtx_tracing
@@ -1633,6 +1636,10 @@ class EngineArgs:
1636
"--kv-cache-metrics-sample",
1637
**observability_kwargs["kv_cache_metrics_sample"],
1638
)
1639
+ observability_group.add_argument(
1640
+ "--custom-histogram-buckets",
1641
+ **observability_kwargs["custom_histogram_buckets"],
1642
+ )
1643
observability_group.add_argument(
1644
"--cudagraph-metrics",
1645
**observability_kwargs["cudagraph_metrics"],
@@ -2126,6 +2133,7 @@ class EngineArgs:
2133
per_request_spec_decode_metrics=self.per_request_spec_decode_metrics,
2134
kv_cache_metrics=self.kv_cache_metrics,
2135
kv_cache_metrics_sample=self.kv_cache_metrics_sample,
2136
+ custom_histogram_buckets=self.custom_histogram_buckets,
2137
cudagraph_metrics=self.cudagraph_metrics,
2138
enable_layerwise_nvtx_tracing=self.enable_layerwise_nvtx_tracing,
2139
enable_mfu_metrics=self.enable_mfu_metrics,
vllm/v1/metrics/buckets.py
+12
-3
@@ -9,6 +9,7 @@ subsystems, such as the KV connector and offloading metrics, keep their own
9
boundaries.
10
"""
11
12
+from collections.abc import Mapping
13
from typing import Literal, get_args
14
15
BucketFamilyKey = Literal[
@@ -205,23 +206,31 @@ def build_1_2_5_buckets(max_value: int) -> list[float]:
206
def histogram_buckets(
207
family: BucketFamilyKey,
208
max_model_len: int | None = None,
209
+ overrides: Mapping[str, list[float]] | None = None,
210
) -> list[float]:
209
- """Return the default bucket boundaries for a histogram family.
211
+ """Return the bucket boundaries for a histogram family.
212
213
Args:
214
family: Canonical bucket family key.
215
max_model_len: Cap for the token-count series of the
216
`request_tokens` family; required for that family and ignored
217
otherwise.
218
+ overrides: Mapping from family key to replacement bucket bounds.
219
+ A present key replaces the family's defaults verbatim. Values
220
+ must already be validated; the single validation point is
221
+ `ObservabilityConfig.custom_histogram_buckets`.
222
223
Returns:
224
A fresh list of bucket upper bounds; callers may mutate it freely.
225
226
Raises:
221
- ValueError: If `family` is `request_tokens` and `max_model_len`
222
- is None.
227
+ ValueError: If `family` is `request_tokens`, no override is given,
228
+ and `max_model_len` is None.
229
+
230
231
"""
232
+ if overrides is not None and (custom := overrides.get(family)) is not None:
233
+ return list(custom)
234
if family == "request_tokens":
235
if max_model_len is None:
236
raise ValueError(
vllm/v1/metrics/loggers.py
+23
-8
@@ -739,15 +739,28 @@ class PrometheusStatLogger(AggregateStatLoggerBase):
739
for idx in engine_indexes
740
}
741
742
+ custom_buckets = vllm_config.observability_config.custom_histogram_buckets
743
request_tokens_buckets = histogram_buckets(
743
- "request_tokens", max_model_len=max_model_len
744
+ "request_tokens", max_model_len=max_model_len, overrides=custom_buckets
745
+ )
746
+ iteration_tokens_buckets = histogram_buckets(
747
+ "iteration_tokens", overrides=custom_buckets
748
+ )
749
+ request_params_n_buckets = histogram_buckets(
750
+ "request_params_n", overrides=custom_buckets
751
+ )
752
+ time_to_first_token_buckets = histogram_buckets(
753
+ "time_to_first_token", overrides=custom_buckets
754
+ )
755
+ inter_token_latency_buckets = histogram_buckets(
756
+ "inter_token_latency", overrides=custom_buckets
757
+ )
758
+ request_latency_buckets = histogram_buckets(
759
+ "request_latency", overrides=custom_buckets
760
+ )
761
+ request_num_preemptions_buckets = histogram_buckets(
762
+ "request_num_preemptions", overrides=custom_buckets
763
)
745
- iteration_tokens_buckets = histogram_buckets("iteration_tokens")
746
- request_params_n_buckets = histogram_buckets("request_params_n")
747
- time_to_first_token_buckets = histogram_buckets("time_to_first_token")
748
- inter_token_latency_buckets = histogram_buckets("inter_token_latency")
749
- request_latency_buckets = histogram_buckets("request_latency")
750
- request_num_preemptions_buckets = histogram_buckets("request_num_preemptions")
764
765
#
766
# Histograms of counts
@@ -925,7 +938,9 @@ class PrometheusStatLogger(AggregateStatLoggerBase):
938
# KV Cache residency metrics
939
#
940
if self.kv_cache_metrics_enabled:
928
- kv_cache_residency_buckets = histogram_buckets("kv_cache_residency")
941
+ kv_cache_residency_buckets = histogram_buckets(
942
+ "kv_cache_residency", overrides=custom_buckets
943
+ )
944
945
histogram_kv_block_lifetime = self._histogram_cls(
946
name="vllm:kv_block_lifetime_seconds",