[Metrics] Consolidate Prometheus histogram bucket defaults into a single module (#48866)
Signed-off-by: Guy Stone <guys@spotify.com> Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
Guy Stone committed
Sep 11, 2026 at 15:12 UTC
d5a9d0f59bbbbb55ac347e6126d01b1456dbd94f
3 files changed
+524
-147
tests/v1/metrics/test_histogram_buckets.py
new
+273
@@ -0,0 +1,273 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+"""Snapshot tests locking the default Prometheus histogram buckets.
4
+
5
+The expected lists are deliberately hard-coded literals rather than imports:
6
+any change to a default bucket boundary must show up as a failing test here.
7
+"""
8
+
9
+import prometheus_client
10
+import pytest
11
+
12
+from vllm.config import ModelConfig, VllmConfig
13
+from vllm.config.observability import ObservabilityConfig
14
+from vllm.v1.metrics.buckets import (
15
+ BUCKET_FAMILY_KEYS,
16
+ build_1_2_5_buckets,
17
+ histogram_buckets,
18
+)
19
+from vllm.v1.metrics.loggers import PrometheusStatLogger
20
+from vllm.v1.metrics.prometheus import unregister_vllm_metrics
21
+
22
+pytestmark = pytest.mark.cpu_test
23
+
24
+DEFAULT_BUCKET_SNAPSHOTS: dict[str, list[float]] = {
25
+ "request_latency": [
26
+ 0.3,
27
+ 0.5,
28
+ 0.8,
29
+ 1.0,
30
+ 1.5,
31
+ 2.0,
32
+ 2.5,
33
+ 5.0,
34
+ 10.0,
35
+ 15.0,
36
+ 20.0,
37
+ 30.0,
38
+ 40.0,
39
+ 50.0,
40
+ 60.0,
41
+ 120.0,
42
+ 240.0,
43
+ 480.0,
44
+ 960.0,
45
+ 1920.0,
46
+ 7680.0,
47
+ ],
48
+ "time_to_first_token": [
49
+ 0.001,
50
+ 0.005,
51
+ 0.01,
52
+ 0.02,
53
+ 0.04,
54
+ 0.06,
55
+ 0.08,
56
+ 0.1,
57
+ 0.25,
58
+ 0.5,
59
+ 0.75,
60
+ 1.0,
61
+ 2.5,
62
+ 5.0,
63
+ 7.5,
64
+ 10.0,
65
+ 20.0,
66
+ 40.0,
67
+ 80.0,
68
+ 160.0,
69
+ 640.0,
70
+ 2560.0,
71
+ ],
72
+ "inter_token_latency": [
73
+ 0.01,
74
+ 0.025,
75
+ 0.05,
76
+ 0.075,
77
+ 0.1,
78
+ 0.15,
79
+ 0.2,
80
+ 0.3,
81
+ 0.4,
82
+ 0.5,
83
+ 0.75,
84
+ 1.0,
85
+ 2.5,
86
+ 5.0,
87
+ 7.5,
88
+ 10.0,
89
+ 20.0,
90
+ 40.0,
91
+ 80.0,
92
+ ],
93
+ "iteration_tokens": [
94
+ 1,
95
+ 8,
96
+ 16,
97
+ 32,
98
+ 64,
99
+ 128,
100
+ 256,
101
+ 512,
102
+ 1024,
103
+ 2048,
104
+ 4096,
105
+ 8192,
106
+ 16384,
107
+ ],
108
+ "request_params_n": [1, 2, 5, 10, 20],
109
+ "request_num_preemptions": [1, 2, 3, 4, 5, 10, 20],
110
+ "kv_cache_residency": [
111
+ 0.001,
112
+ 0.002,
113
+ 0.005,
114
+ 0.01,
115
+ 0.02,
116
+ 0.05,
117
+ 0.1,
118
+ 0.2,
119
+ 0.5,
120
+ 1,
121
+ 2,
122
+ 5,
123
+ 10,
124
+ 20,
125
+ 30,
126
+ 60,
127
+ 120,
128
+ 300,
129
+ 600,
130
+ 1200,
131
+ 1800,
132
+ ],
133
+}
134
+
135
+METRIC_FAMILIES: dict[str, str] = {
136
+ "vllm:e2e_request_latency_seconds": "request_latency",
137
+ "vllm:request_queue_time_seconds": "request_latency",
138
+ "vllm:request_inference_time_seconds": "request_latency",
139
+ "vllm:request_prefill_time_seconds": "request_latency",
140
+ "vllm:request_decode_time_seconds": "request_latency",
141
+ "vllm:time_to_first_token_seconds": "time_to_first_token",
142
+ "vllm:inter_token_latency_seconds": "inter_token_latency",
143
+ "vllm:request_time_per_output_token_seconds": "inter_token_latency",
144
+ "vllm:iteration_tokens_total": "iteration_tokens",
145
+ "vllm:request_params_n": "request_params_n",
146
+ "vllm:request_num_preemptions": "request_num_preemptions",
147
+ "vllm:request_prompt_tokens": "request_tokens",
148
+ "vllm:request_generation_tokens": "request_tokens",
149
+ "vllm:request_max_num_generation_tokens": "request_tokens",
150
+ "vllm:request_params_max_tokens": "request_tokens",
151
+ "vllm:request_prefill_kv_computed_tokens": "request_tokens",
152
+ "vllm:kv_block_lifetime_seconds": "kv_cache_residency",
153
+ "vllm:kv_block_idle_before_evict_seconds": "kv_cache_residency",
154
+ "vllm:kv_block_reuse_gap_seconds": "kv_cache_residency",
155
+}
156
+
157
+TEST_MODEL = "distilbert/distilgpt2"
158
+TEST_MODEL_MAX_LEN = 1024
159
+REQUEST_TOKENS_SNAPSHOT_FOR_TEST_MODEL: list[float] = [
160
+ 1,
161
+ 2,
162
+ 5,
163
+ 10,
164
+ 20,
165
+ 50,
166
+ 100,
167
+ 200,
168
+ 500,
169
+ 1000,
170
+]
171
+
172
+
173
+@pytest.mark.parametrize("family", sorted(DEFAULT_BUCKET_SNAPSHOTS))
174
+def test_default_bucket_snapshots(family):
175
+ """Static families must expose exactly today's default boundaries."""
176
+ assert histogram_buckets(family) == DEFAULT_BUCKET_SNAPSHOTS[family]
177
+
178
+
179
+def test_request_tokens_bucket_snapshot():
180
+ """The 1-2-5 token-count series must be stable for a given model len."""
181
+ assert histogram_buckets("request_tokens", max_model_len=32768) == [
182
+ 1,
183
+ 2,
184
+ 5,
185
+ 10,
186
+ 20,
187
+ 50,
188
+ 100,
189
+ 200,
190
+ 500,
191
+ 1000,
192
+ 2000,
193
+ 5000,
194
+ 10000,
195
+ 20000,
196
+ ]
197
+ assert build_1_2_5_buckets(100) == [1, 2, 5, 10, 20, 50, 100]
198
+
199
+
200
+def test_request_tokens_requires_max_model_len():
201
+ with pytest.raises(ValueError, match="max_model_len is required"):
202
+ histogram_buckets("request_tokens")
203
+
204
+
205
+def test_buckets_return_fresh_copies():
206
+ """Mutating a returned list must not corrupt the shared defaults."""
207
+ first = histogram_buckets("request_latency")
208
+ first.append(1e9)
209
+ second = histogram_buckets("request_latency")
210
+ assert second == DEFAULT_BUCKET_SNAPSHOTS["request_latency"]
211
+ assert second is not first
212
+
213
+
214
+def test_bucket_family_keys():
215
+ """The declared families are exactly these, and each one feeds a histogram."""
216
+ assert {
217
+ "request_latency",
218
+ "time_to_first_token",
219
+ "inter_token_latency",
220
+ "iteration_tokens",
221
+ "request_params_n",
222
+ "request_num_preemptions",
223
+ "request_tokens",
224
+ "kv_cache_residency",
225
+ } == BUCKET_FAMILY_KEYS
226
+ assert set(METRIC_FAMILIES.values()) == BUCKET_FAMILY_KEYS
227
+
228
+
229
+def collect_histogram_buckets() -> dict[str, list[float]]:
230
+ """Read back each vllm:* histogram's finite bucket bounds by metric name."""
231
+ found: dict[str, list[float]] = {}
232
+ for metric in prometheus_client.REGISTRY.collect():
233
+ if metric.name.startswith("vllm:") and metric.type == "histogram":
234
+ found[metric.name] = sorted(
235
+ {
236
+ float(sample.labels["le"])
237
+ for sample in metric.samples
238
+ if sample.name.endswith("_bucket") and sample.labels["le"] != "+Inf"
239
+ }
240
+ )
241
+ return found
242
+
243
+
244
+def build_logger_config(
245
+ observability_config: ObservabilityConfig,
246
+) -> VllmConfig:
247
+ return VllmConfig(
248
+ model_config=ModelConfig(model=TEST_MODEL),
249
+ observability_config=observability_config,
250
+ )
251
+
252
+
253
+def test_prometheus_logger_default_buckets():
254
+ """Every core engine histogram must use its family's defaults.
255
+
256
+ The config deliberately sets no kv_transfer_config: a KV connector would
257
+ register its own histograms, which own their boundaries and are not
258
+ covered by METRIC_FAMILIES.
259
+ """
260
+ config = build_logger_config(ObservabilityConfig(kv_cache_metrics=True))
261
+ assert config.model_config.max_model_len == TEST_MODEL_MAX_LEN
262
+ try:
263
+ PrometheusStatLogger(config)
264
+ found = collect_histogram_buckets()
265
+ assert set(found) == set(METRIC_FAMILIES)
266
+ for metric_name, family in METRIC_FAMILIES.items():
267
+ if family == "request_tokens":
268
+ expected = REQUEST_TOKENS_SNAPSHOT_FOR_TEST_MODEL
269
+ else:
270
+ expected = DEFAULT_BUCKET_SNAPSHOTS[family]
271
+ assert found[metric_name] == [float(b) for b in expected], metric_name
272
+ finally:
273
+ unregister_vllm_metrics()
vllm/v1/metrics/buckets.py
new
+228
@@ -0,0 +1,228 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+"""Default bucket boundaries for the Prometheus histograms emitted by vLLM.
4
+
5
+Every core engine histogram created by the Prometheus stat logger draws its
6
+bucket boundaries from exactly one of the families defined here, so each
7
+default list has a single source of truth. Histograms owned by other
8
+subsystems, such as the KV connector and offloading metrics, keep their own
9
+boundaries.
10
+"""
11
+
12
+from typing import Literal, get_args
13
+
14
+BucketFamilyKey = Literal[
15
+ "request_latency",
16
+ "time_to_first_token",
17
+ "inter_token_latency",
18
+ "iteration_tokens",
19
+ "request_params_n",
20
+ "request_num_preemptions",
21
+ "request_tokens",
22
+ "kv_cache_residency",
23
+]
24
+"""Canonical keys for the histogram bucket families."""
25
+
26
+BUCKET_FAMILY_KEYS: frozenset[str] = frozenset(get_args(BucketFamilyKey))
27
+"""All bucket family keys, for membership checks."""
28
+
29
+REQUEST_LATENCY_BUCKETS: tuple[float, ...] = (
30
+ 0.3,
31
+ 0.5,
32
+ 0.8,
33
+ 1.0,
34
+ 1.5,
35
+ 2.0,
36
+ 2.5,
37
+ 5.0,
38
+ 10.0,
39
+ 15.0,
40
+ 20.0,
41
+ 30.0,
42
+ 40.0,
43
+ 50.0,
44
+ 60.0,
45
+ 120.0,
46
+ 240.0,
47
+ 480.0,
48
+ 960.0,
49
+ 1920.0,
50
+ 7680.0,
51
+)
52
+"""Sub-second scheduling delays through multi-hour batch requests; shared by
53
+the accumulated request phase-timing histograms (e2e, queue, inference,
54
+prefill, decode)."""
55
+
56
+TIME_TO_FIRST_TOKEN_BUCKETS: tuple[float, ...] = (
57
+ 0.001,
58
+ 0.005,
59
+ 0.01,
60
+ 0.02,
61
+ 0.04,
62
+ 0.06,
63
+ 0.08,
64
+ 0.1,
65
+ 0.25,
66
+ 0.5,
67
+ 0.75,
68
+ 1.0,
69
+ 2.5,
70
+ 5.0,
71
+ 7.5,
72
+ 10.0,
73
+ 20.0,
74
+ 40.0,
75
+ 80.0,
76
+ 160.0,
77
+ 640.0,
78
+ 2560.0,
79
+)
80
+"""Millisecond-scale prefill for tiny prompts up to ~40-minute worst cases,
81
+with the densest resolution around interactive (sub-second) latencies."""
82
+
83
+INTER_TOKEN_LATENCY_BUCKETS: tuple[float, ...] = (
84
+ 0.01,
85
+ 0.025,
86
+ 0.05,
87
+ 0.075,
88
+ 0.1,
89
+ 0.15,
90
+ 0.2,
91
+ 0.3,
92
+ 0.4,
93
+ 0.5,
94
+ 0.75,
95
+ 1.0,
96
+ 2.5,
97
+ 5.0,
98
+ 7.5,
99
+ 10.0,
100
+ 20.0,
101
+ 40.0,
102
+ 80.0,
103
+)
104
+"""Per-decode-step latencies: 10 ms fast decode up to multi-second stalls."""
105
+
106
+ITERATION_TOKENS_BUCKETS: tuple[float, ...] = (
107
+ 1,
108
+ 8,
109
+ 16,
110
+ 32,
111
+ 64,
112
+ 128,
113
+ 256,
114
+ 512,
115
+ 1024,
116
+ 2048,
117
+ 4096,
118
+ 8192,
119
+ 16384,
120
+)
121
+"""Tokens processed per engine step: powers of two up to a typical
122
+max-num-batched-tokens budget."""
123
+
124
+REQUEST_PARAMS_N_BUCKETS: tuple[float, ...] = (1, 2, 5, 10, 20)
125
+"""Small integer counts for the ``n`` sampling parameter."""
126
+
127
+REQUEST_NUM_PREEMPTIONS_BUCKETS: tuple[float, ...] = (1, 2, 3, 4, 5, 10, 20)
128
+"""Per-request preemption counts, recorded for every finished request: the
129
+first bucket therefore holds never-preempted requests together with
130
+singly-preempted ones, 2 through 5 are separated individually, and the tail
131
+coarsens to 20. Distinct from `REQUEST_PARAMS_N_BUCKETS` because these are the
132
+boundaries the metric shipped with, not a harmonized choice."""
133
+
134
+KV_CACHE_RESIDENCY_BUCKETS: tuple[float, ...] = (
135
+ 0.001,
136
+ 0.002,
137
+ 0.005,
138
+ 0.01,
139
+ 0.02,
140
+ 0.05,
141
+ 0.1,
142
+ 0.2,
143
+ 0.5,
144
+ 1,
145
+ 2,
146
+ 5,
147
+ 10,
148
+ 20,
149
+ 30,
150
+ 60,
151
+ 120,
152
+ 300,
153
+ 600,
154
+ 1200,
155
+ 1800,
156
+)
157
+"""KV cache block residency times: millisecond-scale reuse gaps up to
158
+30-minute block lifetimes."""
159
+
160
+_STATIC_FAMILY_DEFAULTS: dict[str, tuple[float, ...]] = {
161
+ "request_latency": REQUEST_LATENCY_BUCKETS,
162
+ "time_to_first_token": TIME_TO_FIRST_TOKEN_BUCKETS,
163
+ "inter_token_latency": INTER_TOKEN_LATENCY_BUCKETS,
164
+ "iteration_tokens": ITERATION_TOKENS_BUCKETS,
165
+ "request_params_n": REQUEST_PARAMS_N_BUCKETS,
166
+ "request_num_preemptions": REQUEST_NUM_PREEMPTIONS_BUCKETS,
167
+ "kv_cache_residency": KV_CACHE_RESIDENCY_BUCKETS,
168
+}
169
+
170
+
171
+def build_buckets(mantissa_lst: list[int], max_value: int) -> list[float]:
172
+ """Build buckets from mantissas scaled by increasing powers of 10.
173
+
174
+ Args:
175
+ mantissa_lst: Mantissa values multiplied by each power of 10.
176
+ max_value: Largest bucket value to include.
177
+
178
+ Returns:
179
+ Bucket values in increasing order, capped at `max_value`.
180
+ """
181
+ exponent = 0
182
+ buckets: list[float] = []
183
+ while True:
184
+ for m in mantissa_lst:
185
+ value = m * 10**exponent
186
+ if value <= max_value:
187
+ buckets.append(value)
188
+ else:
189
+ return buckets
190
+ exponent += 1
191
+
192
+
193
+def build_1_2_5_buckets(max_value: int) -> list[float]:
194
+ """Build a 1-2-5 series of buckets capped at `max_value`.
195
+
196
+ Example:
197
+ >>> build_1_2_5_buckets(100)
198
+ [1, 2, 5, 10, 20, 50, 100]
199
+ """
200
+ return build_buckets([1, 2, 5], max_value)
201
+
202
+
203
+def histogram_buckets(
204
+ family: BucketFamilyKey,
205
+ max_model_len: int | None = None,
206
+) -> list[float]:
207
+ """Return the default bucket boundaries for a histogram family.
208
+
209
+ Args:
210
+ family: Canonical bucket family key.
211
+ max_model_len: Cap for the token-count series of the
212
+ `request_tokens` family; required for that family and ignored
213
+ otherwise.
214
+
215
+ Returns:
216
+ A fresh list of bucket upper bounds; callers may mutate it freely.
217
+
218
+ Raises:
219
+ ValueError: If `family` is `request_tokens` and `max_model_len`
220
+ is None.
221
+ """
222
+ if family == "request_tokens":
223
+ if max_model_len is None:
224
+ raise ValueError(
225
+ "max_model_len is required for the 'request_tokens' bucket family"
226
+ )
227
+ return build_1_2_5_buckets(max_model_len)
228
+ return list(_STATIC_FAMILY_DEFAULTS[family])
vllm/v1/metrics/loggers.py
+23
-147
@@ -18,6 +18,7 @@ from vllm.distributed.kv_transfer.kv_connector.v1.metrics import (
18
from vllm.logger import init_logger
19
from vllm.plugins import STAT_LOGGER_PLUGINS_GROUP, load_plugins_by_group
20
from vllm.v1.engine import FinishReason
21
+from vllm.v1.metrics.buckets import histogram_buckets
22
from vllm.v1.metrics.perf import PerfMetricsLogging, PerfMetricsProm
23
from vllm.v1.metrics.prometheus import unregister_vllm_metrics
24
from vllm.v1.metrics.stats import (
@@ -724,13 +725,23 @@ class PrometheusStatLogger(AggregateStatLoggerBase):
725
for idx in engine_indexes
726
}
727
728
+ request_tokens_buckets = histogram_buckets(
729
+ "request_tokens", max_model_len=max_model_len
730
+ )
731
+ iteration_tokens_buckets = histogram_buckets("iteration_tokens")
732
+ request_params_n_buckets = histogram_buckets("request_params_n")
733
+ time_to_first_token_buckets = histogram_buckets("time_to_first_token")
734
+ inter_token_latency_buckets = histogram_buckets("inter_token_latency")
735
+ request_latency_buckets = histogram_buckets("request_latency")
736
+ request_num_preemptions_buckets = histogram_buckets("request_num_preemptions")
737
+
738
#
739
# Histograms of counts
740
#
741
histogram_num_prompt_tokens_request = self._histogram_cls(
742
name="vllm:request_prompt_tokens",
743
documentation="Number of prefill tokens processed.",
733
- buckets=build_1_2_5_buckets(max_model_len),
744
+ buckets=request_tokens_buckets,
745
labelnames=labelnames,
746
)
747
self.histogram_num_prompt_tokens_request = create_metric_per_engine(
@@ -740,7 +751,7 @@ class PrometheusStatLogger(AggregateStatLoggerBase):
751
histogram_num_generation_tokens_request = self._histogram_cls(
752
name="vllm:request_generation_tokens",
753
documentation="Number of generation tokens processed.",
743
- buckets=build_1_2_5_buckets(max_model_len),
754
+ buckets=request_tokens_buckets,
755
labelnames=labelnames,
756
)
757
self.histogram_num_generation_tokens_request = create_metric_per_engine(
@@ -753,7 +764,7 @@ class PrometheusStatLogger(AggregateStatLoggerBase):
764
histogram_iteration_tokens = self._histogram_cls(
765
name="vllm:iteration_tokens_total",
766
documentation="Histogram of number of tokens per engine_step.",
756
- buckets=[1, 8, 16, 32, 64, 128, 256, 512, 1024, 2048, 4096, 8192, 16384],
767
+ buckets=iteration_tokens_buckets,
768
labelnames=labelnames,
769
)
770
self.histogram_iteration_tokens = create_metric_per_engine(
@@ -763,7 +774,7 @@ class PrometheusStatLogger(AggregateStatLoggerBase):
774
histogram_max_num_generation_tokens_request = self._histogram_cls(
775
name="vllm:request_max_num_generation_tokens",
776
documentation="Histogram of maximum number of requested generation tokens.",
766
- buckets=build_1_2_5_buckets(max_model_len),
777
+ buckets=request_tokens_buckets,
778
labelnames=labelnames,
779
)
780
self.histogram_max_num_generation_tokens_request = create_metric_per_engine(
@@ -773,7 +784,7 @@ class PrometheusStatLogger(AggregateStatLoggerBase):
784
histogram_n_request = self._histogram_cls(
785
name="vllm:request_params_n",
786
documentation="Histogram of the n request parameter.",
776
- buckets=[1, 2, 5, 10, 20],
787
+ buckets=request_params_n_buckets,
788
labelnames=labelnames,
789
)
790
self.histogram_n_request = create_metric_per_engine(
@@ -783,7 +794,7 @@ class PrometheusStatLogger(AggregateStatLoggerBase):
794
histogram_max_tokens_request = self._histogram_cls(
795
name="vllm:request_params_max_tokens",
796
documentation="Histogram of the max_tokens request parameter.",
786
- buckets=build_1_2_5_buckets(max_model_len),
797
+ buckets=request_tokens_buckets,
798
labelnames=labelnames,
799
)
800
self.histogram_max_tokens_request = create_metric_per_engine(
@@ -796,30 +807,7 @@ class PrometheusStatLogger(AggregateStatLoggerBase):
807
histogram_time_to_first_token = self._histogram_cls(
808
name="vllm:time_to_first_token_seconds",
809
documentation="Histogram of time to first token in seconds.",
799
- buckets=[
800
- 0.001,
801
- 0.005,
802
- 0.01,
803
- 0.02,
804
- 0.04,
805
- 0.06,
806
- 0.08,
807
- 0.1,
808
- 0.25,
809
- 0.5,
810
- 0.75,
811
- 1.0,
812
- 2.5,
813
- 5.0,
814
- 7.5,
815
- 10.0,
816
- 20.0,
817
- 40.0,
818
- 80.0,
819
- 160.0,
820
- 640.0,
821
- 2560.0,
822
- ],
810
+ buckets=time_to_first_token_buckets,
811
labelnames=labelnames,
812
)
813
self.histogram_time_to_first_token = create_metric_per_engine(
@@ -829,27 +817,7 @@ class PrometheusStatLogger(AggregateStatLoggerBase):
817
histogram_inter_token_latency = self._histogram_cls(
818
name="vllm:inter_token_latency_seconds",
819
documentation="Histogram of inter-token latency in seconds.",
832
- buckets=[
833
- 0.01,
834
- 0.025,
835
- 0.05,
836
- 0.075,
837
- 0.1,
838
- 0.15,
839
- 0.2,
840
- 0.3,
841
- 0.4,
842
- 0.5,
843
- 0.75,
844
- 1.0,
845
- 2.5,
846
- 5.0,
847
- 7.5,
848
- 10.0,
849
- 20.0,
850
- 40.0,
851
- 80.0,
852
- ],
820
+ buckets=inter_token_latency_buckets,
821
labelnames=labelnames,
822
)
823
self.histogram_inter_token_latency = create_metric_per_engine(
@@ -859,56 +827,13 @@ class PrometheusStatLogger(AggregateStatLoggerBase):
827
histogram_request_time_per_output_token = self._histogram_cls(
828
name="vllm:request_time_per_output_token_seconds",
829
documentation="Histogram of time_per_output_token_seconds per request.",
862
- buckets=[
863
- 0.01,
864
- 0.025,
865
- 0.05,
866
- 0.075,
867
- 0.1,
868
- 0.15,
869
- 0.2,
870
- 0.3,
871
- 0.4,
872
- 0.5,
873
- 0.75,
874
- 1.0,
875
- 2.5,
876
- 5.0,
877
- 7.5,
878
- 10.0,
879
- 20.0,
880
- 40.0,
881
- 80.0,
882
- ],
830
+ buckets=inter_token_latency_buckets,
831
labelnames=labelnames,
832
)
833
self.histogram_request_time_per_output_token = create_metric_per_engine(
834
histogram_request_time_per_output_token, per_engine_labelvalues
835
)
836
889
- request_latency_buckets = [
890
- 0.3,
891
- 0.5,
892
- 0.8,
893
- 1.0,
894
- 1.5,
895
- 2.0,
896
- 2.5,
897
- 5.0,
898
- 10.0,
899
- 15.0,
900
- 20.0,
901
- 30.0,
902
- 40.0,
903
- 50.0,
904
- 60.0,
905
- 120.0,
906
- 240.0,
907
- 480.0,
908
- 960.0,
909
- 1920.0,
910
- 7680.0,
911
- ]
837
histogram_e2e_time_request = self._histogram_cls(
838
name="vllm:e2e_request_latency_seconds",
839
documentation="Histogram of e2e request latency in seconds.",
@@ -962,7 +887,7 @@ class PrometheusStatLogger(AggregateStatLoggerBase):
887
histogram_request_num_preemptions = self._histogram_cls(
888
name="vllm:request_num_preemptions",
889
documentation="Histogram of the number of times a request was preempted.",
965
- buckets=[1, 2, 3, 4, 5, 10, 20],
890
+ buckets=request_num_preemptions_buckets,
891
labelnames=labelnames,
892
)
893
self.histogram_request_num_preemptions = create_metric_per_engine(
@@ -975,7 +900,7 @@ class PrometheusStatLogger(AggregateStatLoggerBase):
900
"Histogram of new KV tokens computed during prefill "
901
"(excluding cached tokens)."
902
),
978
- buckets=build_1_2_5_buckets(max_model_len),
903
+ buckets=request_tokens_buckets,
904
labelnames=labelnames,
905
)
906
self.histogram_prefill_kv_computed_request = create_metric_per_engine(
@@ -986,29 +911,7 @@ class PrometheusStatLogger(AggregateStatLoggerBase):
911
# KV Cache residency metrics
912
#
913
if self.kv_cache_metrics_enabled:
989
- kv_cache_residency_buckets = [
990
- 0.001,
991
- 0.002,
992
- 0.005,
993
- 0.01,
994
- 0.02,
995
- 0.05,
996
- 0.1,
997
- 0.2,
998
- 0.5,
999
- 1,
1000
- 2,
1001
- 5,
1002
- 10,
1003
- 20,
1004
- 30,
1005
- 60,
1006
- 120,
1007
- 300,
1008
- 600,
1009
- 1200,
1010
- 1800,
1011
- ]
914
+ kv_cache_residency_buckets = histogram_buckets("kv_cache_residency")
915
916
histogram_kv_block_lifetime = self._histogram_cls(
917
name="vllm:kv_block_lifetime_seconds",
@@ -1294,33 +1197,6 @@ class PrometheusStatLogger(AggregateStatLoggerBase):
1197
self.log_metrics_info("cache_config", self.vllm_config.cache_config)
1198
1199
1297
-def build_buckets(mantissa_lst: list[int], max_value: int) -> list[int]:
1298
- """
1299
- Builds a list of buckets with increasing powers of 10 multiplied by
1300
- mantissa values until the value exceeds the specified maximum.
1301
-
1302
- """
1303
- exponent = 0
1304
- buckets: list[int] = []
1305
- while True:
1306
- for m in mantissa_lst:
1307
- value = m * 10**exponent
1308
- if value <= max_value:
1309
- buckets.append(value)
1310
- else:
1311
- return buckets
1312
- exponent += 1
1313
-
1314
-
1315
-def build_1_2_5_buckets(max_value: int) -> list[int]:
1316
- """
1317
- Example:
1318
- >>> build_1_2_5_buckets(100)
1319
- [1, 2, 5, 10, 20, 50, 100]
1320
- """
1321
- return build_buckets([1, 2, 5], max_value)
1322
-
1323
-
1200
class StatLoggerManager:
1201
"""
1202
StatLoggerManager: