| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | """Snapshot tests locking the default Prometheus histogram buckets. |
| 4 | |
| 5 | The expected lists are deliberately hard-coded literals rather than imports: |
| 6 | any change to a default bucket boundary must show up as a failing test here. |
| 7 | """ |
| 8 | |
| 9 | import prometheus_client |
| 10 | import pytest |
| 11 | |
| 12 | from vllm.config import ModelConfig, VllmConfig |
| 13 | from vllm.config.observability import ObservabilityConfig |
| 14 | from vllm.v1.metrics.buckets import ( |
| 15 | BUCKET_FAMILY_KEYS, |
| 16 | build_1_2_5_buckets, |
| 17 | histogram_buckets, |
| 18 | ) |
| 19 | from vllm.v1.metrics.loggers import PrometheusStatLogger |
| 20 | from vllm.v1.metrics.prometheus import unregister_vllm_metrics |
| 21 | |
| 22 | pytestmark = pytest.mark.cpu_test |
| 23 | |
| 24 | DEFAULT_BUCKET_SNAPSHOTS: dict[str, list[float]] = { |
| 25 | "request_latency": [ |
| 26 | 0.3, |
| 27 | 0.5, |
| 28 | 0.8, |
| 29 | 1.0, |
| 30 | 1.5, |
| 31 | 2.0, |
| 32 | 2.5, |
| 33 | 5.0, |
| 34 | 10.0, |
| 35 | 15.0, |
| 36 | 20.0, |
| 37 | 30.0, |
| 38 | 40.0, |
| 39 | 50.0, |
| 40 | 60.0, |
| 41 | 120.0, |
| 42 | 240.0, |
| 43 | 480.0, |
| 44 | 960.0, |
| 45 | 1920.0, |
| 46 | 7680.0, |
| 47 | ], |
| 48 | "time_to_first_token": [ |
| 49 | 0.001, |
| 50 | 0.005, |
| 51 | 0.01, |
| 52 | 0.02, |
| 53 | 0.04, |
| 54 | 0.06, |
| 55 | 0.08, |
| 56 | 0.1, |
| 57 | 0.25, |
| 58 | 0.5, |
| 59 | 0.75, |
| 60 | 1.0, |
| 61 | 2.5, |
| 62 | 5.0, |
| 63 | 7.5, |
| 64 | 10.0, |
| 65 | 20.0, |
| 66 | 40.0, |
| 67 | 80.0, |
| 68 | 160.0, |
| 69 | 640.0, |
| 70 | 2560.0, |
| 71 | ], |
| 72 | "inter_token_latency": [ |
| 73 | 0.01, |
| 74 | 0.025, |
| 75 | 0.05, |
| 76 | 0.075, |
| 77 | 0.1, |
| 78 | 0.15, |
| 79 | 0.2, |
| 80 | 0.3, |
| 81 | 0.4, |
| 82 | 0.5, |
| 83 | 0.75, |
| 84 | 1.0, |
| 85 | 2.5, |
| 86 | 5.0, |
| 87 | 7.5, |
| 88 | 10.0, |
| 89 | 20.0, |
| 90 | 40.0, |
| 91 | 80.0, |
| 92 | ], |
| 93 | "iteration_tokens": [ |
| 94 | 1, |
| 95 | 8, |
| 96 | 16, |
| 97 | 32, |
| 98 | 64, |
| 99 | 128, |
| 100 | 256, |
| 101 | 512, |
| 102 | 1024, |
| 103 | 2048, |
| 104 | 4096, |
| 105 | 8192, |
| 106 | 16384, |
| 107 | ], |
| 108 | "request_params_n": [1, 2, 5, 10, 20], |
| 109 | "request_num_preemptions": [1, 2, 3, 4, 5, 10, 20], |
| 110 | "kv_cache_residency": [ |
| 111 | 0.001, |
| 112 | 0.002, |
| 113 | 0.005, |
| 114 | 0.01, |
| 115 | 0.02, |
| 116 | 0.05, |
| 117 | 0.1, |
| 118 | 0.2, |
| 119 | 0.5, |
| 120 | 1, |
| 121 | 2, |
| 122 | 5, |
| 123 | 10, |
| 124 | 20, |
| 125 | 30, |
| 126 | 60, |
| 127 | 120, |
| 128 | 300, |
| 129 | 600, |
| 130 | 1200, |
| 131 | 1800, |
| 132 | ], |
| 133 | } |
| 134 | |
| 135 | METRIC_FAMILIES: dict[str, str] = { |
| 136 | "vllm:e2e_request_latency_seconds": "request_latency", |
| 137 | "vllm:request_queue_time_seconds": "request_latency", |
| 138 | "vllm:request_inference_time_seconds": "request_latency", |
| 139 | "vllm:request_prefill_time_seconds": "request_latency", |
| 140 | "vllm:request_decode_time_seconds": "request_latency", |
| 141 | "vllm:time_to_first_token_seconds": "time_to_first_token", |
| 142 | "vllm:inter_token_latency_seconds": "inter_token_latency", |
| 143 | "vllm:request_time_per_output_token_seconds": "inter_token_latency", |
| 144 | "vllm:iteration_tokens_total": "iteration_tokens", |
| 145 | "vllm:request_params_n": "request_params_n", |
| 146 | "vllm:request_num_preemptions": "request_num_preemptions", |
| 147 | "vllm:request_prompt_tokens": "request_tokens", |
| 148 | "vllm:request_generation_tokens": "request_tokens", |
| 149 | "vllm:request_max_num_generation_tokens": "request_tokens", |
| 150 | "vllm:request_params_max_tokens": "request_tokens", |
| 151 | "vllm:request_prefill_kv_computed_tokens": "request_tokens", |
| 152 | "vllm:kv_block_lifetime_seconds": "kv_cache_residency", |
| 153 | "vllm:kv_block_idle_before_evict_seconds": "kv_cache_residency", |
| 154 | "vllm:kv_block_reuse_gap_seconds": "kv_cache_residency", |
| 155 | } |
| 156 | |
| 157 | TEST_MODEL = "distilbert/distilgpt2" |
| 158 | TEST_MODEL_MAX_LEN = 1024 |
| 159 | REQUEST_TOKENS_SNAPSHOT_FOR_TEST_MODEL: list[float] = [ |
| 160 | 1, |
| 161 | 2, |
| 162 | 5, |
| 163 | 10, |
| 164 | 20, |
| 165 | 50, |
| 166 | 100, |
| 167 | 200, |
| 168 | 500, |
| 169 | 1000, |
| 170 | ] |
| 171 | |
| 172 | |
| 173 | @pytest.mark.parametrize("family", sorted(DEFAULT_BUCKET_SNAPSHOTS)) |
| 174 | def test_default_bucket_snapshots(family): |
| 175 | """Static families must expose exactly today's default boundaries.""" |
| 176 | assert histogram_buckets(family) == DEFAULT_BUCKET_SNAPSHOTS[family] |
| 177 | |
| 178 | |
| 179 | def test_request_tokens_bucket_snapshot(): |
| 180 | """The 1-2-5 token-count series must be stable for a given model len.""" |
| 181 | assert histogram_buckets("request_tokens", max_model_len=32768) == [ |
| 182 | 1, |
| 183 | 2, |
| 184 | 5, |
| 185 | 10, |
| 186 | 20, |
| 187 | 50, |
| 188 | 100, |
| 189 | 200, |
| 190 | 500, |
| 191 | 1000, |
| 192 | 2000, |
| 193 | 5000, |
| 194 | 10000, |
| 195 | 20000, |
| 196 | ] |
| 197 | assert build_1_2_5_buckets(100) == [1, 2, 5, 10, 20, 50, 100] |
| 198 | |
| 199 | |
| 200 | def test_request_tokens_requires_max_model_len(): |
| 201 | with pytest.raises(ValueError, match="max_model_len is required"): |
| 202 | histogram_buckets("request_tokens") |
| 203 | |
| 204 | |
| 205 | def test_buckets_return_fresh_copies(): |
| 206 | """Mutating a returned list must not corrupt the shared defaults.""" |
| 207 | first = histogram_buckets("request_latency") |
| 208 | first.append(1e9) |
| 209 | second = histogram_buckets("request_latency") |
| 210 | assert second == DEFAULT_BUCKET_SNAPSHOTS["request_latency"] |
| 211 | assert second is not first |
| 212 | |
| 213 | |
| 214 | def test_bucket_family_keys(): |
| 215 | """The declared families are exactly these, and each one feeds a histogram.""" |
| 216 | assert { |
| 217 | "request_latency", |
| 218 | "time_to_first_token", |
| 219 | "inter_token_latency", |
| 220 | "iteration_tokens", |
| 221 | "request_params_n", |
| 222 | "request_num_preemptions", |
| 223 | "request_tokens", |
| 224 | "kv_cache_residency", |
| 225 | } == BUCKET_FAMILY_KEYS |
| 226 | assert set(METRIC_FAMILIES.values()) == BUCKET_FAMILY_KEYS |
| 227 | |
| 228 | |
| 229 | def collect_histogram_buckets() -> dict[str, list[float]]: |
| 230 | """Read back each vllm:* histogram's finite bucket bounds by metric name.""" |
| 231 | found: dict[str, list[float]] = {} |
| 232 | for metric in prometheus_client.REGISTRY.collect(): |
| 233 | if metric.name.startswith("vllm:") and metric.type == "histogram": |
| 234 | found[metric.name] = sorted( |
| 235 | { |
| 236 | float(sample.labels["le"]) |
| 237 | for sample in metric.samples |
| 238 | if sample.name.endswith("_bucket") and sample.labels["le"] != "+Inf" |
| 239 | } |
| 240 | ) |
| 241 | return found |
| 242 | |
| 243 | |
| 244 | def build_logger_config( |
| 245 | observability_config: ObservabilityConfig, |
| 246 | ) -> VllmConfig: |
| 247 | return VllmConfig( |
| 248 | model_config=ModelConfig(model=TEST_MODEL), |
| 249 | observability_config=observability_config, |
| 250 | ) |
| 251 | |
| 252 | |
| 253 | def test_prometheus_logger_default_buckets(): |
| 254 | """Every core engine histogram must use its family's defaults. |
| 255 | |
| 256 | The config deliberately sets no kv_transfer_config: a KV connector would |
| 257 | register its own histograms, which own their boundaries and are not |
| 258 | covered by METRIC_FAMILIES. |
| 259 | """ |
| 260 | config = build_logger_config(ObservabilityConfig(kv_cache_metrics=True)) |
| 261 | assert config.model_config.max_model_len == TEST_MODEL_MAX_LEN |
| 262 | try: |
| 263 | PrometheusStatLogger(config) |
| 264 | found = collect_histogram_buckets() |
| 265 | assert set(found) == set(METRIC_FAMILIES) |
| 266 | for metric_name, family in METRIC_FAMILIES.items(): |
| 267 | if family == "request_tokens": |
| 268 | expected = REQUEST_TOKENS_SNAPSHOT_FOR_TEST_MODEL |
| 269 | else: |
| 270 | expected = DEFAULT_BUCKET_SNAPSHOTS[family] |
| 271 | assert found[metric_name] == [float(b) for b in expected], metric_name |
| 272 | finally: |
| 273 | unregister_vllm_metrics() |
| 274 | |
| 275 | |
| 276 | def test_histogram_buckets_override_precedence(): |
| 277 | """A present family key wins verbatim; absent keys keep defaults.""" |
| 278 | overrides = { |
| 279 | "request_latency": [0.5, 1.0], |
| 280 | "request_tokens": [16.0, 4096.0], |
| 281 | } |
| 282 | got = histogram_buckets("request_latency", overrides=overrides) |
| 283 | assert got == [0.5, 1.0] |
| 284 | assert got is not overrides["request_latency"] |
| 285 | # An override replaces the max_model_len-derived series entirely. |
| 286 | assert histogram_buckets("request_tokens", overrides=overrides) == [ |
| 287 | 16.0, |
| 288 | 4096.0, |
| 289 | ] |
| 290 | assert ( |
| 291 | histogram_buckets("time_to_first_token", overrides=overrides) |
| 292 | == DEFAULT_BUCKET_SNAPSHOTS["time_to_first_token"] |
| 293 | ) |
| 294 | |
| 295 | |
| 296 | def test_prometheus_logger_applies_overrides(): |
| 297 | """Every family must honour an override, with a distinct list each. |
| 298 | |
| 299 | Overriding all of them at once is what catches a histogram wired to the |
| 300 | defaults instead of the override, and a family wired to the wrong list. |
| 301 | """ |
| 302 | overridden: dict[str, list[float]] = { |
| 303 | family: [index + 1.0, index + 2.0] |
| 304 | for index, family in enumerate(sorted(BUCKET_FAMILY_KEYS)) |
| 305 | } |
| 306 | config = build_logger_config( |
| 307 | ObservabilityConfig( |
| 308 | kv_cache_metrics=True, |
| 309 | custom_histogram_buckets=overridden, |
| 310 | ) |
| 311 | ) |
| 312 | try: |
| 313 | PrometheusStatLogger(config) |
| 314 | found = collect_histogram_buckets() |
| 315 | assert set(found) == set(METRIC_FAMILIES) |
| 316 | for metric_name, family in METRIC_FAMILIES.items(): |
| 317 | assert found[metric_name] == overridden[family], metric_name |
| 318 | finally: |
| 319 | unregister_vllm_metrics() |