[Feat] Add request-level preemption count histogram metric (#49984)
Amy (Yu-Hsuan) Lin committed
Sep 1, 2026 at 03:04 UTC
923949e6e3b6ff5ed2b842c5206de3e973fd5eb0
3 files changed
+21
tests/entrypoints/serve/instrumentator/test_metrics.py
+4
@@ -85,6 +85,7 @@ def _get_expected_values(num_requests: int, prompt_ids: list[int], max_tokens: i
85
"vllm:request_inference_time_seconds": [("_count", num_requests)],
86
"vllm:request_prefill_time_seconds": [("_count", num_requests)],
87
"vllm:request_decode_time_seconds": [("_count", num_requests)],
88
+ "vllm:request_num_preemptions": [("_count", num_requests)],
89
"vllm:request_prompt_tokens": [
90
("_sum", num_requests * num_prompt_tokens),
91
("_count", num_requests),
@@ -225,6 +226,9 @@ EXPECTED_METRICS_V1 = [
226
"vllm:request_decode_time_seconds_sum",
227
"vllm:request_decode_time_seconds_bucket",
228
"vllm:request_decode_time_seconds_count",
229
+ "vllm:request_num_preemptions_sum",
230
+ "vllm:request_num_preemptions_bucket",
231
+ "vllm:request_num_preemptions_count",
232
]
233
234
EXPECTED_METRICS_MM = [
vllm/v1/metrics/loggers.py
+13
@@ -959,6 +959,16 @@ class PrometheusStatLogger(AggregateStatLoggerBase):
959
histogram_decode_time_request, per_engine_labelvalues
960
)
961
962
+ histogram_request_num_preemptions = self._histogram_cls(
963
+ name="vllm:request_num_preemptions",
964
+ documentation="Histogram of the number of times a request was preempted.",
965
+ buckets=[1, 2, 3, 4, 5, 10, 20],
966
+ labelnames=labelnames,
967
+ )
968
+ self.histogram_request_num_preemptions = create_metric_per_engine(
969
+ histogram_request_num_preemptions, per_engine_labelvalues
970
+ )
971
+
972
histogram_prefill_kv_computed_request = self._histogram_cls(
973
name="vllm:request_prefill_kv_computed_tokens",
974
documentation=(
@@ -1237,6 +1247,9 @@ class PrometheusStatLogger(AggregateStatLoggerBase):
1247
self.histogram_decode_time_request[engine_idx].observe(
1248
finished_request.decode_time
1249
)
1250
+ self.histogram_request_num_preemptions[engine_idx].observe(
1251
+ finished_request.num_preemptions
1252
+ )
1253
# Calculate prefill KV compute (excludes cached tokens)
1254
prefill_kv_computed = finished_request.num_prompt_tokens - max(
1255
finished_request.num_cached_tokens, 0
vllm/v1/metrics/stats.py
+4
@@ -219,6 +219,7 @@ class RequestStateStats:
219
"""Stats that need to be tracked across delta updates."""
220
221
num_generation_tokens: int = 0
222
+ num_preemptions: int = 0
223
224
# This is an engine frontend timestamp (wall-clock)
225
arrival_time: float = 0.0
@@ -244,6 +245,7 @@ class FinishedRequestStats:
245
request_id: str | None = None
246
e2e_latency: float = 0.0
247
num_prompt_tokens: int = 0
248
+ num_preemptions: int = 0
249
num_generation_tokens: int = 0
250
max_tokens_param: int | None = None
251
queued_time: float = 0.0
@@ -523,6 +525,7 @@ class IterationStats:
525
lora_states.request_running(req_id, lora_name)
526
elif event.type == EngineCoreEventType.PREEMPTED:
527
self.num_preempted_reqs += 1
528
+ req_stats.num_preemptions += 1
529
lora_states.request_waiting(req_id, lora_name)
530
531
def update_from_finished_request(
@@ -563,6 +566,7 @@ class IterationStats:
566
request_id=request_id,
567
e2e_latency=e2e_latency,
568
num_prompt_tokens=num_prompt_tokens,
569
+ num_preemptions=req_stats.num_preemptions,
570
num_generation_tokens=req_stats.num_generation_tokens,
571
max_tokens_param=max_tokens_param,
572
queued_time=queued_time,