A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 5, 06:00 UTC
[Core][Metrics] expose waiting request breakdown via labeled metric (capacity/deferred) (#38435)
Signed-off-by: Mukesh Baphna <mukesh@hippocraticai.com> Signed-off-by: Mark McLoughlin <markmc@redhat.com> Co-authored-by: Claude Sonnet 4.6 <noreply@anthropic.com> Co-authored-by: Mark McLoughlin <markmc@redhat.com>
mukesh-hai committed
Apr 13, 2026 at 12:30 UTC
5c18b961d692cf45df0a1ecca7d1688f6168ae0f
5 files changed
+101
-4
tests/entrypoints/serve/instrumentator/test_metrics.py
+1
@@ -182,6 +182,7 @@ async def test_metrics_counts(
182
EXPECTED_METRICS_V1 = [
183
"vllm:num_requests_running",
184
"vllm:num_requests_waiting",
185
+ "vllm:num_requests_waiting_by_reason",
186
"vllm:kv_cache_usage_perc",
187
"vllm:prefix_cache_queries",
188
"vllm:prefix_cache_hits",
tests/v1/core/test_scheduler.py
+48
@@ -1039,6 +1039,54 @@ def test_no_spec_tokens_scheduled_for_prefill_chunks():
1039
assert len(output.scheduled_spec_decode_tokens[req.request_id]) == num_spec_tokens
1040
1041
1042
+def test_scheduler_stats_waiting_queues():
1043
+ """Test that scheduler stats correctly report waiting and skipped_waiting queues."""
1044
+ # Create scheduler with limited capacity so we can have waiting requests
1045
+ scheduler = create_scheduler(max_num_batched_tokens=100)
1046
+
1047
+ # Create requests: some will be scheduled, some will wait on capacity,
1048
+ # and some will be blocked by constraints
1049
+ all_requests = create_requests(num_requests=5, num_tokens=50)
1050
+
1051
+ # Add 3 requests - only 2 can be scheduled (2 * 50 = 100 tokens)
1052
+ # The 3rd will remain in waiting queue (capacity constraint)
1053
+ for request in all_requests[:3]:
1054
+ scheduler.add_request(request)
1055
+
1056
+ # Manually add 2 more to skipped_waiting to simulate constraint-blocked
1057
+ for request in all_requests[3:]:
1058
+ request.status = RequestStatus.WAITING_FOR_REMOTE_KVS
1059
+ scheduler.skipped_waiting.add_request(request)
1060
+
1061
+ # Schedule - this will schedule 2 requests, leaving 1 in waiting
1062
+ output = scheduler.schedule()
1063
+
1064
+ # Verify: 2 scheduled, 1 still waiting on capacity, 2 blocked by constraints
1065
+ assert len(output.scheduled_new_reqs) == 2
1066
+ assert len(scheduler.waiting) == 1
1067
+ assert len(scheduler.skipped_waiting) == 2
1068
+
1069
+ # Call update_from_output() to get frontend-facing stat
1070
+ scheduled_req_ids = list(output.num_scheduled_tokens.keys())
1071
+ model_runner_output = ModelRunnerOutput(
1072
+ req_ids=scheduled_req_ids,
1073
+ req_id_to_index={req_id: i for i, req_id in enumerate(scheduled_req_ids)},
1074
+ sampled_token_ids=[[1]] * len(scheduled_req_ids),
1075
+ logprobs=None,
1076
+ prompt_logprobs_dict={},
1077
+ pooler_output=[],
1078
+ )
1079
+ engine_core_outputs = scheduler.update_from_output(output, model_runner_output)
1080
+ assert engine_core_outputs and len(engine_core_outputs) > 0
1081
+ stats = engine_core_outputs[0].scheduler_stats
1082
+ assert stats is not None
1083
+
1084
+ # Verify stats match queue lengths after scheduling
1085
+ assert stats.num_running_reqs == 2 # 2 were scheduled
1086
+ assert stats.num_waiting_reqs == 1 # 1 waiting on capacity
1087
+ assert stats.num_skipped_waiting_reqs == 2 # 2 blocked by constraints
1088
+
1089
+
1090
def _assert_right_scheduler_output(
1091
output: SchedulerOutput,
1092
num_requests: int,
vllm/v1/core/sched/scheduler.py
+2
-1
@@ -1955,7 +1955,8 @@ class Scheduler(SchedulerInterface):
1955
)
1956
return SchedulerStats(
1957
num_running_reqs=len(self.running),
1958
- num_waiting_reqs=len(self.waiting) + len(self.skipped_waiting),
1958
+ num_waiting_reqs=len(self.waiting),
1959
+ num_skipped_waiting_reqs=len(self.skipped_waiting),
1960
kv_cache_usage=self.kv_cache_manager.usage,
1961
encoder_cache_usage=self._get_encoder_cache_usage(),
1962
prefix_cache_stats=prefix_cache_stats,
vllm/v1/metrics/loggers.py
+47
-2
@@ -32,6 +32,10 @@ from vllm.v1.spec_decode.metrics import SpecDecodingLogging, SpecDecodingProm
32
33
logger = init_logger(__name__)
34
35
+# User-facing reason labels for waiting request breakdown
36
+WAITING_REASON_CAPACITY = "capacity"
37
+WAITING_REASON_DEFERRED = "deferred"
38
+
39
PerEngineStatLoggerFactory = Callable[[VllmConfig, int], "StatLoggerBase"]
40
AggregateStatLoggerFactory = type["AggregateStatLoggerBase"]
41
StatLoggerFactory = AggregateStatLoggerFactory | PerEngineStatLoggerFactory
@@ -222,13 +226,21 @@ class LoggingStatLogger(StatLoggerBase):
226
"Running: %d reqs",
227
"Waiting: %d reqs",
228
]
229
+ total_waiting = (
230
+ self.last_scheduler_stats.num_waiting_reqs
231
+ + self.last_scheduler_stats.num_skipped_waiting_reqs
232
+ )
233
log_args: list[int | float | str] = [
234
self.last_prompt_throughput,
235
self.last_generation_throughput,
236
self.last_scheduler_stats.num_running_reqs,
229
- self.last_scheduler_stats.num_waiting_reqs,
237
+ total_waiting,
238
]
239
240
+ if self.last_scheduler_stats.num_skipped_waiting_reqs > 0:
241
+ log_parts.append("Deferred: %d reqs")
242
+ log_args.append(self.last_scheduler_stats.num_skipped_waiting_reqs)
243
+
244
if self.num_preemptions > 0:
245
log_parts.append("Preemptions: %d")
246
log_args.append(self.num_preemptions)
@@ -328,6 +340,9 @@ class AggregatedLoggingStatLogger(LoggingStatLogger, AggregateStatLoggerBase):
340
self.last_scheduler_stats.num_running_reqs += (
341
last_scheduler_stats.num_running_reqs
342
)
343
+ self.last_scheduler_stats.num_skipped_waiting_reqs += (
344
+ last_scheduler_stats.num_skipped_waiting_reqs
345
+ )
346
self.last_scheduler_stats.kv_cache_usage += (
347
last_scheduler_stats.kv_cache_usage
348
)
@@ -453,6 +468,28 @@ class PrometheusStatLogger(AggregateStatLoggerBase):
468
gauge_scheduler_waiting, per_engine_labelvalues
469
)
470
471
+ gauge_waiting_by_reason = self._gauge_cls(
472
+ name="vllm:num_requests_waiting_by_reason",
473
+ documentation=(
474
+ "Number of waiting requests by reason. "
475
+ "Reason labels: 'capacity' = waiting for scheduling capacity; "
476
+ "'deferred' = deferred by transient constraints "
477
+ "(LoRA budget, KV transfer, blocked status). "
478
+ "Sum of all reasons equals vllm:num_requests_waiting."
479
+ ),
480
+ multiprocess_mode="mostrecent",
481
+ labelnames=labelnames + ["reason"],
482
+ )
483
+ self.gauge_waiting_by_reason: dict[str, dict[int, Gauge]] = {}
484
+ for waiting_reason in [WAITING_REASON_CAPACITY, WAITING_REASON_DEFERRED]:
485
+ per_engine_labelvalues_with_reason = {
486
+ idx: labelvalues + [waiting_reason]
487
+ for idx, labelvalues in per_engine_labelvalues.items()
488
+ }
489
+ self.gauge_waiting_by_reason[waiting_reason] = create_metric_per_engine(
490
+ gauge_waiting_by_reason, per_engine_labelvalues_with_reason
491
+ )
492
+
493
gauge_engine_sleep_state = self._gauge_cls(
494
name="vllm:engine_sleep_state",
495
documentation=(
@@ -1030,8 +1067,16 @@ class PrometheusStatLogger(AggregateStatLoggerBase):
1067
self.gauge_scheduler_running[engine_idx].set(
1068
scheduler_stats.num_running_reqs
1069
)
1033
- self.gauge_scheduler_waiting[engine_idx].set(
1070
+ total_waiting = (
1071
scheduler_stats.num_waiting_reqs
1072
+ + scheduler_stats.num_skipped_waiting_reqs
1073
+ )
1074
+ self.gauge_scheduler_waiting[engine_idx].set(total_waiting)
1075
+ self.gauge_waiting_by_reason[WAITING_REASON_CAPACITY][engine_idx].set(
1076
+ scheduler_stats.num_waiting_reqs
1077
+ )
1078
+ self.gauge_waiting_by_reason[WAITING_REASON_DEFERRED][engine_idx].set(
1079
+ scheduler_stats.num_skipped_waiting_reqs
1080
)
1081
self.gauge_kv_cache_usage[engine_idx].set(scheduler_stats.kv_cache_usage)
1082
vllm/v1/metrics/stats.py
+3
-1
@@ -172,7 +172,9 @@ class SchedulerStats:
172
"""Stats associated with the scheduler."""
173
174
num_running_reqs: int = 0
175
- num_waiting_reqs: int = 0
175
+
176
+ num_waiting_reqs: int = 0 # length of the "waiting" request queue
177
+ num_skipped_waiting_reqs: int = 0 # length of the "skipped waiting" queue
178
179
# These are used for internal DP load-balancing.
180
step_counter: int = 0