[Refactor] Move iteration logging to the frontend (#46647)
Signed-off-by: maxyanghu <hyoung2991@gmail.com> Co-authored-by: Roger Wang <hey@rogerw.io> Co-authored-by: Shang Wang <shangw@nvidia.com>
Max Hu committed
Jul 15, 2026 at 20:59 UTC
5a65ba5f179887ec331bb96b1fc3cd1fb045e988
9 files changed
+398
-40
tests/v1/core/test_scheduler.py
+82
@@ -43,6 +43,65 @@ from .utils import EOS_TOKEN_ID, create_requests, create_scheduler, mock_kv
43
pytestmark = pytest.mark.cpu_test
44
45
46
+def test_make_scheduled_encoder_input_stats_output_embeddings():
47
+ scheduler = create_scheduler()
48
+ mm_features = [
49
+ MultiModalFeatureSpec(
50
+ data=MultiModalKwargsItem.dummy(),
51
+ modality="image",
52
+ identifier="image-0",
53
+ mm_position=PlaceholderRange(offset=0, length=196),
54
+ ),
55
+ MultiModalFeatureSpec(
56
+ data=MultiModalKwargsItem.dummy(),
57
+ modality="video",
58
+ identifier="video-0",
59
+ mm_position=PlaceholderRange(offset=200, length=196),
60
+ ),
61
+ MultiModalFeatureSpec(
62
+ data=MultiModalKwargsItem.dummy(),
63
+ modality="audio",
64
+ identifier="audio-0",
65
+ mm_position=PlaceholderRange(offset=400, length=49),
66
+ ),
67
+ ]
68
+ scheduler.requests["req"] = Mock(mm_features=mm_features)
69
+
70
+ stats = scheduler._make_scheduled_encoder_input_stats({"req": [0, 1, 2]})
71
+
72
+ assert stats is not None
73
+ assert stats.num_inputs == 3
74
+ assert stats.output_tokens == 441
75
+
76
+
77
+def test_scheduled_encoder_input_stats_disabled_without_iteration_logging(
78
+ monkeypatch: pytest.MonkeyPatch,
79
+):
80
+ scheduler = create_scheduler()
81
+ make_stats = Mock(side_effect=AssertionError("stats should not be computed"))
82
+ monkeypatch.setattr(scheduler, "_make_scheduled_encoder_input_stats", make_stats)
83
+
84
+ scheduler_output = scheduler.schedule()
85
+
86
+ make_stats.assert_not_called()
87
+ assert scheduler_output.scheduled_encoder_input_stats is None
88
+
89
+
90
+def test_scheduled_encoder_input_stats_disabled_without_log_stats(
91
+ monkeypatch: pytest.MonkeyPatch,
92
+):
93
+ scheduler = create_scheduler()
94
+ scheduler.log_stats = False
95
+ scheduler.observability_config.enable_logging_iteration_details = True
96
+ make_stats = Mock(side_effect=AssertionError("stats should not be computed"))
97
+ monkeypatch.setattr(scheduler, "_make_scheduled_encoder_input_stats", make_stats)
98
+
99
+ scheduler_output = scheduler.schedule()
100
+
101
+ make_stats.assert_not_called()
102
+ assert scheduler_output.scheduled_encoder_input_stats is None
103
+
104
+
105
def test_add_requests():
106
scheduler = create_scheduler()
107
requests = create_requests(num_requests=10)
@@ -108,6 +167,29 @@ def test_schedule(enable_prefix_caching: bool, prompt_logprobs: int | None):
167
assert scheduler.running[i] == request
168
169
170
+def test_scheduler_stats_route_to_existing_output_client():
171
+ scheduler = create_scheduler()
172
+ request = create_requests(num_requests=1)[0]
173
+ request.client_index = 1
174
+ scheduler.add_request(request)
175
+
176
+ scheduler_output = scheduler.schedule()
177
+ model_output = ModelRunnerOutput(
178
+ req_ids=[request.request_id],
179
+ req_id_to_index={request.request_id: 0},
180
+ sampled_token_ids=[[1000]],
181
+ logprobs=None,
182
+ prompt_logprobs_dict={},
183
+ pooler_output=[],
184
+ )
185
+
186
+ engine_core_outputs = scheduler.update_from_output(scheduler_output, model_output)
187
+
188
+ assert 0 not in engine_core_outputs
189
+ assert engine_core_outputs[1].scheduler_stats is not None
190
+ assert len(engine_core_outputs[1].outputs) == 1
191
+
192
+
193
def test_schedule_multimodal_requests():
194
scheduler = create_scheduler(model="llava-hf/llava-1.5-7b-hf")
195
mm_positions = [[PlaceholderRange(offset=i, length=100)] for i in range(10)]
tests/v1/engine/test_iteration_logging.py
new
+88
@@ -0,0 +1,88 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+
4
+import time
5
+from types import SimpleNamespace
6
+
7
+from vllm.v1.engine import EngineCoreOutputs
8
+from vllm.v1.engine.core import EngineCore
9
+from vllm.v1.metrics.stats import SchedulerIterationDetails, SchedulerStats
10
+
11
+
12
+class FakeEngineCore:
13
+ def _make_iteration_details_stats(
14
+ self, iteration_details: SchedulerIterationDetails
15
+ ) -> SchedulerStats:
16
+ return SchedulerStats(iteration_details=iteration_details)
17
+
18
+
19
+def make_iteration_details() -> SchedulerIterationDetails:
20
+ return SchedulerIterationDetails(
21
+ iteration_index=1,
22
+ num_ctx_requests=2,
23
+ num_ctx_tokens=3,
24
+ num_generation_requests=4,
25
+ num_generation_tokens=5,
26
+ elapsed_ms=6.7,
27
+ )
28
+
29
+
30
+def make_fake_engine(log_stats: bool = True) -> SimpleNamespace:
31
+ return SimpleNamespace(
32
+ log_stats=log_stats,
33
+ vllm_config=SimpleNamespace(
34
+ observability_config=SimpleNamespace(
35
+ enable_logging_iteration_details=True,
36
+ )
37
+ ),
38
+ )
39
+
40
+
41
+def test_capture_iteration_details_disabled_without_log_stats():
42
+ engine = make_fake_engine(log_stats=False)
43
+
44
+ with EngineCore.capture_iteration_details(engine, None) as iteration_details:
45
+ assert iteration_details is None
46
+
47
+ assert not hasattr(engine, "_iteration_index")
48
+
49
+
50
+def test_capture_iteration_details_fills_elapsed_time():
51
+ engine = make_fake_engine()
52
+
53
+ with EngineCore.capture_iteration_details(engine, None) as iteration_details:
54
+ assert iteration_details is not None
55
+ assert iteration_details.elapsed_ms == 0.0
56
+ assert iteration_details.is_dummy
57
+ time.sleep(0.001)
58
+
59
+ assert iteration_details is not None
60
+ assert iteration_details.elapsed_ms > 0.0
61
+ assert engine._iteration_index == 1
62
+
63
+
64
+def test_attach_iteration_details_uses_existing_output():
65
+ iteration_details = make_iteration_details()
66
+ outputs = {
67
+ 2: EngineCoreOutputs(scheduler_stats=SchedulerStats()),
68
+ 1: EngineCoreOutputs(scheduler_stats=SchedulerStats()),
69
+ }
70
+
71
+ EngineCore._attach_iteration_details(FakeEngineCore(), outputs, iteration_details)
72
+
73
+ assert 0 not in outputs
74
+ assert outputs[2].scheduler_stats is not None
75
+ assert outputs[2].scheduler_stats.iteration_details == iteration_details
76
+ assert outputs[1].scheduler_stats is not None
77
+ assert outputs[1].scheduler_stats.iteration_details is None
78
+
79
+
80
+def test_attach_iteration_details_falls_back_to_client_zero_without_outputs():
81
+ iteration_details = make_iteration_details()
82
+ outputs: dict[int, EngineCoreOutputs] = {}
83
+
84
+ EngineCore._attach_iteration_details(FakeEngineCore(), outputs, iteration_details)
85
+
86
+ assert set(outputs) == {0}
87
+ assert outputs[0].scheduler_stats is not None
88
+ assert outputs[0].scheduler_stats.iteration_details == iteration_details
tests/v1/metrics/test_stats.py
+45
-1
@@ -1,12 +1,17 @@
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
-from vllm.v1.engine import FinishReason
3
+from vllm.v1.core.sched.output import ScheduledEncoderInputStats, SchedulerOutput
4
+from vllm.v1.engine import EngineCoreOutputs, FinishReason
5
from vllm.v1.metrics.stats import (
6
IterationStats,
7
PrefillStats,
8
PromptTokenStats,
9
RequestStateStats,
10
+ SchedulerIterationDetails,
11
+ SchedulerStats,
12
)
13
+from vllm.v1.serial_utils import MsgpackDecoder, MsgpackEncoder
14
+from vllm.v1.utils import compute_iteration_details
15
16
17
def test_iteration_stats_repr():
@@ -14,6 +19,45 @@ def test_iteration_stats_repr():
19
assert repr(iteration_stats).startswith("IterationStats(")
20
21
22
+def test_scheduler_iteration_details_serialization():
23
+ iteration_details = SchedulerIterationDetails(
24
+ iteration_index=1,
25
+ num_ctx_requests=2,
26
+ num_ctx_tokens=3,
27
+ num_generation_requests=4,
28
+ num_generation_tokens=5,
29
+ elapsed_ms=6.7,
30
+ num_encoder_inputs=2,
31
+ num_encoder_output_tokens=392,
32
+ )
33
+ outputs = EngineCoreOutputs(
34
+ scheduler_stats=SchedulerStats(
35
+ kv_cache_usage=0.5,
36
+ iteration_details=iteration_details,
37
+ )
38
+ )
39
+
40
+ encoded = MsgpackEncoder().encode(outputs)
41
+ decoded = MsgpackDecoder(EngineCoreOutputs).decode(encoded)
42
+
43
+ assert decoded.scheduler_stats is not None
44
+ assert decoded.scheduler_stats.kv_cache_usage == 0.5
45
+ assert decoded.scheduler_stats.iteration_details == iteration_details
46
+
47
+
48
+def test_compute_iteration_details_includes_encoder_stats():
49
+ scheduler_output = SchedulerOutput.make_empty()
50
+ scheduler_output.scheduled_encoder_input_stats = ScheduledEncoderInputStats(
51
+ num_inputs=2,
52
+ output_tokens=392,
53
+ )
54
+
55
+ iteration_details = compute_iteration_details(scheduler_output)
56
+
57
+ assert iteration_details.num_encoder_inputs == 2
58
+ assert iteration_details.num_encoder_output_tokens == 392
59
+
60
+
61
def test_prefill_kv_computed_with_cache():
62
"""Test that prefill KV compute correctly excludes cached tokens."""
63
iteration_stats = IterationStats()
vllm/v1/core/sched/output.py
+10
@@ -179,6 +179,14 @@ class CachedRequestData:
179
)
180
181
182
+@dataclass
183
+class ScheduledEncoderInputStats:
184
+ """Stats for encoder inputs scheduled in one iteration."""
185
+
186
+ num_inputs: int = 0
187
+ output_tokens: int = 0
188
+
189
+
190
@dataclass
191
class SchedulerOutput:
192
# list of the requests that are scheduled for the first time.
@@ -216,6 +224,8 @@ class SchedulerOutput:
224
# freed from the encoder cache.
225
free_encoder_mm_hashes: list[str]
226
227
+ scheduled_encoder_input_stats: ScheduledEncoderInputStats | None = None
228
+
229
# Request IDs that are preempted in this step.
230
# Only used for v2 model runner.
231
preempted_req_ids: set[str] | None = None
vllm/v1/core/sched/scheduler.py
+32
-1
@@ -44,6 +44,7 @@ from vllm.v1.core.sched.output import (
44
CachedRequestData,
45
GrammarOutput,
46
NewRequestData,
47
+ ScheduledEncoderInputStats,
48
SchedulerOutput,
49
)
50
from vllm.v1.core.sched.request_queue import (
@@ -1121,6 +1122,15 @@ class Scheduler(SchedulerInterface):
1122
len(num_scheduled_tokens)
1123
]
1124
1125
+ scheduled_encoder_input_stats = None
1126
+ if (
1127
+ self.log_stats
1128
+ and self.observability_config.enable_logging_iteration_details
1129
+ ):
1130
+ scheduled_encoder_input_stats = self._make_scheduled_encoder_input_stats(
1131
+ scheduled_encoder_inputs
1132
+ )
1133
+
1134
scheduler_output = SchedulerOutput(
1135
scheduled_new_reqs=new_reqs_data,
1136
scheduled_cached_reqs=cached_reqs_data,
@@ -1128,6 +1138,7 @@ class Scheduler(SchedulerInterface):
1138
total_num_scheduled_tokens=total_num_scheduled_tokens,
1139
scheduled_spec_decode_tokens=scheduled_spec_decode_tokens,
1140
scheduled_encoder_inputs=scheduled_encoder_inputs,
1141
+ scheduled_encoder_input_stats=scheduled_encoder_input_stats,
1142
num_common_prefix_blocks=num_common_prefix_blocks,
1143
preempted_req_ids=self.reset_preempted_req_ids,
1144
# finished_req_ids is an existing state in the scheduler,
@@ -1506,6 +1517,23 @@ class Scheduler(SchedulerInterface):
1517
external_load_encoder_input,
1518
)
1519
1520
+ def _make_scheduled_encoder_input_stats(
1521
+ self, scheduled_encoder_inputs: dict[str, list[int]]
1522
+ ) -> ScheduledEncoderInputStats | None:
1523
+ stats = ScheduledEncoderInputStats()
1524
+
1525
+ for req_id, input_ids in scheduled_encoder_inputs.items():
1526
+ request = self.requests.get(req_id)
1527
+ if request is None:
1528
+ continue
1529
+
1530
+ for input_id in input_ids:
1531
+ mm_feature = request.mm_features[input_id]
1532
+ stats.num_inputs += 1
1533
+ stats.output_tokens += mm_feature.mm_position.get_num_embeds()
1534
+
1535
+ return stats if stats.num_inputs else None
1536
+
1537
def get_grammar_bitmask(
1538
self, scheduler_output: SchedulerOutput
1539
) -> GrammarOutput | None:
@@ -1877,7 +1905,10 @@ class Scheduler(SchedulerInterface):
1905
1906
if (
1907
stats := self.make_stats(
1880
- spec_decoding_stats, kv_connector_stats, cudagraph_stats, perf_stats
1908
+ spec_decoding_stats,
1909
+ kv_connector_stats,
1910
+ cudagraph_stats,
1911
+ perf_stats,
1912
)
1913
) is not None:
1914
# Return stats to only one of the front-ends.
vllm/v1/engine/core.py
+74
-37
@@ -79,16 +79,17 @@ from vllm.v1.engine.utils import (
79
)
80
from vllm.v1.executor import Executor
81
from vllm.v1.kv_cache_interface import KVCacheConfig, get_kv_cache_spec_kind
82
-from vllm.v1.metrics.stats import SchedulerStats
82
+from vllm.v1.metrics.stats import SchedulerIterationDetails, SchedulerStats
83
from vllm.v1.outputs import ModelRunnerOutput
84
from vllm.v1.request import Request, RequestStatus
85
from vllm.v1.serial_utils import MsgpackDecoder, MsgpackEncoder
86
from vllm.v1.structured_output import StructuredOutputManager
87
-from vllm.v1.utils import IterationDetails, compute_iteration_details
87
+from vllm.v1.utils import compute_iteration_details
88
from vllm.version import __version__ as VLLM_VERSION
89
90
logger = init_logger(__name__)
91
92
+
93
HANDSHAKE_TIMEOUT_MINS = 5
94
95
_R = TypeVar("_R") # Return type for collective_rpc
@@ -498,45 +499,74 @@ class EngineCore:
499
raise err
500
501
@contextmanager
501
- def log_iteration_details(self, scheduler_output: SchedulerOutput | None):
502
- if not self.vllm_config.observability_config.enable_logging_iteration_details:
503
- yield
502
+ def capture_iteration_details(
503
+ self, scheduler_output: SchedulerOutput | None
504
+ ) -> Generator[SchedulerIterationDetails | None, None, None]:
505
+ enable_details = (
506
+ self.vllm_config.observability_config.enable_logging_iteration_details
507
+ )
508
+ if not self.log_stats or not enable_details:
509
+ yield None
510
return
511
# 0-token step: let the dummy_batch wrapper log it (avoids double-log).
506
- if scheduler_output and scheduler_output.total_num_scheduled_tokens == 0:
507
- yield
512
+ if (
513
+ scheduler_output is not None
514
+ and scheduler_output.total_num_scheduled_tokens == 0
515
+ ):
516
+ yield None
517
return
509
- self._iteration_index = getattr(self, "_iteration_index", 0)
518
+
519
+ iteration_index = getattr(self, "_iteration_index", 0)
520
# scheduler_output=None marks a DP dummy iteration.
521
if scheduler_output is None:
512
- iteration_details = IterationDetails(0, 0, 0, 0)
513
- is_dummy = True
522
+ iteration_details = SchedulerIterationDetails(
523
+ iteration_index=iteration_index,
524
+ num_ctx_requests=0,
525
+ num_ctx_tokens=0,
526
+ num_generation_requests=0,
527
+ num_generation_tokens=0,
528
+ elapsed_ms=0.0,
529
+ is_dummy=True,
530
+ )
531
else:
515
- iteration_details = compute_iteration_details(scheduler_output)
516
- is_dummy = False
517
- before = time.monotonic()
518
- yield
519
- logger.info(
520
- "".join(
521
- [
522
- "Iteration(",
523
- str(self._iteration_index),
524
- "): ",
525
- str(iteration_details.num_ctx_requests),
526
- " context requests, ",
527
- str(iteration_details.num_ctx_tokens),
528
- " context tokens, ",
529
- str(iteration_details.num_generation_requests),
530
- " generation requests, ",
531
- str(iteration_details.num_generation_tokens),
532
- " generation tokens, iteration elapsed time: ",
533
- format((time.monotonic() - before) * 1000, ".2f"),
534
- " ms",
535
- " (dummy)" if is_dummy else "",
536
- ]
532
+ details = compute_iteration_details(scheduler_output)
533
+ iteration_details = SchedulerIterationDetails(
534
+ iteration_index=iteration_index,
535
+ num_ctx_requests=details.num_ctx_requests,
536
+ num_ctx_tokens=details.num_ctx_tokens,
537
+ num_generation_requests=details.num_generation_requests,
538
+ num_generation_tokens=details.num_generation_tokens,
539
+ elapsed_ms=0.0,
540
+ num_encoder_inputs=details.num_encoder_inputs,
541
+ num_encoder_output_tokens=details.num_encoder_output_tokens,
542
)
538
- )
539
- self._iteration_index += 1
543
+
544
+ start_time = time.monotonic()
545
+ yield iteration_details
546
+ iteration_details.elapsed_ms = (time.monotonic() - start_time) * 1000
547
+ self._iteration_index = iteration_index + 1
548
+
549
+ def _make_iteration_details_stats(
550
+ self, iteration_details: SchedulerIterationDetails
551
+ ) -> SchedulerStats:
552
+ stats = self.scheduler.make_stats() or SchedulerStats()
553
+ stats.iteration_details = iteration_details
554
+ return stats
555
+
556
+ def _attach_iteration_details(
557
+ self,
558
+ outputs: dict[int, EngineCoreOutputs],
559
+ iteration_details: SchedulerIterationDetails | None,
560
+ ) -> None:
561
+ if iteration_details is None:
562
+ return
563
+
564
+ if (eco := next(iter(outputs.values()), None)) is None:
565
+ outputs[0] = eco = EngineCoreOutputs()
566
+ if eco.scheduler_stats is None:
567
+ eco.scheduler_stats = self._make_iteration_details_stats(iteration_details)
568
+ else:
569
+ eco.scheduler_stats.iteration_details = iteration_details
570
571
def _should_throttle_prefills(self) -> bool:
572
"""Whether to defer new prefills this step (DP prefill balancing).
@@ -558,8 +588,8 @@ class EngineCore:
588
future = self.model_executor.execute_model(scheduler_output, non_block=True)
589
grammar_output = self.scheduler.get_grammar_bitmask(scheduler_output)
590
with (
591
+ self.capture_iteration_details(scheduler_output) as iteration_details,
592
self.log_error_detail(scheduler_output),
562
- self.log_iteration_details(scheduler_output),
593
):
594
model_output = future.result()
595
if model_output is None:
@@ -571,6 +601,7 @@ class EngineCore:
601
engine_core_outputs = self.scheduler.update_from_output(
602
scheduler_output, model_output
603
)
604
+ self._attach_iteration_details(engine_core_outputs, iteration_details)
605
606
return engine_core_outputs, scheduler_output.total_num_scheduled_tokens > 0
607
@@ -656,8 +687,8 @@ class EngineCore:
687
# Block until the next result is available.
688
future, scheduler_output, exec_model_fut = batch_queue.pop()
689
with (
690
+ self.capture_iteration_details(scheduler_output) as iteration_details,
691
self.log_error_detail(scheduler_output),
660
- self.log_iteration_details(scheduler_output),
692
):
693
model_output = future.result()
694
if model_output is None:
@@ -672,6 +703,7 @@ class EngineCore:
703
engine_core_outputs = self.scheduler.update_from_output(
704
scheduler_output, model_output
705
)
706
+ self._attach_iteration_details(engine_core_outputs, iteration_details)
707
708
# NOTE(nick): We can either handle the deferred tasks here or save
709
# in a field and do it immediately once step_with_batch_queue is
@@ -2019,8 +2051,13 @@ class DPEngineCoreProc(EngineCoreProc):
2051
# Execute a dummy pass when no ready requests ran, unless the
2052
# engine is sleeping.
2053
elif not self.model_executor.is_sleeping:
2022
- with self.log_iteration_details(None):
2054
+ with self.capture_iteration_details(None) as iteration_details:
2055
self.execute_dummy_batch()
2056
+ if iteration_details is not None and not self.has_coordinator:
2057
+ stats = self._make_iteration_details_stats(iteration_details)
2058
+ self.output_queue.put_nowait(
2059
+ (0, EngineCoreOutputs(scheduler_stats=stats))
2060
+ )
2061
2062
# 3) All-reduce operation to determine global unfinished reqs.
2063
self.engines_running = self._has_global_unfinished_reqs(
vllm/v1/metrics/loggers.py
+37
@@ -160,6 +160,42 @@ class LoggingStatLogger(StatLoggerBase):
160
def log_prefix(self):
161
return "Engine {:03d}: ".format(self.engine_index)
162
163
+ def _log_prefix_for_engine(self, engine_idx: int) -> str:
164
+ if self.engine_index == engine_idx:
165
+ return self.log_prefix
166
+ return "Engine {:03d}: ".format(engine_idx)
167
+
168
+ def _log_iteration_details(
169
+ self, scheduler_stats: SchedulerStats, engine_idx: int
170
+ ) -> None:
171
+ details = scheduler_stats.iteration_details
172
+ if details is None:
173
+ return
174
+
175
+ encoder_msg = ""
176
+ if details.num_encoder_inputs:
177
+ encoder_msg = (
178
+ f", encoder inputs: {details.num_encoder_inputs}, "
179
+ f"encoder output embeddings: {details.num_encoder_output_tokens}"
180
+ )
181
+
182
+ logger.info(
183
+ "%sIteration(%d): %d context requests, %d context tokens, "
184
+ "%d generation requests, %d generation tokens, "
185
+ "iteration elapsed time: %.2f ms%s, "
186
+ "GPU KV cache usage: %.1f%%%s",
187
+ self._log_prefix_for_engine(engine_idx),
188
+ details.iteration_index,
189
+ details.num_ctx_requests,
190
+ details.num_ctx_tokens,
191
+ details.num_generation_requests,
192
+ details.num_generation_tokens,
193
+ details.elapsed_ms,
194
+ " (dummy)" if details.is_dummy else "",
195
+ scheduler_stats.kv_cache_usage * 100,
196
+ encoder_msg,
197
+ )
198
+
199
def record(
200
self,
201
scheduler_stats: SchedulerStats | None,
@@ -172,6 +208,7 @@ class LoggingStatLogger(StatLoggerBase):
208
self._track_iteration_stats(iteration_stats)
209
210
if scheduler_stats is not None:
211
+ self._log_iteration_details(scheduler_stats, engine_idx)
212
self.prefix_caching_metrics.observe(scheduler_stats.prefix_cache_stats)
213
214
if scheduler_stats.connector_prefix_cache_stats is not None:
vllm/v1/metrics/stats.py
+16
@@ -167,6 +167,21 @@ class KVCacheEvictionEvent:
167
reuse_gaps_seconds: tuple[float, ...]
168
169
170
+@dataclass
171
+class SchedulerIterationDetails:
172
+ """Scheduler-side details for one engine iteration."""
173
+
174
+ iteration_index: int
175
+ num_ctx_requests: int
176
+ num_ctx_tokens: int
177
+ num_generation_requests: int
178
+ num_generation_tokens: int
179
+ elapsed_ms: float
180
+ num_encoder_inputs: int = 0
181
+ num_encoder_output_tokens: int = 0
182
+ is_dummy: bool = False
183
+
184
+
185
@dataclass
186
class SchedulerStats:
187
"""Stats associated with the scheduler."""
@@ -181,6 +196,7 @@ class SchedulerStats:
196
current_wave: int = 0
197
198
kv_cache_usage: float = 0.0
199
+ iteration_details: SchedulerIterationDetails | None = None
200
201
prefix_cache_stats: PrefixCacheStats = field(default_factory=PrefixCacheStats)
202
connector_prefix_cache_stats: PrefixCacheStats | None = None
vllm/v1/utils.py
+14
-1
@@ -782,12 +782,16 @@ class IterationDetails:
782
num_ctx_tokens: int
783
num_generation_requests: int
784
num_generation_tokens: int
785
+ num_encoder_inputs: int = 0
786
+ num_encoder_output_tokens: int = 0
787
788
def __repr__(self) -> str:
789
return f"IterationDetails(num_ctx_requests={self.num_ctx_requests},\
790
num_ctx_tokens={self.num_ctx_tokens}, \
791
num_generation_requests={self.num_generation_requests}, \
790
- num_generation_tokens={self.num_generation_tokens})"
792
+ num_generation_tokens={self.num_generation_tokens}, \
793
+ num_encoder_inputs={self.num_encoder_inputs}, \
794
+ num_encoder_output_tokens={self.num_encoder_output_tokens})"
795
796
797
def compute_iteration_details(scheduler_output: SchedulerOutput) -> IterationDetails:
@@ -818,9 +822,18 @@ def compute_iteration_details(scheduler_output: SchedulerOutput) -> IterationDet
822
else:
823
num_generation_requests += 1
824
num_generation_tokens += num_tokens
825
+ scheduled_encoder_input_stats = scheduler_output.scheduled_encoder_input_stats
826
+ num_encoder_inputs = 0
827
+ num_encoder_output_tokens = 0
828
+ if scheduled_encoder_input_stats is not None:
829
+ num_encoder_inputs = scheduled_encoder_input_stats.num_inputs
830
+ num_encoder_output_tokens = scheduled_encoder_input_stats.output_tokens
831
+
832
return IterationDetails(
833
num_context_requests,
834
num_context_tokens,
835
num_generation_requests,
836
num_generation_tokens,
837
+ num_encoder_inputs,
838
+ num_encoder_output_tokens,
839
)