[Bugfix][Multimodal] Bound renderer warmup to the prefill token budget (#55448)
Signed-off-by: Luca Motz <luca.motz@icloud.com> Co-authored-by: OpenAI Codex <codex@openai.com>
lucamotz committed
Sep 5, 2026 at 18:43 UTC
f4eccdadefc6501fafeb1a0bf7f171ff24f984b0
4 files changed
+59
-8
tests/multimodal/test_registry.py
+30
@@ -6,9 +6,11 @@ Qwen2.5-VL visual component loading behavior.
6
"""
7
8
from types import SimpleNamespace
9
+from unittest.mock import MagicMock
10
11
import pytest
12
13
+from vllm.config import SchedulerConfig
14
from vllm.multimodal import MULTIMODAL_REGISTRY
15
16
from ..models.utils import build_model_context
@@ -16,6 +18,34 @@ from ..models.utils import build_model_context
18
pytestmark = pytest.mark.cpu_test
19
20
21
+@pytest.mark.parametrize(
22
+ ("chunked_prefill", "max_model_len", "expected_seq_len"),
23
+ [(None, 491520, 491520), (True, 491520, 8192), (True, 128, 128), (False, 128, 128)],
24
+)
25
+def test_dummy_inputs_scheduler_budget(
26
+ chunked_prefill, max_model_len, expected_seq_len
27
+):
28
+ model_config = MagicMock()
29
+ model_config.max_model_len = max_model_len
30
+ processor = MagicMock()
31
+ processor.apply.return_value = {"prompt_token_ids": [7]}
32
+ kwargs = {}
33
+ if chunked_prefill is not None:
34
+ kwargs["scheduler_config"] = SchedulerConfig(
35
+ max_model_len=max_model_len,
36
+ is_encoder_decoder=False,
37
+ max_num_batched_tokens=8192,
38
+ max_num_seqs=1,
39
+ enable_chunked_prefill=chunked_prefill,
40
+ )
41
+ result = MULTIMODAL_REGISTRY.get_dummy_mm_inputs(
42
+ model_config, {"image": 1}, processor=processor, **kwargs
43
+ )
44
+ get_inputs = processor.dummy_inputs.get_dummy_processor_inputs
45
+ assert get_inputs.call_args.kwargs["seq_len"] == expected_seq_len
46
+ assert len(result["prompt_token_ids"]) == expected_seq_len
47
+
48
+
49
@pytest.mark.parametrize(
50
"model_id,limit_mm_per_prompt,expected",
51
[
tests/renderers/test_warmup.py
+16
@@ -45,6 +45,7 @@ def _make_renderer_mock(mm_limits: dict[str, int]) -> MagicMock:
45
# MM processor with configurable limits
46
mm_processor = MagicMock()
47
mm_processor.info.allowed_mm_limits = mm_limits
48
+ mm_processor.apply.return_value = {"prompt_token_ids": [1]}
49
renderer.mm_processor = mm_processor
50
renderer._readonly_mm_processor = None
51
renderer._warmup_mm_processor = BaseRenderer._warmup_mm_processor.__get__(
@@ -65,6 +66,8 @@ def _make_renderer_mock(mm_limits: dict[str, int]) -> MagicMock:
66
renderer._mm_warmup_done = False
67
renderer.clear_mm_cache = MagicMock()
68
renderer.model_config.max_model_len = 128
69
+ renderer.config.scheduler_config.max_num_batched_tokens = 8192
70
+ renderer.config.scheduler_config.enable_chunked_prefill = True
71
renderer.model_config.get_multimodal_config.return_value.limit_per_prompt = {}
72
73
return renderer
@@ -115,6 +118,19 @@ class TestMmWarmupRunsNormally:
118
# MM warmup must run when mm_processor is set and limits > 0; the chat
119
# template warmup must run alongside it.
120
121
+ @pytest.mark.parametrize(
122
+ ("max_model_len", "expected_seq_len"), [(491520, 8192), (128, 128)]
123
+ )
124
+ def test_warmup_sequence_length_is_bounded(self, max_model_len, expected_seq_len):
125
+ renderer = _make_renderer_mock({"image": 1, "video": 1})
126
+ renderer.model_config.max_model_len = max_model_len
127
+
128
+ with patch("vllm.multimodal.processing.TimingContext", autospec=True):
129
+ BaseRenderer.warmup(renderer, ChatParams())
130
+
131
+ get_inputs = renderer.mm_processor.dummy_inputs.get_dummy_processor_inputs
132
+ assert get_inputs.call_args.kwargs["seq_len"] == expected_seq_len
133
+
134
def test_processor_apply_called(self):
135
renderer = _make_renderer_mock({"image": 1})
136
vllm/multimodal/registry.py
+9
-1
@@ -29,7 +29,12 @@ from .processing import (
29
)
30
31
if TYPE_CHECKING:
32
- from vllm.config import ModelConfig, ObservabilityConfig, VllmConfig
32
+ from vllm.config import (
33
+ ModelConfig,
34
+ ObservabilityConfig,
35
+ SchedulerConfig,
36
+ VllmConfig,
37
+ )
38
from vllm.model_executor.models.interfaces import SupportsMultiModal
39
40
logger = init_logger(__name__)
@@ -243,6 +248,7 @@ class MultiModalRegistry:
248
*,
249
cache: BaseMultiModalProcessorCache | None = None,
250
processor: BaseMultiModalProcessor | None = None,
251
+ scheduler_config: "SchedulerConfig | None" = None,
252
) -> MultiModalInput:
253
"""
254
Create dummy data for profiling the memory usage of a model.
@@ -250,6 +256,8 @@ class MultiModalRegistry:
256
The model is identified by `model_config`.
257
"""
258
seq_len = model_config.max_model_len
259
+ if scheduler_config is not None and scheduler_config.enable_chunked_prefill:
260
+ seq_len = min(seq_len, scheduler_config.max_num_batched_tokens)
261
262
if processor is None:
263
processor = self.create_processor(model_config, cache=cache)
vllm/renderers/base.py
+4
-7
@@ -246,19 +246,16 @@ class BaseRenderer(ABC, Generic[_T]):
246
*,
247
log_prefix: str,
248
) -> None:
249
- from vllm.multimodal.processing import TimingContext
250
-
249
model_config = self.model_config
252
- mm_config = model_config.get_multimodal_config()
250
mm_limits = {k: v for k, v in processor.info.allowed_mm_limits.items() if v > 0}
251
252
start_time = time.perf_counter()
256
- processor_inputs = processor.dummy_inputs.get_dummy_processor_inputs(
257
- seq_len=model_config.max_model_len,
253
+ _ = mm_registry.get_dummy_mm_inputs(
254
+ model_config,
255
mm_counts=dict.fromkeys(mm_limits, 1),
259
- mm_options=mm_config.limit_per_prompt,
256
+ processor=processor,
257
+ scheduler_config=self.config.scheduler_config,
258
)
261
- _ = processor.apply(processor_inputs, timing_ctx=TimingContext(enabled=False))
259
260
elapsed = time.perf_counter() - start_time
261
logger.info("%s warmup completed in %.3fs", log_prefix, elapsed)