A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 4, 21:15 UTC
[Bugfix] avoid warmup if text only expectation in multi_modal run (#40409)
Signed-off-by: khushali9 <khushali.desai9@gmail.com>
Khushali Desai committed
Apr 21, 2026 at 17:19 UTC
6ff8dea0756d32f2487b8981ec2626d0402ef0ba
2 files changed
+114
-1
tests/renderers/test_warmup.py
new
+111
@@ -0,0 +1,111 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+"""Unit tests for BaseRenderer.warmup MM-warmup behavior.
4
+
5
+These tests exercise:
6
+ - Zero-limit modalities are filtered from mm_counts passed to
7
+ get_dummy_processor_inputs (e.g. --limit-mm-per-prompt image=0 ...)
8
+ - MM warmup is skipped entirely when mm_processor is None
9
+
10
+No model weights are required: warmup() is called directly on a MagicMock
11
+that acts as the renderer instance.
12
+"""
13
+
14
+from unittest.mock import MagicMock, patch
15
+
16
+from vllm.renderers.base import BaseRenderer
17
+from vllm.renderers.params import ChatParams
18
+
19
+
20
+def _make_renderer_mock(mm_limits: dict[str, int]) -> MagicMock:
21
+ """Return a MagicMock that quacks like a BaseRenderer instance.
22
+
23
+ render_chat is mocked to raise ChatTemplateResolutionError so the chat
24
+ warmup block is skipped cleanly, keeping the test focused on MM warmup.
25
+ """
26
+ from vllm.entrypoints.chat_utils import ChatTemplateResolutionError
27
+
28
+ renderer = MagicMock()
29
+
30
+ # chat warmup: make render_chat raise so we skip past it cleanly
31
+ renderer.render_chat.side_effect = ChatTemplateResolutionError("no template")
32
+
33
+ # MM processor with configurable limits
34
+ mm_processor = MagicMock()
35
+ mm_processor.info.allowed_mm_limits = mm_limits
36
+ renderer.mm_processor = mm_processor
37
+
38
+ return renderer
39
+
40
+
41
+class TestMmWarmupZeroLimitFiltering:
42
+ """Zero-limit modalities must be excluded from mm_counts."""
43
+
44
+ def test_zero_limit_modality_excluded_from_mm_counts(self):
45
+ """A modality with limit=0 must not appear in mm_counts."""
46
+ renderer = _make_renderer_mock({"image": 1, "video": 0})
47
+
48
+ with patch("vllm.multimodal.processing.TimingContext", autospec=True):
49
+ BaseRenderer.warmup(renderer, ChatParams())
50
+
51
+ get_inputs = renderer.mm_processor.dummy_inputs.get_dummy_processor_inputs
52
+ get_inputs.assert_called_once()
53
+ _, kwargs = get_inputs.call_args
54
+ assert "video" not in kwargs["mm_counts"]
55
+ assert kwargs["mm_counts"]["image"] == 1
56
+
57
+ def test_all_zero_limits_passes_empty_mm_counts(self):
58
+ """When all limits are 0, mm_counts must be empty."""
59
+ renderer = _make_renderer_mock({"image": 0, "video": 0})
60
+
61
+ with patch("vllm.multimodal.processing.TimingContext", autospec=True):
62
+ BaseRenderer.warmup(renderer, ChatParams())
63
+
64
+ get_inputs = renderer.mm_processor.dummy_inputs.get_dummy_processor_inputs
65
+ get_inputs.assert_called_once()
66
+ _, kwargs = get_inputs.call_args
67
+ assert kwargs["mm_counts"] == {}
68
+
69
+ def test_positive_limits_all_included_in_mm_counts(self):
70
+ """All modalities with limit > 0 must be present in mm_counts."""
71
+ renderer = _make_renderer_mock({"image": 2, "video": 1})
72
+
73
+ with patch("vllm.multimodal.processing.TimingContext", autospec=True):
74
+ BaseRenderer.warmup(renderer, ChatParams())
75
+
76
+ get_inputs = renderer.mm_processor.dummy_inputs.get_dummy_processor_inputs
77
+ get_inputs.assert_called_once()
78
+ _, kwargs = get_inputs.call_args
79
+ assert kwargs["mm_counts"] == {"image": 1, "video": 1}
80
+
81
+
82
+class TestMmWarmupRunsNormally:
83
+ """MM warmup must run when mm_processor is set and limits > 0."""
84
+
85
+ def test_processor_apply_called(self):
86
+ renderer = _make_renderer_mock({"image": 1})
87
+
88
+ with patch("vllm.multimodal.processing.TimingContext", autospec=True):
89
+ BaseRenderer.warmup(renderer, ChatParams())
90
+
91
+ renderer.mm_processor.apply.assert_called_once()
92
+
93
+ def test_mm_cache_cleared_after_warmup(self):
94
+ renderer = _make_renderer_mock({"image": 1})
95
+
96
+ with patch("vllm.multimodal.processing.TimingContext", autospec=True):
97
+ BaseRenderer.warmup(renderer, ChatParams())
98
+
99
+ renderer.clear_mm_cache.assert_called_once()
100
+
101
+
102
+class TestMmWarmupSkippedWhenNoProcessor:
103
+ """MM warmup must be skipped when mm_processor is None (text-only model)."""
104
+
105
+ def test_no_warmup_without_processor(self):
106
+ renderer = _make_renderer_mock({})
107
+ renderer.mm_processor = None # override to None
108
+
109
+ BaseRenderer.warmup(renderer, ChatParams())
110
+
111
+ renderer.model_config.get_multimodal_config.assert_not_called()
vllm/renderers/base.py
+3
-1
@@ -226,7 +226,9 @@ class BaseRenderer(ABC, Generic[_T]):
226
model_config = self.model_config
227
mm_config = model_config.get_multimodal_config()
228
processor = self.mm_processor
229
- mm_limits = processor.info.allowed_mm_limits
229
+ mm_limits = {
230
+ k: v for k, v in processor.info.allowed_mm_limits.items() if v > 0
231
+ }
232
233
try:
234
logger.debug("Warming up multi-modal processing...")