[MM] Remove renderer_applies_updates flag (#53385)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
Cyrus Leung committed
Aug 23, 2026 at 01:25 UTC
020829f66baa231a3ed549c7f6b4a3171ef1c759
8 files changed
+80
-86
vllm/model_executor/models/nano_nemotron_vl.py
-1
@@ -734,7 +734,6 @@ class NanoNemotronVLMultiModalProcessor(
734
prompt_ids=prompt_ids,
735
mm_kwargs=mm_info.kwargs,
736
mm_prompt_updates=mm_info.prompt_updates,
737
- is_update_applied=self.renderer_applies_updates,
737
)
738
739
mm_placeholder_ranges = {
vllm/model_executor/models/pixtral.py
+23
-2
@@ -40,6 +40,7 @@ from vllm.model_executor.models.utils import WeightsMapper
40
from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalKwargsItems
41
from vllm.multimodal.inputs import (
42
MultiModalFieldConfig,
43
+ MultiModalKwargsOptionalItems,
44
NestedTensors,
45
)
46
from vllm.multimodal.parse import (
@@ -51,6 +52,8 @@ from vllm.multimodal.processing import BaseDummyInputsBuilder
52
from vllm.multimodal.processing.processor import (
53
BaseMultiModalProcessor,
54
BaseProcessingInfo,
55
+ MultiModalPromptUpdates,
56
+ PlaceholderFeaturesInfo,
57
ProcessorInputs,
58
PromptReplacement,
59
PromptUpdate,
@@ -211,8 +214,26 @@ class PixtralDummyInputsBuilder(BaseDummyInputsBuilder[PixtralProcessingInfo]):
214
215
216
class PixtralMultiModalProcessor(BaseMultiModalProcessor[PixtralProcessingInfo]):
214
- # The tokens are already inserted by the chat template
215
- renderer_applies_updates = True
217
+ # The tokens are already inserted by the chat template,
218
+ # so we just double check that they exist
219
+ def _maybe_apply_prompt_updates(
220
+ self,
221
+ mm_items: MultiModalDataItems,
222
+ prompt_ids: list[int],
223
+ mm_kwargs: MultiModalKwargsOptionalItems,
224
+ mm_prompt_updates: MultiModalPromptUpdates,
225
+ ) -> tuple[list[int], Mapping[str, list[PlaceholderFeaturesInfo]]]:
226
+ mm_item_counts = mm_items.get_all_counts()
227
+ self._validate_mm_kwargs(mm_kwargs, mm_item_counts)
228
+ self._validate_mm_updates(mm_prompt_updates, mm_item_counts)
229
+
230
+ mm_placeholders = self._find_mm_placeholders(
231
+ prompt_ids,
232
+ mm_prompt_updates,
233
+ )
234
+ self._validate_mm_placeholders(mm_placeholders, mm_item_counts)
235
+
236
+ return prompt_ids, mm_placeholders
237
238
def _get_mm_fields_config(
239
self,
vllm/model_executor/models/qwen2_5_omni_thinker.py
+15
-29
@@ -575,7 +575,6 @@ class Qwen2_5OmniThinkerMultiModalProcessor(
575
prompt_ids: list[int],
576
mm_kwargs: MultiModalKwargsItems,
577
mm_prompt_updates: MultiModalPromptUpdates,
578
- is_update_applied: bool,
578
) -> tuple[list[int], Mapping[str, list[PlaceholderFeaturesInfo]]]:
579
"""
580
Qwen2.5-Omni reimplements this function to handle `use_audio_in_video`.
@@ -605,40 +604,27 @@ class Qwen2_5OmniThinkerMultiModalProcessor(
604
if video_audio_item_num != video_updates_num + audio_updates_num:
605
use_audio_in_video = True
606
608
- if is_update_applied:
609
- mm_placeholders = self._find_mm_placeholders(
607
+ if use_audio_in_video and "audio" in mm_prompt_updates:
608
+ # Filter out audio updates - they are embedded in video
609
+ filtered_updates = {
610
+ k: v for k, v in mm_prompt_updates.items() if k != "audio"
611
+ }
612
+ prompt_ids, mm_placeholders = self._apply_prompt_updates(
613
prompt_ids,
611
- mm_prompt_updates,
614
+ filtered_updates,
615
)
613
- self._validate_mm_placeholders(
614
- mm_placeholders,
615
- mm_item_counts,
616
+ # Derive audio placeholders from video placeholders
617
+ mm_placeholders = self._derive_audio_from_video_placeholders(
618
+ mm_placeholders, mm_prompt_updates
619
)
620
else:
618
- if use_audio_in_video and "audio" in mm_prompt_updates:
619
- # Filter out audio updates - they are embedded in video
620
- filtered_updates = {
621
- k: v for k, v in mm_prompt_updates.items() if k != "audio"
622
- }
623
- prompt_ids, mm_placeholders = self._apply_prompt_updates(
624
- prompt_ids,
625
- filtered_updates,
626
- )
627
- # Derive audio placeholders from video placeholders
628
- mm_placeholders = self._derive_audio_from_video_placeholders(
629
- mm_placeholders, mm_prompt_updates
630
- )
631
- else:
632
- prompt_ids, mm_placeholders = self._apply_prompt_updates(
633
- prompt_ids,
634
- mm_prompt_updates,
635
- )
636
-
637
- self._validate_mm_placeholders(
638
- mm_placeholders,
639
- mm_item_counts,
621
+ prompt_ids, mm_placeholders = self._apply_prompt_updates(
622
+ prompt_ids,
623
+ mm_prompt_updates,
624
)
625
626
+ self._validate_mm_placeholders(mm_placeholders, mm_item_counts)
627
+
628
return prompt_ids, mm_placeholders
629
630
@classmethod
vllm/model_executor/models/qwen3_asr_realtime.py
-1
@@ -116,7 +116,6 @@ class Qwen3ASRRealtimeMultiModalProcessor(Qwen3ASRMultiModalProcessor):
116
prompt_ids: list[int],
117
mm_kwargs: MultiModalKwargsOptionalItems,
118
mm_prompt_updates: MultiModalPromptUpdates,
119
- is_update_applied: bool,
119
) -> tuple[list[int], Mapping[str, list[PlaceholderFeaturesInfo]]]:
120
audios = mm_kwargs.get("audio", [])
121
assert len(audios) == 1, (
vllm/model_executor/models/qwen3_omni_moe_thinker.py
+14
-29
@@ -1291,7 +1291,6 @@ class Qwen3OmniMoeThinkerMultiModalProcessor(
1291
prompt_ids: list[int],
1292
mm_kwargs: MultiModalKwargsItems,
1293
mm_prompt_updates: MultiModalPromptUpdates,
1294
- is_update_applied: bool,
1294
) -> tuple[list[int], Mapping[str, list[PlaceholderFeaturesInfo]]]:
1295
"""
1296
Qwen3-Omni reimplements this function to handle `use_audio_in_video`.
@@ -1318,40 +1317,26 @@ class Qwen3OmniMoeThinkerMultiModalProcessor(
1317
if video_audio_item_num != video_updates_num + audio_updates_num:
1318
use_audio_in_video = True
1319
1321
- # normal case with `use_audio_in_video=False`
1322
- if is_update_applied:
1323
- mm_placeholders = self._find_mm_placeholders(
1320
+ if use_audio_in_video and "audio" in mm_prompt_updates:
1321
+ filtered_updates = {
1322
+ k: v for k, v in mm_prompt_updates.items() if k != "audio"
1323
+ }
1324
+ prompt_ids, mm_placeholders = self._apply_prompt_updates(
1325
prompt_ids,
1325
- mm_prompt_updates,
1326
+ filtered_updates,
1327
)
1327
- self._validate_mm_placeholders(
1328
- mm_placeholders,
1329
- mm_item_counts,
1328
+ # Derive audio placeholders from video placeholders
1329
+ mm_placeholders = self._derive_audio_from_video_placeholders(
1330
+ mm_placeholders, mm_prompt_updates
1331
)
1332
else:
1332
- if use_audio_in_video and "audio" in mm_prompt_updates:
1333
- filtered_updates = {
1334
- k: v for k, v in mm_prompt_updates.items() if k != "audio"
1335
- }
1336
- prompt_ids, mm_placeholders = self._apply_prompt_updates(
1337
- prompt_ids,
1338
- filtered_updates,
1339
- )
1340
- # Derive audio placeholders from video placeholders
1341
- mm_placeholders = self._derive_audio_from_video_placeholders(
1342
- mm_placeholders, mm_prompt_updates
1343
- )
1344
- else:
1345
- prompt_ids, mm_placeholders = self._apply_prompt_updates(
1346
- prompt_ids,
1347
- mm_prompt_updates,
1348
- )
1349
-
1350
- self._validate_mm_placeholders(
1351
- mm_placeholders,
1352
- mm_item_counts,
1333
+ prompt_ids, mm_placeholders = self._apply_prompt_updates(
1334
+ prompt_ids,
1335
+ mm_prompt_updates,
1336
)
1337
1338
+ self._validate_mm_placeholders(mm_placeholders, mm_item_counts)
1339
+
1340
return prompt_ids, mm_placeholders
1341
1342
def get_updates_use_audio_in_video(
vllm/model_executor/models/voxtral.py
+22
-2
@@ -36,6 +36,7 @@ from vllm.multimodal import MULTIMODAL_REGISTRY
36
from vllm.multimodal.inputs import (
37
MultiModalFieldConfig,
38
MultiModalKwargsItems,
39
+ MultiModalKwargsOptionalItems,
40
NestedTensors,
41
)
42
from vllm.multimodal.parse import (
@@ -47,6 +48,7 @@ from vllm.multimodal.processing import BaseDummyInputsBuilder
48
from vllm.multimodal.processing.processor import (
49
BaseMultiModalProcessor,
50
BaseProcessingInfo,
51
+ MultiModalPromptUpdates,
52
PlaceholderFeaturesInfo,
53
ProcessorInputs,
54
PromptReplacement,
@@ -202,8 +204,26 @@ class VoxtralDummyInputsBuilder(BaseDummyInputsBuilder[VoxtralProcessingInfo]):
204
205
206
class VoxtralMultiModalProcessor(BaseMultiModalProcessor[VoxtralProcessingInfo]):
205
- # The tokens are already inserted by the chat template
206
- renderer_applies_updates = True
207
+ # The tokens are already inserted by the chat template,
208
+ # so we just double check that they exist
209
+ def _maybe_apply_prompt_updates(
210
+ self,
211
+ mm_items: MultiModalDataItems,
212
+ prompt_ids: list[int],
213
+ mm_kwargs: MultiModalKwargsOptionalItems,
214
+ mm_prompt_updates: MultiModalPromptUpdates,
215
+ ) -> tuple[list[int], Mapping[str, list[PlaceholderFeaturesInfo]]]:
216
+ mm_item_counts = mm_items.get_all_counts()
217
+ self._validate_mm_kwargs(mm_kwargs, mm_item_counts)
218
+ self._validate_mm_updates(mm_prompt_updates, mm_item_counts)
219
+
220
+ mm_placeholders = self._find_mm_placeholders(
221
+ prompt_ids,
222
+ mm_prompt_updates,
223
+ )
224
+ self._validate_mm_placeholders(mm_placeholders, mm_item_counts)
225
+
226
+ return prompt_ids, mm_placeholders
227
228
def _get_mm_fields_config(
229
self,
vllm/model_executor/models/voxtral_realtime.py
-1
@@ -63,7 +63,6 @@ class VoxtralRealtimeMultiModalProcessor(VoxtralMultiModalProcessor):
63
prompt_ids: list[int],
64
mm_kwargs: MultiModalKwargsOptionalItems,
65
mm_prompt_updates: MultiModalPromptUpdates,
66
- is_update_applied: bool,
66
) -> tuple[list[int], Mapping[str, list[PlaceholderFeaturesInfo]]]:
67
# there are no placeholder audio tokens for streaming
68
# so we need to build the place placeholder positions manually
vllm/multimodal/processing/processor.py
+6
-21
@@ -8,7 +8,6 @@ from enum import Enum
8
from functools import lru_cache
9
from typing import (
10
TYPE_CHECKING,
11
- ClassVar,
11
Generic,
12
NamedTuple,
13
Protocol,
@@ -1177,12 +1176,6 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
1176
Not to be confused with `transformers.ProcessorMixin`.
1177
"""
1178
1180
- renderer_applies_updates: ClassVar[bool] = False
1181
- """
1182
- Only set to True if the tokenizer or chat template in the Renderer
1183
- (which are run before MM processing) inserts placeholder tokens.
1184
- """
1185
-
1179
def __init__(
1180
self,
1181
info: _I,
@@ -1780,24 +1773,17 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
1773
prompt_ids: list[int],
1774
mm_kwargs: MultiModalKwargsOptionalItems,
1775
mm_prompt_updates: MultiModalPromptUpdates,
1783
- is_update_applied: bool,
1776
) -> tuple[list[int], Mapping[str, list[PlaceholderFeaturesInfo]]]:
1777
mm_item_counts = mm_items.get_all_counts()
1778
self._validate_mm_kwargs(mm_kwargs, mm_item_counts)
1779
self._validate_mm_updates(mm_prompt_updates, mm_item_counts)
1780
1789
- if is_update_applied:
1790
- mm_placeholders = self._find_mm_placeholders(
1791
- prompt_ids,
1792
- mm_prompt_updates,
1793
- )
1794
- self._validate_mm_placeholders(mm_placeholders, mm_item_counts)
1795
- else:
1796
- prompt_ids, mm_placeholders = self._apply_prompt_updates(
1797
- prompt_ids,
1798
- mm_prompt_updates,
1799
- )
1800
- self._validate_mm_placeholders(mm_placeholders, mm_item_counts)
1781
+ prompt_ids, mm_placeholders = self._apply_prompt_updates(
1782
+ prompt_ids,
1783
+ mm_prompt_updates,
1784
+ )
1785
+
1786
+ self._validate_mm_placeholders(mm_placeholders, mm_item_counts)
1787
1788
return prompt_ids, mm_placeholders
1789
@@ -1829,7 +1815,6 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
1815
prompt_ids=prompt_ids,
1816
mm_kwargs=mm_info.kwargs,
1817
mm_prompt_updates=mm_info.prompt_updates,
1832
- is_update_applied=self.renderer_applies_updates,
1818
)
1819
1820
mm_placeholder_ranges = {