[MM] Address comments on #53275 (#53364)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
Cyrus Leung committed
Aug 22, 2026 at 18:27 UTC
040700aaa6cae3ef54dad8ba75db1abfb79db449
4 files changed
+11
-10
vllm/model_executor/models/nano_nemotron_vl.py
+1
-1
@@ -734,7 +734,7 @@ class NanoNemotronVLMultiModalProcessor(
734
prompt_ids=prompt_ids,
735
mm_kwargs=mm_info.kwargs,
736
mm_prompt_updates=mm_info.prompt_updates,
737
- is_update_applied=self.hf_processor_applies_updates,
737
+ is_update_applied=self.renderer_applies_updates,
738
)
739
740
mm_placeholder_ranges = {
vllm/model_executor/models/pixtral.py
+1
-1
@@ -212,7 +212,7 @@ class PixtralDummyInputsBuilder(BaseDummyInputsBuilder[PixtralProcessingInfo]):
212
213
class PixtralMultiModalProcessor(BaseMultiModalProcessor[PixtralProcessingInfo]):
214
# The tokens are already inserted by the chat template
215
- hf_processor_applies_updates = True
215
+ renderer_applies_updates = True
216
217
def _get_mm_fields_config(
218
self,
vllm/model_executor/models/voxtral.py
+1
-1
@@ -203,7 +203,7 @@ class VoxtralDummyInputsBuilder(BaseDummyInputsBuilder[VoxtralProcessingInfo]):
203
204
class VoxtralMultiModalProcessor(BaseMultiModalProcessor[VoxtralProcessingInfo]):
205
# The tokens are already inserted by the chat template
206
- hf_processor_applies_updates = True
206
+ renderer_applies_updates = True
207
208
def _get_mm_fields_config(
209
self,
vllm/multimodal/processing/processor.py
+8
-7
@@ -1177,9 +1177,9 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
1177
Not to be confused with `transformers.ProcessorMixin`.
1178
"""
1179
1180
- hf_processor_applies_updates: ClassVar[bool] = False
1180
+ renderer_applies_updates: ClassVar[bool] = False
1181
"""
1182
- Only set to True if the tokenizer or chat template
1182
+ Only set to True if the tokenizer or chat template in the Renderer
1183
(which are run before MM processing) inserts placeholder tokens.
1184
"""
1185
@@ -1811,12 +1811,13 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
1811
1812
The main steps are:
1813
1814
- 1. Apply HF Processor on prompt text and multi-modal data together,
1815
- outputting token IDs and processed tensors.
1816
- 2. Find and update sequences in the token IDs with placeholder tokens.
1814
+ 1. Perform MM-specific post-processing to the token inputs.
1815
+ 2. Apply HF Processor on multi-modal data together with optional
1816
+ dummy text, outputting processed multi-modal data tensors.
1817
+ 3. Find and update sequences in the token IDs with placeholder tokens.
1818
The number of placeholder tokens equals the feature size of the
1819
multi-modal data outputted by the multi-modal encoder.
1819
- 3. Extract information about the placeholder tokens from the
1820
+ 4. Extract information about the placeholder tokens from the
1821
processed token IDs.
1822
"""
1823
prompt_ids = self._postprocess_prompt(inputs.prompt)
@@ -1828,7 +1829,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
1829
prompt_ids=prompt_ids,
1830
mm_kwargs=mm_info.kwargs,
1831
mm_prompt_updates=mm_info.prompt_updates,
1831
- is_update_applied=self.hf_processor_applies_updates,
1832
+ is_update_applied=self.renderer_applies_updates,
1833
)
1834
1835
mm_placeholder_ranges = {