[MM] Move get_dummy_processor_inputs into MM processor (#57967)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
Cyrus Leung committed
Sep 22, 2026 at 00:39 UTC
7853700d4cdc2b84c728c840114ed42cda7454a7
10 files changed
+130
-134
tests/models/multimodal/processing/test_common.py
+2
-3
@@ -159,7 +159,6 @@ def get_token_prompt(
159
processor: BaseMultiModalProcessor,
160
mm_data: MultiModalDataDict,
161
) -> list[int]:
162
- dummy_inputs = processor.dummy_inputs
162
tokenizer: TokenizerLike = processor.info.get_tokenizer()
163
model_config = processor.info.ctx.model_config
164
@@ -174,7 +173,7 @@ def get_token_prompt(
173
mm_counts = {k: len(vs) for k, vs in parsed_data.items()}
174
175
if is_mistral_tokenizer(tokenizer):
177
- inputs = dummy_inputs.get_dummy_processor_inputs(
176
+ inputs = processor.get_dummy_inputs(
177
model_config.max_model_len,
178
mm_counts,
179
mm_options=MultiModalDummyOptions(),
@@ -182,7 +181,7 @@ def get_token_prompt(
181
mm_data=mm_data, # type: ignore[call-arg]
182
)
183
else:
185
- inputs = dummy_inputs.get_dummy_processor_inputs(
184
+ inputs = processor.get_dummy_inputs(
185
model_config.max_model_len,
186
mm_counts,
187
mm_options=MultiModalDummyOptions(),
tests/models/multimodal/processing/test_tensor_schema.py
+1
-2
@@ -90,13 +90,12 @@ def create_batched_mm_kwargs(
90
size_factors: tuple[float, ...] = (1.0, 0.5, 0.25),
91
) -> Iterable[tuple[str, int, BatchedTensorInputs]]:
92
processing_info = processor.info
93
- dummy_inputs = processor.dummy_inputs
93
supported_mm_limits = processing_info.get_supported_mm_limits()
94
mm_counts = {
95
modality: 3 if limit is None else limit
96
for modality, limit in supported_mm_limits.items()
97
}
99
- processor_inputs = dummy_inputs.get_dummy_processor_inputs(
98
+ processor_inputs = processor.get_dummy_inputs(
99
seq_len=model_config.max_model_len,
100
mm_counts=mm_counts,
101
mm_options=MultiModalDummyOptions(),
tests/renderers/test_warmup.py
+6
-8
@@ -4,7 +4,7 @@
4
5
These tests exercise:
6
- Zero-limit modalities are filtered from mm_counts passed to
7
- get_dummy_processor_inputs (e.g. --limit-mm-per-prompt image=0 ...)
7
+ get_dummy_inputs (e.g. --limit-mm-per-prompt image=0 ...)
8
- The warmup runs through the processor-only cache and clears it afterwards,
9
so its dummy inputs never land in the sender cache used by the serving path
10
- MM warmup is skipped entirely when mm_processor is None
@@ -92,7 +92,7 @@ class TestMmWarmupZeroLimitFiltering:
92
93
BaseRenderer.warmup(renderer, ChatParams())
94
95
- get_inputs = renderer.mm_processor.dummy_inputs.get_dummy_processor_inputs
95
+ get_inputs = renderer.mm_processor.get_dummy_inputs
96
get_inputs.assert_called_once()
97
_, kwargs = get_inputs.call_args
98
assert "video" not in kwargs["mm_counts"]
@@ -104,7 +104,7 @@ class TestMmWarmupZeroLimitFiltering:
104
105
BaseRenderer.warmup(renderer, ChatParams())
106
107
- get_inputs = renderer.mm_processor.dummy_inputs.get_dummy_processor_inputs
107
+ get_inputs = renderer.mm_processor.get_dummy_inputs
108
get_inputs.assert_called_once()
109
_, kwargs = get_inputs.call_args
110
assert kwargs["mm_counts"] == {}
@@ -115,7 +115,7 @@ class TestMmWarmupZeroLimitFiltering:
115
116
BaseRenderer.warmup(renderer, ChatParams())
117
118
- get_inputs = renderer.mm_processor.dummy_inputs.get_dummy_processor_inputs
118
+ get_inputs = renderer.mm_processor.get_dummy_inputs
119
get_inputs.assert_called_once()
120
_, kwargs = get_inputs.call_args
121
assert kwargs["mm_counts"] == {"image": 1, "video": 1}
@@ -134,7 +134,7 @@ class TestMmWarmupRunsNormally:
134
135
BaseRenderer.warmup(renderer, ChatParams())
136
137
- get_inputs = renderer.mm_processor.dummy_inputs.get_dummy_processor_inputs
137
+ get_inputs = renderer.mm_processor.get_dummy_inputs
138
assert get_inputs.call_args.kwargs["seq_len"] == expected_seq_len
139
140
def test_processor_apply_called(self):
@@ -207,9 +207,7 @@ class TestWarmupFaultIsolation:
207
def test_mm_failure_is_swallowed_and_cache_cleared(self):
208
renderer = _make_renderer_mock({"image": 1})
209
# the warmup blows up before apply()
210
- renderer.mm_processor.dummy_inputs.get_dummy_processor_inputs.side_effect = (
211
- RuntimeError("mm boom")
212
- )
210
+ renderer.mm_processor.get_dummy_inputs.side_effect = RuntimeError("mm boom")
211
212
BaseRenderer.warmup(renderer, ChatParams()) # must not raise
213
vllm/model_executor/models/hyperclovax_vision_v2.py
+23
-33
@@ -189,38 +189,6 @@ class HCXVisionV2DummyInputsBuilder(BaseDummyInputsBuilder[HCXVisionV2Processing
189
num_videos = mm_counts.get("video", 0)
190
return V2_IMAGE_TOKEN * num_images + V2_VIDEO_TOKEN * num_videos
191
192
- def get_dummy_processor_inputs(
193
- self,
194
- seq_len: int,
195
- mm_counts: Mapping[str, int],
196
- mm_options: MultiModalDummyOptions | None = None,
197
- mm_processor_kwargs: Mapping[str, object] | None = None,
198
- ) -> ProcessorInputs:
199
- """Build dummy processor inputs for memory profiling."""
200
- num_images = mm_counts.get("image", 0)
201
- num_videos = mm_counts.get("video", 0)
202
- prompt_text = V2_IMAGE_TOKEN * num_images + V2_VIDEO_TOKEN * num_videos
203
-
204
- dummy_mm_data = self.get_dummy_mm_data(
205
- seq_len,
206
- mm_counts,
207
- mm_options,
208
- mm_processor_kwargs=mm_processor_kwargs,
209
- )
210
- dummy_mm_items = self.info.parse_mm_data(dummy_mm_data, validate=False)
211
-
212
- tokenizer = self.info.get_tokenizer()
213
- prompt = tokenizer.encode(
214
- prompt_text,
215
- **self.info.default_tok_params.get_encode_kwargs(),
216
- )
217
-
218
- return ProcessorInputs(
219
- prompt=prompt,
220
- mm_data_items=dummy_mm_items,
221
- hf_processor_mm_kwargs=mm_processor_kwargs or {},
222
- )
223
-
192
def get_dummy_mm_data(
193
self,
194
seq_len: int,
@@ -255,6 +223,28 @@ class HCXVisionV2MultiModalProcessor(
223
):
224
"""Multimodal processor for HyperCLOVAX V2 (32B Think model)."""
225
226
+ def get_dummy_inputs(
227
+ self,
228
+ seq_len: int,
229
+ mm_counts: Mapping[str, int],
230
+ mm_options: MultiModalDummyOptions,
231
+ ) -> ProcessorInputs:
232
+ num_images = mm_counts.get("image", 0)
233
+ num_videos = mm_counts.get("video", 0)
234
+ prompt_text = V2_IMAGE_TOKEN * num_images + V2_VIDEO_TOKEN * num_videos
235
+
236
+ builder = self.dummy_inputs
237
+ dummy_mm_data = builder.get_dummy_mm_data(seq_len, mm_counts, mm_options)
238
+ dummy_mm_items = self.info.parse_mm_data(dummy_mm_data, validate=False)
239
+
240
+ tokenizer = self.info.get_tokenizer()
241
+ prompt = tokenizer.encode(
242
+ prompt_text,
243
+ **self.info.default_tok_params.get_encode_kwargs(),
244
+ )
245
+
246
+ return ProcessorInputs(prompt=prompt, mm_data_items=dummy_mm_items)
247
+
248
def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str:
249
return self.dummy_inputs.get_dummy_text(mm_counts)
250
@@ -267,7 +257,7 @@ class HCXVisionV2MultiModalProcessor(
257
hf_config = self.info.get_hf_config()
258
259
# Use token IDs directly from config.
270
- # This matches what get_dummy_processor_inputs uses, ensuring consistency.
260
+ # This matches what get_dummy_inputs uses, ensuring consistency.
261
placeholder: dict[str, int] = {
262
"image": hf_config.image_token_id, # 128060 for <|IMAGE_PAD|>
263
"video": hf_config.video_token_id, # 128061 for <|VIDEO_PAD|>
vllm/model_executor/models/kimi_audio.py
+24
-23
@@ -176,29 +176,6 @@ class KimiAudioDummyInputsBuilder(BaseDummyInputsBuilder[KimiAudioProcessingInfo
176
),
177
}
178
179
- def get_dummy_processor_inputs(
180
- self,
181
- seq_len: int,
182
- mm_counts: Mapping[str, int],
183
- mm_options: MultiModalDummyOptions,
184
- ) -> ProcessorInputs:
185
- dummy_mm_data = self.get_dummy_mm_data(seq_len, mm_counts, mm_options)
186
- dummy_mm_items = self.info.parse_mm_data(dummy_mm_data)
187
-
188
- num_audios = mm_counts.get("audio", 0)
189
- dummy_tokens = (
190
- [198]
191
- if num_audios == 0
192
- else [
193
- KimiAudioProcessor.KIMIA_MEDIA_BEGIN,
194
- KimiAudioProcessor.KIMIA_TEXT_BLANK,
195
- KimiAudioProcessor.KIMIA_MEDIA_END,
196
- ]
197
- * num_audios
198
- )
199
-
200
- return ProcessorInputs(prompt=dummy_tokens, mm_data_items=dummy_mm_items)
201
-
179
180
# Field config for Kimi-Audio multimodal data
181
_KIMIAUDIO_FIELD_CONFIG = {
@@ -228,6 +205,30 @@ class KimiAudioMultiModalDataParser(MultiModalDataParser):
205
class KimiAudioMultiModalProcessor(BaseMultiModalProcessor[KimiAudioProcessingInfo]):
206
"""vLLM multi-modal processor wrapper for Kimi-Audio."""
207
208
+ def get_dummy_inputs(
209
+ self,
210
+ seq_len: int,
211
+ mm_counts: Mapping[str, int],
212
+ mm_options: MultiModalDummyOptions,
213
+ ) -> ProcessorInputs:
214
+ builder = self.dummy_inputs
215
+ dummy_mm_data = builder.get_dummy_mm_data(seq_len, mm_counts, mm_options)
216
+ dummy_mm_items = self.info.parse_mm_data(dummy_mm_data)
217
+
218
+ num_audios = mm_counts.get("audio", 0)
219
+ dummy_tokens = (
220
+ [198]
221
+ if num_audios == 0
222
+ else [
223
+ KimiAudioProcessor.KIMIA_MEDIA_BEGIN,
224
+ KimiAudioProcessor.KIMIA_TEXT_BLANK,
225
+ KimiAudioProcessor.KIMIA_MEDIA_END,
226
+ ]
227
+ * num_audios
228
+ )
229
+
230
+ return ProcessorInputs(prompt=dummy_tokens, mm_data_items=dummy_mm_items)
231
+
232
def _get_hf_mm_inputs(
233
self,
234
mm_items: MultiModalDataItems,
vllm/model_executor/models/moss_transcribe_diarize.py
+23
-16
@@ -438,31 +438,38 @@ class MossTranscribeDiarizeDummyInputsBuilder(
438
)
439
}
440
441
- def get_dummy_processor_inputs(
441
+
442
+class MossTranscribeDiarizeMultiModalProcessor(
443
+ BaseMultiModalProcessor[MossTranscribeDiarizeProcessingInfo]
444
+):
445
+ def get_dummy_inputs(
446
self,
447
seq_len: int,
448
mm_counts: Mapping[str, int],
449
mm_options: MultiModalDummyOptions,
450
) -> ProcessorInputs:
447
- dummy_mm_data = self.get_dummy_mm_data(seq_len, mm_counts, mm_options)
451
+ builder = self.dummy_inputs
452
+ tokenizer = self.info.get_tokenizer()
453
+
454
+ dummy_mm_data = builder.get_dummy_mm_data(seq_len, mm_counts, mm_options)
455
dummy_mm_items = self.info.parse_mm_data(dummy_mm_data)
456
+
457
num_audios = mm_counts.get("audio", 0)
450
- tokenizer = self.info.get_tokenizer()
451
- prompt = cached_encode(
452
- tokenizer,
453
- AUDIO_PLACEHOLDER * num_audios,
454
- add_special_tokens=False,
455
- ) or cached_encode(
456
- tokenizer,
457
- "\n",
458
- add_special_tokens=False,
459
- )
460
- return ProcessorInputs(prompt=prompt, mm_data_items=dummy_mm_items)
458
+ if num_audios > 0:
459
+ prompt = cached_encode(
460
+ tokenizer,
461
+ AUDIO_PLACEHOLDER * num_audios,
462
+ add_special_tokens=False,
463
+ )
464
+ else:
465
+ prompt = cached_encode(
466
+ tokenizer,
467
+ "\n",
468
+ add_special_tokens=False,
469
+ )
470
471
+ return ProcessorInputs(prompt=prompt, mm_data_items=dummy_mm_items)
472
463
-class MossTranscribeDiarizeMultiModalProcessor(
464
- BaseMultiModalProcessor[MossTranscribeDiarizeProcessingInfo]
465
-):
473
def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str:
474
return self.dummy_inputs.get_dummy_text(mm_counts)
475
vllm/model_executor/models/pixtral.py
+7
-5
@@ -221,18 +221,22 @@ class PixtralDummyInputsBuilder(BaseDummyInputsBuilder[PixtralProcessingInfo]):
221
)
222
}
223
224
- def get_dummy_processor_inputs(
224
+
225
+class PixtralMultiModalProcessor(BaseMultiModalProcessor[PixtralProcessingInfo]):
226
+ def get_dummy_inputs(
227
self,
228
seq_len: int,
229
mm_counts: Mapping[str, int],
230
mm_options: MultiModalDummyOptions,
231
+ # For test_common.py only
232
mm_data: MultiModalDataDict | None = None,
233
) -> ProcessorInputs:
234
+ builder = self.dummy_inputs
235
tokenizer = self.info.get_tokenizer()
236
233
- dummy_text = self.get_dummy_text(mm_counts)
237
+ dummy_text = builder.get_dummy_text(mm_counts)
238
dummy_mm_data = (
235
- self.get_dummy_mm_data(seq_len, mm_counts, mm_options)
239
+ builder.get_dummy_mm_data(seq_len, mm_counts, mm_options)
240
if mm_data is None
241
else mm_data
242
)
@@ -256,8 +260,6 @@ class PixtralDummyInputsBuilder(BaseDummyInputsBuilder[PixtralProcessingInfo]):
260
261
return ProcessorInputs(prompt=dummy_tokens, mm_data_items=dummy_mm_items)
262
259
-
260
-class PixtralMultiModalProcessor(BaseMultiModalProcessor[PixtralProcessingInfo]):
263
# The tokens are already inserted by the chat template,
264
# so we just double check that they exist
265
def _maybe_apply_prompt_updates(
vllm/model_executor/models/voxtral.py
+7
-5
@@ -149,19 +149,23 @@ class VoxtralDummyInputsBuilder(BaseDummyInputsBuilder[VoxtralProcessingInfo]):
149
)
150
}
151
152
- def get_dummy_processor_inputs(
152
+
153
+class VoxtralMultiModalProcessor(BaseMultiModalProcessor[VoxtralProcessingInfo]):
154
+ def get_dummy_inputs(
155
self,
156
seq_len: int,
157
mm_counts: Mapping[str, int],
158
mm_options: MultiModalDummyOptions,
159
+ # For test_common.py only
160
mm_data: MultiModalDataDict | None = None,
161
) -> ProcessorInputs:
162
+ builder = self.dummy_inputs
163
tokenizer = self.info.get_tokenizer()
164
feature_extractor = self.info.get_feature_extractor()
165
162
- dummy_text = self.get_dummy_text(mm_counts)
166
+ dummy_text = builder.get_dummy_text(mm_counts)
167
dummy_mm_data = (
164
- self.get_dummy_mm_data(seq_len, mm_counts, mm_options)
168
+ builder.get_dummy_mm_data(seq_len, mm_counts, mm_options)
169
if mm_data is None
170
else mm_data
171
)
@@ -197,8 +201,6 @@ class VoxtralDummyInputsBuilder(BaseDummyInputsBuilder[VoxtralProcessingInfo]):
201
202
return ProcessorInputs(prompt=dummy_tokens, mm_data_items=dummy_mm_items)
203
200
-
201
-class VoxtralMultiModalProcessor(BaseMultiModalProcessor[VoxtralProcessingInfo]):
204
# The tokens are already inserted by the chat template,
205
# so we just double check that they exist
206
def _maybe_apply_prompt_updates(
vllm/multimodal/processing/dummy_inputs.py
-37
@@ -18,7 +18,6 @@ from vllm.inputs import MultiModalDataDict
18
from vllm.logger import init_logger
19
20
from .context import BaseProcessingInfo
21
-from .inputs import ProcessorInputs
21
22
_I = TypeVar("_I", bound=BaseProcessingInfo)
23
@@ -61,42 +60,6 @@ class BaseDummyInputsBuilder(ABC, Generic[_I]):
60
"""
61
raise NotImplementedError
62
64
- def get_dummy_processor_inputs(
65
- self,
66
- seq_len: int,
67
- mm_counts: Mapping[str, int],
68
- mm_options: MultiModalDummyOptions,
69
- ) -> ProcessorInputs:
70
- """Build the input which, after processing, results in
71
- the maximum possible number of placeholder tokens.
72
-
73
- Args:
74
- seq_len: Sequence length
75
- mm_counts: Count of items per modality
76
- mm_options: Configurable options per modality (optional)
77
-
78
- """
79
- dummy_text = self.get_dummy_text(mm_counts)
80
- dummy_mm_data = self.get_dummy_mm_data(seq_len, mm_counts, mm_options)
81
- dummy_mm_items = self.info.parse_mm_data(dummy_mm_data, validate=False)
82
-
83
- tokenizer = self.info.ctx.tokenizer
84
- dummy_prompt: list[int]
85
- if tokenizer is None:
86
- # Tokenizer-less models (e.g. `skip_tokenizer_init=True`) only
87
- # accept embeddings and have an empty dummy text, so there are no
88
- # prompt tokens.
89
- dummy_prompt = []
90
- else:
91
- from .processor import cached_encode
92
-
93
- dummy_prompt = cached_encode(tokenizer, dummy_text, truncation=False)
94
-
95
- return ProcessorInputs(
96
- prompt=dummy_prompt,
97
- mm_data_items=dummy_mm_items,
98
- )
99
-
63
def _get_dummy_audios(
64
self,
65
*,
vllm/multimodal/processing/processor.py
+37
-2
@@ -39,7 +39,7 @@ from ..inputs import (
39
)
40
from ..parse import MultiModalDataItems, MultiModalUUIDItems
41
from .context import BaseProcessingInfo, TimingContext
42
-from .dummy_inputs import BaseDummyInputsBuilder
42
+from .dummy_inputs import BaseDummyInputsBuilder, MultiModalDummyOptions
43
from .inputs import ProcessorInputs
44
45
if TYPE_CHECKING:
@@ -1055,6 +1055,41 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
1055
1056
self.data_parser = self.info.get_data_parser()
1057
1058
+ def get_dummy_inputs(
1059
+ self,
1060
+ seq_len: int,
1061
+ mm_counts: Mapping[str, int],
1062
+ mm_options: MultiModalDummyOptions,
1063
+ ) -> ProcessorInputs:
1064
+ """Build the input which, after processing, results in
1065
+ the maximum possible number of placeholder tokens.
1066
+
1067
+ Args:
1068
+ seq_len: Sequence length
1069
+ mm_counts: Count of items per modality
1070
+ mm_options: Configurable options per modality (optional)
1071
+
1072
+ """
1073
+ builder = self.dummy_inputs
1074
+ dummy_text = builder.get_dummy_text(mm_counts)
1075
+ dummy_mm_data = builder.get_dummy_mm_data(seq_len, mm_counts, mm_options)
1076
+ dummy_mm_items = self.info.parse_mm_data(dummy_mm_data, validate=False)
1077
+
1078
+ tokenizer = self.info.ctx.tokenizer
1079
+ dummy_prompt: list[int]
1080
+ if tokenizer is None:
1081
+ # Tokenizer-less models (e.g. `skip_tokenizer_init=True`) only
1082
+ # accept embeddings and have an empty dummy text, so there are no
1083
+ # prompt tokens.
1084
+ dummy_prompt = []
1085
+ else:
1086
+ dummy_prompt = cached_encode(tokenizer, dummy_text, truncation=False)
1087
+
1088
+ return ProcessorInputs(
1089
+ prompt=dummy_prompt,
1090
+ mm_data_items=dummy_mm_items,
1091
+ )
1092
+
1093
def get_dummy_mm_inputs(
1094
self,
1095
mm_counts: Mapping[str, int],
@@ -1070,7 +1105,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
1105
1106
mm_config = model_config.get_multimodal_config()
1107
1073
- processor_inputs = self.dummy_inputs.get_dummy_processor_inputs(
1108
+ processor_inputs = self.get_dummy_inputs(
1109
seq_len=seq_len,
1110
mm_counts=mm_counts,
1111
mm_options=mm_config.limit_per_prompt,