@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 0ab06100f4
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 8, 11:15 UTC

[Multimodal] Expose `mm_processor_kwargs` for `DummyInputsBuilder` (#34330)

Signed-off-by: Isotr0py <mozf@mail2.sysu.edu.cn>

Isotr0py committed Feb 12, 2026 at 01:37 UTC 0ab06100f469fe29b8a71cf0311b6b9da99db23e
72 files changed +131 -27
vllm/model_executor/models/aria.py
+1
@@ -445,6 +445,7 @@ class AriaDummyInputsBuilder(BaseDummyInputsBuilder[AriaProcessingInfo]):
445 seq_len: int,
446 mm_counts: Mapping[str, int],
447 mm_options: Mapping[str, BaseDummyOptions] | None = None,
448 + mm_processor_kwargs: Mapping[str, object] | None = None,
449 ) -> MultiModalDataDict:
450 vision_config = self.info.get_vision_config()
451
vllm/model_executor/models/audioflamingo3.py
+4 -1
@@ -253,8 +253,11 @@ class AudioFlamingo3DummyInputsBuilder(
253 seq_len: int,
254 mm_counts: Mapping[str, int],
255 mm_options: Mapping[str, BaseDummyOptions] | None = None,
256 + mm_processor_kwargs: Mapping[str, object] | None = None,
257 ) -> MultiModalDataDict:
257 - feature_extractor = self.info.get_feature_extractor()
258 + feature_extractor = self.info.get_feature_extractor(
259 + **(mm_processor_kwargs or {})
260 + )
261 sampling_rate = feature_extractor.sampling_rate
262 audio_len = MAX_AUDIO_LEN * sampling_rate
263 num_audios = mm_counts.get("audio", 0)
vllm/model_executor/models/aya_vision.py
+1
@@ -192,6 +192,7 @@ class AyaVisionDummyInputsBuilder(BaseDummyInputsBuilder[AyaVisionProcessingInfo
192 seq_len: int,
193 mm_counts: Mapping[str, int],
194 mm_options: Mapping[str, BaseDummyOptions] | None = None,
195 + mm_processor_kwargs: Mapping[str, object] | None = None,
196 ) -> MultiModalDataDict:
197 num_images = mm_counts.get("image", 0)
198 image_size = self.info.get_image_size_with_most_features()
vllm/model_executor/models/bagel.py
+1
@@ -250,6 +250,7 @@ class BagelDummyInputsBuilder(BaseDummyInputsBuilder[BagelProcessingInfo]):
250 seq_len: int,
251 mm_counts: Mapping[str, int],
252 mm_options: Mapping[str, BaseDummyOptions] | None = None,
253 + mm_processor_kwargs: Mapping[str, object] | None = None,
254 ) -> MultiModalDataDict:
255 num_images = mm_counts.get("image", 0)
256 hf_config = self.info.get_hf_config()
vllm/model_executor/models/bee.py
+1
@@ -91,6 +91,7 @@ class BeeDummyInputsBuilder(LlavaDummyInputsBuilder[BeeProcessingInfo]):
91 seq_len: int,
92 mm_counts: Mapping[str, int],
93 mm_options: Mapping[str, BaseDummyOptions] | None = None,
94 + mm_processor_kwargs: Mapping[str, object] | None = None,
95 ) -> MultiModalDataDict:
96 num_images = mm_counts.get("image", 0)
97
vllm/model_executor/models/blip2.py
+1
@@ -446,6 +446,7 @@ class Blip2DummyInputsBuilder(BaseDummyInputsBuilder[Blip2ProcessingInfo]):
446 seq_len: int,
447 mm_counts: Mapping[str, int],
448 mm_options: Mapping[str, BaseDummyOptions] | None = None,
449 + mm_processor_kwargs: Mapping[str, object] | None = None,
450 ) -> MultiModalDataDict:
451 hf_config = self.info.get_hf_config()
452 vision_config = hf_config.vision_config
vllm/model_executor/models/chameleon.py
+1
@@ -117,6 +117,7 @@ class ChameleonDummyInputsBuilder(BaseDummyInputsBuilder[ChameleonProcessingInfo
117 seq_len: int,
118 mm_counts: Mapping[str, int],
119 mm_options: Mapping[str, BaseDummyOptions] | None = None,
120 + mm_processor_kwargs: Mapping[str, object] | None = None,
121 ) -> MultiModalDataDict:
122 config = self.info.get_hf_config()
123
vllm/model_executor/models/clip.py
+1
@@ -171,6 +171,7 @@ class CLIPDummyInputsBuilder(BaseDummyInputsBuilder[CLIPProcessingInfo]):
171 seq_len: int,
172 mm_counts: Mapping[str, int],
173 mm_options: Mapping[str, BaseDummyOptions] | None = None,
174 + mm_processor_kwargs: Mapping[str, object] | None = None,
175 ) -> MultiModalDataDict:
176 num_images = mm_counts.get("image", 0)
177
vllm/model_executor/models/cohere2_vision.py
+1
@@ -221,6 +221,7 @@ class Cohere2VisionDummyInputsBuilder(
221 seq_len: int,
222 mm_counts: Mapping[str, int],
223 mm_options: Mapping[str, BaseDummyOptions] | None = None,
224 + mm_processor_kwargs: Mapping[str, object] | None = None,
225 ) -> MultiModalDataDict:
226 num_images = mm_counts.get("image", 0)
227 image_size = self.info.get_image_size_with_most_features()
vllm/model_executor/models/deepseek_ocr.py
+1
@@ -256,6 +256,7 @@ class DeepseekOCRDummyInputsBuilder(BaseDummyInputsBuilder[DeepseekOCRProcessing
256 seq_len: int,
257 mm_counts: Mapping[str, int],
258 mm_options: Mapping[str, BaseDummyOptions] | None = None,
259 + mm_processor_kwargs: Mapping[str, object] | None = None,
260 ) -> MultiModalDataDict:
261 num_images = mm_counts.get("image", 0)
262
vllm/model_executor/models/deepseek_ocr2.py
+1
@@ -138,6 +138,7 @@ class DeepseekOCR2DummyInputsBuilder(
138 seq_len: int,
139 mm_counts: Mapping[str, int],
140 mm_options: Mapping[str, BaseDummyOptions] | None = None,
141 + mm_processor_kwargs: Mapping[str, object] | None = None,
142 ) -> MultiModalDataDict:
143 num_images = mm_counts.get("image", 0)
144
vllm/model_executor/models/deepseek_vl2.py
+1
@@ -215,6 +215,7 @@ class DeepseekVL2DummyInputsBuilder(BaseDummyInputsBuilder[DeepseekVL2Processing
215 seq_len: int,
216 mm_counts: Mapping[str, int],
217 mm_options: Mapping[str, BaseDummyOptions] | None = None,
218 + mm_processor_kwargs: Mapping[str, object] | None = None,
219 ) -> MultiModalDataDict:
220 num_images = mm_counts.get("image", 0)
221
vllm/model_executor/models/dots_ocr.py
+3
@@ -107,10 +107,13 @@ class DotsOCRDummyInputsBuilder(Qwen2VLDummyInputsBuilder):
107 seq_len: int,
108 mm_counts: Mapping[str, int],
109 mm_options: Mapping[str, BaseDummyOptions] | None = None,
110 + mm_processor_kwargs: Mapping[str, object] | None = None,
111 ) -> MultiModalDataDict:
112 num_images = mm_counts.get("image", 0)
113
114 + mm_processor_kwargs = mm_processor_kwargs or {}
115 target_width, target_height = self.info.get_image_size_with_most_features( # noqa: E501
116 + mm_processor_kwargs.get("max_pixels", None)
117 )
118
119 image_overrides = mm_options.get("image") if mm_options else None
vllm/model_executor/models/ernie45_vl.py
+1
@@ -1153,6 +1153,7 @@ class Ernie4_5_VLDummyInputsBuilder(BaseDummyInputsBuilder[Ernie4_5_VLProcessing
1153 seq_len: int,
1154 mm_counts: Mapping[str, int],
1155 mm_options: Mapping[str, BaseDummyOptions] | None = None,
1156 + mm_processor_kwargs: Mapping[str, object] | None = None,
1157 ) -> MultiModalDataDict:
1158 num_images = mm_counts.get("image", 0)
1159 num_videos = mm_counts.get("video", 0)
vllm/model_executor/models/funasr.py
+4 -1
@@ -745,8 +745,11 @@ class FunASRDummyInputsBuilder(BaseDummyInputsBuilder[FunASRProcessingInfo]):
745 seq_len: int,
746 mm_counts: Mapping[str, int],
747 mm_options: Mapping[str, BaseDummyOptions] | None = None,
748 + mm_processor_kwargs: Mapping[str, object] | None = None,
749 ) -> MultiModalDataDict:
749 - feature_extractor = self.info.get_feature_extractor()
750 + feature_extractor = self.info.get_feature_extractor(
751 + **(mm_processor_kwargs or {})
752 + )
753
754 sampling_rate = feature_extractor.sampling_rate
755 audio_len = feature_extractor.chunk_length * sampling_rate
vllm/model_executor/models/funaudiochat.py
+5 -2
@@ -611,8 +611,11 @@ class FunAudioChatDummyInputsBuilder(
611 seq_len: int,
612 mm_counts: Mapping[str, int],
613 mm_options: Mapping[str, BaseDummyOptions] | None = None,
614 + mm_processor_kwargs: Mapping[str, object] | None = None,
615 ) -> MultiModalDataDict:
615 - feature_extractor = self.info.get_feature_extractor()
616 + feature_extractor = self.info.get_feature_extractor(
617 + **(mm_processor_kwargs or {})
618 + )
619 sampling_rate = int(feature_extractor.sampling_rate)
620
621 # Dummy inputs are used for profiling; construct the worst-case audio
@@ -656,7 +659,7 @@ class FunAudioChatMultiModalProcessor(
659 if not audios:
660 return BatchFeature({"input_ids": input_ids})
661
659 - feature_extractor = self.info.get_feature_extractor()
662 + feature_extractor = self.info.get_feature_extractor(**mm_kwargs)
663 sr = int(feature_extractor.sampling_rate)
664 min_samples = int(getattr(feature_extractor, "n_fft", 400) or 400)
665
vllm/model_executor/models/fuyu.py
+1
@@ -143,6 +143,7 @@ class FuyuDummyInputsBuilder(BaseDummyInputsBuilder[FuyuProcessingInfo]):
143 seq_len: int,
144 mm_counts: Mapping[str, int],
145 mm_options: Mapping[str, BaseDummyOptions] | None = None,
146 + mm_processor_kwargs: Mapping[str, object] | None = None,
147 ) -> MultiModalDataDict:
148 target_width, target_height = self.info.get_image_size_with_most_features()
149 num_images = mm_counts.get("image", 0)
vllm/model_executor/models/gemma3_mm.py
+1
@@ -256,6 +256,7 @@ class Gemma3DummyInputsBuilder(BaseDummyInputsBuilder[Gemma3ProcessingInfo]):
256 seq_len: int,
257 mm_counts: Mapping[str, int],
258 mm_options: Mapping[str, BaseDummyOptions] | None = None,
259 + mm_processor_kwargs: Mapping[str, object] | None = None,
260 ) -> MultiModalDataDict:
261 num_images = mm_counts.get("image", 0)
262
vllm/model_executor/models/gemma3n_mm.py
+1
@@ -182,6 +182,7 @@ class Gemma3nDummyInputsBuilder(BaseDummyInputsBuilder[Gemma3nProcessingInfo]):
182 seq_len: int,
183 mm_counts: Mapping[str, int],
184 mm_options: Mapping[str, BaseDummyOptions] | None = None,
185 + mm_processor_kwargs: Mapping[str, object] | None = None,
186 ) -> MultiModalDataDict:
187 num_images = mm_counts.get("image", 0)
188 num_audios = mm_counts.get("audio", 0)
vllm/model_executor/models/glm4_1v.py
+1
@@ -1143,6 +1143,7 @@ class Glm4vDummyInputsBuilder(BaseDummyInputsBuilder[Glm4vProcessingInfo]):
1143 seq_len: int,
1144 mm_counts: Mapping[str, int],
1145 mm_options: Mapping[str, BaseDummyOptions] | None = None,
1146 + mm_processor_kwargs: Mapping[str, object] | None = None,
1147 ) -> MultiModalDataDict:
1148 num_images = mm_counts.get("image", 0)
1149 num_videos = mm_counts.get("video", 0)
vllm/model_executor/models/glm4v.py
+1
@@ -493,6 +493,7 @@ class GLM4VDummyInputsBuilder(BaseDummyInputsBuilder[GLM4VProcessingInfo]):
493 seq_len: int,
494 mm_counts: Mapping[str, int],
495 mm_options: Mapping[str, BaseDummyOptions] | None = None,
496 + mm_processor_kwargs: Mapping[str, object] | None = None,
497 ) -> MultiModalDataDict:
498 hf_config = self.info.get_hf_config()
499 vision_config = hf_config.vision_config
vllm/model_executor/models/glmasr.py
+4 -1
@@ -727,8 +727,11 @@ class GlmAsrDummyInputsBuilder(BaseDummyInputsBuilder[GlmAsrProcessingInfo]):
727 seq_len: int,
728 mm_counts: Mapping[str, int],
729 mm_options: Mapping[str, BaseDummyOptions] | None = None,
730 + mm_processor_kwargs: Mapping[str, object] | None = None,
731 ) -> MultiModalDataDict:
731 - feature_extractor = self.info.get_feature_extractor()
732 + feature_extractor = self.info.get_feature_extractor(
733 + **(mm_processor_kwargs or {})
734 + )
735 sampling_rate = feature_extractor.sampling_rate
736 num_audios = mm_counts.get("audio", 0)
737 audio_overrides = mm_options.get("audio") if mm_options else None
vllm/model_executor/models/granite_speech.py
+1
@@ -217,6 +217,7 @@ class GraniteSpeechDummyInputsBuilder(
217 seq_len: int,
218 mm_counts: Mapping[str, int],
219 mm_options: Mapping[str, BaseDummyOptions] | None = None,
220 + mm_processor_kwargs: Mapping[str, object] | None = None,
221 ) -> MultiModalDataDict:
222 num_audios = mm_counts.get("audio", 0)
223 audio_overrides = mm_options.get("audio") if mm_options else None
vllm/model_executor/models/hunyuan_vision.py
+1
@@ -702,6 +702,7 @@ class HunYuanVLDummyInputsBuilder(BaseDummyInputsBuilder[HunYuanVLProcessingInfo
702 seq_len: int,
703 mm_counts: Mapping[str, int],
704 mm_options: Mapping[str, BaseDummyOptions] | None = None,
705 + mm_processor_kwargs: Mapping[str, object] | None = None,
706 ) -> MultiModalDataDict:
707 num_images = mm_counts.get("image", 1)
708
vllm/model_executor/models/hyperclovax_vision.py
+1
@@ -166,6 +166,7 @@ class HCXVisionDummyInputsBuilder(BaseDummyInputsBuilder[HCXVisionProcessingInfo
166 seq_len: int,
167 mm_counts: Mapping[str, int],
168 mm_options: Mapping[str, BaseDummyOptions] | None = None,
169 + mm_processor_kwargs: Mapping[str, object] | None = None,
170 ) -> MultiModalDataDict:
171 num_images = mm_counts.get("image", 0)
172 num_videos = mm_counts.get("video", 0)
vllm/model_executor/models/idefics3.py
+3 -11
@@ -42,7 +42,7 @@ from vllm.multimodal.inputs import (
42 MultiModalFieldConfig,
43 MultiModalKwargsItems,
44 )
45 -from vllm.multimodal.parse import ImageProcessorItems, ImageSize, MultiModalDataItems
45 +from vllm.multimodal.parse import ImageProcessorItems, MultiModalDataItems
46 from vllm.multimodal.processing import (
47 BaseDummyInputsBuilder,
48 BaseMultiModalProcessor,
@@ -285,15 +285,6 @@ class Idefics3ProcessingInfo(BaseProcessingInfo):
285
286 return num_patches * processor.image_seq_len
287
288 - def get_image_size_with_most_features(self) -> ImageSize:
289 - processor = self.get_hf_processor()
290 - image_processor: Idefics3ImageProcessor = processor.image_processor
291 -
292 - return ImageSize(
293 - width=image_processor.size["longest_edge"],
294 - height=image_processor.size["longest_edge"],
295 - )
296 -
288
289 class Idefics3DummyInputsBuilder(BaseDummyInputsBuilder[Idefics3ProcessingInfo]):
290 def get_dummy_text(self, mm_counts: Mapping[str, int]) -> str:
@@ -309,9 +300,10 @@ class Idefics3DummyInputsBuilder(BaseDummyInputsBuilder[Idefics3ProcessingInfo])
300 seq_len: int,
301 mm_counts: Mapping[str, int],
302 mm_options: Mapping[str, BaseDummyOptions] | None = None,
303 + mm_processor_kwargs: Mapping[str, object] | None = None,
304 ) -> MultiModalDataDict:
305 num_images = mm_counts.get("image", 0)
314 - hf_processor = self.info.get_hf_processor()
306 + hf_processor = self.info.get_hf_processor(**(mm_processor_kwargs or {}))
307 image_processor: Idefics3ImageProcessor = hf_processor.image_processor
308 longest_edge = image_processor.max_image_size["longest_edge"]
309
vllm/model_executor/models/interns1.py
+1
@@ -298,6 +298,7 @@ class InternS1DummyInputsBuilder(BaseDummyInputsBuilder[InternS1ProcessingInfo])
298 seq_len: int,
299 mm_counts: Mapping[str, int],
300 mm_options: Mapping[str, BaseDummyOptions] | None = None,
301 + mm_processor_kwargs: Mapping[str, object] | None = None,
302 ) -> MultiModalDataDict:
303 target_width, target_height = self.info.get_image_size_with_most_features()
304 target_num_frames = self.info.get_num_frames_with_most_features(
vllm/model_executor/models/internvl.py
+2
@@ -766,6 +766,7 @@ class BaseInternVLDummyInputsBuilder(BaseDummyInputsBuilder[_I]):
766 seq_len: int,
767 mm_counts: Mapping[str, int],
768 mm_options: Mapping[str, BaseDummyOptions] | None = None,
769 + mm_processor_kwargs: Mapping[str, object] | None = None,
770 ) -> MultiModalDataDict:
771 target_width, target_height = self.info.get_image_size_with_most_features()
772 num_images = mm_counts.get("image", 0)
@@ -938,6 +939,7 @@ class InternVLDummyInputsBuilder(
939 seq_len: int,
940 mm_counts: Mapping[str, int],
941 mm_options: Mapping[str, BaseDummyOptions] | None = None,
942 + mm_processor_kwargs: Mapping[str, object] | None = None,
943 ) -> MultiModalDataDict:
944 dummy_image = super().get_dummy_mm_data(
945 seq_len=seq_len, mm_counts=mm_counts, mm_options=mm_options
vllm/model_executor/models/isaac.py
+1
@@ -850,6 +850,7 @@ class IsaacDummyInputsBuilder(BaseDummyInputsBuilder[IsaacProcessingInfo]):
850 seq_len: int,
851 mm_counts: Mapping[str, int],
852 mm_options: Mapping[str] | None = None,
853 + mm_processor_kwargs: Mapping[str, object] | None = None,
854 ) -> MultiModalDataDict:
855 num_images = mm_counts.get("image", 0)
856
vllm/model_executor/models/kanana_v.py
+1
@@ -445,6 +445,7 @@ class KananaVDummyInputsBuilder(BaseDummyInputsBuilder[KananaVProcessingInfo]):
445 seq_len: int,
446 mm_counts: Mapping[str, int],
447 mm_options: Mapping[str, BaseDummyOptions] | None = None,
448 + mm_processor_kwargs: Mapping[str, object] | None = None,
449 ) -> MultiModalDataDict:
450 num_images = mm_counts.get("image", 0)
451 return {
vllm/model_executor/models/keye.py
+1
@@ -1159,6 +1159,7 @@ class KeyeBaseDummyInputsBuilder(BaseDummyInputsBuilder[_I]):
1159 seq_len: int,
1160 mm_counts: Mapping[str, int],
1161 mm_options: Mapping[str, BaseDummyOptions] | None = None,
1162 + mm_processor_kwargs: Mapping[str, object] | None = None,
1163 ) -> MultiModalDataDict:
1164 num_images = mm_counts.get("image", 0)
1165 num_videos = mm_counts.get("video", 0)
vllm/model_executor/models/kimi_k25.py
+1
@@ -238,6 +238,7 @@ class KimiK25DummyInputsBuilder(BaseDummyInputsBuilder[KimiK25ProcessingInfo]):
238 seq_len: int,
239 mm_counts: Mapping[str, int],
240 mm_options: Mapping[str, BaseDummyOptions] | None = None,
241 + mm_processor_kwargs: Mapping[str, object] | None = None,
242 ) -> MultiModalDataDict:
243 # TODO: Support mm_options for vision_chunk to allow user configuration
244 dummy_items = self.get_dummy_mm_items()
vllm/model_executor/models/kimi_vl.py
+1
@@ -216,6 +216,7 @@ class KimiVLDummyInputsBuilder(BaseDummyInputsBuilder[KimiVLProcessingInfo]):
216 seq_len: int,
217 mm_counts: Mapping[str, int],
218 mm_options: Mapping[str, BaseDummyOptions] | None = None,
219 + mm_processor_kwargs: Mapping[str, object] | None = None,
220 ) -> MultiModalDataDict:
221 num_images = mm_counts.get("image", 0)
222
vllm/model_executor/models/lfm2_vl.py
+1
@@ -319,6 +319,7 @@ class Lfm2VLDummyInputsBuilder(BaseDummyInputsBuilder[Lfm2VLProcessingInfo]):
319 seq_len: int,
320 mm_counts: Mapping[str, int],
321 mm_options: Mapping[str, BaseDummyOptions] | None = None,
322 + mm_processor_kwargs: Mapping[str, object] | None = None,
323 ) -> MultiModalDataDict:
324 num_images = mm_counts.get("image", 0)
325
vllm/model_executor/models/llava.py
+1
@@ -232,6 +232,7 @@ class LlavaDummyInputsBuilder(BaseDummyInputsBuilder[_I]):
232 seq_len: int,
233 mm_counts: Mapping[str, int],
234 mm_options: Mapping[str, BaseDummyOptions] | None = None,
235 + mm_processor_kwargs: Mapping[str, object] | None = None,
236 ) -> MultiModalDataDict:
237 num_images = mm_counts.get("image", 0)
238
vllm/model_executor/models/llava_next_video.py
+1
@@ -166,6 +166,7 @@ class LlavaNextVideoDummyInputsBuilder(
166 seq_len: int,
167 mm_counts: Mapping[str, int],
168 mm_options: Mapping[str, BaseDummyOptions] | None = None,
169 + mm_processor_kwargs: Mapping[str, object] | None = None,
170 ) -> MultiModalDataDict:
171 num_videos = mm_counts.get("video", 0)
172
vllm/model_executor/models/llava_onevision.py
+1
@@ -277,6 +277,7 @@ class LlavaOnevisionDummyInputsBuilder(
277 seq_len: int,
278 mm_counts: Mapping[str, int],
279 mm_options: Mapping[str, BaseDummyOptions] | None = None,
280 + mm_processor_kwargs: Mapping[str, object] | None = None,
281 ) -> MultiModalDataDict:
282 num_images = mm_counts.get("image", 0)
283 num_videos = mm_counts.get("video", 0)
vllm/model_executor/models/midashenglm.py
+1
@@ -566,6 +566,7 @@ class MiDashengLMDummyInputsBuilder(BaseDummyInputsBuilder[MiDashengLMProcessing
566 seq_len: int,
567 mm_counts: Mapping[str, int],
568 mm_options: Mapping[str, BaseDummyOptions] | None = None,
569 + mm_processor_kwargs: Mapping[str, object] | None = None,
570 ) -> MultiModalDataDict:
571 num_audios = mm_counts.get("audio", 0)
572
vllm/model_executor/models/minicpmo.py
+1
@@ -302,6 +302,7 @@ class MiniCPMODummyInputsBuilder(MiniCPMVDummyInputsBuilder[MiniCPMOProcessingIn
302 seq_len: int,
303 mm_counts: Mapping[str, int],
304 mm_options: Mapping[str, BaseDummyOptions] | None = None,
305 + mm_processor_kwargs: Mapping[str, object] | None = None,
306 ) -> MultiModalDataDict:
307 num_audios = mm_counts.get("audio", 0)
308 audio_len = (
vllm/model_executor/models/minicpmv.py
+1
@@ -708,6 +708,7 @@ class MiniCPMVDummyInputsBuilder(BaseDummyInputsBuilder[_I]):
708 seq_len: int,
709 mm_counts: Mapping[str, int],
710 mm_options: Mapping[str, BaseDummyOptions] | None = None,
711 + mm_processor_kwargs: Mapping[str, object] | None = None,
712 ) -> MultiModalDataDict:
713 num_images = mm_counts.get("image", 0)
714 num_videos = mm_counts.get("video", 0)
vllm/model_executor/models/mistral3.py
+1
@@ -237,6 +237,7 @@ class Mistral3DummyInputsBuilder(BaseDummyInputsBuilder[_I]):
237 seq_len: int,
238 mm_counts: Mapping[str, int],
239 mm_options: Mapping[str, BaseDummyOptions] | None = None,
240 + mm_processor_kwargs: Mapping[str, object] | None = None,
241 ) -> MultiModalDataDict:
242 num_images = mm_counts.get("image", 0)
243
vllm/model_executor/models/mllama4.py
+1
@@ -704,6 +704,7 @@ class Mllama4DummyInputsBuilder(BaseDummyInputsBuilder[Mllama4ProcessingInfo]):
704 seq_len: int,
705 mm_counts: Mapping[str, int],
706 mm_options: Mapping[str, BaseDummyOptions] | None = None,
707 + mm_processor_kwargs: Mapping[str, object] | None = None,
708 ) -> MultiModalDataDict:
709 num_images = mm_counts.get("image", 0)
710
vllm/model_executor/models/molmo.py
+1
@@ -1278,6 +1278,7 @@ class MolmoDummyInputsBuilder(BaseDummyInputsBuilder[MolmoProcessingInfo]):
1278 seq_len: int,
1279 mm_counts: Mapping[str, int],
1280 mm_options: Mapping[str, BaseDummyOptions] | None = None,
1281 + mm_processor_kwargs: Mapping[str, object] | None = None,
1282 ) -> MultiModalDataDict:
1283 target_width, target_height = self.info.get_image_size_with_most_features()
1284 num_images = mm_counts.get("image", 0)
vllm/model_executor/models/molmo2.py
+1
@@ -2079,6 +2079,7 @@ class Molmo2DummyInputsBuilder(BaseDummyInputsBuilder[Molmo2ProcessingInfo]):
2079 seq_len: int,
2080 mm_counts: Mapping[str, int],
2081 mm_options: Mapping[str, BaseDummyOptions] | None = None,
2082 + mm_processor_kwargs: Mapping[str, object] | None = None,
2083 ) -> MultiModalDataDict:
2084 num_images = mm_counts.get("image", 0)
2085 num_videos = mm_counts.get("video", 0)
vllm/model_executor/models/nano_nemotron_vl.py
+2
@@ -1385,6 +1385,7 @@ class NanoNemotronVLDummyInputsBuilder(BaseDummyInputsBuilder[_I]):
1385 seq_len: int,
1386 mm_counts: Mapping[str, int],
1387 mm_options: Mapping[str, BaseDummyOptions] | None = None,
1388 + mm_processor_kwargs: Mapping[str, object] | None = None,
1389 ) -> MultiModalDataDict:
1390 num_images = mm_counts.get("image", 0)
1391 processor = self.info.get_hf_processor()
@@ -1457,6 +1458,7 @@ class NanoNemotronVLDummyInputsBuilder(
1458 seq_len: int,
1459 mm_counts: Mapping[str, int],
1460 mm_options: Mapping[str, BaseDummyOptions] | None = None,
1461 + mm_processor_kwargs: Mapping[str, object] | None = None,
1462 ) -> MultiModalDataDict:
1463 dummy_image = super().get_dummy_mm_data(
1464 seq_len=seq_len, mm_counts=mm_counts, mm_options=mm_options
vllm/model_executor/models/nemotron_parse.py
+1
@@ -642,6 +642,7 @@ class NemotronParseDummyInputsBuilder(
642 seq_len: int,
643 mm_counts: Mapping[str, int],
644 mm_options: Mapping[str, BaseDummyOptions] | None = None,
645 + mm_processor_kwargs: Mapping[str, object] | None = None,
646 ) -> MultiModalDataDict:
647 num_images = mm_counts.get("image", 0)
648
vllm/model_executor/models/nvlm_d.py
+1
@@ -93,6 +93,7 @@ class NVLMDummyInputsBuilder(BaseInternVLDummyInputsBuilder[NVLMProcessingInfo])
93 seq_len: int,
94 mm_counts: Mapping[str, int],
95 mm_options: Mapping[str, BaseDummyOptions] | None = None,
96 + mm_processor_kwargs: Mapping[str, object] | None = None,
97 ) -> MultiModalDataDict:
98 target_width, target_height = self.info.get_image_size_with_most_features()
99 num_images = mm_counts.get("image", 0)
vllm/model_executor/models/ovis.py
+1
@@ -303,6 +303,7 @@ class OvisDummyInputsBuilder(BaseDummyInputsBuilder[OvisProcessingInfo]):
303 seq_len: int,
304 mm_counts: Mapping[str, int],
305 mm_options: Mapping[str, BaseDummyOptions] | None = None,
306 + mm_processor_kwargs: Mapping[str, object] | None = None,
307 ) -> MultiModalDataDict:
308 num_images = mm_counts.get("image", 0)
309
vllm/model_executor/models/ovis2_5.py
+1
@@ -302,6 +302,7 @@ class Ovis2_5DummyInputsBuilder(BaseDummyInputsBuilder[Ovis2_5ProcessingInfo]):
302 seq_len: int,
303 mm_counts: Mapping[str, int],
304 mm_options: Mapping[str, BaseDummyOptions] | None = None,
305 + mm_processor_kwargs: Mapping[str, object] | None = None,
306 ) -> MultiModalDataDict:
307 num_images = mm_counts.get("image", 0)
308 num_videos = mm_counts.get("video", 0)
vllm/model_executor/models/paddleocr_vl.py
+1
@@ -204,6 +204,7 @@ class PaddleOCRVLDummyInputsBuilder(BaseDummyInputsBuilder[PaddleOCRVLProcessing
204 seq_len: int,
205 mm_counts: Mapping[str, int],
206 mm_options: Mapping[str, BaseDummyOptions] | None = None,
207 + mm_processor_kwargs: Mapping[str, object] | None = None,
208 ) -> MultiModalDataDict:
209 num_images = mm_counts.get("image", 0)
210
vllm/model_executor/models/paligemma.py
+1
@@ -128,6 +128,7 @@ class PaliGemmaDummyInputsBuilder(BaseDummyInputsBuilder[PaliGemmaProcessingInfo
128 seq_len: int,
129 mm_counts: Mapping[str, int],
130 mm_options: Mapping[str, BaseDummyOptions] | None = None,
131 + mm_processor_kwargs: Mapping[str, object] | None = None,
132 ) -> MultiModalDataDict:
133 hf_config = self.info.get_hf_config()
134 vision_config = hf_config.vision_config
vllm/model_executor/models/phi3v.py
+1
@@ -380,6 +380,7 @@ class Phi3VDummyInputsBuilder(BaseDummyInputsBuilder[Phi3VProcessingInfo]):
380 seq_len: int,
381 mm_counts: Mapping[str, int],
382 mm_options: Mapping[str, BaseDummyOptions] | None = None,
383 + mm_processor_kwargs: Mapping[str, object] | None = None,
384 ) -> MultiModalDataDict:
385 num_images = mm_counts.get("image", 0)
386
vllm/model_executor/models/phi4mm.py
+1
@@ -826,6 +826,7 @@ class Phi4MMDummyInputsBuilder(BaseDummyInputsBuilder[Phi4MMProcessingInfo]):
826 seq_len: int,
827 mm_counts: Mapping[str, int],
828 mm_options: Mapping[str, BaseDummyOptions] | None = None,
829 + mm_processor_kwargs: Mapping[str, object] | None = None,
830 ) -> MultiModalDataDict:
831 num_audios = mm_counts.get("audio", 0)
832 num_images = mm_counts.get("image", 0)
vllm/model_executor/models/pixtral.py
+2
@@ -261,6 +261,7 @@ class PixtralDummyInputsBuilder(BaseDummyInputsBuilder[PixtralProcessingInfo]):
261 seq_len: int,
262 mm_counts: Mapping[str, int],
263 mm_options: Mapping[str, BaseDummyOptions] | None = None,
264 + mm_processor_kwargs: Mapping[str, object] | None = None,
265 ) -> MultiModalDataDict:
266 num_images = mm_counts.get("image", 0)
267
@@ -282,6 +283,7 @@ class PixtralDummyInputsBuilder(BaseDummyInputsBuilder[PixtralProcessingInfo]):
283 seq_len: int,
284 mm_counts: Mapping[str, int],
285 mm_options: Mapping[str, BaseDummyOptions] | None = None,
286 + mm_processor_kwargs: Mapping[str, object] | None = None,
287 ) -> ProcessorInputs:
288 tokenizer = self.info.get_tokenizer()
289
vllm/model_executor/models/qwen2_5_omni_thinker.py
+7 -2
@@ -358,12 +358,14 @@ class Qwen2_5OmniThinkerDummyInputsBuilder(
358 seq_len: int,
359 mm_counts: Mapping[str, int],
360 mm_options: Mapping[str, BaseDummyOptions] | None = None,
361 + mm_processor_kwargs: Mapping[str, object] | None = None,
362 ) -> MultiModalDataDict:
363 num_audios = mm_counts.get("audio", 0)
364 num_images = mm_counts.get("image", 0)
365 num_videos = mm_counts.get("video", 0)
366
366 - feature_extractor = self.info.get_feature_extractor()
367 + mm_processor_kwargs = mm_processor_kwargs or {}
368 + feature_extractor = self.info.get_feature_extractor(**mm_processor_kwargs)
369
370 target_audio_length = (
371 min(
@@ -372,7 +374,10 @@ class Qwen2_5OmniThinkerDummyInputsBuilder(
374 )
375 * feature_extractor.sampling_rate
376 )
375 - target_width, target_height = self.info.get_image_size_with_most_features()
377 +
378 + target_width, target_height = self.info.get_image_size_with_most_features(
379 + max_pixels=mm_processor_kwargs.get("max_pixels", None),
380 + )
381 target_num_frames = self.info.get_num_frames_with_most_features(
382 seq_len, mm_counts
383 )
vllm/model_executor/models/qwen2_audio.py
+4 -1
@@ -195,8 +195,11 @@ class Qwen2AudioDummyInputsBuilder(BaseDummyInputsBuilder[Qwen2AudioProcessingIn
195 seq_len: int,
196 mm_counts: Mapping[str, int],
197 mm_options: Mapping[str, BaseDummyOptions] | None = None,
198 + mm_processor_kwargs: Mapping[str, object] | None = None,
199 ) -> MultiModalDataDict:
199 - feature_extractor = self.info.get_feature_extractor()
200 + feature_extractor = self.info.get_feature_extractor(
201 + **(mm_processor_kwargs or {})
202 + )
203
204 sampling_rate = feature_extractor.sampling_rate
205 audio_len = feature_extractor.chunk_length * sampling_rate
vllm/model_executor/models/qwen2_vl.py
+5 -1
@@ -1016,11 +1016,15 @@ class Qwen2VLDummyInputsBuilder(BaseDummyInputsBuilder[Qwen2VLProcessingInfo]):
1016 seq_len: int,
1017 mm_counts: Mapping[str, int],
1018 mm_options: Mapping[str, BaseDummyOptions] | None = None,
1019 + mm_processor_kwargs: Mapping[str, object] | None = None,
1020 ) -> MultiModalDataDict:
1021 num_images = mm_counts.get("image", 0)
1022 num_videos = mm_counts.get("video", 0)
1023
1023 - target_width, target_height = self.info.get_image_size_with_most_features()
1024 + mm_processor_kwargs = mm_processor_kwargs or {}
1025 + target_width, target_height = self.info.get_image_size_with_most_features(
1026 + max_pixels=mm_processor_kwargs.get("max_pixels", None)
1027 + )
1028 target_num_frames = self.info.get_num_frames_with_most_features(
1029 seq_len, mm_counts
1030 )
vllm/model_executor/models/qwen3_asr.py
+4 -1
@@ -147,10 +147,13 @@ class Qwen3ASRDummyInputsBuilder(BaseDummyInputsBuilder[Qwen3ASRProcessingInfo])
147 seq_len: int,
148 mm_counts: Mapping[str, int],
149 mm_options: Mapping[str, BaseDummyOptions] | None = None,
150 + mm_processor_kwargs: Mapping[str, object] | None = None,
151 ) -> MultiModalDataDict:
152 num_audios = mm_counts.get("audio", 0)
153
153 - feature_extractor = self.info.get_feature_extractor()
154 + feature_extractor = self.info.get_feature_extractor(
155 + **(mm_processor_kwargs or {})
156 + )
157
158 target_audio_length = (
159 min(
vllm/model_executor/models/qwen3_omni_moe_thinker.py
+1 -1
@@ -1169,7 +1169,7 @@ class Qwen3OmniMoeThinkerMultiModalProcessor(
1169 return x
1170
1171 # NOTE: WhisperFeatureExtractor cannot handle empty list of audios
1172 - feature_extractor = self.info.get_feature_extractor()
1172 + feature_extractor = self.info.get_feature_extractor(**mm_kwargs)
1173 hop_length = feature_extractor.hop_length
1174 if audios:
1175 # NOTE: Qwen3-Omni processor accept "audio"
vllm/model_executor/models/qwen3_vl.py
+6 -2
@@ -796,14 +796,18 @@ class Qwen3VLDummyInputsBuilder(BaseDummyInputsBuilder[Qwen3VLProcessingInfo]):
796 seq_len: int,
797 mm_counts: Mapping[str, int],
798 mm_options: Mapping[str, BaseDummyOptions] | None = None,
799 + mm_processor_kwargs: Mapping[str, object] | None = None,
800 ) -> MultiModalDataDict:
801 num_images = mm_counts.get("image", 0)
802 num_videos = mm_counts.get("video", 0)
803 image_overrides = mm_options.get("image") if mm_options else None
804 video_overrides = mm_options.get("video") if mm_options else None
805
806 + mm_processor_kwargs = mm_processor_kwargs or {}
807 target_image_width, target_image_height = (
806 - self.info.get_image_size_with_most_features()
808 + self.info.get_image_size_with_most_features(
809 + max_pixels=mm_processor_kwargs.get("max_pixels", None),
810 + )
811 )
812
813 # treat videos as special images
@@ -828,7 +832,7 @@ class Qwen3VLDummyInputsBuilder(BaseDummyInputsBuilder[Qwen3VLProcessingInfo]):
832 target_num_frames = min(target_num_frames, num_frames_override)
833 target_num_frames = max(target_num_frames, 2)
834
831 - video_processor = self.info.get_video_processor()
835 + video_processor = self.info.get_video_processor(**(mm_processor_kwargs or {}))
836 video_max_pixels = video_processor.size["longest_edge"]
837 # video_max_pixels contains the temporal compression factor,
838 # so we divide by 2 to get the maximum number of image pixels.
vllm/model_executor/models/qwen_vl.py
+1
@@ -618,6 +618,7 @@ class QwenVLDummyInputsBuilder(BaseDummyInputsBuilder[QwenVLProcessingInfo]):
618 seq_len: int,
619 mm_counts: Mapping[str, int],
620 mm_options: Mapping[str, BaseDummyOptions] | None = None,
621 + mm_processor_kwargs: Mapping[str, object] | None = None,
622 ) -> MultiModalDataDict:
623 hf_config = self.info.get_hf_config()
624 vision_config = hf_config.visual
vllm/model_executor/models/rvl.py
+1
@@ -41,6 +41,7 @@ class RVLDummyInputsBuilder(LlavaDummyInputsBuilder[RVLProcessingInfo]):
41 seq_len: int,
42 mm_counts: Mapping[str, int],
43 mm_options: Mapping[str, BaseDummyOptions] | None = None,
44 + mm_processor_kwargs: Mapping[str, object] | None = None,
45 ) -> MultiModalDataDict:
46 num_images = mm_counts.get("image", 0)
47
vllm/model_executor/models/siglip.py
+1
@@ -155,6 +155,7 @@ class SiglipDummyInputsBuilder(BaseDummyInputsBuilder[SiglipProcessingInfo]):
155 seq_len: int,
156 mm_counts: Mapping[str, int],
157 mm_options: Mapping[str, BaseDummyOptions] | None = None,
158 + mm_processor_kwargs: Mapping[str, object] | None = None,
159 ) -> MultiModalDataDict:
160 num_images = mm_counts.get("image", 0)
161
vllm/model_executor/models/skyworkr1v.py
+1
@@ -533,6 +533,7 @@ class SkyworkR1VDummyInputsBuilder(BaseDummyInputsBuilder[SkyworkR1VProcessingIn
533 seq_len: int,
534 mm_counts: Mapping[str, int],
535 mm_options: Mapping[str, BaseDummyOptions] | None = None,
536 + mm_processor_kwargs: Mapping[str, object] | None = None,
537 ) -> MultiModalDataDict:
538 target_width, target_height = self.info.get_image_size_with_most_features()
539 num_images = mm_counts.get("image", 0)
vllm/model_executor/models/step3_vl.py
+1
@@ -565,6 +565,7 @@ class Step3VLDummyInputsBuilder(BaseDummyInputsBuilder[Step3VLProcessingInfo]):
565 seq_len: int,
566 mm_counts: Mapping[str, int],
567 mm_options: Mapping[str, BaseDummyOptions] | None = None,
568 + mm_processor_kwargs: Mapping[str, object] | None = None,
569 ) -> MultiModalDataDict:
570 target_width, target_height = self.info.get_image_size_with_most_features()
571 num_images = mm_counts.get("image", 0)
vllm/model_executor/models/terratorch.py
+1
@@ -154,6 +154,7 @@ class TerratorchInputBuilder(BaseDummyInputsBuilder[TerratorchProcessingInfo]):
154 seq_len: int,
155 mm_counts: Mapping[str, int],
156 mm_options: Mapping[str, BaseDummyOptions] | None = None,
157 + mm_processor_kwargs: Mapping[str, object] | None = None,
158 ) -> MultiModalDataDict:
159 # Dummy data is generated based on the 'input' section
160 # defined in the HF configuration file
vllm/model_executor/models/transformers/multimodal.py
+1
@@ -98,6 +98,7 @@ class MultiModalDummyInputsBuilder(BaseDummyInputsBuilder[MultiModalProcessingIn
98 seq_len: int,
99 mm_counts: Mapping[str, int],
100 mm_options: Mapping[str, "BaseDummyOptions"] | None = None,
101 + mm_processor_kwargs: Mapping[str, object] | None = None,
102 ) -> MultiModalDataDict:
103 num_images = mm_counts.get("image", 0)
104
vllm/model_executor/models/ultravox.py
+4 -1
@@ -161,8 +161,11 @@ class UltravoxDummyInputsBuilder(BaseDummyInputsBuilder[UltravoxProcessingInfo])
161 seq_len: int,
162 mm_counts: Mapping[str, int],
163 mm_options: Mapping[str, BaseDummyOptions] | None = None,
164 + mm_processor_kwargs: Mapping[str, object] | None = None,
165 ) -> MultiModalDataDict:
165 - feature_extractor = self.info.get_feature_extractor()
166 + feature_extractor = self.info.get_feature_extractor(
167 + **(mm_processor_kwargs or {})
168 + )
169
170 sampling_rate = feature_extractor.sampling_rate
171 audio_len = (
vllm/model_executor/models/voxtral.py
+2
@@ -220,6 +220,7 @@ class VoxtralDummyInputsBuilder(BaseDummyInputsBuilder[VoxtralProcessingInfo]):
220 seq_len: int,
221 mm_counts: Mapping[str, int],
222 mm_options: Mapping[str, BaseDummyOptions] | None = None,
223 + mm_processor_kwargs: Mapping[str, object] | None = None,
224 ) -> MultiModalDataDict:
225 num_audios = mm_counts.get("audio", 0)
226
@@ -238,6 +239,7 @@ class VoxtralDummyInputsBuilder(BaseDummyInputsBuilder[VoxtralProcessingInfo]):
239 seq_len: int,
240 mm_counts: Mapping[str, int],
241 mm_options: Mapping[str, BaseDummyOptions] | None = None,
242 + mm_processor_kwargs: Mapping[str, object] | None = None,
243 ) -> ProcessorInputs:
244 tokenizer = self.info.get_tokenizer()
245
vllm/model_executor/models/whisper.py
+4 -1
@@ -685,8 +685,11 @@ class WhisperDummyInputsBuilder(BaseDummyInputsBuilder[WhisperProcessingInfo]):
685 seq_len: int,
686 mm_counts: Mapping[str, int],
687 mm_options: Mapping[str, BaseDummyOptions] | None = None,
688 + mm_processor_kwargs: Mapping[str, object] | None = None,
689 ) -> MultiModalDataDict:
689 - feature_extractor = self.info.get_feature_extractor()
690 + feature_extractor = self.info.get_feature_extractor(
691 + **(mm_processor_kwargs or {})
692 + )
693
694 sampling_rate = feature_extractor.sampling_rate
695 audio_len = feature_extractor.chunk_length * sampling_rate
vllm/multimodal/processing/dummy_inputs.py
+11 -1
@@ -63,6 +63,7 @@ class BaseDummyInputsBuilder(ABC, Generic[_I]):
63 seq_len: int,
64 mm_counts: Mapping[str, int],
65 mm_options: Mapping[str, BaseDummyOptions] | None = None,
66 + mm_processor_kwargs: Mapping[str, object] | None = None,
67 ) -> MultiModalDataDict:
68 """
69 Build the multimodal input which, after processing, results in
@@ -83,6 +84,7 @@ class BaseDummyInputsBuilder(ABC, Generic[_I]):
84 seq_len: int,
85 mm_counts: Mapping[str, int],
86 mm_options: Mapping[str, BaseDummyOptions] | None = None,
87 + mm_processor_kwargs: Mapping[str, object] | None = None,
88 ) -> ProcessorInputs:
89 """
90 Build the input which, after processing, results in
@@ -92,9 +94,16 @@ class BaseDummyInputsBuilder(ABC, Generic[_I]):
94 seq_len: Sequence length
95 mm_counts: Count of items per modality
96 mm_options: Configurable options per modality (optional)
97 + mm_processor_kwargs: Additional keyword arguments
98 + for hf_processor (optional)
99 """
100 dummy_text = self.get_dummy_text(mm_counts)
97 - dummy_mm_data = self.get_dummy_mm_data(seq_len, mm_counts, mm_options)
101 + dummy_mm_data = self.get_dummy_mm_data(
102 + seq_len,
103 + mm_counts,
104 + mm_options,
105 + mm_processor_kwargs=mm_processor_kwargs,
106 + )
107 dummy_mm_items = self.info.parse_mm_data(dummy_mm_data, validate=False)
108
109 tokenization_kwargs = {"truncation": False}
@@ -102,6 +111,7 @@ class BaseDummyInputsBuilder(ABC, Generic[_I]):
111 return ProcessorInputs(
112 prompt=dummy_text,
113 mm_items=dummy_mm_items,
114 + hf_processor_mm_kwargs=mm_processor_kwargs or {},
115 tokenization_kwargs=tokenization_kwargs,
116 )
117
vllm/multimodal/registry.py
+2
@@ -257,10 +257,12 @@ class MultiModalRegistry:
257 if processor is None:
258 processor = self.create_processor(model_config, cache=cache)
259
260 + mm_config = model_config.get_multimodal_config()
261 processor_inputs = processor.dummy_inputs.get_dummy_processor_inputs(
262 seq_len=seq_len,
263 mm_counts=mm_counts,
264 mm_options=self._extract_mm_options(model_config),
265 + mm_processor_kwargs=mm_config.mm_processor_kwargs,
266 )
267 mm_inputs = processor.apply(
268 prompt=processor_inputs.prompt,