@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 987506bca6
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 7, 22:45 UTC

[Refactor] Simplify dummy data generation (#35025)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>

Cyrus Leung committed Feb 23, 2026 at 12:55 UTC 987506bca63d95d9ff4ebb3a37e3f2447fa0757c
78 files changed +282 -367
docs/contributing/model/multimodal.md
+11 -11
@@ -293,21 +293,22 @@ Assuming that the memory usage increases with the number of tokens, the dummy in
293 self,
294 seq_len: int,
295 mm_counts: Mapping[str, int],
296 - mm_options: Mapping[str, BaseDummyOptions] | None = None,
296 + mm_options: Mapping[str, BaseDummyOptions],
297 ) -> MultiModalDataDict:
298 num_images = mm_counts.get("image", 0)
299
300 target_width, target_height = \
301 self.info.get_image_size_with_most_features()
302
303 - image_overrides = mm_options.get("image") if mm_options else None
303 + image_overrides = mm_options.get("image")
304
305 return {
306 - "image":
307 - self._get_dummy_images(width=target_width,
308 - height=target_height,
309 - num_images=num_images,
310 - overrides=image_overrides)
306 + "image": self._get_dummy_images(
307 + width=target_width,
308 + height=target_height,
309 + num_images=num_images,
310 + overrides=image_overrides,
311 + )
312 }
313 ```
314
@@ -479,17 +480,16 @@ Assuming that the memory usage increases with the number of tokens, the dummy in
480 self,
481 seq_len: int,
482 mm_counts: Mapping[str, int],
482 - mm_options: Optional[Mapping[str, BaseDummyOptions]] = None,
483 + mm_options: Mapping[str, BaseDummyOptions],
484 ) -> MultiModalDataDict:
485 target_width, target_height = \
486 self.info.get_image_size_with_most_features()
487 num_images = mm_counts.get("image", 0)
488
488 - image_overrides = mm_options.get("image") if mm_options else None
489 + image_overrides = mm_options.get("image")
490
491 return {
491 - "image":
492 - self._get_dummy_images(
492 + "image": self._get_dummy_images(
493 width=target_width,
494 height=target_height,
495 num_images=num_images,
tests/models/multimodal/processing/test_audioflamingo3.py
+1 -1
@@ -116,7 +116,7 @@ def test_dummy_data_generation(mock_ctx):
116 builder = AudioFlamingo3DummyInputsBuilder(info)
117
118 mm_counts = {"audio": 2}
119 - dummy_data = builder.get_dummy_mm_data(100, mm_counts, None)
119 + dummy_data = builder.get_dummy_mm_data(100, mm_counts, {})
120
121 assert "audio" in dummy_data
122 assert len(dummy_data["audio"]) == 2
tests/models/multimodal/processing/test_common.py
+2
@@ -195,6 +195,7 @@ def get_text_token_prompts(
195 inputs = dummy_inputs.get_dummy_processor_inputs(
196 model_config.max_model_len,
197 mm_counts,
198 + mm_options={},
199 )
200 text_prompt = None
201 token_prompt = (
@@ -224,6 +225,7 @@ def get_text_token_prompts(
225 inputs = dummy_inputs.get_dummy_processor_inputs(
226 model_config.max_model_len,
227 mm_counts,
228 + mm_options={},
229 )
230 assert isinstance(inputs.prompt, str)
231
tests/models/multimodal/processing/test_tensor_schema.py
+1
@@ -97,6 +97,7 @@ def create_batched_mm_kwargs(
97 processor_inputs = dummy_inputs.get_dummy_processor_inputs(
98 seq_len=model_config.max_model_len,
99 mm_counts=mm_counts,
100 + mm_options={},
101 )
102 mm_items = processor_inputs.mm_items
103 resized_mm_data = {
vllm/config/multimodal.py
+36 -20
@@ -2,7 +2,7 @@
2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
4 from collections.abc import Mapping
5 -from typing import Any, Literal, TypeAlias
5 +from typing import Any, Literal, TypeAlias, TypedDict, final
6
7 from pydantic import ConfigDict, Field, field_validator, model_validator
8 from pydantic.dataclasses import dataclass
@@ -43,11 +43,29 @@ class AudioDummyOptions(BaseDummyOptions):
43 length: int | None = Field(None, gt=0)
44
45
46 +@final
47 +class MultiModalDummyOptionsBuiltins(TypedDict, total=False):
48 + """Type annotations for modality types predefined by vLLM."""
49 +
50 + image: ImageDummyOptions
51 + """Options for dummy images."""
52 +
53 + video: VideoDummyOptions
54 + """Options for dummy videos."""
55 +
56 + audio: AudioDummyOptions
57 + """Options for dummy audios."""
58 +
59 +
60 MMEncoderTPMode = Literal["weights", "data"]
61 MMCacheType = Literal["shm", "lru"]
48 -DummyOptions: TypeAlias = (
49 - BaseDummyOptions | VideoDummyOptions | ImageDummyOptions | AudioDummyOptions
50 -)
62 +MMDummyOptions: TypeAlias = dict[str, BaseDummyOptions]
63 +"""
64 +A dictionary containing an entry for each modality type of dummy data.
65 +
66 +The built-in modalities are defined by
67 +[`MultiModalDummyOptionsBuiltins`][vllm.config.multimodal.MultiModalDummyOptionsBuiltins].
68 +"""
69
70
71 @config
@@ -57,7 +75,7 @@ class MultiModalConfig:
75 language_model_only: bool = False
76 """If True, disables all multimodal inputs by setting all modality limits to 0.
77 Equivalent to setting `--limit-mm-per-prompt` to 0 for every modality."""
60 - limit_per_prompt: dict[str, DummyOptions] = Field(default_factory=dict)
78 + limit_per_prompt: MMDummyOptions = Field(default_factory=dict)
79 """The maximum number of input items and options allowed per
80 prompt for each modality.
81
@@ -158,22 +176,27 @@ class MultiModalConfig:
176 @field_validator("limit_per_prompt", mode="before")
177 @classmethod
178 def _validate_limit_per_prompt(
161 - cls, value: dict[str, int | dict[str, int]]
162 - ) -> dict[str, DummyOptions]:
179 + cls,
180 + value: dict[str, int | dict[str, int]],
181 + ) -> MMDummyOptions:
182 + out: MMDummyOptions = {}
183 +
184 for k, v in value.items():
185 # Handle legacy format where only count is specified
186 if isinstance(v, int):
187 v = {"count": v}
188 +
189 # Convert to the appropriate DummyOptions subclass
190 if k == "video":
169 - value[k] = VideoDummyOptions(**v)
191 + out[k] = VideoDummyOptions(**v)
192 elif k == "image":
171 - value[k] = ImageDummyOptions(**v)
193 + out[k] = ImageDummyOptions(**v)
194 elif k == "audio":
173 - value[k] = AudioDummyOptions(**v)
195 + out[k] = AudioDummyOptions(**v)
196 else:
175 - value[k] = BaseDummyOptions(**v)
176 - return value
197 + out[k] = BaseDummyOptions(**v)
198 +
199 + return out
200
201 @field_validator("mm_encoder_attn_backend", mode="before")
202 @classmethod
@@ -240,15 +263,8 @@ class MultiModalConfig:
263 if limit_data is None:
264 # Unspecified modality is set to 999 by default
265 return 999
243 - return limit_data.count
266
245 - def get_dummy_options(self, modality: str) -> BaseDummyOptions | None:
246 - """
247 - Get the configurable dummy data options for a modality.
248 - Returns None if no options are configured for this modality.
249 - """
250 - # All values are now DummyOptions after normalization
251 - return self.limit_per_prompt.get(modality)
267 + return limit_data.count
268
269 def merge_mm_processor_kwargs(
270 self,
vllm/model_executor/models/aria.py
+2 -3
@@ -444,15 +444,14 @@ class AriaDummyInputsBuilder(BaseDummyInputsBuilder[AriaProcessingInfo]):
444 self,
445 seq_len: int,
446 mm_counts: Mapping[str, int],
447 - mm_options: Mapping[str, BaseDummyOptions] | None = None,
448 - mm_processor_kwargs: Mapping[str, object] | None = None,
447 + mm_options: Mapping[str, BaseDummyOptions],
448 ) -> MultiModalDataDict:
449 vision_config = self.info.get_vision_config()
450
451 max_image_size = vision_config.image_size
452 num_images = mm_counts.get("image", 0)
453
455 - image_overrides = mm_options.get("image") if mm_options else None
454 + image_overrides = mm_options.get("image")
455
456 return {
457 "image": self._get_dummy_images(
vllm/model_executor/models/audioflamingo3.py
+3 -6
@@ -252,16 +252,13 @@ class AudioFlamingo3DummyInputsBuilder(
252 self,
253 seq_len: int,
254 mm_counts: Mapping[str, int],
255 - mm_options: Mapping[str, BaseDummyOptions] | None = None,
256 - mm_processor_kwargs: Mapping[str, object] | None = None,
255 + mm_options: Mapping[str, BaseDummyOptions],
256 ) -> MultiModalDataDict:
258 - feature_extractor = self.info.get_feature_extractor(
259 - **(mm_processor_kwargs or {})
260 - )
257 + feature_extractor = self.info.get_feature_extractor()
258 sampling_rate = feature_extractor.sampling_rate
259 audio_len = MAX_AUDIO_LEN * sampling_rate
260 num_audios = mm_counts.get("audio", 0)
264 - audio_overrides = mm_options.get("audio") if mm_options else None
261 + audio_overrides = mm_options.get("audio")
262
263 return {
264 "audio": self._get_dummy_audios(
vllm/model_executor/models/aya_vision.py
+2 -3
@@ -191,13 +191,12 @@ class AyaVisionDummyInputsBuilder(BaseDummyInputsBuilder[AyaVisionProcessingInfo
191 self,
192 seq_len: int,
193 mm_counts: Mapping[str, int],
194 - mm_options: Mapping[str, BaseDummyOptions] | None = None,
195 - mm_processor_kwargs: Mapping[str, object] | None = None,
194 + mm_options: Mapping[str, BaseDummyOptions],
195 ) -> MultiModalDataDict:
196 num_images = mm_counts.get("image", 0)
197 image_size = self.info.get_image_size_with_most_features()
198
200 - image_overrides = mm_options.get("image") if mm_options else None
199 + image_overrides = mm_options.get("image")
200
201 return {
202 "image": self._get_dummy_images(
vllm/model_executor/models/bagel.py
+2 -3
@@ -249,8 +249,7 @@ class BagelDummyInputsBuilder(BaseDummyInputsBuilder[BagelProcessingInfo]):
249 self,
250 seq_len: int,
251 mm_counts: Mapping[str, int],
252 - mm_options: Mapping[str, BaseDummyOptions] | None = None,
253 - mm_processor_kwargs: Mapping[str, object] | None = None,
252 + mm_options: Mapping[str, BaseDummyOptions],
253 ) -> MultiModalDataDict:
254 num_images = mm_counts.get("image", 0)
255 hf_config = self.info.get_hf_config()
@@ -258,7 +257,7 @@ class BagelDummyInputsBuilder(BaseDummyInputsBuilder[BagelProcessingInfo]):
257
258 # Use the configured image size
259 image_size = vit_config.image_size
261 - image_overrides = mm_options.get("image") if mm_options else None
260 + image_overrides = mm_options.get("image")
261
262 return {
263 "image": self._get_dummy_images(
vllm/model_executor/models/bee.py
+2 -3
@@ -90,14 +90,13 @@ class BeeDummyInputsBuilder(LlavaDummyInputsBuilder[BeeProcessingInfo]):
90 self,
91 seq_len: int,
92 mm_counts: Mapping[str, int],
93 - mm_options: Mapping[str, BaseDummyOptions] | None = None,
94 - mm_processor_kwargs: Mapping[str, object] | None = None,
93 + mm_options: Mapping[str, BaseDummyOptions],
94 ) -> MultiModalDataDict:
95 num_images = mm_counts.get("image", 0)
96
97 target_width, target_height = self.info.get_image_size_with_most_features()
98
100 - image_overrides = mm_options.get("image") if mm_options else None
99 + image_overrides = mm_options.get("image")
100
101 return {
102 "image": self._get_dummy_images(
vllm/model_executor/models/blip2.py
+2 -3
@@ -445,8 +445,7 @@ class Blip2DummyInputsBuilder(BaseDummyInputsBuilder[Blip2ProcessingInfo]):
445 self,
446 seq_len: int,
447 mm_counts: Mapping[str, int],
448 - mm_options: Mapping[str, BaseDummyOptions] | None = None,
449 - mm_processor_kwargs: Mapping[str, object] | None = None,
448 + mm_options: Mapping[str, BaseDummyOptions],
449 ) -> MultiModalDataDict:
450 hf_config = self.info.get_hf_config()
451 vision_config = hf_config.vision_config
@@ -454,7 +453,7 @@ class Blip2DummyInputsBuilder(BaseDummyInputsBuilder[Blip2ProcessingInfo]):
453 max_image_size = vision_config.image_size
454 num_images = mm_counts.get("image", 0)
455
457 - image_overrides = mm_options.get("image") if mm_options else None
456 + image_overrides = mm_options.get("image")
457
458 return {
459 "image": self._get_dummy_images(
vllm/model_executor/models/chameleon.py
+2 -3
@@ -116,15 +116,14 @@ class ChameleonDummyInputsBuilder(BaseDummyInputsBuilder[ChameleonProcessingInfo
116 self,
117 seq_len: int,
118 mm_counts: Mapping[str, int],
119 - mm_options: Mapping[str, BaseDummyOptions] | None = None,
120 - mm_processor_kwargs: Mapping[str, object] | None = None,
119 + mm_options: Mapping[str, BaseDummyOptions],
120 ) -> MultiModalDataDict:
121 config = self.info.get_hf_config()
122
123 width = height = config.vq_config.resolution
124 num_images = mm_counts.get("image", 0)
125
127 - image_overrides = mm_options.get("image") if mm_options else None
126 + image_overrides = mm_options.get("image")
127
128 return {
129 "image": self._get_dummy_images(
vllm/model_executor/models/clip.py
+2 -3
@@ -174,14 +174,13 @@ class CLIPDummyInputsBuilder(BaseDummyInputsBuilder[CLIPProcessingInfo]):
174 self,
175 seq_len: int,
176 mm_counts: Mapping[str, int],
177 - mm_options: Mapping[str, BaseDummyOptions] | None = None,
178 - mm_processor_kwargs: Mapping[str, object] | None = None,
177 + mm_options: Mapping[str, BaseDummyOptions],
178 ) -> MultiModalDataDict:
179 num_images = mm_counts.get("image", 0)
180
181 target_width, target_height = self.info.get_image_size_with_most_features()
182
184 - image_overrides = mm_options.get("image") if mm_options else None
183 + image_overrides = mm_options.get("image")
184
185 return {
186 "image": self._get_dummy_images(
vllm/model_executor/models/cohere2_vision.py
+2 -3
@@ -197,13 +197,12 @@ class Cohere2VisionDummyInputsBuilder(
197 self,
198 seq_len: int,
199 mm_counts: Mapping[str, int],
200 - mm_options: Mapping[str, BaseDummyOptions] | None = None,
201 - mm_processor_kwargs: Mapping[str, object] | None = None,
200 + mm_options: Mapping[str, BaseDummyOptions],
201 ) -> MultiModalDataDict:
202 num_images = mm_counts.get("image", 0)
203 image_size = self.info.get_image_size_with_most_features()
204
206 - image_overrides = mm_options.get("image") if mm_options else None
205 + image_overrides = mm_options.get("image")
206
207 return {
208 "image": self._get_dummy_images(
vllm/model_executor/models/colmodernvbert.py
+3 -3
@@ -132,12 +132,12 @@ class ColModernVBertDummyInputsBuilder(
132 self,
133 seq_len: int,
134 mm_counts: Mapping[str, int],
135 - mm_options: Mapping[str, BaseDummyOptions] | None = None,
136 - mm_processor_kwargs: Mapping[str, object] | None = None,
135 + mm_options: Mapping[str, BaseDummyOptions],
136 ) -> MultiModalDataDict:
137 num_images = mm_counts.get("image", 0)
138 target_width, target_height = self.info.get_image_size_with_most_features()
140 - image_overrides = mm_options.get("image") if mm_options else None
139 + image_overrides = mm_options.get("image")
140 +
141 return {
142 "image": self._get_dummy_images(
143 width=target_width,
vllm/model_executor/models/deepseek_ocr.py
+1 -2
@@ -255,8 +255,7 @@ class DeepseekOCRDummyInputsBuilder(BaseDummyInputsBuilder[DeepseekOCRProcessing
255 self,
256 seq_len: int,
257 mm_counts: Mapping[str, int],
258 - mm_options: Mapping[str, BaseDummyOptions] | None = None,
259 - mm_processor_kwargs: Mapping[str, object] | None = None,
258 + mm_options: Mapping[str, BaseDummyOptions],
259 ) -> MultiModalDataDict:
260 num_images = mm_counts.get("image", 0)
261
vllm/model_executor/models/deepseek_ocr2.py
+1 -2
@@ -137,8 +137,7 @@ class DeepseekOCR2DummyInputsBuilder(
137 self,
138 seq_len: int,
139 mm_counts: Mapping[str, int],
140 - mm_options: Mapping[str, BaseDummyOptions] | None = None,
141 - mm_processor_kwargs: Mapping[str, object] | None = None,
140 + mm_options: Mapping[str, BaseDummyOptions],
141 ) -> MultiModalDataDict:
142 num_images = mm_counts.get("image", 0)
143
vllm/model_executor/models/deepseek_vl2.py
+2 -3
@@ -214,14 +214,13 @@ class DeepseekVL2DummyInputsBuilder(BaseDummyInputsBuilder[DeepseekVL2Processing
214 self,
215 seq_len: int,
216 mm_counts: Mapping[str, int],
217 - mm_options: Mapping[str, BaseDummyOptions] | None = None,
218 - mm_processor_kwargs: Mapping[str, object] | None = None,
217 + mm_options: Mapping[str, BaseDummyOptions],
218 ) -> MultiModalDataDict:
219 num_images = mm_counts.get("image", 0)
220
221 max_image_size = self.info.get_image_size_with_most_features()
222
224 - image_overrides = mm_options.get("image") if mm_options else None
223 + image_overrides = mm_options.get("image")
224
225 return {
226 "image": self._get_dummy_images(
vllm/model_executor/models/dots_ocr.py
+3 -7
@@ -106,17 +106,13 @@ class DotsOCRDummyInputsBuilder(Qwen2VLDummyInputsBuilder):
106 self,
107 seq_len: int,
108 mm_counts: Mapping[str, int],
109 - mm_options: Mapping[str, BaseDummyOptions] | None = None,
110 - mm_processor_kwargs: Mapping[str, object] | None = None,
109 + mm_options: Mapping[str, BaseDummyOptions],
110 ) -> MultiModalDataDict:
111 num_images = mm_counts.get("image", 0)
112
114 - mm_processor_kwargs = mm_processor_kwargs or {}
115 - target_width, target_height = self.info.get_image_size_with_most_features( # noqa: E501
116 - mm_processor_kwargs.get("max_pixels", None)
117 - )
113 + target_width, target_height = self.info.get_image_size_with_most_features()
114
119 - image_overrides = mm_options.get("image") if mm_options else None
115 + image_overrides = mm_options.get("image")
116
117 return {
118 "image": self._get_dummy_images(
vllm/model_executor/models/ernie45_vl.py
+3 -4
@@ -1168,8 +1168,7 @@ class Ernie4_5_VLDummyInputsBuilder(BaseDummyInputsBuilder[Ernie4_5_VLProcessing
1168 self,
1169 seq_len: int,
1170 mm_counts: Mapping[str, int],
1171 - mm_options: Mapping[str, BaseDummyOptions] | None = None,
1172 - mm_processor_kwargs: Mapping[str, object] | None = None,
1171 + mm_options: Mapping[str, BaseDummyOptions],
1172 ) -> MultiModalDataDict:
1173 num_images = mm_counts.get("image", 0)
1174 num_videos = mm_counts.get("video", 0)
@@ -1179,8 +1178,8 @@ class Ernie4_5_VLDummyInputsBuilder(BaseDummyInputsBuilder[Ernie4_5_VLProcessing
1178 seq_len, mm_counts
1179 )
1180
1182 - image_overrides = mm_options.get("image") if mm_options else None
1183 - video_overrides = mm_options.get("video") if mm_options else None
1181 + image_overrides = mm_options.get("image")
1182 + video_overrides = mm_options.get("video")
1183
1184 return {
1185 "image": self._get_dummy_images(
vllm/model_executor/models/funasr.py
+7 -8
@@ -746,23 +746,22 @@ class FunASRDummyInputsBuilder(BaseDummyInputsBuilder[FunASRProcessingInfo]):
746 self,
747 seq_len: int,
748 mm_counts: Mapping[str, int],
749 - mm_options: Mapping[str, BaseDummyOptions] | None = None,
750 - mm_processor_kwargs: Mapping[str, object] | None = None,
749 + mm_options: Mapping[str, BaseDummyOptions],
750 ) -> MultiModalDataDict:
752 - feature_extractor = self.info.get_feature_extractor(
753 - **(mm_processor_kwargs or {})
754 - )
751 + feature_extractor = self.info.get_feature_extractor()
752
753 sampling_rate = feature_extractor.sampling_rate
754 audio_len = feature_extractor.chunk_length * sampling_rate
755 num_audios = mm_counts.get("audio", 0)
756
760 - audio_overrides = mm_options.get("audio") if mm_options else None
757 + audio_overrides = mm_options.get("audio")
758
759 return {
760 "audio": self._get_dummy_audios(
764 - length=audio_len, num_audios=num_audios, overrides=audio_overrides
765 - )
761 + length=audio_len,
762 + num_audios=num_audios,
763 + overrides=audio_overrides,
764 + ),
765 }
766
767
vllm/model_executor/models/funaudiochat.py
+3 -6
@@ -610,12 +610,9 @@ class FunAudioChatDummyInputsBuilder(
610 self,
611 seq_len: int,
612 mm_counts: Mapping[str, int],
613 - mm_options: Mapping[str, BaseDummyOptions] | None = None,
614 - mm_processor_kwargs: Mapping[str, object] | None = None,
613 + mm_options: Mapping[str, BaseDummyOptions],
614 ) -> MultiModalDataDict:
616 - feature_extractor = self.info.get_feature_extractor(
617 - **(mm_processor_kwargs or {})
618 - )
615 + feature_extractor = self.info.get_feature_extractor()
616 sampling_rate = int(feature_extractor.sampling_rate)
617
618 # Dummy inputs are used for profiling; construct the worst-case audio
@@ -632,7 +629,7 @@ class FunAudioChatDummyInputsBuilder(
629 )
630 num_audios = int(mm_counts.get("audio", 0))
631
635 - audio_overrides = mm_options.get("audio") if mm_options else None
632 + audio_overrides = mm_options.get("audio")
633 return {
634 "audio": self._get_dummy_audios(
635 length=audio_len,
vllm/model_executor/models/fuyu.py
+2 -3
@@ -142,13 +142,12 @@ class FuyuDummyInputsBuilder(BaseDummyInputsBuilder[FuyuProcessingInfo]):
142 self,
143 seq_len: int,
144 mm_counts: Mapping[str, int],
145 - mm_options: Mapping[str, BaseDummyOptions] | None = None,
146 - mm_processor_kwargs: Mapping[str, object] | None = None,
145 + mm_options: Mapping[str, BaseDummyOptions],
146 ) -> MultiModalDataDict:
147 target_width, target_height = self.info.get_image_size_with_most_features()
148 num_images = mm_counts.get("image", 0)
149
151 - image_overrides = mm_options.get("image") if mm_options else None
150 + image_overrides = mm_options.get("image")
151
152 return {
153 "image": self._get_dummy_images(
vllm/model_executor/models/gemma3_mm.py
+2 -3
@@ -241,14 +241,13 @@ class Gemma3DummyInputsBuilder(BaseDummyInputsBuilder[Gemma3ProcessingInfo]):
241 self,
242 seq_len: int,
243 mm_counts: Mapping[str, int],
244 - mm_options: Mapping[str, BaseDummyOptions] | None = None,
245 - mm_processor_kwargs: Mapping[str, object] | None = None,
244 + mm_options: Mapping[str, BaseDummyOptions],
245 ) -> MultiModalDataDict:
246 num_images = mm_counts.get("image", 0)
247
248 target_width, target_height = self.info.get_image_size_with_most_features()
249
251 - image_overrides = mm_options.get("image") if mm_options else None
250 + image_overrides = mm_options.get("image")
251
252 return {
253 "image": self._get_dummy_images(
vllm/model_executor/models/gemma3n_mm.py
+6 -5
@@ -175,8 +175,7 @@ class Gemma3nDummyInputsBuilder(BaseDummyInputsBuilder[Gemma3nProcessingInfo]):
175 self,
176 seq_len: int,
177 mm_counts: Mapping[str, int],
178 - mm_options: Mapping[str, BaseDummyOptions] | None = None,
179 - mm_processor_kwargs: Mapping[str, object] | None = None,
178 + mm_options: Mapping[str, BaseDummyOptions],
179 ) -> MultiModalDataDict:
180 num_images = mm_counts.get("image", 0)
181 num_audios = mm_counts.get("audio", 0)
@@ -189,8 +188,8 @@ class Gemma3nDummyInputsBuilder(BaseDummyInputsBuilder[Gemma3nProcessingInfo]):
188 img_width = image_processor.size.get("width", 224)
189 img_height = image_processor.size.get("height", 224)
190
192 - image_overrides = mm_options.get("image") if mm_options else None
193 - audio_overrides = mm_options.get("audio") if mm_options else None
191 + image_overrides = mm_options.get("image")
192 + audio_overrides = mm_options.get("audio")
193
194 return {
195 "image": self._get_dummy_images(
@@ -200,7 +199,9 @@ class Gemma3nDummyInputsBuilder(BaseDummyInputsBuilder[Gemma3nProcessingInfo]):
199 overrides=image_overrides,
200 ),
201 "audio": self._get_dummy_audios(
203 - length=audio_len, num_audios=num_audios, overrides=audio_overrides
202 + length=audio_len,
203 + num_audios=num_audios,
204 + overrides=audio_overrides,
205 ),
206 }
207
vllm/model_executor/models/glm4_1v.py
+3 -4
@@ -1163,8 +1163,7 @@ class Glm4vDummyInputsBuilder(BaseDummyInputsBuilder[Glm4vProcessingInfo]):
1163 self,
1164 seq_len: int,
1165 mm_counts: Mapping[str, int],
1166 - mm_options: Mapping[str, BaseDummyOptions] | None = None,
1167 - mm_processor_kwargs: Mapping[str, object] | None = None,
1166 + mm_options: Mapping[str, BaseDummyOptions],
1167 ) -> MultiModalDataDict:
1168 num_images = mm_counts.get("image", 0)
1169 num_videos = mm_counts.get("video", 0)
@@ -1174,8 +1173,8 @@ class Glm4vDummyInputsBuilder(BaseDummyInputsBuilder[Glm4vProcessingInfo]):
1173 seq_len, mm_counts
1174 )
1175
1177 - image_overrides = mm_options.get("image") if mm_options else None
1178 - video_overrides = mm_options.get("video") if mm_options else None
1176 + image_overrides = mm_options.get("image")
1177 + video_overrides = mm_options.get("video")
1178
1179 return {
1180 "image": self._get_dummy_images(
vllm/model_executor/models/glm4v.py
+2 -3
@@ -492,8 +492,7 @@ class GLM4VDummyInputsBuilder(BaseDummyInputsBuilder[GLM4VProcessingInfo]):
492 self,
493 seq_len: int,
494 mm_counts: Mapping[str, int],
495 - mm_options: Mapping[str, BaseDummyOptions] | None = None,
496 - mm_processor_kwargs: Mapping[str, object] | None = None,
495 + mm_options: Mapping[str, BaseDummyOptions],
496 ) -> MultiModalDataDict:
497 hf_config = self.info.get_hf_config()
498 vision_config = hf_config.vision_config
@@ -501,7 +500,7 @@ class GLM4VDummyInputsBuilder(BaseDummyInputsBuilder[GLM4VProcessingInfo]):
500 target_width = target_height = vision_config["image_size"]
501 num_images = mm_counts.get("image", 0)
502
504 - image_overrides = mm_options.get("image") if mm_options else None
503 + image_overrides = mm_options.get("image")
504
505 return {
506 "image": self._get_dummy_images(
vllm/model_executor/models/glmasr.py
+6 -7
@@ -726,15 +726,12 @@ class GlmAsrDummyInputsBuilder(BaseDummyInputsBuilder[GlmAsrProcessingInfo]):
726 self,
727 seq_len: int,
728 mm_counts: Mapping[str, int],
729 - mm_options: Mapping[str, BaseDummyOptions] | None = None,
730 - mm_processor_kwargs: Mapping[str, object] | None = None,
729 + mm_options: Mapping[str, BaseDummyOptions],
730 ) -> MultiModalDataDict:
732 - feature_extractor = self.info.get_feature_extractor(
733 - **(mm_processor_kwargs or {})
734 - )
731 + feature_extractor = self.info.get_feature_extractor()
732 sampling_rate = feature_extractor.sampling_rate
733 num_audios = mm_counts.get("audio", 0)
737 - audio_overrides = mm_options.get("audio") if mm_options else None
734 + audio_overrides = mm_options.get("audio")
735
736 max_audio_len = getattr(
737 self.info.get_hf_processor(), "max_audio_len", DEFAULT_MAX_AUDIO_LEN_S
@@ -743,7 +740,9 @@ class GlmAsrDummyInputsBuilder(BaseDummyInputsBuilder[GlmAsrProcessingInfo]):
740
741 return {
742 "audio": self._get_dummy_audios(
746 - length=audio_len, num_audios=num_audios, overrides=audio_overrides
743 + length=audio_len,
744 + num_audios=num_audios,
745 + overrides=audio_overrides,
746 )
747 }
748
vllm/model_executor/models/granite_speech.py
+2 -3
@@ -216,11 +216,10 @@ class GraniteSpeechDummyInputsBuilder(
216 self,
217 seq_len: int,
218 mm_counts: Mapping[str, int],
219 - mm_options: Mapping[str, BaseDummyOptions] | None = None,
220 - mm_processor_kwargs: Mapping[str, object] | None = None,
219 + mm_options: Mapping[str, BaseDummyOptions],
220 ) -> MultiModalDataDict:
221 num_audios = mm_counts.get("audio", 0)
223 - audio_overrides = mm_options.get("audio") if mm_options else None
222 + audio_overrides = mm_options.get("audio")
223
224 return {
225 "audio": self._get_dummy_audios(
vllm/model_executor/models/hunyuan_vision.py
+1 -2
@@ -713,8 +713,7 @@ class HunYuanVLDummyInputsBuilder(BaseDummyInputsBuilder[HunYuanVLProcessingInfo
713 self,
714 seq_len: int,
715 mm_counts: Mapping[str, int],
716 - mm_options: Mapping[str, BaseDummyOptions] | None = None,
717 - mm_processor_kwargs: Mapping[str, object] | None = None,
716 + mm_options: Mapping[str, BaseDummyOptions],
717 ) -> MultiModalDataDict:
718 num_images = mm_counts.get("image", 1)
719
vllm/model_executor/models/hyperclovax_vision.py
+3 -4
@@ -165,8 +165,7 @@ class HCXVisionDummyInputsBuilder(BaseDummyInputsBuilder[HCXVisionProcessingInfo
165 self,
166 seq_len: int,
167 mm_counts: Mapping[str, int],
168 - mm_options: Mapping[str, BaseDummyOptions] | None = None,
169 - mm_processor_kwargs: Mapping[str, object] | None = None,
168 + mm_options: Mapping[str, BaseDummyOptions],
169 ) -> MultiModalDataDict:
170 num_images = mm_counts.get("image", 0)
171 num_videos = mm_counts.get("video", 0)
@@ -174,8 +173,8 @@ class HCXVisionDummyInputsBuilder(BaseDummyInputsBuilder[HCXVisionProcessingInfo
173 target_width, target_height = self.info.get_image_size_with_most_features()
174 target_num_frames = 32
175
177 - image_overrides = mm_options.get("image") if mm_options else None
178 - video_overrides = mm_options.get("video") if mm_options else None
176 + image_overrides = mm_options.get("image")
177 + video_overrides = mm_options.get("video")
178
179 return {
180 "image": self._get_dummy_images(
vllm/model_executor/models/idefics3.py
+3 -4
@@ -277,15 +277,14 @@ class Idefics3DummyInputsBuilder(BaseDummyInputsBuilder[Idefics3ProcessingInfo])
277 self,
278 seq_len: int,
279 mm_counts: Mapping[str, int],
280 - mm_options: Mapping[str, BaseDummyOptions] | None = None,
281 - mm_processor_kwargs: Mapping[str, object] | None = None,
280 + mm_options: Mapping[str, BaseDummyOptions],
281 ) -> MultiModalDataDict:
282 num_images = mm_counts.get("image", 0)
284 - hf_processor = self.info.get_hf_processor(**(mm_processor_kwargs or {}))
283 + hf_processor = self.info.get_hf_processor()
284 image_processor: Idefics3ImageProcessor = hf_processor.image_processor
285 longest_edge = image_processor.max_image_size["longest_edge"]
286
288 - image_overrides = mm_options.get("image") if mm_options else None
287 + image_overrides = mm_options.get("image")
288
289 return {
290 "image": self._get_dummy_images(
vllm/model_executor/models/interns1.py
+3 -4
@@ -297,8 +297,7 @@ class InternS1DummyInputsBuilder(BaseDummyInputsBuilder[InternS1ProcessingInfo])
297 self,
298 seq_len: int,
299 mm_counts: Mapping[str, int],
300 - mm_options: Mapping[str, BaseDummyOptions] | None = None,
301 - mm_processor_kwargs: Mapping[str, object] | None = None,
300 + mm_options: Mapping[str, BaseDummyOptions],
301 ) -> MultiModalDataDict:
302 target_width, target_height = self.info.get_image_size_with_most_features()
303 target_num_frames = self.info.get_num_frames_with_most_features(
@@ -310,8 +309,8 @@ class InternS1DummyInputsBuilder(BaseDummyInputsBuilder[InternS1ProcessingInfo])
309 config = self.info.get_hf_config()
310 image_size_h, image_size_w = config.vision_config.image_size
311
313 - image_overrides = mm_options.get("image") if mm_options else None
314 - video_overrides = mm_options.get("video") if mm_options else None
312 + image_overrides = mm_options.get("image")
313 + video_overrides = mm_options.get("video")
314
315 return {
316 "image": self._get_dummy_images(
vllm/model_executor/models/internvl.py
+5 -9
@@ -762,13 +762,12 @@ class BaseInternVLDummyInputsBuilder(BaseDummyInputsBuilder[_I]):
762 self,
763 seq_len: int,
764 mm_counts: Mapping[str, int],
765 - mm_options: Mapping[str, BaseDummyOptions] | None = None,
766 - mm_processor_kwargs: Mapping[str, object] | None = None,
765 + mm_options: Mapping[str, BaseDummyOptions],
766 ) -> MultiModalDataDict:
767 target_width, target_height = self.info.get_image_size_with_most_features()
768 num_images = mm_counts.get("image", 0)
769
771 - image_overrides = mm_options.get("image") if mm_options else None
770 + image_overrides = mm_options.get("image")
771
772 return {
773 "image": self._get_dummy_images(
@@ -935,12 +934,9 @@ class InternVLDummyInputsBuilder(
934 self,
935 seq_len: int,
936 mm_counts: Mapping[str, int],
938 - mm_options: Mapping[str, BaseDummyOptions] | None = None,
939 - mm_processor_kwargs: Mapping[str, object] | None = None,
937 + mm_options: Mapping[str, BaseDummyOptions],
938 ) -> MultiModalDataDict:
941 - dummy_image = super().get_dummy_mm_data(
942 - seq_len=seq_len, mm_counts=mm_counts, mm_options=mm_options
943 - )
939 + dummy_image = super().get_dummy_mm_data(seq_len, mm_counts, mm_options)
940 if self.info.supports_video:
941 config = self.info.get_hf_config()
942 image_size: int = config.vision_config.image_size
@@ -948,7 +944,7 @@ class InternVLDummyInputsBuilder(
944 seq_len, mm_counts
945 )
946 num_videos = mm_counts.get("video", 0)
951 - video_overrides = mm_options.get("video") if mm_options else None
947 + video_overrides = mm_options.get("video")
948 dummy_video = {
949 "video": self._get_dummy_videos(
950 width=image_size,
vllm/model_executor/models/isaac.py
+3 -3
@@ -18,6 +18,7 @@ from typing_extensions import TypedDict, Unpack
18
19 from vllm.config import VllmConfig
20 from vllm.config.model import ModelConfig
21 +from vllm.config.multimodal import BaseDummyOptions
22 from vllm.distributed import parallel_state
23 from vllm.distributed import utils as dist_utils
24 from vllm.model_executor.layers.attention import MMEncoderAttention
@@ -849,13 +850,12 @@ class IsaacDummyInputsBuilder(BaseDummyInputsBuilder[IsaacProcessingInfo]):
850 self,
851 seq_len: int,
852 mm_counts: Mapping[str, int],
852 - mm_options: Mapping[str] | None = None,
853 - mm_processor_kwargs: Mapping[str, object] | None = None,
853 + mm_options: Mapping[str, BaseDummyOptions],
854 ) -> MultiModalDataDict:
855 num_images = mm_counts.get("image", 0)
856
857 target_width, target_height = self.info.get_image_size_with_most_features()
858 - image_overrides = mm_options.get("image") if mm_options else None
858 + image_overrides = mm_options.get("image")
859
860 return {
861 "image": self._get_dummy_images(
vllm/model_executor/models/kanana_v.py
+1 -2
@@ -444,8 +444,7 @@ class KananaVDummyInputsBuilder(BaseDummyInputsBuilder[KananaVProcessingInfo]):
444 self,
445 seq_len: int,
446 mm_counts: Mapping[str, int],
447 - mm_options: Mapping[str, BaseDummyOptions] | None = None,
448 - mm_processor_kwargs: Mapping[str, object] | None = None,
447 + mm_options: Mapping[str, BaseDummyOptions],
448 ) -> MultiModalDataDict:
449 num_images = mm_counts.get("image", 0)
450 return {
vllm/model_executor/models/keye.py
+3 -4
@@ -1170,8 +1170,7 @@ class KeyeBaseDummyInputsBuilder(BaseDummyInputsBuilder[_I]):
1170 self,
1171 seq_len: int,
1172 mm_counts: Mapping[str, int],
1173 - mm_options: Mapping[str, BaseDummyOptions] | None = None,
1174 - mm_processor_kwargs: Mapping[str, object] | None = None,
1173 + mm_options: Mapping[str, BaseDummyOptions],
1174 ) -> MultiModalDataDict:
1175 num_images = mm_counts.get("image", 0)
1176 num_videos = mm_counts.get("video", 0)
@@ -1179,8 +1178,8 @@ class KeyeBaseDummyInputsBuilder(BaseDummyInputsBuilder[_I]):
1178 target_width, target_height = self.info.get_image_size_with_most_features()
1179 target_num_frames = self.info.get_num_frames_with_most_features(seq_len)
1180
1182 - image_overrides = mm_options.get("image") if mm_options else None
1183 - video_overrides = mm_options.get("video") if mm_options else None
1181 + image_overrides = mm_options.get("image")
1182 + video_overrides = mm_options.get("video")
1183
1184 mm_data = {
1185 "image": self._get_dummy_images(
vllm/model_executor/models/kimi_k25.py
+1 -2
@@ -240,8 +240,7 @@ class KimiK25DummyInputsBuilder(BaseDummyInputsBuilder[KimiK25ProcessingInfo]):
240 self,
241 seq_len: int,
242 mm_counts: Mapping[str, int],
243 - mm_options: Mapping[str, BaseDummyOptions] | None = None,
244 - mm_processor_kwargs: Mapping[str, object] | None = None,
243 + mm_options: Mapping[str, BaseDummyOptions],
244 ) -> MultiModalDataDict:
245 # TODO: Support mm_options for vision_chunk to allow user configuration
246 dummy_items = self.get_dummy_mm_items()
vllm/model_executor/models/kimi_vl.py
+2 -3
@@ -215,12 +215,11 @@ class KimiVLDummyInputsBuilder(BaseDummyInputsBuilder[KimiVLProcessingInfo]):
215 self,
216 seq_len: int,
217 mm_counts: Mapping[str, int],
218 - mm_options: Mapping[str, BaseDummyOptions] | None = None,
219 - mm_processor_kwargs: Mapping[str, object] | None = None,
218 + mm_options: Mapping[str, BaseDummyOptions],
219 ) -> MultiModalDataDict:
220 num_images = mm_counts.get("image", 0)
221
223 - image_overrides = mm_options.get("image") if mm_options else None
222 + image_overrides = mm_options.get("image")
223
224 return {
225 "image": self._get_dummy_images(
vllm/model_executor/models/lfm2_vl.py
+2 -3
@@ -343,14 +343,13 @@ class Lfm2VLDummyInputsBuilder(BaseDummyInputsBuilder[Lfm2VLProcessingInfo]):
343 self,
344 seq_len: int,
345 mm_counts: Mapping[str, int],
346 - mm_options: Mapping[str, BaseDummyOptions] | None = None,
347 - mm_processor_kwargs: Mapping[str, object] | None = None,
346 + mm_options: Mapping[str, BaseDummyOptions],
347 ) -> MultiModalDataDict:
348 num_images = mm_counts.get("image", 0)
349
350 target_width, target_height = self.info.get_image_size_with_most_features()
351
353 - image_overrides = mm_options.get("image") if mm_options else None
352 + image_overrides = mm_options.get("image")
353
354 return {
355 "image": self._get_dummy_images(
vllm/model_executor/models/llava.py
+2 -3
@@ -232,14 +232,13 @@ class LlavaDummyInputsBuilder(BaseDummyInputsBuilder[_I]):
232 self,
233 seq_len: int,
234 mm_counts: Mapping[str, int],
235 - mm_options: Mapping[str, BaseDummyOptions] | None = None,
236 - mm_processor_kwargs: Mapping[str, object] | None = None,
235 + mm_options: Mapping[str, BaseDummyOptions],
236 ) -> MultiModalDataDict:
237 num_images = mm_counts.get("image", 0)
238
239 target_width, target_height = self.info.get_image_size_with_most_features()
240
242 - image_overrides = mm_options.get("image") if mm_options else None
241 + image_overrides = mm_options.get("image")
242
243 return {
244 "image": self._get_dummy_images(
vllm/model_executor/models/llava_next_video.py
+2 -3
@@ -165,8 +165,7 @@ class LlavaNextVideoDummyInputsBuilder(
165 self,
166 seq_len: int,
167 mm_counts: Mapping[str, int],
168 - mm_options: Mapping[str, BaseDummyOptions] | None = None,
169 - mm_processor_kwargs: Mapping[str, object] | None = None,
168 + mm_options: Mapping[str, BaseDummyOptions],
169 ) -> MultiModalDataDict:
170 num_videos = mm_counts.get("video", 0)
171
@@ -175,7 +174,7 @@ class LlavaNextVideoDummyInputsBuilder(
174 seq_len, mm_counts
175 )
176
178 - video_overrides = mm_options.get("video") if mm_options else None
177 + video_overrides = mm_options.get("video")
178
179 return {
180 "video": self._get_dummy_videos(
vllm/model_executor/models/llava_onevision.py
+3 -4
@@ -276,8 +276,7 @@ class LlavaOnevisionDummyInputsBuilder(
276 self,
277 seq_len: int,
278 mm_counts: Mapping[str, int],
279 - mm_options: Mapping[str, BaseDummyOptions] | None = None,
280 - mm_processor_kwargs: Mapping[str, object] | None = None,
279 + mm_options: Mapping[str, BaseDummyOptions],
280 ) -> MultiModalDataDict:
281 num_images = mm_counts.get("image", 0)
282 num_videos = mm_counts.get("video", 0)
@@ -287,8 +286,8 @@ class LlavaOnevisionDummyInputsBuilder(
286 seq_len, mm_counts
287 )
288
290 - image_overrides = mm_options.get("image") if mm_options else None
291 - video_overrides = mm_options.get("video") if mm_options else None
289 + image_overrides = mm_options.get("image")
290 + video_overrides = mm_options.get("video")
291
292 return {
293 "image": self._get_dummy_images(
vllm/model_executor/models/midashenglm.py
+2 -3
@@ -565,12 +565,11 @@ class MiDashengLMDummyInputsBuilder(BaseDummyInputsBuilder[MiDashengLMProcessing
565 self,
566 seq_len: int,
567 mm_counts: Mapping[str, int],
568 - mm_options: Mapping[str, BaseDummyOptions] | None = None,
569 - mm_processor_kwargs: Mapping[str, object] | None = None,
568 + mm_options: Mapping[str, BaseDummyOptions],
569 ) -> MultiModalDataDict:
570 num_audios = mm_counts.get("audio", 0)
571
573 - audio_overrides = mm_options.get("audio") if mm_options else None
572 + audio_overrides = mm_options.get("audio")
573
574 return {
575 "audio": self._get_dummy_audios(
vllm/model_executor/models/minicpmo.py
+5 -4
@@ -301,8 +301,7 @@ class MiniCPMODummyInputsBuilder(MiniCPMVDummyInputsBuilder[MiniCPMOProcessingIn
301 self,
302 seq_len: int,
303 mm_counts: Mapping[str, int],
304 - mm_options: Mapping[str, BaseDummyOptions] | None = None,
305 - mm_processor_kwargs: Mapping[str, object] | None = None,
304 + mm_options: Mapping[str, BaseDummyOptions],
305 ) -> MultiModalDataDict:
306 num_audios = mm_counts.get("audio", 0)
307 audio_len = (
@@ -310,11 +309,13 @@ class MiniCPMODummyInputsBuilder(MiniCPMVDummyInputsBuilder[MiniCPMOProcessingIn
309 * self.info.get_default_audio_sampling_rate()
310 )
311
313 - audio_overrides = mm_options.get("audio") if mm_options else None
312 + audio_overrides = mm_options.get("audio")
313
314 audio_mm_data = {
315 "audio": self._get_dummy_audios(
317 - length=audio_len, num_audios=num_audios, overrides=audio_overrides
316 + length=audio_len,
317 + num_audios=num_audios,
318 + overrides=audio_overrides,
319 )
320 }
321
vllm/model_executor/models/minicpmv.py
+3 -4
@@ -707,8 +707,7 @@ class MiniCPMVDummyInputsBuilder(BaseDummyInputsBuilder[_I]):
707 self,
708 seq_len: int,
709 mm_counts: Mapping[str, int],
710 - mm_options: Mapping[str, BaseDummyOptions] | None = None,
711 - mm_processor_kwargs: Mapping[str, object] | None = None,
710 + mm_options: Mapping[str, BaseDummyOptions],
711 ) -> MultiModalDataDict:
712 num_images = mm_counts.get("image", 0)
713 num_videos = mm_counts.get("video", 0)
@@ -719,8 +718,8 @@ class MiniCPMVDummyInputsBuilder(BaseDummyInputsBuilder[_I]):
718 seq_len, mm_counts
719 )
720
722 - image_overrides = mm_options.get("image") if mm_options else None
723 - video_overrides = mm_options.get("video") if mm_options else None
721 + image_overrides = mm_options.get("image")
722 + video_overrides = mm_options.get("video")
723
724 return {
725 "image": self._get_dummy_images(
vllm/model_executor/models/mistral3.py
+2 -3
@@ -236,14 +236,13 @@ class Mistral3DummyInputsBuilder(BaseDummyInputsBuilder[_I]):
236 self,
237 seq_len: int,
238 mm_counts: Mapping[str, int],
239 - mm_options: Mapping[str, BaseDummyOptions] | None = None,
240 - mm_processor_kwargs: Mapping[str, object] | None = None,
239 + mm_options: Mapping[str, BaseDummyOptions],
240 ) -> MultiModalDataDict:
241 num_images = mm_counts.get("image", 0)
242
243 target_width, target_height = self.info.get_image_size_with_most_features()
244
246 - image_overrides = mm_options.get("image") if mm_options else None
245 + image_overrides = mm_options.get("image")
246
247 return {
248 "image": self._get_dummy_images(
vllm/model_executor/models/mllama4.py
+2 -3
@@ -707,14 +707,13 @@ class Mllama4DummyInputsBuilder(BaseDummyInputsBuilder[Mllama4ProcessingInfo]):
707 self,
708 seq_len: int,
709 mm_counts: Mapping[str, int],
710 - mm_options: Mapping[str, BaseDummyOptions] | None = None,
711 - mm_processor_kwargs: Mapping[str, object] | None = None,
710 + mm_options: Mapping[str, BaseDummyOptions],
711 ) -> MultiModalDataDict:
712 num_images = mm_counts.get("image", 0)
713
714 (target_width, target_height) = self.info.get_image_size_with_most_features()
715
717 - image_overrides = mm_options.get("image") if mm_options else None
716 + image_overrides = mm_options.get("image")
717
718 return {
719 "image": self._get_dummy_images(
vllm/model_executor/models/molmo.py
+2 -3
@@ -1274,13 +1274,12 @@ class MolmoDummyInputsBuilder(BaseDummyInputsBuilder[MolmoProcessingInfo]):
1274 self,
1275 seq_len: int,
1276 mm_counts: Mapping[str, int],
1277 - mm_options: Mapping[str, BaseDummyOptions] | None = None,
1278 - mm_processor_kwargs: Mapping[str, object] | None = None,
1277 + mm_options: Mapping[str, BaseDummyOptions],
1278 ) -> MultiModalDataDict:
1279 target_width, target_height = self.info.get_image_size_with_most_features()
1280 num_images = mm_counts.get("image", 0)
1281
1283 - image_overrides = mm_options.get("image") if mm_options else None
1282 + image_overrides = mm_options.get("image")
1283
1284 return {
1285 "image": self._get_dummy_images(
vllm/model_executor/models/molmo2.py
+3 -4
@@ -2082,8 +2082,7 @@ class Molmo2DummyInputsBuilder(BaseDummyInputsBuilder[Molmo2ProcessingInfo]):
2082 self,
2083 seq_len: int,
2084 mm_counts: Mapping[str, int],
2085 - mm_options: Mapping[str, BaseDummyOptions] | None = None,
2086 - mm_processor_kwargs: Mapping[str, object] | None = None,
2085 + mm_options: Mapping[str, BaseDummyOptions],
2086 ) -> MultiModalDataDict:
2087 num_images = mm_counts.get("image", 0)
2088 num_videos = mm_counts.get("video", 0)
@@ -2094,7 +2093,7 @@ class Molmo2DummyInputsBuilder(BaseDummyInputsBuilder[Molmo2ProcessingInfo]):
2093 if num_images > 0:
2094 target_width, target_height = self.info.get_image_size_with_most_features()
2095
2097 - image_overrides = mm_options.get("image") if mm_options else None
2096 + image_overrides = mm_options.get("image")
2097
2098 dummy_images = self._get_dummy_images(
2099 width=target_width,
@@ -2110,7 +2109,7 @@ class Molmo2DummyInputsBuilder(BaseDummyInputsBuilder[Molmo2ProcessingInfo]):
2109 seq_len, mm_counts
2110 )
2111
2113 - video_overrides = mm_options.get("video") if mm_options else None
2112 + video_overrides = mm_options.get("video")
2113
2114 if video_overrides:
2115 assert isinstance(video_overrides, VideoDummyOptions)
vllm/model_executor/models/nano_nemotron_vl.py
+5 -9
@@ -1388,8 +1388,7 @@ class NanoNemotronVLDummyInputsBuilder(BaseDummyInputsBuilder[_I]):
1388 self,
1389 seq_len: int,
1390 mm_counts: Mapping[str, int],
1391 - mm_options: Mapping[str, BaseDummyOptions] | None = None,
1392 - mm_processor_kwargs: Mapping[str, object] | None = None,
1391 + mm_options: Mapping[str, BaseDummyOptions],
1392 ) -> MultiModalDataDict:
1393 num_images = mm_counts.get("image", 0)
1394 processor = self.info.get_hf_processor()
@@ -1404,7 +1403,7 @@ class NanoNemotronVLDummyInputsBuilder(BaseDummyInputsBuilder[_I]):
1403 max_num_tiles
1404 )
1405
1407 - image_overrides = mm_options.get("image") if mm_options else None
1406 + image_overrides = mm_options.get("image")
1407
1408 return {
1409 "image": self._get_dummy_images(
@@ -1461,12 +1460,9 @@ class NanoNemotronVLDummyInputsBuilder(
1460 self,
1461 seq_len: int,
1462 mm_counts: Mapping[str, int],
1464 - mm_options: Mapping[str, BaseDummyOptions] | None = None,
1465 - mm_processor_kwargs: Mapping[str, object] | None = None,
1463 + mm_options: Mapping[str, BaseDummyOptions],
1464 ) -> MultiModalDataDict:
1467 - dummy_image = super().get_dummy_mm_data(
1468 - seq_len=seq_len, mm_counts=mm_counts, mm_options=mm_options
1469 - )
1465 + dummy_image = super().get_dummy_mm_data(seq_len, mm_counts, mm_options)
1466 if self.info.supports_video:
1467 config = self.info.get_hf_config()
1468 image_size: int = config.force_image_size
@@ -1474,7 +1470,7 @@ class NanoNemotronVLDummyInputsBuilder(
1470 seq_len, mm_counts
1471 )
1472 num_videos = mm_counts.get("video", 0)
1477 - video_overrides = mm_options.get("video") if mm_options else None
1473 + video_overrides = mm_options.get("video")
1474 dummy_video = {
1475 "video": self._get_dummy_videos(
1476 width=image_size,
vllm/model_executor/models/nemotron_parse.py
+1 -2
@@ -645,8 +645,7 @@ class NemotronParseDummyInputsBuilder(
645 self,
646 seq_len: int,
647 mm_counts: Mapping[str, int],
648 - mm_options: Mapping[str, BaseDummyOptions] | None = None,
649 - mm_processor_kwargs: Mapping[str, object] | None = None,
648 + mm_options: Mapping[str, BaseDummyOptions],
649 ) -> MultiModalDataDict:
650 num_images = mm_counts.get("image", 0)
651
vllm/model_executor/models/nvlm_d.py
+2 -3
@@ -92,13 +92,12 @@ class NVLMDummyInputsBuilder(BaseInternVLDummyInputsBuilder[NVLMProcessingInfo])
92 self,
93 seq_len: int,
94 mm_counts: Mapping[str, int],
95 - mm_options: Mapping[str, BaseDummyOptions] | None = None,
96 - mm_processor_kwargs: Mapping[str, object] | None = None,
95 + mm_options: Mapping[str, BaseDummyOptions],
96 ) -> MultiModalDataDict:
97 target_width, target_height = self.info.get_image_size_with_most_features()
98 num_images = mm_counts.get("image", 0)
99
101 - image_overrides = mm_options.get("image") if mm_options else None
100 + image_overrides = mm_options.get("image")
101
102 return {
103 "image": self._get_dummy_images(
vllm/model_executor/models/ovis.py
+2 -3
@@ -306,14 +306,13 @@ class OvisDummyInputsBuilder(BaseDummyInputsBuilder[OvisProcessingInfo]):
306 self,
307 seq_len: int,
308 mm_counts: Mapping[str, int],
309 - mm_options: Mapping[str, BaseDummyOptions] | None = None,
310 - mm_processor_kwargs: Mapping[str, object] | None = None,
309 + mm_options: Mapping[str, BaseDummyOptions],
310 ) -> MultiModalDataDict:
311 num_images = mm_counts.get("image", 0)
312
313 target_width, target_height = self.info.get_image_size_with_most_features()
314
316 - image_overrides = mm_options.get("image") if mm_options else None
315 + image_overrides = mm_options.get("image")
316
317 mm_data = {
318 "image": self._get_dummy_images(
vllm/model_executor/models/ovis2_5.py
+3 -4
@@ -287,8 +287,7 @@ class Ovis2_5DummyInputsBuilder(BaseDummyInputsBuilder[Ovis2_5ProcessingInfo]):
287 self,
288 seq_len: int,
289 mm_counts: Mapping[str, int],
290 - mm_options: Mapping[str, BaseDummyOptions] | None = None,
291 - mm_processor_kwargs: Mapping[str, object] | None = None,
290 + mm_options: Mapping[str, BaseDummyOptions],
291 ) -> MultiModalDataDict:
292 num_images = mm_counts.get("image", 0)
293 num_videos = mm_counts.get("video", 0)
@@ -298,8 +297,8 @@ class Ovis2_5DummyInputsBuilder(BaseDummyInputsBuilder[Ovis2_5ProcessingInfo]):
297 seq_len, mm_counts
298 )
299
301 - image_overrides = mm_options.get("image") if mm_options else None
302 - video_overrides = mm_options.get("video") if mm_options else None
300 + image_overrides = mm_options.get("image")
301 + video_overrides = mm_options.get("video")
302
303 mm_data = {
304 "image": self._get_dummy_images(
vllm/model_executor/models/paddleocr_vl.py
+2 -3
@@ -206,13 +206,12 @@ class PaddleOCRVLDummyInputsBuilder(BaseDummyInputsBuilder[PaddleOCRVLProcessing
206 self,
207 seq_len: int,
208 mm_counts: Mapping[str, int],
209 - mm_options: Mapping[str, BaseDummyOptions] | None = None,
210 - mm_processor_kwargs: Mapping[str, object] | None = None,
209 + mm_options: Mapping[str, BaseDummyOptions],
210 ) -> MultiModalDataDict:
211 num_images = mm_counts.get("image", 0)
212
213 max_image_size = self.info.get_image_size_with_most_features()
215 - image_overrides = mm_options.get("image") if mm_options else None
214 + image_overrides = mm_options.get("image")
215
216 return {
217 "image": self._get_dummy_images(
vllm/model_executor/models/paligemma.py
+2 -3
@@ -131,8 +131,7 @@ class PaliGemmaDummyInputsBuilder(BaseDummyInputsBuilder[PaliGemmaProcessingInfo
131 self,
132 seq_len: int,
133 mm_counts: Mapping[str, int],
134 - mm_options: Mapping[str, BaseDummyOptions] | None = None,
135 - mm_processor_kwargs: Mapping[str, object] | None = None,
134 + mm_options: Mapping[str, BaseDummyOptions],
135 ) -> MultiModalDataDict:
136 hf_config = self.info.get_hf_config()
137 vision_config = hf_config.vision_config
@@ -140,7 +139,7 @@ class PaliGemmaDummyInputsBuilder(BaseDummyInputsBuilder[PaliGemmaProcessingInfo
139
140 num_images = mm_counts.get("image", 0)
141
143 - image_overrides = mm_options.get("image") if mm_options else None
142 + image_overrides = mm_options.get("image")
143
144 return {
145 "image": self._get_dummy_images(
vllm/model_executor/models/phi3v.py
+2 -3
@@ -376,14 +376,13 @@ class Phi3VDummyInputsBuilder(BaseDummyInputsBuilder[Phi3VProcessingInfo]):
376 self,
377 seq_len: int,
378 mm_counts: Mapping[str, int],
379 - mm_options: Mapping[str, BaseDummyOptions] | None = None,
380 - mm_processor_kwargs: Mapping[str, object] | None = None,
379 + mm_options: Mapping[str, BaseDummyOptions],
380 ) -> MultiModalDataDict:
381 num_images = mm_counts.get("image", 0)
382
383 target_width, target_height = self.info.get_image_size_with_most_features()
384
386 - image_overrides = mm_options.get("image") if mm_options else None
385 + image_overrides = mm_options.get("image")
386
387 return {
388 "image": self._get_dummy_images(
vllm/model_executor/models/phi4mm.py
+3 -4
@@ -822,16 +822,15 @@ class Phi4MMDummyInputsBuilder(BaseDummyInputsBuilder[Phi4MMProcessingInfo]):
822 self,
823 seq_len: int,
824 mm_counts: Mapping[str, int],
825 - mm_options: Mapping[str, BaseDummyOptions] | None = None,
826 - mm_processor_kwargs: Mapping[str, object] | None = None,
825 + mm_options: Mapping[str, BaseDummyOptions],
826 ) -> MultiModalDataDict:
827 num_audios = mm_counts.get("audio", 0)
828 num_images = mm_counts.get("image", 0)
829
830 target_width, target_height = self.info.get_image_size_with_most_features()
831
833 - image_overrides = mm_options.get("image") if mm_options else None
834 - audio_overrides = mm_options.get("audio") if mm_options else None
832 + image_overrides = mm_options.get("image")
833 + audio_overrides = mm_options.get("audio")
834
835 mm_data = {
836 "image": self._get_dummy_images(
vllm/model_executor/models/pixtral.py
+3 -5
@@ -249,14 +249,13 @@ class PixtralDummyInputsBuilder(BaseDummyInputsBuilder[PixtralProcessingInfo]):
249 self,
250 seq_len: int,
251 mm_counts: Mapping[str, int],
252 - mm_options: Mapping[str, BaseDummyOptions] | None = None,
253 - mm_processor_kwargs: Mapping[str, object] | None = None,
252 + mm_options: Mapping[str, BaseDummyOptions],
253 ) -> MultiModalDataDict:
254 num_images = mm_counts.get("image", 0)
255
256 target_width, target_height = self.info.get_image_size_with_most_features()
257
259 - image_overrides = mm_options.get("image") if mm_options else None
258 + image_overrides = mm_options.get("image")
259
260 return {
261 "image": self._get_dummy_images(
@@ -271,8 +270,7 @@ class PixtralDummyInputsBuilder(BaseDummyInputsBuilder[PixtralProcessingInfo]):
270 self,
271 seq_len: int,
272 mm_counts: Mapping[str, int],
274 - mm_options: Mapping[str, BaseDummyOptions] | None = None,
275 - mm_processor_kwargs: Mapping[str, object] | None = None,
273 + mm_options: Mapping[str, BaseDummyOptions],
274 ) -> ProcessorInputs:
275 tokenizer = self.info.get_tokenizer()
276
vllm/model_executor/models/qwen2_5_omni_thinker.py
+6 -10
@@ -357,15 +357,13 @@ class Qwen2_5OmniThinkerDummyInputsBuilder(
357 self,
358 seq_len: int,
359 mm_counts: Mapping[str, int],
360 - mm_options: Mapping[str, BaseDummyOptions] | None = None,
361 - mm_processor_kwargs: Mapping[str, object] | None = None,
360 + mm_options: Mapping[str, BaseDummyOptions],
361 ) -> MultiModalDataDict:
362 num_audios = mm_counts.get("audio", 0)
363 num_images = mm_counts.get("image", 0)
364 num_videos = mm_counts.get("video", 0)
365
367 - mm_processor_kwargs = mm_processor_kwargs or {}
368 - feature_extractor = self.info.get_feature_extractor(**mm_processor_kwargs)
366 + feature_extractor = self.info.get_feature_extractor()
367
368 target_audio_length = (
369 min(
@@ -375,16 +373,14 @@ class Qwen2_5OmniThinkerDummyInputsBuilder(
373 * feature_extractor.sampling_rate
374 )
375
378 - target_width, target_height = self.info.get_image_size_with_most_features(
379 - max_pixels=mm_processor_kwargs.get("max_pixels", None),
380 - )
376 + target_width, target_height = self.info.get_image_size_with_most_features()
377 target_num_frames = self.info.get_num_frames_with_most_features(
378 seq_len, mm_counts
379 )
380
385 - image_overrides = mm_options.get("image") if mm_options else None
386 - video_overrides = mm_options.get("video") if mm_options else None
387 - audio_overrides = mm_options.get("audio") if mm_options else None
381 + image_overrides = mm_options.get("image")
382 + video_overrides = mm_options.get("video")
383 + audio_overrides = mm_options.get("audio")
384
385 mm_data = {
386 "audio": self._get_dummy_audios(
vllm/model_executor/models/qwen2_audio.py
+6 -7
@@ -195,22 +195,21 @@ class Qwen2AudioDummyInputsBuilder(BaseDummyInputsBuilder[Qwen2AudioProcessingIn
195 self,
196 seq_len: int,
197 mm_counts: Mapping[str, int],
198 - mm_options: Mapping[str, BaseDummyOptions] | None = None,
199 - mm_processor_kwargs: Mapping[str, object] | None = None,
198 + mm_options: Mapping[str, BaseDummyOptions],
199 ) -> MultiModalDataDict:
201 - feature_extractor = self.info.get_feature_extractor(
202 - **(mm_processor_kwargs or {})
203 - )
200 + feature_extractor = self.info.get_feature_extractor()
201
202 sampling_rate = feature_extractor.sampling_rate
203 audio_len = feature_extractor.chunk_length * sampling_rate
204 num_audios = mm_counts.get("audio", 0)
205
209 - audio_overrides = mm_options.get("audio") if mm_options else None
206 + audio_overrides = mm_options.get("audio")
207
208 return {
209 "audio": self._get_dummy_audios(
213 - length=audio_len, num_audios=num_audios, overrides=audio_overrides
210 + length=audio_len,
211 + num_audios=num_audios,
212 + overrides=audio_overrides,
213 )
214 }
215
vllm/model_executor/models/qwen2_vl.py
+10 -9
@@ -925,9 +925,14 @@ class Qwen2VLProcessingInfo(BaseProcessingInfo):
925 vision_config = hf_config.vision_config
926 patch_size = vision_config.patch_size
927 merge_size = vision_config.spatial_merge_size
928 +
929 if max_pixels is None:
930 image_processor = self.get_image_processor()
930 - max_pixels = image_processor.size["longest_edge"]
931 +
932 + mm_kwargs = self.ctx.get_merged_mm_kwargs({})
933 + size = mm_kwargs.get("size", image_processor.size)
934 + max_pixels = size["longest_edge"]
935 +
936 unit = patch_size * merge_size
937 max_seq_len = max_pixels // (unit * unit)
938
@@ -1027,22 +1032,18 @@ class Qwen2VLDummyInputsBuilder(BaseDummyInputsBuilder[Qwen2VLProcessingInfo]):
1032 self,
1033 seq_len: int,
1034 mm_counts: Mapping[str, int],
1030 - mm_options: Mapping[str, BaseDummyOptions] | None = None,
1031 - mm_processor_kwargs: Mapping[str, object] | None = None,
1035 + mm_options: Mapping[str, BaseDummyOptions],
1036 ) -> MultiModalDataDict:
1037 num_images = mm_counts.get("image", 0)
1038 num_videos = mm_counts.get("video", 0)
1039
1036 - mm_processor_kwargs = mm_processor_kwargs or {}
1037 - target_width, target_height = self.info.get_image_size_with_most_features(
1038 - max_pixels=mm_processor_kwargs.get("max_pixels", None)
1039 - )
1040 + target_width, target_height = self.info.get_image_size_with_most_features()
1041 target_num_frames = self.info.get_num_frames_with_most_features(
1042 seq_len, mm_counts
1043 )
1044
1044 - image_overrides = mm_options.get("image") if mm_options else None
1045 - video_overrides = mm_options.get("video") if mm_options else None
1045 + image_overrides = mm_options.get("image")
1046 + video_overrides = mm_options.get("video")
1047
1048 return {
1049 "image": self._get_dummy_images(
vllm/model_executor/models/qwen3_asr.py
+3 -6
@@ -146,14 +146,11 @@ class Qwen3ASRDummyInputsBuilder(BaseDummyInputsBuilder[Qwen3ASRProcessingInfo])
146 self,
147 seq_len: int,
148 mm_counts: Mapping[str, int],
149 - mm_options: Mapping[str, BaseDummyOptions] | None = None,
150 - mm_processor_kwargs: Mapping[str, object] | None = None,
149 + mm_options: Mapping[str, BaseDummyOptions],
150 ) -> MultiModalDataDict:
151 num_audios = mm_counts.get("audio", 0)
152
154 - feature_extractor = self.info.get_feature_extractor(
155 - **(mm_processor_kwargs or {})
156 - )
153 + feature_extractor = self.info.get_feature_extractor()
154
155 target_audio_length = (
156 min(
@@ -163,7 +160,7 @@ class Qwen3ASRDummyInputsBuilder(BaseDummyInputsBuilder[Qwen3ASRProcessingInfo])
160 * feature_extractor.sampling_rate
161 )
162
166 - audio_overrides = mm_options.get("audio") if mm_options else None
163 + audio_overrides = mm_options.get("audio")
164
165 return {
166 "audio": self._get_dummy_audios(
vllm/model_executor/models/qwen3_vl.py
+23 -13
@@ -703,11 +703,18 @@ class Qwen3VLProcessingInfo(Qwen2VLProcessingInfo):
703 mm_counts: Mapping[str, int],
704 ) -> int:
705 video_processor = self.get_video_processor()
706 - video_max_pixels = video_processor.size["longest_edge"]
706 +
707 + mm_kwargs = self.ctx.get_merged_mm_kwargs({})
708 + video_size = mm_kwargs.get("size", video_processor.size)
709 + temporal_patch_size = mm_kwargs.get(
710 + "temporal_patch_size", video_processor.temporal_patch_size
711 + )
712 +
713 # video_max_pixels contains the temporal compression factor,
714 # so we divide by 2 to get the maximum number of image pixels.
715 + video_max_pixels = video_size["longest_edge"]
716 target_width, target_height = self.get_image_size_with_most_features(
710 - max_pixels=video_max_pixels // video_processor.temporal_patch_size
717 + max_pixels=video_max_pixels // temporal_patch_size
718 )
719 num_video_soft_tokens = self.get_num_video_tokens(
720 image_width=target_width,
@@ -789,19 +796,15 @@ class Qwen3VLDummyInputsBuilder(BaseDummyInputsBuilder[Qwen3VLProcessingInfo]):
796 self,
797 seq_len: int,
798 mm_counts: Mapping[str, int],
792 - mm_options: Mapping[str, BaseDummyOptions] | None = None,
793 - mm_processor_kwargs: Mapping[str, object] | None = None,
799 + mm_options: Mapping[str, BaseDummyOptions],
800 ) -> MultiModalDataDict:
801 num_images = mm_counts.get("image", 0)
802 num_videos = mm_counts.get("video", 0)
797 - image_overrides = mm_options.get("image") if mm_options else None
798 - video_overrides = mm_options.get("video") if mm_options else None
803 + image_overrides = mm_options.get("image")
804 + video_overrides = mm_options.get("video")
805
800 - mm_processor_kwargs = mm_processor_kwargs or {}
806 target_image_width, target_image_height = (
802 - self.info.get_image_size_with_most_features(
803 - max_pixels=mm_processor_kwargs.get("max_pixels", None),
804 - )
807 + self.info.get_image_size_with_most_features()
808 )
809
810 # treat videos as special images
@@ -826,13 +829,20 @@ class Qwen3VLDummyInputsBuilder(BaseDummyInputsBuilder[Qwen3VLProcessingInfo]):
829 target_num_frames = min(target_num_frames, num_frames_override)
830 target_num_frames = max(target_num_frames, 2)
831
829 - video_processor = self.info.get_video_processor(**(mm_processor_kwargs or {}))
830 - video_max_pixels = video_processor.size["longest_edge"]
832 + video_processor = self.info.get_video_processor()
833 +
834 + mm_kwargs = self.info.ctx.get_merged_mm_kwargs({})
835 + video_size = mm_kwargs.get("size", video_processor.size)
836 + temporal_patch_size = mm_kwargs.get(
837 + "temporal_patch_size", video_processor.temporal_patch_size
838 + )
839 +
840 # video_max_pixels contains the temporal compression factor,
841 # so we divide by 2 to get the maximum number of image pixels.
842 + video_max_pixels = video_size["longest_edge"]
843 target_video_width, target_video_height = (
844 self.info.get_image_size_with_most_features(
835 - max_pixels=video_max_pixels // video_processor.temporal_patch_size
845 + max_pixels=video_max_pixels // temporal_patch_size
846 )
847 )
848 target_video_size, _ = self.info._get_vision_info(
vllm/model_executor/models/qwen_vl.py
+2 -3
@@ -617,8 +617,7 @@ class QwenVLDummyInputsBuilder(BaseDummyInputsBuilder[QwenVLProcessingInfo]):
617 self,
618 seq_len: int,
619 mm_counts: Mapping[str, int],
620 - mm_options: Mapping[str, BaseDummyOptions] | None = None,
621 - mm_processor_kwargs: Mapping[str, object] | None = None,
620 + mm_options: Mapping[str, BaseDummyOptions],
621 ) -> MultiModalDataDict:
622 hf_config = self.info.get_hf_config()
623 vision_config = hf_config.visual
@@ -626,7 +625,7 @@ class QwenVLDummyInputsBuilder(BaseDummyInputsBuilder[QwenVLProcessingInfo]):
625 target_width = target_height = vision_config["image_size"]
626 num_images = mm_counts.get("image", 0)
627
629 - image_overrides = mm_options.get("image") if mm_options else None
628 + image_overrides = mm_options.get("image")
629
630 return {
631 "image": self._get_dummy_images(
vllm/model_executor/models/rvl.py
+2 -3
@@ -40,14 +40,13 @@ class RVLDummyInputsBuilder(LlavaDummyInputsBuilder[RVLProcessingInfo]):
40 self,
41 seq_len: int,
42 mm_counts: Mapping[str, int],
43 - mm_options: Mapping[str, BaseDummyOptions] | None = None,
44 - mm_processor_kwargs: Mapping[str, object] | None = None,
43 + mm_options: Mapping[str, BaseDummyOptions],
44 ) -> MultiModalDataDict:
45 num_images = mm_counts.get("image", 0)
46
47 target_width, target_height = self.info.get_image_size_with_most_features()
48
50 - image_overrides = mm_options.get("image") if mm_options else None
49 + image_overrides = mm_options.get("image")
50
51 return {
52 "image": self._get_dummy_images(
vllm/model_executor/models/siglip.py
+2 -3
@@ -158,14 +158,13 @@ class SiglipDummyInputsBuilder(BaseDummyInputsBuilder[SiglipProcessingInfo]):
158 self,
159 seq_len: int,
160 mm_counts: Mapping[str, int],
161 - mm_options: Mapping[str, BaseDummyOptions] | None = None,
162 - mm_processor_kwargs: Mapping[str, object] | None = None,
161 + mm_options: Mapping[str, BaseDummyOptions],
162 ) -> MultiModalDataDict:
163 num_images = mm_counts.get("image", 0)
164
165 target_width, target_height = self.info.get_image_size_with_most_features()
166
168 - image_overrides = mm_options.get("image") if mm_options else None
167 + image_overrides = mm_options.get("image")
168
169 return {
170 "image": self._get_dummy_images(
vllm/model_executor/models/skyworkr1v.py
+2 -3
@@ -529,13 +529,12 @@ class SkyworkR1VDummyInputsBuilder(BaseDummyInputsBuilder[SkyworkR1VProcessingIn
529 self,
530 seq_len: int,
531 mm_counts: Mapping[str, int],
532 - mm_options: Mapping[str, BaseDummyOptions] | None = None,
533 - mm_processor_kwargs: Mapping[str, object] | None = None,
532 + mm_options: Mapping[str, BaseDummyOptions],
533 ) -> MultiModalDataDict:
534 target_width, target_height = self.info.get_image_size_with_most_features()
535 num_images = mm_counts.get("image", 0)
536
538 - image_overrides = mm_options.get("image") if mm_options else None
537 + image_overrides = mm_options.get("image")
538
539 return {
540 "image": self._get_dummy_images(
vllm/model_executor/models/step3_vl.py
+2 -3
@@ -564,13 +564,12 @@ class Step3VLDummyInputsBuilder(BaseDummyInputsBuilder[Step3VLProcessingInfo]):
564 self,
565 seq_len: int,
566 mm_counts: Mapping[str, int],
567 - mm_options: Mapping[str, BaseDummyOptions] | None = None,
568 - mm_processor_kwargs: Mapping[str, object] | None = None,
567 + mm_options: Mapping[str, BaseDummyOptions],
568 ) -> MultiModalDataDict:
569 target_width, target_height = self.info.get_image_size_with_most_features()
570 num_images = mm_counts.get("image", 0)
571
573 - image_overrides = mm_options.get("image") if mm_options else None
572 + image_overrides = mm_options.get("image")
573
574 return {
575 "image": self._get_dummy_images(
vllm/model_executor/models/terratorch.py
+1 -2
@@ -154,8 +154,7 @@ class TerratorchInputBuilder(BaseDummyInputsBuilder[TerratorchProcessingInfo]):
154 self,
155 seq_len: int,
156 mm_counts: Mapping[str, int],
157 - mm_options: Mapping[str, BaseDummyOptions] | None = None,
158 - mm_processor_kwargs: Mapping[str, object] | None = None,
157 + mm_options: Mapping[str, BaseDummyOptions],
158 ) -> MultiModalDataDict:
159 # Dummy data is generated based on the 'input' section
160 # defined in the HF configuration file
vllm/model_executor/models/transformers/multimodal.py
+2 -3
@@ -101,14 +101,13 @@ class MultiModalDummyInputsBuilder(BaseDummyInputsBuilder[MultiModalProcessingIn
101 self,
102 seq_len: int,
103 mm_counts: Mapping[str, int],
104 - mm_options: Mapping[str, "BaseDummyOptions"] | None = None,
105 - mm_processor_kwargs: Mapping[str, object] | None = None,
104 + mm_options: Mapping[str, "BaseDummyOptions"],
105 ) -> MultiModalDataDict:
106 num_images = mm_counts.get("image", 0)
107
108 target_width, target_height = self.info.get_max_image_size()
109
111 - image_overrides = mm_options.get("image") if mm_options else None
110 + image_overrides = mm_options.get("image")
111
112 return {
113 "image": self._get_dummy_images(
vllm/model_executor/models/ultravox.py
+6 -7
@@ -164,12 +164,9 @@ class UltravoxDummyInputsBuilder(BaseDummyInputsBuilder[UltravoxProcessingInfo])
164 self,
165 seq_len: int,
166 mm_counts: Mapping[str, int],
167 - mm_options: Mapping[str, BaseDummyOptions] | None = None,
168 - mm_processor_kwargs: Mapping[str, object] | None = None,
167 + mm_options: Mapping[str, BaseDummyOptions],
168 ) -> MultiModalDataDict:
170 - feature_extractor = self.info.get_feature_extractor(
171 - **(mm_processor_kwargs or {})
172 - )
169 + feature_extractor = self.info.get_feature_extractor()
170
171 sampling_rate = feature_extractor.sampling_rate
172 audio_len = (
@@ -177,11 +174,13 @@ class UltravoxDummyInputsBuilder(BaseDummyInputsBuilder[UltravoxProcessingInfo])
174 )
175 num_audios = mm_counts.get("audio", 0)
176
180 - audio_overrides = mm_options.get("audio") if mm_options else None
177 + audio_overrides = mm_options.get("audio")
178
179 return {
180 "audio": self._get_dummy_audios(
184 - length=audio_len, num_audios=num_audios, overrides=audio_overrides
181 + length=audio_len,
182 + num_audios=num_audios,
183 + overrides=audio_overrides,
184 )
185 }
186
vllm/model_executor/models/voxtral.py
+6 -6
@@ -218,18 +218,19 @@ class VoxtralDummyInputsBuilder(BaseDummyInputsBuilder[VoxtralProcessingInfo]):
218 self,
219 seq_len: int,
220 mm_counts: Mapping[str, int],
221 - mm_options: Mapping[str, BaseDummyOptions] | None = None,
222 - mm_processor_kwargs: Mapping[str, object] | None = None,
221 + mm_options: Mapping[str, BaseDummyOptions],
222 ) -> MultiModalDataDict:
223 num_audios = mm_counts.get("audio", 0)
224
225 target_length = self.info.get_max_audio_array_len()
226
228 - audio_overrides = mm_options.get("audio") if mm_options else None
227 + audio_overrides = mm_options.get("audio")
228
229 return {
230 "audio": self._get_dummy_audios(
232 - length=target_length, num_audios=num_audios, overrides=audio_overrides
231 + length=target_length,
232 + num_audios=num_audios,
233 + overrides=audio_overrides,
234 )
235 }
236
@@ -237,8 +238,7 @@ class VoxtralDummyInputsBuilder(BaseDummyInputsBuilder[VoxtralProcessingInfo]):
238 self,
239 seq_len: int,
240 mm_counts: Mapping[str, int],
240 - mm_options: Mapping[str, BaseDummyOptions] | None = None,
241 - mm_processor_kwargs: Mapping[str, object] | None = None,
241 + mm_options: Mapping[str, BaseDummyOptions],
242 ) -> ProcessorInputs:
243 tokenizer = self.info.get_tokenizer()
244
vllm/model_executor/models/whisper.py
+6 -7
@@ -695,22 +695,21 @@ class WhisperDummyInputsBuilder(BaseDummyInputsBuilder[WhisperProcessingInfo]):
695 self,
696 seq_len: int,
697 mm_counts: Mapping[str, int],
698 - mm_options: Mapping[str, BaseDummyOptions] | None = None,
699 - mm_processor_kwargs: Mapping[str, object] | None = None,
698 + mm_options: Mapping[str, BaseDummyOptions],
699 ) -> MultiModalDataDict:
701 - feature_extractor = self.info.get_feature_extractor(
702 - **(mm_processor_kwargs or {})
703 - )
700 + feature_extractor = self.info.get_feature_extractor()
701
702 sampling_rate = feature_extractor.sampling_rate
703 audio_len = feature_extractor.chunk_length * sampling_rate
704 num_audios = mm_counts.get("audio", 0)
705
709 - audio_overrides = mm_options.get("audio") if mm_options else None
706 + audio_overrides = mm_options.get("audio")
707
708 return {
709 "audio": self._get_dummy_audios(
713 - length=audio_len, num_audios=num_audios, overrides=audio_overrides
710 + length=audio_len,
711 + num_audios=num_audios,
712 + overrides=audio_overrides,
713 )
714 }
715
vllm/multimodal/processing/context.py
+5 -7
@@ -266,11 +266,14 @@ class InputProcessingContext:
266 if isinstance(tokenizer, MistralTokenizer):
267 tokenizer = tokenizer.transformers_tokenizer
268
269 + merged_kwargs = self.get_merged_mm_kwargs(kwargs)
270 + merged_kwargs.pop("tokenizer", None)
271 +
272 return cached_processor_from_config(
273 self.model_config,
274 processor_cls=typ,
275 tokenizer=tokenizer,
273 - **kwargs,
276 + **merged_kwargs,
277 )
278
279 def init_processor(
@@ -283,12 +286,7 @@ class InputProcessingContext:
286 Initialize a HuggingFace-like processor class, merging the
287 keyword arguments with those in the model's configuration.
288 """
286 - mm_config = self.model_config.get_multimodal_config()
287 - base_kwargs = mm_config.mm_processor_kwargs
288 - if base_kwargs is None:
289 - base_kwargs = {}
290 -
291 - merged_kwargs = {**base_kwargs, **kwargs}
289 + merged_kwargs = self.get_merged_mm_kwargs(kwargs)
290
291 return typ(**merged_kwargs)
292
vllm/multimodal/processing/dummy_inputs.py
+3 -13
@@ -62,8 +62,7 @@ class BaseDummyInputsBuilder(ABC, Generic[_I]):
62 self,
63 seq_len: int,
64 mm_counts: Mapping[str, int],
65 - mm_options: Mapping[str, BaseDummyOptions] | None = None,
66 - mm_processor_kwargs: Mapping[str, object] | None = None,
65 + mm_options: Mapping[str, BaseDummyOptions],
66 ) -> MultiModalDataDict:
67 """
68 Build the multimodal input which, after processing, results in
@@ -83,8 +82,7 @@ class BaseDummyInputsBuilder(ABC, Generic[_I]):
82 self,
83 seq_len: int,
84 mm_counts: Mapping[str, int],
86 - mm_options: Mapping[str, BaseDummyOptions] | None = None,
87 - mm_processor_kwargs: Mapping[str, object] | None = None,
85 + mm_options: Mapping[str, BaseDummyOptions],
86 ) -> ProcessorInputs:
87 """
88 Build the input which, after processing, results in
@@ -94,16 +92,9 @@ class BaseDummyInputsBuilder(ABC, Generic[_I]):
92 seq_len: Sequence length
93 mm_counts: Count of items per modality
94 mm_options: Configurable options per modality (optional)
97 - mm_processor_kwargs: Additional keyword arguments
98 - for hf_processor (optional)
95 """
96 dummy_text = self.get_dummy_text(mm_counts)
101 - dummy_mm_data = self.get_dummy_mm_data(
102 - seq_len,
103 - mm_counts,
104 - mm_options,
105 - mm_processor_kwargs=mm_processor_kwargs,
106 - )
97 + dummy_mm_data = self.get_dummy_mm_data(seq_len, mm_counts, mm_options)
98 dummy_mm_items = self.info.parse_mm_data(dummy_mm_data, validate=False)
99
100 tokenization_kwargs = {"truncation": False}
@@ -111,7 +102,6 @@ class BaseDummyInputsBuilder(ABC, Generic[_I]):
102 return ProcessorInputs(
103 prompt=dummy_text,
104 mm_items=dummy_mm_items,
114 - hf_processor_mm_kwargs=mm_processor_kwargs or {},
105 tokenization_kwargs=tokenization_kwargs,
106 )
107
vllm/multimodal/registry.py
+1 -24
@@ -5,7 +5,6 @@ from dataclasses import dataclass
5 from multiprocessing.synchronize import Lock as LockType
6 from typing import TYPE_CHECKING, Generic, Literal, Protocol, TypeVar, cast
7
8 -from vllm.config.multimodal import BaseDummyOptions
8 from vllm.config.observability import ObservabilityConfig
9 from vllm.logger import init_logger
10 from vllm.tokenizers import TokenizerLike, cached_tokenizer_from_config
@@ -99,27 +98,6 @@ class MultiModalRegistry:
98 A registry that dispatches data processing according to the model.
99 """
100
102 - def _extract_mm_options(
103 - self,
104 - model_config: "ModelConfig",
105 - ) -> Mapping[str, BaseDummyOptions] | None:
106 - """
107 - Extract multimodal dummy options from model config.
108 -
109 - Returns None if no configurable options are found, otherwise returns
110 - a mapping of modality names to their dummy options.
111 - """
112 - if not model_config.multimodal_config:
113 - return None
114 -
115 - mm_options = {
116 - m: opt
117 - for m in model_config.multimodal_config.limit_per_prompt
118 - if (opt := model_config.multimodal_config.get_dummy_options(m)) is not None
119 - }
120 -
121 - return mm_options if len(mm_options) > 0 else None
122 -
101 def supports_multimodal_inputs(self, model_config: "ModelConfig") -> bool:
102 """
103 Checks if the model supports multimodal inputs.
@@ -261,8 +239,7 @@ class MultiModalRegistry:
239 processor_inputs = processor.dummy_inputs.get_dummy_processor_inputs(
240 seq_len=seq_len,
241 mm_counts=mm_counts,
264 - mm_options=self._extract_mm_options(model_config),
265 - mm_processor_kwargs=mm_config.mm_processor_kwargs,
242 + mm_options=mm_config.limit_per_prompt,
243 )
244 mm_inputs = processor.apply(
245 prompt=processor_inputs.prompt,