8a5cf54387 · tduncan/Nvidia-Optimization-Stack-VLLM

tests/models/multimodal/processing/test_audioflamingo3.py

+2 -1

@@ -25,6 +25,7 @@ import torch

25 from transformers import PretrainedConfig

26

27 from tests.models.registry import HF_EXAMPLE_MODELS

28 +from vllm.config.multimodal import MultiModalDummyOptions

29

30

31 class MockAudioFlamingo3Config(PretrainedConfig):

@@ -127,7 +128,7 @@ def test_dummy_data_generation(mock_ctx):

128 builder = AudioFlamingo3DummyInputsBuilder(info)

129

130 mm_counts = {"audio": 2}

130 - dummy_data = builder.get_dummy_mm_data(100, mm_counts, {})

131 + dummy_data = builder.get_dummy_mm_data(100, mm_counts, MultiModalDummyOptions())

132

133 assert "audio" in dummy_data

134 assert len(dummy_data["audio"]) == 2

tests/models/multimodal/processing/test_common.py

+9 -6

@@ -13,6 +13,7 @@ from vllm.config.multimodal import (

13 AudioDummyOptions,

14 BaseDummyOptions,

15 ImageDummyOptions,

16 + MultiModalDummyOptions,

17 VideoDummyOptions,

18 )

19 from vllm.inputs import MultiModalDataDict, MultiModalInput

@@ -174,7 +175,7 @@ def get_token_prompt(

175 inputs = dummy_inputs.get_dummy_processor_inputs(

176 model_config.max_model_len,

177 mm_counts,

177 - mm_options={},

178 + mm_options=MultiModalDummyOptions(),

179 # Assume all Mistral models define this extra argument

180 mm_data=mm_data, # type: ignore[call-arg]

181 )

@@ -182,7 +183,7 @@ def get_token_prompt(

183 inputs = dummy_inputs.get_dummy_processor_inputs(

184 model_config.max_model_len,

185 mm_counts,

185 - mm_options={},

186 + mm_options=MultiModalDummyOptions(),

187 )

188

189 if not isinstance(inputs.prompt, list):

@@ -278,10 +279,12 @@ def _test_processing_correctness(

279 return BaseDummyOptions(count=count)

280

281 # Assign normalized DummyOptions to the model config

281 - model_config.get_multimodal_config().limit_per_prompt = {

282 - modality: _to_dummy_options(modality, count)

283 - for modality, count in limit_mm_per_prompt_ints.items()

284 - }

282 + model_config.get_multimodal_config().limit_per_prompt = MultiModalDummyOptions(

283 + {

284 + modality: _to_dummy_options(modality, count)

285 + for modality, count in limit_mm_per_prompt_ints.items()

286 + }

287 + )

288

289 baseline_processor = factories.build_processor(ctx, cache=None)

290 cached_processor = factories.build_processor(ctx, cache=cache)

tests/models/multimodal/processing/test_molmo2.py

+4 -2

@@ -6,7 +6,7 @@ from types import SimpleNamespace

6 import pytest

7 import torch

8

9 -from vllm.config.multimodal import VideoDummyOptions

9 +from vllm.config.multimodal import MultiModalDummyOptions, VideoDummyOptions

10 from vllm.model_executor.models.molmo2 import (

11 Molmo2DummyInputsBuilder,

12 build_flat_image_bool_length,

@@ -83,7 +83,9 @@ def test_dummy_video_num_frames_override_honors_min_of_two(

83 data = builder.get_dummy_mm_data(

84 seq_len=128,

85 mm_counts={"image": 0, "video": 1},

86 - mm_options={"video": VideoDummyOptions(num_frames=num_frames_override)},

86 + mm_options=MultiModalDummyOptions(

87 + video=VideoDummyOptions(num_frames=num_frames_override)

88 + ),

89 )

90

91 video, _metadata = data["video"][0]

tests/models/multimodal/processing/test_qwen3_vl.py

+7 -2

@@ -12,6 +12,7 @@ import numpy as np

12 import pytest

13

14 from vllm.config import ModelConfig

15 +from vllm.config.multimodal import MultiModalDummyOptions

16 from vllm.multimodal import MULTIMODAL_REGISTRY

17

18 from ...registry import HF_EXAMPLE_MODELS

@@ -339,7 +340,9 @@ def test_dummy_video_spreads_budget_when_frame_cap_enabled(model_id: str) -> Non

340 limit_mm_per_prompt={"image": 0, "video": 1},

341 )

342 capped = MULTIMODAL_REGISTRY.create_processor(capped_ctx.model_config)

342 - capped_dummy = capped.dummy_inputs.get_dummy_mm_data(1024, {"video": 1}, {})

343 + capped_dummy = capped.dummy_inputs.get_dummy_mm_data(

344 + 1024, {"video": 1}, MultiModalDummyOptions()

345 + )

346 capped_frames = capped_dummy["video"][0][0].shape[0]

347 assert capped_frames == 16, (

348 f"Expected the dummy to spread the budget over 16 frames, got {capped_frames}"

@@ -351,7 +354,9 @@ def test_dummy_video_spreads_budget_when_frame_cap_enabled(model_id: str) -> Non

354 limit_mm_per_prompt={"image": 0, "video": 1},

355 )

356 uncapped = MULTIMODAL_REGISTRY.create_processor(uncapped_ctx.model_config)

354 - uncapped_dummy = uncapped.dummy_inputs.get_dummy_mm_data(1024, {"video": 1}, {})

357 + uncapped_dummy = uncapped.dummy_inputs.get_dummy_mm_data(

358 + 1024, {"video": 1}, MultiModalDummyOptions()

359 + )

360 uncapped_frames = uncapped_dummy["video"][0][0].shape[0]

361 assert uncapped_frames == 2, (

362 f"Expected the uncapped dummy to keep 2 frames, got {uncapped_frames}"

tests/models/multimodal/processing/test_tensor_schema.py

+8 -5

@@ -18,6 +18,7 @@ from vllm.config.multimodal import (

18 AudioDummyOptions,

19 BaseDummyOptions,

20 ImageDummyOptions,

21 + MultiModalDummyOptions,

22 VideoDummyOptions,

23 )

24 from vllm.distributed import (

@@ -98,7 +99,7 @@ def create_batched_mm_kwargs(

99 processor_inputs = dummy_inputs.get_dummy_processor_inputs(

100 seq_len=model_config.max_model_len,

101 mm_counts=mm_counts,

101 - mm_options={},

102 + mm_options=MultiModalDummyOptions(),

103 )

104 mm_items = processor_inputs.mm_data_items

105 resized_mm_data = {

@@ -248,10 +249,12 @@ def test_model_tensor_schema(model_id: str):

249 return AudioDummyOptions(count=count)

250 return BaseDummyOptions(count=count)

251

251 - model_config.get_multimodal_config().limit_per_prompt = {

252 - modality: _to_dummy_options(modality, count)

253 - for modality, count in limit_mm_per_prompt.items()

254 - }

252 + model_config.get_multimodal_config().limit_per_prompt = MultiModalDummyOptions(

253 + {

254 + modality: _to_dummy_options(modality, count)

255 + for modality, count in limit_mm_per_prompt.items()

256 + }

257 + )

258 processor = factories.build_processor(ctx, cache=None)

259

260 with initialize_dummy_model(model_cls, model_config) as model:

vllm/config/multimodal.py

+57 -22

@@ -3,11 +3,18 @@

3

4 from collections.abc import Mapping

5 from pathlib import Path

6 -from typing import Any, Literal, TypeAlias, TypedDict, final

6 +from typing import Any, Literal, TypeAlias, TypeVar, final, overload

7

8 import torch

9 -from pydantic import ConfigDict, Field, field_validator, model_validator

9 +from pydantic import (

10 + ConfigDict,

11 + Field,

12 + GetCoreSchemaHandler,

13 + field_validator,

14 + model_validator,

15 +)

16 from pydantic.dataclasses import dataclass

17 +from pydantic_core import core_schema

18

19 import vllm.envs as envs

20 from vllm.config.ec_transfer import ECTransferConfig

@@ -18,6 +25,8 @@ from vllm.v1.attention.backends.registry import AttentionBackendEnum

25

26 logger = init_logger(__name__)

27

28 +_T = TypeVar("_T")

29 +

30

31 @dataclass

32 class BaseDummyOptions:

@@ -51,17 +60,50 @@ class AudioDummyOptions(BaseDummyOptions):

60

61

62 @final

54 -class MultiModalDummyOptionsBuiltins(TypedDict, total=False):

55 - """Type annotations for modality types predefined by vLLM."""

63 +class MultiModalDummyOptions(dict[str, BaseDummyOptions]):

64 + """Dummy data options for each modality.

65 +

66 + Lookups of the modalities predefined by vLLM return their own options

67 + class, while any other modality returns

68 + [`BaseDummyOptions`][vllm.config.multimodal.BaseDummyOptions].

69 + """

70 +

71 + @overload # type: ignore[override]

72 + def get(self, key: Literal["image"], /) -> ImageDummyOptions | None: ...

73 +

74 + @overload

75 + def get(self, key: Literal["image"], default: _T, /) -> ImageDummyOptions | _T: ...

76 +

77 + @overload

78 + def get(self, key: Literal["video"], /) -> VideoDummyOptions | None: ...

79 +

80 + @overload

81 + def get(self, key: Literal["video"], default: _T, /) -> VideoDummyOptions | _T: ...

82 +

83 + @overload

84 + def get(self, key: Literal["audio"], /) -> AudioDummyOptions | None: ...

85

57 - image: ImageDummyOptions

58 - """Options for dummy images."""

86 + @overload

87 + def get(self, key: Literal["audio"], default: _T, /) -> AudioDummyOptions | _T: ...

88

60 - video: VideoDummyOptions

61 - """Options for dummy videos."""

89 + @overload

90 + def get(self, key: str, /) -> BaseDummyOptions | None: ...

91

63 - audio: AudioDummyOptions

64 - """Options for dummy audios."""

92 + @overload

93 + def get(self, key: str, default: _T, /) -> BaseDummyOptions | _T: ...

94 +

95 + def get(self, key: str, default: object = None, /) -> object:

96 + return super().get(key, default)

97 +

98 + @classmethod

99 + def __get_pydantic_core_schema__(

100 + cls,

101 + source_type: Any,

102 + handler: GetCoreSchemaHandler,

103 + ) -> core_schema.CoreSchema:

104 + return core_schema.no_info_after_validator_function(

105 + cls, handler.generate_schema(dict[str, BaseDummyOptions])

106 + )

107

108

109 MMEncoderTPMode = Literal["weights", "data"]

@@ -75,15 +117,6 @@ MMProcessorDevice: TypeAlias = str

117 `"xpu"` on XPU). Validated against that set by the CLI."""

118

119

78 -MMDummyOptions: TypeAlias = dict[str, BaseDummyOptions]

79 -"""

80 -A dictionary containing an entry for each modality type of dummy data.

81 -

82 -The built-in modalities are defined by

83 -[`MultiModalDummyOptionsBuiltins`][vllm.config.multimodal.MultiModalDummyOptionsBuiltins].

84 -"""

85 -

86 -

120 @config

121 class MultiModalConfig:

122 """Controls the behavior of multimodal models."""

@@ -91,7 +124,9 @@ class MultiModalConfig:

124 language_model_only: bool = False

125 """If True, disables all multimodal inputs by setting all modality limits to 0.

126 Equivalent to setting `--limit-mm-per-prompt` to 0 for every modality."""

94 - limit_per_prompt: MMDummyOptions = Field(default_factory=dict)

127 + limit_per_prompt: MultiModalDummyOptions = Field(

128 + default_factory=MultiModalDummyOptions

129 + )

130 """The maximum number of input items and options allowed per

131 prompt for each modality.

132

@@ -257,8 +292,8 @@ class MultiModalConfig:

292 def _validate_limit_per_prompt(

293 cls,

294 value: dict[str, int | dict[str, int]],

260 - ) -> MMDummyOptions:

261 - out: MMDummyOptions = {}

295 + ) -> MultiModalDummyOptions:

296 + out = MultiModalDummyOptions()

297

298 for k, v in value.items():

299 # Handle legacy format where only count is specified

vllm/engine/arg_utils.py

+15

@@ -22,6 +22,7 @@ from typing import (

22 cast,

23 get_args,

24 get_origin,

25 + is_typeddict,

26 )

27

28 import huggingface_hub

@@ -183,6 +184,18 @@ def is_type(type_hint: TypeHint, type: TypeHintT) -> TypeIs[TypeHintT]:

184 return type_hint is type or get_origin(type_hint) is type

185

186

187 +def is_dict_subclass(type_hint: TypeHint) -> bool:

188 + """Check if the type hint is a subclass of `dict`.

189 +

190 + `TypedDict`s are excluded because they do not support class checks.

191 + """

192 + return (

193 + isinstance(type_hint, type)

194 + and not is_typeddict(type_hint)

195 + and issubclass(type_hint, dict)

196 + )

197 +

198 +

199 def contains_type(type_hints: set[TypeHint], type: TypeHintT) -> bool:

200 """Check if the type hints contain a specific type."""

201 return any(is_type(type_hint, type) for type_hint in type_hints)

@@ -305,6 +318,8 @@ def _compute_kwargs(cls: ConfigType) -> dict[str, dict[str, Any]]:

318 for field in fields(cls):

319 # Get the set of possible types for the field

320 type_hints: set[TypeHint] = get_type_hints(field.type)

321 + # Subclasses of dict (e.g. MultiModalDummyOptions) are CLI dicts

322 + type_hints = {dict if is_dict_subclass(th) else th for th in type_hints}

323

324 # If the field is a dataclass, we can use the model_validate_json

325 generator = (th for th in type_hints if is_dataclass(th))

vllm/model_executor/models/aria.py

+4 -8

@@ -10,7 +10,7 @@ from transformers.models.aria.modeling_aria import AriaCrossAttention

10 from transformers.models.aria.processing_aria import AriaProcessor

11

12 from vllm.config import VllmConfig

13 -from vllm.config.multimodal import BaseDummyOptions, ImageDummyOptions

13 +from vllm.config.multimodal import MultiModalDummyOptions

14 from vllm.inputs import MultiModalDataDict

15 from vllm.model_executor.layers.activation import get_act_fn

16 from vllm.model_executor.layers.fused_moe import FusedMoEFactory

@@ -315,22 +315,18 @@ class AriaDummyInputsBuilder(BaseDummyInputsBuilder[AriaProcessingInfo]):

315 self,

316 seq_len: int,

317 mm_counts: Mapping[str, int],

318 - mm_options: Mapping[str, BaseDummyOptions],

318 + mm_options: MultiModalDummyOptions,

319 ) -> MultiModalDataDict:

320 vision_config = self.info.get_vision_config()

321

322 max_image_size = vision_config.image_size

323 - num_images = mm_counts.get("image", 0)

324 -

325 - image_overrides = mm_options.get("image")

326 - assert image_overrides is None or isinstance(image_overrides, ImageDummyOptions)

323

324 return {

325 "image": self._get_dummy_images(

326 width=max_image_size,

327 height=max_image_size,

332 - num_images=num_images,

333 - overrides=image_overrides,

328 + num_images=mm_counts.get("image", 0),

329 + overrides=mm_options.get("image"),

330 )

331 }

332

vllm/model_executor/models/audioflamingo3.py

+4 -7

@@ -30,7 +30,7 @@ from transformers.models.audioflamingo3 import (

30 from transformers.models.qwen2_audio import Qwen2AudioEncoder

31

32 from vllm.config import VllmConfig

33 -from vllm.config.multimodal import AudioDummyOptions, BaseDummyOptions

33 +from vllm.config.multimodal import MultiModalDummyOptions

34 from vllm.inputs import ModalityData, MultiModalDataDict

35 from vllm.model_executor.layers.activation import get_act_fn

36 from vllm.model_executor.models.module_mapping import MultiModelKeys

@@ -216,21 +216,18 @@ class AudioFlamingo3DummyInputsBuilder(

216 self,

217 seq_len: int,

218 mm_counts: Mapping[str, int],

219 - mm_options: Mapping[str, BaseDummyOptions],

219 + mm_options: MultiModalDummyOptions,

220 ) -> MultiModalDataDict:

221 hf_processor = self.info.get_hf_processor()

222 feature_extractor = self.info.get_feature_extractor()

223 sampling_rate = feature_extractor.sampling_rate

224 audio_len = int(hf_processor.max_audio_len * sampling_rate)

225 - num_audios = mm_counts.get("audio", 0)

226 - audio_overrides = mm_options.get("audio")

227 - assert audio_overrides is None or isinstance(audio_overrides, AudioDummyOptions)

225

226 return {

227 "audio": self._get_dummy_audios(

228 length=audio_len,

232 - num_audios=num_audios,

233 - overrides=audio_overrides,

229 + num_audios=mm_counts.get("audio", 0),

230 + overrides=mm_options.get("audio"),

231 )

232 }

233

vllm/model_executor/models/bagel.py

+4 -7

@@ -14,7 +14,7 @@ import torch

14 import torch.nn as nn

15

16 from vllm.config import VllmConfig

17 -from vllm.config.multimodal import BaseDummyOptions, ImageDummyOptions

17 +from vllm.config.multimodal import MultiModalDummyOptions

18 from vllm.inputs import MultiModalDataDict

19 from vllm.logger import init_logger

20 from vllm.model_executor.layers.activation import get_act_fn

@@ -248,23 +248,20 @@ class BagelDummyInputsBuilder(BaseDummyInputsBuilder[BagelProcessingInfo]):

248 self,

249 seq_len: int,

250 mm_counts: Mapping[str, int],

251 - mm_options: Mapping[str, BaseDummyOptions],

251 + mm_options: MultiModalDummyOptions,

252 ) -> MultiModalDataDict:

253 - num_images = mm_counts.get("image", 0)

253 hf_config = self.info.get_hf_config()

254 vit_config = hf_config.vit_config

255

256 # Use the configured image size

257 image_size = vit_config.image_size

259 - image_overrides = mm_options.get("image")

260 - assert image_overrides is None or isinstance(image_overrides, ImageDummyOptions)

258

259 return {

260 "image": self._get_dummy_images(

261 width=image_size,

262 height=image_size,

266 - num_images=num_images,

267 - overrides=image_overrides,

263 + num_images=mm_counts.get("image", 0),

264 + overrides=mm_options.get("image"),

265 ),

266 }

267

vllm/model_executor/models/bailing_moe_v3_vl.py

+2 -2

@@ -13,7 +13,7 @@ import torch.nn as nn

13 from transformers import AutoProcessor, BatchFeature

14

15 from vllm.config import VllmConfig

16 -from vllm.config.multimodal import BaseDummyOptions

16 +from vllm.config.multimodal import MultiModalDummyOptions

17 from vllm.inputs import MultiModalDataDict

18 from vllm.model_executor.layers.linear import (

19 ColumnParallelLinear,

@@ -103,7 +103,7 @@ class BailingMoeV3VLDummyInputsBuilder(

103 self,

104 seq_len: int,

105 mm_counts: Mapping[str, int],

106 - mm_options: Mapping[str, BaseDummyOptions],

106 + mm_options: MultiModalDummyOptions,

107 ) -> MultiModalDataDict:

108 target_width, target_height = self.info.get_image_size_with_most_features()

109 return {

vllm/model_executor/models/bee.py

+4 -9

@@ -8,7 +8,7 @@ import torch.nn as nn

8 from transformers.activations import GELUActivation

9

10 from vllm.config import VllmConfig

11 -from vllm.config.multimodal import BaseDummyOptions, ImageDummyOptions

11 +from vllm.config.multimodal import MultiModalDummyOptions

12 from vllm.inputs import MultiModalDataDict

13 from vllm.multimodal import MULTIMODAL_REGISTRY

14

@@ -90,21 +90,16 @@ class BeeDummyInputsBuilder(LlavaDummyInputsBuilder[BeeProcessingInfo]):

90 self,

91 seq_len: int,

92 mm_counts: Mapping[str, int],

93 - mm_options: Mapping[str, BaseDummyOptions],

93 + mm_options: MultiModalDummyOptions,

94 ) -> MultiModalDataDict:

95 - num_images = mm_counts.get("image", 0)

96 -

95 target_width, target_height = self.info.get_image_size_with_most_features()

96

99 - image_overrides = mm_options.get("image")

100 - assert image_overrides is None or isinstance(image_overrides, ImageDummyOptions)

101 -

97 return {

98 "image": self._get_dummy_images(

99 width=target_width,

100 height=target_height,

106 - num_images=num_images,

107 - overrides=image_overrides,

101 + num_images=mm_counts.get("image", 0),

102 + overrides=mm_options.get("image"),

103 ),

104 }

105

vllm/model_executor/models/blip2.py

+4 -8

@@ -15,7 +15,7 @@ from transformers import (

15 )

16

17 from vllm.config import CacheConfig, VllmConfig

18 -from vllm.config.multimodal import BaseDummyOptions, ImageDummyOptions

18 +from vllm.config.multimodal import MultiModalDummyOptions

19 from vllm.inputs import MultiModalDataDict

20 from vllm.model_executor.layers.activation import get_act_fn

21 from vllm.model_executor.layers.quantization import QuantizationConfig

@@ -444,23 +444,19 @@ class Blip2DummyInputsBuilder(BaseDummyInputsBuilder[Blip2ProcessingInfo]):

444 self,

445 seq_len: int,

446 mm_counts: Mapping[str, int],

447 - mm_options: Mapping[str, BaseDummyOptions],

447 + mm_options: MultiModalDummyOptions,

448 ) -> MultiModalDataDict:

449 hf_config = self.info.get_hf_config()

450 vision_config = hf_config.vision_config

451

452 max_image_size = vision_config.image_size

453 - num_images = mm_counts.get("image", 0)

454 -

455 - image_overrides = mm_options.get("image")

456 - assert image_overrides is None or isinstance(image_overrides, ImageDummyOptions)

453

454 return {

455 "image": self._get_dummy_images(

456 width=max_image_size,

457 height=max_image_size,

462 - num_images=num_images,

463 - overrides=image_overrides,

458 + num_images=mm_counts.get("image", 0),

459 + overrides=mm_options.get("image"),

460 )

461 }

462

vllm/model_executor/models/clip.py

+4 -8

@@ -15,7 +15,7 @@ from transformers import (

15 )

16

17 from vllm.config import VllmConfig

18 -from vllm.config.multimodal import BaseDummyOptions

18 +from vllm.config.multimodal import MultiModalDummyOptions

19 from vllm.distributed import divide, get_tensor_model_parallel_world_size

20 from vllm.inputs import MultiModalDataDict, MultiModalInput

21 from vllm.model_executor.layers.activation import get_act_fn

@@ -205,20 +205,16 @@ class CLIPDummyInputsBuilder(BaseDummyInputsBuilder[CLIPProcessingInfo]):

205 self,

206 seq_len: int,

207 mm_counts: Mapping[str, int],

208 - mm_options: Mapping[str, BaseDummyOptions],

208 + mm_options: MultiModalDummyOptions,

209 ) -> MultiModalDataDict:

210 - num_images = mm_counts.get("image", 0)

211 -

210 target_width, target_height = self.info.get_image_size_with_most_features()

211

214 - image_overrides = mm_options.get("image")

215 -

212 return {

213 "image": self._get_dummy_images(

214 width=target_width,

215 height=target_height,

220 - num_images=num_images,

221 - overrides=image_overrides,

216 + num_images=mm_counts.get("image", 0),

217 + overrides=mm_options.get("image"),

218 )

219 }

220

vllm/model_executor/models/cohere2_vision.py

+4 -7

@@ -18,7 +18,7 @@ from transformers.models.cohere2_vision.processing_cohere2_vision import (

18 )

19

20 from vllm.config import VllmConfig

21 -from vllm.config.multimodal import BaseDummyOptions

21 +from vllm.config.multimodal import MultiModalDummyOptions

22 from vllm.inputs import MultiModalDataDict

23 from vllm.model_executor.layers.activation import MulAndSilu

24 from vllm.model_executor.layers.linear import (

@@ -202,19 +202,16 @@ class Cohere2VisionDummyInputsBuilder(

202 self,

203 seq_len: int,

204 mm_counts: Mapping[str, int],

205 - mm_options: Mapping[str, BaseDummyOptions],

205 + mm_options: MultiModalDummyOptions,

206 ) -> MultiModalDataDict:

207 - num_images = mm_counts.get("image", 0)

207 image_size = self.info.get_image_size_with_most_features()

208

210 - image_overrides = mm_options.get("image")

211 -

209 return {

210 "image": self._get_dummy_images(

211 width=image_size.width,

212 height=image_size.height,

216 - num_images=num_images,

217 - overrides=image_overrides,

213 + num_images=mm_counts.get("image", 0),

214 + overrides=mm_options.get("image"),

215 )

216 }

217

vllm/model_executor/models/cohere_asr.py

+2 -2

@@ -12,7 +12,7 @@ from transformers import PretrainedConfig

12

13 from vllm.compilation.decorators import support_torch_compile

14 from vllm.config import CacheConfig, ModelConfig, SpeechToTextConfig, VllmConfig

15 -from vllm.config.multimodal import BaseDummyOptions

15 +from vllm.config.multimodal import MultiModalDummyOptions

16 from vllm.config.speech_to_text import SpeechToTextParams

17 from vllm.distributed import get_tensor_model_parallel_world_size

18 from vllm.inputs import MultiModalDataDict, PromptType, TokensPrompt

@@ -1944,7 +1944,7 @@ class CohereASRDummyInputsBuilder(BaseDummyInputsBuilder[CohereASRProcessingInfo

1944 self,

1945 seq_len: int,

1946 mm_counts: Mapping[str, int],

1947 - mm_options: Mapping[str, BaseDummyOptions],

1947 + mm_options: MultiModalDummyOptions,

1948 mm_processor_kwargs=None,

1949 ) -> MultiModalDataDict:

1950 feature_extractor = self.info.get_feature_extractor()

vllm/model_executor/models/cohere_compass.py

+4 -7

@@ -51,7 +51,7 @@ from transformers.models.cohere_compass.image_processing_cohere_compass import (

51

52 from vllm.compilation.decorators import support_torch_compile

53 from vllm.config import CacheConfig, VllmConfig

54 -from vllm.config.multimodal import BaseDummyOptions

54 +from vllm.config.multimodal import MultiModalDummyOptions

55 from vllm.distributed import (

56 get_pp_group,

57 get_tensor_model_parallel_world_size,

@@ -1238,11 +1238,8 @@ class CohereCompassDummyInputsBuilder(

1238 self,

1239 seq_len: int,

1240 mm_counts: Mapping[str, int],

1241 - mm_options: Mapping[str, BaseDummyOptions],

1241 + mm_options: MultiModalDummyOptions,

1242 ) -> MultiModalDataDict:

1243 - num_images = mm_counts.get("image", 0)

1244 - image_overrides = mm_options.get("image")

1245 -

1243 target_image_width, target_image_height = (

1244 self.info.get_image_size_with_most_features()

1245 )

@@ -1251,8 +1248,8 @@ class CohereCompassDummyInputsBuilder(

1248 "image": self._get_dummy_images(

1249 width=target_image_width,

1250 height=target_image_height,

1254 - num_images=num_images,

1255 - overrides=image_overrides,

1251 + num_images=mm_counts.get("image", 0),

1252 + overrides=mm_options.get("image"),

1253 ),

1254 }

1255

vllm/model_executor/models/colmodernvbert.py

+4 -6

@@ -15,7 +15,7 @@ from torch import nn

15 from transformers import BatchFeature

16

17 from vllm.config import VllmConfig

18 -from vllm.config.multimodal import BaseDummyOptions

18 +from vllm.config.multimodal import MultiModalDummyOptions

19 from vllm.inputs import MultiModalDataDict

20 from vllm.model_executor.layers.pooler.tokwise import pooler_for_token_embed

21 from vllm.multimodal import MULTIMODAL_REGISTRY

@@ -135,18 +135,16 @@ class ColModernVBertDummyInputsBuilder(

135 self,

136 seq_len: int,

137 mm_counts: Mapping[str, int],

138 - mm_options: Mapping[str, BaseDummyOptions],

138 + mm_options: MultiModalDummyOptions,

139 ) -> MultiModalDataDict:

140 - num_images = mm_counts.get("image", 0)

140 target_width, target_height = self.info.get_image_size_with_most_features()

142 - image_overrides = mm_options.get("image")

141

142 return {

143 "image": self._get_dummy_images(

144 width=target_width,

145 height=target_height,

148 - num_images=num_images,

149 - overrides=image_overrides,

146 + num_images=mm_counts.get("image", 0),

147 + overrides=mm_options.get("image"),

148 )

149 }

150

vllm/model_executor/models/deepseek_ocr.py

+3 -5

@@ -11,7 +11,7 @@ import torch.nn as nn

11 from transformers import BatchFeature, CLIPVisionConfig

12

13 from vllm.config import VllmConfig

14 -from vllm.config.multimodal import BaseDummyOptions

14 +from vllm.config.multimodal import MultiModalDummyOptions

15 from vllm.inputs import MultiModalDataDict

16 from vllm.model_executor.models.interfaces import (

17 MultiModalEmbeddings,

@@ -268,17 +268,15 @@ class DeepseekOCRDummyInputsBuilder(BaseDummyInputsBuilder[DeepseekOCRProcessing

268 self,

269 seq_len: int,

270 mm_counts: Mapping[str, int],

271 - mm_options: Mapping[str, BaseDummyOptions],

271 + mm_options: MultiModalDummyOptions,

272 ) -> MultiModalDataDict:

273 - num_images = mm_counts.get("image", 0)

274 -

273 max_image_size = self.info.get_image_size_with_most_features()

274

275 return {

276 "image": self._get_dummy_images(

277 width=max_image_size.width,

278 height=max_image_size.height,

281 - num_images=num_images,

279 + num_images=mm_counts.get("image", 0),

280 )

281 }

282

vllm/model_executor/models/deepseek_ocr2.py

+3 -5

@@ -11,7 +11,7 @@ import torch.nn as nn

11 from transformers import BatchFeature

12

13 from vllm.config import VllmConfig

14 -from vllm.config.multimodal import BaseDummyOptions

14 +from vllm.config.multimodal import MultiModalDummyOptions

15 from vllm.inputs import MultiModalDataDict

16 from vllm.model_executor.models.interfaces import (

17 MultiModalEmbeddings,

@@ -140,17 +140,15 @@ class DeepseekOCR2DummyInputsBuilder(

140 self,

141 seq_len: int,

142 mm_counts: Mapping[str, int],

143 - mm_options: Mapping[str, BaseDummyOptions],

143 + mm_options: MultiModalDummyOptions,

144 ) -> MultiModalDataDict:

145 - num_images = mm_counts.get("image", 0)

146 -

145 max_image_size = self.info.get_image_size_with_most_features()

146

147 return {

148 "image": self._get_dummy_images(

149 width=max_image_size.width,

150 height=max_image_size.height,

153 - num_images=num_images,

151 + num_images=mm_counts.get("image", 0),

152 )

153 }

154

vllm/model_executor/models/deepseek_vl2.py

+4 -8

@@ -15,7 +15,7 @@ from einops import rearrange, repeat

15 from transformers import BatchFeature

16

17 from vllm.config import VllmConfig

18 -from vllm.config.multimodal import BaseDummyOptions

18 +from vllm.config.multimodal import MultiModalDummyOptions

19 from vllm.distributed import get_tensor_model_parallel_world_size

20 from vllm.inputs import MultiModalDataDict

21 from vllm.model_executor.layers.quantization import QuantizationConfig

@@ -212,20 +212,16 @@ class DeepseekVL2DummyInputsBuilder(BaseDummyInputsBuilder[DeepseekVL2Processing

212 self,

213 seq_len: int,

214 mm_counts: Mapping[str, int],

215 - mm_options: Mapping[str, BaseDummyOptions],

215 + mm_options: MultiModalDummyOptions,

216 ) -> MultiModalDataDict:

217 - num_images = mm_counts.get("image", 0)

218 -

217 max_image_size = self.info.get_image_size_with_most_features()

218

221 - image_overrides = mm_options.get("image")

222 -

219 return {

220 "image": self._get_dummy_images(

221 width=max_image_size.width,

222 height=max_image_size.height,

227 - num_images=num_images,

228 - overrides=image_overrides,

223 + num_images=mm_counts.get("image", 0),

224 + overrides=mm_options.get("image"),

225 )

226 }

227

vllm/model_executor/models/dots_ocr.py

+4 -8

@@ -9,7 +9,7 @@ from torch.nn import LayerNorm

9 from transformers.models.qwen2_vl import Qwen2VLProcessor

10

11 from vllm.config import VllmConfig

12 -from vllm.config.multimodal import BaseDummyOptions

12 +from vllm.config.multimodal import MultiModalDummyOptions

13 from vllm.distributed import utils as dist_utils

14 from vllm.distributed.parallel_state import (

15 get_tensor_model_parallel_rank,

@@ -104,20 +104,16 @@ class DotsOCRDummyInputsBuilder(Qwen2VLDummyInputsBuilder):

104 self,

105 seq_len: int,

106 mm_counts: Mapping[str, int],

107 - mm_options: Mapping[str, BaseDummyOptions],

107 + mm_options: MultiModalDummyOptions,

108 ) -> MultiModalDataDict:

109 - num_images = mm_counts.get("image", 0)

110 -

109 target_width, target_height = self.info.get_image_size_with_most_features()

110

113 - image_overrides = mm_options.get("image")

114 -

111 return {

112 "image": self._get_dummy_images(

113 width=target_width,

114 height=target_height,

119 - num_images=num_images,

120 - overrides=image_overrides,

115 + num_images=mm_counts.get("image", 0),

116 + overrides=mm_options.get("image"),

117 ),

118 }

119

vllm/model_executor/models/ernie45_vl.py

+6 -13

@@ -36,7 +36,7 @@ from einops import rearrange

36 from transformers import BaseImageProcessor, BatchFeature

37

38 from vllm.config import VllmConfig

39 -from vllm.config.multimodal import BaseDummyOptions, VideoDummyOptions

39 +from vllm.config.multimodal import MultiModalDummyOptions, VideoDummyOptions

40 from vllm.distributed import parallel_state

41 from vllm.distributed import utils as dist_utils

42 from vllm.inputs import MultiModalDataDict

@@ -1276,33 +1276,26 @@ class Ernie4_5_VLDummyInputsBuilder(BaseDummyInputsBuilder[Ernie4_5_VLProcessing

1276 self,

1277 seq_len: int,

1278 mm_counts: Mapping[str, int],

1279 - mm_options: Mapping[str, BaseDummyOptions],

1279 + mm_options: MultiModalDummyOptions,

1280 ) -> MultiModalDataDict:

1281 - num_images = mm_counts.get("image", 0)

1282 - num_videos = mm_counts.get("video", 0)

1283 -

1281 target_width, target_height = self.info.get_image_size_with_most_features()

1282 target_num_frames = self.info.get_num_frames_with_most_features(

1283 seq_len, mm_counts

1284 )

1285

1289 - image_overrides = mm_options.get("image")

1290 - video_overrides = mm_options.get("video")

1291 - assert video_overrides is None or isinstance(video_overrides, VideoDummyOptions)

1292 -

1286 return {

1287 "image": self._get_dummy_images(

1288 width=target_width,

1289 height=target_height,

1297 - num_images=num_images,

1298 - overrides=image_overrides,

1290 + num_images=mm_counts.get("image", 0),

1291 + overrides=mm_options.get("image"),

1292 ),

1293 "video": self._get_dummy_videos(

1294 width=target_width,

1295 height=target_height,

1296 num_frames=target_num_frames,

1304 - num_videos=num_videos,

1305 - overrides=video_overrides,

1297 + num_videos=mm_counts.get("video", 0),

1298 + overrides=mm_options.get("video"),

1299 ),

1300 }

1301

vllm/model_executor/models/fireredasr2.py

+5 -7

@@ -12,7 +12,7 @@ from transformers import (

12 )

13

14 from vllm.config import ModelConfig, SpeechToTextConfig, VllmConfig

15 -from vllm.config.multimodal import AudioDummyOptions, BaseDummyOptions

15 +from vllm.config.multimodal import MultiModalDummyOptions

16 from vllm.config.speech_to_text import SpeechToTextParams

17 from vllm.inputs import MultiModalDataDict, PromptType

18 from vllm.logger import init_logger

@@ -207,20 +207,18 @@ class FireRedASR2DummyInputsBuilder(BaseDummyInputsBuilder[FireRedASR2Processing

207 self,

208 seq_len: int,

209 mm_counts: Mapping[str, int],

210 - mm_options: Mapping[str, BaseDummyOptions],

210 + mm_options: MultiModalDummyOptions,

211 ) -> MultiModalDataDict:

212 feature_extractor = self.info.get_feature_extractor()

213

214 sampling_rate = feature_extractor.sampling_rate

215 audio_len = feature_extractor.chunk_length * sampling_rate

216 - num_audios = mm_counts.get("audio", 0)

217 -

218 - audio_overrides = mm_options.get("audio")

219 - assert audio_overrides is None or isinstance(audio_overrides, AudioDummyOptions)

216

217 ret = {

218 "audio": self._get_dummy_audios(

223 - length=audio_len, num_audios=num_audios, overrides=audio_overrides

219 + length=audio_len,

220 + num_audios=mm_counts.get("audio", 0),

221 + overrides=mm_options.get("audio"),

222 )

223 }

224 return ret

vllm/model_executor/models/funasr.py

+4 -8

@@ -14,7 +14,7 @@ from transformers import (

14 )

15

16 from vllm.config import ModelConfig, SpeechToTextConfig, VllmConfig

17 -from vllm.config.multimodal import AudioDummyOptions, BaseDummyOptions

17 +from vllm.config.multimodal import MultiModalDummyOptions

18 from vllm.config.speech_to_text import SpeechToTextParams

19 from vllm.distributed import get_tensor_model_parallel_world_size

20 from vllm.inputs import MultiModalDataDict, PromptType

@@ -734,22 +734,18 @@ class FunASRDummyInputsBuilder(BaseDummyInputsBuilder[FunASRProcessingInfo]):

734 self,

735 seq_len: int,

736 mm_counts: Mapping[str, int],

737 - mm_options: Mapping[str, BaseDummyOptions],

737 + mm_options: MultiModalDummyOptions,

738 ) -> MultiModalDataDict:

739 feature_extractor = self.info.get_feature_extractor()

740

741 sampling_rate = feature_extractor.sampling_rate

742 audio_len = feature_extractor.chunk_length * sampling_rate

743 - num_audios = mm_counts.get("audio", 0)

744 -

745 - audio_overrides = mm_options.get("audio")

746 - assert audio_overrides is None or isinstance(audio_overrides, AudioDummyOptions)

743

744 return {

745 "audio": self._get_dummy_audios(

746 length=audio_len,

751 - num_audios=num_audios,

752 - overrides=audio_overrides,

747 + num_audios=mm_counts.get("audio", 0),

748 + overrides=mm_options.get("audio"),

749 ),

750 }

751

vllm/model_executor/models/funaudiochat.py

+4 -7

@@ -26,7 +26,7 @@ from transformers.feature_extraction_utils import BatchFeature

26 from transformers.modeling_outputs import BaseModelOutput

27

28 from vllm.config import VllmConfig

29 -from vllm.config.multimodal import AudioDummyOptions, BaseDummyOptions

29 +from vllm.config.multimodal import MultiModalDummyOptions

30 from vllm.inputs import MultiModalDataDict

31 from vllm.model_executor.layers.attention.mm_encoder_attention import MMEncoderAttention

32 from vllm.model_executor.layers.linear import QKVParallelLinear, RowParallelLinear

@@ -580,7 +580,7 @@ class FunAudioChatDummyInputsBuilder(

580 self,

581 seq_len: int,

582 mm_counts: Mapping[str, int],

583 - mm_options: Mapping[str, BaseDummyOptions],

583 + mm_options: MultiModalDummyOptions,

584 ) -> MultiModalDataDict:

585 feature_extractor = self.info.get_feature_extractor()

586 sampling_rate = int(feature_extractor.sampling_rate)

@@ -597,15 +597,12 @@ class FunAudioChatDummyInputsBuilder(

597 1,

598 (target_num_frames * sampling_rate + token_fps - 1) // token_fps,

599 )

600 - num_audios = int(mm_counts.get("audio", 0))

600

602 - audio_overrides = mm_options.get("audio")

603 - assert audio_overrides is None or isinstance(audio_overrides, AudioDummyOptions)

601 return {

602 "audio": self._get_dummy_audios(

603 length=audio_len,

607 - num_audios=num_audios,

608 - overrides=audio_overrides,

604 + num_audios=int(mm_counts.get("audio", 0)),

605 + overrides=mm_options.get("audio"),

606 )

607 }

608

vllm/model_executor/models/gemma3_mm.py

+4 -8

@@ -11,7 +11,7 @@ from transformers.models.gemma3.image_processing_gemma3 import Gemma3ImageProces

11 from transformers.models.gemma3.processing_gemma3 import Gemma3ProcessorKwargs

12

13 from vllm.config import VllmConfig

14 -from vllm.config.multimodal import BaseDummyOptions

14 +from vllm.config.multimodal import MultiModalDummyOptions

15 from vllm.inputs import MultiModalDataDict

16 from vllm.logger import init_logger

17 from vllm.model_executor.layers.layernorm import GemmaRMSNorm

@@ -244,20 +244,16 @@ class Gemma3DummyInputsBuilder(BaseDummyInputsBuilder[Gemma3ProcessingInfo]):

244 self,

245 seq_len: int,

246 mm_counts: Mapping[str, int],

247 - mm_options: Mapping[str, BaseDummyOptions],

247 + mm_options: MultiModalDummyOptions,

248 ) -> MultiModalDataDict:

249 - num_images = mm_counts.get("image", 0)

250 -

249 target_width, target_height = self.info.get_image_size_with_most_features()

250

253 - image_overrides = mm_options.get("image")

254 -

251 return {

252 "image": self._get_dummy_images(

253 width=target_width,

254 height=target_height,

259 - num_images=num_images,

260 - overrides=image_overrides,

255 + num_images=mm_counts.get("image", 0),

256 + overrides=mm_options.get("image"),

257 )

258 }

259

vllm/model_executor/models/gemma3n_mm.py

+6 -12

@@ -18,7 +18,7 @@ from transformers.models.gemma3n import (

18 from transformers.models.siglip import SiglipImageProcessorFast

19

20 from vllm.config import ModelConfig, SpeechToTextConfig, VllmConfig

21 -from vllm.config.multimodal import AudioDummyOptions, BaseDummyOptions

21 +from vllm.config.multimodal import MultiModalDummyOptions

22 from vllm.config.speech_to_text import SpeechToTextParams

23 from vllm.inputs import MultiModalDataDict, PromptType, TextPrompt

24 from vllm.logger import init_logger

@@ -232,10 +232,8 @@ class Gemma3nDummyInputsBuilder(BaseDummyInputsBuilder[Gemma3nProcessingInfo]):

232 self,

233 seq_len: int,

234 mm_counts: Mapping[str, int],

235 - mm_options: Mapping[str, BaseDummyOptions],

235 + mm_options: MultiModalDummyOptions,

236 ) -> MultiModalDataDict:

237 - num_images = mm_counts.get("image", 0)

238 - num_audios = mm_counts.get("audio", 0)

237 processor = self.info.get_hf_processor()

238 audio_feature_extractor: Gemma3nAudioFeatureExtractor = (

239 processor.feature_extractor

@@ -245,21 +243,17 @@ class Gemma3nDummyInputsBuilder(BaseDummyInputsBuilder[Gemma3nProcessingInfo]):

243 img_width = image_processor.size.get("width", 224)

244 img_height = image_processor.size.get("height", 224)

245

248 - image_overrides = mm_options.get("image")

249 - audio_overrides = mm_options.get("audio")

250 - assert audio_overrides is None or isinstance(audio_overrides, AudioDummyOptions)

251 -

246 return {

247 "image": self._get_dummy_images(

248 width=img_width,

249 height=img_height,

256 - num_images=num_images,

257 - overrides=image_overrides,

250 + num_images=mm_counts.get("image", 0),

251 + overrides=mm_options.get("image"),

252 ),

253 "audio": self._get_dummy_audios(

254 length=audio_len,

261 - num_audios=num_audios,

262 - overrides=audio_overrides,

255 + num_audios=mm_counts.get("audio", 0),

256 + overrides=mm_options.get("audio"),

257 ),

258 }

259

vllm/model_executor/models/gemma4_mm.py

+7 -21

@@ -36,11 +36,7 @@ from transformers.models.gemma4.configuration_gemma4 import (

36

37 from vllm.config import VllmConfig

38 from vllm.config.model import get_served_model_name

39 -from vllm.config.multimodal import (

40 - AudioDummyOptions,

41 - BaseDummyOptions,

42 - VideoDummyOptions,

43 -)

39 +from vllm.config.multimodal import MultiModalDummyOptions, VideoDummyOptions

40 from vllm.inputs import MultiModalDataDict

41 from vllm.logger import init_logger

42 from vllm.model_executor.layers.layernorm import RMSNorm

@@ -273,10 +269,7 @@ class Gemma4ProcessingInfo(BaseProcessingInfo):

269 num_frames = _VIDEO_MAX_FRAMES

270 mm_config = self.ctx.model_config.get_multimodal_config()

271 video_opts = mm_config.limit_per_prompt.get("video")

276 - if (

277 - isinstance(video_opts, VideoDummyOptions)

278 - and video_opts.num_frames is not None

279 - ):

272 + if video_opts is not None and video_opts.num_frames is not None:

273 num_frames = min(num_frames, video_opts.num_frames)

274 tokens["video"] = num_frames * (_VIDEO_MAX_SOFT_TOKENS + 2 + 6)

275 return tokens

@@ -502,9 +495,8 @@ class Gemma4DummyInputsBuilder(BaseDummyInputsBuilder[Gemma4ProcessingInfo]):

495 self,

496 seq_len: int,

497 mm_counts: Mapping[str, int],

505 - mm_options: Mapping[str, BaseDummyOptions] | None = None,

498 + mm_options: MultiModalDummyOptions | None = None,

499 ) -> MultiModalDataDict:

507 - num_images = mm_counts.get("image", 0)

500 num_audios = mm_counts.get("audio", 0)

501 num_videos = mm_counts.get("video", 0)

502 processor = self.info.get_hf_processor()

@@ -517,18 +509,12 @@ class Gemma4DummyInputsBuilder(BaseDummyInputsBuilder[Gemma4ProcessingInfo]):

509 img_width = size.get("width", 224)

510 img_height = size.get("height", 224)

511

520 - image_overrides = mm_options.get("image") if mm_options else None

521 - audio_overrides = mm_options.get("audio") if mm_options else None

522 - video_overrides = mm_options.get("video") if mm_options else None

523 - assert audio_overrides is None or isinstance(audio_overrides, AudioDummyOptions)

524 - assert video_overrides is None or isinstance(video_overrides, VideoDummyOptions)

525 -

512 data: dict[str, Any] = {

513 "image": self._get_dummy_images(

514 width=img_width,

515 height=img_height,

530 - num_images=num_images,

531 - overrides=image_overrides,

516 + num_images=mm_counts.get("image", 0),

517 + overrides=mm_options.get("image") if mm_options else None,

518 ),

519 }

520

@@ -537,7 +523,7 @@ class Gemma4DummyInputsBuilder(BaseDummyInputsBuilder[Gemma4ProcessingInfo]):

523 data["audio"] = self._get_dummy_audios(

524 length=audio_len,

525 num_audios=num_audios,

540 - overrides=audio_overrides,

526 + overrides=mm_options.get("audio") if mm_options else None,

527 )

528

529 if num_videos > 0:

@@ -546,7 +532,7 @@ class Gemma4DummyInputsBuilder(BaseDummyInputsBuilder[Gemma4ProcessingInfo]):

532 height=img_height,

533 num_frames=_VIDEO_MAX_FRAMES,

534 num_videos=num_videos,

549 - overrides=video_overrides,

535 + overrides=mm_options.get("video") if mm_options else None,

536 )

537

538 return data

vllm/model_executor/models/gemma4_unified.py

+1 -5

@@ -28,7 +28,6 @@ from transformers.models.gemma4_unified.processing_gemma4_unified import (

28 )

29

30 from vllm.config import VllmConfig

31 -from vllm.config.multimodal import VideoDummyOptions

31 from vllm.model_executor.layers.linear import ColumnParallelLinear

32 from vllm.model_executor.models.gemma3n_mm import batch_audio_features

33 from vllm.model_executor.models.gemma4 import Gemma4ForCausalLM

@@ -177,10 +176,7 @@ class Gemma4UnifiedProcessingInfo(Gemma4ProcessingInfo):

176 num_frames = _VIDEO_MAX_FRAMES

177 mm_config = self.ctx.model_config.get_multimodal_config()

178 video_opts = mm_config.limit_per_prompt.get("video")

180 - if (

181 - isinstance(video_opts, VideoDummyOptions)

182 - and video_opts.num_frames is not None

183 - ):

179 + if video_opts is not None and video_opts.num_frames is not None:

180 num_frames = min(num_frames, video_opts.num_frames)

181 tokens["video"] = num_frames * (_VIDEO_MAX_SOFT_TOKENS + 2 + 6)

182 return tokens

vllm/model_executor/models/glm4_1v.py

+6 -13

@@ -52,7 +52,7 @@ from transformers.video_processing_utils import BaseVideoProcessor

52 from transformers.video_utils import VideoMetadata

53

54 from vllm.config import VllmConfig

55 -from vllm.config.multimodal import BaseDummyOptions, VideoDummyOptions

55 +from vllm.config.multimodal import MultiModalDummyOptions, VideoDummyOptions

56 from vllm.distributed import get_tensor_model_parallel_world_size, parallel_state

57 from vllm.distributed import utils as dist_utils

58 from vllm.inputs import MultiModalDataDict

@@ -1455,33 +1455,26 @@ class Glm4vDummyInputsBuilder(BaseDummyInputsBuilder[Glm4vProcessingInfo]):

1455 self,

1456 seq_len: int,

1457 mm_counts: Mapping[str, int],

1458 - mm_options: Mapping[str, BaseDummyOptions],

1458 + mm_options: MultiModalDummyOptions,

1459 ) -> MultiModalDataDict:

1460 - num_images = mm_counts.get("image", 0)

1461 - num_videos = mm_counts.get("video", 0)

1462 -

1460 target_width, target_height = self.info.get_image_size_with_most_features()

1461 target_num_frames = self.info.get_num_frames_with_most_features(

1462 seq_len, mm_counts

1463 )

1464

1468 - image_overrides = mm_options.get("image")

1469 - video_overrides = mm_options.get("video")

1470 - assert video_overrides is None or isinstance(video_overrides, VideoDummyOptions)

1471 -

1465 return {

1466 "image": self._get_dummy_images(

1467 width=target_width,

1468 height=target_height,

1476 - num_images=num_images,

1477 - overrides=image_overrides,

1469 + num_images=mm_counts.get("image", 0),

1470 + overrides=mm_options.get("image"),

1471 ),

1472 "video": self._get_dummy_videos(

1473 width=target_width,

1474 height=target_height,

1475 num_frames=target_num_frames,

1483 - num_videos=num_videos,

1484 - overrides=video_overrides,

1476 + num_videos=mm_counts.get("video", 0),

1477 + overrides=mm_options.get("video"),

1478 ),

1479 }

1480

vllm/model_executor/models/glm4v.py

+4 -7

@@ -16,7 +16,7 @@ from torch.nn import LayerNorm

16 from transformers import BatchFeature

17

18 from vllm.config import VllmConfig

19 -from vllm.config.multimodal import BaseDummyOptions

19 +from vllm.config.multimodal import MultiModalDummyOptions

20 from vllm.distributed import get_tensor_model_parallel_world_size

21 from vllm.inputs import MultiModalDataDict

22 from vllm.model_executor.layers.activation import SiluAndMul, get_act_fn

@@ -445,22 +445,19 @@ class GLM4VDummyInputsBuilder(BaseDummyInputsBuilder[GLM4VProcessingInfo]):

445 self,

446 seq_len: int,

447 mm_counts: Mapping[str, int],

448 - mm_options: Mapping[str, BaseDummyOptions],

448 + mm_options: MultiModalDummyOptions,

449 ) -> MultiModalDataDict:

450 hf_config = self.info.get_hf_config()

451 vision_config = hf_config.vision_config

452

453 target_width = target_height = vision_config["image_size"]

454 - num_images = mm_counts.get("image", 0)

455 -

456 - image_overrides = mm_options.get("image")

454

455 return {

456 "image": self._get_dummy_images(

457 width=target_width,

458 height=target_height,

462 - num_images=num_images,

463 - overrides=image_overrides,

459 + num_images=mm_counts.get("image", 0),

460 + overrides=mm_options.get("image"),

461 )

462 }

463

vllm/model_executor/models/glmasr.py

+4 -7

@@ -11,7 +11,7 @@ from transformers.models.glmasr import GlmAsrConfig, GlmAsrProcessor

11 from transformers.models.whisper import WhisperFeatureExtractor

12

13 from vllm.config import ModelConfig, SpeechToTextConfig, VllmConfig

14 -from vllm.config.multimodal import AudioDummyOptions, BaseDummyOptions

14 +from vllm.config.multimodal import MultiModalDummyOptions

15 from vllm.config.speech_to_text import SpeechToTextParams

16 from vllm.distributed.parallel_state import get_tensor_model_parallel_world_size

17 from vllm.inputs import ModalityData, MultiModalDataDict, PromptType, TokensPrompt

@@ -694,13 +694,10 @@ class GlmAsrDummyInputsBuilder(BaseDummyInputsBuilder[GlmAsrProcessingInfo]):

694 self,

695 seq_len: int,

696 mm_counts: Mapping[str, int],

697 - mm_options: Mapping[str, BaseDummyOptions],

697 + mm_options: MultiModalDummyOptions,

698 ) -> MultiModalDataDict:

699 feature_extractor = self.info.get_feature_extractor()

700 sampling_rate = feature_extractor.sampling_rate

701 - num_audios = mm_counts.get("audio", 0)

702 - audio_overrides = mm_options.get("audio")

703 - assert audio_overrides is None or isinstance(audio_overrides, AudioDummyOptions)

701

702 max_audio_len = getattr(

703 self.info.get_hf_processor(), "max_audio_len", DEFAULT_MAX_AUDIO_LEN_S

@@ -710,8 +707,8 @@ class GlmAsrDummyInputsBuilder(BaseDummyInputsBuilder[GlmAsrProcessingInfo]):

707 return {

708 "audio": self._get_dummy_audios(

709 length=audio_len,

713 - num_audios=num_audios,

714 - overrides=audio_overrides,

710 + num_audios=mm_counts.get("audio", 0),

711 + overrides=mm_options.get("audio"),

712 )

713 }

714

vllm/model_executor/models/granite_speech.py

+4 -8

@@ -34,7 +34,7 @@ from torch import nn

34 from transformers import BatchFeature, PretrainedConfig

35

36 from vllm.config import CacheConfig, ModelConfig, SpeechToTextConfig, VllmConfig

37 -from vllm.config.multimodal import AudioDummyOptions, BaseDummyOptions

37 +from vllm.config.multimodal import MultiModalDummyOptions

38 from vllm.config.speech_to_text import SpeechToTextParams

39 from vllm.inputs import MultiModalDataDict, PromptType, TokensPrompt

40 from vllm.model_executor.layers.linear import ColumnParallelLinear, RowParallelLinear

@@ -208,17 +208,13 @@ class GraniteSpeechDummyInputsBuilder(

208 self,

209 seq_len: int,

210 mm_counts: Mapping[str, int],

211 - mm_options: Mapping[str, BaseDummyOptions],

211 + mm_options: MultiModalDummyOptions,

212 ) -> MultiModalDataDict:

213 - num_audios = mm_counts.get("audio", 0)

214 - audio_overrides = mm_options.get("audio")

215 - assert audio_overrides is None or isinstance(audio_overrides, AudioDummyOptions)

216 -

213 return {

214 "audio": self._get_dummy_audios(

215 length=self.info.get_max_audio_len(),

220 - num_audios=num_audios,

221 - overrides=audio_overrides,

216 + num_audios=mm_counts.get("audio", 0),

217 + overrides=mm_options.get("audio"),

218 )

219 }

220

vllm/model_executor/models/hyperclovax_vision_v2.py

+7 -13

@@ -18,7 +18,7 @@ import torch.nn as nn

18 from transformers import BatchFeature

19

20 from vllm.config import VllmConfig

21 -from vllm.config.multimodal import BaseDummyOptions

21 +from vllm.config.multimodal import MultiModalDummyOptions

22 from vllm.forward_context import set_forward_context

23 from vllm.inputs import MultiModalDataDict

24 from vllm.multimodal import MULTIMODAL_REGISTRY

@@ -193,7 +193,7 @@ class HCXVisionV2DummyInputsBuilder(BaseDummyInputsBuilder[HCXVisionV2Processing

193 self,

194 seq_len: int,

195 mm_counts: Mapping[str, int],

196 - mm_options: Mapping[str, BaseDummyOptions] | None = None,

196 + mm_options: MultiModalDummyOptions | None = None,

197 mm_processor_kwargs: Mapping[str, object] | None = None,

198 ) -> ProcessorInputs:

199 """Build dummy processor inputs for memory profiling."""

@@ -225,31 +225,25 @@ class HCXVisionV2DummyInputsBuilder(BaseDummyInputsBuilder[HCXVisionV2Processing

225 self,

226 seq_len: int,

227 mm_counts: Mapping[str, int],

228 - mm_options: Mapping[str, BaseDummyOptions] | None = None,

228 + mm_options: MultiModalDummyOptions | None = None,

229 mm_processor_kwargs: Mapping[str, object] | None = None,

230 ) -> MultiModalDataDict:

231 - num_images = mm_counts.get("image", 0)

232 - num_videos = mm_counts.get("video", 0)

233 -

231 target_width, target_height = self.info.get_image_size_with_most_features()

232 target_num_frames = 16 # Default for video

233

237 - image_overrides = mm_options.get("image") if mm_options else None

238 - video_overrides = mm_options.get("video") if mm_options else None

239 -

234 result: MultiModalDataDict = {

235 "image": self._get_dummy_images(

236 width=target_width,

237 height=target_height,

244 - num_images=num_images,

245 - overrides=image_overrides, # type: ignore

238 + num_images=mm_counts.get("image", 0),

239 + overrides=mm_options.get("image") if mm_options else None,

240 ),

241 "video": self._get_dummy_videos(

242 width=target_width,

243 height=target_height,

244 num_frames=target_num_frames,

251 - num_videos=num_videos,

252 - overrides=video_overrides, # type: ignore

245 + num_videos=mm_counts.get("video", 0),

246 + overrides=mm_options.get("video") if mm_options else None,

247 ),

248 }

249

vllm/model_executor/models/idefics3.py

+4 -7

@@ -29,7 +29,7 @@ from transformers import (

29 )

30

31 from vllm.config import VllmConfig

32 -from vllm.config.multimodal import BaseDummyOptions

32 +from vllm.config.multimodal import MultiModalDummyOptions

33 from vllm.inputs import MultiModalDataDict

34 from vllm.model_executor.layers.linear import ReplicatedLinear

35 from vllm.model_executor.layers.logits_processor import LogitsProcessor

@@ -221,21 +221,18 @@ class Idefics3DummyInputsBuilder(BaseDummyInputsBuilder[Idefics3ProcessingInfo])

221 self,

222 seq_len: int,

223 mm_counts: Mapping[str, int],

224 - mm_options: Mapping[str, BaseDummyOptions],

224 + mm_options: MultiModalDummyOptions,

225 ) -> MultiModalDataDict:

226 - num_images = mm_counts.get("image", 0)

226 hf_processor = self.info.get_hf_processor()

227 image_processor: Idefics3ImageProcessor = hf_processor.image_processor

228 longest_edge = image_processor.max_image_size["longest_edge"]

229

231 - image_overrides = mm_options.get("image")

232 -

230 return {

231 "image": self._get_dummy_images(

232 width=longest_edge,

233 height=longest_edge,

237 - num_images=num_images,

238 - overrides=image_overrides,

234 + num_images=mm_counts.get("image", 0),

235 + overrides=mm_options.get("image"),

236 )

237 }

238

vllm/model_executor/models/interns1.py

+6 -12

@@ -22,7 +22,7 @@ from transformers.models.internvl.video_processing_internvl import (

22 )

23

24 from vllm.config import VllmConfig

25 -from vllm.config.multimodal import BaseDummyOptions, VideoDummyOptions

25 +from vllm.config.multimodal import MultiModalDummyOptions

26 from vllm.inputs import MultiModalDataDict

27 from vllm.model_executor.layers.quantization import QuantizationConfig

28 from vllm.model_executor.models.interns1_vit import InternS1VisionModel

@@ -299,35 +299,29 @@ class InternS1DummyInputsBuilder(BaseDummyInputsBuilder[InternS1ProcessingInfo])

299 self,

300 seq_len: int,

301 mm_counts: Mapping[str, int],

302 - mm_options: Mapping[str, BaseDummyOptions],

302 + mm_options: MultiModalDummyOptions,

303 ) -> MultiModalDataDict:

304 target_width, target_height = self.info.get_image_size_with_most_features()

305 target_num_frames = self.info.get_num_frames_with_most_features(

306 seq_len, mm_counts

307 )

308 - num_images = mm_counts.get("image", 0)

309 - num_videos = mm_counts.get("video", 0)

308

309 config = self.info.get_hf_config()

310 image_size_h, image_size_w = config.vision_config.image_size

311

314 - image_overrides = mm_options.get("image")

315 - video_overrides = mm_options.get("video")

316 - assert video_overrides is None or isinstance(video_overrides, VideoDummyOptions)

317 -

312 return {

313 "image": self._get_dummy_images(

314 width=target_width,

315 height=target_height,

322 - num_images=num_images,

323 - overrides=image_overrides,

316 + num_images=mm_counts.get("image", 0),

317 + overrides=mm_options.get("image"),

318 ),

319 "video": self._get_dummy_videos(

320 width=image_size_w,

321 height=image_size_h,

322 num_frames=target_num_frames,

329 - num_videos=num_videos,

330 - overrides=video_overrides,

323 + num_videos=mm_counts.get("video", 0),

324 + overrides=mm_options.get("video"),

325 ),

326 }

327

vllm/model_executor/models/internvl.py

+5 -11

@@ -17,7 +17,7 @@ import torch.nn as nn

17 from transformers import BatchFeature, PretrainedConfig

18

19 from vllm.config import VllmConfig

20 -from vllm.config.multimodal import BaseDummyOptions, VideoDummyOptions

20 +from vllm.config.multimodal import MultiModalDummyOptions

21 from vllm.inputs import MultiModalDataDict

22 from vllm.model_executor.layers.quantization import QuantizationConfig

23 from vllm.model_executor.layers.quantization.auto_awq import AutoAWQConfig

@@ -203,19 +203,16 @@ class BaseInternVLDummyInputsBuilder(BaseDummyInputsBuilder[_I]):

203 self,

204 seq_len: int,

205 mm_counts: Mapping[str, int],

206 - mm_options: Mapping[str, BaseDummyOptions],

206 + mm_options: MultiModalDummyOptions,

207 ) -> MultiModalDataDict:

208 target_width, target_height = self.info.get_image_size_with_most_features()

209 - num_images = mm_counts.get("image", 0)

210 -

211 - image_overrides = mm_options.get("image")

209

210 return {

211 "image": self._get_dummy_images(

212 width=target_width,

213 height=target_height,

217 - num_images=num_images,

218 - overrides=image_overrides,

214 + num_images=mm_counts.get("image", 0),

215 + overrides=mm_options.get("image"),

216 )

217 }

218

@@ -425,7 +422,7 @@ class InternVLDummyInputsBuilder(

422 self,

423 seq_len: int,

424 mm_counts: Mapping[str, int],

428 - mm_options: Mapping[str, BaseDummyOptions],

425 + mm_options: MultiModalDummyOptions,

426 ) -> MultiModalDataDict:

427 dummy_image = super().get_dummy_mm_data(seq_len, mm_counts, mm_options)

428 if self.info.ctx_video_token:

@@ -436,9 +433,6 @@ class InternVLDummyInputsBuilder(

433 )

434 num_videos = mm_counts.get("video", 0)

435 video_overrides = mm_options.get("video")

439 - assert video_overrides is None or isinstance(

440 - video_overrides, VideoDummyOptions

441 - )

436 dummy_video = {

437 "video": self._get_dummy_videos(

438 width=image_size,

vllm/model_executor/models/isaac.py

+4 -7

@@ -13,7 +13,7 @@ from einops import rearrange

13 from transformers.image_processing_utils import BatchFeature

14

15 from vllm.config import ModelConfig, VllmConfig

16 -from vllm.config.multimodal import BaseDummyOptions

16 +from vllm.config.multimodal import MultiModalDummyOptions

17 from vllm.distributed import parallel_state

18 from vllm.distributed import utils as dist_utils

19 from vllm.inputs import MultiModalDataDict

@@ -386,19 +386,16 @@ class IsaacDummyInputsBuilder(BaseDummyInputsBuilder[IsaacProcessingInfo]):

386 self,

387 seq_len: int,

388 mm_counts: Mapping[str, int],

389 - mm_options: Mapping[str, BaseDummyOptions],

389 + mm_options: MultiModalDummyOptions,

390 ) -> MultiModalDataDict:

391 - num_images = mm_counts.get("image", 0)

392 -

391 target_width, target_height = self.info.get_image_size_with_most_features()

394 - image_overrides = mm_options.get("image")

392

393 return {

394 "image": self._get_dummy_images(

395 width=target_width,

396 height=target_height,

400 - num_images=num_images,

401 - overrides=image_overrides,

397 + num_images=mm_counts.get("image", 0),

398 + overrides=mm_options.get("image"),

399 ),

400 }

401

vllm/model_executor/models/kanana_v.py

+2 -2

@@ -18,7 +18,7 @@ from transformers.modeling_outputs import BaseModelOutput

18 from transformers.models.qwen2_vl.configuration_qwen2_vl import Qwen2VLVisionConfig

19

20 from vllm.config import VllmConfig

21 -from vllm.config.multimodal import BaseDummyOptions

21 +from vllm.config.multimodal import MultiModalDummyOptions

22 from vllm.inputs import MultiModalDataDict

23 from vllm.logger import init_logger

24 from vllm.multimodal import MULTIMODAL_REGISTRY

@@ -442,7 +442,7 @@ class KananaVDummyInputsBuilder(BaseDummyInputsBuilder[KananaVProcessingInfo]):

442 self,

443 seq_len: int,

444 mm_counts: Mapping[str, int],

445 - mm_options: Mapping[str, BaseDummyOptions],

445 + mm_options: MultiModalDummyOptions,

446 ) -> MultiModalDataDict:

447 num_images = mm_counts.get("image", 0)

448 return {

vllm/model_executor/models/keye.py

+6 -12

@@ -17,7 +17,7 @@ from transformers.modeling_outputs import BaseModelOutput, BaseModelOutputWithPo

17 from transformers.utils import torch_int

18

19 from vllm.config import VllmConfig

20 -from vllm.config.multimodal import BaseDummyOptions

20 +from vllm.config.multimodal import MultiModalDummyOptions

21 from vllm.distributed import get_tensor_model_parallel_world_size

22 from vllm.inputs import ModalityData, MultiModalDataDict

23 from vllm.logger import init_logger

@@ -1124,30 +1124,24 @@ class KeyeBaseDummyInputsBuilder(BaseDummyInputsBuilder[_I]):

1124 self,

1125 seq_len: int,

1126 mm_counts: Mapping[str, int],

1127 - mm_options: Mapping[str, BaseDummyOptions],

1127 + mm_options: MultiModalDummyOptions,

1128 ) -> MultiModalDataDict:

1129 - num_images = mm_counts.get("image", 0)

1130 - num_videos = mm_counts.get("video", 0)

1131 -

1129 target_width, target_height = self.info.get_image_size_with_most_features()

1130 target_num_frames = self.info.get_num_frames_with_most_features(seq_len)

1131

1135 - image_overrides = mm_options.get("image")

1136 - video_overrides = mm_options.get("video")

1137 -

1132 mm_data = {

1133 "image": self._get_dummy_images(

1134 width=target_width,

1135 height=target_height,

1142 - num_images=num_images,

1143 - overrides=image_overrides,

1136 + num_images=mm_counts.get("image", 0),

1137 + overrides=mm_options.get("image"),

1138 ),

1139 "video": self._get_dummy_videos(

1140 width=target_width,

1141 height=target_height,

1142 num_frames=target_num_frames,

1149 - num_videos=num_videos,

1150 - overrides=video_overrides,

1143 + num_videos=mm_counts.get("video", 0),

1144 + overrides=mm_options.get("video"),

1145 ),

1146 }

1147

vllm/model_executor/models/kimi_audio.py

+2 -2

@@ -13,7 +13,7 @@ from transformers import BatchFeature

13 from transformers import WhisperConfig as HFWhisperConfig

14

15 from vllm.config import ModelConfig, SpeechToTextConfig, VllmConfig

16 -from vllm.config.multimodal import BaseDummyOptions

16 +from vllm.config.multimodal import MultiModalDummyOptions

17 from vllm.config.speech_to_text import SpeechToTextParams

18 from vllm.inputs import PromptType, TokensPrompt

19 from vllm.model_executor.model_loader import DefaultModelLoader

@@ -180,7 +180,7 @@ class KimiAudioDummyInputsBuilder(BaseDummyInputsBuilder[KimiAudioProcessingInfo

180 self,

181 seq_len: int,

182 mm_counts: Mapping[str, int],

183 - mm_options: Mapping[str, BaseDummyOptions],

183 + mm_options: MultiModalDummyOptions,

184 ) -> ProcessorInputs:

185 dummy_mm_data = self.get_dummy_mm_data(seq_len, mm_counts, mm_options)

186 dummy_mm_items = self.info.parse_mm_data(dummy_mm_data)

vllm/model_executor/models/kimi_k25.py

+2 -2

@@ -14,7 +14,7 @@ from torch import nn

14 from transformers import BatchFeature

15

16 from vllm.config import VllmConfig

17 -from vllm.config.multimodal import BaseDummyOptions

17 +from vllm.config.multimodal import MultiModalDummyOptions

18 from vllm.inputs import MultiModalDataDict

19 from vllm.logger import init_logger

20 from vllm.model_executor.layers.quantization import QuantizationConfig

@@ -216,7 +216,7 @@ class KimiK25DummyInputsBuilder(BaseDummyInputsBuilder[KimiK25ProcessingInfo]):

216 self,

217 seq_len: int,

218 mm_counts: Mapping[str, int],

219 - mm_options: Mapping[str, BaseDummyOptions],

219 + mm_options: MultiModalDummyOptions,

220 ) -> MultiModalDataDict:

221 # TODO: Support mm_options for vision_chunk to allow user configuration

222 dummy_items = self.get_dummy_mm_items()

vllm/model_executor/models/kimi_vl.py

+4 -8

@@ -53,7 +53,7 @@ from transformers import BatchFeature

53 from transformers.activations import GELUActivation

54

55 from vllm.config import VllmConfig

56 -from vllm.config.multimodal import BaseDummyOptions

56 +from vllm.config.multimodal import MultiModalDummyOptions

57 from vllm.inputs import MultiModalDataDict

58 from vllm.model_executor.layers.linear import ReplicatedLinear

59 from vllm.model_executor.models.interfaces import (

@@ -219,18 +219,14 @@ class KimiVLDummyInputsBuilder(BaseDummyInputsBuilder[KimiVLProcessingInfo]):

219 self,

220 seq_len: int,

221 mm_counts: Mapping[str, int],

222 - mm_options: Mapping[str, BaseDummyOptions],

222 + mm_options: MultiModalDummyOptions,

223 ) -> MultiModalDataDict:

224 - num_images = mm_counts.get("image", 0)

225 -

226 - image_overrides = mm_options.get("image")

227 -

224 return {

225 "image": self._get_dummy_images(

226 width=MaxImageTokenMeta.width,

227 height=MaxImageTokenMeta.height,

232 - num_images=num_images,

233 - overrides=image_overrides,

228 + num_images=mm_counts.get("image", 0),

229 + overrides=mm_options.get("image"),

230 )

231 }

232

vllm/model_executor/models/lfm2_vl.py

+4 -8

@@ -21,7 +21,7 @@ from transformers.models.lfm2_vl.image_processing_lfm2_vl_fast import (

21 from typing_extensions import Buffer

22

23 from vllm.config import VllmConfig

24 -from vllm.config.multimodal import BaseDummyOptions

24 +from vllm.config.multimodal import MultiModalDummyOptions

25 from vllm.forward_context import set_forward_context

26 from vllm.inputs import MultiModalDataDict

27 from vllm.model_executor.layers.linear import ReplicatedLinear

@@ -387,20 +387,16 @@ class Lfm2VLDummyInputsBuilder(BaseDummyInputsBuilder[Lfm2VLProcessingInfo]):

387 self,

388 seq_len: int,

389 mm_counts: Mapping[str, int],

390 - mm_options: Mapping[str, BaseDummyOptions],

390 + mm_options: MultiModalDummyOptions,

391 ) -> MultiModalDataDict:

392 - num_images = mm_counts.get("image", 0)

393 -

392 target_width, target_height = self.info.get_image_size_with_most_features()

393

396 - image_overrides = mm_options.get("image")

397 -

394 return {

395 "image": self._get_dummy_images(

396 width=target_width,

397 height=target_height,

402 - num_images=num_images,

403 - overrides=image_overrides,

398 + num_images=mm_counts.get("image", 0),

399 + overrides=mm_options.get("image"),

400 ),

401 }

402

vllm/model_executor/models/llava.py

+4 -8

@@ -19,7 +19,7 @@ from transformers.models.llava import LlavaProcessor

19 from transformers.models.pixtral import PixtralProcessor

20

21 from vllm.config import VllmConfig

22 -from vllm.config.multimodal import BaseDummyOptions

22 +from vllm.config.multimodal import MultiModalDummyOptions

23 from vllm.inputs import MultiModalDataDict

24 from vllm.model_executor.layers.activation import get_act_fn

25 from vllm.model_executor.layers.linear import ColumnParallelLinear, RowParallelLinear

@@ -228,20 +228,16 @@ class LlavaDummyInputsBuilder(BaseDummyInputsBuilder[_I]):

228 self,

229 seq_len: int,

230 mm_counts: Mapping[str, int],

231 - mm_options: Mapping[str, BaseDummyOptions],

231 + mm_options: MultiModalDummyOptions,

232 ) -> MultiModalDataDict:

233 - num_images = mm_counts.get("image", 0)

234 -

233 target_width, target_height = self.info.get_image_size_with_most_features()

234

237 - image_overrides = mm_options.get("image")

238 -

235 return {

236 "image": self._get_dummy_images(

237 width=target_width,

238 height=target_height,

243 - num_images=num_images,

244 - overrides=image_overrides,

239 + num_images=mm_counts.get("image", 0),

240 + overrides=mm_options.get("image"),

241 )

242 }

243

vllm/model_executor/models/llava_next_video.py

+4 -9

@@ -10,7 +10,7 @@ import torch.nn as nn

10 from transformers import BatchFeature, LlavaNextVideoConfig, LlavaNextVideoProcessor

11

12 from vllm.config import VllmConfig

13 -from vllm.config.multimodal import BaseDummyOptions, VideoDummyOptions

13 +from vllm.config.multimodal import MultiModalDummyOptions

14 from vllm.inputs import MultiModalDataDict

15 from vllm.model_executor.layers.activation import get_act_fn

16 from vllm.model_executor.models.clip import CLIPVisionModel

@@ -171,25 +171,20 @@ class LlavaNextVideoDummyInputsBuilder(

171 self,

172 seq_len: int,

173 mm_counts: Mapping[str, int],

174 - mm_options: Mapping[str, BaseDummyOptions],

174 + mm_options: MultiModalDummyOptions,

175 ) -> MultiModalDataDict:

176 - num_videos = mm_counts.get("video", 0)

177 -

176 target_width, target_height = self.info.get_image_size_with_most_features()

177 target_num_frames = self.info.get_num_frames_with_most_features(

178 seq_len, mm_counts

179 )

180

183 - video_overrides = mm_options.get("video")

184 - assert video_overrides is None or isinstance(video_overrides, VideoDummyOptions)

185 -

181 return {

182 "video": self._get_dummy_videos(

183 width=target_width,

184 height=target_height,

185 num_frames=target_num_frames,

191 - num_videos=num_videos,

192 - overrides=video_overrides,

186 + num_videos=mm_counts.get("video", 0),

187 + overrides=mm_options.get("video"),

188 )

189 }

190

vllm/model_executor/models/llava_onevision.py

+6 -13

@@ -14,7 +14,7 @@ from transformers.models.llava_onevision.modeling_llava_onevision import (

14 )

15

16 from vllm.config import VllmConfig

17 -from vllm.config.multimodal import BaseDummyOptions, VideoDummyOptions

17 +from vllm.config.multimodal import MultiModalDummyOptions

18 from vllm.inputs import MultiModalDataDict

19 from vllm.model_executor.layers.activation import get_act_fn

20 from vllm.multimodal import MULTIMODAL_REGISTRY

@@ -278,33 +278,26 @@ class LlavaOnevisionDummyInputsBuilder(

278 self,

279 seq_len: int,

280 mm_counts: Mapping[str, int],

281 - mm_options: Mapping[str, BaseDummyOptions],

281 + mm_options: MultiModalDummyOptions,

282 ) -> MultiModalDataDict:

283 - num_images = mm_counts.get("image", 0)

284 - num_videos = mm_counts.get("video", 0)

285 -

283 target_width, target_height = self.info.get_image_size_with_most_features()

284 target_num_frames = self.info.get_num_frames_with_most_features(

285 seq_len, mm_counts

286 )

287

291 - image_overrides = mm_options.get("image")

292 - video_overrides = mm_options.get("video")

293 - assert video_overrides is None or isinstance(video_overrides, VideoDummyOptions)

294 -

288 return {

289 "image": self._get_dummy_images(

290 width=target_width,

291 height=target_height,

299 - num_images=num_images,

300 - overrides=image_overrides,

292 + num_images=mm_counts.get("image", 0),

293 + overrides=mm_options.get("image"),

294 ),

295 "video": self._get_dummy_videos(

296 width=target_width,

297 height=target_height,

298 num_frames=target_num_frames,

306 - num_videos=num_videos,

307 - overrides=video_overrides,

299 + num_videos=mm_counts.get("video", 0),

300 + overrides=mm_options.get("video"),

301 ),

302 }

303

vllm/model_executor/models/llava_onevision2.py

+2 -2

@@ -54,7 +54,7 @@ from vllm.compilation.decorators import (

54 support_torch_compile,

55 )

56 from vllm.config import VllmConfig

57 -from vllm.config.multimodal import BaseDummyOptions

57 +from vllm.config.multimodal import MultiModalDummyOptions

58 from vllm.distributed import parallel_state

59 from vllm.distributed import utils as dist_utils

60 from vllm.inputs import ModalityData, MultiModalDataDict

@@ -1471,7 +1471,7 @@ class LlavaOnevision2DummyInputsBuilder(

1471 self,

1472 seq_len: int,

1473 mm_counts: Mapping[str, int],

1474 - mm_options: Mapping[str, BaseDummyOptions] | None = None,

1474 + mm_options: MultiModalDummyOptions | None = None,

1475 ) -> MultiModalDataDict:

1476 n_img = mm_counts.get("image", 0)

1477 n_vid = mm_counts.get("video", 0)

vllm/model_executor/models/midashenglm.py

+4 -9

@@ -36,7 +36,7 @@ from torch.nn.functional import scaled_dot_product_attention

36 from transformers import BatchFeature

37

38 from vllm.config import VllmConfig

39 -from vllm.config.multimodal import AudioDummyOptions, BaseDummyOptions

39 +from vllm.config.multimodal import MultiModalDummyOptions

40 from vllm.distributed import get_tensor_model_parallel_world_size

41 from vllm.inputs import MultiModalDataDict

42 from vllm.model_executor.layers.activation import get_act_fn

@@ -564,18 +564,13 @@ class MiDashengLMDummyInputsBuilder(BaseDummyInputsBuilder[MiDashengLMProcessing

564 self,

565 seq_len: int,

566 mm_counts: Mapping[str, int],

567 - mm_options: Mapping[str, BaseDummyOptions],

567 + mm_options: MultiModalDummyOptions,

568 ) -> MultiModalDataDict:

569 - num_audios = mm_counts.get("audio", 0)

570 -

571 - audio_overrides = mm_options.get("audio")

572 - assert audio_overrides is None or isinstance(audio_overrides, AudioDummyOptions)

573 -

569 return {

570 "audio": self._get_dummy_audios(

571 length=self.info.get_max_audio_len(),

577 - num_audios=num_audios,

578 - overrides=audio_overrides,

572 + num_audios=mm_counts.get("audio", 0),

573 + overrides=mm_options.get("audio"),

574 )

575 }

576

vllm/model_executor/models/mimo_v2_omni.py

+6 -17

@@ -15,11 +15,7 @@ from transformers.models.qwen2_vl.image_processing_qwen2_vl import smart_resize

15 from typing_extensions import TypedDict

16

17 from vllm.config import VllmConfig

18 -from vllm.config.multimodal import (

19 - BaseDummyOptions,

20 - ImageDummyOptions,

21 - VideoDummyOptions,

22 -)

18 +from vllm.config.multimodal import MultiModalDummyOptions

19 from vllm.distributed import parallel_state

20 from vllm.distributed import utils as dist_utils

21 from vllm.inputs import MultiModalDataDict

@@ -1184,33 +1180,26 @@ class MiMoV2OmniDummyInputsBuilder(BaseDummyInputsBuilder[MiMoV2OmniProcessingIn

1180 self,

1181 seq_len: int,

1182 mm_counts: Mapping[str, int],

1187 - mm_options: Mapping[str, BaseDummyOptions],

1183 + mm_options: MultiModalDummyOptions,

1184 ) -> MultiModalDataDict:

1189 - num_images = mm_counts.get("image", 0)

1190 - num_videos = mm_counts.get("video", 0)

1191 -

1185 target_width, target_height = self.info.get_image_size_with_most_features()

1186 target_num_frames = self.info.get_num_frames_with_most_features(

1187 seq_len, mm_counts

1188 )

1196 - image_overrides = mm_options.get("image")

1197 - video_overrides = mm_options.get("video")

1198 - assert image_overrides is None or isinstance(image_overrides, ImageDummyOptions)

1199 - assert video_overrides is None or isinstance(video_overrides, VideoDummyOptions)

1189

1190 return {

1191 "image": self._get_dummy_images(

1192 width=target_width,

1193 height=target_height,

1205 - num_images=num_images,

1206 - overrides=image_overrides,

1194 + num_images=mm_counts.get("image", 0),

1195 + overrides=mm_options.get("image"),

1196 ),

1197 "video": self._get_dummy_videos(

1198 width=target_width,

1199 height=target_height,

1200 num_frames=target_num_frames,

1212 - num_videos=num_videos,

1213 - overrides=video_overrides,

1201 + num_videos=mm_counts.get("video", 0),

1202 + overrides=mm_options.get("video"),

1203 ),

1204 }

1205

vllm/model_executor/models/minicpmo.py

+4 -8

@@ -40,7 +40,7 @@ from transformers.models.whisper.modeling_whisper import (

40 )

41

42 from vllm.config import VllmConfig

43 -from vllm.config.multimodal import AudioDummyOptions, BaseDummyOptions

43 +from vllm.config.multimodal import MultiModalDummyOptions

44 from vllm.inputs import ModalityData, MultiModalDataDict

45 from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalKwargsItems

46 from vllm.multimodal.inputs import (

@@ -402,22 +402,18 @@ class MiniCPMODummyInputsBuilder(MiniCPMVDummyInputsBuilder[MiniCPMOProcessingIn

402 self,

403 seq_len: int,

404 mm_counts: Mapping[str, int],

405 - mm_options: Mapping[str, BaseDummyOptions],

405 + mm_options: MultiModalDummyOptions,

406 ) -> MultiModalDataDict:

407 - num_audios = mm_counts.get("audio", 0)

407 audio_len = (

408 self.info.get_max_audio_chunks_with_most_features()

409 * self.info.get_default_audio_sampling_rate()

410 )

411

413 - audio_overrides = mm_options.get("audio")

414 - assert audio_overrides is None or isinstance(audio_overrides, AudioDummyOptions)

415 -

412 audio_mm_data = {

413 "audio": self._get_dummy_audios(

414 length=audio_len,

419 - num_audios=num_audios,

420 - overrides=audio_overrides,

415 + num_audios=mm_counts.get("audio", 0),

416 + overrides=mm_options.get("audio"),

417 )

418 }

419

vllm/model_executor/models/minicpmv.py

+5 -9

@@ -45,9 +45,8 @@ from typing_extensions import TypedDict, TypeVar

45

46 from vllm.config import VllmConfig

47 from vllm.config.multimodal import (

48 - BaseDummyOptions,

48 ImageDummyOptions,

50 - VideoDummyOptions,

49 + MultiModalDummyOptions,

50 )

51 from vllm.inputs import ModalityData, MultiModalDataDict

52 from vllm.model_executor.layers.quantization import QuantizationConfig

@@ -848,9 +847,8 @@ class MiniCPMVDummyInputsBuilder(BaseDummyInputsBuilder[_I]):

847 self,

848 seq_len: int,

849 mm_counts: Mapping[str, int],

851 - mm_options: Mapping[str, BaseDummyOptions],

850 + mm_options: MultiModalDummyOptions,

851 ) -> MultiModalDataDict:

853 - num_images = mm_counts.get("image", 0)

852 num_videos = mm_counts.get("video", 0)

853

854 image_width, image_height = self.info.get_image_size_with_most_features()

@@ -859,14 +857,12 @@ class MiniCPMVDummyInputsBuilder(BaseDummyInputsBuilder[_I]):

857 seq_len, mm_counts

858 )

859

862 - image_overrides = mm_options.get("image")

860 video_overrides = mm_options.get("video")

864 - assert image_overrides is None or isinstance(image_overrides, ImageDummyOptions)

861

862 # Convert video overrides to image overrides for per-frame image generation,

863 # and apply num_frames override to num_video_frames.

864 video_frame_overrides: ImageDummyOptions | None = None

869 - if isinstance(video_overrides, VideoDummyOptions):

865 + if video_overrides is not None:

866 if video_overrides.num_frames:

867 num_video_frames = min(num_video_frames, video_overrides.num_frames)

868 if video_overrides.width or video_overrides.height:

@@ -880,8 +876,8 @@ class MiniCPMVDummyInputsBuilder(BaseDummyInputsBuilder[_I]):

876 "image": self._get_dummy_images(

877 width=image_width,

878 height=image_height,

883 - num_images=num_images,

884 - overrides=image_overrides,

879 + num_images=mm_counts.get("image", 0),

880 + overrides=mm_options.get("image"),

881 ),

882 "video": [

883 self._get_dummy_images(

vllm/model_executor/models/mistral3.py

+4 -8

@@ -11,7 +11,7 @@ from transformers import BatchFeature, Mistral3Config, PixtralVisionConfig

11 from transformers.models.pixtral import PixtralProcessor

12

13 from vllm.config import VllmConfig

14 -from vllm.config.multimodal import BaseDummyOptions

14 +from vllm.config.multimodal import MultiModalDummyOptions

15 from vllm.inputs import MultiModalDataDict

16 from vllm.model_executor.layers.activation import get_act_fn

17 from vllm.model_executor.layers.layernorm import RMSNorm

@@ -261,20 +261,16 @@ class Mistral3DummyInputsBuilder(BaseDummyInputsBuilder):

261 self,

262 seq_len: int,

263 mm_counts: Mapping[str, int],

264 - mm_options: Mapping[str, BaseDummyOptions],

264 + mm_options: MultiModalDummyOptions,

265 ) -> MultiModalDataDict:

266 - num_images = mm_counts.get("image", 0)

267 -

266 target_width, target_height = self.info.get_image_size_with_most_features()

267

270 - image_overrides = mm_options.get("image")

271 -

268 return {

269 "image": self._get_dummy_images(

270 width=target_width,

271 height=target_height,

276 - num_images=num_images,

277 - overrides=image_overrides,

272 + num_images=mm_counts.get("image", 0),

273 + overrides=mm_options.get("image"),

274 )

275 }

276

vllm/model_executor/models/mllama4.py

+4 -8

@@ -36,7 +36,7 @@ from vllm.compilation.decorators import (

36 support_torch_compile,

37 )

38 from vllm.config import VllmConfig, set_current_vllm_config

39 -from vllm.config.multimodal import BaseDummyOptions

39 +from vllm.config.multimodal import MultiModalDummyOptions

40 from vllm.distributed import get_tensor_model_parallel_world_size

41 from vllm.inputs import MultiModalDataDict

42 from vllm.model_executor.layers.attention import MMEncoderAttention

@@ -709,20 +709,16 @@ class Mllama4DummyInputsBuilder(BaseDummyInputsBuilder[Mllama4ProcessingInfo]):

709 self,

710 seq_len: int,

711 mm_counts: Mapping[str, int],

712 - mm_options: Mapping[str, BaseDummyOptions],

712 + mm_options: MultiModalDummyOptions,

713 ) -> MultiModalDataDict:

714 - num_images = mm_counts.get("image", 0)

715 -

714 (target_width, target_height) = self.info.get_image_size_with_most_features()

715

718 - image_overrides = mm_options.get("image")

719 -

716 return {

717 "image": self._get_dummy_images(

718 width=target_width,

719 height=target_height,

724 - num_images=num_images,

725 - overrides=image_overrides,

720 + num_images=mm_counts.get("image", 0),

721 + overrides=mm_options.get("image"),

722 )

723 }

724

vllm/model_executor/models/molmo.py

+4 -7

@@ -21,7 +21,7 @@ from transformers import (

21

22 from vllm.compilation.decorators import support_torch_compile

23 from vllm.config import CacheConfig, VllmConfig

24 -from vllm.config.multimodal import BaseDummyOptions

24 +from vllm.config.multimodal import MultiModalDummyOptions

25 from vllm.distributed import (

26 get_pp_group,

27 get_tensor_model_parallel_rank,

@@ -1104,19 +1104,16 @@ class MolmoDummyInputsBuilder(BaseDummyInputsBuilder[MolmoProcessingInfo]):

1104 self,

1105 seq_len: int,

1106 mm_counts: Mapping[str, int],

1107 - mm_options: Mapping[str, BaseDummyOptions],

1107 + mm_options: MultiModalDummyOptions,

1108 ) -> MultiModalDataDict:

1109 target_width, target_height = self.info.get_image_size_with_most_features()

1110 - num_images = mm_counts.get("image", 0)

1111 -

1112 - image_overrides = mm_options.get("image")

1110

1111 return {

1112 "image": self._get_dummy_images(

1113 width=target_width,

1114 height=target_height,

1118 - num_images=num_images,

1119 - overrides=image_overrides,

1115 + num_images=mm_counts.get("image", 0),

1116 + overrides=mm_options.get("image"),

1117 )

1118 }

1119

vllm/model_executor/models/molmo2.py

+3 -6

@@ -26,7 +26,7 @@ from typing_extensions import TypedDict

26

27 from vllm.compilation.decorators import support_torch_compile

28 from vllm.config import CacheConfig, VllmConfig

29 -from vllm.config.multimodal import BaseDummyOptions, VideoDummyOptions

29 +from vllm.config.multimodal import MultiModalDummyOptions, VideoDummyOptions

30 from vllm.distributed import (

31 get_pp_group,

32 get_tensor_model_parallel_rank,

@@ -1834,7 +1834,7 @@ class Molmo2DummyInputsBuilder(BaseDummyInputsBuilder[Molmo2ProcessingInfo]):

1834 self,

1835 seq_len: int,

1836 mm_counts: Mapping[str, int],

1837 - mm_options: Mapping[str, BaseDummyOptions],

1837 + mm_options: MultiModalDummyOptions,

1838 ) -> MultiModalDataDict:

1839 num_images = mm_counts.get("image", 0)

1840 num_videos = mm_counts.get("video", 0)

@@ -1845,13 +1845,11 @@ class Molmo2DummyInputsBuilder(BaseDummyInputsBuilder[Molmo2ProcessingInfo]):

1845 if num_images > 0:

1846 target_width, target_height = self.info.get_image_size_with_most_features()

1847

1848 - image_overrides = mm_options.get("image")

1849 -

1848 dummy_images = self._get_dummy_images(

1849 width=target_width,

1850 height=target_height,

1851 num_images=num_images,

1854 - overrides=image_overrides,

1852 + overrides=mm_options.get("image"),

1853 )

1854

1855 if num_videos > 0:

@@ -1864,7 +1862,6 @@ class Molmo2DummyInputsBuilder(BaseDummyInputsBuilder[Molmo2ProcessingInfo]):

1862 video_overrides = mm_options.get("video")

1863

1864 if video_overrides:

1867 - assert isinstance(video_overrides, VideoDummyOptions)

1865 num_frames_override = video_overrides.num_frames

1866 if num_frames_override:

1867 if num_frames_override > target_num_frames:

vllm/model_executor/models/moondream3.py

+3 -4

@@ -13,7 +13,7 @@ import torch.nn.functional as F

13 from transformers import BatchFeature

14

15 from vllm.config import VllmConfig

16 -from vllm.config.multimodal import BaseDummyOptions

16 +from vllm.config.multimodal import MultiModalDummyOptions

17 from vllm.distributed import (

18 get_pp_group,

19 get_tensor_model_parallel_rank,

@@ -943,15 +943,14 @@ class Moondream3DummyInputsBuilder(BaseDummyInputsBuilder[Moondream3ProcessingIn

943 self,

944 seq_len: int,

945 mm_counts: Mapping[str, int],

946 - mm_options: Mapping[str, BaseDummyOptions] | None = None,

946 + mm_options: MultiModalDummyOptions | None = None,

947 mm_processor_kwargs: Mapping[str, object] | None = None,

948 ) -> MultiModalDataDict:

949 - num_images = mm_counts.get("image", 0)

949 return {

950 "image": self._get_dummy_images(

951 width=378,

952 height=378,

954 - num_images=num_images,

953 + num_images=mm_counts.get("image", 0),

954 )

955 }

956

vllm/model_executor/models/moss_audio.py

+4 -7

@@ -19,7 +19,7 @@ from typing_extensions import TypedDict

19

20 from vllm.compilation.decorators import support_torch_compile

21 from vllm.config import VllmConfig

22 -from vllm.config.multimodal import AudioDummyOptions, BaseDummyOptions

22 +from vllm.config.multimodal import MultiModalDummyOptions

23 from vllm.distributed import (

24 get_pp_group,

25 get_tensor_model_parallel_world_size,

@@ -1341,16 +1341,13 @@ class MossAudioDummyInputsBuilder(BaseDummyInputsBuilder[MossAudioProcessingInfo

1341 self,

1342 seq_len: int,

1343 mm_counts: Mapping[str, int],

1344 - mm_options: Mapping[str, BaseDummyOptions],

1344 + mm_options: MultiModalDummyOptions,

1345 ) -> MultiModalDataDict:

1346 - num_audios = mm_counts.get("audio", 0)

1347 - audio_overrides = mm_options.get("audio")

1348 - assert audio_overrides is None or isinstance(audio_overrides, AudioDummyOptions)

1346 return {

1347 "audio": self._get_dummy_audios(

1348 length=16000,

1352 - num_audios=num_audios,

1353 - overrides=audio_overrides,

1349 + num_audios=mm_counts.get("audio", 0),

1350 + overrides=mm_options.get("audio"),

1351 )

1352 }

1353

vllm/model_executor/models/moss_transcribe_diarize.py

+4 -6

@@ -20,7 +20,7 @@ from torch import nn

20 from transformers import BatchFeature

21

22 from vllm.config import ModelConfig, SpeechToTextConfig, VllmConfig

23 -from vllm.config.multimodal import AudioDummyOptions, BaseDummyOptions

23 +from vllm.config.multimodal import MultiModalDummyOptions

24 from vllm.config.speech_to_text import SpeechToTextParams

25 from vllm.inputs import ModalityData, MultiModalDataDict, PromptType, TextPrompt

26 from vllm.model_executor.models.interfaces import (

@@ -423,20 +423,18 @@ class MossTranscribeDiarizeDummyInputsBuilder(

423 self,

424 seq_len: int,

425 mm_counts: Mapping[str, int],

426 - mm_options: Mapping[str, BaseDummyOptions],

426 + mm_options: MultiModalDummyOptions,

427 ) -> MultiModalDataDict:

428 num_audios = mm_counts.get("audio", 0)

429 if num_audios == 0:

430 return {}

431

432 feature_extractor = self.info.get_feature_extractor()

433 - audio_overrides = mm_options.get("audio")

434 - assert audio_overrides is None or isinstance(audio_overrides, AudioDummyOptions)

433 return {

434 "audio": self._get_dummy_audios(

435 length=_get_max_audio_samples(feature_extractor),

436 num_audios=num_audios,

439 - overrides=audio_overrides,

437 + overrides=mm_options.get("audio"),

438 )

439 }

440

@@ -444,7 +442,7 @@ class MossTranscribeDiarizeDummyInputsBuilder(

442 self,

443 seq_len: int,

444 mm_counts: Mapping[str, int],

447 - mm_options: Mapping[str, BaseDummyOptions],

445 + mm_options: MultiModalDummyOptions,

446 ) -> ProcessorInputs:

447 dummy_mm_data = self.get_dummy_mm_data(seq_len, mm_counts, mm_options)

448 dummy_mm_items = self.info.parse_mm_data(dummy_mm_data)

vllm/model_executor/models/muse_glimmer.py

+4 -12

@@ -45,11 +45,7 @@ from transformers import BatchFeature

45

46 from vllm.compilation.decorators import support_torch_compile

47 from vllm.config import CacheConfig, VllmConfig

48 -from vllm.config.multimodal import (

49 - BaseDummyOptions,

50 - ImageDummyOptions,

51 - VideoDummyOptions,

52 -)

48 +from vllm.config.multimodal import MultiModalDummyOptions

49 from vllm.distributed import (

50 divide,

51 get_pp_group,

@@ -234,7 +230,7 @@ class MuseGlimmerDummyInputsBuilder(BaseDummyInputsBuilder[MuseGlimmerProcessing

230 self,

231 seq_len: int,

232 mm_counts: Mapping[str, int],

237 - mm_options: Mapping[str, BaseDummyOptions],

233 + mm_options: MultiModalDummyOptions,

234 ) -> MultiModalDataDict:

235 processor = self.info.get_hf_processor()

236 video_processor = processor.video_processor

@@ -245,16 +241,12 @@ class MuseGlimmerDummyInputsBuilder(BaseDummyInputsBuilder[MuseGlimmerProcessing

241 * int(video_processor.downsample_factor)

242 * video_grid

243 )

248 - image_overrides = mm_options.get("image")

249 - video_overrides = mm_options.get("video")

250 - assert image_overrides is None or isinstance(image_overrides, ImageDummyOptions)

251 - assert video_overrides is None or isinstance(video_overrides, VideoDummyOptions)

244 return {

245 "image": self._get_dummy_images(

246 width=image_width,

247 height=image_height,

248 num_images=mm_counts.get("image", 0),

257 - overrides=image_overrides,

249 + overrides=mm_options.get("image"),

250 ),

251 "video": self._get_dummy_videos(

252 width=video_size,

@@ -263,7 +255,7 @@ class MuseGlimmerDummyInputsBuilder(BaseDummyInputsBuilder[MuseGlimmerProcessing

255 seq_len, mm_counts

256 ),

257 num_videos=mm_counts.get("video", 0),

266 - overrides=video_overrides,

258 + overrides=mm_options.get("video"),

259 ),

260 }

261

vllm/model_executor/models/nano_nemotron_vl.py

+4 -14

@@ -21,8 +21,7 @@ from transformers import BatchFeature, PretrainedConfig

21 from vllm import envs

22 from vllm.config import VllmConfig

23 from vllm.config.multimodal import (

24 - AudioDummyOptions,

25 - BaseDummyOptions,

24 + MultiModalDummyOptions,

25 VideoDummyOptions,

26 )

27 from vllm.inputs import MultiModalDataDict, MultiModalInput

@@ -839,22 +838,19 @@ class NanoNemotronVLDummyInputsBuilder(

838 self,

839 seq_len: int,

840 mm_counts: Mapping[str, int],

842 - mm_options: Mapping[str, BaseDummyOptions],

841 + mm_options: MultiModalDummyOptions,

842 ) -> MultiModalDataDict:

844 - num_images = mm_counts.get("image", 0)

843 (target_width, target_height), _ = (

844 self.info.get_dummy_image_size_and_max_tokens(mm_counts)

845 )

846 processor = self.info.get_hf_processor()

847

850 - image_overrides = mm_options.get("image")

851 -

848 dummy_image = {

849 "image": self._get_dummy_images(

850 width=target_width,

851 height=target_height,

856 - num_images=num_images,

857 - overrides=image_overrides,

852 + num_images=mm_counts.get("image", 0),

853 + overrides=mm_options.get("image"),

854 )

855 }

856

@@ -889,9 +885,6 @@ class NanoNemotronVLDummyInputsBuilder(

885 target_num_frames = num_frames

886 num_videos = mm_counts.get("video", 0)

887 video_overrides = mm_options.get("video")

892 - assert video_overrides is None or isinstance(

893 - video_overrides, VideoDummyOptions

894 - )

888 dummy_video = {

889 "video": self._get_dummy_videos(

890 width=video_width,

@@ -907,9 +900,6 @@ class NanoNemotronVLDummyInputsBuilder(

900 if sound_config := self.info.sound_config:

901 num_audios = mm_counts.get("audio", 0)

902 audio_overrides = mm_options.get("audio") if mm_options else None

910 - assert audio_overrides is None or isinstance(

911 - audio_overrides, AudioDummyOptions

912 - )

903 tokens_per_audio = max(1, seq_len // max(num_audios, 1))

904 max_audio_num_samples = MAX_AUDIO_LEN_S * sound_config.sampling_rate

905 calculated_max_audio_num_samples = ParakeetExtractor.audio_length(

vllm/model_executor/models/nemotron_parse.py

+2 -2

@@ -22,7 +22,7 @@ from transformers import (

22

23 from vllm.config import CacheConfig, VllmConfig

24 from vllm.config.lora import LoRAConfig

25 -from vllm.config.multimodal import BaseDummyOptions

25 +from vllm.config.multimodal import MultiModalDummyOptions

26 from vllm.inputs import MultiModalDataDict

27 from vllm.logger import init_logger

28 from vllm.model_executor.layers.activation import get_act_fn

@@ -388,7 +388,7 @@ class NemotronParseDummyInputsBuilder(

388 self,

389 seq_len: int,

390 mm_counts: Mapping[str, int],

391 - mm_options: Mapping[str, BaseDummyOptions],

391 + mm_options: MultiModalDummyOptions,

392 ) -> MultiModalDataDict:

393 num_images = mm_counts.get("image", 0)

394

vllm/model_executor/models/nvlm_d.py

+4 -7

@@ -13,7 +13,7 @@ import torch

13 import torch.nn as nn

14 from transformers import PretrainedConfig

15

16 -from vllm.config.multimodal import BaseDummyOptions

16 +from vllm.config.multimodal import MultiModalDummyOptions

17 from vllm.inputs import MultiModalDataDict

18 from vllm.model_executor.layers.quantization import QuantizationConfig

19 from vllm.multimodal import MULTIMODAL_REGISTRY

@@ -83,19 +83,16 @@ class NVLMDummyInputsBuilder(BaseInternVLDummyInputsBuilder[NVLMProcessingInfo])

83 self,

84 seq_len: int,

85 mm_counts: Mapping[str, int],

86 - mm_options: Mapping[str, BaseDummyOptions],

86 + mm_options: MultiModalDummyOptions,

87 ) -> MultiModalDataDict:

88 target_width, target_height = self.info.get_image_size_with_most_features()

89 - num_images = mm_counts.get("image", 0)

90 -

91 - image_overrides = mm_options.get("image")

89

90 return {

91 "image": self._get_dummy_images(

92 width=target_width,

93 height=target_height,

97 - num_images=num_images,

98 - overrides=image_overrides,

94 + num_images=mm_counts.get("image", 0),

95 + overrides=mm_options.get("image"),

96 )

97 }

98

vllm/model_executor/models/openvla.py

+4 -6

@@ -9,7 +9,7 @@ import torch.nn as nn

9 from transformers import BatchFeature

10

11 from vllm.config import VllmConfig

12 -from vllm.config.multimodal import BaseDummyOptions

12 +from vllm.config.multimodal import MultiModalDummyOptions

13 from vllm.inputs import MultiModalDataDict

14 from vllm.model_executor.layers.activation import get_act_fn

15 from vllm.model_executor.layers.linear import (

@@ -299,18 +299,16 @@ class OpenVLADummyInputsBuilder(BaseDummyInputsBuilder[OpenVLAProcessingInfo]):

299 self,

300 seq_len: int,

301 mm_counts: Mapping[str, int],

302 - mm_options: Mapping[str, BaseDummyOptions],

302 + mm_options: MultiModalDummyOptions,

303 ) -> MultiModalDataDict:

304 - num_images = mm_counts.get("image", 0)

305 - image_overrides = mm_options.get("image")

304 image_size = self.info.get_image_size_with_most_features()

305

306 return {

307 "image": self._get_dummy_images(

308 width=image_size.width,

309 height=image_size.height,

312 - num_images=num_images,

313 - overrides=image_overrides,

310 + num_images=mm_counts.get("image", 0),

311 + overrides=mm_options.get("image"),

312 )

313 }

314

vllm/model_executor/models/ovis.py

+4 -8

@@ -29,7 +29,7 @@ from torch.nn.functional import gumbel_softmax, pad, softmax

29 from transformers import BatchFeature, PretrainedConfig

30

31 from vllm.config import VllmConfig

32 -from vllm.config.multimodal import BaseDummyOptions

32 +from vllm.config.multimodal import MultiModalDummyOptions

33 from vllm.inputs import MultiModalDataDict

34 from vllm.model_executor.layers.linear import ReplicatedLinear

35 from vllm.model_executor.layers.quantization import QuantizationConfig

@@ -311,20 +311,16 @@ class OvisDummyInputsBuilder(BaseDummyInputsBuilder[OvisProcessingInfo]):

311 self,

312 seq_len: int,

313 mm_counts: Mapping[str, int],

314 - mm_options: Mapping[str, BaseDummyOptions],

314 + mm_options: MultiModalDummyOptions,

315 ) -> MultiModalDataDict:

316 - num_images = mm_counts.get("image", 0)

317 -

316 target_width, target_height = self.info.get_image_size_with_most_features()

317

320 - image_overrides = mm_options.get("image")

321 -

318 mm_data = {

319 "image": self._get_dummy_images(

320 width=target_width,

321 height=target_height,

326 - num_images=num_images,

327 - overrides=image_overrides,

322 + num_images=mm_counts.get("image", 0),

323 + overrides=mm_options.get("image"),

324 ),

325 }

326 return mm_data

vllm/model_executor/models/ovis2_5.py

+6 -13

@@ -11,7 +11,7 @@ import torch.nn as nn

11 from transformers import BaseImageProcessor, BatchFeature, PretrainedConfig

12

13 from vllm.config import VllmConfig

14 -from vllm.config.multimodal import BaseDummyOptions, VideoDummyOptions

14 +from vllm.config.multimodal import MultiModalDummyOptions

15 from vllm.inputs import MultiModalDataDict

16 from vllm.model_executor.layers.linear import ReplicatedLinear

17 from vllm.model_executor.layers.quantization import QuantizationConfig

@@ -290,33 +290,26 @@ class Ovis2_5DummyInputsBuilder(BaseDummyInputsBuilder[Ovis2_5ProcessingInfo]):

290 self,

291 seq_len: int,

292 mm_counts: Mapping[str, int],

293 - mm_options: Mapping[str, BaseDummyOptions],

293 + mm_options: MultiModalDummyOptions,

294 ) -> MultiModalDataDict:

295 - num_images = mm_counts.get("image", 0)

296 - num_videos = mm_counts.get("video", 0)

297 -

295 target_width, target_height = self.info.get_image_size_with_most_features()

296 target_num_frames = self.info.get_num_frames_with_most_features(

297 seq_len, mm_counts

298 )

299

303 - image_overrides = mm_options.get("image")

304 - video_overrides = mm_options.get("video")

305 - assert video_overrides is None or isinstance(video_overrides, VideoDummyOptions)

306 -

300 mm_data = {

301 "image": self._get_dummy_images(

302 width=target_width,

303 height=target_height,

311 - num_images=num_images,

312 - overrides=image_overrides,

304 + num_images=mm_counts.get("image", 0),

305 + overrides=mm_options.get("image"),

306 ),

307 "video": self._get_dummy_videos(

308 width=target_width,

309 height=target_height,

310 num_frames=target_num_frames,

318 - num_videos=num_videos,

319 - overrides=video_overrides,

311 + num_videos=mm_counts.get("video", 0),

312 + overrides=mm_options.get("video"),

313 ),

314 }

315 return mm_data

vllm/model_executor/models/paddleocr_vl.py

+4 -7

@@ -32,7 +32,7 @@ from transformers.modeling_outputs import (

32 from transformers.utils import torch_int

33

34 from vllm.config import VllmConfig

35 -from vllm.config.multimodal import BaseDummyOptions

35 +from vllm.config.multimodal import MultiModalDummyOptions

36 from vllm.distributed import parallel_state

37 from vllm.distributed import utils as dist_utils

38 from vllm.inputs import MultiModalDataDict

@@ -223,19 +223,16 @@ class PaddleOCRVLDummyInputsBuilder(BaseDummyInputsBuilder[PaddleOCRVLProcessing

223 self,

224 seq_len: int,

225 mm_counts: Mapping[str, int],

226 - mm_options: Mapping[str, BaseDummyOptions],

226 + mm_options: MultiModalDummyOptions,

227 ) -> MultiModalDataDict:

228 - num_images = mm_counts.get("image", 0)

229 -

228 max_image_size = self.info.get_image_size_with_most_features()

231 - image_overrides = mm_options.get("image")

229

230 return {

231 "image": self._get_dummy_images(

232 width=max_image_size.width,

233 height=max_image_size.height,

237 - num_images=num_images,

238 - overrides=image_overrides,

234 + num_images=mm_counts.get("image", 0),

235 + overrides=mm_options.get("image"),

236 )

237 }

238

vllm/model_executor/models/paligemma.py

+4 -8

@@ -8,7 +8,7 @@ from torch import nn

8 from transformers import BatchFeature, PaliGemmaConfig

9

10 from vllm.config import VllmConfig

11 -from vllm.config.multimodal import BaseDummyOptions

11 +from vllm.config.multimodal import MultiModalDummyOptions

12 from vllm.inputs import MultiModalDataDict, MultiModalInput

13 from vllm.logger import init_logger

14 from vllm.multimodal import MULTIMODAL_REGISTRY

@@ -130,22 +130,18 @@ class PaliGemmaDummyInputsBuilder(BaseDummyInputsBuilder[PaliGemmaProcessingInfo

130 self,

131 seq_len: int,

132 mm_counts: Mapping[str, int],

133 - mm_options: Mapping[str, BaseDummyOptions],

133 + mm_options: MultiModalDummyOptions,

134 ) -> MultiModalDataDict:

135 hf_config = self.info.get_hf_config()

136 vision_config = hf_config.vision_config

137 max_image_size = vision_config.image_size

138

139 - num_images = mm_counts.get("image", 0)

140 -

141 - image_overrides = mm_options.get("image")

142 -

139 return {

140 "image": self._get_dummy_images(

141 width=max_image_size,

142 height=max_image_size,

147 - num_images=num_images,

148 - overrides=image_overrides,

143 + num_images=mm_counts.get("image", 0),

144 + overrides=mm_options.get("image"),

145 )

146 }

147

vllm/model_executor/models/phi3v.py

+4 -8

@@ -29,7 +29,7 @@ from transformers import (

29 )

30

31 from vllm.config import VllmConfig

32 -from vllm.config.multimodal import BaseDummyOptions

32 +from vllm.config.multimodal import MultiModalDummyOptions

33 from vllm.inputs import MultiModalDataDict

34 from vllm.logger import init_logger

35 from vllm.model_executor.layers.quantization import QuantizationConfig

@@ -371,20 +371,16 @@ class Phi3VDummyInputsBuilder(BaseDummyInputsBuilder[Phi3VProcessingInfo]):

371 self,

372 seq_len: int,

373 mm_counts: Mapping[str, int],

374 - mm_options: Mapping[str, BaseDummyOptions],

374 + mm_options: MultiModalDummyOptions,

375 ) -> MultiModalDataDict:

376 - num_images = mm_counts.get("image", 0)

377 -

376 target_width, target_height = self.info.get_image_size_with_most_features()

377

380 - image_overrides = mm_options.get("image")

381 -

378 return {

379 "image": self._get_dummy_images(

380 width=target_width,

381 height=target_height,

386 - num_images=num_images,

387 - overrides=image_overrides,

382 + num_images=mm_counts.get("image", 0),

383 + overrides=mm_options.get("image"),

384 )

385 }

386

vllm/model_executor/models/phi4mm.py

+6 -13

@@ -17,7 +17,7 @@ from transformers import (

17 )

18

19 from vllm.config import VllmConfig

20 -from vllm.config.multimodal import AudioDummyOptions, BaseDummyOptions

20 +from vllm.config.multimodal import MultiModalDummyOptions

21 from vllm.distributed import get_pp_group

22 from vllm.inputs import MultiModalDataDict

23 from vllm.model_executor.layers.logits_processor import LogitsProcessor

@@ -839,28 +839,21 @@ class Phi4MMDummyInputsBuilder(BaseDummyInputsBuilder[Phi4MMProcessingInfo]):

839 self,

840 seq_len: int,

841 mm_counts: Mapping[str, int],

842 - mm_options: Mapping[str, BaseDummyOptions],

842 + mm_options: MultiModalDummyOptions,

843 ) -> MultiModalDataDict:

844 - num_audios = mm_counts.get("audio", 0)

845 - num_images = mm_counts.get("image", 0)

846 -

844 target_width, target_height = self.info.get_image_size_with_most_features()

845

849 - image_overrides = mm_options.get("image")

850 - audio_overrides = mm_options.get("audio")

851 - assert audio_overrides is None or isinstance(audio_overrides, AudioDummyOptions)

852 -

846 mm_data = {

847 "image": self._get_dummy_images(

848 width=target_width,

849 height=target_height,

857 - num_images=num_images,

858 - overrides=image_overrides,

850 + num_images=mm_counts.get("image", 0),

851 + overrides=mm_options.get("image"),

852 ),

853 "audio": self._get_dummy_audios(

854 length=_AUDIO_MAX_SOUNDFILE_SIZE,

862 - num_audios=num_audios,

863 - overrides=audio_overrides,

855 + num_audios=mm_counts.get("audio", 0),

856 + overrides=mm_options.get("audio"),

857 ),

858 }

859

vllm/model_executor/models/phi4siglip.py

+3 -4

@@ -14,7 +14,7 @@ import torch.nn as nn

14 from transformers import BatchFeature, PretrainedConfig, Siglip2VisionConfig

15

16 from vllm.config import VllmConfig

17 -from vllm.config.multimodal import BaseDummyOptions

17 +from vllm.config.multimodal import MultiModalDummyOptions

18 from vllm.inputs import MultiModalDataDict

19 from vllm.logger import init_logger

20 from vllm.multimodal import MULTIMODAL_REGISTRY

@@ -126,15 +126,14 @@ class Phi4SiglipDummyInputsBuilder(

126 self,

127 seq_len: int,

128 mm_counts: Mapping[str, int],

129 - mm_options: Mapping[str, BaseDummyOptions],

129 + mm_options: MultiModalDummyOptions,

130 ) -> MultiModalDataDict:

131 - num_images = mm_counts.get("image", 0)

131 size = self.info.get_image_size_with_most_features()

132 return {

133 "image": self._get_dummy_images(

134 width=size.width,

135 height=size.height,

137 - num_images=num_images,

136 + num_images=mm_counts.get("image", 0),

137 overrides=mm_options.get("image"),

138 ),

139 }

vllm/model_executor/models/pixtral.py

+5 -9

@@ -21,7 +21,7 @@ from transformers.models.pixtral.image_processing_pixtral import (

21 from transformers.models.pixtral.modeling_pixtral import apply_rotary_pos_emb

22

23 from vllm.config import VllmConfig

24 -from vllm.config.multimodal import BaseDummyOptions

24 +from vllm.config.multimodal import MultiModalDummyOptions

25 from vllm.distributed import divide, get_tensor_model_parallel_world_size

26 from vllm.inputs import MultiModalDataDict

27 from vllm.model_executor.layers.activation import SiluAndMul, get_act_and_mul_fn

@@ -208,20 +208,16 @@ class PixtralDummyInputsBuilder(BaseDummyInputsBuilder[PixtralProcessingInfo]):

208 self,

209 seq_len: int,

210 mm_counts: Mapping[str, int],

211 - mm_options: Mapping[str, BaseDummyOptions],

211 + mm_options: MultiModalDummyOptions,

212 ) -> MultiModalDataDict:

213 - num_images = mm_counts.get("image", 0)

214 -

213 target_width, target_height = self.info.get_image_size_with_most_features()

214

217 - image_overrides = mm_options.get("image")

218 -

215 return {

216 "image": self._get_dummy_images(

217 width=target_width,

218 height=target_height,

223 - num_images=num_images,

224 - overrides=image_overrides,

219 + num_images=mm_counts.get("image", 0),

220 + overrides=mm_options.get("image"),

221 )

222 }

223

@@ -229,7 +225,7 @@ class PixtralDummyInputsBuilder(BaseDummyInputsBuilder[PixtralProcessingInfo]):

225 self,

226 seq_len: int,

227 mm_counts: Mapping[str, int],

232 - mm_options: Mapping[str, BaseDummyOptions],

228 + mm_options: MultiModalDummyOptions,

229 mm_data: MultiModalDataDict | None = None,

230 ) -> ProcessorInputs:

231 tokenizer = self.info.get_tokenizer()

vllm/model_executor/models/qwen2_5_omni_thinker.py

+8 -16

@@ -44,7 +44,7 @@ from transformers.models.qwen2_5_omni.processing_qwen2_5_omni import (

44 from transformers.models.whisper import WhisperFeatureExtractor

45

46 from vllm.config import VllmConfig

47 -from vllm.config.multimodal import BaseDummyOptions

47 +from vllm.config.multimodal import MultiModalDummyOptions

48 from vllm.forward_context import set_forward_context

49 from vllm.inputs import ModalityData, MultiModalDataDict

50 from vllm.logger import init_logger

@@ -448,12 +448,8 @@ class Qwen2_5OmniThinkerDummyInputsBuilder(

448 self,

449 seq_len: int,

450 mm_counts: Mapping[str, int],

451 - mm_options: Mapping[str, BaseDummyOptions],

451 + mm_options: MultiModalDummyOptions,

452 ) -> MultiModalDataDict:

453 - num_audios = mm_counts.get("audio", 0)

454 - num_images = mm_counts.get("image", 0)

455 - num_videos = mm_counts.get("video", 0)

456 -

453 feature_extractor = self.info.get_feature_extractor()

454

455 target_audio_length = (

@@ -469,28 +465,24 @@ class Qwen2_5OmniThinkerDummyInputsBuilder(

465 seq_len, mm_counts

466 )

467

472 - image_overrides = mm_options.get("image")

473 - video_overrides = mm_options.get("video")

474 - audio_overrides = mm_options.get("audio")

475 -

468 mm_data = {

469 "audio": self._get_dummy_audios(

470 length=target_audio_length,

479 - num_audios=num_audios,

480 - overrides=audio_overrides,

471 + num_audios=mm_counts.get("audio", 0),

472 + overrides=mm_options.get("audio"),

473 ),

474 "image": self._get_dummy_images(

475 width=target_width,

476 height=target_height,

485 - num_images=num_images,

486 - overrides=image_overrides,

477 + num_images=mm_counts.get("image", 0),

478 + overrides=mm_options.get("image"),

479 ),

480 "video": self._get_dummy_videos(

481 width=target_width,

482 height=target_height,

483 num_frames=target_num_frames,

492 - num_videos=num_videos,

493 - overrides=video_overrides,

484 + num_videos=mm_counts.get("video", 0),

485 + overrides=mm_options.get("video"),

486 ),

487 }

488

vllm/model_executor/models/qwen2_audio.py

+4 -7

@@ -37,7 +37,7 @@ from transformers.models.qwen2_audio import (

37 from transformers.models.whisper import WhisperFeatureExtractor

38

39 from vllm.config import VllmConfig

40 -from vllm.config.multimodal import BaseDummyOptions

40 +from vllm.config.multimodal import MultiModalDummyOptions

41 from vllm.inputs import ModalityData, MultiModalDataDict

42 from vllm.multimodal import MULTIMODAL_REGISTRY

43 from vllm.multimodal.inputs import (

@@ -214,21 +214,18 @@ class Qwen2AudioDummyInputsBuilder(BaseDummyInputsBuilder[Qwen2AudioProcessingIn

214 self,

215 seq_len: int,

216 mm_counts: Mapping[str, int],

217 - mm_options: Mapping[str, BaseDummyOptions],

217 + mm_options: MultiModalDummyOptions,

218 ) -> MultiModalDataDict:

219 feature_extractor = self.info.get_feature_extractor()

220

221 sampling_rate = feature_extractor.sampling_rate

222 audio_len = feature_extractor.chunk_length * sampling_rate

223 - num_audios = mm_counts.get("audio", 0)

224 -

225 - audio_overrides = mm_options.get("audio")

223

224 return {

225 "audio": self._get_dummy_audios(

226 length=audio_len,

230 - num_audios=num_audios,

231 - overrides=audio_overrides,

227 + num_audios=mm_counts.get("audio", 0),

228 + overrides=mm_options.get("audio"),

229 )

230 }

231

vllm/model_executor/models/qwen2_vl.py

+6 -12

@@ -43,7 +43,7 @@ from transformers.models.qwen2_vl.configuration_qwen2_vl import (

43 from transformers.models.qwen2_vl.image_processing_qwen2_vl import smart_resize

44

45 from vllm.config import VllmConfig

46 -from vllm.config.multimodal import BaseDummyOptions

46 +from vllm.config.multimodal import MultiModalDummyOptions

47 from vllm.distributed import parallel_state, tensor_model_parallel_all_gather

48 from vllm.distributed import utils as dist_utils

49 from vllm.inputs import ModalityData, MultiModalDataDict

@@ -1096,32 +1096,26 @@ class Qwen2VLDummyInputsBuilder(BaseDummyInputsBuilder[Qwen2VLProcessingInfo]):

1096 self,

1097 seq_len: int,

1098 mm_counts: Mapping[str, int],

1099 - mm_options: Mapping[str, BaseDummyOptions],

1099 + mm_options: MultiModalDummyOptions,

1100 ) -> MultiModalDataDict:

1101 - num_images = mm_counts.get("image", 0)

1102 - num_videos = mm_counts.get("video", 0)

1103 -

1101 target_width, target_height = self.info.get_image_size_with_most_features()

1102 target_num_frames = self.info.get_num_frames_with_most_features(

1103 seq_len, mm_counts

1104 )

1105

1109 - image_overrides = mm_options.get("image")

1110 - video_overrides = mm_options.get("video")

1111 -

1106 return {

1107 "image": self._get_dummy_images(

1108 width=target_width,

1109 height=target_height,

1116 - num_images=num_images,

1117 - overrides=image_overrides,

1110 + num_images=mm_counts.get("image", 0),

1111 + overrides=mm_options.get("image"),

1112 ),

1113 "video": self._get_dummy_videos(

1114 width=target_width,

1115 height=target_height,

1116 num_frames=target_num_frames,

1123 - num_videos=num_videos,

1124 - overrides=video_overrides,

1117 + num_videos=mm_counts.get("video", 0),

1118 + overrides=mm_options.get("video"),

1119 ),

1120 }

1121

vllm/model_executor/models/qwen3_asr.py

+4 -8

@@ -32,7 +32,7 @@ from transformers.feature_extraction_utils import BatchFeature

32 from transformers.models.whisper import WhisperFeatureExtractor

33

34 from vllm.config import ModelConfig, SpeechToTextConfig, VllmConfig

35 -from vllm.config.multimodal import BaseDummyOptions

35 +from vllm.config.multimodal import MultiModalDummyOptions

36 from vllm.config.speech_to_text import SpeechToTextParams

37 from vllm.inputs import ModalityData, MultiModalDataDict, PromptType, TokensPrompt

38 from vllm.logger import init_logger

@@ -226,10 +226,8 @@ class Qwen3ASRDummyInputsBuilder(BaseDummyInputsBuilder[Qwen3ASRProcessingInfo])

226 self,

227 seq_len: int,

228 mm_counts: Mapping[str, int],

229 - mm_options: Mapping[str, BaseDummyOptions],

229 + mm_options: MultiModalDummyOptions,

230 ) -> MultiModalDataDict:

231 - num_audios = mm_counts.get("audio", 0)

232 -

231 feature_extractor = self.info.get_feature_extractor()

232

233 target_audio_length = (

@@ -240,13 +238,11 @@ class Qwen3ASRDummyInputsBuilder(BaseDummyInputsBuilder[Qwen3ASRProcessingInfo])

238 * feature_extractor.sampling_rate

239 )

240

243 - audio_overrides = mm_options.get("audio")

244 -

241 return {

242 "audio": self._get_dummy_audios(

243 length=target_audio_length,

248 - num_audios=num_audios,

249 - overrides=audio_overrides,

244 + num_audios=mm_counts.get("audio", 0),

245 + overrides=mm_options.get("audio"),

246 ),

247 }

248

vllm/model_executor/models/qwen3_vl.py

+5 -10

@@ -54,8 +54,8 @@ from vllm.compilation.decorators import (

54 )

55 from vllm.config import VllmConfig

56 from vllm.config.multimodal import (

57 - BaseDummyOptions,

57 MultiModalConfig,

58 + MultiModalDummyOptions,

59 VideoDummyOptions,

60 VideoPruningMethod,

61 )

@@ -1119,11 +1119,8 @@ class Qwen3VLDummyInputsBuilder(BaseDummyInputsBuilder[Qwen3VLProcessingInfo]):

1119 self,

1120 seq_len: int,

1121 mm_counts: Mapping[str, int],

1122 - mm_options: Mapping[str, BaseDummyOptions],

1122 + mm_options: MultiModalDummyOptions,

1123 ) -> MultiModalDataDict:

1124 - num_images = mm_counts.get("image", 0)

1125 - num_videos = mm_counts.get("video", 0)

1126 - image_overrides = mm_options.get("image")

1124 video_overrides = mm_options.get("video")

1125

1126 target_image_width, target_image_height = (

@@ -1133,7 +1130,6 @@ class Qwen3VLDummyInputsBuilder(BaseDummyInputsBuilder[Qwen3VLProcessingInfo]):

1130 # treat videos as special images

1131 target_num_frames = 2

1132 if video_overrides:

1136 - assert isinstance(video_overrides, VideoDummyOptions)

1133 num_frames_override = video_overrides.num_frames

1134 if num_frames_override:

1135 if num_frames_override > target_num_frames:

@@ -1204,7 +1200,6 @@ class Qwen3VLDummyInputsBuilder(BaseDummyInputsBuilder[Qwen3VLProcessingInfo]):

1200 target_video_size.height,

1201 )

1202 if video_overrides:

1207 - assert isinstance(video_overrides, VideoDummyOptions)

1203 width_override = video_overrides.width

1204 if width_override:

1205 if width_override > target_video_width:

@@ -1230,14 +1225,14 @@ class Qwen3VLDummyInputsBuilder(BaseDummyInputsBuilder[Qwen3VLProcessingInfo]):

1225 "image": self._get_dummy_images(

1226 width=target_image_width,

1227 height=target_image_height,

1233 - num_images=num_images,

1234 - overrides=image_overrides,

1228 + num_images=mm_counts.get("image", 0),

1229 + overrides=mm_options.get("image"),

1230 ),

1231 "video": self._get_dummy_videos(

1232 width=target_video_width,

1233 height=target_video_height,

1234 num_frames=target_num_frames,

1240 - num_videos=num_videos,

1235 + num_videos=mm_counts.get("video", 0),

1236 ),

1237 }

1238

vllm/model_executor/models/rvl.py

+4 -8

@@ -8,7 +8,7 @@ import torch.nn as nn

8 from transformers.activations import GELUActivation

9

10 from vllm.config import VllmConfig

11 -from vllm.config.multimodal import BaseDummyOptions

11 +from vllm.config.multimodal import MultiModalDummyOptions

12 from vllm.inputs import MultiModalDataDict

13 from vllm.multimodal import MULTIMODAL_REGISTRY

14

@@ -40,20 +40,16 @@ class RVLDummyInputsBuilder(LlavaDummyInputsBuilder[RVLProcessingInfo]):

40 self,

41 seq_len: int,

42 mm_counts: Mapping[str, int],

43 - mm_options: Mapping[str, BaseDummyOptions],

43 + mm_options: MultiModalDummyOptions,

44 ) -> MultiModalDataDict:

45 - num_images = mm_counts.get("image", 0)

46 -

45 target_width, target_height = self.info.get_image_size_with_most_features()

46

49 - image_overrides = mm_options.get("image")

50 -

47 return {

48 "image": self._get_dummy_images(

49 width=target_width,

50 height=target_height,

55 - num_images=num_images,

56 - overrides=image_overrides,

51 + num_images=mm_counts.get("image", 0),

52 + overrides=mm_options.get("image"),

53 ),

54 }

55

vllm/model_executor/models/siglip.py

+4 -8

@@ -16,7 +16,7 @@ from transformers import (

16 )

17

18 from vllm.config import VllmConfig

19 -from vllm.config.multimodal import BaseDummyOptions

19 +from vllm.config.multimodal import MultiModalDummyOptions

20 from vllm.distributed import divide, get_tensor_model_parallel_world_size

21 from vllm.inputs import MultiModalDataDict, MultiModalInput

22 from vllm.model_executor.layers.activation import get_act_fn

@@ -158,20 +158,16 @@ class SiglipDummyInputsBuilder(BaseDummyInputsBuilder[SiglipProcessingInfo]):

158 self,

159 seq_len: int,

160 mm_counts: Mapping[str, int],

161 - mm_options: Mapping[str, BaseDummyOptions],

161 + mm_options: MultiModalDummyOptions,

162 ) -> MultiModalDataDict:

163 - num_images = mm_counts.get("image", 0)

164 -

163 target_width, target_height = self.info.get_image_size_with_most_features()

164

167 - image_overrides = mm_options.get("image")

168 -

165 return {

166 "image": self._get_dummy_images(

167 width=target_width,

168 height=target_height,

173 - num_images=num_images,

174 - overrides=image_overrides,

169 + num_images=mm_counts.get("image", 0),

170 + overrides=mm_options.get("image"),

171 )

172 }

173

vllm/model_executor/models/step3_vl.py

+4 -7

@@ -11,7 +11,7 @@ import torch.nn.functional as F

11 from transformers import BatchFeature

12

13 from vllm.config import VllmConfig

14 -from vllm.config.multimodal import BaseDummyOptions

14 +from vllm.config.multimodal import MultiModalDummyOptions

15 from vllm.distributed import get_tensor_model_parallel_world_size

16 from vllm.inputs import MultiModalDataDict

17 from vllm.model_executor.layers.activation import get_act_fn

@@ -139,19 +139,16 @@ class Step3VLDummyInputsBuilder(BaseDummyInputsBuilder[Step3VLProcessingInfo]):

139 self,

140 seq_len: int,

141 mm_counts: Mapping[str, int],

142 - mm_options: Mapping[str, BaseDummyOptions],

142 + mm_options: MultiModalDummyOptions,

143 ) -> MultiModalDataDict:

144 target_width, target_height = self.info.get_image_size_with_most_features()

145 - num_images = mm_counts.get("image", 0)

146 -

147 - image_overrides = mm_options.get("image")

145

146 return {

147 "image": self._get_dummy_images(

148 width=target_width,

149 height=target_height,

153 - num_images=num_images,

154 - overrides=image_overrides,

150 + num_images=mm_counts.get("image", 0),

151 + overrides=mm_options.get("image"),

152 )

153 }

154

vllm/model_executor/models/terratorch.py

+2 -2

@@ -33,7 +33,7 @@ from terratorch.vllm import (

33 from transformers import BatchFeature

34

35 from vllm.config import VllmConfig

36 -from vllm.config.multimodal import BaseDummyOptions

36 +from vllm.config.multimodal import MultiModalDummyOptions

37 from vllm.inputs import ModalityData, MultiModalDataDict, MultiModalInput, mm_input

38 from vllm.logger import init_logger

39 from vllm.model_executor.layers.pooler import IdentityPooler

@@ -153,7 +153,7 @@ class TerratorchInputBuilder(BaseDummyInputsBuilder[TerratorchProcessingInfo]):

153 self,

154 seq_len: int,

155 mm_counts: Mapping[str, int],

156 - mm_options: Mapping[str, BaseDummyOptions],

156 + mm_options: MultiModalDummyOptions,

157 ) -> MultiModalDataDict:

158 # Dummy data is generated based on the 'input' section

159 # defined in the HF configuration file

vllm/model_executor/models/transformers/multimodal.py

+7 -13

@@ -27,7 +27,6 @@ from packaging.version import Version

27 from transformers.utils.generic import ModelOutput

28

29 from vllm.compilation.decorators import should_torch_compile_mm_encoder

30 -from vllm.config.multimodal import AudioDummyOptions

30 from vllm.config.utils import getattr_iter

31 from vllm.inputs import (

32 MultiModalDataBuiltins,

@@ -75,7 +74,7 @@ if TYPE_CHECKING:

74 from transformers import BatchFeature, PreTrainedModel

75

76 from vllm.config import VllmConfig

78 - from vllm.config.multimodal import BaseDummyOptions

77 + from vllm.config.multimodal import MultiModalDummyOptions

78

79 logger = init_logger(__name__)

80

@@ -195,7 +194,7 @@ class MultiModalDummyInputsBuilder(BaseDummyInputsBuilder[MultiModalProcessingIn

194 self,

195 seq_len: int,

196 mm_counts: Mapping[str, int],

198 - mm_options: Mapping[str, "BaseDummyOptions"],

197 + mm_options: "MultiModalDummyOptions",

198 ) -> MultiModalDataDict:

199 data = MultiModalDataBuiltins()

200 if self.info._is_audio_model() and (num_audios := mm_counts.get("audio", 0)):

@@ -204,21 +203,16 @@ class MultiModalDummyInputsBuilder(BaseDummyInputsBuilder[MultiModalProcessingIn

203 chunk_length = getattr(sub, "chunk_length", None) if sub else None

204 if chunk_length is None:

205 chunk_length = 30

207 - audio_len = int(chunk_length * sampling_rate)

208 - audio_overrides = mm_options.get("audio")

209 - assert audio_overrides is None or isinstance(

210 - audio_overrides, AudioDummyOptions

211 - )

206 data["audio"] = self._get_dummy_audios(

213 - length=audio_len,

207 + length=int(chunk_length * sampling_rate),

208 num_audios=num_audios,

215 - overrides=audio_overrides,

209 + overrides=mm_options.get("audio"),

210 )

211 if self.info._is_image_model() and (num_images := mm_counts.get("image", 0)):

218 - target_width, target_height = self.info.get_image_size_with_most_features()

212 + width, height = self.info.get_image_size_with_most_features()

213 data["image"] = self._get_dummy_images(

220 - width=target_width,

221 - height=target_height,

214 + width=width,

215 + height=height,

216 num_images=num_images,

217 overrides=mm_options.get("image"),

218 )

vllm/model_executor/models/ultravox.py

+4 -7

@@ -16,7 +16,7 @@ from transformers import BatchFeature, ProcessorMixin

16 from transformers.models.whisper import WhisperFeatureExtractor

17

18 from vllm.config import VllmConfig

19 -from vllm.config.multimodal import BaseDummyOptions

19 +from vllm.config.multimodal import MultiModalDummyOptions

20 from vllm.distributed import get_tensor_model_parallel_world_size

21 from vllm.inputs import MultiModalDataDict

22 from vllm.model_executor.layers.activation import MulAndSilu, get_act_fn

@@ -169,7 +169,7 @@ class UltravoxDummyInputsBuilder(BaseDummyInputsBuilder[UltravoxProcessingInfo])

169 self,

170 seq_len: int,

171 mm_counts: Mapping[str, int],

172 - mm_options: Mapping[str, BaseDummyOptions],

172 + mm_options: MultiModalDummyOptions,

173 ) -> MultiModalDataDict:

174 feature_extractor = self.info.get_feature_extractor()

175

@@ -177,15 +177,12 @@ class UltravoxDummyInputsBuilder(BaseDummyInputsBuilder[UltravoxProcessingInfo])

177 audio_len = (

178 feature_extractor.chunk_length * sampling_rate * _MAX_ENCODER_BATCH_SIZE

179 )

180 - num_audios = mm_counts.get("audio", 0)

181 -

182 - audio_overrides = mm_options.get("audio")

180

181 return {

182 "audio": self._get_dummy_audios(

183 length=audio_len,

187 - num_audios=num_audios,

188 - overrides=audio_overrides,

184 + num_audios=mm_counts.get("audio", 0),

185 + overrides=mm_options.get("audio"),

186 )

187 }

188

vllm/model_executor/models/voxtral.py

+5 -9

@@ -19,7 +19,7 @@ from mistral_common.tokens.tokenizers.audio import Audio

19 from transformers import WhisperConfig

20

21 from vllm.config import ModelConfig, SpeechToTextConfig, VllmConfig

22 -from vllm.config.multimodal import BaseDummyOptions

22 +from vllm.config.multimodal import MultiModalDummyOptions

23 from vllm.config.speech_to_text import SpeechToTextParams

24 from vllm.inputs import MultiModalDataDict, PromptType, TokensPrompt

25 from vllm.logger import init_logger

@@ -137,19 +137,15 @@ class VoxtralDummyInputsBuilder(BaseDummyInputsBuilder[VoxtralProcessingInfo]):

137 self,

138 seq_len: int,

139 mm_counts: Mapping[str, int],

140 - mm_options: Mapping[str, BaseDummyOptions],

140 + mm_options: MultiModalDummyOptions,

141 ) -> MultiModalDataDict:

142 - num_audios = mm_counts.get("audio", 0)

143 -

142 target_length = self.info.get_max_audio_array_len()

143

146 - audio_overrides = mm_options.get("audio")

147 -

144 return {

145 "audio": self._get_dummy_audios(

146 length=target_length,

151 - num_audios=num_audios,

152 - overrides=audio_overrides,

147 + num_audios=mm_counts.get("audio", 0),

148 + overrides=mm_options.get("audio"),

149 )

150 }

151

@@ -157,7 +153,7 @@ class VoxtralDummyInputsBuilder(BaseDummyInputsBuilder[VoxtralProcessingInfo]):

153 self,

154 seq_len: int,

155 mm_counts: Mapping[str, int],

160 - mm_options: Mapping[str, BaseDummyOptions],

156 + mm_options: MultiModalDummyOptions,

157 mm_data: MultiModalDataDict | None = None,

158 ) -> ProcessorInputs:

159 tokenizer = self.info.get_tokenizer()

vllm/model_executor/models/whisper.py

+4 -7

@@ -19,7 +19,7 @@ from transformers.models.whisper.modeling_whisper import sinusoids

19

20 from vllm.compilation.decorators import support_torch_compile

21 from vllm.config import CacheConfig, ModelConfig, SpeechToTextConfig, VllmConfig

22 -from vllm.config.multimodal import BaseDummyOptions

22 +from vllm.config.multimodal import MultiModalDummyOptions

23 from vllm.config.speech_to_text import SpeechToTextParams

24 from vllm.distributed import get_tensor_model_parallel_world_size

25 from vllm.inputs import (

@@ -724,21 +724,18 @@ class WhisperDummyInputsBuilder(BaseDummyInputsBuilder[WhisperProcessingInfo]):

724 self,

725 seq_len: int,

726 mm_counts: Mapping[str, int],

727 - mm_options: Mapping[str, BaseDummyOptions],

727 + mm_options: MultiModalDummyOptions,

728 ) -> MultiModalDataDict:

729 feature_extractor = self.info.get_feature_extractor()

730

731 sampling_rate = feature_extractor.sampling_rate

732 audio_len = feature_extractor.chunk_length * sampling_rate

733 - num_audios = mm_counts.get("audio", 0)

734 -

735 - audio_overrides = mm_options.get("audio")

733

734 return {

735 "audio": self._get_dummy_audios(

736 length=audio_len,

740 - num_audios=num_audios,

741 - overrides=audio_overrides,

737 + num_audios=mm_counts.get("audio", 0),

738 + overrides=mm_options.get("audio"),

739 )

740 }

741

vllm/models/deepseek_v4/common/mm_preprocess.py

+4 -4

@@ -22,7 +22,7 @@ reference's out-of-vocab scheme.

22

23 import math

24 from collections.abc import Mapping, Sequence

25 -from typing import Any, cast

25 +from typing import Any

26

27 import numpy as np

28 import torch

@@ -30,7 +30,7 @@ from PIL import Image, ImageOps

30 from transformers import BatchFeature

31 from typing_extensions import assert_never

32

33 -from vllm.config.multimodal import BaseDummyOptions, ImageDummyOptions

33 +from vllm.config.multimodal import MultiModalDummyOptions

34 from vllm.inputs import MultiModalDataDict

35 from vllm.multimodal.inputs import MultiModalFieldConfig, MultiModalKwargsItems

36 from vllm.multimodal.parse import ImageSize, MultiModalDataItems

@@ -375,7 +375,7 @@ class DeepseekV4VLDummyInputsBuilder(

375 self,

376 seq_len: int,

377 mm_counts: Mapping[str, int],

378 - mm_options: Mapping[str, BaseDummyOptions],

378 + mm_options: MultiModalDummyOptions,

379 ) -> MultiModalDataDict:

380 size = self.info.get_image_size_with_most_features()

381 return {

@@ -383,7 +383,7 @@ class DeepseekV4VLDummyInputsBuilder(

383 width=size.width,

384 height=size.height,

385 num_images=mm_counts.get("image", 0),

386 - overrides=cast(ImageDummyOptions | None, mm_options.get("image")),

386 + overrides=mm_options.get("image"),

387 ),

388 }

389

vllm/models/deepseek_v41/common/mm_preprocess.py

+4 -4

@@ -19,14 +19,14 @@ boundaries freely).

19

20 import math

21 from collections.abc import Mapping, Sequence

22 -from typing import Any, cast

22 +from typing import Any

23

24 import numpy as np

25 import torch

26 from PIL import Image, ImageOps

27 from transformers import BatchFeature

28

29 -from vllm.config.multimodal import BaseDummyOptions, ImageDummyOptions

29 +from vllm.config.multimodal import MultiModalDummyOptions

30 from vllm.inputs import MultiModalDataDict

31 from vllm.multimodal.inputs import MultiModalFieldConfig, MultiModalKwargsItems

32 from vllm.multimodal.parse import ImageSize, MultiModalDataItems

@@ -296,7 +296,7 @@ class DeepseekV4VLDummyInputsBuilder(

296 self,

297 seq_len: int,

298 mm_counts: Mapping[str, int],

299 - mm_options: Mapping[str, BaseDummyOptions],

299 + mm_options: MultiModalDummyOptions,

300 ) -> MultiModalDataDict:

301 size = self.info.get_image_size_with_most_features()

302 return {

@@ -304,7 +304,7 @@ class DeepseekV4VLDummyInputsBuilder(

304 width=size.width,

305 height=size.height,

306 num_images=mm_counts.get("image", 0),

307 - overrides=cast(ImageDummyOptions | None, mm_options.get("image")),

307 + overrides=mm_options.get("image"),

308 ),

309 }

310

vllm/models/dots3_note/common/processor.py

+5 -8

@@ -13,10 +13,7 @@ from PIL import Image

13 from transformers import BatchFeature

14

15 from vllm.config.multimodal import (

16 - AudioDummyOptions,

17 - BaseDummyOptions,

18 - ImageDummyOptions,

19 - VideoDummyOptions,

16 + MultiModalDummyOptions,

17 )

18 from vllm.inputs import MultiModalDataDict

19 from vllm.multimodal.inputs import MultiModalFieldConfig, MultiModalKwargsItems

@@ -584,7 +581,7 @@ class Dots3NoteDummyInputsBuilder(BaseDummyInputsBuilder[Dots3NoteProcessingInfo

581 self,

582 seq_len: int,

583 mm_counts: Mapping[str, int],

587 - mm_options: Mapping[str, BaseDummyOptions],

584 + mm_options: MultiModalDummyOptions,

585 ) -> MultiModalDataDict:

586 data: dict[str, Any] = {}

587 num_images = mm_counts.get("image", 0)

@@ -594,7 +591,7 @@ class Dots3NoteDummyInputsBuilder(BaseDummyInputsBuilder[Dots3NoteProcessingInfo

591 width=width,

592 height=height,

593 num_images=num_images,

597 - overrides=cast(ImageDummyOptions | None, mm_options.get("image")),

594 + overrides=mm_options.get("image"),

595 )

596 num_audios = mm_counts.get("audio", 0)

597 if num_audios:

@@ -611,7 +608,7 @@ class Dots3NoteDummyInputsBuilder(BaseDummyInputsBuilder[Dots3NoteProcessingInfo

608 data["audio"] = self._get_dummy_audios(

609 length=min(chunk_samples, max(1, seq_len) * stride),

610 num_audios=num_audios,

614 - overrides=cast(AudioDummyOptions | None, mm_options.get("audio")),

611 + overrides=mm_options.get("audio"),

612 )

613 num_videos = mm_counts.get("video", 0)

614 if num_videos:

@@ -625,7 +622,7 @@ class Dots3NoteDummyInputsBuilder(BaseDummyInputsBuilder[Dots3NoteProcessingInfo

622 height=height,

623 num_frames=num_frames,

624 num_videos=num_videos,

628 - overrides=cast(VideoDummyOptions | None, mm_options.get("video")),

625 + overrides=mm_options.get("video"),

626 )

627 return data

628

vllm/models/inkling/common/mm_preprocess.py

+4 -8

@@ -13,9 +13,7 @@ import torch

13 from transformers.feature_extraction_utils import BatchFeature

14

15 from vllm.config.multimodal import (

16 - AudioDummyOptions,

17 - BaseDummyOptions,

18 - ImageDummyOptions,

16 + MultiModalDummyOptions,

17 )

18 from vllm.inputs import MultiModalDataDict

19 from vllm.multimodal.inputs import (

@@ -145,7 +143,7 @@ class InklingDummyInputsBuilder(BaseDummyInputsBuilder[InklingProcessingInfo]):

143 self,

144 seq_len: int,

145 mm_counts: Mapping[str, int],

148 - mm_options: Mapping[str, BaseDummyOptions],

146 + mm_options: MultiModalDummyOptions,

147 ) -> MultiModalDataDict:

148 config = self.info.get_hf_config()

149 num_images = mm_counts.get("image", 0)

@@ -156,12 +154,11 @@ class InklingDummyInputsBuilder(BaseDummyInputsBuilder[InklingProcessingInfo]):

154 patch_size = getattr(config.vision_config, "patch_size", 40)

155 # A square image ~4 patches wide so the dummy emits several patches.

156 side = patch_size * 4

159 - image_overrides = mm_options.get("image")

157 mm_data["image"] = self._get_dummy_images(

158 width=side,

159 height=side,

160 num_images=num_images,

164 - overrides=cast(ImageDummyOptions | None, image_overrides),

161 + overrides=mm_options.get("image"),

162 )

163 if num_audios:

164 # Size the dummy at the maximum allowed audio so memory/encoder

@@ -169,11 +166,10 @@ class InklingDummyInputsBuilder(BaseDummyInputsBuilder[InklingProcessingInfo]):

166 params = self.info.get_hf_processor().audio_feature_extractor.params

167 hop = int(round(params.audio_token_duration_s * params.sample_rate))

168 audio_len = MAX_AUDIO_TOKENS * hop

172 - audio_overrides = mm_options.get("audio")

169 mm_data["audio"] = self._get_dummy_audios(

170 length=audio_len,

171 num_audios=num_audios,

176 - overrides=cast(AudioDummyOptions | None, audio_overrides),

172 + overrides=mm_options.get("audio"),

173 )

174 return mm_data

175

vllm/models/kimi_k3/common/mm_preprocess.py

+3 -4

@@ -9,7 +9,7 @@ from typing import Any, cast

9 import torch

10 from transformers import BatchFeature

11

12 -from vllm.config.multimodal import BaseDummyOptions, ImageDummyOptions

12 +from vllm.config.multimodal import ImageDummyOptions, MultiModalDummyOptions

13 from vllm.inputs import MultiModalDataDict

14 from vllm.logger import init_logger

15 from vllm.multimodal.inputs import (

@@ -204,7 +204,7 @@ class KimiK3DummyInputsBuilder(BaseDummyInputsBuilder[KimiK3ProcessingInfo]):

204 self,

205 seq_len: int,

206 mm_counts: Mapping[str, int],

207 - mm_options: Mapping[str, BaseDummyOptions] | None = None,

207 + mm_options: MultiModalDummyOptions | None = None,

208 ) -> MultiModalDataDict:

209 media_proc_cfg = self.info.image_processor.media_proc_cfg

210 max_size = self.info.get_max_image_size(

@@ -214,7 +214,6 @@ class KimiK3DummyInputsBuilder(BaseDummyInputsBuilder[KimiK3ProcessingInfo]):

214 media_proc_cfg["patch_limit_on_one_side"],

215 media_proc_cfg["fixed_output_tokens"],

216 )

217 - num_images = mm_counts.get("image", 0)

217 image_overrides = cast(

218 ImageDummyOptions | None,

219 mm_options.get("image") if mm_options else None,

@@ -223,7 +222,7 @@ class KimiK3DummyInputsBuilder(BaseDummyInputsBuilder[KimiK3ProcessingInfo]):

222 "image": self._get_dummy_images(

223 width=max_size.width,

224 height=max_size.height,

226 - num_images=num_images,

225 + num_images=mm_counts.get("image", 0),

226 overrides=image_overrides,

227 )

228 }

vllm/models/minimax_m3/common/mm_preprocess.py

+4 -6

@@ -11,9 +11,7 @@ from transformers import BatchFeature

11 from transformers.video_utils import VideoMetadata

12

13 from vllm.config.multimodal import (

14 - BaseDummyOptions,

15 - ImageDummyOptions,

16 - VideoDummyOptions,

14 + MultiModalDummyOptions,

15 )

16 from vllm.inputs import MultiModalDataDict

17 from vllm.multimodal.inputs import (

@@ -283,7 +281,7 @@ class MiniMaxM3VLDummyInputsBuilder(BaseDummyInputsBuilder[MiniMaxM3VLProcessing

281 self,

282 seq_len: int,

283 mm_counts: Mapping[str, int],

286 - mm_options: Mapping[str, BaseDummyOptions],

284 + mm_options: MultiModalDummyOptions,

285 ) -> MultiModalDataDict:

286 size = self.info.get_image_size_with_most_features()

287 video_size = self.info.get_video_size_with_most_features()

@@ -293,14 +291,14 @@ class MiniMaxM3VLDummyInputsBuilder(BaseDummyInputsBuilder[MiniMaxM3VLProcessing

291 width=size.width,

292 height=size.height,

293 num_images=mm_counts.get("image", 0),

296 - overrides=cast(ImageDummyOptions | None, mm_options.get("image")),

294 + overrides=mm_options.get("image"),

295 ),

296 "video": self._get_dummy_videos(

297 width=video_size.width,

298 height=video_size.height,

299 num_frames=num_frames,

300 num_videos=mm_counts.get("video", 0),

303 - overrides=cast(VideoDummyOptions | None, mm_options.get("video")),

301 + overrides=mm_options.get("video"),

302 ),

303 }

304

vllm/multimodal/processing/dummy_inputs.py

+4 -5

@@ -10,8 +10,8 @@ from PIL import Image

10

11 from vllm.config.multimodal import (

12 AudioDummyOptions,

13 - BaseDummyOptions,

13 ImageDummyOptions,

14 + MultiModalDummyOptions,

15 VideoDummyOptions,

16 )

17 from vllm.inputs import MultiModalDataDict

@@ -45,7 +45,7 @@ class BaseDummyInputsBuilder(ABC, Generic[_I]):

45 self,

46 seq_len: int,

47 mm_counts: Mapping[str, int],

48 - mm_options: Mapping[str, BaseDummyOptions],

48 + mm_options: MultiModalDummyOptions,

49 ) -> MultiModalDataDict:

50 """Build the multimodal input which, after processing, results in

51 the maximum possible number of placeholder tokens.

@@ -65,7 +65,7 @@ class BaseDummyInputsBuilder(ABC, Generic[_I]):

65 self,

66 seq_len: int,

67 mm_counts: Mapping[str, int],

68 - mm_options: Mapping[str, BaseDummyOptions],

68 + mm_options: MultiModalDummyOptions,

69 ) -> ProcessorInputs:

70 """Build the input which, after processing, results in

71 the maximum possible number of placeholder tokens.

@@ -124,9 +124,8 @@ class BaseDummyInputsBuilder(ABC, Generic[_I]):

124 width: int,

125 height: int,

126 num_images: int,

127 - overrides: BaseDummyOptions | None = None,

127 + overrides: ImageDummyOptions | None = None,

128 ) -> list[Image.Image]:

129 - assert overrides is None or isinstance(overrides, ImageDummyOptions)

129 if num_images == 0:

130 return []

131 if overrides:

[Multimodal] Type dummy options per modality (#57576)