79aa244678 · tduncan/Nvidia-Optimization-Stack-VLLM

docs/contributing/model/multimodal.md

+10 -2

@@ -258,17 +258,21 @@ Assuming that the memory usage increases with the number of tokens, the dummy in

258 self,

259 seq_len: int,

260 mm_counts: Mapping[str, int],

261 + mm_options: Optional[Mapping[str, BaseDummyOptions]] = None,

262 ) -> MultiModalDataDict:

263 num_images = mm_counts.get("image", 0)

264

265 target_width, target_height = \

266 self.info.get_image_size_with_most_features()

267

268 + image_overrides = mm_options.get("image") if mm_options else None

269 +

270 return {

271 "image":

272 self._get_dummy_images(width=target_width,

273 height=target_height,

271 - num_images=num_images)

274 + num_images=num_images,

275 + overrides=image_overrides)

276 }

277 ```

278

@@ -438,16 +442,20 @@ Assuming that the memory usage increases with the number of tokens, the dummy in

442 self,

443 seq_len: int,

444 mm_counts: Mapping[str, int],

445 + mm_options: Optional[Mapping[str, BaseDummyOptions]] = None,

446 ) -> MultiModalDataDict:

447 target_width, target_height = \

448 self.info.get_image_size_with_most_features()

449 num_images = mm_counts.get("image", 0)

450

451 + image_overrides = mm_options.get("image") if mm_options else None

452 +

453 return {

454 "image":

455 self._get_dummy_images(width=target_width,

456 height=target_height,

450 - num_images=num_images)

457 + num_images=num_images,

458 + overrides=image_overrides)

459 }

460 ```

461

tests/models/multimodal/processing/test_common.py

+19 -3

@@ -12,6 +12,8 @@ from mistral_common.protocol.instruct.request import ChatCompletionRequest

12 from PIL import Image

13

14 from vllm.config import ModelConfig

15 +from vllm.config.multimodal import (AudioDummyOptions, BaseDummyOptions,

16 + ImageDummyOptions, VideoDummyOptions)

17 from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalDataDict

18 from vllm.multimodal.cache import MultiModalProcessorOnlyCache

19 from vllm.multimodal.inputs import MultiModalInputs

@@ -112,12 +114,26 @@ def _test_processing_correctness(

114

115 processing_info = factories.info(ctx)

116 supported_mm_limits = processing_info.get_supported_mm_limits()

115 - limit_mm_per_prompt = {

117 + # Keep integer limits for local data generation

118 + limit_mm_per_prompt_ints = {

119 modality: 3 if limit is None else limit

120 for modality, limit in supported_mm_limits.items()

121 }

122

120 - model_config.get_multimodal_config().limit_per_prompt = limit_mm_per_prompt

123 + def _to_dummy_options(modality: str, count: int) -> BaseDummyOptions:

124 + if modality == "video":

125 + return VideoDummyOptions(count=count)

126 + if modality == "image":

127 + return ImageDummyOptions(count=count)

128 + if modality == "audio":

129 + return AudioDummyOptions(count=count)

130 + return BaseDummyOptions(count=count)

131 +

132 + # Assign normalized DummyOptions to the model config

133 + model_config.get_multimodal_config().limit_per_prompt = {

134 + modality: _to_dummy_options(modality, count)

135 + for modality, count in limit_mm_per_prompt_ints.items()

136 + }

137

138 baseline_processor = factories.build_processor(ctx, cache=None)

139 cached_processor = factories.build_processor(ctx, cache=cache)

@@ -150,7 +166,7 @@ def _test_processing_correctness(

166 k:

167 [(input_to_hit[k] if rng.rand() < hit_rate else input_factory[k]())

168 for _ in range(rng.randint(limit + 1))]

153 - for k, limit in limit_mm_per_prompt.items()

169 + for k, limit in limit_mm_per_prompt_ints.items()

170 }

171

172 mm_counts = {k: len(vs) for k, vs in mm_data.items()}

tests/models/multimodal/processing/test_mllama4.py

+5 -5

@@ -17,23 +17,23 @@ def test_profiling(model_id: str, max_model_len: int):

17 model_config_kwargs = {

18 "max_model_len": max_model_len,

19 }

20 + mm_counts = {"image": 1}

21 ctx = build_model_context(

22 model_id,

23 model_config_kwargs=model_config_kwargs,

23 - limit_mm_per_prompt={"image": 1},

24 + limit_mm_per_prompt=mm_counts,

25 )

26

26 - mm_config = ctx.get_mm_config()

27 processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)

28 profiler = MultiModalProfiler(processor)

29

30 decoder_dummy_data = profiler.get_decoder_dummy_data(

31 max_model_len,

32 - mm_counts=mm_config.limit_per_prompt,

32 + mm_counts=mm_counts,

33 )

34 dummy_mm_data = processor.dummy_inputs.get_dummy_processor_inputs(

35 max_model_len,

36 - mm_counts=mm_config.limit_per_prompt,

36 + mm_counts=mm_counts,

37 )

38

39 hf_config = ctx.get_hf_config(Llama4Config)

@@ -58,7 +58,7 @@ def test_profiling(model_id: str, max_model_len: int):

58

59 profiled_tokens = profiler.get_mm_max_contiguous_tokens(

60 max_model_len,

61 - mm_counts=mm_config.limit_per_prompt,

61 + mm_counts=mm_counts,

62 )

63

64 assert total_tokens == profiled_tokens["image"]

tests/models/multimodal/processing/test_tensor_schema.py

+16 -1

@@ -15,6 +15,8 @@ from mistral_common.protocol.instruct.request import ChatCompletionRequest

15 from PIL import Image

16

17 from vllm.config import ModelConfig, VllmConfig, set_current_vllm_config

18 +from vllm.config.multimodal import (AudioDummyOptions, BaseDummyOptions,

19 + ImageDummyOptions, VideoDummyOptions)

20 from vllm.distributed import (cleanup_dist_env_and_memory,

21 init_distributed_environment,

22 initialize_model_parallel)

@@ -236,7 +238,20 @@ def test_model_tensor_schema(model_arch: str, model_id: str):

238 modality: 3 if limit is None else limit

239 for modality, limit in supported_mm_limits.items()

240 }

239 - model_config.get_multimodal_config().limit_per_prompt = limit_mm_per_prompt

241 +

242 + def _to_dummy_options(modality: str, count: int) -> BaseDummyOptions:

243 + if modality == "video":

244 + return VideoDummyOptions(count=count)

245 + if modality == "image":

246 + return ImageDummyOptions(count=count)

247 + if modality == "audio":

248 + return AudioDummyOptions(count=count)

249 + return BaseDummyOptions(count=count)

250 +

251 + model_config.get_multimodal_config().limit_per_prompt = {

252 + modality: _to_dummy_options(modality, count)

253 + for modality, count in limit_mm_per_prompt.items()

254 + }

255 processor = factories.build_processor(ctx, cache=None)

256

257 with initialize_dummy_model(model_cls, model_config) as model:

vllm/config/model.py

+3 -1

@@ -276,7 +276,9 @@ class ModelConfig:

276 multimodal_config: Optional[MultiModalConfig] = None

277 """Configuration for multimodal model. If `None`, this will be inferred

278 from the architecture of `self.model`."""

279 - limit_mm_per_prompt: InitVar[Optional[dict[str, int]]] = None

279 + limit_mm_per_prompt: InitVar[Optional[dict[str, Union[int,

280 + dict[str,

281 + int]]]]] = None

282 media_io_kwargs: InitVar[Optional[dict[str, dict[str, Any]]]] = None

283 mm_processor_kwargs: InitVar[Optional[dict[str, Any]]] = None

284 mm_processor_cache_gb: InitVar[Optional[float]] = None

vllm/config/multimodal.py

+83 -12

@@ -4,15 +4,45 @@

4 import hashlib

5 from collections.abc import Mapping

6 from dataclasses import field

7 -from typing import Any, Literal, Optional

7 +from typing import Any, Literal, Optional, Union

8

9 +from pydantic import ConfigDict, Field, field_validator

10 from pydantic.dataclasses import dataclass

11

11 -import vllm.envs as envs

12 from vllm.config.utils import config

13

14 +

15 +@dataclass

16 +class BaseDummyOptions:

17 + """Base options for generating dummy data during profiling."""

18 + count: int = Field(999, ge=0)

19 +

20 +

21 +@dataclass(config=ConfigDict(extra="forbid"))

22 +class VideoDummyOptions(BaseDummyOptions):

23 + """Options for generating dummy video data during profiling."""

24 + num_frames: Optional[int] = Field(None, gt=0)

25 + width: Optional[int] = Field(None, gt=0)

26 + height: Optional[int] = Field(None, gt=0)

27 +

28 +

29 +@dataclass(config=ConfigDict(extra="forbid"))

30 +class ImageDummyOptions(BaseDummyOptions):

31 + """Options for generating dummy image data during profiling."""

32 + width: Optional[int] = Field(None, gt=0)

33 + height: Optional[int] = Field(None, gt=0)

34 +

35 +

36 +@dataclass(config=ConfigDict(extra="forbid"))

37 +class AudioDummyOptions(BaseDummyOptions):

38 + """Options for generating dummy audio data during profiling."""

39 + length: Optional[int] = Field(None, gt=0)

40 +

41 +

42 MMEncoderTPMode = Literal["weights", "data"]

43 MMCacheType = Literal["shm", "lru"]

44 +DummyOptions = Union[BaseDummyOptions, VideoDummyOptions, ImageDummyOptions,

45 + AudioDummyOptions]

46

47

48 @config

@@ -20,12 +50,22 @@ MMCacheType = Literal["shm", "lru"]

50 class MultiModalConfig:

51 """Controls the behavior of multimodal models."""

52

23 - limit_per_prompt: dict[str, int] = field(default_factory=dict)

24 - """The maximum number of input items allowed per prompt for each modality.

25 - Defaults to 1 (V0) or 999 (V1) for each modality.

53 + limit_per_prompt: dict[str, DummyOptions] = field(default_factory=dict)

54 + """The maximum number of input items and options allowed per

55 + prompt for each modality.

56 + Defaults to 999 for each modality.

57 +

58 + Legacy format (count only):

59 + {"image": 16, "video": 2}

60 +

61 + Configurable format (with options):

62 + {"video": {"count": 1, "num_frames": 32, "width": 512, "height": 512},

63 + "image": {"count": 5, "width": 512, "height": 512}}

64

27 - For example, to allow up to 16 images and 2 videos per prompt:

28 - `{"image": 16, "video": 2}`"""

65 + Mixed format (combining both):

66 + {"image": 16, "video": {"count": 1, "num_frames": 32, "width": 512,

67 + "height": 512}}

68 + """

69 media_io_kwargs: dict[str, dict[str, Any]] = field(default_factory=dict)

70 """Additional args passed to process media inputs, keyed by modalities.

71 For example, to set num_frames for video, set

@@ -84,6 +124,27 @@ class MultiModalConfig:

124 from each video to be pruned.

125 """

126

127 + @field_validator("limit_per_prompt", mode="before")

128 + @classmethod

129 + def _validate_limit_per_prompt(

130 + cls, value: dict[str, Union[int,

131 + dict[str,

132 + int]]]) -> dict[str, DummyOptions]:

133 + for k, v in value.items():

134 + # Handle legacy format where only count is specified

135 + if isinstance(v, int):

136 + v = {"count": v}

137 + # Convert to the appropriate DummyOptions subclass

138 + if k == "video":

139 + value[k] = VideoDummyOptions(**v)

140 + elif k == "image":

141 + value[k] = ImageDummyOptions(**v)

142 + elif k == "audio":

143 + value[k] = AudioDummyOptions(**v)

144 + else:

145 + value[k] = BaseDummyOptions(**v)

146 + return value

147 +

148 def compute_hash(self) -> str:

149 """

150 WARNING: Whenever a new field is added to this config,

@@ -106,12 +167,22 @@ class MultiModalConfig:

167 def get_limit_per_prompt(self, modality: str) -> int:

168 """

169 Get the maximum number of input items allowed per prompt

109 - for the given modality.

170 + for the given modality (backward compatible).

171 + """

172 + limit_data = self.limit_per_prompt.get(modality)

173 +

174 + if limit_data is None:

175 + # Unspecified modality is set to 999 by default

176 + return 999

177 + return limit_data.count

178 +

179 + def get_dummy_options(self, modality: str) -> Optional[BaseDummyOptions]:

180 + """

181 + Get the configurable dummy data options for a modality.

182 + Returns None if no options are configured for this modality.

183 """

111 - return self.limit_per_prompt.get(

112 - modality,

113 - 999 if envs.VLLM_USE_V1 else 1,

114 - )

184 + # All values are now DummyOptions after normalization

185 + return self.limit_per_prompt.get(modality)

186

187 def merge_mm_processor_kwargs(

188 self,

vllm/engine/arg_utils.py

+1 -1

@@ -376,7 +376,7 @@ class EngineArgs:

376 quantization: Optional[QuantizationMethods] = ModelConfig.quantization

377 enforce_eager: bool = ModelConfig.enforce_eager

378 disable_custom_all_reduce: bool = ParallelConfig.disable_custom_all_reduce

379 - limit_mm_per_prompt: dict[str, int] = \

379 + limit_mm_per_prompt: dict[str, Union[int, dict[str, int]]] = \

380 get_field(MultiModalConfig, "limit_per_prompt")

381 interleave_mm_strings: bool = MultiModalConfig.interleave_mm_strings

382 media_io_kwargs: dict[str, dict[str,

vllm/model_executor/models/aria.py

+6 -1

@@ -10,6 +10,7 @@ from transformers.models.aria.modeling_aria import AriaCrossAttention

10 from transformers.models.aria.processing_aria import AriaProcessor

11

12 from vllm.config import VllmConfig

13 +from vllm.config.multimodal import BaseDummyOptions

14 from vllm.distributed import get_tensor_model_parallel_rank

15 from vllm.model_executor.layers.activation import get_act_fn

16 from vllm.model_executor.layers.fused_moe import FusedMoE

@@ -431,17 +432,21 @@ class AriaDummyInputsBuilder(BaseDummyInputsBuilder[AriaProcessingInfo]):

432 self,

433 seq_len: int,

434 mm_counts: Mapping[str, int],

435 + mm_options: Optional[Mapping[str, BaseDummyOptions]] = None,

436 ) -> MultiModalDataDict:

437 vision_config = self.info.get_vision_config()

438

439 max_image_size = vision_config.image_size

440 num_images = mm_counts.get("image", 0)

441

442 + image_overrides = mm_options.get("image") if mm_options else None

443 +

444 return {

445 "image":

446 self._get_dummy_images(width=max_image_size,

447 height=max_image_size,

444 - num_images=num_images)

448 + num_images=num_images,

449 + overrides=image_overrides)

450 }

451

452

vllm/model_executor/models/aya_vision.py

+6 -1

@@ -16,6 +16,7 @@ from transformers.models.got_ocr2.image_processing_got_ocr2 import (

16 get_optimal_tiled_canvas)

17

18 from vllm.config import VllmConfig

19 +from vllm.config.multimodal import BaseDummyOptions

20 from vllm.multimodal import MULTIMODAL_REGISTRY

21 from vllm.multimodal.inputs import MultiModalDataDict, MultiModalKwargsItems

22 from vllm.multimodal.parse import (ImageProcessorItems, ImageSize,

@@ -166,16 +167,20 @@ class AyaVisionDummyInputsBuilder(

167 self,

168 seq_len: int,

169 mm_counts: Mapping[str, int],

170 + mm_options: Optional[Mapping[str, BaseDummyOptions]] = None,

171 ) -> MultiModalDataDict:

172 num_images = mm_counts.get("image", 0)

173 image_size = \

174 self.info.get_image_size_with_most_features()

175

176 + image_overrides = mm_options.get("image") if mm_options else None

177 +

178 return {

179 "image":

180 self._get_dummy_images(width=image_size.width,

181 height=image_size.height,

178 - num_images=num_images)

182 + num_images=num_images,

183 + overrides=image_overrides)

184 }

185

186

vllm/model_executor/models/blip2.py

+6 -1

@@ -10,6 +10,7 @@ from transformers import (BatchFeature, Blip2Config, Blip2QFormerConfig,

10 apply_chunking_to_forward)

11

12 from vllm.config import CacheConfig, VllmConfig

13 +from vllm.config.multimodal import BaseDummyOptions

14 from vllm.model_executor.layers.activation import get_act_fn

15 from vllm.model_executor.layers.quantization import QuantizationConfig

16 from vllm.multimodal import MULTIMODAL_REGISTRY

@@ -435,6 +436,7 @@ class Blip2DummyInputsBuilder(BaseDummyInputsBuilder[Blip2ProcessingInfo]):

436 self,

437 seq_len: int,

438 mm_counts: Mapping[str, int],

439 + mm_options: Optional[Mapping[str, BaseDummyOptions]] = None,

440 ) -> MultiModalDataDict:

441 hf_config = self.info.get_hf_config()

442 vision_config = hf_config.vision_config

@@ -442,11 +444,14 @@ class Blip2DummyInputsBuilder(BaseDummyInputsBuilder[Blip2ProcessingInfo]):

444 max_image_size = vision_config.image_size

445 num_images = mm_counts.get("image", 0)

446

447 + image_overrides = mm_options.get("image") if mm_options else None

448 +

449 return {

450 "image":

451 self._get_dummy_images(width=max_image_size,

452 height=max_image_size,

449 - num_images=num_images)

453 + num_images=num_images,

454 + overrides=image_overrides)

455 }

456

457

vllm/model_executor/models/chameleon.py

+6 -1

@@ -14,6 +14,7 @@ from transformers import (BatchFeature, ChameleonConfig, ChameleonProcessor,

14

15 from vllm.attention import Attention

16 from vllm.config import CacheConfig, VllmConfig

17 +from vllm.config.multimodal import BaseDummyOptions

18 from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size

19 from vllm.logger import init_logger

20 from vllm.model_executor.layers.activation import SiluAndMul

@@ -92,17 +93,21 @@ class ChameleonDummyInputsBuilder(

93 self,

94 seq_len: int,

95 mm_counts: Mapping[str, int],

96 + mm_options: Optional[Mapping[str, BaseDummyOptions]] = None,

97 ) -> MultiModalDataDict:

98 config = self.info.get_hf_config()

99

100 width = height = config.vq_config.resolution

101 num_images = mm_counts.get("image", 0)

102

103 + image_overrides = mm_options.get("image") if mm_options else None

104 +

105 return {

106 "image":

107 self._get_dummy_images(width=width,

108 height=height,

105 - num_images=num_images)

109 + num_images=num_images,

110 + overrides=image_overrides)

111 }

112

113

vllm/model_executor/models/cohere2_vision.py

+6 -1

@@ -16,6 +16,7 @@ from transformers.models.cohere2_vision.processing_cohere2_vision import (

16 Cohere2VisionProcessor)

17

18 from vllm.config import VllmConfig

19 +from vllm.config.multimodal import BaseDummyOptions

20 from vllm.model_executor.layers.activation import MulAndSilu

21 from vllm.model_executor.layers.linear import (MergedColumnParallelLinear,

22 RowParallelLinear)

@@ -209,16 +210,20 @@ class Cohere2VisionDummyInputsBuilder(

210 self,

211 seq_len: int,

212 mm_counts: Mapping[str, int],

213 + mm_options: Optional[Mapping[str, BaseDummyOptions]] = None,

214 ) -> MultiModalDataDict:

215 num_images = mm_counts.get("image", 0)

216 image_size = \

217 self.info.get_image_size_with_most_features()

218

219 + image_overrides = mm_options.get("image") if mm_options else None

220 +

221 return {

222 "image":

223 self._get_dummy_images(width=image_size.width,

224 height=image_size.height,

221 - num_images=num_images)

225 + num_images=num_images,

226 + overrides=image_overrides)

227 }

228

229

vllm/model_executor/models/deepseek_vl2.py

+6 -1

@@ -14,6 +14,7 @@ from einops import rearrange, repeat

14 from transformers import BatchFeature

15

16 from vllm.config import VllmConfig

17 +from vllm.config.multimodal import BaseDummyOptions

18 from vllm.distributed import get_tensor_model_parallel_world_size

19 from vllm.model_executor.layers.quantization import QuantizationConfig

20 from vllm.model_executor.model_loader.utils import set_default_torch_dtype

@@ -191,16 +192,20 @@ class DeepseekVL2DummyInputsBuilder(

192 self,

193 seq_len: int,

194 mm_counts: Mapping[str, int],

195 + mm_options: Optional[Mapping[str, BaseDummyOptions]] = None,

196 ) -> MultiModalDataDict:

197 num_images = mm_counts.get("image", 0)

198

199 max_image_size = self.info.get_image_size_with_most_features()

200

201 + image_overrides = mm_options.get("image") if mm_options else None

202 +

203 return {

204 "image":

205 self._get_dummy_images(width=max_image_size.width,

206 height=max_image_size.height,

203 - num_images=num_images)

207 + num_images=num_images,

208 + overrides=image_overrides)

209 }

210

211

vllm/model_executor/models/dots_ocr.py

+6 -1

@@ -13,6 +13,7 @@ from vllm.attention.backends.registry import _Backend

13 from vllm.attention.layer import (check_upstream_fa_availability,

14 maybe_get_vit_flash_attn_backend)

15 from vllm.config import VllmConfig

16 +from vllm.config.multimodal import BaseDummyOptions

17 from vllm.distributed import utils as dist_utils

18 from vllm.distributed.parallel_state import (

19 get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size)

@@ -91,17 +92,21 @@ class DotsOCRDummyInputsBuilder(Qwen2VLDummyInputsBuilder):

92 self,

93 seq_len: int,

94 mm_counts: Mapping[str, int],

95 + mm_options: Optional[Mapping[str, BaseDummyOptions]] = None,

96 ) -> MultiModalDataDict:

97 num_images = mm_counts.get("image", 0)

98

99 target_width, target_height = self.info.get_image_size_with_most_features( # noqa: E501

100 )

101

102 + image_overrides = mm_options.get("image") if mm_options else None

103 +

104 return {

105 "image":

106 self._get_dummy_images(width=target_width,

107 height=target_height,

104 - num_images=num_images),

108 + num_images=num_images,

109 + overrides=image_overrides),

110 }

111

112

vllm/model_executor/models/ernie45_vl.py

+9 -2

@@ -38,6 +38,7 @@ from vllm.attention.backends.registry import _Backend

38 from vllm.attention.layer import (check_upstream_fa_availability,

39 maybe_get_vit_flash_attn_backend)

40 from vllm.config import VllmConfig

41 +from vllm.config.multimodal import BaseDummyOptions

42 from vllm.distributed import parallel_state

43 from vllm.distributed import utils as dist_utils

44 from vllm.logger import init_logger

@@ -1184,6 +1185,7 @@ class Ernie4_5_VLDummyInputsBuilder(

1185 self,

1186 seq_len: int,

1187 mm_counts: Mapping[str, int],

1188 + mm_options: Optional[Mapping[str, BaseDummyOptions]] = None,

1189 ) -> MultiModalDataDict:

1190 num_images = mm_counts.get("image", 0)

1191 num_videos = mm_counts.get("video", 0)

@@ -1193,16 +1195,21 @@ class Ernie4_5_VLDummyInputsBuilder(

1195 target_num_frames = \

1196 self.info.get_num_frames_with_most_features(seq_len, mm_counts)

1197

1198 + image_overrides = mm_options.get("image") if mm_options else None

1199 + video_overrides = mm_options.get("video") if mm_options else None

1200 +

1201 return {

1202 "image":

1203 self._get_dummy_images(width=target_width,

1204 height=target_height,

1200 - num_images=num_images),

1205 + num_images=num_images,

1206 + overrides=image_overrides),

1207 "video":

1208 self._get_dummy_videos(width=target_width,

1209 height=target_height,

1210 num_frames=target_num_frames,

1205 - num_videos=num_videos)

1211 + num_videos=num_videos,

1212 + overrides=video_overrides)

1213 }

1214

1215

vllm/model_executor/models/fuyu.py

+6 -1

@@ -27,6 +27,7 @@ from transformers import (BatchFeature, FuyuConfig, FuyuImageProcessor,

27 FuyuProcessor)

28

29 from vllm.config import VllmConfig

30 +from vllm.config.multimodal import BaseDummyOptions

31 from vllm.model_executor.layers.linear import ColumnParallelLinear

32 from vllm.model_executor.models.persimmon import PersimmonForCausalLM

33 from vllm.multimodal import MULTIMODAL_REGISTRY

@@ -136,16 +137,20 @@ class FuyuDummyInputsBuilder(BaseDummyInputsBuilder[FuyuProcessingInfo]):

137 self,

138 seq_len: int,

139 mm_counts: Mapping[str, int],

140 + mm_options: Optional[Mapping[str, BaseDummyOptions]] = None,

141 ) -> MultiModalDataDict:

142 target_width, target_height = \

143 self.info.get_image_size_with_most_features()

144 num_images = mm_counts.get("image", 0)

145

146 + image_overrides = mm_options.get("image") if mm_options else None

147 +

148 return {

149 "image":

150 self._get_dummy_images(width=target_width,

151 height=target_height,

148 - num_images=num_images)

152 + num_images=num_images,

153 + overrides=image_overrides)

154 }

155

156

vllm/model_executor/models/gemma3_mm.py

+6 -1

@@ -11,6 +11,7 @@ from transformers.models.gemma3.processing_gemma3 import Gemma3ProcessorKwargs

11

12 import vllm.envs as envs

13 from vllm.config import VllmConfig

14 +from vllm.config.multimodal import BaseDummyOptions

15 from vllm.logger import init_logger

16 from vllm.model_executor.layers.layernorm import GemmaRMSNorm

17 from vllm.model_executor.models.module_mapping import MultiModelKeys

@@ -241,17 +242,21 @@ class Gemma3DummyInputsBuilder(BaseDummyInputsBuilder[Gemma3ProcessingInfo]):

242 self,

243 seq_len: int,

244 mm_counts: Mapping[str, int],

245 + mm_options: Optional[Mapping[str, BaseDummyOptions]] = None,

246 ) -> MultiModalDataDict:

247 num_images = mm_counts.get("image", 0)

248

249 target_width, target_height = \

250 self.info.get_image_size_with_most_features()

251

252 + image_overrides = mm_options.get("image") if mm_options else None

253 +

254 return {

255 "image":

256 self._get_dummy_images(width=target_width,

257 height=target_height,

254 - num_images=num_images)

258 + num_images=num_images,

259 + overrides=image_overrides)

260 }

261

262

vllm/model_executor/models/gemma3n_mm.py

+10 -2

@@ -16,6 +16,7 @@ from transformers.models.gemma3n import (Gemma3nAudioConfig,

16 from transformers.models.siglip import SiglipImageProcessorFast

17

18 from vllm.config import ModelConfig, SpeechToTextConfig, VllmConfig

19 +from vllm.config.multimodal import BaseDummyOptions

20 from vllm.inputs.data import PromptType

21 from vllm.logger import init_logger

22 from vllm.model_executor.layers.layernorm import RMSNorm

@@ -153,6 +154,7 @@ class Gemma3nDummyInputsBuilder(BaseDummyInputsBuilder[Gemma3nProcessingInfo]):

154 self,

155 seq_len: int,

156 mm_counts: Mapping[str, int],

157 + mm_options: Optional[Mapping[str, BaseDummyOptions]] = None,

158 ) -> MultiModalDataDict:

159 num_images = mm_counts.get("image", 0)

160 num_audios = mm_counts.get("audio", 0)

@@ -163,13 +165,19 @@ class Gemma3nDummyInputsBuilder(BaseDummyInputsBuilder[Gemma3nProcessingInfo]):

165 img_width = image_processor.size.get("width", 224)

166 img_height = image_processor.size.get("height", 224)

167

168 + image_overrides = mm_options.get("image") if mm_options else None

169 + audio_overrides = mm_options.get("audio") if mm_options else None

170 +

171 return {

172 "image":

173 self._get_dummy_images(width=img_width,

174 height=img_height,

170 - num_images=num_images),

175 + num_images=num_images,

176 + overrides=image_overrides),

177 "audio":

172 - self._get_dummy_audios(length=audio_len, num_audios=num_audios)

178 + self._get_dummy_audios(length=audio_len,

179 + num_audios=num_audios,

180 + overrides=audio_overrides)

181 }

182

183

vllm/model_executor/models/glm4_1v.py

+34 -2

@@ -29,7 +29,7 @@

29 import math

30 from collections.abc import Iterable, Mapping, Sequence

31 from functools import partial

32 -from typing import Annotated, Any, Callable, Literal, Optional, Union

32 +from typing import Annotated, Any, Callable, Literal, Optional, Union, override

33

34 import numpy as np

35 import torch

@@ -50,6 +50,7 @@ from vllm.attention.backends.registry import _Backend

50 from vllm.attention.layer import (check_upstream_fa_availability,

51 maybe_get_vit_flash_attn_backend)

52 from vllm.config import VllmConfig

53 +from vllm.config.multimodal import BaseDummyOptions, VideoDummyOptions

54 from vllm.distributed import (get_tensor_model_parallel_world_size,

55 parallel_state)

56 from vllm.distributed import utils as dist_utils

@@ -1110,6 +1111,7 @@ class Glm4vDummyInputsBuilder(BaseDummyInputsBuilder[Glm4vProcessingInfo]):

1111 self,

1112 seq_len: int,

1113 mm_counts: Mapping[str, int],

1114 + mm_options: Optional[Mapping[str, BaseDummyOptions]] = None,

1115 ) -> MultiModalDataDict:

1116 num_images = mm_counts.get("image", 0)

1117 num_videos = mm_counts.get("video", 0)

@@ -1118,17 +1120,23 @@ class Glm4vDummyInputsBuilder(BaseDummyInputsBuilder[Glm4vProcessingInfo]):

1120 self.info.get_image_size_with_most_features())

1121 target_num_frames = self.info.get_num_frames_with_most_features(

1122 seq_len, mm_counts)

1123 +

1124 + image_overrides = mm_options.get("image") if mm_options else None

1125 + video_overrides = mm_options.get("video") if mm_options else None

1126 +

1127 return {

1128 "image":

1129 self._get_dummy_images(width=target_width,

1130 height=target_height,

1125 - num_images=num_images),

1131 + num_images=num_images,

1132 + overrides=image_overrides),

1133 "video":

1134 self._get_dummy_videos(

1135 width=target_width,

1136 height=target_height,

1137 num_frames=target_num_frames,

1138 num_videos=num_videos,

1139 + overrides=video_overrides,

1140 ),

1141 }

1142

@@ -1139,7 +1147,31 @@ class Glm4vDummyInputsBuilder(BaseDummyInputsBuilder[Glm4vProcessingInfo]):

1147 height: int,

1148 num_frames: int,

1149 num_videos: int,

1150 + overrides: Optional[VideoDummyOptions] = None,

1151 ) -> list[VideoItem]:

1152 + if overrides:

1153 + if overrides.num_frames:

1154 + if overrides.num_frames > num_frames:

1155 + logger.warning(

1156 + "video.num_frames override (%d) exceeds model's "

1157 + "maximum number of frames (%d), will be ignored",

1158 + overrides.num_frames, num_frames)

1159 + num_frames = min(num_frames, overrides.num_frames)

1160 + if overrides.width:

1161 + if overrides.width > width:

1162 + logger.warning(

1163 + "video.width override (%d) exceeds model's "

1164 + "maximum width (%d), will be ignored", overrides.width,

1165 + width)

1166 + width = min(width, overrides.width)

1167 + if overrides.height:

1168 + if overrides.height > height:

1169 + logger.warning(

1170 + "video.height override (%d) exceeds model's "

1171 + "maximum height (%d), will be ignored",

1172 + overrides.height, height)

1173 + height = min(height, override.height)

1174 +

1175 video = np.full((num_frames, width, height, 3), 255, dtype=np.uint8)

1176 video_items = []

1177 for i in range(num_videos):

vllm/model_executor/models/glm4v.py

+6 -1

@@ -19,6 +19,7 @@ from transformers.tokenization_utils_base import TextInput

19

20 from vllm.attention.layer import MultiHeadAttention

21 from vllm.config import VllmConfig

22 +from vllm.config.multimodal import BaseDummyOptions

23 from vllm.distributed import get_tensor_model_parallel_world_size

24 from vllm.model_executor.layers.activation import SiluAndMul, get_act_fn

25 from vllm.model_executor.layers.linear import (ColumnParallelLinear,

@@ -465,6 +466,7 @@ class GLM4VDummyInputsBuilder(BaseDummyInputsBuilder[GLM4VProcessingInfo]):

466 self,

467 seq_len: int,

468 mm_counts: Mapping[str, int],

469 + mm_options: Optional[Mapping[str, BaseDummyOptions]] = None,

470 ) -> MultiModalDataDict:

471 hf_config = self.info.get_hf_config()

472 vision_config = hf_config.vision_config

@@ -472,11 +474,14 @@ class GLM4VDummyInputsBuilder(BaseDummyInputsBuilder[GLM4VProcessingInfo]):

474 target_width = target_height = vision_config["image_size"]

475 num_images = mm_counts.get("image", 0)

476

477 + image_overrides = mm_options.get("image") if mm_options else None

478 +

479 return {

480 "image":

481 self._get_dummy_images(width=target_width,

482 height=target_height,

479 - num_images=num_images)

483 + num_images=num_images,

484 + overrides=image_overrides)

485 }

486

487

vllm/model_executor/models/granite_speech.py

+5

@@ -33,6 +33,7 @@ from torch import nn

33 from transformers import BatchFeature, PretrainedConfig

34

35 from vllm.config import CacheConfig, VllmConfig

36 +from vllm.config.multimodal import BaseDummyOptions

37 from vllm.model_executor.layers.linear import (ColumnParallelLinear,

38 RowParallelLinear)

39 from vllm.model_executor.layers.quantization import QuantizationConfig

@@ -181,13 +182,17 @@ class GraniteSpeechDummyInputsBuilder(

182 self,

183 seq_len: int,

184 mm_counts: Mapping[str, int],

185 + mm_options: Optional[Mapping[str, BaseDummyOptions]] = None,

186 ) -> MultiModalDataDict:

187 num_audios = mm_counts.get("audio", 0)

188 + audio_overrides = mm_options.get("audio") if mm_options else None

189 +

190 return {

191 "audio":

192 self._get_dummy_audios(

193 length=self.info.get_max_audio_len(),

194 num_audios=num_audios,

195 + overrides=audio_overrides,

196 )

197 }

198

vllm/model_executor/models/hyperclovax_vision.py

+8

@@ -29,6 +29,7 @@ from transformers import BatchFeature, CLIPVisionConfig, SiglipVisionConfig

29 from transformers.modeling_utils import no_init_weights

30

31 from vllm.config import VllmConfig

32 +from vllm.config.multimodal import BaseDummyOptions

33 from vllm.model_executor.layers.quantization import QuantizationConfig

34 from vllm.multimodal import MULTIMODAL_REGISTRY

35 from vllm.multimodal.cache import BaseMultiModalProcessorCache

@@ -149,6 +150,7 @@ class HCXVisionDummyInputsBuilder(

150 self,

151 seq_len: int,

152 mm_counts: Mapping[str, int],

153 + mm_options: Optional[Mapping[str, BaseDummyOptions]] = None,

154 ) -> MultiModalDataDict:

155 num_images = mm_counts.get("image", 0)

156 num_videos = mm_counts.get("video", 0)

@@ -156,12 +158,17 @@ class HCXVisionDummyInputsBuilder(

158 target_width, target_height = \

159 self.info.get_image_size_with_most_features()

160 target_num_frames = 32

161 +

162 + image_overrides = mm_options.get("image") if mm_options else None

163 + video_overrides = mm_options.get("video") if mm_options else None

164 +

165 return {

166 "image":

167 self._get_dummy_images(

168 width=target_width,

169 height=target_height,

170 num_images=num_images,

171 + overrides=image_overrides,

172 ),

173 "video":

174 self._get_dummy_videos(

@@ -169,6 +176,7 @@ class HCXVisionDummyInputsBuilder(

176 height=target_height - 1,

177 num_frames=target_num_frames,

178 num_videos=num_videos,

179 + overrides=video_overrides,

180 )

181 }

182

vllm/model_executor/models/idefics3.py

+6 -1

@@ -26,6 +26,7 @@ from transformers import (BatchFeature, Idefics3Config, Idefics3ImageProcessor,

26 Idefics3Processor)

27

28 from vllm.config import VllmConfig

29 +from vllm.config.multimodal import BaseDummyOptions

30 from vllm.model_executor.layers.linear import ReplicatedLinear

31 from vllm.model_executor.layers.logits_processor import LogitsProcessor

32 from vllm.model_executor.layers.quantization import QuantizationConfig

@@ -292,17 +293,21 @@ class Idefics3DummyInputsBuilder(BaseDummyInputsBuilder[Idefics3ProcessingInfo]

293 self,

294 seq_len: int,

295 mm_counts: Mapping[str, int],

296 + mm_options: Optional[Mapping[str, BaseDummyOptions]] = None,

297 ) -> MultiModalDataDict:

298 num_images = mm_counts.get("image", 0)

299 hf_processor = self.info.get_hf_processor()

300 image_processor: Idefics3ImageProcessor = hf_processor.image_processor

301 longest_edge = image_processor.max_image_size['longest_edge']

302

303 + image_overrides = mm_options.get("image") if mm_options else None

304 +

305 return {

306 "image":

307 self._get_dummy_images(width=longest_edge,

308 height=longest_edge,

305 - num_images=num_images)

309 + num_images=num_images,

310 + overrides=image_overrides)

311 }

312

313

vllm/model_executor/models/interns1.py

+9 -2

@@ -20,6 +20,7 @@ from transformers.models.internvl.video_processing_internvl import (

20 InternVLVideoProcessor)

21

22 from vllm.config import VllmConfig

23 +from vllm.config.multimodal import BaseDummyOptions

24 from vllm.model_executor.layers.quantization import QuantizationConfig

25 from vllm.model_executor.models.interns1_vit import InternS1VisionModel

26 from vllm.model_executor.models.module_mapping import MultiModelKeys

@@ -270,6 +271,7 @@ class InternS1DummyInputsBuilder(BaseDummyInputsBuilder[InternS1ProcessingInfo]

271 self,

272 seq_len: int,

273 mm_counts: Mapping[str, int],

274 + mm_options: Optional[Mapping[str, BaseDummyOptions]] = None,

275 ) -> MultiModalDataDict:

276 target_width, target_height = \

277 self.info.get_image_size_with_most_features()

@@ -281,16 +283,21 @@ class InternS1DummyInputsBuilder(BaseDummyInputsBuilder[InternS1ProcessingInfo]

283 config = self.info.get_hf_config()

284 image_size_h, image_size_w = config.vision_config.image_size

285

286 + image_overrides = mm_options.get("image") if mm_options else None

287 + video_overrides = mm_options.get("video") if mm_options else None

288 +

289 return {

290 "image":

291 self._get_dummy_images(width=target_width,

292 height=target_height,

288 - num_images=num_images),

293 + num_images=num_images,

294 + overrides=image_overrides),

295 "video":

296 self._get_dummy_videos(width=image_size_w,

297 height=image_size_h,

298 num_frames=target_num_frames,

293 - num_videos=num_videos),

299 + num_videos=num_videos,

300 + overrides=video_overrides),

301 }

302

303

vllm/model_executor/models/internvl.py

+12 -3

@@ -20,6 +20,7 @@ from PIL import Image

20 from transformers import BatchFeature, PretrainedConfig, TensorType

21

22 from vllm.config import VllmConfig

23 +from vllm.config.multimodal import BaseDummyOptions

24 from vllm.model_executor.layers.quantization import QuantizationConfig

25 from vllm.model_executor.layers.quantization.awq import AWQConfig

26 from vllm.model_executor.models.intern_vit import (InternVisionModel,

@@ -747,16 +748,20 @@ class BaseInternVLDummyInputsBuilder(BaseDummyInputsBuilder[_I]):

748 self,

749 seq_len: int,

750 mm_counts: Mapping[str, int],

751 + mm_options: Optional[Mapping[str, BaseDummyOptions]] = None,

752 ) -> MultiModalDataDict:

753 target_width, target_height = \

754 self.info.get_image_size_with_most_features()

755 num_images = mm_counts.get("image", 0)

756

757 + image_overrides = mm_options.get("image") if mm_options else None

758 +

759 return {

760 "image":

761 self._get_dummy_images(width=target_width,

762 height=target_height,

759 - num_images=num_images)

763 + num_images=num_images,

764 + overrides=image_overrides)

765 }

766

767

@@ -913,21 +918,25 @@ class InternVLDummyInputsBuilder(

918 self,

919 seq_len: int,

920 mm_counts: Mapping[str, int],

921 + mm_options: Optional[Mapping[str, BaseDummyOptions]] = None,

922 ) -> MultiModalDataDict:

923 dummy_image = super().get_dummy_mm_data(seq_len=seq_len,

918 - mm_counts=mm_counts)

924 + mm_counts=mm_counts,

925 + mm_options=mm_options)

926 if self.info.supports_video:

927 config = self.info.get_hf_config()

928 image_size: int = config.vision_config.image_size

929 target_num_frames = \

930 self.info.get_num_frames_with_most_features(seq_len, mm_counts)

931 num_videos = mm_counts.get("video", 0)

932 + video_overrides = mm_options.get("video") if mm_options else None

933 dummy_video = {

934 "video":

935 self._get_dummy_videos(width=image_size,

936 height=image_size,

937 num_frames=target_num_frames,

930 - num_videos=num_videos)

938 + num_videos=num_videos,

939 + overrides=video_overrides)

940 }

941 else:

942 dummy_video = {}

vllm/model_executor/models/keye.py

+7

@@ -20,6 +20,7 @@ from transformers.utils import torch_int

20 from vllm.attention.backends.registry import _Backend

21 from vllm.attention.layer import check_upstream_fa_availability

22 from vllm.config import VllmConfig

23 +from vllm.config.multimodal import BaseDummyOptions

24 from vllm.distributed import get_tensor_model_parallel_world_size

25 from vllm.logger import init_logger

26 from vllm.model_executor.layers.linear import (ColumnParallelLinear,

@@ -1170,6 +1171,7 @@ class KeyeBaseDummyInputsBuilder(BaseDummyInputsBuilder[_I]):

1171 self,

1172 seq_len: int,

1173 mm_counts: Mapping[str, int],

1174 + mm_options: Optional[Mapping[str, BaseDummyOptions]] = None,

1175 ) -> MultiModalDataDict:

1176 num_images = mm_counts.get("image", 0)

1177 num_videos = mm_counts.get("video", 0)

@@ -1179,12 +1181,16 @@ class KeyeBaseDummyInputsBuilder(BaseDummyInputsBuilder[_I]):

1181 target_num_frames = self.info.get_num_frames_with_most_features(

1182 seq_len)

1183

1184 + image_overrides = mm_options.get("image") if mm_options else None

1185 + video_overrides = mm_options.get("video") if mm_options else None

1186 +

1187 mm_data = {

1188 "image":

1189 self._get_dummy_images(

1190 width=target_width,

1191 height=target_height,

1192 num_images=num_images,

1193 + overrides=image_overrides,

1194 ),

1195 "video":

1196 self._get_dummy_videos(

@@ -1192,6 +1198,7 @@ class KeyeBaseDummyInputsBuilder(BaseDummyInputsBuilder[_I]):

1198 height=target_height,

1199 num_frames=target_num_frames,

1200 num_videos=num_videos,

1201 + overrides=video_overrides,

1202 ),

1203 }

1204

vllm/model_executor/models/kimi_vl.py

+6 -1

@@ -54,6 +54,7 @@ from transformers import BatchFeature

54 from transformers.activations import GELUActivation

55

56 from vllm.config import VllmConfig

57 +from vllm.config.multimodal import BaseDummyOptions

58 from vllm.distributed import get_pp_group

59 from vllm.model_executor.layers.fused_moe import FusedMoE

60 from vllm.model_executor.layers.linear import ReplicatedLinear

@@ -212,14 +213,18 @@ class KimiVLDummyInputsBuilder(BaseDummyInputsBuilder[KimiVLProcessingInfo]):

213 self,

214 seq_len: int,

215 mm_counts: Mapping[str, int],

216 + mm_options: Optional[Mapping[str, BaseDummyOptions]] = None,

217 ) -> MultiModalDataDict:

218 num_images = mm_counts.get("image", 0)

219

220 + image_overrides = mm_options.get("image") if mm_options else None

221 +

222 return {

223 "image":

224 self._get_dummy_images(width=MaxImageTokenMeta.width,

225 height=MaxImageTokenMeta.height,

222 - num_images=num_images)

226 + num_images=num_images,

227 + overrides=image_overrides)

228 }

229

230

vllm/model_executor/models/llava.py

+6 -1

@@ -15,6 +15,7 @@ from transformers.models.llava import LlavaProcessor

15 from transformers.models.pixtral import PixtralProcessor

16

17 from vllm.config import VllmConfig

18 +from vllm.config.multimodal import BaseDummyOptions

19 from vllm.model_executor.layers.activation import get_act_fn

20 from vllm.model_executor.layers.linear import (ColumnParallelLinear,

21 RowParallelLinear)

@@ -195,17 +196,21 @@ class LlavaDummyInputsBuilder(BaseDummyInputsBuilder[_I]):

196 self,

197 seq_len: int,

198 mm_counts: Mapping[str, int],

199 + mm_options: Optional[Mapping[str, BaseDummyOptions]] = None,

200 ) -> MultiModalDataDict:

201 num_images = mm_counts.get("image", 0)

202

203 target_width, target_height = \

204 self.info.get_image_size_with_most_features()

205

206 + image_overrides = mm_options.get("image") if mm_options else None

207 +

208 return {

209 "image":

210 self._get_dummy_images(width=target_width,

211 height=target_height,

208 - num_images=num_images)

212 + num_images=num_images,

213 + overrides=image_overrides)

214 }

215

216

vllm/model_executor/models/llava_next_video.py

+5

@@ -11,6 +11,7 @@ from transformers import (BatchFeature, LlavaNextVideoConfig,

11 LlavaNextVideoProcessor)

12

13 from vllm.config import VllmConfig

14 +from vllm.config.multimodal import BaseDummyOptions

15 from vllm.model_executor.layers.activation import get_act_fn

16 from vllm.model_executor.models.clip import CLIPVisionModel

17 from vllm.multimodal import MULTIMODAL_REGISTRY

@@ -150,6 +151,7 @@ class LlavaNextVideoDummyInputsBuilder(

151 self,

152 seq_len: int,

153 mm_counts: Mapping[str, int],

154 + mm_options: Optional[Mapping[str, BaseDummyOptions]] = None,

155 ) -> MultiModalDataDict:

156 num_videos = mm_counts.get("video", 0)

157

@@ -158,6 +160,8 @@ class LlavaNextVideoDummyInputsBuilder(

160 target_num_frames = \

161 self.info.get_num_frames_with_most_features(seq_len, mm_counts)

162

163 + video_overrides = mm_options.get("video") if mm_options else None

164 +

165 return {

166 "video":

167 self._get_dummy_videos(

@@ -165,6 +169,7 @@ class LlavaNextVideoDummyInputsBuilder(

169 height=target_height,

170 num_frames=target_num_frames,

171 num_videos=num_videos,

172 + overrides=video_overrides,

173 )

174 }

175

vllm/model_executor/models/llava_onevision.py

+8 -1

@@ -13,6 +13,7 @@ from transformers.models.llava_onevision.modeling_llava_onevision import (

13 get_anyres_image_grid_shape, unpad_image)

14

15 from vllm.config import VllmConfig

16 +from vllm.config.multimodal import BaseDummyOptions

17 from vllm.model_executor.layers.activation import get_act_fn

18 from vllm.multimodal import MULTIMODAL_REGISTRY

19 from vllm.multimodal.inputs import (MultiModalDataDict, MultiModalFieldConfig,

@@ -254,6 +255,7 @@ class LlavaOnevisionDummyInputsBuilder(

255 self,

256 seq_len: int,

257 mm_counts: Mapping[str, int],

258 + mm_options: Optional[Mapping[str, BaseDummyOptions]] = None,

259 ) -> MultiModalDataDict:

260 num_images = mm_counts.get("image", 0)

261 num_videos = mm_counts.get("video", 0)

@@ -264,17 +266,22 @@ class LlavaOnevisionDummyInputsBuilder(

266 self.info.get_num_frames_with_most_features(seq_len,

267 mm_counts)

268

269 + image_overrides = mm_options.get("image") if mm_options else None

270 + video_overrides = mm_options.get("video") if mm_options else None

271 +

272 return {

273 "image":

274 self._get_dummy_images(width=target_width,

275 height=target_height,

271 - num_images=num_images),

276 + num_images=num_images,

277 + overrides=image_overrides),

278 "video":

279 self._get_dummy_videos(

280 width=target_width,

281 height=target_height,

282 num_frames=target_num_frames,

283 num_videos=num_videos,

284 + overrides=video_overrides,

285 )

286 }

287

vllm/model_executor/models/midashenglm.py

+6 -1

@@ -36,6 +36,7 @@ from torch.nn.functional import scaled_dot_product_attention

36 from transformers import BatchFeature

37

38 from vllm.config import VllmConfig

39 +from vllm.config.multimodal import BaseDummyOptions

40 from vllm.distributed import get_tensor_model_parallel_world_size

41 from vllm.model_executor.layers.activation import get_act_fn

42 from vllm.model_executor.layers.linear import (ColumnParallelLinear,

@@ -539,13 +540,17 @@ class MiDashengLMDummyInputsBuilder(

540 self,

541 seq_len: int,

542 mm_counts: Mapping[str, int],

543 + mm_options: Optional[Mapping[str, BaseDummyOptions]] = None,

544 ) -> MultiModalDataDict:

545 num_audios = mm_counts.get("audio", 0)

546

547 + audio_overrides = mm_options.get("audio") if mm_options else None

548 +

549 return {

550 "audio":

551 self._get_dummy_audios(length=self.info.get_max_audio_len(),

548 - num_audios=num_audios)

552 + num_audios=num_audios,

553 + overrides=audio_overrides)

554 }

555

556

vllm/model_executor/models/minicpmo.py

+8 -2

@@ -36,6 +36,7 @@ from transformers.models.whisper.modeling_whisper import (ACT2FN,

36 WhisperEncoder)

37

38 from vllm.config import VllmConfig

39 +from vllm.config.multimodal import BaseDummyOptions

40 from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalKwargsItems

41 from vllm.multimodal.inputs import (MultiModalDataDict, MultiModalFieldConfig,

42 NestedTensors)

@@ -237,18 +238,23 @@ class MiniCPMODummyInputsBuilder(

238 self,

239 seq_len: int,

240 mm_counts: Mapping[str, int],

241 + mm_options: Optional[Mapping[str, BaseDummyOptions]] = None,

242 ) -> MultiModalDataDict:

243 num_audios = mm_counts.get("audio", 0)

244 audio_len = self.info.get_max_audio_chunks_with_most_features() * \

245 self.info.get_default_audio_sampling_rate()

246

247 + audio_overrides = mm_options.get("audio") if mm_options else None

248 +

249 audio_mm_data = {

250 "audio":

247 - self._get_dummy_audios(length=audio_len, num_audios=num_audios)

251 + self._get_dummy_audios(length=audio_len,

252 + num_audios=num_audios,

253 + overrides=audio_overrides)

254 }

255

256 return {

251 - **super().get_dummy_mm_data(seq_len, mm_counts),

257 + **super().get_dummy_mm_data(seq_len, mm_counts, mm_options),

258 **audio_mm_data,

259 }

260

vllm/model_executor/models/minicpmv.py

+9 -2

@@ -39,6 +39,7 @@ from transformers import BatchFeature, PretrainedConfig

39 from typing_extensions import TypeVar

40

41 from vllm.config import VllmConfig

42 +from vllm.config.multimodal import BaseDummyOptions

43 from vllm.model_executor.layers.quantization import QuantizationConfig

44 from vllm.model_executor.layers.quantization.awq import AWQConfig

45 from vllm.model_executor.layers.quantization.awq_marlin import AWQMarlinConfig

@@ -679,6 +680,7 @@ class MiniCPMVDummyInputsBuilder(BaseDummyInputsBuilder[_I]):

680 self,

681 seq_len: int,

682 mm_counts: Mapping[str, int],

683 + mm_options: Optional[Mapping[str, BaseDummyOptions]] = None,

684 ) -> MultiModalDataDict:

685 num_images = mm_counts.get("image", 0)

686 num_videos = mm_counts.get("video", 0)

@@ -690,15 +692,20 @@ class MiniCPMVDummyInputsBuilder(BaseDummyInputsBuilder[_I]):

692 num_video_frames = \

693 self.info.get_num_frames_with_most_features(seq_len, mm_counts)

694

695 + image_overrides = mm_options.get("image") if mm_options else None

696 + video_overrides = mm_options.get("video") if mm_options else None

697 +

698 return {

699 "image":

700 self._get_dummy_images(width=image_width,

701 height=image_height,

697 - num_images=num_images),

702 + num_images=num_images,

703 + overrides=image_overrides),

704 "video": [

705 self._get_dummy_images(width=video_width,

706 height=video_height,

701 - num_images=num_video_frames)

707 + num_images=num_video_frames,

708 + overrides=video_overrides)

709 ] * num_videos,

710 }

711

vllm/model_executor/models/mistral3.py

+6 -1

@@ -13,6 +13,7 @@ from transformers import (BatchFeature, Mistral3Config, PixtralVisionConfig,

13 from transformers.models.pixtral import PixtralProcessor

14

15 from vllm.config import VllmConfig

16 +from vllm.config.multimodal import BaseDummyOptions

17 from vllm.model_executor.layers.activation import get_act_fn

18 from vllm.model_executor.layers.layernorm import RMSNorm

19 from vllm.model_executor.layers.linear import (ColumnParallelLinear,

@@ -208,17 +209,21 @@ class Mistral3DummyInputsBuilder(BaseDummyInputsBuilder[_I]):

209 self,

210 seq_len: int,

211 mm_counts: Mapping[str, int],

212 + mm_options: Optional[Mapping[str, BaseDummyOptions]] = None,

213 ) -> MultiModalDataDict:

214 num_images = mm_counts.get("image", 0)

215

216 target_width, target_height = \

217 self.info.get_image_size_with_most_features()

218

219 + image_overrides = mm_options.get("image") if mm_options else None

220 +

221 return {

222 "image":

223 self._get_dummy_images(width=target_width,

224 height=target_height,

221 - num_images=num_images)

225 + num_images=num_images,

226 + overrides=image_overrides)

227 }

228

229

vllm/model_executor/models/mllama4.py

+6 -1

@@ -31,6 +31,7 @@ from transformers.models.llama4.image_processing_llama4_fast import (

31

32 from vllm.attention.layer import MultiHeadAttention

33 from vllm.config import VllmConfig

34 +from vllm.config.multimodal import BaseDummyOptions

35 from vllm.distributed import get_tensor_model_parallel_world_size

36 from vllm.model_executor.layers.linear import (ColumnParallelLinear,

37 QKVParallelLinear,

@@ -689,17 +690,21 @@ class Mllama4DummyInputsBuilder(BaseDummyInputsBuilder[Mllama4ProcessingInfo]):

690 self,

691 seq_len: int,

692 mm_counts: Mapping[str, int],

693 + mm_options: Optional[Mapping[str, BaseDummyOptions]] = None,

694 ) -> MultiModalDataDict:

695 num_images = mm_counts.get("image", 0)

696

697 (target_width,

698 target_height) = self.info.get_image_size_with_most_features()

699

700 + image_overrides = mm_options.get("image") if mm_options else None

701 +

702 return {

703 "image":

704 self._get_dummy_images(width=target_width,

705 height=target_height,

702 - num_images=num_images)

706 + num_images=num_images,

707 + overrides=image_overrides)

708 }

709

710

vllm/model_executor/models/molmo.py

+6 -1

@@ -22,6 +22,7 @@ from vllm.attention import Attention

22 from vllm.attention.layer import MultiHeadAttention

23 from vllm.compilation.decorators import support_torch_compile

24 from vllm.config import CacheConfig, VllmConfig

25 +from vllm.config.multimodal import BaseDummyOptions

26 from vllm.distributed import (get_pp_group, get_tensor_model_parallel_rank,

27 get_tensor_model_parallel_world_size,

28 split_tensor_along_last_dim,

@@ -1226,16 +1227,20 @@ class MolmoDummyInputsBuilder(BaseDummyInputsBuilder[MolmoProcessingInfo]):

1227 self,

1228 seq_len: int,

1229 mm_counts: Mapping[str, int],

1230 + mm_options: Optional[Mapping[str, BaseDummyOptions]] = None,

1231 ) -> MultiModalDataDict:

1232 target_width, target_height = \

1233 self.info.get_image_size_with_most_features()

1234 num_images = mm_counts.get("image", 0)

1235

1236 + image_overrides = mm_options.get("image") if mm_options else None

1237 +

1238 return {

1239 "image":

1240 self._get_dummy_images(width=target_width,

1241 height=target_height,

1238 - num_images=num_images)

1242 + num_images=num_images,

1243 + overrides=image_overrides)

1244 }

1245

1246

vllm/model_executor/models/nano_nemotron_vl.py

+12 -3

@@ -21,6 +21,7 @@ from PIL import Image

21 from transformers import BatchFeature, PretrainedConfig, TensorType

22

23 from vllm.config import VllmConfig

24 +from vllm.config.multimodal import BaseDummyOptions

25 from vllm.model_executor.layers.activation import ReLUSquaredActivation

26 from vllm.model_executor.layers.layernorm import RMSNorm

27 from vllm.model_executor.model_loader.weight_utils import default_weight_loader

@@ -809,6 +810,7 @@ class NanoNemotronVLDummyInputsBuilder(BaseDummyInputsBuilder[_I]):

810 self,

811 seq_len: int,

812 mm_counts: Mapping[str, int],

813 + mm_options: Optional[Mapping[str, BaseDummyOptions]] = None,

814 ) -> MultiModalDataDict:

815 # Use default max_num_tiles for dummy data generation

816 max_num_tiles = 12

@@ -816,11 +818,14 @@ class NanoNemotronVLDummyInputsBuilder(BaseDummyInputsBuilder[_I]):

818 self.info.get_image_size_with_most_features(max_num_tiles))

819 num_images = mm_counts.get("image", 0)

820

821 + image_overrides = mm_options.get("image") if mm_options else None

822 +

823 return {

824 "image":

825 self._get_dummy_images(width=target_width,

826 height=target_height,

823 - num_images=num_images)

827 + num_images=num_images,

828 + overrides=image_overrides)

829 }

830

831

@@ -837,21 +842,25 @@ class NanoNemotronVLDummyInputsBuilder(

842 self,

843 seq_len: int,

844 mm_counts: Mapping[str, int],

845 + mm_options: Optional[Mapping[str, BaseDummyOptions]] = None,

846 ) -> MultiModalDataDict:

847 dummy_image = super().get_dummy_mm_data(seq_len=seq_len,

842 - mm_counts=mm_counts)

848 + mm_counts=mm_counts,

849 + mm_options=mm_options)

850 if self.info.supports_video:

851 config = self.info.get_hf_config()

852 image_size: int = config.force_image_size

853 target_num_frames = \

854 self.info.get_num_frames_with_most_features(seq_len, mm_counts)

855 num_videos = mm_counts.get("video", 0)

856 + video_overrides = mm_options.get("video") if mm_options else None

857 dummy_video = {

858 "video":

859 self._get_dummy_videos(width=image_size,

860 height=image_size,

861 num_frames=target_num_frames,

854 - num_videos=num_videos)

862 + num_videos=num_videos,

863 + overrides=video_overrides)

864 }

865 else:

866 dummy_video = {}

vllm/model_executor/models/nvlm_d.py

+6 -1

@@ -14,6 +14,7 @@ import torch

14 import torch.nn as nn

15 from transformers import PretrainedConfig

16

17 +from vllm.config.multimodal import BaseDummyOptions

18 from vllm.model_executor.layers.quantization import QuantizationConfig

19 from vllm.multimodal import MULTIMODAL_REGISTRY

20 from vllm.multimodal.inputs import MultiModalDataDict, MultiModalKwargsItems

@@ -86,16 +87,20 @@ class NVLMDummyInputsBuilder(BaseInternVLDummyInputsBuilder[NVLMProcessingInfo]

87 self,

88 seq_len: int,

89 mm_counts: Mapping[str, int],

90 + mm_options: Optional[Mapping[str, BaseDummyOptions]] = None,

91 ) -> MultiModalDataDict:

92 target_width, target_height = \

93 self.info.get_image_size_with_most_features()

94 num_images = mm_counts.get("image", 0)

95

96 + image_overrides = mm_options.get("image") if mm_options else None

97 +

98 return {

99 "image":

100 self._get_dummy_images(width=target_width,

101 height=target_height,

98 - num_images=num_images)

102 + num_images=num_images,

103 + overrides=image_overrides)

104 }

105

106

vllm/model_executor/models/ovis.py

+6 -1

@@ -28,6 +28,7 @@ from torch.nn.functional import gumbel_softmax, pad, softmax

28 from transformers import BatchFeature, PretrainedConfig

29

30 from vllm.config import VllmConfig

31 +from vllm.config.multimodal import BaseDummyOptions

32 from vllm.model_executor.layers.linear import ReplicatedLinear

33 from vllm.model_executor.layers.quantization import QuantizationConfig

34 from vllm.model_executor.models.aimv2 import AIMv2Model

@@ -283,17 +284,21 @@ class OvisDummyInputsBuilder(BaseDummyInputsBuilder[OvisProcessingInfo]):

284 self,

285 seq_len: int,

286 mm_counts: Mapping[str, int],

287 + mm_options: Optional[Mapping[str, BaseDummyOptions]] = None,

288 ) -> MultiModalDataDict:

289 num_images = mm_counts.get("image", 0)

290

291 target_width, target_height = \

292 self.info.get_image_size_with_most_features()

293

294 + image_overrides = mm_options.get("image") if mm_options else None

295 +

296 mm_data = {

297 "image":

298 self._get_dummy_images(width=target_width,

299 height=target_height,

296 - num_images=num_images),

300 + num_images=num_images,

301 + overrides=image_overrides),

302 }

303 return mm_data

304

vllm/model_executor/models/ovis2_5.py

+9 -1

@@ -10,6 +10,7 @@ import torch.nn as nn

10 from transformers import BaseImageProcessor, BatchFeature, PretrainedConfig

11

12 from vllm.config import VllmConfig

13 +from vllm.config.multimodal import BaseDummyOptions

14 from vllm.model_executor.layers.linear import ReplicatedLinear

15 from vllm.model_executor.layers.quantization import QuantizationConfig

16 from vllm.model_executor.models.ovis import (OvisImagePatchInputs,

@@ -290,6 +291,7 @@ class Ovis2_5DummyInputsBuilder(BaseDummyInputsBuilder[Ovis2_5ProcessingInfo]):

291 self,

292 seq_len: int,

293 mm_counts: Mapping[str, int],

294 + mm_options: Optional[Mapping[str, BaseDummyOptions]] = None,

295 ) -> MultiModalDataDict:

296 num_images = mm_counts.get("image", 0)

297 num_videos = mm_counts.get("video", 0)

@@ -298,17 +300,23 @@ class Ovis2_5DummyInputsBuilder(BaseDummyInputsBuilder[Ovis2_5ProcessingInfo]):

300 self.info.get_image_size_with_most_features()

301 target_num_frames = \

302 self.info.get_num_frames_with_most_features(seq_len, mm_counts)

303 +

304 + image_overrides = mm_options.get("image") if mm_options else None

305 + video_overrides = mm_options.get("video") if mm_options else None

306 +

307 mm_data = {

308 "image":

309 self._get_dummy_images(width=target_width,

310 height=target_height,

305 - num_images=num_images),

311 + num_images=num_images,

312 + overrides=image_overrides),

313 "video":

314 self._get_dummy_videos(

315 width=target_width,

316 height=target_height,

317 num_frames=target_num_frames,

318 num_videos=num_videos,

319 + overrides=video_overrides,

320 )

321 }

322 return mm_data

vllm/model_executor/models/paligemma.py

+6 -1

@@ -8,6 +8,7 @@ from torch import nn

8 from transformers import BatchFeature, PaliGemmaConfig

9

10 from vllm.config import VllmConfig

11 +from vllm.config.multimodal import BaseDummyOptions

12 from vllm.logger import init_logger

13 from vllm.multimodal import MULTIMODAL_REGISTRY

14 from vllm.multimodal.inputs import (MultiModalDataDict, MultiModalFieldConfig,

@@ -106,6 +107,7 @@ class PaliGemmaDummyInputsBuilder(

107 self,

108 seq_len: int,

109 mm_counts: Mapping[str, int],

110 + mm_options: Optional[Mapping[str, BaseDummyOptions]] = None,

111 ) -> MultiModalDataDict:

112 hf_config = self.info.get_hf_config()

113 vision_config = hf_config.vision_config

@@ -113,11 +115,14 @@ class PaliGemmaDummyInputsBuilder(

115

116 num_images = mm_counts.get("image", 0)

117

118 + image_overrides = mm_options.get("image") if mm_options else None

119 +

120 return {

121 "image":

122 self._get_dummy_images(width=max_image_size,

123 height=max_image_size,

120 - num_images=num_images)

124 + num_images=num_images,

125 + overrides=image_overrides)

126 }

127

128

vllm/model_executor/models/phi3v.py

+6 -1

@@ -25,6 +25,7 @@ from transformers import (BatchFeature, CLIPVisionConfig, PretrainedConfig,

25 ProcessorMixin)

26

27 from vllm.config import VllmConfig

28 +from vllm.config.multimodal import BaseDummyOptions

29 from vllm.logger import init_logger

30 from vllm.model_executor.layers.quantization import QuantizationConfig

31 from vllm.model_executor.layers.vocab_parallel_embedding import (

@@ -356,17 +357,21 @@ class Phi3VDummyInputsBuilder(BaseDummyInputsBuilder[Phi3VProcessingInfo]):

357 self,

358 seq_len: int,

359 mm_counts: Mapping[str, int],

360 + mm_options: Optional[Mapping[str, BaseDummyOptions]] = None,

361 ) -> MultiModalDataDict:

362 num_images = mm_counts.get("image", 0)

363

364 target_width, target_height = \

365 self.info.get_image_size_with_most_features()

366

367 + image_overrides = mm_options.get("image") if mm_options else None

368 +

369 return {

370 "image":

371 self._get_dummy_images(width=target_width,

372 height=target_height,

369 - num_images=num_images)

373 + num_images=num_images,

374 + overrides=image_overrides)

375 }

376

377

vllm/model_executor/models/phi4_multimodal.py

+9 -2

@@ -17,6 +17,7 @@ from transformers.models.phi4_multimodal.modeling_phi4_multimodal import (

17 Phi4MultimodalAudioRelativeAttentionBias, adaptive_enc_mask, unfold_tensor)

18

19 from vllm.config import VllmConfig

20 +from vllm.config.multimodal import BaseDummyOptions

21 from vllm.distributed import (divide, get_tensor_model_parallel_rank,

22 get_tensor_model_parallel_world_size)

23 from vllm.model_executor.layers.activation import MulAndSilu, get_act_fn

@@ -980,6 +981,7 @@ class Phi4MMDummyInputsBuilder(BaseDummyInputsBuilder[Phi4MMProcessingInfo]):

981 self,

982 seq_len: int,

983 mm_counts: Mapping[str, int],

984 + mm_options: Optional[Mapping[str, BaseDummyOptions]] = None,

985 ) -> MultiModalDataDict:

986 num_audios = mm_counts.get("audio", 0)

987 num_images = mm_counts.get("image", 0)

@@ -987,14 +989,19 @@ class Phi4MMDummyInputsBuilder(BaseDummyInputsBuilder[Phi4MMProcessingInfo]):

989 target_width, target_height = \

990 self.info.get_image_size_with_most_features()

991

992 + image_overrides = mm_options.get("image") if mm_options else None

993 + audio_overrides = mm_options.get("audio") if mm_options else None

994 +

995 mm_data = {

996 "image":

997 self._get_dummy_images(width=target_width,

998 height=target_height,

994 - num_images=num_images),

999 + num_images=num_images,

1000 + overrides=image_overrides),

1001 "audio":

1002 self._get_dummy_audios(length=_AUDIO_MAX_SOUNDFILE_SIZE,

997 - num_audios=num_audios),

1003 + num_audios=num_audios,

1004 + overrides=audio_overrides),

1005 }

1006

1007 return mm_data

vllm/model_executor/models/phi4mm.py

+9 -2

@@ -11,6 +11,7 @@ from transformers import (BatchFeature, PretrainedConfig, ProcessorMixin,

11 SequenceFeatureExtractor, SiglipVisionConfig)

12

13 from vllm.config import VllmConfig

14 +from vllm.config.multimodal import BaseDummyOptions

15 from vllm.distributed import get_pp_group

16 from vllm.model_executor.layers.logits_processor import LogitsProcessor

17 from vllm.model_executor.layers.quantization import QuantizationConfig

@@ -749,6 +750,7 @@ class Phi4MMDummyInputsBuilder(BaseDummyInputsBuilder[Phi4MMProcessingInfo]):

750 self,

751 seq_len: int,

752 mm_counts: Mapping[str, int],

753 + mm_options: Optional[Mapping[str, BaseDummyOptions]] = None,

754 ) -> MultiModalDataDict:

755 num_audios = mm_counts.get("audio", 0)

756 num_images = mm_counts.get("image", 0)

@@ -756,14 +758,19 @@ class Phi4MMDummyInputsBuilder(BaseDummyInputsBuilder[Phi4MMProcessingInfo]):

758 target_width, target_height = \

759 self.info.get_image_size_with_most_features()

760

761 + image_overrides = mm_options.get("image") if mm_options else None

762 + audio_overrides = mm_options.get("audio") if mm_options else None

763 +

764 mm_data = {

765 "image":

766 self._get_dummy_images(width=target_width,

767 height=target_height,

763 - num_images=num_images),

768 + num_images=num_images,

769 + overrides=image_overrides),

770 "audio":

771 self._get_dummy_audios(length=_AUDIO_MAX_SOUNDFILE_SIZE,

766 - num_audios=num_audios),

772 + num_audios=num_audios,

773 + overrides=audio_overrides),

774 }

775

776 return mm_data

vllm/model_executor/models/pixtral.py

+8 -2

@@ -24,6 +24,7 @@ from transformers.models.pixtral.modeling_pixtral import (

24 from transformers.tokenization_utils_base import TextInput

25

26 from vllm.config import VllmConfig

27 +from vllm.config.multimodal import BaseDummyOptions

28 from vllm.distributed import divide, get_tensor_model_parallel_world_size

29 from vllm.model_executor.layers.activation import get_act_and_mul_fn

30 from vllm.model_executor.layers.layernorm import RMSNorm

@@ -228,28 +229,33 @@ class PixtralDummyInputsBuilder(BaseDummyInputsBuilder[PixtralProcessingInfo]):

229 self,

230 seq_len: int,

231 mm_counts: Mapping[str, int],

232 + mm_options: Optional[Mapping[str, BaseDummyOptions]] = None,

233 ) -> MultiModalDataDict:

234 num_images = mm_counts.get("image", 0)

235

236 target_width, target_height = \

237 self.info.get_image_size_with_most_features()

238

239 + image_overrides = mm_options.get("image") if mm_options else None

240 +

241 return {

242 "image":

243 self._get_dummy_images(width=target_width,

244 height=target_height,

241 - num_images=num_images)

245 + num_images=num_images,

246 + overrides=image_overrides)

247 }

248

249 def get_dummy_processor_inputs(

250 self,

251 seq_len: int,

252 mm_counts: Mapping[str, int],

253 + mm_options: Optional[Mapping[str, BaseDummyOptions]] = None,

254 ) -> ProcessorInputs:

255 tokenizer = self.info.get_tokenizer()

256

257 dummy_text = self.get_dummy_text(mm_counts)

252 - dummy_mm_data = self.get_dummy_mm_data(seq_len, mm_counts)

258 + dummy_mm_data = self.get_dummy_mm_data(seq_len, mm_counts, mm_options)

259 dummy_images = dummy_mm_data.get("image", [])

260 tokenization_kwargs = {"truncation": False}

261

vllm/model_executor/models/qwen2_5_omni_thinker.py

+12 -3

@@ -39,6 +39,7 @@ from transformers.models.qwen2_5_omni.processing_qwen2_5_omni import (

39 from transformers.models.whisper import WhisperFeatureExtractor

40

41 from vllm.config import VllmConfig

42 +from vllm.config.multimodal import BaseDummyOptions

43 from vllm.logger import init_logger

44 from vllm.model_executor.layers.rotary_embedding import MRotaryEmbedding

45 from vllm.model_executor.models.module_mapping import MultiModelKeys

@@ -212,6 +213,7 @@ class Qwen2_5OmniThinkerDummyInputsBuilder(

213 self,

214 seq_len: int,

215 mm_counts: Mapping[str, int],

216 + mm_options: Optional[Mapping[str, BaseDummyOptions]] = None,

217 ) -> MultiModalDataDict:

218 num_audios = mm_counts.get("audio", 0)

219 num_images = mm_counts.get("image", 0)

@@ -228,19 +230,26 @@ class Qwen2_5OmniThinkerDummyInputsBuilder(

230 target_num_frames = \

231 self.info.get_num_frames_with_most_features(seq_len, mm_counts)

232

233 + image_overrides = mm_options.get("image") if mm_options else None

234 + video_overrides = mm_options.get("video") if mm_options else None

235 + audio_overrides = mm_options.get("audio") if mm_options else None

236 +

237 mm_data = {

238 "audio":

239 self._get_dummy_audios(length=target_audio_length,

234 - num_audios=num_audios),

240 + num_audios=num_audios,

241 + overrides=audio_overrides),

242 "image":

243 self._get_dummy_images(width=target_width,

244 height=target_height,

238 - num_images=num_images),

245 + num_images=num_images,

246 + overrides=image_overrides),

247 "video":

248 self._get_dummy_videos(width=target_width,

249 height=target_height,

250 num_frames=target_num_frames,

243 - num_videos=num_videos),

251 + num_videos=num_videos,

252 + overrides=video_overrides),

253 }

254

255 return mm_data

vllm/model_executor/models/qwen2_audio.py

+7 -1

@@ -34,6 +34,7 @@ from transformers.models.qwen2_audio import (Qwen2AudioConfig,

34 from transformers.models.whisper import WhisperFeatureExtractor

35

36 from vllm.config import VllmConfig

37 +from vllm.config.multimodal import BaseDummyOptions

38 from vllm.multimodal import MULTIMODAL_REGISTRY

39 from vllm.multimodal.inputs import (AudioItem, ModalityData,

40 MultiModalDataDict, MultiModalFieldConfig,

@@ -144,6 +145,7 @@ class Qwen2AudioDummyInputsBuilder(

145 self,

146 seq_len: int,

147 mm_counts: Mapping[str, int],

148 + mm_options: Optional[Mapping[str, BaseDummyOptions]] = None,

149 ) -> MultiModalDataDict:

150 feature_extractor = self.info.get_feature_extractor()

151

@@ -151,9 +153,13 @@ class Qwen2AudioDummyInputsBuilder(

153 audio_len = feature_extractor.chunk_length * sampling_rate

154 num_audios = mm_counts.get("audio", 0)

155

156 + audio_overrides = mm_options.get("audio") if mm_options else None

157 +

158 return {

159 "audio":

156 - self._get_dummy_audios(length=audio_len, num_audios=num_audios)

160 + self._get_dummy_audios(length=audio_len,

161 + num_audios=num_audios,

162 + overrides=audio_overrides)

163 }

164

165

vllm/model_executor/models/qwen2_vl.py

+8 -1

@@ -45,6 +45,7 @@ from vllm.attention.backends.registry import _Backend

45 from vllm.attention.layer import (check_upstream_fa_availability,

46 maybe_get_vit_flash_attn_backend)

47 from vllm.config import VllmConfig

48 +from vllm.config.multimodal import BaseDummyOptions

49 from vllm.distributed import parallel_state, tensor_model_parallel_all_gather

50 from vllm.distributed import utils as dist_utils

51 from vllm.logger import init_logger

@@ -1034,6 +1035,7 @@ class Qwen2VLDummyInputsBuilder(BaseDummyInputsBuilder[Qwen2VLProcessingInfo]):

1035 self,

1036 seq_len: int,

1037 mm_counts: Mapping[str, int],

1038 + mm_options: Optional[Mapping[str, BaseDummyOptions]] = None,

1039 ) -> MultiModalDataDict:

1040 num_images = mm_counts.get("image", 0)

1041 num_videos = mm_counts.get("video", 0)

@@ -1043,17 +1045,22 @@ class Qwen2VLDummyInputsBuilder(BaseDummyInputsBuilder[Qwen2VLProcessingInfo]):

1045 target_num_frames = \

1046 self.info.get_num_frames_with_most_features(seq_len, mm_counts)

1047

1048 + image_overrides = mm_options.get("image") if mm_options else None

1049 + video_overrides = mm_options.get("video") if mm_options else None

1050 +

1051 return {

1052 "image":

1053 self._get_dummy_images(width=target_width,

1054 height=target_height,

1050 - num_images=num_images),

1055 + num_images=num_images,

1056 + overrides=image_overrides),

1057 "video":

1058 self._get_dummy_videos(

1059 width=target_width,

1060 height=target_height,

1061 num_frames=target_num_frames,

1062 num_videos=num_videos,

1063 + overrides=video_overrides,

1064 )

1065 }

1066

vllm/model_executor/models/qwen3_vl.py

+9 -1

@@ -47,6 +47,7 @@ from vllm.attention.backends.registry import _Backend

47 from vllm.attention.layer import check_upstream_fa_availability

48 from vllm.compilation.decorators import support_torch_compile

49 from vllm.config import VllmConfig

50 +from vllm.config.multimodal import BaseDummyOptions

51 from vllm.distributed import get_pp_group

52 from vllm.logger import init_logger

53 from vllm.model_executor.layers.activation import _ACTIVATION_REGISTRY

@@ -736,6 +737,7 @@ class Qwen3VLDummyInputsBuilder(BaseDummyInputsBuilder[Qwen3VLProcessingInfo]):

737 self,

738 seq_len: int,

739 mm_counts: Mapping[str, int],

740 + mm_options: Optional[Mapping[str, BaseDummyOptions]] = None,

741 ) -> MultiModalDataDict:

742 num_images = mm_counts.get("image", 0)

743 num_videos = mm_counts.get("video", 0)

@@ -750,17 +752,23 @@ class Qwen3VLDummyInputsBuilder(BaseDummyInputsBuilder[Qwen3VLProcessingInfo]):

752 num_frames=target_num_frames,

753 image_processor=self.info.get_video_processor(),

754 )

755 +

756 + image_overrides = mm_options.get("image") if mm_options else None

757 + video_overrides = mm_options.get("video") if mm_options else None

758 +

759 return {

760 "image":

761 self._get_dummy_images(width=target_width,

762 height=target_height,

757 - num_images=num_images),

763 + num_images=num_images,

764 + overrides=image_overrides),

765 "video":

766 self._get_dummy_videos(

767 width=target_video_size.width,

768 height=target_video_size.height,

769 num_frames=target_num_frames,

770 num_videos=num_videos,

771 + overrides=video_overrides,

772 ),

773 }

774

vllm/model_executor/models/qwen_vl.py

+6 -1

@@ -24,6 +24,7 @@ from transformers.image_utils import ImageInput

24 from transformers.tokenization_utils_base import TextInput

25

26 from vllm.config import VllmConfig

27 +from vllm.config.multimodal import BaseDummyOptions

28 from vllm.model_executor.layers.activation import get_act_fn

29 from vllm.model_executor.layers.linear import (ColumnParallelLinear,

30 ReplicatedLinear,

@@ -567,6 +568,7 @@ class QwenVLDummyInputsBuilder(BaseDummyInputsBuilder[QwenVLProcessingInfo]):

568 self,

569 seq_len: int,

570 mm_counts: Mapping[str, int],

571 + mm_options: Optional[Mapping[str, BaseDummyOptions]] = None,

572 ) -> MultiModalDataDict:

573 hf_config = self.info.get_hf_config()

574 vision_config = hf_config.visual

@@ -574,11 +576,14 @@ class QwenVLDummyInputsBuilder(BaseDummyInputsBuilder[QwenVLProcessingInfo]):

576 target_width = target_height = vision_config["image_size"]

577 num_images = mm_counts.get("image", 0)

578

579 + image_overrides = mm_options.get("image") if mm_options else None

580 +

581 return {

582 "image":

583 self._get_dummy_images(width=target_width,

584 height=target_height,

581 - num_images=num_images)

585 + num_images=num_images,

586 + overrides=image_overrides)

587 }

588

589

vllm/model_executor/models/rvl.py

+7 -1

@@ -2,12 +2,14 @@

2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project

3

4 from collections.abc import Mapping

5 +from typing import Optional

6

7 import torch

8 import torch.nn as nn

9 from transformers.activations import GELUActivation

10

11 from vllm.config import VllmConfig

12 +from vllm.config.multimodal import BaseDummyOptions

13 from vllm.multimodal import MULTIMODAL_REGISTRY

14 from vllm.multimodal.inputs import MultiModalDataDict

15

@@ -38,17 +40,21 @@ class RVLDummyInputsBuilder(LlavaDummyInputsBuilder[RVLProcessingInfo]):

40 self,

41 seq_len: int,

42 mm_counts: Mapping[str, int],

43 + mm_options: Optional[Mapping[str, BaseDummyOptions]] = None,

44 ) -> MultiModalDataDict:

45 num_images = mm_counts.get("image", 0)

46

47 target_width, target_height = (

48 self.info.get_image_size_with_most_features())

49

50 + image_overrides = mm_options.get("image") if mm_options else None

51 +

52 return {

53 "image":

54 self._get_dummy_images(width=target_width,

55 height=target_height,

51 - num_images=num_images),

56 + num_images=num_images,

57 + overrides=image_overrides),

58 }

59

60

vllm/model_executor/models/skyworkr1v.py

+6 -1

@@ -17,6 +17,7 @@ from PIL import Image

17 from transformers import BatchFeature, PretrainedConfig, TensorType

18

19 from vllm.config import VllmConfig

20 +from vllm.config.multimodal import BaseDummyOptions

21 from vllm.model_executor.layers.linear import ReplicatedLinear

22 from vllm.model_executor.layers.quantization import QuantizationConfig

23 from vllm.model_executor.layers.quantization.awq import AWQConfig

@@ -505,16 +506,20 @@ class SkyworkR1VDummyInputsBuilder(

506 self,

507 seq_len: int,

508 mm_counts: Mapping[str, int],

509 + mm_options: Optional[Mapping[str, BaseDummyOptions]] = None,

510 ) -> MultiModalDataDict:

511 target_width, target_height = \

512 self.info.get_image_size_with_most_features()

513 num_images = mm_counts.get("image", 0)

514

515 + image_overrides = mm_options.get("image") if mm_options else None

516 +

517 return {

518 "image":

519 self._get_dummy_images(width=target_width,

520 height=target_height,

517 - num_images=num_images)

521 + num_images=num_images,

522 + overrides=image_overrides)

523 }

524

525

vllm/model_executor/models/step3_vl.py

+6 -1

@@ -17,6 +17,7 @@ from transformers import BatchFeature, PretrainedConfig, TensorType

17

18 from vllm.attention.layer import MultiHeadAttention

19 from vllm.config import VllmConfig

20 +from vllm.config.multimodal import BaseDummyOptions

21 from vllm.distributed import get_tensor_model_parallel_world_size

22 from vllm.model_executor.layers.activation import get_act_fn

23 from vllm.model_executor.layers.linear import (ColumnParallelLinear,

@@ -496,16 +497,20 @@ class Step3VLDummyInputsBuilder(BaseDummyInputsBuilder[Step3VLProcessingInfo]):

497 self,

498 seq_len: int,

499 mm_counts: Mapping[str, int],

500 + mm_options: Optional[Mapping[str, BaseDummyOptions]] = None,

501 ) -> MultiModalDataDict:

502 target_width, target_height = \

503 self.info.get_image_size_with_most_features()

504 num_images = mm_counts.get("image", 0)

505

506 + image_overrides = mm_options.get("image") if mm_options else None

507 +

508 return {

509 "image":

510 self._get_dummy_images(width=target_width,

511 height=target_height,

508 - num_images=num_images)

512 + num_images=num_images,

513 + overrides=image_overrides)

514 }

515

516

vllm/model_executor/models/terratorch.py

+11

@@ -28,6 +28,8 @@ from terratorch.vllm import (DummyDataGenerator, InferenceRunner,

28 from transformers import BatchFeature

29

30 from vllm.config import VllmConfig

31 +from vllm.config.multimodal import BaseDummyOptions

32 +from vllm.logger import init_logger

33 from vllm.model_executor.layers.pooler import DispatchPooler, Pooler

34 from vllm.model_executor.model_loader.weight_utils import default_weight_loader

35 from vllm.model_executor.models.utils import AutoWeightsLoader

@@ -48,6 +50,8 @@ from .interfaces import (IsAttentionFree, MultiModalEmbeddings,

50 SupportsMultiModal)

51 from .interfaces_base import default_pooling_type

52

53 +logger = init_logger(__name__)

54 +

55

56 def _terratorch_field_names(pretrained_cfg: dict):

57 input_definition = InputDefinition(**pretrained_cfg["input"])

@@ -97,9 +101,16 @@ class TerratorchInputBuilder(BaseDummyInputsBuilder[TerratorchProcessingInfo]):

101 self,

102 seq_len: int,

103 mm_counts: Mapping[str, int],

104 + mm_options: Optional[Mapping[str, BaseDummyOptions]] = None,

105 ) -> MultiModalDataDict:

106 # Dummy data is generated based on the 'input' section

107 # defined in the HF configuration file

108 +

109 + if mm_options:

110 + logger.warning("Configurable multimodal profiling "

111 + "options are not supported for Terratorch. "

112 + "They are ignored for now.")

113 +

114 return self.dummy_data_generator.get_dummy_mm_data()

115

116

vllm/model_executor/models/transformers.py

+6 -1

@@ -33,6 +33,7 @@ from vllm.attention import Attention, AttentionType

33 from vllm.compilation.decorators import support_torch_compile

34 from vllm.config import (CacheConfig, DeviceConfig, ModelConfig,

35 ParallelConfig, VllmConfig)

36 +from vllm.config.multimodal import BaseDummyOptions

37 from vllm.config.utils import getattr_iter

38 from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size

39 from vllm.distributed.utils import get_pp_indices

@@ -285,16 +286,20 @@ class MultiModalDummyInputsBuilder(

286 self,

287 seq_len: int,

288 mm_counts: Mapping[str, int],

289 + mm_options: Optional[Mapping[str, BaseDummyOptions]] = None,

290 ) -> MultiModalDataDict:

291 num_images = mm_counts.get("image", 0)

292

293 target_width, target_height = self.info.get_max_image_size()

294

295 + image_overrides = mm_options.get("image") if mm_options else None

296 +

297 return {

298 "image":

299 self._get_dummy_images(width=target_width,

300 height=target_height,

297 - num_images=num_images),

301 + num_images=num_images,

302 + overrides=image_overrides),

303 }

304

305

vllm/model_executor/models/ultravox.py

+7 -1

@@ -14,6 +14,7 @@ from transformers.models.whisper import WhisperFeatureExtractor

14 from transformers.models.whisper.modeling_whisper import WhisperEncoder

15

16 from vllm.config import VllmConfig

17 +from vllm.config.multimodal import BaseDummyOptions

18 from vllm.model_executor.layers.activation import MulAndSilu, get_act_fn

19 from vllm.model_executor.layers.layernorm import RMSNorm

20 from vllm.model_executor.model_loader import DefaultModelLoader

@@ -114,6 +115,7 @@ class UltravoxDummyInputsBuilder(BaseDummyInputsBuilder[UltravoxProcessingInfo]

115 self,

116 seq_len: int,

117 mm_counts: Mapping[str, int],

118 + mm_options: Optional[Mapping[str, BaseDummyOptions]] = None,

119 ) -> MultiModalDataDict:

120 feature_extractor = self.info.get_feature_extractor()

121

@@ -122,9 +124,13 @@ class UltravoxDummyInputsBuilder(BaseDummyInputsBuilder[UltravoxProcessingInfo]

124 _MAX_ENCODER_BATCH_SIZE)

125 num_audios = mm_counts.get("audio", 0)

126

127 + audio_overrides = mm_options.get("audio") if mm_options else None

128 +

129 return {

130 "audio":

127 - self._get_dummy_audios(length=audio_len, num_audios=num_audios)

131 + self._get_dummy_audios(length=audio_len,

132 + num_audios=num_audios,

133 + overrides=audio_overrides)

134 }

135

136

vllm/model_executor/models/voxtral.py

+9 -2

@@ -21,6 +21,7 @@ from transformers import BatchFeature, TensorType, WhisperConfig

21 from transformers.tokenization_utils_base import TextInput

22

23 from vllm.config import ModelConfig, SpeechToTextConfig, VllmConfig

24 +from vllm.config.multimodal import BaseDummyOptions

25 from vllm.inputs.data import PromptType

26 from vllm.logger import init_logger

27 from vllm.model_executor.layers.quantization import QuantizationConfig

@@ -204,25 +205,31 @@ class VoxtralDummyInputsBuilder(BaseDummyInputsBuilder[VoxtralProcessingInfo]):

205 self,

206 seq_len: int,

207 mm_counts: Mapping[str, int],

208 + mm_options: Optional[Mapping[str, BaseDummyOptions]] = None,

209 ) -> MultiModalDataDict:

210 num_audios = mm_counts.get("audio", 0)

211

212 target_length = self.info.get_max_audio_array_len()

213

214 + audio_overrides = mm_options.get("audio") if mm_options else None

215 +

216 return {

217 "audio":

214 - self._get_dummy_audios(length=target_length, num_audios=num_audios)

218 + self._get_dummy_audios(length=target_length,

219 + num_audios=num_audios,

220 + overrides=audio_overrides)

221 }

222

223 def get_dummy_processor_inputs(

224 self,

225 seq_len: int,

226 mm_counts: Mapping[str, int],

227 + mm_options: Optional[Mapping[str, BaseDummyOptions]] = None,

228 ) -> ProcessorInputs:

229 tokenizer = self.info.get_tokenizer()

230

231 dummy_text = self.get_dummy_text(mm_counts)

225 - dummy_mm_data = self.get_dummy_mm_data(seq_len, mm_counts)

232 + dummy_mm_data = self.get_dummy_mm_data(seq_len, mm_counts, mm_options)

233 dummy_audios = dummy_mm_data.get("audio", [])

234

235 audio_chunks: list[AudioChunk] = []

vllm/model_executor/models/whisper.py

+7 -1

@@ -18,6 +18,7 @@ from vllm.attention.layer import MultiHeadAttention

18 from vllm.attention.layers.cross_attention import CrossAttention

19 from vllm.config import (CacheConfig, ModelConfig, SpeechToTextConfig,

20 VllmConfig)

21 +from vllm.config.multimodal import BaseDummyOptions

22 from vllm.distributed import get_tensor_model_parallel_world_size

23 from vllm.inputs.data import PromptType

24 from vllm.logger import init_logger

@@ -691,6 +692,7 @@ class WhisperDummyInputsBuilder(BaseDummyInputsBuilder[WhisperProcessingInfo]):

692 self,

693 seq_len: int,

694 mm_counts: Mapping[str, int],

695 + mm_options: Optional[Mapping[str, BaseDummyOptions]] = None,

696 ) -> MultiModalDataDict:

697 feature_extractor = self.info.get_feature_extractor()

698

@@ -698,9 +700,13 @@ class WhisperDummyInputsBuilder(BaseDummyInputsBuilder[WhisperProcessingInfo]):

700 audio_len = feature_extractor.chunk_length * sampling_rate

701 num_audios = mm_counts.get("audio", 0)

702

703 + audio_overrides = mm_options.get("audio") if mm_options else None

704 +

705 return {

706 "audio":

703 - self._get_dummy_audios(length=audio_len, num_audios=num_audios)

707 + self._get_dummy_audios(length=audio_len,

708 + num_audios=num_audios,

709 + overrides=audio_overrides)

710 }

711

712

vllm/multimodal/profiling.py

+75 -5

@@ -10,6 +10,8 @@ import numpy.typing as npt

10 from PIL import Image

11

12 import vllm.envs as envs

13 +from vllm.config.multimodal import (AudioDummyOptions, BaseDummyOptions,

14 + ImageDummyOptions, VideoDummyOptions)

15 from vllm.logger import init_logger

16

17 from .inputs import (MultiModalDataDict, MultiModalEncDecInputs,

@@ -73,10 +75,19 @@ class BaseDummyInputsBuilder(ABC, Generic[_I]):

75 self,

76 seq_len: int,

77 mm_counts: Mapping[str, int],

78 + mm_options: Optional[Mapping[str, BaseDummyOptions]] = None,

79 ) -> MultiModalDataDict:

80 """

81 Build the multimodal input which, after processing, results in

82 the maximum possible number of placeholder tokens.

83 +

84 + Args:

85 + seq_len: Sequence length

86 + mm_counts: Count of items per modality

87 + mm_options: Configurable options per modality (optional).

88 + If None, use model defaults for backward compatibility.

89 + If provided, models can use these to customize dummy

90 + data generation.

91 """

92 raise NotImplementedError

93

@@ -84,13 +95,22 @@ class BaseDummyInputsBuilder(ABC, Generic[_I]):

95 self,

96 seq_len: int,

97 mm_counts: Mapping[str, int],

98 + mm_options: Optional[Mapping[str, BaseDummyOptions]] = None,

99 ) -> ProcessorInputs:

100 """

101 Build the input which, after processing, results in

102 the maximum possible number of placeholder tokens.

103 +

104 + Args:

105 + seq_len: Sequence length

106 + mm_counts: Count of items per modality

107 + mm_options: Configurable options per modality (optional)

108 """

109 dummy_text = self.get_dummy_text(mm_counts)

93 - dummy_mm_data = self.get_dummy_mm_data(seq_len, mm_counts)

110 +

111 + # Use the unified function for both legacy and configurable cases

112 + dummy_mm_data = self.get_dummy_mm_data(seq_len, mm_counts, mm_options)

113 +

114 tokenization_kwargs = {"truncation": False}

115

116 return ProcessorInputs(prompt=dummy_text,

@@ -102,9 +122,17 @@ class BaseDummyInputsBuilder(ABC, Generic[_I]):

122 *,

123 length: int,

124 num_audios: int,

125 + overrides: Optional[AudioDummyOptions] = None,

126 ) -> list[npt.NDArray]:

127 if num_audios == 0:

128 return []

129 + if overrides and overrides.length:

130 + if overrides.length > length:

131 + logger.warning(

132 + "audio.length override (%d) exceeds model's "

133 + "maximum length (%d), will be ignored", overrides.length,

134 + length)

135 + length = min(length, overrides.length)

136 audio = np.zeros((length, ))

137 return [audio] * num_audios

138

@@ -114,9 +142,25 @@ class BaseDummyInputsBuilder(ABC, Generic[_I]):

142 width: int,

143 height: int,

144 num_images: int,

145 + overrides: Optional[ImageDummyOptions] = None,

146 ) -> list[Image.Image]:

147 if num_images == 0:

148 return []

149 + if overrides:

150 + if overrides.width:

151 + if overrides.width > width:

152 + logger.warning(

153 + "image.width override (%d) exceeds model's "

154 + "maximum width (%d), will be ignored", overrides.width,

155 + width)

156 + width = min(width, overrides.width)

157 + if overrides.height:

158 + if overrides.height > height:

159 + logger.warning(

160 + "image.height override (%d) exceeds model's "

161 + "maximum height (%d), will be ignored",

162 + overrides.height, height)

163 + height = min(height, overrides.height)

164 image = Image.new("RGB", (width, height), color=255)

165 return [image] * num_images

166

@@ -127,9 +171,32 @@ class BaseDummyInputsBuilder(ABC, Generic[_I]):

171 height: int,

172 num_frames: int,

173 num_videos: int,

174 + overrides: Optional[VideoDummyOptions] = None,

175 ) -> list[npt.NDArray]:

176 if num_videos == 0:

177 return []

178 + if overrides:

179 + if overrides.num_frames:

180 + if overrides.num_frames > num_frames:

181 + logger.warning(

182 + "video.num_frames override (%d) exceeds model's "

183 + "maximum number of frames (%d), will be ignored",

184 + overrides.num_frames, num_frames)

185 + num_frames = min(num_frames, overrides.num_frames)

186 + if overrides.width:

187 + if overrides.width > width:

188 + logger.warning(

189 + "video.width override (%d) exceeds model's "

190 + "maximum width (%d), will be ignored", overrides.width,

191 + width)

192 + width = min(width, overrides.width)

193 + if overrides.height:

194 + if overrides.height > height:

195 + logger.warning(

196 + "video.height override (%d) exceeds model's "

197 + "maximum height (%d), will be ignored",

198 + overrides.height, height)

199 + height = min(height, overrides.height)

200 video = np.full((num_frames, width, height, 3), 255)

201 return [video] * num_videos

202

@@ -162,13 +229,14 @@ class MultiModalProfiler(Generic[_I]):

229 self,

230 seq_len: int,

231 mm_counts: Optional[Mapping[str, int]] = None,

232 + mm_options: Optional[Mapping[str, BaseDummyOptions]] = None,

233 ) -> MultiModalInputs:

234 if mm_counts is None:

235 mm_counts = self.get_mm_limits()

236

237 factory = self.dummy_inputs

238 processor_inputs = factory.get_dummy_processor_inputs(

171 - seq_len, mm_counts)

239 + seq_len, mm_counts, mm_options)

240

241 return self.processor.apply(

242 prompt=processor_inputs.prompt,

@@ -195,8 +263,9 @@ class MultiModalProfiler(Generic[_I]):

263 self,

264 seq_len: int,

265 mm_counts: Optional[Mapping[str, int]] = None,

266 + mm_options: Optional[Mapping[str, BaseDummyOptions]] = None,

267 ) -> DummyEncoderData:

199 - mm_inputs = self._get_dummy_mm_inputs(seq_len, mm_counts)

268 + mm_inputs = self._get_dummy_mm_inputs(seq_len, mm_counts, mm_options)

269 mm_inputs = cast(MultiModalEncDecInputs, mm_inputs)

270

271 # For encoder-decoder models, use encoder prompt token ids instead of

@@ -228,8 +297,9 @@ class MultiModalProfiler(Generic[_I]):

297 self,

298 seq_len: int,

299 mm_counts: Optional[Mapping[str, int]] = None,

300 + mm_options: Optional[Mapping[str, BaseDummyOptions]] = None,

301 ) -> DummyDecoderData:

232 - mm_inputs = self._get_dummy_mm_inputs(seq_len, mm_counts)

302 + mm_inputs = self._get_dummy_mm_inputs(seq_len, mm_counts, mm_options)

303

304 prompt_token_ids = mm_inputs["prompt_token_ids"]

305 total_len = len(prompt_token_ids)

@@ -274,7 +344,7 @@ class MultiModalProfiler(Generic[_I]):

344

345 `<im_start> [IMG] [IMG] [IMG] <row_break> [IMG] [IMG] [IMG] <im_end>`

346 Returns 9, even when the number of image embeddings is 6.

277 -

347 +

348 This is important to take into account when profiling and

349 initializing the encoder cache size.

350 """

vllm/multimodal/registry.py

+44 -7

@@ -6,6 +6,7 @@ from typing import TYPE_CHECKING, Generic, Optional, Protocol, TypeVar

6

7 import torch.nn as nn

8

9 +from vllm.config.multimodal import BaseDummyOptions

10 from vllm.logger import init_logger

11 from vllm.transformers_utils.tokenizer import (AnyTokenizer,

12 cached_tokenizer_from_config)

@@ -52,7 +53,7 @@ class DummyInputsBuilderFactory(Protocol[_I]): # type: ignore[misc]

53 ...

54

55

55 -class MultiModalProcessorFactory(Protocol[_I]):

56 +class MultiModalProcessorFactory(Protocol[_I]): # type: ignore[misc]

57 """

58 Constructs a

59 [`BaseMultiModalProcessor`][vllm.multimodal.processing.BaseMultiModalProcessor]

@@ -95,6 +96,28 @@ class MultiModalRegistry:

96 self._processor_factories = ClassRegistry[nn.Module,

97 _ProcessorFactories]()

98

99 + def _extract_mm_options(

100 + self,

101 + model_config: "ModelConfig",

102 + ) -> Optional[Mapping[str, BaseDummyOptions]]:

103 + """

104 + Extract multimodal dummy options from model config.

105 +

106 + Returns None if no configurable options are found, otherwise returns

107 + a mapping of modality names to their dummy options.

108 + """

109 + if not model_config.multimodal_config:

110 + return None

111 +

112 + mm_options = {

113 + m: opt

114 + for m in model_config.multimodal_config.limit_per_prompt

115 + if (opt := model_config.multimodal_config.get_dummy_options(m)

116 + ) is not None

117 + }

118 +

119 + return mm_options if len(mm_options) > 0 else None

120 +

121 def supports_multimodal_inputs(self, model_config: "ModelConfig") -> bool:

122 """

123 Checks if the model supports multimodal inputs.

@@ -135,7 +158,7 @@ class MultiModalRegistry:

158 return {}

159

160 processor = self.create_processor(model_config, cache=cache)

138 - profiler = MultiModalProfiler(processor)

161 + profiler: MultiModalProfiler = MultiModalProfiler(processor)

162

163 seq_len = model_config.max_model_len

164 mm_limits = self.get_mm_limits_per_prompt(model_config, cache=cache)

@@ -189,7 +212,7 @@ class MultiModalRegistry:

212 return {}

213

214 processor = self.create_processor(model_config, cache=cache)

192 - profiler = MultiModalProfiler(processor)

215 + profiler: MultiModalProfiler = MultiModalProfiler(processor)

216 return profiler.get_mm_limits()

217

218 def register_processor(

@@ -285,8 +308,15 @@ class MultiModalRegistry:

308 The model is identified by ``model_config``.

309 """

310 processor = self.create_processor(model_config, cache=cache)

288 - profiler = MultiModalProfiler(processor)

289 - dummy_data = profiler.get_decoder_dummy_data(seq_len, mm_counts)

311 + profiler: MultiModalProfiler = MultiModalProfiler(processor)

312 +

313 + # Extract configurable options from multimodal config.

314 + # Only include modalities that use advanced option types so legacy

315 + # count-only behavior remains unchanged.

316 + mm_options = self._extract_mm_options(model_config)

317 +

318 + dummy_data = profiler.get_decoder_dummy_data(seq_len, mm_counts,

319 + mm_options)

320

321 # Having more tokens is over-conservative but otherwise fine

322 token_ids = dummy_data.prompt_token_ids

@@ -311,8 +341,15 @@ class MultiModalRegistry:

341 The model is identified by ``model_config``.

342 """

343 processor = self.create_processor(model_config, cache=cache)

314 - profiler = MultiModalProfiler(processor)

315 - dummy_data = profiler.get_encoder_dummy_data(seq_len, mm_counts)

344 + profiler: MultiModalProfiler = MultiModalProfiler(processor)

345 +

346 + # Extract configurable options from multimodal config.

347 + # Only include modalities that use advanced option types so legacy

348 + # count-only behavior remains unchanged.

349 + mm_options = self._extract_mm_options(model_config)

350 +

351 + dummy_data = profiler.get_encoder_dummy_data(seq_len, mm_counts,

352 + mm_options)

353

354 # Having more tokens is over-conservative but otherwise fine

355 token_ids = dummy_data.prompt_token_ids

[Multi Modal] Configurable MM Profiling (#25631)