b024a42e93 · tduncan/Nvidia-Optimization-Stack-VLLM

docs/contributing/model/multimodal.md

+16

@@ -10,6 +10,22 @@ This document walks you through the steps to extend a basic model so that it acc

10 It is assumed that you have already implemented the model in vLLM according to [these steps][new-model-basic].

11 Further update the model as follows:

12

13 +- Implement [get_placeholder_str][vllm.model_executor.models.interfaces.SupportsMultiModal.get_placeholder_str] to define the placeholder string which is used to represent the multi-modal item in the text prompt. This should be consistent with the chat template of the model.

14 +

15 + ??? Code

16 +

17 + ```python

18 + class YourModelForImage2Seq(nn.Module):

19 + ...

20 +

21 + @classmethod

22 + def get_placeholder_str(cls, modality: str, i: int) -> Optional[str]:

23 + if modality.startswith("image"):

24 + return "<image>"

25 +

26 + raise ValueError("Only image modality is supported")

27 + ```

28 +

29 - Reserve a keyword parameter in [forward][torch.nn.Module.forward] for each input tensor that corresponds to a multi-modal input, as shown in the following example:

30

31 ```diff

tests/async_engine/test_async_llm_engine.py

-1

@@ -33,7 +33,6 @@ class RequestOutput:

33 class MockModelConfig:

34 use_async_output_proc = True

35 media_io_kwargs: dict[str, dict[str, Any]] = field(default_factory=dict)

36 - mm_placeholder_str_override: dict[str, str] = field(default_factory=dict)

36

37

38 class MockEngine:

tests/engine/test_arg_utils.py

-20

@@ -263,26 +263,6 @@ def test_media_io_kwargs_parser(arg, expected):

263 assert args.media_io_kwargs == expected

264

265

266 -@pytest.mark.parametrize(("arg", "expected"), [

267 - (None, dict()),

268 - ('{"video":"<|video_placeholder|>"}', {

269 - "video": "<|video_placeholder|>"

270 - }),

271 - ('{"video":"<|video_placeholder|>", "image": "<|image_placeholder|>"}', {

272 - "video": "<|video_placeholder|>",

273 - "image": "<|image_placeholder|>"

274 - }),

275 -])

276 -def test_mm_placeholder_str_override_parser(arg, expected):

277 - parser = EngineArgs.add_cli_args(FlexibleArgumentParser())

278 - if arg is None:

279 - args = parser.parse_args([])

280 - else:

281 - args = parser.parse_args(["--mm-placeholder-str-override", arg])

282 -

283 - assert args.mm_placeholder_str_override == expected

284 -

285 -

266 def test_compilation_config():

267 parser = EngineArgs.add_cli_args(FlexibleArgumentParser())

268

tests/entrypoints/openai/test_serving_chat.py

-1

@@ -41,7 +41,6 @@ class MockModelConfig:

41 encoder_config = None

42 generation_config: str = "auto"

43 media_io_kwargs: dict[str, dict[str, Any]] = field(default_factory=dict)

44 - mm_placeholder_str_override: dict[str, str] = field(default_factory=dict)

44

45 def get_diff_sampling_param(self):

46 return self.diff_sampling_param or {}

vllm/config.py

+1 -7

@@ -350,8 +350,6 @@ class ModelConfig:

350 """Additional args passed to process media inputs, keyed by modalities.

351 For example, to set num_frames for video, set

352 `--media-io-kwargs '{"video": {"num_frames": 40} }'` """

353 - mm_placeholder_str_override: dict[str, str] = field(default_factory=dict)

354 - """Optionally override placeholder string for given modalities."""

353 use_async_output_proc: bool = True

354 """Whether to use async output processor."""

355 config_format: Union[str, ConfigFormat] = ConfigFormat.AUTO.value

@@ -661,7 +659,7 @@ class ModelConfig:

659 return self._architecture

660

661 @property

664 - def model_info(self) -> dict[str, Any]:

662 + def model_info(self):

663 return self._model_info

664

665 def maybe_pull_model_tokenizer_for_s3(self, model: str,

@@ -701,7 +699,6 @@ class ModelConfig:

699 return MultiModalConfig(

700 limit_per_prompt=self.limit_mm_per_prompt,

701 media_io_kwargs=self.media_io_kwargs,

704 - mm_placeholder_str_override=self.mm_placeholder_str_override,

702 mm_processor_kwargs=self.mm_processor_kwargs,

703 disable_mm_preprocessor_cache=self.

704 disable_mm_preprocessor_cache)

@@ -3096,9 +3093,6 @@ class MultiModalConfig:

3093 For example, to set num_frames for video, set

3094 `--media-io-kwargs '{"video": {"num_frames": 40} }'` """

3095

3099 - mm_placeholder_str_override: dict[str, str] = field(default_factory=dict)

3100 - """Optionally override placeholder string for given modalities."""

3101 -

3096 mm_processor_kwargs: Optional[dict[str, object]] = None

3097 """

3098 Overrides for the multi-modal processor obtained from

vllm/engine/arg_utils.py

-6

@@ -373,8 +373,6 @@ class EngineArgs:

373 media_io_kwargs: dict[str, dict[str,

374 Any]] = get_field(MultiModalConfig,

375 "media_io_kwargs")

376 - mm_placeholder_str_override: dict[str, str] = \

377 - get_field(MultiModalConfig, "mm_placeholder_str_override")

376 mm_processor_kwargs: Optional[Dict[str, Any]] = \

377 MultiModalConfig.mm_processor_kwargs

378 disable_mm_preprocessor_cache: bool = \

@@ -759,9 +757,6 @@ class EngineArgs:

757 **multimodal_kwargs["limit_per_prompt"])

758 multimodal_group.add_argument("--media-io-kwargs",

759 **multimodal_kwargs["media_io_kwargs"])

762 - multimodal_group.add_argument(

763 - "--mm-placeholder-str-override",

764 - **multimodal_kwargs["mm_placeholder_str_override"])

760 multimodal_group.add_argument(

761 "--mm-processor-kwargs",

762 **multimodal_kwargs["mm_processor_kwargs"])

@@ -987,7 +982,6 @@ class EngineArgs:

982 served_model_name=self.served_model_name,

983 limit_mm_per_prompt=self.limit_mm_per_prompt,

984 media_io_kwargs=self.media_io_kwargs,

990 - mm_placeholder_str_override=self.mm_placeholder_str_override,

985 use_async_output_proc=not self.disable_async_output_proc,

986 config_format=self.config_format,

987 mm_processor_kwargs=self.mm_processor_kwargs,

vllm/entrypoints/chat_utils.py

+9 -85

@@ -6,7 +6,7 @@ import json

6 from abc import ABC, abstractmethod

7 from collections import defaultdict, deque

8 from collections.abc import Awaitable, Iterable

9 -from functools import cache, lru_cache, partial

9 +from functools import cached_property, lru_cache, partial

10 from pathlib import Path

11 from typing import (Any, Callable, Generic, Literal, Optional, TypeVar, Union,

12 cast)

@@ -37,6 +37,8 @@ from typing_extensions import Required, TypeAlias, TypedDict

37

38 from vllm.config import ModelConfig

39 from vllm.logger import init_logger

40 +from vllm.model_executor.model_loader import get_model_cls

41 +from vllm.model_executor.models import SupportsMultiModal

42 from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalDataDict

43 from vllm.multimodal.utils import MediaConnector

44 # yapf: disable

@@ -492,6 +494,10 @@ class BaseMultiModalItemTracker(ABC, Generic[_T]):

494 def model_config(self) -> ModelConfig:

495 return self._model_config

496

497 + @cached_property

498 + def model_cls(self):

499 + return get_model_cls(self.model_config)

500 +

501 @property

502 def allowed_local_media_path(self):

503 return self._model_config.allowed_local_media_path

@@ -500,89 +506,6 @@ class BaseMultiModalItemTracker(ABC, Generic[_T]):

506 def mm_registry(self):

507 return MULTIMODAL_REGISTRY

508

503 - @staticmethod

504 - @cache

505 - def _cached_token_str(tokenizer: AnyTokenizer, token_index: int) -> str:

506 - return tokenizer.decode(token_index)

507 -

508 - def _placeholder_str(self, modality: ModalityStr,

509 - current_count: int) -> Optional[str]:

510 - if modality in self._model_config.mm_placeholder_str_override:

511 - return self._model_config.mm_placeholder_str_override[modality]

512 -

513 - # TODO: Let user specify how to insert image tokens into prompt

514 - # (similar to chat template)

515 - hf_config = self._model_config.hf_config

516 - model_type = hf_config.model_type

517 -

518 - if modality in ("image", "image_embeds"):

519 - if model_type == "chatglm":

520 - return "<|begin_of_image|><|endoftext|><|end_of_image|>"

521 - if model_type == "glm4v":

522 - return "<|begin_of_image|><|image|><|end_of_image|>"

523 - if model_type in ("phi3_v", "phi4mm"):

524 - return f"<|image_{current_count}|>"

525 - if model_type in ("minicpmo", "minicpmv"):

526 - return "(<image>./</image>)"

527 - if model_type in ("blip-2", "florence2", "fuyu", "paligemma",

528 - "pixtral", "mistral3"):

529 - # These models do not use image tokens in the prompt

530 - return None

531 - if model_type == "qwen":

532 - return f"Picture {current_count}: <img></img>"

533 - if model_type.startswith("llava"):

534 - return self._cached_token_str(self._tokenizer,

535 - hf_config.image_token_index)

536 -

537 - if model_type in ("aya_vision", "chameleon", "deepseek_vl_v2",

538 - "internvl_chat", "ovis", "skywork_chat",

539 - "NVLM_D", "h2ovl_chat", "idefics3", "smolvlm"):

540 - return "<image>"

541 - if model_type in ("mllama", "llama4"):

542 - return "<|image|>"

543 - if model_type in ("qwen2_vl", "qwen2_5_vl", "keye", "Keye"):

544 - return "<|vision_start|><|image_pad|><|vision_end|>"

545 - if model_type == "qwen2_5_omni":

546 - return "<|vision_start|><|IMAGE|><|vision_end|>"

547 - if model_type == "molmo":

548 - return ""

549 - if model_type == "aria":

550 - return "<|fim_prefix|><|img|><|fim_suffix|>"

551 - if model_type == "gemma3":

552 - return "<start_of_image>"

553 - if model_type == "kimi_vl":

555 -

556 - raise TypeError(f"Unknown {modality} model type: {model_type}")

557 - elif modality == "audio":

558 - if model_type in ("ultravox", "granite_speech"):

559 - return "<|audio|>"

560 - if model_type == "phi4mm":

561 - return f"<|audio_{current_count}|>"

562 - if model_type in ("qwen2_audio", "qwen2_5_omni"):

563 - return (f"Audio {current_count}: "

564 - f"<|audio_bos|><|AUDIO|><|audio_eos|>")

565 - if model_type == "minicpmo":

566 - return "(<audio>./</audio>)"

567 - raise TypeError(f"Unknown model type: {model_type}")

568 - elif modality == "video":

569 - if model_type == "internvl_chat":

570 - return "<video>"

571 - if model_type == "glm4v":

572 - return "<|begin_of_video|><|video|><|end_of_video|>"

573 - if model_type in ("qwen2_vl", "qwen2_5_vl", "keye", "Keye"):

574 - return "<|vision_start|><|video_pad|><|vision_end|>"

575 - if model_type == "qwen2_5_omni":

576 - return "<|vision_start|><|VIDEO|><|vision_end|>"

577 - if model_type in ("minicpmo", "minicpmv"):

578 - return "(<video>./</video>)"

579 - if model_type.startswith("llava"):

580 - return self._cached_token_str(self._tokenizer,

581 - hf_config.video_token_index)

582 - raise TypeError(f"Unknown {modality} model type: {model_type}")

583 - else:

584 - raise TypeError(f"Unknown modality: {modality}")

585 -

509 def add(self, modality: ModalityStr, item: _T) -> Optional[str]:

510 """

511 Add a multi-modal item to the current prompt and returns the

@@ -590,6 +513,7 @@ class BaseMultiModalItemTracker(ABC, Generic[_T]):

513 """

514 mm_registry = self.mm_registry

515 model_config = self.model_config

516 + model_cls = cast(SupportsMultiModal, self.model_cls)

517

518 input_modality = modality.replace("_embeds", "")

519

@@ -614,7 +538,7 @@ class BaseMultiModalItemTracker(ABC, Generic[_T]):

538

539 self._items_by_modality[modality].append(item)

540

617 - return self._placeholder_str(modality, current_count)

541 + return model_cls.get_placeholder_str(modality, current_count)

542

543 @abstractmethod

544 def create_parser(self) -> "BaseMultiModalContentParser":

vllm/entrypoints/openai/speech_to_text.py

+12 -6

@@ -5,6 +5,7 @@ import io

5 import math

6 import time

7 from collections.abc import AsyncGenerator

8 +from functools import cached_property

9 from math import ceil

10 from typing import Callable, Literal, Optional, TypeVar, Union, cast

11

@@ -24,7 +25,8 @@ from vllm.entrypoints.openai.serving_engine import (OpenAIServing,

25 from vllm.entrypoints.openai.serving_models import OpenAIServingModels

26 from vllm.inputs.data import PromptType

27 from vllm.logger import init_logger

27 -from vllm.model_executor.model_loader.utils import get_model_architecture

28 +from vllm.model_executor.model_loader import get_model_cls

29 +from vllm.model_executor.models import SupportsTranscription

30 from vllm.outputs import RequestOutput

31 from vllm.transformers_utils.processor import cached_get_processor

32 from vllm.utils import PlaceholderModule

@@ -76,24 +78,29 @@ class OpenAISpeechToText(OpenAIServing):

78 self.model_sr = processor.feature_extractor.sampling_rate

79 self.hop_length = processor.feature_extractor.hop_length

80 self.task_type = task_type

79 - self.model_cls, _ = get_model_architecture(model_config)

81

82 if self.default_sampling_params:

83 logger.info(

84 "Overwriting default completion sampling param with: %s",

85 self.default_sampling_params)

86

87 + @cached_property

88 + def model_cls(self):

89 + return get_model_cls(self.model_config)

90 +

91 async def _preprocess_speech_to_text(

92 self,

93 request: SpeechToTextRequest,

94 audio_data: bytes,

95 ) -> tuple[list[PromptType], float]:

96 + model_cls = cast(SupportsTranscription, self.model_cls)

97 +

98 # Validate request

99 # TODO language should be optional and can be guessed.

100 # For now we default to en. See

101 # https://github.com/huggingface/transformers/blob/main/src/transformers/models/whisper/generation_whisper.py#L1520

102 lang = request.language or "en"

96 - self.model_cls.validate_language(lang) # type: ignore[attr-defined]

103 + model_cls.validate_language(lang)

104

105 if len(audio_data) / 1024**2 > MAX_AUDIO_CLIP_FILESIZE_MB:

106 raise ValueError("Maximum file size exceeded.")

@@ -117,9 +124,8 @@ class OpenAISpeechToText(OpenAIServing):

124 },

125 },

126 "decoder_prompt":

120 - self.model_cls.

121 - get_decoder_prompt( # type: ignore[attr-defined]

122 - lang, self.task_type, request.prompt)

127 + model_cls.get_decoder_prompt(lang, self.task_type,

128 + request.prompt)

129 }

130 prompts.append(cast(PromptType, prompt))

131 return prompts, duration

vllm/model_executor/model_loader/__init__.py

+2 -1

@@ -18,7 +18,7 @@ from vllm.model_executor.model_loader.sharded_state_loader import (

18 ShardedStateLoader)

19 from vllm.model_executor.model_loader.tensorizer_loader import TensorizerLoader

20 from vllm.model_executor.model_loader.utils import (

21 - get_architecture_class_name, get_model_architecture)

21 + get_architecture_class_name, get_model_architecture, get_model_cls)

22

23

24 def get_model_loader(load_config: LoadConfig) -> BaseModelLoader:

@@ -65,6 +65,7 @@ __all__ = [

65 "get_model_loader",

66 "get_architecture_class_name",

67 "get_model_architecture",

68 + "get_model_cls",

69 "BaseModelLoader",

70 "BitsAndBytesModelLoader",

71 "GGUFModelLoader",

vllm/model_executor/model_loader/tensorizer.py

+5 -3

@@ -13,7 +13,7 @@ import time

13 from collections.abc import Generator

14 from dataclasses import dataclass

15 from functools import partial

16 -from typing import Any, BinaryIO, Optional, Union

16 +from typing import TYPE_CHECKING, Any, BinaryIO, Optional, Union

17

18 import regex as re

19 import torch

@@ -24,12 +24,14 @@ from transformers import PretrainedConfig

24 import vllm.envs as envs

25 from vllm.config import (ModelConfig, ParallelConfig, VllmConfig,

26 set_current_vllm_config)

27 -from vllm.engine.arg_utils import EngineArgs

27 from vllm.logger import init_logger

28 from vllm.model_executor.layers.vocab_parallel_embedding import (

29 VocabParallelEmbedding)

30 from vllm.utils import FlexibleArgumentParser, PlaceholderModule

31

32 +if TYPE_CHECKING:

33 + from vllm.engine.arg_utils import EngineArgs

34 +

35 try:

36 from tensorizer import (DecryptionParams, EncryptionParams,

37 TensorDeserializer, TensorSerializer)

@@ -503,7 +505,7 @@ def serialize_vllm_model(

505 return model

506

507

506 -def tensorize_vllm_model(engine_args: EngineArgs,

508 +def tensorize_vllm_model(engine_args: "EngineArgs",

509 tensorizer_config: TensorizerConfig,

510 generate_keyfile: bool = True):

511 """Utility to load a model and then serialize it with Tensorizer

vllm/model_executor/model_loader/utils.py

+4

@@ -253,6 +253,10 @@ def get_model_architecture(

253 return model_cls, arch

254

255

256 +def get_model_cls(model_config: ModelConfig) -> type[nn.Module]:

257 + return get_model_architecture(model_config)[0]

258 +

259 +

260 def get_architecture_class_name(model_config: ModelConfig) -> str:

261 return get_model_architecture(model_config)[1]

262

vllm/model_executor/models/__init__.py

+5 -3

@@ -2,9 +2,9 @@

2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project

3

4 from .interfaces import (HasInnerState, SupportsLoRA, SupportsMultiModal,

5 - SupportsPP, SupportsV0Only, has_inner_state,

6 - supports_lora, supports_multimodal, supports_pp,

7 - supports_v0_only)

5 + SupportsPP, SupportsTranscription, SupportsV0Only,

6 + has_inner_state, supports_lora, supports_multimodal,

7 + supports_pp, supports_transcription, supports_v0_only)

8 from .interfaces_base import (VllmModelForPooling, VllmModelForTextGeneration,

9 is_pooling_model, is_text_generation_model)

10 from .registry import ModelRegistry

@@ -23,6 +23,8 @@ __all__ = [

23 "supports_multimodal",

24 "SupportsPP",

25 "supports_pp",

26 + "SupportsTranscription",

27 + "supports_transcription",

28 "SupportsV0Only",

29 "supports_v0_only",

30 ]

vllm/model_executor/models/aria.py

+7

@@ -499,6 +499,13 @@ class AriaForConditionalGeneration(nn.Module, SupportsMultiModal):

499 },

500 )

501

502 + @classmethod

503 + def get_placeholder_str(cls, modality: str, i: int) -> Optional[str]:

504 + if modality.startswith("image"):

505 + return "<|fim_prefix|><|img|><|fim_suffix|>"

506 +

507 + raise ValueError("Only image modality is supported")

508 +

509 def __init__(

510 self,

511 vllm_config: VllmConfig,

vllm/model_executor/models/aya_vision.py

+7

@@ -304,6 +304,13 @@ class AyaVisionForConditionalGeneration(nn.Module, SupportsMultiModal,

304 "lm_head.": "language_model.lm_head.",

305 })

306

307 + @classmethod

308 + def get_placeholder_str(cls, modality: str, i: int) -> Optional[str]:

309 + if modality.startswith("image"):

310 + return "<image>"

311 +

312 + raise ValueError("Only image modality is supported")

313 +

314 def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):

315 super().__init__()

316 config: AyaVisionConfig = vllm_config.model_config.hf_config

vllm/model_executor/models/blip2.py

+7

@@ -507,6 +507,13 @@ class Blip2MultiModalProcessor(BaseMultiModalProcessor[Blip2ProcessingInfo]):

507 class Blip2ForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP,

508 SupportsQuant):

509

510 + @classmethod

511 + def get_placeholder_str(cls, modality: str, i: int) -> Optional[str]:

512 + if modality.startswith("image"):

513 + return None

514 +

515 + raise ValueError("Only image modality is supported")

516 +

517 def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):

518

519 super().__init__()

vllm/model_executor/models/chameleon.py

+7

@@ -933,6 +933,13 @@ class ChameleonForConditionalGeneration(nn.Module, SupportsMultiModal,

933 "gate_up_proj": ["gate_proj", "up_proj"]

934 }

935

936 + @classmethod

937 + def get_placeholder_str(cls, modality: str, i: int) -> Optional[str]:

938 + if modality.startswith("image"):

939 + return "<image>"

940 +

941 + raise ValueError("Only image modality is supported")

942 +

943 def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):

944 super().__init__()

945 config = vllm_config.model_config.hf_config

vllm/model_executor/models/deepseek_vl2.py

+7

@@ -315,6 +315,13 @@ class DeepseekVLV2ForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):

315 "language.": "language_model.",

316 })

317

318 + @classmethod

319 + def get_placeholder_str(cls, modality: str, i: int) -> Optional[str]:

320 + if modality.startswith("image"):

321 + return "<image>"

322 +

323 + raise ValueError("Only image modality is supported")

324 +

325 def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):

326 super().__init__()

327 config: DeepseekVLV2Config = vllm_config.model_config.hf_config

vllm/model_executor/models/florence2.py

+7

@@ -877,6 +877,13 @@ class Florence2MultiModalProcessor(

877 class Florence2ForConditionalGeneration(nn.Module, SupportsMultiModal,

878 SupportsV0Only):

879

880 + @classmethod

881 + def get_placeholder_str(cls, modality: str, i: int) -> Optional[str]:

882 + if modality.startswith("image"):

883 + return None

884 +

885 + raise ValueError("Only image modality is supported")

886 +

887 def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):

888 super().__init__()

889 config = vllm_config.model_config.hf_config

vllm/model_executor/models/fuyu.py

+7

@@ -254,6 +254,13 @@ class FuyuForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):

254 "lm_head.": "language_model.lm_head.",

255 })

256

257 + @classmethod

258 + def get_placeholder_str(cls, modality: str, i: int) -> Optional[str]:

259 + if modality.startswith("image"):

260 + return None

261 +

262 + raise ValueError("Only image modality is supported")

263 +

264 def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):

265 super().__init__()

266 config = vllm_config.model_config.hf_config

vllm/model_executor/models/gemma3_mm.py

+7

@@ -483,6 +483,13 @@ class Gemma3ForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP,

483 "lm_head.": "language_model.lm_head.",

484 })

485

486 + @classmethod

487 + def get_placeholder_str(cls, modality: str, i: int) -> Optional[str]:

488 + if modality.startswith("image"):

489 + return "<start_of_image>"

490 +

491 + raise ValueError("Only image modality is supported")

492 +

493 def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):

494 super().__init__()

495 config = vllm_config.model_config.hf_config

vllm/model_executor/models/glm4_1v.py

+9

@@ -1257,6 +1257,15 @@ class Glm4vForConditionalGeneration(nn.Module, SupportsMultiModal,

1257 "model.visual.": "visual.",

1258 })

1259

1260 + @classmethod

1261 + def get_placeholder_str(cls, modality: str, i: int) -> Optional[str]:

1262 + if modality.startswith("image"):

1263 + return "<|begin_of_image|><|image|><|end_of_image|>"

1264 + if modality.startswith("video"):

1265 + return "<|begin_of_video|><|video|><|end_of_video|>"

1266 +

1267 + raise ValueError("Only image or video modality is supported")

1268 +

1269 def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):

1270 super().__init__()

1271 config: Glm4vConfig = vllm_config.model_config.hf_config

vllm/model_executor/models/glm4v.py

+7

@@ -540,6 +540,13 @@ class GLM4VForCausalLM(ChatGLMBaseModel, SupportsLoRA, SupportsPP,

540 connector="transformer.vision.linear_proj",

541 tower_model="transformer.vision.transformer")

542

543 + @classmethod

544 + def get_placeholder_str(cls, modality: str, i: int) -> Optional[str]:

545 + if modality.startswith("image"):

546 + return "<|begin_of_image|><|endoftext|><|end_of_image|>"

547 +

548 + raise ValueError("Only image modality is supported")

549 +

550 def __init__(

551 self,

552 *,

vllm/model_executor/models/granite_speech.py

+7

@@ -533,6 +533,13 @@ class GraniteSpeechForConditionalGeneration(

533 ],

534 }

535

536 + @classmethod

537 + def get_placeholder_str(cls, modality: str, i: int) -> Optional[str]:

538 + if modality.startswith("audio"):

539 + return "<|audio|>"

540 +

541 + raise ValueError("Only audio modality is supported")

542 +

543 def __init__(self, *, vllm_config: VllmConfig, prefix: str):

544 super().__init__()

545 config = vllm_config.model_config.hf_config

vllm/model_executor/models/idefics3.py

+7

@@ -591,6 +591,13 @@ class Idefics3ForConditionalGeneration(nn.Module, SupportsMultiModal,

591 ],

592 }

593

594 + @classmethod

595 + def get_placeholder_str(cls, modality: str, i: int) -> Optional[str]:

596 + if modality.startswith("image"):

597 + return "<image>"

598 +

599 + raise ValueError("Only image modality is supported")

600 +

601 def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):

602 super().__init__()

603

vllm/model_executor/models/interfaces.py

+7

@@ -46,6 +46,13 @@ class SupportsMultiModal(Protocol):

46 MRO of your model class.

47 """

48

49 + @classmethod

50 + def get_placeholder_str(cls, modality: str, i: int) -> Optional[str]:

51 + """

52 + Get the placeholder text for the `i`th `modality` item in the prompt.

53 + """

54 + ...

55 +

56 def get_multimodal_embeddings(self,

57 **kwargs: object) -> MultiModalEmbeddings:

58 """

vllm/model_executor/models/internvl.py

+9

@@ -1023,6 +1023,15 @@ class InternVLMultiModalProcessor(

1023 class InternVLChatModel(nn.Module, SupportsMultiModal, SupportsPP,

1024 SupportsLoRA):

1025

1026 + @classmethod

1027 + def get_placeholder_str(cls, modality: str, i: int) -> Optional[str]:

1028 + if modality.startswith("image"):

1029 + return "<image>"

1030 + if modality.startswith("video"):

1031 + return "<video>"

1032 +

1033 + raise ValueError("Only image or video modality is supported")

1034 +

1035 def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None:

1036 super().__init__()

1037

vllm/model_executor/models/keye.py

+9

@@ -1343,6 +1343,15 @@ class KeyeForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsLoRA,

1343 "model.": "language_model.model.",

1344 })

1345

1346 + @classmethod

1347 + def get_placeholder_str(cls, modality: str, i: int) -> Optional[str]:

1348 + if modality.startswith("image"):

1349 + return "<|vision_start|><|image_pad|><|vision_end|>"

1350 + if modality.startswith("video"):

1351 + return "<|vision_start|><|video_pad|><|vision_end|>"

1352 +

1353 + raise ValueError("Only image or video modality is supported")

1354 +

1355 def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):

1356 super().__init__()

1357 config: PretrainedConfig = vllm_config.model_config.hf_config

vllm/model_executor/models/kimi_vl.py

+7

@@ -264,6 +264,13 @@ class KimiVLMultiModalProcessor(BaseMultiModalProcessor[KimiVLProcessingInfo]):

264 dummy_inputs=KimiVLDummyInputsBuilder)

265 class KimiVLForConditionalGeneration(nn.Module, SupportsMultiModal):

266

267 + @classmethod

268 + def get_placeholder_str(cls, modality: str, i: int) -> Optional[str]:

269 + if modality.startswith("image"):

271 +

272 + raise ValueError("Only image modality is supported")

273 +

274 def __init__(

275 self,

276 vllm_config: VllmConfig,

vllm/model_executor/models/llava.py

+7

@@ -511,6 +511,13 @@ class LlavaForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP):

511 "lm_head.": "language_model.lm_head.",

512 })

513

514 + @classmethod

515 + def get_placeholder_str(cls, modality: str, i: int) -> Optional[str]:

516 + if modality.startswith("image"):

517 + return "<image>"

518 +

519 + raise ValueError("Only image modality is supported")

520 +

521 def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None:

522 super().__init__()

523

vllm/model_executor/models/llava_next.py

+7

@@ -215,6 +215,13 @@ class LlavaNextForConditionalGeneration(nn.Module, SupportsMultiModal,

215 "lm_head.": "language_model.lm_head.",

216 })

217

218 + @classmethod

219 + def get_placeholder_str(cls, modality: str, i: int) -> Optional[str]:

220 + if modality.startswith("image"):

221 + return "<image>"

222 +

223 + raise ValueError("Only image modality is supported")

224 +

225 def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None:

226 super().__init__()

227 config = vllm_config.model_config.hf_config

vllm/model_executor/models/llava_next_video.py

+9

@@ -281,6 +281,15 @@ class LlavaNextVideoForConditionalGeneration(nn.Module, SupportsMultiModal,

281 "lm_head.": "language_model.lm_head.",

282 })

283

284 + @classmethod

285 + def get_placeholder_str(cls, modality: str, i: int) -> Optional[str]:

286 + if modality.startswith("image"):

287 + return "<image>"

288 + if modality.startswith("video"):

289 + return "<video>"

290 +

291 + raise ValueError("Only image or video modality is supported")

292 +

293 def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None:

294 super().__init__()

295 config = vllm_config.model_config.hf_config

vllm/model_executor/models/llava_onevision.py

+9

@@ -446,6 +446,15 @@ class LlavaOnevisionForConditionalGeneration(nn.Module, SupportsMultiModal,

446 "lm_head.": "language_model.lm_head.",

447 })

448

449 + @classmethod

450 + def get_placeholder_str(cls, modality: str, i: int) -> Optional[str]:

451 + if modality.startswith("image"):

452 + return "<image>"

453 + if modality.startswith("video"):

454 + return "<video>"

455 +

456 + raise ValueError("Only image or video modality is supported")

457 +

458 def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None:

459 super().__init__()

460 config = vllm_config.model_config.hf_config

vllm/model_executor/models/minicpmo.py

+11

@@ -511,6 +511,17 @@ class MiniCPMO(MiniCPMV2_6):

511 ],

512 }

513

514 + @classmethod

515 + def get_placeholder_str(cls, modality: str, i: int) -> Optional[str]:

516 + if modality.startswith("image"):

517 + return "(<image>./</image>)"

518 + if modality.startswith("video"):

519 + return "(<video>./</video>)"

520 + if modality.startswith("audio"):

521 + return "(<audio>./</audio>)"

522 +

523 + raise ValueError("Only image, video or audio modality is supported")

524 +

525 def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):

526 super().__init__(vllm_config=vllm_config, prefix=prefix)

527 self.apm = self.init_audio_module(vllm_config=vllm_config,

vllm/model_executor/models/minicpmv.py

+9

@@ -735,6 +735,15 @@ class MiniCPMVBaseModel(nn.Module, SupportsMultiModal, SupportsPP):

735 instantiated.

736 """

737

738 + @classmethod

739 + def get_placeholder_str(cls, modality: str, i: int) -> Optional[str]:

740 + if modality.startswith("image"):

741 + return "(<image>./</image>)"

742 + if modality.startswith("video"):

743 + return "(<video>./</video>)"

744 +

745 + raise ValueError("Only image or video modality is supported")

746 +

747 def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):

748 config = vllm_config.model_config.hf_config

749 multimodal_config = vllm_config.model_config.multimodal_config

vllm/model_executor/models/minimax_vl_01.py

+7

@@ -158,6 +158,13 @@ class MiniMaxVL01ForConditionalGeneration(nn.Module, SupportsMultiModal,

158 "gate_up_proj": ["gate_proj", "up_proj"]

159 }

160

161 + @classmethod

162 + def get_placeholder_str(cls, modality: str, i: int) -> Optional[str]:

163 + if modality.startswith("image"):

164 + return "<image>"

165 +

166 + raise ValueError("Only image modality is supported")

167 +

168 def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None:

169 super().__init__()

170

vllm/model_executor/models/mistral3.py

+7

@@ -401,6 +401,13 @@ class Mistral3ForConditionalGeneration(nn.Module, SupportsLoRA,

401 "lm_head.": "language_model.lm_head.",

402 })

403

404 + @classmethod

405 + def get_placeholder_str(cls, modality: str, i: int) -> Optional[str]:

406 + if modality.startswith("image"):

407 + return None

408 +

409 + raise ValueError("Only image modality is supported")

410 +

411 def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None:

412 super().__init__()

413

vllm/model_executor/models/mllama.py

+7

@@ -1276,6 +1276,13 @@ class MllamaForConditionalGeneration(nn.Module, SupportsMultiModal,

1276 },

1277 )

1278

1279 + @classmethod

1280 + def get_placeholder_str(cls, modality: str, i: int) -> Optional[str]:

1281 + if modality.startswith("image"):

1282 + return "<|image|>"

1283 +

1284 + raise ValueError("Only image modality is supported")

1285 +

1286 def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):

1287 super().__init__()

1288 config: MllamaConfig = vllm_config.model_config.hf_config

vllm/model_executor/models/mllama4.py

+7

@@ -719,6 +719,13 @@ class Llama4ForConditionalGeneration(nn.Module, SupportsMultiModal,

719 "qkv_proj": ["q_proj", "k_proj", "v_proj"],

720 }

721

722 + @classmethod

723 + def get_placeholder_str(cls, modality: str, i: int) -> Optional[str]:

724 + if modality.startswith("image"):

725 + return "<|image|>"

726 +

727 + raise ValueError("Only image modality is supported")

728 +

729 def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):

730 super().__init__()

731 config = vllm_config.model_config.hf_config

vllm/model_executor/models/molmo.py

+7

@@ -1366,6 +1366,13 @@ class MolmoForCausalLM(nn.Module, SupportsMultiModal, SupportsPP, SupportsLoRA,

1366 "merged_linear": ["gate_proj", "up_proj"] # image_projector

1367 }

1368

1369 + @classmethod

1370 + def get_placeholder_str(cls, modality: str, i: int) -> Optional[str]:

1371 + if modality.startswith("image"):

1372 + return None

1373 +

1374 + raise ValueError("Only image modality is supported")

1375 +

1376 def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):

1377 super().__init__()

1378 config = vllm_config.model_config.hf_config

vllm/model_executor/models/ovis.py

+7

@@ -405,6 +405,13 @@ class OvisMultiModalProcessor(BaseMultiModalProcessor[OvisProcessingInfo]):

405 dummy_inputs=OvisDummyInputsBuilder)

406 class Ovis(nn.Module, SupportsMultiModal, SupportsPP):

407

408 + @classmethod

409 + def get_placeholder_str(cls, modality: str, i: int) -> Optional[str]:

410 + if modality.startswith("image"):

411 + return "<image>"

412 +

413 + raise ValueError("Only image modality is supported")

414 +

415 def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):

416 super().__init__()

417 config = vllm_config.model_config.hf_config

vllm/model_executor/models/paligemma.py

+7

@@ -240,6 +240,13 @@ class PaliGemmaForConditionalGeneration(nn.Module, SupportsMultiModal,

240 "lm_head.": "language_model.lm_head.",

241 })

242

243 + @classmethod

244 + def get_placeholder_str(cls, modality: str, i: int) -> Optional[str]:

245 + if modality.startswith("image"):

246 + return None

247 +

248 + raise ValueError("Only image modality is supported")

249 +

250 def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):

251 super().__init__()

252 config = vllm_config.model_config.hf_config

vllm/model_executor/models/phi3v.py

+7

@@ -520,6 +520,13 @@ class Phi3VForCausalLM(nn.Module, SupportsMultiModal, SupportsPP,

520 "model.": "language_model.model.",

521 })

522

523 + @classmethod

524 + def get_placeholder_str(cls, modality: str, i: int) -> Optional[str]:

525 + if modality.startswith("image"):

526 + return f"<|image_{i}|>"

527 +

528 + raise ValueError("Only image modality is supported")

529 +

530 def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):

531 super().__init__()

532 config = vllm_config.model_config.hf_config

vllm/model_executor/models/phi4mm.py

+9

@@ -902,6 +902,15 @@ class Phi4MMForCausalLM(nn.Module, SupportsLoRA, SupportsMultiModal):

902 },

903 )

904

905 + @classmethod

906 + def get_placeholder_str(cls, modality: str, i: int) -> Optional[str]:

907 + if modality.startswith("image"):

908 + return f"<|image_{i}|>"

909 + if modality.startswith("audio"):

910 + return f"<|audio_{i}|>"

911 +

912 + raise ValueError("Only image or audio modality is supported")

913 +

914 def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):

915 super().__init__()

916 config = vllm_config.model_config.hf_config

vllm/model_executor/models/pixtral.py

+7

@@ -327,6 +327,13 @@ class PixtralMultiModalProcessor(BaseMultiModalProcessor[PixtralProcessingInfo]

327 class PixtralForConditionalGeneration(nn.Module, SupportsMultiModal,

328 SupportsPP):

329

330 + @classmethod

331 + def get_placeholder_str(cls, modality: str, i: int) -> Optional[str]:

332 + if modality.startswith("image"):

333 + return None

334 +

335 + raise ValueError("Only image modality is supported")

336 +

337 def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):

338 super().__init__()

339 config = vllm_config.model_config.hf_config

vllm/model_executor/models/prithvi_geospatial_mae.py

+7

@@ -118,6 +118,13 @@ class PrithviGeoSpatialMAE(nn.Module, IsAttentionFree, SupportsMultiModal,

118 SupportsV0Only):

119 """ Prithvi Masked Autoencoder"""

120

121 + @classmethod

122 + def get_placeholder_str(cls, modality: str, i: int) -> Optional[str]:

123 + if modality.startswith("image"):

124 + return None

125 +

126 + raise ValueError("Only image modality is supported")

127 +

128 def _instantiate_model(self, config: dict) -> Optional[nn.Module]:

129

130 # We might be able/need to support different tasks with this same model

vllm/model_executor/models/qwen2_5_omni_thinker.py

+11

@@ -717,6 +717,17 @@ class Qwen2_5OmniThinkerForConditionalGeneration(

717 "thinker.": "",

718 })

719

720 + @classmethod

721 + def get_placeholder_str(cls, modality: str, i: int) -> Optional[str]:

722 + if modality.startswith("image"):

723 + return "<|vision_start|><|IMAGE|><|vision_end|>"

724 + if modality.startswith("video"):

725 + return "<|vision_start|><|VIDEO|><|vision_end|>"

726 + if modality.startswith("audio"):

727 + return f"Audio {i}: <|audio_bos|><|AUDIO|><|audio_eos|>"

728 +

729 + raise ValueError("Only image, video or audio modality is supported")

730 +

731 def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):

732 super().__init__()

733 thinker_config: Qwen2_5OmniThinkerConfig = (

vllm/model_executor/models/qwen2_5_vl.py

+9

@@ -835,6 +835,15 @@ class Qwen2_5_VLForConditionalGeneration(nn.Module, SupportsMultiModal,

835 "model.": "language_model.model.",

836 })

837

838 + @classmethod

839 + def get_placeholder_str(cls, modality: str, i: int) -> Optional[str]:

840 + if modality.startswith("image"):

841 + return "<|vision_start|><|image_pad|><|vision_end|>"

842 + if modality.startswith("video"):

843 + return "<|vision_start|><|video_pad|><|vision_end|>"

844 +

845 + raise ValueError("Only image or video modality is supported")

846 +

847 def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):

848 super().__init__()

849 config: Qwen2_5_VLConfig = vllm_config.model_config.hf_config

vllm/model_executor/models/qwen2_audio.py

+7

@@ -251,6 +251,13 @@ class Qwen2AudioMultiModalProcessor(

251 class Qwen2AudioForConditionalGeneration(nn.Module, SupportsMultiModal,

252 SupportsPP):

253

254 + @classmethod

255 + def get_placeholder_str(cls, modality: str, i: int) -> Optional[str]:

256 + if modality.startswith("audio"):

257 + return f"Audio {i}: <|audio_bos|><|AUDIO|><|audio_eos|>"

258 +

259 + raise ValueError("Only audio modality is supported")

260 +

261 def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):

262 super().__init__()

263 config = vllm_config.model_config.hf_config

vllm/model_executor/models/qwen2_vl.py

+9

@@ -1096,6 +1096,15 @@ class Qwen2VLForConditionalGeneration(nn.Module, SupportsMultiModal,

1096 "model.": "language_model.model.",

1097 })

1098

1099 + @classmethod

1100 + def get_placeholder_str(cls, modality: str, i: int) -> Optional[str]:

1101 + if modality.startswith("image"):

1102 + return "<|vision_start|><|image_pad|><|vision_end|>"

1103 + if modality.startswith("video"):

1104 + return "<|vision_start|><|video_pad|><|vision_end|>"

1105 +

1106 + raise ValueError("Only image or video modality is supported")

1107 +

1108 def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):

1109 super().__init__()

1110 config: Qwen2VLConfig = vllm_config.model_config.hf_config

vllm/model_executor/models/qwen_vl.py

+7

@@ -675,6 +675,13 @@ class QwenVLForConditionalGeneration(QWenBaseModel, SupportsPP, SupportsLoRA,

675 connector="transformer.visual.attn_pool",

676 tower_model="transformer.visual.transformer")

677

678 + @classmethod

679 + def get_placeholder_str(cls, modality: str, i: int) -> Optional[str]:

680 + if modality.startswith("image"):

681 + return f"Picture {i}: <img></img>"

682 +

683 + raise ValueError("Only image modality is supported")

684 +

685 def __init__(

686 self,

687 *,

vllm/model_executor/models/skyworkr1v.py

+7

@@ -648,6 +648,13 @@ class SkyworkR1VProcessingInfo(BaseSkyworkR1VProcessingInfo):

648 dummy_inputs=SkyworkR1VDummyInputsBuilder)

649 class SkyworkR1VChatModel(nn.Module, SupportsMultiModal, SupportsPP):

650

651 + @classmethod

652 + def get_placeholder_str(cls, modality: str, i: int) -> Optional[str]:

653 + if modality.startswith("image"):

654 + return "<image>"

655 +

656 + raise ValueError("Only image modality is supported")

657 +

658 def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None:

659 super().__init__()

660

vllm/model_executor/models/tarsier.py

+7

@@ -393,6 +393,13 @@ class TarsierForConditionalGeneration(nn.Module, SupportsMultiModal,

393 "gate_up_proj": ["gate_proj", "up_proj"]

394 }

395

396 + @classmethod

397 + def get_placeholder_str(cls, modality: str, i: int) -> Optional[str]:

398 + if modality.startswith("image"):

399 + return "<image>"

400 +

401 + raise ValueError("Only image modality is supported")

402 +

403 def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None:

404 super().__init__()

405 config: TarsierHfConfig = vllm_config.model_config.hf_config

vllm/model_executor/models/ultravox.py

+7

@@ -407,6 +407,13 @@ class UltravoxModel(nn.Module, SupportsMultiModal, SupportsPP, SupportsLoRA):

407 hf_to_vllm_mapper = WeightsMapper(

408 orig_to_new_prefix={"audio_tower.model.encoder.": "audio_tower."})

409

410 + @classmethod

411 + def get_placeholder_str(cls, modality: str, i: int) -> Optional[str]:

412 + if modality.startswith("audio"):

413 + return "<|audio|>"

414 +

415 + raise ValueError("Only audio modality is supported")

416 +

417 def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):

418 super().__init__()

419 config = vllm_config.model_config.hf_config

vllm/model_executor/models/whisper.py

+29 -22

@@ -761,6 +761,35 @@ class WhisperForConditionalGeneration(nn.Module, SupportsTranscription,

761 ".fc2.": ".mlp.fc2."

762 })

763

764 + @classmethod

765 + def validate_language(cls, language: str) -> bool:

766 + if language in ISO639_1_SUPPORTED_LANGS:

767 + return True

768 + elif language in ISO639_1_OTHER_LANGS:

769 + logger.warning(

770 + "The selected language %s has limited accuracy with"

771 + " reported WER>=0.5. Results may be less accurate "

772 + "for this choice.", language)

773 + return True

774 + else:

775 + raise ValueError(f"Unsupported language: {language}."

776 + "Language should be one of:" +

777 + f" {list(ISO639_1_SUPPORTED_LANGS.values())}" +

778 + f"or {list(ISO639_1_OTHER_LANGS.values())}")

779 +

780 + @classmethod

781 + def get_decoder_prompt(cls, language: str, task_type: str,

782 + prompt: str) -> str:

783 + return (f"<|startoftranscript|><|{language}|><|{task_type}|>"

784 + f"<|notimestamps|>{prompt}")

785 +

786 + @classmethod

787 + def get_placeholder_str(cls, modality: str, i: int) -> Optional[str]:

788 + if modality.startswith("audio"):

789 + return None

790 +

791 + raise ValueError("Only audio modality is supported")

792 +

793 def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):

794 super().__init__()

795 config = vllm_config.model_config.hf_config

@@ -840,28 +869,6 @@ class WhisperForConditionalGeneration(nn.Module, SupportsTranscription,

869 weights = _create_fake_bias_for_k_proj(weights)

870 return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)

871

843 - @classmethod

844 - def validate_language(cls, language: str) -> bool:

845 - if language in ISO639_1_SUPPORTED_LANGS:

846 - return True

847 - elif language in ISO639_1_OTHER_LANGS:

848 - logger.warning(

849 - "The selected language %s has limited accuracy with"

850 - " reported WER>=0.5. Results may be less accurate "

851 - "for this choice.", language)

852 - return True

853 - else:

854 - raise ValueError(f"Unsupported language: {language}."

855 - "Language should be one of:" +

856 - f" {list(ISO639_1_SUPPORTED_LANGS.values())}" +

857 - f"or {list(ISO639_1_OTHER_LANGS.values())}")

858 -

859 - @classmethod

860 - def get_decoder_prompt(cls, language: str, task_type: str,

861 - prompt: str) -> str:

862 - return (f"<|startoftranscript|><|{language}|><|{task_type}|>"

863 - f"<|notimestamps|>{prompt}")

864 -

872

873 def _create_fake_bias_for_k_proj(

874 weights: Iterable[tuple[str, torch.Tensor]]

[Core] Move multimodal placeholder from chat utils to model definition (#20355)