@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / c46b932df2
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 11, 22:30 UTC

[Chore] Deprecate `SupportsMultiModal.merge_by_field_config` (#30170)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>

Cyrus Leung committed Dec 6, 2025 at 15:57 UTC c46b932df2b801ba0a6452e436268f086029d82b
61 files changed +23 -110
vllm/model_executor/models/aria.py
-2
@@ -499,8 +499,6 @@ class AriaForConditionalGeneration(nn.Module, SupportsMultiModal):
499 model to perform tasks that involve both image and text inputs.
500 """
501
502 - merge_by_field_config = True
503 -
502 hf_to_vllm_mapper = WeightsMapper(
503 orig_to_new_prefix={
504 # mapping for new names in checkpoint saved after transformers v4.52
vllm/model_executor/models/aya_vision.py
-2
@@ -318,8 +318,6 @@ def _get_layer_index(feature_layer_index: int, num_hidden_layers: int) -> int:
318 dummy_inputs=AyaVisionDummyInputsBuilder,
319 )
320 class AyaVisionForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP):
321 - merge_by_field_config = True
322 -
321 hf_to_vllm_mapper = WeightsMapper(
322 orig_to_new_prefix={
323 # mapping for new names in checkpoint saved after transformers v4.52
vllm/model_executor/models/blip2.py
-2
@@ -523,8 +523,6 @@ class Blip2MultiModalProcessor(BaseMultiModalProcessor[Blip2ProcessingInfo]):
523 class Blip2ForConditionalGeneration(
524 nn.Module, SupportsMultiModal, SupportsPP, SupportsQuant
525 ):
526 - merge_by_field_config = True
527 -
526 @classmethod
527 def get_placeholder_str(cls, modality: str, i: int) -> str | None:
528 if modality.startswith("image"):
vllm/model_executor/models/chameleon.py
-2
@@ -918,8 +918,6 @@ class ChameleonModel(nn.Module):
918 class ChameleonForConditionalGeneration(
919 nn.Module, SupportsMultiModal, SupportsPP, SupportsQuant
920 ):
921 - merge_by_field_config = True
922 -
921 packed_modules_mapping = {
922 "qkv_proj": ["q_proj", "k_proj", "v_proj"],
923 "gate_up_proj": ["gate_proj", "up_proj"],
vllm/model_executor/models/clip.py
-1
@@ -784,7 +784,6 @@ class CLIPEmbeddingModel(nn.Module, SupportsMultiModal, SupportsQuant):
784 is_pooling_model = True
785
786 packed_modules_mapping = {"qkv_proj": ["q_proj", "k_proj", "v_proj"]}
787 - merge_by_field_config = True
787
788 @classmethod
789 def get_placeholder_str(cls, modality: str, i: int) -> str | None:
vllm/model_executor/models/cohere2_vision.py
-2
@@ -331,8 +331,6 @@ class Cohere2VisionMultiModalProcessor(
331 dummy_inputs=Cohere2VisionDummyInputsBuilder,
332 )
333 class Cohere2VisionForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP):
334 - merge_by_field_config = True
335 -
334 hf_to_vllm_mapper = WeightsMapper(
335 orig_to_new_prefix={
336 "model.vision_tower.": "vision_tower.",
vllm/model_executor/models/deepseek_ocr.py
-2
@@ -344,8 +344,6 @@ class DeepseekOCRMultiModalProcessor(
344 dummy_inputs=DeepseekOCRDummyInputsBuilder,
345 )
346 class DeepseekOCRForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):
347 - merge_by_field_config = True
348 -
347 hf_to_vllm_mapper = WeightsMapper(
348 orig_to_new_prefix={
349 # map prefix for language backbone
vllm/model_executor/models/deepseek_vl2.py
-2
@@ -344,8 +344,6 @@ class DeepseekVL2MultiModalProcessor(
344 dummy_inputs=DeepseekVL2DummyInputsBuilder,
345 )
346 class DeepseekVLV2ForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):
347 - merge_by_field_config = True
348 -
347 hf_to_vllm_mapper = WeightsMapper(
348 orig_to_new_prefix={
349 "language.": "language_model.",
vllm/model_executor/models/dots_ocr.py
-2
@@ -690,8 +690,6 @@ class DotsVisionTransformer(nn.Module):
690 dummy_inputs=DotsOCRDummyInputsBuilder,
691 )
692 class DotsOCRForCausalLM(nn.Module, SupportsMultiModal, SupportsPP, SupportsLoRA):
693 - merge_by_field_config = True
694 -
693 hf_to_vllm_mapper = WeightsMapper(
694 orig_to_new_substr={
695 ".attn.qkv_proj.": ".attn.qkv.",
vllm/model_executor/models/ernie45_vl.py
-2
@@ -1254,8 +1254,6 @@ class Ernie4_5_VLDummyInputsBuilder(BaseDummyInputsBuilder[Ernie4_5_VLProcessing
1254 class Ernie4_5_VLMoeForConditionalGeneration(
1255 nn.Module, SupportsMultiModal, SupportsLoRA, SupportsPP, SupportsMRoPE
1256 ):
1257 - merge_by_field_config = True
1258 -
1257 packed_modules_mapping = {
1258 "qkv_proj": [
1259 "q_proj",
vllm/model_executor/models/fuyu.py
-2
@@ -260,8 +260,6 @@ class FuyuMultiModalProcessor(BaseMultiModalProcessor[FuyuProcessingInfo]):
260 dummy_inputs=FuyuDummyInputsBuilder,
261 )
262 class FuyuForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):
263 - merge_by_field_config = True
264 -
263 hf_to_vllm_mapper = WeightsMapper(
264 orig_to_new_prefix={
265 "model.vision_embed_tokens.": "vision_embed_tokens.",
vllm/model_executor/models/gemma3_mm.py
-2
@@ -483,8 +483,6 @@ class Gemma3MultiModalProjector(nn.Module):
483 class Gemma3ForConditionalGeneration(
484 nn.Module, SupportsMultiModal, SupportsPP, SupportsLoRA
485 ):
486 - merge_by_field_config = True
487 -
486 packed_modules_mapping = {
487 "qkv_proj": [
488 "q_proj",
vllm/model_executor/models/gemma3n_mm.py
-1
@@ -463,7 +463,6 @@ class Gemma3nMultimodalEmbedder(nn.Module):
463 class Gemma3nForConditionalGeneration(
464 nn.Module, SupportsMultiModal, SupportsTranscription
465 ):
466 - merge_by_field_config = True
466 supported_languages = ISO639_1_SUPPORTED_LANGS
467
468 packed_modules_mapping = {
vllm/model_executor/models/glm4_1v.py
-2
@@ -1424,8 +1424,6 @@ class Glm4vMultiModalProcessor(BaseMultiModalProcessor[Glm4vProcessingInfo]):
1424 class Glm4vForConditionalGeneration(
1425 nn.Module, SupportsMultiModal, SupportsLoRA, SupportsPP, SupportsMRoPE
1426 ):
1427 - merge_by_field_config = True
1428 -
1427 packed_modules_mapping = {
1428 "qkv_proj": [
1429 "q_proj",
vllm/model_executor/models/glm4v.py
-2
@@ -561,8 +561,6 @@ class GLM4VMultiModalProcessor(BaseMultiModalProcessor[GLM4VProcessingInfo]):
561 class GLM4VForCausalLM(
562 ChatGLMBaseModel, SupportsMultiModal, SupportsLoRA, SupportsPP, SupportsMRoPE
563 ):
564 - merge_by_field_config = True
565 -
564 packed_modules_mapping = {
565 "query_key_value": ["query_key_value"],
566 "dense_h_to_4h": ["dense_h_to_4h"],
vllm/model_executor/models/granite_speech.py
-1
@@ -564,7 +564,6 @@ class GraniteSpeechForConditionalGeneration(
564 SupportsLoRA,
565 SupportsTranscription,
566 ):
567 - merge_by_field_config = True
567 supported_languages = ISO639_1_SUPPORTED_LANGS
568
569 packed_modules_mapping = {
vllm/model_executor/models/hunyuan_vision.py
-1
@@ -786,7 +786,6 @@ class HunYuanVLForConditionalGeneration(
786 SupportsQuant,
787 SupportsXDRoPE,
788 ):
789 - merge_by_field_config = True
789 multimodal_cpu_fields = {"image_grid_thw"}
790
791 # To ensure correct weight loading and mapping.
vllm/model_executor/models/hyperclovax_vision.py
-2
@@ -592,8 +592,6 @@ class HCXVisionCAbstractor(nn.Module):
592 dummy_inputs=HCXVisionDummyInputsBuilder,
593 )
594 class HCXVisionForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):
595 - merge_by_field_config = True
596 -
595 packed_modules_mapping = {
596 "qkv_proj": ["q_proj", "k_proj", "v_proj"],
597 "gate_up_proj": ["gate_proj", "up_proj"],
vllm/model_executor/models/idefics3.py
-2
@@ -576,8 +576,6 @@ class Idefics3Model(nn.Module):
576 dummy_inputs=Idefics3DummyInputsBuilder,
577 )
578 class Idefics3ForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsLoRA):
579 - merge_by_field_config = True
580 -
579 packed_modules_mapping = {
580 "qkv_proj": [
581 "q_proj",
vllm/model_executor/models/interfaces.py
+22 -3
@@ -78,9 +78,9 @@ class SupportsMultiModal(Protocol):
78 `multimodal_config.mm_encoder_tp_mode="data"`.
79 """
80
81 - merge_by_field_config: ClassVar[bool] = True
81 + merge_by_field_config: ClassVar[bool | None] = None
82 """
83 - A flag that indicates which implementation of
83 + [DEPRECATED] A flag that indicates which implementation of
84 `vllm.multimodal.utils.group_mm_kwargs_by_modality` to use.
85 """
86
@@ -260,7 +260,26 @@ def supports_multimodal(model: object) -> TypeIs[SupportsMultiModal]: ...
260 def supports_multimodal(
261 model: type[object] | object,
262 ) -> TypeIs[type[SupportsMultiModal]] | TypeIs[SupportsMultiModal]:
263 - return getattr(model, "supports_multimodal", False)
263 + res = getattr(model, "supports_multimodal", False)
264 +
265 + if res:
266 + # We can remove this starting from v0.14
267 + merge_by_field_config = getattr(model, "merge_by_field_config", None)
268 + if merge_by_field_config is False:
269 + raise ValueError(
270 + "`merge_by_field_config=False` is no longer effective, "
271 + "please update your model to consider the new batching logic "
272 + "in `group_mm_kwargs_by_modality` (refer to "
273 + "https://github.com/vllm-project/vllm/issues/26149), "
274 + "and then remove the override from your model."
275 + )
276 + if merge_by_field_config is True:
277 + logger.warning_once(
278 + "`merge_by_field_config=True` is redundant, "
279 + "please remove the override from your model."
280 + )
281 +
282 + return res
283
284
285 def supports_multimodal_raw_input_only(model: type[object] | object) -> bool:
vllm/model_executor/models/interns1.py
-2
@@ -509,8 +509,6 @@ class InternS1MultiModalProcessor(BaseMultiModalProcessor[InternS1ProcessingInfo
509 class InternS1ForConditionalGeneration(
510 nn.Module, SupportsMultiModal, SupportsPP, SupportsLoRA
511 ):
512 - merge_by_field_config = True
513 -
512 # To ensure correct weight loading and mapping.
513 hf_to_vllm_mapper = WeightsMapper(
514 orig_to_new_prefix={
vllm/model_executor/models/internvl.py
-2
@@ -1074,8 +1074,6 @@ class InternVLMultiModalProcessor(
1074 dummy_inputs=InternVLDummyInputsBuilder,
1075 )
1076 class InternVLChatModel(nn.Module, SupportsMultiModal, SupportsPP, SupportsLoRA):
1077 - merge_by_field_config = True
1078 -
1077 supports_encoder_tp_data = True
1078
1079 @classmethod
vllm/model_executor/models/keye.py
-2
@@ -1292,8 +1292,6 @@ class KeyeMultiModalProcessor(BaseMultiModalProcessor[KeyeProcessingInfo]):
1292
1293
1294 class BaseKeyeModule(nn.Module):
1295 - merge_by_field_config = True
1296 -
1295 packed_modules_mapping = {
1296 "qkv_proj": [
1297 "q_proj",
vllm/model_executor/models/kimi_vl.py
-2
@@ -298,8 +298,6 @@ class KimiVLMultiModalProcessor(BaseMultiModalProcessor[KimiVLProcessingInfo]):
298 dummy_inputs=KimiVLDummyInputsBuilder,
299 )
300 class KimiVLForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP):
301 - merge_by_field_config = True
302 -
301 supports_encoder_tp_data = True
302
303 @classmethod
vllm/model_executor/models/llava.py
-2
@@ -506,8 +506,6 @@ def init_vision_tower_for_llava(
506 dummy_inputs=LlavaDummyInputsBuilder,
507 )
508 class LlavaForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP):
509 - merge_by_field_config = True
510 -
509 packed_modules_mapping = {
510 "qkv_proj": ["q_proj", "k_proj", "v_proj"],
511 "gate_up_proj": ["gate_proj", "up_proj"],
vllm/model_executor/models/llava_next.py
-2
@@ -223,8 +223,6 @@ class LlavaNextMultiModalProcessor(
223 dummy_inputs=LlavaDummyInputsBuilder,
224 )
225 class LlavaNextForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP):
226 - merge_by_field_config = True
227 -
226 hf_to_vllm_mapper = WeightsMapper(
227 orig_to_new_prefix={
228 # mapping for new names in checkpoint saved after transformers v4.52
vllm/model_executor/models/llava_next_video.py
-2
@@ -299,8 +299,6 @@ class LlavaNextMultiModalProjector(nn.Module):
299 dummy_inputs=LlavaNextVideoDummyInputsBuilder,
300 )
301 class LlavaNextVideoForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP):
302 - merge_by_field_config = True
303 -
302 hf_to_vllm_mapper = WeightsMapper(
303 orig_to_new_prefix={
304 # mapping for new names in checkpoint saved after transformers v4.52
vllm/model_executor/models/llava_onevision.py
-2
@@ -479,8 +479,6 @@ class LlavaOnevisionMultiModalProjector(nn.Module):
479 dummy_inputs=LlavaOnevisionDummyInputsBuilder,
480 )
481 class LlavaOnevisionForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP):
482 - merge_by_field_config = True
483 -
482 hf_to_vllm_mapper = WeightsMapper(
483 orig_to_new_prefix={
484 # mapping for new names in checkpoint saved after transformers v4.52
vllm/model_executor/models/midashenglm.py
-2
@@ -683,8 +683,6 @@ class MiDashengLMMultiModalProcessor(
683 dummy_inputs=MiDashengLMDummyInputsBuilder,
684 )
685 class MiDashengLMModel(nn.Module, SupportsMultiModal, SupportsPP):
686 - merge_by_field_config = True
687 -
686 packed_modules_mapping = {
687 "qkv_proj": [
688 "q_proj",
vllm/model_executor/models/minicpmv.py
-2
@@ -1003,8 +1003,6 @@ class MiniCPMVBaseModel(nn.Module, SupportsMultiModal, SupportsPP):
1003 instantiated.
1004 """
1005
1006 - merge_by_field_config = True
1007 -
1006 supports_encoder_tp_data = True
1007
1008 @classmethod
vllm/model_executor/models/minimax_vl_01.py
-2
@@ -179,8 +179,6 @@ class MiniMaxVL01MultiModalProcessor(
179 dummy_inputs=MiniMaxVL01DummyInputsBuilder,
180 )
181 class MiniMaxVL01ForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP):
182 - merge_by_field_config = True
183 -
182 packed_modules_mapping = {
183 "qkv_proj": ["q_proj", "k_proj", "v_proj"],
184 "gate_up_proj": ["gate_proj", "up_proj"],
vllm/model_executor/models/mistral3.py
-2
@@ -423,8 +423,6 @@ def init_vision_tower_for_llava(
423 class Mistral3ForConditionalGeneration(
424 nn.Module, SupportsLoRA, SupportsMultiModal, SupportsPP
425 ):
426 - merge_by_field_config = True
427 -
426 packed_modules_mapping = {
427 "qkv_proj": ["q_proj", "k_proj", "v_proj"],
428 "gate_up_proj": ["gate_proj", "up_proj"],
vllm/model_executor/models/mllama4.py
-2
@@ -741,8 +741,6 @@ class Llama4ForConditionalGeneration(
741 SupportsEagle3,
742 SupportsLoRA,
743 ):
744 - merge_by_field_config = True
745 -
744 packed_modules_mapping = {
745 "qkv_proj": ["q_proj", "k_proj", "v_proj"],
746 "gate_up_proj": ["gate_proj", "up_proj"],
vllm/model_executor/models/molmo.py
-2
@@ -1354,8 +1354,6 @@ class MolmoMultiModalProcessor(BaseMultiModalProcessor[MolmoProcessingInfo]):
1354 class MolmoForCausalLM(
1355 nn.Module, SupportsMultiModal, SupportsPP, SupportsLoRA, SupportsQuant
1356 ):
1357 - merge_by_field_config = True
1358 -
1357 hf_to_vllm_mapper = WeightsMapper(
1358 orig_to_new_substr={
1359 # vision backbone mapping
vllm/model_executor/models/nano_nemotron_vl.py
-2
@@ -1116,8 +1116,6 @@ class NanoNemotronVLDummyInputsBuilder(
1116 class NemotronH_Nano_VL_V2(
1117 nn.Module, HasInnerState, IsHybrid, SupportsMultiModal, SupportsMultiModalPruning
1118 ):
1119 - merge_by_field_config = True
1120 -
1119 @classmethod
1120 def get_placeholder_str(cls, modality: str, i: int) -> str | None:
1121 if modality.startswith("image"):
vllm/model_executor/models/nemotron_vl.py
-2
@@ -358,8 +358,6 @@ class NemotronVLProcessingInfo(BaseInternVLProcessingInfo):
358 dummy_inputs=BaseInternVLDummyInputsBuilder[NemotronVLProcessingInfo],
359 )
360 class LlamaNemotronVLChatModel(nn.Module, SupportsMultiModal, SupportsPP, SupportsLoRA):
361 - merge_by_field_config = True
362 -
361 @classmethod
362 def get_placeholder_str(cls, modality: str, i: int) -> str | None:
363 if modality.startswith("image"):
vllm/model_executor/models/opencua.py
-1
@@ -201,7 +201,6 @@ class OpenCUADummyInputsBuilder(Qwen2VLDummyInputsBuilder):
201 dummy_inputs=OpenCUADummyInputsBuilder,
202 )
203 class OpenCUAForConditionalGeneration(Qwen2_5_VLForConditionalGeneration):
204 - merge_by_field_config = True
204 multimodal_cpu_fields = {"image_grid_thw"}
205
206 packed_modules_mapping = {
vllm/model_executor/models/ovis.py
-2
@@ -414,8 +414,6 @@ class OvisMultiModalProcessor(BaseMultiModalProcessor[OvisProcessingInfo]):
414 dummy_inputs=OvisDummyInputsBuilder,
415 )
416 class Ovis(nn.Module, SupportsMultiModal, SupportsPP):
417 - merge_by_field_config = True
418 -
417 @classmethod
418 def get_placeholder_str(cls, modality: str, i: int) -> str | None:
419 if modality.startswith("image"):
vllm/model_executor/models/ovis2_5.py
-2
@@ -456,8 +456,6 @@ class Ovis2_5MultiModalProcessor(BaseMultiModalProcessor[Ovis2_5ProcessingInfo])
456 dummy_inputs=Ovis2_5DummyInputsBuilder,
457 )
458 class Ovis2_5(nn.Module, SupportsMultiModal, SupportsPP):
459 - merge_by_field_config = True
460 -
459 def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
460 super().__init__()
461 config = vllm_config.model_config.hf_config
vllm/model_executor/models/paddleocr_vl.py
-2
@@ -1103,8 +1103,6 @@ class SiglipVisionModel(nn.Module):
1103 dummy_inputs=PaddleOCRVLDummyInputsBuilder,
1104 )
1105 class PaddleOCRVLForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsMRoPE):
1106 - merge_by_field_config = True
1107 -
1106 hf_to_vllm_mapper = WeightsMapper(
1107 orig_to_new_prefix={
1108 "model.": "language_model.model.",
vllm/model_executor/models/paligemma.py
-2
@@ -251,8 +251,6 @@ class PaliGemmaMultiModalProcessor(BaseMultiModalProcessor[PaliGemmaProcessingIn
251 dummy_inputs=PaliGemmaDummyInputsBuilder,
252 )
253 class PaliGemmaForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP):
254 - merge_by_field_config = True
255 -
254 packed_modules_mapping = {
255 "qkv_proj": [
256 "q_proj",
vllm/model_executor/models/phi3v.py
-2
@@ -562,8 +562,6 @@ class Phi3VMultiModalProcessor(BaseMultiModalProcessor[Phi3VProcessingInfo]):
562 dummy_inputs=Phi3VDummyInputsBuilder,
563 )
564 class Phi3VForCausalLM(nn.Module, SupportsMultiModal, SupportsPP, SupportsQuant):
565 - merge_by_field_config = True
566 -
565 hf_to_vllm_mapper = WeightsMapper(
566 orig_to_new_prefix={
567 "model.vision_embed_tokens.wte": "embed_tokens",
vllm/model_executor/models/phi4mm.py
-2
@@ -984,8 +984,6 @@ class Phi4MMForCausalLM(nn.Module, SupportsLoRA, SupportsMultiModal):
984 Implements the Phi-4-multimodal-instruct model in vLLM.
985 """
986
987 - merge_by_field_config = True
988 -
987 packed_modules_mapping = {
988 "qkv_proj": [
989 "qkv_proj",
vllm/model_executor/models/pixtral.py
-2
@@ -365,8 +365,6 @@ class PixtralMultiModalProcessor(BaseMultiModalProcessor[PixtralProcessingInfo])
365 dummy_inputs=PixtralDummyInputsBuilder,
366 )
367 class PixtralForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP):
368 - merge_by_field_config = True
369 -
368 @classmethod
369 def get_placeholder_str(cls, modality: str, i: int) -> str | None:
370 if modality.startswith("image"):
vllm/model_executor/models/qwen2_5_omni_thinker.py
-2
@@ -773,8 +773,6 @@ class Qwen2_5OmniThinkerForConditionalGeneration(
773 SupportsMRoPE,
774 Qwen2_5OmniConditionalGenerationMixin,
775 ):
776 - merge_by_field_config = True
777 -
776 hf_to_vllm_mapper = WeightsMapper(
777 orig_to_new_prefix={
778 "thinker.lm_head.": "language_model.lm_head.",
vllm/model_executor/models/qwen2_5_vl.py
-1
@@ -1039,7 +1039,6 @@ class Qwen2_5_VLForConditionalGeneration(
1039 SupportsMultiModalPruning,
1040 SupportsMRoPE,
1041 ):
1042 - merge_by_field_config = True
1042 multimodal_cpu_fields = {"image_grid_thw", "video_grid_thw"}
1043
1044 packed_modules_mapping = {
vllm/model_executor/models/qwen2_audio.py
-2
@@ -313,8 +313,6 @@ class Qwen2AudioMultiModalProcessor(BaseMultiModalProcessor[Qwen2AudioProcessing
313 dummy_inputs=Qwen2AudioDummyInputsBuilder,
314 )
315 class Qwen2AudioForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP):
316 - merge_by_field_config = True
317 -
316 @classmethod
317 def get_placeholder_str(cls, modality: str, i: int) -> str | None:
318 if modality.startswith("audio"):
vllm/model_executor/models/qwen2_vl.py
-1
@@ -1131,7 +1131,6 @@ class Qwen2VLMultiModalProcessor(BaseMultiModalProcessor[Qwen2VLProcessingInfo])
1131 class Qwen2VLForConditionalGeneration(
1132 nn.Module, SupportsMultiModal, SupportsLoRA, SupportsPP, SupportsMRoPE
1133 ):
1134 - merge_by_field_config = True
1134 multimodal_cpu_fields = {"image_grid_thw", "video_grid_thw"}
1135
1136 # To ensure correct weight loading and mapping.
vllm/model_executor/models/qwen3_omni_moe_thinker.py
-2
@@ -1131,8 +1131,6 @@ class Qwen3OmniMoeThinkerForConditionalGeneration(
1131 SupportsMRoPE,
1132 Qwen3OmniMoeConditionalGenerationMixin,
1133 ):
1134 - merge_by_field_config = True
1135 -
1134 hf_to_vllm_mapper = WeightsMapper(
1135 orig_to_new_prefix={
1136 "thinker.lm_head.": "language_model.lm_head.",
vllm/model_executor/models/qwen3_vl.py
-1
@@ -1190,7 +1190,6 @@ class Qwen3VLForConditionalGeneration(
1190 SupportsMRoPE,
1191 SupportsEagle3,
1192 ):
1193 - merge_by_field_config = True
1193 multimodal_cpu_fields = {"image_grid_thw", "video_grid_thw"}
1194
1195 packed_modules_mapping = {
vllm/model_executor/models/qwen_vl.py
-2
@@ -703,8 +703,6 @@ class QwenVLMultiModalProcessor(BaseMultiModalProcessor[QwenVLProcessingInfo]):
703 class QwenVLForConditionalGeneration(
704 QWenBaseModel, SupportsPP, SupportsLoRA, SupportsMultiModal
705 ):
706 - merge_by_field_config = True
707 -
706 packed_modules_mapping = {
707 "c_attn": ["c_attn"],
708 "gate_up_proj": [
vllm/model_executor/models/siglip.py
-1
@@ -989,7 +989,6 @@ class SiglipEmbeddingModel(nn.Module, SupportsMultiModal, SupportsQuant):
989 is_pooling_model = True
990
991 packed_modules_mapping = {"qkv_proj": ["q_proj", "k_proj", "v_proj"]}
992 - merge_by_field_config = True
992
993 @classmethod
994 def get_placeholder_str(cls, modality: str, i: int) -> str | None:
vllm/model_executor/models/skyworkr1v.py
-2
@@ -647,8 +647,6 @@ class SkyworkR1VMultiModalProcessor(BaseMultiModalProcessor[SkyworkR1VProcessing
647 dummy_inputs=SkyworkR1VDummyInputsBuilder,
648 )
649 class SkyworkR1VChatModel(nn.Module, SupportsMultiModal, SupportsPP):
650 - merge_by_field_config = True
651 -
650 @classmethod
651 def get_placeholder_str(cls, modality: str, i: int) -> str | None:
652 if modality.startswith("image"):
vllm/model_executor/models/step3_vl.py
-2
@@ -916,8 +916,6 @@ class Step3VisionTransformer(nn.Module):
916 dummy_inputs=Step3VLDummyInputsBuilder,
917 )
918 class Step3VLForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP):
919 - merge_by_field_config = True
920 -
919 hf_to_vllm_mapper = WeightsMapper(
920 orig_to_new_prefix={
921 "model.": "language_model.model.",
vllm/model_executor/models/tarsier.py
-2
@@ -400,8 +400,6 @@ def init_vision_tower_for_tarsier(
400 dummy_inputs=TarsierDummyInputsBuilder,
401 )
402 class TarsierForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP):
403 - merge_by_field_config = True
404 -
403 packed_modules_mapping = {
404 "qkv_proj": ["q_proj", "k_proj", "v_proj"],
405 "gate_up_proj": ["gate_proj", "up_proj"],
vllm/model_executor/models/terratorch.py
-1
@@ -227,7 +227,6 @@ class TerratorchMultiModalProcessor(BaseMultiModalProcessor):
227 dummy_inputs=TerratorchInputBuilder,
228 )
229 class Terratorch(nn.Module, IsAttentionFree, SupportsMultiModal):
230 - merge_by_field_config = True
230 supports_multimodal_raw_input_only = True
231 is_pooling_model = True
232
vllm/model_executor/models/transformers/multimodal.py
+1 -1
@@ -264,7 +264,7 @@ class MultiModalProcessor(BaseMultiModalProcessor[MultiModalProcessingInfo]):
264
265 class MultiModalMixin(SupportsMultiModal, SupportsMRoPE):
266 supports_multimodal_raw_input_only = True
267 - merge_by_field_config = True
267 +
268 # Backwards compatibility for prev released models. State dicts back then
269 # had different formats and cannot be loaded with `AutoModel` mapping as is
270 hf_to_vllm_mapper = WeightsMapper(
vllm/model_executor/models/ultravox.py
-2
@@ -498,8 +498,6 @@ class ModifiedWhisperEncoder(WhisperEncoder):
498 dummy_inputs=UltravoxDummyInputsBuilder,
499 )
500 class UltravoxModel(nn.Module, SupportsMultiModal, SupportsPP, SupportsLoRA):
501 - merge_by_field_config = True
502 -
501 packed_modules_mapping = {
502 "qkv_proj": ["q_proj", "k_proj", "v_proj"],
503 "gate_up_proj": ["gate_proj", "up_proj"],
vllm/model_executor/models/voxtral.py
-2
@@ -330,8 +330,6 @@ class VoxtralMultiModalProcessor(BaseMultiModalProcessor[VoxtralProcessingInfo])
330 class VoxtralForConditionalGeneration(
331 nn.Module, SupportsMultiModal, SupportsPP, SupportsLoRA, SupportsTranscription
332 ):
333 - merge_by_field_config = True
334 -
333 supported_languages = ISO639_1_SUPPORTED_LANGS
334
335 packed_modules_mapping = {
vllm/model_executor/models/whisper.py
-1
@@ -775,7 +775,6 @@ class WhisperMultiModalProcessor(EncDecMultiModalProcessor[WhisperProcessingInfo
775 class WhisperForConditionalGeneration(
776 nn.Module, SupportsTranscription, SupportsMultiModal
777 ):
778 - merge_by_field_config = True
778 packed_modules_mapping = {
779 "self_attn.qkv_proj": [
780 "self_attn.q_proj",
vllm/multimodal/utils.py
-1
@@ -426,7 +426,6 @@ def group_mm_kwargs_by_modality(
426 Yields:
427 A tuple `(modality, num_items, grouped_kwargs)`.
428 """
429 - # TODO: After v0.13, remove merge_by_field_config attribute from model impls
429 if merge_by_field_config is not None:
430 logger.warning_once(
431 "The `merge_by_field_config` argument of `group_mm_kwargs_by_modality` "