d55244df31 · tduncan/Nvidia-Optimization-Stack-VLLM

docs/source/contributing/model/multimodal.md

+11

@@ -79,6 +79,17 @@ Further update the model as follows:

79 return inputs_embeds

80 ```

81

82 +- Implement {meth}`~vllm.model_executor.models.interfaces.SupportsMultiModal.get_language_model` getter to provide stable access to the underlying language model.

83 +

84 + ```python

85 + class YourModelForImage2Seq(nn.Module):

86 + ...

87 +

88 + def get_language_model(self) -> torch.nn.Module:

89 + # Change `language_model` according to your implementation.

90 + return self.language_model

91 + ```

92 +

93 - Once the above steps are done, update the model class with the {class}`~vllm.model_executor.models.interfaces.SupportsMultiModal` interface.

94

95 ```diff

vllm/model_executor/models/aria.py

+3

@@ -605,6 +605,9 @@ class AriaForConditionalGeneration(nn.Module, SupportsMultiModal):

605

606 return self.multi_modal_projector(image_outputs, image_attn_mask)

607

608 + def get_language_model(self) -> torch.nn.Module:

609 + return self.language_model

610 +

611 def get_multimodal_embeddings(

612 self, **kwargs: object) -> Optional[MultiModalEmbeddings]:

613 image_input = self._parse_and_validate_image_input(**kwargs)

vllm/model_executor/models/aya_vision.py

+3

@@ -424,6 +424,9 @@ class AyaVisionForConditionalGeneration(nn.Module, SupportsMultiModal,

424 num_patches=num_patches,

425 )

426

427 + def get_language_model(self) -> torch.nn.Module:

428 + return self.language_model

429 +

430 def get_multimodal_embeddings(

431 self, **kwargs: object) -> Optional[MultiModalEmbeddings]:

432 image_input = self._parse_and_validate_image_input(**kwargs)

vllm/model_executor/models/blip2.py

+3

@@ -627,6 +627,9 @@ class Blip2ForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP,

627

628 return self.language_projection(query_output)

629

630 + def get_language_model(self) -> torch.nn.Module:

631 + return self.language_model

632 +

633 def get_multimodal_embeddings(

634 self, **kwargs: object) -> Optional[MultiModalEmbeddings]:

635 image_input = self._parse_and_validate_image_input(**kwargs)

vllm/model_executor/models/chameleon.py

+3

@@ -988,6 +988,9 @@ class ChameleonForConditionalGeneration(nn.Module, SupportsMultiModal,

988 data=self._validate_pixel_values(pixel_values),

989 )

990

991 + def get_language_model(self) -> torch.nn.Module:

992 + return self.model

993 +

994 def get_multimodal_embeddings(

995 self, **kwargs: object) -> Optional[MultiModalEmbeddings]:

996 image_input = self._parse_and_validate_image_input(**kwargs)

vllm/model_executor/models/deepseek_vl2.py

+3

@@ -604,6 +604,9 @@ class DeepseekVLV2ForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):

604 return self._pixel_values_to_embedding(

605 pixel_values=pixel_values, images_spatial_crop=images_spatial_crop)

606

607 + def get_language_model(self) -> torch.nn.Module:

608 + return self.language_model

609 +

610 def get_multimodal_embeddings(

611 self, **kwargs: object) -> Optional[MultiModalEmbeddings]:

612 image_input = self._parse_and_validate_image_input(**kwargs)

vllm/model_executor/models/florence2.py

+3

@@ -1050,6 +1050,9 @@ class Florence2ForConditionalGeneration(nn.Module, SupportsMultiModal,

1050 pixel_values = image_input["data"]

1051 return self._encode_image(pixel_values)

1052

1053 + def get_language_model(self) -> torch.nn.Module:

1054 + return self.language_model

1055 +

1056 def get_multimodal_embeddings(

1057 self, **kwargs: object) -> Optional[MultiModalEmbeddings]:

1058 image_input = self._parse_and_validate_image_input(**kwargs)

vllm/model_executor/models/fuyu.py

+3

@@ -341,6 +341,9 @@ class FuyuForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):

341

342 return vision_embeddings_flat.split(patches_per_image, dim=0)

343

344 + def get_language_model(self) -> torch.nn.Module:

345 + return self.language_model

346 +

347 def get_multimodal_embeddings(

348 self, **kwargs: object) -> Optional[MultiModalEmbeddings]:

349 image_input = self._parse_and_validate_image_input(**kwargs)

vllm/model_executor/models/gemma3_mm.py

+3

@@ -591,6 +591,9 @@ class Gemma3ForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP,

591 e.flatten(0, 1) for e in image_embeds.split(num_patches.tolist())

592 ]

593

594 + def get_language_model(self) -> torch.nn.Module:

595 + return self.language_model

596 +

597 def get_multimodal_embeddings(

598 self, **kwargs: object) -> Optional[MultiModalEmbeddings]:

599 image_input = self._parse_and_validate_image_input(**kwargs)

vllm/model_executor/models/glm4v.py

+3

@@ -596,6 +596,9 @@ class GLM4VForCausalLM(ChatGLMBaseModel, SupportsLoRA, SupportsPP,

596

597 return self.transformer.vision(pixel_values)

598

599 + def get_language_model(self) -> torch.nn.Module:

600 + return self.transformer

601 +

602 def get_multimodal_embeddings(

603 self, **kwargs: object) -> Optional[MultiModalEmbeddings]:

604 image_input = self._parse_and_validate_image_input(**kwargs)

vllm/model_executor/models/idefics3.py

+3

@@ -710,6 +710,9 @@ class Idefics3ForConditionalGeneration(nn.Module, SupportsMultiModal,

710 e.flatten(0, 1) for e in image_features.split(num_patches.tolist())

711 ]

712

713 + def get_language_model(self) -> torch.nn.Module:

714 + return self.model

715 +

716 def get_multimodal_embeddings(

717 self, **kwargs: object) -> Optional[MultiModalEmbeddings]:

718 image_input = self._parse_and_validate_image_input(**kwargs)

vllm/model_executor/models/interfaces.py

+12

@@ -56,6 +56,18 @@ class SupportsMultiModal(Protocol):

56 """

57 ...

58

59 + def get_language_model(self) -> torch.nn.Module:

60 + """

61 + Returns the underlying language model used for text generation.

62 +

63 + This is typically the `torch.nn.Module` instance responsible for

64 + processing the merged multimodal embeddings and producing hidden states

65 +

66 + Returns:

67 + torch.nn.Module: The core language model component.

68 + """

69 + ...

70 +

71 # Only for models that support v0 chunked prefill

72 # TODO(ywang96): Remove this overload once v0 is deprecated

73 @overload

vllm/model_executor/models/internvl.py

+3

@@ -884,6 +884,9 @@ class InternVLChatModel(nn.Module, SupportsMultiModal, SupportsPP):

884 else:

885 self.visual_token_mask = None

886

887 + def get_language_model(self) -> torch.nn.Module:

888 + return self.language_model

889 +

890 def get_multimodal_embeddings(

891 self, **kwargs: object) -> Optional[MultiModalEmbeddings]:

892 image_input = self._parse_and_validate_image_input(**kwargs)

vllm/model_executor/models/llava.py

+3

@@ -674,6 +674,9 @@ class LlavaForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP):

674 image_embeds = torch.split(image_embeds, feature_sizes)

675 return image_embeds

676

677 + def get_language_model(self) -> torch.nn.Module:

678 + return self.language_model

679 +

680 def get_multimodal_embeddings(

681 self, **kwargs: object) -> Optional[MultiModalEmbeddings]:

682 image_input = self._parse_and_validate_image_input(**kwargs)

vllm/model_executor/models/llava_next.py

+3

@@ -480,6 +480,9 @@ class LlavaNextForConditionalGeneration(nn.Module, SupportsMultiModal,

480 for i, patch_features_batch in enumerate(patch_embeddings)

481 ]

482

483 + def get_language_model(self) -> torch.nn.Module:

484 + return self.language_model

485 +

486 def get_multimodal_embeddings(

487 self, **kwargs: object) -> Optional[MultiModalEmbeddings]:

488 image_input = self._parse_and_validate_image_input(**kwargs)

vllm/model_executor/models/llava_next_video.py

+3

@@ -421,6 +421,9 @@ class LlavaNextVideoForConditionalGeneration(nn.Module, SupportsMultiModal,

421

422 return [e.flatten(0, 1) for e in embeds]

423

424 + def get_language_model(self) -> torch.nn.Module:

425 + return self.language_model

426 +

427 def get_multimodal_embeddings(

428 self, **kwargs: object) -> Optional[MultiModalEmbeddings]:

429 video_input = self._parse_and_validate_video_input(**kwargs)

vllm/model_executor/models/llava_onevision.py

+3

@@ -852,6 +852,9 @@ class LlavaOnevisionForConditionalGeneration(nn.Module, SupportsMultiModal,

852 image_feature = image_feature.view(batch_frames, -1, dim)

853 return image_feature

854

855 + def get_language_model(self) -> torch.nn.Module:

856 + return self.language_model

857 +

858 def get_multimodal_embeddings(

859 self, **kwargs: object) -> Optional[MultiModalEmbeddings]:

860 modalities = self._parse_and_validate_multimodal_inputs(**kwargs)

vllm/model_executor/models/minicpmv.py

+3

@@ -892,6 +892,9 @@ class MiniCPMVBaseModel(nn.Module, SupportsMultiModal, SupportsPP):

892

893 return multimodal_embeddings

894

895 + def get_language_model(self) -> torch.nn.Module:

896 + return self.llm

897 +

898 def get_multimodal_embeddings(

899 self, **kwargs: object) -> Optional[MultiModalEmbeddings]:

900 modalities = self._parse_and_validate_multimodal_inputs(**kwargs)

vllm/model_executor/models/mistral3.py

+3

@@ -514,6 +514,9 @@ class Mistral3ForConditionalGeneration(nn.Module, SupportsMultiModal,

514 image_embeds = (image_embeds, )

515 return image_embeds

516

517 + def get_language_model(self) -> torch.nn.Module:

518 + return self.language_model

519 +

520 def get_multimodal_embeddings(

521 self, **kwargs: object) -> Optional[MultiModalEmbeddings]:

522 image_input = self._parse_and_validate_image_input(**kwargs)

vllm/model_executor/models/mllama.py

+3

@@ -1325,6 +1325,9 @@ class MllamaForConditionalGeneration(nn.Module, SupportsMultiModal,

1325 cross_attention_states = cross_attention_states_flat

1326 return cross_attention_states

1327

1328 + def get_language_model(self) -> torch.nn.Module:

1329 + return self.language_model

1330 +

1331 def get_cross_attention_states(

1332 self,

1333 image_inputs: MllamaImagePixelInputs,

vllm/model_executor/models/mllama4.py

+3

@@ -742,6 +742,9 @@ class Llama4ForConditionalGeneration(nn.Module, SupportsMultiModal,

742 for img in vision_embeddings_flat.split(patches_per_image, dim=0)

743 ]

744

745 + def get_language_model(self) -> torch.nn.Module:

746 + return self.language_model

747 +

748 def get_multimodal_embeddings(self,

749 **kwargs) -> Optional[MultiModalEmbeddings]:

750 image_input = self._parse_and_validate_image_input(**kwargs)

vllm/model_executor/models/molmo.py

+3

@@ -1488,6 +1488,9 @@ class MolmoForCausalLM(nn.Module, SupportsMultiModal, SupportsPP, SupportsLoRA,

1488 )

1489 ]

1490

1491 + def get_language_model(self) -> torch.nn.Module:

1492 + return self.model

1493 +

1494 def get_multimodal_embeddings(

1495 self, **kwargs: object) -> Optional[MultiModalEmbeddings]:

1496 image_input = self._parse_and_validate_image_input(**kwargs)

vllm/model_executor/models/paligemma.py

+3

@@ -323,6 +323,9 @@ class PaliGemmaForConditionalGeneration(nn.Module, SupportsMultiModal,

323

324 return self.multi_modal_projector(image_features)

325

326 + def get_language_model(self) -> torch.nn.Module:

327 + return self.language_model

328 +

329 def get_multimodal_embeddings(

330 self, **kwargs: object) -> Optional[MultiModalEmbeddings]:

331 image_input = self._parse_and_validate_image_input(**kwargs)

vllm/model_executor/models/phi3v.py

+3

@@ -674,6 +674,9 @@ class Phi3VForCausalLM(nn.Module, SupportsMultiModal, SupportsPP,

674

675 return image_embeds

676

677 + def get_language_model(self) -> torch.nn.Module:

678 + return self.language_model

679 +

680 def get_multimodal_embeddings(

681 self, **kwargs: object) -> Optional[MultiModalEmbeddings]:

682 image_input = self._parse_and_validate_image_input(**kwargs)

vllm/model_executor/models/phi4mm.py

+3

@@ -1802,3 +1802,6 @@ class Phi4MMForCausalLM(nn.Module, SupportsLoRA, SupportsMultiModal,

1802 connector=["audio_projection_for_vision", "audio_projection"],

1803 tower_model=["vision_encoder", "embed_tokens_extend"],

1804 )

1805 +

1806 + def get_language_model(self) -> torch.nn.Module:

1807 + return self.model

vllm/model_executor/models/pixtral.py

+3

@@ -396,6 +396,9 @@ class PixtralForConditionalGeneration(nn.Module, SupportsMultiModal,

396 image_embeds = torch.split(image_embeds, feature_sizes)

397 return image_embeds

398

399 + def get_language_model(self) -> torch.nn.Module:

400 + return self.language_model

401 +

402 def get_multimodal_embeddings(

403 self, **kwargs: object) -> Optional[MultiModalEmbeddings]:

404 image_input = self._parse_and_validate_image_input(**kwargs)

vllm/model_executor/models/qwen2_5_vl.py

+3

@@ -967,6 +967,9 @@ class Qwen2_5_VLForConditionalGeneration(nn.Module, SupportsMultiModal,

967 **kwargs)

968 return modalities

969

970 + def get_language_model(self) -> torch.nn.Module:

971 + return self.language_model

972 +

973 def get_multimodal_embeddings(

974 self, **kwargs: object) -> Optional[MultiModalEmbeddings]:

975

vllm/model_executor/models/qwen2_audio.py

+3

@@ -355,6 +355,9 @@ class Qwen2AudioForConditionalGeneration(nn.Module, SupportsMultiModal,

355 return torch.split(masked_audio_features,

356 audio_output_lengths.flatten().tolist())

357

358 + def get_language_model(self) -> torch.nn.Module:

359 + return self.language_model

360 +

361 def get_multimodal_embeddings(

362 self, **kwargs: object) -> Optional[MultiModalEmbeddings]:

363 audio_input = self._parse_and_validate_audio_input(**kwargs)

vllm/model_executor/models/qwen2_vl.py

+3

@@ -1276,6 +1276,9 @@ class Qwen2VLForConditionalGeneration(nn.Module, SupportsMultiModal,

1276

1277 return modalities

1278

1279 + def get_language_model(self) -> torch.nn.Module:

1280 + return self.language_model

1281 +

1282 def get_multimodal_embeddings(

1283 self, **kwargs: object) -> Optional[MultiModalEmbeddings]:

1284

vllm/model_executor/models/qwen_vl.py

+3

@@ -740,6 +740,9 @@ class QwenVLForConditionalGeneration(QWenBaseModel, SupportsPP, SupportsLoRA,

740

741 return self.transformer.visual(image_input["data"])

742

743 + def get_language_model(self) -> torch.nn.Module:

744 + return self.transformer

745 +

746 def get_multimodal_embeddings(

747 self, **kwargs: object) -> Optional[MultiModalEmbeddings]:

748 image_input = self._parse_and_validate_image_input(**kwargs)

vllm/model_executor/models/skyworkr1v.py

+3

@@ -889,6 +889,9 @@ class SkyworkR1VChatModel(nn.Module, SupportsMultiModal, SupportsPP):

889 else:

890 self.visual_token_mask = None

891

892 + def get_language_model(self) -> torch.nn.Module:

893 + return self.language_model

894 +

895 def get_multimodal_embeddings(

896 self, **kwargs: object) -> Optional[MultiModalEmbeddings]:

897 image_input = self._parse_and_validate_image_input(**kwargs)

vllm/model_executor/models/ultravox.py

+3

@@ -563,6 +563,9 @@ class UltravoxModel(nn.Module, SupportsMultiModal, SupportsPP, SupportsLoRA):

563 ]

564 return flattened_embeddings.split(embed_lens)

565

566 + def get_language_model(self) -> torch.nn.Module:

567 + return self.language_model

568 +

569 def get_multimodal_embeddings(

570 self, **kwargs: object) -> Optional[MultiModalEmbeddings]:

571 audio_input = self._parse_and_validate_audio_input(**kwargs)

vllm/model_executor/models/whisper.py

+3

@@ -692,6 +692,9 @@ class WhisperForConditionalGeneration(nn.Module, SupportsTranscription,

692 )

693 return decoder_outputs

694

695 + def get_language_model(self) -> torch.nn.Module:

696 + return self.model.decoder

697 +

698 def get_multimodal_embeddings(

699 self, **kwargs: object) -> Optional[MultiModalEmbeddings]:

700 # TODO: This method does not obey the interface for SupportsMultiModal.

[Model] Add `SupportsMultiModal.get_language_model` interface (#16007)