90f9c2eb5c · tduncan/Nvidia-Optimization-Stack-VLLM

vllm/model_executor/models/aria.py

+3 -3

@@ -601,11 +601,11 @@ class AriaForConditionalGeneration(nn.Module, SupportsMultiModal):

601 def get_language_model(self) -> torch.nn.Module:

602 return self.language_model

603

604 - def get_multimodal_embeddings(

605 - self, **kwargs: object) -> Optional[MultiModalEmbeddings]:

604 + def get_multimodal_embeddings(self,

605 + **kwargs: object) -> MultiModalEmbeddings:

606 image_input = self._parse_and_validate_image_input(**kwargs)

607 if image_input is None:

608 - return None

608 + return []

609 multimodal_embeddings = self._process_image_input(image_input)

610 return multimodal_embeddings

611

vllm/model_executor/models/aya_vision.py

+3 -3

@@ -406,11 +406,11 @@ class AyaVisionForConditionalGeneration(nn.Module, SupportsMultiModal,

406 def get_language_model(self) -> torch.nn.Module:

407 return self.language_model

408

409 - def get_multimodal_embeddings(

410 - self, **kwargs: object) -> Optional[MultiModalEmbeddings]:

409 + def get_multimodal_embeddings(self,

410 + **kwargs: object) -> MultiModalEmbeddings:

411 image_input = self._parse_and_validate_image_input(**kwargs)

412 if image_input is None:

413 - return None

413 + return []

414

415 return self._process_image_input(image_input, **kwargs)

416

vllm/model_executor/models/blip2.py

+3 -3

@@ -627,11 +627,11 @@ class Blip2ForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP,

627 def get_language_model(self) -> torch.nn.Module:

628 return self.language_model

629

630 - def get_multimodal_embeddings(

631 - self, **kwargs: object) -> Optional[MultiModalEmbeddings]:

630 + def get_multimodal_embeddings(self,

631 + **kwargs: object) -> MultiModalEmbeddings:

632 image_input = self._parse_and_validate_image_input(**kwargs)

633 if image_input is None:

634 - return None

634 + return []

635 vision_embeddings = self._process_image_input(image_input)

636 return vision_embeddings

637

vllm/model_executor/models/chameleon.py

+3 -3

@@ -987,11 +987,11 @@ class ChameleonForConditionalGeneration(nn.Module, SupportsMultiModal,

987 def get_language_model(self) -> torch.nn.Module:

988 return self.model

989

990 - def get_multimodal_embeddings(

991 - self, **kwargs: object) -> Optional[MultiModalEmbeddings]:

990 + def get_multimodal_embeddings(self,

991 + **kwargs: object) -> MultiModalEmbeddings:

992 image_input = self._parse_and_validate_image_input(**kwargs)

993 if image_input is None:

994 - return None

994 + return []

995 assert self.model.vqmodel is not None

996 image_tokens = self.model.get_image_tokens(image_input["data"].to(

997 self.config.torch_dtype))

vllm/model_executor/models/deepseek_vl2.py

+3 -3

@@ -586,11 +586,11 @@ class DeepseekVLV2ForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):

586 def get_language_model(self) -> torch.nn.Module:

587 return self.language_model

588

589 - def get_multimodal_embeddings(

590 - self, **kwargs: object) -> Optional[MultiModalEmbeddings]:

589 + def get_multimodal_embeddings(self,

590 + **kwargs: object) -> MultiModalEmbeddings:

591 image_input = self._parse_and_validate_image_input(**kwargs)

592 if image_input is None:

593 - return None

593 + return []

594 vision_embeddings = self._process_image_input(image_input)

595 return vision_embeddings

596

vllm/model_executor/models/florence2.py

+3 -3

@@ -1032,11 +1032,11 @@ class Florence2ForConditionalGeneration(nn.Module, SupportsMultiModal,

1032 def get_language_model(self) -> torch.nn.Module:

1033 return self.language_model

1034

1035 - def get_multimodal_embeddings(

1036 - self, **kwargs: object) -> Optional[MultiModalEmbeddings]:

1035 + def get_multimodal_embeddings(self,

1036 + **kwargs: object) -> MultiModalEmbeddings:

1037 image_input = self._parse_and_validate_image_input(**kwargs)

1038 if image_input is None:

1039 - return None

1039 + return []

1040 vision_embeddings = self._process_image_input(image_input)

1041 return vision_embeddings

1042

vllm/model_executor/models/fuyu.py

+3 -3

@@ -324,11 +324,11 @@ class FuyuForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):

324 def get_language_model(self) -> torch.nn.Module:

325 return self.language_model

326

327 - def get_multimodal_embeddings(

328 - self, **kwargs: object) -> Optional[MultiModalEmbeddings]:

327 + def get_multimodal_embeddings(self,

328 + **kwargs: object) -> MultiModalEmbeddings:

329 image_input = self._parse_and_validate_image_input(**kwargs)

330 if image_input is None:

331 - return None

331 + return []

332

333 return self._process_image_input(image_input)

334

vllm/model_executor/models/gemma3_mm.py

+3 -3

@@ -568,11 +568,11 @@ class Gemma3ForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP,

568 def get_language_model(self) -> torch.nn.Module:

569 return self.language_model

570

571 - def get_multimodal_embeddings(

572 - self, **kwargs: object) -> Optional[MultiModalEmbeddings]:

571 + def get_multimodal_embeddings(self,

572 + **kwargs: object) -> MultiModalEmbeddings:

573 image_input = self._parse_and_validate_image_input(**kwargs)

574 if image_input is None:

575 - return None

575 + return []

576

577 return self._process_image_input(image_input)

578

vllm/model_executor/models/glm4v.py

+3 -3

@@ -593,11 +593,11 @@ class GLM4VForCausalLM(ChatGLMBaseModel, SupportsLoRA, SupportsPP,

593 def get_language_model(self) -> torch.nn.Module:

594 return self.transformer

595

596 - def get_multimodal_embeddings(

597 - self, **kwargs: object) -> Optional[MultiModalEmbeddings]:

596 + def get_multimodal_embeddings(self,

597 + **kwargs: object) -> MultiModalEmbeddings:

598 image_input = self._parse_and_validate_image_input(**kwargs)

599 if image_input is None:

600 - return None

600 + return []

601

602 vision_embeddings = self._process_image_input(image_input)

603 return vision_embeddings

vllm/model_executor/models/granite_speech.py

+2 -1

@@ -706,10 +706,11 @@ class GraniteSpeechForConditionalGeneration(

706 def get_multimodal_embeddings(

707 self,

708 **kwargs: object,

709 - ) -> Optional[MultiModalEmbeddings]:

709 + ) -> MultiModalEmbeddings:

710 """Compute the audio embeddings if audio inputs are present."""

711 audio_input = self._parse_and_validate_audio_input(**kwargs)

712 if audio_input is None:

713 + return []

714 return None

715 audio_features = self._process_audio_input(audio_input)

716 return audio_features

vllm/model_executor/models/idefics3.py

+3 -3

@@ -706,11 +706,11 @@ class Idefics3ForConditionalGeneration(nn.Module, SupportsMultiModal,

706 def get_language_model(self) -> torch.nn.Module:

707 return self.model

708

709 - def get_multimodal_embeddings(

710 - self, **kwargs: object) -> Optional[MultiModalEmbeddings]:

709 + def get_multimodal_embeddings(self,

710 + **kwargs: object) -> MultiModalEmbeddings:

711 image_input = self._parse_and_validate_image_input(**kwargs)

712 if image_input is None:

713 - return None

713 + return []

714

715 return self._process_image_input(image_input)

716

vllm/model_executor/models/interfaces.py

+2 -2

@@ -44,8 +44,8 @@ class SupportsMultiModal(Protocol):

44 MRO of your model class.

45 """

46

47 - def get_multimodal_embeddings(

48 - self, **kwargs: object) -> Optional[MultiModalEmbeddings]:

47 + def get_multimodal_embeddings(self,

48 + **kwargs: object) -> MultiModalEmbeddings:

49 """

50 Returns multimodal embeddings generated from multimodal kwargs

51 to be merged with text embeddings.

vllm/model_executor/models/internvl.py

+3 -2

@@ -1304,11 +1304,12 @@ class InternVLChatModel(nn.Module, SupportsMultiModal, SupportsPP,

1304 def get_language_model(self) -> torch.nn.Module:

1305 return self.language_model

1306

1307 - def get_multimodal_embeddings(

1308 - self, **kwargs: object) -> Optional[MultiModalEmbeddings]:

1307 + def get_multimodal_embeddings(self,

1308 + **kwargs: object) -> MultiModalEmbeddings:

1309

1310 modalities = self._parse_and_validate_multimodal_inputs(**kwargs)

1311 if not modalities:

1312 + return []

1313 return None

1314

1315 # The result multimodal_embeddings is tuple of tensors, with each

vllm/model_executor/models/llava.py

+3 -3

@@ -659,11 +659,11 @@ class LlavaForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP):

659 def get_language_model(self) -> torch.nn.Module:

660 return self.language_model

661

662 - def get_multimodal_embeddings(

663 - self, **kwargs: object) -> Optional[MultiModalEmbeddings]:

662 + def get_multimodal_embeddings(self,

663 + **kwargs: object) -> MultiModalEmbeddings:

664 image_input = self._parse_and_validate_image_input(**kwargs)

665 if image_input is None:

666 - return None

666 + return []

667

668 return self._process_image_input(image_input)

669

vllm/model_executor/models/llava_next.py

+4 -4

@@ -478,11 +478,11 @@ class LlavaNextForConditionalGeneration(nn.Module, SupportsMultiModal,

478 def get_language_model(self) -> torch.nn.Module:

479 return self.language_model

480

481 - def get_multimodal_embeddings(

482 - self, **kwargs: object) -> Optional[MultiModalEmbeddings]:

481 + def get_multimodal_embeddings(self,

482 + **kwargs: object) -> MultiModalEmbeddings:

483 image_input = self._parse_and_validate_image_input(**kwargs)

484 if image_input is None:

485 - return None

485 + return []

486 vision_embeddings = self._process_image_input(image_input)

487 return vision_embeddings

488

@@ -492,7 +492,7 @@ class LlavaNextForConditionalGeneration(nn.Module, SupportsMultiModal,

492 multimodal_embeddings: Optional[MultiModalEmbeddings] = None,

493 ) -> torch.Tensor:

494

495 - if multimodal_embeddings is None:

495 + if not multimodal_embeddings:

496 return self.language_model.get_input_embeddings(input_ids)

497

498 inputs_embeds = embed_multimodal(

vllm/model_executor/models/llava_next_video.py

+3 -3

@@ -401,11 +401,11 @@ class LlavaNextVideoForConditionalGeneration(nn.Module, SupportsMultiModal,

401 def get_language_model(self) -> torch.nn.Module:

402 return self.language_model

403

404 - def get_multimodal_embeddings(

405 - self, **kwargs: object) -> Optional[MultiModalEmbeddings]:

404 + def get_multimodal_embeddings(self,

405 + **kwargs: object) -> MultiModalEmbeddings:

406 video_input = self._parse_and_validate_video_input(**kwargs)

407 if video_input is None:

408 - return None

408 + return []

409 vision_embeddings = self._process_video_pixels(video_input)

410 return vision_embeddings

411

vllm/model_executor/models/llava_onevision.py

+3 -2

@@ -839,11 +839,12 @@ class LlavaOnevisionForConditionalGeneration(nn.Module, SupportsMultiModal,

839 def get_language_model(self) -> torch.nn.Module:

840 return self.language_model

841

842 - def get_multimodal_embeddings(

843 - self, **kwargs: object) -> Optional[MultiModalEmbeddings]:

842 + def get_multimodal_embeddings(self,

843 + **kwargs: object) -> MultiModalEmbeddings:

844 mm_input_by_modality = self._parse_and_validate_multimodal_inputs(

845 **kwargs)

846 if not mm_input_by_modality:

847 + return []

848 return None

849

850 # The result multimodal_embeddings is tuple of tensors, with each

vllm/model_executor/models/minicpmv.py

+3 -3

@@ -878,11 +878,11 @@ class MiniCPMVBaseModel(nn.Module, SupportsMultiModal, SupportsPP):

878 def get_language_model(self) -> torch.nn.Module:

879 return self.llm

880

881 - def get_multimodal_embeddings(

882 - self, **kwargs: object) -> Optional[MultiModalEmbeddings]:

881 + def get_multimodal_embeddings(self,

882 + **kwargs: object) -> MultiModalEmbeddings:

883 modalities = self._parse_and_validate_multimodal_inputs(**kwargs)

884 if not modalities:

885 - return None

885 + return []

886

887 return self._process_multimodal_inputs(modalities)

888

vllm/model_executor/models/minimax_vl_01.py

+3 -3

@@ -318,11 +318,11 @@ class MiniMaxVL01ForConditionalGeneration(nn.Module, SupportsMultiModal,

318

319 raise AssertionError("This line should be unreachable.")

320

321 - def get_multimodal_embeddings(

322 - self, **kwargs: object) -> Optional[MultiModalEmbeddings]:

321 + def get_multimodal_embeddings(self,

322 + **kwargs: object) -> MultiModalEmbeddings:

323 image_input = self._parse_and_validate_image_input(**kwargs)

324 if image_input is None:

325 - return None

325 + return []

326

327 return self._process_image_input(image_input)

328

vllm/model_executor/models/mistral3.py

+3 -3

@@ -495,11 +495,11 @@ class Mistral3ForConditionalGeneration(nn.Module, SupportsLoRA,

495 def get_language_model(self) -> torch.nn.Module:

496 return self.language_model

497

498 - def get_multimodal_embeddings(

499 - self, **kwargs: object) -> Optional[MultiModalEmbeddings]:

498 + def get_multimodal_embeddings(self,

499 + **kwargs: object) -> MultiModalEmbeddings:

500 image_input = self._parse_and_validate_image_input(**kwargs)

501 if image_input is None:

502 - return None

502 + return []

503

504 vision_embeddings = self._process_image_input(image_input)

505

vllm/model_executor/models/mllama4.py

+2 -3

@@ -794,11 +794,10 @@ class Llama4ForConditionalGeneration(nn.Module, SupportsMultiModal,

794 def get_language_model(self) -> torch.nn.Module:

795 return self.language_model

796

797 - def get_multimodal_embeddings(self,

798 - **kwargs) -> Optional[MultiModalEmbeddings]:

797 + def get_multimodal_embeddings(self, **kwargs) -> MultiModalEmbeddings:

798 image_input = self._parse_and_validate_image_input(**kwargs)

799 if image_input is None:

801 - return None

800 + return []

801

802 return self._process_image_input(image_input)

803

vllm/model_executor/models/molmo.py

+3 -3

@@ -1473,11 +1473,11 @@ class MolmoForCausalLM(nn.Module, SupportsMultiModal, SupportsPP, SupportsLoRA,

1473 def get_language_model(self) -> torch.nn.Module:

1474 return self.model

1475

1476 - def get_multimodal_embeddings(

1477 - self, **kwargs: object) -> Optional[MultiModalEmbeddings]:

1476 + def get_multimodal_embeddings(self,

1477 + **kwargs: object) -> MultiModalEmbeddings:

1478 image_input = self._parse_and_validate_image_input(**kwargs)

1479 if image_input is None:

1480 - return None

1480 + return []

1481

1482 return self._process_image_input(image_input)

1483

vllm/model_executor/models/ovis.py

+3 -3

@@ -499,11 +499,11 @@ class Ovis(nn.Module, SupportsMultiModal, SupportsPP):

499

500 return tuple(vision_embeddings)

501

502 - def get_multimodal_embeddings(

503 - self, **kwargs: object) -> Optional[MultiModalEmbeddings]:

502 + def get_multimodal_embeddings(self,

503 + **kwargs: object) -> MultiModalEmbeddings:

504 image_input = self._parse_and_validate_image_input(**kwargs)

505 if image_input is None:

506 - return None

506 + return []

507

508 image_features = self._process_image_input(image_input)

509

vllm/model_executor/models/paligemma.py

+3 -3

@@ -338,11 +338,11 @@ class PaliGemmaForConditionalGeneration(nn.Module, SupportsMultiModal,

338 def get_language_model(self) -> torch.nn.Module:

339 return self.language_model

340

341 - def get_multimodal_embeddings(

342 - self, **kwargs: object) -> Optional[MultiModalEmbeddings]:

341 + def get_multimodal_embeddings(self,

342 + **kwargs: object) -> MultiModalEmbeddings:

343 image_input = self._parse_and_validate_image_input(**kwargs)

344 if image_input is None:

345 - return None

345 + return []

346 vision_embeddings = self._process_image_input(image_input)

347 # https://github.com/huggingface/transformers/blob/main/src/transformers/models/paligemma/modeling_paligemma.py#L294 # noqa

348 vision_embeddings = vision_embeddings * (self.config.hidden_size**-0.5)

vllm/model_executor/models/phi3v.py

+4 -4

@@ -655,11 +655,11 @@ class Phi3VForCausalLM(nn.Module, SupportsMultiModal, SupportsPP,

655 def get_language_model(self) -> torch.nn.Module:

656 return self.language_model

657

658 - def get_multimodal_embeddings(

659 - self, **kwargs: object) -> Optional[MultiModalEmbeddings]:

658 + def get_multimodal_embeddings(self,

659 + **kwargs: object) -> MultiModalEmbeddings:

660 image_input = self._parse_and_validate_image_input(**kwargs)

661 if image_input is None:

662 - return None

662 + return []

663 vision_embeddings = self._process_image_input(image_input)

664 return vision_embeddings

665

@@ -669,7 +669,7 @@ class Phi3VForCausalLM(nn.Module, SupportsMultiModal, SupportsPP,

669 multimodal_embeddings: Optional[MultiModalEmbeddings] = None,

670 ) -> torch.Tensor:

671 inputs_embeds = self.embed_tokens(input_ids)

672 - if multimodal_embeddings is not None:

672 + if multimodal_embeddings:

673 inputs_embeds = merge_multimodal_embeddings(

674 input_ids, inputs_embeds, multimodal_embeddings,

675 self.image_token_id)

vllm/model_executor/models/phi4mm.py

+3 -2

@@ -1112,11 +1112,12 @@ class Phi4MMForCausalLM(nn.Module, SupportsLoRA, SupportsMultiModal):

1112 image_attention_mask)

1113 return image_embeds

1114

1115 - def get_multimodal_embeddings(

1116 - self, **kwargs: object) -> Optional[MultiModalEmbeddings]:

1115 + def get_multimodal_embeddings(self,

1116 + **kwargs: object) -> MultiModalEmbeddings:

1117

1118 modalities = self._parse_and_validate_multimodal_inputs(**kwargs)

1119 if not modalities:

1120 + return []

1121 return None

1122

1123 # The result multimodal_embeddings is tuple of tensors, with each

vllm/model_executor/models/pixtral.py

+3 -3

@@ -409,11 +409,11 @@ class PixtralForConditionalGeneration(nn.Module, SupportsMultiModal,

409 def get_language_model(self) -> torch.nn.Module:

410 return self.language_model

411

412 - def get_multimodal_embeddings(

413 - self, **kwargs: object) -> Optional[MultiModalEmbeddings]:

412 + def get_multimodal_embeddings(self,

413 + **kwargs: object) -> MultiModalEmbeddings:

414 image_input = self._parse_and_validate_image_input(**kwargs)

415 if image_input is None:

416 - return None

416 + return []

417

418 return self._process_image_input(image_input)

419

vllm/model_executor/models/qwen2_5_omni_thinker.py

+3 -3

@@ -772,13 +772,13 @@ class Qwen2_5OmniThinkerForConditionalGeneration(

772 def get_language_model(self) -> torch.nn.Module:

773 return self.language_model

774

775 - def get_multimodal_embeddings(

776 - self, **kwargs: object) -> Optional[MultiModalEmbeddings]:

775 + def get_multimodal_embeddings(self,

776 + **kwargs: object) -> MultiModalEmbeddings:

777

778 mm_input_by_modality = self._parse_and_validate_multimodal_inputs(

779 **kwargs)

780 if not mm_input_by_modality:

781 - return None

781 + return []

782

783 # The result multimodal_embeddings is tuple of tensors, with each

784 # tensor correspoending to a multimodal data item (image or video).

vllm/model_executor/models/qwen2_5_vl.py

+3 -3

@@ -1016,13 +1016,13 @@ class Qwen2_5_VLForConditionalGeneration(nn.Module, SupportsMultiModal,

1016 def get_language_model(self) -> torch.nn.Module:

1017 return self.language_model

1018

1019 - def get_multimodal_embeddings(

1020 - self, **kwargs: object) -> Optional[MultiModalEmbeddings]:

1019 + def get_multimodal_embeddings(self,

1020 + **kwargs: object) -> MultiModalEmbeddings:

1021

1022 mm_input_by_modality = self._parse_and_validate_multimodal_inputs(

1023 **kwargs)

1024 if not mm_input_by_modality:

1025 - return None

1025 + return []

1026

1027 # The result multimodal_embeddings is tuple of tensors, with each

1028 # tensor correspoending to a multimodal data item (image or video).

vllm/model_executor/models/qwen2_audio.py

+3 -3

@@ -350,11 +350,11 @@ class Qwen2AudioForConditionalGeneration(nn.Module, SupportsMultiModal,

350 def get_language_model(self) -> torch.nn.Module:

351 return self.language_model

352

353 - def get_multimodal_embeddings(

354 - self, **kwargs: object) -> Optional[MultiModalEmbeddings]:

353 + def get_multimodal_embeddings(self,

354 + **kwargs: object) -> MultiModalEmbeddings:

355 audio_input = self._parse_and_validate_audio_input(**kwargs)

356 if audio_input is None:

357 - return None

357 + return []

358 masked_audio_features = self._process_audio_input(audio_input)

359 return masked_audio_features

360

vllm/model_executor/models/qwen2_vl.py

+3 -2

@@ -1257,11 +1257,12 @@ class Qwen2VLForConditionalGeneration(nn.Module, SupportsMultiModal,

1257 def get_language_model(self) -> torch.nn.Module:

1258 return self.language_model

1259

1260 - def get_multimodal_embeddings(

1261 - self, **kwargs: object) -> Optional[MultiModalEmbeddings]:

1260 + def get_multimodal_embeddings(self,

1261 + **kwargs: object) -> MultiModalEmbeddings:

1262

1263 modalities = self._parse_and_validate_multimodal_inputs(**kwargs)

1264 if not modalities:

1265 + return []

1266 return None

1267

1268 # The result multimodal_embeddings is tuple of tensors, with each

vllm/model_executor/models/qwen_vl.py

+3 -3

@@ -738,11 +738,11 @@ class QwenVLForConditionalGeneration(QWenBaseModel, SupportsPP, SupportsLoRA,

738 def get_language_model(self) -> torch.nn.Module:

739 return self.transformer

740

741 - def get_multimodal_embeddings(

742 - self, **kwargs: object) -> Optional[MultiModalEmbeddings]:

741 + def get_multimodal_embeddings(self,

742 + **kwargs: object) -> MultiModalEmbeddings:

743 image_input = self._parse_and_validate_image_input(**kwargs)

744 if image_input is None:

745 - return None

745 + return []

746

747 vision_embeddings = self._process_image_input(image_input)

748 return vision_embeddings

vllm/model_executor/models/skyworkr1v.py

+3 -3

@@ -869,11 +869,11 @@ class SkyworkR1VChatModel(nn.Module, SupportsMultiModal, SupportsPP):

869 def get_language_model(self) -> torch.nn.Module:

870 return self.language_model

871

872 - def get_multimodal_embeddings(

873 - self, **kwargs: object) -> Optional[MultiModalEmbeddings]:

872 + def get_multimodal_embeddings(self,

873 + **kwargs: object) -> MultiModalEmbeddings:

874 image_input = self._parse_and_validate_image_input(**kwargs)

875 if image_input is None:

876 - return None

876 + return []

877

878 return self._process_image_input(image_input)

879

vllm/model_executor/models/tarsier.py

+3 -3

@@ -585,11 +585,11 @@ class TarsierForConditionalGeneration(nn.Module, SupportsMultiModal,

585 def get_language_model(self) -> torch.nn.Module:

586 return self.language_model

587

588 - def get_multimodal_embeddings(

589 - self, **kwargs: object) -> Optional[MultiModalEmbeddings]:

588 + def get_multimodal_embeddings(self,

589 + **kwargs: object) -> MultiModalEmbeddings:

590 image_input = self._parse_and_validate_image_input(**kwargs)

591 if image_input is None:

592 - return None

592 + return []

593 return self._process_image_input(image_input)

594

595 def get_input_embeddings(

vllm/model_executor/models/ultravox.py

+3 -3

@@ -546,11 +546,11 @@ class UltravoxModel(nn.Module, SupportsMultiModal, SupportsPP, SupportsLoRA):

546 def get_language_model(self) -> torch.nn.Module:

547 return self.language_model

548

549 - def get_multimodal_embeddings(

550 - self, **kwargs: object) -> Optional[MultiModalEmbeddings]:

549 + def get_multimodal_embeddings(self,

550 + **kwargs: object) -> MultiModalEmbeddings:

551 audio_input = self._parse_and_validate_audio_input(**kwargs)

552 if audio_input is None:

553 - return None

553 + return []

554 audio_embeddings = self._process_audio_input(audio_input)

555 return audio_embeddings

556

vllm/model_executor/models/whisper.py

+2 -2

@@ -687,8 +687,8 @@ class WhisperForConditionalGeneration(nn.Module, SupportsTranscription,

687 def get_language_model(self) -> torch.nn.Module:

688 return self.model.decoder

689

690 - def get_multimodal_embeddings(

691 - self, **kwargs: object) -> Optional[MultiModalEmbeddings]:

690 + def get_multimodal_embeddings(self,

691 + **kwargs: object) -> MultiModalEmbeddings:

692 # TODO: This method does not obey the interface for SupportsMultiModal.

693 # Refactor this once encoder/decoder support is implemented in V1.

694 audio_input = self._parse_and_validate_audio_input(**kwargs)

vllm/v1/worker/utils.py

+2 -1

@@ -4,11 +4,12 @@ from typing import Optional

4

5 import torch

6

7 +from vllm.model_executor.models.interfaces import MultiModalEmbeddings

8 from vllm.v1.kv_cache_interface import KVCacheGroupSpec

9

10

11 def sanity_check_mm_encoder_outputs(

11 - mm_embeddings: object,

12 + mm_embeddings: MultiModalEmbeddings,

13 expected_num_items: int,

14 ) -> None:

15 """

[V1] Change return type on get_multimodal_embeddings() (#19446)