14fdd21d39 · tduncan/Nvidia-Optimization-Stack-VLLM

vllm/model_executor/models/aria.py

+2 -1

@@ -620,7 +620,8 @@ class AriaForConditionalGeneration(nn.Module, SupportsMultiModal):

620 multimodal_embeddings: Optional[MultiModalEmbeddings] = None,

621 ) -> torch.Tensor:

622 inputs_embeds = self.language_model.get_input_embeddings(input_ids)

623 - if multimodal_embeddings is not None:

623 + if multimodal_embeddings is not None \

624 + and len(multimodal_embeddings) != 0:

625 inputs_embeds = merge_multimodal_embeddings(

626 input_ids, inputs_embeds, multimodal_embeddings,

627 self.config.image_token_index)

vllm/model_executor/models/aya_vision.py

+2 -1

@@ -430,7 +430,8 @@ class AyaVisionForConditionalGeneration(nn.Module, SupportsMultiModal,

430 multimodal_embeddings: Optional[MultiModalEmbeddings] = None,

431 ) -> torch.Tensor:

432 inputs_embeds = self.language_model.get_input_embeddings(input_ids)

433 - if multimodal_embeddings is not None:

433 + if multimodal_embeddings is not None \

434 + and len(multimodal_embeddings) != 0:

435 inputs_embeds = merge_multimodal_embeddings(

436 input_ids=input_ids,

437 inputs_embeds=inputs_embeds,

vllm/model_executor/models/blip2.py

+2 -1

@@ -641,7 +641,8 @@ class Blip2ForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP,

641 multimodal_embeddings: Optional[MultiModalEmbeddings] = None,

642 ) -> torch.Tensor:

643 inputs_embeds = self.language_model.get_input_embeddings(input_ids)

644 - if multimodal_embeddings is not None:

644 + if multimodal_embeddings is not None \

645 + and len(multimodal_embeddings) != 0:

646 inputs_embeds = merge_multimodal_embeddings(

647 input_ids, inputs_embeds, multimodal_embeddings,

648 _IMAGE_TOKEN_ID)

vllm/model_executor/models/chameleon.py

+2 -1

@@ -1005,7 +1005,8 @@ class ChameleonForConditionalGeneration(nn.Module, SupportsMultiModal,

1005 ) -> torch.Tensor:

1006

1007 inputs_embeds = self.model.get_input_embeddings(input_ids)

1008 - if multimodal_embeddings is not None:

1008 + if multimodal_embeddings is not None \

1009 + and len(multimodal_embeddings) != 0:

1010 inputs_embeds = merge_multimodal_embeddings(

1011 input_ids, inputs_embeds, multimodal_embeddings,

1012 self.model.vocabulary_mapping.image_token_id)

vllm/model_executor/models/deepseek_vl2.py

+2 -1

@@ -600,7 +600,8 @@ class DeepseekVLV2ForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):

600 multimodal_embeddings: Optional[MultiModalEmbeddings] = None,

601 ) -> torch.Tensor:

602 inputs_embeds = self.language_model.get_input_embeddings(input_ids)

603 - if multimodal_embeddings is not None:

603 + if multimodal_embeddings is not None \

604 + and len(multimodal_embeddings) != 0:

605 inputs_embeds = merge_multimodal_embeddings(

606 input_ids, inputs_embeds, multimodal_embeddings,

607 self.image_token_id)

vllm/model_executor/models/florence2.py

+2 -1

@@ -1046,7 +1046,8 @@ class Florence2ForConditionalGeneration(nn.Module, SupportsMultiModal,

1046 multimodal_embeddings: Optional[MultiModalEmbeddings] = None,

1047 ) -> torch.Tensor:

1048 inputs_embeds = self.language_model.get_input_embeddings(input_ids)

1049 - if multimodal_embeddings is not None:

1049 + if multimodal_embeddings is not None \

1050 + and len(multimodal_embeddings) != 0:

1051 inputs_embeds = merge_multimodal_embeddings(

1052 input_ids, inputs_embeds, multimodal_embeddings,

1053 self.pad_token_id)

vllm/model_executor/models/fuyu.py

+2 -1

@@ -345,7 +345,8 @@ class FuyuForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):

345 multimodal_embeddings: Optional[MultiModalEmbeddings] = None,

346 ) -> torch.Tensor:

347 inputs_embeds = self.language_model.get_input_embeddings(input_ids)

348 - if multimodal_embeddings is not None:

348 + if multimodal_embeddings is not None \

349 + and len(multimodal_embeddings) != 0:

350 inputs_embeds = merge_multimodal_embeddings(

351 input_ids,

352 inputs_embeds,

vllm/model_executor/models/gemma3_mm.py

+2 -1

@@ -592,7 +592,8 @@ class Gemma3ForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP,

592 multimodal_embeddings: Optional[MultiModalEmbeddings] = None,

593 ) -> torch.Tensor:

594 inputs_embeds = self.language_model.get_input_embeddings(input_ids)

595 - if multimodal_embeddings is not None:

595 + if multimodal_embeddings is not None \

596 + and len(multimodal_embeddings) != 0:

597 inputs_embeds = merge_multimodal_embeddings(

598 input_ids,

599 inputs_embeds,

vllm/model_executor/models/glm4v.py

+2 -1

@@ -609,7 +609,8 @@ class GLM4VForCausalLM(ChatGLMBaseModel, SupportsLoRA, SupportsPP,

609 ) -> torch.Tensor:

610 inputs_embeds = self.transformer.get_input_embeddings(input_ids)

611

612 - if multimodal_embeddings is not None:

612 + if multimodal_embeddings is not None \

613 + and len(multimodal_embeddings) != 0:

614 inputs_embeds = merge_multimodal_embeddings(

615 input_ids=input_ids,

616 inputs_embeds=inputs_embeds,

vllm/model_executor/models/granite_speech.py

+2 -1

@@ -721,7 +721,8 @@ class GraniteSpeechForConditionalGeneration(

721 multimodal_embeddings: Optional[MultiModalEmbeddings] = None,

722 ) -> torch.Tensor:

723 """Compute the merged LLM / audio embeddings."""

724 - if multimodal_embeddings is None:

724 + if multimodal_embeddings is None \

725 + or len(multimodal_embeddings) == 0:

726 return self.language_model.get_input_embeddings(input_ids)

727

728 inputs_embeds = embed_multimodal(

vllm/model_executor/models/idefics3.py

+2 -1

@@ -720,7 +720,8 @@ class Idefics3ForConditionalGeneration(nn.Module, SupportsMultiModal,

720 multimodal_embeddings: Optional[MultiModalEmbeddings] = None,

721 ) -> torch.Tensor:

722 inputs_embeds = self.model.get_input_embeddings(input_ids)

723 - if multimodal_embeddings is not None:

723 + if multimodal_embeddings is not None \

724 + and len(multimodal_embeddings) != 0:

725 inputs_embeds = merge_multimodal_embeddings(

726 input_ids,

727 inputs_embeds,

vllm/model_executor/models/internvl.py

+2 -1

@@ -1336,7 +1336,8 @@ class InternVLChatModel(nn.Module, SupportsMultiModal, SupportsPP,

1336 multimodal_embeddings: Optional[MultiModalEmbeddings] = None,

1337 ) -> torch.Tensor:

1338 inputs_embeds = self.language_model.get_input_embeddings(input_ids)

1339 - if multimodal_embeddings is not None:

1339 + if multimodal_embeddings is not None \

1340 + and len(multimodal_embeddings) != 0:

1341 context_token_ids = [

1342 token_id for token_id in (self.img_context_token_id,

1343 self.video_context_token_id)

vllm/model_executor/models/kimi_vl.py

+2 -1

@@ -393,7 +393,8 @@ class KimiVLForConditionalGeneration(nn.Module, SupportsMultiModal):

393 # model as one of the requirements of basic vLLM model implementation.

394 inputs_embeds = self.language_model.get_input_embeddings(input_ids)

395

396 - if multimodal_embeddings is not None:

396 + if multimodal_embeddings is not None and len(

397 + multimodal_embeddings) != 0:

398 inputs_embeds = merge_multimodal_embeddings(

399 input_ids=input_ids,

400 inputs_embeds=inputs_embeds,

vllm/model_executor/models/llava.py

+2 -1

@@ -683,7 +683,8 @@ class LlavaForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP):

683 multimodal_embeddings: Optional[MultiModalEmbeddings] = None,

684 ) -> torch.Tensor:

685 inputs_embeds = self.language_model.get_input_embeddings(input_ids)

686 - if multimodal_embeddings is not None:

686 + if multimodal_embeddings is not None \

687 + and len(multimodal_embeddings) != 0:

688 inputs_embeds = merge_multimodal_embeddings(

689 input_ids,

690 inputs_embeds,

vllm/model_executor/models/llava_next.py

+2 -1

@@ -502,7 +502,8 @@ class LlavaNextForConditionalGeneration(nn.Module, SupportsMultiModal,

502 multimodal_embeddings: Optional[MultiModalEmbeddings] = None,

503 ) -> torch.Tensor:

504

505 - if not multimodal_embeddings:

505 + if multimodal_embeddings is None \

506 + or len(multimodal_embeddings) == 0:

507 return self.language_model.get_input_embeddings(input_ids)

508

509 inputs_embeds = embed_multimodal(

vllm/model_executor/models/llava_next_video.py

+2 -1

@@ -426,7 +426,8 @@ class LlavaNextVideoForConditionalGeneration(nn.Module, SupportsMultiModal,

426 multimodal_embeddings: Optional[MultiModalEmbeddings] = None,

427 ) -> torch.Tensor:

428 inputs_embeds = self.language_model.get_input_embeddings(input_ids)

429 - if multimodal_embeddings is not None:

429 + if multimodal_embeddings is not None \

430 + and len(multimodal_embeddings) != 0:

431 inputs_embeds = merge_multimodal_embeddings(

432 input_ids, inputs_embeds, multimodal_embeddings,

433 self.config.video_token_index)

vllm/model_executor/models/llava_onevision.py

+2 -1

@@ -881,7 +881,8 @@ class LlavaOnevisionForConditionalGeneration(nn.Module, SupportsMultiModal,

881 multimodal_embeddings: Optional[MultiModalEmbeddings] = None,

882 ) -> torch.Tensor:

883 inputs_embeds = self.language_model.get_input_embeddings(input_ids)

884 - if multimodal_embeddings is not None:

884 + if multimodal_embeddings is not None \

885 + and len(multimodal_embeddings) != 0:

886 inputs_embeds = merge_multimodal_embeddings(

887 input_ids, inputs_embeds, multimodal_embeddings,

888 [self.config.image_token_index, self.config.video_token_index])

vllm/model_executor/models/minicpmv.py

+2 -1

@@ -892,7 +892,8 @@ class MiniCPMVBaseModel(nn.Module, SupportsMultiModal, SupportsPP):

892 multimodal_embeddings: Optional[MultiModalEmbeddings] = None,

893 ) -> torch.Tensor:

894 inputs_embeds = self.llm.get_input_embeddings(input_ids)

895 - if multimodal_embeddings is not None:

895 + if multimodal_embeddings is not None \

896 + and len(multimodal_embeddings) != 0:

897 assert len(self.mm_token_ids) > 0

898 inputs_embeds = merge_multimodal_embeddings(

899 input_ids,

vllm/model_executor/models/minimax_vl_01.py

+2 -1

@@ -201,7 +201,8 @@ class MiniMaxVL01ForConditionalGeneration(nn.Module, SupportsMultiModal,

201 multimodal_embeddings: Optional[MultiModalEmbeddings] = None,

202 ) -> torch.Tensor:

203 inputs_embeds = self.language_model.get_input_embeddings(input_ids)

204 - if multimodal_embeddings is not None:

204 + if multimodal_embeddings is not None \

205 + and len(multimodal_embeddings) != 0:

206 inputs_embeds = merge_multimodal_embeddings(

207 input_ids,

208 inputs_embeds,

vllm/model_executor/models/mistral3.py

+2 -1

@@ -521,7 +521,8 @@ class Mistral3ForConditionalGeneration(nn.Module, SupportsLoRA,

521 multimodal_embeddings: Optional[MultiModalEmbeddings] = None,

522 ) -> torch.Tensor:

523 inputs_embeds = self.language_model.get_input_embeddings(input_ids)

524 - if multimodal_embeddings is not None:

524 + if multimodal_embeddings is not None \

525 + and len(multimodal_embeddings) != 0:

526 inputs_embeds = merge_multimodal_embeddings(

527 input_ids,

528 inputs_embeds,

vllm/model_executor/models/mllama4.py

+2 -1

@@ -808,7 +808,8 @@ class Llama4ForConditionalGeneration(nn.Module, SupportsMultiModal,

808 ) -> torch.Tensor:

809 inputs_embeds = self.language_model.get_input_embeddings(input_ids)

810

811 - if multimodal_embeddings is not None:

811 + if multimodal_embeddings is not None and len(

812 + multimodal_embeddings) != 0:

813 inputs_embeds = merge_multimodal_embeddings(

814 input_ids,

815 inputs_embeds,

vllm/model_executor/models/molmo.py

+2 -1

@@ -1487,7 +1487,8 @@ class MolmoForCausalLM(nn.Module, SupportsMultiModal, SupportsPP, SupportsLoRA,

1487 multimodal_embeddings: Optional[MultiModalEmbeddings] = None,

1488 ) -> torch.Tensor:

1489 inputs_embeds = self.model.get_input_embeddings(input_ids)

1490 - if multimodal_embeddings is not None:

1490 + if multimodal_embeddings is not None \

1491 + and len(multimodal_embeddings) != 0:

1492 assert self.img_patch_id is not None

1493

1494 inputs_embeds = merge_multimodal_embeddings(

vllm/model_executor/models/ovis.py

+2 -1

@@ -515,7 +515,8 @@ class Ovis(nn.Module, SupportsMultiModal, SupportsPP):

515 multimodal_embeddings: Optional[MultiModalEmbeddings] = None,

516 ) -> torch.Tensor:

517 inputs_embeds = self.llm.get_input_embeddings(input_ids)

518 - if multimodal_embeddings is not None:

518 + if multimodal_embeddings is not None \

519 + and len(multimodal_embeddings) != 0:

520 inputs_embeds = merge_multimodal_embeddings(

521 input_ids, inputs_embeds, multimodal_embeddings,

522 self.image_pad_token_id)

vllm/model_executor/models/paligemma.py

+2 -1

@@ -364,7 +364,8 @@ class PaliGemmaForConditionalGeneration(nn.Module, SupportsMultiModal,

364 multimodal_embeddings: Optional[MultiModalEmbeddings] = None,

365 ) -> torch.Tensor:

366 inputs_embeds = self.language_model.get_input_embeddings(input_ids)

367 - if multimodal_embeddings is not None:

367 + if multimodal_embeddings is not None \

368 + and len(multimodal_embeddings) != 0:

369 inputs_embeds = merge_multimodal_embeddings(

370 input_ids, inputs_embeds, multimodal_embeddings,

371 self.config.image_token_index)

vllm/model_executor/models/phi3v.py

+2 -1

@@ -669,7 +669,8 @@ class Phi3VForCausalLM(nn.Module, SupportsMultiModal, SupportsPP,

669 multimodal_embeddings: Optional[MultiModalEmbeddings] = None,

670 ) -> torch.Tensor:

671 inputs_embeds = self.embed_tokens(input_ids)

672 - if multimodal_embeddings:

672 + if multimodal_embeddings is not None \

673 + and len(multimodal_embeddings) != 0:

674 inputs_embeds = merge_multimodal_embeddings(

675 input_ids, inputs_embeds, multimodal_embeddings,

676 self.image_token_id)

vllm/model_executor/models/phi4mm.py

+2 -1

@@ -1148,7 +1148,8 @@ class Phi4MMForCausalLM(nn.Module, SupportsLoRA, SupportsMultiModal):

1148 multimodal_embeddings: Optional[MultiModalEmbeddings] = None,

1149 ) -> torch.Tensor:

1150 inputs_embeds = self.model.embed_tokens(input_ids)

1151 - if multimodal_embeddings is not None:

1151 + if multimodal_embeddings is not None and len(

1152 + multimodal_embeddings) != 0:

1153 inputs_embeds = merge_multimodal_embeddings(

1154 input_ids, inputs_embeds, multimodal_embeddings,

1155 [_IMAGE_PLACEHOLDER_TOKEN_ID, _AUDIO_PLACEHOLDER_TOKEN_ID])

vllm/model_executor/models/pixtral.py

+2 -1

@@ -423,7 +423,8 @@ class PixtralForConditionalGeneration(nn.Module, SupportsMultiModal,

423 multimodal_embeddings: Optional[MultiModalEmbeddings] = None,

424 ) -> torch.Tensor:

425 inputs_embeds = self.language_model.get_input_embeddings(input_ids)

426 - if multimodal_embeddings is not None:

426 + if multimodal_embeddings is not None \

427 + and len(multimodal_embeddings) != 0:

428 inputs_embeds = merge_multimodal_embeddings(

429 input_ids,

430 inputs_embeds,

vllm/model_executor/models/qwen2_5_omni_thinker.py

+3 -2

@@ -805,7 +805,8 @@ class Qwen2_5OmniThinkerForConditionalGeneration(

805 multimodal_embeddings: Optional[MultiModalEmbeddings] = None,

806 ) -> torch.Tensor:

807 inputs_embeds = self.language_model.get_input_embeddings(input_ids)

808 - if multimodal_embeddings is not None:

808 + if multimodal_embeddings is not None \

809 + and len(multimodal_embeddings) != 0:

810

811 # TODO (ywang96): support overlapping modalitiy embeddings so that

812 # `use_audio_in_video` will work on V1.

@@ -845,7 +846,7 @@ class Qwen2_5OmniThinkerForConditionalGeneration(

846 multimodal_embeddings: Optional[NestedTensors] = None,

847 ) -> torch.Tensor:

848 inputs_embeds = self.language_model.get_input_embeddings(input_ids)

848 - if multimodal_embeddings is None:

849 + if multimodal_embeddings is None or len(multimodal_embeddings) == 0:

850 return inputs_embeds

851

852 for embeddings, modality in multimodal_embeddings:

vllm/model_executor/models/qwen2_5_vl.py

+2 -1

@@ -1046,7 +1046,8 @@ class Qwen2_5_VLForConditionalGeneration(nn.Module, SupportsMultiModal,

1046 multimodal_embeddings: Optional[MultiModalEmbeddings] = None,

1047 ) -> torch.Tensor:

1048 inputs_embeds = self.language_model.get_input_embeddings(input_ids)

1049 - if multimodal_embeddings is not None:

1049 + if multimodal_embeddings is not None \

1050 + and len(multimodal_embeddings) != 0:

1051 inputs_embeds = merge_multimodal_embeddings(

1052 input_ids, inputs_embeds, multimodal_embeddings,

1053 [self.config.image_token_id, self.config.video_token_id])

vllm/model_executor/models/qwen2_audio.py

+2 -1

@@ -364,7 +364,8 @@ class Qwen2AudioForConditionalGeneration(nn.Module, SupportsMultiModal,

364 multimodal_embeddings: Optional[MultiModalEmbeddings] = None,

365 ) -> torch.Tensor:

366 inputs_embeds = self.language_model.get_input_embeddings(input_ids)

367 - if multimodal_embeddings is not None:

367 + if multimodal_embeddings is not None \

368 + and len(multimodal_embeddings) != 0:

369 inputs_embeds = merge_multimodal_embeddings(

370 input_ids, inputs_embeds, multimodal_embeddings,

371 self.config.audio_token_index)

vllm/model_executor/models/qwen2_vl.py

+2 -1

@@ -1289,7 +1289,8 @@ class Qwen2VLForConditionalGeneration(nn.Module, SupportsMultiModal,

1289 multimodal_embeddings: Optional[MultiModalEmbeddings] = None,

1290 ) -> torch.Tensor:

1291 inputs_embeds = self.language_model.get_input_embeddings(input_ids)

1292 - if multimodal_embeddings is not None:

1292 + if multimodal_embeddings is not None \

1293 + and len(multimodal_embeddings) != 0:

1294 inputs_embeds = merge_multimodal_embeddings(

1295 input_ids, inputs_embeds, multimodal_embeddings,

1296 [self.config.image_token_id, self.config.video_token_id])

vllm/model_executor/models/qwen_vl.py

+2 -1

@@ -754,7 +754,8 @@ class QwenVLForConditionalGeneration(QWenBaseModel, SupportsPP, SupportsLoRA,

754 ) -> torch.Tensor:

755 inputs_embeds = self.transformer.get_input_embeddings(input_ids)

756

757 - if multimodal_embeddings is not None:

757 + if multimodal_embeddings is not None \

758 + and len(multimodal_embeddings) != 0:

759 inputs_embeds = merge_multimodal_embeddings(

760 input_ids, inputs_embeds, multimodal_embeddings,

761 self.transformer.visual.image_pad_id)

vllm/model_executor/models/skyworkr1v.py

+2 -1

@@ -883,7 +883,8 @@ class SkyworkR1VChatModel(nn.Module, SupportsMultiModal, SupportsPP):

883 multimodal_embeddings: Optional[MultiModalEmbeddings] = None,

884 ) -> torch.Tensor:

885 inputs_embeds = self.language_model.get_input_embeddings(input_ids)

886 - if multimodal_embeddings is not None:

886 + if multimodal_embeddings is not None \

887 + and len(multimodal_embeddings) != 0:

888 assert self.img_context_token_id is not None

889 self._set_visual_token_mask(input_ids)

890 inputs_embeds = merge_multimodal_embeddings(

vllm/model_executor/models/tarsier.py

+2 -1

@@ -598,7 +598,8 @@ class TarsierForConditionalGeneration(nn.Module, SupportsMultiModal,

598 multimodal_embeddings: Optional[MultiModalEmbeddings] = None,

599 ) -> torch.Tensor:

600 inputs_embeds = self.language_model.get_input_embeddings(input_ids)

601 - if multimodal_embeddings is not None:

601 + if multimodal_embeddings is not None \

602 + and len(multimodal_embeddings) != 0:

603 inputs_embeds = merge_multimodal_embeddings(

604 input_ids,

605 inputs_embeds,

vllm/model_executor/models/ultravox.py

+2 -1

@@ -560,7 +560,8 @@ class UltravoxModel(nn.Module, SupportsMultiModal, SupportsPP, SupportsLoRA):

560 multimodal_embeddings: Optional[MultiModalEmbeddings] = None,

561 ) -> torch.Tensor:

562 inputs_embeds = self.language_model.get_input_embeddings(input_ids)

563 - if multimodal_embeddings is not None:

563 + if multimodal_embeddings is not None \

564 + and len(multimodal_embeddings) != 0:

565

566 # TODO(ywang96): remove this block after v0 is deprecated.

567 if not envs.VLLM_USE_V1:

[Core] More fixes to MultiModalEmbeddings type handling (#19715)