bd51f78e39 · tduncan/Nvidia-Optimization-Stack-VLLM

vllm/model_executor/models/aya_vision.py

-11

@@ -427,17 +427,6 @@ class AyaVisionForConditionalGeneration(nn.Module, SupportsMultiModal,

427 if intermediate_tensors is not None:

428 inputs_embeds = None

429

430 - # NOTE: In v1, inputs_embeds is always generated at model runner, this

431 - # condition is for v0 compatibility.

432 - elif inputs_embeds is None:

433 - vision_embeddings = self.get_multimodal_embeddings(**kwargs)

434 - inputs_embeds = self.get_input_embeddings(

435 - input_ids,

436 - vision_embeddings,

437 - is_multimodal=input_ids == self.config.image_token_index,

438 - )

439 - input_ids = None

440 -

430 hidden_states = self.language_model.model(

431 input_ids=input_ids,

432 positions=positions,

vllm/model_executor/models/blip2.py

-11

@@ -672,17 +672,6 @@ class Blip2ForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP,

672 if intermediate_tensors is not None:

673 inputs_embeds = None

674

675 - # NOTE: In v1, inputs_embeds is always generated at model runner, this

676 - # condition is for v0 compatibility.

677 - elif inputs_embeds is None:

678 - vision_embeddings = self.get_multimodal_embeddings(**kwargs)

679 - inputs_embeds = self.get_input_embeddings(

680 - input_ids,

681 - vision_embeddings,

682 - is_multimodal=input_ids == _IMAGE_TOKEN_ID,

683 - )

684 - input_ids = None

685 -

675 hidden_states = self.language_model.model(input_ids,

676 positions,

677 intermediate_tensors,

vllm/model_executor/models/chameleon.py

-12

@@ -1014,18 +1014,6 @@ class ChameleonForConditionalGeneration(nn.Module, SupportsMultiModal,

1014 if intermediate_tensors is not None:

1015 inputs_embeds = None

1016

1017 - # NOTE: In v1, inputs_embeds is always generated at model runner, this

1018 - # condition is for v0 compatibility.

1019 - elif inputs_embeds is None:

1020 - vision_embeddings = self.get_multimodal_embeddings(**kwargs)

1021 - image_token_id = self.model.vocabulary_mapping.image_token_id

1022 - inputs_embeds = self.get_input_embeddings(

1023 - input_ids,

1024 - vision_embeddings,

1025 - is_multimodal=input_ids == image_token_id,

1026 - )

1027 - input_ids = None

1028 -

1017 hidden_states = self.model(input_ids,

1018 positions,

1019 intermediate_tensors,

vllm/model_executor/models/cohere2_vision.py

-11

@@ -440,17 +440,6 @@ class Cohere2VisionForConditionalGeneration(nn.Module, SupportsMultiModal,

440 if intermediate_tensors is not None:

441 inputs_embeds = None

442

443 - # NOTE: In v1, inputs_embeds is always generated at model runner, this

444 - # condition is for v0 compatibility.

445 - elif inputs_embeds is None:

446 - vision_embeddings = self.get_multimodal_embeddings(**kwargs)

447 - inputs_embeds = self.get_input_embeddings(

448 - input_ids,

449 - vision_embeddings,

450 - is_multimodal=input_ids == self.config.image_token_id,

451 - )

452 - input_ids = None

453 -

443 hidden_states = self.language_model.model(

444 input_ids=input_ids,

445 positions=positions,

vllm/model_executor/models/deepseek_vl2.py

-11

@@ -614,17 +614,6 @@ class DeepseekVLV2ForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):

614 if intermediate_tensors is not None:

615 inputs_embeds = None

616

617 - # NOTE: In v1, inputs_embeds is always generated at model runner, this

618 - # condition is for v0 compatibility

619 - elif inputs_embeds is None:

620 - vision_embeddings = self.get_multimodal_embeddings(**kwargs)

621 - inputs_embeds = self.get_input_embeddings(

622 - input_ids,

623 - vision_embeddings,

624 - is_multimodal=input_ids == self.image_token_id,

625 - )

626 - input_ids = None

627 -

617 hidden_states = self.language_model(input_ids,

618 positions,

619 intermediate_tensors,

vllm/model_executor/models/fuyu.py

-11

@@ -352,17 +352,6 @@ class FuyuForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):

352 if intermediate_tensors is not None:

353 inputs_embeds = None

354

355 - # NOTE: In v1, inputs_embeds is always generated at model runner, this

356 - # condition is for v0 compatibility.

357 - elif inputs_embeds is None:

358 - vision_embeddings = self.get_multimodal_embeddings(**kwargs)

359 - inputs_embeds = self.get_input_embeddings(

360 - input_ids,

361 - vision_embeddings,

362 - is_multimodal=input_ids == _IMAGE_TOKEN_ID,

363 - )

364 - input_ids = None

365 -

355 hidden_states = self.language_model(

356 input_ids=input_ids,

357 positions=positions,

vllm/model_executor/models/gemma3_mm.py

-19

@@ -596,25 +596,6 @@ class Gemma3ForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP,

596 if intermediate_tensors is not None:

597 inputs_embeds = None

598

599 - # NOTE: In v1, inputs_embeds is always generated at model runner, this

600 - # condition is for v0 compatibility.

601 - elif inputs_embeds is None:

602 - vision_embeddings = self.get_multimodal_embeddings(**kwargs)

603 -

604 - inputs_embeds = self.get_input_embeddings(

605 - input_ids,

606 - vision_embeddings,

607 - is_multimodal=input_ids == self.config.image_token_index,

608 - )

609 - if (vision_embeddings is not None) and len(vision_embeddings) != 0:

610 - kwargs = self.prepare_attn_masks(

611 - input_ids,

612 - positions,

613 - mask_dtype=self.dtype,

614 - **kwargs,

615 - )

616 - input_ids = None

617 -

599 hidden_states = self.language_model.model(input_ids,

600 positions,

601 intermediate_tensors,

vllm/model_executor/models/glm4_1v.py

+1 -49

@@ -71,7 +71,6 @@ from vllm.multimodal.processing import (BaseMultiModalProcessor,

71 from vllm.multimodal.profiling import BaseDummyInputsBuilder

72 from vllm.platforms import _Backend

73 from vllm.sequence import IntermediateTensors

74 -from vllm.transformers_utils.config import uses_mrope

74 from vllm.utils.tensor_schema import TensorSchema, TensorShape

75

76 from ..layers.activation import SiluAndMul

@@ -80,8 +79,7 @@ from .interfaces import (MultiModalEmbeddings, SupportsLoRA,

79 from .qwen2_vl import (_create_qwen2vl_field_factory,

80 apply_rotary_pos_emb_vision)

81 from .utils import (AutoWeightsLoader, WeightsMapper,

83 - init_vllm_registered_model, maybe_prefix,

84 - merge_multimodal_embeddings)

82 + init_vllm_registered_model, maybe_prefix)

83 from .vision import get_vit_attn_backend, run_dp_sharded_mrope_vision_model

84

85 logger = init_logger(__name__)

@@ -1552,32 +1550,6 @@ class Glm4vForConditionalGeneration(nn.Module, SupportsMultiModal,

1550 multimodal_embeddings += video_embeddings

1551 return multimodal_embeddings

1552

1555 - def get_input_embeddings_v0(

1556 - self,

1557 - input_ids: torch.Tensor,

1558 - image_input: Optional[Glm4vImageInputs] = None,

1559 - video_input: Optional[Glm4vVideoInputs] = None,

1560 - ) -> torch.Tensor:

1561 - inputs_embeds = self.get_input_embeddings(input_ids)

1562 - if image_input is not None:

1563 - image_embeds = self._process_image_input(image_input)

1564 - inputs_embeds = merge_multimodal_embeddings(

1565 - input_ids,

1566 - inputs_embeds,

1567 - image_embeds,

1568 - placeholder_token_id=self.config.image_token_id,

1569 - )

1570 -

1571 - if video_input is not None:

1572 - video_embeds = self._process_video_input(video_input)

1573 - inputs_embeds = merge_multimodal_embeddings(

1574 - input_ids,

1575 - inputs_embeds,

1576 - video_embeds,

1577 - placeholder_token_id=self.config.video_token_id,

1578 - )

1579 - return inputs_embeds

1580 -

1553 def forward(

1554 self,

1555 input_ids: torch.Tensor,

@@ -1604,26 +1576,6 @@ class Glm4vForConditionalGeneration(nn.Module, SupportsMultiModal,

1576 if intermediate_tensors is not None:

1577 inputs_embeds = None

1578

1607 - # NOTE: In v1, inputs_embeds is always generated at model runner from

1608 - # `get_multimodal_embeddings` and `get_input_embeddings`, this

1609 - # condition is only for v0 compatibility.

1610 - elif inputs_embeds is None:

1611 - image_input = self._parse_and_validate_image_input(**kwargs)

1612 - video_input = self._parse_and_validate_video_input(**kwargs)

1613 -

1614 - if image_input is None and video_input is None:

1615 - inputs_embeds = None

1616 - else:

1617 - if uses_mrope(self.config):

1618 - assert positions.ndim == 2 and positions.size(0) == 3, (

1619 - "multimodal section rotary embedding requires "

1620 - f"(3, seq_len) positions, but got {positions.size()}")

1621 - inputs_embeds = self.get_input_embeddings_v0(

1622 - input_ids,

1623 - image_input=image_input,

1624 - video_input=video_input)

1625 - input_ids = None

1626 -

1579 hidden_states = self.language_model.model(

1580 input_ids=input_ids,

1581 positions=positions,

vllm/model_executor/models/glm4v.py

+1 -16

@@ -43,7 +43,7 @@ from vllm.utils.tensor_schema import TensorSchema, TensorShape

43 from .chatglm import ChatGLMBaseModel, ChatGLMModel

44 from .interfaces import (MultiModalEmbeddings, SupportsLoRA,

45 SupportsMultiModal, SupportsPP)

46 -from .utils import flatten_bn, isin_list

46 +from .utils import flatten_bn

47

48

49 class GLMVImagePixelInputs(TensorSchema):

@@ -618,21 +618,6 @@ class GLM4VForCausalLM(ChatGLMBaseModel, SupportsLoRA, SupportsPP,

618 if intermediate_tensors is not None:

619 inputs_embeds = None

620

621 - # NOTE: In v1, inputs_embeds is always generated at model runner, this

622 - # condition is for v0 compatibility.

623 - elif inputs_embeds is None:

624 - vision_embeddings = self.get_multimodal_embeddings(**kwargs)

625 - inputs_embeds = self.get_input_embeddings(

626 - input_ids,

627 - vision_embeddings,

628 - is_multimodal=isin_list(input_ids, [

629 - self.config.boi_token_id,

630 - self.config.pad_token_id,

631 - self.config.eoi_token_id,

632 - ]),

633 - )

634 - input_ids = None

635 -

621 hidden_states = self.transformer(input_ids, positions,

622 intermediate_tensors, inputs_embeds)

623

vllm/model_executor/models/granite_speech.py

-11

@@ -765,17 +765,6 @@ class GraniteSpeechForConditionalGeneration(

765 if intermediate_tensors is not None:

766 inputs_embeds = None

767

768 - # NOTE: In v1, inputs_embeds is always generated at model runner, this

769 - # condition is for v0 compatibility.

770 - elif inputs_embeds is None:

771 - audio_embeds = self.get_multimodal_embeddings(**kwargs)

772 - inputs_embeds = self.get_input_embeddings(

773 - input_ids,

774 - audio_embeds,

775 - is_multimodal=input_ids == self.config.audio_token_index,

776 - )

777 - input_ids = None

778 -

768 model_output = self.language_model(input_ids, positions,

769 intermediate_tensors, inputs_embeds)

770 return model_output

vllm/model_executor/models/hyperclovax_vision.py

+1 -14

@@ -45,8 +45,7 @@ from vllm.sequence import IntermediateTensors

45 from .clip import CLIPVisionModel

46 from .interfaces import MultiModalEmbeddings, SupportsMultiModal, SupportsPP

47 from .siglip import SiglipVisionModel

48 -from .utils import (AutoWeightsLoader, init_vllm_registered_model, isin_list,

49 - maybe_prefix)

48 +from .utils import AutoWeightsLoader, init_vllm_registered_model, maybe_prefix

49 from .vision import get_vision_encoder_info

50

51 EOT = "<|endofturn|>"

@@ -747,18 +746,6 @@ class HCXVisionForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):

746 if intermediate_tensors is not None:

747 inputs_embeds = None

748

750 - # NOTE: In v1, inputs_embeds is always generated at model runner, this

751 - # condition is for v0 compatibility.

752 - elif inputs_embeds is None:

753 - multimodal_embeddings = self.get_multimodal_embeddings(**kwargs)

754 - inputs_embeds = self.get_input_embeddings(

755 - input_ids,

756 - multimodal_embeddings,

757 - is_multimodal=isin_list(

758 - input_ids,

759 - [self.config.image_token_id, self.config.video_token_id]),

760 - )

761 - input_ids = None

749 hidden_states = self.language_model.model(input_ids,

750 positions,

751 intermediate_tensors,

vllm/model_executor/models/idefics3.py

-11

@@ -702,17 +702,6 @@ class Idefics3ForConditionalGeneration(nn.Module, SupportsMultiModal,

702 if intermediate_tensors is not None:

703 inputs_embeds = None

704

705 - # NOTE: In v1, inputs_embeds is always generated at model runner, this

706 - # condition is for v0 compatibility.

707 - elif inputs_embeds is None:

708 - vision_embeddings = self.get_multimodal_embeddings(**kwargs)

709 - inputs_embeds = self.get_input_embeddings(

710 - input_ids,

711 - vision_embeddings,

712 - is_multimodal=input_ids == self.config.image_token_id,

713 - )

714 - input_ids = None

715 -

705 hidden_states = self.model.text_model(input_ids,

706 positions,

707 intermediate_tensors,

vllm/model_executor/models/interns1.py

+1 -17

@@ -40,7 +40,7 @@ from vllm.utils.tensor_schema import TensorSchema, TensorShape

40 from .interfaces import (MultiModalEmbeddings, SupportsLoRA,

41 SupportsMultiModal, SupportsPP)

42 from .utils import (AutoWeightsLoader, WeightsMapper, flatten_bn,

43 - init_vllm_registered_model, isin_list, maybe_prefix)

43 + init_vllm_registered_model, maybe_prefix)

44

45

46 class InternS1MultiModalProjector(nn.Module):

@@ -798,22 +798,6 @@ class InternS1ForConditionalGeneration(nn.Module, SupportsMultiModal,

798 input_ids = None

799 inputs_embeds = None

800

801 - # NOTE: In v1, inputs_embeds is always generated at model runner, this

802 - # condition is for v0 compatibility.

803 - elif inputs_embeds is None:

804 - context_token_ids = [

805 - token_id for token_id in (self.img_context_token_id,

806 - self.video_context_token_id)

807 - if token_id is not None

808 - ]

809 - vision_embeddings = self.get_multimodal_embeddings(**kwargs)

810 - inputs_embeds = self.get_input_embeddings(

811 - input_ids,

812 - vision_embeddings,

813 - is_multimodal=isin_list(input_ids, context_token_ids),

814 - )

815 - input_ids = None

816 -

801 forward_kwargs = {

802 "input_ids": input_ids,

803 "positions": positions,

vllm/model_executor/models/internvl.py

+1 -17

@@ -43,7 +43,7 @@ from vllm.utils.tensor_schema import TensorSchema, TensorShape

43 from .interfaces import (MultiModalEmbeddings, SupportsLoRA,

44 SupportsMultiModal, SupportsPP)

45 from .utils import (AutoWeightsLoader, flatten_bn, init_vllm_registered_model,

46 - isin_list, maybe_prefix)

46 + maybe_prefix)

47

48 IMG_START = '<img>'

49 IMG_END = '</img>'

@@ -1371,22 +1371,6 @@ class InternVLChatModel(nn.Module, SupportsMultiModal, SupportsPP,

1371 input_ids = None

1372 inputs_embeds = None

1373

1374 - # NOTE: In v1, inputs_embeds is always generated at model runner, this

1375 - # condition is for v0 compatibility.

1376 - elif inputs_embeds is None:

1377 - context_token_ids = [

1378 - token_id for token_id in (self.img_context_token_id,

1379 - self.video_context_token_id)

1380 - if token_id is not None

1381 - ]

1382 - vision_embeddings = self.get_multimodal_embeddings(**kwargs)

1383 - inputs_embeds = self.get_input_embeddings(

1384 - input_ids,

1385 - vision_embeddings,

1386 - is_multimodal=isin_list(input_ids, context_token_ids),

1387 - )

1388 - input_ids = None

1389 -

1374 forward_kwargs = {

1375 "input_ids": input_ids,

1376 "positions": positions,

vllm/model_executor/models/kimi_vl.py

-16

@@ -433,22 +433,6 @@ class KimiVLForConditionalGeneration(nn.Module, SupportsMultiModal,

433 ) -> IntermediateTensors:

434 if intermediate_tensors is not None:

435 inputs_embeds = None

436 - # NOTE: In v1, inputs_embeds is always generated at model runner from

437 - # `get_multimodal_embeddings` and `get_input_embeddings`, this

438 - # condition is only for v0 compatibility.

439 - elif inputs_embeds is None:

440 - image_input = self._parse_and_validate_image_input(**kwargs)

441 - if image_input is None:

442 - inputs_embeds = None

443 - else:

444 - image_embeds = self._process_image_input(image_input)

445 - inputs_embeds = self.get_input_embeddings(

446 - input_ids,

447 - image_embeds,

448 - is_multimodal=input_ids ==

449 - self.config.media_placeholder_token_id,

450 - )

451 - input_ids = None

436

437 hidden_states = self.language_model(

438 input_ids=input_ids,

vllm/model_executor/models/llava.py

-11

@@ -723,17 +723,6 @@ class LlavaForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP):

723 if intermediate_tensors is not None:

724 inputs_embeds = None

725

726 - # NOTE: In v1, inputs_embeds is always generated at model runner, this

727 - # condition is for v0 compatibility.

728 - elif inputs_embeds is None:

729 - vision_embeddings = self.get_multimodal_embeddings(**kwargs)

730 - inputs_embeds = self.get_input_embeddings(

731 - input_ids,

732 - vision_embeddings,

733 - is_multimodal=input_ids == self.config.image_token_index,

734 - )

735 - input_ids = None

736 -

726 hidden_states = self.language_model.model(input_ids,

727 positions,

728 intermediate_tensors,

vllm/model_executor/models/llava_next.py

-11

@@ -547,17 +547,6 @@ model_executor.models.llava_next.LlavaNextProcessingInfo.get_num_image_tokens].

547 if intermediate_tensors is not None:

548 inputs_embeds = None

549

550 - # NOTE: In v1, inputs_embeds is always generated at model runner, this

551 - # condition is for v0 compatibility.

552 - elif inputs_embeds is None:

553 - vision_embeddings = self.get_multimodal_embeddings(**kwargs)

554 - inputs_embeds = self.get_input_embeddings(

555 - input_ids,

556 - vision_embeddings,

557 - is_multimodal=input_ids == self.config.image_token_index,

558 - )

559 - input_ids = None

560 -

550 hidden_states = self.language_model.model(input_ids,

551 positions,

552 intermediate_tensors,

vllm/model_executor/models/llava_next_video.py

-11

@@ -431,17 +431,6 @@ class LlavaNextVideoForConditionalGeneration(nn.Module, SupportsMultiModal,

431 if intermediate_tensors is not None:

432 inputs_embeds = None

433

434 - # NOTE: In v1, inputs_embeds is always generated at model runner, this

435 - # condition is for v0 compatibility.

436 - elif inputs_embeds is None:

437 - vision_embeddings = self.get_multimodal_embeddings(**kwargs)

438 - inputs_embeds = self.get_input_embeddings(

439 - input_ids,

440 - vision_embeddings,

441 - is_multimodal=input_ids == self.config.video_token_index,

442 - )

443 - input_ids = None

444 -

434 hidden_states = self.language_model.model(input_ids,

435 positions,

436 intermediate_tensors,

vllm/model_executor/models/llava_onevision.py

+1 -45

@@ -30,8 +30,7 @@ from .llava_next import (BaseLlavaNextMultiModalProcessor, LlavaNextLikeConfig,

30 LlavaNextProcessingInfo)

31 from .siglip import SiglipVisionModel

32 from .utils import (AutoWeightsLoader, WeightsMapper, flatten_bn,

33 - init_vllm_registered_model, maybe_prefix,

34 - merge_multimodal_embeddings)

33 + init_vllm_registered_model, maybe_prefix)

34

35 # For profile run

36 _MAX_FRAMES_PER_VIDEO = 16

@@ -850,33 +849,6 @@ class LlavaOnevisionForConditionalGeneration(nn.Module, SupportsMultiModal,

849

850 return multimodal_embeddings

851

853 - def get_input_embeddings_v0(

854 - self,

855 - input_ids: torch.Tensor,

856 - image_input: Optional[LlavaOnevisionImagePixelInputs] = None,

857 - video_input: Optional[LlavaOnevisionVideoPixelInputs] = None,

858 - ) -> torch.Tensor:

859 - inputs_embeds = self.get_input_embeddings(input_ids)

860 - if image_input is not None:

861 - image_embeds = self._process_image_input(image_input)

862 - inputs_embeds = merge_multimodal_embeddings(

863 - input_ids,

864 - inputs_embeds,

865 - image_embeds,

866 - placeholder_token_id=self.config.image_token_index,

867 - )

868 -

869 - if video_input is not None:

870 - video_embeds = self._process_video_pixels(video_input)

871 - inputs_embeds = merge_multimodal_embeddings(

872 - input_ids,

873 - inputs_embeds,

874 - video_embeds,

875 - placeholder_token_id=self.config.video_token_index,

876 - )

877 -

878 - return inputs_embeds

879 -

852 def forward(

853 self,

854 input_ids: torch.Tensor,

@@ -894,22 +866,6 @@ class LlavaOnevisionForConditionalGeneration(nn.Module, SupportsMultiModal,

866 if intermediate_tensors is not None:

867 inputs_embeds = None

868

897 - # NOTE: In v1, inputs_embeds is always generated at model runner from

898 - # `get_multimodal_embeddings` and `get_input_embeddings`, this

899 - # condition is only for v0 compatibility.

900 - elif inputs_embeds is None:

901 - image_input = self._parse_and_validate_image_input(**kwargs)

902 - video_input = self._parse_and_validate_video_input(**kwargs)

903 -

904 - if image_input is None and video_input is None:

905 - inputs_embeds = None

906 - else:

907 - inputs_embeds = self.get_input_embeddings_v0(

908 - input_ids,

909 - image_input=image_input,

910 - video_input=video_input)

911 - input_ids = None

912 -

869 hidden_states = self.language_model.model(input_ids,

870 positions,

871 intermediate_tensors,

vllm/model_executor/models/minicpmv.py

+1 -14

@@ -71,7 +71,7 @@ from vllm.utils.tensor_schema import TensorSchema, TensorShape

71 from .idefics2_vision_model import Idefics2VisionTransformer

72 from .interfaces import (MultiModalEmbeddings, SupportsLoRA,

73 SupportsMultiModal, SupportsPP)

74 -from .utils import AutoWeightsLoader, flatten_bn, isin_list, maybe_prefix

74 +from .utils import AutoWeightsLoader, flatten_bn, maybe_prefix

75

76 # For profile run

77 _MAX_FRAMES_PER_VIDEO = 16

@@ -1154,19 +1154,6 @@ class MiniCPMVBaseModel(nn.Module, SupportsMultiModal, SupportsPP):

1154 if intermediate_tensors is not None:

1155 inputs_embeds = None

1156

1157 - # NOTE: In v1, inputs_embeds is always generated at model runner from

1158 - # `get_multimodal_embeddings` and `get_input_embeddings`, this

1159 - # condition is only for v0 compatibility.

1160 - elif inputs_embeds is None:

1161 - vision_embeddings = self.get_multimodal_embeddings(**kwargs)

1162 -

1163 - inputs_embeds = self.get_input_embeddings(

1164 - input_ids,

1165 - vision_embeddings,

1166 - is_multimodal=isin_list(input_ids, list(self.mm_token_ids)),

1167 - )

1168 - input_ids = None

1169 -

1157 hidden_states = self.llm.model(

1158 input_ids=input_ids,

1159 positions=positions,

vllm/model_executor/models/mistral3.py

-11

@@ -571,17 +571,6 @@ class Mistral3ForConditionalGeneration(nn.Module, SupportsLoRA,

571 if intermediate_tensors is not None:

572 inputs_embeds = None

573

574 - # NOTE: In v1, inputs_embeds is always generated at model runner, this

575 - # condition is for v0 compatibility.

576 - elif inputs_embeds is None:

577 - vision_embeddings = self.get_multimodal_embeddings(**kwargs)

578 - inputs_embeds = self.get_input_embeddings(

579 - input_ids,

580 - vision_embeddings,

581 - is_multimodal=input_ids == self.config.image_token_index,

582 - )

583 - input_ids = None

584 -

574 hidden_states = self.language_model.model(input_ids,

575 positions,

576 intermediate_tensors,

vllm/model_executor/models/mllama4.py

-11

@@ -823,17 +823,6 @@ class Llama4ForConditionalGeneration(nn.Module, SupportsMultiModal,

823 if intermediate_tensors is not None:

824 inputs_embeds = None

825

826 - # NOTE: In v1, inputs_embeds is always generated at model runner,

827 - # this condition is for v0 compatibility.

828 - elif inputs_embeds is None:

829 - vision_embeddings = self.get_multimodal_embeddings(**kwargs)

830 - inputs_embeds = self.get_input_embeddings(

831 - input_ids,

832 - vision_embeddings,

833 - is_multimodal=input_ids == self.config.image_token_index,

834 - )

835 - input_ids = None

836 -

826 return self.language_model(input_ids, positions, intermediate_tensors,

827 inputs_embeds)

828

vllm/model_executor/models/molmo.py

-11

@@ -1490,17 +1490,6 @@ class MolmoForCausalLM(nn.Module, SupportsMultiModal, SupportsPP, SupportsLoRA,

1490 if intermediate_tensors is not None:

1491 inputs_embeds = None

1492

1493 - # NOTE: In v1, inputs_embeds is always generated at model runner, this

1494 - # condition is for v0 compatibility.

1495 - elif inputs_embeds is None:

1496 - vision_embeddings = self.get_multimodal_embeddings(**kwargs)

1497 - inputs_embeds = self.get_input_embeddings(

1498 - input_ids,

1499 - vision_embeddings,

1500 - is_multimodal=input_ids == self.img_patch_id,

1501 - )

1502 - input_ids = None

1503 -

1493 hidden_states = self.model(input_ids,

1494 positions,

1495 intermediate_tensors,

vllm/model_executor/models/nano_nemotron_vl.py

+1 -17

@@ -35,7 +35,7 @@ from vllm.model_executor.models.nemotron_h import NemotronHForCausalLM

35 from vllm.model_executor.models.radio import RadioModel

36 from vllm.model_executor.models.utils import (flatten_bn,

37 init_vllm_registered_model,

38 - isin_list, maybe_prefix)

38 + maybe_prefix)

39 from vllm.multimodal import MULTIMODAL_REGISTRY

40 from vllm.multimodal.inputs import (MultiModalDataDict, MultiModalFieldConfig,

41 MultiModalKwargs, MultiModalKwargsItems,

@@ -1135,22 +1135,6 @@ class NemotronH_Nano_VL_V2(nn.Module, HasInnerState, IsHybrid,

1135 input_ids = None

1136 inputs_embeds = None

1137

1138 - # NOTE: In v1, inputs_embeds is always generated at model runner, this

1139 - # condition is for v0 compatibility.

1140 - elif inputs_embeds is None:

1141 - context_token_ids = [

1142 - token_id for token_id in (self.img_context_token_id,

1143 - self.video_context_token_id)

1144 - if token_id is not None

1145 - ]

1146 - vision_embeddings = self.get_multimodal_embeddings(**kwargs)

1147 - inputs_embeds = self.get_input_embeddings(

1148 - input_ids,

1149 - vision_embeddings,

1150 - is_multimodal=isin_list(input_ids, context_token_ids),

1151 - )

1152 - input_ids = None

1153 -

1138 hidden_states = self.language_model(

1139 input_ids=input_ids,

1140 positions=positions,

vllm/model_executor/models/nemotron_vl.py

-11

@@ -608,17 +608,6 @@ class LlamaNemotronVLChatModel(nn.Module, SupportsMultiModal, SupportsPP,

608 input_ids = None

609 inputs_embeds = None

610

611 - # NOTE: In v1, inputs_embeds is always generated at model runner, this

612 - # condition is for v0 compatibility.

613 - elif inputs_embeds is None:

614 - vision_embeddings = self.get_multimodal_embeddings(**kwargs)

615 - inputs_embeds = self.get_input_embeddings(

616 - input_ids,

617 - vision_embeddings,

618 - is_multimodal=input_ids == self.img_context_token_id,

619 - )

620 - input_ids = None

621 -

611 forward_kwargs = {

612 "input_ids": input_ids,

613 "positions": positions,

vllm/model_executor/models/ovis.py

-11

@@ -511,17 +511,6 @@ class Ovis(nn.Module, SupportsMultiModal, SupportsPP):

511 if intermediate_tensors is not None:

512 inputs_embeds = None

513

514 - # NOTE: In v1, inputs_embeds is always generated at model runner, this

515 - # condition is for v0 compatibility.

516 - elif inputs_embeds is None:

517 - vision_embeddings = self.get_multimodal_embeddings(**kwargs)

518 - inputs_embeds = self.get_input_embeddings(

519 - input_ids,

520 - vision_embeddings,

521 - is_multimodal=input_ids == self.image_pad_token_id,

522 - )

523 - input_ids = None

524 -

514 # up until here we have an inputs_embeds 100% numerical identity

515 # between the OG HF Transformers implementation and ours

516 hidden_states = self.llm(

vllm/model_executor/models/ovis2_5.py

-12

@@ -596,18 +596,6 @@ class Ovis2_5(nn.Module, SupportsMultiModal, SupportsPP):

596 if intermediate_tensors is not None:

597 inputs_embeds = None

598

599 - # NOTE: In v1, inputs_embeds is always generated at model runner, this

600 - # condition is for v0 compatibility.

601 - elif inputs_embeds is None:

602 - vision_embeddings = self.get_multimodal_embeddings(**kwargs)

603 -

604 - inputs_embeds = self.get_input_embeddings(

605 - input_ids,

606 - vision_embeddings,

607 - is_multimodal=input_ids == self.image_pad_token_id,

608 - )

609 - input_ids = None

610 -

599 # up until here we have a inputs_embeds 100% numerical identity

600 # between the OG HF Transformers implementation and ours

601 hidden_states = self.llm(

vllm/model_executor/models/paligemma.py

-11

@@ -370,17 +370,6 @@ class PaliGemmaForConditionalGeneration(nn.Module, SupportsMultiModal,

370 if intermediate_tensors is not None:

371 inputs_embeds = None

372

373 - # NOTE: In v1, inputs_embeds is always generated at model runner, this

374 - # condition is for v0 compatibility.

375 - elif inputs_embeds is None:

376 - vision_embeddings = self.get_multimodal_embeddings(**kwargs)

377 - inputs_embeds = self.get_input_embeddings(

378 - input_ids,

379 - vision_embeddings,

380 - is_multimodal=input_ids == self.config.image_token_index,

381 - )

382 - input_ids = None

383 -

373 hidden_states = self.language_model.model(input_ids,

374 positions,

375 intermediate_tensors,

vllm/model_executor/models/phi3v.py

-11

@@ -679,17 +679,6 @@ class Phi3VForCausalLM(nn.Module, SupportsMultiModal, SupportsPP,

679 if intermediate_tensors is not None:

680 inputs_embeds = None

681

682 - # NOTE: In v1, inputs_embeds is always generated at model runner, this

683 - # condition is for v0 compatibility

684 - elif inputs_embeds is None:

685 - vision_embeddings = self.get_multimodal_embeddings(**kwargs)

686 - inputs_embeds = self.get_input_embeddings(

687 - input_ids,

688 - vision_embeddings,

689 - is_multimodal=self.image_token_id,

690 - )

691 - input_ids = None

692 -

682 hidden_states = self.language_model.model(input_ids,

683 positions,

684 intermediate_tensors,

vllm/model_executor/models/phi4_multimodal.py

-16

@@ -1411,22 +1411,6 @@ class Phi4MultimodalForCausalLM(nn.Module, SupportsLoRA, SupportsMultiModal):

1411 if intermediate_tensors is not None:

1412 inputs_embeds = None

1413

1414 - # NOTE: In v1, inputs_embeds is always generated at model runner from

1415 - # `get_multimodal_embeddings` and `get_input_embeddings`, this

1416 - # condition is only for v0 compatibility.

1417 - elif inputs_embeds is None:

1418 - image_input = self._parse_and_validate_image_input(**kwargs)

1419 - audio_input = self._parse_and_validate_audio_input(**kwargs)

1420 -

1421 - if image_input is None and audio_input is None:

1422 - inputs_embeds = None

1423 - else:

1424 - inputs_embeds = self.get_input_embeddings_v0(

1425 - input_ids,

1426 - image_input=image_input,

1427 - audio_input=audio_input)

1428 - input_ids = None

1429 -

1414 hidden_states = self.language_model(

1415 input_ids,

1416 positions,

vllm/model_executor/models/phi4mm.py

+1 -47

@@ -35,8 +35,7 @@ from vllm.utils.tensor_schema import TensorSchema, TensorShape

35 from .idefics2_vision_model import Idefics2VisionTransformer

36 from .interfaces import MultiModalEmbeddings, SupportsLoRA, SupportsMultiModal

37 from .phi4mm_audio import AudioEmbedding

38 -from .utils import (AutoWeightsLoader, WeightsMapper, flatten_bn, maybe_prefix,

39 - merge_multimodal_embeddings)

38 +from .utils import AutoWeightsLoader, WeightsMapper, flatten_bn, maybe_prefix

39

40 # <|endoftext10|> (see vocab.json in hf model)

41 _IMAGE_PLACEHOLDER_TOKEN_ID = 200010

@@ -1174,35 +1173,6 @@ class Phi4MMForCausalLM(nn.Module, SupportsLoRA, SupportsMultiModal):

1173

1174 return multimodal_embeddings

1175

1177 - def get_input_embeddings_v0(

1178 - self,

1179 - input_ids: torch.Tensor,

1180 - image_input: Optional[Phi4MMImagePixelInputs] = None,

1181 - audio_input: Optional[Phi4MMAudioFeatureInputs] = None,

1182 - ) -> torch.Tensor:

1183 - audio_projection_mode = 'speech'

1184 - inputs_embeds = self.get_input_embeddings(input_ids)

1185 - if image_input is not None:

1186 - image_embeds = self._process_image_input(image_input)

1187 - inputs_embeds = merge_multimodal_embeddings(

1188 - input_ids,

1189 - inputs_embeds,

1190 - image_embeds,

1191 - placeholder_token_id=_IMAGE_PLACEHOLDER_TOKEN_ID,

1192 - )

1193 - audio_projection_mode = 'vision'

1194 -

1195 - if audio_input is not None:

1196 - audio_embeds = self._process_audio_input(

1197 - audio_input, audio_projection_mode=audio_projection_mode)

1198 - inputs_embeds = merge_multimodal_embeddings(

1199 - input_ids,

1200 - inputs_embeds,

1201 - audio_embeds,

1202 - placeholder_token_id=_AUDIO_PLACEHOLDER_TOKEN_ID,

1203 - )

1204 - return inputs_embeds

1205 -

1176 def forward(

1177 self,

1178 input_ids: torch.Tensor,

@@ -1214,22 +1184,6 @@ class Phi4MMForCausalLM(nn.Module, SupportsLoRA, SupportsMultiModal):

1184 if intermediate_tensors is not None:

1185 inputs_embeds = None

1186

1217 - # NOTE: In v1, inputs_embeds is always generated at model runner from

1218 - # `get_multimodal_embeddings` and `get_input_embeddings`, this

1219 - # condition is only for v0 compatibility.

1220 - elif inputs_embeds is None:

1221 - image_input = self._parse_and_validate_image_input(**kwargs)

1222 - audio_input = self._parse_and_validate_audio_input(**kwargs)

1223 -

1224 - if image_input is None and audio_input is None:

1225 - inputs_embeds = None

1226 - else:

1227 - inputs_embeds = self.get_input_embeddings_v0(

1228 - input_ids,

1229 - image_input=image_input,

1230 - audio_input=audio_input)

1231 - input_ids = None

1232 -

1187 hidden_states = self.model(

1188 input_ids,

1189 positions,

vllm/model_executor/models/pixtral.py

-11

@@ -444,17 +444,6 @@ class PixtralForConditionalGeneration(nn.Module, SupportsMultiModal,

444 if intermediate_tensors is not None:

445 inputs_embeds = None

446

447 - # NOTE: In v1, inputs_embeds is always generated at model runner, this

448 - # condition is for v0 compatibility.

449 - elif inputs_embeds is None:

450 - vision_embeddings = self.get_multimodal_embeddings(**kwargs)

451 - inputs_embeds = self.get_input_embeddings(

452 - input_ids,

453 - vision_embeddings,

454 - is_multimodal=input_ids == self.vision_args.image_token_id,

455 - )

456 - input_ids = None

457 -

447 hidden_states = self.language_model.model(input_ids,

448 positions,

449 intermediate_tensors,

vllm/model_executor/models/qwen2_5_omni_thinker.py

+1 -30

@@ -69,8 +69,7 @@ from vllm.utils.tensor_schema import TensorSchema, TensorShape

69 from .interfaces import (MultiModalEmbeddings, SupportsLoRA,

70 SupportsMultiModal, SupportsPP)

71 from .utils import (AutoWeightsLoader, WeightsMapper,

72 - init_vllm_registered_model, maybe_prefix,

73 - merge_multimodal_embeddings)

72 + init_vllm_registered_model, maybe_prefix)

73

74 try:

75 import flash_attn

@@ -908,26 +907,6 @@ class Qwen2_5OmniThinkerForConditionalGeneration(

907 multimodal_embeddings.append((video_embeds, "video"))

908 return multimodal_embeddings

909

911 - def get_input_embeddings_v0(

912 - self,

913 - input_ids: torch.Tensor,

914 - multimodal_embeddings: Optional[NestedTensors] = None,

915 - ) -> torch.Tensor:

916 - inputs_embeds = self.language_model.get_input_embeddings(input_ids)

917 - if multimodal_embeddings is None or len(multimodal_embeddings) == 0:

918 - return inputs_embeds

919 -

920 - for embeddings, modality in multimodal_embeddings:

921 - if modality == "audio":

922 - placeholder_token_id = self.config.audio_token_index

923 - if modality == "image":

924 - placeholder_token_id = self.config.image_token_index

925 - if modality == "video":

926 - placeholder_token_id = self.config.video_token_index

927 - inputs_embeds = merge_multimodal_embeddings(

928 - input_ids, inputs_embeds, embeddings, placeholder_token_id)

929 - return inputs_embeds

930 -

910 def forward(

911 self,

912 input_ids: torch.Tensor,

@@ -939,14 +918,6 @@ class Qwen2_5OmniThinkerForConditionalGeneration(

918 if intermediate_tensors is not None:

919 inputs_embeds = None

920

942 - # NOTE: In v1, inputs_embeds is always generated at model runner, this

943 - # condition is for v0 compatibility.

944 - elif inputs_embeds is None:

945 - multimodal_embeddings = self.get_multimodal_embeddings_v0(**kwargs)

946 - inputs_embeds = self.get_input_embeddings_v0(

947 - input_ids, multimodal_embeddings)

948 - input_ids = None

949 -

921 hidden_states = self.language_model.model(input_ids,

922 positions,

923 intermediate_tensors,

vllm/model_executor/models/qwen2_5_vl.py

+1 -57

@@ -64,7 +64,6 @@ from vllm.multimodal.parse import MultiModalDataItems

64 from vllm.multimodal.processing import PromptReplacement, PromptUpdate

65 from vllm.platforms import _Backend

66 from vllm.sequence import IntermediateTensors

67 -from vllm.transformers_utils.config import uses_mrope

67 from vllm.utils import is_pin_memory_available

68 from vllm.utils.tensor_schema import TensorSchema, TensorShape

69

@@ -75,8 +74,7 @@ from .qwen2_vl import Qwen2VLDummyInputsBuilder as Qwen2_5_VLDummyInputsBuilder

74 from .qwen2_vl import (Qwen2VLMultiModalProcessor, Qwen2VLProcessingInfo,

75 apply_rotary_pos_emb_vision)

76 from .utils import (AutoWeightsLoader, WeightsMapper, cast_overflow_tensors,

78 - init_vllm_registered_model, maybe_prefix,

79 - merge_multimodal_embeddings)

77 + init_vllm_registered_model, maybe_prefix)

78 from .vision import get_vit_attn_backend, run_dp_sharded_mrope_vision_model

79

80 logger = init_logger(__name__)

@@ -1365,40 +1363,6 @@ class Qwen2_5_VLForConditionalGeneration(nn.Module, SupportsMultiModal,

1363 multimodal_embeddings += video_embeddings

1364 return multimodal_embeddings

1365

1368 - def get_input_embeddings_v0(

1369 - self,

1370 - input_ids: torch.Tensor,

1371 - image_input: Optional[Qwen2_5_VLImageInputs] = None,

1372 - video_input: Optional[Qwen2_5_VLVideoInputs] = None,

1373 - ) -> torch.Tensor:

1374 - inputs_embeds = self.get_input_embeddings(input_ids)

1375 - if image_input is not None:

1376 - image_embeds = self._process_image_input(image_input)

1377 - if self.is_multimodal_pruning_enabled:

1378 - image_embeds = self._postprocess_image_embeds_evs(

1379 - image_embeds, image_input

1380 - )

1381 - inputs_embeds = merge_multimodal_embeddings(

1382 - input_ids,

1383 - inputs_embeds,

1384 - image_embeds,

1385 - placeholder_token_id=self.config.image_token_id,

1386 - )

1387 -

1388 - if video_input is not None:

1389 - video_embeds = self._process_video_input(video_input)

1390 - if self.is_multimodal_pruning_enabled:

1391 - video_embeds = self._postprocess_video_embeds_evs(

1392 - video_embeds, video_input

1393 - )

1394 - inputs_embeds = merge_multimodal_embeddings(

1395 - input_ids,

1396 - inputs_embeds,

1397 - video_embeds,

1398 - placeholder_token_id=self.config.video_token_id,

1399 - )

1400 - return inputs_embeds

1401 -

1366 def forward(

1367 self,

1368 input_ids: torch.Tensor,

@@ -1421,26 +1385,6 @@ class Qwen2_5_VLForConditionalGeneration(nn.Module, SupportsMultiModal,

1385 if intermediate_tensors is not None:

1386 inputs_embeds = None

1387

1424 - # NOTE: In v1, inputs_embeds is always generated at model runner from

1425 - # `get_multimodal_embeddings` and `get_input_embeddings`, this

1426 - # condition is only for v0 compatibility.

1427 - elif inputs_embeds is None:

1428 - image_input = self._parse_and_validate_image_input(**kwargs)

1429 - video_input = self._parse_and_validate_video_input(**kwargs)

1430 -

1431 - if image_input is None and video_input is None:

1432 - inputs_embeds = None

1433 - else:

1434 - if uses_mrope(self.config):

1435 - assert positions.ndim == 2 and positions.size(0) == 3, (

1436 - "multimodal section rotary embedding requires "

1437 - f"(3, seq_len) positions, but got {positions.size()}")

1438 - inputs_embeds = self.get_input_embeddings_v0(

1439 - input_ids,

1440 - image_input=image_input,

1441 - video_input=video_input)

1442 - input_ids = None

1443 -

1388 hidden_states = self.language_model.model(

1389 input_ids=input_ids,

1390 positions=positions,

vllm/model_executor/models/qwen2_audio.py

-11

@@ -449,17 +449,6 @@ class Qwen2AudioForConditionalGeneration(nn.Module, SupportsMultiModal,

449 if intermediate_tensors is not None:

450 inputs_embeds = None

451

452 - # NOTE: In v1, inputs_embeds is always generated at model runner, this

453 - # condition is for v0 compatibility.

454 - elif inputs_embeds is None:

455 - multimodal_embeddings = self.get_multimodal_embeddings(**kwargs)

456 - inputs_embeds = self.get_input_embeddings(

457 - input_ids,

458 - multimodal_embeddings,

459 - is_multimodal=input_ids == self.config.audio_token_index,

460 - )

461 - input_ids = None

462 -

452 hidden_states = self.language_model.model(input_ids,

453 positions,

454 intermediate_tensors,

vllm/model_executor/models/qwen2_vl.py

+1 -49

@@ -65,15 +65,13 @@ from vllm.multimodal.processing import (BaseMultiModalProcessor,

65 from vllm.multimodal.profiling import BaseDummyInputsBuilder

66 from vllm.platforms import _Backend, current_platform

67 from vllm.sequence import IntermediateTensors

68 -from vllm.transformers_utils.config import uses_mrope

68 from vllm.transformers_utils.tokenizer import AnyTokenizer

69 from vllm.utils.tensor_schema import TensorSchema, TensorShape

70

71 from .interfaces import (MultiModalEmbeddings, SupportsLoRA, SupportsMRoPE,

72 SupportsMultiModal, SupportsPP)

73 from .utils import (AutoWeightsLoader, WeightsMapper,

75 - init_vllm_registered_model, maybe_prefix,

76 - merge_multimodal_embeddings)

74 + init_vllm_registered_model, maybe_prefix)

75 from .vision import get_vit_attn_backend, run_dp_sharded_mrope_vision_model

76

77 logger = init_logger(__name__)

@@ -1464,32 +1462,6 @@ class Qwen2VLForConditionalGeneration(nn.Module, SupportsMultiModal,

1462

1463 return multimodal_embeddings

1464

1467 - def get_input_embeddings_v0(

1468 - self,

1469 - input_ids: torch.Tensor,

1470 - image_input: Optional[Qwen2VLImagePixelInputs] = None,

1471 - video_input: Optional[Qwen2VLVideoPixelInputs] = None,

1472 - ) -> torch.Tensor:

1473 - inputs_embeds = self.get_input_embeddings(input_ids)

1474 - if image_input is not None:

1475 - image_embeds = self._process_image_input(image_input)

1476 - inputs_embeds = merge_multimodal_embeddings(

1477 - input_ids,

1478 - inputs_embeds,

1479 - image_embeds,

1480 - placeholder_token_id=self.config.image_token_id,

1481 - )

1482 -

1483 - if video_input is not None:

1484 - video_embeds = self._process_video_input(video_input)

1485 - inputs_embeds = merge_multimodal_embeddings(

1486 - input_ids,

1487 - inputs_embeds,

1488 - video_embeds,

1489 - placeholder_token_id=self.config.video_token_id,

1490 - )

1491 - return inputs_embeds

1492 -

1465 def forward(

1466 self,

1467 input_ids: torch.Tensor,

@@ -1515,26 +1487,6 @@ class Qwen2VLForConditionalGeneration(nn.Module, SupportsMultiModal,

1487 if intermediate_tensors is not None:

1488 inputs_embeds = None

1489

1518 - # NOTE: In v1, inputs_embeds is always generated at model runner from

1519 - # `get_multimodal_embeddings` and `get_input_embeddings`, this

1520 - # condition is only for v0 compatibility.

1521 - elif inputs_embeds is None:

1522 - image_input = self._parse_and_validate_image_input(**kwargs)

1523 - video_input = self._parse_and_validate_video_input(**kwargs)

1524 -

1525 - if image_input is None and video_input is None:

1526 - inputs_embeds = None

1527 - else:

1528 - if uses_mrope(self.config):

1529 - assert positions.ndim == 2 and positions.size(0) == 3, (

1530 - "multimodal section rotary embedding requires "

1531 - f"(3, seq_len) positions, but got {positions.size()}")

1532 - inputs_embeds = self.get_input_embeddings_v0(

1533 - input_ids,

1534 - image_input=image_input,

1535 - video_input=video_input)

1536 - input_ids = None

1537 -

1490 hidden_states = self.language_model.model(

1491 input_ids=input_ids,

1492 positions=positions,

vllm/model_executor/models/qwen3_vl.py

+1 -92

@@ -68,7 +68,6 @@ from vllm.multimodal.processing import (BaseMultiModalProcessor,

68 from vllm.multimodal.profiling import BaseDummyInputsBuilder

69 from vllm.platforms import _Backend

70 from vllm.sequence import IntermediateTensors

71 -from vllm.transformers_utils.config import uses_mrope

71 from vllm.utils import is_list_of

72

73 from .interfaces import (MultiModalEmbeddings, SupportsLoRA,

@@ -82,8 +81,7 @@ from .qwen2_5_vl import (Qwen2_5_VisionAttention,

81 from .qwen2_vl import Qwen2VLProcessingInfo

82 from .qwen3 import Qwen3ForCausalLM, Qwen3Model

83 from .utils import (AutoWeightsLoader, PPMissingLayer, WeightsMapper,

85 - _merge_multimodal_embeddings, maybe_prefix,

86 - merge_multimodal_embeddings)

84 + _merge_multimodal_embeddings, maybe_prefix)

85 from .vision import get_vit_attn_backend, run_dp_sharded_mrope_vision_model

86

87 logger = init_logger(__name__)

@@ -1464,75 +1462,6 @@ class Qwen3VLForConditionalGeneration(nn.Module, SupportsMultiModal,

1462

1463 return inputs_embeds

1464

1467 - def get_input_embeddings_v0(

1468 - self,

1469 - input_ids: torch.Tensor,

1470 - image_input: Optional[Qwen2_5_VLImageInputs] = None,

1471 - video_input: Optional[Qwen2_5_VLVideoInputs] = None,

1472 - ) -> torch.Tensor:

1473 - inputs_embeds = self.get_input_embeddings(input_ids)

1474 -

1475 - if self.use_deepstack:

1476 - visual_dim = inputs_embeds.shape[-1]

1477 - deepstack_input_embeds = None

1478 - if image_input is not None or video_input is not None:

1479 - deepstack_input_embeds = torch.zeros_like(

1480 - inputs_embeds).unsqueeze(1).repeat(

1481 - 1, self.deepstack_num_level, 1).flatten(1)

1482 -

1483 - if image_input is not None:

1484 - image_embeds = self._process_image_input(image_input)

1485 - if self.use_deepstack:

1486 - image_embeds = torch.cat(image_embeds)

1487 -

1488 - image_embeds, image_embeds_multiscale = image_embeds.split(

1489 - [visual_dim, visual_dim * self.deepstack_num_level],

1490 - dim=-1)

1491 -

1492 - deepstack_input_embeds = merge_multimodal_embeddings(

1493 - input_ids,

1494 - deepstack_input_embeds,

1495 - image_embeds_multiscale,

1496 - placeholder_token_id=self.config.image_token_id,

1497 - )

1498 -

1499 - inputs_embeds = merge_multimodal_embeddings(

1500 - input_ids,

1501 - inputs_embeds,

1502 - image_embeds,

1503 - placeholder_token_id=self.config.image_token_id,

1504 - )

1505 -

1506 - if video_input is not None:

1507 - video_embeds = self._process_video_input(video_input)

1508 - if self.use_deepstack:

1509 - video_embeds = torch.cat(video_embeds)

1510 -

1511 - video_embeds, video_embeds_multiscale = video_embeds.split(

1512 - [visual_dim, visual_dim * self.deepstack_num_level],

1513 - dim=-1)

1514 -

1515 - deepstack_input_embeds = merge_multimodal_embeddings(

1516 - input_ids,

1517 - deepstack_input_embeds,

1518 - video_embeds_multiscale,

1519 - placeholder_token_id=self.config.video_token_id,

1520 - )

1521 -

1522 - inputs_embeds = merge_multimodal_embeddings(

1523 - input_ids,

1524 - inputs_embeds,

1525 - video_embeds,

1526 - placeholder_token_id=self.config.video_token_id,

1527 - )

1528 -

1529 - if self.use_deepstack and deepstack_input_embeds is not None:

1530 - deepstack_input_embeds = deepstack_input_embeds.view(

1531 - inputs_embeds.shape[0], self.deepstack_num_level,

1532 - visual_dim).permute(1, 0, 2).contiguous()

1533 - self._set_deepstack_input_embeds(deepstack_input_embeds)

1534 - return inputs_embeds

1535 -

1465 def forward(

1466 self,

1467 input_ids: torch.Tensor,

@@ -1568,26 +1497,6 @@ class Qwen3VLForConditionalGeneration(nn.Module, SupportsMultiModal,

1497 if intermediate_tensors is not None:

1498 inputs_embeds = None

1499

1571 - # NOTE: In v1, inputs_embeds is always generated at model runner from

1572 - # `get_multimodal_embeddings` and `get_input_embeddings`, this

1573 - # condition is only for v0 compatibility.

1574 - elif inputs_embeds is None:

1575 - image_input = self._parse_and_validate_image_input(**kwargs)

1576 - video_input = self._parse_and_validate_video_input(**kwargs)

1577 -

1578 - if image_input is None and video_input is None:

1579 - inputs_embeds = None

1580 - else:

1581 - if uses_mrope(self.config):

1582 - assert positions.ndim == 2 and positions.size(0) == 3, (

1583 - "multimodal section rotary embedding requires "

1584 - f"(3, seq_len) positions, but got {positions.size()}")

1585 - inputs_embeds = self.get_input_embeddings_v0(

1586 - input_ids,

1587 - image_input=image_input,

1588 - video_input=video_input)

1589 - input_ids = None

1590 -

1500 if self.use_deepstack and inputs_embeds is not None and get_pp_group(

1501 ).is_first_rank:

1502 deepstack_input_embeds = self._get_deepstack_input_embeds(

vllm/model_executor/models/qwen_vl.py

-12

@@ -767,18 +767,6 @@ class QwenVLForConditionalGeneration(QWenBaseModel, SupportsPP, SupportsLoRA,

767 if intermediate_tensors is not None:

768 inputs_embeds = None

769

770 - # NOTE: In v1, inputs_embeds is always generated at model runner, this

771 - # condition is for v0 compatibility.

772 - elif inputs_embeds is None:

773 - vision_embeddings = self.get_multimodal_embeddings(**kwargs)

774 - inputs_embeds = self.get_input_embeddings(

775 - input_ids,

776 - vision_embeddings,

777 - is_multimodal=input_ids ==

778 - self.transformer.visual.image_pad_id,

779 - )

780 - input_ids = None

781 -

770 hidden_states = self.transformer(input_ids, positions,

771 intermediate_tensors, inputs_embeds)

772 return hidden_states

vllm/model_executor/models/skyworkr1v.py

-11

@@ -874,17 +874,6 @@ class SkyworkR1VChatModel(nn.Module, SupportsMultiModal, SupportsPP):

874 input_ids = None

875 inputs_embeds = None

876

877 - # NOTE: In v1, inputs_embeds is always generated at model runner, this

878 - # condition is for v0 compatibility.

879 - elif inputs_embeds is None:

880 - vision_embeddings = self.get_multimodal_embeddings(**kwargs)

881 - inputs_embeds = self.get_input_embeddings(

882 - input_ids,

883 - vision_embeddings,

884 - is_multimodal=input_ids == self.img_context_token_id,

885 - )

886 - input_ids = None

887 -

877 forward_kwargs = {

878 "input_ids": input_ids,

879 "positions": positions,

vllm/model_executor/models/transformers.py

-13

@@ -881,19 +881,6 @@ class TransformersForMultimodalLM(TransformersForCausalLM, SupportsMultiModal):

881 inputs_embeds: Optional[torch.Tensor] = None,

882 **kwargs: object,

883 ) -> Union[torch.Tensor, IntermediateTensors]:

884 - # NOTE: In v1, inputs_embeds is always generated at model runner from

885 - # `get_multimodal_embeddings` and `get_input_embeddings`, this

886 - # condition is only for v0 compatibility.

887 - if inputs_embeds is None:

888 - multimodal_embeds = self.get_multimodal_embeddings(**kwargs)

889 - if multimodal_embeds is not None:

890 - inputs_embeds = self.get_input_embeddings(

891 - input_ids,

892 - multimodal_embeds,

893 - is_multimodal=input_ids == self.config.image_token_id,

894 - )

895 - input_ids = None

896 -

884 model_output = super().forward(input_ids, positions,

885 intermediate_tensors, inputs_embeds)

886 return model_output

vllm/model_executor/models/ultravox.py

-12

@@ -597,18 +597,6 @@ class UltravoxModel(nn.Module, SupportsMultiModal, SupportsPP, SupportsLoRA):

597 if intermediate_tensors is not None:

598 inputs_embeds = None

599

600 - # NOTE: In v1, inputs_embeds is always generated at model runner, this

601 - # condition is for v0 compatibility.

602 - elif inputs_embeds is None:

603 - multimodal_embeddings = self.get_multimodal_embeddings(**kwargs)

604 -

605 - inputs_embeds = self.get_input_embeddings(

606 - input_ids,

607 - multimodal_embeddings,

608 - is_multimodal=input_ids == self.config.audio_token_index,

609 - )

610 - input_ids = None

611 -

600 language_model = self.language_model

601 if hasattr(language_model, "language_model"):

602 language_model = language_model.language_model

vllm/model_executor/models/voxtral.py

-13

@@ -371,19 +371,6 @@ class VoxtralForConditionalGeneration(nn.Module, SupportsMultiModal,

371 if intermediate_tensors is not None:

372 inputs_embeds = None

373

374 - # NOTE: In v1, inputs_embeds is always generated at model runner, this

375 - # condition is for v0 compatibility.

376 - elif inputs_embeds is None:

377 - audio_encoder = self.tokenizer.instruct.audio_encoder

378 - audio_tok_id = audio_encoder.audio_token

379 - audio_embeddings = self.get_multimodal_embeddings(**kwargs)

380 - inputs_embeds = self.get_input_embeddings(

381 - input_ids,

382 - audio_embeddings,

383 - is_multimodal=input_ids == audio_tok_id,

384 - )

385 - input_ids = None

386 -

374 hidden_states = self.language_model.model(input_ids,

375 positions,

376 intermediate_tensors,

[V0 Deprecation][Models] Remove all V0 condition for mm embeddings merge (#25331)