601bd3268e · tduncan/Nvidia-Optimization-Stack-VLLM

docs/source/contributing/model/multimodal.md

+3 -2

@@ -34,7 +34,8 @@ Further update the model as follows:

34 image_features = self.vision_encoder(image_input)

35 return self.multi_modal_projector(image_features)

36

37 - def get_multimodal_embeddings(self, **kwargs: object) -> Optional[NestedTensors]:

37 + def get_multimodal_embeddings(

38 + self, **kwargs: object) -> Optional[MultiModalEmbeddings]:

39

40 # Validate the multimodal input keyword arguments

41 image_input = self._parse_and_validate_image_input(**kwargs)

@@ -61,7 +62,7 @@ Further update the model as follows:

62 def get_input_embeddings(

63 self,

64 input_ids: torch.Tensor,

64 - multimodal_embeddings: Optional[NestedTensors] = None,

65 + multimodal_embeddings: Optional[MultiModalEmbeddings] = None,

66 ) -> torch.Tensor:

67

68 # `get_input_embeddings` should already be implemented for the language

tests/distributed/test_pipeline_parallel.py

+2 -2

@@ -214,7 +214,7 @@ MULTIMODAL_MODELS = {

214 "llava-hf/llava-onevision-qwen2-0.5b-ov-hf": PPTestSettings.fast(),

215 "openbmb/MiniCPM-Llama3-V-2_5": PPTestSettings.fast(),

216 "allenai/Molmo-7B-D-0924": PPTestSettings.fast(),

217 - "microsoft/Phi-3-vision-128k-instruct": PPTestSettings.fast(),

217 + "microsoft/Phi-3.5-vision-instruct": PPTestSettings.fast(),

218 "mistralai/Pixtral-12B-2409": PPTestSettings.fast(load_format="dummy"),

219 "Qwen/Qwen-VL-Chat": PPTestSettings.fast(),

220 "Qwen/Qwen2-Audio-7B-Instruct": PPTestSettings.fast(),

@@ -237,7 +237,7 @@ TEST_MODELS = [

237 "BAAI/bge-multilingual-gemma2",

238 # [MULTIMODAL GENERATION]

239 "OpenGVLab/InternVL2-1B",

240 - "microsoft/Phi-3-vision-128k-instruct",

240 + "microsoft/Phi-3.5-vision-instruct",

241 "fixie-ai/ultravox-v0_5-llama-3_2-1b",

242 # [LANGUAGE GENERATION - HYBRID ARCH]

243 "ai21labs/Jamba-tiny-dev",

vllm/model_executor/models/aria.py

+4 -6

@@ -21,8 +21,7 @@ from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead

21 from vllm.model_executor.model_loader.weight_utils import (

22 default_weight_loader, maybe_remap_kv_scale_name)

23 from vllm.multimodal import MULTIMODAL_REGISTRY

24 -from vllm.multimodal.inputs import (MultiModalFieldConfig, MultiModalKwargs,

25 - NestedTensors)

24 +from vllm.multimodal.inputs import MultiModalFieldConfig, MultiModalKwargs

25 from vllm.multimodal.parse import MultiModalDataItems

26 from vllm.multimodal.processing import (BaseMultiModalProcessor,

27 BaseProcessingInfo, PromptReplacement,

@@ -35,7 +34,7 @@ from .idefics2_vision_model import Idefics2VisionConfig

34 from .idefics2_vision_model import (

35 Idefics2VisionTransformer as Idefics3VisionTransformer)

36 # yapf: enable

38 -from .interfaces import SupportsMultiModal, SupportsQuant

37 +from .interfaces import MultiModalEmbeddings, SupportsMultiModal, SupportsQuant

38 from .llama import LlamaDecoderLayer, LlamaMLP, LlamaModel

39 from .utils import (AutoWeightsLoader, WeightsMapper, flatten_bn,

40 is_pp_missing_parameter, maybe_prefix,

@@ -607,8 +606,7 @@ class AriaForConditionalGeneration(nn.Module, SupportsMultiModal):

606 return self.multi_modal_projector(image_outputs, image_attn_mask)

607

608 def get_multimodal_embeddings(

610 - self, **kwargs

611 - ) -> Union[list[torch.Tensor], torch.Tensor, tuple[torch.Tensor, ...]]:

609 + self, **kwargs: object) -> Optional[MultiModalEmbeddings]:

610 image_input = self._parse_and_validate_image_input(**kwargs)

611 if image_input is None:

612 return None

@@ -618,7 +616,7 @@ class AriaForConditionalGeneration(nn.Module, SupportsMultiModal):

616 def get_input_embeddings(

617 self,

618 input_ids: torch.Tensor,

621 - multimodal_embeddings: Optional[NestedTensors] = None,

619 + multimodal_embeddings: Optional[MultiModalEmbeddings] = None,

620 ) -> torch.Tensor:

621 inputs_embeds = self.language_model.get_input_embeddings(input_ids)

622 if multimodal_embeddings is not None:

vllm/model_executor/models/blip2.py

+4 -6

@@ -15,8 +15,7 @@ from vllm.model_executor.layers.quantization import QuantizationConfig

15 from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler

16 from vllm.model_executor.sampling_metadata import SamplingMetadata

17 from vllm.multimodal import MULTIMODAL_REGISTRY

18 -from vllm.multimodal.inputs import (MultiModalFieldConfig, MultiModalKwargs,

19 - NestedTensors)

18 +from vllm.multimodal.inputs import MultiModalFieldConfig, MultiModalKwargs

19 from vllm.multimodal.parse import MultiModalDataItems

20 from vllm.multimodal.processing import (BaseMultiModalProcessor,

21 BaseProcessingInfo, PromptIndexTargets,

@@ -25,7 +24,7 @@ from vllm.multimodal.profiling import BaseDummyInputsBuilder, ProcessorInputs

24 from vllm.sequence import IntermediateTensors

25

26 from .blip import BlipVisionModel

28 -from .interfaces import SupportsMultiModal, SupportsPP

27 +from .interfaces import MultiModalEmbeddings, SupportsMultiModal, SupportsPP

28 from .utils import (AutoWeightsLoader, init_vllm_registered_model,

29 maybe_prefix, merge_multimodal_embeddings)

30

@@ -629,8 +628,7 @@ class Blip2ForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP):

628 return self.language_projection(query_output)

629

630 def get_multimodal_embeddings(

632 - self, **kwargs

633 - ) -> Union[list[torch.Tensor], torch.Tensor, tuple[torch.Tensor, ...]]:

631 + self, **kwargs: object) -> Optional[MultiModalEmbeddings]:

632 image_input = self._parse_and_validate_image_input(**kwargs)

633 if image_input is None:

634 return None

@@ -640,7 +638,7 @@ class Blip2ForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP):

638 def get_input_embeddings(

639 self,

640 input_ids: torch.Tensor,

643 - multimodal_embeddings: Optional[NestedTensors] = None,

641 + multimodal_embeddings: Optional[MultiModalEmbeddings] = None,

642 ) -> torch.Tensor:

643 inputs_embeds = self.language_model.get_input_embeddings(input_ids)

644 if multimodal_embeddings is not None:

vllm/model_executor/models/chameleon.py

+4 -6

@@ -30,8 +30,7 @@ from vllm.model_executor.model_loader.weight_utils import (

30 from vllm.model_executor.sampling_metadata import SamplingMetadata

31 from vllm.model_executor.utils import set_weight_attrs

32 from vllm.multimodal import MULTIMODAL_REGISTRY

33 -from vllm.multimodal.inputs import (MultiModalFieldConfig, MultiModalKwargs,

34 - NestedTensors)

33 +from vllm.multimodal.inputs import MultiModalFieldConfig, MultiModalKwargs

34 from vllm.multimodal.parse import MultiModalDataItems

35 from vllm.multimodal.processing import (BaseMultiModalProcessor,

36 BaseProcessingInfo, PromptReplacement,

@@ -39,7 +38,7 @@ from vllm.multimodal.processing import (BaseMultiModalProcessor,

38 from vllm.multimodal.profiling import BaseDummyInputsBuilder, ProcessorInputs

39 from vllm.sequence import IntermediateTensors

40

42 -from .interfaces import SupportsMultiModal, SupportsPP

41 +from .interfaces import MultiModalEmbeddings, SupportsMultiModal, SupportsPP

42 from .utils import (is_pp_missing_parameter,

43 make_empty_intermediate_tensors_factory, make_layers,

44 maybe_prefix, merge_multimodal_embeddings)

@@ -986,8 +985,7 @@ class ChameleonForConditionalGeneration(nn.Module, SupportsMultiModal,

985 )

986

987 def get_multimodal_embeddings(

989 - self, **kwargs

990 - ) -> Union[list[torch.Tensor], torch.Tensor, tuple[torch.Tensor, ...]]:

988 + self, **kwargs: object) -> Optional[MultiModalEmbeddings]:

989 image_input = self._parse_and_validate_image_input(**kwargs)

990 if image_input is None:

991 return None

@@ -1000,7 +998,7 @@ class ChameleonForConditionalGeneration(nn.Module, SupportsMultiModal,

998 def get_input_embeddings(

999 self,

1000 input_ids: torch.Tensor,

1003 - multimodal_embeddings: Optional[NestedTensors] = None,

1001 + multimodal_embeddings: Optional[MultiModalEmbeddings] = None,

1002 ) -> torch.Tensor:

1003

1004 inputs_embeds = self.model.get_input_embeddings(input_ids)

vllm/model_executor/models/deepseek_vl2.py

+3 -4

@@ -36,7 +36,7 @@ from vllm.transformers_utils.processors.deepseek_vl2 import (

36 from vllm.transformers_utils.tokenizer import cached_tokenizer_from_config

37 from vllm.utils import is_list_of

38

39 -from .interfaces import SupportsMultiModal, SupportsPP

39 +from .interfaces import MultiModalEmbeddings, SupportsMultiModal, SupportsPP

40 from .utils import (AutoWeightsLoader, WeightsMapper, flatten_bn,

41 init_vllm_registered_model, maybe_prefix,

42 merge_multimodal_embeddings)

@@ -605,8 +605,7 @@ class DeepseekVLV2ForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):

605 pixel_values=pixel_values, images_spatial_crop=images_spatial_crop)

606

607 def get_multimodal_embeddings(

608 - self, **kwargs: object

609 - ) -> Union[list[torch.Tensor], torch.Tensor, tuple[torch.Tensor, ...]]:

608 + self, **kwargs: object) -> Optional[MultiModalEmbeddings]:

609 image_input = self._parse_and_validate_image_input(**kwargs)

610 if image_input is None:

611 return None

@@ -616,7 +615,7 @@ class DeepseekVLV2ForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):

615 def get_input_embeddings(

616 self,

617 input_ids: torch.Tensor,

619 - multimodal_embeddings: Optional[NestedTensors] = None,

618 + multimodal_embeddings: Optional[MultiModalEmbeddings] = None,

619 ) -> torch.Tensor:

620 inputs_embeds = self.language_model.get_input_embeddings(input_ids)

621 if multimodal_embeddings is not None:

vllm/model_executor/models/florence2.py

+5 -5

@@ -20,7 +20,7 @@ from vllm.model_executor.models.bart import (BartDecoder, BartEncoder,

20 BartParallelLMHead,

21 BartScaledWordEmbedding)

22 from vllm.model_executor.sampling_metadata import SamplingMetadata

23 -from vllm.multimodal import MULTIMODAL_REGISTRY, NestedTensors

23 +from vllm.multimodal import MULTIMODAL_REGISTRY

24 from vllm.multimodal.inputs import MultiModalFieldConfig, MultiModalKwargs

25 from vllm.multimodal.parse import MultiModalDataDict, MultiModalDataItems

26 from vllm.multimodal.processing import (BaseProcessingInfo,

@@ -30,7 +30,8 @@ from vllm.multimodal.processing import (BaseProcessingInfo,

30 from vllm.multimodal.profiling import BaseDummyInputsBuilder, ProcessorInputs

31 from vllm.sequence import IntermediateTensors

32

33 -from .interfaces import SupportsMultiModal, SupportsV0Only

33 +from .interfaces import (MultiModalEmbeddings, SupportsMultiModal,

34 + SupportsV0Only)

35 from .utils import AutoWeightsLoader, flatten_bn, merge_multimodal_embeddings

36

37

@@ -1037,8 +1038,7 @@ class Florence2ForConditionalGeneration(nn.Module, SupportsMultiModal):

1038 return self._encode_image(pixel_values)

1039

1040 def get_multimodal_embeddings(

1040 - self, **kwargs: object

1041 - ) -> Union[list[torch.Tensor], torch.Tensor, tuple[torch.Tensor, ...]]:

1041 + self, **kwargs: object) -> Optional[MultiModalEmbeddings]:

1042 image_input = self._parse_and_validate_image_input(**kwargs)

1043 if image_input is None:

1044 return None

@@ -1048,7 +1048,7 @@ class Florence2ForConditionalGeneration(nn.Module, SupportsMultiModal):

1048 def get_input_embeddings(

1049 self,

1050 input_ids: torch.Tensor,

1051 - multimodal_embeddings: Optional[NestedTensors] = None,

1051 + multimodal_embeddings: Optional[MultiModalEmbeddings] = None,

1052 ) -> torch.Tensor:

1053 inputs_embeds = self.language_model.get_input_embeddings(input_ids)

1054 if multimodal_embeddings is not None:

vllm/model_executor/models/fuyu.py

+4 -5

@@ -18,7 +18,7 @@

18 """ PyTorch Fuyu model."""

19 import math

20 from collections.abc import Iterable, Mapping, Sequence

21 -from typing import List, Literal, Optional, Set, Tuple, TypedDict, Union

21 +from typing import List, Literal, Optional, Set, Tuple, TypedDict

22

23 import torch

24 import torch.nn as nn

@@ -41,7 +41,7 @@ from vllm.multimodal.processing import (BaseMultiModalProcessor,

41 from vllm.multimodal.profiling import BaseDummyInputsBuilder, ProcessorInputs

42 from vllm.sequence import IntermediateTensors

43

44 -from .interfaces import SupportsMultiModal, SupportsPP

44 +from .interfaces import MultiModalEmbeddings, SupportsMultiModal, SupportsPP

45 from .utils import (AutoWeightsLoader, flatten_bn, maybe_prefix,

46 merge_multimodal_embeddings)

47

@@ -327,8 +327,7 @@ class FuyuForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):

327 return vision_embeddings_flat.split(patches_per_image, dim=0)

328

329 def get_multimodal_embeddings(

330 - self, **kwargs

331 - ) -> Union[list[torch.Tensor], torch.Tensor, tuple[torch.Tensor, ...]]:

330 + self, **kwargs: object) -> Optional[MultiModalEmbeddings]:

331 image_input = self._parse_and_validate_image_input(**kwargs)

332 if image_input is None:

333 return None

@@ -338,7 +337,7 @@ class FuyuForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):

337 def get_input_embeddings(

338 self,

339 input_ids: torch.Tensor,

341 - multimodal_embeddings: Optional[NestedTensors] = None,

340 + multimodal_embeddings: Optional[MultiModalEmbeddings] = None,

341 ) -> torch.Tensor:

342 inputs_embeds = self.language_model.get_input_embeddings(input_ids)

343 if multimodal_embeddings is not None:

vllm/model_executor/models/gemma3_mm.py

+5 -5

@@ -14,8 +14,7 @@ from vllm.model_executor.layers.layernorm import GemmaRMSNorm

14 from vllm.model_executor.layers.sampler import SamplerOutput

15 from vllm.model_executor.sampling_metadata import SamplingMetadata

16 from vllm.multimodal import MULTIMODAL_REGISTRY

17 -from vllm.multimodal.inputs import (MultiModalFieldConfig, MultiModalKwargs,

18 - NestedTensors)

17 +from vllm.multimodal.inputs import MultiModalFieldConfig, MultiModalKwargs

18 from vllm.multimodal.parse import (ImageProcessorItems, ImageSize,

19 MultiModalDataItems)

20 from vllm.multimodal.processing import (BaseMultiModalProcessor,

@@ -24,7 +23,7 @@ from vllm.multimodal.processing import (BaseMultiModalProcessor,

23 from vllm.multimodal.profiling import BaseDummyInputsBuilder, ProcessorInputs

24 from vllm.sequence import IntermediateTensors

25

27 -from .interfaces import SupportsMultiModal, SupportsPP

26 +from .interfaces import MultiModalEmbeddings, SupportsMultiModal, SupportsPP

27 from .siglip import SiglipVisionModel

28 from .utils import (AutoWeightsLoader, flatten_bn, init_vllm_registered_model,

29 maybe_prefix, merge_multimodal_embeddings)

@@ -481,7 +480,8 @@ class Gemma3ForConditionalGeneration(nn.Module, SupportsMultiModal,

480 )

481 return self.multi_modal_projector(vision_outputs)

482

484 - def get_multimodal_embeddings(self, **kwargs) -> Optional[NestedTensors]:

483 + def get_multimodal_embeddings(

484 + self, **kwargs: object) -> Optional[MultiModalEmbeddings]:

485 image_input = self._parse_and_validate_image_input(**kwargs)

486 if image_input is None:

487 return None

@@ -491,7 +491,7 @@ class Gemma3ForConditionalGeneration(nn.Module, SupportsMultiModal,

491 def get_input_embeddings(

492 self,

493 input_ids: torch.Tensor,

494 - multimodal_embeddings: Optional[NestedTensors] = None,

494 + multimodal_embeddings: Optional[MultiModalEmbeddings] = None,

495 ) -> torch.Tensor:

496 if multimodal_embeddings is None:

497 inputs_embeds = self.language_model.get_input_embeddings(input_ids)

vllm/model_executor/models/glm4v.py

+5 -5

@@ -28,7 +28,7 @@ from vllm.model_executor.layers.linear import (ColumnParallelLinear,

28 from vllm.model_executor.layers.quantization import QuantizationConfig

29 from vllm.model_executor.models.module_mapping import MultiModelKeys

30 from vllm.multimodal import MULTIMODAL_REGISTRY

31 -from vllm.multimodal.inputs import MultiModalKwargs, NestedTensors

31 +from vllm.multimodal.inputs import MultiModalKwargs

32 from vllm.multimodal.parse import MultiModalDataItems

33 from vllm.multimodal.processing import (BaseMultiModalProcessor,

34 BaseProcessingInfo, BatchFeature,

@@ -39,7 +39,8 @@ from vllm.sequence import IntermediateTensors

39 from vllm.transformers_utils.configs import ChatGLMConfig

40

41 from .chatglm import ChatGLMBaseModel, ChatGLMModel

42 -from .interfaces import SupportsLoRA, SupportsMultiModal, SupportsPP

42 +from .interfaces import (MultiModalEmbeddings, SupportsLoRA,

43 + SupportsMultiModal, SupportsPP)

44 from .utils import flatten_bn, merge_multimodal_embeddings

45

46

@@ -596,8 +597,7 @@ class GLM4VForCausalLM(ChatGLMBaseModel, SupportsLoRA, SupportsPP,

597 return self.transformer.vision(pixel_values)

598

599 def get_multimodal_embeddings(

599 - self, **kwargs

600 - ) -> Union[list[torch.Tensor], torch.Tensor, tuple[torch.Tensor, ...]]:

600 + self, **kwargs: object) -> Optional[MultiModalEmbeddings]:

601 image_input = self._parse_and_validate_image_input(**kwargs)

602 if image_input is None:

603 return None

@@ -608,7 +608,7 @@ class GLM4VForCausalLM(ChatGLMBaseModel, SupportsLoRA, SupportsPP,

608 def get_input_embeddings(

609 self,

610 input_ids: torch.Tensor,

611 - multimodal_embeddings: Optional[NestedTensors] = None,

611 + multimodal_embeddings: Optional[MultiModalEmbeddings] = None,

612 ) -> torch.Tensor:

613 inputs_embeds = self.transformer.get_input_embeddings(input_ids)

614

vllm/model_executor/models/idefics3.py

+3 -4

@@ -49,7 +49,7 @@ from vllm.sequence import IntermediateTensors

49 from .idefics2_vision_model import (

50 Idefics2VisionTransformer as Idefics3VisionTransformer)

51 # yapf: enable

52 -from .interfaces import SupportsLoRA, SupportsMultiModal

52 +from .interfaces import MultiModalEmbeddings, SupportsLoRA, SupportsMultiModal

53 from .llama import LlamaModel

54 from .utils import (AutoWeightsLoader, flatten_bn, maybe_prefix,

55 merge_multimodal_embeddings)

@@ -617,8 +617,7 @@ class Idefics3ForConditionalGeneration(nn.Module, SupportsMultiModal,

617 self.sampler = get_sampler()

618

619 def get_multimodal_embeddings(

620 - self, **kwargs

621 - ) -> Union[list[torch.Tensor], torch.Tensor, tuple[torch.Tensor, ...]]:

620 + self, **kwargs: object) -> Optional[MultiModalEmbeddings]:

621 image_input = self.model._parse_and_validate_image_input(**kwargs)

622 if image_input is None:

623 return None

@@ -628,7 +627,7 @@ class Idefics3ForConditionalGeneration(nn.Module, SupportsMultiModal,

627 def get_input_embeddings(

628 self,

629 input_ids: torch.Tensor,

631 - multimodal_embeddings: Optional[NestedTensors] = None,

630 + multimodal_embeddings: Optional[MultiModalEmbeddings] = None,

631 ) -> torch.Tensor:

632 inputs_embeds = self.model.get_input_embeddings(input_ids)

633 if multimodal_embeddings is not None:

vllm/model_executor/models/interfaces.py

+13 -11

@@ -5,7 +5,7 @@ from typing import (TYPE_CHECKING, ClassVar, Dict, List, Literal, Optional,

5

6 import torch

7 from torch import Tensor

8 -from typing_extensions import TypeIs, TypeVar

8 +from typing_extensions import TypeIs

9

10 from vllm.logger import init_logger

11 from vllm.model_executor.layers.quantization.base_config import (

@@ -20,7 +20,14 @@ if TYPE_CHECKING:

20

21 logger = init_logger(__name__)

22

23 -T = TypeVar("T", default=Union[list[Tensor], Tensor, tuple[Tensor, ...]])

23 +MultiModalEmbeddings = Union[list[Tensor], Tensor, tuple[Tensor, ...]]

24 +"""

25 +The output embeddings must be one of the following formats:

26 +

27 +- A list or tuple of 2D tensors, where each tensor corresponds to

28 + each input multimodal data item (e.g, image).

29 +- A single 3D tensor, with the batch dimension grouping the 2D tensors.

30 +"""

31

32

33 @runtime_checkable

@@ -36,17 +43,12 @@ class SupportsMultiModal(Protocol):

43 MRO of your model class.

44 """

45

39 - def get_multimodal_embeddings(self, **kwargs) -> T:

46 + def get_multimodal_embeddings(

47 + self, **kwargs: object) -> Optional[MultiModalEmbeddings]:

48 """

49 Returns multimodal embeddings generated from multimodal kwargs

50 to be merged with text embeddings.

51

44 - The output embeddings must be one of the following formats:

45 -

46 - - A list or tuple of 2D tensors, where each tensor corresponds to

47 - each input multimodal data item (e.g, image).

48 - - A single 3D tensor, with the batch dimension grouping the 2D tensors.

49 -

52 Note:

53 The returned multimodal embeddings must be in the same order as

54 the appearances of their corresponding multimodal data item in the

@@ -60,7 +62,7 @@ class SupportsMultiModal(Protocol):

62 def get_input_embeddings(

63 self,

64 input_ids: Tensor,

63 - multimodal_embeddings: Optional[T] = None,

65 + multimodal_embeddings: Optional[MultiModalEmbeddings] = None,

66 attn_metadata: Optional["AttentionMetadata"] = None,

67 ) -> Tensor:

68 ...

@@ -69,7 +71,7 @@ class SupportsMultiModal(Protocol):

71 def get_input_embeddings(

72 self,

73 input_ids: Tensor,

72 - multimodal_embeddings: Optional[T] = None,

74 + multimodal_embeddings: Optional[MultiModalEmbeddings] = None,

75 ) -> Tensor:

76 """

77 Returns the input embeddings merged from the text embeddings from

vllm/model_executor/models/internvl.py

+3 -4

@@ -37,7 +37,7 @@ from vllm.multimodal.profiling import BaseDummyInputsBuilder, ProcessorInputs

37 from vllm.sequence import IntermediateTensors

38 from vllm.transformers_utils.tokenizer import AnyTokenizer

39

40 -from .interfaces import SupportsMultiModal, SupportsPP

40 +from .interfaces import MultiModalEmbeddings, SupportsMultiModal, SupportsPP

41 from .utils import (AutoWeightsLoader, flatten_bn, init_vllm_registered_model,

42 maybe_prefix, merge_multimodal_embeddings)

43

@@ -905,8 +905,7 @@ class InternVLChatModel(nn.Module, SupportsMultiModal, SupportsPP):

905 self.visual_token_mask = None

906

907 def get_multimodal_embeddings(

908 - self, **kwargs

909 - ) -> Union[list[torch.Tensor], torch.Tensor, tuple[torch.Tensor, ...]]:

908 + self, **kwargs: object) -> Optional[MultiModalEmbeddings]:

909 image_input = self._parse_and_validate_image_input(**kwargs)

910 if image_input is None:

911 return None

@@ -916,7 +915,7 @@ class InternVLChatModel(nn.Module, SupportsMultiModal, SupportsPP):

915 def get_input_embeddings(

916 self,

917 input_ids: torch.Tensor,

919 - multimodal_embeddings: Optional[NestedTensors] = None,

918 + multimodal_embeddings: Optional[MultiModalEmbeddings] = None,

919 ) -> torch.Tensor:

920 inputs_embeds = self.language_model.get_input_embeddings(input_ids)

921 if multimodal_embeddings is not None:

vllm/model_executor/models/llava.py

+4 -3

@@ -38,7 +38,7 @@ from vllm.sequence import IntermediateTensors

38 from vllm.utils import JSONTree, flatten_2d_lists, json_map_leaves

39

40 from .clip import CLIPVisionModel

41 -from .interfaces import SupportsMultiModal, SupportsPP

41 +from .interfaces import MultiModalEmbeddings, SupportsMultiModal, SupportsPP

42 from .pixtral import (PixtralHFVisionModel,

43 get_pixtral_hf_image_feature_grid_size)

44 from .siglip import SiglipVisionModel

@@ -778,7 +778,8 @@ class LlavaForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP):

778

779 return embeds_in_batch

780

781 - def get_multimodal_embeddings(self, **kwargs) -> Optional[NestedTensors]:

781 + def get_multimodal_embeddings(

782 + self, **kwargs: object) -> Optional[MultiModalEmbeddings]:

783 image_input = self._parse_and_validate_image_input(**kwargs)

784 if image_input is None:

785 return None

@@ -800,7 +801,7 @@ class LlavaForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP):

801 def get_input_embeddings(

802 self,

803 input_ids: torch.Tensor,

803 - multimodal_embeddings: Optional[NestedTensors] = None,

804 + multimodal_embeddings: Optional[MultiModalEmbeddings] = None,

805 ) -> torch.Tensor:

806 inputs_embeds = self.language_model.get_input_embeddings(input_ids)

807 if multimodal_embeddings is not None:

vllm/model_executor/models/llava_next.py

+4 -5

@@ -16,12 +16,12 @@ from vllm.config import VllmConfig

16 from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler

17 from vllm.model_executor.sampling_metadata import SamplingMetadata

18 from vllm.multimodal import MULTIMODAL_REGISTRY

19 -from vllm.multimodal.inputs import MultiModalFieldConfig, NestedTensors

19 +from vllm.multimodal.inputs import MultiModalFieldConfig

20 from vllm.multimodal.parse import ImageSize

21 from vllm.sequence import IntermediateTensors

22

23 from .clip import CLIPVisionModel

24 -from .interfaces import SupportsMultiModal, SupportsPP

24 +from .interfaces import MultiModalEmbeddings, SupportsMultiModal, SupportsPP

25 from .llava import (BaseLlavaMultiModalProcessor, BaseLlavaProcessingInfo,

26 LlavaDummyInputsBuilder, LlavaLikeConfig,

27 LlavaMultiModalProjector, init_vision_tower_for_llava)

@@ -480,8 +480,7 @@ class LlavaNextForConditionalGeneration(nn.Module, SupportsMultiModal,

480 ]

481

482 def get_multimodal_embeddings(

483 - self, **kwargs

484 - ) -> Union[list[torch.Tensor], torch.Tensor, tuple[torch.Tensor, ...]]:

483 + self, **kwargs: object) -> Optional[MultiModalEmbeddings]:

484 image_input = self._parse_and_validate_image_input(**kwargs)

485 if image_input is None:

486 return None

@@ -491,7 +490,7 @@ class LlavaNextForConditionalGeneration(nn.Module, SupportsMultiModal,

490 def get_input_embeddings(

491 self,

492 input_ids: torch.Tensor,

494 - multimodal_embeddings: Optional[NestedTensors] = None,

493 + multimodal_embeddings: Optional[MultiModalEmbeddings] = None,

494 ) -> torch.Tensor:

495

496 if multimodal_embeddings is None:

vllm/model_executor/models/llava_next_video.py

+4 -6

@@ -16,8 +16,7 @@ from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler

16 from vllm.model_executor.models.clip import CLIPVisionModel

17 from vllm.model_executor.sampling_metadata import SamplingMetadata

18 from vllm.multimodal import MULTIMODAL_REGISTRY

19 -from vllm.multimodal.inputs import (MultiModalFieldConfig, MultiModalKwargs,

20 - NestedTensors)

19 +from vllm.multimodal.inputs import MultiModalFieldConfig, MultiModalKwargs

20 from vllm.multimodal.parse import (ImageSize, MultiModalDataItems,

21 VideoEmbeddingItems, VideoProcessorItems)

22 from vllm.multimodal.processing import (BaseMultiModalProcessor,

@@ -27,7 +26,7 @@ from vllm.multimodal.profiling import BaseDummyInputsBuilder, ProcessorInputs

26 from vllm.sequence import IntermediateTensors

27 from vllm.utils import is_list_of

28

30 -from .interfaces import SupportsMultiModal, SupportsPP

29 +from .interfaces import MultiModalEmbeddings, SupportsMultiModal, SupportsPP

30 from .llava import init_vision_tower_for_llava

31 from .siglip import SiglipVisionModel

32 from .utils import (AutoWeightsLoader, init_vllm_registered_model,

@@ -421,8 +420,7 @@ class LlavaNextVideoForConditionalGeneration(nn.Module, SupportsMultiModal,

420 f"Unsupported type of video input {type(video_pixels)}")

421

422 def get_multimodal_embeddings(

424 - self, **kwargs

425 - ) -> Union[list[torch.Tensor], torch.Tensor, tuple[torch.Tensor, ...]]:

423 + self, **kwargs: object) -> Optional[MultiModalEmbeddings]:

424 video_input = self._parse_and_validate_video_input(**kwargs)

425 if video_input is None:

426 return None

@@ -432,7 +430,7 @@ class LlavaNextVideoForConditionalGeneration(nn.Module, SupportsMultiModal,

430 def get_input_embeddings(

431 self,

432 input_ids: torch.Tensor,

435 - multimodal_embeddings: Optional[NestedTensors] = None,

433 + multimodal_embeddings: Optional[MultiModalEmbeddings] = None,

434 ) -> torch.Tensor:

435 inputs_embeds = self.language_model.get_input_embeddings(input_ids)

436 if multimodal_embeddings is not None:

vllm/model_executor/models/llava_onevision.py

+6 -8

@@ -19,8 +19,7 @@ from vllm.model_executor.layers.activation import get_act_fn

19 from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler

20 from vllm.model_executor.sampling_metadata import SamplingMetadata

21 from vllm.multimodal import MULTIMODAL_REGISTRY

22 -from vllm.multimodal.inputs import (MultiModalFieldConfig, MultiModalKwargs,

23 - NestedTensors)

22 +from vllm.multimodal.inputs import MultiModalFieldConfig, MultiModalKwargs

23 from vllm.multimodal.parse import (ImageSize, MultiModalDataItems,

24 VideoEmbeddingItems, VideoProcessorItems)

25 from vllm.multimodal.processing import PromptReplacement, PromptUpdate

@@ -29,7 +28,7 @@ from vllm.sequence import IntermediateTensors

28 from vllm.utils import is_list_of

29

30 from .clip import CLIPVisionModel

32 -from .interfaces import SupportsMultiModal, SupportsPP

31 +from .interfaces import MultiModalEmbeddings, SupportsMultiModal, SupportsPP

32 from .llava import LlavaDummyInputsBuilder, init_vision_tower_for_llava

33 from .llava_next import (BaseLlavaNextMultiModalProcessor, LlavaNextLikeConfig,

34 LlavaNextProcessingInfo)

@@ -856,7 +855,7 @@ class LlavaOnevisionForConditionalGeneration(nn.Module, SupportsMultiModal,

855 return image_feature

856

857 def get_multimodal_embeddings(

859 - self, **kwargs) -> Optional[tuple[torch.Tensor, ...]]:

858 + self, **kwargs: object) -> Optional[MultiModalEmbeddings]:

859 modalities = self._parse_and_validate_multimodal_inputs(**kwargs)

860 if not modalities:

861 return None

@@ -882,7 +881,7 @@ class LlavaOnevisionForConditionalGeneration(nn.Module, SupportsMultiModal,

881 def get_input_embeddings(

882 self,

883 input_ids: torch.Tensor,

885 - multimodal_embeddings: Optional[tuple[torch.Tensor, ...]] = None,

884 + multimodal_embeddings: Optional[MultiModalEmbeddings] = None,

885 ) -> torch.Tensor:

886 inputs_embeds = self.language_model.get_input_embeddings(input_ids)

887 if multimodal_embeddings is not None:

@@ -894,10 +893,9 @@ class LlavaOnevisionForConditionalGeneration(nn.Module, SupportsMultiModal,

893 def get_input_embeddings_v0(

894 self,

895 input_ids: torch.Tensor,

897 - image_input: Optional[NestedTensors] = None,

898 - video_input: Optional[NestedTensors] = None,

896 + image_input: Optional[LlavaOnevisionImagePixelInputs] = None,

897 + video_input: Optional[LlavaOnevisionVideoPixelInputs] = None,

898 ) -> torch.Tensor:

900 -

899 inputs_embeds = self.get_input_embeddings(input_ids)

900 if image_input is not None:

901 image_embeds = self._process_image_input(image_input)

vllm/model_executor/models/molmo.py

+4 -5

@@ -52,8 +52,8 @@ from vllm.multimodal.profiling import BaseDummyInputsBuilder, ProcessorInputs

52 from vllm.sequence import IntermediateTensors

53 from vllm.utils import JSONTree, flatten_2d_lists, json_map_leaves

54

55 -from .interfaces import (SupportsLoRA, SupportsMultiModal, SupportsPP,

56 - SupportsQuant)

55 +from .interfaces import (MultiModalEmbeddings, SupportsLoRA,

56 + SupportsMultiModal, SupportsPP, SupportsQuant)

57 from .utils import (AutoWeightsLoader, WeightsMapper, flatten_bn,

58 is_pp_missing_parameter,

59 make_empty_intermediate_tensors_factory, make_layers,

@@ -1577,8 +1577,7 @@ class MolmoForCausalLM(nn.Module, SupportsMultiModal, SupportsPP, SupportsLoRA,

1577 return embeds_in_batch

1578

1579 def get_multimodal_embeddings(

1580 - self, **kwargs

1581 - ) -> Union[list[torch.Tensor], torch.Tensor, tuple[torch.Tensor, ...]]:

1580 + self, **kwargs: object) -> Optional[MultiModalEmbeddings]:

1581 image_input = self._parse_and_validate_image_input(**kwargs)

1582 if image_input is None:

1583 return None

@@ -1598,7 +1597,7 @@ class MolmoForCausalLM(nn.Module, SupportsMultiModal, SupportsPP, SupportsLoRA,

1597 def get_input_embeddings(

1598 self,

1599 input_ids: torch.Tensor,

1601 - multimodal_embeddings: Optional[NestedTensors] = None,

1600 + multimodal_embeddings: Optional[MultiModalEmbeddings] = None,

1601 ) -> torch.Tensor:

1602 inputs_embeds = self.model.get_input_embeddings(input_ids)

1603 if multimodal_embeddings is not None:

vllm/model_executor/models/paligemma.py

+4 -6

@@ -13,8 +13,7 @@ from vllm.model_executor.layers.sampler import SamplerOutput

13 from vllm.model_executor.sampling_metadata import SamplingMetadata

14 from vllm.multimodal import MULTIMODAL_REGISTRY

15 from vllm.multimodal.inputs import (MultiModalDataDict, MultiModalFieldConfig,

16 - MultiModalInputs, MultiModalKwargs,

17 - NestedTensors)

16 + MultiModalInputs, MultiModalKwargs)

17 from vllm.multimodal.parse import MultiModalDataItems

18 from vllm.multimodal.processing import (BaseMultiModalProcessor,

19 BaseProcessingInfo, PromptIndexTargets,

@@ -23,7 +22,7 @@ from vllm.multimodal.processing import (BaseMultiModalProcessor,

22 from vllm.multimodal.profiling import BaseDummyInputsBuilder, ProcessorInputs

23 from vllm.sequence import IntermediateTensors

24

26 -from .interfaces import SupportsMultiModal, SupportsPP

25 +from .interfaces import MultiModalEmbeddings, SupportsMultiModal, SupportsPP

26 from .siglip import SiglipVisionModel

27 from .utils import (AutoWeightsLoader, init_vllm_registered_model,

28 maybe_prefix, merge_multimodal_embeddings)

@@ -328,8 +327,7 @@ class PaliGemmaForConditionalGeneration(nn.Module, SupportsMultiModal,

327 return self.multi_modal_projector(image_features)

328

329 def get_multimodal_embeddings(

331 - self, **kwargs

332 - ) -> Union[list[torch.Tensor], torch.Tensor, tuple[torch.Tensor, ...]]:

330 + self, **kwargs: object) -> Optional[MultiModalEmbeddings]:

331 image_input = self._parse_and_validate_image_input(**kwargs)

332 if image_input is None:

333 return None

@@ -341,7 +339,7 @@ class PaliGemmaForConditionalGeneration(nn.Module, SupportsMultiModal,

339 def get_input_embeddings(

340 self,

341 input_ids: torch.Tensor,

344 - multimodal_embeddings: Optional[NestedTensors] = None,

342 + multimodal_embeddings: Optional[MultiModalEmbeddings] = None,

343 ) -> torch.Tensor:

344 inputs_embeds = self.language_model.get_input_embeddings(input_ids)

345 if multimodal_embeddings is not None:

vllm/model_executor/models/phi3v.py

+5 -6

@@ -31,8 +31,7 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (

31 VocabParallelEmbedding)

32 from vllm.model_executor.sampling_metadata import SamplingMetadata

33 from vllm.multimodal import MULTIMODAL_REGISTRY

34 -from vllm.multimodal.inputs import (MultiModalFieldConfig, MultiModalKwargs,

35 - NestedTensors)

34 +from vllm.multimodal.inputs import MultiModalFieldConfig, MultiModalKwargs

35 from vllm.multimodal.parse import (ImageEmbeddingItems, ImageProcessorItems,

36 ImageSize, MultiModalDataItems)

37 # yapf conflicts with isort for this block

@@ -48,7 +47,8 @@ from vllm.sequence import IntermediateTensors

47 from vllm.utils import is_list_of

48

49 from .clip import CLIPVisionModel

51 -from .interfaces import SupportsMultiModal, SupportsPP, SupportsQuant

50 +from .interfaces import (MultiModalEmbeddings, SupportsMultiModal, SupportsPP,

51 + SupportsQuant)

52 from .utils import (AutoWeightsLoader, WeightsMapper, flatten_bn,

53 init_vllm_registered_model, maybe_prefix,

54 merge_multimodal_embeddings)

@@ -649,8 +649,7 @@ class Phi3VForCausalLM(nn.Module, SupportsMultiModal, SupportsPP,

649 return image_embeds

650

651 def get_multimodal_embeddings(

652 - self, **kwargs

653 - ) -> Union[list[torch.Tensor], torch.Tensor, tuple[torch.Tensor, ...]]:

652 + self, **kwargs: object) -> Optional[MultiModalEmbeddings]:

653 image_input = self._parse_and_validate_image_input(**kwargs)

654 if image_input is None:

655 return None

@@ -660,7 +659,7 @@ class Phi3VForCausalLM(nn.Module, SupportsMultiModal, SupportsPP,

659 def get_input_embeddings(

660 self,

661 input_ids: torch.Tensor,

663 - multimodal_embeddings: Optional[NestedTensors] = None,

662 + multimodal_embeddings: Optional[MultiModalEmbeddings] = None,

663 ) -> torch.Tensor:

664 inputs_embeds = self.embed_tokens(input_ids)

665 if multimodal_embeddings is not None:

vllm/model_executor/models/pixtral.py

+4 -5

@@ -30,12 +30,12 @@ from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler

30 from vllm.model_executor.model_loader.weight_utils import default_weight_loader

31 from vllm.model_executor.sampling_metadata import SamplingMetadata

32 from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalKwargs

33 -from vllm.multimodal.inputs import NestedTensors, PlaceholderRange

33 +from vllm.multimodal.inputs import PlaceholderRange

34 from vllm.multimodal.utils import consecutive_placeholder_ranges

35 from vllm.sequence import IntermediateTensors, SequenceData

36 from vllm.transformers_utils.tokenizer import cached_tokenizer_from_config

37

38 -from .interfaces import SupportsMultiModal, SupportsPP

38 +from .interfaces import MultiModalEmbeddings, SupportsMultiModal, SupportsPP

39 from .utils import (init_vllm_registered_model, maybe_prefix,

40 merge_multimodal_embeddings)

41 from .vision import VisionEncoderInfo, resolve_visual_encoder_outputs

@@ -221,8 +221,7 @@ class PixtralForConditionalGeneration(nn.Module, SupportsMultiModal,

221 return get_sampler()

222

223 def get_multimodal_embeddings(

224 - self, **kwargs

225 - ) -> Union[list[torch.Tensor], torch.Tensor, tuple[torch.Tensor, ...]]:

224 + self, **kwargs: object) -> Optional[MultiModalEmbeddings]:

225 image_input, image_tokens = self._parse_and_validate_image_input(

226 **kwargs)

227 if image_input is None:

@@ -255,7 +254,7 @@ class PixtralForConditionalGeneration(nn.Module, SupportsMultiModal,

254 def get_input_embeddings(

255 self,

256 input_ids: torch.Tensor,

258 - multimodal_embeddings: Optional[NestedTensors] = None,

257 + multimodal_embeddings: Optional[MultiModalEmbeddings] = None,

258 ) -> torch.Tensor:

259 inputs_embeds = self.language_model.get_input_embeddings(input_ids)

260 if multimodal_embeddings is not None:

vllm/model_executor/models/qwen2_5_vl.py

+6 -6

@@ -59,7 +59,8 @@ from vllm.platforms import _Backend

59 from vllm.sequence import IntermediateTensors

60 from vllm.transformers_utils.config import uses_mrope

61

62 -from .interfaces import SupportsLoRA, SupportsMultiModal, SupportsPP

62 +from .interfaces import (MultiModalEmbeddings, SupportsLoRA,

63 + SupportsMultiModal, SupportsPP)

64 from .qwen2_vl import Qwen2VLDummyInputsBuilder as Qwen2_5_VLDummyInputsBuilder

65 from .qwen2_vl import (Qwen2VLMultiModalProcessor, Qwen2VLProcessingInfo,

66 apply_rotary_pos_emb_vision)

@@ -952,7 +953,7 @@ class Qwen2_5_VLForConditionalGeneration(nn.Module, SupportsMultiModal,

953 return modalities

954

955 def get_multimodal_embeddings(

955 - self, **kwargs) -> Optional[tuple[torch.Tensor, ...]]:

956 + self, **kwargs: object) -> Optional[MultiModalEmbeddings]:

957

958 modalities = self._parse_and_validate_multimodal_inputs(**kwargs)

959 if not modalities:

@@ -978,7 +979,7 @@ class Qwen2_5_VLForConditionalGeneration(nn.Module, SupportsMultiModal,

979 def get_input_embeddings(

980 self,

981 input_ids: torch.Tensor,

981 - multimodal_embeddings: Optional[tuple[torch.Tensor, ...]] = None,

982 + multimodal_embeddings: Optional[MultiModalEmbeddings] = None,

983 ) -> torch.Tensor:

984 inputs_embeds = self.language_model.get_input_embeddings(input_ids)

985 if multimodal_embeddings is not None:

@@ -990,10 +991,9 @@ class Qwen2_5_VLForConditionalGeneration(nn.Module, SupportsMultiModal,

991 def get_input_embeddings_v0(

992 self,

993 input_ids: torch.Tensor,

993 - image_input: Optional[tuple[torch.Tensor, ...]] = None,

994 - video_input: Optional[tuple[torch.Tensor, ...]] = None,

994 + image_input: Optional[Qwen2_5_VLImageInputs] = None,

995 + video_input: Optional[Qwen2_5_VLVideoInputs] = None,

996 ) -> torch.Tensor:

996 -

997 inputs_embeds = self.get_input_embeddings(input_ids)

998 if image_input is not None:

999 image_embeds = self._process_image_input(image_input)

vllm/model_executor/models/qwen2_audio.py

+4 -6

@@ -37,8 +37,7 @@ from vllm.config import VllmConfig

37 from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler

38 from vllm.model_executor.sampling_metadata import SamplingMetadata

39 from vllm.multimodal import MULTIMODAL_REGISTRY

40 -from vllm.multimodal.inputs import (MultiModalFieldConfig, MultiModalKwargs,

41 - NestedTensors)

40 +from vllm.multimodal.inputs import MultiModalFieldConfig, MultiModalKwargs

41 from vllm.multimodal.parse import (AudioProcessorItems, MultiModalDataItems,

42 MultiModalDataParser)

43 from vllm.multimodal.processing import (BaseMultiModalProcessor,

@@ -47,7 +46,7 @@ from vllm.multimodal.processing import (BaseMultiModalProcessor,

46 from vllm.multimodal.profiling import BaseDummyInputsBuilder, ProcessorInputs

47 from vllm.sequence import IntermediateTensors

48

50 -from .interfaces import SupportsMultiModal, SupportsPP

49 +from .interfaces import MultiModalEmbeddings, SupportsMultiModal, SupportsPP

50 from .utils import (AutoWeightsLoader, init_vllm_registered_model,

51 maybe_prefix, merge_multimodal_embeddings)

52

@@ -357,8 +356,7 @@ class Qwen2AudioForConditionalGeneration(nn.Module, SupportsMultiModal,

356 audio_output_lengths.flatten().tolist())

357

358 def get_multimodal_embeddings(

360 - self, **kwargs

361 - ) -> Union[list[torch.Tensor], torch.Tensor, tuple[torch.Tensor, ...]]:

359 + self, **kwargs: object) -> Optional[MultiModalEmbeddings]:

360 audio_input = self._parse_and_validate_audio_input(**kwargs)

361 if audio_input is None:

362 return None

@@ -368,7 +366,7 @@ class Qwen2AudioForConditionalGeneration(nn.Module, SupportsMultiModal,

366 def get_input_embeddings(

367 self,

368 input_ids: torch.Tensor,

371 - multimodal_embeddings: Optional[NestedTensors] = None,

369 + multimodal_embeddings: Optional[MultiModalEmbeddings] = None,

370 ) -> torch.Tensor:

371 inputs_embeds = self.language_model.get_input_embeddings(input_ids)

372 if multimodal_embeddings is not None:

vllm/model_executor/models/qwen2_vl.py

+6 -6

@@ -71,7 +71,8 @@ from vllm.transformers_utils.config import uses_mrope

71 from vllm.transformers_utils.processor import (

72 cached_image_processor_from_config)

73

74 -from .interfaces import SupportsLoRA, SupportsMultiModal, SupportsPP

74 +from .interfaces import (MultiModalEmbeddings, SupportsLoRA,

75 + SupportsMultiModal, SupportsPP)

76 from .utils import (AutoWeightsLoader, WeightsMapper,

77 init_vllm_registered_model, maybe_prefix,

78 merge_multimodal_embeddings)

@@ -1262,7 +1263,7 @@ class Qwen2VLForConditionalGeneration(nn.Module, SupportsMultiModal,

1263 return modalities

1264

1265 def get_multimodal_embeddings(

1265 - self, **kwargs) -> Optional[tuple[torch.Tensor, ...]]:

1266 + self, **kwargs: object) -> Optional[MultiModalEmbeddings]:

1267

1268 modalities = self._parse_and_validate_multimodal_inputs(**kwargs)

1269 if not modalities:

@@ -1289,7 +1290,7 @@ class Qwen2VLForConditionalGeneration(nn.Module, SupportsMultiModal,

1290 def get_input_embeddings(

1291 self,

1292 input_ids: torch.Tensor,

1292 - multimodal_embeddings: Optional[tuple[torch.Tensor, ...]] = None,

1293 + multimodal_embeddings: Optional[MultiModalEmbeddings] = None,

1294 ) -> torch.Tensor:

1295 inputs_embeds = self.language_model.get_input_embeddings(input_ids)

1296 if multimodal_embeddings is not None:

@@ -1301,10 +1302,9 @@ class Qwen2VLForConditionalGeneration(nn.Module, SupportsMultiModal,

1302 def get_input_embeddings_v0(

1303 self,

1304 input_ids: torch.Tensor,

1304 - image_input: Optional[tuple[torch.Tensor, ...]] = None,

1305 - video_input: Optional[tuple[torch.Tensor, ...]] = None,

1305 + image_input: Optional[Qwen2VLImagePixelInputs] = None,

1306 + video_input: Optional[Qwen2VLVideoPixelInputs] = None,

1307 ) -> torch.Tensor:

1307 -

1308 inputs_embeds = self.get_input_embeddings(input_ids)

1309 if image_input is not None:

1310 image_embeds = self._process_image_input(image_input)

vllm/model_executor/models/qwen_vl.py

+5 -6

@@ -32,8 +32,7 @@ from vllm.model_executor.layers.quantization import QuantizationConfig

32 from vllm.model_executor.layers.resampler import Resampler2, get_abs_pos

33 from vllm.model_executor.models.module_mapping import MultiModelKeys

34 from vllm.multimodal import MULTIMODAL_REGISTRY

35 -from vllm.multimodal.inputs import (MultiModalFieldConfig, MultiModalKwargs,

36 - NestedTensors)

35 +from vllm.multimodal.inputs import MultiModalFieldConfig, MultiModalKwargs

36 from vllm.multimodal.parse import MultiModalDataItems

37 from vllm.multimodal.processing import (BaseMultiModalProcessor,

38 BaseProcessingInfo, PromptReplacement,

@@ -41,7 +40,8 @@ from vllm.multimodal.processing import (BaseMultiModalProcessor,

40 from vllm.multimodal.profiling import BaseDummyInputsBuilder, ProcessorInputs

41 from vllm.sequence import IntermediateTensors

42

44 -from .interfaces import SupportsLoRA, SupportsMultiModal, SupportsPP

43 +from .interfaces import (MultiModalEmbeddings, SupportsLoRA,

44 + SupportsMultiModal, SupportsPP)

45 from .qwen import QWenBaseModel, QWenModel

46 from .utils import flatten_bn, merge_multimodal_embeddings

47

@@ -741,8 +741,7 @@ class QwenVLForConditionalGeneration(QWenBaseModel, SupportsPP, SupportsLoRA,

741 return self.transformer.visual(image_input["data"])

742

743 def get_multimodal_embeddings(

744 - self, **kwargs

745 - ) -> Union[list[torch.Tensor], torch.Tensor, tuple[torch.Tensor, ...]]:

744 + self, **kwargs: object) -> Optional[MultiModalEmbeddings]:

745 image_input = self._parse_and_validate_image_input(**kwargs)

746 if image_input is None:

747 return None

@@ -753,7 +752,7 @@ class QwenVLForConditionalGeneration(QWenBaseModel, SupportsPP, SupportsLoRA,

752 def get_input_embeddings(

753 self,

754 input_ids: torch.Tensor,

756 - multimodal_embeddings: Optional[NestedTensors] = None,

755 + multimodal_embeddings: Optional[MultiModalEmbeddings] = None,

756 ) -> torch.Tensor:

757 inputs_embeds = self.transformer.get_input_embeddings(input_ids)

758

vllm/model_executor/models/ultravox.py

+4 -4

@@ -35,7 +35,8 @@ from vllm.multimodal.profiling import BaseDummyInputsBuilder, ProcessorInputs

35 from vllm.sequence import IntermediateTensors

36 from vllm.transformers_utils.configs.ultravox import UltravoxConfig

37

38 -from .interfaces import SupportsLoRA, SupportsMultiModal, SupportsPP

38 +from .interfaces import (MultiModalEmbeddings, SupportsLoRA,

39 + SupportsMultiModal, SupportsPP)

40 from .utils import (AutoWeightsLoader, WeightsMapper, flatten_bn,

41 init_vllm_registered_model, maybe_prefix,

42 merge_multimodal_embeddings,

@@ -555,8 +556,7 @@ class UltravoxModel(nn.Module, SupportsMultiModal, SupportsPP, SupportsLoRA):

556 return flattened_embeddings

557

558 def get_multimodal_embeddings(

558 - self, **kwargs

559 - ) -> Union[list[torch.Tensor], torch.Tensor, tuple[torch.Tensor, ...]]:

559 + self, **kwargs: object) -> Optional[MultiModalEmbeddings]:

560 audio_input = self._parse_and_validate_audio_input(**kwargs)

561 if audio_input is None:

562 return None

@@ -566,7 +566,7 @@ class UltravoxModel(nn.Module, SupportsMultiModal, SupportsPP, SupportsLoRA):

566 def get_input_embeddings(

567 self,

568 input_ids: torch.Tensor,

569 - multimodal_embeddings: Optional[NestedTensors] = None,

569 + multimodal_embeddings: Optional[MultiModalEmbeddings] = None,

570 ) -> torch.Tensor:

571 inputs_embeds = self.language_model.get_input_embeddings(input_ids)

572 if multimodal_embeddings is not None:

vllm/model_executor/models/whisper.py

+3 -4

@@ -34,8 +34,8 @@ from vllm.multimodal.processing import (BaseProcessingInfo,

34 PromptReplacement, PromptUpdate)

35 from vllm.multimodal.profiling import BaseDummyInputsBuilder, ProcessorInputs

36

37 -from .interfaces import (SupportsMultiModal, SupportsTranscription,

38 - SupportsV0Only)

37 +from .interfaces import (MultiModalEmbeddings, SupportsMultiModal,

38 + SupportsTranscription, SupportsV0Only)

39 from .utils import (AutoWeightsLoader, WeightsMapper, cast_overflow_tensors,

40 make_layers)

41

@@ -689,8 +689,7 @@ class WhisperForConditionalGeneration(nn.Module, SupportsTranscription,

689 return decoder_outputs

690

691 def get_multimodal_embeddings(

692 - self, **kwargs

693 - ) -> Union[list[torch.Tensor], torch.Tensor, tuple[torch.Tensor, ...]]:

692 + self, **kwargs: object) -> Optional[MultiModalEmbeddings]:

693 # TODO: This method does not obey the interface for SupportsMultiModal.

694 # Refactor this once encoder/decoder support is implemented in V1.

695 audio_input = self._parse_and_validate_audio_input(**kwargs)

[Misc] Clean up type annotation for `SupportsMultiModal` (#14794)