@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 2f0a0a17a4
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 05:45 UTC

[V1] Refactor model executable interface for multimodal models (#10570)

Signed-off-by: Roger Wang <ywang@roblox.com>

Roger Wang committed Nov 26, 2024 at 12:46 UTC 2f0a0a17a47436fe9709462dfee3bb9d2f91e0a0
18 files changed +581 -306
vllm/model_executor/models/blip2.py
+37 -24
@@ -16,6 +16,7 @@ from vllm.model_executor.layers.quantization import QuantizationConfig
16 from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
17 from vllm.model_executor.sampling_metadata import SamplingMetadata
18 from vllm.multimodal import MULTIMODAL_REGISTRY
19 +from vllm.multimodal.inputs import NestedTensors
20 from vllm.multimodal.utils import consecutive_placeholder_ranges
21 from vllm.sequence import IntermediateTensors, SequenceData
22
@@ -609,6 +610,25 @@ class Blip2ForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP):
610
611 return self.language_projection(query_output)
612
613 + def get_multimodal_embeddings(self, **kwargs) -> Optional[NestedTensors]:
614 + image_input = self._parse_and_validate_image_input(**kwargs)
615 + if image_input is None:
616 + return None
617 + vision_embeddings = self._process_image_input(image_input)
618 + return vision_embeddings
619 +
620 + def get_input_embeddings(
621 + self,
622 + input_ids: torch.Tensor,
623 + multimodal_embeddings: Optional[NestedTensors] = None,
624 + ) -> torch.Tensor:
625 + inputs_embeds = self.language_model.get_input_embeddings(input_ids)
626 + if multimodal_embeddings is not None:
627 + inputs_embeds = merge_multimodal_embeddings(
628 + input_ids, inputs_embeds, multimodal_embeddings,
629 + BLIP2_IMAGE_TOKEN_ID)
630 + return inputs_embeds
631 +
632 def forward(
633 self,
634 input_ids: torch.Tensor,
@@ -616,6 +636,7 @@ class Blip2ForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP):
636 kv_caches: List[torch.Tensor],
637 attn_metadata: AttentionMetadata,
638 intermediate_tensors: Optional[IntermediateTensors] = None,
639 + inputs_embeds: Optional[torch.Tensor] = None,
640 **kwargs: object,
641 ) -> Union[SamplerOutput, IntermediateTensors]:
642 """Run forward pass for BLIP-2.
@@ -648,32 +669,24 @@ class Blip2ForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP):
669 See also:
670 :class:`Blip2ImageInputs`
671 """
672 +
673 if intermediate_tensors is not None:
652 - input_ids = None
674 inputs_embeds = None
654 - else:
655 - image_input = self._parse_and_validate_image_input(**kwargs)
656 -
657 - if image_input is not None:
658 - vision_embeddings = self._process_image_input(image_input)
659 - inputs_embeds = self.language_model.model.get_input_embeddings(
660 - input_ids)
661 -
662 - inputs_embeds = merge_multimodal_embeddings(
663 - input_ids, inputs_embeds, vision_embeddings,
664 - BLIP2_IMAGE_TOKEN_ID)
665 -
666 - input_ids = None
667 - else:
668 - inputs_embeds = None
669 -
670 - hidden_states = self.language_model.model(
671 - input_ids,
672 - positions,
673 - kv_caches,
674 - attn_metadata,
675 - intermediate_tensors=intermediate_tensors,
676 - inputs_embeds=inputs_embeds)
675 +
676 + # NOTE: In v1, inputs_embeds is always generated at model runner, this
677 + # condition is for v0 compatibility.
678 + elif inputs_embeds is None:
679 + vision_embeddings = self.get_multimodal_embeddings(**kwargs)
680 + inputs_embeds = self.get_input_embeddings(input_ids,
681 + vision_embeddings)
682 + input_ids = None
683 +
684 + hidden_states = self.language_model.model(input_ids,
685 + positions,
686 + kv_caches,
687 + attn_metadata,
688 + intermediate_tensors,
689 + inputs_embeds=inputs_embeds)
690
691 return hidden_states
692
vllm/model_executor/models/chameleon.py
+41 -17
@@ -29,6 +29,7 @@ from vllm.model_executor.model_loader.weight_utils import (
29 from vllm.model_executor.sampling_metadata import SamplingMetadata
30 from vllm.model_executor.utils import set_weight_attrs
31 from vllm.multimodal import MULTIMODAL_REGISTRY
32 +from vllm.multimodal.inputs import NestedTensors
33 from vllm.multimodal.utils import (cached_get_tokenizer,
34 consecutive_placeholder_ranges,
35 repeat_and_pad_placeholder_tokens)
@@ -38,7 +39,7 @@ from vllm.utils import print_warning_once
39 from .interfaces import SupportsMultiModal, SupportsPP
40 from .utils import (is_pp_missing_parameter,
41 make_empty_intermediate_tensors_factory, make_layers,
41 - maybe_prefix)
42 + maybe_prefix, merge_multimodal_embeddings)
43
44 # These configs are not part of the model config but the preprocessor
45 # and processor files, so we hardcode them in the model file for now.
@@ -987,6 +988,29 @@ class ChameleonForConditionalGeneration(nn.Module, SupportsMultiModal,
988 data=self._validate_pixel_values(pixel_values),
989 )
990
991 + def get_multimodal_embeddings(self, **kwargs) -> Optional[NestedTensors]:
992 + image_input = self._parse_and_validate_image_input(**kwargs)
993 + if image_input is None:
994 + return None
995 + assert self.model.vqmodel is not None
996 + image_tokens = self.model.get_image_tokens(image_input["data"].to(
997 + self.config.torch_dtype))
998 + vision_embeddings = self.model.get_input_embeddings(image_tokens)
999 + return vision_embeddings
1000 +
1001 + def get_input_embeddings(
1002 + self,
1003 + input_ids: torch.Tensor,
1004 + multimodal_embeddings: Optional[NestedTensors] = None,
1005 + ) -> torch.Tensor:
1006 +
1007 + inputs_embeds = self.model.get_input_embeddings(input_ids)
1008 + if multimodal_embeddings is not None:
1009 + inputs_embeds = merge_multimodal_embeddings(
1010 + input_ids, inputs_embeds, multimodal_embeddings,
1011 + self.model.vocabulary_mapping.image_token_id)
1012 + return inputs_embeds
1013 +
1014 def forward(
1015 self,
1016 input_ids: torch.Tensor,
@@ -994,27 +1018,27 @@ class ChameleonForConditionalGeneration(nn.Module, SupportsMultiModal,
1018 kv_caches: List[torch.Tensor],
1019 attn_metadata: AttentionMetadata,
1020 intermediate_tensors: Optional[IntermediateTensors] = None,
1021 + inputs_embeds: Optional[torch.Tensor] = None,
1022 **kwargs,
1023 ) -> Union[torch.Tensor, IntermediateTensors]:
1024
1025 if intermediate_tensors is not None:
1026 + inputs_embeds = None
1027 +
1028 + # NOTE: In v1, inputs_embeds is always generated at model runner, this
1029 + # condition is for v0 compatibility.
1030 + elif inputs_embeds is None:
1031 + vision_embeddings = self.get_multimodal_embeddings(**kwargs)
1032 + inputs_embeds = self.get_input_embeddings(input_ids,
1033 + vision_embeddings)
1034 input_ids = None
1002 - else:
1003 - image_input = self._parse_and_validate_image_input(**kwargs)
1004 -
1005 - if image_input is not None:
1006 - assert self.model.vqmodel is not None
1007 - image_tokens = self.model.get_image_tokens(
1008 - image_input["data"].to(self.config.torch_dtype))
1009 - image_token_id = self.model.vocabulary_mapping.image_token_id
1010 - special_image_mask = input_ids == image_token_id
1011 - image_tokens = image_tokens.to(input_ids.device,
1012 - input_ids.dtype)
1013 - input_ids = input_ids.masked_scatter(special_image_mask,
1014 - image_tokens)
1015 -
1016 - hidden_states = self.model(input_ids, positions, kv_caches,
1017 - attn_metadata, intermediate_tensors)
1035 +
1036 + hidden_states = self.model(input_ids,
1037 + positions,
1038 + kv_caches,
1039 + attn_metadata,
1040 + intermediate_tensors,
1041 + inputs_embeds=inputs_embeds)
1042 return hidden_states
1043
1044 def compute_logits(
vllm/model_executor/models/chatglm.py
+35 -19
@@ -33,7 +33,8 @@ from vllm.model_executor.models.glm4_vision_encoder import EVA2CLIPModel
33 from vllm.model_executor.models.module_mapping import MultiModelKeys
34 from vllm.model_executor.sampling_metadata import SamplingMetadata
35 from vllm.multimodal import MULTIMODAL_REGISTRY
36 -from vllm.multimodal.inputs import MultiModalData, MultiModalKwargs
36 +from vllm.multimodal.inputs import (MultiModalData, MultiModalKwargs,
37 + NestedTensors)
38 from vllm.multimodal.utils import cached_get_tokenizer
39 from vllm.sequence import (VLLM_TOKEN_ID_ARRAY_TYPE, IntermediateTensors,
40 SequenceData)
@@ -545,6 +546,30 @@ class ChatGLMModel(nn.Module):
546 """)
547 return GLMImagePixelInputs(pixel_values=pixel_values)
548
549 + def get_multimodal_embeddings(self, **kwargs) -> Optional[NestedTensors]:
550 + image_input = self._parse_and_validate_image_input(**kwargs)
551 + if image_input["pixel_values"] is None:
552 + return None
553 + pixel_values = image_input["pixel_values"].to(
554 + dtype=self.config.torch_dtype)
555 + vision_embeddings = self.vision(pixel_values)
556 + return vision_embeddings
557 +
558 + def get_input_embeddings(
559 + self,
560 + input_ids: torch.Tensor,
561 + multimodal_embeddings: Optional[NestedTensors] = None,
562 + ) -> torch.Tensor:
563 + inputs_embeds = self.embedding(input_ids)
564 + if multimodal_embeddings is not None:
565 + inputs_embeds = merge_glm_vision_embeddings(
566 + input_ids=input_ids,
567 + inputs_embeds=inputs_embeds,
568 + vision_embeddings=multimodal_embeddings,
569 + boi_token_id=self.config.boi_token_id,
570 + eoi_token_id=self.config.eoi_token_id)
571 + return inputs_embeds
572 +
573 def forward(
574 self,
575 input_ids: torch.Tensor,
@@ -552,26 +577,17 @@ class ChatGLMModel(nn.Module):
577 kv_caches: List[torch.Tensor],
578 attn_metadata: AttentionMetadata,
579 intermediate_tensors: Optional[IntermediateTensors] = None,
580 + inputs_embeds: Optional[torch.Tensor] = None,
581 **kwargs: object,
582 ) -> torch.Tensor:
557 - if intermediate_tensors is None:
558 - inputs_embeds = self.embedding(input_ids)
559 - image_input = self._parse_and_validate_image_input(**kwargs)
560 -
561 - if image_input["pixel_values"] is not None:
562 - pixel_values = image_input["pixel_values"].to(
563 - dtype=inputs_embeds.dtype)
564 - image_embeds = self.vision(pixel_values)
565 -
566 - boi_token_id = self.config.boi_token_id
567 - eoi_token_id = self.config.eoi_token_id
568 -
569 - inputs_embeds = merge_glm_vision_embeddings(
570 - input_ids=input_ids,
571 - inputs_embeds=inputs_embeds,
572 - vision_embeddings=image_embeds,
573 - boi_token_id=boi_token_id,
574 - eoi_token_id=eoi_token_id)
583 +
584 + # NOTE: In v1, inputs_embeds is always generated at model runner, this
585 + # condition is for v0 compatibility.
586 + if intermediate_tensors is None and inputs_embeds is None:
587 + vision_embeddings = self.get_multimodal_embeddings(**kwargs)
588 + inputs_embeds = self.get_input_embeddings(input_ids,
589 + vision_embeddings)
590 + input_ids = None
591 else:
592 inputs_embeds = intermediate_tensors["hidden_states"]
593
vllm/model_executor/models/fuyu.py
+29 -14
@@ -35,6 +35,7 @@ from vllm.model_executor.models.persimmon import PersimmonForCausalLM
35 from vllm.model_executor.sampling_metadata import SamplingMetadata
36 from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalKwargs
37 from vllm.multimodal.image import cached_get_image_processor
38 +from vllm.multimodal.inputs import NestedTensors
39 from vllm.multimodal.utils import (cached_get_tokenizer,
40 consecutive_placeholder_ranges)
41 from vllm.sequence import (VLLM_TOKEN_ID_ARRAY_TYPE, IntermediateTensors,
@@ -302,6 +303,25 @@ class FuyuForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):
303 vision_embeddings, _ = self.vision_embed_tokens(image_input["data"])
304 return vision_embeddings
305
306 + def get_multimodal_embeddings(self, **kwargs) -> Optional[NestedTensors]:
307 + image_input = self._parse_and_validate_image_input(**kwargs)
308 + if image_input is None:
309 + return None
310 + vision_embeddings = self._process_image_input(image_input)
311 + return vision_embeddings
312 +
313 + def get_input_embeddings(
314 + self,
315 + input_ids: torch.Tensor,
316 + multimodal_embeddings: Optional[NestedTensors] = None,
317 + ) -> torch.Tensor:
318 + inputs_embeds = self.language_model.get_input_embeddings(input_ids)
319 + if multimodal_embeddings is not None:
320 + inputs_embeds = merge_multimodal_embeddings(
321 + input_ids, inputs_embeds, multimodal_embeddings,
322 + _IMAGE_TOKEN_ID)
323 + return inputs_embeds
324 +
325 def forward(
326 self,
327 input_ids: torch.Tensor,
@@ -309,24 +329,19 @@ class FuyuForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):
329 kv_caches: List[torch.Tensor],
330 attn_metadata: AttentionMetadata,
331 intermediate_tensors: Optional[IntermediateTensors] = None,
332 + inputs_embeds: Optional[torch.Tensor] = None,
333 **kwargs: object,
334 ):
335 if intermediate_tensors is not None:
315 - input_ids = None
336 inputs_embeds = None
317 - else:
318 - image_input = self._parse_and_validate_image_input(**kwargs)
319 -
320 - if image_input is not None:
321 - vision_embeddings = self._process_image_input(image_input)
322 - inputs_embeds = self.language_model.model.embed_tokens(
323 - input_ids)
324 - inputs_embeds = merge_multimodal_embeddings(
325 - input_ids, inputs_embeds, vision_embeddings,
326 - self.image_token_id)
327 -
328 - else:
329 - inputs_embeds = None
337 +
338 + # NOTE: In v1, inputs_embeds is always generated at model runner, this
339 + # condition is for v0 compatibility.
340 + elif inputs_embeds is None:
341 + vision_embeddings = self.get_multimodal_embeddings(**kwargs)
342 + inputs_embeds = self.get_input_embeddings(input_ids,
343 + vision_embeddings)
344 + input_ids = None
345
346 hidden_states = self.language_model(
347 input_ids=input_ids,
vllm/model_executor/models/interfaces.py
+35 -1
@@ -2,7 +2,7 @@ from typing import (TYPE_CHECKING, ClassVar, Dict, List, Literal, Optional,
2 Protocol, Type, Union, overload, runtime_checkable)
3
4 import torch
5 -from typing_extensions import TypeIs
5 +from typing_extensions import TypeIs, TypeVar
6
7 from vllm.logger import init_logger
8 from vllm.utils import supports_kw
@@ -10,10 +10,14 @@ from vllm.utils import supports_kw
10 from .interfaces_base import is_embedding_model
11
12 if TYPE_CHECKING:
13 + from vllm.attention import AttentionMetadata
14 + from vllm.multimodal.inputs import NestedTensors # noqa: F401
15 from vllm.sequence import IntermediateTensors
16
17 logger = init_logger(__name__)
18
19 +T = TypeVar("T", default="NestedTensors")
20 +
21
22 @runtime_checkable
23 class SupportsMultiModal(Protocol):
@@ -28,6 +32,36 @@ class SupportsMultiModal(Protocol):
32 MRO of your model class.
33 """
34
35 + def get_multimodal_embeddings(self, **kwargs) -> Optional[T]:
36 + """
37 + Returns multimodal embeddings generated from multimodal kwargs
38 + to be merged with text embeddings.
39 + """
40 + ...
41 +
42 + # Only for models that support v0 chunked prefill
43 + # TODO(ywang96): Remove this overload once v0 is deprecated
44 + @overload
45 + def get_input_embeddings(
46 + self,
47 + input_ids: torch.Tensor,
48 + multimodal_embeddings: Optional[T] = None,
49 + attn_metadata: Optional["AttentionMetadata"] = None,
50 + ) -> torch.Tensor:
51 + ...
52 +
53 + def get_input_embeddings(
54 + self,
55 + input_ids: torch.Tensor,
56 + multimodal_embeddings: Optional[T] = None,
57 + ) -> torch.Tensor:
58 + """
59 + Returns the input embeddings merged from the text embeddings from
60 + input_ids and the multimodal embeddings generated from multimodal
61 + kwargs.
62 + """
63 + ...
64 +
65
66 # We can't use runtime_checkable with ClassVar for issubclass checks
67 # so we need to treat the class as an instance and use isinstance instead
vllm/model_executor/models/internvl.py
+39 -15
@@ -26,6 +26,7 @@ from vllm.model_executor.models.intern_vit import (InternVisionModel,
26 InternVisionPatchModel)
27 from vllm.model_executor.sampling_metadata import SamplingMetadata
28 from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalKwargs
29 +from vllm.multimodal.inputs import NestedTensors
30 from vllm.multimodal.utils import cached_get_tokenizer
31 from vllm.sequence import IntermediateTensors
32 from vllm.utils import is_list_of
@@ -641,6 +642,26 @@ class InternVLChatModel(nn.Module, SupportsMultiModal, SupportsPP):
642 visual_token_mask = None
643 return visual_token_mask
644
645 + def get_multimodal_embeddings(self, **kwargs) -> Optional[NestedTensors]:
646 + image_input = self._parse_and_validate_image_input(**kwargs)
647 + if image_input is None:
648 + return None
649 + vision_embeddings = self._process_image_input(image_input)
650 + return vision_embeddings
651 +
652 + def get_input_embeddings(
653 + self,
654 + input_ids: torch.Tensor,
655 + multimodal_embeddings: Optional[NestedTensors] = None,
656 + ) -> torch.Tensor:
657 + inputs_embeds = self.language_model.get_input_embeddings(input_ids)
658 + if multimodal_embeddings is not None:
659 + assert self.img_context_token_id is not None
660 + inputs_embeds = merge_multimodal_embeddings(
661 + input_ids, inputs_embeds, multimodal_embeddings,
662 + self.img_context_token_id)
663 + return inputs_embeds
664 +
665 def forward(
666 self,
667 input_ids: torch.Tensor,
@@ -648,26 +669,22 @@ class InternVLChatModel(nn.Module, SupportsMultiModal, SupportsPP):
669 kv_caches: List[torch.Tensor],
670 attn_metadata: AttentionMetadata,
671 intermediate_tensors: Optional[IntermediateTensors] = None,
672 + inputs_embeds: Optional[torch.Tensor] = None,
673 **kwargs: object,
674 ) -> Union[SamplerOutput, IntermediateTensors]:
675 +
676 + visual_token_mask = None
677 if intermediate_tensors is not None:
678 input_ids = None
679 inputs_embeds = None
656 - visual_token_mask = None
657 - else:
658 - image_input = self._parse_and_validate_image_input(**kwargs)
659 - if image_input is not None:
660 - inputs_embeds = self.language_model.model.get_input_embeddings(
661 - input_ids)
662 - vision_embeddings = self._process_image_input(image_input)
663 - inputs_embeds = merge_multimodal_embeddings(
664 - input_ids, inputs_embeds, vision_embeddings,
665 - self.img_context_token_id)
666 - visual_token_mask = self._get_visual_token_mask(input_ids)
667 - input_ids = None
668 - else:
669 - inputs_embeds = None
670 - visual_token_mask = None
680 +
681 + # NOTE: In v1, inputs_embeds is always generated at model runner, this
682 + # condition is for v0 compatibility.
683 + elif inputs_embeds is None:
684 + vision_embeddings = self.get_multimodal_embeddings(**kwargs)
685 + inputs_embeds = self.get_input_embeddings(input_ids,
686 + vision_embeddings)
687 + input_ids = None
688
689 forward_kwargs = {
690 "input_ids": input_ids,
@@ -677,6 +694,13 @@ class InternVLChatModel(nn.Module, SupportsMultiModal, SupportsPP):
694 "intermediate_tensors": intermediate_tensors,
695 "inputs_embeds": inputs_embeds,
696 }
697 + if self.img_context_token_id is not None:
698 + visual_token_mask = self._get_visual_token_mask(input_ids)
699 +
700 + # We always overwrite it back to None after computing visual token
701 + # mask so that this doesn't need to depend on encoder output
702 + self.img_context_token_id = None
703 +
704 if self.is_mono:
705 forward_kwargs.update({"visual_token_mask": visual_token_mask})
706
vllm/model_executor/models/llava.py
+8 -7
@@ -478,7 +478,7 @@ class LlavaForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP):
478 image_features = self._process_image_pixels(image_input)
479 return self.multi_modal_projector(image_features)
480
481 - def process_mm_inputs(self, **kwargs):
481 + def get_multimodal_embeddings(self, **kwargs) -> Optional[NestedTensors]:
482 image_input = self._parse_and_validate_image_input(**kwargs)
483 if image_input is None:
484 return None
@@ -488,12 +488,12 @@ class LlavaForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP):
488 def get_input_embeddings(
489 self,
490 input_ids: torch.Tensor,
491 - vision_embeddings: Optional[NestedTensors] = None,
491 + multimodal_embeddings: Optional[NestedTensors] = None,
492 ) -> torch.Tensor:
493 inputs_embeds = self.language_model.get_input_embeddings(input_ids)
494 - if vision_embeddings is not None:
494 + if multimodal_embeddings is not None:
495 inputs_embeds = merge_multimodal_embeddings(
496 - input_ids, inputs_embeds, vision_embeddings,
496 + input_ids, inputs_embeds, multimodal_embeddings,
497 self.config.image_token_index)
498 return inputs_embeds
499
@@ -544,10 +544,11 @@ class LlavaForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP):
544 """
545 if intermediate_tensors is not None:
546 inputs_embeds = None
547 +
548 + # NOTE: In v1, inputs_embeds is always generated at model runner, this
549 + # condition is for v0 compatibility.
550 elif inputs_embeds is None:
548 - vision_embeddings = self.process_mm_inputs(**kwargs)
549 - # always pass the input via `inputs_embeds`
550 - # to make sure the computation graph is consistent
551 + vision_embeddings = self.get_multimodal_embeddings(**kwargs)
552 inputs_embeds = self.get_input_embeddings(input_ids,
553 vision_embeddings)
554 input_ids = None
vllm/model_executor/models/llava_next.py
+33 -18
@@ -19,6 +19,7 @@ from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
19 from vllm.model_executor.pooling_metadata import PoolingMetadata
20 from vllm.model_executor.sampling_metadata import SamplingMetadata
21 from vllm.multimodal import MULTIMODAL_REGISTRY
22 +from vllm.multimodal.inputs import NestedTensors
23 from vllm.sequence import IntermediateTensors, PoolerOutput
24 from vllm.utils import is_list_of
25
@@ -565,6 +566,30 @@ class LlavaNextForConditionalGeneration(nn.Module, SupportsMultiModal,
566 for i, patch_features_batch in enumerate(patch_embeddings)
567 ]
568
569 + def get_multimodal_embeddings(self, **kwargs) -> Optional[NestedTensors]:
570 + image_input = self._parse_and_validate_image_input(**kwargs)
571 + if image_input is None:
572 + return None
573 + vision_embeddings = self._process_image_input(image_input)
574 + return vision_embeddings
575 +
576 + def get_input_embeddings(
577 + self,
578 + input_ids: torch.Tensor,
579 + multimodal_embeddings: Optional[NestedTensors] = None,
580 + ) -> torch.Tensor:
581 +
582 + if multimodal_embeddings is None:
583 + return self.language_model.get_input_embeddings(input_ids)
584 +
585 + inputs_embeds = embed_multimodal(
586 + input_ids,
587 + self.config.image_token_index,
588 + self.language_model.model.get_input_embeddings,
589 + multimodal_embeddings,
590 + )
591 + return inputs_embeds
592 +
593 def forward(
594 self,
595 input_ids: torch.Tensor,
@@ -572,6 +597,7 @@ class LlavaNextForConditionalGeneration(nn.Module, SupportsMultiModal,
597 kv_caches: List[torch.Tensor],
598 attn_metadata: AttentionMetadata,
599 intermediate_tensors: Optional[IntermediateTensors] = None,
600 + inputs_embeds: Optional[torch.Tensor] = None,
601 **kwargs: object,
602 ) -> Union[torch.Tensor, IntermediateTensors]:
603 """Run forward pass for LlaVA-NeXT.
@@ -620,24 +646,14 @@ class LlavaNextForConditionalGeneration(nn.Module, SupportsMultiModal,
646 """
647 if intermediate_tensors is not None:
648 inputs_embeds = None
623 - else:
624 - image_input = self._parse_and_validate_image_input(**kwargs)
625 -
626 - if image_input is not None:
627 - inputs_embeds = embed_multimodal(
628 - input_ids,
629 - self.config.image_token_index,
630 - self.language_model.model.get_input_embeddings,
631 - lambda _: self._process_image_input(image_input),
632 - )
633 - else:
634 - inputs_embeds = self.language_model.model.get_input_embeddings(
635 - input_ids)
649
637 - # always pass the input via `inputs_embeds`
638 - # to make sure the computation graph is consistent
639 - # for `torch.compile` integration
640 - input_ids = None
650 + # NOTE: In v1, inputs_embeds is always generated at model runner, this
651 + # condition is for v0 compatibility.
652 + elif inputs_embeds is None:
653 + vision_embeddings = self.get_multimodal_embeddings(**kwargs)
654 + inputs_embeds = self.get_input_embeddings(input_ids,
655 + vision_embeddings)
656 + input_ids = None
657
658 hidden_states = self.language_model.model(input_ids,
659 positions,
@@ -645,7 +661,6 @@ class LlavaNextForConditionalGeneration(nn.Module, SupportsMultiModal,
661 attn_metadata,
662 intermediate_tensors,
663 inputs_embeds=inputs_embeds)
648 -
664 return hidden_states
665
666 def compute_logits(
vllm/model_executor/models/llava_next_video.py
+29 -15
@@ -18,6 +18,7 @@ from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
18 from vllm.model_executor.models.clip import CLIPVisionModel
19 from vllm.model_executor.sampling_metadata import SamplingMetadata
20 from vllm.multimodal import MULTIMODAL_REGISTRY
21 +from vllm.multimodal.inputs import NestedTensors
22 from vllm.multimodal.utils import (cached_get_tokenizer,
23 repeat_and_pad_placeholder_tokens)
24 from vllm.sequence import IntermediateTensors
@@ -388,6 +389,25 @@ class LlavaNextVideoForConditionalGeneration(nn.Module, SupportsMultiModal,
389 raise ValueError(
390 f"Unsupported type of video input {type(video_pixels)}")
391
392 + def get_multimodal_embeddings(self, **kwargs) -> Optional[NestedTensors]:
393 + video_input = self._parse_and_validate_video_input(**kwargs)
394 + if video_input is None:
395 + return None
396 + vision_embeddings = self._process_video_pixels(video_input)
397 + return vision_embeddings
398 +
399 + def get_input_embeddings(
400 + self,
401 + input_ids: torch.Tensor,
402 + multimodal_embeddings: Optional[NestedTensors] = None,
403 + ) -> torch.Tensor:
404 + inputs_embeds = self.language_model.get_input_embeddings(input_ids)
405 + if multimodal_embeddings is not None:
406 + inputs_embeds = merge_multimodal_embeddings(
407 + input_ids, inputs_embeds, multimodal_embeddings,
408 + self.config.video_token_index)
409 + return inputs_embeds
410 +
411 def forward(
412 self,
413 input_ids: torch.Tensor,
@@ -395,6 +415,7 @@ class LlavaNextVideoForConditionalGeneration(nn.Module, SupportsMultiModal,
415 kv_caches: List[torch.Tensor],
416 attn_metadata: AttentionMetadata,
417 intermediate_tensors: Optional[IntermediateTensors] = None,
418 + inputs_embeds: Optional[torch.Tensor] = None,
419 **kwargs: object,
420 ) -> Union[torch.Tensor, IntermediateTensors]:
421 """Run forward pass for LlaVA-NeXT-Video.
@@ -404,22 +425,15 @@ class LlavaNextVideoForConditionalGeneration(nn.Module, SupportsMultiModal,
425 pixel_values_videos: Pixels in each frames for each input videos.
426 """
427 if intermediate_tensors is not None:
407 - input_ids = None
428 inputs_embeds = None
409 - else:
410 - video_input = self._parse_and_validate_video_input(**kwargs)
411 - if video_input is not None:
412 - video_embeddings = self._process_video_pixels(video_input)
413 - inputs_embeds = self.language_model \
414 - .model.get_input_embeddings(input_ids)
415 -
416 - inputs_embeds = merge_multimodal_embeddings(
417 - input_ids, inputs_embeds, video_embeddings,
418 - self.config.video_token_index)
419 -
420 - input_ids = None
421 - else:
422 - inputs_embeds = None
429 +
430 + # NOTE: In v1, inputs_embeds is always generated at model runner, this
431 + # condition is for v0 compatibility.
432 + elif inputs_embeds is None:
433 + vision_embeddings = self.get_multimodal_embeddings(**kwargs)
434 + inputs_embeds = self.get_input_embeddings(input_ids,
435 + vision_embeddings)
436 + input_ids = None
437
438 hidden_states = self.language_model.model(input_ids,
439 positions,
vllm/model_executor/models/llava_onevision.py
+53 -21
@@ -21,6 +21,7 @@ from vllm.model_executor.layers.activation import get_act_fn
21 from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
22 from vllm.model_executor.sampling_metadata import SamplingMetadata
23 from vllm.multimodal import MULTIMODAL_REGISTRY
24 +from vllm.multimodal.inputs import NestedTensors
25 from vllm.multimodal.utils import (cached_get_tokenizer,
26 repeat_and_pad_placeholder_tokens)
27 from vllm.sequence import IntermediateTensors
@@ -824,6 +825,49 @@ class LlavaOnevisionForConditionalGeneration(nn.Module, SupportsMultiModal,
825 image_feature = image_feature.view(batch_frames, -1, dim)
826 return image_feature
827
828 + def get_multimodal_embeddings(
829 + self, **kwargs) -> Optional[List[Tuple[NestedTensors, str]]]:
830 + modalities = self._parse_and_validate_multimodal_inputs(**kwargs)
831 + if not modalities:
832 + return None
833 +
834 + # We make a tuple of each embedding with its modality string. This is a
835 + # temporary workaround for models to handle mixed modalities when
836 + # get_multimodal_embeddings and get_input_embeddings are called
837 + # separately.
838 + # TODO(ywang96): Add support for mixed-modality inference for v1.
839 + multimodal_embeddings: List[Tuple[NestedTensors, str]] = []
840 +
841 + if "images" in modalities:
842 + image_input = modalities["images"]
843 + vision_embeddings = self._process_image_input(image_input)
844 + multimodal_embeddings.append((vision_embeddings, "image"))
845 + if "videos" in modalities:
846 + video_input = modalities["videos"]
847 + video_embeddings = self._process_video_pixels(video_input)
848 + multimodal_embeddings.append((video_embeddings, "video"))
849 +
850 + return multimodal_embeddings
851 +
852 + def get_input_embeddings(
853 + self,
854 + input_ids: torch.Tensor,
855 + multimodal_embeddings: Optional[List[Tuple[NestedTensors,
856 + str]]] = None,
857 + ) -> torch.Tensor:
858 + inputs_embeds = self.language_model.get_input_embeddings(input_ids)
859 + if multimodal_embeddings is not None:
860 + for embeddings, modality in multimodal_embeddings:
861 + if modality == "image":
862 + inputs_embeds = merge_multimodal_embeddings(
863 + input_ids, inputs_embeds, embeddings,
864 + self.config.image_token_index)
865 + if modality == "video":
866 + inputs_embeds = merge_multimodal_embeddings(
867 + input_ids, inputs_embeds, embeddings,
868 + self.config.video_token_index)
869 + return inputs_embeds
870 +
871 def forward(
872 self,
873 input_ids: torch.Tensor,
@@ -831,6 +875,7 @@ class LlavaOnevisionForConditionalGeneration(nn.Module, SupportsMultiModal,
875 kv_caches: List[torch.Tensor],
876 attn_metadata: AttentionMetadata,
877 intermediate_tensors: Optional[IntermediateTensors] = None,
878 + inputs_embeds: Optional[torch.Tensor] = None,
879 **kwargs: object,
880 ) -> Union[torch.Tensor, IntermediateTensors]:
881 """Run forward pass for LlaVA-Onevision.
@@ -840,28 +885,15 @@ class LlavaOnevisionForConditionalGeneration(nn.Module, SupportsMultiModal,
885 pixel_values_videos: Pixels in each frames for each input videos.
886 """
887 if intermediate_tensors is not None:
843 - input_ids = None
888 inputs_embeds = None
845 - else:
846 - modalities = self._parse_and_validate_multimodal_inputs(**kwargs)
847 - if modalities:
848 - inputs_embeds = self.language_model.model.get_input_embeddings(
849 - input_ids)
850 - if "images" in modalities:
851 - image_input = modalities["images"]
852 - vision_embeddings = self._process_image_input(image_input)
853 - inputs_embeds = merge_multimodal_embeddings(
854 - input_ids, inputs_embeds, vision_embeddings,
855 - self.config.image_token_index)
856 - if "videos" in modalities:
857 - video_input = modalities["videos"]
858 - video_embeddings = self._process_video_pixels(video_input)
859 - inputs_embeds = merge_multimodal_embeddings(
860 - input_ids, inputs_embeds, video_embeddings,
861 - self.config.video_token_index)
862 - input_ids = None
863 - else:
864 - inputs_embeds = None
889 +
890 + # NOTE: In v1, inputs_embeds is always generated at model runner, this
891 + # condition is for v0 compatibility.
892 + elif inputs_embeds is None:
893 + multimodal_embeddings = self.get_multimodal_embeddings(**kwargs)
894 + inputs_embeds = self.get_input_embeddings(input_ids,
895 + multimodal_embeddings)
896 + input_ids = None
897
898 hidden_states = self.language_model.model(input_ids,
899 positions,
vllm/model_executor/models/molmo.py
+46 -42
@@ -3,7 +3,7 @@ import re
3 from array import array
4 from dataclasses import dataclass
5 from functools import lru_cache, partial
6 -from typing import Iterable, List, Mapping, Optional, Tuple, TypedDict, Union
6 +from typing import Iterable, List, Mapping, Optional, Tuple, TypedDict
7
8 import torch
9 from einops import rearrange
@@ -36,6 +36,7 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
36 ParallelLMHead, VocabParallelEmbedding)
37 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
38 from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalKwargs
39 +from vllm.multimodal.inputs import NestedTensors
40 from vllm.multimodal.utils import cached_get_tokenizer
41 from vllm.platforms import _Backend
42 from vllm.sequence import (VLLM_TOKEN_ID_ARRAY_TYPE, IntermediateTensors,
@@ -756,6 +757,12 @@ class MolmoModel(nn.Module):
757 make_empty_intermediate_tensors_factory(
758 ["hidden_states", "residual"], config.hidden_size))
759
760 + def get_input_embeddings(
761 + self,
762 + input_ids: torch.Tensor,
763 + ) -> torch.Tensor:
764 + return self.embed_tokens(input_ids)
765 +
766 def forward(
767 self,
768 input_ids: torch.Tensor,
@@ -1098,19 +1105,16 @@ class MolmoForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):
1105
1106 return image_features
1107
1101 - def _merge_multimodal_embeddings(
1102 - self,
1103 - inputs_embeds: torch.Tensor,
1104 - image_features: torch.Tensor,
1105 - image_input_idx: torch.Tensor,
1106 - seq_len: Union[torch.Tensor, List[torch.Tensor]],
1107 - ) -> torch.Tensor:
1108 + def get_multimodal_embeddings(self, **kwargs) -> Optional[NestedTensors]:
1109 + image_input = self._parse_and_validate_image_input(**kwargs)
1110 + if image_input is None:
1111 + return None
1112 + image_features = self._process_image_input(image_input)
1113 + image_input_idx = image_input["image_input_idx"]
1114 + seq_len = image_input["seq_len"]
1115 batch_size, num_image, num_patch = image_features.shape[:3]
1116 assert image_input_idx.shape == (batch_size, num_image, num_patch)
1117
1111 - image_features = image_features.to(inputs_embeds.device)
1112 - seq_len = seq_len.to(inputs_embeds.device)
1113 -
1118 # insert the image feature into the embedding.
1119 image_features = image_features.view(batch_size, num_image * num_patch,
1120 -1)
@@ -1130,12 +1134,24 @@ class MolmoForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):
1134 image_input_idx = image_input_idx + offset.to(image_input_idx.dtype)
1135 image_input_idx = image_input_idx.flatten()[:, None]
1136 mat = image_input_idx == torch.arange(
1133 - seq_len.sum().item(), device=inputs_embeds.device)[None, :]
1137 + seq_len.sum().item(), device=image_features.device)[None, :]
1138 mat = mat.to(image_features.dtype)
1139
1136 - inputs_embeds = inputs_embeds + torch.einsum('nd,nm->md',
1137 - image_features, mat)
1140 + # Note: In this original implementation from AI2, the final
1141 + # vision_embeddings will be always be the same length
1142 + # of input embedddings, which is not very efficient.
1143 + # TODO(ywang96): see if this can be optimized.
1144 + vision_embeddings = torch.einsum('nd,nm->md', image_features, mat)
1145 + return vision_embeddings
1146
1147 + def get_input_embeddings(
1148 + self,
1149 + input_ids: torch.Tensor,
1150 + multimodal_embeddings: Optional[NestedTensors] = None,
1151 + ) -> torch.Tensor:
1152 + inputs_embeds = self.model.get_input_embeddings(input_ids)
1153 + if multimodal_embeddings is not None:
1154 + inputs_embeds = inputs_embeds + multimodal_embeddings
1155 return inputs_embeds
1156
1157 def forward(
@@ -1145,39 +1161,27 @@ class MolmoForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):
1161 kv_caches: List[torch.Tensor],
1162 attn_metadata: AttentionMetadata,
1163 intermediate_tensors: Optional[IntermediateTensors] = None,
1164 + inputs_embeds: Optional[torch.Tensor] = None,
1165 **kwargs: object,
1166 ) -> SamplerOutput:
1167 +
1168 if intermediate_tensors is not None:
1169 inputs_embeds = None
1152 - else:
1153 - image_input = self._parse_and_validate_image_input(**kwargs)
1154 -
1155 - if image_input is not None:
1156 - inputs_embeds = self.model.embed_tokens(input_ids)
1157 - image_features = self._process_image_input(image_input)
1158 -
1159 - inputs_embeds = self._merge_multimodal_embeddings(
1160 - inputs_embeds,
1161 - image_features,
1162 - image_input["image_input_idx"],
1163 - image_input["seq_len"],
1164 - )
1165 - else:
1166 - inputs_embeds = self.model.embed_tokens(input_ids)
1170
1168 - # always pass the input via `inputs_embeds`
1169 - # to make sure the computation graph is consistent
1170 - # for `torch.compile` integration
1171 - input_ids = None
1172 -
1173 - hidden_states = self.model(
1174 - input_ids=input_ids,
1175 - positions=positions,
1176 - kv_caches=kv_caches,
1177 - attn_metadata=attn_metadata,
1178 - intermediate_tensors=intermediate_tensors,
1179 - inputs_embeds=inputs_embeds,
1180 - )
1171 + # NOTE: In v1, inputs_embeds is always generated at model runner, this
1172 + # condition is for v0 compatibility.
1173 + elif inputs_embeds is None:
1174 + vision_embeddings = self.get_multimodal_embeddings(**kwargs)
1175 + inputs_embeds = self.get_input_embeddings(input_ids,
1176 + vision_embeddings)
1177 + input_ids = None
1178 +
1179 + hidden_states = self.model(input_ids,
1180 + positions,
1181 + kv_caches,
1182 + attn_metadata,
1183 + intermediate_tensors,
1184 + inputs_embeds=inputs_embeds)
1185
1186 return hidden_states
1187
vllm/model_executor/models/paligemma.py
+31 -21
@@ -13,6 +13,7 @@ from vllm.logger import init_logger
13 from vllm.model_executor.layers.sampler import SamplerOutput
14 from vllm.model_executor.sampling_metadata import SamplingMetadata
15 from vllm.multimodal import MULTIMODAL_REGISTRY
16 +from vllm.multimodal.inputs import NestedTensors
17 from vllm.multimodal.utils import cached_get_tokenizer
18 from vllm.sequence import IntermediateTensors
19
@@ -240,36 +241,45 @@ class PaliGemmaForConditionalGeneration(nn.Module, SupportsMultiModal,
241
242 return self.multi_modal_projector(image_features)
243
244 + def get_multimodal_embeddings(self, **kwargs) -> Optional[NestedTensors]:
245 + image_input = self._parse_and_validate_image_input(**kwargs)
246 + if image_input is None:
247 + return None
248 + vision_embeddings = self._process_image_input(image_input)
249 + # https://github.com/huggingface/transformers/blob/main/src/transformers/models/paligemma/modeling_paligemma.py#L294 # noqa
250 + vision_embeddings = vision_embeddings * (self.config.hidden_size**-0.5)
251 + return vision_embeddings
252 +
253 + def get_input_embeddings(
254 + self,
255 + input_ids: torch.Tensor,
256 + multimodal_embeddings: Optional[NestedTensors] = None,
257 + ) -> torch.Tensor:
258 + inputs_embeds = self.language_model.get_input_embeddings(input_ids)
259 + if multimodal_embeddings is not None:
260 + inputs_embeds = merge_multimodal_embeddings(
261 + input_ids, inputs_embeds, multimodal_embeddings,
262 + self.config.image_token_index)
263 + return inputs_embeds
264 +
265 def forward(self,
266 input_ids: torch.Tensor,
267 positions: torch.Tensor,
268 kv_caches: List[torch.Tensor],
269 attn_metadata: AttentionMetadata,
270 intermediate_tensors: Optional[IntermediateTensors] = None,
271 + inputs_embeds: Optional[torch.Tensor] = None,
272 **kwargs: object) -> Union[SamplerOutput, IntermediateTensors]:
273 if intermediate_tensors is not None:
251 - input_ids = None
274 inputs_embeds = None
253 - else:
254 - parsed_image_input = self._parse_and_validate_image_input(**kwargs)
255 -
256 - if parsed_image_input is not None:
257 - vision_embeddings = self._process_image_input(
258 - parsed_image_input)
259 - # https://github.com/huggingface/transformers/blob/main/src/transformers/models/paligemma/modeling_paligemma.py#L294 # noqa
260 - vision_embeddings = vision_embeddings * (
261 - self.config.hidden_size**-0.5)
262 -
263 - inputs_embeds = self.language_model.model.get_input_embeddings(
264 - input_ids)
265 -
266 - inputs_embeds = merge_multimodal_embeddings(
267 - input_ids, inputs_embeds, vision_embeddings,
268 - self.config.image_token_index)
269 -
270 - input_ids = None
271 - else:
272 - inputs_embeds = None
275 +
276 + # NOTE: In v1, inputs_embeds is always generated at model runner, this
277 + # condition is for v0 compatibility.
278 + elif inputs_embeds is None:
279 + vision_embeddings = self.get_multimodal_embeddings(**kwargs)
280 + inputs_embeds = self.get_input_embeddings(input_ids,
281 + vision_embeddings)
282 + input_ids = None
283
284 hidden_states = self.language_model.model(input_ids,
285 positions,
vllm/model_executor/models/phi3v.py
+9 -7
@@ -676,7 +676,7 @@ class Phi3VForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):
676
677 return image_embeds
678
679 - def process_mm_inputs(self, **kwargs):
679 + def get_multimodal_embeddings(self, **kwargs) -> Optional[NestedTensors]:
680 image_input = self._parse_and_validate_image_input(**kwargs)
681 if image_input is None:
682 return None
@@ -686,12 +686,12 @@ class Phi3VForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):
686 def get_input_embeddings(
687 self,
688 input_ids: torch.Tensor,
689 - vision_embeddings: Optional[NestedTensors] = None,
689 + multimodal_embeddings: Optional[NestedTensors] = None,
690 ) -> torch.Tensor:
691 inputs_embeds = self.embed_tokens(input_ids)
692 - if vision_embeddings is not None:
692 + if multimodal_embeddings is not None:
693 inputs_embeds = merge_multimodal_embeddings(
694 - input_ids, inputs_embeds, vision_embeddings,
694 + input_ids, inputs_embeds, multimodal_embeddings,
695 self.image_token_id)
696 return inputs_embeds
697
@@ -703,12 +703,14 @@ class Phi3VForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):
703 intermediate_tensors: Optional[IntermediateTensors] = None,
704 inputs_embeds: Optional[torch.Tensor] = None,
705 **kwargs: object):
706 +
707 if intermediate_tensors is not None:
708 inputs_embeds = None
709 +
710 + # NOTE: In v1, inputs_embeds is always generated at model runner, this
711 + # condition is for v0 compatibility
712 elif inputs_embeds is None:
709 - vision_embeddings = self.process_mm_inputs(**kwargs)
710 - # always pass the input via `inputs_embeds`
711 - # to make sure the computation graph is consistent
713 + vision_embeddings = self.get_multimodal_embeddings(**kwargs)
714 inputs_embeds = self.get_input_embeddings(input_ids,
715 vision_embeddings)
716 input_ids = None
vllm/model_executor/models/qwen2_audio.py
+37 -22
@@ -42,10 +42,12 @@ from vllm.model_executor.model_loader.weight_utils import (
42 from vllm.model_executor.models.qwen2 import Qwen2Model
43 from vllm.model_executor.sampling_metadata import SamplingMetadata
44 from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalKwargs
45 +from vllm.multimodal.inputs import NestedTensors
46 from vllm.multimodal.utils import consecutive_placeholder_ranges
47 from vllm.sequence import IntermediateTensors, SequenceData
48
49 from .interfaces import SupportsMultiModal, SupportsPP
50 +from .utils import merge_multimodal_embeddings
51
52 logger = init_logger(__name__)
53
@@ -371,6 +373,25 @@ class Qwen2AudioForConditionalGeneration(nn.Module, SupportsMultiModal,
373
374 return masked_audio_features
375
376 + def get_multimodal_embeddings(self, **kwargs) -> Optional[NestedTensors]:
377 + audio_input = self._parse_and_validate_audio_input(**kwargs)
378 + if audio_input is None:
379 + return None
380 + masked_audio_features = self._process_audio_input(audio_input)
381 + return masked_audio_features
382 +
383 + def get_input_embeddings(
384 + self,
385 + input_ids: torch.Tensor,
386 + multimodal_embeddings: Optional[NestedTensors] = None,
387 + ) -> torch.Tensor:
388 + inputs_embeds = self.language_model.get_input_embeddings(input_ids)
389 + if multimodal_embeddings is not None:
390 + inputs_embeds = merge_multimodal_embeddings(
391 + input_ids, inputs_embeds, multimodal_embeddings,
392 + self.config.audio_token_index)
393 + return inputs_embeds
394 +
395 def forward(
396 self,
397 input_ids: torch.Tensor,
@@ -378,33 +399,27 @@ class Qwen2AudioForConditionalGeneration(nn.Module, SupportsMultiModal,
399 kv_caches: List[torch.Tensor],
400 attn_metadata: AttentionMetadata,
401 intermediate_tensors: Optional[IntermediateTensors] = None,
402 + inputs_embeds: Optional[torch.Tensor] = None,
403 **kwargs: object,
404 ) -> Union[torch.Tensor, IntermediateTensors]:
405 +
406 if intermediate_tensors is not None:
384 - input_ids = None
407 inputs_embeds = None
386 - else:
387 - audio_input = self._parse_and_validate_audio_input(**kwargs)
408
389 - if audio_input is None:
390 - inputs_embeds = None
391 - else:
392 - inputs_embeds = self.language_model.embed_tokens(input_ids)
393 - masked_audio_features = self._process_audio_input(audio_input)
394 - # merge llm embeddings and audio features
395 - mask = (input_ids == self.config.audio_token_index)
396 - inputs_embeds[mask, :] = masked_audio_features
397 -
398 - input_ids = None
399 -
400 - hidden_states = self.language_model(
401 - input_ids=input_ids,
402 - positions=positions,
403 - kv_caches=kv_caches,
404 - attn_metadata=attn_metadata,
405 - intermediate_tensors=intermediate_tensors,
406 - inputs_embeds=inputs_embeds,
407 - )
409 + # NOTE: In v1, inputs_embeds is always generated at model runner, this
410 + # condition is for v0 compatibility.
411 + elif inputs_embeds is None:
412 + multimodal_embeddings = self.get_multimodal_embeddings(**kwargs)
413 + inputs_embeds = self.get_input_embeddings(input_ids,
414 + multimodal_embeddings)
415 + input_ids = None
416 +
417 + hidden_states = self.language_model(input_ids,
418 + positions,
419 + kv_caches,
420 + attn_metadata,
421 + intermediate_tensors,
422 + inputs_embeds=inputs_embeds)
423 return hidden_states
424
425 def compute_logits(self, hidden_states: torch.Tensor,
vllm/model_executor/models/qwen2_vl.py
+68 -34
@@ -63,7 +63,7 @@ from vllm.model_executor.pooling_metadata import PoolingMetadata
63 from vllm.multimodal import MULTIMODAL_REGISTRY
64 from vllm.multimodal.image import cached_get_image_processor
65 from vllm.multimodal.inputs import (MultiModalData, MultiModalDataDict,
66 - MultiModalKwargs)
66 + MultiModalKwargs, NestedTensors)
67 from vllm.multimodal.utils import cached_get_tokenizer
68 from vllm.platforms import _Backend
69 from vllm.sequence import IntermediateTensors, PoolerOutput, SequenceData
@@ -1238,6 +1238,55 @@ class Qwen2VLForConditionalGeneration(nn.Module, SupportsMultiModal,
1238 inputs_embeds[mask, :] = multimodal_embeddings
1239 return inputs_embeds
1240
1241 + def get_multimodal_embeddings(
1242 + self, **kwargs) -> Optional[List[Tuple[NestedTensors, str]]]:
1243 +
1244 + image_input = self._parse_and_validate_image_input(**kwargs)
1245 + video_input = self._parse_and_validate_video_input(**kwargs)
1246 + if image_input is None and video_input is None:
1247 + return None
1248 +
1249 + # We make a tuple of each embedding with its modality string. This is a
1250 + # temporary workaround for models to handle mixed modalities when
1251 + # get_multimodal_embeddings and get_input_embeddings are called
1252 + # separately.
1253 + # TODO(ywang96): Add support for mixed-modality inference for v1.
1254 + multimodal_embeddings: List[Tuple[NestedTensors, str]] = []
1255 +
1256 + if image_input is not None:
1257 + image_embeds = self._process_image_input(image_input)
1258 + multimodal_embeddings.append((image_embeds, "image"))
1259 + if video_input is not None:
1260 + video_embeds = self._process_video_input(video_input)
1261 + multimodal_embeddings.append((video_embeds, "video"))
1262 +
1263 + return multimodal_embeddings
1264 +
1265 + def get_input_embeddings(
1266 + self,
1267 + input_ids: torch.Tensor,
1268 + multimodal_embeddings: Optional[List[Tuple[NestedTensors,
1269 + str]]] = None,
1270 + ) -> torch.Tensor:
1271 + inputs_embeds = self.model.get_input_embeddings(input_ids)
1272 + if multimodal_embeddings is not None:
1273 + for embeddings, modality in multimodal_embeddings:
1274 + if modality == "image":
1275 + inputs_embeds = self._merge_multimodal_embeddings(
1276 + input_ids,
1277 + inputs_embeds,
1278 + embeddings,
1279 + placeholder_token_id=self.config.image_token_id,
1280 + )
1281 + if modality == "video":
1282 + inputs_embeds = self._merge_multimodal_embeddings(
1283 + input_ids,
1284 + inputs_embeds,
1285 + embeddings,
1286 + placeholder_token_id=self.config.video_token_id,
1287 + )
1288 + return inputs_embeds
1289 +
1290 def forward(
1291 self,
1292 input_ids: torch.Tensor,
@@ -1245,6 +1294,7 @@ class Qwen2VLForConditionalGeneration(nn.Module, SupportsMultiModal,
1294 kv_caches: List[torch.Tensor],
1295 attn_metadata: AttentionMetadata,
1296 intermediate_tensors: Optional[IntermediateTensors] = None,
1297 + inputs_embeds: Optional[torch.Tensor] = None,
1298 **kwargs: object,
1299 ) -> Union[torch.Tensor, IntermediateTensors]:
1300 """Run forward pass for Qwen2-VL.
@@ -1266,42 +1316,26 @@ class Qwen2VLForConditionalGeneration(nn.Module, SupportsMultiModal,
1316 video_grid_thw: Tensor `(n_videos, 3)` of video 3D grid in LLM.
1317 `None` if no videos are passed.
1318 """
1319 +
1320 if intermediate_tensors is not None:
1270 - input_ids = None
1321 inputs_embeds = None
1272 - else:
1273 - image_input = self._parse_and_validate_image_input(**kwargs)
1274 - video_input = self._parse_and_validate_video_input(**kwargs)
1275 -
1276 - if image_input is None and video_input is None:
1277 - inputs_embeds = None
1278 - else:
1279 - if uses_mrope(self.config):
1280 - assert positions.ndim == 2 and positions.size(0) == 3, (
1281 - "multimodal section rotary embedding requires "
1282 - f"(3, seq_len) positions, but got {positions.size()}")
1283 -
1284 - inputs_embeds = self.model.embed_tokens(input_ids)
1285 -
1286 - if image_input is not None:
1287 - image_embeds = self._process_image_input(image_input)
1288 - inputs_embeds = self._merge_multimodal_embeddings(
1289 - input_ids,
1290 - inputs_embeds,
1291 - image_embeds,
1292 - placeholder_token_id=self.config.image_token_id,
1293 - )
1294 -
1295 - if video_input is not None:
1296 - video_embeds = self._process_video_input(video_input)
1297 - inputs_embeds = self._merge_multimodal_embeddings(
1298 - input_ids,
1299 - inputs_embeds,
1300 - video_embeds,
1301 - placeholder_token_id=self.config.video_token_id,
1302 - )
1322
1304 - input_ids = None
1323 + # NOTE: In v1, inputs_embeds is always generated at model runner, this
1324 + # condition is for v0 compatibility.
1325 + elif inputs_embeds is None:
1326 + multimodal_embeddings = self.get_multimodal_embeddings(**kwargs)
1327 +
1328 + # We need to check for usage of mrope here in case there is
1329 + # multimodal data.
1330 + # TODO (ywang96): move this to model runner in V1.
1331 + if multimodal_embeddings is not None and uses_mrope(self.config):
1332 + assert positions.ndim == 2 and positions.size(0) == 3, (
1333 + "multimodal section rotary embedding requires "
1334 + f"(3, seq_len) positions, but got {positions.size()}")
1335 +
1336 + inputs_embeds = self.get_input_embeddings(input_ids,
1337 + multimodal_embeddings)
1338 + input_ids = None
1339
1340 hidden_states = self.model(
1341 input_ids=input_ids,
vllm/model_executor/models/ultravox.py
+48 -24
@@ -449,10 +449,36 @@ class UltravoxModel(nn.Module, SupportsMultiModal, SupportsPP):
449
450 return result
451
452 - def forward(self, input_ids: torch.Tensor, positions: torch.Tensor,
452 + def get_multimodal_embeddings(self, **kwargs) -> Optional[NestedTensors]:
453 + audio_input = self._parse_and_validate_audio_input(**kwargs)
454 + if audio_input is None:
455 + return None
456 + audio_embeddings = self._process_audio_input(audio_input)
457 + return audio_embeddings
458 +
459 + def get_input_embeddings(
460 + self,
461 + input_ids: torch.Tensor,
462 + multimodal_embeddings: Optional[NestedTensors] = None,
463 + attn_metadata: Optional[AttentionMetadata] = None,
464 + ) -> torch.Tensor:
465 + inputs_embeds = self.language_model.get_input_embeddings(input_ids)
466 + if multimodal_embeddings is not None:
467 +
468 + # TODO(ywang96): use merge_multimodal_embeddings after
469 + # v0 is deprecated
470 + merge_multimodal_embeddings_from_map(
471 + inputs_embeds, multimodal_embeddings,
472 + attn_metadata.multi_modal_placeholder_index_maps["audio"])
473 + return inputs_embeds
474 +
475 + def forward(self,
476 + input_ids: torch.Tensor,
477 + positions: torch.Tensor,
478 kv_caches: List[torch.Tensor],
479 attn_metadata: AttentionMetadata,
455 - intermediate_tensors: Optional[torch.Tensor],
480 + intermediate_tensors: Optional[torch.Tensor] = None,
481 + inputs_embeds: Optional[torch.Tensor] = None,
482 **kwargs) -> Union[torch.Tensor, IntermediateTensors]:
483 """Run forward pass for Ultravox
484
@@ -466,30 +492,28 @@ class UltravoxModel(nn.Module, SupportsMultiModal, SupportsPP):
492 Args:
493 audio_features: A batch of audio inputs [B, N, 80, M].
494 """
495 +
496 if intermediate_tensors is not None:
470 - input_ids = None
497 inputs_embeds = None
472 - else:
473 - audio_input = self._parse_and_validate_audio_input(**kwargs)
474 - if audio_input is not None:
475 - audio_embeddings = self._process_audio_input(audio_input)
476 - inputs_embeds = self.language_model.model.get_input_embeddings(
477 - input_ids)
478 -
479 - merge_multimodal_embeddings_from_map(
480 - inputs_embeds, audio_embeddings,
481 - attn_metadata.multi_modal_placeholder_index_maps["audio"])
482 - input_ids = None
483 - else:
484 - inputs_embeds = None
485 -
486 - hidden_states = self.language_model.model(
487 - input_ids=input_ids,
488 - positions=positions,
489 - kv_caches=kv_caches,
490 - attn_metadata=attn_metadata,
491 - intermediate_tensors=intermediate_tensors,
492 - inputs_embeds=inputs_embeds)
498 +
499 + # NOTE: In v1, inputs_embeds is always generated at model runner, this
500 + # condition is for v0 compatibility.
501 + elif inputs_embeds is None:
502 + multimodal_embeddings = self.get_multimodal_embeddings(**kwargs)
503 +
504 + # TODO(ywang96): remove attn_metadata from get_input_embeddings
505 + # after v0 is deprecated
506 + inputs_embeds = self.get_input_embeddings(input_ids,
507 + multimodal_embeddings,
508 + attn_metadata)
509 + input_ids = None
510 +
511 + hidden_states = self.language_model.model(input_ids,
512 + positions,
513 + kv_caches,
514 + attn_metadata,
515 + intermediate_tensors,
516 + inputs_embeds=inputs_embeds)
517 return hidden_states
518
519 def compute_logits(self, hidden_states: torch.Tensor,
vllm/model_executor/models/utils.py
+1 -4
@@ -356,8 +356,7 @@ def embed_multimodal(
356 input_ids: torch.Tensor,
357 multimodal_token_id: int,
358 get_text_embeds: Callable[[torch.Tensor], torch.Tensor],
359 - get_multimodal_embeds: Callable[[torch.Tensor], Union[torch.Tensor,
360 - List[torch.Tensor]]],
359 + multimodal_embeds: Union[torch.Tensor, List[torch.Tensor]],
360 ) -> torch.Tensor:
361 """
362 Embed token IDs and multimodal inputs and combine their embeddings.
@@ -374,8 +373,6 @@ def embed_multimodal(
373 is_text = ~is_multimodal
374
375 text_embeds = get_text_embeds(input_ids[is_text])
377 - multimodal_embeds = get_multimodal_embeds(input_ids[is_multimodal])
378 -
376 merged_embeds = torch.empty(
377 (input_ids.shape[0], text_embeds.shape[1]),
378 dtype=text_embeds.dtype,
vllm/v1/worker/gpu_model_runner.py
+2 -1
@@ -363,7 +363,8 @@ class GPUModelRunner:
363 # 2. A list (length: num_images) of tensors, each of shape
364 # [feature_size, hidden_size] in case when the feature size is
365 # dynamic depending on input images.
366 - encoder_outputs = self.model.process_mm_inputs(**batched_mm_inputs)
366 + encoder_outputs = self.model.get_multimodal_embeddings(
367 + **batched_mm_inputs)
368
369 # Cache the encoder outputs.
370 for (req_id, input_id), output in zip(req_input_ids, encoder_outputs):