A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 05:45 UTC
[V1] Refactor model executable interface for multimodal models (#10570)
Signed-off-by: Roger Wang <ywang@roblox.com>
Roger Wang committed
Nov 26, 2024 at 12:46 UTC
2f0a0a17a47436fe9709462dfee3bb9d2f91e0a0
18 files changed
+581
-306
vllm/model_executor/models/blip2.py
+37
-24
@@ -16,6 +16,7 @@ from vllm.model_executor.layers.quantization import QuantizationConfig
16
from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
17
from vllm.model_executor.sampling_metadata import SamplingMetadata
18
from vllm.multimodal import MULTIMODAL_REGISTRY
19
+from vllm.multimodal.inputs import NestedTensors
20
from vllm.multimodal.utils import consecutive_placeholder_ranges
21
from vllm.sequence import IntermediateTensors, SequenceData
22
@@ -609,6 +610,25 @@ class Blip2ForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP):
610
611
return self.language_projection(query_output)
612
613
+ def get_multimodal_embeddings(self, **kwargs) -> Optional[NestedTensors]:
614
+ image_input = self._parse_and_validate_image_input(**kwargs)
615
+ if image_input is None:
616
+ return None
617
+ vision_embeddings = self._process_image_input(image_input)
618
+ return vision_embeddings
619
+
620
+ def get_input_embeddings(
621
+ self,
622
+ input_ids: torch.Tensor,
623
+ multimodal_embeddings: Optional[NestedTensors] = None,
624
+ ) -> torch.Tensor:
625
+ inputs_embeds = self.language_model.get_input_embeddings(input_ids)
626
+ if multimodal_embeddings is not None:
627
+ inputs_embeds = merge_multimodal_embeddings(
628
+ input_ids, inputs_embeds, multimodal_embeddings,
629
+ BLIP2_IMAGE_TOKEN_ID)
630
+ return inputs_embeds
631
+
632
def forward(
633
self,
634
input_ids: torch.Tensor,
@@ -616,6 +636,7 @@ class Blip2ForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP):
636
kv_caches: List[torch.Tensor],
637
attn_metadata: AttentionMetadata,
638
intermediate_tensors: Optional[IntermediateTensors] = None,
639
+ inputs_embeds: Optional[torch.Tensor] = None,
640
**kwargs: object,
641
) -> Union[SamplerOutput, IntermediateTensors]:
642
"""Run forward pass for BLIP-2.
@@ -648,32 +669,24 @@ class Blip2ForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP):
669
See also:
670
:class:`Blip2ImageInputs`
671
"""
672
+
673
if intermediate_tensors is not None:
652
- input_ids = None
674
inputs_embeds = None
654
- else:
655
- image_input = self._parse_and_validate_image_input(**kwargs)
656
-
657
- if image_input is not None:
658
- vision_embeddings = self._process_image_input(image_input)
659
- inputs_embeds = self.language_model.model.get_input_embeddings(
660
- input_ids)
661
-
662
- inputs_embeds = merge_multimodal_embeddings(
663
- input_ids, inputs_embeds, vision_embeddings,
664
- BLIP2_IMAGE_TOKEN_ID)
665
-
666
- input_ids = None
667
- else:
668
- inputs_embeds = None
669
-
670
- hidden_states = self.language_model.model(
671
- input_ids,
672
- positions,
673
- kv_caches,
674
- attn_metadata,
675
- intermediate_tensors=intermediate_tensors,
676
- inputs_embeds=inputs_embeds)
675
+
676
+ # NOTE: In v1, inputs_embeds is always generated at model runner, this
677
+ # condition is for v0 compatibility.
678
+ elif inputs_embeds is None:
679
+ vision_embeddings = self.get_multimodal_embeddings(**kwargs)
680
+ inputs_embeds = self.get_input_embeddings(input_ids,
681
+ vision_embeddings)
682
+ input_ids = None
683
+
684
+ hidden_states = self.language_model.model(input_ids,
685
+ positions,
686
+ kv_caches,
687
+ attn_metadata,
688
+ intermediate_tensors,
689
+ inputs_embeds=inputs_embeds)
690
691
return hidden_states
692
vllm/model_executor/models/chameleon.py
+41
-17
@@ -29,6 +29,7 @@ from vllm.model_executor.model_loader.weight_utils import (
29
from vllm.model_executor.sampling_metadata import SamplingMetadata
30
from vllm.model_executor.utils import set_weight_attrs
31
from vllm.multimodal import MULTIMODAL_REGISTRY
32
+from vllm.multimodal.inputs import NestedTensors
33
from vllm.multimodal.utils import (cached_get_tokenizer,
34
consecutive_placeholder_ranges,
35
repeat_and_pad_placeholder_tokens)
@@ -38,7 +39,7 @@ from vllm.utils import print_warning_once
39
from .interfaces import SupportsMultiModal, SupportsPP
40
from .utils import (is_pp_missing_parameter,
41
make_empty_intermediate_tensors_factory, make_layers,
41
- maybe_prefix)
42
+ maybe_prefix, merge_multimodal_embeddings)
43
44
# These configs are not part of the model config but the preprocessor
45
# and processor files, so we hardcode them in the model file for now.
@@ -987,6 +988,29 @@ class ChameleonForConditionalGeneration(nn.Module, SupportsMultiModal,
988
data=self._validate_pixel_values(pixel_values),
989
)
990
991
+ def get_multimodal_embeddings(self, **kwargs) -> Optional[NestedTensors]:
992
+ image_input = self._parse_and_validate_image_input(**kwargs)
993
+ if image_input is None:
994
+ return None
995
+ assert self.model.vqmodel is not None
996
+ image_tokens = self.model.get_image_tokens(image_input["data"].to(
997
+ self.config.torch_dtype))
998
+ vision_embeddings = self.model.get_input_embeddings(image_tokens)
999
+ return vision_embeddings
1000
+
1001
+ def get_input_embeddings(
1002
+ self,
1003
+ input_ids: torch.Tensor,
1004
+ multimodal_embeddings: Optional[NestedTensors] = None,
1005
+ ) -> torch.Tensor:
1006
+
1007
+ inputs_embeds = self.model.get_input_embeddings(input_ids)
1008
+ if multimodal_embeddings is not None:
1009
+ inputs_embeds = merge_multimodal_embeddings(
1010
+ input_ids, inputs_embeds, multimodal_embeddings,
1011
+ self.model.vocabulary_mapping.image_token_id)
1012
+ return inputs_embeds
1013
+
1014
def forward(
1015
self,
1016
input_ids: torch.Tensor,
@@ -994,27 +1018,27 @@ class ChameleonForConditionalGeneration(nn.Module, SupportsMultiModal,
1018
kv_caches: List[torch.Tensor],
1019
attn_metadata: AttentionMetadata,
1020
intermediate_tensors: Optional[IntermediateTensors] = None,
1021
+ inputs_embeds: Optional[torch.Tensor] = None,
1022
**kwargs,
1023
) -> Union[torch.Tensor, IntermediateTensors]:
1024
1025
if intermediate_tensors is not None:
1026
+ inputs_embeds = None
1027
+
1028
+ # NOTE: In v1, inputs_embeds is always generated at model runner, this
1029
+ # condition is for v0 compatibility.
1030
+ elif inputs_embeds is None:
1031
+ vision_embeddings = self.get_multimodal_embeddings(**kwargs)
1032
+ inputs_embeds = self.get_input_embeddings(input_ids,
1033
+ vision_embeddings)
1034
input_ids = None
1002
- else:
1003
- image_input = self._parse_and_validate_image_input(**kwargs)
1004
-
1005
- if image_input is not None:
1006
- assert self.model.vqmodel is not None
1007
- image_tokens = self.model.get_image_tokens(
1008
- image_input["data"].to(self.config.torch_dtype))
1009
- image_token_id = self.model.vocabulary_mapping.image_token_id
1010
- special_image_mask = input_ids == image_token_id
1011
- image_tokens = image_tokens.to(input_ids.device,
1012
- input_ids.dtype)
1013
- input_ids = input_ids.masked_scatter(special_image_mask,
1014
- image_tokens)
1015
-
1016
- hidden_states = self.model(input_ids, positions, kv_caches,
1017
- attn_metadata, intermediate_tensors)
1035
+
1036
+ hidden_states = self.model(input_ids,
1037
+ positions,
1038
+ kv_caches,
1039
+ attn_metadata,
1040
+ intermediate_tensors,
1041
+ inputs_embeds=inputs_embeds)
1042
return hidden_states
1043
1044
def compute_logits(
vllm/model_executor/models/chatglm.py
+35
-19
@@ -33,7 +33,8 @@ from vllm.model_executor.models.glm4_vision_encoder import EVA2CLIPModel
33
from vllm.model_executor.models.module_mapping import MultiModelKeys
34
from vllm.model_executor.sampling_metadata import SamplingMetadata
35
from vllm.multimodal import MULTIMODAL_REGISTRY
36
-from vllm.multimodal.inputs import MultiModalData, MultiModalKwargs
36
+from vllm.multimodal.inputs import (MultiModalData, MultiModalKwargs,
37
+ NestedTensors)
38
from vllm.multimodal.utils import cached_get_tokenizer
39
from vllm.sequence import (VLLM_TOKEN_ID_ARRAY_TYPE, IntermediateTensors,
40
SequenceData)
@@ -545,6 +546,30 @@ class ChatGLMModel(nn.Module):
546
""")
547
return GLMImagePixelInputs(pixel_values=pixel_values)
548
549
+ def get_multimodal_embeddings(self, **kwargs) -> Optional[NestedTensors]:
550
+ image_input = self._parse_and_validate_image_input(**kwargs)
551
+ if image_input["pixel_values"] is None:
552
+ return None
553
+ pixel_values = image_input["pixel_values"].to(
554
+ dtype=self.config.torch_dtype)
555
+ vision_embeddings = self.vision(pixel_values)
556
+ return vision_embeddings
557
+
558
+ def get_input_embeddings(
559
+ self,
560
+ input_ids: torch.Tensor,
561
+ multimodal_embeddings: Optional[NestedTensors] = None,
562
+ ) -> torch.Tensor:
563
+ inputs_embeds = self.embedding(input_ids)
564
+ if multimodal_embeddings is not None:
565
+ inputs_embeds = merge_glm_vision_embeddings(
566
+ input_ids=input_ids,
567
+ inputs_embeds=inputs_embeds,
568
+ vision_embeddings=multimodal_embeddings,
569
+ boi_token_id=self.config.boi_token_id,
570
+ eoi_token_id=self.config.eoi_token_id)
571
+ return inputs_embeds
572
+
573
def forward(
574
self,
575
input_ids: torch.Tensor,
@@ -552,26 +577,17 @@ class ChatGLMModel(nn.Module):
577
kv_caches: List[torch.Tensor],
578
attn_metadata: AttentionMetadata,
579
intermediate_tensors: Optional[IntermediateTensors] = None,
580
+ inputs_embeds: Optional[torch.Tensor] = None,
581
**kwargs: object,
582
) -> torch.Tensor:
557
- if intermediate_tensors is None:
558
- inputs_embeds = self.embedding(input_ids)
559
- image_input = self._parse_and_validate_image_input(**kwargs)
560
-
561
- if image_input["pixel_values"] is not None:
562
- pixel_values = image_input["pixel_values"].to(
563
- dtype=inputs_embeds.dtype)
564
- image_embeds = self.vision(pixel_values)
565
-
566
- boi_token_id = self.config.boi_token_id
567
- eoi_token_id = self.config.eoi_token_id
568
-
569
- inputs_embeds = merge_glm_vision_embeddings(
570
- input_ids=input_ids,
571
- inputs_embeds=inputs_embeds,
572
- vision_embeddings=image_embeds,
573
- boi_token_id=boi_token_id,
574
- eoi_token_id=eoi_token_id)
583
+
584
+ # NOTE: In v1, inputs_embeds is always generated at model runner, this
585
+ # condition is for v0 compatibility.
586
+ if intermediate_tensors is None and inputs_embeds is None:
587
+ vision_embeddings = self.get_multimodal_embeddings(**kwargs)
588
+ inputs_embeds = self.get_input_embeddings(input_ids,
589
+ vision_embeddings)
590
+ input_ids = None
591
else:
592
inputs_embeds = intermediate_tensors["hidden_states"]
593
vllm/model_executor/models/fuyu.py
+29
-14
@@ -35,6 +35,7 @@ from vllm.model_executor.models.persimmon import PersimmonForCausalLM
35
from vllm.model_executor.sampling_metadata import SamplingMetadata
36
from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalKwargs
37
from vllm.multimodal.image import cached_get_image_processor
38
+from vllm.multimodal.inputs import NestedTensors
39
from vllm.multimodal.utils import (cached_get_tokenizer,
40
consecutive_placeholder_ranges)
41
from vllm.sequence import (VLLM_TOKEN_ID_ARRAY_TYPE, IntermediateTensors,
@@ -302,6 +303,25 @@ class FuyuForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):
303
vision_embeddings, _ = self.vision_embed_tokens(image_input["data"])
304
return vision_embeddings
305
306
+ def get_multimodal_embeddings(self, **kwargs) -> Optional[NestedTensors]:
307
+ image_input = self._parse_and_validate_image_input(**kwargs)
308
+ if image_input is None:
309
+ return None
310
+ vision_embeddings = self._process_image_input(image_input)
311
+ return vision_embeddings
312
+
313
+ def get_input_embeddings(
314
+ self,
315
+ input_ids: torch.Tensor,
316
+ multimodal_embeddings: Optional[NestedTensors] = None,
317
+ ) -> torch.Tensor:
318
+ inputs_embeds = self.language_model.get_input_embeddings(input_ids)
319
+ if multimodal_embeddings is not None:
320
+ inputs_embeds = merge_multimodal_embeddings(
321
+ input_ids, inputs_embeds, multimodal_embeddings,
322
+ _IMAGE_TOKEN_ID)
323
+ return inputs_embeds
324
+
325
def forward(
326
self,
327
input_ids: torch.Tensor,
@@ -309,24 +329,19 @@ class FuyuForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):
329
kv_caches: List[torch.Tensor],
330
attn_metadata: AttentionMetadata,
331
intermediate_tensors: Optional[IntermediateTensors] = None,
332
+ inputs_embeds: Optional[torch.Tensor] = None,
333
**kwargs: object,
334
):
335
if intermediate_tensors is not None:
315
- input_ids = None
336
inputs_embeds = None
317
- else:
318
- image_input = self._parse_and_validate_image_input(**kwargs)
319
-
320
- if image_input is not None:
321
- vision_embeddings = self._process_image_input(image_input)
322
- inputs_embeds = self.language_model.model.embed_tokens(
323
- input_ids)
324
- inputs_embeds = merge_multimodal_embeddings(
325
- input_ids, inputs_embeds, vision_embeddings,
326
- self.image_token_id)
327
-
328
- else:
329
- inputs_embeds = None
337
+
338
+ # NOTE: In v1, inputs_embeds is always generated at model runner, this
339
+ # condition is for v0 compatibility.
340
+ elif inputs_embeds is None:
341
+ vision_embeddings = self.get_multimodal_embeddings(**kwargs)
342
+ inputs_embeds = self.get_input_embeddings(input_ids,
343
+ vision_embeddings)
344
+ input_ids = None
345
346
hidden_states = self.language_model(
347
input_ids=input_ids,
vllm/model_executor/models/interfaces.py
+35
-1
@@ -2,7 +2,7 @@ from typing import (TYPE_CHECKING, ClassVar, Dict, List, Literal, Optional,
2
Protocol, Type, Union, overload, runtime_checkable)
3
4
import torch
5
-from typing_extensions import TypeIs
5
+from typing_extensions import TypeIs, TypeVar
6
7
from vllm.logger import init_logger
8
from vllm.utils import supports_kw
@@ -10,10 +10,14 @@ from vllm.utils import supports_kw
10
from .interfaces_base import is_embedding_model
11
12
if TYPE_CHECKING:
13
+ from vllm.attention import AttentionMetadata
14
+ from vllm.multimodal.inputs import NestedTensors # noqa: F401
15
from vllm.sequence import IntermediateTensors
16
17
logger = init_logger(__name__)
18
19
+T = TypeVar("T", default="NestedTensors")
20
+
21
22
@runtime_checkable
23
class SupportsMultiModal(Protocol):
@@ -28,6 +32,36 @@ class SupportsMultiModal(Protocol):
32
MRO of your model class.
33
"""
34
35
+ def get_multimodal_embeddings(self, **kwargs) -> Optional[T]:
36
+ """
37
+ Returns multimodal embeddings generated from multimodal kwargs
38
+ to be merged with text embeddings.
39
+ """
40
+ ...
41
+
42
+ # Only for models that support v0 chunked prefill
43
+ # TODO(ywang96): Remove this overload once v0 is deprecated
44
+ @overload
45
+ def get_input_embeddings(
46
+ self,
47
+ input_ids: torch.Tensor,
48
+ multimodal_embeddings: Optional[T] = None,
49
+ attn_metadata: Optional["AttentionMetadata"] = None,
50
+ ) -> torch.Tensor:
51
+ ...
52
+
53
+ def get_input_embeddings(
54
+ self,
55
+ input_ids: torch.Tensor,
56
+ multimodal_embeddings: Optional[T] = None,
57
+ ) -> torch.Tensor:
58
+ """
59
+ Returns the input embeddings merged from the text embeddings from
60
+ input_ids and the multimodal embeddings generated from multimodal
61
+ kwargs.
62
+ """
63
+ ...
64
+
65
66
# We can't use runtime_checkable with ClassVar for issubclass checks
67
# so we need to treat the class as an instance and use isinstance instead
vllm/model_executor/models/internvl.py
+39
-15
@@ -26,6 +26,7 @@ from vllm.model_executor.models.intern_vit import (InternVisionModel,
26
InternVisionPatchModel)
27
from vllm.model_executor.sampling_metadata import SamplingMetadata
28
from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalKwargs
29
+from vllm.multimodal.inputs import NestedTensors
30
from vllm.multimodal.utils import cached_get_tokenizer
31
from vllm.sequence import IntermediateTensors
32
from vllm.utils import is_list_of
@@ -641,6 +642,26 @@ class InternVLChatModel(nn.Module, SupportsMultiModal, SupportsPP):
642
visual_token_mask = None
643
return visual_token_mask
644
645
+ def get_multimodal_embeddings(self, **kwargs) -> Optional[NestedTensors]:
646
+ image_input = self._parse_and_validate_image_input(**kwargs)
647
+ if image_input is None:
648
+ return None
649
+ vision_embeddings = self._process_image_input(image_input)
650
+ return vision_embeddings
651
+
652
+ def get_input_embeddings(
653
+ self,
654
+ input_ids: torch.Tensor,
655
+ multimodal_embeddings: Optional[NestedTensors] = None,
656
+ ) -> torch.Tensor:
657
+ inputs_embeds = self.language_model.get_input_embeddings(input_ids)
658
+ if multimodal_embeddings is not None:
659
+ assert self.img_context_token_id is not None
660
+ inputs_embeds = merge_multimodal_embeddings(
661
+ input_ids, inputs_embeds, multimodal_embeddings,
662
+ self.img_context_token_id)
663
+ return inputs_embeds
664
+
665
def forward(
666
self,
667
input_ids: torch.Tensor,
@@ -648,26 +669,22 @@ class InternVLChatModel(nn.Module, SupportsMultiModal, SupportsPP):
669
kv_caches: List[torch.Tensor],
670
attn_metadata: AttentionMetadata,
671
intermediate_tensors: Optional[IntermediateTensors] = None,
672
+ inputs_embeds: Optional[torch.Tensor] = None,
673
**kwargs: object,
674
) -> Union[SamplerOutput, IntermediateTensors]:
675
+
676
+ visual_token_mask = None
677
if intermediate_tensors is not None:
678
input_ids = None
679
inputs_embeds = None
656
- visual_token_mask = None
657
- else:
658
- image_input = self._parse_and_validate_image_input(**kwargs)
659
- if image_input is not None:
660
- inputs_embeds = self.language_model.model.get_input_embeddings(
661
- input_ids)
662
- vision_embeddings = self._process_image_input(image_input)
663
- inputs_embeds = merge_multimodal_embeddings(
664
- input_ids, inputs_embeds, vision_embeddings,
665
- self.img_context_token_id)
666
- visual_token_mask = self._get_visual_token_mask(input_ids)
667
- input_ids = None
668
- else:
669
- inputs_embeds = None
670
- visual_token_mask = None
680
+
681
+ # NOTE: In v1, inputs_embeds is always generated at model runner, this
682
+ # condition is for v0 compatibility.
683
+ elif inputs_embeds is None:
684
+ vision_embeddings = self.get_multimodal_embeddings(**kwargs)
685
+ inputs_embeds = self.get_input_embeddings(input_ids,
686
+ vision_embeddings)
687
+ input_ids = None
688
689
forward_kwargs = {
690
"input_ids": input_ids,
@@ -677,6 +694,13 @@ class InternVLChatModel(nn.Module, SupportsMultiModal, SupportsPP):
694
"intermediate_tensors": intermediate_tensors,
695
"inputs_embeds": inputs_embeds,
696
}
697
+ if self.img_context_token_id is not None:
698
+ visual_token_mask = self._get_visual_token_mask(input_ids)
699
+
700
+ # We always overwrite it back to None after computing visual token
701
+ # mask so that this doesn't need to depend on encoder output
702
+ self.img_context_token_id = None
703
+
704
if self.is_mono:
705
forward_kwargs.update({"visual_token_mask": visual_token_mask})
706
vllm/model_executor/models/llava.py
+8
-7
@@ -478,7 +478,7 @@ class LlavaForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP):
478
image_features = self._process_image_pixels(image_input)
479
return self.multi_modal_projector(image_features)
480
481
- def process_mm_inputs(self, **kwargs):
481
+ def get_multimodal_embeddings(self, **kwargs) -> Optional[NestedTensors]:
482
image_input = self._parse_and_validate_image_input(**kwargs)
483
if image_input is None:
484
return None
@@ -488,12 +488,12 @@ class LlavaForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP):
488
def get_input_embeddings(
489
self,
490
input_ids: torch.Tensor,
491
- vision_embeddings: Optional[NestedTensors] = None,
491
+ multimodal_embeddings: Optional[NestedTensors] = None,
492
) -> torch.Tensor:
493
inputs_embeds = self.language_model.get_input_embeddings(input_ids)
494
- if vision_embeddings is not None:
494
+ if multimodal_embeddings is not None:
495
inputs_embeds = merge_multimodal_embeddings(
496
- input_ids, inputs_embeds, vision_embeddings,
496
+ input_ids, inputs_embeds, multimodal_embeddings,
497
self.config.image_token_index)
498
return inputs_embeds
499
@@ -544,10 +544,11 @@ class LlavaForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP):
544
"""
545
if intermediate_tensors is not None:
546
inputs_embeds = None
547
+
548
+ # NOTE: In v1, inputs_embeds is always generated at model runner, this
549
+ # condition is for v0 compatibility.
550
elif inputs_embeds is None:
548
- vision_embeddings = self.process_mm_inputs(**kwargs)
549
- # always pass the input via `inputs_embeds`
550
- # to make sure the computation graph is consistent
551
+ vision_embeddings = self.get_multimodal_embeddings(**kwargs)
552
inputs_embeds = self.get_input_embeddings(input_ids,
553
vision_embeddings)
554
input_ids = None
vllm/model_executor/models/llava_next.py
+33
-18
@@ -19,6 +19,7 @@ from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
19
from vllm.model_executor.pooling_metadata import PoolingMetadata
20
from vllm.model_executor.sampling_metadata import SamplingMetadata
21
from vllm.multimodal import MULTIMODAL_REGISTRY
22
+from vllm.multimodal.inputs import NestedTensors
23
from vllm.sequence import IntermediateTensors, PoolerOutput
24
from vllm.utils import is_list_of
25
@@ -565,6 +566,30 @@ class LlavaNextForConditionalGeneration(nn.Module, SupportsMultiModal,
566
for i, patch_features_batch in enumerate(patch_embeddings)
567
]
568
569
+ def get_multimodal_embeddings(self, **kwargs) -> Optional[NestedTensors]:
570
+ image_input = self._parse_and_validate_image_input(**kwargs)
571
+ if image_input is None:
572
+ return None
573
+ vision_embeddings = self._process_image_input(image_input)
574
+ return vision_embeddings
575
+
576
+ def get_input_embeddings(
577
+ self,
578
+ input_ids: torch.Tensor,
579
+ multimodal_embeddings: Optional[NestedTensors] = None,
580
+ ) -> torch.Tensor:
581
+
582
+ if multimodal_embeddings is None:
583
+ return self.language_model.get_input_embeddings(input_ids)
584
+
585
+ inputs_embeds = embed_multimodal(
586
+ input_ids,
587
+ self.config.image_token_index,
588
+ self.language_model.model.get_input_embeddings,
589
+ multimodal_embeddings,
590
+ )
591
+ return inputs_embeds
592
+
593
def forward(
594
self,
595
input_ids: torch.Tensor,
@@ -572,6 +597,7 @@ class LlavaNextForConditionalGeneration(nn.Module, SupportsMultiModal,
597
kv_caches: List[torch.Tensor],
598
attn_metadata: AttentionMetadata,
599
intermediate_tensors: Optional[IntermediateTensors] = None,
600
+ inputs_embeds: Optional[torch.Tensor] = None,
601
**kwargs: object,
602
) -> Union[torch.Tensor, IntermediateTensors]:
603
"""Run forward pass for LlaVA-NeXT.
@@ -620,24 +646,14 @@ class LlavaNextForConditionalGeneration(nn.Module, SupportsMultiModal,
646
"""
647
if intermediate_tensors is not None:
648
inputs_embeds = None
623
- else:
624
- image_input = self._parse_and_validate_image_input(**kwargs)
625
-
626
- if image_input is not None:
627
- inputs_embeds = embed_multimodal(
628
- input_ids,
629
- self.config.image_token_index,
630
- self.language_model.model.get_input_embeddings,
631
- lambda _: self._process_image_input(image_input),
632
- )
633
- else:
634
- inputs_embeds = self.language_model.model.get_input_embeddings(
635
- input_ids)
649
637
- # always pass the input via `inputs_embeds`
638
- # to make sure the computation graph is consistent
639
- # for `torch.compile` integration
640
- input_ids = None
650
+ # NOTE: In v1, inputs_embeds is always generated at model runner, this
651
+ # condition is for v0 compatibility.
652
+ elif inputs_embeds is None:
653
+ vision_embeddings = self.get_multimodal_embeddings(**kwargs)
654
+ inputs_embeds = self.get_input_embeddings(input_ids,
655
+ vision_embeddings)
656
+ input_ids = None
657
658
hidden_states = self.language_model.model(input_ids,
659
positions,
@@ -645,7 +661,6 @@ class LlavaNextForConditionalGeneration(nn.Module, SupportsMultiModal,
661
attn_metadata,
662
intermediate_tensors,
663
inputs_embeds=inputs_embeds)
648
-
664
return hidden_states
665
666
def compute_logits(
vllm/model_executor/models/llava_next_video.py
+29
-15
@@ -18,6 +18,7 @@ from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
18
from vllm.model_executor.models.clip import CLIPVisionModel
19
from vllm.model_executor.sampling_metadata import SamplingMetadata
20
from vllm.multimodal import MULTIMODAL_REGISTRY
21
+from vllm.multimodal.inputs import NestedTensors
22
from vllm.multimodal.utils import (cached_get_tokenizer,
23
repeat_and_pad_placeholder_tokens)
24
from vllm.sequence import IntermediateTensors
@@ -388,6 +389,25 @@ class LlavaNextVideoForConditionalGeneration(nn.Module, SupportsMultiModal,
389
raise ValueError(
390
f"Unsupported type of video input {type(video_pixels)}")
391
392
+ def get_multimodal_embeddings(self, **kwargs) -> Optional[NestedTensors]:
393
+ video_input = self._parse_and_validate_video_input(**kwargs)
394
+ if video_input is None:
395
+ return None
396
+ vision_embeddings = self._process_video_pixels(video_input)
397
+ return vision_embeddings
398
+
399
+ def get_input_embeddings(
400
+ self,
401
+ input_ids: torch.Tensor,
402
+ multimodal_embeddings: Optional[NestedTensors] = None,
403
+ ) -> torch.Tensor:
404
+ inputs_embeds = self.language_model.get_input_embeddings(input_ids)
405
+ if multimodal_embeddings is not None:
406
+ inputs_embeds = merge_multimodal_embeddings(
407
+ input_ids, inputs_embeds, multimodal_embeddings,
408
+ self.config.video_token_index)
409
+ return inputs_embeds
410
+
411
def forward(
412
self,
413
input_ids: torch.Tensor,
@@ -395,6 +415,7 @@ class LlavaNextVideoForConditionalGeneration(nn.Module, SupportsMultiModal,
415
kv_caches: List[torch.Tensor],
416
attn_metadata: AttentionMetadata,
417
intermediate_tensors: Optional[IntermediateTensors] = None,
418
+ inputs_embeds: Optional[torch.Tensor] = None,
419
**kwargs: object,
420
) -> Union[torch.Tensor, IntermediateTensors]:
421
"""Run forward pass for LlaVA-NeXT-Video.
@@ -404,22 +425,15 @@ class LlavaNextVideoForConditionalGeneration(nn.Module, SupportsMultiModal,
425
pixel_values_videos: Pixels in each frames for each input videos.
426
"""
427
if intermediate_tensors is not None:
407
- input_ids = None
428
inputs_embeds = None
409
- else:
410
- video_input = self._parse_and_validate_video_input(**kwargs)
411
- if video_input is not None:
412
- video_embeddings = self._process_video_pixels(video_input)
413
- inputs_embeds = self.language_model \
414
- .model.get_input_embeddings(input_ids)
415
-
416
- inputs_embeds = merge_multimodal_embeddings(
417
- input_ids, inputs_embeds, video_embeddings,
418
- self.config.video_token_index)
419
-
420
- input_ids = None
421
- else:
422
- inputs_embeds = None
429
+
430
+ # NOTE: In v1, inputs_embeds is always generated at model runner, this
431
+ # condition is for v0 compatibility.
432
+ elif inputs_embeds is None:
433
+ vision_embeddings = self.get_multimodal_embeddings(**kwargs)
434
+ inputs_embeds = self.get_input_embeddings(input_ids,
435
+ vision_embeddings)
436
+ input_ids = None
437
438
hidden_states = self.language_model.model(input_ids,
439
positions,
vllm/model_executor/models/llava_onevision.py
+53
-21
@@ -21,6 +21,7 @@ from vllm.model_executor.layers.activation import get_act_fn
21
from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
22
from vllm.model_executor.sampling_metadata import SamplingMetadata
23
from vllm.multimodal import MULTIMODAL_REGISTRY
24
+from vllm.multimodal.inputs import NestedTensors
25
from vllm.multimodal.utils import (cached_get_tokenizer,
26
repeat_and_pad_placeholder_tokens)
27
from vllm.sequence import IntermediateTensors
@@ -824,6 +825,49 @@ class LlavaOnevisionForConditionalGeneration(nn.Module, SupportsMultiModal,
825
image_feature = image_feature.view(batch_frames, -1, dim)
826
return image_feature
827
828
+ def get_multimodal_embeddings(
829
+ self, **kwargs) -> Optional[List[Tuple[NestedTensors, str]]]:
830
+ modalities = self._parse_and_validate_multimodal_inputs(**kwargs)
831
+ if not modalities:
832
+ return None
833
+
834
+ # We make a tuple of each embedding with its modality string. This is a
835
+ # temporary workaround for models to handle mixed modalities when
836
+ # get_multimodal_embeddings and get_input_embeddings are called
837
+ # separately.
838
+ # TODO(ywang96): Add support for mixed-modality inference for v1.
839
+ multimodal_embeddings: List[Tuple[NestedTensors, str]] = []
840
+
841
+ if "images" in modalities:
842
+ image_input = modalities["images"]
843
+ vision_embeddings = self._process_image_input(image_input)
844
+ multimodal_embeddings.append((vision_embeddings, "image"))
845
+ if "videos" in modalities:
846
+ video_input = modalities["videos"]
847
+ video_embeddings = self._process_video_pixels(video_input)
848
+ multimodal_embeddings.append((video_embeddings, "video"))
849
+
850
+ return multimodal_embeddings
851
+
852
+ def get_input_embeddings(
853
+ self,
854
+ input_ids: torch.Tensor,
855
+ multimodal_embeddings: Optional[List[Tuple[NestedTensors,
856
+ str]]] = None,
857
+ ) -> torch.Tensor:
858
+ inputs_embeds = self.language_model.get_input_embeddings(input_ids)
859
+ if multimodal_embeddings is not None:
860
+ for embeddings, modality in multimodal_embeddings:
861
+ if modality == "image":
862
+ inputs_embeds = merge_multimodal_embeddings(
863
+ input_ids, inputs_embeds, embeddings,
864
+ self.config.image_token_index)
865
+ if modality == "video":
866
+ inputs_embeds = merge_multimodal_embeddings(
867
+ input_ids, inputs_embeds, embeddings,
868
+ self.config.video_token_index)
869
+ return inputs_embeds
870
+
871
def forward(
872
self,
873
input_ids: torch.Tensor,
@@ -831,6 +875,7 @@ class LlavaOnevisionForConditionalGeneration(nn.Module, SupportsMultiModal,
875
kv_caches: List[torch.Tensor],
876
attn_metadata: AttentionMetadata,
877
intermediate_tensors: Optional[IntermediateTensors] = None,
878
+ inputs_embeds: Optional[torch.Tensor] = None,
879
**kwargs: object,
880
) -> Union[torch.Tensor, IntermediateTensors]:
881
"""Run forward pass for LlaVA-Onevision.
@@ -840,28 +885,15 @@ class LlavaOnevisionForConditionalGeneration(nn.Module, SupportsMultiModal,
885
pixel_values_videos: Pixels in each frames for each input videos.
886
"""
887
if intermediate_tensors is not None:
843
- input_ids = None
888
inputs_embeds = None
845
- else:
846
- modalities = self._parse_and_validate_multimodal_inputs(**kwargs)
847
- if modalities:
848
- inputs_embeds = self.language_model.model.get_input_embeddings(
849
- input_ids)
850
- if "images" in modalities:
851
- image_input = modalities["images"]
852
- vision_embeddings = self._process_image_input(image_input)
853
- inputs_embeds = merge_multimodal_embeddings(
854
- input_ids, inputs_embeds, vision_embeddings,
855
- self.config.image_token_index)
856
- if "videos" in modalities:
857
- video_input = modalities["videos"]
858
- video_embeddings = self._process_video_pixels(video_input)
859
- inputs_embeds = merge_multimodal_embeddings(
860
- input_ids, inputs_embeds, video_embeddings,
861
- self.config.video_token_index)
862
- input_ids = None
863
- else:
864
- inputs_embeds = None
889
+
890
+ # NOTE: In v1, inputs_embeds is always generated at model runner, this
891
+ # condition is for v0 compatibility.
892
+ elif inputs_embeds is None:
893
+ multimodal_embeddings = self.get_multimodal_embeddings(**kwargs)
894
+ inputs_embeds = self.get_input_embeddings(input_ids,
895
+ multimodal_embeddings)
896
+ input_ids = None
897
898
hidden_states = self.language_model.model(input_ids,
899
positions,
vllm/model_executor/models/molmo.py
+46
-42
@@ -3,7 +3,7 @@ import re
3
from array import array
4
from dataclasses import dataclass
5
from functools import lru_cache, partial
6
-from typing import Iterable, List, Mapping, Optional, Tuple, TypedDict, Union
6
+from typing import Iterable, List, Mapping, Optional, Tuple, TypedDict
7
8
import torch
9
from einops import rearrange
@@ -36,6 +36,7 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
36
ParallelLMHead, VocabParallelEmbedding)
37
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
38
from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalKwargs
39
+from vllm.multimodal.inputs import NestedTensors
40
from vllm.multimodal.utils import cached_get_tokenizer
41
from vllm.platforms import _Backend
42
from vllm.sequence import (VLLM_TOKEN_ID_ARRAY_TYPE, IntermediateTensors,
@@ -756,6 +757,12 @@ class MolmoModel(nn.Module):
757
make_empty_intermediate_tensors_factory(
758
["hidden_states", "residual"], config.hidden_size))
759
760
+ def get_input_embeddings(
761
+ self,
762
+ input_ids: torch.Tensor,
763
+ ) -> torch.Tensor:
764
+ return self.embed_tokens(input_ids)
765
+
766
def forward(
767
self,
768
input_ids: torch.Tensor,
@@ -1098,19 +1105,16 @@ class MolmoForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):
1105
1106
return image_features
1107
1101
- def _merge_multimodal_embeddings(
1102
- self,
1103
- inputs_embeds: torch.Tensor,
1104
- image_features: torch.Tensor,
1105
- image_input_idx: torch.Tensor,
1106
- seq_len: Union[torch.Tensor, List[torch.Tensor]],
1107
- ) -> torch.Tensor:
1108
+ def get_multimodal_embeddings(self, **kwargs) -> Optional[NestedTensors]:
1109
+ image_input = self._parse_and_validate_image_input(**kwargs)
1110
+ if image_input is None:
1111
+ return None
1112
+ image_features = self._process_image_input(image_input)
1113
+ image_input_idx = image_input["image_input_idx"]
1114
+ seq_len = image_input["seq_len"]
1115
batch_size, num_image, num_patch = image_features.shape[:3]
1116
assert image_input_idx.shape == (batch_size, num_image, num_patch)
1117
1111
- image_features = image_features.to(inputs_embeds.device)
1112
- seq_len = seq_len.to(inputs_embeds.device)
1113
-
1118
# insert the image feature into the embedding.
1119
image_features = image_features.view(batch_size, num_image * num_patch,
1120
-1)
@@ -1130,12 +1134,24 @@ class MolmoForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):
1134
image_input_idx = image_input_idx + offset.to(image_input_idx.dtype)
1135
image_input_idx = image_input_idx.flatten()[:, None]
1136
mat = image_input_idx == torch.arange(
1133
- seq_len.sum().item(), device=inputs_embeds.device)[None, :]
1137
+ seq_len.sum().item(), device=image_features.device)[None, :]
1138
mat = mat.to(image_features.dtype)
1139
1136
- inputs_embeds = inputs_embeds + torch.einsum('nd,nm->md',
1137
- image_features, mat)
1140
+ # Note: In this original implementation from AI2, the final
1141
+ # vision_embeddings will be always be the same length
1142
+ # of input embedddings, which is not very efficient.
1143
+ # TODO(ywang96): see if this can be optimized.
1144
+ vision_embeddings = torch.einsum('nd,nm->md', image_features, mat)
1145
+ return vision_embeddings
1146
1147
+ def get_input_embeddings(
1148
+ self,
1149
+ input_ids: torch.Tensor,
1150
+ multimodal_embeddings: Optional[NestedTensors] = None,
1151
+ ) -> torch.Tensor:
1152
+ inputs_embeds = self.model.get_input_embeddings(input_ids)
1153
+ if multimodal_embeddings is not None:
1154
+ inputs_embeds = inputs_embeds + multimodal_embeddings
1155
return inputs_embeds
1156
1157
def forward(
@@ -1145,39 +1161,27 @@ class MolmoForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):
1161
kv_caches: List[torch.Tensor],
1162
attn_metadata: AttentionMetadata,
1163
intermediate_tensors: Optional[IntermediateTensors] = None,
1164
+ inputs_embeds: Optional[torch.Tensor] = None,
1165
**kwargs: object,
1166
) -> SamplerOutput:
1167
+
1168
if intermediate_tensors is not None:
1169
inputs_embeds = None
1152
- else:
1153
- image_input = self._parse_and_validate_image_input(**kwargs)
1154
-
1155
- if image_input is not None:
1156
- inputs_embeds = self.model.embed_tokens(input_ids)
1157
- image_features = self._process_image_input(image_input)
1158
-
1159
- inputs_embeds = self._merge_multimodal_embeddings(
1160
- inputs_embeds,
1161
- image_features,
1162
- image_input["image_input_idx"],
1163
- image_input["seq_len"],
1164
- )
1165
- else:
1166
- inputs_embeds = self.model.embed_tokens(input_ids)
1170
1168
- # always pass the input via `inputs_embeds`
1169
- # to make sure the computation graph is consistent
1170
- # for `torch.compile` integration
1171
- input_ids = None
1172
-
1173
- hidden_states = self.model(
1174
- input_ids=input_ids,
1175
- positions=positions,
1176
- kv_caches=kv_caches,
1177
- attn_metadata=attn_metadata,
1178
- intermediate_tensors=intermediate_tensors,
1179
- inputs_embeds=inputs_embeds,
1180
- )
1171
+ # NOTE: In v1, inputs_embeds is always generated at model runner, this
1172
+ # condition is for v0 compatibility.
1173
+ elif inputs_embeds is None:
1174
+ vision_embeddings = self.get_multimodal_embeddings(**kwargs)
1175
+ inputs_embeds = self.get_input_embeddings(input_ids,
1176
+ vision_embeddings)
1177
+ input_ids = None
1178
+
1179
+ hidden_states = self.model(input_ids,
1180
+ positions,
1181
+ kv_caches,
1182
+ attn_metadata,
1183
+ intermediate_tensors,
1184
+ inputs_embeds=inputs_embeds)
1185
1186
return hidden_states
1187
vllm/model_executor/models/paligemma.py
+31
-21
@@ -13,6 +13,7 @@ from vllm.logger import init_logger
13
from vllm.model_executor.layers.sampler import SamplerOutput
14
from vllm.model_executor.sampling_metadata import SamplingMetadata
15
from vllm.multimodal import MULTIMODAL_REGISTRY
16
+from vllm.multimodal.inputs import NestedTensors
17
from vllm.multimodal.utils import cached_get_tokenizer
18
from vllm.sequence import IntermediateTensors
19
@@ -240,36 +241,45 @@ class PaliGemmaForConditionalGeneration(nn.Module, SupportsMultiModal,
241
242
return self.multi_modal_projector(image_features)
243
244
+ def get_multimodal_embeddings(self, **kwargs) -> Optional[NestedTensors]:
245
+ image_input = self._parse_and_validate_image_input(**kwargs)
246
+ if image_input is None:
247
+ return None
248
+ vision_embeddings = self._process_image_input(image_input)
249
+ # https://github.com/huggingface/transformers/blob/main/src/transformers/models/paligemma/modeling_paligemma.py#L294 # noqa
250
+ vision_embeddings = vision_embeddings * (self.config.hidden_size**-0.5)
251
+ return vision_embeddings
252
+
253
+ def get_input_embeddings(
254
+ self,
255
+ input_ids: torch.Tensor,
256
+ multimodal_embeddings: Optional[NestedTensors] = None,
257
+ ) -> torch.Tensor:
258
+ inputs_embeds = self.language_model.get_input_embeddings(input_ids)
259
+ if multimodal_embeddings is not None:
260
+ inputs_embeds = merge_multimodal_embeddings(
261
+ input_ids, inputs_embeds, multimodal_embeddings,
262
+ self.config.image_token_index)
263
+ return inputs_embeds
264
+
265
def forward(self,
266
input_ids: torch.Tensor,
267
positions: torch.Tensor,
268
kv_caches: List[torch.Tensor],
269
attn_metadata: AttentionMetadata,
270
intermediate_tensors: Optional[IntermediateTensors] = None,
271
+ inputs_embeds: Optional[torch.Tensor] = None,
272
**kwargs: object) -> Union[SamplerOutput, IntermediateTensors]:
273
if intermediate_tensors is not None:
251
- input_ids = None
274
inputs_embeds = None
253
- else:
254
- parsed_image_input = self._parse_and_validate_image_input(**kwargs)
255
-
256
- if parsed_image_input is not None:
257
- vision_embeddings = self._process_image_input(
258
- parsed_image_input)
259
- # https://github.com/huggingface/transformers/blob/main/src/transformers/models/paligemma/modeling_paligemma.py#L294 # noqa
260
- vision_embeddings = vision_embeddings * (
261
- self.config.hidden_size**-0.5)
262
-
263
- inputs_embeds = self.language_model.model.get_input_embeddings(
264
- input_ids)
265
-
266
- inputs_embeds = merge_multimodal_embeddings(
267
- input_ids, inputs_embeds, vision_embeddings,
268
- self.config.image_token_index)
269
-
270
- input_ids = None
271
- else:
272
- inputs_embeds = None
275
+
276
+ # NOTE: In v1, inputs_embeds is always generated at model runner, this
277
+ # condition is for v0 compatibility.
278
+ elif inputs_embeds is None:
279
+ vision_embeddings = self.get_multimodal_embeddings(**kwargs)
280
+ inputs_embeds = self.get_input_embeddings(input_ids,
281
+ vision_embeddings)
282
+ input_ids = None
283
284
hidden_states = self.language_model.model(input_ids,
285
positions,
vllm/model_executor/models/phi3v.py
+9
-7
@@ -676,7 +676,7 @@ class Phi3VForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):
676
677
return image_embeds
678
679
- def process_mm_inputs(self, **kwargs):
679
+ def get_multimodal_embeddings(self, **kwargs) -> Optional[NestedTensors]:
680
image_input = self._parse_and_validate_image_input(**kwargs)
681
if image_input is None:
682
return None
@@ -686,12 +686,12 @@ class Phi3VForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):
686
def get_input_embeddings(
687
self,
688
input_ids: torch.Tensor,
689
- vision_embeddings: Optional[NestedTensors] = None,
689
+ multimodal_embeddings: Optional[NestedTensors] = None,
690
) -> torch.Tensor:
691
inputs_embeds = self.embed_tokens(input_ids)
692
- if vision_embeddings is not None:
692
+ if multimodal_embeddings is not None:
693
inputs_embeds = merge_multimodal_embeddings(
694
- input_ids, inputs_embeds, vision_embeddings,
694
+ input_ids, inputs_embeds, multimodal_embeddings,
695
self.image_token_id)
696
return inputs_embeds
697
@@ -703,12 +703,14 @@ class Phi3VForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):
703
intermediate_tensors: Optional[IntermediateTensors] = None,
704
inputs_embeds: Optional[torch.Tensor] = None,
705
**kwargs: object):
706
+
707
if intermediate_tensors is not None:
708
inputs_embeds = None
709
+
710
+ # NOTE: In v1, inputs_embeds is always generated at model runner, this
711
+ # condition is for v0 compatibility
712
elif inputs_embeds is None:
709
- vision_embeddings = self.process_mm_inputs(**kwargs)
710
- # always pass the input via `inputs_embeds`
711
- # to make sure the computation graph is consistent
713
+ vision_embeddings = self.get_multimodal_embeddings(**kwargs)
714
inputs_embeds = self.get_input_embeddings(input_ids,
715
vision_embeddings)
716
input_ids = None
vllm/model_executor/models/qwen2_audio.py
+37
-22
@@ -42,10 +42,12 @@ from vllm.model_executor.model_loader.weight_utils import (
42
from vllm.model_executor.models.qwen2 import Qwen2Model
43
from vllm.model_executor.sampling_metadata import SamplingMetadata
44
from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalKwargs
45
+from vllm.multimodal.inputs import NestedTensors
46
from vllm.multimodal.utils import consecutive_placeholder_ranges
47
from vllm.sequence import IntermediateTensors, SequenceData
48
49
from .interfaces import SupportsMultiModal, SupportsPP
50
+from .utils import merge_multimodal_embeddings
51
52
logger = init_logger(__name__)
53
@@ -371,6 +373,25 @@ class Qwen2AudioForConditionalGeneration(nn.Module, SupportsMultiModal,
373
374
return masked_audio_features
375
376
+ def get_multimodal_embeddings(self, **kwargs) -> Optional[NestedTensors]:
377
+ audio_input = self._parse_and_validate_audio_input(**kwargs)
378
+ if audio_input is None:
379
+ return None
380
+ masked_audio_features = self._process_audio_input(audio_input)
381
+ return masked_audio_features
382
+
383
+ def get_input_embeddings(
384
+ self,
385
+ input_ids: torch.Tensor,
386
+ multimodal_embeddings: Optional[NestedTensors] = None,
387
+ ) -> torch.Tensor:
388
+ inputs_embeds = self.language_model.get_input_embeddings(input_ids)
389
+ if multimodal_embeddings is not None:
390
+ inputs_embeds = merge_multimodal_embeddings(
391
+ input_ids, inputs_embeds, multimodal_embeddings,
392
+ self.config.audio_token_index)
393
+ return inputs_embeds
394
+
395
def forward(
396
self,
397
input_ids: torch.Tensor,
@@ -378,33 +399,27 @@ class Qwen2AudioForConditionalGeneration(nn.Module, SupportsMultiModal,
399
kv_caches: List[torch.Tensor],
400
attn_metadata: AttentionMetadata,
401
intermediate_tensors: Optional[IntermediateTensors] = None,
402
+ inputs_embeds: Optional[torch.Tensor] = None,
403
**kwargs: object,
404
) -> Union[torch.Tensor, IntermediateTensors]:
405
+
406
if intermediate_tensors is not None:
384
- input_ids = None
407
inputs_embeds = None
386
- else:
387
- audio_input = self._parse_and_validate_audio_input(**kwargs)
408
389
- if audio_input is None:
390
- inputs_embeds = None
391
- else:
392
- inputs_embeds = self.language_model.embed_tokens(input_ids)
393
- masked_audio_features = self._process_audio_input(audio_input)
394
- # merge llm embeddings and audio features
395
- mask = (input_ids == self.config.audio_token_index)
396
- inputs_embeds[mask, :] = masked_audio_features
397
-
398
- input_ids = None
399
-
400
- hidden_states = self.language_model(
401
- input_ids=input_ids,
402
- positions=positions,
403
- kv_caches=kv_caches,
404
- attn_metadata=attn_metadata,
405
- intermediate_tensors=intermediate_tensors,
406
- inputs_embeds=inputs_embeds,
407
- )
409
+ # NOTE: In v1, inputs_embeds is always generated at model runner, this
410
+ # condition is for v0 compatibility.
411
+ elif inputs_embeds is None:
412
+ multimodal_embeddings = self.get_multimodal_embeddings(**kwargs)
413
+ inputs_embeds = self.get_input_embeddings(input_ids,
414
+ multimodal_embeddings)
415
+ input_ids = None
416
+
417
+ hidden_states = self.language_model(input_ids,
418
+ positions,
419
+ kv_caches,
420
+ attn_metadata,
421
+ intermediate_tensors,
422
+ inputs_embeds=inputs_embeds)
423
return hidden_states
424
425
def compute_logits(self, hidden_states: torch.Tensor,
vllm/model_executor/models/qwen2_vl.py
+68
-34
@@ -63,7 +63,7 @@ from vllm.model_executor.pooling_metadata import PoolingMetadata
63
from vllm.multimodal import MULTIMODAL_REGISTRY
64
from vllm.multimodal.image import cached_get_image_processor
65
from vllm.multimodal.inputs import (MultiModalData, MultiModalDataDict,
66
- MultiModalKwargs)
66
+ MultiModalKwargs, NestedTensors)
67
from vllm.multimodal.utils import cached_get_tokenizer
68
from vllm.platforms import _Backend
69
from vllm.sequence import IntermediateTensors, PoolerOutput, SequenceData
@@ -1238,6 +1238,55 @@ class Qwen2VLForConditionalGeneration(nn.Module, SupportsMultiModal,
1238
inputs_embeds[mask, :] = multimodal_embeddings
1239
return inputs_embeds
1240
1241
+ def get_multimodal_embeddings(
1242
+ self, **kwargs) -> Optional[List[Tuple[NestedTensors, str]]]:
1243
+
1244
+ image_input = self._parse_and_validate_image_input(**kwargs)
1245
+ video_input = self._parse_and_validate_video_input(**kwargs)
1246
+ if image_input is None and video_input is None:
1247
+ return None
1248
+
1249
+ # We make a tuple of each embedding with its modality string. This is a
1250
+ # temporary workaround for models to handle mixed modalities when
1251
+ # get_multimodal_embeddings and get_input_embeddings are called
1252
+ # separately.
1253
+ # TODO(ywang96): Add support for mixed-modality inference for v1.
1254
+ multimodal_embeddings: List[Tuple[NestedTensors, str]] = []
1255
+
1256
+ if image_input is not None:
1257
+ image_embeds = self._process_image_input(image_input)
1258
+ multimodal_embeddings.append((image_embeds, "image"))
1259
+ if video_input is not None:
1260
+ video_embeds = self._process_video_input(video_input)
1261
+ multimodal_embeddings.append((video_embeds, "video"))
1262
+
1263
+ return multimodal_embeddings
1264
+
1265
+ def get_input_embeddings(
1266
+ self,
1267
+ input_ids: torch.Tensor,
1268
+ multimodal_embeddings: Optional[List[Tuple[NestedTensors,
1269
+ str]]] = None,
1270
+ ) -> torch.Tensor:
1271
+ inputs_embeds = self.model.get_input_embeddings(input_ids)
1272
+ if multimodal_embeddings is not None:
1273
+ for embeddings, modality in multimodal_embeddings:
1274
+ if modality == "image":
1275
+ inputs_embeds = self._merge_multimodal_embeddings(
1276
+ input_ids,
1277
+ inputs_embeds,
1278
+ embeddings,
1279
+ placeholder_token_id=self.config.image_token_id,
1280
+ )
1281
+ if modality == "video":
1282
+ inputs_embeds = self._merge_multimodal_embeddings(
1283
+ input_ids,
1284
+ inputs_embeds,
1285
+ embeddings,
1286
+ placeholder_token_id=self.config.video_token_id,
1287
+ )
1288
+ return inputs_embeds
1289
+
1290
def forward(
1291
self,
1292
input_ids: torch.Tensor,
@@ -1245,6 +1294,7 @@ class Qwen2VLForConditionalGeneration(nn.Module, SupportsMultiModal,
1294
kv_caches: List[torch.Tensor],
1295
attn_metadata: AttentionMetadata,
1296
intermediate_tensors: Optional[IntermediateTensors] = None,
1297
+ inputs_embeds: Optional[torch.Tensor] = None,
1298
**kwargs: object,
1299
) -> Union[torch.Tensor, IntermediateTensors]:
1300
"""Run forward pass for Qwen2-VL.
@@ -1266,42 +1316,26 @@ class Qwen2VLForConditionalGeneration(nn.Module, SupportsMultiModal,
1316
video_grid_thw: Tensor `(n_videos, 3)` of video 3D grid in LLM.
1317
`None` if no videos are passed.
1318
"""
1319
+
1320
if intermediate_tensors is not None:
1270
- input_ids = None
1321
inputs_embeds = None
1272
- else:
1273
- image_input = self._parse_and_validate_image_input(**kwargs)
1274
- video_input = self._parse_and_validate_video_input(**kwargs)
1275
-
1276
- if image_input is None and video_input is None:
1277
- inputs_embeds = None
1278
- else:
1279
- if uses_mrope(self.config):
1280
- assert positions.ndim == 2 and positions.size(0) == 3, (
1281
- "multimodal section rotary embedding requires "
1282
- f"(3, seq_len) positions, but got {positions.size()}")
1283
-
1284
- inputs_embeds = self.model.embed_tokens(input_ids)
1285
-
1286
- if image_input is not None:
1287
- image_embeds = self._process_image_input(image_input)
1288
- inputs_embeds = self._merge_multimodal_embeddings(
1289
- input_ids,
1290
- inputs_embeds,
1291
- image_embeds,
1292
- placeholder_token_id=self.config.image_token_id,
1293
- )
1294
-
1295
- if video_input is not None:
1296
- video_embeds = self._process_video_input(video_input)
1297
- inputs_embeds = self._merge_multimodal_embeddings(
1298
- input_ids,
1299
- inputs_embeds,
1300
- video_embeds,
1301
- placeholder_token_id=self.config.video_token_id,
1302
- )
1322
1304
- input_ids = None
1323
+ # NOTE: In v1, inputs_embeds is always generated at model runner, this
1324
+ # condition is for v0 compatibility.
1325
+ elif inputs_embeds is None:
1326
+ multimodal_embeddings = self.get_multimodal_embeddings(**kwargs)
1327
+
1328
+ # We need to check for usage of mrope here in case there is
1329
+ # multimodal data.
1330
+ # TODO (ywang96): move this to model runner in V1.
1331
+ if multimodal_embeddings is not None and uses_mrope(self.config):
1332
+ assert positions.ndim == 2 and positions.size(0) == 3, (
1333
+ "multimodal section rotary embedding requires "
1334
+ f"(3, seq_len) positions, but got {positions.size()}")
1335
+
1336
+ inputs_embeds = self.get_input_embeddings(input_ids,
1337
+ multimodal_embeddings)
1338
+ input_ids = None
1339
1340
hidden_states = self.model(
1341
input_ids=input_ids,
vllm/model_executor/models/ultravox.py
+48
-24
@@ -449,10 +449,36 @@ class UltravoxModel(nn.Module, SupportsMultiModal, SupportsPP):
449
450
return result
451
452
- def forward(self, input_ids: torch.Tensor, positions: torch.Tensor,
452
+ def get_multimodal_embeddings(self, **kwargs) -> Optional[NestedTensors]:
453
+ audio_input = self._parse_and_validate_audio_input(**kwargs)
454
+ if audio_input is None:
455
+ return None
456
+ audio_embeddings = self._process_audio_input(audio_input)
457
+ return audio_embeddings
458
+
459
+ def get_input_embeddings(
460
+ self,
461
+ input_ids: torch.Tensor,
462
+ multimodal_embeddings: Optional[NestedTensors] = None,
463
+ attn_metadata: Optional[AttentionMetadata] = None,
464
+ ) -> torch.Tensor:
465
+ inputs_embeds = self.language_model.get_input_embeddings(input_ids)
466
+ if multimodal_embeddings is not None:
467
+
468
+ # TODO(ywang96): use merge_multimodal_embeddings after
469
+ # v0 is deprecated
470
+ merge_multimodal_embeddings_from_map(
471
+ inputs_embeds, multimodal_embeddings,
472
+ attn_metadata.multi_modal_placeholder_index_maps["audio"])
473
+ return inputs_embeds
474
+
475
+ def forward(self,
476
+ input_ids: torch.Tensor,
477
+ positions: torch.Tensor,
478
kv_caches: List[torch.Tensor],
479
attn_metadata: AttentionMetadata,
455
- intermediate_tensors: Optional[torch.Tensor],
480
+ intermediate_tensors: Optional[torch.Tensor] = None,
481
+ inputs_embeds: Optional[torch.Tensor] = None,
482
**kwargs) -> Union[torch.Tensor, IntermediateTensors]:
483
"""Run forward pass for Ultravox
484
@@ -466,30 +492,28 @@ class UltravoxModel(nn.Module, SupportsMultiModal, SupportsPP):
492
Args:
493
audio_features: A batch of audio inputs [B, N, 80, M].
494
"""
495
+
496
if intermediate_tensors is not None:
470
- input_ids = None
497
inputs_embeds = None
472
- else:
473
- audio_input = self._parse_and_validate_audio_input(**kwargs)
474
- if audio_input is not None:
475
- audio_embeddings = self._process_audio_input(audio_input)
476
- inputs_embeds = self.language_model.model.get_input_embeddings(
477
- input_ids)
478
-
479
- merge_multimodal_embeddings_from_map(
480
- inputs_embeds, audio_embeddings,
481
- attn_metadata.multi_modal_placeholder_index_maps["audio"])
482
- input_ids = None
483
- else:
484
- inputs_embeds = None
485
-
486
- hidden_states = self.language_model.model(
487
- input_ids=input_ids,
488
- positions=positions,
489
- kv_caches=kv_caches,
490
- attn_metadata=attn_metadata,
491
- intermediate_tensors=intermediate_tensors,
492
- inputs_embeds=inputs_embeds)
498
+
499
+ # NOTE: In v1, inputs_embeds is always generated at model runner, this
500
+ # condition is for v0 compatibility.
501
+ elif inputs_embeds is None:
502
+ multimodal_embeddings = self.get_multimodal_embeddings(**kwargs)
503
+
504
+ # TODO(ywang96): remove attn_metadata from get_input_embeddings
505
+ # after v0 is deprecated
506
+ inputs_embeds = self.get_input_embeddings(input_ids,
507
+ multimodal_embeddings,
508
+ attn_metadata)
509
+ input_ids = None
510
+
511
+ hidden_states = self.language_model.model(input_ids,
512
+ positions,
513
+ kv_caches,
514
+ attn_metadata,
515
+ intermediate_tensors,
516
+ inputs_embeds=inputs_embeds)
517
return hidden_states
518
519
def compute_logits(self, hidden_states: torch.Tensor,
vllm/model_executor/models/utils.py
+1
-4
@@ -356,8 +356,7 @@ def embed_multimodal(
356
input_ids: torch.Tensor,
357
multimodal_token_id: int,
358
get_text_embeds: Callable[[torch.Tensor], torch.Tensor],
359
- get_multimodal_embeds: Callable[[torch.Tensor], Union[torch.Tensor,
360
- List[torch.Tensor]]],
359
+ multimodal_embeds: Union[torch.Tensor, List[torch.Tensor]],
360
) -> torch.Tensor:
361
"""
362
Embed token IDs and multimodal inputs and combine their embeddings.
@@ -374,8 +373,6 @@ def embed_multimodal(
373
is_text = ~is_multimodal
374
375
text_embeds = get_text_embeds(input_ids[is_text])
377
- multimodal_embeds = get_multimodal_embeds(input_ids[is_multimodal])
378
-
376
merged_embeds = torch.empty(
377
(input_ids.shape[0], text_embeds.shape[1]),
378
dtype=text_embeds.dtype,
vllm/v1/worker/gpu_model_runner.py
+2
-1
@@ -363,7 +363,8 @@ class GPUModelRunner:
363
# 2. A list (length: num_images) of tensors, each of shape
364
# [feature_size, hidden_size] in case when the feature size is
365
# dynamic depending on input images.
366
- encoder_outputs = self.model.process_mm_inputs(**batched_mm_inputs)
366
+ encoder_outputs = self.model.get_multimodal_embeddings(
367
+ **batched_mm_inputs)
368
369
# Cache the encoder outputs.
370
for (req_id, input_id), output in zip(req_input_ids, encoder_outputs):