A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[V0 Deprecation] Remove V0 logic from `get_input_embeddings` interface (#25242)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
Cyrus Leung committed
Sep 19, 2025 at 19:10 UTC
5089fd749cbe4233a29f29ce706d56c47464c117
4 files changed
+22
-84
vllm/model_executor/models/hyperclovax_vision.py
+17
-28
@@ -46,7 +46,8 @@ from vllm.sequence import IntermediateTensors
46
from .clip import CLIPVisionModel
47
from .interfaces import MultiModalEmbeddings, SupportsMultiModal, SupportsPP
48
from .siglip import SiglipVisionModel
49
-from .utils import AutoWeightsLoader, init_vllm_registered_model, maybe_prefix
49
+from .utils import (AutoWeightsLoader, init_vllm_registered_model,
50
+ maybe_prefix, merge_multimodal_embeddings)
51
from .vision import get_vision_encoder_info
52
53
EOT = "<|endofturn|>"
@@ -740,33 +741,20 @@ class HCXVisionForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):
741
self,
742
input_ids: torch.Tensor,
743
multimodal_embeddings: Optional[MultiModalEmbeddings] = None,
743
- **kwargs,
744
) -> torch.Tensor:
745
inputs_embeds = self.language_model.get_input_embeddings(input_ids)
746
- if (kwargs.get("pixel_values_images") is not None
747
- or kwargs.get("pixel_values_videos")
748
- is not None): # v0 compatibility
749
- multimodal_embeddings = self.get_multimodal_embeddings(**kwargs)
750
- if multimodal_embeddings is not None:
751
- multimodal_embeddings = torch.cat(multimodal_embeddings, dim=0)
752
- _mask_image = input_ids == self.config.image_token_id
753
- _mask_video = input_ids == self.config.video_token_id
754
- assert _mask_image.sum() + _mask_video.sum() == len(
755
- multimodal_embeddings)
756
-
757
- if multimodal_embeddings.dtype != inputs_embeds.dtype:
758
- multimodal_embeddings = multimodal_embeddings.to(
759
- dtype=inputs_embeds.dtype)
760
- if multimodal_embeddings.device != inputs_embeds.device:
761
- multimodal_embeddings = multimodal_embeddings.to(
762
- device=inputs_embeds.device)
763
-
764
- if _mask_image.sum() > 0:
765
- inputs_embeds[
766
- _mask_image] = multimodal_embeddings[:sum(_mask_image)]
767
- if _mask_video.sum() > 0:
768
- inputs_embeds[_mask_video] = multimodal_embeddings[
769
- -sum(_mask_video):]
746
+ if multimodal_embeddings is not None \
747
+ and len(multimodal_embeddings) != 0:
748
+ inputs_embeds = merge_multimodal_embeddings(
749
+ input_ids,
750
+ inputs_embeds,
751
+ multimodal_embeddings,
752
+ placeholder_token_id=[
753
+ self.config.image_token_id,
754
+ self.config.video_token_id,
755
+ ],
756
+ )
757
+
758
return inputs_embeds
759
760
def forward(
@@ -783,8 +771,9 @@ class HCXVisionForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):
771
# NOTE: In v1, inputs_embeds is always generated at model runner, this
772
# condition is for v0 compatibility.
773
elif inputs_embeds is None:
786
- inputs_embeds = self.get_input_embeddings(input_ids=input_ids,
787
- **kwargs)
774
+ multimodal_embeddings = self.get_multimodal_embeddings(**kwargs)
775
+ inputs_embeds = self.get_input_embeddings(input_ids,
776
+ multimodal_embeddings)
777
input_ids = None
778
hidden_states = self.language_model.model(input_ids,
779
positions,
vllm/model_executor/models/interfaces.py
-24
@@ -23,7 +23,6 @@ from vllm.utils import supports_kw
23
from .interfaces_base import is_pooling_model
24
25
if TYPE_CHECKING:
26
- from vllm.attention import AttentionMetadata
26
from vllm.config import VllmConfig
27
from vllm.model_executor.models.utils import WeightsMapper
28
from vllm.sequence import IntermediateTensors
@@ -97,33 +96,10 @@ class SupportsMultiModal(Protocol):
96
"""
97
...
98
100
- # Only for models that support v0 chunked prefill
101
- # TODO(ywang96): Remove this overload once v0 is deprecated
102
- @overload
99
def get_input_embeddings(
100
self,
101
input_ids: Tensor,
102
multimodal_embeddings: Optional[MultiModalEmbeddings] = None,
107
- attn_metadata: Optional["AttentionMetadata"] = None,
108
- ) -> Tensor:
109
- ...
110
-
111
- # TODO: Remove this overload once v0 is deprecated
112
- @overload
113
- def get_input_embeddings(
114
- self,
115
- input_ids: Tensor,
116
- multimodal_embeddings: Optional[MultiModalEmbeddings] = None,
117
- ) -> Tensor:
118
- ...
119
-
120
- def get_input_embeddings(
121
- self,
122
- input_ids: Tensor,
123
- multimodal_embeddings: Optional[MultiModalEmbeddings] = None,
124
- # Only necessary so that the v0 overload is valid
125
- # TODO: Remove attn_metadata once v0 is deprecated
126
- attn_metadata: Optional["AttentionMetadata"] = None,
103
) -> Tensor:
104
"""
105
Returns the input embeddings merged from the text embeddings from
vllm/model_executor/models/ultravox.py
+4
-15
@@ -13,9 +13,7 @@ from transformers import BatchFeature, ProcessorMixin
13
from transformers.models.whisper import WhisperFeatureExtractor
14
from transformers.models.whisper.modeling_whisper import WhisperEncoder
15
16
-from vllm import envs
16
from vllm.config import VllmConfig
18
-from vllm.forward_context import get_forward_context
17
from vllm.model_executor.layers.activation import MulAndSilu, get_act_fn
18
from vllm.model_executor.layers.layernorm import RMSNorm
19
from vllm.model_executor.model_loader import DefaultModelLoader
@@ -37,8 +35,7 @@ from .interfaces import (MultiModalEmbeddings, SupportsLoRA,
35
SupportsMultiModal, SupportsPP)
36
from .utils import (AutoWeightsLoader, WeightsMapper, flatten_bn,
37
init_vllm_registered_model, maybe_prefix,
40
- merge_multimodal_embeddings,
41
- merge_multimodal_embeddings_from_map)
38
+ merge_multimodal_embeddings)
39
40
_AUDIO_PLACEHOLDER_OVERRIDE = "<|audio|>"
41
_MAX_ENCODER_BATCH_SIZE = 16
@@ -568,17 +565,9 @@ class UltravoxModel(nn.Module, SupportsMultiModal, SupportsPP, SupportsLoRA):
565
safe_input_ids)
566
if multimodal_embeddings is not None and len(
567
multimodal_embeddings) > 0:
571
-
572
- # TODO(ywang96): remove this block after v0 is deprecated.
573
- if not envs.VLLM_USE_V1:
574
- attn_metadata = get_forward_context().attn_metadata
575
- merge_multimodal_embeddings_from_map(
576
- inputs_embeds, multimodal_embeddings,
577
- attn_metadata.multi_modal_placeholder_index_maps["audio"])
578
- else:
579
- inputs_embeds = merge_multimodal_embeddings(
580
- input_ids, inputs_embeds, multimodal_embeddings,
581
- self.config.audio_token_index)
568
+ inputs_embeds = merge_multimodal_embeddings(
569
+ input_ids, inputs_embeds, multimodal_embeddings,
570
+ self.config.audio_token_index)
571
return inputs_embeds
572
573
def forward(self,
vllm/model_executor/models/utils.py
+1
-17
@@ -15,7 +15,7 @@ import vllm.envs as envs
15
from vllm.config import VllmConfig
16
from vllm.logger import init_logger
17
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
18
-from vllm.multimodal import MultiModalPlaceholderMap, NestedTensors
18
+from vllm.multimodal import NestedTensors
19
from vllm.sequence import IntermediateTensors
20
from vllm.utils import (get_cuda_view_from_cpu_tensor, is_pin_memory_available,
21
is_uva_available)
@@ -389,22 +389,6 @@ def _embedding_count_expression(embeddings: NestedTensors) -> str:
389
_embedding_count_expression(inner) for inner in embeddings)
390
391
392
-def merge_multimodal_embeddings_from_map(
393
- inputs_embeds: torch.Tensor, multimodal_embeddings: NestedTensors,
394
- placeholder_map: MultiModalPlaceholderMap.IndexMap) -> torch.Tensor:
395
- """
396
- Merge ``multimodal_embeddings`` into ``inputs_embeds`` using the provided
397
- placeholder map .
398
-
399
- Note:
400
- This updates ``inputs_embeds`` in place.
401
- """
402
- flattened_embeddings = _flatten_embeddings(multimodal_embeddings)
403
- inputs_embeds[placeholder_map.dest] = flattened_embeddings[
404
- placeholder_map.src].to(dtype=inputs_embeds.dtype)
405
- return inputs_embeds
406
-
407
-
392
def _merge_multimodal_embeddings(
393
inputs_embeds: torch.Tensor,
394
is_multimodal: torch.Tensor,