@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 64bc09ba27

Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 12, 07:15 UTC

[Core] Enable `inputs_embeds_size` separate from `hidden_size` (#29741)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>

Cyrus Leung committed Nov 30, 2025 at 17:31 UTC 64bc09ba279d42bbb7f8c4fba6a049474c438774

10 files changed +123 -18

tests/models/multimodal/pooling/test_siglip.py

+6 -1

@@ -19,7 +19,12 @@ HF_IMAGE_PROMPTS = IMAGE_ASSETS.prompts(

19 }

20 )

22 -MODELS = ["google/siglip-base-patch16-224", "google/siglip2-base-patch16-224"]

22 +MODELS = [

23 + "google/siglip-base-patch16-224",

24 + "google/siglip2-base-patch16-224",

25 + # Different image embedding dim than text_config.hidden_size

26 + "google/siglip2-giant-opt-patch16-384",

27 +]

30 def _run_test(

vllm/config/model.py

+10

@@ -1202,6 +1202,16 @@ class ModelConfig:

1202 def get_hidden_size(self) -> int:

1203 return getattr(self.hf_text_config, "hidden_size", 0)

1204

1205 + def get_inputs_embeds_size(self) -> int:

1206 + # The size of inputs_embeds is usually identical to the size

1207 + # of the hidden states, however there are exceptions, such as

1208 + # embedding models like CLIP and SigLIP

1209 + for target_attr in ("projection_dim", "projection_size"):

1210 + if hasattr(self.hf_text_config, target_attr):

1211 + return getattr(self.hf_text_config, target_attr)

1212 +

1213 + return self.get_hidden_size()

1214 +

1215 @property

1216 def is_deepseek_mla(self) -> bool:

1217 if not hasattr(self.hf_text_config, "model_type"):

vllm/model_executor/models/clip.py

+46 -5

@@ -1,6 +1,6 @@

1 # SPDX-License-Identifier: Apache-2.0

2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project

3 -from collections.abc import Iterable, Mapping, Sequence

3 +from collections.abc import Callable, Iterable, Mapping, Sequence

4 from functools import cached_property

5 from typing import Annotated, Literal

@@ -903,6 +903,41 @@ class CLIPEmbeddingModel(nn.Module, SupportsMultiModal, SupportsQuant):

903 def get_language_model(self) -> torch.nn.Module:

904 return self.text_model

905

906 + def _embed_text_input_ids(

907 + self,

908 + input_ids: torch.Tensor,

909 + embed_input_ids: Callable[[torch.Tensor], torch.Tensor],

910 + *,

911 + is_multimodal: torch.Tensor | None,

912 + handle_oov_mm_token: bool,

913 + ) -> torch.Tensor:

914 + inputs_embeds = super()._embed_text_input_ids(

915 + input_ids,

916 + embed_input_ids,

917 + is_multimodal=is_multimodal,

918 + handle_oov_mm_token=handle_oov_mm_token,

919 + )

920 +

921 + # NOTE: inputs_embeds in model runner has size text_config.projection_dim

922 + # (instead of text_config.hidden_size) to accommodate image embeddings

923 + inputs_embeds_size = self.projection_dim

924 + if inputs_embeds.shape[1] < inputs_embeds_size:

925 + inputs_embeds = torch.cat(

926 + [

927 + inputs_embeds,

928 + inputs_embeds.new_empty(

929 + inputs_embeds.shape[0],

930 + inputs_embeds_size - inputs_embeds.shape[1],

931 + ),

932 + ],

933 + dim=1,

934 + )

935 + elif inputs_embeds.shape[1] > inputs_embeds_size:

936 + # No need to handle this case for now

937 + raise NotImplementedError

938 +

939 + return inputs_embeds

940 +

941 def embed_input_ids(

942 self,

943 input_ids: torch.Tensor,

@@ -949,10 +984,16 @@ class CLIPEmbeddingModel(nn.Module, SupportsMultiModal, SupportsQuant):

984 if not self._is_text_input:

985 return inputs_embeds

986

952 - # Text inputs

953 - return self.get_text_features(

954 - input_ids=input_ids, position_ids=positions, inputs_embeds=inputs_embeds

955 - )

987 + # NOTE: inputs_embeds in model runner has size text_config.projection_dim

988 + # (instead of text_config.hidden_size) to accommodate image embeddings

989 + hidden_size = self.text_embed_dim

990 + if inputs_embeds.shape[1] > hidden_size:

991 + inputs_embeds = inputs_embeds[:, :hidden_size]

992 + elif inputs_embeds.shape[1] < hidden_size:

993 + # No need to handle this case for now

994 + raise NotImplementedError

995 +

996 + return self.get_text_features(input_ids, positions, inputs_embeds)

997

998 def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]):

999 loader = AutoWeightsLoader(

vllm/model_executor/models/siglip.py

+46 -3

@@ -1,10 +1,8 @@

1 # SPDX-License-Identifier: Apache-2.0

2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project

3 -"""Implementation of SiglipVisionModel intended to be only used

4 -within a vision language model."""

4 import math

7 -from collections.abc import Iterable, Mapping

5 +from collections.abc import Callable, Iterable, Mapping

6 from functools import cached_property

7 from typing import Annotated, Literal

@@ -976,6 +974,7 @@ class SiglipTextEmbeddings(nn.Module):

974

975 position_embeddings = self.position_embedding(position_ids)

976 embeddings = inputs_embeds + position_embeddings

977 +

978 return embeddings

979

980

@@ -1145,6 +1144,41 @@ class SiglipEmbeddingModel(nn.Module, SupportsMultiModal, SupportsQuant):

1144 def get_language_model(self) -> torch.nn.Module:

1145 return self.text_model

1146

1147 + def _embed_text_input_ids(

1148 + self,

1149 + input_ids: torch.Tensor,

1150 + embed_input_ids: Callable[[torch.Tensor], torch.Tensor],

1151 + *,

1152 + is_multimodal: torch.Tensor | None,

1153 + handle_oov_mm_token: bool,

1154 + ) -> torch.Tensor:

1155 + inputs_embeds = super()._embed_text_input_ids(

1156 + input_ids,

1157 + embed_input_ids,

1158 + is_multimodal=is_multimodal,

1159 + handle_oov_mm_token=handle_oov_mm_token,

1160 + )

1161 +

1162 + # NOTE: inputs_embeds in model runner has size text_config.projection_size

1163 + # (instead of text_config.hidden_size) to accommodate image embeddings

1164 + inputs_embeds_size = self.text_projection_size

1165 + if inputs_embeds.shape[1] < inputs_embeds_size:

1166 + inputs_embeds = torch.cat(

1167 + [

1168 + inputs_embeds,

1169 + inputs_embeds.new_empty(

1170 + inputs_embeds.shape[0],

1171 + inputs_embeds_size - inputs_embeds.shape[1],

1172 + ),

1173 + ],

1174 + dim=1,

1175 + )

1176 + elif inputs_embeds.shape[1] > inputs_embeds_size:

1177 + # No need to handle this case for now

1178 + raise NotImplementedError

1179 +

1180 + return inputs_embeds

1181 +

1182 def embed_input_ids(

1183 self,

1184 input_ids: torch.Tensor,

@@ -1190,6 +1224,15 @@ class SiglipEmbeddingModel(nn.Module, SupportsMultiModal, SupportsQuant):

1224 if not self._is_text_input:

1225 return inputs_embeds

1226

1227 + # NOTE: inputs_embeds in model runner has size text_config.projection_size

1228 + # (instead of text_config.hidden_size) to accommodate image embeddings

1229 + hidden_size = self.text_embed_dim

1230 + if inputs_embeds.shape[1] > hidden_size:

1231 + inputs_embeds = inputs_embeds[:, :hidden_size]

1232 + elif inputs_embeds.shape[1] < hidden_size:

1233 + # No need to handle this case for now

1234 + raise NotImplementedError

1235 +

1236 return self.get_text_features(input_ids, positions, inputs_embeds)

1237

1238 def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]):

vllm/v1/spec_decode/eagle.py

+4 -1

@@ -80,6 +80,7 @@ class EagleProposer:

80 # the draft model's hidden size can be different from the target model's

81 # hidden size (e.g., Llama 3.3 70B).

82 self.hidden_size = self.draft_model_config.get_hidden_size()

83 + self.inputs_embeds_size = self.draft_model_config.get_inputs_embeds_size()

85 # Multi-modal data support

86 self.mm_registry = MULTIMODAL_REGISTRY

@@ -151,7 +152,9 @@ class EagleProposer:

152 )

153

154 self.inputs_embeds = torch.zeros(

154 - (self.max_num_tokens, self.hidden_size), dtype=self.dtype, device=device

155 + (self.max_num_tokens, self.inputs_embeds_size),

156 + dtype=self.dtype,

157 + device=device,

158 )

159

160 self.backup_next_token_ids = CpuGpuBuffer(

vllm/v1/worker/gpu/input_batch.py

+1 -1

@@ -17,7 +17,7 @@ class InputBuffers:

17 self,

18 max_num_reqs: int,

19 max_num_tokens: int,

20 - hidden_size: int,

20 + inputs_embeds_size: int,

21 vocab_size: int,

22 dtype: torch.dtype,

23 device: torch.device,

vllm/v1/worker/gpu/model_runner.py

+2 -2

@@ -98,7 +98,7 @@ class GPUModelRunner(LoRAModelRunnerMixin, KVConnectorModelRunnerMixin):

98 self.max_model_len = self.model_config.max_model_len

99 self.max_num_tokens = self.scheduler_config.max_num_batched_tokens

100 self.max_num_reqs = self.scheduler_config.max_num_seqs

101 - self.hidden_size = self.model_config.get_hidden_size()

101 + self.inputs_embeds_size = self.model_config.get_inputs_embeds_size()

102

103 self.dp_size = self.parallel_config.data_parallel_size

104 self.dp_rank = self.parallel_config.data_parallel_rank

@@ -134,7 +134,7 @@ class GPUModelRunner(LoRAModelRunnerMixin, KVConnectorModelRunnerMixin):

134 self.input_buffers = InputBuffers(

135 max_num_reqs=self.max_num_reqs,

136 max_num_tokens=self.max_num_tokens,

137 - hidden_size=self.hidden_size,

137 + inputs_embeds_size=self.inputs_embeds_size,

138 vocab_size=self.vocab_size,

139 dtype=self.dtype,

140 device=self.device,

vllm/v1/worker/gpu/spec_decode/eagle.py

+2 -1

@@ -44,6 +44,7 @@ class EagleSpeculator:

44 # the draft model's hidden size can be different from the target model's

45 # hidden size (e.g., Llama 3.3 70B).

46 self.hidden_size = self.draft_model_config.get_hidden_size()

47 + self.inputs_embeds_size = self.draft_model_config.get_inputs_embeds_size()

48 self.vocab_size = self.draft_model_config.get_vocab_size()

49 self.pin_memory = is_pin_memory_available()

50 self.dtype = vllm_config.model_config.dtype

@@ -51,7 +52,7 @@ class EagleSpeculator:

52 self.input_buffers = InputBuffers(

53 max_num_reqs=self.max_num_reqs,

54 max_num_tokens=self.max_num_tokens,

54 - hidden_size=self.hidden_size,

55 + inputs_embeds_size=self.inputs_embeds_size,

56 vocab_size=self.vocab_size,

57 dtype=self.dtype,

58 device=device,

vllm/v1/worker/gpu_model_runner.py

+2 -2

@@ -320,7 +320,7 @@ class GPUModelRunner(

320

321 # Model-related.

322 self.num_query_heads = model_config.get_num_attention_heads(parallel_config)

323 - self.hidden_size = model_config.get_hidden_size()

323 + self.inputs_embeds_size = model_config.get_inputs_embeds_size()

324 self.attention_chunk_size = model_config.attention_chunk_size

325 # Only relevant for models using ALiBi (e.g, MPT)

326 self.use_alibi = model_config.uses_alibi

@@ -485,7 +485,7 @@ class GPUModelRunner(

485 # version of this tensor, avoid a RuntimeError by not creating a

486 # numpy buffer.

487 self.inputs_embeds = self._make_buffer(

488 - self.max_num_tokens, self.hidden_size, dtype=self.dtype, numpy=False

488 + self.max_num_tokens, self.inputs_embeds_size, dtype=self.dtype, numpy=False

489 )

490 self.is_token_ids = self._make_buffer(self.max_num_tokens, dtype=torch.bool)

491 self.discard_request_mask = self._make_buffer(

vllm/v1/worker/tpu_model_runner.py

+4 -2

@@ -215,7 +215,7 @@ class TPUModelRunner(LoRAModelRunnerMixin, KVConnectorModelRunnerMixin):

215 self.num_query_heads = model_config.get_num_attention_heads(parallel_config)

216 self.num_kv_heads = model_config.get_num_kv_heads(parallel_config)

217 self.head_size = model_config.get_head_size()

218 - self.hidden_size = model_config.get_hidden_size()

218 + self.inputs_embeds_size = model_config.get_inputs_embeds_size()

219 self.vocab_size = model_config.get_vocab_size()

220

221 # Multi-modal data support

@@ -1406,7 +1406,9 @@ class TPUModelRunner(LoRAModelRunnerMixin, KVConnectorModelRunnerMixin):

1406 if self.supports_mm_inputs:

1407 input_ids = None

1408 inputs_embeds = torch.zeros(

1409 - (num_tokens, self.hidden_size), dtype=self.dtype, device=self.device

1409 + (num_tokens, self.inputs_embeds_size),

1410 + dtype=self.dtype,

1411 + device=self.device,

1412 )

1413 else:

1414 input_ids = torch.zeros((num_tokens), dtype=torch.int32).to(self.device)