A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[Bugfix] Token type and position embeddings fail to be applied to `inputs_embeds` (#25922)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
Cyrus Leung committed
Oct 1, 2025 at 00:23 UTC
9f1c4ecaf2aa6fa2a53d870304126d1d950842c5
2 files changed
+14
-9
vllm/model_executor/models/bert.py
+10
-7
@@ -61,11 +61,13 @@ class BertEmbedding(nn.Module):
61
self,
62
input_ids: torch.Tensor,
63
position_ids: torch.Tensor,
64
+ inputs_embeds: Optional[torch.Tensor] = None,
65
) -> torch.Tensor:
65
-
66
token_type_ids = _decode_token_type_ids(input_ids)
67
68
- inputs_embeds = self.word_embeddings(input_ids)
68
+ if inputs_embeds is None:
69
+ inputs_embeds = self.word_embeddings(input_ids)
70
+
71
position_embeddings = self.position_embeddings(position_ids)
72
73
token_type_embeddings = self.token_type_embeddings(token_type_ids)
@@ -358,11 +360,12 @@ class BertModel(nn.Module, SupportsQuant):
360
intermediate_tensors: Optional[IntermediateTensors] = None,
361
inputs_embeds: Optional[torch.Tensor] = None,
362
) -> torch.Tensor:
361
- if inputs_embeds is not None:
362
- hidden_states = inputs_embeds
363
- else:
364
- hidden_states = self.embeddings(input_ids=input_ids,
365
- position_ids=positions)
363
+ hidden_states = self.embeddings(
364
+ input_ids=input_ids,
365
+ position_ids=positions,
366
+ inputs_embeds=inputs_embeds,
367
+ )
368
+
369
return self.encoder(hidden_states)
370
371
def _load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]):
vllm/model_executor/models/roberta.py
+4
-2
@@ -56,11 +56,13 @@ class RobertaEmbedding(nn.Module):
56
self,
57
input_ids: torch.Tensor,
58
position_ids: torch.Tensor,
59
+ inputs_embeds: Optional[torch.Tensor] = None,
60
) -> torch.Tensor:
60
-
61
token_type_ids = _decode_token_type_ids(input_ids)
62
63
- inputs_embeds = self.word_embeddings(input_ids)
63
+ if inputs_embeds is None:
64
+ inputs_embeds = self.word_embeddings(input_ids)
65
+
66
position_embeddings = self.position_embeddings(position_ids)
67
68
token_type_embeddings = self.token_type_embeddings(token_type_ids)