A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
Fix implementation divergence for BLOOM models between vLLM and HuggingFace when using prompt embeds (#24686)
Signed-off-by: Andrew Sansom <andrew@protopia.ai>
Andrew Sansom committed
Sep 11, 2025 at 23:35 UTC
ddcec289c7f3da664aa8b74f122d0330d17da3bc
2 files changed
+4
-5
tests/models/language/generation/test_common.py
+2
-4
@@ -1,6 +1,5 @@
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
-import os
3
from typing import Optional
4
5
import pytest
@@ -99,9 +98,10 @@ AITER_MODEL_LIST = [
98
@pytest.mark.parametrize("num_logprobs", [5])
99
@pytest.mark.parametrize(
100
"use_rocm_aiter", [True, False] if current_platform.is_rocm() else [False])
101
+@pytest.mark.parametrize("use_prompt_embeds", [True, False])
102
def test_models(hf_runner, vllm_runner, example_prompts, model: str,
103
max_tokens: int, num_logprobs: int, use_rocm_aiter: bool,
104
- monkeypatch) -> None:
104
+ use_prompt_embeds: bool, monkeypatch) -> None:
105
106
model_info = HF_EXAMPLE_MODELS.find_hf_info(model)
107
model_info.check_available_online(on_fail="skip")
@@ -119,8 +119,6 @@ def test_models(hf_runner, vllm_runner, example_prompts, model: str,
119
# in parts of the operators
120
pytest.skip(f"Skipping '{model}' model test with AITER kernel.")
121
122
- use_prompt_embeds = os.getenv("VLLM_USE_V1") == "0"
123
-
122
with hf_runner(model) as hf_model:
123
hf_outputs = hf_model.generate_greedy_logprobs_limit(
124
example_prompts, max_tokens, num_logprobs)
vllm/model_executor/models/bloom.py
+2
-1
@@ -257,7 +257,7 @@ class BloomModel(nn.Module):
257
config.hidden_size))
258
259
def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
260
- return self.word_embeddings_layernorm(self.word_embeddings(input_ids))
260
+ return self.word_embeddings(input_ids)
261
262
def forward(
263
self,
@@ -271,6 +271,7 @@ class BloomModel(nn.Module):
271
hidden_states = inputs_embeds
272
else:
273
hidden_states = self.get_input_embeddings(input_ids)
274
+ hidden_states = self.word_embeddings_layernorm(hidden_states)
275
else:
276
assert intermediate_tensors is not None
277
hidden_states = intermediate_tensors["hidden_states"]