A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
Mark prompt logprobs as incompatible with prompt embeds at API level (#25077)
Signed-off-by: Andrew Sansom <andrew@protopia.ai>
Andrew Sansom committed
Sep 17, 2025 at 23:25 UTC
bec060fd99e371b1adc53f65636061f702fa8e61
3 files changed
+29
-4
tests/entrypoints/openai/test_completion_with_prompt_embeds.py
+17
@@ -228,3 +228,20 @@ async def test_completions_with_logprobs_and_prompt_embeds(
228
assert max(logprobs_arg,
229
1) <= len(top_logprobs) <= logprobs_arg + 1
230
assert len(logprobs.tokens) == 5
231
+
232
+
233
+@pytest.mark.asyncio
234
+async def test_prompt_logprobs_raises_error(
235
+ client_with_prompt_embeds: openai.AsyncOpenAI):
236
+ with pytest.raises(BadRequestError, match="not compatible"):
237
+ encoded_embeds = create_dummy_embeds()
238
+ await client_with_prompt_embeds.completions.create(
239
+ model=MODEL_NAME,
240
+ prompt="",
241
+ max_tokens=5,
242
+ temperature=0.0,
243
+ extra_body={
244
+ "prompt_embeds": encoded_embeds,
245
+ "prompt_logprobs": True
246
+ },
247
+ )
vllm/engine/llm_engine.py
+7
-4
@@ -671,10 +671,13 @@ class LLMEngine:
671
arrival_time = time.time()
672
673
if (isinstance(prompt, dict)
674
- and prompt.get("prompt_embeds", None) is not None
675
- and not prompt.get("prompt_token_ids", None)):
676
- seq_len = prompt["prompt_embeds"].shape[0]
677
- prompt["prompt_token_ids"] = [0] * seq_len
674
+ and prompt.get("prompt_embeds", None) is not None):
675
+ if not prompt.get("prompt_token_ids", None):
676
+ seq_len = prompt["prompt_embeds"].shape[0]
677
+ prompt["prompt_token_ids"] = [0] * seq_len
678
+ if params.prompt_logprobs is not None:
679
+ raise ValueError(
680
+ "prompt_logprobs is not compatible with prompt embeds.")
681
682
processed_inputs = self.input_preprocessor.preprocess(
683
prompt,
vllm/entrypoints/openai/serving_completion.py
+5
@@ -112,6 +112,11 @@ class OpenAIServingCompletion(OpenAIServing):
112
return self.create_error_response(
113
"Echo is unsupported with prompt embeds.")
114
115
+ if (request.prompt_logprobs is not None
116
+ and request.prompt_embeds is not None):
117
+ return self.create_error_response(
118
+ "prompt_logprobs is not compatible with prompt embeds.")
119
+
120
request_id = (
121
f"cmpl-"
122
f"{self._base_request_id(raw_request, request.request_id)}")