A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[Frontend] Support returning all prompt logprobs (#24956)
Signed-off-by: chaunceyjiang <chaunceyjiang@gmail.com>
Chauncey committed
Sep 17, 2025 at 17:03 UTC
544fe76b95aacdb6d0636c41813bee6236fb0027
2 files changed
+38
-8
tests/entrypoints/openai/test_chat_echo.py
+22
@@ -22,6 +22,8 @@ def server():
22
"--enforce-eager",
23
"--max-model-len",
24
"4080",
25
+ "--max-logprobs", # test prompt_logprobs equal to -1
26
+ "151936"
27
]
28
29
with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:
@@ -77,3 +79,23 @@ async def test_chat_session_with_echo_and_continue_final_message(
79
else:
80
assert message.content is not None and saying not in message.content
81
assert message.role == "assistant"
82
+
83
+
84
+@pytest.mark.asyncio
85
+async def test_prompt_logprobs(client: openai.AsyncOpenAI):
86
+ messages = [{
87
+ "role": "system",
88
+ "content": "You are a helpful assistant."
89
+ }, {
90
+ "role": "user",
91
+ "content": "Beijing is the capital of which country?"
92
+ }]
93
+
94
+ completion = await client.chat.completions.create(
95
+ model=MODEL_NAME,
96
+ messages=messages,
97
+ extra_body={"prompt_logprobs": -1},
98
+ )
99
+
100
+ assert completion.prompt_logprobs is not None
101
+ assert len(completion.prompt_logprobs) > 0
vllm/entrypoints/openai/protocol.py
+16
-8
@@ -822,13 +822,17 @@ class ChatCompletionRequest(OpenAIBaseModel):
822
@classmethod
823
def check_logprobs(cls, data):
824
if (prompt_logprobs := data.get("prompt_logprobs")) is not None:
825
- if data.get("stream") and prompt_logprobs > 0:
825
+ if data.get("stream") and (prompt_logprobs > 0
826
+ or prompt_logprobs == -1):
827
raise ValueError(
828
"`prompt_logprobs` are not available when `stream=True`.")
829
829
- if prompt_logprobs < 0:
830
- raise ValueError("`prompt_logprobs` must be a positive value.")
831
-
830
+ if prompt_logprobs < 0 and prompt_logprobs != -1:
831
+ raise ValueError(
832
+ "`prompt_logprobs` must be a positive value or -1.")
833
+ if prompt_logprobs == -1 and not envs.VLLM_USE_V1:
834
+ raise ValueError("`prompt_logprobs=-1` is only supported with "
835
+ "vLLM engine V1.")
836
if (top_logprobs := data.get("top_logprobs")) is not None:
837
if top_logprobs < 0:
838
raise ValueError("`top_logprobs` must be a positive value.")
@@ -1246,13 +1250,17 @@ class CompletionRequest(OpenAIBaseModel):
1250
@classmethod
1251
def check_logprobs(cls, data):
1252
if (prompt_logprobs := data.get("prompt_logprobs")) is not None:
1249
- if data.get("stream") and prompt_logprobs > 0:
1253
+ if data.get("stream") and (prompt_logprobs > 0
1254
+ or prompt_logprobs == -1):
1255
raise ValueError(
1256
"`prompt_logprobs` are not available when `stream=True`.")
1257
1253
- if prompt_logprobs < 0:
1254
- raise ValueError("`prompt_logprobs` must be a positive value.")
1255
-
1258
+ if prompt_logprobs < 0 and prompt_logprobs != -1:
1259
+ raise ValueError(
1260
+ "`prompt_logprobs` must be a positive value or -1.")
1261
+ if prompt_logprobs == -1 and not envs.VLLM_USE_V1:
1262
+ raise ValueError("`prompt_logprobs=-1` is only supported with "
1263
+ "vLLM engine V1.")
1264
if (logprobs := data.get("logprobs")) is not None and logprobs < 0:
1265
raise ValueError("`logprobs` must be a positive value.")
1266