A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[Frontend] Support setting logprobs to -1 (#25031)
Signed-off-by: chaunceyjiang <chaunceyjiang@gmail.com>
Chauncey committed
Sep 18, 2025 at 18:34 UTC
cc935fdd7e0c466cd556b6515e435dddd78677e0
2 files changed
+28
-3
tests/entrypoints/openai/test_chat_echo.py
+23
@@ -99,3 +99,26 @@ async def test_prompt_logprobs(client: openai.AsyncOpenAI):
99
100
assert completion.prompt_logprobs is not None
101
assert len(completion.prompt_logprobs) > 0
102
+
103
+
104
+@pytest.mark.asyncio
105
+async def test_top_logprobs(client: openai.AsyncOpenAI):
106
+ messages = [{
107
+ "role": "system",
108
+ "content": "You are a helpful assistant."
109
+ }, {
110
+ "role": "user",
111
+ "content": "Beijing is the capital of which country?"
112
+ }]
113
+
114
+ completion = await client.chat.completions.create(
115
+ model=MODEL_NAME,
116
+ messages=messages,
117
+ extra_body={
118
+ "top_logprobs": -1,
119
+ "logprobs": "true",
120
+ },
121
+ )
122
+ assert completion.choices[0].logprobs is not None
123
+ assert completion.choices[0].logprobs.content is not None
124
+ assert len(completion.choices[0].logprobs.content) > 0
vllm/entrypoints/openai/protocol.py
+5
-3
@@ -832,10 +832,12 @@ class ChatCompletionRequest(OpenAIBaseModel):
832
raise ValueError("`prompt_logprobs=-1` is only supported with "
833
"vLLM engine V1.")
834
if (top_logprobs := data.get("top_logprobs")) is not None:
835
- if top_logprobs < 0:
836
- raise ValueError("`top_logprobs` must be a positive value.")
835
+ if top_logprobs < 0 and top_logprobs != -1:
836
+ raise ValueError(
837
+ "`top_logprobs` must be a positive value or -1.")
838
838
- if top_logprobs > 0 and not data.get("logprobs"):
839
+ if (top_logprobs == -1
840
+ or top_logprobs > 0) and not data.get("logprobs"):
841
raise ValueError(
842
"when using `top_logprobs`, `logprobs` must be set to true."
843
)