A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[Bugfix] fixed top_logprobs: -1 does not appear to work as intended (#26470)
Signed-off-by: chaunceyjiang <chaunceyjiang@gmail.com>
Chauncey committed
Oct 11, 2025 at 00:41 UTC
910abdbd0866b203df2eb4e5419cada905d3803d
2 files changed
+16
-1
tests/entrypoints/openai/test_chat_echo.py
+15
@@ -7,12 +7,23 @@ import openai # use the official client for correctness check
7
import pytest
8
import pytest_asyncio
9
10
+from vllm.config import ModelConfig
11
+
12
from ...utils import RemoteOpenAIServer
13
14
# # any model with a chat template should work here
15
MODEL_NAME = "Qwen/Qwen2-1.5B-Instruct"
16
17
18
+def get_vocab_size(model_name):
19
+ config = ModelConfig(
20
+ model=model_name,
21
+ seed=0,
22
+ dtype="float16",
23
+ )
24
+ return config.get_vocab_size()
25
+
26
+
27
@pytest.fixture(scope="module")
28
def server():
29
args = [
@@ -107,6 +118,7 @@ async def test_top_logprobs(client: openai.AsyncOpenAI):
118
completion = await client.chat.completions.create(
119
model=MODEL_NAME,
120
messages=messages,
121
+ max_tokens=1,
122
extra_body={
123
"top_logprobs": -1,
124
"logprobs": "true",
@@ -115,3 +127,6 @@ async def test_top_logprobs(client: openai.AsyncOpenAI):
127
assert completion.choices[0].logprobs is not None
128
assert completion.choices[0].logprobs.content is not None
129
assert len(completion.choices[0].logprobs.content) > 0
130
+ assert len(
131
+ completion.choices[0].logprobs.content[0].top_logprobs
132
+ ) == get_vocab_size(MODEL_NAME)
vllm/entrypoints/openai/serving_chat.py
+1
-1
@@ -1643,7 +1643,7 @@ class OpenAIServingChat(OpenAIServing):
1643
bytes=list(token.encode("utf-8", errors="replace")),
1644
)
1645
for i, p in enumerate(logprobs.items())
1646
- if top_logprobs and i < top_logprobs
1646
+ if (top_logprobs and i < top_logprobs or top_logprobs == -1)
1647
]
1648
1649
def _create_chat_logprobs(