A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 8, 17:45 UTC
[Fix] Fix `logprobs=0` handling for `/inference/v1/generate` endpoint (#34010)
Signed-off-by: SumanthRH <sumanthrh99@gmail.com>
Sumanth R Hegde committed
Feb 6, 2026 at 12:33 UTC
ae2e93f89b06529dba41b5c0adc1a6d27e921320
2 files changed
+29
-2
tests/entrypoints/openai/test_serving_tokens.py
+26
@@ -87,6 +87,32 @@ async def test_generate_endpoint(client):
87
assert "choices" in data
88
89
90
+@pytest.mark.asyncio
91
+@pytest.mark.parametrize("logprobs_value", [0, 1, 5])
92
+async def test_generate_logprobs(client, logprobs_value):
93
+ payload = {
94
+ "model": MODEL_NAME,
95
+ "token_ids": [1, 2, 3],
96
+ "sampling_params": {
97
+ "max_tokens": 5,
98
+ "temperature": 0.0,
99
+ "logprobs": logprobs_value,
100
+ },
101
+ "stream": False,
102
+ }
103
+ resp = await client.post(GEN_ENDPOINT, json=payload)
104
+ resp.raise_for_status()
105
+ data = resp.json()
106
+ choice = data["choices"][0]
107
+ assert choice["logprobs"] is not None
108
+ logprobs_content = choice["logprobs"]["content"]
109
+ assert len(logprobs_content) == len(choice["token_ids"])
110
+ for entry in logprobs_content:
111
+ assert "logprob" in entry
112
+ assert len(entry["top_logprobs"]) >= 1
113
+ assert len(entry["top_logprobs"]) == max(logprobs_value, 1)
114
+
115
+
116
@pytest.mark.asyncio
117
async def test_same_response_as_chat_completions(client, tokenizer, messages):
118
token_ids = tokenizer.apply_chat_template(
vllm/entrypoints/serve/disagg/serving.py
+3
-2
@@ -184,7 +184,7 @@ class ServingTokens(OpenAIServing):
184
out_logprobs = output.logprobs
185
186
# This is top_logprobs in completions API
187
- if sampling_params.logprobs:
187
+ if sampling_params.logprobs is not None:
188
assert out_logprobs is not None, "Did not output logprobs"
189
logprobs = self._create_tokens_logprobs(
190
token_ids=token_ids,
@@ -284,7 +284,8 @@ class ServingTokens(OpenAIServing):
284
logprob=max(p[1].logprob, -9999.0),
285
)
286
for i, p in enumerate(step_top_logprobs.items())
287
- if num_output_top_logprobs and i < num_output_top_logprobs
287
+ if num_output_top_logprobs is not None
288
+ and i < max(num_output_top_logprobs, 1)
289
],
290
)
291
)