A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 3, 08:00 UTC
[Bugfix] Fix top logprobs token placeholders in `/inference/v1/generate` (#42887)
Signed-off-by: Sage Ahrac <sagiahrak@gmail.com>
Sage committed
May 19, 2026 at 13:21 UTC
a78b842d0e85d287176031334f4721cd96b6e47d
2 files changed
+33
-3
tests/entrypoints/serve/disagg/test_tokens_logprobs.py
new
+28
@@ -0,0 +1,28 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+
4
+from vllm.entrypoints.serve.disagg.serving import ServingTokens
5
+from vllm.logprobs import Logprob
6
+
7
+
8
+def test_top_logprobs_alternatives_have_own_token_ids():
9
+ """Each top_logprobs alternative must carry its own token_id placeholder."""
10
+ result = ServingTokens._create_tokens_logprobs(
11
+ None,
12
+ token_ids=[262],
13
+ top_logprobs=[{262: Logprob(-0.1), 257: Logprob(-1.2), 428: Logprob(-2.3)}],
14
+ num_output_top_logprobs=2,
15
+ )
16
+ tokens = {e.token for e in result.content[0].top_logprobs}
17
+ assert tokens == {"token_id:262", "token_id:257"}, f"got {tokens}"
18
+
19
+
20
+def test_logprobs_zero_emits_sampled_token():
21
+ """logprobs=0 must still emit 1 entry (the sampled token)."""
22
+ result = ServingTokens._create_tokens_logprobs(
23
+ None,
24
+ token_ids=[7],
25
+ top_logprobs=[{7: Logprob(-0.9), 8: Logprob(-1.1)}],
26
+ num_output_top_logprobs=0,
27
+ )
28
+ assert len(result.content[0].top_logprobs) == 1
vllm/entrypoints/serve/disagg/serving.py
+5
-3
@@ -460,10 +460,12 @@ class ServingTokens(OpenAIServing):
460
logprob=max(step_token.logprob, -9999.0),
461
top_logprobs=[
462
ChatCompletionLogProb(
463
- token=token,
464
- logprob=max(p[1].logprob, -9999.0),
463
+ token=f"token_id:{token_id}",
464
+ logprob=max(logprob.logprob, -9999.0),
465
+ )
466
+ for i, (token_id, logprob) in enumerate(
467
+ step_top_logprobs.items()
468
)
466
- for i, p in enumerate(step_top_logprobs.items())
469
if num_output_top_logprobs is not None
470
and i < max(num_output_top_logprobs, 1)
471
],