A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 11, 13:45 UTC
[bugfix] fix bug when top_logprobs=0 with spec decoding (#30059)
Signed-off-by: realliujiaxu <realliujiaxu@163.com>
realliujiaxu committed
Dec 13, 2025 at 01:03 UTC
d2c919dcc20b1ea77a94fa01e813ebbb31f8a66a
3 files changed
+5
-3
tests/v1/sample/test_logprobs.py
+3
-1
@@ -528,9 +528,11 @@ def test_logprobs_mode(logprobs_mode: LogprobsMode):
528
),
529
],
530
)
531
+@pytest.mark.parametrize("top_logprobs", [0, 3])
532
def test_spec_decode_logprobs(
533
logprobs_mode: LogprobsMode,
534
model_setup: tuple[str, str, str],
535
+ top_logprobs: int,
536
):
537
"""Spec decode logprobs should match those of the base model.
538
@@ -543,7 +545,7 @@ def test_spec_decode_logprobs(
545
546
prompt = "Hello world " * 50
547
sampling_params = SamplingParams(
546
- temperature=0, logprobs=3, max_tokens=10, ignore_eos=False
548
+ temperature=0, logprobs=top_logprobs, max_tokens=10, ignore_eos=False
549
)
550
method, model_name, spec_model_name = model_setup
551
max_model_len = 256
tests/v1/sample/test_rejection_sampler.py
+1
-1
@@ -111,7 +111,7 @@ def create_sampling_metadata(
111
top_p=top_p,
112
top_k=top_k,
113
generators=generators,
114
- max_num_logprobs=0,
114
+ max_num_logprobs=None,
115
no_penalties=no_penalties,
116
prompt_token_ids=prompt_token_ids,
117
frequency_penalties=frequency_penalties,
vllm/v1/sample/rejection_sampler.py
+1
-1
@@ -145,7 +145,7 @@ class RejectionSampler(nn.Module):
145
)
146
147
logprobs_tensors = None
148
- if sampling_metadata.max_num_logprobs:
148
+ if sampling_metadata.max_num_logprobs is not None:
149
logprobs_tensors = self._get_logprobs_tensors(
150
sampling_metadata.max_num_logprobs,
151
metadata,