A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[Bugfix] Fix error with penalties when speculative decoding and structural output are enabled (#26586)
Signed-off-by: southfreebird <yvorott@gmail.com>
Sergei Skvortsov committed
Oct 19, 2025 at 20:24 UTC
f6fdacd82c98cb04368a6f8b75bfb1d383057425
1 file changed
+8
-2
vllm/v1/worker/gpu_model_runner.py
+8
-2
@@ -780,7 +780,7 @@ class GPUModelRunner(LoRAModelRunnerMixin, KVConnectorModelRunnerMixin):
780
781
# Add spec_token_ids to token_ids_cpu.
782
spec_token_ids = scheduler_output.scheduled_spec_decode_tokens.get(
783
- req_id, ()
783
+ req_id, []
784
)
785
if spec_token_ids:
786
num_spec_tokens = len(spec_token_ids)
@@ -791,7 +791,13 @@ class GPUModelRunner(LoRAModelRunnerMixin, KVConnectorModelRunnerMixin):
791
] = spec_token_ids
792
# NOTE(woosuk): `num_tokens` here may include spec tokens.
793
self.input_batch.num_tokens[req_index] += num_spec_tokens
794
- self.input_batch.spec_token_ids[req_index] = spec_token_ids
794
+
795
+ # When speculative decoding is used with structured output,
796
+ # the scheduler can drop draft tokens that do not
797
+ # conform to the schema. This can result in
798
+ # scheduler_output.scheduled_spec_decode_tokens being empty,
799
+ # even when speculative decoding is enabled.
800
+ self.input_batch.spec_token_ids[req_index] = spec_token_ids
801
802
# Add the new or resumed requests to the persistent batch.
803
# The smaller empty indices are filled first.