A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 11, 05:00 UTC
[BugFix] Fix spec decode + structured outputs + preemption edge case (#30916)
Signed-off-by: Nick Hill <nhill@redhat.com>
Nick Hill committed
Dec 18, 2025 at 12:59 UTC
b0b77c46551bb5762a520f2b6e3af73a4defdfab
1 file changed
+5
-1
vllm/v1/worker/gpu_model_runner.py
+5
-1
@@ -3393,9 +3393,13 @@ class GPUModelRunner(
3393
return async_output
3394
3395
def take_draft_token_ids(self) -> DraftTokenIds | None:
3396
- if self._draft_token_ids is None:
3396
+ if not self.num_spec_tokens:
3397
return None
3398
+
3399
req_ids = self.input_batch.req_ids
3400
+ if self._draft_token_ids is None:
3401
+ return DraftTokenIds(req_ids, [[] for _ in req_ids])
3402
+
3403
if isinstance(self._draft_token_ids, torch.Tensor):
3404
draft_token_ids = self._draft_token_ids.tolist()
3405
else: