A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[Bugfix] skip cuda graph for drafter when running with eager (#26821)
Signed-off-by: Benjamin Chislett <bchislett@nvidia.com>
Benjamin Chislett committed
Oct 21, 2025 at 18:39 UTC
19748806f04e6e390b42c2318a5ca76ffb6c1368
1 file changed
+4
-1
vllm/v1/worker/gpu_model_runner.py
+4
-1
@@ -3482,7 +3482,10 @@ class GPUModelRunner(LoRAModelRunnerMixin, KVConnectorModelRunnerMixin):
3482
3483
if self.speculative_config and self.speculative_config.use_eagle():
3484
assert isinstance(self.drafter, EagleProposer)
3485
- use_cudagraphs = cudagraph_runtime_mode == CUDAGraphMode.PIECEWISE
3485
+ use_cudagraphs = (
3486
+ cudagraph_runtime_mode == CUDAGraphMode.PIECEWISE
3487
+ and not self.speculative_config.enforce_eager
3488
+ )
3489
self.drafter.dummy_run(num_tokens, use_cudagraphs=use_cudagraphs)
3490
3491
# This is necessary to avoid blocking DP.