A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 13, 14:30 UTC
[bugfix] support eagle with lora cudagraph specialization (#28318)
Signed-off-by: gnovack <gnovack@amazon.com>
gnovack committed
Nov 7, 2025 at 19:25 UTC
70af44fd1051b629ff22d98ebbba723e47221886
1 file changed
+12
-1
vllm/v1/worker/gpu_model_runner.py
+12
-1
@@ -3602,7 +3602,18 @@ class GPUModelRunner(LoRAModelRunnerMixin, KVConnectorModelRunnerMixin):
3602
cudagraph_runtime_mode == CUDAGraphMode.PIECEWISE
3603
and not self.speculative_config.enforce_eager
3604
)
3605
- self.drafter.dummy_run(num_tokens, use_cudagraphs=use_cudagraphs)
3605
+
3606
+ # Note(gnovack) - We need to disable cudagraphs for one of the two
3607
+ # lora cases when cudagraph_specialize_lora is enabled. This is a
3608
+ # short term mitigation for issue mentioned in
3609
+ # https://github.com/vllm-project/vllm/issues/28334
3610
+ if self.compilation_config.cudagraph_specialize_lora and activate_lora:
3611
+ use_cudagraphs = False
3612
+
3613
+ self.drafter.dummy_run(
3614
+ num_tokens,
3615
+ use_cudagraphs=use_cudagraphs,
3616
+ )
3617
3618
# This is necessary to avoid blocking DP.
3619
# For dummy runs, we typically skip EPLB since we don't have any real