A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 9, 20:45 UTC
[Bugfix] Fix potential EAGLE spec decode segfault during graph capture (#32818)
Signed-off-by: Matthew Wong <Matthew.Wong2@amd.com>
Matt committed
Jan 21, 2026 at 21:11 UTC
c5487e2b96b2627e8c2def872449e3e43246f6a8
1 file changed
+8
-4
vllm/v1/spec_decode/eagle.py
+8
-4
@@ -1222,10 +1222,14 @@ class SpecDecodeBaseProposer:
1222
num_tokens_dp_padded, num_tokens_across_dp = self._pad_batch_across_dp(
1223
num_tokens_unpadded=num_tokens, num_tokens_padded=num_tokens
1224
)
1225
- cudagraph_runtime_mode, batch_desc = self.cudagraph_dispatcher.dispatch(
1226
- num_tokens_dp_padded
1227
- )
1228
- num_input_tokens = batch_desc.num_tokens
1225
+ if use_cudagraphs:
1226
+ cudagraph_runtime_mode, batch_desc = (
1227
+ self.cudagraph_dispatcher.dispatch(num_tokens_dp_padded)
1228
+ )
1229
+ num_input_tokens = batch_desc.num_tokens
1230
+ else:
1231
+ cudagraph_runtime_mode = CUDAGraphMode.NONE
1232
+ num_input_tokens = num_tokens_dp_padded
1233
if num_tokens_across_dp is not None:
1234
num_tokens_across_dp[self.dp_rank] = num_input_tokens
1235