A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 13, 22:00 UTC
[Bugfix] DeepSeek V3.2 MTP metadata & CUDA graph issues (#26779)
Signed-off-by: xiaohajiayou <923390377@qq.com>
Haco committed
Nov 1, 2025 at 22:52 UTC
d811b442d305b33b3aca2836c5d7f761effe76de
1 file changed
+8
-5
vllm/v1/spec_decode/eagle.py
+8
-5
@@ -109,6 +109,7 @@ class EagleProposer:
109
else []
110
)
111
112
+ self.use_cuda_graph = self.use_cuda_graph and bool(self.cudagraph_batch_sizes)
113
# persistent buffers for cuda graph
114
self.input_ids = torch.zeros(
115
self.max_num_tokens, dtype=torch.int32, device=device
@@ -939,7 +940,7 @@ class EagleProposer:
940
self.vllm_config, DeepseekV32IndexerCache
941
)
942
draft_indexer_layer_names = indexer_layers.keys() - target_indexer_layer_names
942
- self.attn_layer_names = list(draft_attn_layer_names)
943
+ self.attn_layer_names = list(draft_attn_layer_names - draft_indexer_layer_names)
944
self.indexer_layer_names = list(draft_indexer_layer_names)
945
946
if self.indexer_layer_names:
@@ -1050,16 +1051,18 @@ class EagleProposer:
1051
num_tokens: int,
1052
use_cudagraphs=True,
1053
) -> None:
1053
- if use_cudagraphs and num_tokens <= self.cudagraph_batch_sizes[-1]:
1054
+ # Determine if CUDA graphs should be used for this run.
1055
+ cudagraphs_enabled = use_cudagraphs and self.use_cuda_graph
1056
+ if cudagraphs_enabled and num_tokens <= self.cudagraph_batch_sizes[-1]:
1057
num_tokens = self.vllm_config.pad_for_cudagraph(num_tokens)
1058
1059
with set_forward_context(
1060
None,
1061
self.vllm_config,
1062
num_tokens=num_tokens,
1060
- cudagraph_runtime_mode=CUDAGraphMode.PIECEWISE
1061
- if use_cudagraphs
1062
- else CUDAGraphMode.NONE,
1063
+ cudagraph_runtime_mode=(
1064
+ CUDAGraphMode.PIECEWISE if cudagraphs_enabled else CUDAGraphMode.NONE
1065
+ ),
1066
):
1067
if self.supports_mm_inputs:
1068
input_ids = None