A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[Qwen3Next] Fixes the cuda graph capture conditions under large batch sizes (#24660) (#24667)
Signed-off-by: Tao He <linzhu.ht@alibaba-inc.com>
Tao He committed
Sep 13, 2025 at 06:31 UTC
8226dd56bf64f80e04f2aa90c97401d8783aa2bf
1 file changed
+2
-1
vllm/v1/attention/backends/gdn_attn.py
+2
-1
@@ -209,7 +209,8 @@ class GDNAttentionMetadataBuilder(
209
210
# prepare tensors for cudagraph
211
if (self.use_full_cuda_graph and num_prefills == 0 and num_decodes == 0
212
- and num_spec_decodes <= self.decode_cudagraph_max_bs):
212
+ and num_spec_decodes <= self.decode_cudagraph_max_bs
213
+ and m.num_actual_tokens <= self.decode_cudagraph_max_bs):
214
num_total_tokens = self.vllm_config.pad_for_cudagraph(
215
m.num_actual_tokens)
216
batch_size = num_total_tokens // (self.num_spec + 1)