A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 11, 17:30 UTC
[Bugfix] Fix cuda graph sizes when running with speculative decoding (#30330)
Signed-off-by: Patryk Saffer <patryk.saffer99@gmail.com> Signed-off-by: PatrykSaffer <patryk.saffer@mistral.ai> Co-authored-by: Patryk Saffer <patryk.saffer99@gmail.com>
PatrykSaffer committed
Dec 10, 2025 at 01:47 UTC
4c2e10ea19b9053924d66f30f3d7121fbd9684f8
1 file changed
+7
-1
vllm/config/vllm.py
+7
-1
@@ -1047,8 +1047,14 @@ class VllmConfig:
1047
self.compilation_config.max_cudagraph_capture_size
1048
)
1049
if max_cudagraph_capture_size is None:
1050
+ decode_query_len = 1
1051
+ if (
1052
+ self.speculative_config
1053
+ and self.speculative_config.num_speculative_tokens
1054
+ ):
1055
+ decode_query_len += self.speculative_config.num_speculative_tokens
1056
max_cudagraph_capture_size = min(
1051
- self.scheduler_config.max_num_seqs * 2, 512
1057
+ self.scheduler_config.max_num_seqs * decode_query_len * 2, 512
1058
)
1059
max_num_tokens = self.scheduler_config.max_num_batched_tokens
1060
max_cudagraph_capture_size = min(max_num_tokens, max_cudagraph_capture_size)