A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[V0] Correct CUDA Graph capture for encoder-decoder models (#22630)
Sugar-zsg committed
Aug 12, 2025 at 17:01 UTC
8d17fa633e319c4a585f6ae1258000a40750e127
1 file changed
+11
-1
vllm/config/__init__.py
+11
-1
@@ -1164,8 +1164,18 @@ class ModelConfig:
1164
"non-quantized models.", self.quantization)
1165
1166
def _verify_cuda_graph(self) -> None:
1167
+ # The `max_seq_len_to_capture` was incorrectly
1168
+ # based on the encoder's input length (448)
1169
+ # but not the decoder's larger input length (1500).
1170
+ # This change ensures the CUDA Graph captures the correct,
1171
+ # larger sequence length, allowing it to work as intended.
1172
+ effective_max_seq_len = self.max_model_len
1173
+ if self.is_encoder_decoder:
1174
+ effective_max_seq_len = max(
1175
+ effective_max_seq_len,
1176
+ getattr(self.hf_config, "max_source_positions", 0))
1177
self.max_seq_len_to_capture = min(self.max_seq_len_to_capture,
1168
- self.max_model_len)
1178
+ effective_max_seq_len)
1179
# CUDAGraph capture not supported for enc-dec models and mllama on ROCm
1180
ROCM_UNSUPPORTED_MODELS = ['mllama']
1181
unsupported_rocm = (self.hf_config.model_type