A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 09:45 UTC
[Fix][ROCm] Enforce eager for all encoder-decoder models on ROCm (#18154)
Signed-off-by: Luka Govedič <lgovedic@redhat.com>
Luka Govedič committed
May 15, 2025 at 01:04 UTC
83f74c698f1f7c781ae02e3c533a52432799e717
1 file changed
+8
-3
vllm/config.py
+8
-3
@@ -906,12 +906,17 @@ class ModelConfig:
906
def _verify_cuda_graph(self) -> None:
907
self.max_seq_len_to_capture = min(self.max_seq_len_to_capture,
908
self.max_model_len)
909
+ # CUDAGraph capture not supported for enc-dec models and mllama on ROCm
910
ROCM_UNSUPPORTED_MODELS = ['mllama']
910
- if (self.hf_config.model_type in ROCM_UNSUPPORTED_MODELS
911
- and not self.enforce_eager and current_platform.is_rocm()):
911
+ unsupported_rocm = (self.hf_config.model_type
912
+ in ROCM_UNSUPPORTED_MODELS
913
+ or self.is_encoder_decoder)
914
+
915
+ if (unsupported_rocm and not self.enforce_eager
916
+ and current_platform.is_rocm()):
917
logger.warning(
918
"CUDA graph is not supported for %s on ROCm yet, fallback "
914
- "to the eager mode.", self.hf_config.model_type)
919
+ "to eager mode.", self.hf_config.model_type)
920
self.enforce_eager = True
921
922
def _verify_bnb_config(self) -> None: