A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 16:00 UTC
[ROCm][Bugfix] Bring back fallback to eager mode removed in #14917, but for ROCm only (#15413)
Signed-off-by: Gregory Shtrasberg <Gregory.Shtrasberg@amd.com>
Gregory Shtrasberg committed
Apr 4, 2025 at 12:40 UTC
a6d042df0a1f62ffab206693cc798dd695b89d7c
1 file changed
+8
-1
vllm/config.py
+8
-1
@@ -29,7 +29,7 @@ from vllm.logger import init_logger
29
from vllm.model_executor.layers.quantization import (QUANTIZATION_METHODS,
30
get_quantization_config)
31
from vllm.model_executor.models import ModelRegistry
32
-from vllm.platforms import CpuArchEnum
32
+from vllm.platforms import CpuArchEnum, current_platform
33
from vllm.sampling_params import GuidedDecodingParams
34
from vllm.tracing import is_otel_available, otel_import_error_traceback
35
from vllm.transformers_utils.config import (
@@ -684,6 +684,13 @@ class ModelConfig:
684
self.max_seq_len_to_capture = self.max_model_len
685
self.max_seq_len_to_capture = min(self.max_seq_len_to_capture,
686
self.max_model_len)
687
+ ROCM_UNSUPPORTED_MODELS = ['mllama']
688
+ if (self.hf_config.model_type in ROCM_UNSUPPORTED_MODELS
689
+ and not self.enforce_eager and current_platform.is_rocm()):
690
+ logger.warning(
691
+ "CUDA graph is not supported for %s on ROCm yet, fallback "
692
+ "to the eager mode.", self.hf_config.model_type)
693
+ self.enforce_eager = True
694
695
def _verify_bnb_config(self) -> None:
696
"""