@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 6c6f7fe8a8
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 04:30 UTC

[Platform] Move model arch check to platform (#11503)

Signed-off-by: Mengqing Cao <cmq0113@163.com>

Mengqing Cao committed Dec 27, 2024 at 16:45 UTC 6c6f7fe8a850ca08f9a8774de020163a2a7c2164
3 files changed +51 -37
vllm/model_executor/models/registry.py
+1 -36
@@ -187,31 +187,6 @@ _VLLM_MODELS = {
187 **_SPECULATIVE_DECODING_MODELS,
188 }
189
190 -# Models not supported by ROCm.
191 -_ROCM_UNSUPPORTED_MODELS: List[str] = []
192 -
193 -# Models partially supported by ROCm.
194 -# Architecture -> Reason.
195 -_ROCM_SWA_REASON = ("Sliding window attention (SWA) is not yet supported in "
196 - "Triton flash attention. For half-precision SWA support, "
197 - "please use CK flash attention by setting "
198 - "`VLLM_USE_TRITON_FLASH_ATTN=0`")
199 -_ROCM_PARTIALLY_SUPPORTED_MODELS: Dict[str, str] = {
200 - "Qwen2ForCausalLM":
201 - _ROCM_SWA_REASON,
202 - "MistralForCausalLM":
203 - _ROCM_SWA_REASON,
204 - "MixtralForCausalLM":
205 - _ROCM_SWA_REASON,
206 - "PaliGemmaForConditionalGeneration":
207 - ("ROCm flash attention does not yet "
208 - "fully support 32-bit precision on PaliGemma"),
209 - "Phi3VForCausalLM":
210 - ("ROCm Triton flash attention may run into compilation errors due to "
211 - "excessive use of shared memory. If this happens, disable Triton FA "
212 - "by setting `VLLM_USE_TRITON_FLASH_ATTN=0`")
213 -}
214 -
190
191 @dataclass(frozen=True)
192 class _ModelInfo:
@@ -297,17 +272,7 @@ def _try_load_model_cls(
272 model_arch: str,
273 model: _BaseRegisteredModel,
274 ) -> Optional[Type[nn.Module]]:
300 - if current_platform.is_rocm():
301 - if model_arch in _ROCM_UNSUPPORTED_MODELS:
302 - raise ValueError(f"Model architecture '{model_arch}' is not "
303 - "supported by ROCm for now.")
304 -
305 - if model_arch in _ROCM_PARTIALLY_SUPPORTED_MODELS:
306 - msg = _ROCM_PARTIALLY_SUPPORTED_MODELS[model_arch]
307 - logger.warning(
308 - "Model architecture '%s' is partially "
309 - "supported by ROCm: %s", model_arch, msg)
310 -
275 + current_platform.verify_model_arch(model_arch)
276 try:
277 return model.load_model_cls()
278 except Exception:
vllm/platforms/interface.py
+12
@@ -199,6 +199,18 @@ class Platform:
199 """
200 pass
201
202 + @classmethod
203 + def verify_model_arch(cls, model_arch: str) -> None:
204 + """
205 + Verify whether the current platform supports the specified model
206 + architecture.
207 +
208 + - This will raise an Error or Warning based on the model support on
209 + the current platform.
210 + - By default all models are considered supported.
211 + """
212 + pass
213 +
214 @classmethod
215 def verify_quantization(cls, quant: str) -> None:
216 """
vllm/platforms/rocm.py
+38 -1
@@ -1,6 +1,6 @@
1 import os
2 from functools import lru_cache
3 -from typing import TYPE_CHECKING, Optional
3 +from typing import TYPE_CHECKING, Dict, List, Optional
4
5 import torch
6
@@ -33,6 +33,31 @@ if os.environ.get("VLLM_WORKER_MULTIPROC_METHOD", None) in ["fork", None]:
33 " `spawn` instead.")
34 os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
35
36 +# Models not supported by ROCm.
37 +_ROCM_UNSUPPORTED_MODELS: List[str] = []
38 +
39 +# Models partially supported by ROCm.
40 +# Architecture -> Reason.
41 +_ROCM_SWA_REASON = ("Sliding window attention (SWA) is not yet supported in "
42 + "Triton flash attention. For half-precision SWA support, "
43 + "please use CK flash attention by setting "
44 + "`VLLM_USE_TRITON_FLASH_ATTN=0`")
45 +_ROCM_PARTIALLY_SUPPORTED_MODELS: Dict[str, str] = {
46 + "Qwen2ForCausalLM":
47 + _ROCM_SWA_REASON,
48 + "MistralForCausalLM":
49 + _ROCM_SWA_REASON,
50 + "MixtralForCausalLM":
51 + _ROCM_SWA_REASON,
52 + "PaliGemmaForConditionalGeneration":
53 + ("ROCm flash attention does not yet "
54 + "fully support 32-bit precision on PaliGemma"),
55 + "Phi3VForCausalLM":
56 + ("ROCm Triton flash attention may run into compilation errors due to "
57 + "excessive use of shared memory. If this happens, disable Triton FA "
58 + "by setting `VLLM_USE_TRITON_FLASH_ATTN=0`")
59 +}
60 +
61
62 class RocmPlatform(Platform):
63 _enum = PlatformEnum.ROCM
@@ -102,6 +127,18 @@ class RocmPlatform(Platform):
127 else:
128 parallel_config.worker_cls = "vllm.worker.worker.Worker"
129
130 + @classmethod
131 + def verify_model_arch(cls, model_arch: str) -> None:
132 + if model_arch in _ROCM_UNSUPPORTED_MODELS:
133 + raise ValueError(f"Model architecture '{model_arch}' is not "
134 + "supported by ROCm for now.")
135 +
136 + if model_arch in _ROCM_PARTIALLY_SUPPORTED_MODELS:
137 + msg = _ROCM_PARTIALLY_SUPPORTED_MODELS[model_arch]
138 + logger.warning(
139 + "Model architecture '%s' is partially "
140 + "supported by ROCm: %s", model_arch, msg)
141 +
142 @classmethod
143 def verify_quantization(cls, quant: str) -> None:
144 super().verify_quantization(quant)