A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 04:30 UTC
[Platform] Move model arch check to platform (#11503)
Signed-off-by: Mengqing Cao <cmq0113@163.com>
Mengqing Cao committed
Dec 27, 2024 at 16:45 UTC
6c6f7fe8a850ca08f9a8774de020163a2a7c2164
3 files changed
+51
-37
vllm/model_executor/models/registry.py
+1
-36
@@ -187,31 +187,6 @@ _VLLM_MODELS = {
187
**_SPECULATIVE_DECODING_MODELS,
188
}
189
190
-# Models not supported by ROCm.
191
-_ROCM_UNSUPPORTED_MODELS: List[str] = []
192
-
193
-# Models partially supported by ROCm.
194
-# Architecture -> Reason.
195
-_ROCM_SWA_REASON = ("Sliding window attention (SWA) is not yet supported in "
196
- "Triton flash attention. For half-precision SWA support, "
197
- "please use CK flash attention by setting "
198
- "`VLLM_USE_TRITON_FLASH_ATTN=0`")
199
-_ROCM_PARTIALLY_SUPPORTED_MODELS: Dict[str, str] = {
200
- "Qwen2ForCausalLM":
201
- _ROCM_SWA_REASON,
202
- "MistralForCausalLM":
203
- _ROCM_SWA_REASON,
204
- "MixtralForCausalLM":
205
- _ROCM_SWA_REASON,
206
- "PaliGemmaForConditionalGeneration":
207
- ("ROCm flash attention does not yet "
208
- "fully support 32-bit precision on PaliGemma"),
209
- "Phi3VForCausalLM":
210
- ("ROCm Triton flash attention may run into compilation errors due to "
211
- "excessive use of shared memory. If this happens, disable Triton FA "
212
- "by setting `VLLM_USE_TRITON_FLASH_ATTN=0`")
213
-}
214
-
190
191
@dataclass(frozen=True)
192
class _ModelInfo:
@@ -297,17 +272,7 @@ def _try_load_model_cls(
272
model_arch: str,
273
model: _BaseRegisteredModel,
274
) -> Optional[Type[nn.Module]]:
300
- if current_platform.is_rocm():
301
- if model_arch in _ROCM_UNSUPPORTED_MODELS:
302
- raise ValueError(f"Model architecture '{model_arch}' is not "
303
- "supported by ROCm for now.")
304
-
305
- if model_arch in _ROCM_PARTIALLY_SUPPORTED_MODELS:
306
- msg = _ROCM_PARTIALLY_SUPPORTED_MODELS[model_arch]
307
- logger.warning(
308
- "Model architecture '%s' is partially "
309
- "supported by ROCm: %s", model_arch, msg)
310
-
275
+ current_platform.verify_model_arch(model_arch)
276
try:
277
return model.load_model_cls()
278
except Exception:
vllm/platforms/interface.py
+12
@@ -199,6 +199,18 @@ class Platform:
199
"""
200
pass
201
202
+ @classmethod
203
+ def verify_model_arch(cls, model_arch: str) -> None:
204
+ """
205
+ Verify whether the current platform supports the specified model
206
+ architecture.
207
+
208
+ - This will raise an Error or Warning based on the model support on
209
+ the current platform.
210
+ - By default all models are considered supported.
211
+ """
212
+ pass
213
+
214
@classmethod
215
def verify_quantization(cls, quant: str) -> None:
216
"""
vllm/platforms/rocm.py
+38
-1
@@ -1,6 +1,6 @@
1
import os
2
from functools import lru_cache
3
-from typing import TYPE_CHECKING, Optional
3
+from typing import TYPE_CHECKING, Dict, List, Optional
4
5
import torch
6
@@ -33,6 +33,31 @@ if os.environ.get("VLLM_WORKER_MULTIPROC_METHOD", None) in ["fork", None]:
33
" `spawn` instead.")
34
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
35
36
+# Models not supported by ROCm.
37
+_ROCM_UNSUPPORTED_MODELS: List[str] = []
38
+
39
+# Models partially supported by ROCm.
40
+# Architecture -> Reason.
41
+_ROCM_SWA_REASON = ("Sliding window attention (SWA) is not yet supported in "
42
+ "Triton flash attention. For half-precision SWA support, "
43
+ "please use CK flash attention by setting "
44
+ "`VLLM_USE_TRITON_FLASH_ATTN=0`")
45
+_ROCM_PARTIALLY_SUPPORTED_MODELS: Dict[str, str] = {
46
+ "Qwen2ForCausalLM":
47
+ _ROCM_SWA_REASON,
48
+ "MistralForCausalLM":
49
+ _ROCM_SWA_REASON,
50
+ "MixtralForCausalLM":
51
+ _ROCM_SWA_REASON,
52
+ "PaliGemmaForConditionalGeneration":
53
+ ("ROCm flash attention does not yet "
54
+ "fully support 32-bit precision on PaliGemma"),
55
+ "Phi3VForCausalLM":
56
+ ("ROCm Triton flash attention may run into compilation errors due to "
57
+ "excessive use of shared memory. If this happens, disable Triton FA "
58
+ "by setting `VLLM_USE_TRITON_FLASH_ATTN=0`")
59
+}
60
+
61
62
class RocmPlatform(Platform):
63
_enum = PlatformEnum.ROCM
@@ -102,6 +127,18 @@ class RocmPlatform(Platform):
127
else:
128
parallel_config.worker_cls = "vllm.worker.worker.Worker"
129
130
+ @classmethod
131
+ def verify_model_arch(cls, model_arch: str) -> None:
132
+ if model_arch in _ROCM_UNSUPPORTED_MODELS:
133
+ raise ValueError(f"Model architecture '{model_arch}' is not "
134
+ "supported by ROCm for now.")
135
+
136
+ if model_arch in _ROCM_PARTIALLY_SUPPORTED_MODELS:
137
+ msg = _ROCM_PARTIALLY_SUPPORTED_MODELS[model_arch]
138
+ logger.warning(
139
+ "Model architecture '%s' is partially "
140
+ "supported by ROCm: %s", model_arch, msg)
141
+
142
@classmethod
143
def verify_quantization(cls, quant: str) -> None:
144
super().verify_quantization(quant)