A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 11:00 UTC
[Misc] Auto fallback to float16 for pre-Ampere GPUs when detected bfloat16 config (#17265)
Signed-off-by: Isotr0py <2037008807@qq.com>
Isotr0py committed
May 10, 2025 at 01:16 UTC
5c4c08f6f1609960b047c8b9d6aa003e9afc2897
4 files changed
+57
-26
vllm/config.py
+21
-25
@@ -7,7 +7,6 @@ import hashlib
7
import inspect
8
import json
9
import re
10
-import sys
10
import textwrap
11
import warnings
12
from collections import Counter
@@ -34,7 +33,7 @@ from vllm.model_executor.layers.quantization import (QUANTIZATION_METHODS,
33
QuantizationMethods,
34
get_quantization_config)
35
from vllm.model_executor.models import ModelRegistry
37
-from vllm.platforms import CpuArchEnum, current_platform
36
+from vllm.platforms import current_platform
37
from vllm.tracing import is_otel_available, otel_import_error_traceback
38
from vllm.transformers_utils.config import (
39
ConfigFormat, get_config, get_hf_image_processor_config,
@@ -2988,6 +2987,7 @@ def _get_and_verify_dtype(
2987
if isinstance(dtype, str):
2988
dtype = dtype.lower()
2989
if dtype == "auto":
2990
+ # Set default dtype from model config
2991
if config_dtype == torch.float32:
2992
# Following common practice, we use float16 for float32 models
2993
torch_dtype = torch.float16
@@ -2995,37 +2995,33 @@ def _get_and_verify_dtype(
2995
torch_dtype = config_dtype
2996
2997
if config.model_type == "plamo2":
2998
- logger.info(
2998
+ logger.warning(
2999
"For PLaMo2, we cast models to bfloat16 instead of using "
3000
"float16 by default. This is because float16 does not work."
3001
)
3002
torch_dtype = torch.bfloat16
3003
3004
+ # Deal with torch dtype fallback for device compatibility.
3005
from vllm.platforms import current_platform
3005
- if (current_platform.is_cpu()
3006
- and current_platform.get_cpu_architecture()
3007
- == CpuArchEnum.POWERPC
3008
- and (config_dtype == torch.float16
3009
- or config_dtype == torch.float32)):
3010
- logger.info(
3011
- "For POWERPC, we cast models to bfloat16 instead of "
3012
- "using float16 by default. Float16 is not currently "
3013
- "supported for POWERPC.")
3014
- torch_dtype = torch.bfloat16
3006
+ if torch_dtype not in current_platform.supported_dtypes:
3007
+ device_name = current_platform.get_device_name()
3008
3016
- # TODO: change this condition to check if the platform support bf16
3017
- # instead of checking the OS. For instance M2 shall supports bf16
3018
- # already. But we need to modify `cpu_extension.cmake` to activate
3019
- # the feature in the build.
3020
- if (current_platform.is_cpu() and sys.platform.startswith("darwin")
3021
- and current_platform.get_cpu_architecture()
3022
- == CpuArchEnum.ARM and config_dtype == torch.bfloat16):
3023
- logger.info("For macOS with Apple Silicon, currently bfloat16 "
3024
- "is not supported. Setting dtype to float16.")
3025
- torch_dtype = torch.float16
3009
+ if ((capability := current_platform.get_device_capability())
3010
+ is None):
3011
+ compute_str = ""
3012
+ else:
3013
+ version_str = capability.as_version_str()
3014
+ compute_str = f" (with compute capability {version_str})"
3015
+ fallback_dtype = current_platform.supported_dtypes[0]
3016
+ logger.warning(
3017
+ "Your %s device%s doesn't support %s. " \
3018
+ "Falling back to %s for compatibility.",
3019
+ device_name, compute_str, torch_dtype, fallback_dtype
3020
+ )
3021
+ torch_dtype = fallback_dtype
3022
3027
- if current_platform.is_hpu() and config_dtype == torch.float16:
3028
- logger.info(
3023
+ if current_platform.is_hpu() and torch_dtype == torch.float16:
3024
+ logger.warning(
3025
"For HPU, we cast models to bfloat16 instead of "
3026
"using float16 by default. Please specify `dtype` if you "
3027
"want to use float16.")
vllm/platforms/cpu.py
+15
-1
@@ -10,7 +10,7 @@ import torch
10
11
from vllm.logger import init_logger
12
13
-from .interface import Platform, PlatformEnum, _Backend
13
+from .interface import CpuArchEnum, Platform, PlatformEnum, _Backend
14
15
logger = init_logger(__name__)
16
@@ -26,6 +26,20 @@ class CpuPlatform(Platform):
26
device_type: str = "cpu"
27
dispatch_key: str = "CPU"
28
29
+ @property
30
+ def supported_dtypes(self) -> list:
31
+ if self.get_cpu_architecture() == CpuArchEnum.POWERPC:
32
+ return [torch.bfloat16, torch.float32]
33
+ elif sys.platform.startswith(
34
+ "darwin") and self.get_cpu_architecture() == CpuArchEnum.ARM:
35
+ # TODO: change this condition to check if the platform support bf16
36
+ # instead of checking the OS. For instance M2 shall supports bf16
37
+ # already. But we need to modify `cpu_extension.cmake` to activate
38
+ # the feature in the build.
39
+ return [torch.bfloat16, torch.float32]
40
+ # x86/aarch64 CPU has supported both bf16 and fp16 natively.
41
+ return [torch.bfloat16, torch.float16, torch.float32]
42
+
43
@classmethod
44
def get_device_name(cls, device_id: int = 0) -> str:
45
return "cpu"
vllm/platforms/cuda.py
+13
@@ -73,6 +73,19 @@ class CudaPlatformBase(Platform):
73
ray_device_key: str = "GPU"
74
device_control_env_var: str = "CUDA_VISIBLE_DEVICES"
75
76
+ @property
77
+ def supported_dtypes(self) -> List[torch.dtype]:
78
+ if self.has_device_capability(80):
79
+ # Ampere and Hopper or later NVIDIA GPUs.
80
+ return [torch.bfloat16, torch.float16, torch.float32]
81
+ elif (not self.has_device_capability(80)
82
+ ) and self.has_device_capability(60):
83
+ # Pascal, Volta and Turing NVIDIA GPUs, BF16 is not supported
84
+ return [torch.float16, torch.float32]
85
+ # Kepler and Maxwell NVIDIA GPUs, only FP32 is supported,
86
+ # though vLLM doesn't support these GPUs.
87
+ return [torch.float32]
88
+
89
@classmethod
90
def get_device_capability(cls,
91
device_id: int = 0
vllm/platforms/interface.py
+8
@@ -122,6 +122,14 @@ class Platform:
122
123
additional_env_vars: list[str] = []
124
125
+ @property
126
+ def supported_dtypes(self) -> list[torch.dtype]:
127
+ """Returns the supported dtypes for the current platform."""
128
+ # Be careful with the order of the dtypes. The first dtype will
129
+ # be used as the default dtype fallback for the current platform,
130
+ # when encountering unsupported dtypes in "auto" dtype.
131
+ return [torch.bfloat16, torch.float16, torch.float32]
132
+
133
def is_cuda(self) -> bool:
134
return self._enum == PlatformEnum.CUDA
135