A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 04:30 UTC
[Bugfix][Hardware][POWERPC] Fix auto dtype failure in case of POWER10 (#11331)
Signed-off-by: Akash Kaothalkar <0052v2@linux.vnet.ibm.com>
Akash kaothalkar committed
Dec 20, 2024 at 07:02 UTC
48edab8041741a82a1fd2f4d463cc0f393561b05
1 file changed
+12
-1
vllm/config.py
+12
-1
@@ -22,7 +22,7 @@ from vllm.logger import init_logger
22
from vllm.model_executor.layers.quantization import (QUANTIZATION_METHODS,
23
get_quantization_config)
24
from vllm.model_executor.models import ModelRegistry
25
-from vllm.platforms import current_platform
25
+from vllm.platforms import current_platform, interface
26
from vllm.tracing import is_otel_available, otel_import_error_traceback
27
from vllm.transformers_utils.config import (
28
ConfigFormat, get_config, get_hf_image_processor_config,
@@ -2199,6 +2199,17 @@ def _get_and_verify_dtype(
2199
else:
2200
torch_dtype = config_dtype
2201
2202
+ if (current_platform.is_cpu()
2203
+ and current_platform.get_cpu_architecture()
2204
+ == interface.CpuArchEnum.POWERPC
2205
+ and (config_dtype == torch.float16
2206
+ or config_dtype == torch.float32)):
2207
+ logger.info(
2208
+ "For POWERPC, we cast models to bfloat16 instead of "
2209
+ "using float16 by default. Float16 is not currently "
2210
+ "supported for POWERPC.")
2211
+ torch_dtype = torch.bfloat16
2212
+
2213
if current_platform.is_hpu() and config_dtype == torch.float16:
2214
logger.info(
2215
"For HPU, we cast models to bfloat16 instead of"