@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 48edab8041
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 04:30 UTC

[Bugfix][Hardware][POWERPC] Fix auto dtype failure in case of POWER10 (#11331)

Signed-off-by: Akash Kaothalkar <0052v2@linux.vnet.ibm.com>

Akash kaothalkar committed Dec 20, 2024 at 07:02 UTC 48edab8041741a82a1fd2f4d463cc0f393561b05
1 file changed +12 -1
vllm/config.py
+12 -1
@@ -22,7 +22,7 @@ from vllm.logger import init_logger
22 from vllm.model_executor.layers.quantization import (QUANTIZATION_METHODS,
23 get_quantization_config)
24 from vllm.model_executor.models import ModelRegistry
25 -from vllm.platforms import current_platform
25 +from vllm.platforms import current_platform, interface
26 from vllm.tracing import is_otel_available, otel_import_error_traceback
27 from vllm.transformers_utils.config import (
28 ConfigFormat, get_config, get_hf_image_processor_config,
@@ -2199,6 +2199,17 @@ def _get_and_verify_dtype(
2199 else:
2200 torch_dtype = config_dtype
2201
2202 + if (current_platform.is_cpu()
2203 + and current_platform.get_cpu_architecture()
2204 + == interface.CpuArchEnum.POWERPC
2205 + and (config_dtype == torch.float16
2206 + or config_dtype == torch.float32)):
2207 + logger.info(
2208 + "For POWERPC, we cast models to bfloat16 instead of "
2209 + "using float16 by default. Float16 is not currently "
2210 + "supported for POWERPC.")
2211 + torch_dtype = torch.bfloat16
2212 +
2213 if current_platform.is_hpu() and config_dtype == torch.float16:
2214 logger.info(
2215 "For HPU, we cast models to bfloat16 instead of"