A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 00:45 UTC
[Hardware][Intel GPU] add XPU bf16 support (#12392)
Signed-off-by: Kunshang Ji <kunshang.ji@intel.com>
Kunshang Ji committed
Feb 2, 2025 at 18:17 UTC
f256ebe4df6757d76f1f1642d7e110268a2f8190
2 files changed
+21
-4
docs/source/getting_started/installation/gpu/xpu.inc.md
+1
-1
@@ -36,7 +36,7 @@ VLLM_TARGET_DEVICE=xpu python setup.py install
36
37
:::{note}
38
- FP16 is the default data type in the current XPU backend. The BF16 data
39
- type will be supported in the future.
39
+ type is supported on Intel Data Center GPU, not supported on Intel Arc GPU yet.
40
:::
41
42
## Set up using Docker
vllm/platforms/xpu.py
+20
-3
@@ -66,9 +66,14 @@ class XPUPlatform(Platform):
66
# check and update model config
67
model_config = vllm_config.model_config
68
if model_config.dtype == torch.bfloat16:
69
- logger.warning(
70
- "bfloat16 is not fully supported on XPU, casting to float16.")
71
- model_config.dtype = torch.float16
69
+ bf16_supported = cls.device_support_bf16()
70
+ if not bf16_supported:
71
+ logger.warning(
72
+ "bfloat16 is only supported on Intel Data Center GPU, "
73
+ "Intel Arc GPU is not supported yet. Your device is %s,"
74
+ "which is not supported. will fallback to float16",
75
+ cls.get_device_name())
76
+ model_config.dtype = torch.float16
77
if not model_config.enforce_eager:
78
logger.warning(
79
"CUDA graph is not supported on XPU, fallback to the eager "
@@ -116,3 +121,15 @@ class XPUPlatform(Platform):
121
) -> float:
122
torch.xpu.reset_peak_memory_stats(device)
123
return torch.xpu.max_memory_allocated(device)
124
+
125
+ @classmethod
126
+ def device_support_bf16(cls) -> bool:
127
+ device_name = cls.get_device_name().lower()
128
+ if device_name.count("arc") > 0:
129
+ return False
130
+ elif device_name.count("data center gpu") > 0:
131
+ return True
132
+ else:
133
+ logger.warning("Unknown device name %s, always use float16",
134
+ device_name)
135
+ return False