@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / f256ebe4df
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 00:45 UTC

[Hardware][Intel GPU] add XPU bf16 support (#12392)

Signed-off-by: Kunshang Ji <kunshang.ji@intel.com>

Kunshang Ji committed Feb 2, 2025 at 18:17 UTC f256ebe4df6757d76f1f1642d7e110268a2f8190
2 files changed +21 -4
docs/source/getting_started/installation/gpu/xpu.inc.md
+1 -1
@@ -36,7 +36,7 @@ VLLM_TARGET_DEVICE=xpu python setup.py install
36
37 :::{note}
38 - FP16 is the default data type in the current XPU backend. The BF16 data
39 - type will be supported in the future.
39 + type is supported on Intel Data Center GPU, not supported on Intel Arc GPU yet.
40 :::
41
42 ## Set up using Docker
vllm/platforms/xpu.py
+20 -3
@@ -66,9 +66,14 @@ class XPUPlatform(Platform):
66 # check and update model config
67 model_config = vllm_config.model_config
68 if model_config.dtype == torch.bfloat16:
69 - logger.warning(
70 - "bfloat16 is not fully supported on XPU, casting to float16.")
71 - model_config.dtype = torch.float16
69 + bf16_supported = cls.device_support_bf16()
70 + if not bf16_supported:
71 + logger.warning(
72 + "bfloat16 is only supported on Intel Data Center GPU, "
73 + "Intel Arc GPU is not supported yet. Your device is %s,"
74 + "which is not supported. will fallback to float16",
75 + cls.get_device_name())
76 + model_config.dtype = torch.float16
77 if not model_config.enforce_eager:
78 logger.warning(
79 "CUDA graph is not supported on XPU, fallback to the eager "
@@ -116,3 +121,15 @@ class XPUPlatform(Platform):
121 ) -> float:
122 torch.xpu.reset_peak_memory_stats(device)
123 return torch.xpu.max_memory_allocated(device)
124 +
125 + @classmethod
126 + def device_support_bf16(cls) -> bool:
127 + device_name = cls.get_device_name().lower()
128 + if device_name.count("arc") > 0:
129 + return False
130 + elif device_name.count("data center gpu") > 0:
131 + return True
132 + else:
133 + logger.warning("Unknown device name %s, always use float16",
134 + device_name)
135 + return False