A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 2, 23:15 UTC
[XPU]feat: enable FP8 block-scaled quantization on XPU (#42952)
Signed-off-by: Ma Jian <jian1.ma@intel.com> Co-authored-by: Kunshang Ji <kunshang.ji@intel.com>
Ma Jian committed
May 23, 2026 at 12:33 UTC
a5bbd81e2e872eba255da9b9f8b86063c0d0cef0
4 files changed
+23
-5
vllm/model_executor/kernels/linear/__init__.py
+3
@@ -307,6 +307,9 @@ _POSSIBLE_FP8_BLOCK_KERNELS: dict[
307
PlatformEnum.CPU: [
308
CPUFp8BlockScaledMMKernel,
309
],
310
+ PlatformEnum.XPU: [
311
+ TritonFp8BlockScaledMMKernel,
312
+ ],
313
}
314
315
_POSSIBLE_WFP8A16_KERNELS: dict[PlatformEnum, list[type[FP8ScaledMMLinearKernel]]] = {
vllm/model_executor/kernels/linear/scaled_mm/triton.py
+2
-2
@@ -159,8 +159,8 @@ class TritonInt8ScaledMMLinearKernel(CutlassInt8ScaledMMLinearKernel):
159
class TritonFp8BlockScaledMMKernel(Fp8BlockScaledMMLinearKernel):
160
@classmethod
161
def is_supported(cls, compute_capability=None):
162
- if not current_platform.is_cuda_alike():
163
- return False, "only cuda like devices are supported."
162
+ if not (current_platform.is_cuda_alike() or current_platform.is_xpu()):
163
+ return False, "only cuda-like and xpu devices are supported."
164
return True, None
165
166
def apply_block_scaled_mm(
vllm/model_executor/layers/quantization/input_quant_fp8.py
+10
-1
@@ -173,7 +173,16 @@ class QuantFP8(CustomOp):
173
scale_ub: torch.Tensor | None = None,
174
use_triton: bool = False,
175
) -> tuple[torch.Tensor, torch.Tensor]:
176
- # XPU can use same code path as CUDA.
176
+ if self.is_group_quant and not self.static:
177
+ from vllm.model_executor.layers.quantization.utils import fp8_utils
178
+
179
+ return fp8_utils.per_token_group_quant_fp8(
180
+ x,
181
+ group_size=self.group_size,
182
+ column_major_scales=self.column_major_scales,
183
+ dtype=_FP8_DTYPE,
184
+ use_ue8m0=self.use_ue8m0,
185
+ )
186
return self.forward_cuda(x, scale, scale_ub, use_triton)
187
188
def forward_native(
vllm/model_executor/layers/quantization/utils/fp8_utils.py
+8
-2
@@ -573,7 +573,7 @@ def per_token_group_quant_fp8(
573
shape = x.shape[:-1] + (x.shape[-1] // group_size,)
574
x_s = torch.empty(shape, device=x.device, dtype=torch.float32)
575
576
- # prefer CUDA kernel if available
576
+ # prefer CUDA/XPU kernel if available
577
# TODO(bnell): this causes some fp8 moe test to fail.
578
if current_platform.is_cuda() and x.is_contiguous():
579
torch.ops._C.per_token_group_fp8_quant(
@@ -590,6 +590,12 @@ def per_token_group_quant_fp8(
590
)
591
return x_q, x_s
592
593
+ if current_platform.is_xpu() and x.is_contiguous():
594
+ torch.ops._C.per_token_group_fp8_quant(
595
+ x, x_q, x_s, group_size, eps, fp8_min, fp8_max, use_ue8m0
596
+ )
597
+ return x_q, x_s
598
+
599
# TRITON FALLBACK
600
M = x.numel() // group_size
601
N = group_size
@@ -861,7 +867,7 @@ def w8a8_triton_block_scaled_mm(
867
# Triton cannot currently bind E8M0 scale tensors directly. On ROCm,
868
# DeepSeek-V4 checkpoints store block scales in exponent-only E8M0 format,
869
# so decode them to fp32 before launching the kernel.
864
- if current_platform.is_rocm():
870
+ if current_platform.is_rocm() or current_platform.is_xpu():
871
if As.dtype == torch.float8_e8m0fnu:
872
As = _upcast_e8m0_to_fp32(As).contiguous()
873
if Bs.dtype == torch.float8_e8m0fnu: