[XPU] [Linear] add torch as xpu linear backend (#49664)
Signed-off-by: Zhu, Zufang <zufang.zhu@intel.com> Co-authored-by: Kunshang Ji <kunshang.ji@intel.com>
zofia committed
Aug 3, 2026 at 14:30 UTC
e481da9508066465566ce9f0ea843ac3cd15cff9
5 files changed
+15
-8
.buildkite/intel_jobs/test-intel.yaml
+3
-1
@@ -70,8 +70,10 @@ steps:
70
- >-
71
bash .buildkite/scripts/hardware_ci/run-intel-test.sh
72
'python3 examples/basic/offline_inference/generate.py --linear-backend xpu --model RedHatAI/Meta-Llama-3.1-8B-Instruct-FP8 --enforce-eager --max-model-len 4096 &&
73
+ python3 examples/basic/offline_inference/generate.py --linear-backend torch --model RedHatAI/Meta-Llama-3.1-8B-Instruct-FP8 --enforce-eager --max-model-len 4096 &&
74
python3 examples/basic/offline_inference/generate.py --linear-backend xpu --model neuralmagic/Llama-3.2-1B-Instruct-FP8-dynamic --enforce-eager --max-model-len 4096 &&
74
- python3 examples/basic/offline_inference/generate.py --linear-backend xpu --model meta-llama/Llama-3.2-1B-Instruct --quantization fp8 --enforce-eager --max-model-len 4096
75
+ python3 examples/basic/offline_inference/generate.py --linear-backend xpu --model meta-llama/Llama-3.2-1B-Instruct --quantization fp8 --enforce-eager --max-model-len 4096 &&
76
+ python3 examples/basic/offline_inference/generate.py --linear-backend torch --model meta-llama/Llama-3.2-1B-Instruct --quantization fp8 --enforce-eager --max-model-len 4096
77
'
78
- label: "XPU V1 test"
79
depends_on:
docs/features/quantization/online.md
+1
-1
@@ -62,7 +62,7 @@ weight name. Unset fields fall back to the `--quantization` shorthand's
62
defaults, or for already-quantized checkpoints to whatever the checkpoint
63
declares.
64
65
-On XPU, non-block FP8 scaled-mm linear layers default to W8A16; setting `--linear-backend xpu` forces W8A8. Use `--linear-backend xpu_woq` to explicitly select weight-only quantization (W8A16).
65
+On XPU, non-block FP8 scaled-mm linear layers default to W8A16; setting `--linear-backend xpu` forces W8A8. Use `--linear-backend xpu_woq` to explicitly select weight-only quantization (W8A16). Setting `--linear-backend torch` also forces W8A8 but runs the GEMM through `torch._scaled_mm` instead of the custom XPU kernel.
66
67
The CLI accepts the same shape as JSON or as dotted keys:
68
vllm/model_executor/kernels/linear/__init__.py
+2
@@ -363,6 +363,8 @@ _POSSIBLE_FP8_KERNELS: dict[PlatformEnum, list[type[FP8ScaledMMLinearKernel]]] =
363
PlatformEnum.XPU: [
364
XPUW8A16FP8LinearKernel,
365
XPUW8A8FP8LinearKernel,
366
+ PerTensorTorchFP8ScaledMMLinearKernel,
367
+ ChannelWiseTorchFP8ScaledMMLinearKernel,
368
],
369
}
370
vllm/model_executor/kernels/linear/scaled_mm/pytorch.py
+6
-2
@@ -32,8 +32,12 @@ class TorchFP8ScaledMMLinearKernel(FP8ScaledMMLinearKernel):
32
def is_supported(
33
cls, compute_capability: int | None = None
34
) -> tuple[bool, str | None]:
35
- if not (current_platform.is_cuda_alike() or current_platform.is_cpu()):
36
- return False, "requires ROCm, CUDA or CPU."
35
+ if not (
36
+ current_platform.is_cuda_alike()
37
+ or current_platform.is_cpu()
38
+ or current_platform.is_xpu()
39
+ ):
40
+ return False, "requires ROCm, CUDA, CPU or XPU."
41
42
if compute_capability is not None and compute_capability < 89:
43
return False, "requires compute capability 89 and above."
vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors.py
+3
-4
@@ -811,12 +811,11 @@ class CompressedTensorsConfig(QuantizationConfig):
811
if act_quant_format:
812
if self._is_fp8_w8a8(weight_quant, input_quant):
813
if current_platform.is_xpu():
814
- # On XPU, --linear-backend xpu opts into W8A8 FP8
814
+ # On XPU, --linear-backend xpu or torch opts into W8A8 FP8
815
# linear kernel; otherwise default to W8A16.
816
config = get_current_vllm_config_or_none()
817
- is_fp8_w8a8_supported = (
818
- config is not None
819
- and config.kernel_config.linear_backend == "xpu"
817
+ is_fp8_w8a8_supported = config is not None and (
818
+ config.kernel_config.linear_backend in ("xpu", "torch")
819
)
820
else:
821
is_fp8_w8a8_supported = self._check_scheme_supported(