A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 11, 23:45 UTC
[Compressed Tensors] Add XPU `wNa16` support (#29484)
Signed-off-by: yiliu30 <yi4.liu@intel.com>
Yi Liu committed
Dec 5, 2025 at 22:02 UTC
0d8a7d8a264354ed53f822821b29cb0485bfa70f
3 files changed
+102
.buildkite/scripts/hardware_ci/run-xpu-test.sh
+1
@@ -38,6 +38,7 @@ docker run \
38
python3 examples/offline_inference/basic/generate.py --model facebook/opt-125m --block-size 64 -O3 -cc.cudagraph_mode=NONE
39
python3 examples/offline_inference/basic/generate.py --model facebook/opt-125m --block-size 64 --enforce-eager -tp 2 --distributed-executor-backend ray
40
python3 examples/offline_inference/basic/generate.py --model facebook/opt-125m --block-size 64 --enforce-eager -tp 2 --distributed-executor-backend mp
41
+ python3 examples/offline_inference/basic/generate.py --model Intel/Qwen2.5-0.5B-W4A16-G128-AutoRound-LLMC-TEST-ONLY --enforce-eager
42
VLLM_ATTENTION_BACKEND=TRITON_ATTN python3 examples/offline_inference/basic/generate.py --model facebook/opt-125m --block-size 64 --enforce-eager
43
cd tests
44
pytest -v -s v1/core
vllm/model_executor/layers/quantization/kernels/mixed_precision/__init__.py
+4
@@ -30,6 +30,9 @@ from vllm.model_executor.layers.quantization.kernels.mixed_precision.MPLinearKer
30
MPLinearKernel,
31
MPLinearLayerConfig,
32
)
33
+from vllm.model_executor.layers.quantization.kernels.mixed_precision.xpu import ( # noqa: E501
34
+ XPUwNa16LinearKernel,
35
+)
36
from vllm.platforms import current_platform
37
38
# in priority/performance order (when available)
@@ -42,6 +45,7 @@ _POSSIBLE_KERNELS: list[type[MPLinearKernel]] = [
45
BitBLASLinearKernel,
46
ConchLinearKernel,
47
ExllamaLinearKernel,
48
+ XPUwNa16LinearKernel,
49
]
50
51
vllm/model_executor/layers/quantization/kernels/mixed_precision/xpu.py
new
+97
@@ -0,0 +1,97 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+
4
+
5
+import torch
6
+
7
+from vllm.platforms import current_platform
8
+
9
+from .MPLinearKernel import MPLinearKernel, MPLinearLayerConfig
10
+
11
+
12
+class XPUwNa16LinearKernel(MPLinearKernel):
13
+ @classmethod
14
+ def get_min_capability(cls) -> int:
15
+ return 0
16
+
17
+ @classmethod
18
+ def can_implement(cls, c: MPLinearLayerConfig) -> tuple[bool, str | None]:
19
+ if not current_platform.is_xpu():
20
+ return False, "IPEX wNa16 only supported on XPU/CPU devices"
21
+
22
+ # TODO: (yiliu30) relax these restrictions in later PRs
23
+ if c.zero_points:
24
+ return False, "Zero points not supported for Now"
25
+
26
+ return True, None
27
+
28
+ def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
29
+ from packaging import version
30
+
31
+ MIN_IPEX_VERSION = "2.6.0"
32
+ bias = layer.bias if not layer.skip_bias_add else None
33
+
34
+ try:
35
+ import intel_extension_for_pytorch as ipex
36
+
37
+ if version.parse(ipex.__version__) < version.parse(MIN_IPEX_VERSION):
38
+ raise ImportError(
39
+ "intel_extension_for_pytorch version is "
40
+ "wrong. Please install "
41
+ f"intel_extension_for_pytorch>={MIN_IPEX_VERSION}."
42
+ )
43
+ except ImportError as err:
44
+ raise ImportError(
45
+ "Please install "
46
+ f"intel_extension_for_pytorch>={MIN_IPEX_VERSION} via "
47
+ f"`pip install intel_extension_for_pytorch>={MIN_IPEX_VERSION}`"
48
+ " to use IPEX-AWQ linear method."
49
+ ) from err
50
+ # Using the compute dtype (lowp_mode) as INT8 to leverage instructions
51
+ # with better performance.
52
+ lowp_mode = ipex.quantization.WoqLowpMode.INT8
53
+ # The weight will be de-packed from INT4 to INT8.
54
+ weight_dtype = ipex.quantization.WoqWeightDtype.INT4
55
+ # The float activation will be quantized (dynamic, per-token) to INT8.
56
+ act_quant_mode = ipex.quantization.WoqActQuantMode.PER_BATCH
57
+
58
+ qconfig = ipex.quantization.get_weight_only_quant_qconfig_mapping(
59
+ weight_dtype=weight_dtype,
60
+ lowp_mode=lowp_mode,
61
+ act_quant_mode=act_quant_mode,
62
+ group_size=self.config.group_size,
63
+ weight_qscheme=ipex.quantization.WoqWeightQScheme.SYMMETRIC,
64
+ )
65
+ qweight = layer.weight_packed
66
+ g_idx = layer.weight_g_idx if self.config.has_g_idx else None
67
+ scales = layer.weight_scale
68
+ qzeros = None
69
+ if self.config.zero_points:
70
+ qzeros = layer.weight_zero_point.contiguous()
71
+ qweight = qweight.t().contiguous()
72
+ scales = scales.t().contiguous()
73
+ layer.ipex_output_size = self.config.partition_weight_shape[1]
74
+ layer.ipex_qlinear = (
75
+ ipex.llm.quantization.woq_linear.IPEXWeightOnlyQuantizedLinear.from_weight(
76
+ qweight,
77
+ scales,
78
+ qzeros,
79
+ in_features=self.config.partition_weight_shape[0],
80
+ out_features=self.config.partition_weight_shape[1],
81
+ qconfig=qconfig,
82
+ g_idx=g_idx,
83
+ bias=bias,
84
+ group_size=self.config.group_size,
85
+ quant_method=0, # `0` stands for the IPEX GPTQ
86
+ )
87
+ )
88
+
89
+ def apply_weights(
90
+ self,
91
+ layer: torch.nn.Module,
92
+ x: torch.Tensor,
93
+ bias: torch.Tensor | None = None,
94
+ ) -> torch.Tensor:
95
+ reshaped_x = x.reshape(-1, x.shape[-1])
96
+ out = layer.ipex_qlinear(reshaped_x)
97
+ return out.reshape(x.shape[:-1] + (layer.ipex_output_size,))