A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 3, 09:15 UTC
[XPU] add gptq(int4) support (#37844)
Signed-off-by: Kunshang Ji <kunshang.ji@intel.com>
Kunshang Ji committed
May 19, 2026 at 03:17 UTC
36dcaf25d8e091ea0f47b9ce7dcfca05de56f16d
4 files changed
+55
-13
.buildkite/intel_jobs/test-intel.yaml
+3
-1
@@ -39,7 +39,9 @@ steps:
39
python3 examples/basic/offline_inference/generate.py --model nvidia/Llama-3.1-8B-Instruct-FP8 --block-size 64 --enforce-eager --quantization modelopt --kv-cache-dtype fp8 --attention-backend TRITON_ATTN --max-model-len 4096 &&
40
python3 examples/basic/offline_inference/generate.py --model superjob/Qwen3-4B-Instruct-2507-GPTQ-Int4 --block-size 64 --enforce-eager --max-model-len 8192 &&
41
python3 examples/basic/offline_inference/generate.py --model ibm-research/PowerMoE-3b --block-size 64 --enforce-eager -tp 2 &&
42
- python3 examples/basic/offline_inference/generate.py --model ibm-research/PowerMoE-3b --block-size 64 --enforce-eager -tp 2 --enable-expert-parallel'
42
+ python3 examples/basic/offline_inference/generate.py --model ibm-research/PowerMoE-3b --block-size 64 --enforce-eager -tp 2 --enable-expert-parallel &&
43
+ python3 examples/basic/offline_inference/generate.py --model superjob/Qwen3-4B-Instruct-2507-GPTQ-Int4 --max-model-len 8192
44
+ '
45
- label: "XPU V1 test"
46
depends_on:
47
- image-build-xpu
vllm/model_executor/kernels/linear/mixed_precision/MPLinearKernel.py
+2
-2
@@ -50,8 +50,8 @@ class MPLinearKernel(ABC):
50
assert w_zp_param_name is not None
51
if c.has_g_idx:
52
assert w_gidx_param_name is not None
53
- self.w_zp_name = w_zp_param_name
54
- self.w_gidx_name = w_gidx_param_name
53
+ self.w_zp_name: str | None = w_zp_param_name
54
+ self.w_gidx_name: str | None = w_gidx_param_name
55
56
@abstractmethod
57
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
vllm/model_executor/kernels/linear/mixed_precision/xpu.py
+47
-10
@@ -61,17 +61,53 @@ class XPUwNa16LinearKernel(MPLinearKernel):
61
return True, None
62
63
def process_weights_after_loading(self, layer: torch.nn.Module):
64
- layer.weight_scale.data = layer.weight_scale.t().contiguous()
64
+ # Default names since marlin requires empty parameters for these,
65
+ # TODO: remove this requirement from marlin (allow optional tensors)
66
+ if self.w_gidx_name is None:
67
+ self.w_gidx_name = "g_idx"
68
+ if self.w_zp_name is None:
69
+ self.w_zp_name = "w_zp"
70
+
71
+ need_transpose = False
72
+ qweight_shape = getattr(layer, self.w_q_name).shape
73
+ scale_shape = getattr(layer, self.w_s_name).shape
74
+ # gptq marlin and compressed tensors wna16 expect different default
75
+ # layouts for weight and scale, so we check the shapes to determine
76
+ # if we need to transpose
77
+ if qweight_shape[0] != scale_shape[0]:
78
+ need_transpose = True
79
+
80
+ if need_transpose:
81
+ getattr(layer, self.w_q_name).data = (
82
+ getattr(layer, self.w_q_name).data.t().contiguous()
83
+ )
84
+ getattr(layer, self.w_s_name).data = getattr(layer, self.w_s_name).data
85
+ else:
86
+ getattr(layer, self.w_s_name).data = (
87
+ getattr(layer, self.w_s_name).data.t().contiguous()
88
+ )
89
90
if self.config.zero_points:
67
- layer.weight_zero_point.data = layer.weight_zero_point.t().contiguous()
91
+ # (FIXME): maybe zero points should also be transposed.
92
+ getattr(layer, self.w_zp_name).data = (
93
+ getattr(layer, self.w_zp_name).data.t().contiguous()
94
+ )
95
else:
96
weight_zero_point = torch.Tensor([8]).to(torch.int8).to("xpu")
70
- layer.weight_zero_point = Parameter(weight_zero_point, requires_grad=False)
97
+ setattr(
98
+ layer, self.w_zp_name, Parameter(weight_zero_point, requires_grad=False)
99
+ )
100
if self.config.has_g_idx:
72
- layer.g_idx.data = layer.g_idx.t().contiguous()
101
+ setattr(
102
+ layer,
103
+ self.w_gidx_name,
104
+ Parameter(
105
+ getattr(layer, self.w_gidx_name).data.t().contiguous(),
106
+ requires_grad=False,
107
+ ),
108
+ )
109
else:
74
- layer.g_idx = None
110
+ setattr(layer, self.w_gidx_name, None)
111
112
def apply_weights(
113
self,
@@ -80,14 +116,15 @@ class XPUwNa16LinearKernel(MPLinearKernel):
116
bias: torch.Tensor | None = None,
117
) -> torch.Tensor:
118
reshaped_x = x.reshape(-1, x.shape[-1])
119
+ w_q, w_s, w_zp, w_gidx = self._get_weight_params(layer)
120
out = torch.ops._xpu_C.int4_gemm_w4a16(
121
reshaped_x,
85
- layer.weight_packed.t(),
86
- bias,
87
- layer.weight_scale,
88
- layer.weight_zero_point,
122
+ w_q.t(),
123
+ bias if bias is not None else None,
124
+ w_s,
125
+ w_zp,
126
self.config.group_size,
90
- layer.g_idx,
127
+ w_gidx,
128
)
129
return out
130
vllm/model_executor/layers/quantization/utils/marlin_utils.py
+3
@@ -47,6 +47,9 @@ def query_marlin_supported_quant_types(
47
if current_platform.is_cpu():
48
return _query_cpu_marlin_supported_quant_types(has_zp, include_fp_type)
49
50
+ if current_platform.is_xpu():
51
+ return [scalar_types.uint4, scalar_types.uint4b8]
52
+
53
if not current_platform.is_rocm():
54
if device_capability is None:
55
capability_tuple = current_platform.get_device_capability()