A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 6, 08:15 UTC
[Bugfix] Fix ConchLinearKernel channelwise quantization (group_size=-1) (#37329)
Signed-off-by: Matthias Gehre <matthias.gehre@amd.com>
Matthias Gehre committed
Mar 20, 2026 at 16:32 UTC
5e806bcf541c0a90619bbf4fab3ab721c98b12a1
1 file changed
+9
-1
vllm/model_executor/kernels/linear/mixed_precision/conch.py
+9
-1
@@ -124,6 +124,14 @@ class ConchLinearKernel(MPLinearKernel):
124
125
w_q, w_s, w_zp, _ = self._get_weight_params(layer)
126
127
+ # Map channelwise group_size=-1 to the actual input dimension K.
128
+ # The conch kernel computes stride_mul = block_k / group_size;
129
+ # passing -1 produces a negative stride that reads out-of-bounds
130
+ # scale values for all K-blocks after the first.
131
+ group_size = self.config.group_size
132
+ if group_size == -1:
133
+ group_size = x.shape[-1]
134
+
135
output = mixed_precision_gemm(
136
x=x,
137
w_q_packed=w_q.data,
@@ -131,7 +139,7 @@ class ConchLinearKernel(MPLinearKernel):
139
w_zp=w_zp.data if w_zp is not None else None,
140
weight_size_bits=self.config.weight_type.size_bits,
141
weight_bias=self.config.weight_type.bias,
134
- group_size=self.config.group_size,
142
+ group_size=group_size,
143
)
144
145
if bias is not None: