A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 4, 13:45 UTC
[Bugfix][ROCm] Fix gemm_a4w4 call to use updated AITER API signature (#40754)
Signed-off-by: cheiluno <cheiluno@amd.com>
chelnnexy committed
Apr 28, 2026 at 19:04 UTC
d109eacd05f774008c7e1d17afc76fc48c4fcbc5
1 file changed
+2
-11
vllm/model_executor/layers/quantization/quark/schemes/quark_ocp_mx.py
+2
-11
@@ -96,21 +96,12 @@ try:
96
x_q = x
97
x_s = x_scales
98
99
- # 32 alignment is enough for dim0 padding of output for
100
- # gemm_a4w4 kernel
101
- y = torch.empty(
102
- (M + 31) // 32 * 32,
103
- weight.shape[0],
104
- device=x_q.device,
105
- dtype=out_dtype,
106
- )
107
-
108
- gemm_a4w4(
99
+ y = gemm_a4w4(
100
x_q,
101
weight.view(x_q.dtype),
102
x_s,
103
weight_scale.view(x_s.dtype),
113
- y,
104
+ dtype=out_dtype,
105
bpreshuffle=True,
106
)
107
return y[:M]