A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 8, 17:45 UTC
[Rocm][Bugfix] Fix dtype not same for gemm_a4w4 op (#33734)
Signed-off-by: charlifu <charlifu@amd.com>
Charlie Fu committed
Feb 6, 2026 at 13:09 UTC
fe5438200b1cd5e496cf50f095d208eb1fb18fdd
1 file changed
+6
-1
vllm/model_executor/layers/quantization/quark/schemes/quark_ocp_mx.py
+6
-1
@@ -118,7 +118,12 @@ try:
118
)
119
120
gemm_a4w4(
121
- x_q, weight, x_s, weight_scale.view(x_s.dtype), y, bpreshuffle=True
121
+ x_q,
122
+ weight.view(x_q.dtype),
123
+ x_s,
124
+ weight_scale.view(x_s.dtype),
125
+ y,
126
+ bpreshuffle=True,
127
)
128
return y[:M]
129
else: