A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 12, 07:15 UTC
[Bugfix] Fix mismatched nvfp4 gemm output shape (#29742)
Signed-off-by: Isotr0py <mozf@mail2.sysu.edu.cn>
Isotr0py committed
Nov 30, 2025 at 17:15 UTC
47539cfd3e5006159e427ee5bc32823f6cef7ec3
1 file changed
+1
-1
vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_w4a4_nvfp4.py
+1
-1
@@ -184,7 +184,7 @@ class CompressedTensorsW4A4Fp4(CompressedTensorsScheme):
184
return out
185
186
output_dtype = x.dtype
187
- output_shape = [x.shape[0], layer.weight_packed.shape[0]]
187
+ output_shape = [*x.shape[:-1], layer.weight_packed.shape[0]]
188
189
# quantize BF16 or FP16 to (FP4 and interleaved block scale)
190
x_fp4, x_blockscale = scaled_fp4_quant(x, layer.input_global_scale)