A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[Bugfix] Enable padded FP4 quantization (#25947)
Signed-off-by: Roi Koren <roik@nvidia.com>
roikoren755 committed
Oct 9, 2025 at 20:59 UTC
4069db3f2e3fad65251ca5ed659c83ba48894ef2
2 files changed
+1
-3
vllm/_custom_ops.py
+1
-1
@@ -1384,7 +1384,7 @@ def scaled_fp4_quant(
1384
rounded_m = round_up(m, 128)
1385
scale_n = n // block_size
1386
rounded_n = round_up(scale_n, 4)
1387
- output_scale = torch.empty(
1387
+ output_scale = torch.zeros(
1388
(rounded_m, rounded_n // 4), device=device, dtype=torch.int32
1389
)
1390
vllm/utils/flashinfer.py
-2
@@ -386,8 +386,6 @@ def flashinfer_scaled_fp4_mm(
386
assert block_scale_a.ndim == 2 and block_scale_b.ndim == 2
387
assert a.stride(-1) == 1 and b.stride(-1) == 1
388
assert a.shape[1] == b.shape[1]
389
- assert block_scale_a.shape[1] == a.shape[1] // 8
390
- assert block_scale_b.shape[1] == b.shape[1] // 8
389
390
if backend == "cutlass":
391
block_scale_a = block_scale_a.view(torch.uint8)