A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 11, 18:45 UTC
[bugfix][quantization] Fix fp8 per_tensor scale shape (#30257)
Signed-off-by: Haoyang Li <lihaoyang0109@gmail.com>
haoyangli-amd committed
Dec 9, 2025 at 19:28 UTC
03416eada6c01770fb71c3d988fc3c74958d8f5e
1 file changed
+1
-1
vllm/_custom_ops.py
+1
-1
@@ -1726,7 +1726,7 @@ def scaled_fp8_quant(
1726
output, input, scale, scale_ub
1727
)
1728
else:
1729
- scale = torch.empty((1, 1), device=input.device, dtype=torch.float32)
1729
+ scale = torch.empty(1, device=input.device, dtype=torch.float32)
1730
torch.ops._C.dynamic_scaled_fp8_quant(output, input, scale)
1731
else:
1732
assert scale.numel() == 1, f"{scale.shape}"