A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 04:45 UTC
static_scaled_fp8_quant should not run when scale.numel is not 1 (#20076)
Eldar Kurtić committed
Jun 25, 2025 at 21:08 UTC
8b8c209e352899c870fe348013a99a91262bf1e4
1 file changed
+1
-1
vllm/_custom_ops.py
+1
-1
@@ -1276,7 +1276,7 @@ def scaled_fp8_quant(
1276
torch.ops._C.dynamic_scaled_fp8_quant(output, input, scale)
1277
else:
1278
# num_token_padding not implemented for this case
1279
- assert (scale.numel() == 1 or num_token_padding is None)
1279
+ assert (scale.numel() == 1 and num_token_padding is None)
1280
torch.ops._C.static_scaled_fp8_quant(output, input, scale)
1281
1282
return output, scale