A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 04:45 UTC
Fix num_token_padding support for static per-tensor scaled_fp8_quant (#20188)
Signed-off-by: mgoin <mgoin64@gmail.com>
Michael Goin committed
Jun 28, 2025 at 14:48 UTC
a29e62ea3452bc6b1d4f3eeac2dc9a6b30357c4d
1 file changed
+1
-2
vllm/_custom_ops.py
+1
-2
@@ -1274,8 +1274,7 @@ def scaled_fp8_quant(
1274
scale = torch.zeros(1, device=input.device, dtype=torch.float32)
1275
torch.ops._C.dynamic_scaled_fp8_quant(output, input, scale)
1276
else:
1277
- # num_token_padding not implemented for this case
1278
- assert (scale.numel() == 1 and num_token_padding is None)
1277
+ assert scale.numel() == 1
1278
torch.ops._C.static_scaled_fp8_quant(output, input, scale)
1279
1280
return output, scale