A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 10, 22:45 UTC
[Bugfix][Hardware][AMD] Fix FP8 dtype in silu_mul quantization (#31179)
Signed-off-by: c0de128 <kevin.mckay@outlook.com>
Kevin McKay committed
Dec 24, 2025 at 09:37 UTC
66c98874401b1e246e6c649942156720da671acd
1 file changed
+8
-4
vllm/model_executor/layers/quantization/utils/fp8_utils.py
+8
-4
@@ -625,8 +625,9 @@ def silu_mul_per_token_group_quant_fp8_colmajor(
625
M, N = input.size()
626
N_2 = N // 2
627
628
+ fp8_dtype = current_platform.fp8_dtype()
629
if output is None:
629
- output = torch.empty((M, N_2), dtype=torch.float8_e4m3fn, device=input.device)
630
+ output = torch.empty((M, N_2), dtype=fp8_dtype, device=input.device)
631
632
output_scales = torch.empty(
633
((N_2 // GROUP_SIZE), M), dtype=torch.float32, device=input.device
@@ -637,9 +638,12 @@ def silu_mul_per_token_group_quant_fp8_colmajor(
638
assert M % BLOCK_M == 0
639
assert N_2 % BLOCK_N == 0
640
640
- finfo = torch.finfo(torch.float8_e4m3fn)
641
- fp8_min = finfo.min
642
- fp8_max = finfo.max
641
+ # Using the default value (240.0) from pytorch will cause accuracy
642
+ # issue on dynamic quantization models. Here use 224.0 for fnuz on ROCm
643
+ # platforms that use the torch.float8_e4m3fnuz dtype.
644
+ finfo = torch.finfo(fp8_dtype)
645
+ fp8_min = -224.0 if current_platform.is_fp8_fnuz() else finfo.min
646
+ fp8_max = 224.0 if current_platform.is_fp8_fnuz() else finfo.max
647
648
# Force even division so we can avoid edgecases within the kernel.
649
assert M % BLOCK_M == 0