A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 11, 13:45 UTC
[CI/Build][Kernel][BugFix][AMD] Fix per_token_group_quant_fp8 to use correct fp8 min/max values and update atol/rtol in test_quantfp8_group_functionality (#30292)
Signed-off-by: Randall Smith <ransmith@amd.com> Co-authored-by: Randall Smith <ransmith@amd.com>
rasmith committed
Dec 12, 2025 at 17:41 UTC
08f8a5627e6306312202d5c0fda8b7255d2f27fc
2 files changed
+7
-4
tests/kernels/quantization/test_fp8_quant_group.py
+2
-2
@@ -62,7 +62,7 @@ def test_quantfp8_group_functionality(
62
assert scales_col.stride(1) == batch_size
63
64
# Test column-major scales consistency
65
- assert torch.allclose(scales_col, scales_native, rtol=1e-9, atol=1e-8)
65
+ torch.testing.assert_close(scales_col, scales_native, rtol=1e-9, atol=1e-8)
66
67
# 3. Test CUDA implementation (only for divisible dimensions)
68
if is_divisible:
@@ -71,7 +71,7 @@ def test_quantfp8_group_functionality(
71
assert scales_cuda.shape == (batch_size, expected_num_groups)
72
73
# Verify CUDA/native consistency
74
- assert torch.allclose(scales_cuda, scales_native, rtol=1e-9, atol=1e-8)
74
+ torch.testing.assert_close(scales_cuda, scales_native, rtol=2e-7, atol=2e-8)
75
76
# Quantized values should mostly match
77
diff_count = (x_quant_cuda != x_quant_native).sum().item()
vllm/model_executor/layers/quantization/utils/fp8_utils.py
+5
-2
@@ -762,9 +762,12 @@ def per_token_group_quant_fp8(
762
)
763
assert x.stride(-1) == 1, "`x` groups must be contiguous"
764
765
+ # Using the default value (240.0) from pytorch will cause accuracy
766
+ # issue on dynamic quantization models. Here use 224.0 for fnuz on ROCm
767
+ # platforms that use the torch.float8_e4mefnuz dtype.
768
finfo = torch.finfo(dtype)
766
- fp8_min = finfo.min
767
- fp8_max = finfo.max
769
+ fp8_min = -224.0 if current_platform.is_fp8_fnuz() else finfo.min
770
+ fp8_max = 224.0 if current_platform.is_fp8_fnuz() else finfo.max
771
772
assert out_q is None or out_q.shape == x.shape
773
x_q = out_q