A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 10, 04:00 UTC
[CI][AMD][Quantization][BugFix] Fix fp8 max in quant_utils.py and update test_fp8_quant.::test_static_fp8_quant_group_2d to use correct fp8 dtype and adjust atol/rtol (#32201)
Signed-off-by: Randall Smith <ransmith@amd.com>
rasmith committed
Jan 14, 2026 at 23:04 UTC
3c2685645ed602e3db8010cd62d88d3130115f06
2 files changed
+4
-3
tests/kernels/quantization/test_fp8_quant.py
+2
-2
@@ -178,12 +178,12 @@ def test_static_fp8_quant_group_2d(
178
179
x = torch.rand(num_tokens, hidden_size, dtype=dtype, device="cuda")
180
ref_out, scale = scaled_quantize(
181
- x, group_shape, FP8_DTYPE, compute_dtype=torch.float32
181
+ x, group_shape, current_platform.fp8_dtype(), compute_dtype=torch.float32
182
)
183
ops_out, ops_scale = ops.scaled_fp8_quant(x, scale=scale, group_shape=group_shape)
184
185
torch.testing.assert_close(scale, ops_scale)
186
- torch.testing.assert_close(ref_out.float(), ops_out.float(), rtol=0.12, atol=0.0)
186
+ torch.testing.assert_close(ref_out.float(), ops_out.float(), rtol=1.2e-1, atol=1e-5)
187
188
opcheck_fp8_quant(ops_out, x, scale=scale)
189
vllm/model_executor/layers/quantization/utils/quant_utils.py
+2
-1
@@ -221,7 +221,8 @@ def scaled_quantize(
221
# Compute scales
222
min_val, max_val = x_blkd_permd.aminmax(dim=-1)
223
amax = torch.maximum(min_val.abs(), max_val.abs()).clamp(min=1e-12)
224
- scale = finfo.max / amax
224
+ _, fp8_max = get_fp8_min_max()
225
+ scale = fp8_max / amax
226
227
# Apply scale and convert form:
228
# (BLK_M, BLK_N, BLOCK_SIZE_M * BLOCK_SIZE_N) to (M, N)