A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 10, 15:15 UTC
[Bugfix][Hardware][AMD] Consolidate FP8 min/max values helper function (#31106)
Signed-off-by: c0de128 <kevin.mckay@outlook.com> Signed-off-by: Kevin McKay <kevin@example.com> Co-authored-by: Claude Sonnet 4.5 <noreply@anthropic.com>
Kevin McKay committed
Jan 7, 2026 at 00:55 UTC
4614c5a539282e6a8055d3b370b0f58c6f094c25
6 files changed
+102
-39
tests/kernels/quant_utils.py
+11
-25
@@ -4,13 +4,13 @@
4
5
import torch
6
7
-from vllm.model_executor.layers.quantization.utils.quant_utils import group_broadcast
7
+from vllm.model_executor.layers.quantization.utils.quant_utils import (
8
+ get_fp8_min_max,
9
+ group_broadcast,
10
+)
11
from vllm.platforms import current_platform
12
from vllm.utils.math_utils import round_up
13
11
-# Using the default value (240.0) from pytorch will cause accuracy
12
-# issue on dynamic quantization models. Here use 224.0 for rocm.
13
-ROCM_FP8FNUZ_MAX = 224.0
14
FP8_DTYPE = current_platform.fp8_dtype()
15
16
@@ -25,16 +25,12 @@ def ref_dynamic_per_token_quant(
25
if scale_ub is not None:
26
assert quant_dtype == FP8_DTYPE
27
28
- qtype_traits = (
29
- torch.iinfo(quant_dtype)
30
- if quant_dtype == torch.int8
31
- else torch.finfo(quant_dtype)
32
- )
33
- use_fp8fnuz = (
34
- current_platform.is_fp8_fnuz() and quant_dtype == current_platform.fp8_dtype()
35
- )
36
- qtype_traits_max = ROCM_FP8FNUZ_MAX if use_fp8fnuz else qtype_traits.max
37
- qtype_traits_min = -ROCM_FP8FNUZ_MAX if use_fp8fnuz else qtype_traits.min
28
+ if quant_dtype == torch.int8:
29
+ qtype_traits = torch.iinfo(quant_dtype)
30
+ qtype_traits_min = qtype_traits.min
31
+ qtype_traits_max = qtype_traits.max
32
+ else:
33
+ qtype_traits_min, qtype_traits_max = get_fp8_min_max()
34
qtype_max = as_float32_tensor(qtype_traits_max)
35
s_1 = as_float32_tensor(1.0)
36
s_512 = as_float32_tensor(512.0)
@@ -72,17 +68,7 @@ def ref_dynamic_per_token_quant(
68
def ref_dynamic_per_tensor_fp8_quant(
69
x: torch.Tensor,
70
) -> tuple[torch.Tensor, torch.Tensor]:
75
- fp8_traits = torch.finfo(FP8_DTYPE)
76
- fp8_traits_max = (
77
- ROCM_FP8FNUZ_MAX
78
- if current_platform.is_rocm() and current_platform.is_fp8_fnuz()
79
- else fp8_traits.max
80
- )
81
- fp8_traits_min = (
82
- -ROCM_FP8FNUZ_MAX
83
- if current_platform.is_rocm() and current_platform.is_fp8_fnuz()
84
- else fp8_traits.min
85
- )
71
+ fp8_traits_min, fp8_traits_max = get_fp8_min_max()
72
fp8_max = as_float32_tensor(fp8_traits_max)
73
one = as_float32_tensor(1.0)
74
tests/kernels/quantization/test_fp8_min_max_helper.py
new
+65
@@ -0,0 +1,65 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+"""
4
+Unit tests for the get_fp8_min_max() helper function.
5
+
6
+These tests verify the FP8 min/max value logic for both standard
7
+and fnuz (ROCm MI300) dtype handling.
8
+"""
9
+
10
+from unittest.mock import patch
11
+
12
+import pytest
13
+import torch
14
+
15
+from vllm.model_executor.layers.quantization.utils.quant_utils import (
16
+ get_fp8_min_max,
17
+)
18
+
19
+
20
+class TestGetFp8MinMax:
21
+ """Test cases for get_fp8_min_max() function."""
22
+
23
+ @patch("vllm.model_executor.layers.quantization.utils.quant_utils.current_platform")
24
+ def test_standard_fp8_platform(self, mock_platform):
25
+ """Test that standard FP8 platform uses PyTorch's finfo values."""
26
+ mock_platform.is_fp8_fnuz.return_value = False
27
+ mock_platform.fp8_dtype.return_value = torch.float8_e4m3fn
28
+
29
+ fp8_min, fp8_max = get_fp8_min_max()
30
+ finfo = torch.finfo(torch.float8_e4m3fn)
31
+
32
+ # Standard FP8 max is 448.0 for e4m3fn
33
+ assert fp8_max == finfo.max, f"Expected finfo.max={finfo.max}, got {fp8_max}"
34
+ assert fp8_min == finfo.min, f"Expected finfo.min={finfo.min}, got {fp8_min}"
35
+
36
+ @patch("vllm.model_executor.layers.quantization.utils.quant_utils.current_platform")
37
+ def test_fnuz_platform_returns_224(self, mock_platform):
38
+ """Test that fnuz platform returns 224.0."""
39
+ mock_platform.is_fp8_fnuz.return_value = True
40
+
41
+ fp8_min, fp8_max = get_fp8_min_max()
42
+
43
+ # fnuz on ROCm MI300 should return 224.0, not 240.0
44
+ assert fp8_max == 224.0, f"Expected 224.0 for fnuz platform, got {fp8_max}"
45
+ assert fp8_min == -224.0, f"Expected -224.0 for fnuz platform, got {fp8_min}"
46
+
47
+ @patch("vllm.model_executor.layers.quantization.utils.quant_utils.current_platform")
48
+ def test_non_fnuz_platform_uses_finfo(self, mock_platform):
49
+ """Test that non-fnuz platform uses finfo values."""
50
+ mock_platform.is_fp8_fnuz.return_value = False
51
+ mock_platform.fp8_dtype.return_value = torch.float8_e4m3fn
52
+
53
+ fp8_min, fp8_max = get_fp8_min_max()
54
+ finfo = torch.finfo(torch.float8_e4m3fn)
55
+
56
+ assert fp8_max == finfo.max, (
57
+ f"Non-fnuz platform should use finfo.max={finfo.max}, got {fp8_max}"
58
+ )
59
+ assert fp8_min == finfo.min, (
60
+ f"Non-fnuz platform should use finfo.min={finfo.min}, got {fp8_min}"
61
+ )
62
+
63
+
64
+if __name__ == "__main__":
65
+ pytest.main([__file__, "-v"])
vllm/model_executor/layers/quantization/input_quant_fp8.py
+5
-6
@@ -7,15 +7,14 @@ import torch.nn.functional as F
7
from vllm import _custom_ops as ops
8
from vllm._aiter_ops import rocm_aiter_ops
9
from vllm.model_executor.custom_op import CustomOp
10
-from vllm.model_executor.layers.quantization.utils.quant_utils import GroupShape
10
+from vllm.model_executor.layers.quantization.utils.quant_utils import (
11
+ GroupShape,
12
+ get_fp8_min_max,
13
+)
14
from vllm.platforms import current_platform
15
13
-# Using the default value (240.0) from pytorch will cause accuracy
14
-# issue on dynamic quantization models. Here use 224.0 for fnuz on ROCm.
16
_FP8_DTYPE = current_platform.fp8_dtype()
16
-_FP8_FINFO = torch.finfo(_FP8_DTYPE)
17
-_FP8_MAX = 224.0 if current_platform.is_fp8_fnuz() else _FP8_FINFO.max
18
-_FP8_MIN = -224.0 if current_platform.is_fp8_fnuz() else _FP8_FINFO.min
17
+_FP8_MIN, _FP8_MAX = get_fp8_min_max()
18
_FP8_MIN_SCALING_FACTOR = 1.0 / (_FP8_MAX * 512.0)
19
20
vllm/model_executor/layers/quantization/utils/fp8_utils.py
+5
-7
@@ -15,7 +15,10 @@ from vllm import _custom_ops as ops
15
from vllm._aiter_ops import rocm_aiter_ops
16
from vllm.logger import init_logger
17
from vllm.model_executor.layers.quantization.input_quant_fp8 import QuantFP8
18
-from vllm.model_executor.layers.quantization.utils.quant_utils import GroupShape
18
+from vllm.model_executor.layers.quantization.utils.quant_utils import (
19
+ GroupShape,
20
+ get_fp8_min_max,
21
+)
22
from vllm.model_executor.layers.quantization.utils.w8a8_utils import (
23
CUTLASS_BLOCK_FP8_SUPPORTED,
24
)
@@ -748,12 +751,7 @@ def per_token_group_quant_fp8(
751
)
752
assert x.stride(-1) == 1, "`x` groups must be contiguous"
753
751
- # Using the default value (240.0) from pytorch will cause accuracy
752
- # issue on dynamic quantization models. Here use 224.0 for fnuz on ROCm
753
- # platforms that use the torch.float8_e4mefnuz dtype.
754
- finfo = torch.finfo(dtype)
755
- fp8_min = -224.0 if current_platform.is_fp8_fnuz() else finfo.min
756
- fp8_max = 224.0 if current_platform.is_fp8_fnuz() else finfo.max
754
+ fp8_min, fp8_max = get_fp8_min_max()
755
756
assert out_q is None or out_q.shape == x.shape
757
x_q = out_q
vllm/model_executor/layers/quantization/utils/quant_utils.py
+11
@@ -19,6 +19,17 @@ FP8_DTYPE = current_platform.fp8_dtype()
19
FP4_DTYPE = torch.uint8
20
21
22
+def get_fp8_min_max() -> tuple[float, float]:
23
+ """Get the min and max values for FP8 quantization."""
24
+ # Using the default value (240.0) from pytorch will cause accuracy
25
+ # issue on dynamic quantization models on ROCm. Here, use 224.0 for fnuz
26
+ # on ROCm platforms that use the torch.float8_e4m3fnuz dtype.
27
+ if current_platform.is_fp8_fnuz():
28
+ return -224.0, 224.0
29
+ finfo = torch.finfo(current_platform.fp8_dtype())
30
+ return finfo.min, finfo.max
31
+
32
+
33
# Use proxy as NamedTuple direct subclasses cannot have static members
34
class _GroupShape(NamedTuple):
35
row: int
vllm/utils/deep_gemm.py
+5
-1
@@ -16,6 +16,9 @@ import torch
16
17
import vllm.envs as envs
18
from vllm.logger import logger
19
+from vllm.model_executor.layers.quantization.utils.quant_utils import (
20
+ get_fp8_min_max,
21
+)
22
from vllm.platforms import current_platform
23
from vllm.utils.import_utils import has_deep_gemm
24
from vllm.utils.math_utils import cdiv
@@ -355,7 +358,8 @@ def per_block_cast_to_fp8(
358
x_padded[:m, :n] = x
359
x_view = x_padded.view(-1, block_m, x_padded.size(1) // block_n, block_n)
360
x_amax = x_view.abs().float().amax(dim=(1, 3), keepdim=True).clamp(1e-4)
358
- sf = x_amax / 224.0 if current_platform.is_fp8_fnuz() else x_amax / 448.0
361
+ _, fp8_max = get_fp8_min_max()
362
+ sf = x_amax / fp8_max
363
sf = _ceil_to_ue8m0(sf) if use_ue8m0 else sf
364
x_scaled = (x_view * (1.0 / sf)).to(fp8_dtype)
365
return x_scaled.view_as(x_padded)[:m, :n].contiguous(), sf.view(