A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[Misc] Removed force_fp8_e4m3fnuz from FP8LinearOp (#23725)
Signed-off-by: Julien Lin <jullin@nvidia.com> Signed-off-by: Luka Govedič <ProExpertProg@users.noreply.github.com> Co-authored-by: Luka Govedič <ProExpertProg@users.noreply.github.com>
nvjullin committed
Sep 4, 2025 at 21:25 UTC
37241077d563d5808ce8f36ebce4eec25e1c1f88
5 files changed
+45
-30
tests/compile/test_fusion.py
+16
-11
@@ -15,9 +15,10 @@ from vllm.model_executor.layers.layernorm import RMSNorm
15
from vllm.model_executor.layers.quantization.utils.quant_utils import (
16
GroupShape, QuantKey, ScaleDesc)
17
from vllm.model_executor.layers.quantization.utils.w8a8_utils import (
18
- Fp8LinearOp, maybe_create_device_identity)
18
+ Fp8LinearOp, cutlass_fp8_supported, maybe_create_device_identity)
19
from vllm.platforms import current_platform
20
21
+from ..utils import override_cutlass_fp8_supported
22
from .backend import TestBackend
23
24
FP8_DTYPE = current_platform.fp8_dtype()
@@ -26,9 +27,9 @@ FP8_DTYPE = current_platform.fp8_dtype()
27
class TestModel(torch.nn.Module):
28
29
def __init__(self, hidden_size: int, eps: float, static: bool,
29
- force_fp8_e4m3fnuz: bool, *args, **kwargs):
30
+ cuda_force_torch: bool, *args, **kwargs):
31
super().__init__(*args, **kwargs)
31
- self.force_fp8_e4m3fnuz = force_fp8_e4m3fnuz
32
+ self.cuda_force_torch = cuda_force_torch
33
self.norm = [RMSNorm(hidden_size, eps) for _ in range(3)]
34
self.wscale = [torch.rand(1, dtype=torch.float32) for _ in range(2)]
35
group_shape = GroupShape.PER_TENSOR if static else GroupShape.PER_TOKEN
@@ -42,11 +43,12 @@ class TestModel(torch.nn.Module):
43
torch.rand(hidden_size, hidden_size).to(dtype=FP8_DTYPE).t()
44
for _ in range(2)
45
]
45
- self.fp8_linear = Fp8LinearOp(
46
- force_fp8_e4m3fnuz=force_fp8_e4m3fnuz,
47
- act_quant_static=static,
48
- act_quant_group_shape=group_shape,
49
- )
46
+
47
+ with override_cutlass_fp8_supported(not cuda_force_torch):
48
+ self.fp8_linear = Fp8LinearOp(
49
+ act_quant_static=static,
50
+ act_quant_group_shape=group_shape,
51
+ )
52
53
def forward(self, x):
54
resid = torch.sqrt(x)
@@ -81,11 +83,14 @@ class TestModel(torch.nn.Module):
83
@pytest.mark.parametrize("num_tokens", [7, 256, 533, 2048, 2049])
84
@pytest.mark.parametrize("eps", [1e-5, 1e-6])
85
@pytest.mark.parametrize("static", [True, False])
84
-@pytest.mark.parametrize("force_fp8_e4m3fnuz", [True, False])
86
+# cuda_force_torch used to test torch code path on platforms that
87
+# cutlass_fp8_supported() == True.
88
+@pytest.mark.parametrize("cuda_force_torch",
89
+ [True, False] if cutlass_fp8_supported() else [True])
90
@pytest.mark.skipif(envs.VLLM_TARGET_DEVICE not in ["cuda", "rocm"],
91
reason="Only test on CUDA and ROCm")
92
def test_fusion_rmsnorm_quant(dtype, hidden_size, num_tokens, eps, static,
88
- force_fp8_e4m3fnuz):
93
+ cuda_force_torch):
94
torch.set_default_device("cuda")
95
torch.set_default_dtype(dtype)
96
torch.manual_seed(1)
@@ -102,7 +107,7 @@ def test_fusion_rmsnorm_quant(dtype, hidden_size, num_tokens, eps, static,
107
fusion_pass = FusionPass.instance(vllm_config)
108
109
backend = TestBackend(noop_pass, fusion_pass)
105
- model = TestModel(hidden_size, eps, static, force_fp8_e4m3fnuz)
110
+ model = TestModel(hidden_size, eps, static, cuda_force_torch)
111
112
# First dimension dynamic
113
x = torch.rand(num_tokens, hidden_size)
tests/compile/test_silu_mul_quant_fusion.py
+15
-12
@@ -17,9 +17,10 @@ from vllm.model_executor.layers.activation import SiluAndMul
17
from vllm.model_executor.layers.quantization.utils.quant_utils import (
18
GroupShape, kFp8StaticTensorSym, kNvfp4Quant)
19
from vllm.model_executor.layers.quantization.utils.w8a8_utils import (
20
- Fp8LinearOp)
20
+ Fp8LinearOp, cutlass_fp8_supported)
21
from vllm.platforms import current_platform
22
23
+from ..utils import override_cutlass_fp8_supported
24
from .backend import TestBackend
25
26
FP8_DTYPE = current_platform.fp8_dtype()
@@ -32,7 +33,7 @@ def is_nvfp4_supported():
33
34
class TestSiluMulFp8QuantModel(torch.nn.Module):
35
35
- def __init__(self, hidden_size: int, force_fp8_e4m3fnuz: bool, **kwargs):
36
+ def __init__(self, hidden_size: int, cuda_force_torch: bool, **kwargs):
37
super().__init__()
38
self.silu_and_mul = SiluAndMul()
39
self.wscale = torch.rand(1, dtype=torch.float32)
@@ -40,11 +41,11 @@ class TestSiluMulFp8QuantModel(torch.nn.Module):
41
42
self.w = torch.rand(hidden_size, hidden_size).to(dtype=FP8_DTYPE).t()
43
43
- self.fp8_linear = Fp8LinearOp(
44
- force_fp8_e4m3fnuz=force_fp8_e4m3fnuz,
45
- act_quant_static=True,
46
- act_quant_group_shape=GroupShape.PER_TENSOR,
47
- )
44
+ with override_cutlass_fp8_supported(not cuda_force_torch):
45
+ self.fp8_linear = Fp8LinearOp(
46
+ act_quant_static=True,
47
+ act_quant_group_shape=GroupShape.PER_TENSOR,
48
+ )
49
50
def forward(self, x):
51
y = self.silu_and_mul(x)
@@ -96,12 +97,15 @@ class TestSiluMulNvfp4QuantModel(torch.nn.Module):
97
@pytest.mark.parametrize(
98
"model_class", [TestSiluMulFp8QuantModel, TestSiluMulNvfp4QuantModel]
99
if is_nvfp4_supported() else [TestSiluMulFp8QuantModel])
99
-@pytest.mark.parametrize("force_fp8_e4m3fnuz", [True, False])
100
+# cuda_force_torch used to test torch code path on platforms that
101
+# cutlass_fp8_supported() == True.
102
+@pytest.mark.parametrize("cuda_force_torch",
103
+ [True, False] if cutlass_fp8_supported() else [True])
104
@pytest.mark.skipif(envs.VLLM_TARGET_DEVICE not in ["cuda", "rocm"],
105
reason="Only test on CUDA and ROCm")
106
def test_fusion_silu_and_mul_quant(num_tokens, hidden_size, model_class,
103
- force_fp8_e4m3fnuz):
104
- if model_class == TestSiluMulNvfp4QuantModel and force_fp8_e4m3fnuz:
107
+ cuda_force_torch):
108
+ if model_class == TestSiluMulNvfp4QuantModel and cuda_force_torch:
109
pytest.skip("Duplicate tests for NVFP4")
110
111
torch.set_default_device("cuda")
@@ -114,8 +118,7 @@ def test_fusion_silu_and_mul_quant(num_tokens, hidden_size, model_class,
118
fusion_pass = ActivationQuantFusionPass(config)
119
120
backend = TestBackend(NoOpEliminationPass(config), fusion_pass)
117
- model = model_class(hidden_size=hidden_size,
118
- force_fp8_e4m3fnuz=force_fp8_e4m3fnuz)
121
+ model = model_class(hidden_size, cuda_force_torch)
122
123
# First dimension dynamic
124
x = torch.rand(num_tokens, hidden_size * 2)
tests/utils.py
+9
@@ -17,6 +17,7 @@ from contextlib import contextmanager, suppress
17
from multiprocessing import Process
18
from pathlib import Path
19
from typing import Any, Callable, Literal, Optional, Union
20
+from unittest.mock import patch
21
22
import cloudpickle
23
import httpx
@@ -1077,3 +1078,11 @@ def get_attn_backend_list_based_on_platform() -> list[str]:
1078
return attn_backend_list
1079
else:
1080
raise ValueError("Unsupported platform")
1081
+
1082
+
1083
+@contextmanager
1084
+def override_cutlass_fp8_supported(value: bool):
1085
+ with patch(
1086
+ "vllm.model_executor.layers.quantization.utils.w8a8_utils.cutlass_fp8_supported",
1087
+ return_value=value):
1088
+ yield
vllm/model_executor/layers/quantization/ptpc_fp8.py
+3
-3
@@ -92,13 +92,13 @@ class PTPCFp8LinearMethod(Fp8LinearMethod):
92
"""
93
94
def __init__(self, quant_config: PTPCFp8Config):
95
+ assert current_platform.is_rocm(), \
96
+ "PTPCFp8LinearMethod is only supported on ROCm."
97
super().__init__(quant_config=quant_config)
98
# Force weight quantization
99
self.quant_config.is_checkpoint_fp8_serialized = False
100
self.fp8_linear = Fp8LinearOp(
99
- act_quant_static=False,
100
- act_quant_group_shape=GroupShape.PER_TOKEN,
101
- force_fp8_e4m3fnuz=True)
101
+ act_quant_static=False, act_quant_group_shape=GroupShape.PER_TOKEN)
102
103
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
104
layer.weight = torch.nn.Parameter(layer.weight.data,
vllm/model_executor/layers/quantization/utils/w8a8_utils.py
+2
-4
@@ -355,12 +355,10 @@ class Fp8LinearOp:
355
def __init__(self,
356
act_quant_static: bool,
357
act_quant_group_shape: GroupShape = GroupShape.PER_TENSOR,
358
- pad_output: Optional[bool] = None,
359
- force_fp8_e4m3fnuz: bool = False):
358
+ pad_output: Optional[bool] = None):
359
if current_platform.is_rocm():
360
self.preferred_backend = "rocm"
362
- elif current_platform.is_cuda(
363
- ) and not force_fp8_e4m3fnuz and cutlass_fp8_supported():
361
+ elif current_platform.is_cuda() and cutlass_fp8_supported():
362
if has_flashinfer() and current_platform.has_device_capability(
363
100):
364
self.preferred_backend = "flashinfer"