| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | import itertools |
| 4 | from functools import partial |
| 5 | |
| 6 | import pytest |
| 7 | import torch |
| 8 | |
| 9 | import vllm.envs as envs |
| 10 | from tests.compile.backend import TestBackend |
| 11 | from tests.kernels.quantization.nvfp4_utils import quant_nvfp4_tensor |
| 12 | from tests.utils import TestFP8Layer |
| 13 | from vllm._aiter_ops import IS_AITER_FOUND, rocm_aiter_ops |
| 14 | from vllm._custom_ops import cutlass_scaled_fp4_mm, scaled_fp4_quant |
| 15 | from vllm.compilation.passes.fusion.act_quant_fusion import ( |
| 16 | FUSED_OPS, |
| 17 | SILU_MUL_OP, |
| 18 | ActivationQuantFusionPass, |
| 19 | ) |
| 20 | from vllm.compilation.passes.fusion.rms_quant_fusion import QUANT_OPS |
| 21 | from vllm.compilation.passes.utility.noop_elimination import NoOpEliminationPass |
| 22 | from vllm.compilation.passes.utility.post_cleanup import PostCleanupPass |
| 23 | from vllm.config import ( |
| 24 | CompilationConfig, |
| 25 | CompilationMode, |
| 26 | PassConfig, |
| 27 | VllmConfig, |
| 28 | set_current_vllm_config, |
| 29 | ) |
| 30 | from vllm.model_executor.kernels.linear import ( |
| 31 | CutlassFP8ScaledMMLinearKernel, |
| 32 | FlashInferFP8ScaledMMLinearKernel, |
| 33 | FP8ScaledMMLinearKernel, |
| 34 | PerTensorTorchFP8ScaledMMLinearKernel, |
| 35 | ROCmFP8ScaledMMLinearKernel, |
| 36 | ) |
| 37 | from vllm.model_executor.layers.activation import SiluAndMul |
| 38 | from vllm.model_executor.layers.quantization.input_quant_fp8 import QuantFP8 |
| 39 | from vllm.model_executor.layers.quantization.utils.quant_utils import ( |
| 40 | GroupShape, |
| 41 | create_fp8_quant_key, |
| 42 | kFp8Dynamic128Sym, |
| 43 | kFp8StaticTensorSym, |
| 44 | kNvfp4Dynamic, |
| 45 | ) |
| 46 | from vllm.platforms import current_platform |
| 47 | from vllm.utils.deep_gemm import is_deep_gemm_supported |
| 48 | |
| 49 | FP8_DTYPE = current_platform.fp8_dtype() |
| 50 | FP4_DTYPE = torch.uint8 |
| 51 | |
| 52 | |
| 53 | def is_nvfp4_supported(): |
| 54 | return current_platform.has_device_capability(100) |
| 55 | |
| 56 | |
| 57 | class TestSiluMulFp8QuantModel(torch.nn.Module): |
| 58 | quant_key = kFp8StaticTensorSym |
| 59 | |
| 60 | def __init__( |
| 61 | self, |
| 62 | hidden_size: int, |
| 63 | force_kernel: type[FP8ScaledMMLinearKernel], |
| 64 | dtype: torch.dtype, |
| 65 | **kwargs, |
| 66 | ): |
| 67 | super().__init__() |
| 68 | self.silu_and_mul = SiluAndMul() |
| 69 | |
| 70 | self.fp8_linear = TestFP8Layer( |
| 71 | weight_shape=(hidden_size, hidden_size), |
| 72 | activation_quant_key=self.quant_key, |
| 73 | weight_quant_key=self.quant_key, |
| 74 | force_kernel=force_kernel, |
| 75 | input_dtype=dtype, |
| 76 | ) |
| 77 | |
| 78 | self.enable_silu_mul_custom_op = self.silu_and_mul.enabled() |
| 79 | self.enable_quant_fp8_custom_op = self.fp8_linear.is_quant_fp8_enabled() |
| 80 | |
| 81 | def forward(self, x): |
| 82 | y = self.silu_and_mul(x) |
| 83 | x2 = self.fp8_linear(y) |
| 84 | return x2 |
| 85 | |
| 86 | def ops_in_model_before(self): |
| 87 | return [ |
| 88 | SILU_MUL_OP if self.enable_silu_mul_custom_op else torch.ops.aten.mul, |
| 89 | ( |
| 90 | QUANT_OPS[kFp8StaticTensorSym] |
| 91 | if self.enable_quant_fp8_custom_op |
| 92 | else torch.ops.aten.reciprocal |
| 93 | ), |
| 94 | ] |
| 95 | |
| 96 | def ops_in_model_after(self): |
| 97 | return [FUSED_OPS[kFp8StaticTensorSym]] |
| 98 | |
| 99 | |
| 100 | class TestSiluMulNvfp4QuantModel(torch.nn.Module): |
| 101 | def __init__(self, hidden_size: int, x: torch.Tensor, **kwargs): |
| 102 | super().__init__() |
| 103 | from vllm.compilation.passes.fusion.act_quant_fusion import ( |
| 104 | silu_and_mul_nvfp4_quant_supported, |
| 105 | ) |
| 106 | |
| 107 | assert silu_and_mul_nvfp4_quant_supported |
| 108 | |
| 109 | self.silu_and_mul = SiluAndMul() |
| 110 | self.enable_silu_mul_custom_op = self.silu_and_mul.enabled() |
| 111 | |
| 112 | # create nvfp4 weight |
| 113 | w = torch.rand((hidden_size, hidden_size)) |
| 114 | self.w, self.w_block_scale, self.w_global_scale = quant_nvfp4_tensor(w) |
| 115 | |
| 116 | # get global scale offline |
| 117 | _, _, self.y_global_scale = quant_nvfp4_tensor(self.silu_and_mul(x)) |
| 118 | |
| 119 | self.alpha = 1.0 / (self.w_global_scale * self.y_global_scale) |
| 120 | |
| 121 | def forward(self, x): |
| 122 | y = self.silu_and_mul(x) |
| 123 | y_quant, y_block_scale = scaled_fp4_quant(y, self.y_global_scale) |
| 124 | out = cutlass_scaled_fp4_mm( |
| 125 | a=y_quant, |
| 126 | b=self.w, |
| 127 | block_scale_a=y_block_scale, |
| 128 | block_scale_b=self.w_block_scale, |
| 129 | alpha=self.alpha, |
| 130 | out_dtype=y.dtype, |
| 131 | ) |
| 132 | return out |
| 133 | |
| 134 | def ops_in_model_before(self): |
| 135 | return [ |
| 136 | SILU_MUL_OP if self.enable_silu_mul_custom_op else torch.ops.aten.mul, |
| 137 | QUANT_OPS[kNvfp4Dynamic], |
| 138 | ] |
| 139 | |
| 140 | def ops_in_model_after(self): |
| 141 | return [FUSED_OPS[kNvfp4Dynamic]] |
| 142 | |
| 143 | |
| 144 | class TestSiluMulGroupFp8QuantModel(torch.nn.Module): |
| 145 | act_quant_key = kFp8Dynamic128Sym |
| 146 | |
| 147 | def __init__(self, hidden_size: int, dtype: torch.dtype, **kwargs): |
| 148 | super().__init__() |
| 149 | self.silu_and_mul = SiluAndMul() |
| 150 | self.weight_quant_key = create_fp8_quant_key( |
| 151 | static=True, group_shape=GroupShape(hidden_size, hidden_size) |
| 152 | ) |
| 153 | |
| 154 | self.w8a8_block_fp8_linear = TestFP8Layer( |
| 155 | weight_shape=(hidden_size, hidden_size), |
| 156 | weight_quant_key=self.weight_quant_key, |
| 157 | activation_quant_key=self.act_quant_key, |
| 158 | input_dtype=dtype, |
| 159 | ) |
| 160 | |
| 161 | self.enable_silu_mul_custom_op = self.silu_and_mul.enabled() |
| 162 | |
| 163 | def forward(self, x): |
| 164 | y = self.silu_and_mul(x) |
| 165 | x2 = self.w8a8_block_fp8_linear(y) |
| 166 | return x2 |
| 167 | |
| 168 | def ops_in_model_before(self): |
| 169 | return [ |
| 170 | SILU_MUL_OP if self.enable_silu_mul_custom_op else torch.ops.aten.mul, |
| 171 | rocm_aiter_ops.get_group_quant_op(), |
| 172 | ] |
| 173 | |
| 174 | def ops_in_model_after(self): |
| 175 | return [torch.ops.vllm.rocm_aiter_act_mul_and_fp8_group_quant] |
| 176 | |
| 177 | |
| 178 | class TestSiluMulBlockQuantModel(torch.nn.Module): |
| 179 | quant_key = kFp8Dynamic128Sym |
| 180 | |
| 181 | def __init__(self, hidden_size: int, is_scale_transposed: bool = False, **kwargs): |
| 182 | super().__init__() |
| 183 | self.silu_and_mul = SiluAndMul() |
| 184 | self.is_scale_transposed = is_scale_transposed |
| 185 | self.quant_fp8 = QuantFP8( |
| 186 | static=False, |
| 187 | group_shape=GroupShape(1, 128), |
| 188 | column_major_scales=is_scale_transposed, |
| 189 | compile_native=False, |
| 190 | ) |
| 191 | |
| 192 | self.enable_silu_mul_custom_op = self.silu_and_mul.enabled() |
| 193 | self.enable_quant_fp8_custom_op = self.quant_fp8.enabled() |
| 194 | |
| 195 | def forward(self, x): |
| 196 | y = self.silu_and_mul(x) |
| 197 | out, scale = self.quant_fp8(y) |
| 198 | group_size = self.quant_key.scale.group_shape[1] |
| 199 | scale_expanded = scale.repeat_interleave(group_size, dim=1) |
| 200 | dequant = out.to(dtype=torch.float32) * scale_expanded |
| 201 | return (dequant,) |
| 202 | |
| 203 | def ops_in_model_before(self): |
| 204 | ops = [] |
| 205 | if self.enable_silu_mul_custom_op: |
| 206 | ops.append(SILU_MUL_OP) |
| 207 | # When silu custom op is disabled, aten.mul.Tensor also appears |
| 208 | # in dequant code, so we skip checking it to avoid false positives. |
| 209 | ops.append( |
| 210 | QUANT_OPS[self.quant_key] |
| 211 | if self.enable_quant_fp8_custom_op |
| 212 | else torch.ops.aten.reciprocal.default |
| 213 | ) |
| 214 | return ops |
| 215 | |
| 216 | def ops_in_model_after(self): |
| 217 | return [FUSED_OPS[self.quant_key]] |
| 218 | |
| 219 | |
| 220 | ROCM_KERNELS = [ROCmFP8ScaledMMLinearKernel, PerTensorTorchFP8ScaledMMLinearKernel] |
| 221 | CUDA_KERNELS = [ |
| 222 | FlashInferFP8ScaledMMLinearKernel, |
| 223 | CutlassFP8ScaledMMLinearKernel, |
| 224 | PerTensorTorchFP8ScaledMMLinearKernel, |
| 225 | ] |
| 226 | TEST_KERNELS = ROCM_KERNELS if current_platform.is_rocm() else CUDA_KERNELS |
| 227 | |
| 228 | |
| 229 | @pytest.mark.parametrize("num_tokens", [32, 64]) |
| 230 | @pytest.mark.parametrize("hidden_size", [128, 256]) |
| 231 | @pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float16]) |
| 232 | @pytest.mark.parametrize("enable_silu_mul_custom_op", [True, False]) |
| 233 | @pytest.mark.parametrize( |
| 234 | "model_class, enable_quant_fp8_custom_op, force_kernel", |
| 235 | list(itertools.product([TestSiluMulFp8QuantModel], [True, False], TEST_KERNELS)) |
| 236 | + [ |
| 237 | pytest.param( |
| 238 | TestSiluMulNvfp4QuantModel, |
| 239 | False, |
| 240 | None, |
| 241 | marks=pytest.mark.skipif( |
| 242 | not current_platform.is_cuda(), reason="CUDA only" |
| 243 | ), |
| 244 | ), |
| 245 | # GroupFP8Quant fusion only works with AITER on ROCm. |
| 246 | # and the enable_quant_fp8_custom_op must be True. |
| 247 | pytest.param( |
| 248 | TestSiluMulGroupFp8QuantModel, |
| 249 | True, |
| 250 | None, |
| 251 | marks=pytest.mark.skipif( |
| 252 | not current_platform.is_rocm(), reason="ROCm only" |
| 253 | ), |
| 254 | ), |
| 255 | # Block quant fusion for per-group FP8 (CUDA only). |
| 256 | *[ |
| 257 | pytest.param( |
| 258 | partial(TestSiluMulBlockQuantModel, is_scale_transposed=transposed), |
| 259 | True, |
| 260 | None, |
| 261 | marks=pytest.mark.skipif( |
| 262 | not current_platform.is_cuda(), reason="CUDA only" |
| 263 | ), |
| 264 | id=f"TestSiluMulBlockQuant-transposed={transposed}", |
| 265 | ) |
| 266 | for transposed in [False, True] |
| 267 | ], |
| 268 | ], |
| 269 | ) |
| 270 | @pytest.mark.skipif( |
| 271 | envs.VLLM_TARGET_DEVICE not in ["cuda", "rocm"], reason="Only test on CUDA and ROCm" |
| 272 | ) |
| 273 | def test_fusion_silu_and_mul_quant( |
| 274 | num_tokens: int, |
| 275 | hidden_size: int, |
| 276 | dtype: torch.dtype, |
| 277 | model_class: type[ |
| 278 | TestSiluMulFp8QuantModel |
| 279 | | TestSiluMulNvfp4QuantModel |
| 280 | | TestSiluMulGroupFp8QuantModel |
| 281 | | TestSiluMulBlockQuantModel |
| 282 | ], |
| 283 | enable_silu_mul_custom_op: bool, |
| 284 | enable_quant_fp8_custom_op: bool, |
| 285 | force_kernel: type[FP8ScaledMMLinearKernel] | None, |
| 286 | monkeypatch: pytest.MonkeyPatch, |
| 287 | ): |
| 288 | if model_class is TestSiluMulNvfp4QuantModel and not is_nvfp4_supported(): |
| 289 | pytest.skip("NVFP4 is not supported on this GPU.") |
| 290 | if model_class is TestSiluMulGroupFp8QuantModel and not IS_AITER_FOUND: |
| 291 | pytest.skip("AITER is not supported on this GPU.") |
| 292 | if ( |
| 293 | isinstance(model_class, partial) |
| 294 | and model_class.func is TestSiluMulBlockQuantModel |
| 295 | and is_deep_gemm_supported() |
| 296 | ): |
| 297 | pytest.skip("SiluMul+BlockQuant fusion not applicable with DeepGemm") |
| 298 | |
| 299 | torch.set_default_device("cuda") |
| 300 | torch.set_default_dtype(dtype) |
| 301 | |
| 302 | x = torch.rand(num_tokens, hidden_size * 2) |
| 303 | |
| 304 | # Reshape pass is needed for the fusion pass to work |
| 305 | custom_ops = ["none"] |
| 306 | if enable_silu_mul_custom_op: |
| 307 | custom_ops.append("+silu_and_mul") |
| 308 | if enable_quant_fp8_custom_op: |
| 309 | custom_ops.append("+quant_fp8") |
| 310 | config = VllmConfig( |
| 311 | compilation_config=CompilationConfig( |
| 312 | mode=CompilationMode.VLLM_COMPILE, |
| 313 | custom_ops=custom_ops, |
| 314 | backend="eager", # avoid compilation for SiluAndMul and QuantFP8 |
| 315 | pass_config=PassConfig(fuse_act_quant=True, eliminate_noops=True), |
| 316 | ), |
| 317 | ) |
| 318 | |
| 319 | with set_current_vllm_config(config), monkeypatch.context() as m: |
| 320 | fusion_passes = [ActivationQuantFusionPass(config)] |
| 321 | if IS_AITER_FOUND and model_class is TestSiluMulGroupFp8QuantModel: |
| 322 | from vllm.compilation.passes.fusion.rocm_aiter_fusion import ( |
| 323 | RocmAiterSiluMulFp8GroupQuantFusionPass, |
| 324 | ) |
| 325 | |
| 326 | m.setenv("VLLM_ROCM_USE_AITER", "1") |
| 327 | rocm_aiter_ops.refresh_env_variables() |
| 328 | fusion_passes += [RocmAiterSiluMulFp8GroupQuantFusionPass(config)] |
| 329 | |
| 330 | passes = [NoOpEliminationPass(config), *fusion_passes, PostCleanupPass(config)] |
| 331 | backend = TestBackend(*passes) |
| 332 | model = model_class( |
| 333 | hidden_size=hidden_size, |
| 334 | force_kernel=force_kernel, |
| 335 | x=x, |
| 336 | dtype=dtype, |
| 337 | ) |
| 338 | |
| 339 | # First dimension dynamic |
| 340 | torch._dynamo.mark_dynamic(x, 0) |
| 341 | |
| 342 | result = model(x) |
| 343 | |
| 344 | model2 = torch.compile(model, backend=backend) |
| 345 | result2 = model2(x) |
| 346 | |
| 347 | # Check that it gives the same answer |
| 348 | if isinstance(model, TestSiluMulFp8QuantModel): |
| 349 | atol, rtol = 1e-3, 1e-3 |
| 350 | elif isinstance(model, TestSiluMulNvfp4QuantModel): |
| 351 | atol, rtol = 1e-1, 1e-1 |
| 352 | elif isinstance(model, TestSiluMulGroupFp8QuantModel): |
| 353 | atol, rtol = 5e-2, 5e-2 |
| 354 | elif isinstance(model, TestSiluMulBlockQuantModel): |
| 355 | if current_platform.is_rocm(): |
| 356 | atol, rtol = 1e-3, 1e-3 |
| 357 | else: |
| 358 | # CUDA fused kernel computes silu*mul in fp32 while the reference |
| 359 | # goes through bf16/fp16 storage, so group maxima (and thus scales) |
| 360 | # can shift by one FP8-e4m3 code (~1/8 relative step). |
| 361 | atol, rtol = 5e-2, 5e-2 |
| 362 | |
| 363 | torch.testing.assert_close( |
| 364 | result[0].to(dtype=dtype), result2[0].to(dtype=dtype), atol=atol, rtol=rtol |
| 365 | ) |
| 366 | |
| 367 | assert sum([p.matched_count for p in fusion_passes]) == 1 |
| 368 | |
| 369 | # In pre-nodes, quant op should be present and fused kernels should not |
| 370 | backend.check_before_ops(model.ops_in_model_before()) |
| 371 | |
| 372 | # In post-nodes, fused kernels should be present and quant op should not |
| 373 | backend.check_after_ops(model.ops_in_model_after()) |