| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | import pytest |
| 4 | import torch |
| 5 | from torch import nn |
| 6 | |
| 7 | import vllm.kernels # noqa: F401 to register kernels |
| 8 | from vllm import ir |
| 9 | from vllm.compilation.passes.ir.lowering_pass import ( |
| 10 | VllmIRLoweringPass, |
| 11 | ) |
| 12 | from vllm.config import get_current_vllm_config |
| 13 | from vllm.ir import ops |
| 14 | from vllm.model_executor.layers.activation import GeluAndMulSparse |
| 15 | from vllm.platforms import current_platform |
| 16 | |
| 17 | from ...backend import TestBackend |
| 18 | |
| 19 | |
| 20 | class Model(nn.Module): |
| 21 | def __init__(self, hidden_size=16, *args, **kwargs): |
| 22 | super().__init__(*args, **kwargs) |
| 23 | self.hidden_size = hidden_size |
| 24 | self.weight = torch.ones(hidden_size, dtype=torch.bfloat16) |
| 25 | |
| 26 | def forward(self, x): |
| 27 | x1 = x + 4.0 |
| 28 | x2 = ops.rms_norm(x1, self.weight, 1e-5) |
| 29 | x3 = x2 * 5.0 |
| 30 | # no weight |
| 31 | x4 = ops.rms_norm(x3, None, 1e-5) |
| 32 | x5 = x4 / 2.0 |
| 33 | # dispatch to native due to variance_size parameter |
| 34 | x6 = ops.rms_norm(x5, self.weight, 1e-5, self.hidden_size // 2) |
| 35 | return x6 + 3.0 |
| 36 | |
| 37 | |
| 38 | class SparseActivationModel(nn.Module): |
| 39 | def __init__(self): |
| 40 | super().__init__() |
| 41 | self.activation = GeluAndMulSparse(0.95, "tanh") |
| 42 | |
| 43 | def forward(self, x): |
| 44 | return self.activation(x) |
| 45 | |
| 46 | |
| 47 | @pytest.mark.parametrize("rms_provider", ops.rms_norm.supported_providers()) |
| 48 | def test_lowering_rms_norm(rms_provider, default_vllm_config): |
| 49 | torch.set_default_device(current_platform.device_type) |
| 50 | |
| 51 | lowering_pass = VllmIRLoweringPass(get_current_vllm_config()) |
| 52 | backend = TestBackend(lowering_pass) |
| 53 | backend_unlowered = TestBackend() |
| 54 | |
| 55 | model = Model() |
| 56 | x = torch.randn(8, 16, dtype=torch.bfloat16) |
| 57 | with ( |
| 58 | ops.rms_norm.set_priority([rms_provider, "native"]), |
| 59 | ir.enable_torch_wrap(True), |
| 60 | ): |
| 61 | compiled_model = torch.compile(model, backend=backend, fullgraph=True) |
| 62 | compiled_unlowered_model = torch.compile( |
| 63 | model, backend=backend_unlowered, fullgraph=True |
| 64 | ) |
| 65 | output = compiled_model(x) |
| 66 | output_unlowered = compiled_unlowered_model(x) |
| 67 | |
| 68 | selected = lowering_pass.selected_impls["rms_norm"] |
| 69 | assert len(selected) == 3 |
| 70 | assert selected["rms_norm"] == rms_provider |
| 71 | assert selected["rms_norm_1"] == rms_provider |
| 72 | assert selected["rms_norm_2"] == "native" |
| 73 | |
| 74 | # Compiled function guards on global value, avoid recompilation |
| 75 | with ir.enable_torch_wrap(True): |
| 76 | output2 = compiled_model(x) |
| 77 | |
| 78 | torch.testing.assert_close(output_unlowered, output) |
| 79 | torch.testing.assert_close(output_unlowered, output2) |
| 80 | |
| 81 | |
| 82 | @pytest.mark.skipif(not current_platform.is_cuda(), reason="CUDA-only provider") |
| 83 | def test_lowering_gelu_and_mul_sparse(default_vllm_config): |
| 84 | torch.set_default_device("cuda") |
| 85 | lowering_pass = VllmIRLoweringPass(get_current_vllm_config()) |
| 86 | backend = TestBackend(lowering_pass) |
| 87 | model = SparseActivationModel() |
| 88 | x = torch.randn(8, 32768, dtype=torch.bfloat16) |
| 89 | |
| 90 | with ( |
| 91 | ops.gelu_and_mul_sparse.set_priority(["triton", "native"]), |
| 92 | ir.enable_torch_wrap(True), |
| 93 | ): |
| 94 | compiled_model = torch.compile(model, backend=backend, fullgraph=True) |
| 95 | output = compiled_model(x) |
| 96 | |
| 97 | selected = lowering_pass.selected_impls["gelu_and_mul_sparse"] |
| 98 | assert selected == {"gelu_and_mul_sparse": "triton"} |
| 99 | |
| 100 | expected = ops.gelu_and_mul_sparse.impls["native"].impl_fn( |
| 101 | x, 1.6448536269514722, "tanh" |
| 102 | ) |
| 103 | torch.testing.assert_close(output, expected, rtol=1.6e-2, atol=1e-3) |