| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | """Tests for MoE with non-gated activations (*_no_mul). |
| 4 | |
| 5 | These tests verify that MoE layers work correctly with activations like |
| 6 | silu_no_mul, gelu_no_mul, relu2_no_mul where the activation output dimension |
| 7 | equals N (not N // 2 like gated activations). |
| 8 | """ |
| 9 | |
| 10 | import pytest |
| 11 | import torch |
| 12 | |
| 13 | from tests.kernels.moe.utils import make_dummy_moe_config |
| 14 | from vllm.model_executor.layers.fused_moe.activation import ( |
| 15 | ApplyMoEActivationConfig, |
| 16 | MoEActivation, |
| 17 | apply_moe_activation, |
| 18 | apply_moe_activation_supported, |
| 19 | ) |
| 20 | from vllm.model_executor.layers.fused_moe.config import ( |
| 21 | FUSED_MOE_UNQUANTIZED_CONFIG, |
| 22 | ) |
| 23 | from vllm.model_executor.layers.fused_moe.experts.triton_moe import TritonExperts |
| 24 | from vllm.platforms import current_platform |
| 25 | |
| 26 | DEVICE = current_platform.device_type |
| 27 | |
| 28 | pytestmark = [ |
| 29 | pytest.mark.skipif( |
| 30 | not (current_platform.is_cuda_alike() or current_platform.is_xpu()), |
| 31 | reason="Triton MoE kernels require a CUDA-alike or XPU device.", |
| 32 | ), |
| 33 | pytest.mark.skipif( |
| 34 | current_platform.is_cuda_alike() |
| 35 | and not current_platform.has_device_capability(80), |
| 36 | reason="Triton MoE kernels require compute capability >= 8.0 on CUDA/ROCm.", |
| 37 | ), |
| 38 | ] |
| 39 | |
| 40 | # Test parameters |
| 41 | M_SIZES = [1, 16, 64] |
| 42 | N_SIZES = [128, 256] |
| 43 | K_SIZES = [64, 128] |
| 44 | TOPK_VALUES = [1, 2] |
| 45 | NUM_EXPERTS = 8 |
| 46 | NO_MUL_ACTIVATIONS = [ |
| 47 | MoEActivation.SILU_NO_MUL, |
| 48 | MoEActivation.GELU_NO_MUL, |
| 49 | MoEActivation.RELU2_NO_MUL, |
| 50 | ] |
| 51 | |
| 52 | |
| 53 | def test_apply_moe_activation_supported_contract(): |
| 54 | supported = { |
| 55 | activation |
| 56 | for activation in MoEActivation |
| 57 | if apply_moe_activation_supported(activation) |
| 58 | } |
| 59 | |
| 60 | assert supported == set(MoEActivation) - {MoEActivation.RELU2} |
| 61 | |
| 62 | |
| 63 | @pytest.mark.skipif(not current_platform.is_cuda(), reason="Requires CUDA") |
| 64 | @pytest.mark.parametrize( |
| 65 | "activation", |
| 66 | [ |
| 67 | activation |
| 68 | for activation in MoEActivation |
| 69 | if apply_moe_activation_supported(activation) |
| 70 | ], |
| 71 | ) |
| 72 | @torch.inference_mode() |
| 73 | def test_supported_apply_moe_activation_executes(activation: MoEActivation): |
| 74 | output_width = 64 |
| 75 | input_width = output_width * 2 if activation.is_gated else output_width |
| 76 | input = torch.randn(2, input_width, device="cuda", dtype=torch.bfloat16) |
| 77 | output = torch.empty(2, output_width, device="cuda", dtype=torch.bfloat16) |
| 78 | activation_config = ApplyMoEActivationConfig() |
| 79 | if activation == MoEActivation.SITU: |
| 80 | activation_config = ApplyMoEActivationConfig(activation_situ_beta=1.0) |
| 81 | elif activation == MoEActivation.SWIGLUOAI_UNINTERLEAVE: |
| 82 | activation_config = ApplyMoEActivationConfig(clamp_limit=7.0) |
| 83 | |
| 84 | apply_moe_activation(activation, output, input, activation_config=activation_config) |
| 85 | |
| 86 | assert torch.isfinite(output).all() |
| 87 | |
| 88 | |
| 89 | @pytest.mark.parametrize("activation", list(MoEActivation)) |
| 90 | def test_triton_activation_metadata_tracks_shared_apply(activation: MoEActivation): |
| 91 | assert TritonExperts._supports_activation( |
| 92 | activation |
| 93 | ) == apply_moe_activation_supported(activation) |
| 94 | |
| 95 | |
| 96 | def make_test_tensors( |
| 97 | m: int, |
| 98 | n: int, |
| 99 | k: int, |
| 100 | num_experts: int, |
| 101 | topk: int, |
| 102 | dtype: torch.dtype = torch.bfloat16, |
| 103 | device: str = DEVICE, |
| 104 | ): |
| 105 | """Create test tensors for MoE with non-gated activation. |
| 106 | |
| 107 | For non-gated activations (*_no_mul): |
| 108 | - w1: (E, N, K) - projects from K to N |
| 109 | - w2: (E, K, N) - projects from N back to K (note: N, not N//2) |
| 110 | """ |
| 111 | hidden_states = torch.randn(m, k, dtype=dtype, device=device) |
| 112 | |
| 113 | # For non-gated: w1 projects K -> N, w2 projects N -> K |
| 114 | w1 = torch.randn(num_experts, n, k, dtype=dtype, device=device) * 0.1 |
| 115 | w2 = torch.randn(num_experts, k, n, dtype=dtype, device=device) * 0.1 |
| 116 | |
| 117 | topk_weights = torch.ones(m, topk, dtype=torch.float32, device=device) / topk |
| 118 | topk_ids = torch.randint(0, num_experts, (m, topk), device=device) |
| 119 | |
| 120 | return hidden_states, w1, w2, topk_weights, topk_ids |
| 121 | |
| 122 | |
| 123 | @pytest.mark.parametrize("m", M_SIZES) |
| 124 | @pytest.mark.parametrize("n", N_SIZES) |
| 125 | @pytest.mark.parametrize("k", K_SIZES) |
| 126 | @pytest.mark.parametrize("topk", TOPK_VALUES) |
| 127 | @pytest.mark.parametrize("activation", NO_MUL_ACTIVATIONS) |
| 128 | @torch.inference_mode() |
| 129 | def test_triton_experts_no_mul_activation( |
| 130 | m: int, |
| 131 | n: int, |
| 132 | k: int, |
| 133 | topk: int, |
| 134 | activation: MoEActivation, |
| 135 | ): |
| 136 | hidden_states, w1, w2, topk_weights, topk_ids = make_test_tensors( |
| 137 | m, n, k, NUM_EXPERTS, topk |
| 138 | ) |
| 139 | |
| 140 | experts = TritonExperts( |
| 141 | moe_config=make_dummy_moe_config(), |
| 142 | quant_config=FUSED_MOE_UNQUANTIZED_CONFIG, |
| 143 | ) |
| 144 | |
| 145 | ws1_shape, ws2_shape, out_shape = experts.workspace_shapes( |
| 146 | M=m, |
| 147 | N=n, |
| 148 | K=k, |
| 149 | topk=topk, |
| 150 | global_num_experts=NUM_EXPERTS, |
| 151 | local_num_experts=NUM_EXPERTS, |
| 152 | expert_tokens_meta=None, |
| 153 | activation=activation, |
| 154 | ) |
| 155 | |
| 156 | # Verify workspace shapes are correct for no_mul activation |
| 157 | # workspace1 should handle activation_out_dim = N (not N//2) |
| 158 | assert ws1_shape == (m, topk, max(n, k)), ( |
| 159 | f"workspace1 shape mismatch: expected {(m, topk, max(n, k))}, got {ws1_shape}" |
| 160 | ) |
| 161 | # workspace2 should handle max(N, K) for intermediate_cache1/cache3 |
| 162 | assert ws2_shape == (m, topk, max(n, k)), ( |
| 163 | f"workspace2 shape mismatch: expected {(m, topk, max(n, k))}, got {ws2_shape}" |
| 164 | ) |
| 165 | assert out_shape == (m, k), ( |
| 166 | f"output shape mismatch: expected {(m, k)}, got {out_shape}" |
| 167 | ) |
| 168 | |
| 169 | workspace1 = torch.empty( |
| 170 | ws1_shape[0] * ws1_shape[1] * ws1_shape[2], |
| 171 | dtype=hidden_states.dtype, |
| 172 | device=hidden_states.device, |
| 173 | ) |
| 174 | workspace2 = torch.empty( |
| 175 | ws2_shape[0] * ws2_shape[1] * ws2_shape[2], |
| 176 | dtype=hidden_states.dtype, |
| 177 | device=hidden_states.device, |
| 178 | ) |
| 179 | output = torch.zeros(m, k, dtype=hidden_states.dtype, device=hidden_states.device) |
| 180 | |
| 181 | experts.apply( |
| 182 | output=output, |
| 183 | hidden_states=hidden_states, |
| 184 | w1=w1, |
| 185 | w2=w2, |
| 186 | topk_weights=topk_weights, |
| 187 | topk_ids=topk_ids, |
| 188 | activation=activation, |
| 189 | global_num_experts=NUM_EXPERTS, |
| 190 | expert_map=None, |
| 191 | a1q_scale=None, |
| 192 | a2_scale=None, |
| 193 | workspace13=workspace1, |
| 194 | workspace2=workspace2, |
| 195 | expert_tokens_meta=None, |
| 196 | apply_router_weight_on_input=False, |
| 197 | ) |
| 198 | |
| 199 | assert output.shape == (m, k), f"Expected shape {(m, k)}, got {output.shape}" |
| 200 | assert not torch.isnan(output).any(), "Output contains NaN" |
| 201 | assert not torch.isinf(output).any(), "Output contains Inf" |
| 202 | assert output.abs().sum() > 0, "Output is all zeros" |
| 203 | |
| 204 | |
| 205 | @torch.inference_mode() |
| 206 | def test_workspace_shapes_no_mul_vs_gated(): |
| 207 | """Test that workspace shapes differ correctly between gated and non-gated.""" |
| 208 | from vllm.model_executor.layers.fused_moe.experts.triton_moe import TritonExperts |
| 209 | |
| 210 | M, N, K, topk = 64, 256, 128, 2 |
| 211 | |
| 212 | experts = TritonExperts( |
| 213 | moe_config=make_dummy_moe_config(), |
| 214 | quant_config=FUSED_MOE_UNQUANTIZED_CONFIG, |
| 215 | ) |
| 216 | |
| 217 | ws1_no_mul, _, out_no_mul = experts.workspace_shapes( |
| 218 | M, N, K, topk, 8, 8, None, MoEActivation.SILU_NO_MUL |
| 219 | ) |
| 220 | |
| 221 | ws1_gated, _, out_gated = experts.workspace_shapes( |
| 222 | M, N, K, topk, 8, 8, None, MoEActivation.SILU |
| 223 | ) |
| 224 | |
| 225 | # For no_mul: activation_out_dim = N |
| 226 | # For gated: activation_out_dim = N // 2 |
| 227 | # workspace1 should use max(activation_out_dim, K) |
| 228 | activation_out_dim_no_mul = N |
| 229 | activation_out_dim_gated = N // 2 |
| 230 | |
| 231 | assert ws1_no_mul[2] == max(activation_out_dim_no_mul, K), ( |
| 232 | f"no_mul workspace1 last dim should be max({activation_out_dim_no_mul}, {K})" |
| 233 | ) |
| 234 | assert ws1_gated[2] == max(activation_out_dim_gated, K), ( |
| 235 | f"gated workspace1 last dim should be max({activation_out_dim_gated}, {K})" |
| 236 | ) |
| 237 | |
| 238 | # Output shapes should be the same |
| 239 | assert out_no_mul == out_gated == (M, K) |
| 240 | |
| 241 | |
| 242 | @torch.inference_mode() |
| 243 | def test_adjust_n_for_activation(): |
| 244 | """Test the adjust_N_for_activation method.""" |
| 245 | from vllm.model_executor.layers.fused_moe.experts.triton_moe import TritonExperts |
| 246 | |
| 247 | experts = TritonExperts( |
| 248 | moe_config=make_dummy_moe_config(), |
| 249 | quant_config=FUSED_MOE_UNQUANTIZED_CONFIG, |
| 250 | ) |
| 251 | |
| 252 | N = 256 |
| 253 | |
| 254 | # Gated activations should return N // 2 |
| 255 | assert experts.adjust_N_for_activation(N, MoEActivation.SILU) == N // 2 |
| 256 | assert experts.adjust_N_for_activation(N, MoEActivation.GELU) == N // 2 |
| 257 | |
| 258 | # Non-gated activations should return N |
| 259 | assert experts.adjust_N_for_activation(N, MoEActivation.SILU_NO_MUL) == N |
| 260 | assert experts.adjust_N_for_activation(N, MoEActivation.GELU_NO_MUL) == N |
| 261 | assert experts.adjust_N_for_activation(N, MoEActivation.RELU2_NO_MUL) == N |