| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | """Dense FFN and MoE blocks for HY V4 (NVIDIA).""" |
| 4 | |
| 5 | import torch |
| 6 | from torch import nn |
| 7 | from transformers import PreTrainedConfig |
| 8 | |
| 9 | from vllm.config import get_current_vllm_config |
| 10 | from vllm.distributed import get_ep_group, get_tensor_model_parallel_world_size |
| 11 | from vllm.logger import init_logger |
| 12 | from vllm.model_executor.layers.activation import SiluAndMul |
| 13 | from vllm.model_executor.layers.fused_moe import FusedMoEFactory, GateLinear |
| 14 | from vllm.model_executor.layers.linear import ( |
| 15 | MergedColumnParallelLinear, |
| 16 | RowParallelLinear, |
| 17 | ) |
| 18 | from vllm.model_executor.layers.quantization import QuantizationConfig |
| 19 | |
| 20 | logger = init_logger(__name__) |
| 21 | |
| 22 | |
| 23 | class HYV4FeedForward(nn.Module): |
| 24 | """Dense SwiGLU feed-forward block. |
| 25 | |
| 26 | Used both for the dense decoder layers and for the MoE shared experts. |
| 27 | The routed-expert SwiGLU clamp does not apply here, matching the reference |
| 28 | implementation. |
| 29 | """ |
| 30 | |
| 31 | def __init__( |
| 32 | self, |
| 33 | hidden_size: int, |
| 34 | intermediate_size: int, |
| 35 | hidden_act: str, |
| 36 | quant_config: QuantizationConfig | None = None, |
| 37 | reduce_results: bool = True, |
| 38 | prefix: str = "", |
| 39 | ) -> None: |
| 40 | super().__init__() |
| 41 | if hidden_act != "silu": |
| 42 | raise ValueError( |
| 43 | f"Unsupported activation: {hidden_act}. Only silu is supported for now." |
| 44 | ) |
| 45 | self.gate_up_proj = MergedColumnParallelLinear( |
| 46 | hidden_size, |
| 47 | [intermediate_size] * 2, |
| 48 | bias=False, |
| 49 | quant_config=quant_config, |
| 50 | prefix=f"{prefix}.gate_up_proj", |
| 51 | ) |
| 52 | self.down_proj = RowParallelLinear( |
| 53 | intermediate_size, |
| 54 | hidden_size, |
| 55 | bias=False, |
| 56 | quant_config=quant_config, |
| 57 | reduce_results=reduce_results, |
| 58 | prefix=f"{prefix}.down_proj", |
| 59 | ) |
| 60 | self.act_fn = SiluAndMul() |
| 61 | |
| 62 | def forward(self, x: torch.Tensor) -> torch.Tensor: |
| 63 | gate_up, _ = self.gate_up_proj(x) |
| 64 | out = self.act_fn(gate_up) |
| 65 | out, _ = self.down_proj(out) |
| 66 | return out |
| 67 | |
| 68 | |
| 69 | class HYV4MoEFused(nn.Module): |
| 70 | """HY V4 MoE layer with optional SwiGLU clamp support. |
| 71 | |
| 72 | When ``config.swiglu_limit > 0`` the routed experts use a clamped SwiGLU:: |
| 73 | |
| 74 | gate = clamp(gate, max=limit) |
| 75 | up = clamp(up, -limit, limit) |
| 76 | output = silu(gate) * up |
| 77 | |
| 78 | Dense layers and shared experts are NOT clamped. The clamp is forwarded to |
| 79 | ``FusedMoEFactory`` as ``swiglu_limit`` so it travels through the quant |
| 80 | config to every expert backend instead of mutating global state. |
| 81 | |
| 82 | NOTE: The reference implementation additionally supports HPC gate kernels, |
| 83 | TPCP/MLPSP sharded shared experts and a side-stream overlap for the shared |
| 84 | experts. Those depend on infrastructure that is absent here, so this port |
| 85 | keeps only the standard path. TODO: restore once available. |
| 86 | """ |
| 87 | |
| 88 | def __init__( |
| 89 | self, |
| 90 | config: PreTrainedConfig, |
| 91 | quant_config: QuantizationConfig | None = None, |
| 92 | prefix: str = "", |
| 93 | enable_eplb: bool = False, |
| 94 | ): |
| 95 | super().__init__() |
| 96 | self.tp_size = get_tensor_model_parallel_world_size() |
| 97 | self.ep_group = get_ep_group().device_group |
| 98 | self.ep_rank = get_ep_group().rank_in_group |
| 99 | self.ep_size = self.ep_group.size() |
| 100 | self.n_routed_experts = config.num_experts |
| 101 | if self.tp_size > config.num_experts: |
| 102 | raise ValueError( |
| 103 | f"Tensor parallel size {self.tp_size} is greater than " |
| 104 | f"the number of experts {config.num_experts}." |
| 105 | ) |
| 106 | top_k = config.num_experts_per_tok |
| 107 | intermediate_size = config.expert_hidden_dim |
| 108 | router_scaling_factor = getattr(config, "router_scaling_factor", 1.0) |
| 109 | vllm_config = get_current_vllm_config() |
| 110 | eplb_config = vllm_config.parallel_config.eplb_config |
| 111 | self.enable_eplb = enable_eplb |
| 112 | |
| 113 | self.n_logical_experts = self.n_routed_experts |
| 114 | self.n_redundant_experts = eplb_config.num_redundant_experts |
| 115 | self.n_physical_experts = self.n_logical_experts + self.n_redundant_experts |
| 116 | self.n_local_physical_experts = self.n_physical_experts // self.ep_size |
| 117 | self.physical_expert_start = self.ep_rank * self.n_local_physical_experts |
| 118 | self.physical_expert_end = ( |
| 119 | self.physical_expert_start + self.n_local_physical_experts |
| 120 | ) |
| 121 | |
| 122 | self.gate = GateLinear( |
| 123 | config.hidden_size, |
| 124 | config.num_experts, |
| 125 | bias=False, |
| 126 | out_dtype=torch.float32, |
| 127 | params_dtype=torch.float32, |
| 128 | prefix=f"{prefix}.gate", |
| 129 | ) |
| 130 | |
| 131 | self.shared_experts: HYV4FeedForward | None |
| 132 | if config.num_shared_experts > 0: |
| 133 | self.shared_experts = HYV4FeedForward( |
| 134 | hidden_size=config.hidden_size, |
| 135 | intermediate_size=config.expert_hidden_dim * config.num_shared_experts, |
| 136 | hidden_act=config.hidden_act, |
| 137 | quant_config=quant_config, |
| 138 | prefix=f"{prefix}.shared_experts", |
| 139 | reduce_results=False, |
| 140 | ) |
| 141 | else: |
| 142 | self.shared_experts = None |
| 143 | |
| 144 | self.expert_bias = nn.Parameter( |
| 145 | torch.empty(config.num_experts, dtype=torch.float32) |
| 146 | ) |
| 147 | |
| 148 | raw_swiglu_limit = getattr(config, "swiglu_limit", 0) |
| 149 | use_swiglu_clamp = bool(raw_swiglu_limit) and float(raw_swiglu_limit) > 0 |
| 150 | moe_swiglu_limit = float(raw_swiglu_limit) if use_swiglu_clamp else None |
| 151 | |
| 152 | self.experts = FusedMoEFactory( |
| 153 | num_experts=self.n_routed_experts, |
| 154 | top_k=top_k, |
| 155 | hidden_size=config.hidden_size, |
| 156 | intermediate_size=intermediate_size, |
| 157 | renormalize=config.route_norm, |
| 158 | quant_config=quant_config, |
| 159 | prefix=f"{prefix}.experts", |
| 160 | enable_eplb=self.enable_eplb, |
| 161 | num_redundant_experts=self.n_redundant_experts, |
| 162 | scoring_func="sigmoid", |
| 163 | use_grouped_topk=True, |
| 164 | num_expert_group=1, |
| 165 | topk_group=1, |
| 166 | routed_scaling_factor=router_scaling_factor, |
| 167 | e_score_correction_bias=self.expert_bias, |
| 168 | shared_experts=self.shared_experts, |
| 169 | swiglu_limit=moe_swiglu_limit, |
| 170 | ) |
| 171 | self.prefix = prefix |
| 172 | if use_swiglu_clamp: |
| 173 | logger.debug_once( |
| 174 | "HYV4MoEFused: swiglu_limit=%.1f enabled for routed experts", |
| 175 | moe_swiglu_limit, |
| 176 | ) |
| 177 | |
| 178 | def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: |
| 179 | orig_shape = hidden_states.shape |
| 180 | hidden_dim = hidden_states.shape[-1] |
| 181 | hidden_states = hidden_states.view(-1, hidden_dim) |
| 182 | |
| 183 | # router_logits: (num_tokens, n_experts); the gate stores fp32 weights |
| 184 | # and emits fp32 logits, so no explicit upcast is needed here. |
| 185 | router_logits, _ = self.gate(hidden_states) |
| 186 | |
| 187 | final_hidden_states = self.experts( |
| 188 | hidden_states=hidden_states, router_logits=router_logits |
| 189 | ) |
| 190 | return final_hidden_states.view(orig_shape) |