| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | """Weight preparation helpers for b12x MoE kernels.""" |
| 4 | |
| 5 | import torch |
| 6 | import torch.nn.functional as F |
| 7 | |
| 8 | from vllm.model_executor.layers.quantization.utils.flashinfer_utils import ( |
| 9 | swap_w13_to_w31, |
| 10 | ) |
| 11 | from vllm.model_executor.layers.quantization.utils.nvfp4_utils import ( |
| 12 | swizzle_blockscale, |
| 13 | ) |
| 14 | from vllm.utils.math_utils import round_up |
| 15 | |
| 16 | |
| 17 | def _pad_gated_weights( |
| 18 | w13: torch.Tensor, |
| 19 | w13_scale: torch.Tensor, |
| 20 | w2: torch.Tensor, |
| 21 | w2_scale: torch.Tensor, |
| 22 | ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: |
| 23 | if w13_scale.size(1) != w13.size(1): |
| 24 | raise ValueError("w13 weight and scale row counts must match") |
| 25 | half_size = w13.size(1) // 2 |
| 26 | if half_size * 2 != w13.size(1): |
| 27 | raise ValueError("gated NVFP4 MoE weights must have even row counts") |
| 28 | if w2.size(2) * 2 != half_size: |
| 29 | raise ValueError("w2 shape does not match gated w13") |
| 30 | if w2_scale.size(2) * 16 != half_size: |
| 31 | raise ValueError("w2 scale shape does not match gated w13") |
| 32 | |
| 33 | half_pad_size = round_up(half_size, 64) - half_size |
| 34 | if half_pad_size == 0: |
| 35 | return w13, w13_scale, w2, w2_scale |
| 36 | |
| 37 | def pad_rows(tensor: torch.Tensor) -> torch.Tensor: |
| 38 | shape = tensor.shape |
| 39 | return F.pad( |
| 40 | tensor.reshape(shape[0], 2, half_size, *shape[2:]), |
| 41 | (0, 0, 0, half_pad_size), |
| 42 | ).flatten(1, 2) |
| 43 | |
| 44 | return ( |
| 45 | pad_rows(w13), |
| 46 | pad_rows(w13_scale), |
| 47 | F.pad(w2, (0, half_pad_size // 2)), |
| 48 | F.pad(w2_scale, (0, half_pad_size // 16)), |
| 49 | ) |
| 50 | |
| 51 | |
| 52 | def _per_expert_scale( |
| 53 | scale: torch.Tensor, |
| 54 | num_experts: int, |
| 55 | name: str, |
| 56 | ) -> torch.Tensor: |
| 57 | if scale.dim() == 0: |
| 58 | scale = scale.expand(num_experts) |
| 59 | elif scale.dim() == 2: |
| 60 | scale = scale.amax(dim=1) |
| 61 | if scale.dim() != 1 or scale.numel() != num_experts: |
| 62 | raise ValueError(f"{name} must contain one value per expert") |
| 63 | return scale.to(torch.float32).contiguous() |
| 64 | |
| 65 | |
| 66 | def prepare_nvfp4_moe_layer_for_b12x( |
| 67 | w13: torch.Tensor, |
| 68 | w13_scale: torch.Tensor, |
| 69 | w13_scale_2: torch.Tensor, |
| 70 | a13_scale: torch.Tensor, |
| 71 | w2: torch.Tensor, |
| 72 | w2_scale: torch.Tensor, |
| 73 | w2_scale_2: torch.Tensor, |
| 74 | a2_scale: torch.Tensor, |
| 75 | is_act_and_mul: bool, |
| 76 | reorder_w13: bool = False, |
| 77 | ) -> tuple[ |
| 78 | torch.Tensor, |
| 79 | torch.Tensor, |
| 80 | torch.Tensor, |
| 81 | torch.Tensor, |
| 82 | torch.Tensor, |
| 83 | torch.Tensor, |
| 84 | torch.Tensor, |
| 85 | torch.Tensor, |
| 86 | ]: |
| 87 | """Prepare b12x NVFP4 MoE weights and scales.""" |
| 88 | num_experts = w13.shape[0] |
| 89 | a13_scale = _per_expert_scale(a13_scale, num_experts, "a13_scale") |
| 90 | a2_scale = _per_expert_scale(a2_scale, num_experts, "a2_scale") |
| 91 | |
| 92 | if reorder_w13 and is_act_and_mul: |
| 93 | w13 = swap_w13_to_w31(w13) |
| 94 | w13_scale = swap_w13_to_w31(w13_scale) |
| 95 | if is_act_and_mul: |
| 96 | w13, w13_scale, w2, w2_scale = _pad_gated_weights(w13, w13_scale, w2, w2_scale) |
| 97 | |
| 98 | w13_scale = swizzle_blockscale(w13_scale) |
| 99 | pad_size = w13_scale.size(1) - w13.size(1) |
| 100 | if pad_size > 0: |
| 101 | if is_act_and_mul: |
| 102 | raise RuntimeError("gated NVFP4 MoE padding must precede scale swizzling") |
| 103 | w13 = F.pad(w13, (0, 0, 0, pad_size)) |
| 104 | w2 = F.pad(w2, (0, pad_size // 2, 0, 0)) |
| 105 | w2_scale = F.pad(w2_scale, (0, pad_size // 16)) |
| 106 | |
| 107 | w2_scale = swizzle_blockscale(w2_scale) |
| 108 | return w13, w13_scale, w13_scale_2, a13_scale, w2, w2_scale, w2_scale_2, a2_scale |