| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | |
| 4 | import torch |
| 5 | from torch.nn.parameter import Parameter |
| 6 | |
| 7 | from vllm.model_executor.layers.fused_moe.experts.cutlass_moe import ( |
| 8 | swizzle_mxfp4_scales, |
| 9 | ) |
| 10 | from vllm.model_executor.layers.quantization.utils.quant_utils import ( |
| 11 | kMxfp4Dynamic, |
| 12 | ) |
| 13 | from vllm.platforms import current_platform |
| 14 | from vllm.utils.flashinfer import has_flashinfer_cutedsl |
| 15 | |
| 16 | from .base import MxFp4LinearKernel, MxFp4LinearLayerConfig |
| 17 | |
| 18 | _MXFP4_GROUP_SIZE = 32 |
| 19 | |
| 20 | |
| 21 | class FlashInferMxFp4LinearKernel(MxFp4LinearKernel): |
| 22 | """MXFP4 W4A4 GEMM via FlashInfer CUTLASS (SM100+).""" |
| 23 | |
| 24 | @classmethod |
| 25 | def is_supported( |
| 26 | cls, compute_capability: int | None = None |
| 27 | ) -> tuple[bool, str | None]: |
| 28 | if current_platform.has_device_capability(100) and has_flashinfer_cutedsl(): |
| 29 | return True, None |
| 30 | return False, "FlashInfer + >=sm_100 (Blackwell) required" |
| 31 | |
| 32 | @classmethod |
| 33 | def can_implement(cls, config: MxFp4LinearLayerConfig) -> tuple[bool, str | None]: |
| 34 | if config.activation_quant_key != kMxfp4Dynamic: |
| 35 | return False, "only supports MXFP4 dynamic activation" |
| 36 | return True, None |
| 37 | |
| 38 | def process_weights_after_loading(self, layer: torch.nn.Module) -> None: |
| 39 | N, scale_K = layer.weight_scale.shape |
| 40 | K = scale_K * _MXFP4_GROUP_SIZE |
| 41 | |
| 42 | # swizzle pads N to the next multiple of 128 for CUTLASS tiling |
| 43 | padded_N = ((N + 127) // 128) * 128 |
| 44 | layer.weight_scale = Parameter( |
| 45 | swizzle_mxfp4_scales(layer.weight_scale.data, N, K).reshape(padded_N, -1), |
| 46 | requires_grad=False, |
| 47 | ) |
| 48 | |
| 49 | def apply_weights( |
| 50 | self, |
| 51 | layer: torch.nn.Module, |
| 52 | x: torch.Tensor, |
| 53 | bias: torch.Tensor | None = None, |
| 54 | ) -> torch.Tensor: |
| 55 | from vllm.utils.flashinfer import ( |
| 56 | flashinfer_mxfp4_quantize, |
| 57 | flashinfer_scaled_fp4_mm, |
| 58 | ) |
| 59 | |
| 60 | weight = layer.weight |
| 61 | out_shape = x.shape[:-1] + (layer.output_size_per_partition,) |
| 62 | x_2d = x.reshape(-1, x.shape[-1]) |
| 63 | |
| 64 | x_fp4, x_scale = flashinfer_mxfp4_quantize( |
| 65 | x_2d.contiguous(), backend="cute-dsl" |
| 66 | ) |
| 67 | out = flashinfer_scaled_fp4_mm( |
| 68 | x_fp4, |
| 69 | weight, |
| 70 | x_scale, |
| 71 | layer.weight_scale, |
| 72 | alpha=None, |
| 73 | out_dtype=x.dtype, |
| 74 | backend="cute-dsl", |
| 75 | block_size=_MXFP4_GROUP_SIZE, |
| 76 | use_nvfp4=False, |
| 77 | ) |
| 78 | |
| 79 | if bias is not None: |
| 80 | out = out + bias |
| 81 | return out.view(out_shape) |