| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | |
| 4 | import torch |
| 5 | |
| 6 | from vllm._custom_ops import ( |
| 7 | cutlass_scaled_fp4_mm, |
| 8 | scaled_fp4_quant, |
| 9 | ) |
| 10 | from vllm.model_executor.layers.quantization.utils.nvfp4_utils import ( |
| 11 | cutlass_fp4_supported, |
| 12 | nvfp4_weight_padding_bytes, |
| 13 | pad_nvfp4_weight_for_cutlass, |
| 14 | slice_nvfp4_output, |
| 15 | swizzle_blockscale, |
| 16 | ) |
| 17 | |
| 18 | from .base import NvFp4LinearKernel, NvFp4LinearLayerConfig |
| 19 | |
| 20 | |
| 21 | class CutlassNvFp4LinearKernel(NvFp4LinearKernel): |
| 22 | """NVFP4 GEMM via the vLLM CUTLASS kernel.""" |
| 23 | |
| 24 | @classmethod |
| 25 | def is_supported( |
| 26 | cls, compute_capability: int | None = None |
| 27 | ) -> tuple[bool, str | None]: |
| 28 | if cutlass_fp4_supported(): |
| 29 | return True, None |
| 30 | return False, "CUTLASS FP4 kernels not available" |
| 31 | |
| 32 | @classmethod |
| 33 | def can_implement(cls, config: NvFp4LinearLayerConfig) -> tuple[bool, str | None]: |
| 34 | return True, None |
| 35 | |
| 36 | def process_weights_after_loading(self, layer: torch.nn.Module) -> None: |
| 37 | layer.weight_scale = torch.nn.Parameter( |
| 38 | swizzle_blockscale(layer.weight_scale.data), requires_grad=False |
| 39 | ) |
| 40 | padded_weight, _ = pad_nvfp4_weight_for_cutlass(layer.weight.data) |
| 41 | layer.weight = torch.nn.Parameter(padded_weight, requires_grad=False) |
| 42 | |
| 43 | def apply_weights( |
| 44 | self, |
| 45 | layer: torch.nn.Module, |
| 46 | x: torch.Tensor, |
| 47 | bias: torch.Tensor | None = None, |
| 48 | ) -> torch.Tensor: |
| 49 | output_size = layer.output_size_per_partition |
| 50 | output_dtype = x.dtype |
| 51 | output_shape = [*x.shape[:-1], output_size] |
| 52 | weights_padding_bytes = nvfp4_weight_padding_bytes(layer) |
| 53 | |
| 54 | x_fp4, x_blockscale = scaled_fp4_quant( |
| 55 | x, |
| 56 | layer.input_global_scale_inv, |
| 57 | is_sf_swizzled_layout=True, |
| 58 | backend="cutlass", |
| 59 | padded_n=x.shape[-1] + weights_padding_bytes * 2, |
| 60 | ) |
| 61 | |
| 62 | out = cutlass_scaled_fp4_mm( |
| 63 | x_fp4, |
| 64 | layer.weight, |
| 65 | x_blockscale, |
| 66 | layer.weight_scale, |
| 67 | layer.alpha, |
| 68 | output_dtype, |
| 69 | ) |
| 70 | |
| 71 | out = slice_nvfp4_output(out, output_size) |
| 72 | |
| 73 | if bias is not None: |
| 74 | out = out + bias |
| 75 | return out.view(*output_shape) |