| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | |
| 4 | import torch |
| 5 | |
| 6 | from vllm.model_executor.layers.quantization.utils.fp8_utils import ( |
| 7 | deepgemm_post_process_fp8_weight_block, |
| 8 | per_token_group_quant_fp8_packed_for_deepgemm, |
| 9 | ) |
| 10 | from vllm.model_executor.utils import replace_parameter |
| 11 | from vllm.platforms import current_platform |
| 12 | from vllm.utils.deep_gemm import ( |
| 13 | fp8_einsum, |
| 14 | is_deep_gemm_supported, |
| 15 | ) |
| 16 | from vllm.utils.torch_utils import direct_register_custom_op |
| 17 | |
| 18 | from .Mxfp8LinearKernel import Mxfp8LinearKernel, Mxfp8LinearLayerConfig |
| 19 | |
| 20 | |
| 21 | class DeepGemmMxfp8BmmLinearKernel(Mxfp8LinearKernel): |
| 22 | """Grouped MXFP8 einsum with per-row, 32-element weight scales.""" |
| 23 | |
| 24 | def __init__(self, config: Mxfp8LinearLayerConfig): |
| 25 | super().__init__(config) |
| 26 | self.recipe = (1, 1, 32) |
| 27 | |
| 28 | @classmethod |
| 29 | def is_supported( |
| 30 | cls, compute_capability: int | None = None |
| 31 | ) -> tuple[bool, str | None]: |
| 32 | if not current_platform.is_cuda() or not is_deep_gemm_supported(): |
| 33 | return False, "DeepGEMM BMM requires a supported CUDA device." |
| 34 | if not current_platform.is_device_capability_family(100): |
| 35 | return False, "DeepGEMM MXFP8 BMM requires Blackwell." |
| 36 | return True, None |
| 37 | |
| 38 | @classmethod |
| 39 | def can_implement(cls, config: Mxfp8LinearLayerConfig) -> tuple[bool, str | None]: |
| 40 | if config.bmm_batch_size is None or config.bmm_batch_size <= 0: |
| 41 | return False, "DeepGEMM BMM requires a positive batch size." |
| 42 | return True, None |
| 43 | |
| 44 | def process_weights_after_loading(self, layer: torch.nn.Module) -> None: |
| 45 | weight = layer.weight.data |
| 46 | if weight.ndim == 3: |
| 47 | return |
| 48 | scale = layer.weight_scale.data |
| 49 | assert weight.dtype == torch.float8_e4m3fn and weight.ndim == 2 |
| 50 | assert scale.shape == (weight.shape[0], weight.shape[1] // 32) |
| 51 | assert self.config.bmm_batch_size is not None |
| 52 | weight, scale = deepgemm_post_process_fp8_weight_block( |
| 53 | wq=weight, |
| 54 | ws=scale, |
| 55 | quant_block_shape=(1, 32), |
| 56 | use_e8m0=False, |
| 57 | is_bmm=True, |
| 58 | bmm_batch_size=self.config.bmm_batch_size, |
| 59 | ) |
| 60 | replace_parameter(layer, "weight", weight) |
| 61 | replace_parameter(layer, "weight_scale", scale) |
| 62 | layer.weight_block_size = [1, 32] |
| 63 | |
| 64 | def apply_weights( |
| 65 | self, |
| 66 | layer: torch.nn.Module, |
| 67 | x: torch.Tensor | tuple[torch.Tensor, torch.Tensor], |
| 68 | bias: torch.Tensor | None = None, |
| 69 | ) -> torch.Tensor: |
| 70 | """Project [tokens, groups, K] input to [tokens, groups, N].""" |
| 71 | if isinstance(x, tuple): |
| 72 | q_input, input_scale = x |
| 73 | else: |
| 74 | tokens, groups, k = x.shape |
| 75 | q_input, input_scale = per_token_group_quant_fp8_packed_for_deepgemm( |
| 76 | x.reshape(tokens, groups * k), |
| 77 | group_size=32, |
| 78 | use_ue8m0=True, |
| 79 | ) |
| 80 | q_input = q_input.view(tokens, groups, k) |
| 81 | input_scale = input_scale.view(tokens, groups, k // 128) |
| 82 | output = torch.ops.vllm.deepgemm_mxfp8_bmm( |
| 83 | q_input, input_scale, layer.weight, layer.weight_scale, list(self.recipe) |
| 84 | ) |
| 85 | if bias is not None: |
| 86 | output = output + bias.view(output.shape[1:]) |
| 87 | return output |
| 88 | |
| 89 | |
| 90 | def _deepgemm_mxfp8_bmm_fake( |
| 91 | x: torch.Tensor, |
| 92 | x_scale: torch.Tensor, |
| 93 | weight: torch.Tensor, |
| 94 | weight_scale: torch.Tensor, |
| 95 | recipe: list[int], |
| 96 | ) -> torch.Tensor: |
| 97 | return torch.empty( |
| 98 | (x.shape[0], weight.shape[0], weight.shape[1]), |
| 99 | device=x.device, |
| 100 | dtype=torch.bfloat16, |
| 101 | ) |
| 102 | |
| 103 | |
| 104 | def _deepgemm_mxfp8_bmm( |
| 105 | x: torch.Tensor, |
| 106 | x_scale: torch.Tensor, |
| 107 | weight: torch.Tensor, |
| 108 | weight_scale: torch.Tensor, |
| 109 | recipe: list[int], |
| 110 | ) -> torch.Tensor: |
| 111 | output = torch.empty( |
| 112 | (x.shape[0], weight.shape[0], weight.shape[1]), |
| 113 | device=x.device, |
| 114 | dtype=torch.bfloat16, |
| 115 | ) |
| 116 | fp8_einsum( |
| 117 | "bhr,hdr->bhd", |
| 118 | (x, x_scale), |
| 119 | (weight, weight_scale), |
| 120 | output, |
| 121 | recipe=tuple(recipe), |
| 122 | ) |
| 123 | return output |
| 124 | |
| 125 | |
| 126 | direct_register_custom_op( |
| 127 | "deepgemm_mxfp8_bmm", _deepgemm_mxfp8_bmm, fake_impl=_deepgemm_mxfp8_bmm_fake |
| 128 | ) |