| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | |
| 4 | from abc import ABC, abstractmethod |
| 5 | from dataclasses import dataclass |
| 6 | from typing import ClassVar |
| 7 | |
| 8 | import torch |
| 9 | from typing_extensions import Self |
| 10 | |
| 11 | from vllm.model_executor.layers.quantization.input_quant_fp8 import QuantFP8 |
| 12 | from vllm.model_executor.layers.quantization.utils.fp8_utils import ( |
| 13 | process_fp8_weight_block_strategy, |
| 14 | ) |
| 15 | from vllm.model_executor.utils import replace_parameter |
| 16 | |
| 17 | from ..base import ( |
| 18 | FP8Params, |
| 19 | MMLinearKernel, |
| 20 | ) |
| 21 | from .ScaledMMLinearKernel import FP8ScaledMMLinearLayerConfig |
| 22 | |
| 23 | |
| 24 | @dataclass |
| 25 | class FP8BlockParams(FP8Params): |
| 26 | weight_scale_inv: torch.Tensor | None |
| 27 | weight_scale: torch.Tensor | None |
| 28 | |
| 29 | WEIGHT_SCALE_INV: ClassVar[str] = "weight_scale_inv" |
| 30 | |
| 31 | @classmethod |
| 32 | def from_layer(cls, layer: torch.nn.Module) -> Self: |
| 33 | return cls( |
| 34 | weight=getattr(layer, cls.WEIGHT), |
| 35 | weight_scale_inv=getattr(layer, cls.WEIGHT_SCALE_INV, None), |
| 36 | weight_scale=getattr(layer, cls.WEIGHT_SCALE, None), |
| 37 | input_scale=getattr(layer, cls.INPUT_SCALE, None), |
| 38 | input_scale_ub=getattr(layer, cls.INPUT_SCALE_UB, None), |
| 39 | ) |
| 40 | |
| 41 | @property |
| 42 | def block_scale_attr(self) -> str: |
| 43 | """Fp8LinearMethod registers the block scale as ``weight_scale_inv``, |
| 44 | compressed-tensors as ``weight_scale``.""" |
| 45 | return ( |
| 46 | self.WEIGHT_SCALE |
| 47 | if self.weight_scale_inv is None |
| 48 | else self.WEIGHT_SCALE_INV |
| 49 | ) |
| 50 | |
| 51 | @property |
| 52 | def block_scale(self) -> torch.Tensor: |
| 53 | scale = getattr(self, self.block_scale_attr) |
| 54 | assert scale is not None |
| 55 | return scale |
| 56 | |
| 57 | |
| 58 | class Fp8BlockScaledMMLinearKernel( |
| 59 | MMLinearKernel[FP8ScaledMMLinearLayerConfig, FP8BlockParams], ABC |
| 60 | ): |
| 61 | # Set to False in subclasses that accept BF16 input directly (e.g. FlashInfer) |
| 62 | # and therefore do not need the input quantization step in apply_weights. |
| 63 | apply_input_quant: ClassVar[bool] = True |
| 64 | |
| 65 | def __init__(self, config: FP8ScaledMMLinearLayerConfig) -> None: |
| 66 | super().__init__(config) |
| 67 | act_scale_descriptor = config.activation_quant_key.scale |
| 68 | self.weight_group_shape = config.weight_quant_key.scale.group_shape |
| 69 | self.quant_fp8 = QuantFP8( |
| 70 | static=act_scale_descriptor.static, |
| 71 | group_shape=act_scale_descriptor.group_shape, |
| 72 | num_token_padding=self.get_output_padding(), |
| 73 | use_ue8m0=False, |
| 74 | ) |
| 75 | self.use_triton = False |
| 76 | |
| 77 | @classmethod |
| 78 | def can_implement(cls, config: FP8ScaledMMLinearLayerConfig): |
| 79 | act_quant_key = config.activation_quant_key |
| 80 | if act_quant_key.scale.static: |
| 81 | return ( |
| 82 | False, |
| 83 | "Only dynamic per token group activation quantization is supported.", |
| 84 | ) |
| 85 | |
| 86 | return True, None |
| 87 | |
| 88 | def _get_layer_params(self, layer: torch.nn.Module, **kwargs) -> FP8BlockParams: |
| 89 | return FP8BlockParams.from_layer(layer) |
| 90 | |
| 91 | def process_weights_after_loading(self, layer: torch.nn.Module): |
| 92 | params = self._get_layer_params(layer) |
| 93 | new_weight, new_weight_scale = process_fp8_weight_block_strategy( |
| 94 | params.weight, |
| 95 | params.block_scale, |
| 96 | ) |
| 97 | |
| 98 | replace_parameter(layer, params.WEIGHT, new_weight.data) |
| 99 | replace_parameter(layer, params.block_scale_attr, new_weight_scale.data) |
| 100 | |
| 101 | def apply_weights( |
| 102 | self, |
| 103 | layer: torch.nn.Module, |
| 104 | x: torch.Tensor, |
| 105 | bias: torch.Tensor | None = None, |
| 106 | **kwargs, |
| 107 | ) -> torch.Tensor: |
| 108 | out_dtype = self.config.out_dtype |
| 109 | params = self._get_layer_params(layer) |
| 110 | weight = params.weight |
| 111 | weight_scale = params.block_scale |
| 112 | input_scale = params.input_scale |
| 113 | scale_up = params.input_scale_ub |
| 114 | |
| 115 | # View input as 2D matrix for fp8 methods |
| 116 | input_2d = x.view(-1, x.shape[-1]) |
| 117 | output_shape = [*x.shape[:-1], weight.shape[0]] |
| 118 | |
| 119 | if self.apply_input_quant: |
| 120 | q_input, input_scale = self.quant_fp8( |
| 121 | input_2d, input_scale, scale_up, use_triton=self.use_triton |
| 122 | ) |
| 123 | else: |
| 124 | q_input = input_2d |
| 125 | # Provide a concrete placeholder so apply_block_scaled_mm args are |
| 126 | # always Tensors. Subclasses with apply_input_quant=False must not |
| 127 | # use As in apply_block_scaled_mm. |
| 128 | input_scale = ( |
| 129 | input_scale if input_scale is not None else input_2d.new_empty(1) |
| 130 | ) |
| 131 | |
| 132 | output = self.apply_block_scaled_mm( |
| 133 | A=q_input, |
| 134 | B=weight, |
| 135 | As=input_scale, |
| 136 | Bs=weight_scale, |
| 137 | ) |
| 138 | |
| 139 | if bias is not None: |
| 140 | output = output + bias |
| 141 | return output.to(dtype=out_dtype).view(*output_shape) |
| 142 | |
| 143 | @abstractmethod |
| 144 | def apply_block_scaled_mm( |
| 145 | self, |
| 146 | A: torch.Tensor, |
| 147 | B: torch.Tensor, |
| 148 | As: torch.Tensor, |
| 149 | Bs: torch.Tensor, |
| 150 | ) -> torch.Tensor: |
| 151 | raise NotImplementedError |
| 152 | |
| 153 | |
| 154 | class Fp8BlockScaledDynamicMMLinearKernel(Fp8BlockScaledMMLinearKernel, ABC): |
| 155 | """Dynamic FP8 block-scaled kernel that dispatches at runtime. |
| 156 | |
| 157 | Extends Fp8BlockScaledMMLinearKernel to inherit apply_weights and overrides |
| 158 | apply_block_scaled_mm to dispatch between two sub-kernels using torch.cond. |
| 159 | |
| 160 | Subclasses must define: |
| 161 | base_type: The primary kernel class. |
| 162 | fallback_type: The fallback kernel class. |
| 163 | """ |
| 164 | |
| 165 | base_type: ClassVar[type[Fp8BlockScaledMMLinearKernel]] |
| 166 | fallback_type: ClassVar[type[Fp8BlockScaledMMLinearKernel]] |
| 167 | |
| 168 | def __init__(self, config: "FP8ScaledMMLinearLayerConfig") -> None: |
| 169 | super().__init__(config) |
| 170 | self.base = self.base_type(config) |
| 171 | self.fallback = self.fallback_type(config) |
| 172 | |
| 173 | @classmethod |
| 174 | def is_supported( |
| 175 | cls, compute_capability: int | None = None |
| 176 | ) -> tuple[bool, str | None]: |
| 177 | is_base_supported, reason_1 = cls.base_type.is_supported(compute_capability) |
| 178 | is_fallback_supported, reason_2 = cls.fallback_type.is_supported( |
| 179 | compute_capability |
| 180 | ) |
| 181 | if is_base_supported and is_fallback_supported: |
| 182 | return True, None |
| 183 | if not is_base_supported and not is_fallback_supported: |
| 184 | return ( |
| 185 | False, |
| 186 | f"base is not supported due to {reason_1}; " |
| 187 | f"fallback is not supported due to {reason_2}", |
| 188 | ) |
| 189 | if not is_base_supported: |
| 190 | return False, f"base is not supported due to {reason_1}" |
| 191 | return False, f"fallback is not supported due to {reason_2}" |
| 192 | |
| 193 | @classmethod |
| 194 | def can_implement( |
| 195 | cls, config: "FP8ScaledMMLinearLayerConfig" |
| 196 | ) -> tuple[bool, str | None]: |
| 197 | can_implement_base, reason_1 = cls.base_type.can_implement(config) |
| 198 | can_implement_fallback, reason_2 = cls.fallback_type.can_implement(config) |
| 199 | if can_implement_base and can_implement_fallback: |
| 200 | return True, None |
| 201 | if not can_implement_base and not can_implement_fallback: |
| 202 | return ( |
| 203 | False, |
| 204 | f"base cannot implement due to {reason_1}; " |
| 205 | f"fallback cannot implement due to {reason_2}", |
| 206 | ) |
| 207 | if not can_implement_base: |
| 208 | return False, f"base cannot implement due to {reason_1}" |
| 209 | return False, f"fallback cannot implement due to {reason_2}" |