| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | |
| 4 | from abc import ABC, abstractmethod |
| 5 | from dataclasses import dataclass |
| 6 | |
| 7 | import torch |
| 8 | |
| 9 | from vllm.model_executor.layers.quantization.utils.quant_utils import QuantKey |
| 10 | |
| 11 | |
| 12 | @dataclass |
| 13 | class MxFp6LinearLayerConfig: |
| 14 | """Configuration for an MXFP6 linear layer. |
| 15 | |
| 16 | All MXFP6 layers share the same structure: packed uint8 weights (4 FP6 values |
| 17 | per 3 bytes) and per-block weight scales (group size 32). |
| 18 | |
| 19 | Attributes: |
| 20 | weight_quant_key: Identifies the weight quantization format. Can be |
| 21 | kMxfp6E2M3Static or kMxfp6E3M2Static. |
| 22 | activation_quant_key: Identifies the activation quantization format, |
| 23 | or `None` when activations must not be quantized. |
| 24 | |
| 25 | """ |
| 26 | |
| 27 | weight_quant_key: QuantKey |
| 28 | activation_quant_key: QuantKey | None = None |
| 29 | |
| 30 | |
| 31 | class MxFp6LinearKernel(ABC): |
| 32 | """Base class for MXFP6 quantized linear kernels. |
| 33 | |
| 34 | Each subclass implements a specific GEMM backend (CUTLASS, Marlin, etc). |
| 35 | The kernel selection mechanism iterates over registered subclasses in |
| 36 | priority order,calling ``is_supported`` and ``can_implement`` to find the best |
| 37 | match for the current hardware. |
| 38 | """ |
| 39 | |
| 40 | def __init__(self, config: MxFp6LinearLayerConfig) -> None: |
| 41 | assert self.can_implement(config)[0] |
| 42 | assert self.is_supported()[0] |
| 43 | self.config = config |
| 44 | |
| 45 | @classmethod |
| 46 | @abstractmethod |
| 47 | def is_supported( |
| 48 | cls, compute_capability: int | None = None |
| 49 | ) -> tuple[bool, str | None]: |
| 50 | """Return whether this kernel can run on the current platform.""" |
| 51 | raise NotImplementedError |
| 52 | |
| 53 | @classmethod |
| 54 | @abstractmethod |
| 55 | def can_implement(cls, config: MxFp6LinearLayerConfig) -> tuple[bool, str | None]: |
| 56 | """Return whether this kernel can handle *config*.""" |
| 57 | raise NotImplementedError |
| 58 | |
| 59 | @abstractmethod |
| 60 | def process_weights_after_loading(self, layer: torch.nn.Module) -> None: |
| 61 | """Transform weights into the format required by this kernel. |
| 62 | |
| 63 | Called once after checkpoint weights have been loaded onto the |
| 64 | device. Implementations should repack / swizzle / pad weights |
| 65 | and scales in-place on *layer*. |
| 66 | """ |
| 67 | raise NotImplementedError |
| 68 | |
| 69 | @abstractmethod |
| 70 | def apply_weights( |
| 71 | self, |
| 72 | layer: torch.nn.Module, |
| 73 | x: torch.Tensor, |
| 74 | bias: torch.Tensor | None = None, |
| 75 | ) -> torch.Tensor: |
| 76 | """Run the quantized GEMM.""" |
| 77 | raise NotImplementedError |