| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | |
| 4 | from collections.abc import Callable |
| 5 | from functools import partial |
| 6 | |
| 7 | import torch |
| 8 | import torch.nn.functional as F |
| 9 | from torch.nn.parameter import Parameter |
| 10 | |
| 11 | import vllm.envs as envs |
| 12 | from vllm.logger import init_logger |
| 13 | from vllm.model_executor.layers.quantization.utils.mxfp4_utils import ( |
| 14 | dequant_mxfp4, |
| 15 | quant_dequant_mxfp4, |
| 16 | ) |
| 17 | from vllm.model_executor.layers.quantization.utils.mxfp6_utils import ( |
| 18 | quant_dequant_mxfp6, |
| 19 | ) |
| 20 | from vllm.model_executor.layers.quantization.utils.quant_utils import ( |
| 21 | QuantKey, |
| 22 | kMxfp4Dynamic, |
| 23 | kMxfp6E2M3Dynamic, |
| 24 | kMxfp6E3M2Dynamic, |
| 25 | ) |
| 26 | from vllm.platforms import current_platform |
| 27 | from vllm.utils.import_utils import has_quark |
| 28 | |
| 29 | from .base import MxFp4LinearKernel, MxFp4LinearLayerConfig |
| 30 | |
| 31 | logger = init_logger(__name__) |
| 32 | |
| 33 | _ACTIVATION_QUANT_DEQUANT_FUNCS: dict[ |
| 34 | QuantKey, Callable[[torch.Tensor], torch.Tensor] |
| 35 | ] = { |
| 36 | kMxfp4Dynamic: quant_dequant_mxfp4, |
| 37 | kMxfp6E3M2Dynamic: partial(quant_dequant_mxfp6, quant_dtype="fp6_e3m2"), |
| 38 | kMxfp6E2M3Dynamic: partial(quant_dequant_mxfp6, quant_dtype="fp6_e2m3"), |
| 39 | } |
| 40 | |
| 41 | |
| 42 | class EmulationMxfp4LinearKernel(MxFp4LinearKernel): |
| 43 | """Software emulation fallback for OCP MXFP4/MXFP6 (dequant + F.linear).""" |
| 44 | |
| 45 | def __init__(self, config: MxFp4LinearLayerConfig) -> None: |
| 46 | super().__init__(config) |
| 47 | if config.activation_quant_key is None: |
| 48 | # no input Q/DQ for weight-only |
| 49 | self.quant_dequant_func: Callable[[torch.Tensor], torch.Tensor] = ( |
| 50 | lambda x: x |
| 51 | ) |
| 52 | else: |
| 53 | self.quant_dequant_func = _ACTIVATION_QUANT_DEQUANT_FUNCS[ |
| 54 | config.activation_quant_key |
| 55 | ] |
| 56 | |
| 57 | @classmethod |
| 58 | def is_supported( |
| 59 | cls, compute_capability: int | None = None |
| 60 | ) -> tuple[bool, str | None]: |
| 61 | return True, None |
| 62 | |
| 63 | @classmethod |
| 64 | def can_implement(cls, config: MxFp4LinearLayerConfig) -> tuple[bool, str | None]: |
| 65 | if config.activation_quant_key not in ( |
| 66 | None, |
| 67 | kMxfp4Dynamic, |
| 68 | kMxfp6E3M2Dynamic, |
| 69 | kMxfp6E2M3Dynamic, |
| 70 | ): |
| 71 | return False, "only supports MXFP4 or MXFP6 or unquantized activations" |
| 72 | |
| 73 | if not has_quark(): |
| 74 | return False, "amd-quark package not available" |
| 75 | |
| 76 | if ( |
| 77 | current_platform.is_rocm() |
| 78 | and current_platform.supports_mx() |
| 79 | and config.activation_quant_key != kMxfp4Dynamic |
| 80 | ): |
| 81 | logger.warning_once( |
| 82 | "The current platform supports native MXFP4/MXFP6 computation, " |
| 83 | f"but kernels for activation_quant_key={config.activation_quant_key} " |
| 84 | f"are not yet integrated in vLLM. Using EmulationMxfp4LinearKernel, " |
| 85 | "with simulated weight dequantization and activation " |
| 86 | "QDQ (quantize and dequantize), with the linear " |
| 87 | "layers computed in high precision." |
| 88 | ) |
| 89 | |
| 90 | if not current_platform.supports_mx(): |
| 91 | logger.warning_once( |
| 92 | "The current platform does not support native MXFP4 " |
| 93 | "computation. Using EmulationMxfp4LinearKernel, with simulated weight " |
| 94 | "dequantization and activation QDQ (quantize and dequantize), with " |
| 95 | "the linear layers computed in high precision." |
| 96 | ) |
| 97 | |
| 98 | return True, None |
| 99 | |
| 100 | def process_weights_after_loading(self, layer: torch.nn.Module) -> None: |
| 101 | weight_scale = Parameter(layer.weight_scale.data, requires_grad=False) |
| 102 | |
| 103 | # Match the MXFP8 emulation path when explicitly enabled: dequantize |
| 104 | # once at load time so inference uses a plain high-precision linear. |
| 105 | # Otherwise retain packed weights and dequantize per invocation. |
| 106 | if envs.VLLM_MXFP4_EMULATION_DEQUANT_AT_LOAD: |
| 107 | weight = dequant_mxfp4(layer.weight, weight_scale, torch.bfloat16) |
| 108 | layer.weight = Parameter(weight.contiguous(), requires_grad=False) |
| 109 | |
| 110 | layer.weight_scale = weight_scale |
| 111 | |
| 112 | def apply_weights( |
| 113 | self, |
| 114 | layer: torch.nn.Module, |
| 115 | x: torch.Tensor, |
| 116 | bias: torch.Tensor | None = None, |
| 117 | ) -> torch.Tensor: |
| 118 | qdq_x = self.quant_dequant_func(x) |
| 119 | |
| 120 | # Packed MXFP4 weights use one byte for two values. A weight with |
| 121 | # elements wider than one byte was already dequantized at load time. |
| 122 | if layer.weight.element_size() >= 2: |
| 123 | return F.linear(qdq_x, layer.weight.to(x.dtype), bias) |
| 124 | |
| 125 | dq_w = dequant_mxfp4(layer.weight, layer.weight_scale, x.dtype) |
| 126 | return F.linear(qdq_x, dq_w, bias) |