| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | """Quantization config for DeepSeek V4.""" |
| 4 | |
| 5 | from __future__ import annotations |
| 6 | |
| 7 | from typing import TYPE_CHECKING, cast |
| 8 | |
| 9 | from vllm.config import get_current_vllm_config |
| 10 | from vllm.model_executor.layers.fused_moe import ( |
| 11 | RoutedExperts, |
| 12 | UnquantizedFusedMoEMethod, |
| 13 | ) |
| 14 | from vllm.model_executor.layers.quantization import QuantizationMethods |
| 15 | from vllm.model_executor.layers.quantization.fp8 import Fp8Config |
| 16 | from vllm.model_executor.layers.quantization.mxfp4 import Mxfp4MoEMethod |
| 17 | from vllm.model_executor.layers.quantization.utils.quant_utils import ( |
| 18 | is_layer_skipped, |
| 19 | ) |
| 20 | |
| 21 | _DEEPSEEK_V4_EXPERT_DTYPES = ("fp4", "fp8") |
| 22 | |
| 23 | if TYPE_CHECKING: |
| 24 | from vllm.model_executor.layers.quantization.modelopt import ( |
| 25 | ModelOptNvFp4Config, |
| 26 | ) |
| 27 | |
| 28 | |
| 29 | class DeepseekV4FP8Config(Fp8Config): |
| 30 | """FP8 config for DeepSeek V4 with expert-dtype-aware MoE dispatch. |
| 31 | |
| 32 | DeepSeek V4 checkpoints always use FP8 block quantization for |
| 33 | linear/attention layers. The MoE expert weights vary by checkpoint: |
| 34 | - ``expert_dtype="fp4"`` (e.g. DeepSeek-V4-Flash): MXFP4 experts |
| 35 | with ue8m0 (e8m0fnu) FP8 linear scales. |
| 36 | - ``expert_dtype="fp8"`` (e.g. DeepSeek-V4-Flash-Base): FP8 block |
| 37 | experts with float32 FP8 linear scales. |
| 38 | |
| 39 | The dispatch and the linear scale dtype are both keyed off |
| 40 | ``expert_dtype`` from the model's hf_config; missing values default |
| 41 | to ``"fp4"`` so existing FP4 checkpoints stay unchanged. |
| 42 | |
| 43 | NOTE: ``expert_dtype`` is resolved lazily because this config is |
| 44 | constructed during VllmConfig setup, before ``set_current_vllm_config`` |
| 45 | is active. Reading hf_config eagerly in ``__init__`` would always see |
| 46 | the default ``"fp4"`` and silently misroute Flash-Base checkpoints. |
| 47 | """ |
| 48 | |
| 49 | def __init__(self, *args, **kwargs): |
| 50 | super().__init__(*args, **kwargs) |
| 51 | self._resolved_expert_dtype: str | None = None |
| 52 | self._resolved_moe_quant_algo: str | None = None |
| 53 | self._nvfp4_config: ModelOptNvFp4Config | None = None |
| 54 | # ``is_scale_e8m0`` is a property that resolves on first read, |
| 55 | # by which time the current vllm_config has been set. |
| 56 | |
| 57 | @property |
| 58 | def expert_dtype(self) -> str: |
| 59 | if self._resolved_expert_dtype is None: |
| 60 | try: |
| 61 | hf_config = get_current_vllm_config().model_config.hf_config |
| 62 | except Exception: |
| 63 | # vllm_config not yet set; defer the decision until a |
| 64 | # later call lands inside set_current_vllm_config. |
| 65 | return "fp4" |
| 66 | expert_dtype = getattr(hf_config, "expert_dtype", "fp4") |
| 67 | if expert_dtype not in _DEEPSEEK_V4_EXPERT_DTYPES: |
| 68 | raise ValueError( |
| 69 | f"Unsupported DeepSeek V4 expert_dtype={expert_dtype!r}; " |
| 70 | f"expected one of {_DEEPSEEK_V4_EXPERT_DTYPES}." |
| 71 | ) |
| 72 | self._resolved_expert_dtype = expert_dtype |
| 73 | from vllm.logger import init_logger |
| 74 | |
| 75 | init_logger(__name__).info_once( |
| 76 | "DeepSeek V4 expert_dtype resolved to %r", expert_dtype |
| 77 | ) |
| 78 | return self._resolved_expert_dtype |
| 79 | |
| 80 | @property |
| 81 | def is_scale_e8m0(self) -> bool: |
| 82 | # FP4 checkpoints store FP8 linear scales as e8m0fnu; FP8 expert |
| 83 | # checkpoints (Flash-Base) store them as float32. |
| 84 | return self.expert_dtype == "fp4" |
| 85 | |
| 86 | def _resolve_moe_overrides(self) -> None: |
| 87 | if self._resolved_moe_quant_algo is not None: |
| 88 | return |
| 89 | try: |
| 90 | hf_config = get_current_vllm_config().model_config.hf_config |
| 91 | except Exception: |
| 92 | return |
| 93 | quant_cfg = getattr(hf_config, "quantization_config", None) or {} |
| 94 | algo = (quant_cfg.get("moe_quant_algo") or "").upper() or None |
| 95 | self._resolved_moe_quant_algo = algo or "" |
| 96 | |
| 97 | @property |
| 98 | def moe_quant_algo(self) -> str: |
| 99 | self._resolve_moe_overrides() |
| 100 | return self._resolved_moe_quant_algo or "" |
| 101 | |
| 102 | def _get_nvfp4_config(self) -> ModelOptNvFp4Config: |
| 103 | if self._nvfp4_config is None: |
| 104 | from vllm.model_executor.layers.quantization.modelopt import ( |
| 105 | ModelOptNvFp4Config, |
| 106 | ) |
| 107 | |
| 108 | self._nvfp4_config = ModelOptNvFp4Config( |
| 109 | is_checkpoint_nvfp4_serialized=True, |
| 110 | kv_cache_quant_algo=None, |
| 111 | exclude_modules=[], |
| 112 | group_size=16, |
| 113 | ) |
| 114 | return self._nvfp4_config |
| 115 | |
| 116 | @classmethod |
| 117 | def get_name(cls) -> QuantizationMethods: |
| 118 | return "deepseek_v4_fp8" |
| 119 | |
| 120 | @staticmethod |
| 121 | def _is_quark_mxfp4_ocp(hf_quant_cfg: dict) -> bool: |
| 122 | """True for AMD-Quark exports whose global scheme is MXFP4.""" |
| 123 | weight = (hf_quant_cfg.get("global_quant_config") or {}).get("weight") |
| 124 | # A non-dict weight (e.g. a list of multiple specs) means not an OCP |
| 125 | # MXFP4 scheme (e.g. NVFP4 with 2-level scale). |
| 126 | if not isinstance(weight, dict): |
| 127 | return False |
| 128 | return ( |
| 129 | weight.get("dtype") == "fp4" |
| 130 | and weight.get("qscheme") == "per_group" |
| 131 | and weight.get("group_size") == 32 |
| 132 | ) |
| 133 | |
| 134 | @classmethod |
| 135 | def override_quantization_method( |
| 136 | cls, hf_quant_cfg, user_quant, hf_config=None |
| 137 | ) -> QuantizationMethods | None: |
| 138 | # Quark checkpoints are always handled by QuarkConfig. |
| 139 | if ( |
| 140 | isinstance(hf_quant_cfg, dict) |
| 141 | and hf_quant_cfg.get("quant_method") == "quark" |
| 142 | ): |
| 143 | return None |
| 144 | if not ( |
| 145 | isinstance(hf_quant_cfg, dict) |
| 146 | and hf_quant_cfg.get("quant_method") in ("fp8", "deepseek_v4_fp8") |
| 147 | ): |
| 148 | return None |
| 149 | model_type = getattr(hf_config, "model_type", None) |
| 150 | if model_type == "deepseek_v4" or user_quant == "deepseek_v4_fp8": |
| 151 | return "deepseek_v4_fp8" |
| 152 | return None |
| 153 | |
| 154 | @classmethod |
| 155 | def from_config(cls, config: dict) -> DeepseekV4FP8Config: |
| 156 | # Reroute AMD-Quark fused shared expert MXFP4 checkpoints onto the fp8 |
| 157 | # path: the runtime layout matches the DeepSeek-native fp8 checkpoint, |
| 158 | # so translate the schema into format Fp8Config.from_config expects. |
| 159 | if config.get("quant_method") == "quark": |
| 160 | quark_exclude = config.get("exclude") or [] |
| 161 | config = { |
| 162 | "quant_method": "fp8", |
| 163 | "activation_scheme": "dynamic", |
| 164 | "fmt": "e4m3", |
| 165 | "scale_fmt": "ue8m0", |
| 166 | "weight_block_size": [128, 128], |
| 167 | "ignored_layers": [ |
| 168 | name for name in quark_exclude if isinstance(name, str) |
| 169 | ], |
| 170 | } |
| 171 | return cast("DeepseekV4FP8Config", super().from_config(config)) |
| 172 | |
| 173 | def get_quant_method(self, layer, prefix): |
| 174 | if isinstance(layer, RoutedExperts): |
| 175 | if is_layer_skipped( |
| 176 | prefix=prefix, |
| 177 | ignored_layers=self.ignored_layers, |
| 178 | fused_mapping=self.packed_modules_mapping, |
| 179 | ): |
| 180 | return UnquantizedFusedMoEMethod(layer.moe_config) |
| 181 | if self.expert_dtype == "fp4": |
| 182 | if self.moe_quant_algo == "NVFP4": |
| 183 | from vllm.model_executor.layers.quantization.modelopt import ( |
| 184 | ModelOptNvFp4FusedMoE, |
| 185 | ) |
| 186 | |
| 187 | return ModelOptNvFp4FusedMoE( |
| 188 | quant_config=self._get_nvfp4_config(), |
| 189 | moe_config=layer.moe_config, |
| 190 | ) |
| 191 | return Mxfp4MoEMethod(layer.moe_config) |
| 192 | # expert_dtype == "fp8": fall through to Fp8Config which |
| 193 | # returns Fp8MoEMethod with block-wise float32 scales. |
| 194 | return super().get_quant_method(layer, prefix) |