Fix Quark mxfp4 quantized model loading issue under mtp (#46757)
Signed-off-by: Xiao Yu <xiao.yu.dc@outlook.com> Co-authored-by: Andreas Karatzas <akaratza@amd.com>
xiao-llm committed
Jul 16, 2026 at 14:58 UTC
02bf9c7907b491146e2a79c3839af243c4478e00
2 files changed
+26
-2
vllm/model_executor/layers/quantization/quark/quark.py
+10
-2
@@ -9,7 +9,10 @@ from transformers import PretrainedConfig
9
10
from vllm.logger import init_logger
11
from vllm.model_executor.layers.attention import Attention
12
-from vllm.model_executor.layers.fused_moe import RoutedExperts
12
+from vllm.model_executor.layers.fused_moe import (
13
+ RoutedExperts,
14
+ UnquantizedFusedMoEMethod,
15
+)
16
from vllm.model_executor.layers.linear import (
17
LinearBase,
18
LinearMethodBase,
@@ -146,8 +149,13 @@ class QuarkConfig(QuantizationConfig):
149
# Check if the layer is skipped for quantization.
150
exclude_layers = cast(list[str], self.quant_config.get("exclude"))
151
if should_ignore_layer(
149
- prefix, ignore=exclude_layers, fused_mapping=self.packed_modules_mapping
152
+ prefix,
153
+ ignore=exclude_layers,
154
+ fused_mapping=self.packed_modules_mapping,
155
+ check_children=isinstance(layer, RoutedExperts),
156
):
157
+ if isinstance(layer, RoutedExperts):
158
+ return UnquantizedFusedMoEMethod(layer.moe_config)
159
if (
160
"self_attn" not in prefix # only quantize attention projections
161
or not getattr(self, "dynamic_mxfp4_quant", False)
vllm/model_executor/layers/quantization/quark/utils.py
+16
@@ -27,10 +27,26 @@ def should_ignore_layer(
27
layer_name: str | None,
28
ignore: Iterable[str],
29
fused_mapping: Mapping[str, list[str]] = MappingProxyType({}),
30
+ *,
31
+ check_children: bool = False,
32
) -> bool:
33
if layer_name is None:
34
return False
35
36
+ # MoE layers are currently all-or-nothing: if any child is ignored,
37
+ # the parent layer must be ignored as well. For example, the
38
+ # amd/GLM-5.2-MXFP4 config ignores children like
39
+ # model.layers.78.mlp.experts.*.down_proj, while the layer checked
40
+ # here is the parent model.layers.N.mlp.experts.
41
+ # See:
42
+ # https://huggingface.co/amd/GLM-5.2-MXFP4/blob/main/config.json#L793-L795
43
+ if check_children and any(
44
+ target == layer_name or target.startswith(layer_name + ".")
45
+ for target in ignore
46
+ if not target.startswith("re:")
47
+ ):
48
+ return True
49
+
50
# layer_name = model.layers.0.self_attn.qkv_proj
51
# proj_name = qkv_proj
52
proj_name = layer_name.split(".")[-1]