[BugFix] Fix ModelOpt quantization inference for fused siblings (#47445)
Signed-off-by: jasonlizhengjian <jasonlizhengjian@gmail.com>
Jason Li committed
Jul 7, 2026 at 18:19 UTC
d99adcebdc9402c26346230ef2480d1cb72e654e
2 files changed
+42
tests/quantization/test_modelopt.py
+32
@@ -133,6 +133,38 @@ def test_modelopt_mixed_precision_quantizes_parallel_lm_head():
133
assert isinstance(method, ModelOptNvFp4LinearMethod)
134
135
136
+def test_modelopt_mixed_precision_resolves_declared_packed_projection():
137
+ config = _mixed_precision_config(
138
+ {
139
+ "model.layers.0.self_attn.q_proj": {"quant_algo": "MXFP8"},
140
+ "model.layers.0.self_attn.k_proj": {"quant_algo": "MXFP8"},
141
+ "model.layers.0.self_attn.v_proj": {"quant_algo": "MXFP8"},
142
+ }
143
+ )
144
+ config.packed_modules_mapping = {"qkv_proj": ["q_proj", "k_proj", "v_proj"]}
145
+
146
+ assert config._resolve_quant_algo("model.layers.0.self_attn.qkv_proj") == "MXFP8"
147
+
148
+
149
+def test_modelopt_mixed_precision_does_not_quantize_unlisted_fused_sibling():
150
+ config = _mixed_precision_config(
151
+ {
152
+ "model.layers.0.linear_attn.in_proj_qkv": {"quant_algo": "FP8"},
153
+ "model.layers.0.linear_attn.in_proj_z": {"quant_algo": "FP8"},
154
+ "model.layers.0.linear_attn.out_proj": {"quant_algo": "FP8"},
155
+ }
156
+ )
157
+ config.packed_modules_mapping = {
158
+ "in_proj_qkvz": ["in_proj_qkv", "in_proj_z"],
159
+ "in_proj_ba": ["in_proj_b", "in_proj_a"],
160
+ }
161
+
162
+ assert (
163
+ config._resolve_quant_algo("model.layers.0.linear_attn.in_proj_qkvz") == "FP8"
164
+ )
165
+ assert config._resolve_quant_algo("model.layers.0.linear_attn.in_proj_ba") is None
166
+
167
+
168
def test_modelopt_mixed_precision_infers_fused_gate_up_projection():
169
from vllm.model_executor.layers.linear import LinearBase
170
vllm/models/minimax_m3/nvidia/model.py
+10
@@ -957,6 +957,11 @@ class MiniMaxM3Model(nn.Module, EagleModelMixin):
957
class MiniMaxM3SparseForCausalLM(nn.Module, SupportsPP, SupportsEagle3):
958
"""MiniMax M3 (sparse/dense backbone) for causal language modeling."""
959
960
+ packed_modules_mapping = {
961
+ "qkv_proj": ["q_proj", "k_proj", "v_proj"],
962
+ "gate_up_proj": ["gate_proj", "up_proj"],
963
+ }
964
+
965
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
966
super().__init__()
967
config = vllm_config.model_config.hf_text_config
@@ -1024,6 +1029,11 @@ class MiniMaxM3SparseForConditionalGeneration(
1029
# ranks (see ``_process_image_input`` / ``_process_video_input``).
1030
supports_encoder_tp_data = True
1031
1032
+ packed_modules_mapping = {
1033
+ "qkv_proj": ["q_proj", "k_proj", "v_proj"],
1034
+ "gate_up_proj": ["gate_proj", "up_proj"],
1035
+ }
1036
+
1037
hf_to_vllm_mapper = WeightsMapper(
1038
orig_to_new_prefix={
1039
"multi_modal_projector.": "vision_tower.multi_modal_projector.",