A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 5, 19:45 UTC
[ROCm][Bugfix] fix exception related to trust_remote_code for MiniMax-M2.1-MXFP4 (#37698)
Signed-off-by: Hongxia Yang <hongxiay.yang@amd.com> Co-authored-by: Hongxia Yang <hongxiay.yang@amd.com>
Hongxia Yang committed
Mar 30, 2026 at 11:49 UTC
dbdd9ae06738bb8dc5b3fe6c99b3dd5bcd5d5526
9 files changed
+142
-19
tests/quantization/test_quark_maybe_update_config.py
new
+63
@@ -0,0 +1,63 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+"""Unit tests for QuarkConfig.maybe_update_config.
4
+
5
+Fetches real HF configs (metadata only, no model weights) to verify
6
+that dynamic_mxfp4_quant is only enabled for DeepSeek-V3-family models.
7
+
8
+Run: pytest tests/quantization/test_quark_maybe_update_config.py -v
9
+"""
10
+
11
+import pytest
12
+from transformers import AutoConfig
13
+
14
+from vllm.model_executor.layers.quantization.quark.quark import QuarkConfig
15
+
16
+
17
+def _make_quark_config() -> QuarkConfig:
18
+ """Create a minimal QuarkConfig for testing."""
19
+ return QuarkConfig(quant_config={}, kv_cache_group=[], pack_method="reorder")
20
+
21
+
22
+# ---------------------------------------------------------------------------
23
+# Non-deepseek models must not flip dynamic_mxfp4_quant
24
+# ---------------------------------------------------------------------------
25
+@pytest.mark.parametrize(
26
+ "model_name",
27
+ ["amd/MiniMax-M2.1-MXFP4"],
28
+)
29
+def test_non_deepseek_model_stays_false(model_name: str):
30
+ """Non-deepseek_v3 models must not enable dynamic_mxfp4_quant."""
31
+ hf_config = AutoConfig.from_pretrained(model_name, trust_remote_code=True)
32
+ qcfg = _make_quark_config()
33
+
34
+ qcfg.maybe_update_config(model_name, hf_config=hf_config)
35
+
36
+ assert qcfg.dynamic_mxfp4_quant is False
37
+
38
+
39
+# ---------------------------------------------------------------------------
40
+# DeepSeek-V3 family + fp4 must enable dynamic_mxfp4_quant
41
+# ---------------------------------------------------------------------------
42
+@pytest.mark.parametrize(
43
+ "model_name",
44
+ ["amd/DeepSeek-R1-MXFP4-ASQ"],
45
+)
46
+def test_deepseek_family_fp4_enables_flag(model_name: str):
47
+ hf_config = AutoConfig.from_pretrained(model_name, trust_remote_code=True)
48
+ qcfg = _make_quark_config()
49
+
50
+ qcfg.maybe_update_config(model_name, hf_config=hf_config)
51
+
52
+ assert qcfg.dynamic_mxfp4_quant is True
53
+
54
+
55
+# ---------------------------------------------------------------------------
56
+# Missing hf_config → warn and stay False
57
+# ---------------------------------------------------------------------------
58
+def test_missing_hf_config_stays_false():
59
+ qcfg = _make_quark_config()
60
+
61
+ qcfg.maybe_update_config("some/model")
62
+
63
+ assert qcfg.dynamic_mxfp4_quant is False
vllm/config/vllm.py
+4
-1
@@ -526,7 +526,10 @@ class VllmConfig:
526
f"method {model_config.quantization}. Supported dtypes: "
527
f"{supported_dtypes}"
528
)
529
- quant_config.maybe_update_config(model_config.model)
529
+ quant_config.maybe_update_config(
530
+ model_config.model,
531
+ hf_config=model_config.hf_config,
532
+ )
533
return quant_config
534
return None
535
vllm/model_executor/layers/quantization/awq.py
+7
-1
@@ -5,6 +5,7 @@ from typing import TYPE_CHECKING, Any, Union
5
6
import torch
7
from safetensors.torch import _TYPES as _SAFETENSORS_TO_TORCH_DTYPE
8
+from transformers import PretrainedConfig
9
10
from vllm import _custom_ops as ops
11
from vllm.logger import init_logger
@@ -146,7 +147,12 @@ class AWQConfig(QuantizationConfig):
147
self.modules_to_not_convert
148
)
149
149
- def maybe_update_config(self, model_name: str, revision: str | None = None):
150
+ def maybe_update_config(
151
+ self,
152
+ model_name: str,
153
+ hf_config: PretrainedConfig | None = None,
154
+ revision: str | None = None,
155
+ ):
156
if self.modules_to_not_convert:
157
return
158
vllm/model_executor/layers/quantization/awq_marlin.py
+7
-1
@@ -6,6 +6,7 @@ from typing import TYPE_CHECKING, Any
6
import torch
7
from safetensors.torch import _TYPES as _SAFETENSORS_TO_TORCH_DTYPE
8
from torch.nn import Parameter
9
+from transformers import PretrainedConfig
10
11
import vllm.model_executor.layers.fused_moe # noqa
12
from vllm import _custom_ops as ops
@@ -332,7 +333,12 @@ class AWQMarlinConfig(QuantizationConfig):
333
self.modules_to_not_convert
334
)
335
335
- def maybe_update_config(self, model_name: str, revision: str | None = None):
336
+ def maybe_update_config(
337
+ self,
338
+ model_name: str,
339
+ hf_config: PretrainedConfig | None = None,
340
+ revision: str | None = None,
341
+ ):
342
if self.modules_to_not_convert:
343
return
344
vllm/model_executor/layers/quantization/base_config.py
+15
-1
@@ -7,6 +7,7 @@ from typing import TYPE_CHECKING, Any
7
8
import torch
9
from torch import nn
10
+from transformers import PretrainedConfig
11
12
if TYPE_CHECKING:
13
from vllm.model_executor.layers.quantization import QuantizationMethods
@@ -168,10 +169,23 @@ class QuantizationConfig(ABC):
169
# TODO (@kylesayrs): add implementations for all subclasses
170
pass
171
171
- def maybe_update_config(self, model_name: str): # noqa: B027
172
+ def maybe_update_config( # noqa: B027
173
+ self,
174
+ model_name: str,
175
+ hf_config: PretrainedConfig | None = None,
176
+ revision: str | None = None,
177
+ ):
178
"""
179
Interface to update values after config initialization.
180
+
181
+ Args:
182
+ model_name: The name of the model
183
+ hf_config: The Hugging Face config of the model
184
+ revision: The revision of the model
185
+ Returns:
186
"""
187
+ # TODO: revision is never passed currently in vllm.py,
188
+ # but is used in subclasses, should we remove this parameter?
189
pass
190
191
def is_mxfp4_quant(self, prefix: str, layer: torch.nn.Module) -> bool:
vllm/model_executor/layers/quantization/cpu_wna16.py
+7
-1
@@ -5,6 +5,7 @@ from typing import Any
5
6
import torch
7
from safetensors.torch import _TYPES as _SAFETENSORS_TO_TORCH_DTYPE
8
+from transformers import PretrainedConfig
9
10
from vllm._custom_ops import (
11
cpu_gemm_wna16,
@@ -133,7 +134,12 @@ class CPUAWQConfig(QuantizationConfig):
134
self.modules_to_not_convert
135
)
136
136
- def maybe_update_config(self, model_name: str, revision: str | None = None):
137
+ def maybe_update_config(
138
+ self,
139
+ model_name: str,
140
+ hf_config: PretrainedConfig | None = None,
141
+ revision: str | None = None,
142
+ ):
143
if self.modules_to_not_convert:
144
return
145
vllm/model_executor/layers/quantization/gptq.py
+7
-1
@@ -9,6 +9,7 @@ from typing import TYPE_CHECKING, Any, Union
9
import torch
10
from safetensors.torch import _TYPES as _SAFETENSORS_TO_TORCH_DTYPE
11
from torch.nn.parameter import Parameter
12
+from transformers import PretrainedConfig
13
14
from vllm import _custom_ops as ops
15
from vllm.logger import init_logger
@@ -193,7 +194,12 @@ class GPTQConfig(QuantizationConfig):
194
self.modules_in_block_to_quantize
195
)
196
196
- def maybe_update_config(self, model_name: str, revision: str | None = None):
197
+ def maybe_update_config(
198
+ self,
199
+ model_name: str,
200
+ hf_config: PretrainedConfig | None = None,
201
+ revision: str | None = None,
202
+ ):
203
if self.modules_in_block_to_quantize:
204
if is_list_of(self.modules_in_block_to_quantize, list):
205
# original modules_in_block_to_quantize: list[list[str]]
vllm/model_executor/layers/quantization/gptq_marlin.py
+7
-1
@@ -6,6 +6,7 @@ from typing import Any
6
7
import torch
8
from safetensors.torch import _TYPES as _SAFETENSORS_TO_TORCH_DTYPE
9
+from transformers import PretrainedConfig
10
11
import vllm.model_executor.layers.fused_moe # noqa
12
from vllm import _custom_ops as ops
@@ -299,7 +300,12 @@ class GPTQMarlinConfig(QuantizationConfig):
300
self.modules_in_block_to_quantize
301
)
302
302
- def maybe_update_config(self, model_name: str, revision: str | None = None):
303
+ def maybe_update_config(
304
+ self,
305
+ model_name: str,
306
+ hf_config: PretrainedConfig | None = None,
307
+ revision: str | None = None,
308
+ ):
309
if self.modules_in_block_to_quantize:
310
if is_list_of(self.modules_in_block_to_quantize, list):
311
# original modules_in_block_to_quantize: list[list[str]]
vllm/model_executor/layers/quantization/quark/quark.py
+25
-12
@@ -5,6 +5,7 @@ import fnmatch
5
from typing import TYPE_CHECKING, Any, cast
6
7
import torch
8
+from transformers import PretrainedConfig
9
10
from vllm.logger import init_logger
11
from vllm.model_executor.layers.attention import Attention
@@ -36,7 +37,6 @@ from vllm.model_executor.layers.quantization.quark.utils import (
37
)
38
from vllm.model_executor.models.utils import WeightsMapper
39
from vllm.platforms import current_platform
39
-from vllm.transformers_utils.config import get_config
40
41
if TYPE_CHECKING:
42
from vllm.model_executor.models.utils import WeightsMapper
@@ -45,6 +45,10 @@ __all__ = ["QuarkLinearMethod"]
45
46
logger = init_logger(__name__)
47
48
+# model_type values that use dynamic MXFP4 re-quantization for
49
+# OCP MX fp4 Quark checkpoints
50
+_DEEPSEEK_V3_FAMILY_MODEL_TYPES = frozenset({"deepseek_v3"})
51
+
52
53
class QuarkConfig(QuantizationConfig):
54
def __init__(
@@ -63,19 +67,28 @@ class QuarkConfig(QuantizationConfig):
67
self.pack_method = pack_method
68
self.dynamic_mxfp4_quant = False
69
66
- def maybe_update_config(self, model_name: str, revision: str | None = None):
67
- self.hf_config = get_config(
68
- model=model_name,
69
- trust_remote_code=False, # or get from model_config if available
70
- revision=revision,
71
- config_format="auto",
72
- )
70
+ def maybe_update_config(
71
+ self,
72
+ model_name: str,
73
+ hf_config: PretrainedConfig | None = None,
74
+ revision: str | None = None,
75
+ ):
76
+ """Enable dynamic MXFP4 only for DeepSeek-V3-family + fp4 Quark checkpoints."""
77
74
- quant_config = getattr(self.hf_config, "quantization_config", None)
78
+ if (
79
+ getattr(hf_config, "model_type", None)
80
+ not in _DEEPSEEK_V3_FAMILY_MODEL_TYPES
81
+ ):
82
+ return
83
+
84
+ quant_config = getattr(hf_config, "quantization_config", None)
85
if quant_config is not None:
76
- quant_dtype = quant_config["global_quant_config"]["weight"]["dtype"]
77
- model_type = self.hf_config.model_type
78
- if quant_dtype == "fp4" and model_type == "deepseek_v3":
86
+ quant_dtype = (
87
+ quant_config.get("global_quant_config", {})
88
+ .get("weight", {})
89
+ .get("dtype")
90
+ )
91
+ if quant_dtype == "fp4":
92
self.dynamic_mxfp4_quant = True
93
94
def get_linear_method(self) -> "QuarkLinearMethod":