A high-throughput and memory-efficient inference and serving engine for LLMs
[Bugfix] Add deepseek_v32 to Quark dynamic MXFP4 model type check (#39498)
Signed-off-by: Shantipriya Parida <shantipriya.parida@amd.com>
Shantipriya Parida committed
Jun 10, 2026 at 12:52 UTC
a1ec011a833e5155ac7dcb8a412a5d3853a32806
1 file changed
+32
vllm/model_executor/layers/quantization/quark/quark.py
+32
@@ -5,6 +5,7 @@ import fnmatch
5
from typing import TYPE_CHECKING, Any, cast
6
7
import torch
8
+from transformers import PretrainedConfig
9
10
from vllm.logger import init_logger
11
from vllm.model_executor.layers.attention import Attention
@@ -45,6 +46,10 @@ __all__ = ["QuarkLinearMethod"]
46
47
logger = init_logger(__name__)
48
49
+# model_type values that use dynamic MXFP4 re-quantization for
50
+# OCP MX fp4 Quark checkpoints
51
+_DEEPSEEK_V3_FAMILY_MODEL_TYPES = frozenset({"deepseek_v3", "deepseek_v32"})
52
+
53
54
class QuarkConfig(QuantizationConfig):
55
def __init__(
@@ -67,6 +72,33 @@ class QuarkConfig(QuantizationConfig):
72
# we want to re-enable it in the future.
73
self.dynamic_mxfp4_quant = False
74
75
+ def maybe_update_config(
76
+ self,
77
+ model_name: str,
78
+ hf_config: PretrainedConfig | None = None,
79
+ revision: str | None = None,
80
+ ):
81
+ """Enable dynamic MXFP4 only for DeepSeek-V3-family fp4 checkpoints."""
82
+
83
+ if hf_config is None:
84
+ return
85
+
86
+ if (
87
+ getattr(hf_config, "model_type", None)
88
+ not in _DEEPSEEK_V3_FAMILY_MODEL_TYPES
89
+ ):
90
+ return
91
+
92
+ quant_config = getattr(hf_config, "quantization_config", None)
93
+ if isinstance(quant_config, dict):
94
+ quant_dtype = (
95
+ quant_config.get("global_quant_config", {})
96
+ .get("weight", {})
97
+ .get("dtype")
98
+ )
99
+ if quant_dtype == "fp4":
100
+ self.dynamic_mxfp4_quant = True
101
+
102
def get_linear_method(self) -> "QuarkLinearMethod":
103
return QuarkLinearMethod(self)
104