[Quark] Remove quark-specific silent online quantization (#51800)
Signed-off-by: Felix Marty <Felix.Marty@amd.com> Signed-off-by: mgoin <mgoin64@gmail.com> Co-authored-by: mgoin <mgoin64@gmail.com>
fxmarty-amd committed
Sep 23, 2026 at 17:35 UTC
0549e8d0ab88d3edf152147e14171f964c6e6f07
3 files changed
+45
-133
tests/quantization/test_quark.py
+14
-15
@@ -855,7 +855,7 @@ def test_quant_method_dispatch_ignored(default_vllm_config):
855
UnquantizedFusedMoEMethod,
856
)
857
858
- dynamic_mxfp4_config = _make_qtensor_config(
858
+ mxfp4_config = _make_qtensor_config(
859
{
860
"dtype": "fp4",
861
"qscheme": "per_group",
@@ -866,24 +866,23 @@ def test_quant_method_dispatch_ignored(default_vllm_config):
866
None,
867
exclude=["self_attn.q_proj", "mlp.down_proj"],
868
)
869
- dynamic_mxfp4_config.dynamic_mxfp4_quant = True
870
-
871
- assert dynamic_mxfp4_config.get_quant_method_target(
872
- "self_attn.q_proj", LinearBase
873
- ) == (kMxfp4Static, None, QuarkLinearMethod)
874
- attention_proj = TestLinear()
869
+ assert mxfp4_config.get_quant_method_target("self_attn.q_proj", LinearBase) == (
870
+ None,
871
+ None,
872
+ UnquantizedLinearMethod,
873
+ )
874
assert isinstance(
876
- dynamic_mxfp4_config.get_quant_method(attention_proj, "self_attn.q_proj"),
877
- QuarkLinearMethod,
875
+ mxfp4_config.get_quant_method(TestLinear(), "self_attn.q_proj"),
876
+ UnquantizedLinearMethod,
877
)
879
- assert isinstance(attention_proj.scheme, QuarkOCP_MX)
880
- assert attention_proj.scheme.dynamic_mxfp4_quant
878
882
- assert dynamic_mxfp4_config.get_quant_method_target(
883
- "mlp.down_proj", LinearBase
884
- ) == (None, None, UnquantizedLinearMethod)
879
+ assert mxfp4_config.get_quant_method_target("mlp.down_proj", LinearBase) == (
880
+ None,
881
+ None,
882
+ UnquantizedLinearMethod,
883
+ )
884
assert isinstance(
886
- dynamic_mxfp4_config.get_quant_method(TestLinear(), "mlp.down_proj"),
885
+ mxfp4_config.get_quant_method(TestLinear(), "mlp.down_proj"),
886
UnquantizedLinearMethod,
887
)
888
vllm/model_executor/layers/quantization/quark/quark.py
-57
@@ -5,7 +5,6 @@ import fnmatch
5
from typing import TYPE_CHECKING, Any, NamedTuple, cast
6
7
import torch
8
-from transformers import PretrainedConfig
8
9
from vllm.logger import init_logger
10
from vllm.model_executor.layers.attention import Attention
@@ -82,10 +81,6 @@ __all__ = ["QuarkLinearMethod"]
81
82
logger = init_logger(__name__)
83
85
-# model_type values that use dynamic MXFP4 re-quantization for
86
-# OCP MX fp4 Quark checkpoints
87
-_DEEPSEEK_V3_FAMILY_MODEL_TYPES = frozenset({"deepseek_v3", "deepseek_v32"})
88
-
84
85
class QuantKeyMatch(NamedTuple):
86
matches: bool
@@ -119,37 +114,6 @@ class QuarkConfig(QuantizationConfig):
114
self.kv_cache_group = kv_cache_group
115
self.kv_cache_config = kv_cache_config
116
self.pack_method = pack_method
122
- # Note : this flag is kept disabled because the overhead of
123
- # dynamic mxfp4 quantization negates the performance gains
124
- # that come from shifting to mxfp4. It is left here in case
125
- # we want to re-enable it in the future.
126
- self.dynamic_mxfp4_quant = False
127
-
128
- def maybe_update_config(
129
- self,
130
- model_name: str,
131
- hf_config: PretrainedConfig | None = None,
132
- revision: str | None = None,
133
- ):
134
- """Enable dynamic MXFP4 only for DeepSeek-V3-family fp4 checkpoints."""
135
- if hf_config is None:
136
- return
137
-
138
- if (
139
- getattr(hf_config, "model_type", None)
140
- not in _DEEPSEEK_V3_FAMILY_MODEL_TYPES
141
- ):
142
- return
143
-
144
- quant_config = getattr(hf_config, "quantization_config", None)
145
- if isinstance(quant_config, dict):
146
- quant_dtype = (
147
- quant_config.get("global_quant_config", {})
148
- .get("weight", {})
149
- .get("dtype")
150
- )
151
- if quant_dtype == "fp4":
152
- self.dynamic_mxfp4_quant = True
117
118
def get_linear_method(self) -> "QuarkLinearMethod":
119
return QuarkLinearMethod(self)
@@ -201,17 +165,6 @@ class QuarkConfig(QuantizationConfig):
165
self.get_quant_method_target(prefix, type(layer))
166
)
167
204
- exclude_layers = cast(list[str], self.quant_config.get("exclude"))
205
- is_ignored = should_ignore_layer(
206
- prefix,
207
- ignore=exclude_layers,
208
- fused_mapping=self.packed_modules_mapping,
209
- check_children=isinstance(layer, RoutedExperts),
210
- )
211
- dynamic_mxfp4_quant = (
212
- is_ignored and "self_attn" in prefix and self.dynamic_mxfp4_quant
213
- )
214
-
168
if method_cls is UnquantizedFusedMoEMethod:
169
return UnquantizedFusedMoEMethod(layer.moe_config)
170
if method_cls is UnquantizedLinearMethod:
@@ -222,7 +175,6 @@ class QuarkConfig(QuantizationConfig):
175
scheme_cls,
176
weight_quant_key=weight_quant_key,
177
activation_quant_key=activation_quant_key,
225
- dynamic_mxfp4_quant=dynamic_mxfp4_quant,
178
weight_config=self._find_matched_config(prefix, type(layer)).get(
179
"weight"
180
),
@@ -274,11 +226,6 @@ class QuarkConfig(QuantizationConfig):
226
if is_routed_experts:
227
return None, None, UnquantizedFusedMoEMethod
228
if issubclass(layer_type, LinearBase):
277
- if "self_attn" in prefix and self.dynamic_mxfp4_quant:
278
- weight_quant_key, activation_key, _ = self.get_scheme_cls(
279
- layer_type, prefix
280
- )
281
- return weight_quant_key, activation_key, QuarkLinearMethod
229
return None, None, UnquantizedLinearMethod
230
return None, None, None
231
if issubclass(layer_type, LinearBase):
@@ -956,7 +903,6 @@ class QuarkConfig(QuantizationConfig):
903
scheme_cls: type["QuarkScheme"],
904
weight_quant_key: QuantKey | None,
905
activation_quant_key: QuantKey | None,
959
- dynamic_mxfp4_quant: bool = False,
906
weight_config: dict[str, Any] | None = None,
907
) -> "QuarkScheme":
908
"""Construct a Quark scheme selected by get_scheme_cls."""
@@ -996,9 +942,6 @@ class QuarkConfig(QuantizationConfig):
942
):
943
kwargs["weight_quant_key"] = weight_quant_key
944
999
- if scheme_cls is QuarkOCP_MX:
1000
- kwargs["dynamic_mxfp4_quant"] = dynamic_mxfp4_quant
1001
-
945
scheme = scheme_cls(**kwargs)
946
947
# Raise error if device does not support the scheme
vllm/model_executor/layers/quantization/quark/schemes/quark_ocp_mx.py
+31
-61
@@ -26,10 +26,8 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import (
26
)
27
from vllm.model_executor.parameter import (
28
GroupQuantScaleParameter,
29
- ModelWeightParameter,
29
PackedvLLMParameter,
30
)
32
-from vllm.model_executor.utils import set_weight_attrs
31
from vllm.platforms import current_platform
32
33
from .quark_scheme import QuarkScheme
@@ -46,10 +44,9 @@ class QuarkOCP_MX(QuarkScheme):
44
self,
45
weight_quant_key: QuantKey,
46
activation_quant_key: QuantKey | None,
49
- dynamic_mxfp4_quant: bool = False,
47
):
48
super().__init__(weight_quant_key, activation_quant_key)
52
- self.dynamic_mxfp4_quant = dynamic_mxfp4_quant
49
+
50
self.weight_dtype = next(
51
dtype
52
for dtype, quant_key in _WEIGHT_QUANT_KEY_MAP.items()
@@ -109,21 +106,9 @@ class QuarkOCP_MX(QuarkScheme):
106
def get_min_capability(cls) -> int:
107
return 70
108
112
- def process_dynamic_mxfp4_weights_after_loading(
113
- self, layer: torch.nn.Module
114
- ) -> None:
115
- from aiter.ops.triton.quant import dynamic_mxfp4_quant
116
-
117
- w_q, w_s = dynamic_mxfp4_quant(layer.weight)
118
- layer.weight_scale = torch.nn.Parameter(w_s, requires_grad=False)
119
- layer.weight = torch.nn.Parameter(w_q, requires_grad=False)
120
-
109
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
110
layer.weight = torch.nn.Parameter(layer.weight.data, requires_grad=False)
111
124
- if self.dynamic_mxfp4_quant:
125
- self.process_dynamic_mxfp4_weights_after_loading(layer)
126
-
112
self.ocp_mx_linear.process_weights_after_loading(layer)
113
114
def create_weights(
@@ -145,51 +130,36 @@ class QuarkOCP_MX(QuarkScheme):
130
"tensor-parallel sharding for this layer."
131
)
132
148
- if self.dynamic_mxfp4_quant:
149
- weight = ModelWeightParameter(
150
- data=torch.empty(
151
- sum(output_partition_sizes),
152
- input_size_per_partition,
153
- dtype=params_dtype,
154
- ),
155
- input_dim=1,
156
- output_dim=0,
157
- weight_loader=weight_loader,
158
- )
159
-
160
- layer.register_parameter("weight", weight)
161
- set_weight_attrs(weight, kwargs)
162
- else:
163
- output_size_per_partition = sum(output_partition_sizes)
164
- layer.logical_widths = output_partition_sizes
165
-
166
- # WEIGHT
167
- weight = PackedvLLMParameter(
168
- data=torch.empty(
169
- output_size_per_partition,
170
- self.get_packed_dim(input_size_per_partition, self.weight_dtype),
171
- dtype=torch.uint8,
172
- ),
173
- input_dim=1,
174
- output_dim=0,
175
- packed_dim=1,
176
- packed_factor=self.packed_factor,
177
- weight_loader=weight_loader,
178
- )
179
- layer.register_parameter("weight", weight)
180
-
181
- # WEIGHT SCALE
182
- weight_scale = GroupQuantScaleParameter(
183
- data=torch.empty(
184
- output_size_per_partition,
185
- input_size_per_partition // OCP_MX_BLOCK_SIZE,
186
- dtype=torch.uint8,
187
- ),
188
- input_dim=1,
189
- output_dim=0,
190
- weight_loader=weight_loader,
191
- )
192
- layer.register_parameter("weight_scale", weight_scale)
133
+ output_size_per_partition = sum(output_partition_sizes)
134
+ layer.logical_widths = output_partition_sizes
135
+
136
+ # WEIGHT
137
+ weight = PackedvLLMParameter(
138
+ data=torch.empty(
139
+ output_size_per_partition,
140
+ self.get_packed_dim(input_size_per_partition, self.weight_dtype),
141
+ dtype=torch.uint8,
142
+ ),
143
+ input_dim=1,
144
+ output_dim=0,
145
+ packed_dim=1,
146
+ packed_factor=self.packed_factor,
147
+ weight_loader=weight_loader,
148
+ )
149
+ layer.register_parameter("weight", weight)
150
+
151
+ # WEIGHT SCALE
152
+ weight_scale = GroupQuantScaleParameter(
153
+ data=torch.empty(
154
+ output_size_per_partition,
155
+ input_size_per_partition // OCP_MX_BLOCK_SIZE,
156
+ dtype=torch.uint8,
157
+ ),
158
+ input_dim=1,
159
+ output_dim=0,
160
+ weight_loader=weight_loader,
161
+ )
162
+ layer.register_parameter("weight_scale", weight_scale)
163
164
if self.weight_quant_key == kMxfp4Static:
165
self.ocp_mx_linear = init_mxfp4_linear_kernel(