A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[BUGFIX] GPTQ quantization compatibility for Qwen3 MOE models (AutoGPTQ and AutoRound-GPTQ) (#23994)
Signed-off-by: JartX <sagformas@epdcenter.es> Signed-off-by: Isotr0py <mozf@mail2.sysu.edu.cn> Co-authored-by: Isotr0py <mozf@mail2.sysu.edu.cn>
JartX committed
Sep 1, 2025 at 05:33 UTC
183a70967a90ef06e614ad020ead7d27e87b7688
3 files changed
+17
-4
vllm/model_executor/layers/quantization/gptq.py
+7
-1
@@ -37,6 +37,7 @@ class GPTQConfig(QuantizationConfig):
37
desc_act: bool,
38
lm_head_quantized: bool,
39
dynamic: dict[str, dict[str, Union[int, bool]]],
40
+ autoround_version: str = "",
41
) -> None:
42
# GPTQModel use `dynamic` config property to allow per module
43
# quantization config so each module can be individually optimized.
@@ -74,6 +75,9 @@ class GPTQConfig(QuantizationConfig):
75
"Currently, only 2/3/4/8-bit weight quantization is "
76
f"supported for GPTQ, but got {self.weight_bits} bits.")
77
78
+ # used to identify GPTQ model quantized by autoround
79
+ self.autoround_version = autoround_version
80
+
81
def __repr__(self) -> str:
82
return (f"GPTQConfig(weight_bits={self.weight_bits}, "
83
f"group_size={self.group_size}, "
@@ -108,8 +112,10 @@ class GPTQConfig(QuantizationConfig):
112
desc_act = cls.get_from_keys(config, ["desc_act"])
113
lm_head_quantized = cls.get_from_keys_or(config, ["lm_head"],
114
default=False)
115
+ autoround_version = cls.get_from_keys_or(config, ["autoround_version"],
116
+ default="")
117
return cls(weight_bits, group_size, desc_act, lm_head_quantized,
112
- dynamic)
118
+ dynamic, autoround_version)
119
120
def get_quant_method(
121
self, layer: torch.nn.Module, prefix: str
vllm/model_executor/layers/quantization/gptq_marlin.py
+3
@@ -119,6 +119,9 @@ class GPTQMarlinConfig(QuantizationConfig):
119
120
self.quant_type = self.TYPE_MAP[(weight_bits, is_sym)]
121
122
+ # used to identify GPTQ model quantized by autoround
123
+ self.autoround_version = full_config.get("autoround_version", "")
124
+
125
def __repr__(self) -> str:
126
return (f"GPTQMarlinConfig(quant_type={self.quant_type}, "
127
f"group_size={self.group_size}, "
vllm/model_executor/models/qwen3_moe.py
+7
-3
@@ -159,9 +159,13 @@ class Qwen3MoeSparseMoeBlock(nn.Module):
159
160
def _maybe_ignore_quant_config(self, quant_config: QuantizationConfig):
161
# GPTQ configs do not have a list of ignored modules, however AutoGPTQ
162
- # seems to avoid gate quantization.
163
- # See: https://huggingface.co/Qwen/Qwen3-30B-A3B-GPTQ-Int4
164
- if isinstance(quant_config, (GPTQConfig, GPTQMarlinConfig)):
162
+ # seems to avoid gate quantization while AutoRound does.
163
+ # See: https://huggingface.co/Qwen/Qwen3-30B-A3B-GPTQ-Int4,
164
+ # and https://huggingface.co/jart25/Qwen3-Coder-30B-A3B-Instruct-Int4-gptq
165
+ if isinstance(
166
+ quant_config,
167
+ (GPTQConfig,
168
+ GPTQMarlinConfig)) and not quant_config.autoround_version:
169
return None
170
return quant_config
171