A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[Bugfix] Fix Qwen3 MoE GPTQ inference (#23490)
Signed-off-by: Isotr0py <mozf@mail2.sysu.edu.cn>
Isotr0py committed
Aug 25, 2025 at 21:40 UTC
a9082a4d144e516d0ee00bacfa0dca9609b6b2c3
1 file changed
+18
-6
vllm/model_executor/models/qwen3_moe.py
+18
-6
@@ -45,6 +45,9 @@ from vllm.model_executor.layers.linear import (MergedColumnParallelLinear,
45
RowParallelLinear)
46
from vllm.model_executor.layers.logits_processor import LogitsProcessor
47
from vllm.model_executor.layers.quantization import QuantizationConfig
48
+from vllm.model_executor.layers.quantization.gptq import GPTQConfig
49
+from vllm.model_executor.layers.quantization.gptq_marlin import (
50
+ GPTQMarlinConfig)
51
from vllm.model_executor.layers.rotary_embedding import get_rope
52
from vllm.model_executor.layers.vocab_parallel_embedding import (
53
ParallelLMHead, VocabParallelEmbedding)
@@ -146,11 +149,20 @@ class Qwen3MoeSparseMoeBlock(nn.Module):
149
enable_eplb=self.enable_eplb,
150
num_redundant_experts=self.n_redundant_experts)
151
149
- self.gate = ReplicatedLinear(config.hidden_size,
150
- config.num_experts,
151
- bias=False,
152
- quant_config=quant_config,
153
- prefix=f"{prefix}.gate")
152
+ self.gate = ReplicatedLinear(
153
+ config.hidden_size,
154
+ config.num_experts,
155
+ bias=False,
156
+ quant_config=self._maybe_ignore_quant_config(quant_config),
157
+ prefix=f"{prefix}.gate")
158
+
159
+ def _maybe_ignore_quant_config(self, quant_config: QuantizationConfig):
160
+ # GPTQ configs do not have a list of ignored modules, however AutoGPTQ
161
+ # seems to avoid gate quantization.
162
+ # See: https://huggingface.co/Qwen/Qwen3-30B-A3B-GPTQ-Int4
163
+ if isinstance(quant_config, (GPTQConfig, GPTQMarlinConfig)):
164
+ return None
165
+ return quant_config
166
167
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
168
# NOTE: hidden_states can have either 1D or 2D shape.
@@ -682,4 +694,4 @@ class Qwen3MoeForCausalLM(nn.Module, SupportsPP, SupportsLoRA,
694
return loader.load_weights(weights)
695
696
def get_expert_mapping(self) -> list[tuple[str, str, int, str]]:
685
- return self.model.get_expert_mapping()
\ No newline at end of file
697
+ return self.model.get_expert_mapping()