A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[Bugfix][Qwen3-Next] add prefixes to shared_expert in qwen3-next and mlp in qwen2moe to successfully load ignored params in quantized models (#24960)
Signed-off-by: toncao <cpatonn@gmail.com> Co-authored-by: toncao <cpatonn@gmail.com> Co-authored-by: Jee Jee Li <pandaleefree@gmail.com>
toncao committed
Sep 18, 2025 at 11:08 UTC
027d37df389b00ed2e7d874113f869267533a2ab
2 files changed
+26
-23
vllm/model_executor/models/qwen2_moe.py
+25
-23
@@ -72,17 +72,20 @@ class Qwen2MoeMLP(nn.Module):
72
hidden_act: str,
73
quant_config: Optional[QuantizationConfig] = None,
74
reduce_results: bool = True,
75
+ prefix: str = "",
76
) -> None:
77
super().__init__()
78
self.gate_up_proj = MergedColumnParallelLinear(
79
hidden_size, [intermediate_size] * 2,
80
bias=False,
80
- quant_config=quant_config)
81
+ quant_config=quant_config,
82
+ prefix=f"{prefix}.gate_up_proj")
83
self.down_proj = RowParallelLinear(intermediate_size,
84
hidden_size,
85
bias=False,
86
quant_config=quant_config,
85
- reduce_results=reduce_results)
87
+ reduce_results=reduce_results,
88
+ prefix=f"{prefix}.down_proj")
89
if hidden_act != "silu":
90
raise ValueError(f"Unsupported activation: {hidden_act}. "
91
"Only silu is supported for now.")
@@ -123,7 +126,8 @@ class Qwen2MoeSparseMoeBlock(nn.Module):
126
self.gate = ReplicatedLinear(config.hidden_size,
127
config.num_experts,
128
bias=False,
126
- quant_config=None)
129
+ quant_config=None,
130
+ prefix=f"{prefix}.gate")
131
if config.shared_expert_intermediate_size > 0:
132
self.shared_expert = Qwen2MoeMLP(
133
hidden_size=config.hidden_size,
@@ -132,6 +136,7 @@ class Qwen2MoeSparseMoeBlock(nn.Module):
136
quant_config=quant_config,
137
reduce_results=self.experts.must_reduce_shared_expert_outputs(
138
),
139
+ prefix=f"{prefix}.shared_expert",
140
)
141
else:
142
self.shared_expert = None
@@ -203,21 +208,19 @@ class Qwen2MoeAttention(nn.Module):
208
self.max_position_embeddings = max_position_embeddings
209
self.dual_chunk_attention_config = dual_chunk_attention_config
210
206
- self.qkv_proj = QKVParallelLinear(
207
- hidden_size,
208
- self.head_dim,
209
- self.total_num_heads,
210
- self.total_num_kv_heads,
211
- bias=True,
212
- quant_config=quant_config,
213
- )
211
+ self.qkv_proj = QKVParallelLinear(hidden_size,
212
+ self.head_dim,
213
+ self.total_num_heads,
214
+ self.total_num_kv_heads,
215
+ bias=True,
216
+ quant_config=quant_config,
217
+ prefix=f"{prefix}.qkv_proj")
218
215
- self.o_proj = RowParallelLinear(
216
- self.total_num_heads * self.head_dim,
217
- hidden_size,
218
- bias=False,
219
- quant_config=quant_config,
220
- )
219
+ self.o_proj = RowParallelLinear(self.total_num_heads * self.head_dim,
220
+ hidden_size,
221
+ bias=False,
222
+ quant_config=quant_config,
223
+ prefix=f"{prefix}.o_proj")
224
225
self.rotary_emb = get_rope(
226
self.head_dim,
@@ -296,12 +299,11 @@ class Qwen2MoeDecoderLayer(nn.Module):
299
quant_config=quant_config,
300
prefix=f"{prefix}.mlp")
301
else:
299
- self.mlp = Qwen2MoeMLP(
300
- hidden_size=config.hidden_size,
301
- intermediate_size=config.intermediate_size,
302
- hidden_act=config.hidden_act,
303
- quant_config=quant_config,
304
- )
302
+ self.mlp = Qwen2MoeMLP(hidden_size=config.hidden_size,
303
+ intermediate_size=config.intermediate_size,
304
+ hidden_act=config.hidden_act,
305
+ quant_config=quant_config,
306
+ prefix=f"{prefix}.mlp")
307
self.input_layernorm = RMSNorm(config.hidden_size,
308
eps=config.rms_norm_eps)
309
self.post_attention_layernorm = RMSNorm(config.hidden_size,
vllm/model_executor/models/qwen3_next.py
+1
@@ -138,6 +138,7 @@ class Qwen3NextSparseMoeBlock(nn.Module):
138
quant_config=quant_config,
139
reduce_results=self.experts.must_reduce_shared_expert_outputs(
140
),
141
+ prefix=f"{prefix}.shared_expert",
142
)
143
else:
144
self.shared_expert = None