A high-throughput and memory-efficient inference and serving engine for LLMs
[Bugfix] fix qwen3.5 ep weight loading (#45002)
Signed-off-by: zjy0516 <riverclouds.zhu@qq.com>
Jiangyun Zhu committed
Jun 10, 2026 at 04:55 UTC
ca4cfd873163cc1911f98385e776fba6e7300f9f
3 files changed
+32
-14
tests/evals/gsm8k/configs/Qwen3.5-35B-A3B-DEP2.yaml
+1
@@ -7,3 +7,4 @@ server_args: >-
7
--max-model-len 4096
8
--data-parallel-size 2
9
--enable-expert-parallel
10
+ --no-enable-flashinfer-autotune
\ No newline at end of file
vllm/model_executor/models/qwen3_5.py
+17
-7
@@ -36,6 +36,9 @@ from vllm.distributed import (
36
get_pp_group,
37
)
38
from vllm.logger import init_logger
39
+from vllm.model_executor.layers.fused_moe import (
40
+ fused_moe_make_expert_params_mapping,
41
+)
42
from vllm.model_executor.layers.layernorm import (
43
GemmaRMSNorm as Qwen3_5RMSNorm,
44
)
@@ -294,13 +297,20 @@ class Qwen3_5Model(Qwen3NextModel):
297
loaded_params: set[str] = set()
298
expert_params_mapping = self.get_expert_mapping()
299
is_fused_expert = False
297
- base_layer = (
298
- "base_layer." if any(".base_layer." in name for name in params_dict) else ""
299
- )
300
- fused_expert_params_mapping = [
301
- (f"experts.{base_layer}w13_weight", "experts.gate_up_proj", 0, "w1"),
302
- (f"experts.{base_layer}w2_weight", "experts.down_proj", 0, "w2"),
303
- ]
300
+ fused_expert_params_mapping: list[tuple[str, str, int, str]] = []
301
+ for param_name, ckpt_name, _, shard_id in fused_moe_make_expert_params_mapping(
302
+ self,
303
+ ckpt_gate_proj_name="gate_up_proj",
304
+ ckpt_down_proj_name="down_proj",
305
+ ckpt_up_proj_name="gate_up_proj",
306
+ num_experts=1,
307
+ ):
308
+ if shard_id == "w3":
309
+ continue
310
+ parts = ckpt_name.split(".")
311
+ fused_expert_params_mapping.append(
312
+ (f"{param_name}weight", f"{parts[0]}.{parts[2]}", 0, shard_id)
313
+ )
314
num_experts = (
315
self.config.num_experts if hasattr(self.config, "num_experts") else 0
316
)
vllm/model_executor/models/qwen3_5_mtp.py
+14
-7
@@ -209,13 +209,20 @@ class Qwen3_5MultiTokenPredictor(nn.Module):
209
params_dict = dict(self.named_parameters())
210
loaded_params: set[str] = set()
211
is_fused_expert = False
212
- base_layer = (
213
- "base_layer." if any(".base_layer." in name for name in params_dict) else ""
214
- )
215
- fused_expert_params_mapping = [
216
- (f"experts.{base_layer}w13_weight", "experts.gate_up_proj", 0, "w1"),
217
- (f"experts.{base_layer}w2_weight", "experts.down_proj", 0, "w2"),
218
- ]
212
+ fused_expert_params_mapping: list[tuple[str, str, int, str]] = []
213
+ for param_name, ckpt_name, _, shard_id in fused_moe_make_expert_params_mapping(
214
+ self,
215
+ ckpt_gate_proj_name="gate_up_proj",
216
+ ckpt_down_proj_name="down_proj",
217
+ ckpt_up_proj_name="gate_up_proj",
218
+ num_experts=1,
219
+ ):
220
+ if shard_id == "w3":
221
+ continue
222
+ parts = ckpt_name.split(".")
223
+ fused_expert_params_mapping.append(
224
+ (f"{param_name}weight", f"{parts[0]}.{parts[2]}", 0, shard_id)
225
+ )
226
num_experts = (
227
self.config.num_experts if hasattr(self.config, "num_experts") else 0
228
)