A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 4, 21:15 UTC
[Bugfix] LoRA: extend expert base_layer loading to Qwen3.5 and Step3.x (#37114)
Signed-off-by: Hollow Man <hollowman@opensuse.org>
ℍ𝕠𝕝𝕝𝕠𝕨 𝕄𝕒𝕟 committed
Apr 21, 2026 at 17:17 UTC
908a713488db21470da3a52fcf65d2dea854271a
6 files changed
+34
-16
vllm/model_executor/models/qwen3_5.py
+5
-2
@@ -306,9 +306,12 @@ class Qwen3_5Model(Qwen3NextModel):
306
loaded_params: set[str] = set()
307
expert_params_mapping = self.get_expert_mapping()
308
is_fused_expert = False
309
+ base_layer = (
310
+ "base_layer." if any(".base_layer." in name for name in params_dict) else ""
311
+ )
312
fused_expert_params_mapping = [
310
- ("experts.w13_weight", "experts.gate_up_proj", 0, "w1"),
311
- ("experts.w2_weight", "experts.down_proj", 0, "w2"),
313
+ (f"experts.{base_layer}w13_weight", "experts.gate_up_proj", 0, "w1"),
314
+ (f"experts.{base_layer}w2_weight", "experts.down_proj", 0, "w2"),
315
]
316
num_experts = (
317
self.config.num_experts if hasattr(self.config, "num_experts") else 0
vllm/model_executor/models/qwen3_5_mtp.py
+5
-2
@@ -207,9 +207,12 @@ class Qwen3_5MultiTokenPredictor(nn.Module):
207
params_dict = dict(self.named_parameters())
208
loaded_params: set[str] = set()
209
is_fused_expert = False
210
+ base_layer = (
211
+ "base_layer." if any(".base_layer." in name for name in params_dict) else ""
212
+ )
213
fused_expert_params_mapping = [
211
- ("experts.w13_weight", "experts.gate_up_proj", 0, "w1"),
212
- ("experts.w2_weight", "experts.down_proj", 0, "w2"),
214
+ (f"experts.{base_layer}w13_weight", "experts.gate_up_proj", 0, "w1"),
215
+ (f"experts.{base_layer}w2_weight", "experts.down_proj", 0, "w2"),
216
]
217
num_experts = (
218
self.config.num_experts if hasattr(self.config, "num_experts") else 0
vllm/model_executor/models/qwen3_vl_moe.py
+5
-2
@@ -183,9 +183,12 @@ class Qwen3MoeLLMModel(Qwen3MoeModel):
183
loaded_params: set[str] = set()
184
expert_params_mapping = self.get_expert_mapping()
185
is_fused_expert = False
186
+ base_layer = (
187
+ "base_layer." if any(".base_layer." in name for name in params_dict) else ""
188
+ )
189
fused_expert_params_mapping = [
187
- ("experts.w13_weight", "experts.gate_up_proj", 0, "w1"),
188
- ("experts.w2_weight", "experts.down_proj", 0, "w2"),
190
+ (f"experts.{base_layer}w13_weight", "experts.gate_up_proj", 0, "w1"),
191
+ (f"experts.{base_layer}w2_weight", "experts.down_proj", 0, "w2"),
192
]
193
num_experts = self.config.num_experts
194
for name, loaded_weight in weights:
vllm/model_executor/models/step3_text.py
+6
-3
@@ -463,11 +463,14 @@ class Step3TextForCausalLM(nn.Module, SupportsPP):
463
]
464
params_dict = dict(self.named_parameters())
465
loaded_params: set[str] = set()
466
+ base_layer = (
467
+ "base_layer." if any(".base_layer." in name for name in params_dict) else ""
468
+ )
469
470
expert_params_mapping = [
468
- (".moe.experts.w13_weight", ".moe.gate_proj.weight", "w1"),
469
- (".moe.experts.w13_weight", ".moe.up_proj.weight", "w3"),
470
- (".moe.experts.w2_weight", ".moe.down_proj.weight", "w2"),
471
+ (f".moe.experts.{base_layer}w13_weight", ".moe.gate_proj.weight", "w1"),
472
+ (f".moe.experts.{base_layer}w13_weight", ".moe.up_proj.weight", "w3"),
473
+ (f".moe.experts.{base_layer}w2_weight", ".moe.down_proj.weight", "w2"),
474
]
475
476
disable_moe_stacked_params = [data[1] for data in expert_params_mapping]
vllm/model_executor/models/step3p5.py
+6
-3
@@ -626,12 +626,15 @@ class Step3p5Model(nn.Module):
626
627
params_dict = dict(self.named_parameters())
628
loaded_params: set[str] = set()
629
+ base_layer = (
630
+ "base_layer." if any(".base_layer." in name for name in params_dict) else ""
631
+ )
632
633
# Old packed 3D format: .moe.gate_proj.weight [num_experts, out, in]
634
expert_params_mapping = [
632
- (".moe.experts.w13_weight", ".moe.gate_proj.weight", "w1"),
633
- (".moe.experts.w13_weight", ".moe.up_proj.weight", "w3"),
634
- (".moe.experts.w2_weight", ".moe.down_proj.weight", "w2"),
635
+ (f".moe.experts.{base_layer}w13_weight", ".moe.gate_proj.weight", "w1"),
636
+ (f".moe.experts.{base_layer}w13_weight", ".moe.up_proj.weight", "w3"),
637
+ (f".moe.experts.{base_layer}w2_weight", ".moe.down_proj.weight", "w2"),
638
]
639
640
# New per-expert format: .moe.experts.E.gate_proj.weight_packed [out, in]
vllm/model_executor/models/step3p5_mtp.py
+7
-4
@@ -181,14 +181,17 @@ class Step3p5MTP(nn.Module):
181
("gate_up_proj", "gate_proj", 0),
182
("gate_up_proj", "up_proj", 1),
183
]
184
+ params_dict = dict(self.named_parameters())
185
+ base_layer = (
186
+ "base_layer." if any(".base_layer." in name for name in params_dict) else ""
187
+ )
188
189
expert_params_mapping = [
186
- (".moe.experts.w13_weight", ".moe.gate_proj.weight", "w1"),
187
- (".moe.experts.w13_weight", ".moe.up_proj.weight", "w3"),
188
- (".moe.experts.w2_weight", ".moe.down_proj.weight", "w2"),
190
+ (f".moe.experts.{base_layer}w13_weight", ".moe.gate_proj.weight", "w1"),
191
+ (f".moe.experts.{base_layer}w13_weight", ".moe.up_proj.weight", "w3"),
192
+ (f".moe.experts.{base_layer}w2_weight", ".moe.down_proj.weight", "w2"),
193
]
194
191
- params_dict = dict(self.named_parameters())
195
loaded_params: set[str] = set()
196
for name, loaded_weight in weights:
197
if "rotary_emb.inv_freq" in name: