A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
Fix GGUF loader for Qwen3 MoE. (#22785)
Signed-off-by: Gh0u1L5 <Gh0u1L5@outlook.com>
Gh0u1L5 committed
Aug 13, 2025 at 21:08 UTC
b159c0a67aaafe865c785d289335c3760e01a62f
2 files changed
+12
vllm/model_executor/model_loader/gguf_loader.py
+11
@@ -74,6 +74,17 @@ class GGUFModelLoader(BaseModelLoader):
74
f"model.layers.{idx}.mlp.experts.0.gate_proj.weight"
75
gguf_to_hf_name_map[f"blk.{idx}.ffn_up_exps.weight"] = \
76
f"model.layers.{idx}.mlp.experts.0.up_proj.weight"
77
+ if model_type in ("qwen2_moe", "qwen3_moe"):
78
+ model_type = model_type.replace("_", "")
79
+ # GGUF layer map assumes that we will have a merged expert weights
80
+ # so we need to map them manually
81
+ for idx in range(config.num_hidden_layers):
82
+ gguf_to_hf_name_map[f"blk.{idx}.ffn_down_exps.weight"] = \
83
+ f"model.layers.{idx}.mlp.experts.0.down_proj.weight"
84
+ gguf_to_hf_name_map[f"blk.{idx}.ffn_gate_exps.weight"] = \
85
+ f"model.layers.{idx}.mlp.experts.0.gate_proj.weight"
86
+ gguf_to_hf_name_map[f"blk.{idx}.ffn_up_exps.weight"] = \
87
+ f"model.layers.{idx}.mlp.experts.0.up_proj.weight"
88
89
arch = None
90
for key, value in gguf.MODEL_ARCH_NAMES.items():
vllm/model_executor/models/qwen3_moe.py
+1
@@ -375,6 +375,7 @@ class Qwen3MoeModel(nn.Module):
375
self.embed_tokens = VocabParallelEmbedding(
376
config.vocab_size,
377
config.hidden_size,
378
+ quant_config=quant_config,
379
prefix=f"{prefix}.embed_tokens")
380
self.start_layer, self.end_layer, self.layers = make_layers(
381
config.num_hidden_layers,