A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 11, 17:30 UTC
[Model][Quantization] Fix / Add GGUF support for Qwen2 MoE models (#30307)
Signed-off-by: Tsukasa OI <floss_llm@irq.a4lg.com>
Tsukasa OI committed
Dec 10, 2025 at 04:13 UTC
73a484caa1ad320d6e695f098c25c479a71e6774
1 file changed
+8
vllm/model_executor/models/qwen2_moe.py
+8
@@ -367,6 +367,8 @@ class Qwen2MoeModel(nn.Module):
367
self.embed_tokens = VocabParallelEmbedding(
368
config.vocab_size,
369
config.hidden_size,
370
+ quant_config=quant_config,
371
+ prefix=f"{prefix}.embed_tokens",
372
)
373
self.start_layer, self.end_layer, self.layers = make_layers(
374
config.num_hidden_layers,
@@ -512,6 +514,12 @@ class Qwen2MoeModel(nn.Module):
514
continue
515
else:
516
name = remapped_kv_scale_name
517
+ # GGUF: make sure that shared_expert_gate is a 2D tensor.
518
+ if (
519
+ "mlp.shared_expert_gate" in name
520
+ and len(loaded_weight.shape) == 1
521
+ ):
522
+ loaded_weight = loaded_weight[None, :]
523
param = params_dict[name]
524
weight_loader = getattr(
525
param, "weight_loader", default_weight_loader