A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 11, 18:45 UTC
[Model][Quantization] Restore MoE + GGUF models support (incl. Qwen3 MoE) by allowing Sideload Parameters (#30116)
Signed-off-by: Tsukasa OI <floss_llm@irq.a4lg.com> Co-authored-by: Isotr0py <mozf@mail2.sysu.edu.cn>
Tsukasa OI committed
Dec 9, 2025 at 14:30 UTC
58d5b3f51455706bf4f1f2360a0feb83d161147e
2 files changed
+24
-1
vllm/model_executor/layers/quantization/gguf.py
+1
@@ -82,6 +82,7 @@ class GGUFConfig(QuantizationConfig):
82
return UnquantizedEmbeddingMethod()
83
return GGUFEmbeddingMethod(self)
84
elif isinstance(layer, FusedMoE):
85
+ # TODO: Select UnquantizedFusedMoEMethod on unquantized layers.
86
return GGUFMoEMethod(self, layer.moe_config)
87
return None
88
vllm/model_executor/model_loader/gguf_loader.py
+23
-1
@@ -4,6 +4,7 @@ import os
4
from collections.abc import Generator
5
6
import gguf
7
+import regex as re
8
import torch
9
import torch.nn as nn
10
from huggingface_hub import hf_hub_download
@@ -94,6 +95,7 @@ class GGUFModelLoader(BaseModelLoader):
95
hasattr(config, "vision_config") and config.vision_config is not None
96
)
97
gguf_to_hf_name_map = {}
98
+ sideload_params: list[re.Pattern] = []
99
# hack: ggufs have a different name than transformers
100
if model_type == "cohere":
101
model_type = "command-r"
@@ -118,6 +120,12 @@ class GGUFModelLoader(BaseModelLoader):
120
gguf_to_hf_name_map[f"blk.{idx}.ffn_up_exps.weight"] = (
121
f"model.layers.{idx}.mlp.experts.0.up_proj.weight"
122
)
123
+ sideload_params.append(
124
+ re.compile(
125
+ f"model\\.layers\\.{idx}"
126
+ r"\.mlp\.experts\.[0-9]+\.(gate|up|down)_proj\.weight"
127
+ )
128
+ )
129
if model_type in ("qwen2_moe", "qwen3_moe"):
130
model_type = model_type.replace("_", "")
131
# GGUF layer map assumes that we will have a merged expert weights
@@ -132,6 +140,12 @@ class GGUFModelLoader(BaseModelLoader):
140
gguf_to_hf_name_map[f"blk.{idx}.ffn_up_exps.weight"] = (
141
f"model.layers.{idx}.mlp.experts.0.up_proj.weight"
142
)
143
+ sideload_params.append(
144
+ re.compile(
145
+ f"model\\.layers\\.{idx}"
146
+ r"\.mlp\.experts\.[0-9]+\.(gate|up|down)_proj\.weight"
147
+ )
148
+ )
149
150
arch = None
151
for key, value in gguf.MODEL_ARCH_NAMES.items():
@@ -241,7 +255,15 @@ class GGUFModelLoader(BaseModelLoader):
255
# Parameter not in manual overrides either
256
unmapped_params.append(hf_name)
257
244
- # All parameters must be mapped: both vision/projector and backbone
258
+ # All parameters (except those initialized by other means) must be mapped:
259
+ # both vision/projector and backbone
260
+ if unmapped_params:
261
+ unmapped_params = list(
262
+ filter(
263
+ lambda x: not any(re.fullmatch(p, x) for p in sideload_params),
264
+ unmapped_params,
265
+ )
266
+ )
267
if unmapped_params:
268
raise RuntimeError(
269
f"Failed to map GGUF parameters "