@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 63babd17f1

Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 5, 21:00 UTC

[Model][Quantization] Add GGUF support for MiniMax-M2.1 (#36965)

Signed-off-by: kangletian <Letian.Kang@amd.com>

PikaPikachu committed Mar 30, 2026 at 14:24 UTC 63babd17f1b110e267e1ad801a9b9d4ccf5bbe7d

5 files changed +137 -10

vllm/config/model.py

@@ -948,6 +948,7 @@ class ModelConfig:

948 # imports during override detection (e.g., MXFP4 imports Triton)

949 "mxfp4",

950 "cpu_awq",

951 + "gguf",

952 ]

953 quantization_methods = [

954 q for q in supported_quantization if q not in overrides

vllm/model_executor/layers/quantization/gguf.py

+14 -1

@@ -3,7 +3,10 @@

4 from collections.abc import Mapping

5 from types import MappingProxyType

6 -from typing import Any

6 +from typing import TYPE_CHECKING, Any

7 +

8 +if TYPE_CHECKING:

9 + from vllm.model_executor.layers.quantization import QuantizationMethods

11 import gguf

12 import torch

@@ -79,6 +82,16 @@ class GGUFConfig(QuantizationConfig):

82 def from_config(cls, config: dict[str, Any]) -> "GGUFConfig":

83 return cls()

85 + @classmethod

86 + def override_quantization_method(

87 + cls, hf_quant_cfg: dict[str, Any], user_quant: str | None

88 + ) -> "QuantizationMethods | None":

89 + # When user explicitly specifies --quantization gguf, override

90 + # whatever quantization method is in the HF model config (e.g. fp8).

91 + if user_quant == "gguf":

92 + return "gguf"

93 + return None

94 +

95 def get_quant_method(

96 self, layer: torch.nn.Module, prefix: str

97 ) -> "QuantizeMethodBase | None":

vllm/model_executor/model_loader/gguf_loader.py

+74 -7

@@ -24,6 +24,7 @@ from vllm.model_executor.model_loader.weight_utils import (

24 get_gguf_extra_tensor_names,

25 get_gguf_weight_type_map,

26 gguf_quant_weights_iterator,

27 + gguf_quant_weights_iterator_multi,

28 )

29 from vllm.transformers_utils.gguf_utils import detect_gguf_multimodal

30 from vllm.utils.torch_utils import set_default_torch_dtype

@@ -74,6 +75,31 @@ class GGUFModelLoader(BaseModelLoader):

75 "or <repo_id>:<quant_type>)"

76 )

78 + @staticmethod

79 + def _get_all_gguf_files(model_path: str) -> list[str]:

80 + """Discover all GGUF shard files from a single shard path.

81 +

82 + Supports variable-width shard indices by dynamically detecting

83 + the padding from the original filename.

84 + E.g. ``*-00001-of-00005.gguf`` → all 5 shards,

85 + ``*-01-of-15.gguf`` → all 15 shards.

86 + """

87 + match = re.search(r"-(\d+)-of-(\d+)\.gguf$", model_path)

88 + if not match:

89 + return [model_path]

90 + total = int(match.group(2))

91 + num_digits = len(match.group(1))

92 + prefix = model_path[: match.start(1)]

93 + suffix = model_path[match.end(2) :]

94 + files = []

95 + for i in range(1, total + 1):

96 + shard_path = f"{prefix}{i:0{num_digits}d}-of-{total:0{num_digits}d}{suffix}"

97 + if os.path.isfile(shard_path):

98 + files.append(shard_path)

99 + if files:

100 + logger.info("Discovered %d GGUF shard files", len(files))

101 + return files if files else [model_path]

102 +

103 def _get_gguf_weights_map(self, model_config: ModelConfig):

104 """

105 GGUF uses this naming convention for their tensors from HF checkpoint:

@@ -145,6 +171,29 @@ class GGUFModelLoader(BaseModelLoader):

171 r"\.mlp\.experts\.[0-9]+\.(gate|up|down)_proj\.weight"

172 )

173 )

174 + if model_type == "minimax_m2":

175 + model_type = "minimax-m2"

176 + # GGUF layer map assumes merged expert weights

177 + # map them manually like deepseek2

178 + for idx in range(config.num_hidden_layers):

179 + gguf_to_hf_name_map[f"blk.{idx}.exp_probs_b.bias"] = (

180 + f"model.layers.{idx}.block_sparse_moe.e_score_correction_bias"

181 + )

182 + gguf_to_hf_name_map[f"blk.{idx}.ffn_down_exps.weight"] = (

183 + f"model.layers.{idx}.block_sparse_moe.experts.0.w2.weight"

184 + )

185 + gguf_to_hf_name_map[f"blk.{idx}.ffn_gate_exps.weight"] = (

186 + f"model.layers.{idx}.block_sparse_moe.experts.0.w1.weight"

187 + )

188 + gguf_to_hf_name_map[f"blk.{idx}.ffn_up_exps.weight"] = (

189 + f"model.layers.{idx}.block_sparse_moe.experts.0.w3.weight"

190 + )

191 + sideload_params.append(

192 + re.compile(

193 + f"model\\.layers\\.{idx}"

194 + r"\.block_sparse_moe\.experts\.(gate_up_proj|down_proj)"

195 + )

196 + )

197

198 arch = None

199 for key, value in gguf.MODEL_ARCH_NAMES.items():

@@ -190,6 +239,13 @@ class GGUFModelLoader(BaseModelLoader):

239 revert_hf_rename(name): tensor for name, tensor in state_dict.items()

240 }

241

242 + if model_type == "minimax-m2" and not hf_checkpoint_map:

243 + # Reverse HF convention: mlp -> block_sparse_moe

244 + state_dict = {

245 + name.replace(".mlp.", ".block_sparse_moe."): tensor

246 + for name, tensor in state_dict.items()

247 + }

248 +

249 def find_hf_name_in_tensor_map(hf_name: str) -> str | None:

250 """

251 Map HuggingFace parameter name to GGUF tensor name.

@@ -277,9 +333,10 @@ class GGUFModelLoader(BaseModelLoader):

333 model_name_or_path: str,

334 gguf_to_hf_name_map: dict[str, str],

335 ) -> dict[str, str]:

280 - weight_type_map = get_gguf_weight_type_map(

281 - model_name_or_path, gguf_to_hf_name_map

282 - )

336 + gguf_files = self._get_all_gguf_files(model_name_or_path)

337 + weight_type_map = {}

338 + for f in gguf_files:

339 + weight_type_map.update(get_gguf_weight_type_map(f, gguf_to_hf_name_map))

340 is_multimodal = hasattr(model_config.hf_config, "vision_config")

341 if is_multimodal:

342 mmproj_file = detect_gguf_multimodal(model_name_or_path)

@@ -321,7 +378,15 @@ class GGUFModelLoader(BaseModelLoader):

378 )

379 yield from gguf_quant_weights_iterator(mmproj_file, gguf_to_hf_name_map)

380

324 - yield from gguf_quant_weights_iterator(model_name_or_path, gguf_to_hf_name_map)

381 + gguf_files = self._get_all_gguf_files(model_name_or_path)

382 + if len(gguf_files) > 1:

383 + yield from gguf_quant_weights_iterator_multi(

384 + gguf_files, gguf_to_hf_name_map

385 + )

386 + else:

387 + yield from gguf_quant_weights_iterator(

388 + model_name_or_path, gguf_to_hf_name_map

389 + )

390

391 def download_model(self, model_config: ModelConfig) -> None:

392 self._prepare_weights(model_config)

@@ -340,9 +405,11 @@ class GGUFModelLoader(BaseModelLoader):

405 local_model_path = self._prepare_weights(model_config)

406 gguf_weights_map = self._get_gguf_weights_map(model_config)

407 # we can only know if tie word embeddings after mapping weights

343 - if "lm_head.weight" in get_gguf_extra_tensor_names(

344 - local_model_path, gguf_weights_map

345 - ):

408 + gguf_files = self._get_all_gguf_files(local_model_path)

409 + all_extra_names = []

410 + for f in gguf_files:

411 + all_extra_names.extend(get_gguf_extra_tensor_names(f, gguf_weights_map))

412 + if "lm_head.weight" in all_extra_names:

413 model_config.hf_config.update({"tie_word_embeddings": True})

414

415 weight_type_map = self._get_gguf_weight_type(

vllm/model_executor/model_loader/weight_utils.py

+43

@@ -1222,6 +1222,49 @@ def gguf_quant_weights_iterator(

1222 yield name, param

1223

1224

1225 +def gguf_quant_weights_iterator_multi(

1226 + gguf_files: list[str], gguf_to_hf_name_map: dict[str, str]

1227 +) -> Generator[tuple[str, torch.Tensor], None, None]:

1228 + """

1229 + Iterate over the quant weights across multiple GGUF shard files

1230 + and convert them to torch tensors.

1231 +

1232 + Like gguf_quant_weights_iterator, we yield all weight types first

1233 + before yielding any weights data to avoid issues with packed layers

1234 + that have different quant types.

1235 + """

1236 + readers = [gguf.GGUFReader(f) for f in gguf_files]

1237 +

1238 + # First pass: yield all weight types across all shards

1239 + for reader in readers:

1240 + for tensor in reader.tensors:

1241 + if tensor.name in gguf_to_hf_name_map:

1242 + weight_type = tensor.tensor_type

1243 + name = gguf_to_hf_name_map[tensor.name]

1244 + if weight_type.name not in ("F32", "BF16", "F16"):

1245 + weight_type_name = name.replace("weight", "qweight_type")

1246 + weight_type = torch.tensor(weight_type)

1247 + yield weight_type_name, weight_type

1248 +

1249 + # Second pass: yield all weight data across all shards

1250 + for reader in readers:

1251 + for tensor in reader.tensors:

1252 + if tensor.name in gguf_to_hf_name_map:

1253 + weight = tensor.data

1254 + weight_type = tensor.tensor_type

1255 + name = gguf_to_hf_name_map[tensor.name]

1256 + if weight_type.name not in ("F32", "BF16", "F16"):

1257 + name = name.replace("weight", "qweight")

1258 + if weight_type.name == "BF16" and tensor.data.dtype == np.uint8:

1259 + weight = weight.view(np.uint16)

1260 + if reader.byte_order == "S":

1261 + weight = weight.byteswap()

1262 + param = torch.tensor(weight).view(torch.bfloat16)

1263 + else:

1264 + param = torch.tensor(weight)

1265 + yield name, param

1266 +

1267 +

1268 def convert_pyslice_to_tensor(x: Any) -> torch.Tensor:

1269 """convert PySafeSlice object from safetensors to torch.Tensor

1270

vllm/model_executor/models/minimax_m2.py

+5 -2

@@ -331,7 +331,7 @@ class MiniMaxM2Model(nn.Module):

331 self.embed_tokens = VocabParallelEmbedding(

332 config.vocab_size,

333 config.hidden_size,

334 - quant_config=None,

334 + quant_config=quant_config,

335 prefix=f"{prefix}.embed_tokens",

336 )

337 else:

@@ -518,7 +518,10 @@ class MiniMaxM2ForCausalLM(nn.Module, SupportsLoRA, SupportsPP):

518 )

519 if get_pp_group().is_last_rank:

520 self.lm_head = ParallelLMHead(

521 - config.vocab_size, config.hidden_size, quant_config=None

521 + config.vocab_size,

522 + config.hidden_size,

523 + quant_config=quant_config,

524 + prefix=maybe_prefix(prefix, "lm_head"),

525 )

526 else:

527 self.lm_head = PPMissingLayer()