A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 11, 12:30 UTC
[Model][Quantization] Override HF defaults to GGUF ones (incl. Qwen3 MoE) (#30118)
Signed-off-by: Tsukasa OI <floss_llm@irq.a4lg.com>
Tsukasa OI committed
Dec 15, 2025 at 00:01 UTC
9e33a1a75b032e035b9129d7876d33c37596c6fe
1 file changed
+22
vllm/transformers_utils/config.py
+22
@@ -617,6 +617,28 @@ def get_config(
617
hf_overrides=hf_overrides_kw,
618
**kwargs,
619
)
620
+
621
+ # Patching defaults for GGUF models
622
+ if _is_gguf:
623
+ # Some models have different default values between GGUF and HF.
624
+ def apply_gguf_default(key: str, gguf_default: Any):
625
+ """
626
+ Apply GGUF defaults unless explicitly configured.
627
+
628
+ This function reads/writes external `config` and `config_dict`.
629
+ If the specified `key` is not in `config_dict` (i.e. not explicitly
630
+ configured and the default HF value is used), it updates the
631
+ corresponding `config` value to `gguf_default`.
632
+ """
633
+ if key not in config_dict:
634
+ config.update({key: gguf_default})
635
+
636
+ # Apply architecture-specific GGUF defaults.
637
+ if config.model_type in {"qwen3_moe"}:
638
+ # Qwen3 MoE: norm_topk_prob is always true.
639
+ # Note that, this parameter is always false (HF default) on Qwen2 MoE.
640
+ apply_gguf_default("norm_topk_prob", True)
641
+
642
# Special architecture mapping check for GGUF models
643
if _is_gguf:
644
if config.model_type not in MODEL_FOR_CAUSAL_LM_MAPPING_NAMES: