A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 5, 14:45 UTC
[Bugfix] Fix NVFP4+MTP crash: force unquantized mtp.fc for Qwen3.5 (#38832)
Signed-off-by: Vadim Gimpelson <vadim.gimpelson@gmail.com>
Vadim Gimpelson committed
Apr 3, 2026 at 04:45 UTC
771913e4a024945a31c9a4ad607b81993704582c
1 file changed
+10
-1
vllm/model_executor/models/qwen3_5_mtp.py
+10
-1
@@ -75,13 +75,22 @@ class Qwen3_5MultiTokenPredictor(nn.Module):
75
config.hidden_size,
76
)
77
78
+ # Workaround: mtp.fc is stored as BF16 in NVFP4 checkpoints but is
79
+ # missing from hf_quant_config.json exclude_modules. Force unquantized.
80
+ # Ref: https://github.com/vllm-project/vllm/pull/38650
81
+ # Ref: https://github.com/NVIDIA/Model-Optimizer/pull/1124
82
+ fc_quant = (
83
+ None
84
+ if (quant_config and quant_config.get_name() == "modelopt_fp4")
85
+ else quant_config
86
+ )
87
self.fc = ColumnParallelLinear(
88
self.config.hidden_size * 2,
89
self.config.hidden_size,
90
gather_output=True,
91
bias=False,
92
return_bias=False,
84
- quant_config=quant_config,
93
+ quant_config=fc_quant,
94
prefix=f"{prefix}.fc",
95
)
96