Fix GPTQ quantized Qwen3.5 MTP weight loading with spec decode (#48816)
Signed-off-by: vllmellm <vllm.ellm@embeddedllm.com> Co-authored-by: TJian <tunjian.tan@embeddedllm.com> Co-authored-by: noobHappylife <64898326+noobHappylife@users.noreply.github.com>
vllmellm committed
Jul 23, 2026 at 21:59 UTC
c8db00b16cc188b46b7b9517a5836a0da4aa8c3e
1 file changed
+12
-2
vllm/model_executor/models/qwen3_5_mtp.py
+12
-2
@@ -103,6 +103,16 @@ class Qwen3_5MultiTokenPredictor(nn.Module):
103
prefix=f"{prefix}.fc",
104
)
105
106
+ # GPTQ: quantized checkpoints may exclude MTP from quantization via
107
+ # quantization_config.dynamic with "-:pattern" entries. When detected,
108
+ # disable quantization for MTP layers so they use unquantized params.
109
+ original_quant = vllm_config.quant_config
110
+ if quant_config and quant_config.get_name() not in ("modelopt_fp4",):
111
+ hf_qc = getattr(model_config.hf_config, "quantization_config", None)
112
+ if isinstance(hf_qc, dict):
113
+ dynamic = hf_qc.get("dynamic", {})
114
+ if any(k.startswith("-:") and "mtp" in k for k in dynamic):
115
+ vllm_config.quant_config = None
116
self.layers = torch.nn.ModuleList(
117
Qwen3_5DecoderLayer(
118
vllm_config,
@@ -111,11 +121,10 @@ class Qwen3_5MultiTokenPredictor(nn.Module):
121
)
122
for idx in range(self.num_mtp_layers)
123
)
114
-
124
+ vllm_config.quant_config = original_quant
125
self.make_empty_intermediate_tensors = make_empty_intermediate_tensors_factory(
126
["hidden_states", "residual"], config.hidden_size
127
)
118
-
128
self.norm = Qwen3_5RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
129
self.pre_fc_norm_hidden = Qwen3_5RMSNorm(
130
config.hidden_size, eps=config.rms_norm_eps
@@ -170,6 +179,7 @@ class Qwen3_5MultiTokenPredictor(nn.Module):
179
positions.shape[-1],
180
self.config.hidden_size,
181
)
182
+
183
hidden_states, _ = self.norm(hidden_states, residual)
184
return hidden_states
185