[Bugfix][Spec Decode] Resolve n_predict from text_config for Qwen3.5 multimodal MTP (#55369)
Signed-off-by: SOUMYAJIT GHOSH <23051387@kiit.ac.in> Signed-off-by: Soumyajit Ghosh <jobsoumyajit6124@gmail.com> Co-authored-by: SOUMYAJIT GHOSH <23051387@kiit.ac.in> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
Soumyajit Ghosh committed
Sep 7, 2026 at 18:58 UTC
b339d75a410e4e889049dff59042f385909bad67
3 files changed
+101
-9
.buildkite/test_areas/models_basic.yaml
+2
-1
@@ -134,7 +134,8 @@ steps:
134
- tests/models/test_utils.py
135
- tests/models/test_vision.py
136
- tests/models/test_adapters.py
137
+ - tests/models/test_qwen3_5_mtp_config.py
138
- tests/models/transformers/fusers/
139
device: cpu-small
140
commands:
140
- - pytest -v -s models/test_utils.py models/test_vision.py models/test_adapters.py models/transformers/fusers/
141
+ - pytest -v -s models/test_utils.py models/test_vision.py models/test_adapters.py models/test_qwen3_5_mtp_config.py models/transformers/fusers/
tests/models/test_qwen3_5_mtp_config.py
+96
-8
@@ -1,19 +1,45 @@
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
-"""CPU-only tests for Qwen3.5 text-only MTP speculative decoding."""
3
+"""CPU-only tests for Qwen3.5 MTP speculative decoding config overrides."""
4
+
5
+from typing import Any
6
7
import pytest
6
-from transformers import PretrainedConfig
8
+from transformers import AutoConfig, PretrainedConfig
9
10
from vllm.config.speculative import SpeculativeConfig
11
12
+_CHECKPOINTS = {
13
+ "qwen3_5": "Qwen/Qwen3.8-27B",
14
+ "qwen3_5_moe": "Qwen/Qwen3.6-35B-A3B",
15
+}
16
+
17
18
def _mtp_config(model_type: str) -> PretrainedConfig:
12
- return PretrainedConfig(
13
- model_type=model_type,
14
- architectures=["SomeArch"],
15
- mtp_num_hidden_layers=1,
16
- )
19
+ """Create a top-level MTP configuration with mtp_num_hidden_layers."""
20
+ kwargs: dict[str, Any] = {
21
+ "model_type": model_type,
22
+ "architectures": ["SomeArch"],
23
+ "mtp_num_hidden_layers": 1,
24
+ }
25
+ return PretrainedConfig(**kwargs)
26
+
27
+
28
+def _multimodal_wrapper_mtp_config(
29
+ model_type: str, mtp_layers: int = 1
30
+) -> PretrainedConfig:
31
+ """Download a multimodal wrapper config via AutoConfig.from_pretrained
32
+ and configure mtp_num_hidden_layers in text_config.
33
+
34
+ Uses real-world Hugging Face Hub checkpoints:
35
+ - Dense (qwen3_5): Qwen/Qwen3.8-27B
36
+ - MoE (qwen3_5_moe): Qwen/Qwen3.6-35B-A3B
37
+ """
38
+ repo = _CHECKPOINTS[model_type]
39
+ config: PretrainedConfig = AutoConfig.from_pretrained(repo)
40
+ text_config = config.get_text_config()
41
+ text_config.mtp_num_hidden_layers = mtp_layers
42
+ return config
43
44
45
@pytest.mark.parametrize(
@@ -26,8 +52,70 @@ def _mtp_config(model_type: str) -> PretrainedConfig:
52
("qwen3_5_moe_text", "Qwen3_5MoeMTP"),
53
],
54
)
29
-def test_mtp_override_recognizes_text_only_types(model_type, expected_arch):
55
+def test_mtp_override_recognizes_text_only_types(
56
+ model_type: str, expected_arch: str
57
+) -> None:
58
+ """Verify that text-only config variants map to the expected MTP architectures."""
59
cfg = SpeculativeConfig.hf_config_override(_mtp_config(model_type))
60
assert cfg.model_type == "qwen3_5_mtp"
61
assert cfg.architectures == [expected_arch]
62
assert cfg.n_predict == 1
63
+
64
+
65
+@pytest.mark.parametrize(
66
+ "model_type,expected_arch",
67
+ [
68
+ ("qwen3_5", "Qwen3_5MTP"),
69
+ ("qwen3_5_moe", "Qwen3_5MoeMTP"),
70
+ ],
71
+)
72
+def test_mtp_override_extracts_n_predict_from_multimodal_wrapper(
73
+ model_type: str, expected_arch: str
74
+) -> None:
75
+ """Verify that multimodal wrapper checkpoints with mtp_num_hidden_layers
76
+ in text_config resolve n_predict and architecture correctly."""
77
+ cfg = SpeculativeConfig.hf_config_override(
78
+ _multimodal_wrapper_mtp_config(model_type, mtp_layers=2)
79
+ )
80
+ assert cfg.model_type == "qwen3_5_mtp"
81
+ assert cfg.architectures == [expected_arch]
82
+ assert cfg.n_predict == 2
83
+
84
+
85
+@pytest.mark.parametrize(
86
+ "model_type,expected_arch",
87
+ [
88
+ ("qwen3_5", "Qwen3_5MTP"),
89
+ ("qwen3_5_moe", "Qwen3_5MoeMTP"),
90
+ ],
91
+)
92
+def test_mtp_override_top_level_precedence_over_nested_text_config(
93
+ model_type: str, expected_arch: str
94
+) -> None:
95
+ """Verify that an explicit top-level mtp_num_hidden_layers takes precedence
96
+ over a nested text_config value."""
97
+ cfg = _multimodal_wrapper_mtp_config(model_type, mtp_layers=2)
98
+ cfg.mtp_num_hidden_layers = 3
99
+ overridden = SpeculativeConfig.hf_config_override(cfg)
100
+ assert overridden.model_type == "qwen3_5_mtp"
101
+ assert overridden.architectures == [expected_arch]
102
+ assert overridden.n_predict == 3
103
+
104
+
105
+@pytest.mark.parametrize(
106
+ "model_id,expected_arch",
107
+ [
108
+ ("Qwen/Qwen3.8-27B", "Qwen3_5MTP"),
109
+ ("Qwen/Qwen3.6-35B-A3B", "Qwen3_5MoeMTP"),
110
+ ],
111
+)
112
+def test_mtp_override_downloads_real_hf_hub_configs(
113
+ model_id: str, expected_arch: str
114
+) -> None:
115
+ """Verify that unmodified real-world checkpoints downloaded via
116
+ AutoConfig.from_pretrained resolve n_predict=1 from text_config."""
117
+ hf_config = AutoConfig.from_pretrained(model_id)
118
+ cfg = SpeculativeConfig.hf_config_override(hf_config)
119
+ assert cfg.model_type == "qwen3_5_mtp"
120
+ assert cfg.architectures == [expected_arch]
121
+ assert cfg.n_predict == 1
vllm/config/speculative.py
+3
@@ -897,6 +897,9 @@ class SpeculativeConfig:
897
is_moe = hf_config.model_type in ("qwen3_5_moe", "qwen3_5_moe_text")
898
hf_config.model_type = "qwen3_5_mtp"
899
n_predict = getattr(hf_config, "mtp_num_hidden_layers", None)
900
+ if n_predict is None:
901
+ text_config = get_hf_text_config(hf_config)
902
+ n_predict = getattr(text_config, "mtp_num_hidden_layers", None)
903
hf_config.update(
904
{
905
"n_predict": n_predict,