A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 4, 11:15 UTC
[CI] Add MTP coverage: Qwen3.5 correctness + no-sync spec decode (#40472)
Signed-off-by: Stefano Castagnetta <scastagnetta@nvidia.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
Stefano Castagnetta committed
Apr 30, 2026 at 21:24 UTC
92a7c121b62a1484b68c0a27d1ecefd1a84f78fc
3 files changed
+53
-1
.buildkite/test_areas/spec_decode.yaml
+13
@@ -97,3 +97,16 @@ steps:
97
commands:
98
- export VLLM_ALLOW_INSECURE_SERIALIZATION=1
99
- pytest -v -s v1/spec_decode/test_speculators_dflash.py -m slow_test
100
+
101
+- label: Spec Decode MTP hybrid (B200)
102
+ timeout_in_minutes: 30
103
+ device: b200
104
+ optional: true
105
+ source_file_dependencies:
106
+ - vllm/v1/spec_decode/
107
+ - vllm/v1/worker/gpu/spec_decode/
108
+ - vllm/model_executor/models/qwen3_5.py
109
+ - vllm/model_executor/models/qwen3_5_mtp.py
110
+ - tests/v1/e2e/spec_decode/
111
+ commands:
112
+ - pytest -v -s v1/e2e/spec_decode -k "qwen3_5-hybrid"
tests/v1/e2e/spec_decode/test_async_spec_decode.py
+16
@@ -82,6 +82,13 @@ SPEC_DECODE_CONFIGS = [
82
2,
83
id="eagle-mla-deepseek",
84
),
85
+ pytest.param(
86
+ "Qwen/Qwen3.5-0.8B-Base",
87
+ "Qwen/Qwen3.5-0.8B-Base",
88
+ "mtp",
89
+ 1,
90
+ id="mtp-qwen3_5-hybrid",
91
+ ),
92
]
93
94
@@ -104,6 +111,14 @@ def test_no_sync_with_spec_decode(
111
from vllm import LLM, SamplingParams
112
from vllm.distributed import cleanup_dist_env_and_memory
113
114
+ # Qwen3.5 is a VLM; without this, profile_run runs the ViT warmup
115
+ # and peaks well above the 18GB MIG slice used by one of the CI lanes.
116
+ # This test only exercises text generation, so the vision tower is
117
+ # never needed.
118
+ extra_kwargs: dict = {}
119
+ if "Qwen3.5" in model:
120
+ extra_kwargs["limit_mm_per_prompt"] = {"image": 0, "video": 0}
121
+
122
llm = LLM(
123
model=model,
124
max_model_len=256,
@@ -114,6 +129,7 @@ def test_no_sync_with_spec_decode(
129
},
130
enforce_eager=True,
131
async_scheduling=True,
132
+ **extra_kwargs,
133
)
134
135
# Assert async scheduling is actually active before running inference.
tests/v1/e2e/spec_decode/test_spec_decode.py
+24
-1
@@ -1,5 +1,6 @@
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+import os
4
import random
5
from collections.abc import Iterable
6
from dataclasses import dataclass
@@ -723,8 +724,13 @@ def test_eagle_correctness_heavy(
724
[
725
(("mtp", "XiaomiMiMo/MiMo-7B-Base", 1), False, 0.5), # ref: 65%-70%
726
(("mtp", "ZixiQi/DeepSeek-V3-4layers-MTP-FP8", 1), False, 0.0), # dummy model
727
+ (
728
+ ("mtp", "Qwen/Qwen3.5-0.8B-Base", 1),
729
+ False,
730
+ 0.20,
731
+ ), # hybrid + MTP, ref: ~34%-35%
732
],
727
- ids=["mimo", "deepseek"],
733
+ ids=["mimo", "deepseek", "qwen3_5-hybrid"],
734
)
735
@single_gpu_only
736
@large_gpu_mark(min_gb=20)
@@ -750,13 +756,29 @@ def test_mtp_correctness(
756
method, model_name, tp_size = model_setup
757
_skip_if_insufficient_gpus_for_tp(tp_size)
758
759
+ if "Qwen3.5" in model_name and os.environ.get("VLLM_USE_V2_MODEL_RUNNER"):
760
+ pytest.skip(
761
+ "Model Runner V2 does not yet support hybrid models "
762
+ "(Qwen3.5 mixes Mamba-style GDN with attention layers)."
763
+ )
764
+
765
attn_backend = "TRITON_ATTN" if current_platform.is_rocm() else "auto"
766
+
767
+ # Qwen3.5 is a VLM; without this, profile_run runs the ViT warmup
768
+ # and peaks well above the 18GB MIG slice used by one of the CI
769
+ # lanes. This test only exercises text generation, so the vision
770
+ # tower is never needed.
771
+ extra_kwargs: dict[str, Any] = {}
772
+ if "Qwen3.5" in model_name:
773
+ extra_kwargs["limit_mm_per_prompt"] = {"image": 0, "video": 0}
774
+
775
ref_llm = LLM(
776
model=model_name,
777
max_model_len=2048,
778
tensor_parallel_size=tp_size,
779
trust_remote_code=True,
780
attention_backend=attn_backend,
781
+ **extra_kwargs,
782
)
783
ref_outputs = ref_llm.chat(test_prompts, sampling_config)
784
evaluate_llm_for_gsm8k(
@@ -777,6 +799,7 @@ def test_mtp_correctness(
799
},
800
max_model_len=2048,
801
attention_backend=attn_backend,
802
+ **extra_kwargs,
803
)
804
# MTP supports async scheduling; assert it is active by default.
805
assert spec_llm.llm_engine.vllm_config.scheduler_config.async_scheduling