[Model][NVIDIA] Route DSA models to the CUDA non-compiled path (#52861)
Woosuk Kwon committed
Aug 19, 2026 at 00:13 UTC
b09bd69b5bf14911abef9a0e8e493b83c8a38fa6
16 files changed
+260
-146
tests/compile/fusions_e2e/conftest.py
+1
-20
@@ -90,11 +90,8 @@ def run_e2e_fusion_test(monkeypatch, caplog_mp_spawn):
90
backend_name = attn_backend.backend.name.lower()
91
requires_mla = "deepseek" in model_name.lower()
92
is_mla = "mla" in backend_name
93
- # DeepSeek V3.2 uses sparse MLA
94
- requires_sparse = "v3.2" in model_name.lower()
95
- is_sparse = "sparse" in backend_name
93
97
- if requires_mla != is_mla or requires_sparse != is_sparse:
94
+ if requires_mla != is_mla:
95
pytest.skip(
96
f"Incompatible model '{model_name}' and "
97
f"attention backend '{attn_backend.backend.name}'"
@@ -128,22 +125,6 @@ def run_e2e_fusion_test(monkeypatch, caplog_mp_spawn):
125
# models (e.g. Llama-4-Scout-FP8) at 16384 tokens may trigger OOM.
126
model_kwargs.setdefault("max_num_batched_tokens", 8192)
127
131
- # Sparse MLA models (DSv3.2) hit an over-strict inductor assertion in
132
- # decompose_auto_functionalized when +rotary_embedding is forced into
133
- # the compile graph. Disable qk_norm+rope fusion (which auto-enables
134
- # +rotary_embedding) for this combo to avoid the known torch bug.
135
- # TODO: remove once upstream torch fix lands.
136
- if requires_sparse:
137
- if "pass_config" in compilation_config:
138
- compilation_config["pass_config"].enable_qk_norm_rope_fusion = False
139
- matches_check = [m for m in matches_check if m != "norm_rope_fusion"]
140
- # DSv3.2 sparse indexer uses persistent_topk with k=config.index_topk
141
- # (2048 for the default config). max_model_len must be >= index_topk
142
- # or the topk kernel raises "k out of range" at runtime.
143
- model_kwargs["max_model_len"] = max(
144
- model_kwargs.get("max_model_len", 0), 2048
145
- )
146
-
128
# Always compile the full graph instead of piecewise
129
if not compilation_config["use_inductor_graph_partition"]:
130
compilation_config["splitting_ops"] = []
tests/compile/fusions_e2e/models.py
-24
@@ -58,18 +58,6 @@ TRITON_MLA_ATTN = pytest.param(
58
id="TRITON_MLA",
59
)
60
61
-FLASHMLA_SPARSE_ATTN = pytest.param(
62
- AttentionBackendCase(
63
- backend=AttentionBackendEnum.FLASHMLA_SPARSE,
64
- model_kwargs=dict(kv_cache_dtype="fp8_ds_mla"),
65
- ),
66
- id="FLASHMLA_SPARSE",
67
- marks=pytest.mark.skipif(
68
- not is_blackwell(),
69
- reason="FlashMLA Sparse requires Blackwell",
70
- ),
71
-)
72
-
61
# Models
62
llama3_8b = ModelFusionInfo(
63
model_name="meta-llama/Llama-3.1-8B-Instruct",
@@ -197,18 +185,6 @@ deepseek_r1_fp4 = ModelFusionInfo(
185
),
186
)
187
200
-deepseek_v32_fp4 = ModelFusionInfo(
201
- model_name="nvidia/DeepSeek-V3.2-NVFP4",
202
- matches=lambda n_layers: Matches(
203
- rms_quant_fusion=0,
204
- # silu+quant on dense layers only; MoE hides the act+quant site
205
- act_quant_fusion=min(3, n_layers),
206
- # MLA attn + NVFP4 output quant fuses on sparse MLA output path
207
- attn_quant_fusion=n_layers,
208
- ar_rms_fusion=n_layers * 2 + 1,
209
- ),
210
-)
211
-
188
gpt_oss_20b = ModelFusionInfo(
189
model_name="openai/gpt-oss-20b",
190
matches=lambda n_layers: Matches(
tests/compile/fusions_e2e/test_tp1_quant.py
+2
-4
@@ -18,7 +18,6 @@ from .common import (
18
from .models import (
19
FLASHINFER_ATTN,
20
FLASHINFER_MLA_ATTN,
21
- FLASHMLA_SPARSE_ATTN,
21
ROCM_AITER_UNIFIED_ATTN,
22
ROCM_ATTN,
23
TRITON_ATTN,
@@ -26,7 +25,6 @@ from .models import (
25
deepseek_coder_v2_lite_fp8,
26
deepseek_r1_fp4,
27
deepseek_v3_fp8,
29
- deepseek_v32_fp4,
28
llama3_8b_fp4,
29
llama3_8b_fp8,
30
llama4_scout_fp4,
@@ -149,11 +147,11 @@ def test_tp1_fp8_fusions(
147
148
@pytest.mark.parametrize(
149
"model_name, matches_fn, model_kwargs, hf_overrides",
152
- [llama3_8b_fp4, llama4_scout_fp4, deepseek_r1_fp4, deepseek_v32_fp4],
150
+ [llama3_8b_fp4, llama4_scout_fp4, deepseek_r1_fp4],
151
)
152
@pytest.mark.parametrize(
153
"attn_backend",
156
- [FLASHINFER_ATTN, FLASHINFER_MLA_ATTN, FLASHMLA_SPARSE_ATTN],
154
+ [FLASHINFER_ATTN, FLASHINFER_MLA_ATTN],
155
)
156
@pytest.mark.parametrize("n_layers", [6])
157
@pytest.mark.parametrize("custom_ops", custom_ops_combos("rms_norm"))
tests/compile/fusions_e2e/test_tp2_ar_rms.py
+2
-4
@@ -18,14 +18,12 @@ from .common import (
18
from .models import (
19
FLASHINFER_ATTN,
20
FLASHINFER_MLA_ATTN,
21
- FLASHMLA_SPARSE_ATTN,
21
ROCM_AITER_UNIFIED_ATTN,
22
ROCM_ATTN,
23
TRITON_ATTN,
24
deepseek_coder_v2_lite_fp8,
25
deepseek_r1_fp4,
26
deepseek_v3_fp8,
28
- deepseek_v32_fp4,
27
gpt_oss_20b,
28
llama3_8b,
29
llama3_8b_fp4,
@@ -120,11 +118,11 @@ def test_tp2_ar_rms_fp8_fusions(
118
@multi_gpu_test(num_gpus=2)
119
@pytest.mark.parametrize(
120
"model_name, matches_fn, model_kwargs, hf_overrides",
123
- [llama3_8b_fp4, llama4_scout_fp4, deepseek_r1_fp4, deepseek_v32_fp4],
121
+ [llama3_8b_fp4, llama4_scout_fp4, deepseek_r1_fp4],
122
)
123
@pytest.mark.parametrize(
124
"attn_backend",
127
- [FLASHINFER_ATTN, FLASHINFER_MLA_ATTN, FLASHMLA_SPARSE_ATTN],
125
+ [FLASHINFER_ATTN, FLASHINFER_MLA_ATTN],
126
)
127
@pytest.mark.parametrize("n_layers", [4])
128
@pytest.mark.parametrize("custom_ops", custom_ops_combos("rms_norm"))
tests/compile/h100/test_startup.py
-13
@@ -132,19 +132,6 @@ MODEL_SPECS = [
132
),
133
id="gpt_oss_120b",
134
),
135
- # NOTE: DeepSeek-V3.2 requires sparse MLA (index_topk) which needs
136
- # Hopper+ GPUs. This test must run on H100 (see pytorch.yaml).
137
- pytest.param(
138
- ModelStartupSpec(
139
- model="deepseek-ai/DeepSeek-V3.2",
140
- hf_overrides=_SMALL_MOE_OVERRIDES,
141
- cold_artifacts_saved=9,
142
- # https://github.com/vllm-project/vllm/issues/38051
143
- warm_artifacts_saved=0 if is_torch_equal_or_newer("2.12.0") else 9,
144
- warm_artifacts_loaded=9 if is_torch_equal_or_newer("2.12.0") else 0,
145
- ),
146
- id="deepseek_v3.2",
147
- ),
135
pytest.param(
136
ModelStartupSpec(
137
model="moonshotai/Kimi-K2.5",
tests/kernels/test_fused_deepseek_v32_norm_rope.py
+3
-4
@@ -146,8 +146,8 @@ def assert_fp8(got: torch.Tensor, ref: torch.Tensor, msg: str):
146
147
@pytest.mark.parametrize("num_tokens", [1, 4, 17, 512, 4096])
148
@pytest.mark.parametrize("index_interleave", [True, False])
149
-@pytest.mark.parametrize("mla_fp8", [False, True])
150
-def test_fused_norm_rope(num_tokens: int, index_interleave: bool, mla_fp8: bool):
149
+@pytest.mark.parametrize("mla_dtype", ["auto", "bfloat16", "fp8"])
150
+def test_fused_norm_rope(num_tokens: int, index_interleave: bool, mla_dtype: str):
151
torch.manual_seed(0)
152
dev = "cuda"
153
max_pos = 8192
@@ -167,13 +167,12 @@ def test_fused_norm_rope(num_tokens: int, index_interleave: bool, mla_fp8: bool)
167
168
bs = max_pos # single block covering all tokens
169
mla_dim = KV_LORA + ROPE_DIM
170
+ mla_fp8 = mla_dtype == "fp8"
171
if mla_fp8:
172
mla_cache = torch.zeros(1, bs, mla_dim, device=dev, dtype=torch.uint8)
172
- mla_dtype = "fp8"
173
mla_k_scale = torch.tensor([0.3], device=dev, dtype=torch.float32)
174
else:
175
mla_cache = torch.zeros(1, bs, mla_dim, device=dev, dtype=torch.bfloat16)
176
- mla_dtype = "auto"
176
mla_k_scale = None
177
idx_row = INDEX_HEAD_DIM + INDEX_HEAD_DIM // 128 * 4 # 132
178
idx_cache = torch.zeros(1, bs, idx_row, device=dev, dtype=torch.uint8)
tests/models/registry.py
+5
@@ -1679,6 +1679,11 @@ _SPECULATIVE_DECODING_EXAMPLE_MODELS = {
1679
speculative_model="luccafong/deepseek_mtp_draft_random",
1680
trust_remote_code=True,
1681
),
1682
+ "DeepseekV32MTPModel": _HfExamplesInfo(
1683
+ "nvidia/DeepSeek-V3.2-NVFP4",
1684
+ speculative_model="nvidia/DeepSeek-V3.2-NVFP4",
1685
+ is_available_online=False,
1686
+ ),
1687
"DeepSeekV4MTPModel": _HfExamplesInfo(
1688
"deepseek-ai/DeepSeek-V4-Flash",
1689
speculative_model="deepseek-ai/DeepSeek-V4-Flash",
tests/test_config.py
+140
-4
@@ -113,18 +113,154 @@ def test_v2_model_runner_env_tri_state(monkeypatch, env_value, expected):
113
assert envs.VLLM_USE_V2_MODEL_RUNNER is expected
114
115
116
-def test_rocm_defaults_deepseek_v4_to_mrv1(monkeypatch):
117
- """ROCm keeps DeepSeek V4 on MRV1, which is still faster there."""
118
- from vllm.config.vllm import default_v2_model_runner_architectures
116
+def test_rocm_keeps_compiled_deepseek_defaults(monkeypatch):
117
+ """ROCm keeps DeepSeek V3.2 and V4 on their compiled MRV1 paths."""
118
+ from vllm.config.vllm import (
119
+ default_breakable_cudagraph_architectures,
120
+ default_v2_model_runner_architectures,
121
+ )
122
from vllm.platforms import current_platform
123
124
monkeypatch.setattr(current_platform, "is_rocm", lambda: True)
125
# The lookup is lru_cached against a fixed platform.
126
default_v2_model_runner_architectures.cache_clear()
127
+ default_breakable_cudagraph_architectures.cache_clear()
128
+ try:
129
+ v2_architectures = default_v2_model_runner_architectures()
130
+ breakable_architectures = default_breakable_cudagraph_architectures()
131
+
132
+ assert "DeepseekV32ForCausalLM" not in v2_architectures
133
+ assert "DeepseekV4ForCausalLM" not in v2_architectures
134
+ assert "DeepseekV32ForCausalLM" not in breakable_architectures
135
+ assert "DeepseekV32MTPModel" not in breakable_architectures
136
+ finally:
137
+ default_v2_model_runner_architectures.cache_clear()
138
+ default_breakable_cudagraph_architectures.cache_clear()
139
+
140
+
141
+@pytest.mark.parametrize(
142
+ ("model", "architecture"),
143
+ [
144
+ ("nvidia/GLM-5.2-NVFP4", "GlmMoeDsaForCausalLM"),
145
+ ("zai-org/GLM-5.2-FP8", "GlmMoeDsaForCausalLM"),
146
+ ("nvidia/DeepSeek-V3.2-NVFP4", "DeepseekV32ForCausalLM"),
147
+ ],
148
+)
149
+@pytest.mark.parametrize("with_mtp", [False, True], ids=["no-mtp", "mtp"])
150
+def test_dsa_models_default_to_mrv2_and_breakable_cudagraph(
151
+ monkeypatch, model, architecture, with_mtp
152
+):
153
+ from vllm.compilation.breakable_cudagraph import (
154
+ is_breakable_cudagraph_enabled,
155
+ )
156
+ from vllm.config.vllm import (
157
+ default_breakable_cudagraph_architectures,
158
+ default_v2_model_runner_architectures,
159
+ )
160
+ from vllm.platforms import current_platform
161
+
162
+ monkeypatch.delenv("VLLM_USE_BREAKABLE_CUDAGRAPH", raising=False)
163
+ monkeypatch.delenv("VLLM_USE_V2_MODEL_RUNNER", raising=False)
164
+ monkeypatch.setattr(vllm_config_module, "HAS_TRITON", True)
165
+ monkeypatch.setattr(current_platform, "is_rocm", lambda: False)
166
+ default_v2_model_runner_architectures.cache_clear()
167
+ default_breakable_cudagraph_architectures.cache_clear()
168
+
169
+ model_config = SimpleNamespace(
170
+ model=model,
171
+ architectures=[architecture],
172
+ runner_type="generate",
173
+ is_moe=True,
174
+ is_hybrid=False,
175
+ is_attention_free=False,
176
+ is_diffusion=False,
177
+ )
178
+ config = SimpleNamespace(
179
+ model_config=model_config,
180
+ speculative_config=SimpleNamespace(method="mtp") if with_mtp else None,
181
+ parallel_config=SimpleNamespace(prefill_context_parallel_size=1),
182
+ compilation_config=CompilationConfig(
183
+ cudagraph_mode=CUDAGraphMode.FULL_AND_PIECEWISE
184
+ ),
185
+ )
186
+ config._dflash_needs_multi_kv_group = lambda: False
187
+ config._is_default_v2_model_runner_model = lambda: (
188
+ VllmConfig._is_default_v2_model_runner_model(config)
189
+ )
190
+ config._get_v2_model_runner_unsupported_features = lambda: []
191
+ config._uses_breakable_cudagraph_by_default = lambda: (
192
+ VllmConfig._uses_breakable_cudagraph_by_default(config)
193
+ )
194
+
195
try:
125
- assert "DeepseekV4ForCausalLM" not in default_v2_model_runner_architectures()
196
+ assert VllmConfig.use_v2_model_runner.fget(config)
197
+ assert VllmConfig._maybe_enable_breakable_cudagraph(config)
198
+ assert is_breakable_cudagraph_enabled()
199
+ assert config.compilation_config.mode == CompilationMode.NONE
200
+ assert config.compilation_config.cudagraph_mode.has_piecewise_cudagraphs()
201
finally:
202
+ os.environ.pop("VLLM_USE_BREAKABLE_CUDAGRAPH", None)
203
default_v2_model_runner_architectures.cache_clear()
204
+ default_breakable_cudagraph_architectures.cache_clear()
205
+
206
+
207
+@pytest.mark.parametrize(
208
+ ("architecture", "is_rocm", "expected"),
209
+ [
210
+ ("DeepseekV32ForCausalLM", False, True),
211
+ ("DeepseekV32ForCausalLM", True, False),
212
+ ("DeepseekV32MTPModel", False, True),
213
+ ("DeepseekV32MTPModel", True, False),
214
+ ("GlmMoeDsaForCausalLM", False, True),
215
+ ("GlmMoeDsaForCausalLM", True, True),
216
+ ],
217
+)
218
+def test_dsa_breakable_cudagraph_platform_default(
219
+ monkeypatch, architecture, is_rocm, expected
220
+):
221
+ from vllm.config.vllm import default_breakable_cudagraph_architectures
222
+ from vllm.platforms import current_platform
223
+
224
+ monkeypatch.delenv("VLLM_USE_BREAKABLE_CUDAGRAPH", raising=False)
225
+ monkeypatch.setattr(current_platform, "is_rocm", lambda: is_rocm)
226
+ default_breakable_cudagraph_architectures.cache_clear()
227
+ config = SimpleNamespace(
228
+ model_config=SimpleNamespace(architectures=[architecture]),
229
+ compilation_config=CompilationConfig(),
230
+ )
231
+ config._uses_breakable_cudagraph_by_default = lambda: (
232
+ VllmConfig._uses_breakable_cudagraph_by_default(config)
233
+ )
234
+
235
+ try:
236
+ assert VllmConfig._maybe_enable_breakable_cudagraph(config) is expected
237
+ if expected:
238
+ assert config.compilation_config.mode == CompilationMode.NONE
239
+ finally:
240
+ os.environ.pop("VLLM_USE_BREAKABLE_CUDAGRAPH", None)
241
+ default_breakable_cudagraph_architectures.cache_clear()
242
+
243
+
244
+@pytest.mark.parametrize(
245
+ ("model_type", "expected_architecture"),
246
+ [
247
+ ("deepseek_v32", "DeepseekV32MTPModel"),
248
+ ("glm_moe_dsa", "DeepseekV32MTPModel"),
249
+ ("deepseek_v3", "DeepSeekMTPModel"),
250
+ ],
251
+)
252
+def test_dsa_models_select_matching_mtp(model_type, expected_architecture):
253
+ from transformers import PretrainedConfig
254
+
255
+ hf_config = PretrainedConfig(
256
+ architectures=["DeepseekV32ForCausalLM"],
257
+ num_nextn_predict_layers=1,
258
+ )
259
+ hf_config.model_type = model_type
260
+
261
+ SpeculativeConfig.hf_config_override(hf_config)
262
+
263
+ assert hf_config.architectures == [expected_architecture]
264
265
266
@pytest.mark.parametrize(
vllm/config/speculative.py
+7
-5
@@ -341,6 +341,7 @@ class SpeculativeConfig:
341
@staticmethod
342
def hf_config_override(hf_config: PretrainedConfig) -> PretrainedConfig:
343
initial_architecture = hf_config.architectures[0]
344
+ use_v32_mtp = hf_config.model_type in ("deepseek_v32", "glm_moe_dsa")
345
if hf_config.model_type == "dots3_note":
346
n_predict = getattr(hf_config, "num_nextn_predict_layers", 1)
347
mtp_layer_types = getattr(hf_config, "mtp_layer_types", None)
@@ -365,7 +366,12 @@ class SpeculativeConfig:
366
if hf_config.model_type == "deepseek_mtp":
367
n_predict = getattr(hf_config, "num_nextn_predict_layers", None)
368
hf_config.update(
368
- {"n_predict": n_predict, "architectures": ["DeepSeekMTPModel"]}
369
+ {
370
+ "n_predict": n_predict,
371
+ "architectures": [
372
+ "DeepseekV32MTPModel" if use_v32_mtp else "DeepSeekMTPModel"
373
+ ],
374
+ }
375
)
376
if hf_config.model_type == "deepseek_v4":
377
hf_config.model_type = "deepseek_mtp"
@@ -759,10 +765,6 @@ class SpeculativeConfig:
765
if self.method == "mtp":
766
if self.target_model_config is None:
767
raise ValueError("target_model_config must be present for mtp")
762
- if self.target_model_config.hf_text_config.model_type == "deepseek_v32":
763
- # FIXME(luccafong): cudagraph with v32 MTP is not supported,
764
- # remove this when the issue is fixed.
765
- self.enforce_eager = True
768
# use the draft model from the same model:
769
self.model = self.target_model_config.model
770
# Align the quantization of draft model for cases such as
vllm/config/vllm.py
+67
-39
@@ -69,7 +69,9 @@ logger = init_logger(__name__)
69
DEFAULT_V2_MODEL_RUNNER_ARCHITECTURES = frozenset(
70
{
71
"DeepseekV2ForCausalLM",
72
+ "DeepseekV32ForCausalLM",
73
"DeepseekV4ForCausalLM",
74
+ "GlmMoeDsaForCausalLM",
75
"GraniteMoeForCausalLM",
76
"InklingForCausalLM",
77
"InklingForConditionalGeneration",
@@ -79,6 +81,23 @@ DEFAULT_V2_MODEL_RUNNER_ARCHITECTURES = frozenset(
81
}
82
)
83
84
+DEFAULT_BREAKABLE_CUDAGRAPH_ARCHITECTURES = frozenset(
85
+ {
86
+ "DeepseekV32MTPModel",
87
+ "DeepseekV32ForCausalLM",
88
+ "DeepseekV4ForCausalLM",
89
+ "DeepSeekV4MTPModel",
90
+ "GlmMoeDsaForCausalLM",
91
+ "InklingForCausalLM",
92
+ "InklingForConditionalGeneration",
93
+ "KimiK3ForConditionalGeneration",
94
+ "KimiK3MTPModel",
95
+ "KimiLinearForCausalLM",
96
+ "MiniMaxM3SparseForCausalLM",
97
+ "MiniMaxM3SparseForConditionalGeneration",
98
+ }
99
+)
100
+
101
102
@lru_cache
103
def default_v2_model_runner_architectures() -> frozenset[str]:
@@ -86,13 +105,28 @@ def default_v2_model_runner_architectures() -> frozenset[str]:
105
from vllm.platforms import current_platform
106
107
if current_platform.is_rocm():
89
- # TODO(rocm): DeepSeek V4 is still faster on MRV1 on ROCm. The
90
- # attention layer picks the eager cudagraph region MRV1 needs, so
91
- # this is a perf default only; drop it once MRV2 catches up.
92
- return DEFAULT_V2_MODEL_RUNNER_ARCHITECTURES - {"DeepseekV4ForCausalLM"}
108
+ # TODO(rocm): These models are either unsupported by MRV2 or slower with
109
+ # MRV2 on AMD GPUs.
110
+ return DEFAULT_V2_MODEL_RUNNER_ARCHITECTURES - {
111
+ "DeepseekV32ForCausalLM",
112
+ "DeepseekV4ForCausalLM",
113
+ }
114
return DEFAULT_V2_MODEL_RUNNER_ARCHITECTURES
115
116
117
+@lru_cache
118
+def default_breakable_cudagraph_architectures() -> frozenset[str]:
119
+ """Architectures defaulting to breakable CUDA graphs on this platform."""
120
+ from vllm.platforms import current_platform
121
+
122
+ if current_platform.is_rocm():
123
+ return DEFAULT_BREAKABLE_CUDAGRAPH_ARCHITECTURES - {
124
+ "DeepseekV32ForCausalLM",
125
+ "DeepseekV32MTPModel",
126
+ }
127
+ return DEFAULT_BREAKABLE_CUDAGRAPH_ARCHITECTURES
128
+
129
+
130
class OptimizationLevel(IntEnum):
131
"""Optimization level enum."""
132
@@ -691,6 +725,34 @@ class VllmConfig:
725
return False
726
return is_default_v2_architecture or not model_config.is_moe
727
728
+ def _uses_breakable_cudagraph_by_default(self) -> bool:
729
+ model_config = self.model_config
730
+ if model_config is None:
731
+ return False
732
+
733
+ architectures = set(model_config.architectures)
734
+ return bool(architectures & default_breakable_cudagraph_architectures())
735
+
736
+ def _maybe_enable_breakable_cudagraph(self) -> bool:
737
+ if (
738
+ "VLLM_USE_BREAKABLE_CUDAGRAPH" not in os.environ
739
+ and self._uses_breakable_cudagraph_by_default()
740
+ ):
741
+ os.environ["VLLM_USE_BREAKABLE_CUDAGRAPH"] = "1"
742
+ logger.info_once(
743
+ "Auto-enabling VLLM_USE_BREAKABLE_CUDAGRAPH=1. "
744
+ "Set VLLM_USE_BREAKABLE_CUDAGRAPH=0 to opt out."
745
+ )
746
+
747
+ from vllm.compilation.breakable_cudagraph import (
748
+ is_breakable_cudagraph_enabled,
749
+ )
750
+
751
+ enabled = is_breakable_cudagraph_enabled()
752
+ if enabled:
753
+ self.compilation_config.mode = CompilationMode.NONE
754
+ return enabled
755
+
756
@property
757
def needs_dp_coordinator(self) -> bool:
758
"""
@@ -1288,41 +1350,7 @@ class VllmConfig:
1350
)
1351
self.compilation_config.mode = CompilationMode.NONE
1352
1291
- # For model classes don't carry @support_torch_compile —
1292
- # the breakable cudagraph is the supported PIECEWISE path. Auto-enable
1293
- # it unless the user has explicitly opted out via the env var.
1294
- if (
1295
- self.model_config is not None
1296
- and "VLLM_USE_BREAKABLE_CUDAGRAPH" not in os.environ
1297
- and any(
1298
- a
1299
- in (
1300
- "DeepseekV4ForCausalLM",
1301
- "DeepSeekV4MTPModel",
1302
- "InklingForCausalLM",
1303
- "InklingForConditionalGeneration",
1304
- "KimiK3ForConditionalGeneration",
1305
- "KimiK3MTPModel",
1306
- "KimiLinearForCausalLM",
1307
- "MiniMaxM3SparseForCausalLM",
1308
- "MiniMaxM3SparseForConditionalGeneration",
1309
- )
1310
- for a in self.model_config.architectures
1311
- )
1312
- ):
1313
- os.environ["VLLM_USE_BREAKABLE_CUDAGRAPH"] = "1"
1314
- logger.info_once(
1315
- "Auto-enabling VLLM_USE_BREAKABLE_CUDAGRAPH=1. "
1316
- "Set VLLM_USE_BREAKABLE_CUDAGRAPH=0 to opt out."
1317
- )
1318
-
1319
- from vllm.compilation.breakable_cudagraph import (
1320
- is_breakable_cudagraph_enabled,
1321
- )
1322
-
1323
- breakable_cudagraph_enabled = is_breakable_cudagraph_enabled()
1324
- if breakable_cudagraph_enabled:
1325
- self.compilation_config.mode = CompilationMode.NONE
1353
+ breakable_cudagraph_enabled = self._maybe_enable_breakable_cudagraph()
1354
1355
if not breakable_cudagraph_enabled and (
1356
self.compilation_config.backend == "eager"
vllm/model_executor/models/registry.py
+6
-2
@@ -92,7 +92,10 @@ _TEXT_GENERATION_MODELS = {
92
"DeepseekForCausalLM": ("deepseek_v2", "DeepseekForCausalLM"),
93
"DeepseekV2ForCausalLM": ("deepseek_v2", "DeepseekV2ForCausalLM"),
94
"DeepseekV3ForCausalLM": ("deepseek_v2", "DeepseekV3ForCausalLM"),
95
- "DeepseekV32ForCausalLM": ("deepseek_v2", "DeepseekV3ForCausalLM"),
95
+ "DeepseekV32ForCausalLM": (
96
+ "vllm.models.deepseek_v32",
97
+ "DeepseekV32ForCausalLM",
98
+ ),
99
"DeepseekV4ForCausalLM": ("vllm.models.deepseek_v4", "DeepseekV4ForCausalLM"),
100
"Ernie4_5ForCausalLM": ("ernie45", "Ernie4_5ForCausalLM"),
101
"Ernie4_5_MoeForCausalLM": ("ernie45_moe", "Ernie4_5_MoeForCausalLM"),
@@ -115,7 +118,7 @@ _TEXT_GENERATION_MODELS = {
118
"Glm4ForCausalLM": ("glm4", "Glm4ForCausalLM"),
119
"Glm4MoeForCausalLM": ("glm4_moe", "Glm4MoeForCausalLM"),
120
"Glm4MoeLiteForCausalLM": ("glm4_moe_lite", "Glm4MoeLiteForCausalLM"),
118
- "GlmMoeDsaForCausalLM": ("deepseek_v2", "GlmMoeDsaForCausalLM"),
121
+ "GlmMoeDsaForCausalLM": ("vllm.models.deepseek_v32", "GlmMoeDsaForCausalLM"),
122
"GptOssForCausalLM": ("gpt_oss", "GptOssForCausalLM"),
123
"GPT2LMHeadModel": ("gpt2", "GPT2LMHeadModel"),
124
"GPTJForCausalLM": ("gpt_j", "GPTJForCausalLM"),
@@ -661,6 +664,7 @@ _SPECULATIVE_DECODING_MODELS = {
664
"Eagle3DeepseekV3ForCausalLM": ("deepseek_eagle3", "Eagle3DeepseekV2ForCausalLM"),
665
"EagleDeepSeekMTPModel": ("deepseek_eagle", "EagleDeepseekV3ForCausalLM"),
666
"DeepSeekMTPModel": ("deepseek_mtp", "DeepSeekMTP"),
667
+ "DeepseekV32MTPModel": ("vllm.models.deepseek_v32", "DeepseekV32MTP"),
668
"Dots3NoteMTPModel": ("vllm.models.dots3_note", "Dots3NoteMTP"),
669
"DeepSeekV4MTPModel": ("vllm.models.deepseek_v4", "DeepSeekV4MTP"),
670
"BailingMoeV3MTPModel": ("bailing_moe_v3_mtp", "BailingMoeV3MTPModel"),
vllm/models/deepseek_v32/__init__.py
+17
-8
@@ -1,26 +1,35 @@
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
-"""DeepSeek V3.2 (``deepseek_v32``) model — hardware-isolated entry point.
3
+"""DeepSeek V3.2 (``deepseek_v32``) platform entry point.
4
5
DeepSeek V3.2 introduced the DeepSeek Sparse Attention (DSA) architecture:
6
MLA + a "lightning indexer" that selects the top-k tokens for a sparse MLA
7
attend. The same model code serves any DSA checkpoint, including GLM-5.2
8
(``glm_moe_dsa``), which reuses this architecture.
9
+
10
+The CUDA implementation selects capability-specific kernels internally and
11
+falls back when an optimization is unavailable. Other platforms use the
12
+generic implementation by default.
13
"""
14
15
from vllm.platforms import current_platform
16
13
-if current_platform.is_rocm():
14
- from .amd.model import DeepseekV32ForCausalLM
15
- from .amd.mtp import DeepseekV32MTP
16
-elif current_platform.is_xpu():
17
- raise NotImplementedError("deepseek_v32 does not yet support XPU.")
18
-else:
19
- # Covers Blackwell (sm100) and all other CUDA devices.
17
+if current_platform.is_cuda():
18
+ # GLM-5.2 (glm_moe_dsa) reuses the CUDA DSA module. Individual optimized
19
+ # kernels remain gated on the device capabilities they support.
20
from .nvidia.model import DeepseekV32ForCausalLM
21
+ from .nvidia.model import DeepseekV32ForCausalLM as GlmMoeDsaForCausalLM
22
from .nvidia.mtp import DeepseekV32MTP
23
+else:
24
+ # ROCm, XPU, and CPU keep the generic implementation.
25
+ from vllm.model_executor.models.deepseek_mtp import DeepSeekMTP as DeepseekV32MTP
26
+ from vllm.model_executor.models.deepseek_v2 import (
27
+ DeepseekV3ForCausalLM as DeepseekV32ForCausalLM,
28
+ )
29
+ from vllm.model_executor.models.deepseek_v2 import GlmMoeDsaForCausalLM
30
31
__all__ = [
32
"DeepseekV32ForCausalLM",
33
"DeepseekV32MTP",
34
+ "GlmMoeDsaForCausalLM",
35
]
vllm/models/deepseek_v32/amd/rocm.py
-1
@@ -38,7 +38,6 @@ class DeepseekV32ROCmIndexer(DeepseekV32Indexer):
38
39
40
class DeepseekV32MLAAttention(DeepseekV32Attention):
41
- require_fp8_kv_cache: bool = False
41
indexer_cls = DeepseekV32ROCmIndexer
42
43
def __init__(self, vllm_config, config, prefix, topk_indices_buffer=None):
vllm/models/deepseek_v32/attention.py
+3
-16
@@ -170,7 +170,6 @@ class DeepseekV32Attention(MLAAttention):
170
indexer: "DeepseekV32Indexer | None"
171
indexer_cls: "type[DeepseekV32Indexer]" = DeepseekV32Indexer
172
173
- require_fp8_kv_cache: bool = True
173
supports_dense_mha_prefill = False
174
175
def __init__(
@@ -283,21 +282,9 @@ class DeepseekV32Attention(MLAAttention):
282
self.layer_name if enable_short_prefill_scoring_skip else ""
283
)
284
286
- if self.require_fp8_kv_cache:
287
- assert is_quantized_kv_cache(self.kv_cache_dtype), (
288
- "deepseek_v32 (nvidia) requires an fp8 KV cache served by a sparse "
289
- "MLA backend. Launch with --kv-cache-dtype fp8 (FlashInfer sparse) "
290
- "or --kv-cache-dtype fp8_ds_mla (FlashMLA sparse)."
291
- )
292
- self._fp8_query = self.impl.supports_quant_query_input
293
- if not self._fp8_query:
294
- assert self.kv_cache_dtype == "fp8_ds_mla", (
295
- "deepseek_v32 (nvidia) on a bf16-query sparse MLA backend "
296
- "(FlashMLA sparse) requires the fp8_ds_mla KV cache layout. "
297
- "Launch with --kv-cache-dtype fp8_ds_mla."
298
- )
299
-
300
- self._fp8_kv_needs_view = self.kv_cache_dtype != "fp8_ds_mla"
285
+ fp8_attention = is_quantized_kv_cache(self.kv_cache_dtype)
286
+ self._fp8_query = fp8_attention and self.impl.supports_quant_query_input
287
+ self._fp8_kv_needs_view = fp8_attention and self.kv_cache_dtype != "fp8_ds_mla"
288
289
self._index_rope_interleave = getattr(config, "indexer_rope_interleave", False)
290
vllm/models/deepseek_v32/common/kernels.py
+2
-1
@@ -6,6 +6,7 @@ import torch
6
7
from vllm.platforms import current_platform
8
from vllm.triton_utils import tl, triton
9
+from vllm.utils.torch_utils import is_quantized_kv_cache
10
11
# Cache of tiny 1-element dummy tensors (per device, dtype) reused by the
12
# has_indexer=False path so the indexer args don't allocate every call.
@@ -473,7 +474,7 @@ def fused_norm_rope(
474
475
# --- MLA KV cache setup ---
476
mla_cache_ds_mla = mla_kv_cache_dtype == "fp8_ds_mla"
476
- mla_cache_fp8 = mla_kv_cache_dtype not in ("auto", "fp8_ds_mla")
477
+ mla_cache_fp8 = is_quantized_kv_cache(mla_kv_cache_dtype) and not mla_cache_ds_mla
478
mla_num_tiles = 1
479
mla_ds_scale_view = torch.empty(0, dtype=torch.float32, device=device)
480
mla_ds_rope_view = torch.empty(0, dtype=torch.bfloat16, device=device)
vllm/v1/spec_decode/llm_base_proposer.py
+5
-1
@@ -1018,7 +1018,11 @@ class SpecDecodeBaseProposer:
1018
# feedback into the next draft step.
1019
architectures = self.draft_model_config.hf_config.architectures or []
1020
return bool(
1021
- {"DeepSeekMTPModel", "KimiK3MTPModel"}.intersection(architectures)
1021
+ {
1022
+ "DeepSeekMTPModel",
1023
+ "DeepseekV32MTPModel",
1024
+ "KimiK3MTPModel",
1025
+ }.intersection(architectures)
1026
)
1027
return self.method not in ("mtp", "draft_model", "dflash")
1028