[Bugfix] Fall back to full decode graphs for noncompiled models (#55095)
Signed-off-by: Andreas Karatzas <Andreas.Karatzas@amd.com> Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com> Signed-off-by: Andreas Karatzas <akaratza@amd.com> Co-authored-by: OpenAI Codex <codex@openai.com> Co-authored-by: Lucas Wilkinson <lwilkins@redhat.com>
Andreas Karatzas committed
Sep 11, 2026 at 03:56 UTC
d0dfe587d5b20609af4de8e8d968713116b80eb6
7 files changed
+237
-18
tests/test_config.py
+122
@@ -273,6 +273,44 @@ def test_rocm_keeps_compiled_deepseek_defaults(monkeypatch):
273
default_breakable_cudagraph_architectures.cache_clear()
274
275
276
+@pytest.mark.parametrize(
277
+ ("architecture", "use_v2", "mode", "expected"),
278
+ [
279
+ ("DeepseekV4ForCausalLM", True, None, CUDAGraphMode.NONE),
280
+ ("DeepseekV4ForConditionalGeneration", True, None, CUDAGraphMode.NONE),
281
+ ("DeepseekV4ForCausalLM", False, None, None),
282
+ ("LlamaForCausalLM", True, None, None),
283
+ (
284
+ "DeepseekV4ForCausalLM",
285
+ True,
286
+ CUDAGraphMode.FULL_DECODE_ONLY,
287
+ CUDAGraphMode.FULL_DECODE_ONLY,
288
+ ),
289
+ ],
290
+)
291
+def test_rocm_gfx950_deepseek_v4_cudagraph_default(
292
+ monkeypatch, architecture, use_v2, mode, expected
293
+):
294
+ from vllm._aiter_ops import rocm_aiter_ops
295
+ from vllm.platforms import rocm
296
+
297
+ monkeypatch.setattr(rocm, "on_gfx950", lambda: True)
298
+ monkeypatch.setattr(rocm_aiter_ops, "is_fused_moe_enabled", lambda: False)
299
+ monkeypatch.setattr(rocm_aiter_ops, "is_linear_fp8_enabled", lambda: False)
300
+ monkeypatch.setattr(
301
+ rocm_aiter_ops, "is_fusion_moe_shared_experts_enabled", lambda: False
302
+ )
303
+ config = SimpleNamespace(
304
+ compilation_config=CompilationConfig(cudagraph_mode=mode),
305
+ model_config=SimpleNamespace(architecture=architecture),
306
+ use_v2_model_runner=use_v2,
307
+ )
308
+
309
+ rocm.RocmPlatform.apply_config_platform_defaults(config)
310
+
311
+ assert config.compilation_config.cudagraph_mode == expected
312
+
313
+
314
@pytest.mark.parametrize(
315
("model", "architecture"),
316
[
@@ -463,6 +501,66 @@ def test_resolve_cudagraph_mode_adjusts_spec_decode_sizes_only_for_v1(
501
assert compilation_config.cudagraph_capture_sizes == expected_capture_sizes
502
503
504
+@pytest.mark.parametrize(
505
+ ("mode", "piecewise_capture_available", "attention_support", "expected"),
506
+ [
507
+ ("PIECEWISE", False, "ALWAYS", "NONE"),
508
+ ("FULL_AND_PIECEWISE", False, "ALWAYS", "FULL_DECODE_ONLY"),
509
+ ("FULL_DECODE_ONLY", False, "ALWAYS", "FULL_DECODE_ONLY"),
510
+ ("FULL_DECODE_ONLY", False, "NEVER", "NONE"),
511
+ ],
512
+)
513
+def test_resolve_cudagraph_mode_uses_loaded_piecewise_provider(
514
+ mode, piecewise_capture_available, attention_support, expected
515
+):
516
+ compilation_config = CompilationConfig(
517
+ mode=CompilationMode.VLLM_COMPILE,
518
+ cudagraph_mode=CUDAGraphMode[mode],
519
+ use_inductor_graph_partition=True,
520
+ )
521
+
522
+ resolved = compilation_config.resolve_cudagraph_mode_and_sizes(
523
+ AttentionCGSupport[attention_support],
524
+ "FakeAttentionBackend",
525
+ piecewise_capture_available=piecewise_capture_available,
526
+ )
527
+
528
+ assert resolved.name == expected
529
+ assert compilation_config.cudagraph_mode == resolved
530
+
531
+
532
+@pytest.mark.skipif(
533
+ not current_platform.is_cuda_alike(), reason="Requires CUDA graph support"
534
+)
535
+@pytest.mark.parametrize(
536
+ "engine_kwargs",
537
+ [
538
+ {"runner": "pooling", "convert": "embed"},
539
+ pytest.param(
540
+ {"prefill_context_parallel_size": 2, "tensor_parallel_size": 2},
541
+ marks=pytest.mark.skipif(
542
+ not current_platform.is_rocm(), reason="ROCm PCP graph restriction"
543
+ ),
544
+ ),
545
+ ],
546
+)
547
+def test_late_piecewise_restrictions_without_compilation(monkeypatch, engine_kwargs):
548
+ """Late compatibility overrides must not restore unavailable piecewise graphs."""
549
+ from vllm.engine.arg_utils import EngineArgs
550
+
551
+ monkeypatch.setenv("VLLM_USE_BREAKABLE_CUDAGRAPH", "0")
552
+ monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", "1")
553
+ config = EngineArgs(
554
+ model="facebook/opt-125m",
555
+ compilation_config=CompilationConfig(mode=CompilationMode.NONE),
556
+ **engine_kwargs,
557
+ ).create_engine_config()
558
+
559
+ assert config.compilation_config.cudagraph_mode == CUDAGraphMode.NONE
560
+ assert config.compilation_config.cudagraph_capture_sizes == []
561
+ assert config.compilation_config.max_cudagraph_capture_size == 0
562
+
563
+
564
def test_resolve_cudagraph_mode_skips_mamba_block_check_while_profiling():
565
"""Cudagraph memory profiling uses a minimal KV cache, so the Mamba
566
block-count guard must only fire for the real cache sizing."""
@@ -498,6 +596,30 @@ def test_resolve_cudagraph_mode_skips_mamba_block_check_while_profiling():
596
assert cudagraph_mode == CUDAGraphMode.FULL_AND_PIECEWISE
597
598
599
+@pytest.mark.parametrize(
600
+ ("graph_mode", "should_raise"),
601
+ [
602
+ (CUDAGraphMode.FULL_DECODE_ONLY, False),
603
+ (CUDAGraphMode.FULL_AND_PIECEWISE, False),
604
+ (CUDAGraphMode.NONE, True),
605
+ (CUDAGraphMode.PIECEWISE, True),
606
+ ],
607
+)
608
+def test_adaptive_verification_requires_full_cudagraphs(graph_mode, should_raise):
609
+ config = SimpleNamespace(
610
+ speculative_config=SimpleNamespace(enable_adaptive_verification=True),
611
+ lora_config=None,
612
+ compilation_config=CompilationConfig(cudagraph_mode=graph_mode),
613
+ parallel_config=SimpleNamespace(pipeline_parallel_size=1),
614
+ )
615
+
616
+ if should_raise:
617
+ with pytest.raises(ValueError, match="requires full CUDA graphs"):
618
+ VllmConfig._validate_adaptive_verification(config)
619
+ else:
620
+ VllmConfig._validate_adaptive_verification(config)
621
+
622
+
623
@pytest.mark.parametrize(
624
("model_config", "expected"),
625
[
tests/v1/spec_decode/test_adaptive_verification.py
+22
@@ -4,7 +4,9 @@
4
from types import SimpleNamespace
5
6
import numpy as np
7
+import pytest
8
9
+from vllm.config.compilation import CUDAGraphMode
10
from vllm.v1.attention.backend import AttentionCGSupport
11
from vllm.v1.worker.gpu.async_utils import StepTimingSample
12
from vllm.v1.worker.gpu.attn_utils import AttentionCGSupportInfo
@@ -12,6 +14,7 @@ from vllm.v1.worker.gpu.spec_decode import adaptive_verification as adaptive_mod
14
from vllm.v1.worker.gpu.spec_decode.adaptive_verification import (
15
AdaptiveVerificationManager,
16
maybe_create_adaptive_verification_manager,
17
+ resolve_adaptive_cudagraph_mode,
18
)
19
from vllm.v1.worker.gpu.structured_outputs import _build_grammar_mapping
20
@@ -35,6 +38,25 @@ def make_manager(
38
return manager
39
40
41
+@pytest.mark.parametrize(
42
+ ("mode", "piecewise_capture_available", "expected"),
43
+ [
44
+ ("FULL_DECODE_ONLY", True, "FULL_DECODE_ONLY"),
45
+ ("FULL", True, "FULL_AND_PIECEWISE"),
46
+ ("FULL", False, "FULL_DECODE_ONLY"),
47
+ ("FULL_AND_PIECEWISE", True, "FULL_AND_PIECEWISE"),
48
+ ("FULL_AND_PIECEWISE", False, "FULL_DECODE_ONLY"),
49
+ ],
50
+)
51
+def test_resolve_adaptive_cudagraph_mode(mode, piecewise_capture_available, expected):
52
+ assert (
53
+ resolve_adaptive_cudagraph_mode(
54
+ CUDAGraphMode[mode], piecewise_capture_available=piecewise_capture_available
55
+ )
56
+ == CUDAGraphMode[expected]
57
+ )
58
+
59
+
60
def test_manager_scopes_varlen_check_without_weakening_runner_cg_mode(monkeypatch):
61
class Backend:
62
@classmethod
vllm/config/compilation.py
+22
@@ -77,6 +77,13 @@ class CUDAGraphMode(enum.Enum):
77
def requires_piecewise_compilation(self) -> bool:
78
return self.has_mode(CUDAGraphMode.PIECEWISE)
79
80
+ def without_piecewise(self) -> "CUDAGraphMode":
81
+ if self == CUDAGraphMode.PIECEWISE:
82
+ return CUDAGraphMode.NONE
83
+ if self == CUDAGraphMode.FULL_AND_PIECEWISE:
84
+ return CUDAGraphMode.FULL_DECODE_ONLY
85
+ return self
86
+
87
def max_cudagraph_mode(self) -> "CUDAGraphMode":
88
return CUDAGraphMode(max(self.value)) if self.separate_routine() else self
89
@@ -1382,6 +1389,7 @@ class CompilationConfig:
1389
kv_cache_config: "KVCacheConfig | None" = None,
1390
max_num_reqs: int | None = None,
1391
is_profiling: bool = False,
1392
+ piecewise_capture_available: bool = True,
1393
) -> CUDAGraphMode:
1394
from vllm.v1.attention.backend import AttentionCGSupport
1395
@@ -1464,6 +1472,20 @@ class CompilationConfig:
1472
cudagraph_mode = CUDAGraphMode.NONE
1473
logger.warning(msg)
1474
1475
+ if (
1476
+ not piecewise_capture_available
1477
+ and cudagraph_mode.requires_piecewise_compilation()
1478
+ ):
1479
+ fallback_mode = cudagraph_mode.without_piecewise()
1480
+ logger.warning_once(
1481
+ "Cudagraph mode %s requires piecewise capture, but the loaded "
1482
+ "model provides neither a compiled submodule nor breakable CUDA "
1483
+ "graphs. Overriding to %s.",
1484
+ cudagraph_mode,
1485
+ fallback_mode,
1486
+ )
1487
+ cudagraph_mode = fallback_mode
1488
+
1489
# double check that we can support full cudagraph if they are requested
1490
# even after automatic downgrades
1491
if (
vllm/config/vllm.py
+33
-17
@@ -991,6 +991,28 @@ class VllmConfig:
991
)
992
speculative_config.num_speculative_tokens_per_batch_size = None
993
994
+ def _normalize_piecewise_cudagraph_mode(
995
+ self, *, breakable_cudagraph_enabled: bool
996
+ ) -> None:
997
+ compilation_config = self.compilation_config
998
+ if (
999
+ compilation_config.cudagraph_mode.requires_piecewise_compilation()
1000
+ and compilation_config.mode != CompilationMode.VLLM_COMPILE
1001
+ and not breakable_cudagraph_enabled
1002
+ ):
1003
+ fallback_mode = compilation_config.cudagraph_mode.without_piecewise()
1004
+ logger.info_once(
1005
+ "Cudagraph mode %s is not compatible with compilation mode %s. "
1006
+ "Overriding to %s.",
1007
+ compilation_config.cudagraph_mode,
1008
+ compilation_config.mode,
1009
+ fallback_mode,
1010
+ )
1011
+ compilation_config.cudagraph_mode = fallback_mode
1012
+ if fallback_mode == CUDAGraphMode.NONE:
1013
+ compilation_config.max_cudagraph_capture_size = 0
1014
+ compilation_config.cudagraph_capture_sizes = []
1015
+
1016
def _post_init_kv_transfer_config(self) -> None:
1017
"""Update KVTransferConfig based on top-level configs in VllmConfig.
1018
@@ -1536,18 +1558,9 @@ class VllmConfig:
1558
self._maybe_disable_dynamic_sd_for_data_parallel()
1559
self._maybe_override_dynamic_sd_cudagraph_mode()
1560
1539
- if (
1540
- self.compilation_config.cudagraph_mode.requires_piecewise_compilation()
1541
- and self.compilation_config.mode != CompilationMode.VLLM_COMPILE
1542
- and not envs.VLLM_USE_BREAKABLE_CUDAGRAPH
1543
- ):
1544
- logger.info_once(
1545
- "Cudagraph mode %s is not compatible with compilation mode %s."
1546
- "Overriding to NONE.",
1547
- self.compilation_config.cudagraph_mode,
1548
- self.compilation_config.mode,
1549
- )
1550
- self.compilation_config.cudagraph_mode = CUDAGraphMode.NONE
1561
+ self._normalize_piecewise_cudagraph_mode(
1562
+ breakable_cudagraph_enabled=breakable_cudagraph_enabled
1563
+ )
1564
1565
# async tp is built on top of sequence parallelism and requires it.
1566
pass_config = self.compilation_config.pass_config
@@ -1716,6 +1729,10 @@ class VllmConfig:
1729
)
1730
current_platform.check_and_update_config(self)
1731
1732
+ self._normalize_piecewise_cudagraph_mode(
1733
+ breakable_cudagraph_enabled=breakable_cudagraph_enabled
1734
+ )
1735
+
1736
self._resolve_mm_embedding_inputs()
1737
self._resolve_mm_processor_device()
1738
self._validate_mm_processor_device()
@@ -2728,12 +2745,11 @@ class VllmConfig:
2745
"Adaptive verification is not currently compatible with LoRA"
2746
)
2747
2731
- if self.compilation_config.cudagraph_mode == CUDAGraphMode.NONE:
2732
- # The draft budget divides by step costs profiled from captured
2733
- # cudagraphs; eager execution captures none.
2748
+ if not self.compilation_config.cudagraph_mode.has_full_cudagraphs():
2749
raise ValueError(
2735
- "Adaptive verification is not currently compatible with "
2736
- "enforce_eager/cudagraph_mode=none"
2750
+ "Adaptive verification requires full CUDA graphs. Use cudagraph "
2751
+ "mode FULL, FULL_DECODE_ONLY, or FULL_AND_PIECEWISE, or disable "
2752
+ "adaptive verification."
2753
)
2754
2755
if self.parallel_config.pipeline_parallel_size > 1:
vllm/platforms/rocm.py
+17
@@ -880,8 +880,25 @@ class RocmPlatform(Platform):
880
@classmethod
881
def apply_config_platform_defaults(cls, vllm_config: "VllmConfig") -> None:
882
from vllm._aiter_ops import rocm_aiter_ops
883
+ from vllm.config.compilation import CUDAGraphMode
884
885
compilation_config = vllm_config.compilation_config
886
+ model_config = vllm_config.model_config
887
+ if (
888
+ compilation_config.cudagraph_mode is None
889
+ and model_config is not None
890
+ and on_gfx950()
891
+ and vllm_config.use_v2_model_runner
892
+ and model_config.architecture
893
+ in {
894
+ "DeepseekV4ForCausalLM",
895
+ "DeepseekV4ForConditionalGeneration",
896
+ }
897
+ ):
898
+ # Default to eager after reported gfx950/MRV2 accuracy regressions:
899
+ # https://github.com/vllm-project/vllm/issues/52644
900
+ compilation_config.cudagraph_mode = CUDAGraphMode.NONE
901
+
902
use_aiter_fused_moe = rocm_aiter_ops.is_fused_moe_enabled()
903
use_aiter_fp8_linear = rocm_aiter_ops.is_linear_fp8_enabled()
904
use_aiter_fused_se = rocm_aiter_ops.is_fusion_moe_shared_experts_enabled()
vllm/v1/worker/gpu/model_runner.py
+10
-1
@@ -104,6 +104,7 @@ from vllm.v1.worker.gpu.cp_utils import maybe_prepare_dcp_local_seq_lens
104
from vllm.v1.worker.gpu.cudagraph_utils import (
105
BatchExecutionDescriptor,
106
ModelCudaGraphManager,
107
+ has_compiled_submodule,
108
make_cudagraph_stats,
109
)
110
from vllm.v1.worker.gpu.cudagraph_utils import (
@@ -152,6 +153,7 @@ from vllm.v1.worker.gpu.spec_decode import init_speculator
153
from vllm.v1.worker.gpu.spec_decode.adaptive_verification import (
154
AdaptiveVerificationManager,
155
maybe_create_adaptive_verification_manager,
156
+ resolve_adaptive_cudagraph_mode,
157
)
158
from vllm.v1.worker.gpu.spec_decode.eagle.eagle3_utils import (
159
set_eagle3_aux_hidden_state_layers,
@@ -674,8 +676,14 @@ class GPUModelRunner(LoRAModelRunnerMixin):
676
self.kv_cache_config,
677
use_replayssm=self.cache_config.use_replayssm,
678
)
679
+ piecewise_capture_available = bool(
680
+ envs.VLLM_USE_BREAKABLE_CUDAGRAPH or has_compiled_submodule(self.model)
681
+ )
682
if self.adaptive_verification is not None:
678
- self.compilation_config.cudagraph_mode = CUDAGraphMode.FULL_AND_PIECEWISE
683
+ self.compilation_config.cudagraph_mode = resolve_adaptive_cudagraph_mode(
684
+ self.compilation_config.cudagraph_mode,
685
+ piecewise_capture_available=piecewise_capture_available,
686
+ )
687
cudagraph_mode = self.compilation_config.resolve_cudagraph_mode_and_sizes(
688
attn_cg_support.min_cg_support,
689
attn_cg_support.min_cg_attn_backend,
@@ -685,6 +693,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
693
kv_cache_config=self.kv_cache_config,
694
max_num_reqs=self.max_num_reqs,
695
is_profiling=is_profiling,
696
+ piecewise_capture_available=piecewise_capture_available,
697
)
698
self.cudagraph_manager = ModelCudaGraphManager(
699
self.vllm_config,
vllm/v1/worker/gpu/spec_decode/adaptive_verification.py
+11
@@ -10,6 +10,7 @@ import numpy as np
10
import torch
11
12
import vllm.envs as envs
13
+from vllm.config.compilation import CUDAGraphMode
14
from vllm.distributed.parallel_state import get_tp_group
15
from vllm.logger import init_logger
16
from vllm.utils.gpu_sync_debug import gpu_sync_allowed
@@ -33,6 +34,16 @@ if TYPE_CHECKING:
34
from vllm.v1.worker.utils import AttentionGroup
35
36
37
+def resolve_adaptive_cudagraph_mode(
38
+ mode: CUDAGraphMode, *, piecewise_capture_available: bool
39
+) -> CUDAGraphMode:
40
+ """Select a separate decode route for variable-length verification."""
41
+ assert mode.has_full_cudagraphs()
42
+ if mode == CUDAGraphMode.FULL_DECODE_ONLY or not piecewise_capture_available:
43
+ return CUDAGraphMode.FULL_DECODE_ONLY
44
+ return CUDAGraphMode.FULL_AND_PIECEWISE
45
+
46
+
47
def _assign_draft_token_budget(
48
confidence_probs: torch.Tensor,
49
idx_mapping: torch.Tensor,