@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / f075693da7
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 14, 05:30 UTC

[V1] address post issues related to #20059 (part 1) (#23046)

Signed-off-by: fhl2000 <63384265+fhl2000@users.noreply.github.com> Co-authored-by: Luka Govedič <ProExpertProg@users.noreply.github.com>

fhl2000 committed Sep 27, 2025 at 03:58 UTC f075693da767a8246274219c284ef687c4fcaf51
13 files changed +346 -290
tests/compile/piecewise/test_full_cudagraph.py
+1 -85
@@ -3,12 +3,11 @@
3 import contextlib
4 import os
5 import weakref
6 -from dataclasses import dataclass
7 -from typing import Optional
6
7 import pytest
8
9 from tests.utils import wait_for_gpu_memory_to_clear
10 +from tests.v1.attention.utils import full_cg_backend_configs as backend_configs
11 from vllm import LLM, SamplingParams
12 from vllm.config import CompilationConfig
13 from vllm.platforms import current_platform
@@ -33,89 +32,6 @@ def temporary_environ(env_vars):
32 os.environ[k] = v
33
34
36 -@dataclass
37 -class BackendConfig:
38 - name: str
39 - env_vars: dict
40 - comp_config: dict
41 - specific_gpu_arch: Optional[tuple] = None
42 -
43 -
44 -# Define all backend configurations of full cudagraph to be tested
45 -backend_configs = {
46 - # FA3 on Hopper
47 - "FA3":
48 - BackendConfig(name="FA3",
49 - env_vars={
50 - "VLLM_FLASH_ATTN_VERSION": "3",
51 - "VLLM_FLASH_ATTN_MAX_NUM_SPLITS_FOR_CUDA_GRAPH": "16",
52 - },
53 - comp_config={
54 - "cudagraph_mode": "FULL",
55 - },
56 - specific_gpu_arch=(9, 0)),
57 - # FlashMLA on Hopper
58 - "FlashMLA":
59 - BackendConfig(name="FlashMLA",
60 - env_vars={
61 - "VLLM_ATTENTION_BACKEND": "FLASHMLA",
62 - },
63 - comp_config={
64 - "cudagraph_mode": "FULL_AND_PIECEWISE",
65 - },
66 - specific_gpu_arch=(9, 0)),
67 - # FlashAttention MLA on Hopper
68 - "FlashAttentionMLA":
69 - BackendConfig(name="FlashAttentionMLA",
70 - env_vars={
71 - "VLLM_ATTENTION_BACKEND": "FLASH_ATTN_MLA",
72 - "VLLM_FLASH_ATTN_MAX_NUM_SPLITS_FOR_CUDA_GRAPH": "16",
73 - },
74 - comp_config={
75 - "cudagraph_mode": "FULL_DECODE_ONLY",
76 - },
77 - specific_gpu_arch=(9, 0)),
78 - # Cutlass MLA on Blackwell
79 - "CutlassMLA":
80 - BackendConfig(
81 - name="CutlassMLA",
82 - env_vars={
83 - "VLLM_USE_V1": "1",
84 - "VLLM_ATTENTION_BACKEND": "CUTLASS_MLA",
85 - "FORCE_NUM_KV_SPLITS":
86 - "1", # TODO: remove this when hang issue is fixed
87 - },
88 - comp_config={
89 - "cudagraph_mode": "FULL_AND_PIECEWISE",
90 - "cudagraph_capture_sizes": [16, 32, 64, 128, 256, 512],
91 - },
92 - specific_gpu_arch=(10, 0)),
93 - # FA2
94 - "FA2":
95 - BackendConfig(name="FA2",
96 - env_vars={
97 - "VLLM_FLASH_ATTN_VERSION": "2",
98 - "VLLM_FLASH_ATTN_MAX_NUM_SPLITS_FOR_CUDA_GRAPH": "16",
99 - },
100 - comp_config={
101 - "cudagraph_mode": "FULL",
102 - }),
103 - # Triton Attention
104 - "TritonAttn":
105 - BackendConfig(name="TritonAttn",
106 - env_vars={"VLLM_ATTENTION_BACKEND": "TRITON_ATTN"},
107 - comp_config={
108 - "cudagraph_mode": "FULL",
109 - }),
110 - # FlashInfer
111 - "FlashInfer":
112 - BackendConfig(name="FlashInfer",
113 - env_vars={"VLLM_ATTENTION_BACKEND": "FLASHINFER"},
114 - comp_config={
115 - "cudagraph_mode": "FULL_AND_PIECEWISE",
116 - }),
117 -}
118 -
35 test_params_full_cudagraph = []
36
37 # deepseek-ai/DeepSeek-V2-Lite with MLA
tests/compile/test_config.py
+65 -2
@@ -4,7 +4,7 @@ import pytest
4
5 import vllm
6 from vllm.compilation.counter import compilation_counter
7 -from vllm.config import CompilationConfig, VllmConfig
7 +from vllm.config import CompilationConfig, CUDAGraphMode, VllmConfig
8 from vllm.utils import _is_torch_equal_or_newer
9
10
@@ -106,7 +106,6 @@ def test_dynamo_as_is(vllm_runner, monkeypatch):
106 def test_no_compilation(vllm_runner, monkeypatch):
107 # Disable multiprocessing so that the counter is in the same process
108 monkeypatch.setenv('VLLM_ENABLE_V1_MULTIPROCESSING', '0')
109 -
109 with (
110 compilation_counter.expect(num_graphs_seen=0,
111 dynamo_as_is_count=0),
@@ -131,3 +130,67 @@ def test_enforce_eager(vllm_runner, monkeypatch):
130 enforce_eager=True,
131 gpu_memory_utilization=0.4) as _):
132 pass
133 +
134 +
135 +def test_splitting_ops_dynamic():
136 + # Default config
137 + config = VllmConfig()
138 + assert config.compilation_config.cudagraph_mode == \
139 + CUDAGraphMode.FULL_AND_PIECEWISE
140 + assert config.compilation_config.splitting_ops_contain_attention()
141 +
142 + # When use_inductor_graph_partition=True
143 + if _is_torch_equal_or_newer('2.9.0.dev'):
144 + # inductor graph partition is only available in PyTorch 2.9+.
145 + # this is a fast config check so we are not using pytest.skip.
146 + config = VllmConfig(compilation_config=CompilationConfig(
147 + use_inductor_graph_partition=True,
148 + splitting_ops=["silly_attention"]))
149 + # should ignore splitting_ops
150 + assert config.compilation_config.splitting_ops == []
151 +
152 + # When attn_fusion pass enabled.
153 + config = VllmConfig(compilation_config=CompilationConfig(
154 + pass_config={
155 + "enable_attn_fusion": True,
156 + "enable_noop": True
157 + },
158 + custom_ops=["+quant_fp8"],
159 + cudagraph_mode=CUDAGraphMode.PIECEWISE,
160 + ))
161 + assert config.compilation_config.splitting_ops == []
162 + # cudagraph mode also fall back to FULL
163 + assert config.compilation_config.cudagraph_mode == \
164 + CUDAGraphMode.FULL
165 +
166 + # splitting_ops can not contain attention ops when attn_fusion
167 + # pass enabled.
168 + with pytest.raises(AssertionError):
169 + config = VllmConfig(compilation_config=CompilationConfig(
170 + pass_config={
171 + "enable_attn_fusion": True,
172 + "enable_noop": True
173 + },
174 + custom_ops=["+quant_fp8"],
175 + cudagraph_mode=CUDAGraphMode.PIECEWISE,
176 + # work around for accessing all attntion ops
177 + splitting_ops=CompilationConfig()._attention_ops,
178 + ))
179 +
180 + # When both use_inductor_graph_partition and attn_fusion pass enabled.
181 + if _is_torch_equal_or_newer('2.9.0.dev'):
182 + config = VllmConfig(compilation_config=CompilationConfig(
183 + use_inductor_graph_partition=True,
184 + pass_config={
185 + "enable_attn_fusion": True,
186 + "enable_noop": True
187 + },
188 + custom_ops=["+quant_fp8"],
189 + cudagraph_mode=CUDAGraphMode.PIECEWISE,
190 + ))
191 + assert config.compilation_config.splitting_ops == []
192 + # enable_attn_fusion is directly support under
193 + # use_inductor_graph_partition=True, and cudagraph_mode
194 + # is unchanged.
195 + assert config.compilation_config.cudagraph_mode == \
196 + CUDAGraphMode.PIECEWISE
tests/v1/attention/utils.py
+86 -1
@@ -3,7 +3,7 @@
3 """Utility functions for attention-related v1 tests."""
4
5 from dataclasses import dataclass
6 -from typing import Union
6 +from typing import Optional, Union
7
8 import pytest
9 import torch
@@ -260,3 +260,88 @@ def create_dummy_kv_cache(block_size: int,
260 dtype=dtype,
261 device=device)
262 return kv_cache
263 +
264 +
265 +@dataclass
266 +class BackendConfig:
267 + name: str
268 + env_vars: dict
269 + comp_config: dict # compilation config
270 + specific_gpu_arch: Optional[tuple] = None
271 +
272 +
273 +# Define all backend configurations of full cudagraph to be tested
274 +full_cg_backend_configs = {
275 + # FA3 on Hopper
276 + "FA3":
277 + BackendConfig(name="FA3",
278 + env_vars={
279 + "VLLM_ATTENTION_BACKEND": "FLASH_ATTN",
280 + "VLLM_FLASH_ATTN_VERSION": "3",
281 + "VLLM_FLASH_ATTN_MAX_NUM_SPLITS_FOR_CUDA_GRAPH": "16",
282 + },
283 + comp_config={
284 + "cudagraph_mode": "FULL",
285 + },
286 + specific_gpu_arch=(9, 0)),
287 + # FlashMLA on Hopper
288 + "FlashMLA":
289 + BackendConfig(name="FlashMLA",
290 + env_vars={
291 + "VLLM_ATTENTION_BACKEND": "FLASHMLA",
292 + },
293 + comp_config={
294 + "cudagraph_mode": "FULL_AND_PIECEWISE",
295 + },
296 + specific_gpu_arch=(9, 0)),
297 + # Cutlass MLA on Blackwell
298 + "CutlassMLA":
299 + BackendConfig(
300 + name="CutlassMLA",
301 + env_vars={
302 + "VLLM_USE_V1": "1",
303 + "VLLM_ATTENTION_BACKEND": "CUTLASS_MLA",
304 + "FORCE_NUM_KV_SPLITS":
305 + "1", # TODO: remove this when hang issue is fixed
306 + },
307 + comp_config={
308 + "cudagraph_mode": "FULL_AND_PIECEWISE",
309 + },
310 + specific_gpu_arch=(10, 0)),
311 + # FlashAttention MLA on Hopper
312 + "FlashAttentionMLA":
313 + BackendConfig(name="FlashAttentionMLA",
314 + env_vars={
315 + "VLLM_ATTENTION_BACKEND": "FLASH_ATTN_MLA",
316 + "VLLM_FLASH_ATTN_MAX_NUM_SPLITS_FOR_CUDA_GRAPH": "16",
317 + },
318 + comp_config={
319 + "cudagraph_mode": "FULL_DECODE_ONLY",
320 + },
321 + specific_gpu_arch=(9, 0)),
322 + # FA2
323 + "FA2":
324 + BackendConfig(name="FA2",
325 + env_vars={
326 + "VLLM_ATTENTION_BACKEND": "FLASH_ATTN",
327 + "VLLM_FLASH_ATTN_VERSION": "2",
328 + "VLLM_FLASH_ATTN_MAX_NUM_SPLITS_FOR_CUDA_GRAPH": "16",
329 + },
330 + comp_config={
331 + "cudagraph_mode": "FULL_AND_PIECEWISE",
332 + }),
333 + # Triton Attention
334 + "TritonAttn":
335 + BackendConfig(name="TritonAttn",
336 + env_vars={"VLLM_ATTENTION_BACKEND": "TRITON_ATTN"},
337 + comp_config={
338 + "cudagraph_mode": "FULL_AND_PIECEWISE",
339 + }),
340 + # FlashInfer
341 + "FlashInfer":
342 + BackendConfig(name="FlashInfer",
343 + env_vars={"VLLM_ATTENTION_BACKEND": "FLASHINFER"},
344 + comp_config={
345 + "cudagraph_mode": "FULL_AND_PIECEWISE",
346 + }),
347 +}
tests/v1/cudagraph/test_cudagraph_dispatch.py
+26 -35
@@ -45,39 +45,22 @@ def _create_vllm_config(compilation_config: CompilationConfig,
45 class TestCudagraphDispatcher:
46
47 @pytest.mark.parametrize(
48 - "params",
48 + "case_id,cudagraph_mode_str,compilation_level",
49 [
50 # Test case 0: Full CG for mixed batches, no separate routine
51 - {
52 - "case_id": 0,
53 - "cudagraph_mode": "FULL",
54 - "compilation_level": CompilationLevel.NO_COMPILATION,
55 - },
51 + (0, "FULL", CompilationLevel.NO_COMPILATION),
52 # Test case 1: Full CG for uniform batches, piecewise for mixed
57 - {
58 - "case_id": 1,
59 - "cudagraph_mode": "FULL_AND_PIECEWISE",
60 - "compilation_level": CompilationLevel.PIECEWISE,
61 - },
53 + (1, "FULL_AND_PIECEWISE", CompilationLevel.NO_COMPILATION),
54 # Test case 2: Full CG for uniform batches, no CG for mixed
63 - {
64 - "case_id": 2,
65 - "cudagraph_mode": "FULL_DECODE_ONLY",
66 - "compilation_level": CompilationLevel.NO_COMPILATION,
67 - },
55 + (2, "FULL_DECODE_ONLY", CompilationLevel.NO_COMPILATION),
56 # Test case 3: Piecewise for all
69 - {
70 - "case_id": 3,
71 - "cudagraph_mode": "PIECEWISE",
72 - "compilation_level": CompilationLevel.PIECEWISE,
73 - },
57 + (3, "PIECEWISE", CompilationLevel.PIECEWISE),
58 ])
75 - def test_dispatcher(self, params):
59 + def test_dispatcher(self, cudagraph_mode_str, compilation_level):
60 # Setup dispatcher
77 - comp_config = CompilationConfig(
78 - cudagraph_mode=params["cudagraph_mode"],
79 - level=params["compilation_level"],
80 - cudagraph_capture_sizes=[1, 8])
61 + comp_config = CompilationConfig(cudagraph_mode=cudagraph_mode_str,
62 + level=compilation_level,
63 + cudagraph_capture_sizes=[1, 8])
64
65 config = _create_vllm_config(comp_config, max_num_seqs=8)
66 dispatcher = CudagraphDispatcher(config)
@@ -86,11 +69,11 @@ class TestCudagraphDispatcher:
69 uniform_decode_query_len=1)
70
71 # Verify the key is initialized correctly
89 - if params["cudagraph_mode"] in ["FULL_AND_PIECEWISE", "PIECEWISE"]:
72 + if cudagraph_mode_str in ["FULL_AND_PIECEWISE", "PIECEWISE"]:
73 assert len(dispatcher.cudagraph_keys[CUDAGraphMode.PIECEWISE]) == 2
74 else:
75 assert len(dispatcher.cudagraph_keys[CUDAGraphMode.PIECEWISE]) == 0
93 - if params["cudagraph_mode"] not in ["NONE", "PIECEWISE"]:
76 + if cudagraph_mode_str not in ["NONE", "PIECEWISE"]:
77 assert len(dispatcher.cudagraph_keys[CUDAGraphMode.FULL]) == 2
78 else:
79 assert len(dispatcher.cudagraph_keys[CUDAGraphMode.FULL]) == 0
@@ -99,10 +82,10 @@ class TestCudagraphDispatcher:
82 # 1. non-uniform batch, size in cudagraph size list
83 desc_full_exact = BatchDescriptor(num_tokens=8, uniform_decode=False)
84 rt_mode, key = dispatcher.dispatch(desc_full_exact)
102 - if params["cudagraph_mode"] == "FULL":
85 + if cudagraph_mode_str == "FULL":
86 assert rt_mode == CUDAGraphMode.FULL
87 assert key == desc_full_exact
105 - elif params["cudagraph_mode"] in ["FULL_AND_PIECEWISE", "PIECEWISE"]:
88 + elif cudagraph_mode_str in ["FULL_AND_PIECEWISE", "PIECEWISE"]:
89 assert rt_mode == CUDAGraphMode.PIECEWISE
90 assert key == desc_full_exact
91 else:
@@ -111,15 +94,13 @@ class TestCudagraphDispatcher:
94 # 2. uniform decode batch, size in cudagraph size list
95 desc_uniform_exact = BatchDescriptor(num_tokens=8, uniform_decode=True)
96 rt_mode, key = dispatcher.dispatch(desc_uniform_exact)
114 - if params["cudagraph_mode"] == "FULL":
97 + if cudagraph_mode_str == "FULL":
98 assert rt_mode == CUDAGraphMode.FULL
99 assert key == desc_uniform_exact.non_uniform
117 - elif params["cudagraph_mode"] in [
118 - "FULL_DECODE_ONLY", "FULL_AND_PIECEWISE"
119 - ]:
100 + elif cudagraph_mode_str in ["FULL_DECODE_ONLY", "FULL_AND_PIECEWISE"]:
101 assert rt_mode == CUDAGraphMode.FULL
102 assert key == desc_uniform_exact
122 - elif params["cudagraph_mode"] == "PIECEWISE":
103 + elif cudagraph_mode_str == "PIECEWISE":
104 assert rt_mode == CUDAGraphMode.PIECEWISE
105 assert key == desc_uniform_exact.non_uniform
106 else:
@@ -131,6 +112,16 @@ class TestCudagraphDispatcher:
112 assert rt_mode == CUDAGraphMode.NONE
113 assert key is None
114
115 + # 4. Cascade attention should have a fall back mode
116 + desc_full_exact = BatchDescriptor(num_tokens=8, uniform_decode=False)
117 + rt_mode, key = dispatcher.dispatch(desc_full_exact,
118 + use_cascade_attn=True)
119 + if "PIECEWISE" in cudagraph_mode_str: # string contains check
120 + assert rt_mode == CUDAGraphMode.PIECEWISE
121 + assert key == desc_full_exact.non_uniform
122 + else:
123 + assert rt_mode == CUDAGraphMode.NONE
124 +
125
126 @pytest.mark.skipif(not current_platform.is_cuda(), reason="Skip if not cuda")
127 class TestCUDAGraphWrapper:
tests/v1/cudagraph/test_cudagraph_mode.py
+9 -75
@@ -4,12 +4,11 @@ import contextlib
4 import os
5 import weakref
6 from contextlib import ExitStack
7 -from dataclasses import dataclass
8 -from typing import Optional
7
8 import pytest
9
10 from tests.utils import wait_for_gpu_memory_to_clear
11 +from tests.v1.attention.utils import full_cg_backend_configs as backend_configs
12 from vllm import LLM
13 from vllm.config import CompilationConfig
14 from vllm.platforms import current_platform
@@ -34,74 +33,6 @@ def temporary_environ(env_vars):
33 os.environ[k] = v
34
35
37 -@dataclass
38 -class BackendConfig:
39 - name: str
40 - env_vars: dict
41 - comp_config: dict
42 - specific_gpu_arch: Optional[tuple] = None
43 -
44 -
45 -# Define all backend configurations of full cudagraph to be tested
46 -backend_configs = {
47 - # FA3 on Hopper
48 - "FA3":
49 - BackendConfig(name="FA3",
50 - env_vars={
51 - "VLLM_FLASH_ATTN_VERSION": "3",
52 - "VLLM_FLASH_ATTN_MAX_NUM_SPLITS_FOR_CUDA_GRAPH": "16",
53 - },
54 - comp_config={
55 - "cudagraph_mode": "FULL",
56 - },
57 - specific_gpu_arch=(9, 0)),
58 - # FlashMLA on Hopper
59 - "FlashMLA":
60 - BackendConfig(name="FlashMLA",
61 - env_vars={
62 - "VLLM_ATTENTION_BACKEND": "FLASHMLA",
63 - },
64 - comp_config={
65 - "cudagraph_mode": "FULL_AND_PIECEWISE",
66 - },
67 - specific_gpu_arch=(9, 0)),
68 - # FlashAttention MLA on Hopper
69 - "FlashAttentionMLA":
70 - BackendConfig(name="FlashAttentionMLA",
71 - env_vars={
72 - "VLLM_ATTENTION_BACKEND": "FLASH_ATTN_MLA",
73 - "VLLM_FLASH_ATTN_MAX_NUM_SPLITS_FOR_CUDA_GRAPH": "16",
74 - },
75 - comp_config={
76 - "cudagraph_mode": "FULL_DECODE_ONLY",
77 - },
78 - specific_gpu_arch=(9, 0)),
79 - # FA2
80 - "FA2":
81 - BackendConfig(name="FA2",
82 - env_vars={
83 - "VLLM_FLASH_ATTN_VERSION": "2",
84 - "VLLM_FLASH_ATTN_MAX_NUM_SPLITS_FOR_CUDA_GRAPH": "16",
85 - },
86 - comp_config={
87 - "cudagraph_mode": "FULL_AND_PIECEWISE",
88 - }),
89 - # Triton Attention
90 - "TritonAttn":
91 - BackendConfig(name="TritonAttn",
92 - env_vars={"VLLM_ATTENTION_BACKEND": "TRITON_ATTN"},
93 - comp_config={
94 - "cudagraph_mode": "FULL_AND_PIECEWISE",
95 - }),
96 - # FlashInfer
97 - "FlashInfer":
98 - BackendConfig(name="FlashInfer",
99 - env_vars={"VLLM_ATTENTION_BACKEND": "FLASHINFER"},
100 - comp_config={
101 - "cudagraph_mode": "FULL_AND_PIECEWISE",
102 - }),
103 -}
104 -
36 # test attention backend and cudagraph_mode combo
37 # (backend_name, cudagraph_mode, supported)
38 combo_cases_1 = [
@@ -114,9 +45,10 @@ combo_cases_1 = [
45 ]
46
47
117 -@pytest.mark.parametrize("combo_case", combo_cases_1)
118 -def test_backend_and_cudagraph_mode_combo(combo_case):
119 - backend_name, cudagraph_mode, supported = combo_case
48 +@pytest.mark.parametrize("backend_name, cudagraph_mode, supported",
49 + combo_cases_1)
50 +def test_backend_and_cudagraph_mode_combo(backend_name, cudagraph_mode,
51 + supported):
52 if backend_name == "FlashInfer":
53 try:
54 import flashinfer # noqa: F401
@@ -142,7 +74,7 @@ def test_backend_and_cudagraph_mode_combo(combo_case):
74 compilation_config=CompilationConfig(
75 level=3, cudagraph_mode=cudagraph_mode))
76 llm.generate(["Hello, my name is"] * 10)
145 -
77 + # when above code raises, `llm` may be undefined, so we need to catch that
78 try:
79 llm = weakref.proxy(llm)
80 del llm
@@ -173,7 +105,8 @@ combo_cases_2 = [
105 ]
106
107
176 -@pytest.mark.parametrize("combo_case", combo_cases_2)
108 +@pytest.mark.parametrize("backend_name,cudagraph_mode,compilation_level,"\
109 + "supported", combo_cases_2)
110 def test_cudagraph_compilation_combo(combo_case):
111 backend_name, cudagraph_mode, compilation_level, supported\
112 = combo_case
@@ -192,6 +125,7 @@ def test_cudagraph_compilation_combo(combo_case):
125 compilation_config=CompilationConfig(
126 level=compilation_level, cudagraph_mode=cudagraph_mode))
127 llm.generate(["Hello, my name is"] * 10)
128 + # when above code raises, `llm` may be undefined, so we need to catch that
129 try:
130 llm = weakref.proxy(llm)
131 del llm
vllm/compilation/backends.py
+3 -3
@@ -340,15 +340,15 @@ class PiecewiseCompileInterpreter(torch.fx.Interpreter):
340 num_graphs=len(self.compile_submod_names),
341 runtime_shape=None)
342 # Lazy import here to avoid circular import
343 - from .cuda_piecewise_backend import PiecewiseBackend
343 + from .piecewise_backend import PiecewiseBackend
344
345 piecewise_backend = PiecewiseBackend(
346 submod, self.vllm_config, index,
347 len(self.compile_submod_names), sym_shape_indices,
348 compiled_graph_for_dynamic_shape, self.vllm_backend)
349
350 - if (self.compilation_config.cudagraph_mode != CUDAGraphMode.NONE
351 - and
350 + if (self.compilation_config.cudagraph_mode.\
351 + has_piecewise_cudagraphs() and
352 not self.compilation_config.use_inductor_graph_partition):
353 # We're using Dynamo-based piecewise splitting, so we wrap
354 # the whole subgraph with a static graph wrapper.
vllm/compilation/decorators.py
+2 -2
@@ -336,7 +336,7 @@ def maybe_use_cudagraph_partition_wrapper(vllm_config: VllmConfig):
336 from vllm.config import CUDAGraphMode
337
338 compilation_config = vllm_config.compilation_config
339 - if (compilation_config.cudagraph_mode != CUDAGraphMode.NONE
339 + if (compilation_config.cudagraph_mode.has_piecewise_cudagraphs()
340 and compilation_config.use_inductor_graph_partition):
341 from torch._inductor.utils import CUDAGraphWrapperMetadata
342
@@ -365,7 +365,7 @@ def maybe_use_cudagraph_partition_wrapper(vllm_config: VllmConfig):
365
366 yield
367
368 - if (compilation_config.cudagraph_mode != CUDAGraphMode.NONE
368 + if (compilation_config.cudagraph_mode.has_piecewise_cudagraphs()
369 and compilation_config.use_inductor_graph_partition):
370 torch._inductor.utils.set_customized_partition_wrappers(None)
371
vllm/compilation/piecewise_backend.py renamed
vllm/config/__init__.py
+25 -12
@@ -459,15 +459,22 @@ class VllmConfig:
459 "to True to enable.")
460 current_platform.check_and_update_config(self)
461
462 - # final check of cudagraph mode after platform-specific update
462 + # Do this after all the updates to compilation_config.level
463 + if envs.VLLM_USE_V1 and \
464 + self.compilation_config.level == CompilationLevel.PIECEWISE:
465 + self.compilation_config.set_splitting_ops_for_v1()
466 +
467 + # final check of cudagraph mode after all possible updates
468 if envs.VLLM_USE_V1 and current_platform.is_cuda_alike():
464 - if self.compilation_config.cudagraph_mode == CUDAGraphMode.FULL \
469 + if self.compilation_config.cudagraph_mode.has_full_cudagraphs()\
470 and self.model_config is not None and \
466 - not self.model_config.disable_cascade_attn:
467 - logger.info("CUDAGraphMode.FULL is not supported with "
468 - "cascade attention currently. Disabling cascade"
469 - "attention.")
470 - self.model_config.disable_cascade_attn = True
471 + not self.model_config.disable_cascade_attn and\
472 + not self.compilation_config.cudagraph_mode.\
473 + has_piecewise_cudagraphs():
474 + logger.warning_once(
475 + "No piecewise cudagraph for executing cascade attention."
476 + " Will fall back to eager execution if a batch runs "
477 + "into cascade attentions")
478
479 if self.compilation_config.cudagraph_mode\
480 .requires_piecewise_compilation():
@@ -477,6 +484,12 @@ class VllmConfig:
484 "when cudagraph_mode piecewise cudagraphs is used, "\
485 f"cudagraph_mode={self.compilation_config.cudagraph_mode}"
486
487 + # final migrate the deprecated flags
488 + self.compilation_config.use_cudagraph = self.compilation_config.\
489 + cudagraph_mode!= CUDAGraphMode.NONE
490 + self.compilation_config.full_cuda_graph = self.compilation_config.\
491 + cudagraph_mode.has_full_cudagraphs()
492 +
493 if self.parallel_config.enable_dbo:
494 a2a_backend = envs.VLLM_ALL2ALL_BACKEND
495 assert a2a_backend in \
@@ -487,14 +500,14 @@ class VllmConfig:
500 "variable to deepep_low_latency or deepep_high_throughput and "\
501 "install the DeepEP kernels."
502
503 + if not self.model_config.disable_cascade_attn:
504 + self.model_config.disable_cascade_attn = True
505 + logger.warning_once(
506 + "Disabling cascade attention when DBO is enabled.")
507 +
508 if not self.instance_id:
509 self.instance_id = random_uuid()[:5]
510
493 - # Do this after all the updates to compilation_config.level
494 - if envs.VLLM_USE_V1 and \
495 - self.compilation_config.level == CompilationLevel.PIECEWISE:
496 - self.compilation_config.set_splitting_ops_for_v1()
497 -
511 if (envs.VLLM_USE_V1
512 and not self.scheduler_config.disable_hybrid_kv_cache_manager):
513 # logger should only print warning message for hybrid models. As we
vllm/config/compilation.py
+76 -37
@@ -61,9 +61,17 @@ class CUDAGraphMode(enum.Enum):
61 def has_full_cudagraphs(self) -> bool:
62 return self.max_cudagraph_mode() == CUDAGraphMode.FULL
63
64 + def has_piecewise_cudagraphs(self) -> bool:
65 + return self.requires_piecewise_compilation()
66 +
67 def separate_routine(self) -> bool:
68 return isinstance(self.value, tuple)
69
70 + def valid_runtime_modes(self) -> bool:
71 + return self in [
72 + CUDAGraphMode.NONE, CUDAGraphMode.PIECEWISE, CUDAGraphMode.FULL
73 + ]
74 +
75
76 @config
77 @dataclass
@@ -269,7 +277,8 @@ class CompilationConfig:
277 Note that this is orthogonal to the cudagraph capture logic
278 outside of compilation.
279 Warning: This flag is deprecated and will be removed in the next major or
272 - minor release, i.e. v0.11.0 or v1.0.0. Please use cudagraph_mode instead.
280 + minor release, i.e. v0.11.0 or v1.0.0. Please use cudagraph_mode=PIECEWISE
281 + instead.
282 """
283 cudagraph_num_of_warmups: int = 0
284 """Number of warmup runs for cudagraph.
@@ -294,7 +303,8 @@ class CompilationConfig:
303 flag cannot be used together with splitting_ops. This may provide
304 performance benefits for smaller models.
305 Warning: This flag is deprecated and will be removed in the next major or
297 - minor release, i.e. v0.11.0 or v1.0.0. Please use cudagraph_mode instead.
306 + minor release, i.e. v0.11.0 or v1.0.0. Please use cudagraph_mode=
307 + FULL_AND_PIECEWISE instead.
308 """
309
310 use_inductor_graph_partition: bool = False
@@ -464,7 +474,8 @@ class CompilationConfig:
474 if not self.use_cudagraph:
475 logger.warning("use_cudagraph is deprecated, use "
476 "cudagraph_mode=NONE instead.")
467 - if self.cudagraph_mode is not None:
477 + if self.cudagraph_mode is not None and \
478 + self.cudagraph_mode != CUDAGraphMode.NONE:
479 raise ValueError(
480 "use_cudagraph and cudagraph_mode are mutually"
481 " exclusive, prefer cudagraph_mode since "
@@ -473,7 +484,8 @@ class CompilationConfig:
484 if self.full_cuda_graph:
485 logger.warning("full_cuda_graph is deprecated, use "
486 "cudagraph_mode=FULL instead.")
476 - if self.cudagraph_mode is not None:
487 + if self.cudagraph_mode is not None and \
488 + not self.cudagraph_mode.has_full_cudagraphs():
489 raise ValueError("full_cuda_graph and cudagraph_mode are "
490 "mutually exclusive, prefer cudagraph_mode "
491 "since full_cuda_graph is deprecated.")
@@ -570,48 +582,75 @@ class CompilationConfig:
582 "set_splitting_ops_for_v1 should only be called when "
583 "level is CompilationLevel.PIECEWISE")
584
573 - use_inductor_graph_partition_msg = (
574 - "When use_inductor_graph_partition=True, splitting_ops "
575 - "are ignored and set to an empty list. Instead, "
576 - "\"tags=(torch._C.Tag.cudagraph_unsafe, ),\" is "
577 - "used to annotate custom ops for graph partition.")
585 + if self.use_inductor_graph_partition:
586 + self.set_splitting_ops_for_inductor_graph_partition()
587 + return
588 +
589 + if self.pass_config.enable_attn_fusion:
590 + # here use_inductor_graph_partition is False
591 + self.set_splitting_ops_for_attn_fusion()
592 + return
593
594 if self.splitting_ops is None:
580 - if self.use_inductor_graph_partition:
581 - # When using inductor graph partition, we set splitting_ops
582 - # to be empty and rely on torch._C.Tag.cudagraph_unsafe to
583 - # annotate custom ops as splitting ops.
584 - logger.warning_once(use_inductor_graph_partition_msg)
585 - self.splitting_ops = []
586 - else:
587 - # NOTE: When using full cudagraph, instead of setting an empty
588 - # list and capture the full cudagraph inside the flattened fx
589 - # graph, we keep the piecewise fx graph structure but capture
590 - # the full cudagraph outside the fx graph. This reduces some
591 - # cpu overhead when the runtime batch_size is not cudagraph
592 - # captured. see https://github.com/vllm-project/vllm/pull/20059
593 - # for details. make a copy to avoid mutating the class-level
594 - # list via reference.
595 - self.splitting_ops = list(self._attention_ops)
595 + # NOTE: When using full cudagraph, instead of setting an empty
596 + # list and capture the full cudagraph inside the flattened fx
597 + # graph, we keep the piecewise fx graph structure but capture
598 + # the full cudagraph outside the fx graph. This reduces some
599 + # cpu overhead when the runtime batch_size is not cudagraph
600 + # captured. see https://github.com/vllm-project/vllm/pull/20059
601 + # for details. Make a copy to avoid mutating the class-level
602 + # list via reference.
603 + self.splitting_ops = list(self._attention_ops)
604 elif len(self.splitting_ops) == 0:
605 logger.warning_once(
598 - "Using piecewise compilation with empty "
599 - "splitting_ops and use_inductor_graph_partition"
600 - f"={self.use_inductor_graph_partition}.")
601 - if (self.cudagraph_mode == CUDAGraphMode.PIECEWISE
602 - and not self.use_inductor_graph_partition):
606 + "Using piecewise compilation with empty splitting_ops")
607 + if self.cudagraph_mode == CUDAGraphMode.PIECEWISE:
608 + logger.warning_once(
609 + "Piecewise compilation with empty splitting_ops do not" \
610 + "contains piecewise cudagraph. Setting cudagraph_"
611 + "mode to NONE. Hint: If you are using attention backends "
612 + "that support cudagraph, consider manually setting "
613 + "cudagraph_mode to FULL or FULL_DECODE_ONLY to enable "
614 + "full cudagraphs.")
615 + self.cudagraph_mode = CUDAGraphMode.NONE
616 + elif self.cudagraph_mode == CUDAGraphMode.FULL_AND_PIECEWISE:
617 logger.warning_once(
604 - "When compilation level is piecewise with empty "
605 - "splitting_ops, PIECEWISE cudagraph_mode will be "
606 - "treated as FULL cudagraph_mode. Please ensure you are "
607 - "using attention backends that support cudagraph or set "
608 - "cudagraph_mode to NONE explicitly if encountering "
609 - "any problems.")
618 + "Piecewise compilation with empty splitting_ops do not "
619 + "contains piecewise cudagraph. Setting cudagraph_mode "
620 + "to FULL.")
621 self.cudagraph_mode = CUDAGraphMode.FULL
622 self.splitting_ops = []
612 - elif self.use_inductor_graph_partition:
623 +
624 + def set_splitting_ops_for_inductor_graph_partition(self):
625 + assert self.use_inductor_graph_partition
626 + use_inductor_graph_partition_msg = (
627 + "When use_inductor_graph_partition=True, splitting_ops "
628 + "are ignored and set to an empty list. Instead, "
629 + "\"tags=(torch._C.Tag.cudagraph_unsafe, ),\" is "
630 + "used to annotate custom ops for graph partition.")
631 + if self.splitting_ops is not None and \
632 + len(self.splitting_ops) > 0:
633 logger.warning_once(use_inductor_graph_partition_msg)
634 + self.splitting_ops = []
635 +
636 + def set_splitting_ops_for_attn_fusion(self):
637 + assert self.pass_config.enable_attn_fusion
638 + if self.splitting_ops is None:
639 self.splitting_ops = []
640 + if self.cudagraph_mode.has_piecewise_cudagraphs():
641 + logger.warning_once(
642 + "enable_attn_fusion is incompatible with piecewise "
643 + "cudagraph when use_inductor_graph_partition is off."
644 + "In this case, splitting_ops will be set to empty "
645 + "list, and cudagraph_mode will be set to FULL. "
646 + "Please ensure you are using attention backends that "
647 + "support cudagraph or set cudagraph_mode to NONE "
648 + "explicitly if encountering any problems.")
649 + self.cudagraph_mode = CUDAGraphMode.FULL
650 +
651 + assert not self.splitting_ops_contain_attention(), (
652 + "attention ops should not be in splitting_ops "
653 + "when enable_attn_fusion is True")
654
655 def splitting_ops_contain_attention(self) -> bool:
656 return self.splitting_ops is not None and all(
vllm/forward_context.py
+1 -2
@@ -246,8 +246,7 @@ class ForwardContext:
246 ubatch_slices: Optional[UBatchSlices] = None
247
248 def __post_init__(self):
249 - assert self.cudagraph_runtime_mode in [
250 - CUDAGraphMode.NONE, CUDAGraphMode.PIECEWISE, CUDAGraphMode.FULL], \
249 + assert self.cudagraph_runtime_mode.valid_runtime_modes(), \
250 f"Invalid cudagraph runtime mode: {self.cudagraph_runtime_mode}"
251
252
vllm/v1/cudagraph_dispatcher.py
+21 -20
@@ -22,10 +22,10 @@ class CudagraphDispatcher:
22
23 At runtime, the dispatch method generates the runtime cudagraph mode (FULL,
24 PIECEWISE, or NONE for no cudagraph) and the valid key (batch descriptor)
25 - based on the input key. After dispatching (communicate via forward context),
26 - the cudagraph wrappers will trust the dispatch key to do either capturing
27 - or replaying (if mode matched), or pass through to the underlying runnable
28 - without cudagraph (if mode no match or mode is NONE).
25 + based on the input key. After dispatching (communicated via forward
26 + context), the cudagraph wrappers will trust the dispatch key to either
27 + capture or replay (if the mode matches), or pass through to the underlying
28 + runnable without cudagraph (if the mode does not match or mode is NONE).
29 """
30
31 def __init__(self, vllm_config: VllmConfig):
@@ -57,19 +57,15 @@ class CudagraphDispatcher:
57 def add_cudagraph_key(self, runtime_mode: CUDAGraphMode,
58 batch_descriptor: BatchDescriptor):
59 assert runtime_mode in [CUDAGraphMode.PIECEWISE, CUDAGraphMode.FULL], \
60 - f"Invalid cudagraph runtime mode: {runtime_mode}"
60 + f"Invalid cudagraph runtime mode for keys: {runtime_mode}"
61 self.cudagraph_keys[runtime_mode].add(batch_descriptor)
62
63 def initialize_cudagraph_keys(self, cudagraph_mode: CUDAGraphMode,
64 uniform_decode_query_len: int):
65 # This should be called only after attention backend is initialized.
66
67 - # Note: we create all valid keys possible for cudagraph but do not
68 - # guarantee all keys would be used. For example, we create keys for
69 - # piecewise cudagraphs when it is piecewise compilation, which is always
70 - # valid, but for attention backend support unified routine, we may not
71 - # trigger capturing/replaying the piecewise cudagraphs depending on
72 - # CompilationConfig.cudagraph_mode. In addition, if we allow lazy
67 + # Note: we create all valid keys for cudagraph here but do not
68 + # guarantee all keys would be used. For example, if we allow lazy
69 # capturing in future PR, some keys may never be triggered.
70 if cudagraph_mode.mixed_mode() != CUDAGraphMode.NONE:
71 for bs in self.compilation_config.cudagraph_capture_sizes:
@@ -94,10 +90,13 @@ class CudagraphDispatcher:
90 self.keys_initialized = True
91
92 def dispatch(
97 - self, batch_descriptor: BatchDescriptor
93 + self,
94 + batch_descriptor: BatchDescriptor,
95 + use_cascade_attn: bool = False
96 ) -> tuple[CUDAGraphMode, Optional[BatchDescriptor]]:
97 """
100 - Given a batch descriptor, dispatch to a cudagraph mode.
98 + Given conditions(e.g.,batch descriptor and if using cascade attention),
99 + dispatch to a cudagraph runtime mode and the valid batch descriptor.
100 A new batch descriptor is returned as we might dispatch a uniform batch
101 to a graph that supports a more general batch (uniform to non-uniform).
102 """
@@ -107,14 +106,16 @@ class CudagraphDispatcher:
106 "initialized. No cudagraph will be used.")
107 return CUDAGraphMode.NONE, None
108
110 - # check if key exists for full cudagraph
111 - if batch_descriptor in self.cudagraph_keys[CUDAGraphMode.FULL]:
112 - return CUDAGraphMode.FULL, batch_descriptor
113 -
114 - # otherwise, check if non-uniform key exists
109 non_uniform_key = batch_descriptor.non_uniform
116 - if non_uniform_key in self.cudagraph_keys[CUDAGraphMode.FULL]:
117 - return CUDAGraphMode.FULL, non_uniform_key
110 + # if a batch use cascade attention, bypass checking full cudagraphs
111 + if not use_cascade_attn:
112 + # check if key exists for full cudagraph
113 + if batch_descriptor in self.cudagraph_keys[CUDAGraphMode.FULL]:
114 + return CUDAGraphMode.FULL, batch_descriptor
115 +
116 + # otherwise, check if non-uniform key exists
117 + if non_uniform_key in self.cudagraph_keys[CUDAGraphMode.FULL]:
118 + return CUDAGraphMode.FULL, non_uniform_key
119
120 # also check if non-uniform key exists for more "general"
121 # piecewise cudagraph
vllm/v1/worker/gpu_model_runner.py
+31 -16
@@ -923,11 +923,13 @@ class GPUModelRunner(LoRAModelRunnerMixin, KVConnectorModelRunnerMixin):
923 ) -> tuple[PerLayerAttnMetadata, torch.Tensor,
924 Optional[SpecDecodeMetadata], np.ndarray,
925 Optional[CommonAttentionMetadata], int, Optional[UBatchSlices],
926 - Optional[torch.Tensor]]:
926 + Optional[torch.Tensor], bool]:
927 """
928 :return: tuple[
929 attn_metadata: layer-to-attention_metadata mapping,
930 - logits_indices, spec_decode_metadata
930 + logits_indices, spec_decode_metadata,
931 + num_scheduled_tokens, spec_decode_common_attn_metadata,
932 + max_num_scheduled_tokens, use_cascade_attn
933 ]
934 """
935 total_num_scheduled_tokens = scheduler_output.total_num_scheduled_tokens
@@ -1135,6 +1137,7 @@ class GPUModelRunner(LoRAModelRunnerMixin, KVConnectorModelRunnerMixin):
1137 attn_metadata: PerLayerAttnMetadata = {}
1138 if ubatch_slices is not None:
1139 attn_metadata = [dict() for _ in range(len(ubatch_slices))]
1140 + use_cascade_attn = False
1141
1142 # Used in the below loop.
1143 query_start_loc_cpu = self.query_start_loc.cpu[:num_reqs + 1]
@@ -1251,9 +1254,15 @@ class GPUModelRunner(LoRAModelRunnerMixin, KVConnectorModelRunnerMixin):
1254 common_prefix_len=common_prefix_len,
1255 common_attn_metadata=common_attn_metadata,
1256 **extra_attn_metadata_args)
1257 + use_cascade_attn |= getattr(attn_metadata_i, "use_cascade",
1258 + False)
1259 for layer_name in attn_group.layer_names:
1260 attn_metadata[layer_name] = attn_metadata_i
1261
1262 + # disable cascade attention when DBO
1263 + if ubatch_slices is not None:
1264 + use_cascade_attn = False
1265 +
1266 # Hot-Swap lora model
1267 if self.lora_config:
1268 self.set_active_loras(self.input_batch, num_scheduled_tokens)
@@ -1261,7 +1270,7 @@ class GPUModelRunner(LoRAModelRunnerMixin, KVConnectorModelRunnerMixin):
1270 return (attn_metadata, logits_indices, spec_decode_metadata,
1271 num_scheduled_tokens, spec_decode_common_attn_metadata,
1272 max_num_scheduled_tokens, ubatch_slices,
1264 - num_tokens_after_padding)
1273 + num_tokens_after_padding, use_cascade_attn)
1274
1275 def _compute_cascade_attn_prefix_len(
1276 self,
@@ -2251,8 +2260,8 @@ class GPUModelRunner(LoRAModelRunnerMixin, KVConnectorModelRunnerMixin):
2260 # Prepare the decoder inputs.
2261 (attn_metadata, logits_indices, spec_decode_metadata,
2262 num_scheduled_tokens_np, spec_decode_common_attn_metadata,
2254 - max_query_len, ubatch_slices, num_tokens_after_padding
2255 - ) = self._prepare_inputs(scheduler_output)
2263 + max_query_len, ubatch_slices, num_tokens_after_padding,
2264 + use_cascade_attn) = self._prepare_inputs(scheduler_output)
2265
2266 (
2267 num_scheduled_tokens,
@@ -2273,7 +2282,8 @@ class GPUModelRunner(LoRAModelRunnerMixin, KVConnectorModelRunnerMixin):
2282 batch_descriptor = BatchDescriptor(num_tokens=num_input_tokens,
2283 uniform_decode=uniform_decode)
2284 cudagraph_runtime_mode, batch_descriptor = \
2276 - self.cudagraph_dispatcher.dispatch(batch_descriptor)
2285 + self.cudagraph_dispatcher.dispatch(batch_descriptor,
2286 + use_cascade_attn)
2287
2288 # This is currently to get around the assert in the DPMetadata
2289 # where it wants `num_tokens_across_dp` to align with `num_tokens`
@@ -2701,16 +2711,15 @@ class GPUModelRunner(LoRAModelRunnerMixin, KVConnectorModelRunnerMixin):
2711 "Cannot reload weights before model is loaded."
2712 model_loader = get_model_loader(self.load_config)
2713 logger.info("Reloading weights inplace...")
2704 - model = self.get_model()
2705 - model_loader.load_weights(model, model_config=self.model_config)
2714 + model_loader.load_weights(self.get_model(),
2715 + model_config=self.model_config)
2716
2717 def save_tensorized_model(
2718 self,
2719 tensorizer_config: "TensorizerConfig",
2720 ) -> None:
2711 - model = self.get_model()
2721 TensorizerLoader.save_model(
2713 - model,
2722 + self.get_model(),
2723 tensorizer_config=tensorizer_config,
2724 model_config=self.model_config,
2725 )
@@ -2926,9 +2935,8 @@ class GPUModelRunner(LoRAModelRunnerMixin, KVConnectorModelRunnerMixin):
2935 (1 token) and prefill (multiple tokens) requests.
2936 remove_lora: If False, dummy LoRAs are not destroyed after the run
2937 """
2929 - assert cudagraph_runtime_mode is None or cudagraph_runtime_mode in {
2930 - CUDAGraphMode.NONE, CUDAGraphMode.PIECEWISE, CUDAGraphMode.FULL
2931 - }
2938 + assert cudagraph_runtime_mode is None or \
2939 + cudagraph_runtime_mode.valid_runtime_modes()
2940
2941 # If cudagraph_mode.decode_mode() == FULL and
2942 # cudagraph_mode.separate_routine(). This means that we are using
@@ -3113,7 +3121,8 @@ class GPUModelRunner(LoRAModelRunnerMixin, KVConnectorModelRunnerMixin):
3121 # filter out the valid batch descriptor
3122 _cg_mode, batch_descriptor = self.cudagraph_dispatcher.dispatch(
3123 BatchDescriptor(num_tokens=num_tokens,
3116 - uniform_decode=uniform_decode))
3124 + uniform_decode=uniform_decode)) \
3125 + if not is_profile else (CUDAGraphMode.NONE, None)
3126 if cudagraph_runtime_mode is not None:
3127 # we allow forcing NONE when the dispatcher disagrees to support
3128 # warm ups for cudagraph capture
@@ -3453,8 +3462,8 @@ class GPUModelRunner(LoRAModelRunnerMixin, KVConnectorModelRunnerMixin):
3462 cudagraph_runtime_mode: CUDAGraphMode,
3463 uniform_decode: bool):
3464 assert cudagraph_runtime_mode != CUDAGraphMode.NONE and \
3456 - cudagraph_runtime_mode in [CUDAGraphMode.FULL,
3457 - CUDAGraphMode.PIECEWISE]
3465 + cudagraph_runtime_mode.valid_runtime_modes(), \
3466 + f"Invalid cudagraph runtime mode: {cudagraph_runtime_mode}"
3467
3468 # Only rank 0 should print progress bar during capture
3469 if is_global_first_rank():
@@ -3585,6 +3594,12 @@ class GPUModelRunner(LoRAModelRunnerMixin, KVConnectorModelRunnerMixin):
3594 self.calculate_reorder_batch_threshold()
3595
3596 def initialize_cudagraph_capture(self) -> None:
3597 + """
3598 + Resolve the cudagraph_mode when there are multiple attention
3599 + backends with potential conflicting CUDA graph support.
3600 + Then initialize the cudagraph_dispatcher based on the resolved
3601 + cudagraph_mode.
3602 + """
3603 min_cg_support = AttentionCGSupport.ALWAYS
3604 min_cg_builder_name = None
3605