@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / a742134cc5

Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 13, 09:30 UTC

Remove deprecated fields from `CompilationConfig` (#27593)

Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>

Harry Mellor committed Nov 12, 2025 at 16:10 UTC a742134cc5fbdec6c2af1ef383704aac5c445fbd

13 files changed +122 -164

.buildkite/test-pipeline.yaml

@@ -443,6 +443,7 @@ steps:

443 - vllm/

444 - tests/compile

445 commands:

446 + - pytest -v -s compile/test_config.py

447 - pytest -v -s compile/test_pass_manager.py

448 - pytest -v -s compile/test_fusion.py

449 - pytest -v -s compile/test_fusion_attn.py

docs/design/cuda_graphs.md

-10

@@ -218,16 +218,6 @@ outputs = model.generate(

218 )

219 ```

220

221 -### Migration from legacy flags

222 -

223 -Legacy `use_cudagraph` and `full_cuda_graph` are unified by `cudagraph_mode`:

224 -

225 -* `use_cudagraph=False` → `NONE`.

226 -* `use_cudagraph=True` and `full_cuda_graph=False` → `PIECEWISE`.

227 -* `full_cuda_graph=True` → directly set `FULL` and rely on the graceful fallback policy.

228 -

229 -As they are deprecated and will be removed in the next major or minor release, i.e., v0.11.0 or v1.0.0, we recommend using cudagraph_mode instead.

230 -

221 ### Piecewise compilation and full graph custom passes (attention fusion, sequence parallelism)

222

223 Unfortunately, some custom compile passes have to see the whole graph to be effective and hence aren't compatible with piecewise compilation. This includes `AttnFusionPass` and `SequenceParallelismPass`. As a short-term solution, we automatically disable piecewise compilation (by setting `splitting_ops=[]`) when attention fusion is enabled. We use CUDA Graph modes `FULL` or `FULL_DECODE_ONLY` (depending on backend support). However, this leads to another optimization incompatibility and confusing performance tradeoffs.

tests/compile/piecewise/test_multiple_graphs.py

+2 -2

@@ -203,7 +203,7 @@ def test_multi_graph_piecewise_compile(use_inductor_graph_partition: bool):

203 vllm_config = VllmConfig(

204 compilation_config=CompilationConfig(

205 mode=CompilationMode.VLLM_COMPILE,

206 - use_cudagraph=True,

206 + cudagraph_mode=CUDAGraphMode.PIECEWISE,

207 splitting_ops=["silly::attention"],

208 cudagraph_capture_sizes=[1, 2],

209 use_inductor_graph_partition=use_inductor_graph_partition,

@@ -281,7 +281,7 @@ def test_multi_graph_piecewise_compile(use_inductor_graph_partition: bool):

281 vllm_config = VllmConfig(

282 compilation_config=CompilationConfig(

283 mode=CompilationMode.VLLM_COMPILE,

284 - use_cudagraph=False,

284 + cudagraph_mode=CUDAGraphMode.NONE,

285 splitting_ops=["silly::attention"],

286 use_inductor_graph_partition=use_inductor_graph_partition,

287 )

tests/compile/piecewise/test_simple.py

-1

@@ -62,7 +62,6 @@ def _run_simple_model(

62 vllm_config = VllmConfig(

63 compilation_config=CompilationConfig(

64 mode=CompilationMode.VLLM_COMPILE,

65 - use_cudagraph=True,

65 use_inductor=use_inductor,

66 splitting_ops=splitting_ops,

67 use_inductor_graph_partition=use_inductor_graph_partition,

tests/compile/piecewise/test_toy_llama.py

-1

@@ -449,7 +449,6 @@ def benchmark():

449 if piecewise:

450 compilation_config = CompilationConfig(

451 mode=CompilationMode.VLLM_COMPILE,

452 - use_cudagraph=True,

452 splitting_ops=["silly::attention"],

453 cudagraph_capture_sizes=cudagraph_sizes,

454 )

tests/compile/test_config.py

+91 -60

@@ -2,8 +2,10 @@

2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project

3 import copy

4 from contextlib import nullcontext

5 +from unittest.mock import patch

7 import pytest

8 +from pydantic import ValidationError

10 from vllm.compilation.counter import compilation_counter

11 from vllm.compilation.fix_functionalization import FixFunctionalizationPass

@@ -11,7 +13,7 @@ from vllm.config import CompilationConfig, CUDAGraphMode, VllmConfig

13 from vllm.config.compilation import CompilationMode

14 from vllm.engine.arg_utils import EngineArgs

15 from vllm.platforms import current_platform

14 -from vllm.utils.torch_utils import _is_torch_equal_or_newer, is_torch_equal_or_newer

16 +from vllm.utils.torch_utils import _is_torch_equal_or_newer

19 def test_version():

@@ -23,14 +25,6 @@ def test_version():

25 assert not _is_torch_equal_or_newer("2.7.1", "2.8.0.dev")

26 -def test_use_cudagraphs_dynamic():

27 - vllm_config = VllmConfig()

28 - # Default V1 configuration now starts without cudagraphs enabled; the

29 - # engine decides when to capture based on runtime settings instead of a

30 - # blanket default.

31 - assert vllm_config.compilation_config.use_cudagraph

32 -

33 -

28 def test_copy_pass():

29 vllm_config = VllmConfig()

30 inductor_pass = FixFunctionalizationPass(vllm_config)

@@ -65,7 +59,7 @@ def test_VLLM_DISABLE_COMPILE_CACHE(vllm_runner, monkeypatch, val):

59 monkeypatch.setenv("VLLM_DISABLE_COMPILE_CACHE", val)

61 compilation_config = {

68 - "use_cudagraph": False, # speed things up a bit

62 + "cudagraph_mode": CUDAGraphMode.NONE, # speed things up a bit

63 }

64 with (

65 compilation_counter.expect(

@@ -83,20 +77,31 @@ def test_VLLM_DISABLE_COMPILE_CACHE(vllm_runner, monkeypatch, val):

78 # forked needed to workaround https://github.com/vllm-project/vllm/issues/21073

79 @pytest.mark.forked

86 -@pytest.mark.parametrize("enabled", [True, False])

87 -def test_use_cudagraphs(vllm_runner, monkeypatch, enabled):

80 +@pytest.mark.parametrize(

81 + "cudagraph_mode,num_cudagraph_captured",

82 + [

83 + (CUDAGraphMode.NONE, 0),

84 + (CUDAGraphMode.FULL_DECODE_ONLY, 1),

85 + (CUDAGraphMode.PIECEWISE, 13),

86 + (CUDAGraphMode.FULL_AND_PIECEWISE, 14),

87 + ],

88 +)

89 +def test_use_cudagraphs(

90 + vllm_runner, monkeypatch, cudagraph_mode, num_cudagraph_captured

91 +):

92 # Disable multiprocessing so that the counter is in the same process

93 monkeypatch.setenv("VLLM_ENABLE_V1_MULTIPROCESSING", "0")

95 compilation_config = {

96 "cudagraph_capture_sizes": [100],

93 - "use_cudagraph": enabled,

97 + "cudagraph_mode": cudagraph_mode,

98 }

99 + num_gpu_runner_capture_triggers = 1 if cudagraph_mode != CUDAGraphMode.NONE else 0

100 with (

101 compilation_counter.expect(

102 num_graphs_seen=1,

98 - num_gpu_runner_capture_triggers=1 if enabled else 0,

99 - num_cudagraph_captured=13 if enabled else 0,

103 + num_gpu_runner_capture_triggers=num_gpu_runner_capture_triggers,

104 + num_cudagraph_captured=num_cudagraph_captured,

105 ),

106 # loading the model causes compilation (if enabled) to happen

107 vllm_runner(

@@ -168,19 +173,18 @@ def test_splitting_ops_dynamic():

173 assert not config.compilation_config.splitting_ops_contain_attention()

174

175 # When use_inductor_graph_partition=True

171 - if is_torch_equal_or_newer("2.9.0.dev"):

172 - config = VllmConfig(

173 - compilation_config=CompilationConfig(

174 - mode=CompilationMode.VLLM_COMPILE,

175 - use_inductor_graph_partition=True,

176 - splitting_ops=["vllm::unified_attention"],

177 - )

176 + config = VllmConfig(

177 + compilation_config=CompilationConfig(

178 + mode=CompilationMode.VLLM_COMPILE,

179 + use_inductor_graph_partition=True,

180 + splitting_ops=["vllm::unified_attention"],

181 )

179 - # with inductor partition we use splitting_ops directly for

180 - # partition rules

181 - assert config.compilation_config.splitting_ops == ["vllm::unified_attention"]

182 + )

183 + # with inductor partition we use splitting_ops directly for

184 + # partition rules

185 + assert config.compilation_config.splitting_ops == ["vllm::unified_attention"]

186

183 - # When attn_fusion pass enabled, splitting_ops now default to attention ops.

187 + # When attn_fusion pass enabled.

188 config = VllmConfig(

189 compilation_config=CompilationConfig(

190 mode=CompilationMode.VLLM_COMPILE,

@@ -189,29 +193,41 @@ def test_splitting_ops_dynamic():

193 cudagraph_mode=CUDAGraphMode.PIECEWISE,

194 )

195 )

192 - # With the new simplified logic, attention fusion works with splitting_ops

193 - assert config.compilation_config.splitting_ops_contain_attention()

194 - # cudagraph mode remains PIECEWISE

195 - assert config.compilation_config.cudagraph_mode == CUDAGraphMode.PIECEWISE

196 + assert config.compilation_config.splitting_ops == []

197 + # cudagraph mode also fall back to FULL

198 + assert config.compilation_config.cudagraph_mode == CUDAGraphMode.FULL

199

197 - # When both use_inductor_graph_partition and attn_fusion pass enabled.

198 - if is_torch_equal_or_newer("2.9.0.dev"):

200 + # splitting_ops can not contain attention ops when attn_fusion

201 + # pass enabled.

202 + with pytest.raises(ValidationError):

203 config = VllmConfig(

204 compilation_config=CompilationConfig(

205 mode=CompilationMode.VLLM_COMPILE,

202 - use_inductor_graph_partition=True,

206 pass_config={"enable_attn_fusion": True, "enable_noop": True},

207 custom_ops=["+quant_fp8"],

208 cudagraph_mode=CUDAGraphMode.PIECEWISE,

209 + # work around for accessing all attntion ops

210 + splitting_ops=CompilationConfig()._attention_ops,

211 )

212 )

208 - # With inductor graph partition, attn_fusion and splitting_ops

209 - # work together. Default splitting_ops include attention ops.

210 - assert config.compilation_config.splitting_ops_contain_attention()

211 - # enable_attn_fusion is directly supported under

212 - # use_inductor_graph_partition=True, and cudagraph_mode

213 - # is unchanged.

214 - assert config.compilation_config.cudagraph_mode == CUDAGraphMode.PIECEWISE

213 +

214 + # When both use_inductor_graph_partition and attn_fusion pass enabled.

215 + config = VllmConfig(

216 + compilation_config=CompilationConfig(

217 + mode=CompilationMode.VLLM_COMPILE,

218 + use_inductor_graph_partition=True,

219 + pass_config={"enable_attn_fusion": True, "enable_noop": True},

220 + custom_ops=["+quant_fp8"],

221 + cudagraph_mode=CUDAGraphMode.PIECEWISE,

222 + )

223 + )

224 + # With inductor graph partition, attn_fusion and splitting_ops

225 + # work together. Default splitting_ops include attention ops.

226 + assert config.compilation_config.splitting_ops_contain_attention()

227 + # enable_attn_fusion is directly supported under

228 + # use_inductor_graph_partition=True, and cudagraph_mode

229 + # is unchanged.

230 + assert config.compilation_config.cudagraph_mode == CUDAGraphMode.PIECEWISE

231

232

233 def test_should_split():

@@ -293,25 +309,36 @@ def test_should_split():

309 "tp_size",

310 "enable_sequence_parallelism",

311 "max_num_batched_tokens",

296 - "use_cudagraph",

312 + "cudagraph_mode",

313 "expected_max_size",

314 ),

315 [

300 - (None, None, 1, False, 2048, True, 512),

301 - ([1, 2, 4], 4, 1, False, 2048, True, 4),

302 - ([1, 2, 4], 8, 1, False, 2048, True, RuntimeError),

303 - ([1, 256], None, 1, False, 2048, 256),

304 - ([], None, 1, False, 2048, False, 0),

305 - (None, 0, 1, False, 2048, False, 0),

316 + (None, None, 1, False, 2048, CUDAGraphMode.FULL_AND_PIECEWISE, 256),

317 + ([1, 2, 4], 4, 1, False, 2048, CUDAGraphMode.FULL_AND_PIECEWISE, 4),

318 + (

319 + [1, 2, 4],

320 + 8,

321 + 1,

322 + False,

323 + 2048,

324 + CUDAGraphMode.FULL_AND_PIECEWISE,

325 + ValidationError,

326 + ),

327 + ([1, 256], None, 1, False, 2048, CUDAGraphMode.FULL_AND_PIECEWISE, 256),

328 + ([], None, 1, False, 2048, CUDAGraphMode.NONE, 0),

329 + (None, 0, 1, False, 2048, CUDAGraphMode.NONE, 0),

330 # truncated to nearest multiple of 8 or 16

307 - (None, 257, 1, False, 2048, True, 256),

308 - ([1, 2, 4, 15], None, 1, False, 2048, True, 15), # max from list

309 - ([1, 2, 4, 15], None, 2, True, 2048, True, 4), # filtered out 15 due to SP

310 - ([1, 2, 4, 15], None, 1, False, 8, True, 4), # limited by the max_tokens

331 + (None, 257, 1, False, 2048, CUDAGraphMode.FULL_AND_PIECEWISE, 256),

332 + # max from list

333 + ([1, 2, 4, 15], None, 1, False, 2048, CUDAGraphMode.FULL_AND_PIECEWISE, 15),

334 + # filtered out 15 due to SP

335 + ([1, 2, 4, 15], None, 2, True, 2048, CUDAGraphMode.FULL_AND_PIECEWISE, 4),

336 + # limited by the max_tokens

337 + ([1, 2, 4, 15], None, 1, False, 8, CUDAGraphMode.FULL_AND_PIECEWISE, 4),

338 # the list should contain at least 1 element when use cudagraph

312 - ([], None, 1, False, 2048, True, RuntimeError),

339 + ([], None, 1, False, 2048, CUDAGraphMode.FULL_AND_PIECEWISE, ValidationError),

340 # the max capturing size should be >= 1 when use cudagraph

314 - (None, 0, 1, False, 2048, True, RuntimeError),

341 + (None, 0, 1, False, 2048, CUDAGraphMode.FULL_AND_PIECEWISE, ValidationError),

342 ],

343 )

344 def test_cudagraph_sizes_post_init(

@@ -320,15 +347,17 @@ def test_cudagraph_sizes_post_init(

347 tp_size,

348 enable_sequence_parallelism,

349 max_num_batched_tokens,

323 - use_cudagraph,

350 + cudagraph_mode,

351 expected_max_size,

352 ):

353 ctx = nullcontext()

327 - if isinstance(expected_max_size, Exception):

354 + if expected_max_size == ValidationError:

355 ctx = pytest.raises(expected_max_size)

356

330 - cudagraph_mode = CUDAGraphMode.PIECEWISE if use_cudagraph else CUDAGraphMode.NONE

331 - with ctx:

357 + with (

358 + ctx,

359 + patch("vllm.config.parallel.cuda_device_count_stateless", return_value=tp_size),

360 + ):

361 compilation_config = CompilationConfig(

362 cudagraph_capture_sizes=cudagraph_capture_sizes,

363 max_cudagraph_capture_size=max_cudagraph_capture_size,

@@ -342,11 +371,13 @@ def test_cudagraph_sizes_post_init(

371 engine_args = EngineArgs(

372 model="facebook/opt-125m",

373 tensor_parallel_size=tp_size,

374 + max_num_seqs=min(max_num_batched_tokens, 128),

375 max_num_batched_tokens=max_num_batched_tokens,

376 compilation_config=compilation_config,

377 )

378 vllm_config = engine_args.create_engine_config()

379

350 - assert (

351 - vllm_config.compilation_config.max_cudagraph_capture_size == expected_max_size

352 - )

380 + assert (

381 + vllm_config.compilation_config.max_cudagraph_capture_size

382 + == expected_max_size

383 + )

tests/compile/test_decorator.py

-3

@@ -80,7 +80,6 @@ def test_ignore_torch_compile_decorator(use_inductor_graph_partition, monkeypatc

80 vllm_config = VllmConfig(

81 compilation_config=CompilationConfig(

82 mode=CompilationMode.VLLM_COMPILE,

83 - use_cudagraph=True,

83 splitting_ops=["silly::attention"],

84 cudagraph_capture_sizes=[1, 2],

85 use_inductor_graph_partition=use_inductor_graph_partition,

@@ -215,7 +214,6 @@ def test_conditional_compile_enable_if(use_inductor_graph_partition, monkeypatch

214 ),

215 compilation_config=CompilationConfig(

216 mode=CompilationMode.VLLM_COMPILE,

218 - use_cudagraph=True,

217 splitting_ops=["silly::attention"],

218 cudagraph_capture_sizes=[1, 2],

219 use_inductor_graph_partition=use_inductor_graph_partition,

@@ -257,7 +255,6 @@ def test_conditional_compile_enable_if(use_inductor_graph_partition, monkeypatch

255 ),

256 compilation_config=CompilationConfig(

257 mode=CompilationMode.VLLM_COMPILE,

260 - use_cudagraph=True,

258 splitting_ops=["silly::attention"],

259 cudagraph_capture_sizes=[1, 2],

260 use_inductor_graph_partition=use_inductor_graph_partition,

tests/models/multimodal/generation/test_qwen2_5_vl.py

-2

@@ -61,10 +61,8 @@ def test_qwen2_5_vl_evs_functionality(

61 model,

62 runner="generate",

63 max_model_len=4000,

64 - max_num_seqs=1,

64 dtype=dtype,

65 limit_mm_per_prompt={"video": 1},

67 - tensor_parallel_size=1,

66 video_pruning_rate=video_pruning_rate,

67 ) as vllm_model:

68 # Generate output - this should not crash

vllm/config/compilation.py

+22 -73

@@ -206,7 +206,6 @@ class CompilationConfig:

206 - [`splitting_ops`][vllm.config.CompilationConfig.splitting_ops]

207 - [`compile_mm_encoder`][vllm.config.CompilationConfig.compile_mm_encoder]

208 - CudaGraph capture:

209 - - [`use_cudagraph`][vllm.config.CompilationConfig.use_cudagraph]

209 - [`cudagraph_mode`][vllm.config.CompilationConfig.cudagraph_mode]

210 - [`cudagraph_capture_sizes`]

211 [vllm.config.CompilationConfig.cudagraph_capture_sizes]

@@ -216,7 +215,6 @@ class CompilationConfig:

215 [vllm.config.CompilationConfig.cudagraph_num_of_warmups]

216 - [`cudagraph_copy_inputs`]

217 [vllm.config.CompilationConfig.cudagraph_copy_inputs]

219 - - [`full_cuda_graph`][vllm.config.CompilationConfig.full_cuda_graph]

218 - Inductor compilation:

219 - [`use_inductor`][vllm.config.CompilationConfig.use_inductor]

220 - [`compile_sizes`][vllm.config.CompilationConfig.compile_sizes]

@@ -396,18 +394,6 @@ class CompilationConfig:

394 Warning: This flag is new and subject to change in addition

395 more modes may be added.

396 """

399 - use_cudagraph: bool = True

400 - """Whether to use cudagraph inside compilation:

401 -

402 - - False: cudagraph inside compilation is not used.\n

403 - - True: cudagraph inside compilation is used. It requires

404 - that all input buffers have fixed addresses, and all

405 - splitting ops write their outputs to input buffers.

406 -

407 - Warning: This flag is deprecated and will be removed in the next major or

408 - minor release, i.e. v0.11.0 or v1.0.0. Please use cudagraph_mode=FULL_AND

409 - _PIECEWISE instead.

410 - """

397 cudagraph_num_of_warmups: int = 0

398 """Number of warmup runs for cudagraph.

399 It means the first several runs will be treated as warmup runs.

@@ -425,15 +411,6 @@ class CompilationConfig:

411 internally managed buffer. Default is False.

412 Note that this flag is only effective when cudagraph_mode is PIECEWISE.

413 """

428 - full_cuda_graph: bool | None = False

429 - """whether to use a full cuda graph for the entire forward pass rather than

430 - splitting certain operations such as attention into subgraphs. Thus this

431 - flag cannot be used together with splitting_ops. This may provide

432 - performance benefits for smaller models.

433 - Warning: This flag is deprecated and will be removed in the next major or

434 - minor release, i.e. v0.11.0 or v1.0.0. Please use cudagraph_mode=

435 - FULL_AND_PIECEWISE instead.

436 - """

414 cudagraph_specialize_lora: bool = True

415 """Whether to create separate cuda graphs for cases with and without active

416 LoRA adapters. When set to False, the LoRA-enabled cuda graph will be used

@@ -603,13 +580,19 @@ class CompilationConfig:

580 @field_validator("cudagraph_mode", mode="before")

581 @classmethod

582 def validate_cudagraph_mode_before(cls, value: Any) -> Any:

606 - """

607 - enable parse the `cudagraph_mode` enum type from string

608 - """

583 + """Enable parsing of the `cudagraph_mode` enum type from string."""

584 if isinstance(value, str):

585 return CUDAGraphMode[value.upper()]

586 return value

587

588 + @field_validator("pass_config", mode="before")

589 + @classmethod

590 + def validate_pass_config_before(cls, value: Any) -> Any:

591 + """Enable parsing of the `pass_config` field from a dictionary."""

592 + if isinstance(value, dict):

593 + return PassConfig(**value)

594 + return value

595 +

596 @field_validator("compile_cache_save_format")

597 @classmethod

598 def validate_compile_cache_save_format(cls, value: str) -> str:

@@ -666,9 +649,6 @@ class CompilationConfig:

649 func if isinstance(func, InductorPass) else CallableInductorPass(func)

650 )

651

669 - if isinstance(self.pass_config, dict):

670 - self.pass_config = PassConfig(**self.pass_config)

671 -

652 if self.pass_config.enable_qk_norm_rope_fusion:

653 # TODO(zhuhaoran): support rope native forward match and remove this.

654 # Linked issue: https://github.com/vllm-project/vllm/issues/28042

@@ -684,36 +664,6 @@ class CompilationConfig:

664 self.inductor_compile_config["combo_kernels"] = True

665 self.inductor_compile_config["benchmark_combo_kernel"] = True

666

687 - # migrate the deprecated flags

688 - if not self.use_cudagraph:

689 - logger.warning(

690 - "use_cudagraph is deprecated, use cudagraph_mode=NONE instead."

691 - )

692 - if (

693 - self.cudagraph_mode is not None

694 - and self.cudagraph_mode != CUDAGraphMode.NONE

695 - ):

696 - raise ValueError(

697 - "use_cudagraph and cudagraph_mode are mutually"

698 - " exclusive, prefer cudagraph_mode since "

699 - "use_cudagraph is deprecated."

700 - )

701 - self.cudagraph_mode = CUDAGraphMode.NONE

702 - if self.full_cuda_graph:

703 - logger.warning(

704 - "full_cuda_graph is deprecated, use cudagraph_mode=FULL instead."

705 - )

706 - if (

707 - self.cudagraph_mode is not None

708 - and not self.cudagraph_mode.has_full_cudagraphs()

709 - ):

710 - raise ValueError(

711 - "full_cuda_graph and cudagraph_mode are "

712 - "mutually exclusive, prefer cudagraph_mode "

713 - "since full_cuda_graph is deprecated."

714 - )

715 - self.cudagraph_mode = CUDAGraphMode.FULL

716 -

667 if self.use_inductor_graph_partition and not is_torch_equal_or_newer(

668 "2.9.0.dev"

669 ):

@@ -891,20 +841,19 @@ class CompilationConfig:

841

842 def set_splitting_ops_for_attn_fusion(self):

843 assert self.pass_config.enable_attn_fusion

894 - # For dynamo-partition (non-inductor) attention fusion,

895 - # set splitting_ops to empty to avoid splitting at attention ops

896 - self.splitting_ops = []

897 - if self.cudagraph_mode.has_piecewise_cudagraphs():

898 - logger.warning_once(

899 - "enable_attn_fusion is incompatible with piecewise "

900 - "cudagraph when use_inductor_graph_partition is off. "

901 - "In this case, splitting_ops will be set to empty "

902 - "list, and cudagraph_mode will be set to FULL. "

903 - "Please ensure you are using attention backends that "

904 - "support cudagraph or set cudagraph_mode to NONE "

905 - "explicitly if encountering any problems."

906 - )

907 - self.cudagraph_mode = CUDAGraphMode.FULL

844 + if self.splitting_ops is None:

845 + self.splitting_ops = []

846 + if self.cudagraph_mode.has_piecewise_cudagraphs():

847 + logger.warning_once(

848 + "enable_attn_fusion is incompatible with piecewise "

849 + "cudagraph when use_inductor_graph_partition is off. "

850 + "In this case, splitting_ops will be set to empty "

851 + "list, and cudagraph_mode will be set to FULL. "

852 + "Please ensure you are using attention backends that "

853 + "support cudagraph or set cudagraph_mode to NONE "

854 + "explicitly if encountering any problems."

855 + )

856 + self.cudagraph_mode = CUDAGraphMode.FULL

857

858 assert not self.splitting_ops_contain_attention(), (

859 "attention ops should not be in splitting_ops "

vllm/config/vllm.py

+3 -9

@@ -656,14 +656,6 @@ class VllmConfig:

656 f"cudagraph_mode={self.compilation_config.cudagraph_mode}"

657 )

658

659 - # final migrate the deprecated flags

660 - self.compilation_config.use_cudagraph = (

661 - self.compilation_config.cudagraph_mode != CUDAGraphMode.NONE

662 - )

663 - self.compilation_config.full_cuda_graph = (

664 - self.compilation_config.cudagraph_mode.has_full_cudagraphs()

665 - )

666 -

659 if self.parallel_config.enable_dbo:

660 a2a_backend = self.parallel_config.all2all_backend

661 assert a2a_backend in ["deepep_low_latency", "deepep_high_throughput"], (

@@ -853,7 +845,9 @@ class VllmConfig:

845 )

846 # de-duplicate the sizes provided by the config

847 dedup_sizes = list(set(self.compilation_config.cudagraph_capture_sizes))

856 - cudagraph_capture_sizes = dedup_sizes

848 + cudagraph_capture_sizes = [

849 + i for i in dedup_sizes if i <= max_num_tokens

850 + ]

851 # sort to make sure the sizes are in ascending order

852 cudagraph_capture_sizes.sort()

853 else:

vllm/v1/attention/backends/mamba1_attn.py

+1 -1

@@ -123,7 +123,7 @@ class Mamba1AttentionMetadataBuilder(

123 elif (

124 num_decodes > 0

125 and num_decodes <= self.decode_cudagraph_max_bs

126 - and self.compilation_config.full_cuda_graph

126 + and self.compilation_config.cudagraph_mode.has_full_cudagraphs()

127 ):

128 padded_decodes = self.vllm_config.pad_for_cudagraph(num_decodes)

129 self.state_indices_tensor[:num_decodes].copy_(

vllm/v1/attention/backends/mamba2_attn.py

+1 -1

@@ -302,7 +302,7 @@ class Mamba2AttentionMetadataBuilder(

302

303 elif (

304 num_decodes <= self.decode_cudagraph_max_bs

305 - and self.compilation_config.full_cuda_graph

305 + and self.compilation_config.cudagraph_mode.has_full_cudagraphs()

306 ):

307 # Pad state tensor for CUDA graph

308 num_input_tokens = self.vllm_config.pad_for_cudagraph(num_decodes)

vllm/v1/attention/backends/short_conv_attn.py

+1 -1

@@ -81,7 +81,7 @@ class ShortConvAttentionMetadataBuilder(

81 elif (

82 num_decodes > 0

83 and num_decodes <= self.decode_cudagraph_max_bs

84 - and self.compilation_config.full_cuda_graph

84 + and self.compilation_config.cudagraph_mode.has_full_cudagraphs()

85 ):

86 num_input_tokens = self.vllm_config.pad_for_cudagraph(num_decodes)

87 self.state_indices_tensor[:num_decodes].copy_(