@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 96b9aa5aa0

Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 14, 05:30 UTC

[Frontend][torch.compile] CompilationConfig Overhaul (#20283): name change compilation level to compilation mode, deprecation compilation level (#26355)

Signed-off-by: morrison-turnansky <mturnans@redhat.com> Signed-off-by: Morrison Turnansky <mturnans@redhat.com> Co-authored-by: Luka Govedič <ProExpertProg@users.noreply.github.com>

Morrison Turnansky committed Oct 14, 2025 at 22:51 UTC 96b9aa5aa076e64c68765232aec343e4d0006e2a

42 files changed +271 -249

docs/configuration/conserving_memory.md

+2 -2

@@ -58,12 +58,12 @@ You can adjust `compilation_config` to achieve a better balance between inferenc

59 ```python

60 from vllm import LLM

61 - from vllm.config import CompilationConfig, CompilationLevel

61 + from vllm.config import CompilationConfig, CompilationMode

63 llm = LLM(

64 model="meta-llama/Llama-3.1-8B-Instruct",

65 compilation_config=CompilationConfig(

66 - level=CompilationLevel.PIECEWISE,

66 + mode=CompilationMode.VLLM_COMPILE,

67 # By default, it goes up to max_num_seqs

68 cudagraph_capture_sizes=[1, 2, 4, 8, 16],

69 ),

docs/design/cuda_graphs.md

+2 -2

@@ -167,7 +167,7 @@ class AttentionCGSupport(enum.Enum):

167 """NO CUDA Graphs support"""

168 ```

169

170 -Suppose we have hybrid attention backends (e.g., in mamba mixer models). In that case, we seek the minimum capability of all backends to determine the final capability of the model, and we might resolve the incompatible CUDA Graphs mode by downgrading the mode to the best fit one. For example, downgrading `FULL` mode to `FULL_AND_PIECEWISE` mode if the minimum capability is `UNIFORM_BATCH`, or `PIECEWISE` mode if the minimum capability is `NEVER` for -O3 compilation level. For the complete fallback policy, please see the code of [initialize_cudagraph_capture][vllm.v1.worker.gpu_model_runner.GPUModelRunner.initialize_cudagraph_capture].

170 +Suppose we have hybrid attention backends (e.g., in mamba mixer models). In that case, we seek the minimum capability of all backends to determine the final capability of the model, and we might resolve the incompatible CUDA Graphs mode by downgrading the mode to the best fit one. For example, downgrading `FULL` mode to `FULL_AND_PIECEWISE` mode if the minimum capability is `UNIFORM_BATCH`, or `PIECEWISE` mode if the minimum capability is `NEVER` for -O3 compilation mode. For the complete fallback policy, please see the code of [initialize_cudagraph_capture][vllm.v1.worker.gpu_model_runner.GPUModelRunner.initialize_cudagraph_capture].

171

172 The following table lists backends that support full CUDA Graphs at the time of writing.

173

@@ -202,7 +202,7 @@ os.environ.setdefault("VLLM_LOGGING_LEVEL", "DEBUG")

202 import vllm

203 from vllm.config import CUDAGraphMode

204

205 -compilation_config = {"level": 3, "cudagraph_mode": "FULL_AND_PIECEWISE"}

205 +compilation_config = {"mode": 3, "cudagraph_mode": "FULL_AND_PIECEWISE"}

206 model = vllm.LLM(

207 model="meta-llama/Llama-3.1-8B-Instruct",

208 dtype="auto",

examples/offline_inference/data_parallel.py

+1 -1

@@ -95,7 +95,7 @@ def parse_args():

95 parser.add_argument(

96 "--compilation-config",

97 type=int,

98 - help=("Compilation optimization (O) level 0-3."),

98 + help=("Compilation optimization (O) mode 0-3."),

99 )

100 parser.add_argument(

101 "--quantization",

tests/compile/piecewise/test_multiple_graphs.py

+5 -5

@@ -14,7 +14,7 @@ from vllm.compilation.counter import compilation_counter

14 from vllm.compilation.decorators import ignore_torch_compile, support_torch_compile

15 from vllm.config import (

16 CompilationConfig,

17 - CompilationLevel,

17 + CompilationMode,

18 CUDAGraphMode,

19 VllmConfig,

20 set_current_vllm_config,

@@ -199,10 +199,10 @@ def test_multi_graph_piecewise_compile(use_inductor_graph_partition: bool):

199

200 outputs = []

201

202 - # piecewise compile

202 + # vllmcompile compile

203 vllm_config = VllmConfig(

204 compilation_config=CompilationConfig(

205 - level=CompilationLevel.PIECEWISE,

205 + mode=CompilationMode.VLLM_COMPILE,

206 use_cudagraph=True,

207 splitting_ops=["silly::attention"],

208 cudagraph_capture_sizes=[1, 2],

@@ -251,7 +251,7 @@ def test_multi_graph_piecewise_compile(use_inductor_graph_partition: bool):

251 # no compile or cudagraph

252 vllm_config = VllmConfig(

253 compilation_config=CompilationConfig(

254 - level=CompilationLevel.NO_COMPILATION,

254 + mode=CompilationMode.NONE,

255 )

256 )

257 cudagraph_runtime_mode = CUDAGraphMode.NONE

@@ -280,7 +280,7 @@ def test_multi_graph_piecewise_compile(use_inductor_graph_partition: bool):

280 # piecewise compile without CUDA graph

281 vllm_config = VllmConfig(

282 compilation_config=CompilationConfig(

283 - level=CompilationLevel.PIECEWISE,

283 + mode=CompilationMode.VLLM_COMPILE,

284 use_cudagraph=False,

285 splitting_ops=["silly::attention"],

286 use_inductor_graph_partition=use_inductor_graph_partition,

tests/compile/piecewise/test_simple.py

+2 -2

@@ -13,7 +13,7 @@ from vllm.compilation.counter import compilation_counter

13 from vllm.compilation.decorators import support_torch_compile

14 from vllm.config import (

15 CompilationConfig,

16 - CompilationLevel,

16 + CompilationMode,

17 CUDAGraphMode,

18 VllmConfig,

19 set_current_vllm_config,

@@ -61,7 +61,7 @@ def _run_simple_model(

61 ):

62 vllm_config = VllmConfig(

63 compilation_config=CompilationConfig(

64 - level=CompilationLevel.PIECEWISE,

64 + mode=CompilationMode.VLLM_COMPILE,

65 use_cudagraph=True,

66 use_inductor=use_inductor,

67 splitting_ops=splitting_ops,

tests/compile/piecewise/test_toy_llama.py

+5 -5

@@ -21,7 +21,7 @@ from vllm.compilation.counter import compilation_counter

21 from vllm.compilation.decorators import support_torch_compile

22 from vllm.config import (

23 CompilationConfig,

24 - CompilationLevel,

24 + CompilationMode,

25 CUDAGraphMode,

26 VllmConfig,

27 set_current_vllm_config,

@@ -356,13 +356,13 @@ def test_toy_llama(

356 )

357

358 compile_config_no_compile = CompilationConfig(

359 - level=CompilationLevel.NO_COMPILATION,

359 + level=CompilationMode.NONE,

360 cudagraph_mode=CUDAGraphMode.NONE,

361 backend="eager",

362 )

363

364 compile_config_no_split = CompilationConfig(

365 - level=CompilationLevel.PIECEWISE,

365 + level=CompilationMode.VLLM_COMPILE,

366 use_inductor_graph_partition=use_inductor_graph_partition,

367 cudagraph_mode=CUDAGraphMode.PIECEWISE,

368 backend=backend,

@@ -458,14 +458,14 @@ def benchmark():

458 for piecewise in [False, True]:

459 if piecewise:

460 compilation_config = CompilationConfig(

461 - level=CompilationLevel.PIECEWISE,

461 + mode=CompilationMode.VLLM_COMPILE,

462 use_cudagraph=True,

463 splitting_ops=["silly::attention"],

464 cudagraph_capture_sizes=cudagraph_sizes,

465 )

466 else:

467 compilation_config = CompilationConfig(

468 - level=CompilationLevel.PIECEWISE,

468 + mode=CompilationMode.VLLM_COMPILE,

469 cudagraph_capture_sizes=cudagraph_sizes,

470 )

471

tests/compile/test_aot_compile.py

+2 -2

@@ -10,7 +10,7 @@ import torch

10 from vllm.compilation.decorators import support_torch_compile

11 from vllm.config import (

12 CompilationConfig,

13 - CompilationLevel,

13 + CompilationMode,

14 VllmConfig,

15 set_current_vllm_config,

16 )

@@ -38,7 +38,7 @@ class CompiledMod(torch.nn.Module):

38 def make_vllm_config() -> VllmConfig:

39 return VllmConfig(

40 compilation_config=CompilationConfig(

41 - level=CompilationLevel.PIECEWISE,

41 + level=CompilationMode.VLLM_COMPILE,

42 )

43 )

tests/compile/test_async_tp.py

+2 -1

@@ -10,6 +10,7 @@ import vllm.envs as envs

10 from vllm.compilation.collective_fusion import AsyncTPPass

11 from vllm.config import (

12 CompilationConfig,

13 + CompilationMode,

14 DeviceConfig,

15 ModelConfig,

16 PassConfig,

@@ -400,7 +401,7 @@ def test_async_tp_pass_correctness(

401 common_args.append("--enforce-eager")

402

403 compilation_config = {

403 - "level": 3,

404 + "mode": CompilationMode.VLLM_COMPILE,

405 "compile_sizes": [2, 4, 8],

406 "splitting_ops": [],

407 "pass_config": {"enable_async_tp": async_tp_enabled},

tests/compile/test_basic_correctness.py

+14 -16

@@ -4,7 +4,7 @@ import dataclasses

5 import pytest

7 -from vllm.config import CompilationLevel

7 +from vllm.config import CompilationMode

8 from vllm.utils import cuda_device_count_stateless

10 from ..utils import compare_all_settings

@@ -21,7 +21,7 @@ class TestSetting:

23 # we cannot afford testing the full Cartesian product

24 -# of all models and all levels

24 +# of all models and all modes

25 @pytest.mark.parametrize(

26 "test_setting",

27 [

@@ -121,15 +121,13 @@ def test_compile_correctness(

121 all_args: list[list[str]] = []

122 all_envs: list[dict[str, str] | None] = []

123

124 - for comp_level in [

125 - CompilationLevel.DYNAMO_AS_IS,

126 - CompilationLevel.DYNAMO_ONCE,

127 - CompilationLevel.PIECEWISE,

124 + for comp_mode in [

125 + CompilationMode.STOCK_TORCH_COMPILE,

126 + CompilationMode.DYNAMO_TRACE_ONCE,

127 + CompilationMode.VLLM_COMPILE,

128 ]:

129 - for level in [CompilationLevel.NO_COMPILATION, comp_level]:

130 - all_args.append(

131 - final_args + [f"-O.level={level}", "-O.backend=inductor"]

132 - )

129 + for mode in [CompilationMode.NONE, comp_mode]:

130 + all_args.append(final_args + [f"-O.mode={mode}", "-O.backend=inductor"])

131

132 # inductor will change the output, so we only compare if the output

133 # is close, not exactly the same.

@@ -142,13 +140,13 @@ def test_compile_correctness(

140 all_envs.clear()

141 all_args.clear()

142

145 - for level in [

146 - CompilationLevel.NO_COMPILATION,

147 - CompilationLevel.DYNAMO_AS_IS,

148 - CompilationLevel.DYNAMO_ONCE,

149 - CompilationLevel.PIECEWISE,

143 + for mode in [

144 + CompilationMode.NONE,

145 + CompilationMode.STOCK_TORCH_COMPILE,

146 + CompilationMode.DYNAMO_TRACE_ONCE,

147 + CompilationMode.VLLM_COMPILE,

148 ]:

151 - all_args.append(final_args + [f"-O.level={level}", "-O.backend=eager"])

149 + all_args.append(final_args + [f"-O.mode={mode}", "-O.backend=eager"])

150 all_envs.append({})

151 all_envs.append({})

152

tests/compile/test_config.py

+10 -10

@@ -4,7 +4,7 @@ import pytest

5 from vllm.compilation.counter import compilation_counter

6 from vllm.config import CompilationConfig, CUDAGraphMode, VllmConfig

7 -from vllm.config.compilation import CompilationLevel

7 +from vllm.config.compilation import CompilationMode

8 from vllm.utils import _is_torch_equal_or_newer, is_torch_equal_or_newer

@@ -90,16 +90,16 @@ def test_use_cudagraphs(vllm_runner, monkeypatch, enabled):

91 # forked needed to workaround https://github.com/vllm-project/vllm/issues/21073

92 @pytest.mark.forked

93 -def test_dynamo_as_is(vllm_runner, monkeypatch):

93 +def test_stock_torch_compile(vllm_runner, monkeypatch):

94 # Disable multiprocessing so that the counter is in the same process

95 monkeypatch.setenv("VLLM_ENABLE_V1_MULTIPROCESSING", "0")

97 with (

98 - compilation_counter.expect(dynamo_as_is_count=1),

98 + compilation_counter.expect(stock_torch_compile_count=1),

99 # loading the model causes compilation (if enabled) to happen

100 vllm_runner(

101 "facebook/opt-125m",

102 - compilation_config={"level": 1},

102 + compilation_config={"mode": CompilationMode.STOCK_TORCH_COMPILE},

103 gpu_memory_utilization=0.4,

104 ) as _,

105 ):

@@ -112,11 +112,11 @@ def test_no_compilation(vllm_runner, monkeypatch):

112 # Disable multiprocessing so that the counter is in the same process

113 monkeypatch.setenv("VLLM_ENABLE_V1_MULTIPROCESSING", "0")

114 with (

115 - compilation_counter.expect(num_graphs_seen=0, dynamo_as_is_count=0),

115 + compilation_counter.expect(num_graphs_seen=0, stock_torch_compile_count=0),

116 # loading the model causes compilation (if enabled) to happen

117 vllm_runner(

118 "facebook/opt-125m",

119 - compilation_config={"level": 0},

119 + compilation_config={"mode": CompilationMode.NONE},

120 gpu_memory_utilization=0.4,

121 ) as _,

122 ):

@@ -130,7 +130,7 @@ def test_enforce_eager(vllm_runner, monkeypatch):

130 monkeypatch.setenv("VLLM_ENABLE_V1_MULTIPROCESSING", "0")

131

132 with (

133 - compilation_counter.expect(num_graphs_seen=0, dynamo_as_is_count=0),

133 + compilation_counter.expect(num_graphs_seen=0, stock_torch_compile_count=0),

134 # loading the model causes compilation (if enabled) to happen

135 vllm_runner(

136 "facebook/opt-125m", enforce_eager=True, gpu_memory_utilization=0.4

@@ -151,7 +151,7 @@ def test_splitting_ops_dynamic():

151 if is_torch_equal_or_newer("2.9.0.dev"):

152 config = VllmConfig(

153 compilation_config=CompilationConfig(

154 - level=CompilationLevel.PIECEWISE,

154 + level=CompilationMode.VLLM_COMPILE,

155 use_inductor_graph_partition=True,

156 splitting_ops=["vllm::unified_attention"],

157 )

@@ -163,7 +163,7 @@ def test_splitting_ops_dynamic():

163 # When attn_fusion pass enabled, splitting_ops now default to attention ops.

164 config = VllmConfig(

165 compilation_config=CompilationConfig(

166 - level=CompilationLevel.PIECEWISE,

166 + level=CompilationMode.VLLM_COMPILE,

167 pass_config={"enable_attn_fusion": True, "enable_noop": True},

168 custom_ops=["+quant_fp8"],

169 cudagraph_mode=CUDAGraphMode.PIECEWISE,

@@ -178,7 +178,7 @@ def test_splitting_ops_dynamic():

178 if is_torch_equal_or_newer("2.9.0.dev"):

179 config = VllmConfig(

180 compilation_config=CompilationConfig(

181 - level=CompilationLevel.PIECEWISE,

181 + level=CompilationMode.VLLM_COMPILE,

182 use_inductor_graph_partition=True,

183 pass_config={"enable_attn_fusion": True, "enable_noop": True},

184 custom_ops=["+quant_fp8"],

tests/compile/test_decorator.py

+5 -5

@@ -8,7 +8,7 @@ from vllm.compilation.decorators import ignore_torch_compile, support_torch_comp

8 from vllm.config import (

9 CacheConfig,

10 CompilationConfig,

11 - CompilationLevel,

11 + CompilationMode,

12 CUDAGraphMode,

13 VllmConfig,

14 set_current_vllm_config,

@@ -66,10 +66,10 @@ def run_model(

68 def test_ignore_torch_compile_decorator():

69 - # piecewise

69 + # vllmcompile

70 vllm_config = VllmConfig(

71 compilation_config=CompilationConfig(

72 - level=CompilationLevel.PIECEWISE,

72 + mode=CompilationMode.VLLM_COMPILE,

73 use_cudagraph=True,

74 splitting_ops=["silly::attention"],

75 cudagraph_capture_sizes=[1, 2],

@@ -185,7 +185,7 @@ def test_conditional_compile_enable_if():

185 kv_sharing_fast_prefill=True,

186 ),

187 compilation_config=CompilationConfig(

188 - level=CompilationLevel.PIECEWISE,

188 + mode=CompilationMode.VLLM_COMPILE,

189 use_cudagraph=True,

190 splitting_ops=["silly::attention"],

191 cudagraph_capture_sizes=[1, 2],

@@ -218,7 +218,7 @@ def test_conditional_compile_enable_if():

218 kv_sharing_fast_prefill=False,

219 ),

220 compilation_config=CompilationConfig(

221 - level=CompilationLevel.PIECEWISE,

221 + mode=CompilationMode.VLLM_COMPILE,

222 use_cudagraph=True,

223 splitting_ops=["silly::attention"],

224 cudagraph_capture_sizes=[1, 2],

tests/compile/test_full_graph.py

+15 -14

@@ -12,7 +12,7 @@ from tests.quantization.utils import is_quant_method_supported

12 from vllm import LLM, SamplingParams

13 from vllm.attention.backends.registry import _Backend

14 from vllm.attention.selector import global_force_attn_backend_context_manager

15 -from vllm.config import CompilationConfig, CompilationLevel, CUDAGraphMode, PassConfig

15 +from vllm.config import CompilationConfig, CompilationMode, CUDAGraphMode, PassConfig

16 from vllm.platforms import current_platform

17 from vllm.utils import is_torch_equal_or_newer

@@ -80,22 +80,22 @@ def models_list(*, all: bool = True, keywords: list[str] | None = None):

82 @pytest.mark.parametrize(

83 - "optimization_level",

84 - [CompilationLevel.DYNAMO_ONCE, CompilationLevel.PIECEWISE],

83 + "compilation_mode",

84 + [CompilationMode.DYNAMO_TRACE_ONCE, CompilationMode.VLLM_COMPILE],

85 )

86 @pytest.mark.parametrize("model_info", models_list(all=True))

87 @create_new_process_for_each_test()

88 def test_full_graph(

89 monkeypatch: pytest.MonkeyPatch,

90 model_info: tuple[str, dict[str, Any]],

91 - optimization_level: int,

91 + compilation_mode: int,

92 ):

93 model, model_kwargs = model_info

95 with monkeypatch.context():

96 print(f"MODEL={model}")

98 - run_model(optimization_level, model, model_kwargs)

98 + run_model(compilation_mode, model, model_kwargs)

100

101 # TODO(luka) add other supported compilation config scenarios here

@@ -104,7 +104,7 @@ def test_full_graph(

104 [

105 # additional compile sizes, only some of the models

106 (

107 - CompilationConfig(level=CompilationLevel.PIECEWISE, compile_sizes=[1, 2]),

107 + CompilationConfig(mode=CompilationMode.VLLM_COMPILE, compile_sizes=[1, 2]),

108 model,

109 )

110 for model in models_list(all=False)

@@ -113,7 +113,7 @@ def test_full_graph(

113 # RMSNorm + quant fusion, only 8-bit quant models

114 (

115 CompilationConfig(

116 - level=CompilationLevel.PIECEWISE,

116 + mode=CompilationMode.VLLM_COMPILE,

117 custom_ops=["+rms_norm"],

118 pass_config=PassConfig(enable_fusion=True, enable_noop=True),

119 ),

@@ -125,7 +125,8 @@ def test_full_graph(

125 # Test depyf integration works

126 (

127 CompilationConfig(

128 - level=CompilationLevel.PIECEWISE, debug_dump_path=tempfile.gettempdir()

128 + mode=CompilationMode.VLLM_COMPILE,

129 + debug_dump_path=tempfile.gettempdir(),

130 ),

131 ("facebook/opt-125m", {}),

132 ),

@@ -134,7 +135,7 @@ def test_full_graph(

135 # graph inductor partition

136 (

137 CompilationConfig(

137 - level=CompilationLevel.PIECEWISE,

138 + mode=CompilationMode.VLLM_COMPILE,

139 # inductor graph partition uses

140 # torch._C.Tag.cudagraph_unsafe to specify splitting ops

141 use_inductor_graph_partition=True,

@@ -164,10 +165,10 @@ def test_custom_compile_config(

165

166

167 @pytest.mark.parametrize(

167 - "optimization_level",

168 - [CompilationLevel.NO_COMPILATION, CompilationLevel.PIECEWISE],

168 + "compilation_mode",

169 + [CompilationMode.NONE, CompilationMode.VLLM_COMPILE],

170 )

170 -def test_fp8_kv_scale_compile(optimization_level: int):

171 +def test_fp8_kv_scale_compile(compilation_mode: int):

172 model = "Qwen/Qwen2-0.5B"

173 model_kwargs = {

174 "quantization": "fp8",

@@ -175,7 +176,7 @@ def test_fp8_kv_scale_compile(optimization_level: int):

176 "calculate_kv_scales": True,

177 "max_model_len": 512,

178 }

178 - run_model(optimization_level, model, model_kwargs)

179 + run_model(compilation_mode, model, model_kwargs)

180

181

182 def test_inductor_graph_partition_attn_fusion(caplog_vllm):

@@ -184,7 +185,7 @@ def test_inductor_graph_partition_attn_fusion(caplog_vllm):

185

186 model = "nvidia/Llama-4-Scout-17B-16E-Instruct-FP8"

187 compilation_config = CompilationConfig(

187 - level=CompilationLevel.PIECEWISE,

188 + mode=CompilationMode.VLLM_COMPILE,

189 use_inductor_graph_partition=True,

190 cudagraph_mode=CUDAGraphMode.PIECEWISE,

191 custom_ops=["+quant_fp8"],

tests/compile/test_fusion.py

+2 -2

@@ -13,7 +13,7 @@ from vllm.compilation.fusion import (

13 )

14 from vllm.compilation.noop_elimination import NoOpEliminationPass

15 from vllm.compilation.post_cleanup import PostCleanupPass

16 -from vllm.config import CompilationConfig, CompilationLevel, PassConfig, VllmConfig

16 +from vllm.config import CompilationConfig, CompilationMode, PassConfig, VllmConfig

17 from vllm.model_executor.layers.layernorm import RMSNorm

18 from vllm.model_executor.layers.quantization.utils.quant_utils import (

19 GroupShape,

@@ -114,7 +114,7 @@ def test_fusion_rmsnorm_quant(

114

115 vllm_config = VllmConfig(

116 compilation_config=CompilationConfig(

117 - level=CompilationLevel.PIECEWISE,

117 + mode=CompilationMode.VLLM_COMPILE,

118 custom_ops=["+rms_norm", "+quant_fp8"],

119 pass_config=PassConfig(enable_fusion=True, enable_noop=True),

120 )

tests/compile/test_fusion_all_reduce.py

+2 -2

@@ -12,7 +12,7 @@ from vllm.compilation.noop_elimination import NoOpEliminationPass

12 from vllm.compilation.post_cleanup import PostCleanupPass

13 from vllm.config import (

14 CompilationConfig,

15 - CompilationLevel,

15 + CompilationMode,

16 DeviceConfig,

17 ModelConfig,

18 PassConfig,

@@ -219,7 +219,7 @@ def all_reduce_fusion_pass_on_test_model(

219

220 vllm_config = VllmConfig(

221 compilation_config=CompilationConfig(

222 - level=CompilationLevel.PIECEWISE, custom_ops=["+rms_norm", "+quant_fp8"]

222 + mode=CompilationMode.VLLM_COMPILE, custom_ops=["+rms_norm", "+quant_fp8"]

223 )

224 )

225 vllm_config.compilation_config.pass_config = PassConfig(

tests/compile/test_fusion_attn.py

+2 -2

@@ -19,7 +19,7 @@ from vllm.compilation.post_cleanup import PostCleanupPass

19 from vllm.config import (

20 CacheConfig,

21 CompilationConfig,

22 - CompilationLevel,

22 + CompilationMode,

23 ModelConfig,

24 PassConfig,

25 SchedulerConfig,

@@ -321,7 +321,7 @@ def test_attention_quant_pattern(

321 ),

322 scheduler_config=SchedulerConfig(max_num_seqs=1024),

323 compilation_config=CompilationConfig(

324 - level=CompilationLevel.PIECEWISE,

324 + mode=CompilationMode.VLLM_COMPILE,

325 custom_ops=["+quant_fp8"],

326 use_inductor_graph_partition=use_inductor_graph_partition,

327 ),

tests/compile/test_noop_elimination.py

+3 -3

@@ -6,7 +6,7 @@ import torch

7 import vllm

8 from vllm.compilation.noop_elimination import NoOpEliminationPass

9 -from vllm.config import CompilationConfig, CompilationLevel, PassConfig, VllmConfig

9 +from vllm.config import CompilationConfig, CompilationMode, PassConfig, VllmConfig

11 from .backend import TestBackend

@@ -50,7 +50,7 @@ def test_noop_elimination(dtype, num_tokens, hidden_size, buffer_size):

51 vllm_config = VllmConfig(

52 compilation_config=CompilationConfig(

53 - level=CompilationLevel.PIECEWISE,

53 + mode=CompilationMode.VLLM_COMPILE,

54 pass_config=PassConfig(enable_noop=True),

55 )

56 )

@@ -98,7 +98,7 @@ def test_non_noop_slice_preserved():

99 vllm_config = VllmConfig(

100 compilation_config=CompilationConfig(

101 - level=CompilationLevel.PIECEWISE,

101 + mode=CompilationMode.VLLM_COMPILE,

102 pass_config=PassConfig(enable_noop=True),

103 )

104 )

tests/compile/test_wrapper.py

+2 -2

@@ -5,7 +5,7 @@

5 import torch

7 from vllm.compilation.wrapper import TorchCompileWrapperWithCustomDispatcher

8 -from vllm.config import CompilationLevel

8 +from vllm.config import CompilationMode

11 class MyMod(torch.nn.Module):

@@ -20,7 +20,7 @@ class MyWrapper(TorchCompileWrapperWithCustomDispatcher):

20 self.model = model

21 compiled_callable = torch.compile(self.forward, backend="eager")

22 super().__init__(

23 - compiled_callable, compilation_level=CompilationLevel.DYNAMO_ONCE

23 + compiled_callable, compilation_mode=CompilationMode.DYNAMO_TRACE_ONCE

24 )

26 def forward(self, x: torch.Tensor, cache: torch.Tensor | None = None):

tests/distributed/test_sequence_parallel.py

+2 -1

@@ -15,6 +15,7 @@ from typing import Literal, NamedTuple

16 import pytest

18 +from vllm.config.compilation import CompilationMode

19 from vllm.config.model import RunnerOption

20 from vllm.logger import init_logger

@@ -234,7 +235,7 @@ def _compare_sp(

235 common_args.append("--skip-tokenizer-init")

236

237 compilation_config = {

237 - "level": 3,

238 + "mode": CompilationMode.VLLM_COMPILE,

239 "custom_ops": ["+rms_norm"],

240 "compile_sizes": [4, 8],

241 "pass_config": {

tests/engine/test_arg_utils.py

+10 -10

@@ -226,30 +226,30 @@ def test_compilation_config():

226

227 # set to O3

228 args = parser.parse_args(["-O0"])

229 - assert args.compilation_config.level == 0

229 + assert args.compilation_config.mode == 0

230

231 # set to O 3 (space)

232 args = parser.parse_args(["-O", "1"])

233 - assert args.compilation_config.level == 1

233 + assert args.compilation_config.mode == 1

234

235 # set to O 3 (equals)

236 args = parser.parse_args(["-O=2"])

237 - assert args.compilation_config.level == 2

237 + assert args.compilation_config.mode == 2

238

239 - # set to O.level 3

240 - args = parser.parse_args(["-O.level", "3"])

241 - assert args.compilation_config.level == 3

239 + # set to O.mode 3

240 + args = parser.parse_args(["-O.mode", "3"])

241 + assert args.compilation_config.mode == 3

242

243 # set to string form of a dict

244 args = parser.parse_args(

245 [

246 "-O",

247 - '{"level": 3, "cudagraph_capture_sizes": [1, 2, 4, 8], '

247 + '{"mode": 3, "cudagraph_capture_sizes": [1, 2, 4, 8], '

248 '"use_inductor": false}',

249 ]

250 )

251 assert (

252 - args.compilation_config.level == 3

252 + args.compilation_config.mode == 3

253 and args.compilation_config.cudagraph_capture_sizes == [1, 2, 4, 8]

254 and not args.compilation_config.use_inductor

255 )

@@ -258,12 +258,12 @@ def test_compilation_config():

258 args = parser.parse_args(

259 [

260 "--compilation-config="

261 - '{"level": 3, "cudagraph_capture_sizes": [1, 2, 4, 8], '

261 + '{"mode": 3, "cudagraph_capture_sizes": [1, 2, 4, 8], '

262 '"use_inductor": true}',

263 ]

264 )

265 assert (

266 - args.compilation_config.level == 3

266 + args.compilation_config.mode == 3

267 and args.compilation_config.cudagraph_capture_sizes == [1, 2, 4, 8]

268 and args.compilation_config.use_inductor

269 )

tests/tpu/test_custom_dispatcher.py

+3 -3

@@ -3,7 +3,7 @@

4 import pytest

6 -from vllm.config import CompilationLevel

6 +from vllm.config import CompilationMode

8 from ..utils import compare_two_settings

@@ -21,13 +21,13 @@ def test_custom_dispatcher(monkeypatch: pytest.MonkeyPatch):

21 "--max-model-len=256",

22 "--max-num-seqs=32",

23 "--enforce-eager",

24 - f"-O{CompilationLevel.DYNAMO_ONCE}",

24 + f"-O{CompilationMode.DYNAMO_TRACE_ONCE}",

25 ],

26 arg2=[

27 "--max-model-len=256",

28 "--max-num-seqs=32",

29 "--enforce-eager",

30 - f"-O{CompilationLevel.DYNAMO_AS_IS}",

30 + f"-O{CompilationMode.STOCK_TORCH_COMPILE}",

31 ],

32 env1={},

33 env2={},

tests/utils_/test_utils.py

+5 -5

@@ -299,7 +299,7 @@ def test_dict_args(parser):

299 "val2",

300 "--hf-overrides.key2.key4",

301 "val3",

302 - # Test compile config and compilation level

302 + # Test compile config and compilation mode

303 "-O.use_inductor=true",

304 "-O.backend",

305 "custom",

@@ -352,7 +352,7 @@ def test_dict_args(parser):

352 },

353 }

354 assert parsed_args.compilation_config == {

355 - "level": 1,

355 + "mode": 1,

356 "use_inductor": True,

357 "backend": "custom",

358 "custom_ops": ["-quant_fp8", "+silu_mul", "-rms_norm"],

@@ -367,7 +367,7 @@ def test_duplicate_dict_args(caplog_vllm, parser):

367 "--hf-overrides.key1",

368 "val2",

369 "-O1",

370 - "-O.level",

370 + "-O.mode",

371 "2",

372 "-O3",

373 ]

@@ -375,12 +375,12 @@ def test_duplicate_dict_args(caplog_vllm, parser):

375 parsed_args = parser.parse_args(args)

376 # Should be the last value

377 assert parsed_args.hf_overrides == {"key1": "val2"}

378 - assert parsed_args.compilation_config == {"level": 3}

378 + assert parsed_args.compilation_config == {"mode": 3}

379

380 assert len(caplog_vllm.records) == 1

381 assert "duplicate" in caplog_vllm.text

382 assert "--hf-overrides.key1" in caplog_vllm.text

383 - assert "-O.level" in caplog_vllm.text

383 + assert "-O.mode" in caplog_vllm.text

384

385

386 @pytest.mark.parametrize(

tests/v1/cudagraph/test_cudagraph_dispatch.py

+11 -11

@@ -11,7 +11,7 @@ from vllm.compilation.cuda_graph import CUDAGraphWrapper

11 from vllm.compilation.monitor import set_cudagraph_capturing_enabled

12 from vllm.config import (

13 CompilationConfig,

14 - CompilationLevel,

14 + CompilationMode,

15 CUDAGraphMode,

16 ParallelConfig,

17 SchedulerConfig,

@@ -42,7 +42,7 @@ def _create_vllm_config(

42 mock_config.parallel_config = ParallelConfig()

44 # Mimic the behavior of VllmConfig.__post_init__()

45 - if compilation_config.level == CompilationLevel.PIECEWISE:

45 + if compilation_config.mode == CompilationMode.VLLM_COMPILE:

46 compilation_config.set_splitting_ops_for_v1()

48 return mock_config

@@ -50,23 +50,23 @@ def _create_vllm_config(

51 class TestCudagraphDispatcher:

52 @pytest.mark.parametrize(

53 - "case_id,cudagraph_mode_str,compilation_level",

53 + "case_id,cudagraph_mode_str,compilation_mode",

54 [

55 # Test case 0: Full CG for mixed batches, no separate routine

56 - (0, "FULL", CompilationLevel.NO_COMPILATION),

56 + (0, "FULL", CompilationMode.NONE),

57 # Test case 1: Full CG for uniform batches, piecewise for mixed

58 - (1, "FULL_AND_PIECEWISE", CompilationLevel.NO_COMPILATION),

58 + (1, "FULL_AND_PIECEWISE", CompilationMode.NONE),

59 # Test case 2: Full CG for uniform batches, no CG for mixed

60 - (2, "FULL_DECODE_ONLY", CompilationLevel.NO_COMPILATION),

61 - # Test case 3: Piecewise for all

62 - (3, "PIECEWISE", CompilationLevel.PIECEWISE),

60 + (2, "FULL_DECODE_ONLY", CompilationMode.NONE),

61 + # Test case 3: PIECEWISE for all

62 + (3, "PIECEWISE", CompilationMode.VLLM_COMPILE),

63 ],

64 )

65 - def test_dispatcher(self, cudagraph_mode_str, compilation_level):

65 + def test_dispatcher(self, cudagraph_mode_str, compilation_mode):

66 # Setup dispatcher

67 comp_config = CompilationConfig(

68 cudagraph_mode=cudagraph_mode_str,

69 - level=compilation_level,

69 + mode=compilation_mode,

70 cudagraph_capture_sizes=[1, 8],

71 )

@@ -242,7 +242,7 @@ class TestCudagraphIntegration:

242 def setup_method(self):

243 # only FULL mode for non-uniform batches

244 self.comp_config = CompilationConfig(

245 - level=CompilationLevel.PIECEWISE,

245 + mode=CompilationMode.VLLM_COMPILE,

246 cudagraph_mode="FULL",

247 cudagraph_capture_sizes=[10, 20],

248 )

tests/v1/cudagraph/test_cudagraph_mode.py

+17 -22

@@ -10,7 +10,7 @@ import pytest

10 from tests.utils import wait_for_gpu_memory_to_clear

11 from tests.v1.attention.utils import full_cg_backend_configs as backend_configs

12 from vllm import LLM

13 -from vllm.config import CompilationConfig

13 +from vllm.config import CompilationConfig, CompilationMode

14 from vllm.platforms import current_platform

@@ -73,7 +73,7 @@ def test_backend_and_cudagraph_mode_combo(backend_name, cudagraph_mode, supporte

73 gpu_memory_utilization=0.45,

74 max_model_len=1024,

75 compilation_config=CompilationConfig(

76 - level=3, cudagraph_mode=cudagraph_mode

76 + mode=CompilationMode.VLLM_COMPILE, cudagraph_mode=cudagraph_mode

77 ),

78 )

79 llm.generate(["Hello, my name is"] * 10)

@@ -90,32 +90,27 @@ def test_backend_and_cudagraph_mode_combo(backend_name, cudagraph_mode, supporte

90 )

93 -# test cudagraph_mode with different compilation level.

94 -# (backend_name, cudagraph_mode, compilation_level, supported)

93 +# test cudagraph_mode with different compilation mode.

94 +# (backend_name, cudagraph_mode, compilation_mode, supported)

95 combo_cases_2 = [

96 - ("FA2", "FULL", 0, True), # no compilation + full cudagraph

97 - ("FA2", "FULL", 3, True), # piecewise compilation + full cudagraph

98 - ("FA2", "PIECEWISE", 0, False), # no compilation + piecewise cudagraph

99 - ("FA2", "PIECEWISE", 3, True), # piecewise compilation + piecewise cudagraph

100 - (

101 - "FA2",

102 - "FULL_AND_PIECEWISE",

103 - 0,

104 - False,

105 - ), # piecewise cudagraph not supported without piecewise compilation

106 - ("FA2", "FULL_AND_PIECEWISE", 3, True),

107 - ("FA2", "FULL_DECODE_ONLY", 0, True),

108 - ("FA2", "FULL_DECODE_ONLY", 3, True),

109 - ("FA2", "NONE", 0, True), # no compilation + no cudagraph

110 - ("FA2", "NONE", 3, True), # piecewise compilation + no cudagraph

96 + ("FA2", "FULL", CompilationMode.NONE, True),

97 + ("FA2", "FULL", CompilationMode.VLLM_COMPILE, True),

98 + ("FA2", "PIECEWISE", CompilationMode.NONE, False),

99 + ("FA2", "PIECEWISE", CompilationMode.VLLM_COMPILE, True),

100 + ("FA2", "FULL_AND_PIECEWISE", CompilationMode.NONE, False),

101 + ("FA2", "FULL_AND_PIECEWISE", CompilationMode.VLLM_COMPILE, True),

102 + ("FA2", "FULL_DECODE_ONLY", CompilationMode.NONE, True),

103 + ("FA2", "FULL_DECODE_ONLY", CompilationMode.VLLM_COMPILE, True),

104 + ("FA2", "NONE", CompilationMode.NONE, True),

105 + ("FA2", "NONE", CompilationMode.VLLM_COMPILE, True),

106 ]

107

108

109 @pytest.mark.parametrize(

115 - "backend_name,cudagraph_mode,compilation_level,supported", combo_cases_2

110 + "backend_name,cudagraph_mode,compilation_mode,supported", combo_cases_2

111 )

112 def test_cudagraph_compilation_combo(combo_case):

118 - backend_name, cudagraph_mode, compilation_level, supported = combo_case

113 + backend_name, cudagraph_mode, compilation_mode, supported = combo_case

114

115 env_vars = backend_configs[backend_name].env_vars

116

@@ -130,7 +125,7 @@ def test_cudagraph_compilation_combo(combo_case):

125 gpu_memory_utilization=0.45,

126 max_model_len=1024,

127 compilation_config=CompilationConfig(

133 - level=compilation_level, cudagraph_mode=cudagraph_mode

128 + mode=compilation_mode, cudagraph_mode=cudagraph_mode

129 ),

130 )

131 llm.generate(["Hello, my name is"] * 10)

tests/v1/e2e/test_kv_sharing_fast_prefill.py

+3 -3

@@ -7,7 +7,7 @@ import pytest

7 import torch

9 from vllm import LLM, SamplingParams

10 -from vllm.config import CompilationConfig, CompilationLevel

10 +from vllm.config import CompilationConfig, CompilationMode

11 from vllm.distributed import cleanup_dist_env_and_memory

13 from ...utils import fork_new_process_for_each_test

@@ -75,9 +75,9 @@ def test_kv_sharing_fast_prefill(

75 # This allows vLLM compilation backend to handle allocating and

76 # managing buffers for cudagraph

77 cudagraph_copy_inputs=True,

78 - level=CompilationLevel.PIECEWISE

78 + mode=CompilationMode.VLLM_COMPILE

79 if not enforce_eager

80 - else CompilationLevel.NO_COMPILATION,

80 + else CompilationMode.NONE,

81 )

83 with monkeypatch.context() as m:

vllm/compilation/backends.py

+2 -2

@@ -56,7 +56,7 @@ def make_compiler(compilation_config: CompilationConfig) -> CompilerInterface:

56 return InductorAdaptor()

57 else:

58 assert compilation_config.backend == "eager", (

59 - "Custom backends not supported with CompilationLevel.PIECEWISE"

59 + "Custom backends not supported with CompilationMode.VLLM_COMPILE"

60 )

62 logger.debug("Using EagerAdaptor")

@@ -481,7 +481,7 @@ def set_model_tag(tag: str):

481

482 class VllmBackend:

483 """The compilation backend for `torch.compile` with vLLM.

484 - It is used for compilation level of `CompilationLevel.PIECEWISE`,

484 + It is used for compilation mode of `CompilationMode.VLLM_COMPILE`,

485 where we customize the compilation.

486

487 The major work of this backend is to split the graph into

vllm/compilation/compiler_interface.py

+1 -1

@@ -575,7 +575,7 @@ class InductorAdaptor(CompilerInterface):

575

576 Because it is re-entrant, we always set it (even if entering via Dynamo

577 and the context was already entered). We might want to revisit if it

578 - should be set at a different level of compilation.

578 + should be set at a different mode of compilation.

579

580 This is likely a bug in PyTorch: public APIs should not rely on

581 manually setting up internal contexts. But we also rely on non-public

vllm/compilation/counter.py

+2 -2

@@ -27,8 +27,8 @@ class CompilationCounter:

27 num_cache_entries_updated: int = 0

28 # The number of standalone_compile compiled artifacts saved

29 num_compiled_artifacts_saved: int = 0

30 - # Number of times a model was loaded with CompilationLevel.DYNAMO_AS_IS

31 - dynamo_as_is_count: int = 0

30 + # Number of times a model was loaded with CompilationMode.STOCK_TORCH_COMPILE

31 + stock_torch_compile_count: int = 0

33 def clone(self) -> "CompilationCounter":

34 return copy.deepcopy(self)

vllm/compilation/decorators.py

+5 -5

@@ -18,7 +18,7 @@ from torch._dynamo.symbolic_convert import InliningInstructionTranslator

18 import vllm.envs as envs

19 from vllm.compilation.counter import compilation_counter

20 from vllm.compilation.wrapper import TorchCompileWrapperWithCustomDispatcher

21 -from vllm.config import CompilationLevel, VllmConfig, set_current_vllm_config

21 +from vllm.config import CompilationMode, VllmConfig, set_current_vllm_config

22 from vllm.logger import init_logger

23 from vllm.sequence import IntermediateTensors

24 from vllm.utils import resolve_obj_by_qualname, supports_dynamo

@@ -233,11 +233,11 @@ def _support_torch_compile(

233 old_init(self, vllm_config=vllm_config, prefix=prefix, **kwargs)

234 self.vllm_config = vllm_config

235 enable_compile = enable_if is None or enable_if(vllm_config)

236 - # for CompilationLevel.DYNAMO_AS_IS , the upper level model runner

236 + # for CompilationMode.STOCK_TORCH_COMPILE , the upper level model runner

237 # will handle the compilation, so we don't need to do anything here.

238 self.do_not_compile = (

239 - vllm_config.compilation_config.level

240 - in [CompilationLevel.NO_COMPILATION, CompilationLevel.DYNAMO_AS_IS]

239 + vllm_config.compilation_config.mode

240 + in [CompilationMode.NONE, CompilationMode.STOCK_TORCH_COMPILE]

241 or not supports_dynamo()

242 or _should_ignore_torch_compile(self.__class__)

243 or not enable_compile

@@ -247,7 +247,7 @@ def _support_torch_compile(

247

248 compilation_counter.num_models_seen += 1

249 TorchCompileWrapperWithCustomDispatcher.__init__(

250 - self, compilation_level=vllm_config.compilation_config.level

250 + self, compilation_mode=vllm_config.compilation_config.mode

251 )

252

253 cls.__init__ = __init__

vllm/compilation/monitor.py

+3 -3

@@ -3,7 +3,7 @@

4 import time

6 -from vllm.config import CompilationConfig, CompilationLevel, VllmConfig

6 +from vllm.config import CompilationConfig, CompilationMode, VllmConfig

7 from vllm.logger import init_logger

9 logger = init_logger(__name__)

@@ -18,7 +18,7 @@ def start_monitoring_torch_compile(vllm_config: VllmConfig):

19 compilation_config: CompilationConfig = vllm_config.compilation_config

20 path = vllm_config.compile_debug_dump_path()

21 - if compilation_config.level == CompilationLevel.PIECEWISE and path:

21 + if compilation_config.mode == CompilationMode.VLLM_COMPILE and path:

22 import depyf

24 path.mkdir(parents=True, exist_ok=True)

@@ -29,7 +29,7 @@ def start_monitoring_torch_compile(vllm_config: VllmConfig):

30 def end_monitoring_torch_compile(vllm_config: VllmConfig):

31 compilation_config: CompilationConfig = vllm_config.compilation_config

32 - if compilation_config.level == CompilationLevel.PIECEWISE:

32 + if compilation_config.mode == CompilationMode.VLLM_COMPILE:

33 logger.info(

34 "torch.compile takes %.2f s in total", compilation_config.compilation_time

35 )

vllm/compilation/wrapper.py

+4 -4

@@ -11,7 +11,7 @@ from types import CodeType

11 import torch

13 import vllm.envs as envs

14 -from vllm.config import CompilationLevel, CUDAGraphMode, get_current_vllm_config

14 +from vllm.config import CompilationMode, CUDAGraphMode, get_current_vllm_config

15 from vllm.logger import init_logger

17 logger = init_logger(__name__)

@@ -31,7 +31,7 @@ class TorchCompileWrapperWithCustomDispatcher:

31 """

33 def __init__(

34 - self, compiled_callable: Callable | None = None, compilation_level: int = 0

34 + self, compiled_callable: Callable | None = None, compilation_mode: int = 0

35 ):

36 vllm_config = get_current_vllm_config()

37 self.vllm_config = vllm_config

@@ -72,7 +72,7 @@ class TorchCompileWrapperWithCustomDispatcher:

72 # subclasses can use this to switch between the custom dispatcher

73 # and the default Dynamo guard mechanism.

74 self.use_custom_dispatcher: bool = (

75 - compilation_level >= CompilationLevel.DYNAMO_ONCE

75 + compilation_mode >= CompilationMode.DYNAMO_TRACE_ONCE

76 )

78 def aot_compile(self, *args, **kwargs):

@@ -85,7 +85,7 @@ class TorchCompileWrapperWithCustomDispatcher:

85 return self.compiled_callable.aot_compile((args, kwargs))

87 def __call__(self, *args, **kwargs):

88 - """Implement the dispatch logic here, beyond the torch.compile level.

88 + """Implement the dispatch logic here, beyond the torch.compile mode.

89 NOTE: this function can have additional arguments beyond the forward

90 method, for directly dispatching to the compiled code.

91 """

vllm/config/__init__.py

+2 -2

@@ -4,7 +4,7 @@

4 from vllm.config.cache import CacheConfig

5 from vllm.config.compilation import (

6 CompilationConfig,

7 - CompilationLevel,

7 + CompilationMode,

8 CUDAGraphMode,

9 PassConfig,

10 )

@@ -49,7 +49,7 @@ __all__ = [

49 "CacheConfig",

50 # From vllm.config.compilation

51 "CompilationConfig",

52 - "CompilationLevel",

52 + "CompilationMode",

53 "CUDAGraphMode",

54 "PassConfig",

55 # From vllm.config.device

vllm/config/compilation.py

+70 -38

@@ -26,12 +26,20 @@ else:

26 logger = init_logger(__name__)

29 -class CompilationLevel:

30 - # constants for the levels of the compilation process

31 - NO_COMPILATION = 0

32 - DYNAMO_AS_IS = 1

33 - DYNAMO_ONCE = 2

34 - PIECEWISE = 3

29 +class CompilationMode:

30 + """The compilation approach used for torch.compile-based compilation of the

31 + model."""

32 +

33 + NONE = 0

34 + """No torch.compile compilation is applied, model runs in fully eager pytorch mode.

35 + The model runs as-is."""

36 + STOCK_TORCH_COMPILE = 1

37 + """The standard `torch.compile` compilation pipeline."""

38 + DYNAMO_TRACE_ONCE = 2

39 + """Single Dynamo trace through the model, avoiding recompilation."""

40 + VLLM_COMPILE = 3

41 + """Custom vLLM Inductor-based backend with caching, piecewise compilation,

42 + shape specialization, and custom passes."""

45 class CUDAGraphMode(enum.Enum):

@@ -134,7 +142,7 @@ class CompilationConfig:

142 """Configuration for compilation. It has three parts:

143

144 - Top-level Compilation control:

137 - - [`level`][vllm.config.CompilationConfig.level]

145 + - [`mode`][vllm.config.CompilationConfig.mode]

146 - [`debug_dump_path`][vllm.config.CompilationConfig.debug_dump_path]

147 - [`cache_dir`][vllm.config.CompilationConfig.cache_dir]

148 - [`backend`][vllm.config.CompilationConfig.backend]

@@ -171,14 +179,26 @@ class CompilationConfig:

179

180 # Top-level Compilation control

181 level: int | None = None

174 - """The level of compilation:

175 -

176 - - None: If None, we will select the default compilation level.

177 - For V1 engine this is 3, for V0 engine this is 0.

178 - - 0: no compilation.

179 - - 1: dynamo as is.

180 - - 2: dynamo once.

181 - - 3: piecewise compilation."""

182 + """

183 + Level is deprecated and will be removed in the next release,

184 + either 0.12.0 or 0.11.2 whichever is soonest.

185 + Please use mode. Currently all levels are mapped to mode.

186 + """

187 + # Top-level Compilation control

188 + mode: int | None = None

189 + """The compilation approach used for torch.compile-based compilation of the

190 + model.

191 +

192 + - None: If None, we will select the default compilation mode.

193 + For V1 engine this is 3.

194 + - 0: NONE: No torch.compile compilation is applied, model runs in fully

195 + eager pytorch mode. The model runs as-is.

196 + - 1: STOCK_TORCH_COMPILE: The standard `torch.compile` compilation pipeline.

197 + - 2: DYNAMO_TRACE_ONCE: Single Dynamo trace through the model, avoiding

198 + recompilation by removing guards.

199 + Requires no dynamic-shape-dependent control-flow.

200 + - 3: VLLM_COMPILE: Custom vLLM Inductor-based backend with caching,

201 + piecewise compilation, shape specialization, and custom passes."""

202 debug_dump_path: Path | None = None

203 """The path to dump the debug information."""

204 cache_dir: str = ""

@@ -195,11 +215,11 @@ class CompilationConfig:

215

216 backend function.

217 We use string to avoid serialization issues when using compilation in a

198 - distributed setting. When the compilation level is 1 or 2, the backend is

218 + distributed setting. When the compilation mode is 1 or 2, the backend is

219 used for the compilation directly (it sees the whole graph). When the

200 - compilation level is 3, the backend is used for the piecewise compilation

220 + compilation mode is 3, the backend is used for the piecewise compilation

221 (it sees a part of the graph). The backend can not be custom for compilation

202 - level 3, i.e. the backend must be either eager or inductor. Furthermore,

222 + mode 3, i.e. the backend must be either eager or inductor. Furthermore,

223 compilation is only piecewise if splitting ops is set accordingly and

224 use_inductor_graph_partition is off. Note that the default options for

225 splitting ops are sufficient for piecewise compilation.

@@ -214,7 +234,7 @@ class CompilationConfig:

234 - 'none,+op1,+op2' to enable only op1 and op2

235

236 By default, all custom ops are enabled when running without Inductor and

217 - disabled when running with Inductor: level>=PIECEWISE and use_inductor=True.

237 + disabled when running with Inductor: mode>=VLLM_COMPILE and use_inductor=True.

238 Inductor generates (fused) Triton kernels for disabled custom ops."""

239 splitting_ops: list[str] | None = None

240 """A list of ops to exclude from cudagraphs, used in piecewise compilation.

@@ -249,7 +269,7 @@ class CompilationConfig:

269 One graph for symbolic shape and one graph per size in compile_sizes

270 are compiled using configurations in inductor_compile_config.

271

252 - This setting is ignored if level<PIECEWISE.

272 + This setting is ignored if mode<VLLM_COMPILE.

273

274 For future compatibility:

275 If use_inductor is True, backend="inductor" otherwise backend="eager".

@@ -299,7 +319,7 @@ class CompilationConfig:

319 Currently, the cudagraph mode is only used for the v1 engine.

320 Note that the cudagraph logic is generally orthogonal to the

321 compilation logic. While piecewise cudagraphs require piecewise

302 - compilation (level=PIECEWISE and non-empty splitting_ops), full

322 + compilation (mode=VLLM_COMPILE and non-empty splitting_ops), full

323 cudagraphs are supported with and without compilation.

324

325 Warning: This flag is new and subject to change in addition

@@ -312,7 +332,7 @@ class CompilationConfig:

332 that all input buffers have fixed addresses, and all

333 splitting ops write their outputs to input buffers.

334 In the vLLM V1 Engine, this flag only applies for

315 - CompilationLevel.PIECEWISE (aka -O3).

335 + CompilationMode.VLLM_COMPILE (aka -O3).

336 Note that this is orthogonal to the cudagraph capture logic

337 outside of compilation.

338 Warning: This flag is deprecated and will be removed in the next major or

@@ -426,7 +446,7 @@ class CompilationConfig:

446 the final hidden states.

447 """

448 factors: list[Any] = []

429 - factors.append(self.level)

449 + factors.append(self.mode)

450 factors.append(self.backend)

451 factors.append(self.custom_ops)

452 factors.append(self.splitting_ops)

@@ -477,6 +497,17 @@ class CompilationConfig:

497 return value

498

499 def __post_init__(self) -> None:

500 + if self.level is not None:

501 + logger.warning(

502 + "Level is deprecated and will be removed in the next release,"

503 + "either 0.12.0 or 0.11.2 whichever is soonest."

504 + "Use mode instead."

505 + "If both level and mode are given,"

506 + "only mode will be used."

507 + )

508 + if self.mode is None:

509 + self.mode = self.level

510 +

511 count_none = self.custom_ops.count("none")

512 count_all = self.custom_ops.count("all")

513 assert count_none + count_all <= 1, "Can only specify 'none' or 'all'"

@@ -574,7 +605,7 @@ class CompilationConfig:

605 # Currently only eager and inductor backend are supported.

606 # for piecewise compilation. Custom backends are not suppported for

607 # piecewise compilation. Update when more backends are supported.

577 - if self.level == CompilationLevel.PIECEWISE and self.backend not in [

608 + if self.mode == CompilationMode.VLLM_COMPILE and self.backend not in [

609 "",

610 "eager",

611 "inductor",

@@ -602,24 +633,27 @@ class CompilationConfig:

633 Returns:

634 The backend for the compilation config.

635 """

605 - if self.level is None:

636 + if self.mode is None:

637 raise ValueError(

607 - "No compilation level is set. This method should only be \

638 + "No compilation mode is set. This method should only be \

639 called via vllm config where the level is set if none is \

640 provided."

641 )

611 - if self.level == CompilationLevel.NO_COMPILATION:

612 - raise ValueError("No compilation level is set.")

642 + if self.mode == CompilationMode.NONE:

643 + raise ValueError("No compilation mode is set.")

644

645 from torch._dynamo.backends.registry import list_backends

646

647 torch_backends = list_backends(exclude_tags=tuple())

617 - if self.level in [CompilationLevel.DYNAMO_AS_IS, CompilationLevel.DYNAMO_ONCE]:

648 + if self.mode in [

649 + CompilationMode.STOCK_TORCH_COMPILE,

650 + CompilationMode.DYNAMO_TRACE_ONCE,

651 + ]:

652 if self.backend in torch_backends:

653 return self.backend

654 return resolve_obj_by_qualname(self.backend)

655

622 - assert self.level == CompilationLevel.PIECEWISE

656 + assert self.mode == CompilationMode.VLLM_COMPILE

657 if self.backend not in ["eager", "inductor"]:

658 raise ValueError(

659 f"Invalid backend for piecewise compilation: {self.backend}"

@@ -684,11 +718,11 @@ class CompilationConfig:

718 self.bs_to_padded_graph_size[self.max_capture_size] = self.max_capture_size

719

720 def set_splitting_ops_for_v1(self):

687 - # NOTE: this function needs to be called only when level is

688 - # CompilationLevel.PIECEWISE

689 - assert self.level == CompilationLevel.PIECEWISE, (

721 + # NOTE: this function needs to be called only when mode is

722 + # CompilationMode.VLLM_COMPILE

723 + assert self.mode == CompilationMode.VLLM_COMPILE, (

724 "set_splitting_ops_for_v1 should only be called when "

691 - "level is CompilationLevel.PIECEWISE"

725 + "mode is CompilationMode.VLLM_COMPILE"

726 )

727

728 if self.use_inductor_graph_partition:

@@ -769,12 +803,10 @@ class CompilationConfig:

803

804 if not self.use_inductor_graph_partition:

805 # Dynamo-level FX split case

772 - return self.level == CompilationLevel.PIECEWISE

806 + return self.mode == CompilationMode.VLLM_COMPILE

807

808 # Inductor partition case

775 - return (

776 - self.backend == "inductor" and self.level > CompilationLevel.NO_COMPILATION

777 - )

809 + return self.backend == "inductor" and self.mode > CompilationMode.NONE

810

811 def custom_op_log_check(self):

812 """

vllm/config/vllm.py

+22 -28

@@ -22,7 +22,7 @@ from vllm.transformers_utils.runai_utils import is_runai_obj_uri

22 from vllm.utils import random_uuid

24 from .cache import CacheConfig

25 -from .compilation import CompilationConfig, CompilationLevel, CUDAGraphMode

25 +from .compilation import CompilationConfig, CompilationMode, CUDAGraphMode

26 from .device import DeviceConfig

27 from .kv_events import KVEventsConfig

28 from .kv_transfer import KVTransferConfig

@@ -84,17 +84,11 @@ class VllmConfig:

84 compilation_config: CompilationConfig = Field(default_factory=CompilationConfig)

85 """`torch.compile` and cudagraph capture configuration for the model.

87 - As a shorthand, `-O<n>` can be used to directly specify the compilation

88 - level `n`: `-O3` is equivalent to `-O.level=3` (same as `-O='{"level":3}'`).

89 - Currently, -O <n> and -O=<n> are supported as well but this will likely be

90 - removed in favor of clearer -O<n> syntax in the future.

91 -

92 - NOTE: level 0 is the default level without any optimization. level 1 and 2

93 - are for internal testing only. level 3 is the recommended level for

94 - production, also default in V1.

87 + As a shorthand, one can append compilation arguments via

88 + -0.parameter=arguement such as `-O.mode=3` (same as `-O='{"mode":3}'`).

90 You can specify the full compilation config like so:

97 - `{"level": 3, "cudagraph_capture_sizes": [1, 2, 4, 8]}`

91 + `{"mode": 3, "cudagraph_capture_sizes": [1, 2, 4, 8]}`

92 """

93 kv_transfer_config: KVTransferConfig | None = None

94 """The configurations for distributed KV cache transfer."""

@@ -305,33 +299,33 @@ class VllmConfig:

299 "precision for chunked prefill triton kernels."

300 )

301

308 - # If the user does not explicitly set a compilation level, then

309 - # we use the default level. The default level depends on other

302 + # If the user does not explicitly set a compilation mode, then

303 + # we use the default mode. The default mode depends on other

304 # settings (see the below code).

311 - if self.compilation_config.level is None:

305 + if self.compilation_config.mode is None:

306 if envs.VLLM_USE_V1:

307 if (

308 self.model_config is not None

309 and not self.model_config.enforce_eager

310 ):

317 - self.compilation_config.level = CompilationLevel.PIECEWISE

311 + self.compilation_config.mode = CompilationMode.VLLM_COMPILE

312 else:

319 - self.compilation_config.level = CompilationLevel.NO_COMPILATION

313 + self.compilation_config.mode = CompilationMode.NONE

314

315 else:

322 - # NB: Passing both --enforce-eager and a compilation level

323 - # in V0 means the compilation level wins out.

324 - self.compilation_config.level = CompilationLevel.NO_COMPILATION

316 + # NB: Passing both --enforce-eager and a compilation mode

317 + # in V0 means the compilation mode wins out.

318 + self.compilation_config.mode = CompilationMode.NONE

319 else:

326 - assert self.compilation_config.level >= CompilationLevel.NO_COMPILATION

327 - assert self.compilation_config.level <= CompilationLevel.PIECEWISE

320 + assert self.compilation_config.mode >= CompilationMode.NONE

321 + assert self.compilation_config.mode <= CompilationMode.VLLM_COMPILE

322

323 # If user does not set custom ops via none or all set it here based on

330 - # compilation level and backend.

324 + # compilation mode and backend.

325 if all(s not in self.compilation_config.custom_ops for s in ("all", "none")):

326 if (

327 self.compilation_config.backend == "inductor"

334 - and self.compilation_config.level > CompilationLevel.NO_COMPILATION

328 + and self.compilation_config.mode > CompilationMode.NONE

329 ):

330 self.compilation_config.custom_ops.append("none")

331 else:

@@ -350,7 +344,7 @@ class VllmConfig:

344 if self.compilation_config.cudagraph_mode is None:

345 if (

346 envs.VLLM_USE_V1

353 - and self.compilation_config.level == CompilationLevel.PIECEWISE

347 + and self.compilation_config.mode == CompilationMode.VLLM_COMPILE

348 ):

349 # default to full and piecewise for most models

350 self.compilation_config.cudagraph_mode = (

@@ -486,10 +480,10 @@ class VllmConfig:

480 )

481 current_platform.check_and_update_config(self)

482

489 - # Do this after all the updates to compilation_config.level

483 + # Do this after all the updates to compilation_config.mode

484 if (

485 envs.VLLM_USE_V1

492 - and self.compilation_config.level == CompilationLevel.PIECEWISE

486 + and self.compilation_config.mode == CompilationMode.VLLM_COMPILE

487 ):

488 self.compilation_config.set_splitting_ops_for_v1()

489

@@ -508,8 +502,8 @@ class VllmConfig:

502 )

503

504 if self.compilation_config.cudagraph_mode.requires_piecewise_compilation():

511 - assert self.compilation_config.level == CompilationLevel.PIECEWISE, (

512 - "Compilation level should be CompilationLevel.PIECEWISE "

505 + assert self.compilation_config.mode == CompilationMode.VLLM_COMPILE, (

506 + "Compilation mode should be CompilationMode.VLLM_COMPILE "

507 "when cudagraph_mode piecewise cudagraphs is used, "

508 f"cudagraph_mode={self.compilation_config.cudagraph_mode}"

509 )

@@ -837,7 +831,7 @@ def set_current_vllm_config(

831

832 if (

833 check_compile

840 - and vllm_config.compilation_config.level == CompilationLevel.PIECEWISE

834 + and vllm_config.compilation_config.mode == CompilationMode.VLLM_COMPILE

835 and compilation_counter.num_models_seen == num_models_seen

836 ):

837 # If the model supports compilation,

vllm/entrypoints/llm.py

+2 -4

@@ -176,7 +176,7 @@ class LLM:

176 argument is deprecated and will be removed in v0.12.0 or v1.0.0,

177 whichever is sooner.

178 compilation_config: Either an integer or a dictionary. If it is an

179 - integer, it is used as the level of compilation optimization. If it

179 + integer, it is used as the mode of compilation optimization. If it

180 is a dictionary, it can specify the full compilation configuration.

181 **kwargs: Arguments for [`EngineArgs`][vllm.EngineArgs].

182

@@ -257,9 +257,7 @@ class LLM:

257

258 if compilation_config is not None:

259 if isinstance(compilation_config, int):

260 - compilation_config_instance = CompilationConfig(

261 - level=compilation_config

262 - )

260 + compilation_config_instance = CompilationConfig(mode=compilation_config)

261 elif isinstance(compilation_config, dict):

262 compilation_config_instance = CompilationConfig(

263 **{

vllm/model_executor/layers/quantization/utils/w8a8_utils.py

+2 -2

@@ -8,7 +8,7 @@ from packaging import version

9 from vllm import _custom_ops as ops

10 from vllm import envs

11 -from vllm.config import CompilationLevel, get_current_vllm_config

11 +from vllm.config import CompilationMode, get_current_vllm_config

12 from vllm.model_executor.layers.quantization.input_quant_fp8 import QuantFP8

13 from vllm.model_executor.layers.quantization.utils.quant_utils import GroupShape

14 from vllm.platforms import current_platform

@@ -419,7 +419,7 @@ class Fp8LinearOp:

419 if pad_output is None:

420 config = get_current_vllm_config().compilation_config

421 pad_output = (

422 - config.level < CompilationLevel.PIECEWISE

422 + config.mode < CompilationMode.VLLM_COMPILE

423 and self.preferred_backend == "torch"

424 )

425

vllm/platforms/cpu.py

+4 -4

@@ -247,12 +247,12 @@ class CpuPlatform(Platform):

247 parallel_config.enable_dbo = False

248

249 # Note: workaround for v1 gpu_model_runner

250 - from vllm.config import CompilationLevel

250 + from vllm.config import CompilationMode

251

252 vllm_config.compilation_config.cudagraph_capture_sizes = []

253

254 compilation_config = vllm_config.compilation_config

255 - if vllm_config.compilation_config.level == CompilationLevel.PIECEWISE:

255 + if vllm_config.compilation_config.mode == CompilationMode.VLLM_COMPILE:

256 # Note: vLLM V1 is using PIECEWISE level compilation, which will

257 # take time to compile kernels just-in-time with the inductor

258 # backend. For CPU CI tests, most of them are executed fast and

@@ -265,7 +265,7 @@ class CpuPlatform(Platform):

265 else:

266 backend = "inductor"

267

268 - compilation_config.level = CompilationLevel.DYNAMO_ONCE

268 + compilation_config.mode = CompilationMode.DYNAMO_TRACE_ONCE

269 compilation_config.backend = backend

270 compilation_config.inductor_compile_config.update(

271 {

@@ -277,7 +277,7 @@ class CpuPlatform(Platform):

277 )

278

279 if vllm_config.lora_config is not None:

280 - compilation_config.level = CompilationLevel.NO_COMPILATION

280 + compilation_config.mode = CompilationMode.NONE

281

282 assert vllm_config.device_config.device_type == "cpu"

283

vllm/platforms/tpu.py

+6 -5

@@ -114,7 +114,7 @@ class TpuPlatform(Platform):

114

115 @classmethod

116 def check_and_update_config(cls, vllm_config: VllmConfig) -> None:

117 - from vllm.config import CompilationLevel, CUDAGraphMode

117 + from vllm.config import CompilationMode, CUDAGraphMode

118

119 cache_config = vllm_config.cache_config

120 # For v0, the default block size is 16.

@@ -122,12 +122,13 @@ class TpuPlatform(Platform):

122 cache_config.block_size = cast(BlockSize, 16)

123 compilation_config = vllm_config.compilation_config

124

125 - # TPU only supports DYNAMO_ONCE compilation level

126 - if compilation_config.level != CompilationLevel.DYNAMO_ONCE:

125 + # TPU only supports DYNAMO_TRACE_ONCE compilation mode

126 + if compilation_config.mode != CompilationMode.DYNAMO_TRACE_ONCE:

127 logger.info(

128 - "[TPU] Forcing DYNAMO_ONCE compilation level, and disabling cudagraph."

128 + "[TPU] Forcing DYNAMO_TRACE_ONCE compilation mode, and\

129 + disabling cudagraph."

130 )

130 - compilation_config.level = CompilationLevel.DYNAMO_ONCE

131 + compilation_config.mode = CompilationMode.DYNAMO_TRACE_ONCE

132

133 if (

134 compilation_config.cudagraph_mode is None

vllm/platforms/xpu.py

+2 -2

@@ -144,7 +144,7 @@ class XPUPlatform(Platform):

144 cache_config.block_size = 64

145

146 # lazy import to avoid circular import

147 - from vllm.config import CompilationLevel, CUDAGraphMode

147 + from vllm.config import CompilationMode, CUDAGraphMode

148

149 compilation_config = vllm_config.compilation_config

150 if compilation_config.compile_sizes is None:

@@ -155,7 +155,7 @@ class XPUPlatform(Platform):

155 )

156

157 if vllm_config.lora_config is not None:

158 - compilation_config.level = CompilationLevel.NO_COMPILATION

158 + compilation_config.mode = CompilationMode.NONE

159

160 # check and update parallel config

161 parallel_config = vllm_config.parallel_config

vllm/utils/__init__.py

+5 -5

@@ -1686,16 +1686,16 @@ class FlexibleArgumentParser(ArgumentParser):

1686 elif arg.startswith("-O") and arg != "-O" and arg[2] != ".":

1687 # allow -O flag to be used without space, e.g. -O3 or -Odecode

1688 # -O.<...> handled later

1689 - # also handle -O=<level> here

1690 - level = arg[3:] if arg[2] == "=" else arg[2:]

1691 - processed_args.append(f"-O.level={level}")

1689 + # also handle -O=<mode> here

1690 + mode = arg[3:] if arg[2] == "=" else arg[2:]

1691 + processed_args.append(f"-O.mode={mode}")

1692 elif (

1693 arg == "-O"

1694 and i + 1 < len(args)

1695 and args[i + 1] in {"0", "1", "2", "3"}

1696 ):

1697 - # Convert -O <n> to -O.level <n>

1698 - processed_args.append("-O.level")

1697 + # Convert -O <n> to -O.mode <n>

1698 + processed_args.append("-O.mode")

1699 else:

1700 processed_args.append(arg)

1701

vllm/v1/cudagraph_dispatcher.py

+2 -2

@@ -43,12 +43,12 @@ class CudagraphDispatcher:

43 not_use_piecewise_compilation

44 or self.compilation_config.is_attention_compiled_piecewise()

45 ), (

46 - "Compilation level should be CompilationLevel.PIECEWISE when "

46 + "Compilation mode should be CompilationMode.VLLM_COMPILE when "

47 "cudagraph_mode piecewise cudagraphs is used, "

48 "and attention should be in splitting_ops or "

49 "inductor splitting should be used. "

50 f"cudagraph_mode={self.cudagraph_mode}, "

51 - f"compilation_level={self.compilation_config.level}, "

51 + f"compilation_mode={self.compilation_config.mode}, "

52 f"splitting_ops={self.compilation_config.splitting_ops}"

53 )

vllm/v1/spec_decode/eagle.py

+2 -2

@@ -9,7 +9,7 @@ import torch

9 import torch.nn as nn

11 from vllm.config import (

12 - CompilationLevel,

12 + CompilationMode,

13 CUDAGraphMode,

14 VllmConfig,

15 get_layers_from_vllm_config,

@@ -86,7 +86,7 @@ class EagleProposer:

86 self.use_cuda_graph = False

88 compilation_config = self.vllm_config.compilation_config

89 - if compilation_config.level == CompilationLevel.PIECEWISE:

89 + if compilation_config.mode == CompilationMode.VLLM_COMPILE:

90 cudagraph_mode = compilation_config.cudagraph_mode

91 if cudagraph_mode != CUDAGraphMode.NONE and not cudagraph_mode.has_mode(

92 CUDAGraphMode.PIECEWISE

vllm/v1/worker/gpu_model_runner.py

+8 -7

@@ -25,7 +25,7 @@ from vllm.compilation.counter import compilation_counter

25 from vllm.compilation.cuda_graph import CUDAGraphWrapper

26 from vllm.compilation.monitor import set_cudagraph_capturing_enabled

27 from vllm.config import (

28 - CompilationLevel,

28 + CompilationMode,

29 CUDAGraphMode,

30 VllmConfig,

31 get_layers_from_vllm_config,

@@ -2927,14 +2927,15 @@ class GPUModelRunner(LoRAModelRunnerMixin, KVConnectorModelRunnerMixin):

2927 )

2928

2929 if (

2930 - self.vllm_config.compilation_config.level == CompilationLevel.DYNAMO_AS_IS

2930 + self.vllm_config.compilation_config.mode

2931 + == CompilationMode.STOCK_TORCH_COMPILE

2932 and supports_dynamo()

2933 ):

2934 backend = self.vllm_config.compilation_config.init_backend(self.vllm_config)

2934 - compilation_counter.dynamo_as_is_count += 1

2935 + compilation_counter.stock_torch_compile_count += 1

2936 self.model.compile(fullgraph=True, backend=backend)

2937 return

2937 - # for other compilation levels, cudagraph behavior is controlled by

2938 + # for other compilation modes, cudagraph behavior is controlled by

2939 # CudagraphWraper and CudagraphDispatcher of vllm.

2940

2941 # wrap the model with full cudagraph wrapper if needed.

@@ -3985,7 +3986,7 @@ class GPUModelRunner(LoRAModelRunnerMixin, KVConnectorModelRunnerMixin):

3986 # if not supported any full cudagraphs, just raise it.

3987 msg += (

3988 "; please try cudagraph_mode=PIECEWISE, and "

3988 - "make sure compilation level is piecewise"

3989 + "make sure compilation mode is VLLM_COMPILE"

3990 )

3991 raise ValueError(msg)

3992

@@ -4012,7 +4013,7 @@ class GPUModelRunner(LoRAModelRunnerMixin, KVConnectorModelRunnerMixin):

4013 f"with {min_cg_builder_name} backend (support: "

4014 f"{min_cg_support})"

4015 )

4015 - if self.compilation_config.level == CompilationLevel.PIECEWISE and (

4016 + if self.compilation_config.mode == CompilationMode.VLLM_COMPILE and (

4017 self.compilation_config.splitting_ops_contain_attention()

4018 or self.compilation_config.use_inductor_graph_partition

4019 ):

@@ -4068,7 +4069,7 @@ class GPUModelRunner(LoRAModelRunnerMixin, KVConnectorModelRunnerMixin):

4069 f"supported with {min_cg_builder_name} backend ("

4070 f"support:{min_cg_support}) "

4071 "; please try cudagraph_mode=PIECEWISE, "

4071 - "and make sure compilation level is piecewise"

4072 + "and make sure compilation mode is VLLM_COMPILE"

4073 )

4074

4075 # Trigger cudagraph dispatching keys initialization here (after