A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[Frontend] CompilationConfig overhaul (#20283): deprecate use_inductor in favor of backend, simplify custom_ops (#26113)
Signed-off-by: morrison-turnansky <mturnans@redhat.com> Signed-off-by: Morrison Turnansky <mturnans@redhat.com> Signed-off-by: Luka Govedič <ProExpertProg@users.noreply.github.com> Co-authored-by: Luka Govedič <ProExpertProg@users.noreply.github.com> Co-authored-by: Jiangyun Zhu <riverclouds.zhu@qq.com>
Morrison Turnansky committed
Oct 7, 2025 at 15:53 UTC
0c824fc46fe6f19723ba7a930581951d1c2b35f2
7 files changed
+126
-63
tests/compile/piecewise/test_toy_llama.py
+11
-20
@@ -258,13 +258,13 @@ def tractable_computation(
258
259
@torch.inference_mode
260
def run_model(
261
- llama_config, use_compile: bool, use_inductor: bool, split_attn: bool = False
261
+ llama_config, use_compile: bool, backend: str, split_attn: bool = False
262
) -> torch.Tensor:
263
if use_compile:
264
compilation_config = CompilationConfig(
265
level=CompilationLevel.PIECEWISE,
266
use_cudagraph=True,
267
- use_inductor=use_inductor,
267
+ backend=backend,
268
cudagraph_capture_sizes=[1, 2],
269
)
270
if split_attn:
@@ -338,8 +338,8 @@ def run_model(
338
return output.cpu()
339
340
341
-@pytest.mark.parametrize("use_inductor", [True, False])
342
-def test_toy_llama(use_inductor: bool):
341
+@pytest.mark.parametrize("backend", ["inductor", "eager"])
342
+def test_toy_llama(backend: str):
343
# compare output with and without piecewise compilation
344
345
llama_config = LlamaConfig(
@@ -358,10 +358,10 @@ def test_toy_llama(use_inductor: bool):
358
num_backend_compilations=0,
359
num_cudagraph_captured=0,
360
):
361
- outputs.append(run_model(llama_config, use_inductor=False, use_compile=False))
362
- run_model(tractable_config, use_inductor=False, use_compile=False)
361
+ outputs.append(run_model(llama_config, backend="eager", use_compile=False))
362
+ run_model(tractable_config, backend="eager", use_compile=False)
363
364
- if use_inductor:
364
+ if backend == "inductor":
365
kwargs = {"num_inductor_compiles": 1, "num_eager_compiles": 0}
366
else:
367
kwargs = {"num_eager_compiles": 1, "num_inductor_compiles": 0}
@@ -377,10 +377,8 @@ def test_toy_llama(use_inductor: bool):
377
num_cudagraph_captured=2,
378
**kwargs,
379
):
380
- outputs.append(
381
- run_model(llama_config, use_inductor=use_inductor, use_compile=True)
382
- )
383
- run_model(tractable_config, use_inductor=use_inductor, use_compile=True)
380
+ outputs.append(run_model(llama_config, backend=backend, use_compile=True))
381
+ run_model(tractable_config, backend=backend, use_compile=True)
382
383
with compilation_counter.expect(
384
num_graphs_seen=1, # one graph for the model
@@ -395,16 +393,9 @@ def test_toy_llama(use_inductor: bool):
393
), # num_cudagraph_sizes * num_piecewise_capturable_graphs_seen
394
):
395
outputs.append(
398
- run_model(
399
- llama_config,
400
- use_inductor=use_inductor,
401
- use_compile=True,
402
- split_attn=True,
403
- )
396
+ run_model(llama_config, backend=backend, use_compile=True, split_attn=True)
397
)
405
- run_model(
406
- tractable_config, use_inductor=use_inductor, use_compile=True, split_attn=True
407
- )
398
+ run_model(tractable_config, backend=backend, use_compile=True, split_attn=True)
399
400
for i in range(1, len(outputs)):
401
assert torch.allclose(outputs[0], outputs[i])
tests/model_executor/test_enabled_custom_ops.py
+21
-19
@@ -37,57 +37,59 @@ class Relu3(ReLUSquaredActivation):
37
38
39
@pytest.mark.parametrize(
40
- "env, torch_level, use_inductor, ops_enabled, default_on",
40
+ "env, torch_level, backend, ops_enabled, default_on",
41
[
42
# Default values based on compile level
43
# - All by default (no Inductor compilation)
44
- (None, 0, False, [True] * 4, True),
45
- (None, 1, True, [True] * 4, True),
46
- (None, 2, False, [True] * 4, True),
44
+ (None, 0, "eager", [True] * 4, True),
45
+ (None, 1, "eager", [True] * 4, True),
46
+ (None, 2, "eager", [True] * 4, True),
47
+ (None, 3, "eager", [True] * 4, True),
48
# - None by default (with Inductor)
48
- (None, 3, True, [False] * 4, False),
49
- (None, 4, True, [False] * 4, False),
50
- # - All by default (without Inductor)
51
- (None, 3, False, [True] * 4, True),
52
- (None, 4, False, [True] * 4, True),
49
+ (None, 0, "inductor", [True] * 4, True),
50
+ # - None by default (with Inductor)
51
+ (None, 1, "inductor", [False] * 4, False),
52
+ (None, 2, "inductor", [False] * 4, False),
53
+ (None, 3, "inductor", [False] * 4, False),
54
# Explicitly enabling/disabling
55
#
56
# Default: all
57
#
58
# All but SiluAndMul
58
- ("+rms_norm,-silu_and_mul", 0, True, [1, 0, 1, 1], True),
59
+ ("+rms_norm,-silu_and_mul", 0, "inductor", [1, 0, 1, 1], True),
60
# Only ReLU3
60
- ("none,-rms_norm,+relu3", 1, False, [0, 0, 0, 1], False),
61
+ ("none,-rms_norm,+relu3", 1, "eager", [0, 0, 0, 1], False),
62
# All but SiluAndMul
62
- ("all,-silu_and_mul", 2, True, [1, 0, 1, 1], True),
63
+ ("all,-silu_and_mul", 2, "inductor", [1, 0, 1, 1], True),
64
# All but ReLU3 (even if ReLU2 is on)
64
- ("-relu3,+relu2", 3, False, [1, 1, 1, 0], True),
65
+ ("-relu3,+relu2", 3, "eager", [1, 1, 1, 0], True),
66
# RMSNorm and SiluAndMul
66
- ("none,-relu3,+rms_norm,+silu_and_mul", 4, False, [1, 1, 0, 0], False),
67
+ ("none,-relu3,+rms_norm,+silu_and_mul", 3, "eager", [1, 1, 0, 0], False),
68
# All but RMSNorm
68
- ("-rms_norm", 3, False, [0, 1, 1, 1], True),
69
+ ("-rms_norm", 3, "eager", [0, 1, 1, 1], True),
70
#
71
# Default: none
72
#
73
# Only ReLU3
73
- ("-silu_and_mul,+relu3", 3, True, [0, 0, 0, 1], False),
74
+ ("none,+relu3", 3, "inductor", [0, 0, 0, 1], False),
75
# All but RMSNorm
75
- ("all,-rms_norm", 4, True, [0, 1, 1, 1], True),
76
+ ("all,-rms_norm", 3, "inductor", [0, 1, 1, 1], True),
77
],
78
)
79
def test_enabled_ops(
80
env: Optional[str],
81
torch_level: int,
81
- use_inductor: bool,
82
+ backend: str,
83
ops_enabled: list[int],
84
default_on: bool,
85
):
86
custom_ops = env.split(",") if env else []
87
vllm_config = VllmConfig(
88
compilation_config=CompilationConfig(
88
- use_inductor=bool(use_inductor), level=torch_level, custom_ops=custom_ops
89
+ backend=backend, level=torch_level, custom_ops=custom_ops
90
)
91
)
92
+ # breakpoint()
93
with set_current_vllm_config(vllm_config):
94
assert CustomOp.default_on() == default_on
95
vllm/compilation/backends.py
+5
-1
@@ -34,7 +34,7 @@ logger = init_logger(__name__)
34
35
36
def make_compiler(compilation_config: CompilationConfig) -> CompilerInterface:
37
- if compilation_config.use_inductor:
37
+ if compilation_config.backend == "inductor":
38
# Use standalone compile only if requested, version is new enough,
39
# and the symbol actually exists in this PyTorch build.
40
if (
@@ -48,6 +48,10 @@ def make_compiler(compilation_config: CompilationConfig) -> CompilerInterface:
48
logger.debug("Using InductorAdaptor")
49
return InductorAdaptor()
50
else:
51
+ assert compilation_config.backend == "eager", (
52
+ "Custom backends not supported with CompilationLevel.PIECEWISE"
53
+ )
54
+
55
logger.debug("Using EagerAdaptor")
56
return EagerAdaptor()
57
vllm/config/compilation.py
+60
-11
@@ -180,10 +180,11 @@ class CompilationConfig:
180
"""The directory to store the compiled graph, to accelerate Inductor
181
compilation. By default, it will use model-related information to generate
182
a cache directory."""
183
- backend: str = ""
183
+ backend: str = "inductor"
184
"""The backend for compilation. It needs to be a string:
185
186
- - "" (empty string): use the default backend.
186
+ - "" (empty string): use the default backend ("inductor" on CUDA-alike
187
+ platforms).
188
- "eager"/"openxla"/...: use the specified backend registered in PyTorch.
189
- "full.module.name": a qualified name which can be used to import the
190
@@ -192,7 +193,11 @@ class CompilationConfig:
193
distributed setting. When the compilation level is 1 or 2, the backend is
194
used for the compilation directly (it sees the whole graph). When the
195
compilation level is 3, the backend is used for the piecewise compilation
195
- (it sees a part of the graph)."""
196
+ (it sees a part of the graph). The backend can not be custom for compilation
197
+ level 3. Furthermore, compilation is only piecewise if splitting ops is set
198
+ accordingly and use_inductor_cudagraphs_partition is off. Note that the
199
+ default options for splitting ops are sufficient for piecewise compilation.
200
+ """
201
custom_ops: list[str] = field(default_factory=list)
202
"""Fine-grained control over which custom ops to enable/disable. Use 'all'
203
to enable all, 'none' to disable all. Also specify a list of custom op
@@ -210,8 +215,12 @@ class CompilationConfig:
215
compilation."""
216
217
# Inductor capture
213
- use_inductor: bool = True
214
- """Whether to use inductor compilation:
218
+ use_inductor: Optional[bool] = None
219
+ """
220
+ Whether to use inductor compilation.
221
+
222
+ This flag is deprecated and will be removed.
223
+ Please use the 'backend' option instead.
224
225
- False: inductor compilation is not used. graph runs in eager
226
(custom_ops enabled by default).
@@ -219,7 +228,11 @@ class CompilationConfig:
228
One graph for symbolic shape and one graph per size in compile_sizes
229
are compiled using configurations in inductor_compile_config.
230
222
- This setting is ignored if level<PIECEWISE."""
231
+ This setting is ignored if level<PIECEWISE.
232
+
233
+ For future compatibility:
234
+ If use_inductor is True, backend="inductor" otherwise backend="eager".
235
+ """
236
compile_sizes: Optional[list[Union[int, str]]] = None
237
"""Sizes to compile for inductor. In addition
238
to integers, it also supports "cudagraph_capture_sizes" to
@@ -523,7 +536,43 @@ class CompilationConfig:
536
"(where 'op' is the registered op name)"
537
)
538
539
+ # Currently only eager and inductor backend are supported.
540
+ # for piecewise compilation. Custom backends are not suppported for
541
+ # piecewise compilation. Update when more backends are supported.
542
+ if self.level == CompilationLevel.PIECEWISE and self.backend not in [
543
+ "",
544
+ "eager",
545
+ "inductor",
546
+ ]:
547
+ raise ValueError(
548
+ f"Invalid backend for piecewise compilation: {self.backend}"
549
+ )
550
+
551
+ if self.use_inductor is not None:
552
+ logger.warning_once(
553
+ "The 'use_inductor' flag is deprecated and will be\
554
+ removed in a future release."
555
+ "Please use the 'backend' option instead.",
556
+ )
557
+ self.backend = "inductor" if self.use_inductor else "eager"
558
+
559
+ if self.backend == "":
560
+ self.backend = "inductor"
561
+
562
def init_backend(self, vllm_config: "VllmConfig") -> Union[str, Callable]:
563
+ """
564
+ Initialize the backend for the compilation config from a vllm config.
565
+ Arguments:
566
+ vllm_config: The vllm config to initialize the backend from.
567
+ Returns:
568
+ The backend for the compilation config.
569
+ """
570
+ if self.level is None:
571
+ raise ValueError(
572
+ "No compilation level is set. This method should only be \
573
+ called via vllm config where the level is set if none is \
574
+ provided."
575
+ )
576
if self.level == CompilationLevel.NO_COMPILATION:
577
raise ValueError("No compilation level is set.")
578
@@ -531,15 +580,15 @@ class CompilationConfig:
580
581
torch_backends = list_backends(exclude_tags=tuple())
582
if self.level in [CompilationLevel.DYNAMO_AS_IS, CompilationLevel.DYNAMO_ONCE]:
534
- if self.backend == "":
535
- return "eager"
583
if self.backend in torch_backends:
584
return self.backend
585
return resolve_obj_by_qualname(self.backend)
586
540
- # TODO: pass user-specified backend to piecewise compilation
541
- # merge with the config use_inductor
587
assert self.level == CompilationLevel.PIECEWISE
588
+ if self.backend not in ["eager", "inductor"]:
589
+ raise ValueError(
590
+ f"Invalid backend for piecewise compilation: {self.backend}"
591
+ )
592
593
from vllm.compilation.backends import VllmBackend
594
@@ -692,7 +741,7 @@ class CompilationConfig:
741
)
742
743
inductor_used = (
695
- self.level == CompilationLevel.PIECEWISE and self.use_inductor
744
+ self.level == CompilationLevel.PIECEWISE and self.backend == "inductor"
745
) or (
746
self.level >= CompilationLevel.DYNAMO_AS_IS and self.backend == "inductor"
747
)
vllm/config/vllm.py
+19
@@ -318,6 +318,25 @@ class VllmConfig:
318
# NB: Passing both --enforce-eager and a compilation level
319
# in V0 means the compilation level wins out.
320
self.compilation_config.level = CompilationLevel.NO_COMPILATION
321
+ else:
322
+ assert self.compilation_config.level >= CompilationLevel.NO_COMPILATION
323
+ assert self.compilation_config.level <= CompilationLevel.PIECEWISE
324
+ assert self.compilation_config.level <= 3
325
+
326
+ # If user does not set custom ops via none or all set it here based on
327
+ # compilation level and backend.
328
+ if (
329
+ self.compilation_config.custom_ops.count("none")
330
+ + self.compilation_config.custom_ops.count("all")
331
+ == 0
332
+ ):
333
+ if (
334
+ self.compilation_config.level > 0
335
+ and self.compilation_config.backend != "eager"
336
+ ):
337
+ self.compilation_config.custom_ops.append("none")
338
+ else:
339
+ self.compilation_config.custom_ops.append("all")
340
341
# async tp is built on top of sequence parallelism
342
# and requires it to be enabled.
vllm/model_executor/custom_op.py
+10
-10
@@ -114,7 +114,9 @@ class CustomOp(nn.Module):
114
custom_ops = compilation_config.custom_ops
115
if not hasattr(cls, "name"):
116
logger.warning_once(
117
- "Custom op %s was not registered, which means it won't appear in the op registry. It will be enabled/disabled based on the global settings.", # noqa: E501
117
+ "Custom op %s was not registered, which means it won't appear\
118
+ in the op registry. It will be enabled/disabled based on the\
119
+ global settings.", # noqa: E501
120
cls.__name__,
121
)
122
return CustomOp.default_on()
@@ -128,19 +130,17 @@ class CustomOp(nn.Module):
130
@staticmethod
131
def default_on() -> bool:
132
"""
131
- On by default if PyTorch Inductor is not used.
132
- Specifying 'all' or 'none' in custom_op takes precedence.
133
+ Behavior controlled by `CompilationConfig.custom_ops`: On by default if
134
+ 'all', off by default if 'none'.
135
+ When PyTorch Inductor is used, 'none' is the default value,
136
+ otherwise 'all'.
137
"""
134
- from vllm.config import CompilationLevel
135
-
138
compilation_config = get_cached_compilation_config()
137
- default_on = (
138
- compilation_config.level < CompilationLevel.PIECEWISE
139
- or not compilation_config.use_inductor
140
- )
139
count_none = compilation_config.custom_ops.count("none")
140
count_all = compilation_config.custom_ops.count("all")
143
- return default_on and not count_none > 0 or count_all > 0
141
+ assert count_none + count_all == 1
142
+
143
+ return not count_none > 0 or count_all > 0
144
145
# Dictionary of all custom ops (classes, indexed by registered name).
146
# To check if an op with a name is enabled, call .enabled() on the class.
vllm/platforms/cpu.py
-2
@@ -274,8 +274,6 @@ class CpuPlatform(Platform):
274
"epilogue_fusion": True,
275
}
276
)
277
- if compilation_config.use_inductor:
278
- compilation_config.custom_ops = ["none"]
277
278
if vllm_config.lora_config is not None:
279
compilation_config.level = CompilationLevel.NO_COMPILATION