A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
Revert #26113 "[Frontend] CompilationConfig overhaul (#20283): deprecate use_inductor in favor of backend, simplify custom_ops" (#26472)
Signed-off-by: zjy0516 <riverclouds.zhu@qq.com>
Jiangyun Zhu committed
Oct 9, 2025 at 20:43 UTC
5728da11ea0ee02868db017e8d197acfa2f7edda
7 files changed
+63
-126
tests/compile/piecewise/test_toy_llama.py
+20
-11
@@ -258,13 +258,13 @@ def tractable_computation(
258
259
@torch.inference_mode
260
def run_model(
261
- llama_config, use_compile: bool, backend: str, split_attn: bool = False
261
+ llama_config, use_compile: bool, use_inductor: bool, split_attn: bool = False
262
) -> torch.Tensor:
263
if use_compile:
264
compilation_config = CompilationConfig(
265
level=CompilationLevel.PIECEWISE,
266
use_cudagraph=True,
267
- backend=backend,
267
+ use_inductor=use_inductor,
268
cudagraph_capture_sizes=[1, 2],
269
)
270
if split_attn:
@@ -338,8 +338,8 @@ def run_model(
338
return output.cpu()
339
340
341
-@pytest.mark.parametrize("backend", ["inductor", "eager"])
342
-def test_toy_llama(backend: str):
341
+@pytest.mark.parametrize("use_inductor", [True, False])
342
+def test_toy_llama(use_inductor: bool):
343
# compare output with and without piecewise compilation
344
345
llama_config = LlamaConfig(
@@ -358,10 +358,10 @@ def test_toy_llama(backend: str):
358
num_backend_compilations=0,
359
num_cudagraph_captured=0,
360
):
361
- outputs.append(run_model(llama_config, backend="eager", use_compile=False))
362
- run_model(tractable_config, backend="eager", use_compile=False)
361
+ outputs.append(run_model(llama_config, use_inductor=False, use_compile=False))
362
+ run_model(tractable_config, use_inductor=False, use_compile=False)
363
364
- if backend == "inductor":
364
+ if use_inductor:
365
kwargs = {"num_inductor_compiles": 1, "num_eager_compiles": 0}
366
else:
367
kwargs = {"num_eager_compiles": 1, "num_inductor_compiles": 0}
@@ -377,8 +377,10 @@ def test_toy_llama(backend: str):
377
num_cudagraph_captured=2,
378
**kwargs,
379
):
380
- outputs.append(run_model(llama_config, backend=backend, use_compile=True))
381
- run_model(tractable_config, backend=backend, use_compile=True)
380
+ outputs.append(
381
+ run_model(llama_config, use_inductor=use_inductor, use_compile=True)
382
+ )
383
+ run_model(tractable_config, use_inductor=use_inductor, use_compile=True)
384
385
with compilation_counter.expect(
386
num_graphs_seen=1, # one graph for the model
@@ -393,9 +395,16 @@ def test_toy_llama(backend: str):
395
), # num_cudagraph_sizes * num_piecewise_capturable_graphs_seen
396
):
397
outputs.append(
396
- run_model(llama_config, backend=backend, use_compile=True, split_attn=True)
398
+ run_model(
399
+ llama_config,
400
+ use_inductor=use_inductor,
401
+ use_compile=True,
402
+ split_attn=True,
403
+ )
404
)
398
- run_model(tractable_config, backend=backend, use_compile=True, split_attn=True)
405
+ run_model(
406
+ tractable_config, use_inductor=use_inductor, use_compile=True, split_attn=True
407
+ )
408
409
for i in range(1, len(outputs)):
410
assert torch.allclose(outputs[0], outputs[i])
tests/model_executor/test_enabled_custom_ops.py
+19
-21
@@ -37,59 +37,57 @@ class Relu3(ReLUSquaredActivation):
37
38
39
@pytest.mark.parametrize(
40
- "env, torch_level, backend, ops_enabled, default_on",
40
+ "env, torch_level, use_inductor, ops_enabled, default_on",
41
[
42
# Default values based on compile level
43
# - All by default (no Inductor compilation)
44
- (None, 0, "eager", [True] * 4, True),
45
- (None, 1, "eager", [True] * 4, True),
46
- (None, 2, "eager", [True] * 4, True),
47
- (None, 3, "eager", [True] * 4, True),
44
+ (None, 0, False, [True] * 4, True),
45
+ (None, 1, True, [True] * 4, True),
46
+ (None, 2, False, [True] * 4, True),
47
# - None by default (with Inductor)
49
- (None, 0, "inductor", [True] * 4, True),
50
- # - None by default (with Inductor)
51
- (None, 1, "inductor", [False] * 4, False),
52
- (None, 2, "inductor", [False] * 4, False),
53
- (None, 3, "inductor", [False] * 4, False),
48
+ (None, 3, True, [False] * 4, False),
49
+ (None, 4, True, [False] * 4, False),
50
+ # - All by default (without Inductor)
51
+ (None, 3, False, [True] * 4, True),
52
+ (None, 4, False, [True] * 4, True),
53
# Explicitly enabling/disabling
54
#
55
# Default: all
56
#
57
# All but SiluAndMul
59
- ("+rms_norm,-silu_and_mul", 0, "inductor", [1, 0, 1, 1], True),
58
+ ("+rms_norm,-silu_and_mul", 0, True, [1, 0, 1, 1], True),
59
# Only ReLU3
61
- ("none,-rms_norm,+relu3", 1, "eager", [0, 0, 0, 1], False),
60
+ ("none,-rms_norm,+relu3", 1, False, [0, 0, 0, 1], False),
61
# All but SiluAndMul
63
- ("all,-silu_and_mul", 2, "inductor", [1, 0, 1, 1], True),
62
+ ("all,-silu_and_mul", 2, True, [1, 0, 1, 1], True),
63
# All but ReLU3 (even if ReLU2 is on)
65
- ("-relu3,+relu2", 3, "eager", [1, 1, 1, 0], True),
64
+ ("-relu3,+relu2", 3, False, [1, 1, 1, 0], True),
65
# RMSNorm and SiluAndMul
67
- ("none,-relu3,+rms_norm,+silu_and_mul", 3, "eager", [1, 1, 0, 0], False),
66
+ ("none,-relu3,+rms_norm,+silu_and_mul", 4, False, [1, 1, 0, 0], False),
67
# All but RMSNorm
69
- ("-rms_norm", 3, "eager", [0, 1, 1, 1], True),
68
+ ("-rms_norm", 3, False, [0, 1, 1, 1], True),
69
#
70
# Default: none
71
#
72
# Only ReLU3
74
- ("none,+relu3", 3, "inductor", [0, 0, 0, 1], False),
73
+ ("-silu_and_mul,+relu3", 3, True, [0, 0, 0, 1], False),
74
# All but RMSNorm
76
- ("all,-rms_norm", 3, "inductor", [0, 1, 1, 1], True),
75
+ ("all,-rms_norm", 4, True, [0, 1, 1, 1], True),
76
],
77
)
78
def test_enabled_ops(
79
env: Optional[str],
80
torch_level: int,
82
- backend: str,
81
+ use_inductor: bool,
82
ops_enabled: list[int],
83
default_on: bool,
84
):
85
custom_ops = env.split(",") if env else []
86
vllm_config = VllmConfig(
87
compilation_config=CompilationConfig(
89
- backend=backend, level=torch_level, custom_ops=custom_ops
88
+ use_inductor=bool(use_inductor), level=torch_level, custom_ops=custom_ops
89
)
90
)
92
- # breakpoint()
91
with set_current_vllm_config(vllm_config):
92
assert CustomOp.default_on() == default_on
93
vllm/compilation/backends.py
+1
-5
@@ -34,7 +34,7 @@ logger = init_logger(__name__)
34
35
36
def make_compiler(compilation_config: CompilationConfig) -> CompilerInterface:
37
- if compilation_config.backend == "inductor":
37
+ if compilation_config.use_inductor:
38
# Use standalone compile only if requested, version is new enough,
39
# and the symbol actually exists in this PyTorch build.
40
if (
@@ -48,10 +48,6 @@ def make_compiler(compilation_config: CompilationConfig) -> CompilerInterface:
48
logger.debug("Using InductorAdaptor")
49
return InductorAdaptor()
50
else:
51
- assert compilation_config.backend == "eager", (
52
- "Custom backends not supported with CompilationLevel.PIECEWISE"
53
- )
54
-
51
logger.debug("Using EagerAdaptor")
52
return EagerAdaptor()
53
vllm/config/compilation.py
+11
-60
@@ -180,11 +180,10 @@ class CompilationConfig:
180
"""The directory to store the compiled graph, to accelerate Inductor
181
compilation. By default, it will use model-related information to generate
182
a cache directory."""
183
- backend: str = "inductor"
183
+ backend: str = ""
184
"""The backend for compilation. It needs to be a string:
185
186
- - "" (empty string): use the default backend ("inductor" on CUDA-alike
187
- platforms).
186
+ - "" (empty string): use the default backend.
187
- "eager"/"openxla"/...: use the specified backend registered in PyTorch.
188
- "full.module.name": a qualified name which can be used to import the
189
@@ -193,11 +192,7 @@ class CompilationConfig:
192
distributed setting. When the compilation level is 1 or 2, the backend is
193
used for the compilation directly (it sees the whole graph). When the
194
compilation level is 3, the backend is used for the piecewise compilation
196
- (it sees a part of the graph). The backend can not be custom for compilation
197
- level 3. Furthermore, compilation is only piecewise if splitting ops is set
198
- accordingly and use_inductor_cudagraphs_partition is off. Note that the
199
- default options for splitting ops are sufficient for piecewise compilation.
200
- """
195
+ (it sees a part of the graph)."""
196
custom_ops: list[str] = field(default_factory=list)
197
"""Fine-grained control over which custom ops to enable/disable. Use 'all'
198
to enable all, 'none' to disable all. Also specify a list of custom op
@@ -215,12 +210,8 @@ class CompilationConfig:
210
compilation."""
211
212
# Inductor capture
218
- use_inductor: Optional[bool] = None
219
- """
220
- Whether to use inductor compilation.
221
-
222
- This flag is deprecated and will be removed.
223
- Please use the 'backend' option instead.
213
+ use_inductor: bool = True
214
+ """Whether to use inductor compilation:
215
216
- False: inductor compilation is not used. graph runs in eager
217
(custom_ops enabled by default).
@@ -228,11 +219,7 @@ class CompilationConfig:
219
One graph for symbolic shape and one graph per size in compile_sizes
220
are compiled using configurations in inductor_compile_config.
221
231
- This setting is ignored if level<PIECEWISE.
232
-
233
- For future compatibility:
234
- If use_inductor is True, backend="inductor" otherwise backend="eager".
235
- """
222
+ This setting is ignored if level<PIECEWISE."""
223
compile_sizes: Optional[list[Union[int, str]]] = None
224
"""Sizes to compile for inductor. In addition
225
to integers, it also supports "cudagraph_capture_sizes" to
@@ -538,43 +525,7 @@ class CompilationConfig:
525
"(where 'op' is the registered op name)"
526
)
527
541
- # Currently only eager and inductor backend are supported.
542
- # for piecewise compilation. Custom backends are not suppported for
543
- # piecewise compilation. Update when more backends are supported.
544
- if self.level == CompilationLevel.PIECEWISE and self.backend not in [
545
- "",
546
- "eager",
547
- "inductor",
548
- ]:
549
- raise ValueError(
550
- f"Invalid backend for piecewise compilation: {self.backend}"
551
- )
552
-
553
- if self.use_inductor is not None:
554
- logger.warning_once(
555
- "The 'use_inductor' flag is deprecated and will be\
556
- removed in a future release."
557
- "Please use the 'backend' option instead.",
558
- )
559
- self.backend = "inductor" if self.use_inductor else "eager"
560
-
561
- if self.backend == "":
562
- self.backend = "inductor"
563
-
528
def init_backend(self, vllm_config: "VllmConfig") -> Union[str, Callable]:
565
- """
566
- Initialize the backend for the compilation config from a vllm config.
567
- Arguments:
568
- vllm_config: The vllm config to initialize the backend from.
569
- Returns:
570
- The backend for the compilation config.
571
- """
572
- if self.level is None:
573
- raise ValueError(
574
- "No compilation level is set. This method should only be \
575
- called via vllm config where the level is set if none is \
576
- provided."
577
- )
529
if self.level == CompilationLevel.NO_COMPILATION:
530
raise ValueError("No compilation level is set.")
531
@@ -582,15 +533,15 @@ class CompilationConfig:
533
534
torch_backends = list_backends(exclude_tags=tuple())
535
if self.level in [CompilationLevel.DYNAMO_AS_IS, CompilationLevel.DYNAMO_ONCE]:
536
+ if self.backend == "":
537
+ return "eager"
538
if self.backend in torch_backends:
539
return self.backend
540
return resolve_obj_by_qualname(self.backend)
541
542
+ # TODO: pass user-specified backend to piecewise compilation
543
+ # merge with the config use_inductor
544
assert self.level == CompilationLevel.PIECEWISE
590
- if self.backend not in ["eager", "inductor"]:
591
- raise ValueError(
592
- f"Invalid backend for piecewise compilation: {self.backend}"
593
- )
545
546
from vllm.compilation.backends import VllmBackend
547
@@ -743,7 +694,7 @@ class CompilationConfig:
694
)
695
696
inductor_used = (
746
- self.level == CompilationLevel.PIECEWISE and self.backend == "inductor"
697
+ self.level == CompilationLevel.PIECEWISE and self.use_inductor
698
) or (
699
self.level >= CompilationLevel.DYNAMO_AS_IS and self.backend == "inductor"
700
)
vllm/config/vllm.py
-19
@@ -318,25 +318,6 @@ class VllmConfig:
318
# NB: Passing both --enforce-eager and a compilation level
319
# in V0 means the compilation level wins out.
320
self.compilation_config.level = CompilationLevel.NO_COMPILATION
321
- else:
322
- assert self.compilation_config.level >= CompilationLevel.NO_COMPILATION
323
- assert self.compilation_config.level <= CompilationLevel.PIECEWISE
324
- assert self.compilation_config.level <= 3
325
-
326
- # If user does not set custom ops via none or all set it here based on
327
- # compilation level and backend.
328
- if (
329
- self.compilation_config.custom_ops.count("none")
330
- + self.compilation_config.custom_ops.count("all")
331
- == 0
332
- ):
333
- if (
334
- self.compilation_config.level > 0
335
- and self.compilation_config.backend != "eager"
336
- ):
337
- self.compilation_config.custom_ops.append("none")
338
- else:
339
- self.compilation_config.custom_ops.append("all")
321
322
# async tp is built on top of sequence parallelism
323
# and requires it to be enabled.
vllm/model_executor/custom_op.py
+10
-10
@@ -114,9 +114,7 @@ class CustomOp(nn.Module):
114
custom_ops = compilation_config.custom_ops
115
if not hasattr(cls, "name"):
116
logger.warning_once(
117
- "Custom op %s was not registered, which means it won't appear\
118
- in the op registry. It will be enabled/disabled based on the\
119
- global settings.", # noqa: E501
117
+ "Custom op %s was not registered, which means it won't appear in the op registry. It will be enabled/disabled based on the global settings.", # noqa: E501
118
cls.__name__,
119
)
120
return CustomOp.default_on()
@@ -130,17 +128,19 @@ class CustomOp(nn.Module):
128
@staticmethod
129
def default_on() -> bool:
130
"""
133
- Behavior controlled by `CompilationConfig.custom_ops`: On by default if
134
- 'all', off by default if 'none'.
135
- When PyTorch Inductor is used, 'none' is the default value,
136
- otherwise 'all'.
131
+ On by default if PyTorch Inductor is not used.
132
+ Specifying 'all' or 'none' in custom_op takes precedence.
133
"""
134
+ from vllm.config import CompilationLevel
135
+
136
compilation_config = get_cached_compilation_config()
137
+ default_on = (
138
+ compilation_config.level < CompilationLevel.PIECEWISE
139
+ or not compilation_config.use_inductor
140
+ )
141
count_none = compilation_config.custom_ops.count("none")
142
count_all = compilation_config.custom_ops.count("all")
141
- assert count_none + count_all == 1
142
-
143
- return not count_none > 0 or count_all > 0
143
+ return default_on and not count_none > 0 or count_all > 0
144
145
# Dictionary of all custom ops (classes, indexed by registered name).
146
# To check if an op with a name is enabled, call .enabled() on the class.
vllm/platforms/cpu.py
+2
@@ -274,6 +274,8 @@ class CpuPlatform(Platform):
274
"epilogue_fusion": True,
275
}
276
)
277
+ if compilation_config.use_inductor:
278
+ compilation_config.custom_ops = ["none"]
279
280
if vllm_config.lora_config is not None:
281
compilation_config.level = CompilationLevel.NO_COMPILATION