@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / e3fdb627d9
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 14, 05:30 UTC

[FrontEnd] UNREVERT CompilationConfig overhaul (#20283): deprecate use_inductor in favor of backend, simplify custom_ops (#26502)

Signed-off-by: morrison-turnansky <mturnans@redhat.com> Signed-off-by: Morrison Turnansky <mturnans@redhat.com> Signed-off-by: Luka Govedič <ProExpertProg@users.noreply.github.com> Co-authored-by: Luka Govedič <ProExpertProg@users.noreply.github.com> Co-authored-by: Jiangyun Zhu <riverclouds.zhu@qq.com>

Morrison Turnansky committed Oct 13, 2025 at 18:47 UTC e3fdb627d98d0c4e8313988a9a81eac0935dbcab
8 files changed +153 -86
tests/compile/piecewise/test_toy_llama.py
+11 -20
@@ -258,13 +258,13 @@ def tractable_computation(
258
259 @torch.inference_mode
260 def run_model(
261 - llama_config, use_compile: bool, use_inductor: bool, split_attn: bool = False
261 + llama_config, use_compile: bool, backend: str, split_attn: bool = False
262 ) -> torch.Tensor:
263 if use_compile:
264 compilation_config = CompilationConfig(
265 level=CompilationLevel.PIECEWISE,
266 use_cudagraph=True,
267 - use_inductor=use_inductor,
267 + backend=backend,
268 cudagraph_capture_sizes=[1, 2],
269 )
270 if split_attn:
@@ -338,8 +338,8 @@ def run_model(
338 return output.cpu()
339
340
341 -@pytest.mark.parametrize("use_inductor", [True, False])
342 -def test_toy_llama(use_inductor: bool):
341 +@pytest.mark.parametrize("backend", ["inductor", "eager"])
342 +def test_toy_llama(backend: str):
343 # compare output with and without piecewise compilation
344
345 llama_config = LlamaConfig(
@@ -358,10 +358,10 @@ def test_toy_llama(use_inductor: bool):
358 num_backend_compilations=0,
359 num_cudagraph_captured=0,
360 ):
361 - outputs.append(run_model(llama_config, use_inductor=False, use_compile=False))
362 - run_model(tractable_config, use_inductor=False, use_compile=False)
361 + outputs.append(run_model(llama_config, backend="eager", use_compile=False))
362 + run_model(tractable_config, backend="eager", use_compile=False)
363
364 - if use_inductor:
364 + if backend == "inductor":
365 kwargs = {"num_inductor_compiles": 1, "num_eager_compiles": 0}
366 else:
367 kwargs = {"num_eager_compiles": 1, "num_inductor_compiles": 0}
@@ -377,10 +377,8 @@ def test_toy_llama(use_inductor: bool):
377 num_cudagraph_captured=2,
378 **kwargs,
379 ):
380 - outputs.append(
381 - run_model(llama_config, use_inductor=use_inductor, use_compile=True)
382 - )
383 - run_model(tractable_config, use_inductor=use_inductor, use_compile=True)
380 + outputs.append(run_model(llama_config, backend=backend, use_compile=True))
381 + run_model(tractable_config, backend=backend, use_compile=True)
382
383 with compilation_counter.expect(
384 num_graphs_seen=1, # one graph for the model
@@ -395,16 +393,9 @@ def test_toy_llama(use_inductor: bool):
393 ), # num_cudagraph_sizes * num_piecewise_capturable_graphs_seen
394 ):
395 outputs.append(
398 - run_model(
399 - llama_config,
400 - use_inductor=use_inductor,
401 - use_compile=True,
402 - split_attn=True,
403 - )
396 + run_model(llama_config, backend=backend, use_compile=True, split_attn=True)
397 )
405 - run_model(
406 - tractable_config, use_inductor=use_inductor, use_compile=True, split_attn=True
407 - )
398 + run_model(tractable_config, backend=backend, use_compile=True, split_attn=True)
399
400 for i in range(1, len(outputs)):
401 assert torch.allclose(outputs[0], outputs[i])
tests/compile/test_basic_correctness.py
+30 -24
@@ -77,14 +77,15 @@ class TestSetting:
77 method="encode",
78 ),
79 # vision language model
80 - TestSetting(
81 - model="microsoft/Phi-3.5-vision-instruct",
82 - model_args=["--trust-remote-code", "--max-model-len", "2048"],
83 - pp_size=2,
84 - tp_size=1,
85 - attn_backend="FLASH_ATTN",
86 - method="generate_with_image",
87 - ),
80 + # See https://github.com/vllm-project/vllm/issues/26716.
81 + # TestSetting(
82 + # model="microsoft/Phi-3.5-vision-instruct",
83 + # model_args=["--trust-remote-code", "--max-model-len", "2048"],
84 + # pp_size=2,
85 + # tp_size=1,
86 + # attn_backend="FLASH_ATTN",
87 + # method="generate_with_image",
88 + # ),
89 ],
90 )
91 def test_compile_correctness(
@@ -109,41 +110,46 @@ def test_compile_correctness(
110 with monkeypatch.context() as m:
111 m.setenv("VLLM_ATTENTION_BACKEND", attn_backend)
112 final_args = [
112 - "--enforce-eager",
113 *model_args,
114 "-pp",
115 str(pp_size),
116 "-tp",
117 str(tp_size),
118 + "-O.cudagraph_mode=none",
119 ]
120
121 all_args: list[list[str]] = []
122 all_envs: list[dict[str, str] | None] = []
123
123 - for level in [
124 - CompilationLevel.NO_COMPILATION,
124 + for comp_level in [
125 + CompilationLevel.DYNAMO_AS_IS,
126 + CompilationLevel.DYNAMO_ONCE,
127 CompilationLevel.PIECEWISE,
128 ]:
127 - all_args.append(final_args + [f"-O{level}"])
128 - all_envs.append({})
129 + for level in [CompilationLevel.NO_COMPILATION, comp_level]:
130 + all_args.append(
131 + final_args + [f"-O.level={level}", "-O.backend=inductor"]
132 + )
133
130 - # inductor will change the output, so we only compare if the output
131 - # is close, not exactly the same.
132 - compare_all_settings(
133 - model,
134 - all_args,
135 - all_envs,
136 - method=method if method != "generate" else "generate_close",
137 - )
138 - all_envs.clear()
139 - all_args.clear()
134 + # inductor will change the output, so we only compare if the output
135 + # is close, not exactly the same.
136 + compare_all_settings(
137 + model,
138 + all_args,
139 + all_envs,
140 + method=method if method != "generate" else "generate_close",
141 + )
142 + all_envs.clear()
143 + all_args.clear()
144
145 for level in [
146 CompilationLevel.NO_COMPILATION,
147 CompilationLevel.DYNAMO_AS_IS,
148 CompilationLevel.DYNAMO_ONCE,
149 + CompilationLevel.PIECEWISE,
150 ]:
146 - all_args.append(final_args + [f"-O{level}"])
151 + all_args.append(final_args + [f"-O.level={level}", "-O.backend=eager"])
152 + all_envs.append({})
153 all_envs.append({})
154
155 compare_all_settings(model, all_args * 3, all_envs, method=method)
tests/model_executor/test_enabled_custom_ops.py
+20 -19
@@ -36,55 +36,56 @@ class Relu3(ReLUSquaredActivation):
36
37
38 @pytest.mark.parametrize(
39 - "env, torch_level, use_inductor, ops_enabled, default_on",
39 + "env, torch_level, backend, ops_enabled, default_on",
40 [
41 # Default values based on compile level
42 # - All by default (no Inductor compilation)
43 - (None, 0, False, [True] * 4, True),
44 - (None, 1, True, [True] * 4, True),
45 - (None, 2, False, [True] * 4, True),
43 + (None, 0, "eager", [True] * 4, True),
44 + (None, 1, "eager", [True] * 4, True),
45 + (None, 2, "eager", [True] * 4, True),
46 + (None, 3, "eager", [True] * 4, True),
47 # - None by default (with Inductor)
47 - (None, 3, True, [False] * 4, False),
48 - (None, 4, True, [False] * 4, False),
49 - # - All by default (without Inductor)
50 - (None, 3, False, [True] * 4, True),
51 - (None, 4, False, [True] * 4, True),
48 + (None, 0, "inductor", [True] * 4, True),
49 + # - None by default (with Inductor)
50 + (None, 1, "inductor", [False] * 4, False),
51 + (None, 2, "inductor", [False] * 4, False),
52 + (None, 3, "inductor", [False] * 4, False),
53 # Explicitly enabling/disabling
54 #
55 # Default: all
56 #
57 # All but SiluAndMul
57 - ("+rms_norm,-silu_and_mul", 0, True, [1, 0, 1, 1], True),
58 + ("+rms_norm,-silu_and_mul", 0, "inductor", [1, 0, 1, 1], True),
59 # Only ReLU3
59 - ("none,-rms_norm,+relu3", 1, False, [0, 0, 0, 1], False),
60 + ("none,-rms_norm,+relu3", 1, "eager", [0, 0, 0, 1], False),
61 # All but SiluAndMul
61 - ("all,-silu_and_mul", 2, True, [1, 0, 1, 1], True),
62 + ("all,-silu_and_mul", 2, "inductor", [1, 0, 1, 1], True),
63 # All but ReLU3 (even if ReLU2 is on)
63 - ("-relu3,+relu2", 3, False, [1, 1, 1, 0], True),
64 + ("-relu3,+relu2", 3, "eager", [1, 1, 1, 0], True),
65 # RMSNorm and SiluAndMul
65 - ("none,-relu3,+rms_norm,+silu_and_mul", 4, False, [1, 1, 0, 0], False),
66 + ("none,-relu3,+rms_norm,+silu_and_mul", 3, "eager", [1, 1, 0, 0], False),
67 # All but RMSNorm
67 - ("-rms_norm", 3, False, [0, 1, 1, 1], True),
68 + ("-rms_norm", 3, "eager", [0, 1, 1, 1], True),
69 #
70 # Default: none
71 #
72 # Only ReLU3
72 - ("-silu_and_mul,+relu3", 3, True, [0, 0, 0, 1], False),
73 + ("none,+relu3", 3, "inductor", [0, 0, 0, 1], False),
74 # All but RMSNorm
74 - ("all,-rms_norm", 4, True, [0, 1, 1, 1], True),
75 + ("all,-rms_norm", 3, "inductor", [0, 1, 1, 1], True),
76 ],
77 )
78 def test_enabled_ops(
79 env: str | None,
80 torch_level: int,
80 - use_inductor: bool,
81 + backend: str,
82 ops_enabled: list[int],
83 default_on: bool,
84 ):
85 custom_ops = env.split(",") if env else []
86 vllm_config = VllmConfig(
87 compilation_config=CompilationConfig(
87 - use_inductor=bool(use_inductor), level=torch_level, custom_ops=custom_ops
88 + backend=backend, level=torch_level, custom_ops=custom_ops
89 )
90 )
91 with set_current_vllm_config(vllm_config):
vllm/compilation/backends.py
+5 -1
@@ -41,7 +41,7 @@ logger = init_logger(__name__)
41
42
43 def make_compiler(compilation_config: CompilationConfig) -> CompilerInterface:
44 - if compilation_config.use_inductor:
44 + if compilation_config.backend == "inductor":
45 # Use standalone compile only if requested, version is new enough,
46 # and the symbol actually exists in this PyTorch build.
47 if (
@@ -55,6 +55,10 @@ def make_compiler(compilation_config: CompilationConfig) -> CompilerInterface:
55 logger.debug("Using InductorAdaptor")
56 return InductorAdaptor()
57 else:
58 + assert compilation_config.backend == "eager", (
59 + "Custom backends not supported with CompilationLevel.PIECEWISE"
60 + )
61 +
62 logger.debug("Using EagerAdaptor")
63 return EagerAdaptor()
64
vllm/config/compilation.py
+63 -10
@@ -15,6 +15,7 @@ from pydantic.dataclasses import dataclass
15 from vllm.compilation.inductor_pass import CallableInductorPass, InductorPass
16 from vllm.config.utils import config
17 from vllm.logger import init_logger
18 +from vllm.platforms import current_platform
19 from vllm.utils import is_torch_equal_or_newer, resolve_obj_by_qualname
20
21 if TYPE_CHECKING:
@@ -187,7 +188,8 @@ class CompilationConfig:
188 backend: str = ""
189 """The backend for compilation. It needs to be a string:
190
190 - - "" (empty string): use the default backend.
191 + - "" (empty string): use the default backend ("inductor" on CUDA-alike
192 + platforms).
193 - "eager"/"openxla"/...: use the specified backend registered in PyTorch.
194 - "full.module.name": a qualified name which can be used to import the
195
@@ -196,7 +198,12 @@ class CompilationConfig:
198 distributed setting. When the compilation level is 1 or 2, the backend is
199 used for the compilation directly (it sees the whole graph). When the
200 compilation level is 3, the backend is used for the piecewise compilation
199 - (it sees a part of the graph)."""
201 + (it sees a part of the graph). The backend can not be custom for compilation
202 + level 3, i.e. the backend must be either eager or inductor. Furthermore,
203 + compilation is only piecewise if splitting ops is set accordingly and
204 + use_inductor_cudagraphs_partition is off. Note that the default options for
205 + splitting ops are sufficient for piecewise compilation.
206 + """
207 custom_ops: list[str] = field(default_factory=list)
208 """Fine-grained control over which custom ops to enable/disable. Use 'all'
209 to enable all, 'none' to disable all. Also specify a list of custom op
@@ -229,8 +236,12 @@ class CompilationConfig:
236 If empty list [], no ops are excluded (suitable for full cudagraphs)."""
237
238 # Inductor capture
232 - use_inductor: bool = True
233 - """Whether to use inductor compilation:
239 + use_inductor: bool | None = None
240 + """
241 + Whether to use inductor compilation.
242 +
243 + This flag is deprecated and will be removed in the next release 0.12.0.
244 + Please use the 'backend' option instead.
245
246 - False: inductor compilation is not used. graph runs in eager
247 (custom_ops enabled by default).
@@ -238,7 +249,11 @@ class CompilationConfig:
249 One graph for symbolic shape and one graph per size in compile_sizes
250 are compiled using configurations in inductor_compile_config.
251
241 - This setting is ignored if level<PIECEWISE."""
252 + This setting is ignored if level<PIECEWISE.
253 +
254 + For future compatibility:
255 + If use_inductor is True, backend="inductor" otherwise backend="eager".
256 + """
257 compile_sizes: list[int | str] | None = None
258 """Sizes to compile for inductor. In addition
259 to integers, it also supports "cudagraph_capture_sizes" to
@@ -545,7 +560,43 @@ class CompilationConfig:
560 "(where 'op' is the registered op name)"
561 )
562
563 + # Currently only eager and inductor backend are supported.
564 + # for piecewise compilation. Custom backends are not suppported for
565 + # piecewise compilation. Update when more backends are supported.
566 + if self.level == CompilationLevel.PIECEWISE and self.backend not in [
567 + "",
568 + "eager",
569 + "inductor",
570 + ]:
571 + raise ValueError(
572 + f"Invalid backend for piecewise compilation: {self.backend}"
573 + )
574 +
575 + if self.use_inductor is not None:
576 + logger.warning_once(
577 + "The 'use_inductor' flag is deprecated and will be "
578 + "removed in the next release (v0.12.0). "
579 + "Please use the 'backend' option instead.",
580 + )
581 + self.backend = "inductor" if self.use_inductor else "eager"
582 +
583 + if self.backend == "":
584 + self.backend = current_platform.simple_compile_backend
585 +
586 def init_backend(self, vllm_config: "VllmConfig") -> str | Callable:
587 + """
588 + Initialize the backend for the compilation config from a vllm config.
589 + Arguments:
590 + vllm_config: The vllm config to initialize the backend from.
591 + Returns:
592 + The backend for the compilation config.
593 + """
594 + if self.level is None:
595 + raise ValueError(
596 + "No compilation level is set. This method should only be \
597 + called via vllm config where the level is set if none is \
598 + provided."
599 + )
600 if self.level == CompilationLevel.NO_COMPILATION:
601 raise ValueError("No compilation level is set.")
602
@@ -553,15 +604,15 @@ class CompilationConfig:
604
605 torch_backends = list_backends(exclude_tags=tuple())
606 if self.level in [CompilationLevel.DYNAMO_AS_IS, CompilationLevel.DYNAMO_ONCE]:
556 - if self.backend == "":
557 - return "eager"
607 if self.backend in torch_backends:
608 return self.backend
609 return resolve_obj_by_qualname(self.backend)
610
562 - # TODO: pass user-specified backend to piecewise compilation
563 - # merge with the config use_inductor
611 assert self.level == CompilationLevel.PIECEWISE
612 + if self.backend not in ["eager", "inductor"]:
613 + raise ValueError(
614 + f"Invalid backend for piecewise compilation: {self.backend}"
615 + )
616
617 from vllm.compilation.backends import VllmBackend
618
@@ -710,7 +761,9 @@ class CompilationConfig:
761 return self.level == CompilationLevel.PIECEWISE
762
763 # Inductor partition case
713 - return self.level > CompilationLevel.NO_COMPILATION and self.use_inductor
764 + return (
765 + self.backend == "inductor" and self.level > CompilationLevel.NO_COMPILATION
766 + )
767
768 def custom_op_log_check(self):
769 """
vllm/config/vllm.py
+14
@@ -322,6 +322,20 @@ class VllmConfig:
322 # NB: Passing both --enforce-eager and a compilation level
323 # in V0 means the compilation level wins out.
324 self.compilation_config.level = CompilationLevel.NO_COMPILATION
325 + else:
326 + assert self.compilation_config.level >= CompilationLevel.NO_COMPILATION
327 + assert self.compilation_config.level <= CompilationLevel.PIECEWISE
328 +
329 + # If user does not set custom ops via none or all set it here based on
330 + # compilation level and backend.
331 + if all(s not in self.compilation_config.custom_ops for s in ("all", "none")):
332 + if (
333 + self.compilation_config.backend == "inductor"
334 + and self.compilation_config.level > CompilationLevel.NO_COMPILATION
335 + ):
336 + self.compilation_config.custom_ops.append("none")
337 + else:
338 + self.compilation_config.custom_ops.append("all")
339
340 # async tp is built on top of sequence parallelism
341 # and requires it to be enabled.
vllm/model_executor/custom_op.py
+10 -10
@@ -113,7 +113,9 @@ class CustomOp(nn.Module):
113 custom_ops = compilation_config.custom_ops
114 if not hasattr(cls, "name"):
115 logger.warning_once(
116 - "Custom op %s was not registered, which means it won't appear in the op registry. It will be enabled/disabled based on the global settings.", # noqa: E501
116 + "Custom op %s was not registered, which means it won't appear "
117 + "in the op registry. It will be enabled/disabled based on the "
118 + "global settings.",
119 cls.__name__,
120 )
121 return CustomOp.default_on()
@@ -127,19 +129,17 @@ class CustomOp(nn.Module):
129 @staticmethod
130 def default_on() -> bool:
131 """
130 - On by default if PyTorch Inductor is not used.
131 - Specifying 'all' or 'none' in custom_op takes precedence.
132 + Behavior controlled by `CompilationConfig.custom_ops`: On by default if
133 + 'all', off by default if 'none'.
134 + When PyTorch Inductor is used, 'none' is the default value,
135 + otherwise 'all'.
136 """
133 - from vllm.config import CompilationLevel
134 -
137 compilation_config = get_cached_compilation_config()
136 - default_on = (
137 - compilation_config.level < CompilationLevel.PIECEWISE
138 - or not compilation_config.use_inductor
139 - )
138 count_none = compilation_config.custom_ops.count("none")
139 count_all = compilation_config.custom_ops.count("all")
142 - return default_on and not count_none > 0 or count_all > 0
140 + assert count_none + count_all == 1
141 +
142 + return not count_none > 0 or count_all > 0
143
144 # Dictionary of all custom ops (classes, indexed by registered name).
145 # To check if an op with a name is enabled, call .enabled() on the class.
vllm/platforms/cpu.py
-2
@@ -275,8 +275,6 @@ class CpuPlatform(Platform):
275 "epilogue_fusion": True,
276 }
277 )
278 - if compilation_config.use_inductor:
279 - compilation_config.custom_ops = ["none"]
278
279 if vllm_config.lora_config is not None:
280 compilation_config.level = CompilationLevel.NO_COMPILATION