@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / d7284a2604
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 12, 03:30 UTC

[Core] Rename PassConfig flags as per RFC #27995 (#29646)

Signed-off-by: arpitkh101 <arpit5khandelwal@gmail.com> Co-authored-by: Luka Govedič <ProExpertProg@users.noreply.github.com>

Arpit Khandelwal committed Dec 2, 2025 at 22:38 UTC d7284a2604ef3fe96f0779309caafb59860704bb
22 files changed +318 -123
tests/compile/distributed/test_async_tp.py
+2 -2
@@ -326,7 +326,7 @@ def async_tp_pass_on_test_model(
326 vllm_config = VllmConfig()
327 vllm_config.compilation_config = CompilationConfig(
328 pass_config=PassConfig(
329 - enable_async_tp=True,
329 + fuse_gemm_comms=True,
330 ),
331 )
332 vllm_config.device_config = DeviceConfig(device=torch.device("cuda"))
@@ -413,7 +413,7 @@ def test_async_tp_pass_correctness(
413 "mode": CompilationMode.VLLM_COMPILE,
414 "compile_sizes": [2, 4, 8],
415 "splitting_ops": [],
416 - "pass_config": {"enable_async_tp": async_tp_enabled},
416 + "pass_config": {"fuse_gemm_comms": async_tp_enabled},
417 }
418
419 async_tp_args = [
tests/compile/distributed/test_fusion_all_reduce.py
+1 -1
@@ -295,7 +295,7 @@ def all_reduce_fusion_pass_on_test_model(
295 )
296 )
297 vllm_config.compilation_config.pass_config = PassConfig(
298 - enable_fi_allreduce_fusion=True, enable_noop=True
298 + fuse_allreduce_rms=True, eliminate_noops=True
299 )
300 vllm_config.device_config = DeviceConfig(device=torch.device("cuda"))
301 vllm_config.parallel_config.rank = local_rank # Setup rank for debug path
tests/compile/distributed/test_fusions_e2e.py
+8 -8
@@ -192,7 +192,7 @@ def test_attn_quant(
192 splitting_ops=splitting_ops,
193 # Common
194 mode=CompilationMode.VLLM_COMPILE,
195 - pass_config=PassConfig(enable_attn_fusion=True, enable_noop=True),
195 + pass_config=PassConfig(fuse_attn_quant=True, eliminate_noops=True),
196 # Inductor caches custom passes by default as well via uuid
197 inductor_compile_config={"force_disable_caches": True},
198 )
@@ -282,9 +282,9 @@ def test_tp2_attn_quant_allreduce_rmsnorm(
282 # Common
283 mode=CompilationMode.VLLM_COMPILE,
284 pass_config=PassConfig(
285 - enable_attn_fusion=True,
286 - enable_noop=True,
287 - enable_fi_allreduce_fusion=True,
285 + fuse_attn_quant=True,
286 + eliminate_noops=True,
287 + fuse_allreduce_rms=True,
288 ),
289 # Inductor caches custom passes by default as well via uuid
290 inductor_compile_config={"force_disable_caches": True},
@@ -384,10 +384,10 @@ def test_tp2_attn_quant_async_tp(
384 # Common
385 level=CompilationMode.VLLM_COMPILE,
386 pass_config=PassConfig(
387 - enable_attn_fusion=True,
388 - enable_noop=True,
389 - enable_sequence_parallelism=True,
390 - enable_async_tp=True,
387 + fuse_attn_quant=True,
388 + eliminate_noops=True,
389 + enable_sp=True,
390 + fuse_gemm_comms=True,
391 ),
392 # Inductor caches custom passes by default as well via uuid
393 inductor_compile_config={"force_disable_caches": True},
tests/compile/distributed/test_sequence_parallelism.py
+9 -9
@@ -153,7 +153,7 @@ class TestAllReduceRMSNormStaticQuantFP8Model(torch.nn.Module):
153 ]
154
155 def ops_in_model(self):
156 - if self.vllm_config.compilation_config.pass_config.enable_fusion:
156 + if self.vllm_config.compilation_config.pass_config.fuse_norm_quant:
157 return [torch.ops._C.fused_add_rms_norm_static_fp8_quant.default]
158 elif RMSNorm.enabled():
159 return [
@@ -183,7 +183,7 @@ class TestAllReduceRMSNormStaticQuantFP8Model(torch.nn.Module):
183 @pytest.mark.parametrize("seq_len", [16])
184 @pytest.mark.parametrize("hidden_size", [16])
185 @pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16])
186 -@pytest.mark.parametrize("enable_fusion", [True, False])
186 +@pytest.mark.parametrize("fuse_norm_quant", [True, False])
187 @pytest.mark.parametrize("dynamic", [False, True])
188 @pytest.mark.skipif(envs.VLLM_TARGET_DEVICE not in ["cuda"], reason="Only test on CUDA")
189 def test_sequence_parallelism_pass(
@@ -193,7 +193,7 @@ def test_sequence_parallelism_pass(
193 seq_len: int,
194 hidden_size: int,
195 dtype: torch.dtype,
196 - enable_fusion: bool,
196 + fuse_norm_quant: bool,
197 dynamic: bool,
198 ):
199 num_processes = 2
@@ -211,7 +211,7 @@ def test_sequence_parallelism_pass(
211 seq_len,
212 hidden_size,
213 dtype,
214 - enable_fusion,
214 + fuse_norm_quant,
215 dynamic,
216 ),
217 nprocs=nprocs,
@@ -229,7 +229,7 @@ def sequence_parallelism_pass_on_test_model(
229 seq_len: int,
230 hidden_size: int,
231 dtype: torch.dtype,
232 - enable_fusion: bool,
232 + fuse_norm_quant: bool,
233 dynamic: bool,
234 ):
235 current_platform.seed_everything(0)
@@ -260,9 +260,9 @@ def sequence_parallelism_pass_on_test_model(
260 cudagraph_mode=CUDAGraphMode.NONE, # avoid piecewise warnings
261 custom_ops=custom_ops_list,
262 pass_config=PassConfig(
263 - enable_sequence_parallelism=True,
264 - enable_fusion=enable_fusion,
265 - enable_noop=True,
263 + enable_sp=True,
264 + fuse_norm_quant=fuse_norm_quant,
265 + eliminate_noops=True,
266 ),
267 ) # NoOp needed for fusion
268 device_config = DeviceConfig(device=torch.device("cuda"))
@@ -297,7 +297,7 @@ def sequence_parallelism_pass_on_test_model(
297 sequence_parallelism_pass,
298 ]
299
300 - if enable_fusion:
300 + if fuse_norm_quant:
301 fusion_pass = RMSNormQuantFusionPass(vllm_config)
302 passes_for_backend.append(fusion_pass)
303
tests/compile/fullgraph/test_full_graph.py
+3 -1
@@ -122,7 +122,9 @@ def test_full_graph(
122 CompilationConfig(
123 mode=CompilationMode.VLLM_COMPILE,
124 custom_ops=["+rms_norm"],
125 - pass_config=PassConfig(enable_fusion=True, enable_noop=True),
125 + pass_config=PassConfig(
126 + fuse_norm_quant=True, fuse_act_quant=True, eliminate_noops=True
127 + ),
128 ),
129 *model_info,
130 )
tests/compile/test_config.py
+65 -12
@@ -1,6 +1,7 @@
1 # SPDX-License-Identifier: Apache-2.0
2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 import copy
4 +import logging
5 from contextlib import nullcontext
6 from unittest.mock import patch
7
@@ -10,8 +11,9 @@ from pydantic import ValidationError
11 from vllm.compilation.counter import compilation_counter
12 from vllm.compilation.fix_functionalization import FixFunctionalizationPass
13 from vllm.config import CompilationConfig, CUDAGraphMode, VllmConfig
13 -from vllm.config.compilation import CompilationMode
14 +from vllm.config.compilation import CompilationMode, PassConfig
15 from vllm.engine.arg_utils import EngineArgs
16 +from vllm.logger import _print_warning_once
17 from vllm.platforms import current_platform
18 from vllm.utils.torch_utils import _is_torch_equal_or_newer
19
@@ -191,7 +193,7 @@ def test_splitting_ops_dynamic():
193 config = VllmConfig(
194 compilation_config=CompilationConfig(
195 mode=CompilationMode.VLLM_COMPILE,
194 - pass_config={"enable_attn_fusion": True, "enable_noop": True},
196 + pass_config=PassConfig(fuse_attn_quant=True, eliminate_noops=True),
197 custom_ops=["+quant_fp8"],
198 cudagraph_mode=CUDAGraphMode.PIECEWISE,
199 )
@@ -206,7 +208,7 @@ def test_splitting_ops_dynamic():
208 config = VllmConfig(
209 compilation_config=CompilationConfig(
210 mode=CompilationMode.VLLM_COMPILE,
209 - pass_config={"enable_attn_fusion": True, "enable_noop": True},
211 + pass_config=PassConfig(fuse_attn_quant=True, eliminate_noops=True),
212 custom_ops=["+quant_fp8"],
213 cudagraph_mode=CUDAGraphMode.PIECEWISE,
214 # work around for accessing all attntion ops
@@ -219,7 +221,7 @@ def test_splitting_ops_dynamic():
221 compilation_config=CompilationConfig(
222 mode=CompilationMode.VLLM_COMPILE,
223 use_inductor_graph_partition=True,
222 - pass_config={"enable_attn_fusion": True, "enable_noop": True},
224 + pass_config=PassConfig(fuse_attn_quant=True, eliminate_noops=True),
225 custom_ops=["+quant_fp8"],
226 cudagraph_mode=CUDAGraphMode.PIECEWISE,
227 )
@@ -227,7 +229,7 @@ def test_splitting_ops_dynamic():
229 # With inductor graph partition, attn_fusion and splitting_ops
230 # work together. Default splitting_ops include attention ops.
231 assert config.compilation_config.splitting_ops_contain_attention()
230 - # enable_attn_fusion is directly supported under
232 + # fuse_attn_quant is directly supported under
233 # use_inductor_graph_partition=True, and cudagraph_mode
234 # is unchanged.
235 assert config.compilation_config.cudagraph_mode == CUDAGraphMode.PIECEWISE
@@ -301,7 +303,7 @@ def test_should_split():
303 "cudagraph_capture_sizes",
304 "max_cudagraph_capture_size",
305 "tp_size",
304 - "enable_sequence_parallelism",
306 + "enable_sp",
307 "max_num_batched_tokens",
308 "cudagraph_mode",
309 "expected_max_size",
@@ -339,7 +341,7 @@ def test_cudagraph_sizes_post_init(
341 cudagraph_capture_sizes,
342 max_cudagraph_capture_size,
343 tp_size,
342 - enable_sequence_parallelism,
344 + enable_sp,
345 max_num_batched_tokens,
346 cudagraph_mode,
347 expected_max_size,
@@ -355,11 +357,12 @@ def test_cudagraph_sizes_post_init(
357 compilation_config = CompilationConfig(
358 cudagraph_capture_sizes=cudagraph_capture_sizes,
359 max_cudagraph_capture_size=max_cudagraph_capture_size,
358 - pass_config={
359 - "enable_sequence_parallelism": enable_sequence_parallelism,
360 - "enable_fusion": True,
361 - "enable_noop": True,
362 - },
360 + pass_config=PassConfig(
361 + enable_sp=enable_sp,
362 + fuse_norm_quant=True,
363 + fuse_act_quant=True,
364 + eliminate_noops=True,
365 + ),
366 cudagraph_mode=cudagraph_mode,
367 )
368 engine_args = EngineArgs(
@@ -375,3 +378,53 @@ def test_cudagraph_sizes_post_init(
378 vllm_config.compilation_config.max_cudagraph_capture_size
379 == expected_max_size
380 )
381 +
382 +
383 +def test_pass_config_deprecation(caplog_vllm):
384 + caplog_vllm.set_level(logging.WARNING)
385 +
386 + # Clear cache to ensure warnings are re-issued
387 + _print_warning_once.cache_clear()
388 +
389 + # Test enable_fusion -> fuse_norm_quant, fuse_act_quant
390 + caplog_vllm.clear()
391 + config = PassConfig(enable_fusion=True)
392 + assert "enable_fusion is deprecated" in caplog_vllm.text
393 + assert config.fuse_norm_quant is True
394 + assert config.fuse_act_quant is True
395 + assert config.enable_fusion is None
396 +
397 + # Test enable_attn_fusion -> fuse_attn_quant
398 + caplog_vllm.clear()
399 + config = PassConfig(enable_attn_fusion=True)
400 + assert "enable_attn_fusion is deprecated" in caplog_vllm.text
401 + assert config.fuse_attn_quant is True
402 + assert config.enable_attn_fusion is None
403 +
404 + # Test enable_noop -> eliminate_noops
405 + caplog_vllm.clear()
406 + config = PassConfig(enable_noop=True)
407 + assert "enable_noop is deprecated" in caplog_vllm.text
408 + assert config.eliminate_noops is True
409 + assert config.enable_noop is None
410 +
411 + # Test enable_sequence_parallelism -> enable_sp
412 + caplog_vllm.clear()
413 + config = PassConfig(enable_sequence_parallelism=True)
414 + assert "enable_sequence_parallelism is deprecated" in caplog_vllm.text
415 + assert config.enable_sp is True
416 + assert config.enable_sequence_parallelism is None
417 +
418 + # Test enable_async_tp -> fuse_gemm_comms
419 + caplog_vllm.clear()
420 + config = PassConfig(enable_async_tp=True)
421 + assert "enable_async_tp is deprecated" in caplog_vllm.text
422 + assert config.fuse_gemm_comms is True
423 + assert config.enable_async_tp is None
424 +
425 + # Test enable_fi_allreduce_fusion -> fuse_allreduce_rms
426 + caplog_vllm.clear()
427 + config = PassConfig(enable_fi_allreduce_fusion=True)
428 + assert "enable_fi_allreduce_fusion is deprecated" in caplog_vllm.text
429 + assert config.fuse_allreduce_rms is True
430 + assert config.enable_fi_allreduce_fusion is None
tests/compile/test_functionalization.py
+5 -1
@@ -223,7 +223,11 @@ def test_fix_functionalization(
223 model_config=ModelConfig(dtype=dtype),
224 compilation_config=CompilationConfig(
225 custom_ops=["all"],
226 - pass_config=PassConfig(enable_fusion=do_fusion, enable_noop=True),
226 + pass_config=PassConfig(
227 + fuse_norm_quant=do_fusion,
228 + fuse_act_quant=do_fusion,
229 + eliminate_noops=True,
230 + ),
231 ),
232 )
233
tests/compile/test_fusion.py
+3 -1
@@ -159,7 +159,9 @@ def test_fusion_rmsnorm_quant(
159 compilation_config=CompilationConfig(
160 mode=CompilationMode.VLLM_COMPILE,
161 custom_ops=custom_ops,
162 - pass_config=PassConfig(enable_fusion=True, enable_noop=True),
162 + pass_config=PassConfig(
163 + fuse_norm_quant=True, fuse_act_quant=True, eliminate_noops=True
164 + ),
165 ),
166 )
167 with vllm.config.set_current_vllm_config(vllm_config):
tests/compile/test_fusion_attn.py
+1 -1
@@ -373,7 +373,7 @@ def test_attention_quant_pattern(
373
374 # Run model with attn fusion enabled
375 vllm_config.compilation_config.pass_config = PassConfig(
376 - enable_attn_fusion=True, enable_noop=True
376 + fuse_attn_quant=True, eliminate_noops=True
377 )
378 with (
379 set_current_vllm_config(vllm_config),
tests/compile/test_noop_elimination.py
+2 -2
@@ -51,7 +51,7 @@ def test_noop_elimination(dtype, num_tokens, hidden_size, buffer_size):
51 vllm_config = VllmConfig(
52 compilation_config=CompilationConfig(
53 mode=CompilationMode.VLLM_COMPILE,
54 - pass_config=PassConfig(enable_noop=True),
54 + pass_config=PassConfig(eliminate_noops=True),
55 )
56 )
57 with vllm.config.set_current_vllm_config(vllm_config):
@@ -99,7 +99,7 @@ def test_non_noop_slice_preserved():
99 vllm_config = VllmConfig(
100 compilation_config=CompilationConfig(
101 mode=CompilationMode.VLLM_COMPILE,
102 - pass_config=PassConfig(enable_noop=True),
102 + pass_config=PassConfig(eliminate_noops=True),
103 )
104 )
105 with vllm.config.set_current_vllm_config(vllm_config):
tests/compile/test_pass_manager.py
+5 -2
@@ -64,8 +64,11 @@ def test_pass_manager_uuid(callable):
64
65 # UUID should be different due to config change
66 config2 = copy.deepcopy(config)
67 - config2.compilation_config.pass_config.enable_fusion = (
68 - not config2.compilation_config.pass_config.enable_fusion
67 + config2.compilation_config.pass_config.fuse_norm_quant = (
68 + not config2.compilation_config.pass_config.fuse_norm_quant
69 + )
70 + config2.compilation_config.pass_config.fuse_act_quant = (
71 + not config2.compilation_config.pass_config.fuse_act_quant
72 )
73 pass_manager3 = PostGradPassManager()
74 pass_manager3.configure(config2)
tests/compile/test_qk_norm_rope_fusion.py
+1 -1
@@ -140,7 +140,7 @@ def test_qk_norm_rope_fusion(
140 custom_ops=custom_ops,
141 pass_config=PassConfig(
142 enable_qk_norm_rope_fusion=True,
143 - enable_noop=True,
143 + eliminate_noops=True,
144 ),
145 ),
146 )
tests/compile/test_silu_mul_quant_fusion.py
+1 -1
@@ -168,7 +168,7 @@ def test_fusion_silu_and_mul_quant(
168 compilation_config=CompilationConfig(
169 mode=CompilationMode.VLLM_COMPILE,
170 custom_ops=custom_ops,
171 - pass_config=PassConfig(enable_fusion=True, enable_noop=True),
171 + pass_config=PassConfig(fuse_act_quant=True, eliminate_noops=True),
172 ),
173 )
174
tests/distributed/test_sequence_parallel.py
+20 -14
@@ -32,7 +32,8 @@ VLLM_MULTI_NODE = os.getenv("VLLM_MULTI_NODE", "0") == "1"
32 class ParallelSetup(NamedTuple):
33 tp_size: int
34 pp_size: int
35 - enable_fusion: bool
35 + fuse_norm_quant: bool
36 + fuse_act_quant: bool
37 eager_mode: bool
38 chunked_prefill: bool
39
@@ -66,7 +67,8 @@ class SPTestSettings:
67 ParallelSetup(
68 tp_size=tp_base,
69 pp_size=pp_multiplier * pp_base,
69 - enable_fusion=False,
70 + fuse_norm_quant=False,
71 + fuse_act_quant=False,
72 eager_mode=eager_mode_val,
73 chunked_prefill=chunked_prefill_val,
74 )
@@ -97,7 +99,8 @@ class SPTestSettings:
99 ParallelSetup(
100 tp_size=tp_base,
101 pp_size=pp_multiplier * pp_base,
100 - enable_fusion=False,
102 + fuse_norm_quant=False,
103 + fuse_act_quant=False,
104 eager_mode=eager_mode_val,
105 chunked_prefill=chunked_prefill_val,
106 )
@@ -126,7 +129,8 @@ class SPTestSettings:
129 ParallelSetup(
130 tp_size=tp_base,
131 pp_size=pp_base,
129 - enable_fusion=fusion_val,
132 + fuse_norm_quant=fusion_val,
133 + fuse_act_quant=fusion_val,
134 eager_mode=True,
135 chunked_prefill=False,
136 )
@@ -162,7 +166,7 @@ def _compare_sp(
166 test_options: SPTestOptions,
167 num_gpus_available: int,
168 use_inductor_graph_partition: bool,
165 - enable_async_tp: bool,
169 + fuse_gemm_comms: bool,
170 *,
171 method: Literal["generate", "encode"],
172 is_multimodal: bool,
@@ -170,7 +174,8 @@ def _compare_sp(
174 (
175 tp_size,
176 pp_size,
173 - enable_fusion,
177 + fuse_norm_quant,
178 + fuse_act_quant,
179 eager_mode,
180 chunked_prefill,
181 ) = parallel_setup
@@ -248,10 +253,11 @@ def _compare_sp(
253 "mode": CompilationMode.VLLM_COMPILE,
254 "compile_sizes": [4, 8],
255 "pass_config": {
251 - "enable_sequence_parallelism": True,
252 - "enable_async_tp": enable_async_tp,
253 - "enable_fusion": enable_fusion,
254 - "enable_noop": True,
256 + "enable_sp": True,
257 + "fuse_gemm_comms": fuse_gemm_comms,
258 + "fuse_norm_quant": fuse_norm_quant,
259 + "fuse_act_quant": fuse_act_quant,
260 + "eliminate_noops": True,
261 },
262 "use_inductor_graph_partition": use_inductor_graph_partition,
263 }
@@ -309,7 +315,7 @@ SP_TEST_MODELS = [
315 ],
316 )
317 @pytest.mark.parametrize("use_inductor_graph_partition", [True, False])
312 -@pytest.mark.parametrize("enable_async_tp", [False]) # TODO: enable async TP
318 +@pytest.mark.parametrize("fuse_gemm_comms", [False]) # TODO: enable async TP
319 @create_new_process_for_each_test()
320 def test_tp_sp_generation(
321 model_id: str,
@@ -319,7 +325,7 @@ def test_tp_sp_generation(
325 test_options: SPTestOptions,
326 num_gpus_available,
327 use_inductor_graph_partition: bool,
322 - enable_async_tp: bool,
328 + fuse_gemm_comms: bool,
329 ):
330 if use_inductor_graph_partition and not is_torch_equal_or_newer("2.9.0.dev"):
331 pytest.skip("inductor graph partition is only available in PyTorch 2.9+")
@@ -328,7 +334,7 @@ def test_tp_sp_generation(
334 if (
335 "fp8" in model_id.lower()
336 and current_platform.get_device_capability() < (9, 0)
331 - and (not enable_async_tp)
337 + and (not fuse_gemm_comms)
338 ):
339 pytest.skip("FP8 reduction support begins with sm90 capable devices.")
340
@@ -340,7 +346,7 @@ def test_tp_sp_generation(
346 test_options,
347 num_gpus_available,
348 use_inductor_graph_partition,
343 - enable_async_tp=enable_async_tp,
349 + fuse_gemm_comms=fuse_gemm_comms,
350 method="generate",
351 is_multimodal=False,
352 )
tests/test_config.py
+6 -6
@@ -1023,17 +1023,17 @@ def test_vllm_config_explicit_overrides():
1023 assert config.compilation_config.cudagraph_mode == CUDAGraphMode.NONE
1024
1025 # Explicit pass config flags to override defaults
1026 - pass_config = PassConfig(enable_noop=True, enable_attn_fusion=True)
1026 + pass_config = PassConfig(eliminate_noops=True, fuse_attn_quant=True)
1027 compilation_config = CompilationConfig(pass_config=pass_config)
1028 config = VllmConfig(
1029 optimization_level=OptimizationLevel.O0,
1030 compilation_config=compilation_config,
1031 )
1032 - assert config.compilation_config.pass_config.enable_noop is True
1033 - assert config.compilation_config.pass_config.enable_attn_fusion is True
1032 + assert config.compilation_config.pass_config.eliminate_noops is True
1033 + assert config.compilation_config.pass_config.fuse_attn_quant is True
1034
1035 # Explicit cudagraph mode override on quantized model at O2
1036 - pass_config = PassConfig(enable_async_tp=True)
1036 + pass_config = PassConfig(fuse_gemm_comms=True)
1037 compilation_config = CompilationConfig(
1038 cudagraph_mode=CUDAGraphMode.NONE, pass_config=pass_config
1039 )
@@ -1043,7 +1043,7 @@ def test_vllm_config_explicit_overrides():
1043 compilation_config=compilation_config,
1044 )
1045 assert config.compilation_config.cudagraph_mode == CUDAGraphMode.NONE
1046 - assert config.compilation_config.pass_config.enable_async_tp is True
1046 + assert config.compilation_config.pass_config.fuse_gemm_comms is True
1047 # Mode should still use default for O2
1048 assert config.compilation_config.mode == CompilationMode.VLLM_COMPILE
1049
@@ -1093,7 +1093,7 @@ def test_vllm_config_explicit_overrides():
1093 compilation_config=compilation_config,
1094 )
1095 # Explicit override should be respected
1096 - assert config.compilation_config.pass_config.enable_noop is False
1096 + assert config.compilation_config.pass_config.eliminate_noops is False
1097 # Other fields should still use defaults
1098 assert config.compilation_config.mode == CompilationMode.VLLM_COMPILE
1099 assert config.compilation_config.cudagraph_mode == CUDAGraphMode.FULL_AND_PIECEWISE
vllm/compilation/pass_manager.py
+7 -6
@@ -92,22 +92,23 @@ class PostGradPassManager(CustomGraphPass):
92
93 # Set the current vllm config to allow tracing CustomOp instances
94 with set_current_vllm_config(config, check_compile=False):
95 - if self.pass_config.enable_noop:
95 + if self.pass_config.eliminate_noops:
96 self.passes += [NoOpEliminationPass(config)]
97
98 - if self.pass_config.enable_sequence_parallelism:
98 + if self.pass_config.enable_sp:
99 self.passes += [SequenceParallelismPass(config)]
100 - if self.pass_config.enable_async_tp:
100 + if self.pass_config.fuse_gemm_comms:
101 self.passes += [AsyncTPPass(config)]
102
103 - if self.pass_config.enable_fi_allreduce_fusion:
103 + if self.pass_config.fuse_allreduce_rms:
104 self.passes += [AllReduceFusionPass(config)]
105
106 - if self.pass_config.enable_fusion:
106 + if self.pass_config.fuse_norm_quant:
107 self.passes += [RMSNormQuantFusionPass(config)]
108 + if self.pass_config.fuse_act_quant:
109 self.passes += [ActivationQuantFusionPass(config)]
110
110 - if self.pass_config.enable_attn_fusion:
111 + if self.pass_config.fuse_attn_quant:
112 self.passes += [AttnFusionPass(config)]
113
114 if self.pass_config.enable_qk_norm_rope_fusion:
vllm/config/compilation.py
+102 -18
@@ -13,7 +13,7 @@ from pydantic.dataclasses import dataclass
13
14 import vllm.envs as envs
15 from vllm.compilation.inductor_pass import CallableInductorPass, InductorPass
16 -from vllm.config.utils import config
16 +from vllm.config.utils import config, handle_deprecated
17 from vllm.logger import init_logger
18 from vllm.platforms import current_platform
19 from vllm.utils.import_utils import resolve_obj_by_qualname
@@ -105,18 +105,43 @@ class PassConfig:
105 improper state.
106 """
107
108 + # New flags
109 + fuse_norm_quant: bool = Field(default=None)
110 + """Fuse the custom RMSNorm + quant ops."""
111 + fuse_act_quant: bool = Field(default=None)
112 + """Fuse the custom SiluMul + quant ops."""
113 + fuse_attn_quant: bool = Field(default=None)
114 + """Fuse the custom attention + quant ops."""
115 + eliminate_noops: bool = Field(default=None)
116 + """Eliminate no-op ops."""
117 + enable_sp: bool = Field(default=None)
118 + """Enable sequence parallelism."""
119 + fuse_gemm_comms: bool = Field(default=None)
120 + """Enable async TP."""
121 + fuse_allreduce_rms: bool = Field(default=None)
122 + """Enable flashinfer allreduce fusion."""
123 +
124 + # Deprecated flags
125 enable_fusion: bool = Field(default=None)
109 - """Whether to enable the custom fusion (RMSNorm/SiluMul+quant) pass."""
126 + """Deprecated in: v0.12.0. Use fuse_norm_quant and fuse_act_quant
127 + instead. Will be removed in v0.13.0 or v1.0.0, whichever is sooner.
128 + """
129 enable_attn_fusion: bool = Field(default=None)
111 - """Whether to enable the custom attention+quant fusion pass."""
130 + """Deprecated in: v0.12.0. Use fuse_attn_quant instead.
131 + Will be removed in v0.13.0 or v1.0.0, whichever is sooner."""
132 enable_noop: bool = Field(default=None)
113 - """Whether to enable the custom no-op elimination pass."""
133 + """Deprecated in: v0.12.0. Use eliminate_noops instead.
134 + Will be removed in v0.13.0 or v1.0.0, whichever is sooner."""
135 enable_sequence_parallelism: bool = Field(default=None)
115 - """Whether to enable sequence parallelism."""
136 + """Deprecated in: v0.12.0. Use enable_sp instead.
137 + Will be removed in v0.13.0 or v1.0.0, whichever is sooner."""
138 enable_async_tp: bool = Field(default=None)
117 - """Whether to enable async TP."""
139 + """Deprecated in: v0.12.0. Use fuse_gemm_comms instead.
140 + Will be removed in v0.13.0 or v1.0.0, whichever is sooner."""
141 enable_fi_allreduce_fusion: bool = Field(default=None)
119 - """Whether to enable flashinfer allreduce fusion."""
142 + """Deprecated in: v0.12.0. Use fuse_allreduce_rms instead.
143 + Will be removed in v0.13.0 or v1.0.0, whichever is sooner."""
144 +
145 fi_allreduce_fusion_max_size_mb: float | None = None
146 """The threshold of the communicated tensor sizes under which
147 vllm should use flashinfer fused allreduce. Specified as a
@@ -136,7 +161,7 @@ class PassConfig:
161 },
162 }, where key is the device capability"""
163 enable_qk_norm_rope_fusion: bool = False
139 - """Whether to enable the fused Q/K RMSNorm + RoPE pass."""
164 + """Enable fused Q/K RMSNorm + RoPE pass."""
165
166 # TODO(luka) better pass enabling system.
167
@@ -174,6 +199,13 @@ class PassConfig:
199 return InductorPass.hash_dict(asdict(self))
200
201 @field_validator(
202 + "fuse_norm_quant",
203 + "fuse_act_quant",
204 + "fuse_attn_quant",
205 + "eliminate_noops",
206 + "enable_sp",
207 + "fuse_gemm_comms",
208 + "fuse_allreduce_rms",
209 "enable_fusion",
210 "enable_attn_fusion",
211 "enable_noop",
@@ -190,18 +222,71 @@ class PassConfig:
222 return handler(value)
223
224 def __post_init__(self) -> None:
193 - if not self.enable_noop:
194 - if self.enable_fusion:
225 + # Handle deprecation and defaults
226 +
227 + # Map old flags to new flags and issue warnings
228 + handle_deprecated(
229 + self,
230 + "enable_fusion",
231 + ["fuse_norm_quant", "fuse_act_quant"],
232 + "v0.13.0 or v1.0.0, whichever is sooner",
233 + )
234 +
235 + handle_deprecated(
236 + self,
237 + "enable_attn_fusion",
238 + "fuse_attn_quant",
239 + "v0.13.0 or v1.0.0, whichever is sooner",
240 + )
241 +
242 + handle_deprecated(
243 + self,
244 + "enable_sequence_parallelism",
245 + "enable_sp",
246 + "v0.13.0 or v1.0.0, whichever is sooner",
247 + )
248 +
249 + handle_deprecated(
250 + self,
251 + "enable_async_tp",
252 + "fuse_gemm_comms",
253 + "v0.13.0 or v1.0.0, whichever is sooner",
254 + )
255 +
256 + handle_deprecated(
257 + self,
258 + "enable_fi_allreduce_fusion",
259 + "fuse_allreduce_rms",
260 + "v0.13.0 or v1.0.0, whichever is sooner",
261 + )
262 +
263 + handle_deprecated(
264 + self,
265 + "enable_noop",
266 + "eliminate_noops",
267 + "v0.13.0 or v1.0.0, whichever is sooner",
268 + )
269 +
270 + # Force old flags to None to ensure they are not used
271 + self.enable_fusion = None
272 + self.enable_attn_fusion = None
273 + self.enable_noop = None
274 + self.enable_sequence_parallelism = None
275 + self.enable_async_tp = None
276 + self.enable_fi_allreduce_fusion = None
277 +
278 + if not self.eliminate_noops:
279 + if self.fuse_norm_quant or self.fuse_act_quant:
280 logger.warning_once(
281 "Fusion enabled but reshape elimination disabled. "
282 "RMSNorm/SiluMul + quant (fp8) fusion might not work"
283 )
199 - if self.enable_attn_fusion:
284 + if self.fuse_attn_quant:
285 logger.warning_once(
286 "Fusion enabled but reshape elimination disabled. "
287 "Attention + quant (fp8) fusion might not work"
288 )
204 - if self.enable_fi_allreduce_fusion:
289 + if self.fuse_allreduce_rms:
290 logger.warning_once(
291 "Fusion enabled but reshape elimination disabled. "
292 "Allreduce + rms norm + quant (fp8) fusion might not work"
@@ -873,7 +958,7 @@ class CompilationConfig:
958 self.set_splitting_ops_for_inductor_graph_partition()
959 return
960
876 - if self.pass_config.enable_attn_fusion:
961 + if self.pass_config.fuse_attn_quant:
962 # here use_inductor_graph_partition is False
963 self.set_splitting_ops_for_attn_fusion()
964 return
@@ -915,12 +1000,12 @@ class CompilationConfig:
1000 self.splitting_ops = list(self._attention_ops)
1001
1002 def set_splitting_ops_for_attn_fusion(self):
918 - assert self.pass_config.enable_attn_fusion
1003 + assert self.pass_config.fuse_attn_quant
1004 if self.splitting_ops is None:
1005 self.splitting_ops = []
1006 if self.cudagraph_mode.has_piecewise_cudagraphs():
1007 logger.warning_once(
923 - "enable_attn_fusion is incompatible with piecewise "
1008 + "fuse_attn_quant is incompatible with piecewise "
1009 "cudagraph when use_inductor_graph_partition is off. "
1010 "In this case, splitting_ops will be set to empty "
1011 "list, and cudagraph_mode will be set to FULL. "
@@ -931,8 +1016,7 @@ class CompilationConfig:
1016 self.cudagraph_mode = CUDAGraphMode.FULL
1017
1018 assert not self.splitting_ops_contain_attention(), (
934 - "attention ops should not be in splitting_ops "
935 - "when enable_attn_fusion is True"
1019 + "attention ops should not be in splitting_ops when fuse_attn_quant is True"
1020 )
1021
1022 def splitting_ops_contain_attention(self) -> bool:
@@ -1008,7 +1092,7 @@ class CompilationConfig:
1092 self, uniform_decode_query_len: int, tensor_parallel_size: int
1093 ):
1094 multiple_of = uniform_decode_query_len
1011 - if tensor_parallel_size > 1 and self.pass_config.enable_sequence_parallelism:
1095 + if tensor_parallel_size > 1 and self.pass_config.enable_sp:
1096 multiple_of = max(uniform_decode_query_len, tensor_parallel_size)
1097 if (
1098 multiple_of % uniform_decode_query_len != 0
vllm/config/utils.py
+29
@@ -19,6 +19,10 @@ import torch
19 from pydantic.fields import FieldInfo
20 from typing_extensions import runtime_checkable
21
22 +from vllm.logger import init_logger
23 +
24 +logger = init_logger(__name__)
25 +
26 if TYPE_CHECKING:
27 from _typeshed import DataclassInstance
28 else:
@@ -293,3 +297,28 @@ def get_hash_factors(config: ConfigT, ignored_factors: set[str]) -> dict[str, ob
297 def hash_factors(items: dict[str, object]) -> str:
298 """Return a SHA-256 hex digest of the canonical items structure."""
299 return hashlib.sha256(json.dumps(items, sort_keys=True).encode()).hexdigest()
300 +
301 +
302 +def handle_deprecated(
303 + config: ConfigT,
304 + old_name: str,
305 + new_name_or_names: str | list[str],
306 + removal_version: str,
307 +) -> None:
308 + old_val = getattr(config, old_name)
309 + if old_val is None:
310 + return
311 +
312 + if isinstance(new_name_or_names, str):
313 + new_names = [new_name_or_names]
314 + else:
315 + new_names = new_name_or_names
316 +
317 + msg = (
318 + f"{old_name} is deprecated and will be removed in {removal_version}. "
319 + f"Use {', '.join(new_names)} instead."
320 + )
321 + logger.warning(msg)
322 +
323 + for new_name in new_names:
324 + setattr(config, new_name, old_val)
vllm/config/vllm.py
+45 -31
@@ -83,22 +83,33 @@ IS_DENSE = False
83 # See https://github.com/vllm-project/vllm/issues/25689.
84
85
86 -def enable_fusion(cfg: "VllmConfig") -> bool:
87 - """Returns True if RMS norm or quant FP8 is enabled."""
86 +def enable_norm_fusion(cfg: "VllmConfig") -> bool:
87 + """Enable if either RMS norm or quant FP8 custom op is active;
88 + otherwise Inductor handles fusion."""
89 +
90 return cfg.compilation_config.is_custom_op_enabled(
91 "rms_norm"
92 ) or cfg.compilation_config.is_custom_op_enabled("quant_fp8")
93
94
95 +def enable_act_fusion(cfg: "VllmConfig") -> bool:
96 + """Enable if either SiLU+Mul or quant FP8 custom op is active;
97 + otherwise Inductor handles fusion."""
98 + return cfg.compilation_config.is_custom_op_enabled(
99 + "silu_and_mul"
100 + ) or cfg.compilation_config.is_custom_op_enabled("quant_fp8")
101 +
102 +
103 OPTIMIZATION_LEVEL_00 = {
104 "compilation_config": {
105 "pass_config": {
96 - "enable_noop": False,
97 - "enable_fusion": False,
98 - "enable_fi_allreduce_fusion": False,
99 - "enable_attn_fusion": False,
100 - "enable_sequence_parallelism": False,
101 - "enable_async_tp": False,
106 + "eliminate_noops": False,
107 + "fuse_norm_quant": False,
108 + "fuse_act_quant": False,
109 + "fuse_allreduce_rms": False,
110 + "fuse_attn_quant": False,
111 + "enable_sp": False,
112 + "fuse_gemm_comms": False,
113 },
114 "cudagraph_mode": CUDAGraphMode.NONE,
115 "use_inductor_graph_partition": False,
@@ -107,12 +118,13 @@ OPTIMIZATION_LEVEL_00 = {
118 OPTIMIZATION_LEVEL_01 = {
119 "compilation_config": {
120 "pass_config": {
110 - "enable_noop": True,
111 - "enable_fusion": enable_fusion,
112 - "enable_fi_allreduce_fusion": False,
113 - "enable_attn_fusion": False,
114 - "enable_sequence_parallelism": False,
115 - "enable_async_tp": False,
121 + "eliminate_noops": True,
122 + "fuse_norm_quant": enable_norm_fusion,
123 + "fuse_act_quant": enable_act_fusion,
124 + "fuse_allreduce_rms": False,
125 + "fuse_attn_quant": False,
126 + "enable_sp": False,
127 + "fuse_gemm_comms": False,
128 },
129 "cudagraph_mode": CUDAGraphMode.PIECEWISE,
130 "use_inductor_graph_partition": False,
@@ -121,12 +133,13 @@ OPTIMIZATION_LEVEL_01 = {
133 OPTIMIZATION_LEVEL_02 = {
134 "compilation_config": {
135 "pass_config": {
124 - "enable_noop": True,
125 - "enable_fusion": enable_fusion,
126 - "enable_fi_allreduce_fusion": False,
127 - "enable_attn_fusion": IS_QUANTIZED,
128 - "enable_sequence_parallelism": IS_DENSE,
129 - "enable_async_tp": IS_DENSE,
136 + "eliminate_noops": True,
137 + "fuse_norm_quant": enable_norm_fusion,
138 + "fuse_act_quant": enable_act_fusion,
139 + "fuse_allreduce_rms": False,
140 + "fuse_attn_quant": IS_QUANTIZED,
141 + "enable_sp": IS_DENSE,
142 + "fuse_gemm_comms": IS_DENSE,
143 },
144 "cudagraph_mode": CUDAGraphMode.FULL_AND_PIECEWISE,
145 "use_inductor_graph_partition": False,
@@ -135,12 +148,13 @@ OPTIMIZATION_LEVEL_02 = {
148 OPTIMIZATION_LEVEL_03 = {
149 "compilation_config": {
150 "pass_config": {
138 - "enable_noop": True,
139 - "enable_fusion": enable_fusion,
140 - "enable_fi_allreduce_fusion": False,
141 - "enable_attn_fusion": IS_QUANTIZED,
142 - "enable_sequence_parallelism": IS_DENSE,
143 - "enable_async_tp": IS_DENSE,
151 + "eliminate_noops": True,
152 + "fuse_norm_quant": enable_norm_fusion,
153 + "fuse_act_quant": enable_act_fusion,
154 + "fuse_allreduce_rms": False,
155 + "fuse_attn_quant": IS_QUANTIZED,
156 + "enable_sp": IS_DENSE,
157 + "fuse_gemm_comms": IS_DENSE,
158 },
159 "cudagraph_mode": CUDAGraphMode.FULL_AND_PIECEWISE,
160 "use_inductor_graph_partition": False,
@@ -645,9 +659,9 @@ class VllmConfig:
659
660 # async tp is built on top of sequence parallelism
661 # and requires it to be enabled.
648 - if self.compilation_config.pass_config.enable_async_tp:
649 - self.compilation_config.pass_config.enable_sequence_parallelism = True
650 - if self.compilation_config.pass_config.enable_sequence_parallelism:
662 + if self.compilation_config.pass_config.fuse_gemm_comms:
663 + self.compilation_config.pass_config.enable_sp = True
664 + if self.compilation_config.pass_config.enable_sp:
665 if "-rms_norm" in self.compilation_config.custom_ops:
666 logger.warning(
667 "RMS norm force disabled, sequence parallelism might break"
@@ -797,7 +811,7 @@ class VllmConfig:
811 # Do this after all the updates to compilation_config.mode
812 self.compilation_config.set_splitting_ops_for_v1()
813
800 - if self.compilation_config.pass_config.enable_sequence_parallelism:
814 + if self.compilation_config.pass_config.enable_sp:
815 # With pipeline parallelism or dynamo partitioning,
816 # native rms norm tracing errors due to incorrect residual shape.
817 # Use custom rms norm to unblock. In the future,
@@ -1062,7 +1076,7 @@ class VllmConfig:
1076
1077 if (
1078 self.parallel_config.tensor_parallel_size > 1
1065 - and self.compilation_config.pass_config.enable_sequence_parallelism
1079 + and self.compilation_config.pass_config.enable_sp
1080 ):
1081 cudagraph_capture_sizes = self.update_sizes_for_sequence_parallelism(
1082 cudagraph_capture_sizes
vllm/v1/worker/gpu_model_runner.py
+1 -4
@@ -2417,10 +2417,7 @@ class GPUModelRunner(
2417 # Pad tokens to multiple of tensor_parallel_size when
2418 # enabled collective fusion for SP
2419 tp_size = self.vllm_config.parallel_config.tensor_parallel_size
2420 - if (
2421 - self.compilation_config.pass_config.enable_sequence_parallelism
2422 - and tp_size > 1
2423 - ):
2420 + if self.compilation_config.pass_config.enable_sp and tp_size > 1:
2421 return round_up(num_scheduled_tokens, tp_size)
2422 return num_scheduled_tokens
2423
vllm/v1/worker/gpu_worker.py
+1 -1
@@ -552,7 +552,7 @@ class Worker(WorkerBase):
552
553 if (
554 parallel_config.pipeline_parallel_size > 1
555 - and compilation_config.pass_config.enable_sequence_parallelism
555 + and compilation_config.pass_config.enable_sp
556 and forward_pass
557 ):
558 # currently only supported by V1 GPUModelRunner
vllm/v1/worker/utils.py
+1 -1
@@ -342,7 +342,7 @@ def is_residual_scattered_for_sp(
342 partition), SP is always applied
343 - Otherwise, SP is only applied for specific shapes in compile_sizes
344 """
345 - if not vllm_config.compilation_config.pass_config.enable_sequence_parallelism:
345 + if not vllm_config.compilation_config.pass_config.enable_sp:
346 return False
347
348 tp = vllm_config.parallel_config.tensor_parallel_size