@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 0838b52e2e
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 12, 12:15 UTC

[Frontend][torch.compile] CompilationConfig Overhaul (#20283): Set up -O infrastructure (#26847)

Signed-off-by: morrison-turnansky <mturnans@redhat.com> Signed-off-by: adabeyta <aabeyta@redhat.com> Signed-off-by: Morrison Turnansky <mturnans@redhat.com> Co-authored-by: adabeyta <aabeyta@redhat.com> Co-authored-by: Luka Govedič <ProExpertProg@users.noreply.github.com> Co-authored-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>

Morrison Turnansky committed Nov 27, 2025 at 04:55 UTC 0838b52e2eff77d1aaf4ee9d0da19522b9a5749c
13 files changed +735 -64
docs/design/optimization_levels.md new
+69
@@ -0,0 +1,69 @@
1 +<!-- markdownlint-disable -->
2 +
3 +# Optimization Levels
4 +
5 +## Overview
6 +
7 +vLLM now supports optimization levels (`-O0`, `-O1`, `-O2`, `-O3`). Optimization levels provide an intuitive mechnaism for users to trade startup time for performance. Higher levels have better performance but worse startup time. These optimization levels have associated defaults to help users get desired out of the box performance. Importantly, defaults set by optimization levels are purely defaults; explicit user settings will not be overwritten.
8 +
9 +## Level Summaries and Usage Examples
10 +```bash
11 +# CLI usage
12 +python -m vllm.entrypoints.api_server --model RedHatAI/Llama-3.2-1B-FP8 -O0
13 +
14 +# Python API usage
15 +from vllm.entrypoints.llm import LLM
16 +
17 +llm = LLM(
18 + model="RedHatAI/Llama-3.2-1B-FP8",
19 + optimization_level=0
20 +)
21 +```
22 +
23 +#### `-O1`: Quick Optimizations
24 +- **Startup**: Moderate startup time
25 +- **Performance**: Inductor compilation, CUDAGraphMode.PIECEWISE
26 +- **Use case**: Balance for most development scenarios
27 +
28 +```bash
29 +# CLI usage
30 +python -m vllm.entrypoints.api_server --model RedHatAI/Llama-3.2-1B-FP8 -O1
31 +
32 +# Python API usage
33 +from vllm.entrypoints.llm import LLM
34 +
35 +llm = LLM(
36 + model="RedHatAI/Llama-3.2-1B-FP8",
37 + optimization_level=1
38 +)
39 +```
40 +
41 +#### `-O2`: Full Optimizations (Default)
42 +- **Startup**: Longer startup time
43 +- **Performance**: `-O1` + CUDAGraphMode.FULL_AND_PIECEWISE
44 +- **Use case**: Production workloads where performance is important. This is the default use case. It is also very similar to the previous default. The primary difference is that noop & fusion flags are enabled.
45 +
46 +```bash
47 +# CLI usage (default, so optional)
48 +python -m vllm.entrypoints.api_server --model RedHatAI/Llama-3.2-1B-FP8 -O2
49 +
50 +# Python API usage
51 +from vllm.entrypoints.llm import LLM
52 +
53 +llm = LLM(
54 + model="RedHatAI/Llama-3.2-1B-FP8",
55 + optimization_level=2 # This is the default
56 +)
57 +```
58 +
59 +#### `-O3`: Full Optimization
60 +Still in development. Added infrastructure to prevent changing API in future
61 +release. Currently behaves the same O2.
62 +
63 +## Troubleshooting
64 +
65 +### Common Issues
66 +
67 +1. **Startup Time Too Long**: Use `-O0` or `-O1` for faster startup
68 +2. **Compilation Errors**: Use `debug_dump_path` for additional debugging information
69 +3. **Performance Issues**: Ensure using `-O2` for production
\ No newline at end of file
tests/compile/test_config.py
+2 -2
@@ -172,8 +172,8 @@ def test_splitting_ops_dynamic():
172 config = VllmConfig()
173 # Default V1 config leaves cudagraph mode unset; splitting ops are only
174 # populated when the engine decides to use piecewise compilation.
175 - assert config.compilation_config.cudagraph_mode == CUDAGraphMode.NONE
176 - assert not config.compilation_config.splitting_ops_contain_attention()
175 + assert config.compilation_config.cudagraph_mode == CUDAGraphMode.FULL_AND_PIECEWISE
176 + assert config.compilation_config.splitting_ops_contain_attention()
177
178 # When use_inductor_graph_partition=True
179 config = VllmConfig(
tests/engine/test_arg_utils.py
+41 -16
@@ -222,28 +222,53 @@ def test_media_io_kwargs_parser(arg, expected):
222 assert args.media_io_kwargs == expected
223
224
225 -def test_compilation_config():
225 +@pytest.mark.parametrize(
226 + ("args", "expected"),
227 + [
228 + (["-O", "1"], "1"),
229 + (["-O", "2"], "2"),
230 + (["-O", "3"], "3"),
231 + (["-O0"], "0"),
232 + (["-O1"], "1"),
233 + (["-O2"], "2"),
234 + (["-O3"], "3"),
235 + ],
236 +)
237 +def test_optimization_level(args, expected):
238 + """
239 + Test space-separated optimization levels (-O 1, -O 2, -O 3) map to
240 + optimization_level.
241 + """
242 parser = EngineArgs.add_cli_args(FlexibleArgumentParser())
243 + parsed_args = parser.parse_args(args)
244 + assert parsed_args.optimization_level == expected
245 + assert parsed_args.compilation_config.mode is None
246
228 - # default value
229 - args = parser.parse_args([])
230 - assert args.compilation_config == CompilationConfig()
247
232 - # set to O3
233 - args = parser.parse_args(["-O0"])
234 - assert args.compilation_config.mode == 0
248 +@pytest.mark.parametrize(
249 + ("args", "expected"),
250 + [
251 + (["-O.mode=0"], 0),
252 + (["-O.mode=1"], 1),
253 + (["-O.mode=2"], 2),
254 + (["-O.mode=3"], 3),
255 + ],
256 +)
257 +def test_mode_parser(args, expected):
258 + """
259 + Test compilation config modes (-O.mode=int) map to compilation_config.
260 + """
261 + parser = EngineArgs.add_cli_args(FlexibleArgumentParser())
262 + parsed_args = parser.parse_args(args)
263 + assert parsed_args.compilation_config.mode == expected
264
236 - # set to O 3 (space)
237 - args = parser.parse_args(["-O", "1"])
238 - assert args.compilation_config.mode == 1
265
240 - # set to O 3 (equals)
241 - args = parser.parse_args(["-O=2"])
242 - assert args.compilation_config.mode == 2
266 +def test_compilation_config():
267 + parser = EngineArgs.add_cli_args(FlexibleArgumentParser())
268
244 - # set to O.mode 3
245 - args = parser.parse_args(["-O.mode", "3"])
246 - assert args.compilation_config.mode == 3
269 + # default value
270 + args = parser.parse_args([])
271 + assert args.compilation_config == CompilationConfig()
272
273 # set to string form of a dict
274 args = parser.parse_args(
tests/model_executor/test_enabled_custom_ops.py
+7 -1
@@ -5,7 +5,12 @@ import pytest
5 import torch
6
7 from vllm._aiter_ops import rocm_aiter_ops
8 -from vllm.config import CompilationConfig, VllmConfig, set_current_vllm_config
8 +from vllm.config import (
9 + CompilationConfig,
10 + VllmConfig,
11 + get_cached_compilation_config,
12 + set_current_vllm_config,
13 +)
14 from vllm.model_executor.custom_op import CustomOp
15 from vllm.model_executor.layers.activation import (
16 GeluAndMul,
@@ -86,6 +91,7 @@ def test_enabled_ops(
91 backend=backend, mode=compilation_mode, custom_ops=custom_ops
92 )
93 )
94 + get_cached_compilation_config.cache_clear()
95 with set_current_vllm_config(vllm_config):
96 assert CustomOp.default_on() == default_on
97
tests/test_config.py
+306 -1
@@ -8,9 +8,20 @@ from unittest.mock import patch
8 import pytest
9
10 from vllm.compilation.backends import VllmBackend
11 -from vllm.config import ModelConfig, PoolerConfig, VllmConfig, update_config
11 +from vllm.config import (
12 + CompilationConfig,
13 + ModelConfig,
14 + PoolerConfig,
15 + VllmConfig,
16 + update_config,
17 +)
18 +from vllm.config.compilation import CompilationMode, CUDAGraphMode
19 from vllm.config.load import LoadConfig
20 from vllm.config.utils import get_field
21 +from vllm.config.vllm import (
22 + OPTIMIZATION_LEVEL_TO_CONFIG,
23 + OptimizationLevel,
24 +)
25 from vllm.model_executor.layers.pooler import PoolingType
26 from vllm.platforms import current_platform
27
@@ -235,6 +246,43 @@ def test_default_pooling_type(model_id, default_pooling_type, pooling_type):
246 assert model_config.pooler_config.pooling_type == pooling_type
247
248
249 +@pytest.mark.parametrize(
250 + ("model_id", "expected_is_moe_model"),
251 + [
252 + ("RedHatAI/Qwen3-8B-speculator.eagle3", False),
253 + ("RedHatAI/Llama-3.1-8B-Instruct-NVFP4", False),
254 + ("RedHatAI/Llama-3.2-1B-FP8", False),
255 + ("RedHatAI/Mistral-Small-24B-Instruct-2501-quantized.w8a8", False),
256 + ("RedHatAI/gpt-oss-20b", True),
257 + ("RedHatAI/DeepSeek-V2.5-1210-FP8", True),
258 + ("RedHatAI/Llama-4-Scout-17B-16E-Instruct", True),
259 + ("RedHatAI/Mixtral-8x7B-Instruct-v0.1", True),
260 + ],
261 +)
262 +def test_moe_model_detection(model_id, expected_is_moe_model):
263 + model_config = ModelConfig(model_id)
264 + # Just check that is_moe_model field exists and is a boolean
265 + assert model_config.is_model_moe() == expected_is_moe_model
266 +
267 +
268 +@pytest.mark.parametrize(
269 + ("model_id", "quantized"),
270 + [
271 + ("RedHatAI/Qwen3-8B-speculator.eagle3", False),
272 + ("RedHatAI/Llama-3.1-8B-Instruct-NVFP4", True),
273 + ("RedHatAI/Llama-3.2-1B-FP8", True),
274 + ("RedHatAI/Mistral-Small-24B-Instruct-2501-quantized.w8a8", True),
275 + ("RedHatAI/gpt-oss-20b", True),
276 + ("RedHatAI/DeepSeek-V2.5-1210-FP8", True),
277 + ("RedHatAI/Mixtral-8x7B-Instruct-v0.1", False),
278 + ],
279 +)
280 +def test_is_quantized(model_id, quantized):
281 + model_config = ModelConfig(model_id)
282 + # Just check that quantized field exists and is a boolean
283 + assert model_config.is_quantized() == quantized
284 +
285 +
286 @pytest.mark.skipif(
287 current_platform.is_rocm(), reason="Xformers backend is not supported on ROCm."
288 )
@@ -552,3 +600,260 @@ def test_s3_url_different_models_create_different_directories(mock_pull_files):
600 assert os.path.exists(config1.tokenizer) and os.path.isdir(config1.tokenizer)
601 assert os.path.exists(config2.model) and os.path.isdir(config2.model)
602 assert os.path.exists(config2.tokenizer) and os.path.isdir(config2.tokenizer)
603 +
604 +
605 +@pytest.mark.parametrize(
606 + ("backend", "custom_ops", "expected"),
607 + [
608 + ("eager", [], True),
609 + ("eager", ["+fused_layernorm"], True),
610 + ("eager", ["all", "-fused_layernorm"], False),
611 + ("inductor", [], False),
612 + ("inductor", ["none", "+fused_layernorm"], True),
613 + ("inductor", ["none", "-fused_layernorm"], False),
614 + ],
615 +)
616 +def test_is_custom_op_enabled(backend: str, custom_ops: list[str], expected: bool):
617 + """Test that is_custom_op_enabled works correctly."""
618 + config = VllmConfig(
619 + compilation_config=CompilationConfig(backend=backend, custom_ops=custom_ops)
620 + )
621 + assert config.compilation_config.is_custom_op_enabled("fused_layernorm") is expected
622 +
623 +
624 +def test_vllm_config_defaults_are_none():
625 + """Verify that optimization-level defaults are None when not set by user."""
626 + # Test all optimization levels to ensure defaults work correctly
627 + for opt_level in OptimizationLevel:
628 + config = object.__new__(VllmConfig)
629 + config.compilation_config = CompilationConfig()
630 + config.optimization_level = opt_level
631 + config.model_config = None
632 +
633 + # Use the global optimization level defaults
634 + default_config = OPTIMIZATION_LEVEL_TO_CONFIG[opt_level]
635 +
636 + # Verify that all pass_config values are None before defaults are applied
637 + for pass_k in default_config["compilation_config"]["pass_config"]:
638 + assert getattr(config.compilation_config.pass_config, pass_k) is None
639 +
640 + # Verify that other config values are None before defaults are applied
641 + for k in default_config["compilation_config"]:
642 + if k != "pass_config":
643 + assert getattr(config.compilation_config, k) is None
644 +
645 +
646 +@pytest.mark.parametrize(
647 + ("model_id", "compiliation_config", "optimization_level"),
648 + [
649 + (
650 + None,
651 + CompilationConfig(backend="eager", custom_ops=["+quant_fp8"]),
652 + OptimizationLevel.O0,
653 + ),
654 + (None, CompilationConfig(), OptimizationLevel.O0),
655 + (None, CompilationConfig(), OptimizationLevel.O1),
656 + (None, CompilationConfig(), OptimizationLevel.O2),
657 + (None, CompilationConfig(), OptimizationLevel.O3),
658 + (
659 + "RedHatAI/Qwen3-8B-speculator.eagle3",
660 + CompilationConfig(backend="inductor", custom_ops=["+quant_fp8"]),
661 + OptimizationLevel.O2,
662 + ),
663 + (
664 + "RedHatAI/Qwen3-8B-speculator.eagle3",
665 + CompilationConfig(),
666 + OptimizationLevel.O0,
667 + ),
668 + (
669 + "RedHatAI/Qwen3-8B-speculator.eagle3",
670 + CompilationConfig(),
671 + OptimizationLevel.O1,
672 + ),
673 + (
674 + "RedHatAI/Qwen3-8B-speculator.eagle3",
675 + CompilationConfig(),
676 + OptimizationLevel.O2,
677 + ),
678 + (
679 + "RedHatAI/Qwen3-8B-speculator.eagle3",
680 + CompilationConfig(),
681 + OptimizationLevel.O3,
682 + ),
683 + ("RedHatAI/DeepSeek-V2.5-1210-FP8", CompilationConfig(), OptimizationLevel.O0),
684 + ("RedHatAI/DeepSeek-V2.5-1210-FP8", CompilationConfig(), OptimizationLevel.O1),
685 + ("RedHatAI/DeepSeek-V2.5-1210-FP8", CompilationConfig(), OptimizationLevel.O2),
686 + ("RedHatAI/DeepSeek-V2.5-1210-FP8", CompilationConfig(), OptimizationLevel.O3),
687 + ],
688 +)
689 +def test_vllm_config_defaults(model_id, compiliation_config, optimization_level):
690 + """Test that optimization-level defaults are correctly applied."""
691 +
692 + model_config = None
693 + if model_id is not None:
694 + model_config = ModelConfig(model_id)
695 + vllm_config = VllmConfig(
696 + model_config=model_config,
697 + compilation_config=compiliation_config,
698 + optimization_level=optimization_level,
699 + )
700 + else:
701 + vllm_config = VllmConfig(
702 + compilation_config=compiliation_config,
703 + optimization_level=optimization_level,
704 + )
705 + # Use the global optimization level defaults
706 + default_config = OPTIMIZATION_LEVEL_TO_CONFIG[optimization_level]
707 +
708 + # Verify pass_config defaults (nested under compilation_config)
709 + pass_config_dict = default_config["compilation_config"]["pass_config"]
710 + for pass_k, pass_v in pass_config_dict.items():
711 + actual = getattr(vllm_config.compilation_config.pass_config, pass_k)
712 + expected = pass_v(vllm_config) if callable(pass_v) else pass_v
713 + assert actual == expected, (
714 + f"pass_config.{pass_k}: expected {expected}, got {actual}"
715 + )
716 +
717 + # Verify other compilation_config defaults
718 + compilation_config_dict = default_config["compilation_config"]
719 + for k, v in compilation_config_dict.items():
720 + if k != "pass_config":
721 + actual = getattr(vllm_config.compilation_config, k)
722 + expected = v(vllm_config) if callable(v) else v
723 + assert actual == expected, (
724 + f"compilation_config.{k}: expected {expected}, got {actual}"
725 + )
726 +
727 +
728 +def test_vllm_config_callable_defaults():
729 + """Test that callable defaults work in the config system.
730 +
731 + Verifies that lambdas in default configs can inspect VllmConfig properties
732 + (e.g., is_quantized, is_model_moe) to conditionally set optimization flags.
733 + """
734 + config_no_model = VllmConfig(optimization_level=OptimizationLevel.O2)
735 +
736 + # Callable that checks if model exists
737 + has_model = lambda cfg: cfg.model_config is not None
738 + assert has_model(config_no_model) is False
739 +
740 + # Test with quantized model
741 + quantized_model = ModelConfig("RedHatAI/Llama-3.2-1B-FP8")
742 + config_quantized = VllmConfig(
743 + model_config=quantized_model, optimization_level=OptimizationLevel.O2
744 + )
745 + enable_if_quantized = lambda cfg: (
746 + cfg.model_config is not None and cfg.model_config.is_quantized()
747 + )
748 + assert enable_if_quantized(config_quantized) is True
749 + assert enable_if_quantized(config_no_model) is False
750 +
751 + # Test with MoE model
752 + moe_model = ModelConfig("deepseek-ai/DeepSeek-V2-Lite")
753 + config_moe = VllmConfig(
754 + model_config=moe_model, optimization_level=OptimizationLevel.O2
755 + )
756 + enable_if_sequential = lambda cfg: (
757 + cfg.model_config is not None and not cfg.model_config.is_model_moe()
758 + )
759 + assert enable_if_sequential(config_moe) is False
760 + assert enable_if_sequential(config_quantized) is True
761 +
762 +
763 +def test_vllm_config_explicit_overrides():
764 + """Test that explicit property overrides work correctly with callable defaults.
765 +
766 + When users explicitly set configuration properties, those values
767 + take precedence over callable defaults, across different models and
768 + optimization levels.
769 + """
770 + from vllm.config.compilation import PassConfig
771 +
772 + quantized_model = ModelConfig("RedHatAI/Llama-3.2-1B-FP8")
773 + moe_model = ModelConfig("deepseek-ai/DeepSeek-V2-Lite")
774 + regular_model = ModelConfig("Qwen/Qwen1.5-7B")
775 +
776 + # Explicit compilation mode override on O0 (where default is NONE)
777 + compilation_config = CompilationConfig(mode=CompilationMode.VLLM_COMPILE)
778 + config = VllmConfig(
779 + optimization_level=OptimizationLevel.O0,
780 + compilation_config=compilation_config,
781 + )
782 + assert config.compilation_config.mode == CompilationMode.VLLM_COMPILE
783 + assert config.compilation_config.cudagraph_mode == CUDAGraphMode.NONE
784 +
785 + # Explicit pass config flags to override defaults
786 + pass_config = PassConfig(enable_noop=True, enable_attn_fusion=True)
787 + compilation_config = CompilationConfig(pass_config=pass_config)
788 + config = VllmConfig(
789 + optimization_level=OptimizationLevel.O0,
790 + compilation_config=compilation_config,
791 + )
792 + assert config.compilation_config.pass_config.enable_noop is True
793 + assert config.compilation_config.pass_config.enable_attn_fusion is True
794 +
795 + # Explicit cudagraph mode override on quantized model at O2
796 + pass_config = PassConfig(enable_async_tp=True)
797 + compilation_config = CompilationConfig(
798 + cudagraph_mode=CUDAGraphMode.NONE, pass_config=pass_config
799 + )
800 + config = VllmConfig(
801 + model_config=quantized_model,
802 + optimization_level=OptimizationLevel.O2,
803 + compilation_config=compilation_config,
804 + )
805 + assert config.compilation_config.cudagraph_mode == CUDAGraphMode.NONE
806 + assert config.compilation_config.pass_config.enable_async_tp is True
807 + # Mode should still use default for O2
808 + assert config.compilation_config.mode == CompilationMode.VLLM_COMPILE
809 +
810 + # Different optimization levels with same model
811 + config_o0 = VllmConfig(
812 + model_config=regular_model, optimization_level=OptimizationLevel.O0
813 + )
814 + config_o2 = VllmConfig(
815 + model_config=regular_model, optimization_level=OptimizationLevel.O2
816 + )
817 + assert config_o0.compilation_config.mode == CompilationMode.NONE
818 + assert config_o2.compilation_config.mode == CompilationMode.VLLM_COMPILE
819 + assert config_o0.compilation_config.cudagraph_mode == CUDAGraphMode.NONE
820 + assert (
821 + config_o2.compilation_config.cudagraph_mode == CUDAGraphMode.FULL_AND_PIECEWISE
822 + )
823 +
824 + # Same optimization level across different model types
825 + config_moe_o2 = VllmConfig(
826 + model_config=moe_model, optimization_level=OptimizationLevel.O2
827 + )
828 + config_regular_o2 = VllmConfig(
829 + model_config=regular_model, optimization_level=OptimizationLevel.O2
830 + )
831 + config_quantized_o2 = VllmConfig(
832 + model_config=quantized_model, optimization_level=OptimizationLevel.O2
833 + )
834 + # All should have same base compilation settings at O2
835 + assert config_moe_o2.compilation_config.mode == CompilationMode.VLLM_COMPILE
836 + assert config_regular_o2.compilation_config.mode == CompilationMode.VLLM_COMPILE
837 + assert config_quantized_o2.compilation_config.mode == CompilationMode.VLLM_COMPILE
838 + assert (
839 + config_moe_o2.compilation_config.cudagraph_mode
840 + == CUDAGraphMode.FULL_AND_PIECEWISE
841 + )
842 + assert (
843 + config_regular_o2.compilation_config.cudagraph_mode
844 + == CUDAGraphMode.FULL_AND_PIECEWISE
845 + )
846 +
847 + # Override one field but not others
848 + pass_config = PassConfig(enable_noop=False)
849 + compilation_config = CompilationConfig(pass_config=pass_config)
850 + config = VllmConfig(
851 + model_config=regular_model,
852 + optimization_level=OptimizationLevel.O2,
853 + compilation_config=compilation_config,
854 + )
855 + # Explicit override should be respected
856 + assert config.compilation_config.pass_config.enable_noop is False
857 + # Other fields should still use defaults
858 + assert config.compilation_config.mode == CompilationMode.VLLM_COMPILE
859 + assert config.compilation_config.cudagraph_mode == CUDAGraphMode.FULL_AND_PIECEWISE
tests/utils_/test_argparse_utils.py
+6 -4
@@ -28,6 +28,7 @@ def parser():
28 parser.add_argument("--enable-feature", action="store_true")
29 parser.add_argument("--hf-overrides", type=json.loads)
30 parser.add_argument("-O", "--compilation-config", type=json.loads)
31 + parser.add_argument("--optimization-level", type=int)
32 return parser
33
34
@@ -217,8 +218,8 @@ def test_dict_args(parser):
218 "key15": "-minus.and.dot",
219 },
220 }
221 + assert parsed_args.optimization_level == 1
222 assert parsed_args.compilation_config == {
221 - "mode": 1,
223 "use_inductor_graph_partition": True,
224 "backend": "custom",
225 "custom_ops": ["-quant_fp8", "+silu_mul", "-rms_norm"],
@@ -241,12 +242,13 @@ def test_duplicate_dict_args(caplog_vllm, parser):
242 parsed_args = parser.parse_args(args)
243 # Should be the last value
244 assert parsed_args.hf_overrides == {"key1": "val2"}
244 - assert parsed_args.compilation_config == {"mode": 3}
245 + assert parsed_args.optimization_level == 3
246 + assert parsed_args.compilation_config == {"mode": 2}
247
248 assert len(caplog_vllm.records) == 1
249 assert "duplicate" in caplog_vllm.text
250 assert "--hf-overrides.key1" in caplog_vllm.text
249 - assert "-O.mode" in caplog_vllm.text
251 + assert "--optimization-level" in caplog_vllm.text
252
253
254 def test_model_specification(
@@ -383,7 +385,7 @@ def test_compilation_mode_string_values(parser):
385 assert args.compilation_config == {"mode": 0}
386
387 args = parser.parse_args(["-O3"])
386 - assert args.compilation_config == {"mode": 3}
388 + assert args.optimization_level == 3
389
390 args = parser.parse_args(["-O.mode=NONE"])
391 assert args.compilation_config == {"mode": "NONE"}
tests/v1/cudagraph/test_cudagraph_mode.py
+2 -2
@@ -117,9 +117,9 @@ else:
117 combo_cases_2 = [
118 ("FA2", "FULL", CompilationMode.NONE, True),
119 ("FA2", "FULL", CompilationMode.VLLM_COMPILE, True),
120 - ("FA2", "PIECEWISE", CompilationMode.NONE, False),
120 + ("FA2", "PIECEWISE", CompilationMode.NONE, True),
121 ("FA2", "PIECEWISE", CompilationMode.VLLM_COMPILE, True),
122 - ("FA2", "FULL_AND_PIECEWISE", CompilationMode.NONE, False),
122 + ("FA2", "FULL_AND_PIECEWISE", CompilationMode.NONE, True),
123 ("FA2", "FULL_AND_PIECEWISE", CompilationMode.VLLM_COMPILE, True),
124 ("FA2", "FULL_DECODE_ONLY", CompilationMode.NONE, True),
125 ("FA2", "FULL_DECODE_ONLY", CompilationMode.VLLM_COMPILE, True),
vllm/config/compilation.py
+62 -13
@@ -8,7 +8,7 @@ from dataclasses import asdict, field
8 from pathlib import Path
9 from typing import TYPE_CHECKING, Any, ClassVar, Literal
10
11 -from pydantic import TypeAdapter, field_validator
11 +from pydantic import Field, TypeAdapter, field_validator
12 from pydantic.dataclasses import dataclass
13
14 import vllm.envs as envs
@@ -97,19 +97,25 @@ class PassConfig:
97
98 This is separate from general `CompilationConfig` so that inductor passes
99 don't all have access to full configuration - that would create a cycle as
100 - the `PassManager` is set as a property of config."""
100 + the `PassManager` is set as a property of config.
101
102 - enable_fusion: bool = False
102 + You must pass PassConfig to VLLMConfig constructor via the CompilationConfig
103 + constructor. VLLMConfig's post_init does further initialization.
104 + If used outside of the VLLMConfig, some fields may be left in an
105 + improper state.
106 + """
107 +
108 + enable_fusion: bool = Field(default=None)
109 """Whether to enable the custom fusion (RMSNorm/SiluMul+quant) pass."""
104 - enable_attn_fusion: bool = False
110 + enable_attn_fusion: bool = Field(default=None)
111 """Whether to enable the custom attention+quant fusion pass."""
106 - enable_noop: bool = False
112 + enable_noop: bool = Field(default=None)
113 """Whether to enable the custom no-op elimination pass."""
108 - enable_sequence_parallelism: bool = False
114 + enable_sequence_parallelism: bool = Field(default=None)
115 """Whether to enable sequence parallelism."""
110 - enable_async_tp: bool = False
116 + enable_async_tp: bool = Field(default=None)
117 """Whether to enable async TP."""
112 - enable_fi_allreduce_fusion: bool = False
118 + enable_fi_allreduce_fusion: bool = Field(default=None)
119 """Whether to enable flashinfer allreduce fusion."""
120 fi_allreduce_fusion_max_size_mb: float | None = None
121 """The threshold of the communicated tensor sizes under which
@@ -167,6 +173,22 @@ class PassConfig:
173 """
174 return InductorPass.hash_dict(asdict(self))
175
176 + @field_validator(
177 + "enable_fusion",
178 + "enable_attn_fusion",
179 + "enable_noop",
180 + "enable_sequence_parallelism",
181 + "enable_async_tp",
182 + "enable_fi_allreduce_fusion",
183 + mode="wrap",
184 + )
185 + @classmethod
186 + def _skip_none_validation(cls, value: Any, handler: Callable) -> Any:
187 + """Skip validation if the value is `None` when initialisation is delayed."""
188 + if value is None:
189 + return value
190 + return handler(value)
191 +
192 def __post_init__(self) -> None:
193 if not self.enable_noop:
194 if self.enable_fusion:
@@ -243,7 +265,13 @@ class DynamicShapesConfig:
265 @config
266 @dataclass
267 class CompilationConfig:
246 - """Configuration for compilation. It has three parts:
268 + """Configuration for compilation.
269 +
270 + You must pass CompilationConfig to VLLMConfig constructor.
271 + VLLMConfig's post_init does further initialization. If used outside of the
272 + VLLMConfig, some fields will be left in an improper state.
273 +
274 + It has three parts:
275
276 - Top-level Compilation control:
277 - [`mode`][vllm.config.CompilationConfig.mode]
@@ -282,14 +310,14 @@ class CompilationConfig:
310 """
311
312 # Top-level Compilation control
285 - level: int | None = None
313 + level: int = Field(default=None)
314 """
315 Level is deprecated and will be removed in the next release,
316 either 0.12.0 or 0.11.2 whichever is soonest.
317 Please use mode. Currently all levels are mapped to mode.
318 """
319 # Top-level Compilation control
292 - mode: CompilationMode | None = None
320 + mode: CompilationMode = Field(default=None)
321 """The compilation approach used for torch.compile-based compilation of the
322 model.
323
@@ -390,7 +418,7 @@ class CompilationConfig:
418 constructor, e.g. `CompilationConfig(inductor_passes={"a": func})`."""
419
420 # CudaGraph compilation
393 - cudagraph_mode: CUDAGraphMode | None = None
421 + cudagraph_mode: CUDAGraphMode = Field(default=None)
422 """
423 The mode of the cudagraph:
424
@@ -452,7 +480,7 @@ class CompilationConfig:
480 When `enable_lora` is False, this option has no effect.
481 """
482
455 - use_inductor_graph_partition: bool = False
483 + use_inductor_graph_partition: bool = Field(default=None)
484 """Use inductor graph partition to split the graph at cudagraph_unsafe ops.
485 This partition happens at inductor codegen time after all passes and fusions
486 are finished. It generates a single `call` function which wraps
@@ -648,6 +676,20 @@ class CompilationConfig:
676 )
677 return value
678
679 + @field_validator(
680 + "level",
681 + "mode",
682 + "cudagraph_mode",
683 + "use_inductor_graph_partition",
684 + mode="wrap",
685 + )
686 + @classmethod
687 + def _skip_none_validation(cls, value: Any, handler: Callable) -> Any:
688 + """Skip validation if the value is `None` when initialisation is delayed."""
689 + if value is None:
690 + return value
691 + return handler(value)
692 +
693 def __post_init__(self) -> None:
694 if self.level is not None:
695 logger.warning(
@@ -948,6 +990,13 @@ class CompilationConfig:
990 op,
991 )
992
993 + def is_custom_op_enabled(self, op: str) -> bool:
994 + if "all" in self.custom_ops:
995 + return f"-{op}" not in self.custom_ops
996 +
997 + assert "none" in self.custom_ops
998 + return f"+{op}" in self.custom_ops
999 +
1000 def adjust_cudagraph_sizes_for_spec_decode(
1001 self, uniform_decode_query_len: int, tensor_parallel_size: int
1002 ):
vllm/config/model.py
+8
@@ -1752,6 +1752,14 @@ class ModelConfig:
1752 logger.info("Using max model len %s", max_model_len)
1753 return max_model_len
1754
1755 + def is_model_moe(
1756 + self,
1757 + ) -> bool:
1758 + return self.get_num_experts() > 1
1759 +
1760 + def is_quantized(self) -> bool:
1761 + return getattr(self.hf_config, "quantization_config", None) is not None
1762 +
1763
1764 def get_served_model_name(model: str, served_model_name: str | list[str] | None):
1765 """
vllm/config/vllm.py
+205 -18
@@ -9,8 +9,9 @@ import tempfile
9 import threading
10 import time
11 from contextlib import contextmanager
12 -from dataclasses import replace
12 +from dataclasses import is_dataclass, replace
13 from datetime import datetime
14 +from enum import IntEnum
15 from functools import lru_cache
16 from pathlib import Path
17 from typing import TYPE_CHECKING, Any, TypeVar, get_args
@@ -57,6 +58,103 @@ else:
58 logger = init_logger(__name__)
59
60
61 +class OptimizationLevel(IntEnum):
62 + """Optimization level enum."""
63 +
64 + O0 = 0
65 + """O0 : No optimization. no compilation, no cudagraphs, no other
66 + optimization, just starting up immediately"""
67 + O1 = 1
68 + """O1: Quick optimizations. Dynamo+Inductor compilation and Piecewise
69 + cudagraphs"""
70 + O2 = 2
71 + """O2: Full optimizations. -O1 as well as Full and Piecewise cudagraphs."""
72 + O3 = 3
73 + """O3: Currently the same as -O2s."""
74 +
75 +
76 +IS_QUANTIZED = False
77 +IS_DENSE = False
78 +# The optimizations that depend on these properties currently set to False
79 +# in all cases.
80 +# if model_config is not None:
81 +# IS_QUANTIZED = lambda c: c.model_config.is_quantized()
82 +# IS_DENSE = lambda c: not c.model_config.is_model_moe()
83 +# See https://github.com/vllm-project/vllm/issues/25689.
84 +
85 +
86 +def enable_fusion(cfg: "VllmConfig") -> bool:
87 + """Returns True if RMS norm or quant FP8 is enabled."""
88 + return cfg.compilation_config.is_custom_op_enabled(
89 + "rms_norm"
90 + ) or cfg.compilation_config.is_custom_op_enabled("quant_fp8")
91 +
92 +
93 +OPTIMIZATION_LEVEL_00 = {
94 + "compilation_config": {
95 + "pass_config": {
96 + "enable_noop": False,
97 + "enable_fusion": False,
98 + "enable_fi_allreduce_fusion": False,
99 + "enable_attn_fusion": False,
100 + "enable_sequence_parallelism": False,
101 + "enable_async_tp": False,
102 + },
103 + "cudagraph_mode": CUDAGraphMode.NONE,
104 + "use_inductor_graph_partition": False,
105 + },
106 +}
107 +OPTIMIZATION_LEVEL_01 = {
108 + "compilation_config": {
109 + "pass_config": {
110 + "enable_noop": True,
111 + "enable_fusion": enable_fusion,
112 + "enable_fi_allreduce_fusion": False,
113 + "enable_attn_fusion": False,
114 + "enable_sequence_parallelism": False,
115 + "enable_async_tp": False,
116 + },
117 + "cudagraph_mode": CUDAGraphMode.PIECEWISE,
118 + "use_inductor_graph_partition": False,
119 + },
120 +}
121 +OPTIMIZATION_LEVEL_02 = {
122 + "compilation_config": {
123 + "pass_config": {
124 + "enable_noop": True,
125 + "enable_fusion": enable_fusion,
126 + "enable_fi_allreduce_fusion": False,
127 + "enable_attn_fusion": IS_QUANTIZED,
128 + "enable_sequence_parallelism": IS_DENSE,
129 + "enable_async_tp": IS_DENSE,
130 + },
131 + "cudagraph_mode": CUDAGraphMode.FULL_AND_PIECEWISE,
132 + "use_inductor_graph_partition": False,
133 + },
134 +}
135 +OPTIMIZATION_LEVEL_03 = {
136 + "compilation_config": {
137 + "pass_config": {
138 + "enable_noop": True,
139 + "enable_fusion": enable_fusion,
140 + "enable_fi_allreduce_fusion": False,
141 + "enable_attn_fusion": IS_QUANTIZED,
142 + "enable_sequence_parallelism": IS_DENSE,
143 + "enable_async_tp": IS_DENSE,
144 + },
145 + "cudagraph_mode": CUDAGraphMode.FULL_AND_PIECEWISE,
146 + "use_inductor_graph_partition": False,
147 + },
148 +}
149 +
150 +OPTIMIZATION_LEVEL_TO_CONFIG = {
151 + OptimizationLevel.O0: OPTIMIZATION_LEVEL_00,
152 + OptimizationLevel.O1: OPTIMIZATION_LEVEL_01,
153 + OptimizationLevel.O2: OPTIMIZATION_LEVEL_02,
154 + OptimizationLevel.O3: OPTIMIZATION_LEVEL_03,
155 +}
156 +
157 +
158 @config
159 @dataclass(config=ConfigDict(arbitrary_types_allowed=True))
160 class VllmConfig:
@@ -116,6 +214,11 @@ class VllmConfig:
214 you are using. Contents must be hashable."""
215 instance_id: str = ""
216 """The ID of the vLLM instance."""
217 + optimization_level: OptimizationLevel = OptimizationLevel.O2
218 + """The optimization level. These levels trade startup time cost for
219 + performance, with -O0 having the best startup time and -O3 having the best
220 + performance. -02 is used by defult. See OptimizationLevel for full
221 + description."""
222
223 def compute_hash(self) -> str:
224 """
@@ -297,6 +400,50 @@ class VllmConfig:
400
401 return replace(self, model_config=model_config)
402
403 + def _set_config_default(self, config_obj: Any, key: str, value: Any) -> None:
404 + """Set config attribute to default if not already set by user.
405 +
406 + Args:
407 + config_obj: Configuration object to update.
408 + key: Attribute name.
409 + value: Default value (static or callable).
410 + """
411 + if getattr(config_obj, key) is None:
412 + # Some config values are known before initialization and are
413 + # hard coded.
414 + # Other values depend on the user given configuration, so they are
415 + # implemented with lambda functions and decided at run time.
416 + setattr(config_obj, key, value(self) if callable(value) else value)
417 +
418 + def _apply_optimization_level_defaults(self, defaults: dict[str, Any]) -> None:
419 + """Apply optimization level defaults using self as root.
420 +
421 + Recursively applies values from defaults into nested config objects.
422 + Only fields present in defaults are overwritten.
423 +
424 + If the user configuration does not specify a value for a default field
425 + and if the default field is still None after all user selections are
426 + applied, then default values will be applied to the field. User speciied
427 + fields will not be overridden by the default.
428 +
429 + Args:
430 + defaults: Dictionary of default values to apply.
431 + """
432 +
433 + def apply_recursive(config_obj: Any, config_defaults: dict[str, Any]) -> None:
434 + """Recursively apply defaults to config_obj, using self as root."""
435 + for key, value in config_defaults.items():
436 + if not hasattr(config_obj, key):
437 + continue
438 +
439 + current = getattr(config_obj, key)
440 + if isinstance(value, dict) and is_dataclass(current):
441 + apply_recursive(current, value)
442 + else:
443 + self._set_config_default(config_obj, key, value)
444 +
445 + apply_recursive(self, defaults)
446 +
447 def _post_init_kv_transfer_config(self) -> None:
448 """Update KVTransferConfig based on top-level configs in VllmConfig.
449
@@ -434,17 +581,47 @@ class VllmConfig:
581 "precision for chunked prefill triton kernels."
582 )
583
437 - # If the user does not explicitly set a compilation mode, then
438 - # we use the default mode. The default mode depends on other
439 - # settings (see the below code).
584 + if (
585 + self.optimization_level > OptimizationLevel.O0
586 + and self.model_config is not None
587 + and self.model_config.enforce_eager
588 + ):
589 + logger.warning("Enforce eager set, overriding optimization level to -O0")
590 + self.optimization_level = OptimizationLevel.O0
591 +
592 + if self.compilation_config.backend == "eager" or (
593 + self.compilation_config.mode is not None
594 + and self.compilation_config.mode != CompilationMode.VLLM_COMPILE
595 + ):
596 + logger.warning(
597 + "Inductor compilation was disabled by user settings,"
598 + "Optimizations settings that are only active during"
599 + "Inductor compilation will be ignored."
600 + )
601 +
602 + def has_blocked_weights():
603 + if self.quant_config is not None:
604 + if hasattr(self.quant_config, "weight_block_size"):
605 + return self.quant_config.weight_block_size is not None
606 + elif hasattr(self.quant_config, "has_blocked_weights"):
607 + return self.quant_config.has_blocked_weights()
608 + return False
609 +
610 + # Enable quant_fp8 CUDA ops (TODO disable in follow up)
611 + # On H100 the CUDA kernel is faster than
612 + # native implementation
613 + # https://github.com/vllm-project/vllm/issues/25094
614 + if has_blocked_weights():
615 + custom_ops = self.compilation_config.custom_ops
616 + if "-quant_fp8" not in custom_ops:
617 + custom_ops.append("+quant_fp8")
618 +
619 if self.compilation_config.mode is None:
441 - if self.model_config is not None and not self.model_config.enforce_eager:
620 + if self.optimization_level > OptimizationLevel.O0:
621 self.compilation_config.mode = CompilationMode.VLLM_COMPILE
622 else:
623 self.compilation_config.mode = CompilationMode.NONE
624
446 - # If user does not set custom ops via none or all set it here based on
447 - # compilation mode and backend.
625 if all(s not in self.compilation_config.custom_ops for s in ("all", "none")):
626 if (
627 self.compilation_config.backend == "inductor"
@@ -454,23 +631,33 @@ class VllmConfig:
631 else:
632 self.compilation_config.custom_ops.append("all")
633
634 + default_config = OPTIMIZATION_LEVEL_TO_CONFIG[self.optimization_level]
635 + self._apply_optimization_level_defaults(default_config)
636 + if (
637 + self.compilation_config.cudagraph_mode != CUDAGraphMode.NONE
638 + and self.compilation_config.mode != CompilationMode.VLLM_COMPILE
639 + ):
640 + logger.info(
641 + "Cudagraph mode %s is not compatible with compilation mode %s."
642 + "Overriding to NONE.",
643 + self.compilation_config.cudagraph_mode,
644 + self.compilation_config.mode,
645 + )
646 + self.compilation_config.cudagraph_mode = CUDAGraphMode.NONE
647 +
648 # async tp is built on top of sequence parallelism
649 # and requires it to be enabled.
650 if self.compilation_config.pass_config.enable_async_tp:
651 self.compilation_config.pass_config.enable_sequence_parallelism = True
652 + if self.compilation_config.pass_config.enable_sequence_parallelism:
653 + if "-rms_norm" in self.compilation_config.custom_ops:
654 + logger.warning(
655 + "RMS norm force disabled, sequence parallelism might break"
656 + )
657 + else:
658 + self.compilation_config.custom_ops.append("+rms_norm")
659
660 if current_platform.support_static_graph_mode():
463 - # if cudagraph_mode is not explicitly set by users, set default
464 - # value
465 - if self.compilation_config.cudagraph_mode is None:
466 - if self.compilation_config.mode == CompilationMode.VLLM_COMPILE:
467 - # default to full and piecewise for most models
468 - self.compilation_config.cudagraph_mode = (
469 - CUDAGraphMode.FULL_AND_PIECEWISE
470 - )
471 - else:
472 - self.compilation_config.cudagraph_mode = CUDAGraphMode.NONE
473 -
661 # if cudagraph_mode has full cudagraphs, we need to check support
662 if self.compilation_config.cudagraph_mode.has_full_cudagraphs():
663 # decode context parallel does not support full cudagraphs
vllm/engine/arg_utils.py
+7 -1
@@ -77,6 +77,7 @@ from vllm.config.observability import DetailedTraceModules
77 from vllm.config.parallel import DistributedExecutorBackend, ExpertPlacementStrategy
78 from vllm.config.scheduler import SchedulerPolicy
79 from vllm.config.utils import get_field
80 +from vllm.config.vllm import OptimizationLevel
81 from vllm.logger import init_logger, suppress_logging
82 from vllm.platforms import CpuArchEnum, current_platform
83 from vllm.plugins import load_general_plugins
@@ -560,6 +561,7 @@ class EngineArgs:
561 stream_interval: int = SchedulerConfig.stream_interval
562
563 kv_sharing_fast_prefill: bool = CacheConfig.kv_sharing_fast_prefill
564 + optimization_level: OptimizationLevel = VllmConfig.optimization_level
565
566 kv_offloading_size: float | None = CacheConfig.kv_offloading_size
567 kv_offloading_backend: KVOffloadingBackend | None = (
@@ -1114,6 +1116,10 @@ class EngineArgs:
1116 "--structured-outputs-config", **vllm_kwargs["structured_outputs_config"]
1117 )
1118
1119 + vllm_group.add_argument(
1120 + "--optimization-level", **vllm_kwargs["optimization_level"]
1121 + )
1122 +
1123 # Other arguments
1124 parser.add_argument(
1125 "--disable-log-stats",
@@ -1733,7 +1739,6 @@ class EngineArgs:
1739 compilation_config.max_cudagraph_capture_size = (
1740 self.max_cudagraph_capture_size
1741 )
1736 -
1742 config = VllmConfig(
1743 model_config=model_config,
1744 cache_config=cache_config,
@@ -1750,6 +1755,7 @@ class EngineArgs:
1755 kv_events_config=self.kv_events_config,
1756 ec_transfer_config=self.ec_transfer_config,
1757 additional_config=self.additional_config,
1758 + optimization_level=self.optimization_level,
1759 )
1760
1761 return config
vllm/utils/argparse_utils.py
+19 -5
@@ -247,16 +247,16 @@ class FlexibleArgumentParser(ArgumentParser):
247 elif arg.startswith("-O") and arg != "-O" and arg[2] != ".":
248 # allow -O flag to be used without space, e.g. -O3 or -Odecode
249 # -O.<...> handled later
250 - # also handle -O=<mode> here
251 - mode = arg[3:] if arg[2] == "=" else arg[2:]
252 - processed_args.append(f"-O.mode={mode}")
250 + # also handle -O=<optimization_level> here
251 + optimization_level = arg[3:] if arg[2] == "=" else arg[2:]
252 + processed_args += ["--optimization-level", optimization_level]
253 elif (
254 arg == "-O"
255 and i + 1 < len(args)
256 and args[i + 1] in {"0", "1", "2", "3"}
257 ):
258 - # Convert -O <n> to -O.mode <n>
259 - processed_args.append("-O.mode")
258 + # Convert -O <n> to --optimization-level <n>
259 + processed_args.append("--optimization-level")
260 else:
261 processed_args.append(arg)
262
@@ -294,10 +294,24 @@ class FlexibleArgumentParser(ArgumentParser):
294 delete = set[int]()
295 dict_args = defaultdict[str, dict[str, Any]](dict)
296 duplicates = set[str]()
297 + # Track regular arguments (non-dict args) for duplicate detection
298 + regular_args_seen = set[str]()
299 for i, processed_arg in enumerate(processed_args):
300 if i in delete: # skip if value from previous arg
301 continue
302
303 + if processed_arg.startswith("--") and "." not in processed_arg:
304 + if "=" in processed_arg:
305 + arg_name = processed_arg.split("=", 1)[0]
306 + else:
307 + arg_name = processed_arg
308 +
309 + if arg_name in regular_args_seen:
310 + duplicates.add(arg_name)
311 + else:
312 + regular_args_seen.add(arg_name)
313 + continue
314 +
315 if processed_arg.startswith("-") and "." in processed_arg:
316 if "=" in processed_arg:
317 processed_arg, value_str = processed_arg.split("=", 1)
vllm/v1/worker/gpu/cudagraph_utils.py
+1 -1
@@ -37,7 +37,7 @@ class CudaGraphManager:
37 self.dp_size = vllm_config.parallel_config.data_parallel_size
38 self.compilation_config = vllm_config.compilation_config
39 assert self.compilation_config is not None
40 -
40 + self.cudagraph_mode: CUDAGraphMode
41 if self.compilation_config.cudagraph_mode is None:
42 self.cudagraph_mode = CUDAGraphMode.NONE
43 else: