@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 4fd9375028
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 07:00 UTC

[2/N][torch.compile] make compilation cfg part of vllm cfg (#10383)

Signed-off-by: youkaichao <youkaichao@gmail.com>

youkaichao committed Nov 16, 2024 at 18:02 UTC 4fd937502827a7e06c54ded1f9d9b70ff640e222
27 files changed +359 -283
tests/compile/piecewise/test_simple.py
+5 -3
@@ -11,8 +11,8 @@ from torch.library import Library
11 from vllm.compilation.compile_context import set_compile_context
12 from vllm.compilation.counter import compilation_counter
13 from vllm.compilation.decorators import support_torch_compile
14 -from vllm.compilation.levels import CompilationLevel
15 -from vllm.config import VllmConfig
14 +from vllm.config import CompilationLevel, VllmConfig
15 +from vllm.plugins import set_current_vllm_config
16 from vllm.utils import direct_register_custom_op
17
18 global_counter = 0
@@ -82,7 +82,9 @@ def test_simple_piecewise_compile():
82 os.environ["VLLM_TORCH_COMPILE_CONFIG"] = config
83 os.environ["VLLM_TORCH_COMPILE_LEVEL"] = str(CompilationLevel.PIECEWISE)
84
85 - model = SillyModel(vllm_config=VllmConfig(), prefix='')
85 + vllm_config = VllmConfig()
86 + with set_current_vllm_config(vllm_config):
87 + model = SillyModel(vllm_config=vllm_config, prefix='')
88
89 inputs = torch.randn(100).cuda()
90
tests/compile/piecewise/test_toy_llama.py
+12 -10
@@ -15,12 +15,10 @@ from torch import nn
15 from torch.library import Library
16
17 from vllm.compilation.compile_context import set_compile_context
18 -from vllm.compilation.config import CompilationConfig
18 from vllm.compilation.counter import compilation_counter
19 from vllm.compilation.decorators import support_torch_compile
21 -from vllm.compilation.levels import CompilationLevel
22 -from vllm.config import VllmConfig
23 -from vllm.plugins import set_compilation_config
20 +from vllm.config import CompilationConfig, CompilationLevel, VllmConfig
21 +from vllm.plugins import set_compilation_config, set_current_vllm_config
22 from vllm.utils import direct_register_custom_op
23
24 # create a library to hold the custom op
@@ -272,9 +270,11 @@ def run_model(llama_config,
270 CompilationLevel.NO_COMPILATION)
271 set_compilation_config(None)
272
275 - model = LlamaModel(config=llama_config,
276 - vllm_config=VllmConfig(),
277 - prefix="").eval().cuda()
273 + vllm_config = VllmConfig()
274 + with set_current_vllm_config(vllm_config):
275 + model = LlamaModel(config=llama_config,
276 + vllm_config=vllm_config,
277 + prefix="").eval().cuda()
278
279 B = 16 # max batch size
280 input_ids = torch.randint(0, llama_config.vocab_size, (B, )).cuda()
@@ -395,9 +395,11 @@ def benchmark():
395 else:
396 set_compilation_config(None)
397
398 - model = LlamaModel(config=llama_config,
399 - vllm_config=VllmConfig(),
400 - prefix="").eval().cuda().to(torch.bfloat16)
398 + vllm_config = VllmConfig()
399 + with set_current_vllm_config(vllm_config):
400 + model = LlamaModel(config=llama_config,
401 + vllm_config=vllm_config,
402 + prefix="").eval().cuda().to(torch.bfloat16)
403
404 B = 256 # max batch size
405 input_ids = torch.randint(0, llama_config.vocab_size, (B, )).cuda()
tests/compile/test_basic_correctness.py
+1 -1
@@ -3,7 +3,7 @@ from typing import Dict, List, Optional
3
4 import pytest
5
6 -from vllm.compilation.levels import CompilationLevel
6 +from vllm.config import CompilationLevel
7 from vllm.utils import cuda_device_count_stateless
8
9 from ..utils import compare_all_settings
tests/compile/test_full_graph.py
+1 -1
@@ -1,6 +1,6 @@
1 import pytest
2
3 -from vllm.compilation.levels import CompilationLevel
3 +from vllm.config import CompilationLevel
4
5 from ..utils import fork_new_process_for_each_test
6 from .utils import TEST_MODELS, check_full_graph_support
tests/compile/test_fusion.py
+1 -1
@@ -3,10 +3,10 @@ import torch
3 from compressed_tensors.quantization import FP8_DTYPE
4
5 import vllm.envs as envs
6 -from vllm.compilation.config import CompilationConfig
6 from vllm.compilation.fusion import (FusionPass, find_auto_fn,
7 find_auto_fn_maybe)
8 from vllm.compilation.reshapes import RedundantReshapesPass
9 +from vllm.config import CompilationConfig
10 from vllm.model_executor.layers.layernorm import RMSNorm
11 from vllm.model_executor.layers.quantization.utils.w8a8_utils import (
12 apply_fp8_linear)
tests/compile/test_wrapper.py
+3 -1
@@ -3,6 +3,7 @@ from typing import Optional
3 import torch
4
5 from vllm.compilation.wrapper import TorchCompileWrapperWithCustomDispatcher
6 +from vllm.config import CompilationLevel
7
8
9 class MyMod(torch.nn.Module):
@@ -18,7 +19,8 @@ class MyWrapper(TorchCompileWrapperWithCustomDispatcher):
19 def __init__(self, model):
20 self.model = model
21 compiled_callable = torch.compile(self.forward, backend="eager")
21 - super().__init__(compiled_callable)
22 + super().__init__(compiled_callable,
23 + compilation_level=CompilationLevel.DYNAMO_ONCE)
24
25 def forward(self, x: torch.Tensor, cache: Optional[torch.Tensor] = None):
26 # this is the function to be compiled
tests/compile/utils.py
+1 -1
@@ -4,7 +4,7 @@ import torch
4
5 from tests.quantization.utils import is_quant_method_supported
6 from vllm import LLM, SamplingParams
7 -from vllm.compilation.levels import CompilationLevel
7 +from vllm.config import CompilationLevel
8 from vllm.platforms import current_platform
9
10 TEST_MODELS = [
tests/model_executor/test_enabled_custom_ops.py
+26 -26
@@ -3,11 +3,13 @@ from typing import List
3
4 import pytest
5
6 +from vllm.config import CompilationConfig, VllmConfig
7 from vllm.model_executor.custom_op import CustomOp
8 from vllm.model_executor.layers.activation import (GeluAndMul,
9 ReLUSquaredActivation,
10 SiluAndMul)
11 from vllm.model_executor.layers.layernorm import RMSNorm
12 +from vllm.plugins import set_current_vllm_config
13
14
15 # Registered subclass for test
@@ -51,42 +53,40 @@ class Relu3(ReLUSquaredActivation):
53 ])
54 def test_enabled_ops(env: str, torch_level: int, ops_enabled: List[int],
55 default_on: bool):
54 - os.environ["VLLM_CUSTOM_OPS"] = env
56 os.environ["VLLM_TORCH_COMPILE_LEVEL"] = str(torch_level)
57 + vllm_config = VllmConfig(compilation_config=CompilationConfig(
58 + custom_ops=env.split(",")))
59 + with set_current_vllm_config(vllm_config):
60 + assert CustomOp.default_on() == default_on
61
57 - # Reset default_on (computed once):
58 - CustomOp.default_on.cache_clear()
62 + ops_enabled = [bool(x) for x in ops_enabled]
63
60 - assert CustomOp.default_on() == default_on
64 + assert RMSNorm(1024).enabled() == ops_enabled[0]
65 + assert CustomOp.op_registry["rms_norm"].enabled() == ops_enabled[0]
66
62 - ops_enabled = [bool(x) for x in ops_enabled]
67 + assert SiluAndMul().enabled() == ops_enabled[1]
68 + assert CustomOp.op_registry["silu_and_mul"].enabled() == ops_enabled[1]
69
64 - assert RMSNorm(1024).enabled() == ops_enabled[0]
65 - assert CustomOp.op_registry["rms_norm"].enabled() == ops_enabled[0]
70 + assert GeluAndMul().enabled() == ops_enabled[2]
71 + assert CustomOp.op_registry["gelu_and_mul"].enabled() == ops_enabled[2]
72
67 - assert SiluAndMul().enabled() == ops_enabled[1]
68 - assert CustomOp.op_registry["silu_and_mul"].enabled() == ops_enabled[1]
73 + # If registered, subclasses should follow their own name
74 + assert Relu3().enabled() == ops_enabled[3]
75 + assert CustomOp.op_registry["relu3"].enabled() == ops_enabled[3]
76
70 - assert GeluAndMul().enabled() == ops_enabled[2]
71 - assert CustomOp.op_registry["gelu_and_mul"].enabled() == ops_enabled[2]
77 + # Unregistered subclass
78 + class SiluAndMul2(SiluAndMul):
79 + pass
80
73 - # If registered, subclasses should follow their own name
74 - assert Relu3().enabled() == ops_enabled[3]
75 - assert CustomOp.op_registry["relu3"].enabled() == ops_enabled[3]
76 -
77 - # Unregistered subclass
78 - class SiluAndMul2(SiluAndMul):
79 - pass
80 -
81 - # Subclasses should not require registration
82 - assert SiluAndMul2().enabled() == SiluAndMul().enabled()
81 + # Subclasses should not require registration
82 + assert SiluAndMul2().enabled() == SiluAndMul().enabled()
83
84
85 @pytest.mark.parametrize(
86 "env", ["all,none", "all,+rms_norm,all", "+rms_norm,-rms_norm"])
87 def test_enabled_ops_invalid(env: str):
88 - os.environ["VLLM_CUSTOM_OPS"] = env
89 - CustomOp.default_on.cache_clear()
90 -
91 - with pytest.raises(AssertionError):
92 - RMSNorm(1024).enabled()
88 + with pytest.raises(Exception): # noqa
89 + vllm_config = VllmConfig(compilation_config=CompilationConfig(
90 + custom_ops=env.split(",")))
91 + with set_current_vllm_config(vllm_config):
92 + RMSNorm(1024).enabled()
tests/tpu/test_compilation.py
+1 -1
@@ -5,7 +5,7 @@ import tempfile
5
6 import depyf
7
8 -from vllm.compilation.levels import CompilationLevel
8 +from vllm.config import CompilationLevel
9
10 # disable custom dispatcher, let Dynamo takes over
11 # all the control
tests/tpu/test_custom_dispatcher.py
+1 -1
@@ -1,6 +1,6 @@
1 import os
2
3 -from vllm.compilation.levels import CompilationLevel
3 +from vllm.config import CompilationLevel
4
5 from ..utils import compare_two_settings
6
vllm/compilation/backends.py
+13 -7
@@ -10,13 +10,12 @@ import torch
10 import torch.fx as fx
11
12 import vllm.envs as envs
13 +from vllm.config import CompilationConfig, CompilationLevel
14 from vllm.logger import init_logger
15 from vllm.utils import combine_fx_passes, weak_ref_tensors
16
16 -from .config import CompilationConfig
17 from .counter import compilation_counter
18 from .fusion import FusionPass
19 -from .levels import CompilationLevel
19 from .reshapes import RedundantReshapesPass
20
21 logger = init_logger(__name__)
@@ -392,7 +391,10 @@ class VllmBackend:
391 sym_tensor_indices: List[int]
392 input_buffers: List[torch.Tensor]
393
395 - def __init__(self, post_grad_passes: Sequence[Callable] = ()):
394 + def __init__(
395 + self,
396 + compilation_configs: CompilationConfig,
397 + ):
398 global global_graph_pool
399 if global_graph_pool is None:
400 global_graph_pool = torch.cuda.graph_pool_handle()
@@ -401,11 +403,13 @@ class VllmBackend:
403 # streams, it might not be safe to share a global pool.
404 # only investigate this when we use multiple streams
405 self.graph_pool = global_graph_pool
404 - self.post_grad_passes = post_grad_passes
406 + self.post_grad_passes = []
407
408 self.sym_tensor_indices = []
409 self.input_buffers = []
410
411 + self.compilation_configs = compilation_configs
412 +
413 # `torch.compile` is JIT compiled, so we don't need to
414 # do anything here
415
@@ -437,10 +441,10 @@ class VllmBackend:
441 assert not self._called, "VllmBackend can only be called once"
442
443 self.graph = graph
440 - # config is read now, because only here can
444 + # config is updated now, because only here can
445 # we get the sizes to capture for cudagraph
446 # from compilation context
443 - self.compilation_configs = CompilationConfig.select_and_init_config()
447 + self.compilation_configs.init_during_runtime()
448 self.add_passes_to_config()
449
450 self.split_gm, self.piecewise_graphs = split_graph(
@@ -688,4 +692,6 @@ def select_default_backend(level: int) -> Union[str, Callable]:
692 return backend_str
693 assert level == CompilationLevel.PIECEWISE
694
691 - return VllmBackend()
695 + from vllm.plugins import get_current_vllm_config
696 + compilation_config = get_current_vllm_config().compilation_config
697 + return VllmBackend(compilation_config)
vllm/compilation/config.py deleted
-159
@@ -1,159 +0,0 @@
1 -import copy
2 -from pathlib import Path
3 -from typing import Any, Dict, List, Optional
4 -
5 -from pydantic import BaseModel, Field, PrivateAttr
6 -
7 -import vllm.envs as envs
8 -from vllm.logger import init_logger
9 -
10 -from .compile_context import get_compile_context
11 -
12 -logger = init_logger(__name__)
13 -
14 -
15 -class CompilationConfig(BaseModel):
16 - """
17 - Configuration for compilation.
18 - It has two parts:
19 - - CudaGraph capture:
20 - - use_cudagraph: whether to use cudagraph inside compilation.
21 - - False: cudagraph inside compilation is not used.
22 - - True: cudagraph inside compilation is used. It requires
23 - that all input buffers have fixed addresses.
24 - Note that this is orthogonal to the cudagraph capture out
25 - side of compilation.
26 - TODO: move outside cudagraph logic into compilation.
27 - torch.compile will handle cudagraph capture logic in the future.
28 - - cudagraph_capture_sizes: sizes to capture cudagraph.
29 - - None: capture sizes are inferred from compilation context.
30 - - List[int]: capture sizes are specified.
31 - - cudagraph_num_of_warmups: number of warmup runs for cudagraph.
32 - It means the first several runs will be treated as warmup runs.
33 - Only after that, the execution will be recorded, and the recorded
34 - cudagraph will be used for subsequent runs.
35 - - cudagraph_copy_inputs: whether to copy input tensors for
36 - cudagraph. If the caller can guarantee that the same input buffers
37 - are always used, it can set this to False. Otherwise, it should
38 - set this to True, and the compiler will copy the input to an
39 - internally managed buffer. Default is False.
40 - - Inductor compilation:
41 - - use_inductor: whether to use inductor compilation.
42 - - False: inductor compilation is not used. graph runs in eager.
43 - - True: inductor compilation is used. one graph for symbolic shape
44 - is compiled. In addition, compile for different sizes specified
45 - in inductor_compile_sizes, using configurations
46 - in inductor_compile_config.
47 - - inductor_compile_sizes: sizes to compile for inductor.
48 - - inductor_specialize_for_cudagraph_no_more_than: an optional integer
49 - to specialize inductor for cudagraph sizes no more than the
50 - specified size. It is useful when we want to specialize inductor
51 - with a subset of cudagraph sizes.
52 - - inductor_compile_config: additional configurations for inductor.
53 - - None: use default configurations.
54 - - inductor_passes: additional passes for inductor. It is a dictionary
55 - from pass name to pass function qualified name. We use function
56 - name because the config uses json format. If we pass the config
57 - from Python, functions can also be passed directly via Python object
58 - constructor, e.g. `CompilationConfig(inductor_passes={"a": func})`
59 - - Custom inductor passes:
60 - - dump_graph_stages: list of stages for which we want to dump the graph.
61 - Each pass defines its own stages (before, after, maybe in-between).
62 - - dump_graph_dir: directory to dump the graph. Default is .
63 - - enable_fusion: whether to enable the custom fusion pass.
64 - TODO better pass enabling system.
65 -
66 - Why we have different sizes for cudagraph and inductor:
67 - - cudagraph: a cudagraph captured for a specific size can only be used
68 - for the same size. We need to capture all the sizes we want to use.
69 - - inductor: a graph compiled by inductor for a general shape can be used
70 - for different sizes. Inductor can also compile for specific sizes,
71 - where it can have more information to optimize the graph with fully
72 - static shapes. However, we find the general shape compilation is
73 - sufficient for most cases. It might be beneficial to compile for
74 - certain small batchsizes, where inductor is good at optimizing.
75 - """
76 - use_inductor: bool = True
77 - inductor_specialize_for_cudagraph_no_more_than: Optional[int] = None
78 - inductor_compile_sizes: Optional[List[int]] = Field(default_factory=dict)
79 - inductor_compile_config: Dict = Field(default_factory=dict)
80 - inductor_passes: Dict[str, str] = Field(default_factory=dict)
81 -
82 - use_cudagraph: bool = False
83 - non_cudagraph_ops: List[str] = Field(default_factory=list)
84 - cudagraph_num_of_warmups: int = 0
85 - cudagraph_capture_sizes: Optional[List[int]] = None
86 - cudagraph_copy_inputs: bool = False
87 -
88 - dump_graph_stages: List[str] = Field(default_factory=list)
89 - dump_graph_dir: Path = Field(default=Path("."))
90 - enable_fusion: bool = True
91 -
92 - # not configurable, computed after init
93 - compile_sizes: List[int] = PrivateAttr
94 - capture_sizes: List[int] = PrivateAttr
95 -
96 - def model_post_init(self, __context: Any) -> None:
97 - for k, v in self.inductor_passes.items():
98 - if not isinstance(v, str):
99 - assert callable(v), (
100 - f"pass {k} should be a function or a qualified name")
101 - self.inductor_compile_config[k] = v
102 - continue
103 -
104 - # resolve function from qualified name
105 - names = v.split(".")
106 - module = ".".join(names[:-1])
107 - func_name = names[-1]
108 - func = __import__(module).__dict__[func_name]
109 - self.inductor_compile_config[k] = func
110 -
111 - def init_during_runtime(self):
112 - """To complete the initialization of config,
113 - we need to know the compile context, which is only available
114 - during the first run of the model.
115 - """
116 - context = get_compile_context()
117 - context = copy.deepcopy(context) if context is not None else []
118 - sizes_to_specialize: List[int] = context
119 - if self.cudagraph_capture_sizes is None:
120 - self.capture_sizes = sizes_to_specialize
121 - else:
122 - self.capture_sizes = self.cudagraph_capture_sizes
123 - logger.info(("cudagraph sizes specified by model runner"
124 - " %s is overridden by config %s"),
125 - sizes_to_specialize, self.cudagraph_capture_sizes)
126 - if self.inductor_specialize_for_cudagraph_no_more_than is not None:
127 - assert self.inductor_compile_sizes is None, (
128 - "inductor_compile_sizes should be None when "
129 - "inductor_specialize_for_cudagraph_no_more_than is not None")
130 - self.compile_sizes = [
131 - x for x in self.capture_sizes
132 - if x <= self.inductor_specialize_for_cudagraph_no_more_than
133 - ]
134 - else:
135 - assert self.inductor_compile_sizes is not None, (
136 - "inductor_compile_sizes should not be None when "
137 - "inductor_specialize_for_cudagraph_no_more_than is None")
138 - self.compile_sizes = self.inductor_compile_sizes
139 -
140 - @staticmethod
141 - def select_and_init_config() -> "CompilationConfig":
142 - """The order of selecting config is:
143 - 1. Use the config specified in environment variable.
144 - 2. Use the config specified in plugins.
145 - 3. Use the default config.
146 - """
147 - config_path = envs.VLLM_TORCH_COMPILE_CONFIG
148 - if config_path is not None:
149 - with open(config_path) as json_file:
150 - config = CompilationConfig.model_validate_json(
151 - json_file.read())
152 - else:
153 - from vllm.plugins import get_compilation_config
154 - predefined_config = get_compilation_config()
155 - config = predefined_config if predefined_config is not None else (
156 - CompilationConfig())
157 -
158 - config.init_during_runtime()
159 - return config
vllm/compilation/decorators.py
+5 -5
@@ -3,10 +3,8 @@ from typing import Dict, List, Optional, Union
3
4 import torch
5
6 -import vllm.envs as envs
7 -from vllm.compilation.levels import CompilationLevel
6 from vllm.compilation.wrapper import TorchCompileWrapperWithCustomDispatcher
9 -from vllm.config import VllmConfig
7 +from vllm.config import CompilationLevel, VllmConfig
8 from vllm.logger import init_logger
9 from vllm.sequence import IntermediateTensors
10 from vllm.utils import supports_dynamo
@@ -126,12 +124,14 @@ def _support_torch_compile(cls: type,
124 old_init(self, vllm_config=vllm_config, prefix=prefix, **kwargs)
125 # for CompilationLevel.DYNAMO_AS_IS , the upper level model runner
126 # will handle the compilation, so we don't need to do anything here.
129 - self.do_not_compile = envs.VLLM_TORCH_COMPILE_LEVEL in [
127 + self.do_not_compile = \
128 + vllm_config.compilation_config.level in [
129 CompilationLevel.NO_COMPILATION, CompilationLevel.DYNAMO_AS_IS
130 ] or not supports_dynamo()
131 if self.do_not_compile:
132 return
134 - TorchCompileWrapperWithCustomDispatcher.__init__(self)
133 + TorchCompileWrapperWithCustomDispatcher.__init__(
134 + self, compilation_level=vllm_config.compilation_config.level)
135
136 cls.__init__ = __init__ # type: ignore
137
vllm/compilation/fusion.py
+1 -1
@@ -6,8 +6,8 @@ from torch._higher_order_ops.auto_functionalize import auto_functionalized
6 from torch._inductor.pattern_matcher import (Match, PatternMatcherPass,
7 fwd_only, register_replacement)
8
9 -from vllm.compilation.config import CompilationConfig
9 from vllm.compilation.inductor_pass import InductorPass
10 +from vllm.config import CompilationConfig
11 from vllm.logger import init_logger
12
13 logger = init_logger(__name__)
vllm/compilation/inductor_pass.py
+1 -1
@@ -2,7 +2,7 @@ from abc import ABC, abstractmethod
2
3 import torch
4
5 -from vllm.compilation.config import CompilationConfig
5 +from vllm.config import CompilationConfig
6 # yapf: disable
7 from vllm.distributed import get_tensor_model_parallel_rank as get_tp_rank
8 from vllm.distributed import (
vllm/compilation/levels.py deleted
-8
@@ -1,8 +0,0 @@
1 -# constants for the levels of the compilation process
2 -
3 -
4 -class CompilationLevel:
5 - NO_COMPILATION = 0
6 - DYNAMO_AS_IS = 1
7 - DYNAMO_ONCE = 2
8 - PIECEWISE = 3
vllm/compilation/wrapper.py
+6 -5
@@ -8,8 +8,7 @@ from typing import Callable, List, Optional
8 import torch
9
10 import vllm.envs as envs
11 -
12 -from .levels import CompilationLevel
11 +from vllm.config import CompilationLevel
12
13
14 class TorchCompileWrapperWithCustomDispatcher:
@@ -25,7 +24,9 @@ class TorchCompileWrapperWithCustomDispatcher:
24 `torch.compile` over the forward method.
25 """
26
28 - def __init__(self, compiled_callable: Optional[Callable] = None):
27 + def __init__(self,
28 + compiled_callable: Optional[Callable] = None,
29 + compilation_level: int = 0):
30
31 if compiled_callable is None:
32 # default compilation settings
@@ -38,7 +39,7 @@ class TorchCompileWrapperWithCustomDispatcher:
39 backend = get_torch_compile_backend()
40 if backend is None:
41 from vllm.compilation.backends import select_default_backend
41 - backend = select_default_backend(envs.VLLM_TORCH_COMPILE_LEVEL)
42 + backend = select_default_backend(compilation_level)
43
44 compiled_callable = torch.compile(
45 self.forward,
@@ -54,7 +55,7 @@ class TorchCompileWrapperWithCustomDispatcher:
55 # subclasses can use this to switch between the custom dispatcher
56 # and the default Dynamo guard mechanism.
57 self.use_custom_dispatcher: bool = \
57 - envs.VLLM_TORCH_COMPILE_LEVEL >= CompilationLevel.DYNAMO_ONCE
58 + compilation_level >= CompilationLevel.DYNAMO_ONCE
59
60 def __call__(self, *args, **kwargs):
61 """Implement the dispatch logic here, beyond the torch.compile level.
vllm/config.py
+189
@@ -3,10 +3,12 @@ import enum
3 import json
4 import warnings
5 from dataclasses import dataclass, field, replace
6 +from pathlib import Path
7 from typing import (TYPE_CHECKING, Any, Callable, ClassVar, Dict, Final, List,
8 Literal, Mapping, Optional, Set, Tuple, Type, Union)
9
10 import torch
11 +from pydantic import BaseModel, Field, PrivateAttr
12 from transformers import PretrainedConfig
13
14 import vllm.envs as envs
@@ -2052,6 +2054,185 @@ class ObservabilityConfig:
2054 f"installed. Original error:\n{otel_import_error_traceback}")
2055
2056
2057 +class CompilationLevel:
2058 + # constants for the levels of the compilation process
2059 + NO_COMPILATION = 0
2060 + DYNAMO_AS_IS = 1
2061 + DYNAMO_ONCE = 2
2062 + PIECEWISE = 3
2063 +
2064 +
2065 +class CompilationConfig(BaseModel):
2066 + """
2067 + Configuration for compilation.
2068 + It has three parts:
2069 + - Top-level Compilation control:
2070 + - level: the level of compilation.
2071 + - 0: no compilation.
2072 + - 1: dynamo as is.
2073 + - 2: dynamo once.
2074 + - 3: piecewise compilation.
2075 + - custom_ops: fine-grained control over which custom ops to enable/disable.
2076 + Use 'all' to enable all, 'none' to disable all.
2077 + Also specify a list of custom op names to enable (prefixed with a '+'),
2078 + or disable (prefixed with a '-').
2079 + Examples:
2080 + - 'all,-op1' to enable all except op1
2081 + - 'none,+op1,+op2' to enable only op1 and op2
2082 + By default, all custom ops are enabled when running without Inductor
2083 + and disabled when running with Inductor (compile_level >= Inductor).
2084 + - CudaGraph capture:
2085 + - use_cudagraph: whether to use cudagraph inside compilation.
2086 + - False: cudagraph inside compilation is not used.
2087 + - True: cudagraph inside compilation is used. It requires
2088 + that all input buffers have fixed addresses.
2089 + Note that this is orthogonal to the cudagraph capture out
2090 + side of compilation.
2091 + TODO: move outside cudagraph logic into compilation.
2092 + torch.compile will handle cudagraph capture logic in the future.
2093 + - cudagraph_capture_sizes: sizes to capture cudagraph.
2094 + - None: capture sizes are inferred from compilation context.
2095 + - List[int]: capture sizes are specified.
2096 + - cudagraph_num_of_warmups: number of warmup runs for cudagraph.
2097 + It means the first several runs will be treated as warmup runs.
2098 + Only after that, the execution will be recorded, and the recorded
2099 + cudagraph will be used for subsequent runs.
2100 + - cudagraph_copy_inputs: whether to copy input tensors for
2101 + cudagraph. If the caller can guarantee that the same input buffers
2102 + are always used, it can set this to False. Otherwise, it should
2103 + set this to True, and the compiler will copy the input to an
2104 + internally managed buffer. Default is False.
2105 + - Inductor compilation:
2106 + - use_inductor: whether to use inductor compilation.
2107 + - False: inductor compilation is not used. graph runs in eager.
2108 + - True: inductor compilation is used. one graph for symbolic shape
2109 + is compiled. In addition, compile for different sizes specified
2110 + in inductor_compile_sizes, using configurations
2111 + in inductor_compile_config.
2112 + - inductor_compile_sizes: sizes to compile for inductor.
2113 + - inductor_specialize_for_cudagraph_no_more_than: an optional integer
2114 + to specialize inductor for cudagraph sizes no more than the
2115 + specified size. It is useful when we want to specialize inductor
2116 + with a subset of cudagraph sizes.
2117 + - inductor_compile_config: additional configurations for inductor.
2118 + - None: use default configurations.
2119 + - inductor_passes: additional passes for inductor. It is a dictionary
2120 + from pass name to pass function qualified name. We use function
2121 + name because the config uses json format. If we pass the config
2122 + from Python, functions can also be passed directly via Python object
2123 + constructor, e.g. `CompilationConfig(inductor_passes={"a": func})`
2124 + - custom inductor passes:
2125 + - dump_graph_stages: list of stages for which we want to dump the graph.
2126 + Each pass defines its own stages (before, after, maybe in-between).
2127 + - dump_graph_dir: directory to dump the graph. Default is .
2128 + - enable_fusion: whether to enable the custom fusion pass.
2129 + TODO better pass enabling system.
2130 +
2131 + Why we have different sizes for cudagraph and inductor:
2132 + - cudagraph: a cudagraph captured for a specific size can only be used
2133 + for the same size. We need to capture all the sizes we want to use.
2134 + - inductor: a graph compiled by inductor for a general shape can be used
2135 + for different sizes. Inductor can also compile for specific sizes,
2136 + where it can have more information to optimize the graph with fully
2137 + static shapes. However, we find the general shape compilation is
2138 + sufficient for most cases. It might be beneficial to compile for
2139 + certain small batchsizes, where inductor is good at optimizing.
2140 + """ # noqa
2141 + level: int = 0
2142 + custom_ops: List[str] = Field(default_factory=list)
2143 +
2144 + use_inductor: bool = True
2145 + inductor_specialize_for_cudagraph_no_more_than: Optional[int] = None
2146 + inductor_compile_sizes: Optional[List[int]] = Field(default_factory=dict)
2147 + inductor_compile_config: Dict = Field(default_factory=dict)
2148 + inductor_passes: Dict[str, str] = Field(default_factory=dict)
2149 +
2150 + use_cudagraph: bool = False
2151 + non_cudagraph_ops: List[str] = Field(default_factory=list)
2152 + cudagraph_num_of_warmups: int = 0
2153 + cudagraph_capture_sizes: Optional[List[int]] = None
2154 + cudagraph_copy_inputs: bool = False
2155 +
2156 + dump_graph_stages: List[str] = Field(default_factory=list)
2157 + dump_graph_dir: Path = Field(default=Path("."))
2158 + enable_fusion: bool = True
2159 +
2160 + # not configurable, computed after init
2161 + compile_sizes: List[int] = PrivateAttr
2162 + capture_sizes: List[int] = PrivateAttr
2163 +
2164 + def model_post_init(self, __context: Any) -> None:
2165 + self.level = envs.VLLM_TORCH_COMPILE_LEVEL
2166 +
2167 + count_none = self.custom_ops.count("none")
2168 + count_all = self.custom_ops.count("all")
2169 + assert count_none + count_all <= 1, "Can only specify 'none' or 'all'"
2170 +
2171 + for k, v in self.inductor_passes.items():
2172 + if not isinstance(v, str):
2173 + assert callable(v), (
2174 + f"pass {k} should be a function or a qualified name")
2175 + self.inductor_compile_config[k] = v
2176 + continue
2177 +
2178 + # resolve function from qualified name
2179 + names = v.split(".")
2180 + module = ".".join(names[:-1])
2181 + func_name = names[-1]
2182 + func = __import__(module).__dict__[func_name]
2183 + self.inductor_compile_config[k] = func
2184 +
2185 + def init_during_runtime(self):
2186 + """To complete the initialization of config,
2187 + we need to know the compile context, which is only available
2188 + during the first run of the model.
2189 + """
2190 + from vllm.compilation.compile_context import get_compile_context
2191 + context = get_compile_context()
2192 + context = copy.deepcopy(context) if context is not None else []
2193 + sizes_to_specialize: List[int] = context
2194 + if self.cudagraph_capture_sizes is None:
2195 + self.capture_sizes = sizes_to_specialize
2196 + else:
2197 + self.capture_sizes = self.cudagraph_capture_sizes
2198 + logger.info(("cudagraph sizes specified by model runner"
2199 + " %s is overridden by config %s"),
2200 + sizes_to_specialize, self.cudagraph_capture_sizes)
2201 + if self.inductor_specialize_for_cudagraph_no_more_than is not None:
2202 + assert self.inductor_compile_sizes is None, (
2203 + "inductor_compile_sizes should be None when "
2204 + "inductor_specialize_for_cudagraph_no_more_than is not None")
2205 + self.compile_sizes = [
2206 + x for x in self.capture_sizes
2207 + if x <= self.inductor_specialize_for_cudagraph_no_more_than
2208 + ]
2209 + else:
2210 + assert self.inductor_compile_sizes is not None, (
2211 + "inductor_compile_sizes should not be None when "
2212 + "inductor_specialize_for_cudagraph_no_more_than is None")
2213 + self.compile_sizes = self.inductor_compile_sizes
2214 +
2215 + @staticmethod
2216 + def select_and_init_config() -> "CompilationConfig":
2217 + """The order of selecting config is:
2218 + 1. Use the config specified in environment variable.
2219 + 2. Use the config specified in plugins.
2220 + 3. Use the default config.
2221 + """
2222 + config_path = envs.VLLM_TORCH_COMPILE_CONFIG
2223 + if config_path is not None:
2224 + with open(config_path) as json_file:
2225 + config = CompilationConfig.model_validate_json(
2226 + json_file.read())
2227 + else:
2228 + from vllm.plugins import get_compilation_config
2229 + predefined_config = get_compilation_config()
2230 + config = predefined_config if predefined_config is not None else (
2231 + CompilationConfig())
2232 +
2233 + return config
2234 +
2235 +
2236 @dataclass
2237 class VllmConfig:
2238 """Dataclass which contains all vllm-related configuration. This
@@ -2073,6 +2254,8 @@ class VllmConfig:
2254 observability_config: Optional[ObservabilityConfig] = None
2255 prompt_adapter_config: Optional[PromptAdapterConfig] = None
2256 quant_config: Optional[QuantizationConfig] = None
2257 + compilation_config: CompilationConfig = field(default=None,
2258 + init=True) # type: ignore
2259
2260 @staticmethod
2261 def _get_quantization_config(
@@ -2133,6 +2316,12 @@ class VllmConfig:
2316 self.quant_config = VllmConfig._get_quantization_config(
2317 self.model_config, self.load_config)
2318
2319 + if self.compilation_config is None:
2320 + self.compilation_config = CompilationConfig.select_and_init_config(
2321 + )
2322 +
2323 + current_platform.check_and_update_config(self)
2324 +
2325 def __str__(self):
2326 return ("model=%r, speculative_config=%r, tokenizer=%r, "
2327 "skip_tokenizer_init=%s, tokenizer_mode=%s, revision=%s, "
vllm/envs.py
-13
@@ -69,7 +69,6 @@ if TYPE_CHECKING:
69 VLLM_SKIP_P2P_CHECK: bool = False
70 VLLM_TORCH_COMPILE_LEVEL: int = 0
71 VLLM_TORCH_COMPILE_CONFIG: Optional[str] = None
72 - VLLM_CUSTOM_OPS: List[str] = []
72 VLLM_DISABLED_KERNELS: List[str] = []
73 VLLM_USE_V1: bool = False
74 VLLM_ENABLE_V1_MULTIPROCESSING: bool = False
@@ -217,18 +216,6 @@ environment_variables: Dict[str, Callable[[], Any]] = {
216 "VLLM_TORCH_COMPILE_CONFIG":
217 lambda: os.environ.get("VLLM_TORCH_COMPILE_CONFIG", None),
218
220 - # Fine-grained control over which custom ops to enable/disable.
221 - # Use 'all' to enable all, 'none' to disable all.
222 - # Also specify a list of custom op names to enable (prefixed with a '+'),
223 - # or disable (prefixed with a '-').
224 - # Examples:
225 - # - 'all,-op1' to enable all except op1
226 - # - 'none,+op1,+op2' to enable only op1 and op2
227 - # By default, all custom ops are enabled when running without Inductor
228 - # and disabled when running with Inductor (compile_level >= Inductor).
229 - "VLLM_CUSTOM_OPS":
230 - lambda: os.environ.get("VLLM_CUSTOM_OPS", "").replace(" ", "").split(","),
231 -
219 # local rank of the process in the distributed setting, used to determine
220 # the GPU device id
221 "LOCAL_RANK":
vllm/model_executor/custom_op.py
+15 -12
@@ -1,12 +1,10 @@
1 -from functools import lru_cache
1 from typing import Dict, Type
2
3 import torch.nn as nn
4
6 -import vllm.envs as envs
7 -from vllm.compilation.levels import CompilationLevel
5 from vllm.logger import init_logger
6 from vllm.platforms import current_platform
7 +from vllm.plugins import get_current_vllm_config
8 from vllm.utils import print_warning_once
9
10 logger = init_logger(__name__)
@@ -87,6 +85,8 @@ class CustomOp(nn.Module):
85 @classmethod
86 def enabled(cls) -> bool:
87 # if no name, then it was not registered
88 + compilation_config = get_current_vllm_config().compilation_config
89 + custom_ops = compilation_config.custom_ops
90 if not hasattr(cls, "name"):
91 print_warning_once(
92 f"Custom op {cls.__name__} was not registered, "
@@ -94,22 +94,25 @@ class CustomOp(nn.Module):
94 f"It will be enabled/disabled based on the global settings.")
95 return CustomOp.default_on()
96
97 - enabled = f"+{cls.name}" in envs.VLLM_CUSTOM_OPS
98 - disabled = f"-{cls.name}" in envs.VLLM_CUSTOM_OPS
97 + enabled = f"+{cls.name}" in custom_ops
98 + disabled = f"-{cls.name}" in custom_ops
99 assert not (enabled
100 and disabled), f"Cannot enable and disable {cls.name}"
101
102 return (CustomOp.default_on() or enabled) and not disabled
103
104 - # On by default if VLLM_TORCH_COMPILE_LEVEL < CompilationLevel.PIECEWISE
105 - # Specifying 'all' or 'none' in VLLM_CUSTOM_OPS takes precedence.
104 @staticmethod
107 - @lru_cache
105 def default_on() -> bool:
109 - count_none = envs.VLLM_CUSTOM_OPS.count("none")
110 - count_all = envs.VLLM_CUSTOM_OPS.count("all")
111 - assert count_none + count_all <= 1, "Can only specify 'none' or 'all'"
112 - return envs.VLLM_TORCH_COMPILE_LEVEL < CompilationLevel.PIECEWISE and \
106 + """
107 + On by default if level < CompilationLevel.PIECEWISE
108 + Specifying 'all' or 'none' in custom_op takes precedence.
109 + """
110 + from vllm.config import CompilationLevel
111 + compilation_config = get_current_vllm_config().compilation_config
112 + custom_ops = compilation_config.custom_ops
113 + count_none = custom_ops.count("none")
114 + count_all = custom_ops.count("all")
115 + return compilation_config.level < CompilationLevel.PIECEWISE and \
116 not count_none > 0 or count_all > 0
117
118 # Dictionary of all custom ops (classes, indexed by registered name).
vllm/model_executor/model_loader/loader.py
+5 -2
@@ -42,6 +42,7 @@ from vllm.model_executor.model_loader.weight_utils import (
42 safetensors_weights_iterator)
43 from vllm.model_executor.utils import set_weight_attrs
44 from vllm.platforms import current_platform
45 +from vllm.plugins import set_current_vllm_config
46 from vllm.utils import is_pin_memory_available
47
48
@@ -97,7 +98,8 @@ def _initialize_model(vllm_config: VllmConfig, prefix: str = "") -> nn.Module:
98 all_params = [param.name for param in signatures.parameters.values()]
99 if "vllm_config" in all_params and "prefix" in all_params:
100 # new-style model class
100 - return model_class(vllm_config=vllm_config, prefix=prefix)
101 + with set_current_vllm_config(vllm_config):
102 + return model_class(vllm_config=vllm_config, prefix=prefix)
103 msg = ("vLLM model class should accept `vllm_config` and `prefix` as "
104 "input arguments. Possibly you have an old-style model class"
105 " registered from out of tree and it is used for new vLLM version. "
@@ -121,7 +123,8 @@ def _initialize_model(vllm_config: VllmConfig, prefix: str = "") -> nn.Module:
123 kwargs["lora_config"] = vllm_config.lora_config
124 if "scheduler_config" in all_params:
125 kwargs["scheduler_config"] = vllm_config.scheduler_config
124 - return model_class(**kwargs)
126 + with set_current_vllm_config(vllm_config):
127 + return model_class(**kwargs)
128
129
130 class BaseModelLoader(ABC):
vllm/platforms/interface.py
+19 -1
@@ -1,10 +1,15 @@
1 import enum
2 import random
3 -from typing import NamedTuple, Optional, Tuple, Union
3 +from typing import TYPE_CHECKING, NamedTuple, Optional, Tuple, Union
4
5 import numpy as np
6 import torch
7
8 +if TYPE_CHECKING:
9 + from vllm.config import VllmConfig
10 +else:
11 + VllmConfig = None
12 +
13
14 class PlatformEnum(enum.Enum):
15 CUDA = enum.auto()
@@ -129,6 +134,19 @@ class Platform:
134 np.random.seed(seed)
135 torch.manual_seed(seed)
136
137 + @classmethod
138 + def check_and_update_config(cls, vllm_config: VllmConfig) -> None:
139 + """
140 + Check and update the configuration for the current platform.
141 +
142 + It can raise an exception if the configuration is not compatible with
143 + the current platform, or it can update the configuration to make it
144 + compatible with the current platform.
145 +
146 + The config is passed by reference, so it can be modified in place.
147 + """
148 + pass
149 +
150
151 class UnspecifiedPlatform(Platform):
152 _enum = PlatformEnum.UNSPECIFIED
vllm/platforms/tpu.py
+14 -7
@@ -1,18 +1,16 @@
1 import os
2 +from typing import TYPE_CHECKING
3
4 import torch
5
5 -import vllm.envs as envs
6 -from vllm.compilation.levels import CompilationLevel
6 from vllm.plugins import set_torch_compile_backend
7
8 from .interface import Platform, PlatformEnum
9
11 -if "VLLM_TORCH_COMPILE_LEVEL" not in os.environ:
12 - os.environ["VLLM_TORCH_COMPILE_LEVEL"] = str(CompilationLevel.DYNAMO_ONCE)
13 -
14 -assert envs.VLLM_TORCH_COMPILE_LEVEL < CompilationLevel.PIECEWISE,\
15 - "TPU does not support Inductor."
10 +if TYPE_CHECKING:
11 + from vllm.config import VllmConfig
12 +else:
13 + VllmConfig = None
14
15 set_torch_compile_backend("openxla")
16
@@ -31,3 +29,12 @@ class TpuPlatform(Platform):
29 @classmethod
30 def inference_mode(cls):
31 return torch.no_grad()
32 +
33 + @classmethod
34 + def check_and_update_config(cls, vllm_config: VllmConfig) -> None:
35 + from vllm.config import CompilationLevel
36 + compilation_config = vllm_config.compilation_config
37 + if "VLLM_TORCH_COMPILE_LEVEL" not in os.environ:
38 + compilation_config.level = CompilationLevel.DYNAMO_ONCE
39 + assert compilation_config.level < CompilationLevel.PIECEWISE,\
40 + "TPU does not support Inductor."
vllm/plugins/__init__.py
+28 -2
@@ -1,11 +1,11 @@
1 import logging
2 +from contextlib import contextmanager
3 from typing import TYPE_CHECKING, Callable, Optional, Union
4
5 import vllm.envs as envs
6
7 if TYPE_CHECKING:
7 - from vllm.compilation.config import CompilationConfig
8 - from vllm.config import VllmConfig
8 + from vllm.config import CompilationConfig, VllmConfig
9 else:
10 CompilationConfig = None
11 VllmConfig = None
@@ -72,3 +72,29 @@ def set_compilation_config(config: Optional[CompilationConfig]):
72
73 def get_compilation_config() -> Optional[CompilationConfig]:
74 return _compilation_config
75 +
76 +
77 +_current_vllm_config: Optional[VllmConfig] = None
78 +
79 +
80 +@contextmanager
81 +def set_current_vllm_config(vllm_config: VllmConfig):
82 + """
83 + Temporarily set the current VLLM config.
84 + Used during model initialization.
85 + We save the current VLLM config in a global variable,
86 + so that all modules can access it, e.g. custom ops
87 + can access the VLLM config to determine how to dispatch.
88 + """
89 + global _current_vllm_config
90 + old_vllm_config = _current_vllm_config
91 + try:
92 + _current_vllm_config = vllm_config
93 + yield
94 + finally:
95 + _current_vllm_config = old_vllm_config
96 +
97 +
98 +def get_current_vllm_config() -> VllmConfig:
99 + assert _current_vllm_config is not None, "Current VLLM config is not set."
100 + return _current_vllm_config
vllm/v1/worker/gpu_model_runner.py
+3 -7
@@ -1,4 +1,3 @@
1 -import os
1 import time
2 from dataclasses import dataclass
3 from typing import TYPE_CHECKING, Dict, List, Optional, Set, Tuple
@@ -8,11 +7,8 @@ import torch
7 import torch.distributed
8 import torch.nn as nn
9
11 -from vllm import envs
10 from vllm.compilation.compile_context import set_compile_context
13 -from vllm.compilation.config import CompilationConfig
14 -from vllm.compilation.levels import CompilationLevel
15 -from vllm.config import VllmConfig
11 +from vllm.config import CompilationConfig, CompilationLevel, VllmConfig
12 from vllm.forward_context import set_forward_context
13 from vllm.inputs import INPUT_REGISTRY, InputRegistry
14 from vllm.logger import init_logger
@@ -99,7 +95,7 @@ class GPUModelRunner:
95 pin_memory=self.pin_memory,
96 )
97
102 - self.use_cuda_graph = (envs.VLLM_TORCH_COMPILE_LEVEL
98 + self.use_cuda_graph = (self.vllm_config.compilation_config.level
99 == CompilationLevel.PIECEWISE
100 and not self.model_config.enforce_eager)
101 # TODO(woosuk): Provide an option to tune the max cudagraph batch size.
@@ -517,9 +513,9 @@ class GPUModelRunner:
513 # CUDA graphs do not work properly with the custom CUDA kernels.
514 # FIXME(woosuk): Disable inductor to reduce the compilation time
515 # and avoid any potential issues with the inductor.
520 - os.environ["VLLM_CUSTOM_OPS"] = "none"
516 set_compilation_config(
517 CompilationConfig(
518 + custom_ops=["none"],
519 use_cudagraph=True,
520 non_cudagraph_ops=["vllm.unified_v1_flash_attention"],
521 use_inductor=True,
vllm/worker/model_runner.py
+3 -4
@@ -19,8 +19,7 @@ from vllm.attention import AttentionMetadata, get_attn_backend
19 from vllm.attention.backends.abstract import AttentionState
20 from vllm.attention.backends.utils import CommonAttentionState
21 from vllm.compilation.compile_context import set_compile_context
22 -from vllm.compilation.levels import CompilationLevel
23 -from vllm.config import VllmConfig
22 +from vllm.config import CompilationLevel, VllmConfig
23 from vllm.core.scheduler import SchedulerOutputs
24 from vllm.distributed import get_pp_group
25 from vllm.distributed.parallel_state import graph_capture
@@ -1142,8 +1141,8 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
1141 "provided. Defaulting to scaling factors of 1.0. "
1142 "This may lead to less accurate results!")
1143
1145 - if envs.VLLM_TORCH_COMPILE_LEVEL == CompilationLevel.DYNAMO_AS_IS \
1146 - and supports_dynamo():
1144 + if self.vllm_config.compilation_config.level ==\
1145 + CompilationLevel.DYNAMO_AS_IS and supports_dynamo():
1146 from vllm.plugins import get_torch_compile_backend
1147 backend = get_torch_compile_backend() or "eager"
1148 self.model = torch.compile(
vllm/worker/tpu_model_runner.py
+5 -3
@@ -140,7 +140,7 @@ class TPUModelRunner(ModelRunnerBase[ModelInputForTPU]):
140 model = get_model(vllm_config=self.vllm_config)
141 model = model.eval()
142 xm.wait_device_ops()
143 - self.model = ModelWrapper(model)
143 + self.model = ModelWrapper(model, self.vllm_config)
144
145 def _dummy_run(
146 self,
@@ -669,13 +669,15 @@ class TPUModelRunner(ModelRunnerBase[ModelInputForTPU]):
669
670 class ModelWrapper(TorchCompileWrapperWithCustomDispatcher):
671
672 - def __init__(self, model: nn.Module):
672 + def __init__(self, model: nn.Module, vllm_config: VllmConfig):
673 self.model = model
674 compiled_callable = torch.compile(self.forward,
675 backend="openxla",
676 fullgraph=True,
677 dynamic=False)
678 - super().__init__(compiled_callable)
678 + super().__init__(
679 + compiled_callable,
680 + compilation_level=vllm_config.compilation_config.level)
681
682 def __call__(self, *args, is_prompt: bool, **kwargs):
683 if len(self.compiled_codes) < 3 or not self.use_custom_dispatcher: