A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 07:00 UTC
[2/N][torch.compile] make compilation cfg part of vllm cfg (#10383)
Signed-off-by: youkaichao <youkaichao@gmail.com>
youkaichao committed
Nov 16, 2024 at 18:02 UTC
4fd937502827a7e06c54ded1f9d9b70ff640e222
27 files changed
+359
-283
tests/compile/piecewise/test_simple.py
+5
-3
@@ -11,8 +11,8 @@ from torch.library import Library
11
from vllm.compilation.compile_context import set_compile_context
12
from vllm.compilation.counter import compilation_counter
13
from vllm.compilation.decorators import support_torch_compile
14
-from vllm.compilation.levels import CompilationLevel
15
-from vllm.config import VllmConfig
14
+from vllm.config import CompilationLevel, VllmConfig
15
+from vllm.plugins import set_current_vllm_config
16
from vllm.utils import direct_register_custom_op
17
18
global_counter = 0
@@ -82,7 +82,9 @@ def test_simple_piecewise_compile():
82
os.environ["VLLM_TORCH_COMPILE_CONFIG"] = config
83
os.environ["VLLM_TORCH_COMPILE_LEVEL"] = str(CompilationLevel.PIECEWISE)
84
85
- model = SillyModel(vllm_config=VllmConfig(), prefix='')
85
+ vllm_config = VllmConfig()
86
+ with set_current_vllm_config(vllm_config):
87
+ model = SillyModel(vllm_config=vllm_config, prefix='')
88
89
inputs = torch.randn(100).cuda()
90
tests/compile/piecewise/test_toy_llama.py
+12
-10
@@ -15,12 +15,10 @@ from torch import nn
15
from torch.library import Library
16
17
from vllm.compilation.compile_context import set_compile_context
18
-from vllm.compilation.config import CompilationConfig
18
from vllm.compilation.counter import compilation_counter
19
from vllm.compilation.decorators import support_torch_compile
21
-from vllm.compilation.levels import CompilationLevel
22
-from vllm.config import VllmConfig
23
-from vllm.plugins import set_compilation_config
20
+from vllm.config import CompilationConfig, CompilationLevel, VllmConfig
21
+from vllm.plugins import set_compilation_config, set_current_vllm_config
22
from vllm.utils import direct_register_custom_op
23
24
# create a library to hold the custom op
@@ -272,9 +270,11 @@ def run_model(llama_config,
270
CompilationLevel.NO_COMPILATION)
271
set_compilation_config(None)
272
275
- model = LlamaModel(config=llama_config,
276
- vllm_config=VllmConfig(),
277
- prefix="").eval().cuda()
273
+ vllm_config = VllmConfig()
274
+ with set_current_vllm_config(vllm_config):
275
+ model = LlamaModel(config=llama_config,
276
+ vllm_config=vllm_config,
277
+ prefix="").eval().cuda()
278
279
B = 16 # max batch size
280
input_ids = torch.randint(0, llama_config.vocab_size, (B, )).cuda()
@@ -395,9 +395,11 @@ def benchmark():
395
else:
396
set_compilation_config(None)
397
398
- model = LlamaModel(config=llama_config,
399
- vllm_config=VllmConfig(),
400
- prefix="").eval().cuda().to(torch.bfloat16)
398
+ vllm_config = VllmConfig()
399
+ with set_current_vllm_config(vllm_config):
400
+ model = LlamaModel(config=llama_config,
401
+ vllm_config=vllm_config,
402
+ prefix="").eval().cuda().to(torch.bfloat16)
403
404
B = 256 # max batch size
405
input_ids = torch.randint(0, llama_config.vocab_size, (B, )).cuda()
tests/compile/test_basic_correctness.py
+1
-1
@@ -3,7 +3,7 @@ from typing import Dict, List, Optional
3
4
import pytest
5
6
-from vllm.compilation.levels import CompilationLevel
6
+from vllm.config import CompilationLevel
7
from vllm.utils import cuda_device_count_stateless
8
9
from ..utils import compare_all_settings
tests/compile/test_full_graph.py
+1
-1
@@ -1,6 +1,6 @@
1
import pytest
2
3
-from vllm.compilation.levels import CompilationLevel
3
+from vllm.config import CompilationLevel
4
5
from ..utils import fork_new_process_for_each_test
6
from .utils import TEST_MODELS, check_full_graph_support
tests/compile/test_fusion.py
+1
-1
@@ -3,10 +3,10 @@ import torch
3
from compressed_tensors.quantization import FP8_DTYPE
4
5
import vllm.envs as envs
6
-from vllm.compilation.config import CompilationConfig
6
from vllm.compilation.fusion import (FusionPass, find_auto_fn,
7
find_auto_fn_maybe)
8
from vllm.compilation.reshapes import RedundantReshapesPass
9
+from vllm.config import CompilationConfig
10
from vllm.model_executor.layers.layernorm import RMSNorm
11
from vllm.model_executor.layers.quantization.utils.w8a8_utils import (
12
apply_fp8_linear)
tests/compile/test_wrapper.py
+3
-1
@@ -3,6 +3,7 @@ from typing import Optional
3
import torch
4
5
from vllm.compilation.wrapper import TorchCompileWrapperWithCustomDispatcher
6
+from vllm.config import CompilationLevel
7
8
9
class MyMod(torch.nn.Module):
@@ -18,7 +19,8 @@ class MyWrapper(TorchCompileWrapperWithCustomDispatcher):
19
def __init__(self, model):
20
self.model = model
21
compiled_callable = torch.compile(self.forward, backend="eager")
21
- super().__init__(compiled_callable)
22
+ super().__init__(compiled_callable,
23
+ compilation_level=CompilationLevel.DYNAMO_ONCE)
24
25
def forward(self, x: torch.Tensor, cache: Optional[torch.Tensor] = None):
26
# this is the function to be compiled
tests/compile/utils.py
+1
-1
@@ -4,7 +4,7 @@ import torch
4
5
from tests.quantization.utils import is_quant_method_supported
6
from vllm import LLM, SamplingParams
7
-from vllm.compilation.levels import CompilationLevel
7
+from vllm.config import CompilationLevel
8
from vllm.platforms import current_platform
9
10
TEST_MODELS = [
tests/model_executor/test_enabled_custom_ops.py
+26
-26
@@ -3,11 +3,13 @@ from typing import List
3
4
import pytest
5
6
+from vllm.config import CompilationConfig, VllmConfig
7
from vllm.model_executor.custom_op import CustomOp
8
from vllm.model_executor.layers.activation import (GeluAndMul,
9
ReLUSquaredActivation,
10
SiluAndMul)
11
from vllm.model_executor.layers.layernorm import RMSNorm
12
+from vllm.plugins import set_current_vllm_config
13
14
15
# Registered subclass for test
@@ -51,42 +53,40 @@ class Relu3(ReLUSquaredActivation):
53
])
54
def test_enabled_ops(env: str, torch_level: int, ops_enabled: List[int],
55
default_on: bool):
54
- os.environ["VLLM_CUSTOM_OPS"] = env
56
os.environ["VLLM_TORCH_COMPILE_LEVEL"] = str(torch_level)
57
+ vllm_config = VllmConfig(compilation_config=CompilationConfig(
58
+ custom_ops=env.split(",")))
59
+ with set_current_vllm_config(vllm_config):
60
+ assert CustomOp.default_on() == default_on
61
57
- # Reset default_on (computed once):
58
- CustomOp.default_on.cache_clear()
62
+ ops_enabled = [bool(x) for x in ops_enabled]
63
60
- assert CustomOp.default_on() == default_on
64
+ assert RMSNorm(1024).enabled() == ops_enabled[0]
65
+ assert CustomOp.op_registry["rms_norm"].enabled() == ops_enabled[0]
66
62
- ops_enabled = [bool(x) for x in ops_enabled]
67
+ assert SiluAndMul().enabled() == ops_enabled[1]
68
+ assert CustomOp.op_registry["silu_and_mul"].enabled() == ops_enabled[1]
69
64
- assert RMSNorm(1024).enabled() == ops_enabled[0]
65
- assert CustomOp.op_registry["rms_norm"].enabled() == ops_enabled[0]
70
+ assert GeluAndMul().enabled() == ops_enabled[2]
71
+ assert CustomOp.op_registry["gelu_and_mul"].enabled() == ops_enabled[2]
72
67
- assert SiluAndMul().enabled() == ops_enabled[1]
68
- assert CustomOp.op_registry["silu_and_mul"].enabled() == ops_enabled[1]
73
+ # If registered, subclasses should follow their own name
74
+ assert Relu3().enabled() == ops_enabled[3]
75
+ assert CustomOp.op_registry["relu3"].enabled() == ops_enabled[3]
76
70
- assert GeluAndMul().enabled() == ops_enabled[2]
71
- assert CustomOp.op_registry["gelu_and_mul"].enabled() == ops_enabled[2]
77
+ # Unregistered subclass
78
+ class SiluAndMul2(SiluAndMul):
79
+ pass
80
73
- # If registered, subclasses should follow their own name
74
- assert Relu3().enabled() == ops_enabled[3]
75
- assert CustomOp.op_registry["relu3"].enabled() == ops_enabled[3]
76
-
77
- # Unregistered subclass
78
- class SiluAndMul2(SiluAndMul):
79
- pass
80
-
81
- # Subclasses should not require registration
82
- assert SiluAndMul2().enabled() == SiluAndMul().enabled()
81
+ # Subclasses should not require registration
82
+ assert SiluAndMul2().enabled() == SiluAndMul().enabled()
83
84
85
@pytest.mark.parametrize(
86
"env", ["all,none", "all,+rms_norm,all", "+rms_norm,-rms_norm"])
87
def test_enabled_ops_invalid(env: str):
88
- os.environ["VLLM_CUSTOM_OPS"] = env
89
- CustomOp.default_on.cache_clear()
90
-
91
- with pytest.raises(AssertionError):
92
- RMSNorm(1024).enabled()
88
+ with pytest.raises(Exception): # noqa
89
+ vllm_config = VllmConfig(compilation_config=CompilationConfig(
90
+ custom_ops=env.split(",")))
91
+ with set_current_vllm_config(vllm_config):
92
+ RMSNorm(1024).enabled()
tests/tpu/test_compilation.py
+1
-1
@@ -5,7 +5,7 @@ import tempfile
5
6
import depyf
7
8
-from vllm.compilation.levels import CompilationLevel
8
+from vllm.config import CompilationLevel
9
10
# disable custom dispatcher, let Dynamo takes over
11
# all the control
tests/tpu/test_custom_dispatcher.py
+1
-1
@@ -1,6 +1,6 @@
1
import os
2
3
-from vllm.compilation.levels import CompilationLevel
3
+from vllm.config import CompilationLevel
4
5
from ..utils import compare_two_settings
6
vllm/compilation/backends.py
+13
-7
@@ -10,13 +10,12 @@ import torch
10
import torch.fx as fx
11
12
import vllm.envs as envs
13
+from vllm.config import CompilationConfig, CompilationLevel
14
from vllm.logger import init_logger
15
from vllm.utils import combine_fx_passes, weak_ref_tensors
16
16
-from .config import CompilationConfig
17
from .counter import compilation_counter
18
from .fusion import FusionPass
19
-from .levels import CompilationLevel
19
from .reshapes import RedundantReshapesPass
20
21
logger = init_logger(__name__)
@@ -392,7 +391,10 @@ class VllmBackend:
391
sym_tensor_indices: List[int]
392
input_buffers: List[torch.Tensor]
393
395
- def __init__(self, post_grad_passes: Sequence[Callable] = ()):
394
+ def __init__(
395
+ self,
396
+ compilation_configs: CompilationConfig,
397
+ ):
398
global global_graph_pool
399
if global_graph_pool is None:
400
global_graph_pool = torch.cuda.graph_pool_handle()
@@ -401,11 +403,13 @@ class VllmBackend:
403
# streams, it might not be safe to share a global pool.
404
# only investigate this when we use multiple streams
405
self.graph_pool = global_graph_pool
404
- self.post_grad_passes = post_grad_passes
406
+ self.post_grad_passes = []
407
408
self.sym_tensor_indices = []
409
self.input_buffers = []
410
411
+ self.compilation_configs = compilation_configs
412
+
413
# `torch.compile` is JIT compiled, so we don't need to
414
# do anything here
415
@@ -437,10 +441,10 @@ class VllmBackend:
441
assert not self._called, "VllmBackend can only be called once"
442
443
self.graph = graph
440
- # config is read now, because only here can
444
+ # config is updated now, because only here can
445
# we get the sizes to capture for cudagraph
446
# from compilation context
443
- self.compilation_configs = CompilationConfig.select_and_init_config()
447
+ self.compilation_configs.init_during_runtime()
448
self.add_passes_to_config()
449
450
self.split_gm, self.piecewise_graphs = split_graph(
@@ -688,4 +692,6 @@ def select_default_backend(level: int) -> Union[str, Callable]:
692
return backend_str
693
assert level == CompilationLevel.PIECEWISE
694
691
- return VllmBackend()
695
+ from vllm.plugins import get_current_vllm_config
696
+ compilation_config = get_current_vllm_config().compilation_config
697
+ return VllmBackend(compilation_config)
vllm/compilation/config.py
deleted
-159
@@ -1,159 +0,0 @@
1
-import copy
2
-from pathlib import Path
3
-from typing import Any, Dict, List, Optional
4
-
5
-from pydantic import BaseModel, Field, PrivateAttr
6
-
7
-import vllm.envs as envs
8
-from vllm.logger import init_logger
9
-
10
-from .compile_context import get_compile_context
11
-
12
-logger = init_logger(__name__)
13
-
14
-
15
-class CompilationConfig(BaseModel):
16
- """
17
- Configuration for compilation.
18
- It has two parts:
19
- - CudaGraph capture:
20
- - use_cudagraph: whether to use cudagraph inside compilation.
21
- - False: cudagraph inside compilation is not used.
22
- - True: cudagraph inside compilation is used. It requires
23
- that all input buffers have fixed addresses.
24
- Note that this is orthogonal to the cudagraph capture out
25
- side of compilation.
26
- TODO: move outside cudagraph logic into compilation.
27
- torch.compile will handle cudagraph capture logic in the future.
28
- - cudagraph_capture_sizes: sizes to capture cudagraph.
29
- - None: capture sizes are inferred from compilation context.
30
- - List[int]: capture sizes are specified.
31
- - cudagraph_num_of_warmups: number of warmup runs for cudagraph.
32
- It means the first several runs will be treated as warmup runs.
33
- Only after that, the execution will be recorded, and the recorded
34
- cudagraph will be used for subsequent runs.
35
- - cudagraph_copy_inputs: whether to copy input tensors for
36
- cudagraph. If the caller can guarantee that the same input buffers
37
- are always used, it can set this to False. Otherwise, it should
38
- set this to True, and the compiler will copy the input to an
39
- internally managed buffer. Default is False.
40
- - Inductor compilation:
41
- - use_inductor: whether to use inductor compilation.
42
- - False: inductor compilation is not used. graph runs in eager.
43
- - True: inductor compilation is used. one graph for symbolic shape
44
- is compiled. In addition, compile for different sizes specified
45
- in inductor_compile_sizes, using configurations
46
- in inductor_compile_config.
47
- - inductor_compile_sizes: sizes to compile for inductor.
48
- - inductor_specialize_for_cudagraph_no_more_than: an optional integer
49
- to specialize inductor for cudagraph sizes no more than the
50
- specified size. It is useful when we want to specialize inductor
51
- with a subset of cudagraph sizes.
52
- - inductor_compile_config: additional configurations for inductor.
53
- - None: use default configurations.
54
- - inductor_passes: additional passes for inductor. It is a dictionary
55
- from pass name to pass function qualified name. We use function
56
- name because the config uses json format. If we pass the config
57
- from Python, functions can also be passed directly via Python object
58
- constructor, e.g. `CompilationConfig(inductor_passes={"a": func})`
59
- - Custom inductor passes:
60
- - dump_graph_stages: list of stages for which we want to dump the graph.
61
- Each pass defines its own stages (before, after, maybe in-between).
62
- - dump_graph_dir: directory to dump the graph. Default is .
63
- - enable_fusion: whether to enable the custom fusion pass.
64
- TODO better pass enabling system.
65
-
66
- Why we have different sizes for cudagraph and inductor:
67
- - cudagraph: a cudagraph captured for a specific size can only be used
68
- for the same size. We need to capture all the sizes we want to use.
69
- - inductor: a graph compiled by inductor for a general shape can be used
70
- for different sizes. Inductor can also compile for specific sizes,
71
- where it can have more information to optimize the graph with fully
72
- static shapes. However, we find the general shape compilation is
73
- sufficient for most cases. It might be beneficial to compile for
74
- certain small batchsizes, where inductor is good at optimizing.
75
- """
76
- use_inductor: bool = True
77
- inductor_specialize_for_cudagraph_no_more_than: Optional[int] = None
78
- inductor_compile_sizes: Optional[List[int]] = Field(default_factory=dict)
79
- inductor_compile_config: Dict = Field(default_factory=dict)
80
- inductor_passes: Dict[str, str] = Field(default_factory=dict)
81
-
82
- use_cudagraph: bool = False
83
- non_cudagraph_ops: List[str] = Field(default_factory=list)
84
- cudagraph_num_of_warmups: int = 0
85
- cudagraph_capture_sizes: Optional[List[int]] = None
86
- cudagraph_copy_inputs: bool = False
87
-
88
- dump_graph_stages: List[str] = Field(default_factory=list)
89
- dump_graph_dir: Path = Field(default=Path("."))
90
- enable_fusion: bool = True
91
-
92
- # not configurable, computed after init
93
- compile_sizes: List[int] = PrivateAttr
94
- capture_sizes: List[int] = PrivateAttr
95
-
96
- def model_post_init(self, __context: Any) -> None:
97
- for k, v in self.inductor_passes.items():
98
- if not isinstance(v, str):
99
- assert callable(v), (
100
- f"pass {k} should be a function or a qualified name")
101
- self.inductor_compile_config[k] = v
102
- continue
103
-
104
- # resolve function from qualified name
105
- names = v.split(".")
106
- module = ".".join(names[:-1])
107
- func_name = names[-1]
108
- func = __import__(module).__dict__[func_name]
109
- self.inductor_compile_config[k] = func
110
-
111
- def init_during_runtime(self):
112
- """To complete the initialization of config,
113
- we need to know the compile context, which is only available
114
- during the first run of the model.
115
- """
116
- context = get_compile_context()
117
- context = copy.deepcopy(context) if context is not None else []
118
- sizes_to_specialize: List[int] = context
119
- if self.cudagraph_capture_sizes is None:
120
- self.capture_sizes = sizes_to_specialize
121
- else:
122
- self.capture_sizes = self.cudagraph_capture_sizes
123
- logger.info(("cudagraph sizes specified by model runner"
124
- " %s is overridden by config %s"),
125
- sizes_to_specialize, self.cudagraph_capture_sizes)
126
- if self.inductor_specialize_for_cudagraph_no_more_than is not None:
127
- assert self.inductor_compile_sizes is None, (
128
- "inductor_compile_sizes should be None when "
129
- "inductor_specialize_for_cudagraph_no_more_than is not None")
130
- self.compile_sizes = [
131
- x for x in self.capture_sizes
132
- if x <= self.inductor_specialize_for_cudagraph_no_more_than
133
- ]
134
- else:
135
- assert self.inductor_compile_sizes is not None, (
136
- "inductor_compile_sizes should not be None when "
137
- "inductor_specialize_for_cudagraph_no_more_than is None")
138
- self.compile_sizes = self.inductor_compile_sizes
139
-
140
- @staticmethod
141
- def select_and_init_config() -> "CompilationConfig":
142
- """The order of selecting config is:
143
- 1. Use the config specified in environment variable.
144
- 2. Use the config specified in plugins.
145
- 3. Use the default config.
146
- """
147
- config_path = envs.VLLM_TORCH_COMPILE_CONFIG
148
- if config_path is not None:
149
- with open(config_path) as json_file:
150
- config = CompilationConfig.model_validate_json(
151
- json_file.read())
152
- else:
153
- from vllm.plugins import get_compilation_config
154
- predefined_config = get_compilation_config()
155
- config = predefined_config if predefined_config is not None else (
156
- CompilationConfig())
157
-
158
- config.init_during_runtime()
159
- return config
vllm/compilation/decorators.py
+5
-5
@@ -3,10 +3,8 @@ from typing import Dict, List, Optional, Union
3
4
import torch
5
6
-import vllm.envs as envs
7
-from vllm.compilation.levels import CompilationLevel
6
from vllm.compilation.wrapper import TorchCompileWrapperWithCustomDispatcher
9
-from vllm.config import VllmConfig
7
+from vllm.config import CompilationLevel, VllmConfig
8
from vllm.logger import init_logger
9
from vllm.sequence import IntermediateTensors
10
from vllm.utils import supports_dynamo
@@ -126,12 +124,14 @@ def _support_torch_compile(cls: type,
124
old_init(self, vllm_config=vllm_config, prefix=prefix, **kwargs)
125
# for CompilationLevel.DYNAMO_AS_IS , the upper level model runner
126
# will handle the compilation, so we don't need to do anything here.
129
- self.do_not_compile = envs.VLLM_TORCH_COMPILE_LEVEL in [
127
+ self.do_not_compile = \
128
+ vllm_config.compilation_config.level in [
129
CompilationLevel.NO_COMPILATION, CompilationLevel.DYNAMO_AS_IS
130
] or not supports_dynamo()
131
if self.do_not_compile:
132
return
134
- TorchCompileWrapperWithCustomDispatcher.__init__(self)
133
+ TorchCompileWrapperWithCustomDispatcher.__init__(
134
+ self, compilation_level=vllm_config.compilation_config.level)
135
136
cls.__init__ = __init__ # type: ignore
137
vllm/compilation/fusion.py
+1
-1
@@ -6,8 +6,8 @@ from torch._higher_order_ops.auto_functionalize import auto_functionalized
6
from torch._inductor.pattern_matcher import (Match, PatternMatcherPass,
7
fwd_only, register_replacement)
8
9
-from vllm.compilation.config import CompilationConfig
9
from vllm.compilation.inductor_pass import InductorPass
10
+from vllm.config import CompilationConfig
11
from vllm.logger import init_logger
12
13
logger = init_logger(__name__)
vllm/compilation/inductor_pass.py
+1
-1
@@ -2,7 +2,7 @@ from abc import ABC, abstractmethod
2
3
import torch
4
5
-from vllm.compilation.config import CompilationConfig
5
+from vllm.config import CompilationConfig
6
# yapf: disable
7
from vllm.distributed import get_tensor_model_parallel_rank as get_tp_rank
8
from vllm.distributed import (
vllm/compilation/levels.py
deleted
-8
@@ -1,8 +0,0 @@
1
-# constants for the levels of the compilation process
2
-
3
-
4
-class CompilationLevel:
5
- NO_COMPILATION = 0
6
- DYNAMO_AS_IS = 1
7
- DYNAMO_ONCE = 2
8
- PIECEWISE = 3
vllm/compilation/wrapper.py
+6
-5
@@ -8,8 +8,7 @@ from typing import Callable, List, Optional
8
import torch
9
10
import vllm.envs as envs
11
-
12
-from .levels import CompilationLevel
11
+from vllm.config import CompilationLevel
12
13
14
class TorchCompileWrapperWithCustomDispatcher:
@@ -25,7 +24,9 @@ class TorchCompileWrapperWithCustomDispatcher:
24
`torch.compile` over the forward method.
25
"""
26
28
- def __init__(self, compiled_callable: Optional[Callable] = None):
27
+ def __init__(self,
28
+ compiled_callable: Optional[Callable] = None,
29
+ compilation_level: int = 0):
30
31
if compiled_callable is None:
32
# default compilation settings
@@ -38,7 +39,7 @@ class TorchCompileWrapperWithCustomDispatcher:
39
backend = get_torch_compile_backend()
40
if backend is None:
41
from vllm.compilation.backends import select_default_backend
41
- backend = select_default_backend(envs.VLLM_TORCH_COMPILE_LEVEL)
42
+ backend = select_default_backend(compilation_level)
43
44
compiled_callable = torch.compile(
45
self.forward,
@@ -54,7 +55,7 @@ class TorchCompileWrapperWithCustomDispatcher:
55
# subclasses can use this to switch between the custom dispatcher
56
# and the default Dynamo guard mechanism.
57
self.use_custom_dispatcher: bool = \
57
- envs.VLLM_TORCH_COMPILE_LEVEL >= CompilationLevel.DYNAMO_ONCE
58
+ compilation_level >= CompilationLevel.DYNAMO_ONCE
59
60
def __call__(self, *args, **kwargs):
61
"""Implement the dispatch logic here, beyond the torch.compile level.
vllm/config.py
+189
@@ -3,10 +3,12 @@ import enum
3
import json
4
import warnings
5
from dataclasses import dataclass, field, replace
6
+from pathlib import Path
7
from typing import (TYPE_CHECKING, Any, Callable, ClassVar, Dict, Final, List,
8
Literal, Mapping, Optional, Set, Tuple, Type, Union)
9
10
import torch
11
+from pydantic import BaseModel, Field, PrivateAttr
12
from transformers import PretrainedConfig
13
14
import vllm.envs as envs
@@ -2052,6 +2054,185 @@ class ObservabilityConfig:
2054
f"installed. Original error:\n{otel_import_error_traceback}")
2055
2056
2057
+class CompilationLevel:
2058
+ # constants for the levels of the compilation process
2059
+ NO_COMPILATION = 0
2060
+ DYNAMO_AS_IS = 1
2061
+ DYNAMO_ONCE = 2
2062
+ PIECEWISE = 3
2063
+
2064
+
2065
+class CompilationConfig(BaseModel):
2066
+ """
2067
+ Configuration for compilation.
2068
+ It has three parts:
2069
+ - Top-level Compilation control:
2070
+ - level: the level of compilation.
2071
+ - 0: no compilation.
2072
+ - 1: dynamo as is.
2073
+ - 2: dynamo once.
2074
+ - 3: piecewise compilation.
2075
+ - custom_ops: fine-grained control over which custom ops to enable/disable.
2076
+ Use 'all' to enable all, 'none' to disable all.
2077
+ Also specify a list of custom op names to enable (prefixed with a '+'),
2078
+ or disable (prefixed with a '-').
2079
+ Examples:
2080
+ - 'all,-op1' to enable all except op1
2081
+ - 'none,+op1,+op2' to enable only op1 and op2
2082
+ By default, all custom ops are enabled when running without Inductor
2083
+ and disabled when running with Inductor (compile_level >= Inductor).
2084
+ - CudaGraph capture:
2085
+ - use_cudagraph: whether to use cudagraph inside compilation.
2086
+ - False: cudagraph inside compilation is not used.
2087
+ - True: cudagraph inside compilation is used. It requires
2088
+ that all input buffers have fixed addresses.
2089
+ Note that this is orthogonal to the cudagraph capture out
2090
+ side of compilation.
2091
+ TODO: move outside cudagraph logic into compilation.
2092
+ torch.compile will handle cudagraph capture logic in the future.
2093
+ - cudagraph_capture_sizes: sizes to capture cudagraph.
2094
+ - None: capture sizes are inferred from compilation context.
2095
+ - List[int]: capture sizes are specified.
2096
+ - cudagraph_num_of_warmups: number of warmup runs for cudagraph.
2097
+ It means the first several runs will be treated as warmup runs.
2098
+ Only after that, the execution will be recorded, and the recorded
2099
+ cudagraph will be used for subsequent runs.
2100
+ - cudagraph_copy_inputs: whether to copy input tensors for
2101
+ cudagraph. If the caller can guarantee that the same input buffers
2102
+ are always used, it can set this to False. Otherwise, it should
2103
+ set this to True, and the compiler will copy the input to an
2104
+ internally managed buffer. Default is False.
2105
+ - Inductor compilation:
2106
+ - use_inductor: whether to use inductor compilation.
2107
+ - False: inductor compilation is not used. graph runs in eager.
2108
+ - True: inductor compilation is used. one graph for symbolic shape
2109
+ is compiled. In addition, compile for different sizes specified
2110
+ in inductor_compile_sizes, using configurations
2111
+ in inductor_compile_config.
2112
+ - inductor_compile_sizes: sizes to compile for inductor.
2113
+ - inductor_specialize_for_cudagraph_no_more_than: an optional integer
2114
+ to specialize inductor for cudagraph sizes no more than the
2115
+ specified size. It is useful when we want to specialize inductor
2116
+ with a subset of cudagraph sizes.
2117
+ - inductor_compile_config: additional configurations for inductor.
2118
+ - None: use default configurations.
2119
+ - inductor_passes: additional passes for inductor. It is a dictionary
2120
+ from pass name to pass function qualified name. We use function
2121
+ name because the config uses json format. If we pass the config
2122
+ from Python, functions can also be passed directly via Python object
2123
+ constructor, e.g. `CompilationConfig(inductor_passes={"a": func})`
2124
+ - custom inductor passes:
2125
+ - dump_graph_stages: list of stages for which we want to dump the graph.
2126
+ Each pass defines its own stages (before, after, maybe in-between).
2127
+ - dump_graph_dir: directory to dump the graph. Default is .
2128
+ - enable_fusion: whether to enable the custom fusion pass.
2129
+ TODO better pass enabling system.
2130
+
2131
+ Why we have different sizes for cudagraph and inductor:
2132
+ - cudagraph: a cudagraph captured for a specific size can only be used
2133
+ for the same size. We need to capture all the sizes we want to use.
2134
+ - inductor: a graph compiled by inductor for a general shape can be used
2135
+ for different sizes. Inductor can also compile for specific sizes,
2136
+ where it can have more information to optimize the graph with fully
2137
+ static shapes. However, we find the general shape compilation is
2138
+ sufficient for most cases. It might be beneficial to compile for
2139
+ certain small batchsizes, where inductor is good at optimizing.
2140
+ """ # noqa
2141
+ level: int = 0
2142
+ custom_ops: List[str] = Field(default_factory=list)
2143
+
2144
+ use_inductor: bool = True
2145
+ inductor_specialize_for_cudagraph_no_more_than: Optional[int] = None
2146
+ inductor_compile_sizes: Optional[List[int]] = Field(default_factory=dict)
2147
+ inductor_compile_config: Dict = Field(default_factory=dict)
2148
+ inductor_passes: Dict[str, str] = Field(default_factory=dict)
2149
+
2150
+ use_cudagraph: bool = False
2151
+ non_cudagraph_ops: List[str] = Field(default_factory=list)
2152
+ cudagraph_num_of_warmups: int = 0
2153
+ cudagraph_capture_sizes: Optional[List[int]] = None
2154
+ cudagraph_copy_inputs: bool = False
2155
+
2156
+ dump_graph_stages: List[str] = Field(default_factory=list)
2157
+ dump_graph_dir: Path = Field(default=Path("."))
2158
+ enable_fusion: bool = True
2159
+
2160
+ # not configurable, computed after init
2161
+ compile_sizes: List[int] = PrivateAttr
2162
+ capture_sizes: List[int] = PrivateAttr
2163
+
2164
+ def model_post_init(self, __context: Any) -> None:
2165
+ self.level = envs.VLLM_TORCH_COMPILE_LEVEL
2166
+
2167
+ count_none = self.custom_ops.count("none")
2168
+ count_all = self.custom_ops.count("all")
2169
+ assert count_none + count_all <= 1, "Can only specify 'none' or 'all'"
2170
+
2171
+ for k, v in self.inductor_passes.items():
2172
+ if not isinstance(v, str):
2173
+ assert callable(v), (
2174
+ f"pass {k} should be a function or a qualified name")
2175
+ self.inductor_compile_config[k] = v
2176
+ continue
2177
+
2178
+ # resolve function from qualified name
2179
+ names = v.split(".")
2180
+ module = ".".join(names[:-1])
2181
+ func_name = names[-1]
2182
+ func = __import__(module).__dict__[func_name]
2183
+ self.inductor_compile_config[k] = func
2184
+
2185
+ def init_during_runtime(self):
2186
+ """To complete the initialization of config,
2187
+ we need to know the compile context, which is only available
2188
+ during the first run of the model.
2189
+ """
2190
+ from vllm.compilation.compile_context import get_compile_context
2191
+ context = get_compile_context()
2192
+ context = copy.deepcopy(context) if context is not None else []
2193
+ sizes_to_specialize: List[int] = context
2194
+ if self.cudagraph_capture_sizes is None:
2195
+ self.capture_sizes = sizes_to_specialize
2196
+ else:
2197
+ self.capture_sizes = self.cudagraph_capture_sizes
2198
+ logger.info(("cudagraph sizes specified by model runner"
2199
+ " %s is overridden by config %s"),
2200
+ sizes_to_specialize, self.cudagraph_capture_sizes)
2201
+ if self.inductor_specialize_for_cudagraph_no_more_than is not None:
2202
+ assert self.inductor_compile_sizes is None, (
2203
+ "inductor_compile_sizes should be None when "
2204
+ "inductor_specialize_for_cudagraph_no_more_than is not None")
2205
+ self.compile_sizes = [
2206
+ x for x in self.capture_sizes
2207
+ if x <= self.inductor_specialize_for_cudagraph_no_more_than
2208
+ ]
2209
+ else:
2210
+ assert self.inductor_compile_sizes is not None, (
2211
+ "inductor_compile_sizes should not be None when "
2212
+ "inductor_specialize_for_cudagraph_no_more_than is None")
2213
+ self.compile_sizes = self.inductor_compile_sizes
2214
+
2215
+ @staticmethod
2216
+ def select_and_init_config() -> "CompilationConfig":
2217
+ """The order of selecting config is:
2218
+ 1. Use the config specified in environment variable.
2219
+ 2. Use the config specified in plugins.
2220
+ 3. Use the default config.
2221
+ """
2222
+ config_path = envs.VLLM_TORCH_COMPILE_CONFIG
2223
+ if config_path is not None:
2224
+ with open(config_path) as json_file:
2225
+ config = CompilationConfig.model_validate_json(
2226
+ json_file.read())
2227
+ else:
2228
+ from vllm.plugins import get_compilation_config
2229
+ predefined_config = get_compilation_config()
2230
+ config = predefined_config if predefined_config is not None else (
2231
+ CompilationConfig())
2232
+
2233
+ return config
2234
+
2235
+
2236
@dataclass
2237
class VllmConfig:
2238
"""Dataclass which contains all vllm-related configuration. This
@@ -2073,6 +2254,8 @@ class VllmConfig:
2254
observability_config: Optional[ObservabilityConfig] = None
2255
prompt_adapter_config: Optional[PromptAdapterConfig] = None
2256
quant_config: Optional[QuantizationConfig] = None
2257
+ compilation_config: CompilationConfig = field(default=None,
2258
+ init=True) # type: ignore
2259
2260
@staticmethod
2261
def _get_quantization_config(
@@ -2133,6 +2316,12 @@ class VllmConfig:
2316
self.quant_config = VllmConfig._get_quantization_config(
2317
self.model_config, self.load_config)
2318
2319
+ if self.compilation_config is None:
2320
+ self.compilation_config = CompilationConfig.select_and_init_config(
2321
+ )
2322
+
2323
+ current_platform.check_and_update_config(self)
2324
+
2325
def __str__(self):
2326
return ("model=%r, speculative_config=%r, tokenizer=%r, "
2327
"skip_tokenizer_init=%s, tokenizer_mode=%s, revision=%s, "
vllm/envs.py
-13
@@ -69,7 +69,6 @@ if TYPE_CHECKING:
69
VLLM_SKIP_P2P_CHECK: bool = False
70
VLLM_TORCH_COMPILE_LEVEL: int = 0
71
VLLM_TORCH_COMPILE_CONFIG: Optional[str] = None
72
- VLLM_CUSTOM_OPS: List[str] = []
72
VLLM_DISABLED_KERNELS: List[str] = []
73
VLLM_USE_V1: bool = False
74
VLLM_ENABLE_V1_MULTIPROCESSING: bool = False
@@ -217,18 +216,6 @@ environment_variables: Dict[str, Callable[[], Any]] = {
216
"VLLM_TORCH_COMPILE_CONFIG":
217
lambda: os.environ.get("VLLM_TORCH_COMPILE_CONFIG", None),
218
220
- # Fine-grained control over which custom ops to enable/disable.
221
- # Use 'all' to enable all, 'none' to disable all.
222
- # Also specify a list of custom op names to enable (prefixed with a '+'),
223
- # or disable (prefixed with a '-').
224
- # Examples:
225
- # - 'all,-op1' to enable all except op1
226
- # - 'none,+op1,+op2' to enable only op1 and op2
227
- # By default, all custom ops are enabled when running without Inductor
228
- # and disabled when running with Inductor (compile_level >= Inductor).
229
- "VLLM_CUSTOM_OPS":
230
- lambda: os.environ.get("VLLM_CUSTOM_OPS", "").replace(" ", "").split(","),
231
-
219
# local rank of the process in the distributed setting, used to determine
220
# the GPU device id
221
"LOCAL_RANK":
vllm/model_executor/custom_op.py
+15
-12
@@ -1,12 +1,10 @@
1
-from functools import lru_cache
1
from typing import Dict, Type
2
3
import torch.nn as nn
4
6
-import vllm.envs as envs
7
-from vllm.compilation.levels import CompilationLevel
5
from vllm.logger import init_logger
6
from vllm.platforms import current_platform
7
+from vllm.plugins import get_current_vllm_config
8
from vllm.utils import print_warning_once
9
10
logger = init_logger(__name__)
@@ -87,6 +85,8 @@ class CustomOp(nn.Module):
85
@classmethod
86
def enabled(cls) -> bool:
87
# if no name, then it was not registered
88
+ compilation_config = get_current_vllm_config().compilation_config
89
+ custom_ops = compilation_config.custom_ops
90
if not hasattr(cls, "name"):
91
print_warning_once(
92
f"Custom op {cls.__name__} was not registered, "
@@ -94,22 +94,25 @@ class CustomOp(nn.Module):
94
f"It will be enabled/disabled based on the global settings.")
95
return CustomOp.default_on()
96
97
- enabled = f"+{cls.name}" in envs.VLLM_CUSTOM_OPS
98
- disabled = f"-{cls.name}" in envs.VLLM_CUSTOM_OPS
97
+ enabled = f"+{cls.name}" in custom_ops
98
+ disabled = f"-{cls.name}" in custom_ops
99
assert not (enabled
100
and disabled), f"Cannot enable and disable {cls.name}"
101
102
return (CustomOp.default_on() or enabled) and not disabled
103
104
- # On by default if VLLM_TORCH_COMPILE_LEVEL < CompilationLevel.PIECEWISE
105
- # Specifying 'all' or 'none' in VLLM_CUSTOM_OPS takes precedence.
104
@staticmethod
107
- @lru_cache
105
def default_on() -> bool:
109
- count_none = envs.VLLM_CUSTOM_OPS.count("none")
110
- count_all = envs.VLLM_CUSTOM_OPS.count("all")
111
- assert count_none + count_all <= 1, "Can only specify 'none' or 'all'"
112
- return envs.VLLM_TORCH_COMPILE_LEVEL < CompilationLevel.PIECEWISE and \
106
+ """
107
+ On by default if level < CompilationLevel.PIECEWISE
108
+ Specifying 'all' or 'none' in custom_op takes precedence.
109
+ """
110
+ from vllm.config import CompilationLevel
111
+ compilation_config = get_current_vllm_config().compilation_config
112
+ custom_ops = compilation_config.custom_ops
113
+ count_none = custom_ops.count("none")
114
+ count_all = custom_ops.count("all")
115
+ return compilation_config.level < CompilationLevel.PIECEWISE and \
116
not count_none > 0 or count_all > 0
117
118
# Dictionary of all custom ops (classes, indexed by registered name).
vllm/model_executor/model_loader/loader.py
+5
-2
@@ -42,6 +42,7 @@ from vllm.model_executor.model_loader.weight_utils import (
42
safetensors_weights_iterator)
43
from vllm.model_executor.utils import set_weight_attrs
44
from vllm.platforms import current_platform
45
+from vllm.plugins import set_current_vllm_config
46
from vllm.utils import is_pin_memory_available
47
48
@@ -97,7 +98,8 @@ def _initialize_model(vllm_config: VllmConfig, prefix: str = "") -> nn.Module:
98
all_params = [param.name for param in signatures.parameters.values()]
99
if "vllm_config" in all_params and "prefix" in all_params:
100
# new-style model class
100
- return model_class(vllm_config=vllm_config, prefix=prefix)
101
+ with set_current_vllm_config(vllm_config):
102
+ return model_class(vllm_config=vllm_config, prefix=prefix)
103
msg = ("vLLM model class should accept `vllm_config` and `prefix` as "
104
"input arguments. Possibly you have an old-style model class"
105
" registered from out of tree and it is used for new vLLM version. "
@@ -121,7 +123,8 @@ def _initialize_model(vllm_config: VllmConfig, prefix: str = "") -> nn.Module:
123
kwargs["lora_config"] = vllm_config.lora_config
124
if "scheduler_config" in all_params:
125
kwargs["scheduler_config"] = vllm_config.scheduler_config
124
- return model_class(**kwargs)
126
+ with set_current_vllm_config(vllm_config):
127
+ return model_class(**kwargs)
128
129
130
class BaseModelLoader(ABC):
vllm/platforms/interface.py
+19
-1
@@ -1,10 +1,15 @@
1
import enum
2
import random
3
-from typing import NamedTuple, Optional, Tuple, Union
3
+from typing import TYPE_CHECKING, NamedTuple, Optional, Tuple, Union
4
5
import numpy as np
6
import torch
7
8
+if TYPE_CHECKING:
9
+ from vllm.config import VllmConfig
10
+else:
11
+ VllmConfig = None
12
+
13
14
class PlatformEnum(enum.Enum):
15
CUDA = enum.auto()
@@ -129,6 +134,19 @@ class Platform:
134
np.random.seed(seed)
135
torch.manual_seed(seed)
136
137
+ @classmethod
138
+ def check_and_update_config(cls, vllm_config: VllmConfig) -> None:
139
+ """
140
+ Check and update the configuration for the current platform.
141
+
142
+ It can raise an exception if the configuration is not compatible with
143
+ the current platform, or it can update the configuration to make it
144
+ compatible with the current platform.
145
+
146
+ The config is passed by reference, so it can be modified in place.
147
+ """
148
+ pass
149
+
150
151
class UnspecifiedPlatform(Platform):
152
_enum = PlatformEnum.UNSPECIFIED
vllm/platforms/tpu.py
+14
-7
@@ -1,18 +1,16 @@
1
import os
2
+from typing import TYPE_CHECKING
3
4
import torch
5
5
-import vllm.envs as envs
6
-from vllm.compilation.levels import CompilationLevel
6
from vllm.plugins import set_torch_compile_backend
7
8
from .interface import Platform, PlatformEnum
9
11
-if "VLLM_TORCH_COMPILE_LEVEL" not in os.environ:
12
- os.environ["VLLM_TORCH_COMPILE_LEVEL"] = str(CompilationLevel.DYNAMO_ONCE)
13
-
14
-assert envs.VLLM_TORCH_COMPILE_LEVEL < CompilationLevel.PIECEWISE,\
15
- "TPU does not support Inductor."
10
+if TYPE_CHECKING:
11
+ from vllm.config import VllmConfig
12
+else:
13
+ VllmConfig = None
14
15
set_torch_compile_backend("openxla")
16
@@ -31,3 +29,12 @@ class TpuPlatform(Platform):
29
@classmethod
30
def inference_mode(cls):
31
return torch.no_grad()
32
+
33
+ @classmethod
34
+ def check_and_update_config(cls, vllm_config: VllmConfig) -> None:
35
+ from vllm.config import CompilationLevel
36
+ compilation_config = vllm_config.compilation_config
37
+ if "VLLM_TORCH_COMPILE_LEVEL" not in os.environ:
38
+ compilation_config.level = CompilationLevel.DYNAMO_ONCE
39
+ assert compilation_config.level < CompilationLevel.PIECEWISE,\
40
+ "TPU does not support Inductor."
vllm/plugins/__init__.py
+28
-2
@@ -1,11 +1,11 @@
1
import logging
2
+from contextlib import contextmanager
3
from typing import TYPE_CHECKING, Callable, Optional, Union
4
5
import vllm.envs as envs
6
7
if TYPE_CHECKING:
7
- from vllm.compilation.config import CompilationConfig
8
- from vllm.config import VllmConfig
8
+ from vllm.config import CompilationConfig, VllmConfig
9
else:
10
CompilationConfig = None
11
VllmConfig = None
@@ -72,3 +72,29 @@ def set_compilation_config(config: Optional[CompilationConfig]):
72
73
def get_compilation_config() -> Optional[CompilationConfig]:
74
return _compilation_config
75
+
76
+
77
+_current_vllm_config: Optional[VllmConfig] = None
78
+
79
+
80
+@contextmanager
81
+def set_current_vllm_config(vllm_config: VllmConfig):
82
+ """
83
+ Temporarily set the current VLLM config.
84
+ Used during model initialization.
85
+ We save the current VLLM config in a global variable,
86
+ so that all modules can access it, e.g. custom ops
87
+ can access the VLLM config to determine how to dispatch.
88
+ """
89
+ global _current_vllm_config
90
+ old_vllm_config = _current_vllm_config
91
+ try:
92
+ _current_vllm_config = vllm_config
93
+ yield
94
+ finally:
95
+ _current_vllm_config = old_vllm_config
96
+
97
+
98
+def get_current_vllm_config() -> VllmConfig:
99
+ assert _current_vllm_config is not None, "Current VLLM config is not set."
100
+ return _current_vllm_config
vllm/v1/worker/gpu_model_runner.py
+3
-7
@@ -1,4 +1,3 @@
1
-import os
1
import time
2
from dataclasses import dataclass
3
from typing import TYPE_CHECKING, Dict, List, Optional, Set, Tuple
@@ -8,11 +7,8 @@ import torch
7
import torch.distributed
8
import torch.nn as nn
9
11
-from vllm import envs
10
from vllm.compilation.compile_context import set_compile_context
13
-from vllm.compilation.config import CompilationConfig
14
-from vllm.compilation.levels import CompilationLevel
15
-from vllm.config import VllmConfig
11
+from vllm.config import CompilationConfig, CompilationLevel, VllmConfig
12
from vllm.forward_context import set_forward_context
13
from vllm.inputs import INPUT_REGISTRY, InputRegistry
14
from vllm.logger import init_logger
@@ -99,7 +95,7 @@ class GPUModelRunner:
95
pin_memory=self.pin_memory,
96
)
97
102
- self.use_cuda_graph = (envs.VLLM_TORCH_COMPILE_LEVEL
98
+ self.use_cuda_graph = (self.vllm_config.compilation_config.level
99
== CompilationLevel.PIECEWISE
100
and not self.model_config.enforce_eager)
101
# TODO(woosuk): Provide an option to tune the max cudagraph batch size.
@@ -517,9 +513,9 @@ class GPUModelRunner:
513
# CUDA graphs do not work properly with the custom CUDA kernels.
514
# FIXME(woosuk): Disable inductor to reduce the compilation time
515
# and avoid any potential issues with the inductor.
520
- os.environ["VLLM_CUSTOM_OPS"] = "none"
516
set_compilation_config(
517
CompilationConfig(
518
+ custom_ops=["none"],
519
use_cudagraph=True,
520
non_cudagraph_ops=["vllm.unified_v1_flash_attention"],
521
use_inductor=True,
vllm/worker/model_runner.py
+3
-4
@@ -19,8 +19,7 @@ from vllm.attention import AttentionMetadata, get_attn_backend
19
from vllm.attention.backends.abstract import AttentionState
20
from vllm.attention.backends.utils import CommonAttentionState
21
from vllm.compilation.compile_context import set_compile_context
22
-from vllm.compilation.levels import CompilationLevel
23
-from vllm.config import VllmConfig
22
+from vllm.config import CompilationLevel, VllmConfig
23
from vllm.core.scheduler import SchedulerOutputs
24
from vllm.distributed import get_pp_group
25
from vllm.distributed.parallel_state import graph_capture
@@ -1142,8 +1141,8 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
1141
"provided. Defaulting to scaling factors of 1.0. "
1142
"This may lead to less accurate results!")
1143
1145
- if envs.VLLM_TORCH_COMPILE_LEVEL == CompilationLevel.DYNAMO_AS_IS \
1146
- and supports_dynamo():
1144
+ if self.vllm_config.compilation_config.level ==\
1145
+ CompilationLevel.DYNAMO_AS_IS and supports_dynamo():
1146
from vllm.plugins import get_torch_compile_backend
1147
backend = get_torch_compile_backend() or "eager"
1148
self.model = torch.compile(
vllm/worker/tpu_model_runner.py
+5
-3
@@ -140,7 +140,7 @@ class TPUModelRunner(ModelRunnerBase[ModelInputForTPU]):
140
model = get_model(vllm_config=self.vllm_config)
141
model = model.eval()
142
xm.wait_device_ops()
143
- self.model = ModelWrapper(model)
143
+ self.model = ModelWrapper(model, self.vllm_config)
144
145
def _dummy_run(
146
self,
@@ -669,13 +669,15 @@ class TPUModelRunner(ModelRunnerBase[ModelInputForTPU]):
669
670
class ModelWrapper(TorchCompileWrapperWithCustomDispatcher):
671
672
- def __init__(self, model: nn.Module):
672
+ def __init__(self, model: nn.Module, vllm_config: VllmConfig):
673
self.model = model
674
compiled_callable = torch.compile(self.forward,
675
backend="openxla",
676
fullgraph=True,
677
dynamic=False)
678
- super().__init__(compiled_callable)
678
+ super().__init__(
679
+ compiled_callable,
680
+ compilation_level=vllm_config.compilation_config.level)
681
682
def __call__(self, *args, is_prompt: bool, **kwargs):
683
if len(self.compiled_codes) < 3 or not self.use_custom_dispatcher: