A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 08:15 UTC
[1/N] torch.compile user interface design (#10237)
Signed-off-by: youkaichao <youkaichao@gmail.com>
youkaichao committed
Nov 11, 2024 at 18:01 UTC
eea55cca5b0896eab7fa213291090f70c858a3bc
4 files changed
+55
-37
tests/compile/piecewise/test_simple.py
+9
-5
@@ -12,10 +12,9 @@ from vllm.compilation.compile_context import set_compile_context
12
from vllm.compilation.counter import compilation_counter
13
from vllm.compilation.decorators import support_torch_compile
14
from vllm.compilation.levels import CompilationLevel
15
+from vllm.config import VllmConfig
16
from vllm.utils import direct_register_custom_op
17
17
-os.environ["VLLM_TORCH_COMPILE_LEVEL"] = str(CompilationLevel.PIECEWISE)
18
-
18
global_counter = 0
19
20
# create a library to hold the custom op
@@ -48,7 +47,11 @@ direct_register_custom_op(
47
@support_torch_compile
48
class SillyModel(nn.Module):
49
51
- def __init__(self) -> None:
50
+ def __init__(self,
51
+ *,
52
+ vllm_config: VllmConfig,
53
+ prefix: str = '',
54
+ **kwargs) -> None:
55
super().__init__()
56
57
def forward(self, x: torch.Tensor) -> torch.Tensor:
@@ -74,11 +77,12 @@ class SillyModel(nn.Module):
77
78
def test_simple_piecewise_compile():
79
77
- model = SillyModel()
78
-
80
directory = os.path.dirname(__file__)
81
config = os.path.join(directory, "piecewise_compilation_config.json")
82
os.environ["VLLM_TORCH_COMPILE_CONFIG"] = config
83
+ os.environ["VLLM_TORCH_COMPILE_LEVEL"] = str(CompilationLevel.PIECEWISE)
84
+
85
+ model = SillyModel(vllm_config=VllmConfig(), prefix='')
86
87
inputs = torch.randn(100).cuda()
88
tests/compile/piecewise/test_toy_llama.py
+14
-7
@@ -19,6 +19,7 @@ from vllm.compilation.config import CompilationConfig
19
from vllm.compilation.counter import compilation_counter
20
from vllm.compilation.decorators import support_torch_compile
21
from vllm.compilation.levels import CompilationLevel
22
+from vllm.config import VllmConfig
23
from vllm.plugins import set_compilation_config
24
from vllm.utils import direct_register_custom_op
25
@@ -195,9 +196,15 @@ class LlamaDecoderLayer(nn.Module):
196
return hidden_states, residual
197
198
199
+@support_torch_compile
200
class LlamaModel(nn.Module):
201
200
- def __init__(self, config: LlamaConfig) -> None:
202
+ def __init__(self,
203
+ *,
204
+ vllm_config: VllmConfig,
205
+ config: LlamaConfig,
206
+ prefix: str = '',
207
+ **kwargs) -> None:
208
super().__init__()
209
self.embedding_tokens = nn.Embedding(
210
num_embeddings=config.vocab_size,
@@ -265,10 +272,9 @@ def run_model(llama_config,
272
CompilationLevel.NO_COMPILATION)
273
set_compilation_config(None)
274
268
- cls = LlamaModel
269
- if use_compile:
270
- cls = support_torch_compile(LlamaModel)
271
- model = cls(llama_config).eval().cuda()
275
+ model = LlamaModel(config=llama_config,
276
+ vllm_config=VllmConfig(),
277
+ prefix="").eval().cuda()
278
279
B = 16 # max batch size
280
input_ids = torch.randint(0, llama_config.vocab_size, (B, )).cuda()
@@ -357,7 +363,6 @@ def test_toy_llama():
363
def benchmark():
364
os.environ["VLLM_TORCH_COMPILE_LEVEL"] = str(CompilationLevel.PIECEWISE)
365
from triton.testing import do_bench
360
- cls = support_torch_compile(LlamaModel)
366
367
# similar to llama 3.1-8B
368
llama_config = LlamaConfig(hidden_size=4096,
@@ -390,7 +395,9 @@ def benchmark():
395
else:
396
set_compilation_config(None)
397
393
- model = cls(llama_config).eval().cuda().to(torch.bfloat16)
398
+ model = LlamaModel(config=llama_config,
399
+ vllm_config=VllmConfig(),
400
+ prefix="").eval().cuda().to(torch.bfloat16)
401
402
B = 256 # max batch size
403
input_ids = torch.randint(0, llama_config.vocab_size, (B, )).cuda()
vllm/compilation/decorators.py
+14
-13
@@ -6,6 +6,7 @@ import torch
6
import vllm.envs as envs
7
from vllm.compilation.levels import CompilationLevel
8
from vllm.compilation.wrapper import TorchCompileWrapperWithCustomDispatcher
9
+from vllm.config import VllmConfig
10
from vllm.logger import init_logger
11
from vllm.sequence import IntermediateTensors
12
from vllm.utils import supports_dynamo
@@ -110,26 +111,26 @@ def _support_torch_compile(cls: type,
111
"""
112
A decorator to add support for compiling the forward method of a class.
113
"""
113
-
114
- # for CompilationLevel.DYNAMO_AS_IS , the upper level model runner
115
- # will handle the compilation, so we don't need to do anything here.
116
- if envs.VLLM_TORCH_COMPILE_LEVEL in [
117
- CompilationLevel.NO_COMPILATION, CompilationLevel.DYNAMO_AS_IS
118
- ] or not supports_dynamo():
114
+ if TorchCompileWrapperWithCustomDispatcher in cls.__bases__:
115
+ # support decorating multiple times
116
return cls
117
118
# take care of method resolution order
119
# make sure super().__init__ is called on the base class
120
# other than TorchCompileWrapperWithCustomDispatcher
124
- if TorchCompileWrapperWithCustomDispatcher not in cls.__bases__:
125
- # support decorating multiple times
126
- cls.__bases__ = cls.__bases__ + (
127
- TorchCompileWrapperWithCustomDispatcher, )
121
+ cls.__bases__ = cls.__bases__ + (TorchCompileWrapperWithCustomDispatcher, )
122
123
old_init = cls.__init__ # type: ignore
124
131
- def __init__(self, *args, **kwargs):
132
- old_init(self, *args, **kwargs)
125
+ def __init__(self, *, vllm_config: VllmConfig, prefix: str = '', **kwargs):
126
+ old_init(self, vllm_config=vllm_config, prefix=prefix, **kwargs)
127
+ # for CompilationLevel.DYNAMO_AS_IS , the upper level model runner
128
+ # will handle the compilation, so we don't need to do anything here.
129
+ self.do_not_compile = envs.VLLM_TORCH_COMPILE_LEVEL in [
130
+ CompilationLevel.NO_COMPILATION, CompilationLevel.DYNAMO_AS_IS
131
+ ] or not supports_dynamo()
132
+ if self.do_not_compile:
133
+ return
134
TorchCompileWrapperWithCustomDispatcher.__init__(self)
135
136
cls.__init__ = __init__ # type: ignore
@@ -138,7 +139,7 @@ def _support_torch_compile(cls: type,
139
# torch.compiler.is_compiling() means we are inside the compilation
140
# e.g. TPU has the compilation logic in model runner, so we don't
141
# need to compile the model inside.
141
- if torch.compiler.is_compiling():
142
+ if self.do_not_compile or torch.compiler.is_compiling():
143
return self.forward(*args, **kwargs)
144
145
# the first compilation needs to have dynamic shapes marked
vllm/config.py
+18
-12
@@ -2041,12 +2041,15 @@ class VllmConfig:
2041
simplifies passing around the distinct configurations in the codebase.
2042
"""
2043
2044
- model_config: ModelConfig
2045
- cache_config: CacheConfig
2046
- parallel_config: ParallelConfig
2047
- scheduler_config: SchedulerConfig
2048
- device_config: DeviceConfig
2049
- load_config: LoadConfig
2044
+ model_config: ModelConfig = field(default=None, init=True) # type: ignore
2045
+ cache_config: CacheConfig = field(default=None, init=True) # type: ignore
2046
+ parallel_config: ParallelConfig = field(default=None,
2047
+ init=True) # type: ignore
2048
+ scheduler_config: SchedulerConfig = field(default=None,
2049
+ init=True) # type: ignore
2050
+ device_config: DeviceConfig = field(default=None,
2051
+ init=True) # type: ignore
2052
+ load_config: LoadConfig = field(default=None, init=True) # type: ignore
2053
lora_config: Optional[LoRAConfig] = None
2054
speculative_config: Optional[SpeculativeConfig] = None
2055
decoding_config: Optional[DecodingConfig] = None
@@ -2091,11 +2094,14 @@ class VllmConfig:
2094
def __post_init__(self):
2095
"""Verify configs are valid & consistent with each other.
2096
"""
2094
- self.model_config.verify_async_output_proc(self.parallel_config,
2095
- self.speculative_config,
2096
- self.device_config)
2097
- self.model_config.verify_with_parallel_config(self.parallel_config)
2098
- self.cache_config.verify_with_parallel_config(self.parallel_config)
2097
+ if self.model_config is not None:
2098
+ self.model_config.verify_async_output_proc(self.parallel_config,
2099
+ self.speculative_config,
2100
+ self.device_config)
2101
+ self.model_config.verify_with_parallel_config(self.parallel_config)
2102
+
2103
+ if self.cache_config is not None:
2104
+ self.cache_config.verify_with_parallel_config(self.parallel_config)
2105
2106
if self.lora_config:
2107
self.lora_config.verify_with_model_config(self.model_config)
@@ -2149,4 +2155,4 @@ class VllmConfig:
2155
self.scheduler_config.num_scheduler_steps,
2156
self.cache_config.enable_prefix_caching,
2157
self.model_config.use_async_output_proc,
2152
- self.model_config.mm_processor_kwargs)
\ No newline at end of file
2158
+ self.model_config.mm_processor_kwargs)