@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / eea55cca5b
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 08:15 UTC

[1/N] torch.compile user interface design (#10237)

Signed-off-by: youkaichao <youkaichao@gmail.com>

youkaichao committed Nov 11, 2024 at 18:01 UTC eea55cca5b0896eab7fa213291090f70c858a3bc
4 files changed +55 -37
tests/compile/piecewise/test_simple.py
+9 -5
@@ -12,10 +12,9 @@ from vllm.compilation.compile_context import set_compile_context
12 from vllm.compilation.counter import compilation_counter
13 from vllm.compilation.decorators import support_torch_compile
14 from vllm.compilation.levels import CompilationLevel
15 +from vllm.config import VllmConfig
16 from vllm.utils import direct_register_custom_op
17
17 -os.environ["VLLM_TORCH_COMPILE_LEVEL"] = str(CompilationLevel.PIECEWISE)
18 -
18 global_counter = 0
19
20 # create a library to hold the custom op
@@ -48,7 +47,11 @@ direct_register_custom_op(
47 @support_torch_compile
48 class SillyModel(nn.Module):
49
51 - def __init__(self) -> None:
50 + def __init__(self,
51 + *,
52 + vllm_config: VllmConfig,
53 + prefix: str = '',
54 + **kwargs) -> None:
55 super().__init__()
56
57 def forward(self, x: torch.Tensor) -> torch.Tensor:
@@ -74,11 +77,12 @@ class SillyModel(nn.Module):
77
78 def test_simple_piecewise_compile():
79
77 - model = SillyModel()
78 -
80 directory = os.path.dirname(__file__)
81 config = os.path.join(directory, "piecewise_compilation_config.json")
82 os.environ["VLLM_TORCH_COMPILE_CONFIG"] = config
83 + os.environ["VLLM_TORCH_COMPILE_LEVEL"] = str(CompilationLevel.PIECEWISE)
84 +
85 + model = SillyModel(vllm_config=VllmConfig(), prefix='')
86
87 inputs = torch.randn(100).cuda()
88
tests/compile/piecewise/test_toy_llama.py
+14 -7
@@ -19,6 +19,7 @@ from vllm.compilation.config import CompilationConfig
19 from vllm.compilation.counter import compilation_counter
20 from vllm.compilation.decorators import support_torch_compile
21 from vllm.compilation.levels import CompilationLevel
22 +from vllm.config import VllmConfig
23 from vllm.plugins import set_compilation_config
24 from vllm.utils import direct_register_custom_op
25
@@ -195,9 +196,15 @@ class LlamaDecoderLayer(nn.Module):
196 return hidden_states, residual
197
198
199 +@support_torch_compile
200 class LlamaModel(nn.Module):
201
200 - def __init__(self, config: LlamaConfig) -> None:
202 + def __init__(self,
203 + *,
204 + vllm_config: VllmConfig,
205 + config: LlamaConfig,
206 + prefix: str = '',
207 + **kwargs) -> None:
208 super().__init__()
209 self.embedding_tokens = nn.Embedding(
210 num_embeddings=config.vocab_size,
@@ -265,10 +272,9 @@ def run_model(llama_config,
272 CompilationLevel.NO_COMPILATION)
273 set_compilation_config(None)
274
268 - cls = LlamaModel
269 - if use_compile:
270 - cls = support_torch_compile(LlamaModel)
271 - model = cls(llama_config).eval().cuda()
275 + model = LlamaModel(config=llama_config,
276 + vllm_config=VllmConfig(),
277 + prefix="").eval().cuda()
278
279 B = 16 # max batch size
280 input_ids = torch.randint(0, llama_config.vocab_size, (B, )).cuda()
@@ -357,7 +363,6 @@ def test_toy_llama():
363 def benchmark():
364 os.environ["VLLM_TORCH_COMPILE_LEVEL"] = str(CompilationLevel.PIECEWISE)
365 from triton.testing import do_bench
360 - cls = support_torch_compile(LlamaModel)
366
367 # similar to llama 3.1-8B
368 llama_config = LlamaConfig(hidden_size=4096,
@@ -390,7 +395,9 @@ def benchmark():
395 else:
396 set_compilation_config(None)
397
393 - model = cls(llama_config).eval().cuda().to(torch.bfloat16)
398 + model = LlamaModel(config=llama_config,
399 + vllm_config=VllmConfig(),
400 + prefix="").eval().cuda().to(torch.bfloat16)
401
402 B = 256 # max batch size
403 input_ids = torch.randint(0, llama_config.vocab_size, (B, )).cuda()
vllm/compilation/decorators.py
+14 -13
@@ -6,6 +6,7 @@ import torch
6 import vllm.envs as envs
7 from vllm.compilation.levels import CompilationLevel
8 from vllm.compilation.wrapper import TorchCompileWrapperWithCustomDispatcher
9 +from vllm.config import VllmConfig
10 from vllm.logger import init_logger
11 from vllm.sequence import IntermediateTensors
12 from vllm.utils import supports_dynamo
@@ -110,26 +111,26 @@ def _support_torch_compile(cls: type,
111 """
112 A decorator to add support for compiling the forward method of a class.
113 """
113 -
114 - # for CompilationLevel.DYNAMO_AS_IS , the upper level model runner
115 - # will handle the compilation, so we don't need to do anything here.
116 - if envs.VLLM_TORCH_COMPILE_LEVEL in [
117 - CompilationLevel.NO_COMPILATION, CompilationLevel.DYNAMO_AS_IS
118 - ] or not supports_dynamo():
114 + if TorchCompileWrapperWithCustomDispatcher in cls.__bases__:
115 + # support decorating multiple times
116 return cls
117
118 # take care of method resolution order
119 # make sure super().__init__ is called on the base class
120 # other than TorchCompileWrapperWithCustomDispatcher
124 - if TorchCompileWrapperWithCustomDispatcher not in cls.__bases__:
125 - # support decorating multiple times
126 - cls.__bases__ = cls.__bases__ + (
127 - TorchCompileWrapperWithCustomDispatcher, )
121 + cls.__bases__ = cls.__bases__ + (TorchCompileWrapperWithCustomDispatcher, )
122
123 old_init = cls.__init__ # type: ignore
124
131 - def __init__(self, *args, **kwargs):
132 - old_init(self, *args, **kwargs)
125 + def __init__(self, *, vllm_config: VllmConfig, prefix: str = '', **kwargs):
126 + old_init(self, vllm_config=vllm_config, prefix=prefix, **kwargs)
127 + # for CompilationLevel.DYNAMO_AS_IS , the upper level model runner
128 + # will handle the compilation, so we don't need to do anything here.
129 + self.do_not_compile = envs.VLLM_TORCH_COMPILE_LEVEL in [
130 + CompilationLevel.NO_COMPILATION, CompilationLevel.DYNAMO_AS_IS
131 + ] or not supports_dynamo()
132 + if self.do_not_compile:
133 + return
134 TorchCompileWrapperWithCustomDispatcher.__init__(self)
135
136 cls.__init__ = __init__ # type: ignore
@@ -138,7 +139,7 @@ def _support_torch_compile(cls: type,
139 # torch.compiler.is_compiling() means we are inside the compilation
140 # e.g. TPU has the compilation logic in model runner, so we don't
141 # need to compile the model inside.
141 - if torch.compiler.is_compiling():
142 + if self.do_not_compile or torch.compiler.is_compiling():
143 return self.forward(*args, **kwargs)
144
145 # the first compilation needs to have dynamic shapes marked
vllm/config.py
+18 -12
@@ -2041,12 +2041,15 @@ class VllmConfig:
2041 simplifies passing around the distinct configurations in the codebase.
2042 """
2043
2044 - model_config: ModelConfig
2045 - cache_config: CacheConfig
2046 - parallel_config: ParallelConfig
2047 - scheduler_config: SchedulerConfig
2048 - device_config: DeviceConfig
2049 - load_config: LoadConfig
2044 + model_config: ModelConfig = field(default=None, init=True) # type: ignore
2045 + cache_config: CacheConfig = field(default=None, init=True) # type: ignore
2046 + parallel_config: ParallelConfig = field(default=None,
2047 + init=True) # type: ignore
2048 + scheduler_config: SchedulerConfig = field(default=None,
2049 + init=True) # type: ignore
2050 + device_config: DeviceConfig = field(default=None,
2051 + init=True) # type: ignore
2052 + load_config: LoadConfig = field(default=None, init=True) # type: ignore
2053 lora_config: Optional[LoRAConfig] = None
2054 speculative_config: Optional[SpeculativeConfig] = None
2055 decoding_config: Optional[DecodingConfig] = None
@@ -2091,11 +2094,14 @@ class VllmConfig:
2094 def __post_init__(self):
2095 """Verify configs are valid & consistent with each other.
2096 """
2094 - self.model_config.verify_async_output_proc(self.parallel_config,
2095 - self.speculative_config,
2096 - self.device_config)
2097 - self.model_config.verify_with_parallel_config(self.parallel_config)
2098 - self.cache_config.verify_with_parallel_config(self.parallel_config)
2097 + if self.model_config is not None:
2098 + self.model_config.verify_async_output_proc(self.parallel_config,
2099 + self.speculative_config,
2100 + self.device_config)
2101 + self.model_config.verify_with_parallel_config(self.parallel_config)
2102 +
2103 + if self.cache_config is not None:
2104 + self.cache_config.verify_with_parallel_config(self.parallel_config)
2105
2106 if self.lora_config:
2107 self.lora_config.verify_with_model_config(self.model_config)
@@ -2149,4 +2155,4 @@ class VllmConfig:
2155 self.scheduler_config.num_scheduler_steps,
2156 self.cache_config.enable_prefix_caching,
2157 self.model_config.use_async_output_proc,
2152 - self.model_config.mm_processor_kwargs)
\ No newline at end of file
2158 + self.model_config.mm_processor_kwargs)