A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 10:45 UTC
[torch.compile] generic decorators (#9258)
youkaichao committed
Oct 10, 2024 at 15:54 UTC
e00c094f15e79c5a113fdf975df1ee9018cb65b3
3 files changed
+74
-34
vllm/compilation/decorators.py
+58
-30
@@ -1,20 +1,54 @@
1
-from typing import List, Optional, Union
1
+import inspect
2
+from typing import Dict, List, Union
3
4
import torch
5
6
import vllm.envs as envs
6
-from vllm.attention import AttentionMetadata
7
from vllm.compilation.levels import CompilationLevel
8
from vllm.compilation.wrapper import TorchCompileWrapperWithCustomDispatcher
9
from vllm.sequence import IntermediateTensors
10
from vllm.utils import supports_dynamo
11
12
13
-def support_compile_llama_style(cls: type):
13
+def support_torch_compile(dynamic_arg_dims: Dict[str, Union[int, List[int]]]):
14
+ """
15
+ A decorator to add support for compiling the forward method of a class.
16
+
17
+ `dynamic_arg_dims` is a dictionary that maps argument names to the dynamic
18
+ dimensions of the argument. The dynamic dimensions can be either a single
19
+ integer or a list of integers.
20
+
21
+ Depending on the value of arguments:
22
+
23
+ - if it is a single integer, the corresponding dimension of the argument
24
+ will be marked as dynamic.
25
+ - if it is `None`, ignored.
26
+ - if it is `IntermediateTensors`, all the tensors in the intermediate
27
+ tensors will be marked as dynamic.
28
+ - otherwise, it will raise an error.
29
+
30
+ NOTE: if an argument is `None`, it should always be passed as `None` during
31
+ the lifetime of the model, otherwise, it cannot be captured as a single
32
+ computation graph.
33
+ """
34
+
35
+ def cls_decorator_helper(cls: type):
36
+ # helper to pass `dynamic_arg_dims`` to `_support_torch_compile``
37
+ # to avoid too much indentation for `_support_torch_compile``
38
+ sig = inspect.signature(cls.forward)
39
+ for k in dynamic_arg_dims:
40
+ if k not in sig.parameters:
41
+ raise ValueError(
42
+ f"Argument {k} not found in the forward method of {cls}")
43
+ return _support_torch_compile(cls, dynamic_arg_dims)
44
+
45
+ return cls_decorator_helper
46
+
47
+
48
+def _support_torch_compile(cls: type,
49
+ dynamic_arg_dims: Dict[str, Union[int, List[int]]]):
50
"""
51
A decorator to add support for compiling the forward method of a class.
16
- If a module's **forward signature** is compatible with llama, this
17
- decorator can be used to enable the compilation of the forward method.
52
"""
53
54
# for CompilationLevel.DYNAMO_AS_IS , the upper level model runner
@@ -37,48 +71,42 @@ def support_compile_llama_style(cls: type):
71
72
cls.__init__ = __init__
73
40
- def __call__(
41
- self,
42
- input_ids: Optional[torch.Tensor],
43
- positions: torch.Tensor,
44
- kv_caches: List[torch.Tensor],
45
- attn_metadata: AttentionMetadata,
46
- intermediate_tensors: Optional[IntermediateTensors],
47
- inputs_embeds: Optional[torch.Tensor] = None,
48
- ) -> Union[torch.Tensor, IntermediateTensors]:
74
+ def __call__(self, *args, **kwargs):
75
# torch.compiler.is_compiling() means we are inside the compilation
76
# e.g. TPU has the compilation logic in model runner, so we don't
77
# need to compile the model inside.
78
if torch.compiler.is_compiling():
53
- return self.forward(input_ids, positions, kv_caches, attn_metadata,
54
- intermediate_tensors, inputs_embeds)
79
+ return self.forward(*args, **kwargs)
80
81
# the first compilation needs to have dynamic shapes marked
82
if len(self.compiled_codes) < 1:
58
- if input_ids is not None:
59
- torch._dynamo.mark_dynamic(input_ids, 0)
60
- torch._dynamo.mark_dynamic(positions, 0)
61
- if inputs_embeds is not None:
62
- torch._dynamo.mark_dynamic(inputs_embeds, 0)
63
- if intermediate_tensors is not None:
64
- for tensors in intermediate_tensors.tensors.values():
65
- torch._dynamo.mark_dynamic(tensors, 0)
83
+ sig = inspect.signature(self.__class__.forward)
84
+ bound_args = sig.bind(self, *args, **kwargs)
85
+ bound_args.apply_defaults()
86
+ for k, dims in dynamic_arg_dims.items():
87
+ arg = bound_args.arguments.get(k)
88
+ if arg is not None:
89
+ if isinstance(arg, torch.Tensor):
90
+ torch._dynamo.mark_dynamic(arg, dims)
91
+ elif isinstance(arg, IntermediateTensors):
92
+ for tensor in arg.tensors.values():
93
+ torch._dynamo.mark_dynamic(tensor, dims)
94
+ else:
95
+ raise ValueError(
96
+ "Unsupported dynamic dimensions"
97
+ f" {dims} for argument {k} with type {type(arg)}.")
98
99
# if we don't use custom dispatcher, we can directly call the
100
# compiled function and let torch.compile handle the dispatching,
101
# with the overhead of guard evaluation and recompilation.
102
if len(self.compiled_codes) < 1 or not self.use_custom_dispatcher:
71
- return self.compiled_callable(input_ids, positions, kv_caches,
72
- attn_metadata, intermediate_tensors,
73
- inputs_embeds)
103
+ return self.compiled_callable(*args, **kwargs)
104
105
# usually, capturing the model once is enough, and then we can
106
# dispatch to the compiled code directly, without going through
107
# the Dynamo guard mechanism.
108
with self.dispatch_to_code(0):
79
- model_output = self.forward(input_ids, positions, kv_caches,
80
- attn_metadata, intermediate_tensors,
81
- inputs_embeds)
109
+ model_output = self.forward(*args, **kwargs)
110
return model_output
111
112
cls.__call__ = __call__
vllm/model_executor/models/gemma2.py
+8
-2
@@ -21,7 +21,7 @@ from torch import nn
21
from transformers import Gemma2Config
22
23
from vllm.attention import Attention, AttentionMetadata
24
-from vllm.compilation.decorators import support_compile_llama_style
24
+from vllm.compilation.decorators import support_torch_compile
25
from vllm.config import CacheConfig, LoRAConfig
26
from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size
27
from vllm.logger import init_logger
@@ -239,7 +239,13 @@ class Gemma2DecoderLayer(nn.Module):
239
return hidden_states, residual
240
241
242
-@support_compile_llama_style
242
+@support_torch_compile(
243
+ dynamic_arg_dims={
244
+ "input_ids": 0,
245
+ "positions": 0,
246
+ "inputs_embeds": 0,
247
+ "intermediate_tensors": 0,
248
+ })
249
class Gemma2Model(nn.Module):
250
251
def __init__(
vllm/model_executor/models/llama.py
+8
-2
@@ -28,7 +28,7 @@ from torch import nn
28
from transformers import LlamaConfig
29
30
from vllm.attention import Attention, AttentionMetadata
31
-from vllm.compilation.decorators import support_compile_llama_style
31
+from vllm.compilation.decorators import support_torch_compile
32
from vllm.config import CacheConfig, LoRAConfig
33
from vllm.distributed import (get_pp_group, get_tensor_model_parallel_rank,
34
get_tensor_model_parallel_world_size)
@@ -266,7 +266,13 @@ class LlamaDecoderLayer(nn.Module):
266
return hidden_states, residual
267
268
269
-@support_compile_llama_style
269
+@support_torch_compile(
270
+ dynamic_arg_dims={
271
+ "input_ids": 0,
272
+ "positions": 0,
273
+ "inputs_embeds": 0,
274
+ "intermediate_tensors": 0,
275
+ })
276
class LlamaModel(nn.Module):
277
278
def __init__(