A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 09:30 UTC
[torch.compile] auto infer dynamic_arg_dims from type annotation (#9589)
youkaichao committed
Oct 22, 2024 at 13:43 UTC
17c79f3c364be166b68923bced94f902c00bd8bb
3 files changed
+65
-19
vllm/compilation/decorators.py
+63
-5
@@ -1,24 +1,58 @@
1
import inspect
2
-from typing import Dict, List, Union
2
+from typing import Dict, List, Optional, Union
3
4
import torch
5
6
import vllm.envs as envs
7
from vllm.compilation.levels import CompilationLevel
8
from vllm.compilation.wrapper import TorchCompileWrapperWithCustomDispatcher
9
+from vllm.logger import init_logger
10
from vllm.sequence import IntermediateTensors
11
from vllm.utils import supports_dynamo
12
13
+logger = init_logger(__name__)
14
13
-def support_torch_compile(dynamic_arg_dims: Dict[str, Union[int, List[int]]]):
15
+
16
+def support_torch_compile(
17
+ cls: Optional[type] = None,
18
+ dynamic_arg_dims: Optional[Dict[str, Union[int, List[int]]]] = None):
19
"""
20
A decorator to add support for compiling the forward method of a class.
21
22
+ Usage 1: use directly as a decorator without arguments:
23
+
24
+ ```python
25
+ @support_torch_compile
26
+ class MyModel(nn.Module):
27
+ def forward(self, x: torch.Tensor, y: Optional[torch.Tensor]):
28
+ ...
29
+ ```
30
+
31
+ Usage 2: use as a decorator with arguments:
32
+
33
+ ```python
34
+ @support_torch_compile(dynamic_arg_dims={"x": 0, "y": 0})
35
+ class MyModel(nn.Module):
36
+ def forward(self, x: torch.Tensor, y: Optional[torch.Tensor]):
37
+ ...
38
+ ```
39
+
40
`dynamic_arg_dims` is a dictionary that maps argument names to the dynamic
41
dimensions of the argument. The dynamic dimensions can be either a single
42
integer or a list of integers.
43
21
- Depending on the value of arguments:
44
+ if `dynamic_arg_dims` is `None`, it is inferred from the type annotation
45
+ of the `forward` method, based on the following default rules:
46
+
47
+ - if the argument is annotated as `torch.Tensor` or
48
+ `Optional[torch.Tensor]`, the first dimension will be
49
+ marked as dynamic.
50
+ - if the argument is annotated as `IntermediateTensors`, the first
51
+ dimension of all the tensors in the intermediate tensors
52
+ will be marked as dynamic.
53
+
54
+ During runtime, when we actually mark dimensions of tensors,
55
+ it depends on the value of arguments:
56
57
- if it is a single integer, the corresponding dimension of the argument
58
will be marked as dynamic.
@@ -38,11 +72,35 @@ def support_torch_compile(dynamic_arg_dims: Dict[str, Union[int, List[int]]]):
72
if not hasattr(cls, 'forward'):
73
raise TypeError("decorated class should have a forward method.")
74
sig = inspect.signature(cls.forward)
41
- for k in dynamic_arg_dims:
75
+ inferred_dynamic_arg_dims = dynamic_arg_dims
76
+ if inferred_dynamic_arg_dims is None:
77
+ inferred_dynamic_arg_dims = {}
78
+ for k, v in sig.parameters.items():
79
+ if v.annotation in [
80
+ torch.Tensor, Optional[torch.Tensor],
81
+ IntermediateTensors, Optional[IntermediateTensors]
82
+ ]:
83
+ inferred_dynamic_arg_dims[k] = 0
84
+
85
+ logger.debug(("Inferred dynamic dimensions for "
86
+ "forward method of %s: %s"), cls,
87
+ list(inferred_dynamic_arg_dims.keys()))
88
+
89
+ if len(inferred_dynamic_arg_dims) == 0:
90
+ raise ValueError(
91
+ "No dynamic dimensions found in the forward method of "
92
+ f"{cls}. Please provide dynamic_arg_dims explicitly.")
93
+
94
+ for k in inferred_dynamic_arg_dims:
95
if k not in sig.parameters:
96
raise ValueError(
97
f"Argument {k} not found in the forward method of {cls}")
45
- return _support_torch_compile(cls, dynamic_arg_dims)
98
+ return _support_torch_compile(cls, inferred_dynamic_arg_dims)
99
+
100
+ if cls is not None:
101
+ # use `support_torch_compile` as a decorator without arguments
102
+ assert isinstance(cls, type)
103
+ return cls_decorator_helper(cls)
104
105
return cls_decorator_helper
106
vllm/model_executor/models/gemma2.py
+1
-7
@@ -241,13 +241,7 @@ class Gemma2DecoderLayer(nn.Module):
241
return hidden_states, residual
242
243
244
-@support_torch_compile(
245
- dynamic_arg_dims={
246
- "input_ids": 0,
247
- "positions": 0,
248
- "inputs_embeds": 0,
249
- "intermediate_tensors": 0,
250
- })
244
+@support_torch_compile
245
class Gemma2Model(nn.Module):
246
247
def __init__(
vllm/model_executor/models/llama.py
+1
-7
@@ -268,13 +268,7 @@ class LlamaDecoderLayer(nn.Module):
268
return hidden_states, residual
269
270
271
-@support_torch_compile(
272
- dynamic_arg_dims={
273
- "input_ids": 0,
274
- "positions": 0,
275
- "inputs_embeds": 0,
276
- "intermediate_tensors": 0,
277
- })
271
+@support_torch_compile
272
class LlamaModel(nn.Module):
273
274
def __init__(