A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 05:45 UTC
[Misc] Improve type annotations for `support_torch_compile` (#10763)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
Cyrus Leung committed
Dec 1, 2024 at 09:48 UTC
f877a7d12a0490705e6bea0987c89548d1a015ea
1 file changed
+29
-9
vllm/compilation/decorators.py
+29
-9
@@ -1,7 +1,8 @@
1
import inspect
2
-from typing import Dict, List, Optional, Union
2
+from typing import Callable, Dict, List, Optional, TypeVar, Union, overload
3
4
import torch
5
+import torch.nn as nn
6
7
from vllm.compilation.counter import compilation_counter
8
from vllm.compilation.wrapper import TorchCompileWrapperWithCustomDispatcher
@@ -12,10 +13,27 @@ from vllm.utils import supports_dynamo
13
14
logger = init_logger(__name__)
15
16
+_T = TypeVar("_T", bound=type[nn.Module])
17
+
18
+
19
+@overload
20
+def support_torch_compile(
21
+ *,
22
+ dynamic_arg_dims: Optional[Dict[str, Union[int, List[int]]]],
23
+) -> Callable[[_T], _T]:
24
+ ...
25
+
26
+
27
+@overload
28
+def support_torch_compile(cls: _T) -> _T:
29
+ ...
30
+
31
32
def support_torch_compile(
17
- cls: Optional[type] = None,
18
- dynamic_arg_dims: Optional[Dict[str, Union[int, List[int]]]] = None):
33
+ cls: Optional[_T] = None,
34
+ *,
35
+ dynamic_arg_dims: Optional[Dict[str, Union[int, List[int]]]] = None,
36
+) -> Union[Callable[[_T], _T], _T]:
37
"""
38
A decorator to add support for compiling the forward method of a class.
39
@@ -66,7 +84,7 @@ def support_torch_compile(
84
computation graph.
85
"""
86
69
- def cls_decorator_helper(cls: type):
87
+ def cls_decorator_helper(cls: _T) -> _T:
88
# helper to pass `dynamic_arg_dims`` to `_support_torch_compile``
89
# to avoid too much indentation for `_support_torch_compile``
90
if not hasattr(cls, 'forward'):
@@ -105,8 +123,10 @@ def support_torch_compile(
123
return cls_decorator_helper
124
125
108
-def _support_torch_compile(cls: type,
109
- dynamic_arg_dims: Dict[str, Union[int, List[int]]]):
126
+def _support_torch_compile(
127
+ cls: _T,
128
+ dynamic_arg_dims: Dict[str, Union[int, List[int]]],
129
+) -> _T:
130
"""
131
A decorator to add support for compiling the forward method of a class.
132
"""
@@ -119,7 +139,7 @@ def _support_torch_compile(cls: type,
139
# other than TorchCompileWrapperWithCustomDispatcher
140
cls.__bases__ = cls.__bases__ + (TorchCompileWrapperWithCustomDispatcher, )
141
122
- old_init = cls.__init__ # type: ignore
142
+ old_init = cls.__init__
143
144
def __init__(self, *, vllm_config: VllmConfig, prefix: str = '', **kwargs):
145
old_init(self, vllm_config=vllm_config, prefix=prefix, **kwargs)
@@ -135,7 +155,7 @@ def _support_torch_compile(cls: type,
155
TorchCompileWrapperWithCustomDispatcher.__init__(
156
self, compilation_level=vllm_config.compilation_config.level)
157
138
- cls.__init__ = __init__ # type: ignore
158
+ cls.__init__ = __init__
159
160
def __call__(self, *args, **kwargs):
161
# torch.compiler.is_compiling() means we are inside the compilation
@@ -180,5 +200,5 @@ def _support_torch_compile(cls: type,
200
model_output = self.forward(*args, **kwargs)
201
return model_output
202
183
- cls.__call__ = __call__ # type: ignore
203
+ cls.__call__ = __call__
204
return cls