@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / f877a7d12a
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 05:45 UTC

[Misc] Improve type annotations for `support_torch_compile` (#10763)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>

Cyrus Leung committed Dec 1, 2024 at 09:48 UTC f877a7d12a0490705e6bea0987c89548d1a015ea
1 file changed +29 -9
vllm/compilation/decorators.py
+29 -9
@@ -1,7 +1,8 @@
1 import inspect
2 -from typing import Dict, List, Optional, Union
2 +from typing import Callable, Dict, List, Optional, TypeVar, Union, overload
3
4 import torch
5 +import torch.nn as nn
6
7 from vllm.compilation.counter import compilation_counter
8 from vllm.compilation.wrapper import TorchCompileWrapperWithCustomDispatcher
@@ -12,10 +13,27 @@ from vllm.utils import supports_dynamo
13
14 logger = init_logger(__name__)
15
16 +_T = TypeVar("_T", bound=type[nn.Module])
17 +
18 +
19 +@overload
20 +def support_torch_compile(
21 + *,
22 + dynamic_arg_dims: Optional[Dict[str, Union[int, List[int]]]],
23 +) -> Callable[[_T], _T]:
24 + ...
25 +
26 +
27 +@overload
28 +def support_torch_compile(cls: _T) -> _T:
29 + ...
30 +
31
32 def support_torch_compile(
17 - cls: Optional[type] = None,
18 - dynamic_arg_dims: Optional[Dict[str, Union[int, List[int]]]] = None):
33 + cls: Optional[_T] = None,
34 + *,
35 + dynamic_arg_dims: Optional[Dict[str, Union[int, List[int]]]] = None,
36 +) -> Union[Callable[[_T], _T], _T]:
37 """
38 A decorator to add support for compiling the forward method of a class.
39
@@ -66,7 +84,7 @@ def support_torch_compile(
84 computation graph.
85 """
86
69 - def cls_decorator_helper(cls: type):
87 + def cls_decorator_helper(cls: _T) -> _T:
88 # helper to pass `dynamic_arg_dims`` to `_support_torch_compile``
89 # to avoid too much indentation for `_support_torch_compile``
90 if not hasattr(cls, 'forward'):
@@ -105,8 +123,10 @@ def support_torch_compile(
123 return cls_decorator_helper
124
125
108 -def _support_torch_compile(cls: type,
109 - dynamic_arg_dims: Dict[str, Union[int, List[int]]]):
126 +def _support_torch_compile(
127 + cls: _T,
128 + dynamic_arg_dims: Dict[str, Union[int, List[int]]],
129 +) -> _T:
130 """
131 A decorator to add support for compiling the forward method of a class.
132 """
@@ -119,7 +139,7 @@ def _support_torch_compile(cls: type,
139 # other than TorchCompileWrapperWithCustomDispatcher
140 cls.__bases__ = cls.__bases__ + (TorchCompileWrapperWithCustomDispatcher, )
141
122 - old_init = cls.__init__ # type: ignore
142 + old_init = cls.__init__
143
144 def __init__(self, *, vllm_config: VllmConfig, prefix: str = '', **kwargs):
145 old_init(self, vllm_config=vllm_config, prefix=prefix, **kwargs)
@@ -135,7 +155,7 @@ def _support_torch_compile(cls: type,
155 TorchCompileWrapperWithCustomDispatcher.__init__(
156 self, compilation_level=vllm_config.compilation_config.level)
157
138 - cls.__init__ = __init__ # type: ignore
158 + cls.__init__ = __init__
159
160 def __call__(self, *args, **kwargs):
161 # torch.compiler.is_compiling() means we are inside the compilation
@@ -180,5 +200,5 @@ def _support_torch_compile(cls: type,
200 model_output = self.forward(*args, **kwargs)
201 return model_output
202
183 - cls.__call__ = __call__ # type: ignore
203 + cls.__call__ = __call__
204 return cls