A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 04:30 UTC
Mypy checking for vllm/compilation (#11496)
Signed-off-by: lucast2021 <lucast2021@headroyce.org> Co-authored-by: lucast2021 <lucast2021@headroyce.org>
Lucas Tucker committed
Dec 25, 2024 at 23:04 UTC
dbeac95dbbf898bcc0965528fc767e9cadbbe0c5
3 files changed
+9
-8
vllm/compilation/backends.py
+5
-5
@@ -141,14 +141,14 @@ class AlwaysHitShapeEnv:
141
return ""
142
143
144
-def wrap_inductor(graph,
144
+def wrap_inductor(graph: fx.GraphModule,
145
example_inputs,
146
additional_inductor_config,
147
compilation_config: CompilationConfig,
148
graph_index: int = 0,
149
num_graphs: int = 1,
150
runtime_shape: Optional[int] = None,
151
- use_inductor: bool = True):
151
+ use_inductor: bool = True) -> Any:
152
if graph_index == 0:
153
# before compiling the first graph, record the start time
154
global compilation_start_time
@@ -209,7 +209,7 @@ def wrap_inductor(graph,
209
returns_tuple = graph_returns_tuple(graph)
210
211
# this is the graph we return to Dynamo to run
212
- def compiled_graph(*args):
212
+ def compiled_graph(*args) -> Optional[fx.CompiledFxGraph]:
213
# convert args to list
214
list_args = list(args)
215
graph_output = inductor_compiled_graph(list_args)
@@ -247,7 +247,7 @@ def wrap_inductor(graph,
247
# see https://github.com/pytorch/pytorch/blob/9f5ebf3fc609105a74eab4ccc24932d6353ff566/torch/_inductor/codecache.py#L1221 # noqa
248
return
249
250
- def _get_shape_env():
250
+ def _get_shape_env() -> AlwaysHitShapeEnv:
251
return AlwaysHitShapeEnv()
252
253
with patch(# for hijacking the hash of the compiled graph
@@ -537,7 +537,7 @@ class VllmBackend:
537
example_inputs[x].clone() for x in self.sym_tensor_indices
538
]
539
540
- def copy_and_call(*args):
540
+ def copy_and_call(*args) -> fx.GraphModule:
541
list_args = list(args)
542
for i, index in enumerate(self.sym_tensor_indices):
543
runtime_tensor = list_args[index]
vllm/compilation/multi_output_match.py
+2
-1
@@ -7,6 +7,7 @@ from torch import fx
7
from torch._higher_order_ops.auto_functionalize import auto_functionalized
8
from torch._inductor import pattern_matcher as pm
9
from torch._ops import OpOverload
10
+from torch.fx import Node
11
12
from vllm.compilation.fx_utils import find_auto_fn
13
@@ -97,7 +98,7 @@ class MultiOutputMatch(abc.ABC):
98
self.graph.call_function(operator.getitem, (tuple_node, idx))
99
for idx in indices)
100
100
- def insert_auto_fn(self, op: OpOverload, kwargs):
101
+ def insert_auto_fn(self, op: OpOverload, kwargs) -> Node:
102
"""
103
Insert an auto_functionalized node with the given op and kwargs.
104
"""
vllm/compilation/pass_manager.py
+2
-2
@@ -1,4 +1,4 @@
1
-from typing import List
1
+from typing import Any, Dict, List
2
3
from torch import fx as fx
4
@@ -53,7 +53,7 @@ class PostGradPassManager:
53
assert isinstance(pass_, InductorPass)
54
self.passes.append(pass_)
55
56
- def __getstate__(self):
56
+ def __getstate__(self) -> Dict[str, List[Any]]:
57
"""
58
Custom pickling for the pass manager, as some passes cannot be pickled.
59
Pickling occurs because the pass manager is set as the value of