@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / dbeac95dbb
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 04:30 UTC

Mypy checking for vllm/compilation (#11496)

Signed-off-by: lucast2021 <lucast2021@headroyce.org> Co-authored-by: lucast2021 <lucast2021@headroyce.org>

Lucas Tucker committed Dec 25, 2024 at 23:04 UTC dbeac95dbbf898bcc0965528fc767e9cadbbe0c5
3 files changed +9 -8
vllm/compilation/backends.py
+5 -5
@@ -141,14 +141,14 @@ class AlwaysHitShapeEnv:
141 return ""
142
143
144 -def wrap_inductor(graph,
144 +def wrap_inductor(graph: fx.GraphModule,
145 example_inputs,
146 additional_inductor_config,
147 compilation_config: CompilationConfig,
148 graph_index: int = 0,
149 num_graphs: int = 1,
150 runtime_shape: Optional[int] = None,
151 - use_inductor: bool = True):
151 + use_inductor: bool = True) -> Any:
152 if graph_index == 0:
153 # before compiling the first graph, record the start time
154 global compilation_start_time
@@ -209,7 +209,7 @@ def wrap_inductor(graph,
209 returns_tuple = graph_returns_tuple(graph)
210
211 # this is the graph we return to Dynamo to run
212 - def compiled_graph(*args):
212 + def compiled_graph(*args) -> Optional[fx.CompiledFxGraph]:
213 # convert args to list
214 list_args = list(args)
215 graph_output = inductor_compiled_graph(list_args)
@@ -247,7 +247,7 @@ def wrap_inductor(graph,
247 # see https://github.com/pytorch/pytorch/blob/9f5ebf3fc609105a74eab4ccc24932d6353ff566/torch/_inductor/codecache.py#L1221 # noqa
248 return
249
250 - def _get_shape_env():
250 + def _get_shape_env() -> AlwaysHitShapeEnv:
251 return AlwaysHitShapeEnv()
252
253 with patch(# for hijacking the hash of the compiled graph
@@ -537,7 +537,7 @@ class VllmBackend:
537 example_inputs[x].clone() for x in self.sym_tensor_indices
538 ]
539
540 - def copy_and_call(*args):
540 + def copy_and_call(*args) -> fx.GraphModule:
541 list_args = list(args)
542 for i, index in enumerate(self.sym_tensor_indices):
543 runtime_tensor = list_args[index]
vllm/compilation/multi_output_match.py
+2 -1
@@ -7,6 +7,7 @@ from torch import fx
7 from torch._higher_order_ops.auto_functionalize import auto_functionalized
8 from torch._inductor import pattern_matcher as pm
9 from torch._ops import OpOverload
10 +from torch.fx import Node
11
12 from vllm.compilation.fx_utils import find_auto_fn
13
@@ -97,7 +98,7 @@ class MultiOutputMatch(abc.ABC):
98 self.graph.call_function(operator.getitem, (tuple_node, idx))
99 for idx in indices)
100
100 - def insert_auto_fn(self, op: OpOverload, kwargs):
101 + def insert_auto_fn(self, op: OpOverload, kwargs) -> Node:
102 """
103 Insert an auto_functionalized node with the given op and kwargs.
104 """
vllm/compilation/pass_manager.py
+2 -2
@@ -1,4 +1,4 @@
1 -from typing import List
1 +from typing import Any, Dict, List
2
3 from torch import fx as fx
4
@@ -53,7 +53,7 @@ class PostGradPassManager:
53 assert isinstance(pass_, InductorPass)
54 self.passes.append(pass_)
55
56 - def __getstate__(self):
56 + def __getstate__(self) -> Dict[str, List[Any]]:
57 """
58 Custom pickling for the pass manager, as some passes cannot be pickled.
59 Pickling occurs because the pass manager is set as the value of