A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 04:30 UTC
[misc] fix typing (#11540)
Signed-off-by: youkaichao <youkaichao@gmail.com>
youkaichao committed
Dec 27, 2024 at 11:05 UTC
eb881ed006ca458b052905e33f0d16dbb428063a
1 file changed
+4
-3
vllm/compilation/backends.py
+4
-3
@@ -208,8 +208,8 @@ def wrap_inductor(graph: fx.GraphModule,
208
from torch._inductor.compile_fx import graph_returns_tuple
209
returns_tuple = graph_returns_tuple(graph)
210
211
- # this is the graph we return to Dynamo to run
212
- def compiled_graph(*args) -> Optional[fx.CompiledFxGraph]:
211
+ # this is the callable we return to Dynamo to run
212
+ def compiled_graph(*args):
213
# convert args to list
214
list_args = list(args)
215
graph_output = inductor_compiled_graph(list_args)
@@ -537,7 +537,8 @@ class VllmBackend:
537
example_inputs[x].clone() for x in self.sym_tensor_indices
538
]
539
540
- def copy_and_call(*args) -> fx.GraphModule:
540
+ # this is the callable we return to Dynamo to run
541
+ def copy_and_call(*args):
542
list_args = list(args)
543
for i, index in enumerate(self.sym_tensor_indices):
544
runtime_tensor = list_args[index]