@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / eb881ed006
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 04:30 UTC

[misc] fix typing (#11540)

Signed-off-by: youkaichao <youkaichao@gmail.com>

youkaichao committed Dec 27, 2024 at 11:05 UTC eb881ed006ca458b052905e33f0d16dbb428063a
1 file changed +4 -3
vllm/compilation/backends.py
+4 -3
@@ -208,8 +208,8 @@ def wrap_inductor(graph: fx.GraphModule,
208 from torch._inductor.compile_fx import graph_returns_tuple
209 returns_tuple = graph_returns_tuple(graph)
210
211 - # this is the graph we return to Dynamo to run
212 - def compiled_graph(*args) -> Optional[fx.CompiledFxGraph]:
211 + # this is the callable we return to Dynamo to run
212 + def compiled_graph(*args):
213 # convert args to list
214 list_args = list(args)
215 graph_output = inductor_compiled_graph(list_args)
@@ -537,7 +537,8 @@ class VllmBackend:
537 example_inputs[x].clone() for x in self.sym_tensor_indices
538 ]
539
540 - def copy_and_call(*args) -> fx.GraphModule:
540 + # this is the callable we return to Dynamo to run
541 + def copy_and_call(*args):
542 list_args = list(args)
543 for i, index in enumerate(self.sym_tensor_indices):
544 runtime_tensor = list_args[index]