@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / ba4be44c32
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 01:45 UTC

[BugFix] Fix return type of executor execute_model methods (#4402)

Nick Hill committed Apr 27, 2024 at 11:17 UTC ba4be44c32761d30f1e17656b863d2cc078af9e4
6 files changed +9 -8
vllm/executor/cpu_executor.py
+1 -1
@@ -109,7 +109,7 @@ class CPUExecutorAsync(CPUExecutor, ExecutorAsyncBase):
109 blocks_to_swap_in: Dict[int, int],
110 blocks_to_swap_out: Dict[int, int],
111 blocks_to_copy: Dict[int, List[int]],
112 - ) -> SamplerOutput:
112 + ) -> List[SamplerOutput]:
113 output = await make_async(self.driver_worker.execute_model)(
114 seq_group_metadata_list=seq_group_metadata_list,
115 blocks_to_swap_in=blocks_to_swap_in,
vllm/executor/distributed_gpu_executor.py
+4 -3
@@ -1,5 +1,5 @@
1 from abc import abstractmethod
2 -from typing import Any, Dict, Optional, Set, Tuple
2 +from typing import Any, Dict, List, Optional, Set, Tuple
3
4 from vllm.executor.executor_base import ExecutorAsyncBase
5 from vllm.executor.gpu_executor import GPUExecutor
@@ -52,7 +52,7 @@ class DistributedGPUExecutor(GPUExecutor):
52 num_gpu_blocks=num_gpu_blocks,
53 num_cpu_blocks=num_cpu_blocks)
54
55 - def execute_model(self, *args, **kwargs) -> SamplerOutput:
55 + def execute_model(self, *args, **kwargs) -> List[SamplerOutput]:
56 all_outputs = self._run_workers("execute_model",
57 driver_args=args,
58 driver_kwargs=kwargs)
@@ -105,7 +105,8 @@ class DistributedGPUExecutorAsync(DistributedGPUExecutor, ExecutorAsyncBase):
105 """Runs the given method on all workers."""
106 raise NotImplementedError
107
108 - async def execute_model_async(self, *args, **kwargs) -> SamplerOutput:
108 + async def execute_model_async(self, *args,
109 + **kwargs) -> List[SamplerOutput]:
110 all_outputs = await self._run_workers_async("execute_model",
111 driver_args=args,
112 driver_kwargs=kwargs)
vllm/executor/executor_base.py
+1 -1
@@ -112,7 +112,7 @@ class ExecutorAsyncBase(ExecutorBase):
112 blocks_to_swap_in: Dict[int, int],
113 blocks_to_swap_out: Dict[int, int],
114 blocks_to_copy: Dict[int, List[int]],
115 - ) -> SamplerOutput:
115 + ) -> List[SamplerOutput]:
116 """Executes one model step on the given sequences."""
117 raise NotImplementedError
118
vllm/executor/gpu_executor.py
+1 -1
@@ -163,7 +163,7 @@ class GPUExecutorAsync(GPUExecutor, ExecutorAsyncBase):
163 blocks_to_swap_in: Dict[int, int],
164 blocks_to_swap_out: Dict[int, int],
165 blocks_to_copy: Dict[int, List[int]],
166 - ) -> SamplerOutput:
166 + ) -> List[SamplerOutput]:
167 output = await make_async(self.driver_worker.execute_model)(
168 seq_group_metadata_list=seq_group_metadata_list,
169 blocks_to_swap_in=blocks_to_swap_in,
vllm/executor/neuron_executor.py
+1 -1
@@ -84,7 +84,7 @@ class NeuronExecutorAsync(NeuronExecutor, ExecutorAsyncBase):
84 blocks_to_swap_in: Dict[int, int],
85 blocks_to_swap_out: Dict[int, int],
86 blocks_to_copy: Dict[int, List[int]],
87 - ) -> SamplerOutput:
87 + ) -> List[SamplerOutput]:
88 output = await make_async(self.driver_worker.execute_model)(
89 seq_group_metadata_list=seq_group_metadata_list, )
90 return output
vllm/executor/ray_gpu_executor.py
+1 -1
@@ -188,7 +188,7 @@ class RayGPUExecutor(DistributedGPUExecutor):
188 blocks_to_swap_in: Dict[int, int],
189 blocks_to_swap_out: Dict[int, int],
190 blocks_to_copy: Dict[int, List[int]],
191 - num_lookahead_slots: int = 0) -> SamplerOutput:
191 + num_lookahead_slots: int = 0) -> List[SamplerOutput]:
192 all_outputs = self._run_workers(
193 "execute_model",
194 driver_kwargs={