A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 01:45 UTC
[BugFix] Fix return type of executor execute_model methods (#4402)
Nick Hill committed
Apr 27, 2024 at 11:17 UTC
ba4be44c32761d30f1e17656b863d2cc078af9e4
6 files changed
+9
-8
vllm/executor/cpu_executor.py
+1
-1
@@ -109,7 +109,7 @@ class CPUExecutorAsync(CPUExecutor, ExecutorAsyncBase):
109
blocks_to_swap_in: Dict[int, int],
110
blocks_to_swap_out: Dict[int, int],
111
blocks_to_copy: Dict[int, List[int]],
112
- ) -> SamplerOutput:
112
+ ) -> List[SamplerOutput]:
113
output = await make_async(self.driver_worker.execute_model)(
114
seq_group_metadata_list=seq_group_metadata_list,
115
blocks_to_swap_in=blocks_to_swap_in,
vllm/executor/distributed_gpu_executor.py
+4
-3
@@ -1,5 +1,5 @@
1
from abc import abstractmethod
2
-from typing import Any, Dict, Optional, Set, Tuple
2
+from typing import Any, Dict, List, Optional, Set, Tuple
3
4
from vllm.executor.executor_base import ExecutorAsyncBase
5
from vllm.executor.gpu_executor import GPUExecutor
@@ -52,7 +52,7 @@ class DistributedGPUExecutor(GPUExecutor):
52
num_gpu_blocks=num_gpu_blocks,
53
num_cpu_blocks=num_cpu_blocks)
54
55
- def execute_model(self, *args, **kwargs) -> SamplerOutput:
55
+ def execute_model(self, *args, **kwargs) -> List[SamplerOutput]:
56
all_outputs = self._run_workers("execute_model",
57
driver_args=args,
58
driver_kwargs=kwargs)
@@ -105,7 +105,8 @@ class DistributedGPUExecutorAsync(DistributedGPUExecutor, ExecutorAsyncBase):
105
"""Runs the given method on all workers."""
106
raise NotImplementedError
107
108
- async def execute_model_async(self, *args, **kwargs) -> SamplerOutput:
108
+ async def execute_model_async(self, *args,
109
+ **kwargs) -> List[SamplerOutput]:
110
all_outputs = await self._run_workers_async("execute_model",
111
driver_args=args,
112
driver_kwargs=kwargs)
vllm/executor/executor_base.py
+1
-1
@@ -112,7 +112,7 @@ class ExecutorAsyncBase(ExecutorBase):
112
blocks_to_swap_in: Dict[int, int],
113
blocks_to_swap_out: Dict[int, int],
114
blocks_to_copy: Dict[int, List[int]],
115
- ) -> SamplerOutput:
115
+ ) -> List[SamplerOutput]:
116
"""Executes one model step on the given sequences."""
117
raise NotImplementedError
118
vllm/executor/gpu_executor.py
+1
-1
@@ -163,7 +163,7 @@ class GPUExecutorAsync(GPUExecutor, ExecutorAsyncBase):
163
blocks_to_swap_in: Dict[int, int],
164
blocks_to_swap_out: Dict[int, int],
165
blocks_to_copy: Dict[int, List[int]],
166
- ) -> SamplerOutput:
166
+ ) -> List[SamplerOutput]:
167
output = await make_async(self.driver_worker.execute_model)(
168
seq_group_metadata_list=seq_group_metadata_list,
169
blocks_to_swap_in=blocks_to_swap_in,
vllm/executor/neuron_executor.py
+1
-1
@@ -84,7 +84,7 @@ class NeuronExecutorAsync(NeuronExecutor, ExecutorAsyncBase):
84
blocks_to_swap_in: Dict[int, int],
85
blocks_to_swap_out: Dict[int, int],
86
blocks_to_copy: Dict[int, List[int]],
87
- ) -> SamplerOutput:
87
+ ) -> List[SamplerOutput]:
88
output = await make_async(self.driver_worker.execute_model)(
89
seq_group_metadata_list=seq_group_metadata_list, )
90
return output
vllm/executor/ray_gpu_executor.py
+1
-1
@@ -188,7 +188,7 @@ class RayGPUExecutor(DistributedGPUExecutor):
188
blocks_to_swap_in: Dict[int, int],
189
blocks_to_swap_out: Dict[int, int],
190
blocks_to_copy: Dict[int, List[int]],
191
- num_lookahead_slots: int = 0) -> SamplerOutput:
191
+ num_lookahead_slots: int = 0) -> List[SamplerOutput]:
192
all_outputs = self._run_workers(
193
"execute_model",
194
driver_kwargs={