A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 06:45 UTC
Run default _AsyncLLMEngine._run_workers_async in threadpool (#1628)
Dominik Schwabe committed
Nov 11, 2023 at 23:50 UTC
1b290ace4f0c6b74d7536b1acc831e43e9771527
1 file changed
+6
-7
vllm/engine/async_llm_engine.py
+6
-7
@@ -206,18 +206,17 @@ class _AsyncLLMEngine(LLMEngine):
206
**kwargs,
207
) -> Any:
208
"""Runs the given method on all workers."""
209
- all_outputs = []
209
+ coros = []
210
for worker in self.workers:
211
if self.parallel_config.worker_use_ray:
212
- executor = partial(worker.execute_method.remote, method)
212
+ coros.append(
213
+ worker.execute_method.remote(method, *args, **kwargs))
214
else:
215
executor = getattr(worker, method)
216
+ coros.append(asyncio.get_event_loop().run_in_executor(
217
+ None, partial(executor, *args, **kwargs)))
218
216
- output = executor(*args, **kwargs)
217
- all_outputs.append(output)
218
-
219
- if self.parallel_config.worker_use_ray:
220
- all_outputs = await asyncio.gather(*all_outputs)
219
+ all_outputs = await asyncio.gather(*coros)
220
221
if get_all_outputs:
222
return all_outputs