@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 1b290ace4f
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 06:45 UTC

Run default _AsyncLLMEngine._run_workers_async in threadpool (#1628)

Dominik Schwabe committed Nov 11, 2023 at 23:50 UTC 1b290ace4f0c6b74d7536b1acc831e43e9771527
1 file changed +6 -7
vllm/engine/async_llm_engine.py
+6 -7
@@ -206,18 +206,17 @@ class _AsyncLLMEngine(LLMEngine):
206 **kwargs,
207 ) -> Any:
208 """Runs the given method on all workers."""
209 - all_outputs = []
209 + coros = []
210 for worker in self.workers:
211 if self.parallel_config.worker_use_ray:
212 - executor = partial(worker.execute_method.remote, method)
212 + coros.append(
213 + worker.execute_method.remote(method, *args, **kwargs))
214 else:
215 executor = getattr(worker, method)
216 + coros.append(asyncio.get_event_loop().run_in_executor(
217 + None, partial(executor, *args, **kwargs)))
218
216 - output = executor(*args, **kwargs)
217 - all_outputs.append(output)
218 -
219 - if self.parallel_config.worker_use_ray:
220 - all_outputs = await asyncio.gather(*all_outputs)
219 + all_outputs = await asyncio.gather(*coros)
220
221 if get_all_outputs:
222 return all_outputs