@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 080438477f
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 08:00 UTC

Start background task in `AsyncLLMEngine.generate` (#988)

Co-authored-by: Zhuohan Li <zhuohan123@gmail.com>

Antoni Baum committed Sep 8, 2023 at 00:03 UTC 080438477f319149db4f09f3a8835dde23609f7a
4 files changed +16 -24
tests/async_engine/api_server_async_engine.py
+1 -2
@@ -40,8 +40,7 @@ if __name__ == "__main__":
40 args = parser.parse_args()
41
42 engine_args = AsyncEngineArgs.from_cli_args(args)
43 - engine = AsyncLLMEngineWithStats.from_engine_args(engine_args,
44 - start_engine_loop=False)
43 + engine = AsyncLLMEngineWithStats.from_engine_args(engine_args)
44 vllm.entrypoints.api_server.engine = engine
45 uvicorn.run(
46 app,
vllm/engine/async_llm_engine.py
+13 -9
@@ -230,6 +230,8 @@ class AsyncLLMEngine:
230 async frontend will be executed in a separate process as the
231 model workers.
232 log_requests: Whether to log the requests.
233 + start_engine_loop: If True, the background task to run the engine
234 + will be automatically started in the generate call.
235 *args, *kwargs: Arguments for LLMEngine.
236 """
237
@@ -240,7 +242,7 @@ class AsyncLLMEngine:
242 engine_use_ray: bool,
243 *args,
244 log_requests: bool = True,
243 - start_engine_loop: bool = False,
245 + start_engine_loop: bool = True,
246 **kwargs) -> None:
247 self.worker_use_ray = worker_use_ray
248 self.engine_use_ray = engine_use_ray
@@ -249,8 +251,7 @@ class AsyncLLMEngine:
251
252 self.request_tracker: RequestTracker = RequestTracker()
253 self.background_loop = None
252 - if start_engine_loop:
253 - self.start_background_loop()
254 + self.start_engine_loop = start_engine_loop
255
256 @property
257 def is_running(self) -> bool:
@@ -330,11 +331,14 @@ class AsyncLLMEngine:
331 f"prompt token ids: {prompt_token_ids}.")
332
333 if not self.is_running:
333 - raise AsyncEngineDeadError(
334 - "Background loop is not running. If it was running, "
335 - "inspect the output to find the stacktrace of the "
336 - "error that caused the background loop to stop "
337 - "(AsyncEngineDeadError).")
334 + if self.start_engine_loop:
335 + self.start_background_loop()
336 + else:
337 + raise AsyncEngineDeadError(
338 + "Background loop is not running. If it was running, "
339 + "inspect the output to find the stacktrace of the "
340 + "error that caused the background loop to stop "
341 + "(AsyncEngineDeadError).")
342
343 stream = self.request_tracker.add_request(
344 request_id,
@@ -426,7 +430,7 @@ class AsyncLLMEngine:
430 @classmethod
431 def from_engine_args(cls,
432 engine_args: AsyncEngineArgs,
429 - start_engine_loop: bool = False) -> "AsyncLLMEngine":
433 + start_engine_loop: bool = True) -> "AsyncLLMEngine":
434 """Creates an async LLM engine from the engine arguments."""
435 # Create the engine configs.
436 engine_configs = engine_args.create_engine_configs()
vllm/entrypoints/api_server.py
+1 -5
@@ -32,9 +32,6 @@ async def generate(request: Request) -> Response:
32 sampling_params = SamplingParams(**request_dict)
33 request_id = random_uuid()
34
35 - if not engine.is_running:
36 - engine.start_background_loop()
37 -
35 results_generator = engine.generate(prompt, sampling_params, request_id)
36
37 # Streaming case
@@ -80,8 +77,7 @@ if __name__ == "__main__":
77 args = parser.parse_args()
78
79 engine_args = AsyncEngineArgs.from_cli_args(args)
83 - engine = AsyncLLMEngine.from_engine_args(engine_args,
84 - start_engine_loop=False)
80 + engine = AsyncLLMEngine.from_engine_args(engine_args)
81
82 uvicorn.run(app,
83 host=args.host,
vllm/entrypoints/openai/api_server.py
+1 -8
@@ -192,9 +192,6 @@ async def create_chat_completion(request: ChatCompletionRequest,
192 """
193 logger.info(f"Received chat completion request: {request}")
194
195 - if not engine.is_running:
196 - engine.start_background_loop()
197 -
195 error_check_ret = await check_model(request)
196 if error_check_ret is not None:
197 return error_check_ret
@@ -367,9 +364,6 @@ async def create_completion(request: CompletionRequest, raw_request: Request):
364 """
365 logger.info(f"Received completion request: {request}")
366
370 - if not engine.is_running:
371 - engine.start_background_loop()
372 -
367 error_check_ret = await check_model(request)
368 if error_check_ret is not None:
369 return error_check_ret
@@ -627,8 +621,7 @@ if __name__ == "__main__":
621 served_model = args.model
622
623 engine_args = AsyncEngineArgs.from_cli_args(args)
630 - engine = AsyncLLMEngine.from_engine_args(engine_args,
631 - start_engine_loop=False)
624 + engine = AsyncLLMEngine.from_engine_args(engine_args)
625 engine_model_config = asyncio.run(engine.get_model_config())
626 max_model_len = engine_model_config.get_max_model_len()
627