A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 08:00 UTC
Start background task in `AsyncLLMEngine.generate` (#988)
Co-authored-by: Zhuohan Li <zhuohan123@gmail.com>
Antoni Baum committed
Sep 8, 2023 at 00:03 UTC
080438477f319149db4f09f3a8835dde23609f7a
4 files changed
+16
-24
tests/async_engine/api_server_async_engine.py
+1
-2
@@ -40,8 +40,7 @@ if __name__ == "__main__":
40
args = parser.parse_args()
41
42
engine_args = AsyncEngineArgs.from_cli_args(args)
43
- engine = AsyncLLMEngineWithStats.from_engine_args(engine_args,
44
- start_engine_loop=False)
43
+ engine = AsyncLLMEngineWithStats.from_engine_args(engine_args)
44
vllm.entrypoints.api_server.engine = engine
45
uvicorn.run(
46
app,
vllm/engine/async_llm_engine.py
+13
-9
@@ -230,6 +230,8 @@ class AsyncLLMEngine:
230
async frontend will be executed in a separate process as the
231
model workers.
232
log_requests: Whether to log the requests.
233
+ start_engine_loop: If True, the background task to run the engine
234
+ will be automatically started in the generate call.
235
*args, *kwargs: Arguments for LLMEngine.
236
"""
237
@@ -240,7 +242,7 @@ class AsyncLLMEngine:
242
engine_use_ray: bool,
243
*args,
244
log_requests: bool = True,
243
- start_engine_loop: bool = False,
245
+ start_engine_loop: bool = True,
246
**kwargs) -> None:
247
self.worker_use_ray = worker_use_ray
248
self.engine_use_ray = engine_use_ray
@@ -249,8 +251,7 @@ class AsyncLLMEngine:
251
252
self.request_tracker: RequestTracker = RequestTracker()
253
self.background_loop = None
252
- if start_engine_loop:
253
- self.start_background_loop()
254
+ self.start_engine_loop = start_engine_loop
255
256
@property
257
def is_running(self) -> bool:
@@ -330,11 +331,14 @@ class AsyncLLMEngine:
331
f"prompt token ids: {prompt_token_ids}.")
332
333
if not self.is_running:
333
- raise AsyncEngineDeadError(
334
- "Background loop is not running. If it was running, "
335
- "inspect the output to find the stacktrace of the "
336
- "error that caused the background loop to stop "
337
- "(AsyncEngineDeadError).")
334
+ if self.start_engine_loop:
335
+ self.start_background_loop()
336
+ else:
337
+ raise AsyncEngineDeadError(
338
+ "Background loop is not running. If it was running, "
339
+ "inspect the output to find the stacktrace of the "
340
+ "error that caused the background loop to stop "
341
+ "(AsyncEngineDeadError).")
342
343
stream = self.request_tracker.add_request(
344
request_id,
@@ -426,7 +430,7 @@ class AsyncLLMEngine:
430
@classmethod
431
def from_engine_args(cls,
432
engine_args: AsyncEngineArgs,
429
- start_engine_loop: bool = False) -> "AsyncLLMEngine":
433
+ start_engine_loop: bool = True) -> "AsyncLLMEngine":
434
"""Creates an async LLM engine from the engine arguments."""
435
# Create the engine configs.
436
engine_configs = engine_args.create_engine_configs()
vllm/entrypoints/api_server.py
+1
-5
@@ -32,9 +32,6 @@ async def generate(request: Request) -> Response:
32
sampling_params = SamplingParams(**request_dict)
33
request_id = random_uuid()
34
35
- if not engine.is_running:
36
- engine.start_background_loop()
37
-
35
results_generator = engine.generate(prompt, sampling_params, request_id)
36
37
# Streaming case
@@ -80,8 +77,7 @@ if __name__ == "__main__":
77
args = parser.parse_args()
78
79
engine_args = AsyncEngineArgs.from_cli_args(args)
83
- engine = AsyncLLMEngine.from_engine_args(engine_args,
84
- start_engine_loop=False)
80
+ engine = AsyncLLMEngine.from_engine_args(engine_args)
81
82
uvicorn.run(app,
83
host=args.host,
vllm/entrypoints/openai/api_server.py
+1
-8
@@ -192,9 +192,6 @@ async def create_chat_completion(request: ChatCompletionRequest,
192
"""
193
logger.info(f"Received chat completion request: {request}")
194
195
- if not engine.is_running:
196
- engine.start_background_loop()
197
-
195
error_check_ret = await check_model(request)
196
if error_check_ret is not None:
197
return error_check_ret
@@ -367,9 +364,6 @@ async def create_completion(request: CompletionRequest, raw_request: Request):
364
"""
365
logger.info(f"Received completion request: {request}")
366
370
- if not engine.is_running:
371
- engine.start_background_loop()
372
-
367
error_check_ret = await check_model(request)
368
if error_check_ret is not None:
369
return error_check_ret
@@ -627,8 +621,7 @@ if __name__ == "__main__":
621
served_model = args.model
622
623
engine_args = AsyncEngineArgs.from_cli_args(args)
630
- engine = AsyncLLMEngine.from_engine_args(engine_args,
631
- start_engine_loop=False)
624
+ engine = AsyncLLMEngine.from_engine_args(engine_args)
625
engine_model_config = asyncio.run(engine.get_model_config())
626
max_model_len = engine_model_config.get_max_model_len()
627