A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 08:00 UTC
Initialize AsyncLLMEngine bg loop correctly (#943)
Antoni Baum committed
Sep 4, 2023 at 17:41 UTC
1696725879f25de03ca36cde764102bba60ff681
3 files changed
+25
-7
vllm/engine/async_llm_engine.py
+11
-5
@@ -155,11 +155,15 @@ class AsyncLLMEngine:
155
self.finished_requests: Set[str] = set()
156
self.background_loop = None
157
if start_engine_loop:
158
- self._start_background_loop()
158
+ self.start_background_loop()
159
160
- def _start_background_loop(self) -> None:
160
+ @property
161
+ def is_running(self) -> bool:
162
+ return self.background_loop is not None
163
+
164
+ def start_background_loop(self) -> None:
165
"""Start the background loop."""
162
- if self.background_loop is not None:
166
+ if self.is_running:
167
raise RuntimeError("Background loop is already running.")
168
self.background_loop = asyncio.get_event_loop().create_task(
169
self.run_engine_loop())
@@ -323,7 +327,8 @@ class AsyncLLMEngine:
327
328
@classmethod
329
def from_engine_args(cls,
326
- engine_args: AsyncEngineArgs) -> "AsyncLLMEngine":
330
+ engine_args: AsyncEngineArgs,
331
+ start_engine_loop: bool = False) -> "AsyncLLMEngine":
332
"""Creates an async LLM engine from the engine arguments."""
333
# Create the engine configs.
334
engine_configs = engine_args.create_engine_configs()
@@ -338,5 +343,6 @@ class AsyncLLMEngine:
343
distributed_init_method,
344
placement_group,
345
log_requests=not engine_args.disable_log_requests,
341
- log_stats=not engine_args.disable_log_stats)
346
+ log_stats=not engine_args.disable_log_stats,
347
+ start_engine_loop=start_engine_loop)
348
return engine
vllm/entrypoints/api_server.py
+6
-1
@@ -30,6 +30,10 @@ async def generate(request: Request) -> Response:
30
stream = request_dict.pop("stream", False)
31
sampling_params = SamplingParams(**request_dict)
32
request_id = random_uuid()
33
+
34
+ if not engine.is_running:
35
+ engine.start_background_loop()
36
+
37
results_generator = engine.generate(prompt, sampling_params, request_id)
38
39
# Streaming case
@@ -75,7 +79,8 @@ if __name__ == "__main__":
79
args = parser.parse_args()
80
81
engine_args = AsyncEngineArgs.from_cli_args(args)
78
- engine = AsyncLLMEngine.from_engine_args(engine_args)
82
+ engine = AsyncLLMEngine.from_engine_args(engine_args,
83
+ start_engine_loop=False)
84
85
uvicorn.run(app,
86
host=args.host,
vllm/entrypoints/openai/api_server.py
+8
-1
@@ -191,6 +191,9 @@ async def create_chat_completion(request: ChatCompletionRequest,
191
"""
192
logger.info(f"Received chat completion request: {request}")
193
194
+ if not engine.is_running:
195
+ engine.start_background_loop()
196
+
197
error_check_ret = await check_model(request)
198
if error_check_ret is not None:
199
return error_check_ret
@@ -363,6 +366,9 @@ async def create_completion(request: CompletionRequest, raw_request: Request):
366
"""
367
logger.info(f"Received completion request: {request}")
368
369
+ if not engine.is_running:
370
+ engine.start_background_loop()
371
+
372
error_check_ret = await check_model(request)
373
if error_check_ret is not None:
374
return error_check_ret
@@ -620,7 +626,8 @@ if __name__ == "__main__":
626
served_model = args.model
627
628
engine_args = AsyncEngineArgs.from_cli_args(args)
623
- engine = AsyncLLMEngine.from_engine_args(engine_args)
629
+ engine = AsyncLLMEngine.from_engine_args(engine_args,
630
+ start_engine_loop=False)
631
engine_model_config = asyncio.run(engine.get_model_config())
632
max_model_len = engine_model_config.get_max_model_len()
633