@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 1696725879
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 08:00 UTC

Initialize AsyncLLMEngine bg loop correctly (#943)

Antoni Baum committed Sep 4, 2023 at 17:41 UTC 1696725879f25de03ca36cde764102bba60ff681
3 files changed +25 -7
vllm/engine/async_llm_engine.py
+11 -5
@@ -155,11 +155,15 @@ class AsyncLLMEngine:
155 self.finished_requests: Set[str] = set()
156 self.background_loop = None
157 if start_engine_loop:
158 - self._start_background_loop()
158 + self.start_background_loop()
159
160 - def _start_background_loop(self) -> None:
160 + @property
161 + def is_running(self) -> bool:
162 + return self.background_loop is not None
163 +
164 + def start_background_loop(self) -> None:
165 """Start the background loop."""
162 - if self.background_loop is not None:
166 + if self.is_running:
167 raise RuntimeError("Background loop is already running.")
168 self.background_loop = asyncio.get_event_loop().create_task(
169 self.run_engine_loop())
@@ -323,7 +327,8 @@ class AsyncLLMEngine:
327
328 @classmethod
329 def from_engine_args(cls,
326 - engine_args: AsyncEngineArgs) -> "AsyncLLMEngine":
330 + engine_args: AsyncEngineArgs,
331 + start_engine_loop: bool = False) -> "AsyncLLMEngine":
332 """Creates an async LLM engine from the engine arguments."""
333 # Create the engine configs.
334 engine_configs = engine_args.create_engine_configs()
@@ -338,5 +343,6 @@ class AsyncLLMEngine:
343 distributed_init_method,
344 placement_group,
345 log_requests=not engine_args.disable_log_requests,
341 - log_stats=not engine_args.disable_log_stats)
346 + log_stats=not engine_args.disable_log_stats,
347 + start_engine_loop=start_engine_loop)
348 return engine
vllm/entrypoints/api_server.py
+6 -1
@@ -30,6 +30,10 @@ async def generate(request: Request) -> Response:
30 stream = request_dict.pop("stream", False)
31 sampling_params = SamplingParams(**request_dict)
32 request_id = random_uuid()
33 +
34 + if not engine.is_running:
35 + engine.start_background_loop()
36 +
37 results_generator = engine.generate(prompt, sampling_params, request_id)
38
39 # Streaming case
@@ -75,7 +79,8 @@ if __name__ == "__main__":
79 args = parser.parse_args()
80
81 engine_args = AsyncEngineArgs.from_cli_args(args)
78 - engine = AsyncLLMEngine.from_engine_args(engine_args)
82 + engine = AsyncLLMEngine.from_engine_args(engine_args,
83 + start_engine_loop=False)
84
85 uvicorn.run(app,
86 host=args.host,
vllm/entrypoints/openai/api_server.py
+8 -1
@@ -191,6 +191,9 @@ async def create_chat_completion(request: ChatCompletionRequest,
191 """
192 logger.info(f"Received chat completion request: {request}")
193
194 + if not engine.is_running:
195 + engine.start_background_loop()
196 +
197 error_check_ret = await check_model(request)
198 if error_check_ret is not None:
199 return error_check_ret
@@ -363,6 +366,9 @@ async def create_completion(request: CompletionRequest, raw_request: Request):
366 """
367 logger.info(f"Received completion request: {request}")
368
369 + if not engine.is_running:
370 + engine.start_background_loop()
371 +
372 error_check_ret = await check_model(request)
373 if error_check_ret is not None:
374 return error_check_ret
@@ -620,7 +626,8 @@ if __name__ == "__main__":
626 served_model = args.model
627
628 engine_args = AsyncEngineArgs.from_cli_args(args)
623 - engine = AsyncLLMEngine.from_engine_args(engine_args)
629 + engine = AsyncLLMEngine.from_engine_args(engine_args,
630 + start_engine_loop=False)
631 engine_model_config = asyncio.run(engine.get_model_config())
632 max_model_len = engine_model_config.get_max_model_len()
633