@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 04cef2c6ab
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 08:15 UTC

[Bugfix] Fix `MQLLMEngine` hanging (#9973)

Signed-off-by: rshaw@neuralmagic.com <rshaw@neuralmagic.com>

Robert Shaw committed Nov 4, 2024 at 16:01 UTC 04cef2c6ab0ea47bb1dfa73d3343985499fe1c4b
3 files changed +42 -23
vllm/engine/multiprocessing/client.py
+10 -2
@@ -112,7 +112,11 @@ class MQLLMEngineClient(EngineClient):
112
113 # Stream for each individual request.
114 self.output_queues: Dict[str, asyncio.Queue] = {}
115 - self.output_loop = asyncio.create_task(self.run_output_handler_loop())
115 +
116 + # Loop to handle output of the LLMEngine periodically.
117 + # Started after the MQLLMEngine is ready so that we can
118 + # build the Client in an executor to enable clean shutdown.
119 + self.output_loop: Optional[asyncio.Task] = None
120
121 # Loop to check health of the LLMEngine periodically.
122 # Started after the MQLLMEngine is ready.
@@ -247,6 +251,9 @@ class MQLLMEngineClient(EngineClient):
251 async def setup(self):
252 """Setup the client before it starts sending server requests."""
253
254 + # Start output_loop
255 + self.output_loop = asyncio.create_task(self.run_output_handler_loop())
256 +
257 with self.get_data_socket() as socket:
258 # Wait until server is ready.
259 response = await self._wait_for_server_rpc(socket)
@@ -265,7 +272,8 @@ class MQLLMEngineClient(EngineClient):
272 # Cancel background tasks.
273 if self.health_loop is not None:
274 self.health_loop.cancel()
268 - self.output_loop.cancel()
275 + if self.output_loop is not None:
276 + self.output_loop.cancel()
277
278 def _set_errored(self, e: BaseException):
279 logger.exception(repr(e))
vllm/engine/multiprocessing/engine.py
+15 -9
@@ -349,16 +349,22 @@ class MQLLMEngine:
349 self.engine.model_executor._run_workers("stop_profile")
350
351
352 +def signal_handler(*_) -> None:
353 + raise KeyboardInterrupt("MQLLMEngine terminated")
354 +
355 +
356 def run_mp_engine(engine_args: AsyncEngineArgs, usage_context: UsageContext,
353 - ipc_path: str):
357 + ipc_path: str, engine_alive):
358 + try:
359 + engine = MQLLMEngine.from_engine_args(engine_args=engine_args,
360 + usage_context=usage_context,
361 + ipc_path=ipc_path)
362
355 - def signal_handler(*_) -> None:
356 - # Interrupt server on sigterm
357 - raise KeyboardInterrupt("MQLLMEngine terminated")
363 + signal.signal(signal.SIGTERM, signal_handler)
364
359 - signal.signal(signal.SIGTERM, signal_handler)
365 + engine.start()
366
361 - engine = MQLLMEngine.from_engine_args(engine_args=engine_args,
362 - usage_context=usage_context,
363 - ipc_path=ipc_path)
364 - engine.start()
367 + except BaseException as e:
368 + logger.exception(e)
369 + engine_alive.value = False
370 + raise e
vllm/entrypoints/openai/api_server.py
+17 -12
@@ -171,39 +171,44 @@ async def build_async_engine_client_from_engine_args(
171 # so we need to spawn a new process
172 context = multiprocessing.get_context("spawn")
173
174 + # The Process can raise an exception during startup, which may
175 + # not actually result in an exitcode being reported. As a result
176 + # we use a shared variable to communicate the information.
177 + engine_alive = multiprocessing.Value('b', True, lock=False)
178 engine_process = context.Process(target=run_mp_engine,
179 args=(engine_args,
180 UsageContext.OPENAI_API_SERVER,
177 - ipc_path))
181 + ipc_path, engine_alive))
182 engine_process.start()
183 engine_pid = engine_process.pid
180 - assert engine_pid is not None, "Engine process failed to start"
184 + assert engine_pid is not None, "Engine process failed to start."
185 logger.info("Started engine process with PID %d", engine_pid)
186
187 # Build RPCClient, which conforms to EngineClient Protocol.
184 - # NOTE: Actually, this is not true yet. We still need to support
185 - # embedding models via RPC (see TODO above)
188 engine_config = engine_args.create_engine_config()
187 - mp_engine_client = MQLLMEngineClient(ipc_path, engine_config,
188 - engine_pid)
189 -
189 + build_client = partial(MQLLMEngineClient, ipc_path, engine_config,
190 + engine_pid)
191 + mq_engine_client = await asyncio.get_running_loop().run_in_executor(
192 + None, build_client)
193 try:
194 while True:
195 try:
193 - await mp_engine_client.setup()
196 + await mq_engine_client.setup()
197 break
198 except TimeoutError:
196 - if not engine_process.is_alive():
199 + if (not engine_process.is_alive()
200 + or not engine_alive.value):
201 raise RuntimeError(
198 - "Engine process failed to start") from None
202 + "Engine process failed to start. See stack "
203 + "trace for the root cause.") from None
204
200 - yield mp_engine_client # type: ignore[misc]
205 + yield mq_engine_client # type: ignore[misc]
206 finally:
207 # Ensure rpc server process was terminated
208 engine_process.terminate()
209
210 # Close all open connections to the backend
206 - mp_engine_client.close()
211 + mq_engine_client.close()
212
213 # Wait for engine process to join
214 engine_process.join(4)