A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 08:15 UTC
[Bugfix] Fix `MQLLMEngine` hanging (#9973)
Signed-off-by: rshaw@neuralmagic.com <rshaw@neuralmagic.com>
Robert Shaw committed
Nov 4, 2024 at 16:01 UTC
04cef2c6ab0ea47bb1dfa73d3343985499fe1c4b
3 files changed
+42
-23
vllm/engine/multiprocessing/client.py
+10
-2
@@ -112,7 +112,11 @@ class MQLLMEngineClient(EngineClient):
112
113
# Stream for each individual request.
114
self.output_queues: Dict[str, asyncio.Queue] = {}
115
- self.output_loop = asyncio.create_task(self.run_output_handler_loop())
115
+
116
+ # Loop to handle output of the LLMEngine periodically.
117
+ # Started after the MQLLMEngine is ready so that we can
118
+ # build the Client in an executor to enable clean shutdown.
119
+ self.output_loop: Optional[asyncio.Task] = None
120
121
# Loop to check health of the LLMEngine periodically.
122
# Started after the MQLLMEngine is ready.
@@ -247,6 +251,9 @@ class MQLLMEngineClient(EngineClient):
251
async def setup(self):
252
"""Setup the client before it starts sending server requests."""
253
254
+ # Start output_loop
255
+ self.output_loop = asyncio.create_task(self.run_output_handler_loop())
256
+
257
with self.get_data_socket() as socket:
258
# Wait until server is ready.
259
response = await self._wait_for_server_rpc(socket)
@@ -265,7 +272,8 @@ class MQLLMEngineClient(EngineClient):
272
# Cancel background tasks.
273
if self.health_loop is not None:
274
self.health_loop.cancel()
268
- self.output_loop.cancel()
275
+ if self.output_loop is not None:
276
+ self.output_loop.cancel()
277
278
def _set_errored(self, e: BaseException):
279
logger.exception(repr(e))
vllm/engine/multiprocessing/engine.py
+15
-9
@@ -349,16 +349,22 @@ class MQLLMEngine:
349
self.engine.model_executor._run_workers("stop_profile")
350
351
352
+def signal_handler(*_) -> None:
353
+ raise KeyboardInterrupt("MQLLMEngine terminated")
354
+
355
+
356
def run_mp_engine(engine_args: AsyncEngineArgs, usage_context: UsageContext,
353
- ipc_path: str):
357
+ ipc_path: str, engine_alive):
358
+ try:
359
+ engine = MQLLMEngine.from_engine_args(engine_args=engine_args,
360
+ usage_context=usage_context,
361
+ ipc_path=ipc_path)
362
355
- def signal_handler(*_) -> None:
356
- # Interrupt server on sigterm
357
- raise KeyboardInterrupt("MQLLMEngine terminated")
363
+ signal.signal(signal.SIGTERM, signal_handler)
364
359
- signal.signal(signal.SIGTERM, signal_handler)
365
+ engine.start()
366
361
- engine = MQLLMEngine.from_engine_args(engine_args=engine_args,
362
- usage_context=usage_context,
363
- ipc_path=ipc_path)
364
- engine.start()
367
+ except BaseException as e:
368
+ logger.exception(e)
369
+ engine_alive.value = False
370
+ raise e
vllm/entrypoints/openai/api_server.py
+17
-12
@@ -171,39 +171,44 @@ async def build_async_engine_client_from_engine_args(
171
# so we need to spawn a new process
172
context = multiprocessing.get_context("spawn")
173
174
+ # The Process can raise an exception during startup, which may
175
+ # not actually result in an exitcode being reported. As a result
176
+ # we use a shared variable to communicate the information.
177
+ engine_alive = multiprocessing.Value('b', True, lock=False)
178
engine_process = context.Process(target=run_mp_engine,
179
args=(engine_args,
180
UsageContext.OPENAI_API_SERVER,
177
- ipc_path))
181
+ ipc_path, engine_alive))
182
engine_process.start()
183
engine_pid = engine_process.pid
180
- assert engine_pid is not None, "Engine process failed to start"
184
+ assert engine_pid is not None, "Engine process failed to start."
185
logger.info("Started engine process with PID %d", engine_pid)
186
187
# Build RPCClient, which conforms to EngineClient Protocol.
184
- # NOTE: Actually, this is not true yet. We still need to support
185
- # embedding models via RPC (see TODO above)
188
engine_config = engine_args.create_engine_config()
187
- mp_engine_client = MQLLMEngineClient(ipc_path, engine_config,
188
- engine_pid)
189
-
189
+ build_client = partial(MQLLMEngineClient, ipc_path, engine_config,
190
+ engine_pid)
191
+ mq_engine_client = await asyncio.get_running_loop().run_in_executor(
192
+ None, build_client)
193
try:
194
while True:
195
try:
193
- await mp_engine_client.setup()
196
+ await mq_engine_client.setup()
197
break
198
except TimeoutError:
196
- if not engine_process.is_alive():
199
+ if (not engine_process.is_alive()
200
+ or not engine_alive.value):
201
raise RuntimeError(
198
- "Engine process failed to start") from None
202
+ "Engine process failed to start. See stack "
203
+ "trace for the root cause.") from None
204
200
- yield mp_engine_client # type: ignore[misc]
205
+ yield mq_engine_client # type: ignore[misc]
206
finally:
207
# Ensure rpc server process was terminated
208
engine_process.terminate()
209
210
# Close all open connections to the backend
206
- mp_engine_client.close()
211
+ mq_engine_client.close()
212
213
# Wait for engine process to join
214
engine_process.join(4)