@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / bccc43c033

Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 14, 23:45 UTC

[Bugfix]check health for engine core process exiting unexpectedly (#21728)

Signed-off-by: wuhang <wuhang6@huawei.com>

wuhang committed Jul 28, 2025 at 21:17 UTC bccc43c0332cbd2e9b2cc6f7c83d319062f7cccd

2 files changed +84 -2

vllm/v1/engine/core_client.py

+41

@@ -2,6 +2,7 @@

2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project

3 import asyncio

4 import contextlib

5 +import multiprocessing

6 import queue

7 import sys

8 import uuid

@@ -476,6 +477,9 @@ class MPClient(EngineCoreClient):

477 # underlying data.

478 self.pending_messages = deque[tuple[zmq.MessageTracker, Any]]()

479

480 + # Start monitoring engine core processes for unexpected failures

481 + self.start_engine_core_monitor()

482 +

483 success = True

484 finally:

485 if not success:

@@ -505,6 +509,41 @@ class MPClient(EngineCoreClient):

509 def dp_engines_running(self) -> bool:

510 return self.engines_running

511

512 + def start_engine_core_monitor(self):

513 + """Start a monitor thread for engine core processes."""

514 + engine_manager = self.resources.engine_manager

515 + if (engine_manager is None or not hasattr(engine_manager, 'processes')

516 + or not engine_manager.processes):

517 + # No engine processes to monitor

518 + return

519 +

520 + engine_processes = engine_manager.processes

521 + self_ref = weakref.ref(self)

522 +

523 + # Monitor engine core process liveness. If any die unexpectedly,

524 + # logs an error, shuts down the client and invokes the failure

525 + # callback to inform the engine.

526 + def monitor_engine_cores():

527 + sentinels = [proc.sentinel for proc in engine_processes]

528 + died = multiprocessing.connection.wait(sentinels)

529 + _self = self_ref()

530 + if not _self or _self.resources.engine_dead:

531 + return

532 + _self.resources.engine_dead = True

533 + proc_name = next(proc.name for proc in engine_processes

534 + if proc.sentinel == died[0])

535 + logger.error(

536 + "Engine core proc %s died unexpectedly, "

537 + "shutting down client.", proc_name)

538 + _self.shutdown()

539 + # Note: For MPClient, we don't have a failure callback mechanism

540 + # like MultiprocExecutor, but we set engine_dead flag which will

541 + # cause subsequent operations to raise EngineDeadError

542 +

543 + Thread(target=monitor_engine_cores,

544 + daemon=True,

545 + name="MPClientEngineMonitor").start()

546 +

547

548 def _process_utility_output(output: UtilityOutput,

549 utility_results: dict[int, AnyFuture]):

@@ -749,6 +788,8 @@ class AsyncMPClient(MPClient):

788 outputs_queue.put_nowait(outputs)

789 except Exception as e:

790 outputs_queue.put_nowait(e)

791 + except asyncio.CancelledError:

792 + outputs_queue.put_nowait(EngineDeadError())

793

794 resources.output_queue_task = asyncio.create_task(

795 process_outputs_socket(), name="EngineCoreOutputQueueTask")

vllm/v1/executor/multiproc_executor.py

+43 -2

@@ -104,8 +104,12 @@ class MultiprocExecutor(Executor):

104 finally:

105 if not success:

106 # Clean up the worker procs if there was a failure.

107 + # Close death_writers first to signal workers to exit

108 + for uw in unready_workers:

109 + if uw.death_writer is not None:

110 + uw.death_writer.close()

111 self._ensure_worker_termination(

108 - [w.proc for w in unready_workers])

112 + [uw.proc for uw in unready_workers])

113

114 # For pipeline parallel, we use a thread pool for asynchronous

115 # execute_model.

@@ -282,6 +286,10 @@ class MultiprocExecutor(Executor):

286

287 if workers := getattr(self, 'workers', None):

288 for w in workers:

289 + # Close death_writer to signal child processes to exit

290 + if w.death_writer is not None:

291 + w.death_writer.close()

292 + w.death_writer = None

293 w.worker_response_mq = None

294 self._ensure_worker_termination([w.proc for w in workers])

295

@@ -316,6 +324,7 @@ class UnreadyWorkerProcHandle:

324 proc: BaseProcess

325 rank: int

326 ready_pipe: Connection

327 + death_writer: Optional[Connection] = None

328

329

330 @dataclass

@@ -323,6 +332,7 @@ class WorkerProcHandle:

332 proc: BaseProcess

333 rank: int

334 worker_response_mq: MessageQueue # The worker process writes to this MQ

335 + death_writer: Optional[Connection] = None

336

337 @classmethod

338 def from_unready_handle(

@@ -332,6 +342,7 @@ class WorkerProcHandle:

342 proc=unready_handle.proc,

343 rank=unready_handle.rank,

344 worker_response_mq=worker_response_mq,

345 + death_writer=unready_handle.death_writer,

346 )

347

348

@@ -396,6 +407,9 @@ class WorkerProc:

407 # (reader, writer)

408 reader, writer = context.Pipe(duplex=False)

409

410 + # Create death pipe to detect parent process exit

411 + death_reader, death_writer = context.Pipe(duplex=False)

412 +

413 process_kwargs = {

414 "vllm_config": vllm_config,

415 "local_rank": local_rank,

@@ -403,6 +417,7 @@ class WorkerProc:

417 "distributed_init_method": distributed_init_method,

418 "input_shm_handle": input_shm_handle,

419 "ready_pipe": (reader, writer),

420 + "death_pipe": death_reader,

421 }

422 # Run EngineCore busy loop in background process.

423 proc = context.Process(target=WorkerProc.worker_main,

@@ -412,7 +427,9 @@ class WorkerProc:

427

428 proc.start()

429 writer.close()

415 - return UnreadyWorkerProcHandle(proc, rank, reader)

430 + # Keep death_writer open in parent - when parent exits,

431 + # death_reader in child will get EOFError

432 + return UnreadyWorkerProcHandle(proc, rank, reader, death_writer)

433

434 @staticmethod

435 def wait_for_ready(

@@ -483,6 +500,28 @@ class WorkerProc:

500 worker = None

501 # tuple[Connection, Connection]

502 reader, ready_writer = kwargs.pop("ready_pipe")

503 + death_pipe = kwargs.pop("death_pipe", None)

504 +

505 + # Start death monitoring thread if death_pipe is provided

506 + if death_pipe is not None:

507 +

508 + def monitor_parent_death():

509 + try:

510 + # This will block until parent process exits (pipe closes)

511 + death_pipe.recv()

512 + except EOFError:

513 + # Parent process has exited, terminate this worker

514 + logger.info("Parent process exited, terminating worker")

515 + # Send signal to self to trigger clean shutdown

516 + os.kill(os.getpid(), signal.SIGTERM)

517 + except Exception as e:

518 + logger.warning("Death monitoring error: %s", e)

519 +

520 + death_monitor = Thread(target=monitor_parent_death,

521 + daemon=True,

522 + name="WorkerDeathMonitor")

523 + death_monitor.start()

524 +

525 try:

526 reader.close()

527 worker = WorkerProc(*args, **kwargs)

@@ -523,6 +562,8 @@ class WorkerProc:

562 finally:

563 if ready_writer is not None:

564 ready_writer.close()

565 + if death_pipe is not None:

566 + death_pipe.close()

567 # Clean up once worker exits busy loop

568 if worker is not None:

569 worker.shutdown()