A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 5, 19:45 UTC
[Refactor] Unify engine process monitoring in engine manager and add Ray backend support (#35862)
Signed-off-by: fangyuchu <fangyuchu@qq.com> Signed-off-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: Nick Hill <nickhill123@gmail.com>
fangyuchu committed
Mar 31, 2026 at 01:16 UTC
b5e608258e7b5e4abadf84ffee36e584d7e00b7d
4 files changed
+111
-42
vllm/entrypoints/cli/serve.py
+1
-1
@@ -225,7 +225,7 @@ def run_headless(args: argparse.Namespace):
225
)
226
227
try:
228
- engine_manager.join_first()
228
+ engine_manager.monitor_engine_liveness()
229
finally:
230
timeout = None
231
if shutdown_requested:
vllm/v1/engine/core_client.py
+6
-18
@@ -2,7 +2,6 @@
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
import asyncio
4
import contextlib
5
-import multiprocessing
5
import queue
6
import sys
7
import uuid
@@ -640,34 +639,20 @@ class MPClient(EngineCoreClient):
639
def start_engine_core_monitor(self):
640
"""Start a monitor thread for engine core processes."""
641
engine_manager = self.resources.engine_manager
643
- if (
644
- engine_manager is None
645
- or not hasattr(engine_manager, "processes")
646
- or not engine_manager.processes
647
- ):
642
+ if engine_manager is None:
643
# No engine processes to monitor
644
return
645
651
- engine_processes = engine_manager.processes
646
self_ref = weakref.ref(self)
647
648
# Monitor engine core process liveness. If any die unexpectedly,
655
- # logs an error, shuts down the client and invokes the failure
656
- # callback to inform the engine.
649
+ # marks the engine as dead, and shuts down the client.
650
def monitor_engine_cores():
658
- sentinels = [proc.sentinel for proc in engine_processes]
659
- died = multiprocessing.connection.wait(sentinels)
651
+ engine_manager.monitor_engine_liveness()
652
_self = self_ref()
653
if not _self or not _self._finalizer.alive or _self.resources.engine_dead:
654
return
655
_self.resources.engine_dead = True
664
- proc_name = next(
665
- proc.name for proc in engine_processes if proc.sentinel == died[0]
666
- )
667
- logger.error(
668
- "Engine core proc %s died unexpectedly, shutting down client.",
669
- proc_name,
670
- )
656
_self.shutdown()
657
# Note: For MPClient, we don't have a failure callback mechanism
658
# like MultiprocExecutor, but we set engine_dead flag which will
@@ -1634,6 +1619,9 @@ class DPLBAsyncMPClient(DPAsyncMPClient):
1619
parallel_config = self.vllm_config.parallel_config
1620
ip, coord_store_port = self._setup_elastic_ep_reconfig_bootstrap()
1621
1622
+ removed_dp_size = cur_data_parallel_size - new_data_parallel_size
1623
+ assert isinstance(self.resources.engine_manager, CoreEngineActorManager)
1624
+ self.resources.engine_manager.remove_run_refs_for_scale_down(removed_dp_size)
1625
reconfig_futures = []
1626
for cur_dp_rank, engine in enumerate(self.core_engines):
1627
reconfig_request = ReconfigureDistributedRequest(
vllm/v1/engine/utils.py
+81
-6
@@ -11,7 +11,7 @@ from enum import Enum, auto
11
from multiprocessing import Process, connection
12
from multiprocessing.process import BaseProcess
13
from multiprocessing.queues import Queue
14
-from typing import TYPE_CHECKING
14
+from typing import TYPE_CHECKING, cast
15
from unittest.mock import patch
16
17
import msgspec
@@ -133,6 +133,8 @@ class CoreEngineProcManager:
133
)
134
135
self._finalizer = weakref.finalize(self, shutdown, self.processes)
136
+ self.manager_stopped = threading.Event()
137
+ self.failed_proc_name: str | None = None
138
139
try:
140
for proc, local_dp_rank in zip(self.processes, local_dp_ranks):
@@ -154,12 +156,31 @@ class CoreEngineProcManager:
156
157
def shutdown(self, timeout: float | None = None) -> None:
158
"""Shutdown engine core processes with configurable timeout."""
159
+ self.manager_stopped.set()
160
if self._finalizer.detach() is not None:
161
shutdown(self.processes, timeout=timeout)
162
160
- def join_first(self):
161
- """Wait for any process to exit."""
162
- connection.wait(proc.sentinel for proc in self.processes)
163
+ def monitor_engine_liveness(self) -> None:
164
+ """Monitor engine core process liveness."""
165
+
166
+ sentinel_to_proc = {proc.sentinel: proc for proc in self.processes}
167
+ sentinels = set(sentinel_to_proc.keys())
168
+
169
+ while sentinels and not self.manager_stopped.is_set():
170
+ died_sentinels = connection.wait(sentinels, timeout=1)
171
+
172
+ for sentinel in died_sentinels:
173
+ proc = sentinel_to_proc.pop(cast(int, sentinel))
174
+ exitcode = proc.exitcode
175
+ if exitcode != 0 and not self.manager_stopped.is_set():
176
+ self.failed_proc_name = proc.name
177
+ if died_sentinels:
178
+ # Any engine exit currently triggers a shutdown. Future
179
+ # work (e.g., Elastic and fault-tolerant EP) will add finer-grained
180
+ # handling for different exit scenarios.
181
+ break
182
+
183
+ self.shutdown()
184
185
def sentinels(self) -> list:
186
return [proc.sentinel for proc in self.processes]
@@ -298,6 +319,8 @@ class CoreEngineActorManager:
319
self.log_stats = log_stats
320
local_engine_count = vllm_config.parallel_config.data_parallel_size_local
321
world_size = vllm_config.parallel_config.world_size
322
+ self.manager_stopped = threading.Event()
323
+ self.failed_proc_name: str | None = None
324
325
if ray.is_initialized():
326
logger.info("Ray is already initialized. Skipping Ray initialization.")
@@ -395,8 +418,11 @@ class CoreEngineActorManager:
418
419
ray.get(refs)
420
self.run_refs = []
421
+ self.actor_run_ref_dict = dict()
422
for actor in self.local_engine_actors + self.remote_engine_actors:
399
- self.run_refs.append(actor.run.remote())
423
+ ref = actor.run.remote()
424
+ self.run_refs.append(ref)
425
+ self.actor_run_ref_dict[actor] = ref
426
427
@staticmethod
428
def create_dp_placement_groups(
@@ -776,7 +802,9 @@ class CoreEngineActorManager:
802
) + self.remote_engine_actors[-(len(placement_groups) - new_local_engines) :]
803
804
for actor in actors:
779
- self.run_refs.append(actor.run.remote())
805
+ ref = actor.run.remote()
806
+ self.run_refs.append(ref)
807
+ self.actor_run_ref_dict[actor] = ref
808
809
cur_vllm_config.parallel_config.data_parallel_size = new_data_parallel_size
810
# Update old_vllm_config with new data_parallel_size_local if any new
@@ -805,12 +833,59 @@ class CoreEngineActorManager:
833
self.remote_engine_actors.pop()
834
ray.util.remove_placement_group(pg)
835
836
+ def remove_run_refs_for_scale_down(self, removed_dp_size: int) -> None:
837
+ if removed_dp_size <= 0:
838
+ return
839
+ flags = self.placement_group_is_local[-removed_dp_size:]
840
+ li = len(self.local_engine_actors) - 1
841
+ ri = len(self.remote_engine_actors) - 1
842
+ for is_local in reversed(flags):
843
+ if is_local:
844
+ actor = self.local_engine_actors[li]
845
+ li -= 1
846
+ else:
847
+ actor = self.remote_engine_actors[ri]
848
+ ri -= 1
849
+ ref = self.actor_run_ref_dict.pop(actor)
850
+ self.run_refs.remove(ref)
851
+
852
def get_run_refs(self):
853
return self.run_refs
854
855
+ def monitor_engine_liveness(self) -> None:
856
+ import ray
857
+
858
+ while not self.manager_stopped.is_set():
859
+ actor_run_refs = list(self.get_run_refs())
860
+ if not actor_run_refs:
861
+ logger.info(
862
+ "There are no actors to monitor currently. "
863
+ "The monitoring function is about to terminate."
864
+ )
865
+ break
866
+ actor_done_refs, _ = ray.wait(actor_run_refs, timeout=5)
867
+ unexpected_failure = False
868
+ for actor_ref in actor_done_refs:
869
+ if self.manager_stopped.is_set():
870
+ break
871
+ if actor_ref not in self.get_run_refs():
872
+ # The run refs may have been updated by elastic scale-down.
873
+ continue
874
+ try:
875
+ ray.get(actor_ref)
876
+ except ray.exceptions.RayActorError:
877
+ self.failed_proc_name = f"Actor {actor_ref}"
878
+ unexpected_failure = True
879
+
880
+ if unexpected_failure:
881
+ break
882
+
883
+ self.shutdown()
884
+
885
def shutdown(self, timeout: float | None = None) -> None:
886
import ray
887
888
+ self.manager_stopped.set()
889
for actor in self.local_engine_actors + self.remote_engine_actors:
890
ray.kill(actor)
891
for pg in self.created_placement_groups:
vllm/v1/utils.py
+23
-17
@@ -3,6 +3,7 @@
3
import argparse
4
import contextlib
5
import multiprocessing
6
+import threading
7
import time
8
import weakref
9
from collections.abc import Callable, Sequence
@@ -269,8 +270,6 @@ def wait_for_completion_or_failure(
270
coordinator: The coordinator for data parallel.
271
"""
272
272
- from vllm.v1.engine.utils import CoreEngineActorManager, CoreEngineProcManager
273
-
273
try:
274
logger.info("Waiting for API servers to complete ...")
275
# Create a mapping of sentinels to their corresponding processes
@@ -282,33 +281,40 @@ def wait_for_completion_or_failure(
281
if coordinator:
282
sentinel_to_proc[coordinator.proc.sentinel] = coordinator.proc
283
285
- actor_run_refs = []
286
- if isinstance(engine_manager, CoreEngineProcManager):
287
- for proc in engine_manager.processes:
288
- sentinel_to_proc[proc.sentinel] = proc
289
- elif isinstance(engine_manager, CoreEngineActorManager):
290
- actor_run_refs = engine_manager.get_run_refs()
284
+ if engine_manager:
285
+ core_shutdown_recv, core_shutdown_send = connection.Pipe(duplex=False)
286
+
287
+ def monitor_engines():
288
+ try:
289
+ engine_manager.monitor_engine_liveness()
290
+ finally:
291
+ core_shutdown_send.close()
292
+ core_shutdown_recv.close()
293
+
294
+ # start monitor for engine liveness
295
+ threading.Thread(target=monitor_engines, daemon=True).start()
296
+ sentinel_to_proc[core_shutdown_recv] = None # type: ignore[assignment]
297
298
# Check if any process terminates
293
- while sentinel_to_proc or actor_run_refs:
294
- # Wait for any process to terminate
295
- ready_sentinels: list[Any] = connection.wait(sentinel_to_proc, timeout=5)
299
+ while sentinel_to_proc:
300
+ # Wait for any process to terminate (or engine shutdown signal)
301
+ ready_sentinels: list[Any] = connection.wait(sentinel_to_proc)
302
303
# Process any terminated processes
304
for sentinel in ready_sentinels:
305
proc = sentinel_to_proc.pop(sentinel)
306
307
# Check if process exited with error
302
- if proc.exitcode != 0:
308
+ if proc is not None and proc.exitcode != 0:
309
raise RuntimeError(
310
f"Process {proc.name} (PID: {proc.pid}) "
311
f"died with exit code {proc.exitcode}"
312
)
307
-
308
- if actor_run_refs:
309
- import ray
310
-
311
- _, actor_run_refs = ray.wait(actor_run_refs, timeout=5)
313
+ if engine_manager and engine_manager.failed_proc_name is not None:
314
+ raise RuntimeError(
315
+ f"Engine core process {engine_manager.failed_proc_name} "
316
+ "died unexpectedly."
317
+ )
318
319
except KeyboardInterrupt:
320
logger.info("Received KeyboardInterrupt, shutting down API servers...")