A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[RFC] allow cancelation after shutdown in blocking collective_rpc (#23390)
Signed-off-by: Shiyan Deng <dsy842974287@meta.com>
Shiyan Deng committed
Sep 5, 2025 at 14:14 UTC
9dfbeb41e510ad04e90b03ef2f437f476c5abe28
1 file changed
+8
-6
vllm/v1/executor/multiproc_executor.py
+8
-6
@@ -253,7 +253,8 @@ class MultiprocExecutor(Executor):
253
if not non_block:
254
result = result.result()
255
elif not non_block:
256
- result = get_response(w, dequeue_timeout)
256
+ result = get_response(w, dequeue_timeout,
257
+ self.shutdown_event)
258
else:
259
raise RuntimeError("non_block can only be used when"
260
" max_concurrent_batches > 1")
@@ -295,12 +296,8 @@ class MultiprocExecutor(Executor):
296
"""Properly shut down the executor and its workers"""
297
if not getattr(self, 'shutting_down', False):
298
self.shutting_down = True
298
- self.shutdown_event.set()
299
-
300
- if self.io_thread_pool is not None:
301
- self.io_thread_pool.shutdown(wait=False, cancel_futures=True)
302
- self.io_thread_pool = None
299
300
+ # Make sure all the worker processes are terminated first.
301
if workers := getattr(self, 'workers', None):
302
for w in workers:
303
# Close death_writer to signal child processes to exit
@@ -310,6 +307,11 @@ class MultiprocExecutor(Executor):
307
w.worker_response_mq = None
308
self._ensure_worker_termination([w.proc for w in workers])
309
310
+ self.shutdown_event.set()
311
+ if self.io_thread_pool is not None:
312
+ self.io_thread_pool.shutdown(wait=False, cancel_futures=True)
313
+ del self.io_thread_pool
314
+
315
self.rpc_broadcast_mq = None
316
317
def check_health(self) -> None: