@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / ccede2b264
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 04:30 UTC

[Core] cleanup zmq ipc sockets on exit (#11115)

Signed-off-by: Russell Bryant <rbryant@redhat.com>

Russell Bryant committed Dec 11, 2024 at 22:12 UTC ccede2b264668d854cba4fce7f8fbbf203908f60
4 files changed +57 -17
vllm/entrypoints/openai/api_server.py
+9
@@ -1,4 +1,5 @@
1 import asyncio
2 +import atexit
3 import importlib
4 import inspect
5 import multiprocessing
@@ -196,6 +197,14 @@ async def build_async_engine_client_from_engine_args(
197 assert engine_pid is not None, "Engine process failed to start."
198 logger.info("Started engine process with PID %d", engine_pid)
199
200 + def _cleanup_ipc_path():
201 + socket_path = ipc_path.replace("ipc://", "")
202 + if os.path.exists(socket_path):
203 + os.remove(socket_path)
204 +
205 + # Ensure we clean up the local IPC socket file on exit.
206 + atexit.register(_cleanup_ipc_path)
207 +
208 # Build RPCClient, which conforms to EngineClient Protocol.
209 engine_config = engine_args.create_engine_config()
210 build_client = partial(MQLLMEngineClient, ipc_path, engine_config,
vllm/v1/engine/core.py
+14 -2
@@ -4,6 +4,7 @@ import queue
4 import signal
5 import threading
6 import time
7 +from dataclasses import dataclass
8 from multiprocessing.process import BaseProcess
9 from typing import List, Tuple, Type, Union
10
@@ -129,6 +130,14 @@ class EngineCore:
130 self.model_executor.profile(is_start)
131
132
133 +@dataclass
134 +class EngineCoreProcHandle:
135 + proc: BaseProcess
136 + ready_path: str
137 + input_path: str
138 + output_path: str
139 +
140 +
141 class EngineCoreProc(EngineCore):
142 """ZMQ-wrapper for running EngineCore in background process."""
143
@@ -200,7 +209,7 @@ class EngineCoreProc(EngineCore):
209 input_path: str,
210 output_path: str,
211 ready_path: str,
203 - ) -> BaseProcess:
212 + ) -> EngineCoreProcHandle:
213 # The current process might have CUDA context,
214 # so we need to spawn a new process.
215 # NOTE(rob): this is a problem for using EngineCoreProc w/
@@ -222,7 +231,10 @@ class EngineCoreProc(EngineCore):
231
232 # Wait for startup
233 EngineCoreProc.wait_for_startup(proc, ready_path)
225 - return proc
234 + return EngineCoreProcHandle(proc=proc,
235 + ready_path=ready_path,
236 + input_path=input_path,
237 + output_path=output_path)
238
239 @staticmethod
240 def run_engine_core(*args, **kwargs):
vllm/v1/engine/core_client.py
+20 -8
@@ -1,4 +1,5 @@
1 import atexit
2 +import os
3 from typing import List, Union
4
5 import msgspec
@@ -148,7 +149,7 @@ class MPClient(EngineCoreClient):
149 self.input_socket.bind(input_path)
150
151 # Start EngineCore in background process.
151 - self.proc = EngineCoreProc.make_engine_core_process(
152 + self.proc_handle = EngineCoreProc.make_engine_core_process(
153 *args,
154 input_path=input_path,
155 output_path=output_path,
@@ -161,13 +162,24 @@ class MPClient(EngineCoreClient):
162 # Shut down the zmq context.
163 self.ctx.destroy(linger=0)
164
164 - # Shutdown the process if needed.
165 - if hasattr(self, "proc") and self.proc.is_alive():
166 - self.proc.terminate()
167 - self.proc.join(5)
168 -
169 - if self.proc.is_alive():
170 - kill_process_tree(self.proc.pid)
165 + if hasattr(self, "proc_handle"):
166 + # Shutdown the process if needed.
167 + if self.proc_handle.proc.is_alive():
168 + self.proc_handle.proc.terminate()
169 + self.proc_handle.proc.join(5)
170 +
171 + if self.proc_handle.proc.is_alive():
172 + kill_process_tree(self.proc_handle.proc.pid)
173 +
174 + # Remove zmq ipc socket files
175 + ipc_sockets = [
176 + self.proc_handle.ready_path, self.proc_handle.output_path,
177 + self.proc_handle.input_path
178 + ]
179 + for ipc_socket in ipc_sockets:
180 + socket_file = ipc_socket.replace("ipc://", "")
181 + if os.path.exists(socket_file):
182 + os.remove(socket_file)
183
184 def __del__(self):
185 self.shutdown()
vllm/v1/executor/multiproc_executor.py
+14 -7
@@ -172,16 +172,23 @@ class MultiprocExecutor:
172
173 # Send SIGTERM if still running
174 active_procs = [w.proc for w in self.workers if w.proc.is_alive()]
175 - self.workers = None
175 for p in active_procs:
176 p.terminate()
178 - if wait_for_termination(active_procs, 4):
179 - return
177 + if not wait_for_termination(active_procs, 4):
178 + # Send SIGKILL if still running
179 + active_procs = [p for p in active_procs if p.is_alive()]
180 + for p in active_procs:
181 + p.kill()
182
181 - # Send SIGKILL if still running
182 - active_procs = [p for p in active_procs if p.is_alive()]
183 - for p in active_procs:
184 - p.kill()
183 + self._cleanup_sockets()
184 + self.workers = None
185 +
186 + def _cleanup_sockets(self):
187 + for w in self.workers:
188 + # Remove the zmq ipc socket file
189 + socket_path = w.ready_path.replace("ipc://", "")
190 + if os.path.exists(socket_path):
191 + os.remove(socket_path)
192
193 def shutdown(self):
194 """Properly shut down the executor and its workers"""