A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 04:30 UTC
[Core] cleanup zmq ipc sockets on exit (#11115)
Signed-off-by: Russell Bryant <rbryant@redhat.com>
Russell Bryant committed
Dec 11, 2024 at 22:12 UTC
ccede2b264668d854cba4fce7f8fbbf203908f60
4 files changed
+57
-17
vllm/entrypoints/openai/api_server.py
+9
@@ -1,4 +1,5 @@
1
import asyncio
2
+import atexit
3
import importlib
4
import inspect
5
import multiprocessing
@@ -196,6 +197,14 @@ async def build_async_engine_client_from_engine_args(
197
assert engine_pid is not None, "Engine process failed to start."
198
logger.info("Started engine process with PID %d", engine_pid)
199
200
+ def _cleanup_ipc_path():
201
+ socket_path = ipc_path.replace("ipc://", "")
202
+ if os.path.exists(socket_path):
203
+ os.remove(socket_path)
204
+
205
+ # Ensure we clean up the local IPC socket file on exit.
206
+ atexit.register(_cleanup_ipc_path)
207
+
208
# Build RPCClient, which conforms to EngineClient Protocol.
209
engine_config = engine_args.create_engine_config()
210
build_client = partial(MQLLMEngineClient, ipc_path, engine_config,
vllm/v1/engine/core.py
+14
-2
@@ -4,6 +4,7 @@ import queue
4
import signal
5
import threading
6
import time
7
+from dataclasses import dataclass
8
from multiprocessing.process import BaseProcess
9
from typing import List, Tuple, Type, Union
10
@@ -129,6 +130,14 @@ class EngineCore:
130
self.model_executor.profile(is_start)
131
132
133
+@dataclass
134
+class EngineCoreProcHandle:
135
+ proc: BaseProcess
136
+ ready_path: str
137
+ input_path: str
138
+ output_path: str
139
+
140
+
141
class EngineCoreProc(EngineCore):
142
"""ZMQ-wrapper for running EngineCore in background process."""
143
@@ -200,7 +209,7 @@ class EngineCoreProc(EngineCore):
209
input_path: str,
210
output_path: str,
211
ready_path: str,
203
- ) -> BaseProcess:
212
+ ) -> EngineCoreProcHandle:
213
# The current process might have CUDA context,
214
# so we need to spawn a new process.
215
# NOTE(rob): this is a problem for using EngineCoreProc w/
@@ -222,7 +231,10 @@ class EngineCoreProc(EngineCore):
231
232
# Wait for startup
233
EngineCoreProc.wait_for_startup(proc, ready_path)
225
- return proc
234
+ return EngineCoreProcHandle(proc=proc,
235
+ ready_path=ready_path,
236
+ input_path=input_path,
237
+ output_path=output_path)
238
239
@staticmethod
240
def run_engine_core(*args, **kwargs):
vllm/v1/engine/core_client.py
+20
-8
@@ -1,4 +1,5 @@
1
import atexit
2
+import os
3
from typing import List, Union
4
5
import msgspec
@@ -148,7 +149,7 @@ class MPClient(EngineCoreClient):
149
self.input_socket.bind(input_path)
150
151
# Start EngineCore in background process.
151
- self.proc = EngineCoreProc.make_engine_core_process(
152
+ self.proc_handle = EngineCoreProc.make_engine_core_process(
153
*args,
154
input_path=input_path,
155
output_path=output_path,
@@ -161,13 +162,24 @@ class MPClient(EngineCoreClient):
162
# Shut down the zmq context.
163
self.ctx.destroy(linger=0)
164
164
- # Shutdown the process if needed.
165
- if hasattr(self, "proc") and self.proc.is_alive():
166
- self.proc.terminate()
167
- self.proc.join(5)
168
-
169
- if self.proc.is_alive():
170
- kill_process_tree(self.proc.pid)
165
+ if hasattr(self, "proc_handle"):
166
+ # Shutdown the process if needed.
167
+ if self.proc_handle.proc.is_alive():
168
+ self.proc_handle.proc.terminate()
169
+ self.proc_handle.proc.join(5)
170
+
171
+ if self.proc_handle.proc.is_alive():
172
+ kill_process_tree(self.proc_handle.proc.pid)
173
+
174
+ # Remove zmq ipc socket files
175
+ ipc_sockets = [
176
+ self.proc_handle.ready_path, self.proc_handle.output_path,
177
+ self.proc_handle.input_path
178
+ ]
179
+ for ipc_socket in ipc_sockets:
180
+ socket_file = ipc_socket.replace("ipc://", "")
181
+ if os.path.exists(socket_file):
182
+ os.remove(socket_file)
183
184
def __del__(self):
185
self.shutdown()
vllm/v1/executor/multiproc_executor.py
+14
-7
@@ -172,16 +172,23 @@ class MultiprocExecutor:
172
173
# Send SIGTERM if still running
174
active_procs = [w.proc for w in self.workers if w.proc.is_alive()]
175
- self.workers = None
175
for p in active_procs:
176
p.terminate()
178
- if wait_for_termination(active_procs, 4):
179
- return
177
+ if not wait_for_termination(active_procs, 4):
178
+ # Send SIGKILL if still running
179
+ active_procs = [p for p in active_procs if p.is_alive()]
180
+ for p in active_procs:
181
+ p.kill()
182
181
- # Send SIGKILL if still running
182
- active_procs = [p for p in active_procs if p.is_alive()]
183
- for p in active_procs:
184
- p.kill()
183
+ self._cleanup_sockets()
184
+ self.workers = None
185
+
186
+ def _cleanup_sockets(self):
187
+ for w in self.workers:
188
+ # Remove the zmq ipc socket file
189
+ socket_path = w.ready_path.replace("ipc://", "")
190
+ if os.path.exists(socket_path):
191
+ os.remove(socket_path)
192
193
def shutdown(self):
194
"""Properly shut down the executor and its workers"""