A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 03:15 UTC
[V1] Simplify Shutdown (#11659)
Robert Shaw committed
Jan 3, 2025 at 12:25 UTC
80c751e7f68ade3d4c6391a0f3fce9ce970ddad0
7 files changed
+42
-60
tests/v1/engine/test_engine_core_client.py
-6
@@ -142,9 +142,6 @@ def test_engine_core_client(monkeypatch, multiprocessing_mode: bool):
142
143
client.abort_requests([request.request_id])
144
145
- # Shutdown the client.
146
- client.shutdown()
147
-
145
146
@pytest.mark.asyncio
147
async def test_engine_core_client_asyncio(monkeypatch):
@@ -200,6 +197,3 @@ async def test_engine_core_client_asyncio(monkeypatch):
197
else:
198
assert len(outputs[req_id]) == MAX_TOKENS, (
199
f"{len(outputs[req_id])=}, {MAX_TOKENS=}")
203
-
204
- # Shutdown the client.
205
- client.shutdown()
vllm/entrypoints/llm.py
-5
@@ -232,11 +232,6 @@ class LLM:
232
233
self.request_counter = Counter()
234
235
- def __del__(self):
236
- if hasattr(self, 'llm_engine') and self.llm_engine and hasattr(
237
- self.llm_engine, "shutdown"):
238
- self.llm_engine.shutdown()
239
-
235
@staticmethod
236
def get_engine_class() -> Type[LLMEngine]:
237
if envs.VLLM_USE_V1:
vllm/v1/engine/async_llm.py
-3
@@ -103,9 +103,6 @@ class AsyncLLM(EngineClient):
103
104
self.output_handler: Optional[asyncio.Task] = None
105
106
- def __del__(self):
107
- self.shutdown()
108
-
106
@classmethod
107
def from_engine_args(
108
cls,
vllm/v1/engine/core.py
-1
@@ -203,7 +203,6 @@ class EngineCoreProc(EngineCore):
203
finally:
204
if engine_core is not None:
205
engine_core.shutdown()
206
- engine_core = None
206
207
def run_busy_loop(self):
208
"""Core busy loop of the EngineCore."""
vllm/v1/engine/core_client.py
+18
-16
@@ -1,4 +1,6 @@
1
-from typing import List, Optional, Type
1
+import weakref
2
+from abc import ABC, abstractmethod
3
+from typing import List, Type
4
5
import msgspec
6
import zmq
@@ -18,7 +20,7 @@ from vllm.v1.utils import BackgroundProcHandle
20
logger = init_logger(__name__)
21
22
21
-class EngineCoreClient:
23
+class EngineCoreClient(ABC):
24
"""
25
EngineCoreClient: subclasses handle different methods for pushing
26
and pulling from the EngineCore for asyncio / multiprocessing.
@@ -52,8 +54,9 @@ class EngineCoreClient:
54
55
return InprocClient(vllm_config, executor_class, log_stats)
56
57
+ @abstractmethod
58
def shutdown(self):
56
- pass
59
+ ...
60
61
def get_output(self) -> List[EngineCoreOutput]:
62
raise NotImplementedError
@@ -107,9 +110,6 @@ class InprocClient(EngineCoreClient):
110
def shutdown(self):
111
self.engine_core.shutdown()
112
110
- def __del__(self):
111
- self.shutdown()
112
-
113
def profile(self, is_start: bool = True) -> None:
114
self.engine_core.profile(is_start)
115
@@ -139,10 +139,14 @@ class MPClient(EngineCoreClient):
139
self.decoder = msgspec.msgpack.Decoder(EngineCoreOutputs)
140
141
# ZMQ setup.
142
- if asyncio_mode:
143
- self.ctx = zmq.asyncio.Context()
144
- else:
145
- self.ctx = zmq.Context() # type: ignore[attr-defined]
142
+ self.ctx = (
143
+ zmq.asyncio.Context() # type: ignore[attr-defined]
144
+ if asyncio_mode else zmq.Context()) # type: ignore[attr-defined]
145
+
146
+ # Note(rob): shutdown function cannot be a bound method,
147
+ # else the gc cannot collect the object.
148
+ self._finalizer = weakref.finalize(self, lambda x: x.destroy(linger=0),
149
+ self.ctx)
150
151
# Paths and sockets for IPC.
152
output_path = get_open_zmq_ipc_path()
@@ -153,7 +157,6 @@ class MPClient(EngineCoreClient):
157
zmq.constants.PUSH)
158
159
# Start EngineCore in background process.
156
- self.proc_handle: Optional[BackgroundProcHandle]
160
self.proc_handle = BackgroundProcHandle(
161
input_path=input_path,
162
output_path=output_path,
@@ -166,12 +169,11 @@ class MPClient(EngineCoreClient):
169
})
170
171
def shutdown(self):
169
- # Shut down the zmq context.
170
- self.ctx.destroy(linger=0)
171
-
172
- if hasattr(self, "proc_handle") and self.proc_handle:
172
+ """Clean up background resources."""
173
+ if hasattr(self, "proc_handle"):
174
self.proc_handle.shutdown()
174
- self.proc_handle = None
175
+
176
+ self._finalizer()
177
178
179
class SyncMPClient(MPClient):
vllm/v1/engine/llm_engine.py
-7
@@ -205,10 +205,3 @@ class LLMEngine:
205
f"found type: {type(tokenizer_group)}")
206
207
return tokenizer_group
208
-
209
- def __del__(self):
210
- self.shutdown()
211
-
212
- def shutdown(self):
213
- if engine_core := getattr(self, "engine_core", None):
214
- engine_core.shutdown()
vllm/v1/utils.py
+24
-22
@@ -1,3 +1,4 @@
1
+import multiprocessing
2
import os
3
import weakref
4
from collections.abc import Sequence
@@ -91,8 +92,6 @@ class BackgroundProcHandle:
92
target_fn: Callable,
93
process_kwargs: Dict[Any, Any],
94
):
94
- self._finalizer = weakref.finalize(self, self.shutdown)
95
-
95
context = get_mp_context()
96
reader, writer = context.Pipe(duplex=False)
97
@@ -102,11 +101,11 @@ class BackgroundProcHandle:
101
process_kwargs["ready_pipe"] = writer
102
process_kwargs["input_path"] = input_path
103
process_kwargs["output_path"] = output_path
105
- self.input_path = input_path
106
- self.output_path = output_path
104
108
- # Run Detokenizer busy loop in background process.
105
+ # Run busy loop in background process.
106
self.proc = context.Process(target=target_fn, kwargs=process_kwargs)
107
+ self._finalizer = weakref.finalize(self, shutdown, self.proc,
108
+ input_path, output_path)
109
self.proc.start()
110
111
# Wait for startup.
@@ -114,21 +113,24 @@ class BackgroundProcHandle:
113
raise RuntimeError(f"{process_name} initialization failed. "
114
"See root cause above.")
115
117
- def __del__(self):
118
- self.shutdown()
119
-
116
def shutdown(self):
121
- # Shutdown the process if needed.
122
- if hasattr(self, "proc") and self.proc.is_alive():
123
- self.proc.terminate()
124
- self.proc.join(5)
125
-
126
- if self.proc.is_alive():
127
- kill_process_tree(self.proc.pid)
128
-
129
- # Remove zmq ipc socket files
130
- ipc_sockets = [self.output_path, self.input_path]
131
- for ipc_socket in ipc_sockets:
132
- socket_file = ipc_socket.replace("ipc://", "")
133
- if os and os.path.exists(socket_file):
134
- os.remove(socket_file)
117
+ self._finalizer()
118
+
119
+
120
+# Note(rob): shutdown function cannot be a bound method,
121
+# else the gc cannot collect the object.
122
+def shutdown(proc: multiprocessing.Process, input_path: str, output_path: str):
123
+ # Shutdown the process.
124
+ if proc.is_alive():
125
+ proc.terminate()
126
+ proc.join(5)
127
+
128
+ if proc.is_alive():
129
+ kill_process_tree(proc.pid)
130
+
131
+ # Remove zmq ipc socket files.
132
+ ipc_sockets = [output_path, input_path]
133
+ for ipc_socket in ipc_sockets:
134
+ socket_file = ipc_socket.replace("ipc://", "")
135
+ if os and os.path.exists(socket_file):
136
+ os.remove(socket_file)