@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 80c751e7f6
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 03:15 UTC

[V1] Simplify Shutdown (#11659)

Robert Shaw committed Jan 3, 2025 at 12:25 UTC 80c751e7f68ade3d4c6391a0f3fce9ce970ddad0
7 files changed +42 -60
tests/v1/engine/test_engine_core_client.py
-6
@@ -142,9 +142,6 @@ def test_engine_core_client(monkeypatch, multiprocessing_mode: bool):
142
143 client.abort_requests([request.request_id])
144
145 - # Shutdown the client.
146 - client.shutdown()
147 -
145
146 @pytest.mark.asyncio
147 async def test_engine_core_client_asyncio(monkeypatch):
@@ -200,6 +197,3 @@ async def test_engine_core_client_asyncio(monkeypatch):
197 else:
198 assert len(outputs[req_id]) == MAX_TOKENS, (
199 f"{len(outputs[req_id])=}, {MAX_TOKENS=}")
203 -
204 - # Shutdown the client.
205 - client.shutdown()
vllm/entrypoints/llm.py
-5
@@ -232,11 +232,6 @@ class LLM:
232
233 self.request_counter = Counter()
234
235 - def __del__(self):
236 - if hasattr(self, 'llm_engine') and self.llm_engine and hasattr(
237 - self.llm_engine, "shutdown"):
238 - self.llm_engine.shutdown()
239 -
235 @staticmethod
236 def get_engine_class() -> Type[LLMEngine]:
237 if envs.VLLM_USE_V1:
vllm/v1/engine/async_llm.py
-3
@@ -103,9 +103,6 @@ class AsyncLLM(EngineClient):
103
104 self.output_handler: Optional[asyncio.Task] = None
105
106 - def __del__(self):
107 - self.shutdown()
108 -
106 @classmethod
107 def from_engine_args(
108 cls,
vllm/v1/engine/core.py
-1
@@ -203,7 +203,6 @@ class EngineCoreProc(EngineCore):
203 finally:
204 if engine_core is not None:
205 engine_core.shutdown()
206 - engine_core = None
206
207 def run_busy_loop(self):
208 """Core busy loop of the EngineCore."""
vllm/v1/engine/core_client.py
+18 -16
@@ -1,4 +1,6 @@
1 -from typing import List, Optional, Type
1 +import weakref
2 +from abc import ABC, abstractmethod
3 +from typing import List, Type
4
5 import msgspec
6 import zmq
@@ -18,7 +20,7 @@ from vllm.v1.utils import BackgroundProcHandle
20 logger = init_logger(__name__)
21
22
21 -class EngineCoreClient:
23 +class EngineCoreClient(ABC):
24 """
25 EngineCoreClient: subclasses handle different methods for pushing
26 and pulling from the EngineCore for asyncio / multiprocessing.
@@ -52,8 +54,9 @@ class EngineCoreClient:
54
55 return InprocClient(vllm_config, executor_class, log_stats)
56
57 + @abstractmethod
58 def shutdown(self):
56 - pass
59 + ...
60
61 def get_output(self) -> List[EngineCoreOutput]:
62 raise NotImplementedError
@@ -107,9 +110,6 @@ class InprocClient(EngineCoreClient):
110 def shutdown(self):
111 self.engine_core.shutdown()
112
110 - def __del__(self):
111 - self.shutdown()
112 -
113 def profile(self, is_start: bool = True) -> None:
114 self.engine_core.profile(is_start)
115
@@ -139,10 +139,14 @@ class MPClient(EngineCoreClient):
139 self.decoder = msgspec.msgpack.Decoder(EngineCoreOutputs)
140
141 # ZMQ setup.
142 - if asyncio_mode:
143 - self.ctx = zmq.asyncio.Context()
144 - else:
145 - self.ctx = zmq.Context() # type: ignore[attr-defined]
142 + self.ctx = (
143 + zmq.asyncio.Context() # type: ignore[attr-defined]
144 + if asyncio_mode else zmq.Context()) # type: ignore[attr-defined]
145 +
146 + # Note(rob): shutdown function cannot be a bound method,
147 + # else the gc cannot collect the object.
148 + self._finalizer = weakref.finalize(self, lambda x: x.destroy(linger=0),
149 + self.ctx)
150
151 # Paths and sockets for IPC.
152 output_path = get_open_zmq_ipc_path()
@@ -153,7 +157,6 @@ class MPClient(EngineCoreClient):
157 zmq.constants.PUSH)
158
159 # Start EngineCore in background process.
156 - self.proc_handle: Optional[BackgroundProcHandle]
160 self.proc_handle = BackgroundProcHandle(
161 input_path=input_path,
162 output_path=output_path,
@@ -166,12 +169,11 @@ class MPClient(EngineCoreClient):
169 })
170
171 def shutdown(self):
169 - # Shut down the zmq context.
170 - self.ctx.destroy(linger=0)
171 -
172 - if hasattr(self, "proc_handle") and self.proc_handle:
172 + """Clean up background resources."""
173 + if hasattr(self, "proc_handle"):
174 self.proc_handle.shutdown()
174 - self.proc_handle = None
175 +
176 + self._finalizer()
177
178
179 class SyncMPClient(MPClient):
vllm/v1/engine/llm_engine.py
-7
@@ -205,10 +205,3 @@ class LLMEngine:
205 f"found type: {type(tokenizer_group)}")
206
207 return tokenizer_group
208 -
209 - def __del__(self):
210 - self.shutdown()
211 -
212 - def shutdown(self):
213 - if engine_core := getattr(self, "engine_core", None):
214 - engine_core.shutdown()
vllm/v1/utils.py
+24 -22
@@ -1,3 +1,4 @@
1 +import multiprocessing
2 import os
3 import weakref
4 from collections.abc import Sequence
@@ -91,8 +92,6 @@ class BackgroundProcHandle:
92 target_fn: Callable,
93 process_kwargs: Dict[Any, Any],
94 ):
94 - self._finalizer = weakref.finalize(self, self.shutdown)
95 -
95 context = get_mp_context()
96 reader, writer = context.Pipe(duplex=False)
97
@@ -102,11 +101,11 @@ class BackgroundProcHandle:
101 process_kwargs["ready_pipe"] = writer
102 process_kwargs["input_path"] = input_path
103 process_kwargs["output_path"] = output_path
105 - self.input_path = input_path
106 - self.output_path = output_path
104
108 - # Run Detokenizer busy loop in background process.
105 + # Run busy loop in background process.
106 self.proc = context.Process(target=target_fn, kwargs=process_kwargs)
107 + self._finalizer = weakref.finalize(self, shutdown, self.proc,
108 + input_path, output_path)
109 self.proc.start()
110
111 # Wait for startup.
@@ -114,21 +113,24 @@ class BackgroundProcHandle:
113 raise RuntimeError(f"{process_name} initialization failed. "
114 "See root cause above.")
115
117 - def __del__(self):
118 - self.shutdown()
119 -
116 def shutdown(self):
121 - # Shutdown the process if needed.
122 - if hasattr(self, "proc") and self.proc.is_alive():
123 - self.proc.terminate()
124 - self.proc.join(5)
125 -
126 - if self.proc.is_alive():
127 - kill_process_tree(self.proc.pid)
128 -
129 - # Remove zmq ipc socket files
130 - ipc_sockets = [self.output_path, self.input_path]
131 - for ipc_socket in ipc_sockets:
132 - socket_file = ipc_socket.replace("ipc://", "")
133 - if os and os.path.exists(socket_file):
134 - os.remove(socket_file)
117 + self._finalizer()
118 +
119 +
120 +# Note(rob): shutdown function cannot be a bound method,
121 +# else the gc cannot collect the object.
122 +def shutdown(proc: multiprocessing.Process, input_path: str, output_path: str):
123 + # Shutdown the process.
124 + if proc.is_alive():
125 + proc.terminate()
126 + proc.join(5)
127 +
128 + if proc.is_alive():
129 + kill_process_tree(proc.pid)
130 +
131 + # Remove zmq ipc socket files.
132 + ipc_sockets = [output_path, input_path]
133 + for ipc_socket in ipc_sockets:
134 + socket_file = ipc_socket.replace("ipc://", "")
135 + if os and os.path.exists(socket_file):
136 + os.remove(socket_file)