@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 2bfdbf2a36
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 04:30 UTC

[V1][Core] Use weakref.finalize instead of atexit (#11242)

Signed-off-by: Tyler Michael Smith <tyler@neuralmagic.com>

Tyler Michael Smith committed Dec 17, 2024 at 01:11 UTC 2bfdbf2a36256bb08547cea3d4ef83b5d27c4b04
2 files changed +5 -18
vllm/v1/engine/core_client.py
+2 -11
@@ -1,5 +1,5 @@
1 -import atexit
1 import os
2 +import weakref
3 from typing import List, Optional
4
5 import msgspec
@@ -165,15 +165,9 @@ class MPClient(EngineCoreClient):
165 ready_path=ready_path, # type: ignore[misc]
166 **kwargs,
167 )
168 - atexit.register(self.shutdown)
168 + self._finalizer = weakref.finalize(self, self.shutdown)
169
170 def shutdown(self):
171 - # During final garbage collection in process shutdown, atexit may be
172 - # None.
173 - if atexit:
174 - # in case shutdown gets called via __del__ first
175 - atexit.unregister(self.shutdown)
176 -
171 # Shut down the zmq context.
172 self.ctx.destroy(linger=0)
173
@@ -197,9 +191,6 @@ class MPClient(EngineCoreClient):
191 os.remove(socket_file)
192 self.proc_handle = None
193
200 - def __del__(self):
201 - self.shutdown()
202 -
194
195 class SyncMPClient(MPClient):
196 """Synchronous client for multi-proc EngineCore."""
vllm/v1/executor/multiproc_executor.py
+3 -7
@@ -1,9 +1,9 @@
1 -import atexit
1 import os
2 import pickle
3 import signal
4 import sys
5 import time
6 +import weakref
7 from dataclasses import dataclass
8 from enum import Enum, auto
9 from multiprocessing.process import BaseProcess
@@ -37,7 +37,7 @@ class MultiprocExecutor(Executor):
37 def __init__(self, vllm_config: VllmConfig) -> None:
38 # Call self.shutdown at exit to clean up
39 # and ensure workers will be terminated.
40 - atexit.register(self.shutdown)
40 + self._finalizer = weakref.finalize(self, self.shutdown)
41
42 self.vllm_config = vllm_config
43 self.parallel_config = vllm_config.parallel_config
@@ -195,14 +195,10 @@ class MultiprocExecutor(Executor):
195 os.remove(socket_path)
196
197 def shutdown(self):
198 - if atexit:
199 - # in case shutdown was called explicitly, we don't need to call it
200 - # again
201 - atexit.unregister(self.shutdown)
198 """Properly shut down the executor and its workers"""
199 if getattr(self, 'shutting_down', False):
200 self.shutting_down = True
205 - for w in self.workers: #TODO: not sure if needed
201 + for w in self.workers:
202 w.worker_response_mq = None
203 self._ensure_worker_termination()
204