A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 04:30 UTC
[V1][Core] Use weakref.finalize instead of atexit (#11242)
Signed-off-by: Tyler Michael Smith <tyler@neuralmagic.com>
Tyler Michael Smith committed
Dec 17, 2024 at 01:11 UTC
2bfdbf2a36256bb08547cea3d4ef83b5d27c4b04
2 files changed
+5
-18
vllm/v1/engine/core_client.py
+2
-11
@@ -1,5 +1,5 @@
1
-import atexit
1
import os
2
+import weakref
3
from typing import List, Optional
4
5
import msgspec
@@ -165,15 +165,9 @@ class MPClient(EngineCoreClient):
165
ready_path=ready_path, # type: ignore[misc]
166
**kwargs,
167
)
168
- atexit.register(self.shutdown)
168
+ self._finalizer = weakref.finalize(self, self.shutdown)
169
170
def shutdown(self):
171
- # During final garbage collection in process shutdown, atexit may be
172
- # None.
173
- if atexit:
174
- # in case shutdown gets called via __del__ first
175
- atexit.unregister(self.shutdown)
176
-
171
# Shut down the zmq context.
172
self.ctx.destroy(linger=0)
173
@@ -197,9 +191,6 @@ class MPClient(EngineCoreClient):
191
os.remove(socket_file)
192
self.proc_handle = None
193
200
- def __del__(self):
201
- self.shutdown()
202
-
194
195
class SyncMPClient(MPClient):
196
"""Synchronous client for multi-proc EngineCore."""
vllm/v1/executor/multiproc_executor.py
+3
-7
@@ -1,9 +1,9 @@
1
-import atexit
1
import os
2
import pickle
3
import signal
4
import sys
5
import time
6
+import weakref
7
from dataclasses import dataclass
8
from enum import Enum, auto
9
from multiprocessing.process import BaseProcess
@@ -37,7 +37,7 @@ class MultiprocExecutor(Executor):
37
def __init__(self, vllm_config: VllmConfig) -> None:
38
# Call self.shutdown at exit to clean up
39
# and ensure workers will be terminated.
40
- atexit.register(self.shutdown)
40
+ self._finalizer = weakref.finalize(self, self.shutdown)
41
42
self.vllm_config = vllm_config
43
self.parallel_config = vllm_config.parallel_config
@@ -195,14 +195,10 @@ class MultiprocExecutor(Executor):
195
os.remove(socket_path)
196
197
def shutdown(self):
198
- if atexit:
199
- # in case shutdown was called explicitly, we don't need to call it
200
- # again
201
- atexit.unregister(self.shutdown)
198
"""Properly shut down the executor and its workers"""
199
if getattr(self, 'shutting_down', False):
200
self.shutting_down = True
205
- for w in self.workers: #TODO: not sure if needed
201
+ for w in self.workers:
202
w.worker_response_mq = None
203
self._ensure_worker_termination()
204