A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 04:30 UTC
[Core] v1: Use atexit to handle engine core client shutdown (#11076)
Signed-off-by: Russell Bryant <rbryant@redhat.com>
Russell Bryant committed
Dec 11, 2024 at 04:26 UTC
61b1d2f6aef8e29c6a0d795a9c6682d525f4d8cc
1 file changed
+2
vllm/v1/engine/core_client.py
+2
@@ -1,3 +1,4 @@
1
+import atexit
2
import multiprocessing
3
from typing import List, Union
4
@@ -157,6 +158,7 @@ class MPClient(EngineCoreClient):
158
should_shutdown=self.should_shutdown,
159
**kwargs,
160
)
161
+ atexit.register(self.shutdown)
162
163
def shutdown(self):
164
# Send shutdown signal to background process.