A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 04:30 UTC
[V1][Core] Remove should_shutdown to simplify core process termination (#11113)
Signed-off-by: Tyler Michael Smith <tyler@neuralmagic.com>
Tyler Michael Smith committed
Dec 11, 2024 at 18:34 UTC
452a723bf2e8410ee9b47f82f90c7ea48aa6d14f
2 files changed
+2
-17
vllm/v1/engine/core.py
+2
-11
@@ -5,7 +5,6 @@ import signal
5
import threading
6
import time
7
from multiprocessing.process import BaseProcess
8
-from multiprocessing.sharedctypes import Synchronized
8
from typing import List, Tuple, Type, Union
9
10
import zmq
@@ -133,13 +132,9 @@ class EngineCoreProc(EngineCore):
132
input_path: str,
133
output_path: str,
134
ready_path: str,
136
- should_shutdown: Synchronized,
135
):
136
super().__init__(vllm_config, executor_class, usage_context)
137
140
- # Signal from main process to shutdown (multiprocessing.Value).
141
- self.should_shutdown = should_shutdown
142
-
138
# Background Threads and Queues for IO. These enable us to
139
# overlap ZMQ socket IO with GPU since they release the GIL,
140
# and to overlap some serialization/deserialization with the
@@ -195,7 +190,6 @@ class EngineCoreProc(EngineCore):
190
input_path: str,
191
output_path: str,
192
ready_path: str,
198
- should_shutdown: Synchronized,
193
) -> BaseProcess:
194
# The current process might have CUDA context,
195
# so we need to spawn a new process.
@@ -210,7 +204,6 @@ class EngineCoreProc(EngineCore):
204
"vllm_config": vllm_config,
205
"executor_class": executor_class,
206
"usage_context": usage_context,
213
- "should_shutdown": should_shutdown
207
}
208
# Run EngineCore busy loop in background process.
209
proc = context.Process(target=EngineCoreProc.run_engine_core,
@@ -260,8 +253,8 @@ class EngineCoreProc(EngineCore):
253
def run_busy_loop(self):
254
"""Core busy loop of the EngineCore."""
255
263
- # Loop until we get a shutdown signal.
264
- while not self.should_shutdown:
256
+ # Loop until process is sent a SIGINT or SIGTERM
257
+ while True:
258
# 1) Poll the input queue until there is work to do.
259
if not self.scheduler.has_unfinished_requests():
260
while True:
@@ -272,8 +265,6 @@ class EngineCoreProc(EngineCore):
265
except queue.Empty:
266
self._log_stats()
267
logger.debug("EngineCore busy loop waiting.")
275
- if self.should_shutdown:
276
- return
268
except BaseException:
269
raise
270
vllm/v1/engine/core_client.py
-6
@@ -1,5 +1,4 @@
1
import atexit
2
-import multiprocessing
2
from typing import List, Union
3
4
import msgspec
@@ -149,21 +148,16 @@ class MPClient(EngineCoreClient):
148
self.input_socket.bind(input_path)
149
150
# Start EngineCore in background process.
152
- self.should_shutdown = multiprocessing.Value('b', False, lock=False)
151
self.proc = EngineCoreProc.make_engine_core_process(
152
*args,
153
input_path=input_path,
154
output_path=output_path,
155
ready_path=ready_path,
158
- should_shutdown=self.should_shutdown,
156
**kwargs,
157
)
158
atexit.register(self.shutdown)
159
160
def shutdown(self):
164
- # Send shutdown signal to background process.
165
- self.should_shutdown = True
166
-
161
# Shut down the zmq context.
162
self.ctx.destroy(linger=0)
163