@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 452a723bf2
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 04:30 UTC

[V1][Core] Remove should_shutdown to simplify core process termination (#11113)

Signed-off-by: Tyler Michael Smith <tyler@neuralmagic.com>

Tyler Michael Smith committed Dec 11, 2024 at 18:34 UTC 452a723bf2e8410ee9b47f82f90c7ea48aa6d14f
2 files changed +2 -17
vllm/v1/engine/core.py
+2 -11
@@ -5,7 +5,6 @@ import signal
5 import threading
6 import time
7 from multiprocessing.process import BaseProcess
8 -from multiprocessing.sharedctypes import Synchronized
8 from typing import List, Tuple, Type, Union
9
10 import zmq
@@ -133,13 +132,9 @@ class EngineCoreProc(EngineCore):
132 input_path: str,
133 output_path: str,
134 ready_path: str,
136 - should_shutdown: Synchronized,
135 ):
136 super().__init__(vllm_config, executor_class, usage_context)
137
140 - # Signal from main process to shutdown (multiprocessing.Value).
141 - self.should_shutdown = should_shutdown
142 -
138 # Background Threads and Queues for IO. These enable us to
139 # overlap ZMQ socket IO with GPU since they release the GIL,
140 # and to overlap some serialization/deserialization with the
@@ -195,7 +190,6 @@ class EngineCoreProc(EngineCore):
190 input_path: str,
191 output_path: str,
192 ready_path: str,
198 - should_shutdown: Synchronized,
193 ) -> BaseProcess:
194 # The current process might have CUDA context,
195 # so we need to spawn a new process.
@@ -210,7 +204,6 @@ class EngineCoreProc(EngineCore):
204 "vllm_config": vllm_config,
205 "executor_class": executor_class,
206 "usage_context": usage_context,
213 - "should_shutdown": should_shutdown
207 }
208 # Run EngineCore busy loop in background process.
209 proc = context.Process(target=EngineCoreProc.run_engine_core,
@@ -260,8 +253,8 @@ class EngineCoreProc(EngineCore):
253 def run_busy_loop(self):
254 """Core busy loop of the EngineCore."""
255
263 - # Loop until we get a shutdown signal.
264 - while not self.should_shutdown:
256 + # Loop until process is sent a SIGINT or SIGTERM
257 + while True:
258 # 1) Poll the input queue until there is work to do.
259 if not self.scheduler.has_unfinished_requests():
260 while True:
@@ -272,8 +265,6 @@ class EngineCoreProc(EngineCore):
265 except queue.Empty:
266 self._log_stats()
267 logger.debug("EngineCore busy loop waiting.")
275 - if self.should_shutdown:
276 - return
268 except BaseException:
269 raise
270
vllm/v1/engine/core_client.py
-6
@@ -1,5 +1,4 @@
1 import atexit
2 -import multiprocessing
2 from typing import List, Union
3
4 import msgspec
@@ -149,21 +148,16 @@ class MPClient(EngineCoreClient):
148 self.input_socket.bind(input_path)
149
150 # Start EngineCore in background process.
152 - self.should_shutdown = multiprocessing.Value('b', False, lock=False)
151 self.proc = EngineCoreProc.make_engine_core_process(
152 *args,
153 input_path=input_path,
154 output_path=output_path,
155 ready_path=ready_path,
158 - should_shutdown=self.should_shutdown,
156 **kwargs,
157 )
158 atexit.register(self.shutdown)
159
160 def shutdown(self):
164 - # Send shutdown signal to background process.
165 - self.should_shutdown = True
166 -
161 # Shut down the zmq context.
162 self.ctx.destroy(linger=0)
163