@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 3b3f1e7436
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 09:30 UTC

[Bugfix][core] replace heartbeat with pid check (#9818)

Signed-off-by: Joe Runde <Joseph.Runde@ibm.com>

Joe Runde committed Oct 30, 2024 at 10:34 UTC 3b3f1e743631667795469946a33d8352fcc74efd
5 files changed +62 -62
tests/mq_llm_engine/test_error_handling.py
+26 -1
@@ -21,7 +21,7 @@ from vllm.usage.usage_lib import UsageContext
21 from vllm.utils import FlexibleArgumentParser
22
23 MODEL = "google/gemma-1.1-2b-it"
24 -ENGINE_ARGS = AsyncEngineArgs(model=MODEL)
24 +ENGINE_ARGS = AsyncEngineArgs(model=MODEL, enforce_eager=True)
25 RAISED_ERROR = KeyError
26 RAISED_VALUE = "foo"
27
@@ -266,3 +266,28 @@ async def test_mp_cuda_init():
266
267 async with build_async_engine_client(args):
268 pass
269 +
270 +
271 +@pytest.mark.asyncio
272 +async def test_engine_process_death(tmp_socket):
273 + with RemoteMQLLMEngine(engine_args=ENGINE_ARGS,
274 + ipc_path=tmp_socket) as engine:
275 +
276 + client = await engine.make_client()
277 + assert client.is_running
278 +
279 + # kill the engine process
280 + engine.proc.kill()
281 +
282 + # Generate call should fail
283 + with pytest.raises(MQEngineDeadError):
284 + async for _ in client.generate(prompt="Hello my name is",
285 + sampling_params=SamplingParams(),
286 + request_id=uuid.uuid4()):
287 + pass
288 +
289 + # And the health check should show the engine is dead
290 + with pytest.raises(RuntimeError, match="Engine process .* died"):
291 + await client.check_health()
292 +
293 + client.close()
tests/mq_llm_engine/utils.py
+1 -1
@@ -68,7 +68,7 @@ class RemoteMQLLMEngine:
68
69 async def make_client(self) -> MQLLMEngineClient:
70 engine_config = self.engine_args.create_engine_config()
71 - client = MQLLMEngineClient(self.ipc_path, engine_config)
71 + client = MQLLMEngineClient(self.ipc_path, engine_config, self.proc.pid)
72 while True:
73 try:
74 await client.setup()
vllm/engine/multiprocessing/client.py
+19 -10
@@ -6,6 +6,7 @@ from typing import (Any, AsyncGenerator, Dict, Iterator, List, Mapping,
6 Optional, Union, cast, overload)
7
8 import cloudpickle
9 +import psutil
10 import zmq
11 import zmq.asyncio
12 from zmq import Frame # type: ignore[attr-defined]
@@ -77,7 +78,8 @@ class MQLLMEngineClient(EngineClient):
78 every N seconds, confirming the engine is healthy
79 """
80
80 - def __init__(self, ipc_path: str, engine_config: EngineConfig):
81 + def __init__(self, ipc_path: str, engine_config: EngineConfig,
82 + engine_pid: int):
83 self.context = zmq.asyncio.Context()
84 self._errored_with: Optional[BaseException] = None
85
@@ -115,6 +117,7 @@ class MQLLMEngineClient(EngineClient):
117 # Loop to check health of the LLMEngine periodically.
118 # Started after the MQLLMEngine is ready.
119 self.health_loop: Optional[asyncio.Task] = None
120 + self._engine_process = psutil.Process(engine_pid)
121
122 @staticmethod
123 def is_unsupported_config(engine_args: AsyncEngineArgs):
@@ -131,21 +134,22 @@ class MQLLMEngineClient(EngineClient):
134 socket.close(linger=0)
135
136 async def run_heartbeat_loop(self, timeout: int):
134 - """Background loop that continually listens to the RPCServer for
135 - heartbeats.
137 + """Background loop that continually checks to ensure the engine process
138 + is still alive.
139 """
140 try:
141 while True:
139 - if await self.heartbeat_socket.poll(timeout=timeout) == 0:
140 - # No heartbeat was received. Set error and exit the loop
142 + # Check if the engine process is running:
143 + if not self._engine_process.is_running() or (
144 + self._engine_process.status() == psutil.STATUS_ZOMBIE):
145 + # NB: is_running() returns True for zombies
146 self._set_errored(
142 - TimeoutError("No heartbeat received "
143 - "from MQLLMEngine"))
144 - logger.debug("Shutting down MQLLMEngineClient check "
145 - "health loop due to timeout")
147 + RuntimeError(
148 + f"Engine process (pid {self._engine_process.pid}) "
149 + "died."))
150 break
151
148 - else:
152 + if await self.heartbeat_socket.poll(timeout=timeout):
153 # Heartbeat received- check the message
154 await self._check_success(
155 error_message="Heartbeat failed.",
@@ -156,6 +160,11 @@ class MQLLMEngineClient(EngineClient):
160 except asyncio.CancelledError:
161 logger.debug("Shutting down MQLLMEngineClient check health loop.")
162
163 + except psutil.NoSuchProcess:
164 + self._set_errored(
165 + RuntimeError(
166 + f"Engine process (pid {self._engine_process.pid}) died."))
167 +
168 except Exception as e:
169 self._set_errored(e)
170
vllm/engine/multiprocessing/engine.py
+11 -48
@@ -1,7 +1,5 @@
1 import pickle
2 import signal
3 -import threading
4 -import time
3 from contextlib import contextmanager
4 from typing import Iterator, List, Optional, Union
5
@@ -21,7 +19,7 @@ from vllm.engine.multiprocessing import (ENGINE_DEAD_ERROR, IPC_DATA_EXT,
19 RPCStartupRequest, RPCStartupResponse,
20 RPCUProfileRequest)
21 # yapf: enable
24 -from vllm.envs import VLLM_RPC_TIMEOUT, VLLM_USE_V1
22 +from vllm.envs import VLLM_USE_V1
23 from vllm.executor.gpu_executor import GPUExecutor
24 from vllm.logger import init_logger
25 from vllm.outputs import RequestOutput
@@ -108,20 +106,6 @@ class MQLLMEngine:
106 # Error state.
107 self._errored_with: Optional[BaseException] = None
108
111 - # Heartbeat thread
112 - self.heartbeat_thread = threading.Thread(target=self._heartbeat_loop,
113 - daemon=True)
114 - self._heartbeat_stop_event = threading.Event()
115 - # The heartbeat needs to be faster than what the client will wait for
116 - # The VLLM_RPC_TIMEOUT duration is in ms, and we need one in seconds
117 - self.heartbeat_interval_seconds = VLLM_RPC_TIMEOUT / 5000.0
118 -
119 - self._last_alive_time = time.time()
120 - # The heartbeats can tolerate a long period of the engine chugging
121 - # away at a generation request.
122 - # The VLLM_RPC_TIMEOUT duration is in ms, and we need one in seconds
123 - self.last_alive_threshold = VLLM_RPC_TIMEOUT * 3.0 / 1000.0
124 -
109 @property
110 def dead_error(self) -> BaseException:
111 if self._errored_with is not None:
@@ -157,8 +141,6 @@ class MQLLMEngine:
141 try:
142 logger.debug("Starting Startup Loop.")
143 self.run_startup_loop()
160 - logger.debug("Starting heartbeat thread")
161 - self.heartbeat_thread.start()
144 logger.debug("Starting Engine Loop.")
145 self.run_engine_loop()
146 except Exception as e:
@@ -172,7 +154,6 @@ class MQLLMEngine:
154 def cleanup(self):
155 """Cleanup zeromq state on shutdown."""
156 # Closes all sockets and destroys context.
175 - self._heartbeat_stop_event.set()
157 self.ctx.destroy(linger=0)
158 del self.engine
159
@@ -211,11 +192,12 @@ class MQLLMEngine:
192 """Core busy loop of the LLMEngine."""
193
194 while True:
214 - self._alive()
195 if not self.engine.has_unfinished_requests():
196 # Poll until there is work to do.
197 while self.input_socket.poll(timeout=POLLING_TIMEOUT_MS) == 0:
218 - self._alive()
198 + # When there's no work, check on engine health and send
199 + # health status back to client
200 + self._health_check()
201 self.engine.do_log_stats()
202 logger.debug("Waiting for new requests in engine loop.")
203
@@ -314,32 +296,16 @@ class MQLLMEngine:
296 if self.log_requests:
297 logger.info("Aborted request %s.", request.request_id)
298
317 - def _heartbeat_loop(self):
318 - while not self._heartbeat_stop_event.wait(
319 - timeout=self.heartbeat_interval_seconds):
320 - # Loops until the stop event is set
321 - self._heartbeat()
322 -
323 - logger.debug("Exiting MQLLMEngine heartbeat thread")
324 -
325 - def _heartbeat(self):
299 + def _health_check(self):
300 # Send unhealthy if engine has already errored
301 if self._errored_with is not None:
302 self._send_unhealthy(self._errored_with)
329 -
330 - # Check for life of the main loop
331 - elif time.time() - self._last_alive_time > self.last_alive_threshold:
332 - self._send_unhealthy(RuntimeError("Engine loop has died"))
333 -
334 - else:
335 - # Otherwise- check health of the engine
336 - # self.engine.check_health() raises on unhealthy
337 - try:
338 - self.engine.check_health()
339 - self._send_healthy()
340 - except Exception as e:
341 - self._set_errored(e)
342 - self._send_unhealthy(e)
303 + try:
304 + self.engine.check_health()
305 + self._send_healthy()
306 + except Exception as e:
307 + self._set_errored(e)
308 + self._send_unhealthy(e)
309
310 def _send_outputs(self, outputs: REQUEST_OUTPUTS_T):
311 """Send List of RequestOutput to RPCClient."""
@@ -369,9 +335,6 @@ class MQLLMEngine:
335 if self._errored_with is None:
336 self._errored_with = e
337
372 - def _alive(self):
373 - self._last_alive_time = time.time()
374 -
338 def start_profile(self) -> None:
339 if type(self.engine.model_executor) is GPUExecutor:
340 self.engine.model_executor.start_profile()
vllm/entrypoints/openai/api_server.py
+5 -2
@@ -176,13 +176,16 @@ async def build_async_engine_client_from_engine_args(
176 UsageContext.OPENAI_API_SERVER,
177 ipc_path))
178 engine_process.start()
179 - logger.info("Started engine process with PID %d", engine_process.pid)
179 + engine_pid = engine_process.pid
180 + assert engine_pid is not None, "Engine process failed to start"
181 + logger.info("Started engine process with PID %d", engine_pid)
182
183 # Build RPCClient, which conforms to EngineClient Protocol.
184 # NOTE: Actually, this is not true yet. We still need to support
185 # embedding models via RPC (see TODO above)
186 engine_config = engine_args.create_engine_config()
185 - mp_engine_client = MQLLMEngineClient(ipc_path, engine_config)
187 + mp_engine_client = MQLLMEngineClient(ipc_path, engine_config,
188 + engine_pid)
189
190 try:
191 while True: