A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 09:30 UTC
[Bugfix][core] replace heartbeat with pid check (#9818)
Signed-off-by: Joe Runde <Joseph.Runde@ibm.com>
Joe Runde committed
Oct 30, 2024 at 10:34 UTC
3b3f1e743631667795469946a33d8352fcc74efd
5 files changed
+62
-62
tests/mq_llm_engine/test_error_handling.py
+26
-1
@@ -21,7 +21,7 @@ from vllm.usage.usage_lib import UsageContext
21
from vllm.utils import FlexibleArgumentParser
22
23
MODEL = "google/gemma-1.1-2b-it"
24
-ENGINE_ARGS = AsyncEngineArgs(model=MODEL)
24
+ENGINE_ARGS = AsyncEngineArgs(model=MODEL, enforce_eager=True)
25
RAISED_ERROR = KeyError
26
RAISED_VALUE = "foo"
27
@@ -266,3 +266,28 @@ async def test_mp_cuda_init():
266
267
async with build_async_engine_client(args):
268
pass
269
+
270
+
271
+@pytest.mark.asyncio
272
+async def test_engine_process_death(tmp_socket):
273
+ with RemoteMQLLMEngine(engine_args=ENGINE_ARGS,
274
+ ipc_path=tmp_socket) as engine:
275
+
276
+ client = await engine.make_client()
277
+ assert client.is_running
278
+
279
+ # kill the engine process
280
+ engine.proc.kill()
281
+
282
+ # Generate call should fail
283
+ with pytest.raises(MQEngineDeadError):
284
+ async for _ in client.generate(prompt="Hello my name is",
285
+ sampling_params=SamplingParams(),
286
+ request_id=uuid.uuid4()):
287
+ pass
288
+
289
+ # And the health check should show the engine is dead
290
+ with pytest.raises(RuntimeError, match="Engine process .* died"):
291
+ await client.check_health()
292
+
293
+ client.close()
tests/mq_llm_engine/utils.py
+1
-1
@@ -68,7 +68,7 @@ class RemoteMQLLMEngine:
68
69
async def make_client(self) -> MQLLMEngineClient:
70
engine_config = self.engine_args.create_engine_config()
71
- client = MQLLMEngineClient(self.ipc_path, engine_config)
71
+ client = MQLLMEngineClient(self.ipc_path, engine_config, self.proc.pid)
72
while True:
73
try:
74
await client.setup()
vllm/engine/multiprocessing/client.py
+19
-10
@@ -6,6 +6,7 @@ from typing import (Any, AsyncGenerator, Dict, Iterator, List, Mapping,
6
Optional, Union, cast, overload)
7
8
import cloudpickle
9
+import psutil
10
import zmq
11
import zmq.asyncio
12
from zmq import Frame # type: ignore[attr-defined]
@@ -77,7 +78,8 @@ class MQLLMEngineClient(EngineClient):
78
every N seconds, confirming the engine is healthy
79
"""
80
80
- def __init__(self, ipc_path: str, engine_config: EngineConfig):
81
+ def __init__(self, ipc_path: str, engine_config: EngineConfig,
82
+ engine_pid: int):
83
self.context = zmq.asyncio.Context()
84
self._errored_with: Optional[BaseException] = None
85
@@ -115,6 +117,7 @@ class MQLLMEngineClient(EngineClient):
117
# Loop to check health of the LLMEngine periodically.
118
# Started after the MQLLMEngine is ready.
119
self.health_loop: Optional[asyncio.Task] = None
120
+ self._engine_process = psutil.Process(engine_pid)
121
122
@staticmethod
123
def is_unsupported_config(engine_args: AsyncEngineArgs):
@@ -131,21 +134,22 @@ class MQLLMEngineClient(EngineClient):
134
socket.close(linger=0)
135
136
async def run_heartbeat_loop(self, timeout: int):
134
- """Background loop that continually listens to the RPCServer for
135
- heartbeats.
137
+ """Background loop that continually checks to ensure the engine process
138
+ is still alive.
139
"""
140
try:
141
while True:
139
- if await self.heartbeat_socket.poll(timeout=timeout) == 0:
140
- # No heartbeat was received. Set error and exit the loop
142
+ # Check if the engine process is running:
143
+ if not self._engine_process.is_running() or (
144
+ self._engine_process.status() == psutil.STATUS_ZOMBIE):
145
+ # NB: is_running() returns True for zombies
146
self._set_errored(
142
- TimeoutError("No heartbeat received "
143
- "from MQLLMEngine"))
144
- logger.debug("Shutting down MQLLMEngineClient check "
145
- "health loop due to timeout")
147
+ RuntimeError(
148
+ f"Engine process (pid {self._engine_process.pid}) "
149
+ "died."))
150
break
151
148
- else:
152
+ if await self.heartbeat_socket.poll(timeout=timeout):
153
# Heartbeat received- check the message
154
await self._check_success(
155
error_message="Heartbeat failed.",
@@ -156,6 +160,11 @@ class MQLLMEngineClient(EngineClient):
160
except asyncio.CancelledError:
161
logger.debug("Shutting down MQLLMEngineClient check health loop.")
162
163
+ except psutil.NoSuchProcess:
164
+ self._set_errored(
165
+ RuntimeError(
166
+ f"Engine process (pid {self._engine_process.pid}) died."))
167
+
168
except Exception as e:
169
self._set_errored(e)
170
vllm/engine/multiprocessing/engine.py
+11
-48
@@ -1,7 +1,5 @@
1
import pickle
2
import signal
3
-import threading
4
-import time
3
from contextlib import contextmanager
4
from typing import Iterator, List, Optional, Union
5
@@ -21,7 +19,7 @@ from vllm.engine.multiprocessing import (ENGINE_DEAD_ERROR, IPC_DATA_EXT,
19
RPCStartupRequest, RPCStartupResponse,
20
RPCUProfileRequest)
21
# yapf: enable
24
-from vllm.envs import VLLM_RPC_TIMEOUT, VLLM_USE_V1
22
+from vllm.envs import VLLM_USE_V1
23
from vllm.executor.gpu_executor import GPUExecutor
24
from vllm.logger import init_logger
25
from vllm.outputs import RequestOutput
@@ -108,20 +106,6 @@ class MQLLMEngine:
106
# Error state.
107
self._errored_with: Optional[BaseException] = None
108
111
- # Heartbeat thread
112
- self.heartbeat_thread = threading.Thread(target=self._heartbeat_loop,
113
- daemon=True)
114
- self._heartbeat_stop_event = threading.Event()
115
- # The heartbeat needs to be faster than what the client will wait for
116
- # The VLLM_RPC_TIMEOUT duration is in ms, and we need one in seconds
117
- self.heartbeat_interval_seconds = VLLM_RPC_TIMEOUT / 5000.0
118
-
119
- self._last_alive_time = time.time()
120
- # The heartbeats can tolerate a long period of the engine chugging
121
- # away at a generation request.
122
- # The VLLM_RPC_TIMEOUT duration is in ms, and we need one in seconds
123
- self.last_alive_threshold = VLLM_RPC_TIMEOUT * 3.0 / 1000.0
124
-
109
@property
110
def dead_error(self) -> BaseException:
111
if self._errored_with is not None:
@@ -157,8 +141,6 @@ class MQLLMEngine:
141
try:
142
logger.debug("Starting Startup Loop.")
143
self.run_startup_loop()
160
- logger.debug("Starting heartbeat thread")
161
- self.heartbeat_thread.start()
144
logger.debug("Starting Engine Loop.")
145
self.run_engine_loop()
146
except Exception as e:
@@ -172,7 +154,6 @@ class MQLLMEngine:
154
def cleanup(self):
155
"""Cleanup zeromq state on shutdown."""
156
# Closes all sockets and destroys context.
175
- self._heartbeat_stop_event.set()
157
self.ctx.destroy(linger=0)
158
del self.engine
159
@@ -211,11 +192,12 @@ class MQLLMEngine:
192
"""Core busy loop of the LLMEngine."""
193
194
while True:
214
- self._alive()
195
if not self.engine.has_unfinished_requests():
196
# Poll until there is work to do.
197
while self.input_socket.poll(timeout=POLLING_TIMEOUT_MS) == 0:
218
- self._alive()
198
+ # When there's no work, check on engine health and send
199
+ # health status back to client
200
+ self._health_check()
201
self.engine.do_log_stats()
202
logger.debug("Waiting for new requests in engine loop.")
203
@@ -314,32 +296,16 @@ class MQLLMEngine:
296
if self.log_requests:
297
logger.info("Aborted request %s.", request.request_id)
298
317
- def _heartbeat_loop(self):
318
- while not self._heartbeat_stop_event.wait(
319
- timeout=self.heartbeat_interval_seconds):
320
- # Loops until the stop event is set
321
- self._heartbeat()
322
-
323
- logger.debug("Exiting MQLLMEngine heartbeat thread")
324
-
325
- def _heartbeat(self):
299
+ def _health_check(self):
300
# Send unhealthy if engine has already errored
301
if self._errored_with is not None:
302
self._send_unhealthy(self._errored_with)
329
-
330
- # Check for life of the main loop
331
- elif time.time() - self._last_alive_time > self.last_alive_threshold:
332
- self._send_unhealthy(RuntimeError("Engine loop has died"))
333
-
334
- else:
335
- # Otherwise- check health of the engine
336
- # self.engine.check_health() raises on unhealthy
337
- try:
338
- self.engine.check_health()
339
- self._send_healthy()
340
- except Exception as e:
341
- self._set_errored(e)
342
- self._send_unhealthy(e)
303
+ try:
304
+ self.engine.check_health()
305
+ self._send_healthy()
306
+ except Exception as e:
307
+ self._set_errored(e)
308
+ self._send_unhealthy(e)
309
310
def _send_outputs(self, outputs: REQUEST_OUTPUTS_T):
311
"""Send List of RequestOutput to RPCClient."""
@@ -369,9 +335,6 @@ class MQLLMEngine:
335
if self._errored_with is None:
336
self._errored_with = e
337
372
- def _alive(self):
373
- self._last_alive_time = time.time()
374
-
338
def start_profile(self) -> None:
339
if type(self.engine.model_executor) is GPUExecutor:
340
self.engine.model_executor.start_profile()
vllm/entrypoints/openai/api_server.py
+5
-2
@@ -176,13 +176,16 @@ async def build_async_engine_client_from_engine_args(
176
UsageContext.OPENAI_API_SERVER,
177
ipc_path))
178
engine_process.start()
179
- logger.info("Started engine process with PID %d", engine_process.pid)
179
+ engine_pid = engine_process.pid
180
+ assert engine_pid is not None, "Engine process failed to start"
181
+ logger.info("Started engine process with PID %d", engine_pid)
182
183
# Build RPCClient, which conforms to EngineClient Protocol.
184
# NOTE: Actually, this is not true yet. We still need to support
185
# embedding models via RPC (see TODO above)
186
engine_config = engine_args.create_engine_config()
185
- mp_engine_client = MQLLMEngineClient(ipc_path, engine_config)
187
+ mp_engine_client = MQLLMEngineClient(ipc_path, engine_config,
188
+ engine_pid)
189
190
try:
191
while True: