A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 15:45 UTC
[Bugfix] Use heartbeats instead of health checks (#8583)
Joe Runde committed
Sep 24, 2024 at 21:37 UTC
6e0c9d6bd07464b311eb098e2dac8196eed16721
4 files changed
+87
-63
tests/mq_llm_engine/test_error_handling.py
+4
-11
@@ -153,27 +153,20 @@ async def test_failed_abort(tmp_socket):
153
await client.check_health()
154
155
# Trigger an abort on the client side.
156
- async def bad_abort_after_2s():
157
- await asyncio.sleep(2.0)
158
- await client.abort(request_id="foo")
156
+ # This request ID does not exist, and will cause the engine to error
157
+ await client.abort(request_id="foo")
158
160
- # Trigger an abort in 2s from now.
161
- abort_task = asyncio.create_task(bad_abort_after_2s())
162
-
163
- # Exception in abort() will happen during this generation.
164
- # This will kill the engine and should return ENGINE_DEAD_ERROR
159
+ # Future generation requests will now fail
160
# with reference to the original KeyError("foo")
161
with pytest.raises(MQEngineDeadError) as execinfo:
162
async for _ in client.generate(
163
inputs="Hello my name is",
169
- sampling_params=SamplingParams(max_tokens=2000),
164
+ sampling_params=SamplingParams(max_tokens=10),
165
request_id=uuid.uuid4()):
166
pass
167
assert "KeyError" in repr(execinfo.value)
168
assert client.errored
169
175
- await abort_task
176
-
170
# This should raise the original error.
171
with pytest.raises(RAISED_ERROR):
172
await client.check_health()
vllm/engine/multiprocessing/__init__.py
+1
-6
@@ -43,10 +43,6 @@ class RPCAbortRequest:
43
request_id: str
44
45
46
-class RPCHealthRequest:
47
- pass
48
-
49
-
46
class RPCStartupRequest(Enum):
47
IS_SERVER_READY = 1
48
@@ -56,8 +52,7 @@ class RPCStartupResponse:
52
tracing_enabled: bool
53
54
59
-RPC_REQUEST_T = Union[RPCProcessRequest, RPCAbortRequest, RPCHealthRequest,
60
- RPCStartupRequest]
55
+RPC_REQUEST_T = Union[RPCProcessRequest, RPCAbortRequest, RPCStartupRequest]
56
57
REQUEST_OUTPUTS_T = Union[List[RequestOutput], RPCError]
58
vllm/engine/multiprocessing/client.py
+22
-29
@@ -20,9 +20,8 @@ from vllm.engine.multiprocessing import (ENGINE_DEAD_ERROR, IPC_DATA_EXT,
20
IPC_HEALTH_EXT, IPC_INPUT_EXT,
21
IPC_OUTPUT_EXT, RPC_REQUEST_T,
22
VLLM_RPC_SUCCESS_STR, RPCAbortRequest,
23
- RPCError, RPCHealthRequest,
24
- RPCProcessRequest, RPCStartupRequest,
25
- RPCStartupResponse)
23
+ RPCError, RPCProcessRequest,
24
+ RPCStartupRequest, RPCStartupResponse)
25
# yapf: enable
26
from vllm.envs import VLLM_RPC_TIMEOUT
27
from vllm.inputs import PromptInputs
@@ -95,9 +94,9 @@ class MQLLMEngineClient:
94
self.output_socket: Socket = self.context.socket(zmq.constants.PULL)
95
self.output_socket.connect(f"{ipc_path}{IPC_OUTPUT_EXT}")
96
98
- # IPC path for ack of check_health requests.
99
- self.health_socket: Socket = self.context.socket(zmq.constants.PULL)
100
- self.health_socket.connect(f"{ipc_path}{IPC_HEALTH_EXT}")
97
+ # IPC path for acking heartbeats.
98
+ self.heartbeat_socket: Socket = self.context.socket(zmq.constants.PULL)
99
+ self.heartbeat_socket.connect(f"{ipc_path}{IPC_HEALTH_EXT}")
100
101
# IPC path for the data socket.
102
self.data_ipc_path = f"{ipc_path}{IPC_DATA_EXT}"
@@ -124,34 +123,28 @@ class MQLLMEngineClient:
123
finally:
124
socket.close(linger=0)
125
127
- async def run_check_health_loop(self, timeout: int):
128
- """Background loop that continually probes the RPCServer for health.
129
-
130
- The loop sends CHECK_HEALTH requests to the INPUT_SOCKET, which
131
- the MQLLMEngine server is blocking on.
132
-
133
- The Server replies on the HEALTH_SOCKET (rather than on the
134
- OUTPUT_SOCKET such that the messages are not intermingled with
135
- output streaming).
126
+ async def run_heartbeat_loop(self, timeout: int):
127
+ """Background loop that continually listens to the RPCServer for
128
+ heartbeats.
129
"""
137
-
130
try:
131
while True:
140
- if await self.health_socket.poll(timeout=timeout) == 0:
141
- # Wakeup every N seconds and do a health probe.
142
- await self._send_one_way_rpc_request(
143
- RPCHealthRequest(), self.input_socket)
144
-
145
- # Wait for ack from the health socket.
146
- await self._await_ack(error_message="Health check failed.",
147
- socket=self.health_socket)
132
+ if await self.heartbeat_socket.poll(timeout=timeout) == 0:
133
+ # No heartbeat was received. Set error and exit the loop
134
+ self._set_errored(
135
+ TimeoutError("No heartbeat received "
136
+ "from MQLLMEngine"))
137
+ logger.debug("Shutting down MQLLMEngineClient check "
138
+ "health loop due to timeout")
139
+ break
140
+
141
else:
149
- # Server sent a health status message unprompted.
142
+ # Heartbeat received- check the message
143
await self._check_success(
151
- error_message="Health check failed.",
152
- socket=self.health_socket)
144
+ error_message="Heartbeat failed.",
145
+ socket=self.heartbeat_socket)
146
154
- logger.debug("Health probe successful.")
147
+ logger.debug("Heartbeat successful.")
148
149
except asyncio.CancelledError:
150
logger.debug("Shutting down MQLLMEngineClient check health loop.")
@@ -234,7 +227,7 @@ class MQLLMEngineClient:
227
228
# Start health_loop.
229
self.health_loop = asyncio.create_task(
237
- self.run_check_health_loop(timeout=VLLM_RPC_TIMEOUT))
230
+ self.run_heartbeat_loop(timeout=VLLM_RPC_TIMEOUT))
231
232
def close(self):
233
"""Destroy the ZeroMQ Context."""
vllm/engine/multiprocessing/engine.py
+60
-17
@@ -1,5 +1,7 @@
1
import pickle
2
import signal
3
+import threading
4
+import time
5
from contextlib import contextmanager
6
from typing import Iterator, List, Optional, Union
7
@@ -15,10 +17,10 @@ from vllm.engine.multiprocessing import (ENGINE_DEAD_ERROR, IPC_DATA_EXT,
17
IPC_HEALTH_EXT, IPC_INPUT_EXT,
18
IPC_OUTPUT_EXT, REQUEST_OUTPUTS_T,
19
VLLM_RPC_SUCCESS_STR, RPCAbortRequest,
18
- RPCError, RPCHealthRequest,
19
- RPCProcessRequest, RPCStartupRequest,
20
- RPCStartupResponse)
20
+ RPCError, RPCProcessRequest,
21
+ RPCStartupRequest, RPCStartupResponse)
22
# yapf: enable
23
+from vllm.envs import VLLM_RPC_TIMEOUT
24
from vllm.logger import init_logger
25
from vllm.outputs import RequestOutput
26
from vllm.usage.usage_lib import UsageContext
@@ -91,9 +93,9 @@ class MQLLMEngine:
93
self.output_socket = self.ctx.socket(zmq.constants.PUSH)
94
self.output_socket.bind(f"{ipc_path}{IPC_OUTPUT_EXT}")
95
94
- # Send health status back to client.
95
- self.health_socket = self.ctx.socket(zmq.constants.PUSH)
96
- self.health_socket.bind(f"{ipc_path}{IPC_HEALTH_EXT}")
96
+ # Send heartbeats back to client.
97
+ self.heartbeat_socket = self.ctx.socket(zmq.constants.PUSH)
98
+ self.heartbeat_socket.bind(f"{ipc_path}{IPC_HEALTH_EXT}")
99
100
# IPC path for the data socket.
101
self.data_ipc_path = f"{ipc_path}{IPC_DATA_EXT}"
@@ -101,6 +103,20 @@ class MQLLMEngine:
103
# Error state.
104
self._errored_with: Optional[BaseException] = None
105
106
+ # Heartbeat thread
107
+ self.heartbeat_thread = threading.Thread(target=self._heartbeat_loop,
108
+ daemon=True)
109
+ self._heartbeat_stop_event = threading.Event()
110
+ # The heartbeat needs to be faster than what the client will wait for
111
+ # The VLLM_RPC_TIMEOUT duration is in ms, and we need one in seconds
112
+ self.heartbeat_interval_seconds = VLLM_RPC_TIMEOUT / 5000.0
113
+
114
+ self._last_alive_time = time.time()
115
+ # The heartbeats can tolerate a long period of the engine chugging
116
+ # away at a generation request.
117
+ # The VLLM_RPC_TIMEOUT duration is in ms, and we need one in seconds
118
+ self.last_alive_threshold = VLLM_RPC_TIMEOUT * 3.0 / 1000.0
119
+
120
@property
121
def dead_error(self) -> BaseException:
122
if self._errored_with is not None:
@@ -131,6 +147,8 @@ class MQLLMEngine:
147
try:
148
logger.debug("Starting Startup Loop.")
149
self.run_startup_loop()
150
+ logger.debug("Starting heartbeat thread")
151
+ self.heartbeat_thread.start()
152
logger.debug("Starting Engine Loop.")
153
self.run_engine_loop()
154
except Exception as e:
@@ -144,6 +162,7 @@ class MQLLMEngine:
162
def cleanup(self):
163
"""Cleanup zeromq state on shutdown."""
164
# Closes all sockets and destroys context.
165
+ self._heartbeat_stop_event.set()
166
self.ctx.destroy(linger=0)
167
del self.engine
168
@@ -182,9 +201,11 @@ class MQLLMEngine:
201
"""Core busy loop of the LLMEngine."""
202
203
while True:
204
+ self._alive()
205
if not self.engine.has_unfinished_requests():
206
# Poll until there is work to do.
207
while self.input_socket.poll(timeout=POLLING_TIMEOUT_MS) == 0:
208
+ self._alive()
209
self.engine.do_log_stats()
210
logger.debug("Waiting for new requests in engine loop.")
211
@@ -200,7 +221,6 @@ class MQLLMEngine:
221
222
def engine_step(self) -> List[RequestOutput]:
223
"""Engine step wrapper with error handling."""
203
-
224
try:
225
return self.engine.step()
226
except SystemExit:
@@ -229,10 +249,9 @@ class MQLLMEngine:
249
self._handle_process_request(request)
250
elif isinstance(request, RPCAbortRequest):
251
self._handle_abort_request(request)
232
- elif isinstance(request, RPCHealthRequest):
233
- self._handle_health_request()
252
else:
235
- raise ValueError("Unknown RPCRequest Type: {request}")
253
+ raise ValueError("Unknown RPCRequest Type: "
254
+ f"{type(request)}")
255
256
except Exception as e:
257
self._set_errored(e)
@@ -279,13 +298,32 @@ class MQLLMEngine:
298
if self.log_requests:
299
logger.info("Aborted request %s.", request.request_id)
300
282
- def _handle_health_request(self):
301
+ def _heartbeat_loop(self):
302
+ while not self._heartbeat_stop_event.wait(
303
+ timeout=self.heartbeat_interval_seconds):
304
+ # Loops until the stop event is set
305
+ self._heartbeat()
306
+
307
+ logger.debug("Exiting MQLLMEngine heartbeat thread")
308
+
309
+ def _heartbeat(self):
310
+ # Send unhealthy if engine has already errored
311
if self._errored_with is not None:
312
self._send_unhealthy(self._errored_with)
313
286
- # Raises error if unhealthy.
287
- self.engine.check_health()
288
- self._send_healthy()
314
+ # Check for life of the main loop
315
+ elif time.time() - self._last_alive_time > self.last_alive_threshold:
316
+ self._send_unhealthy(RuntimeError("Engine loop has died"))
317
+
318
+ else:
319
+ # Otherwise- check health of the engine
320
+ # self.engine.check_health() raises on unhealthy
321
+ try:
322
+ self.engine.check_health()
323
+ self._send_healthy()
324
+ except Exception as e:
325
+ self._set_errored(e)
326
+ self._send_unhealthy(e)
327
328
def _send_outputs(self, outputs: REQUEST_OUTPUTS_T):
329
"""Send List of RequestOutput to RPCClient."""
@@ -295,12 +333,14 @@ class MQLLMEngine:
333
334
def _send_healthy(self):
335
"""Send HEALTHY message to RPCClient."""
298
- self.health_socket.send_multipart(HEALTHY_RESPONSE, copy=False)
336
+ if not self.heartbeat_socket.closed:
337
+ self.heartbeat_socket.send_multipart(HEALTHY_RESPONSE, copy=False)
338
339
def _send_unhealthy(self, error: BaseException):
340
"""Send UNHEALTHY message to RPCClient."""
302
- error_bytes = pickle.dumps(error)
303
- self.health_socket.send_multipart((error_bytes, ), copy=False)
341
+ if not self.heartbeat_socket.closed:
342
+ error_bytes = pickle.dumps(error)
343
+ self.heartbeat_socket.send_multipart((error_bytes, ), copy=False)
344
345
def _async_socket_engine_callback(self,
346
request_outputs: REQUEST_OUTPUTS_T):
@@ -313,6 +353,9 @@ class MQLLMEngine:
353
if self._errored_with is None:
354
self._errored_with = e
355
356
+ def _alive(self):
357
+ self._last_alive_time = time.time()
358
+
359
360
def run_mp_engine(engine_args: AsyncEngineArgs, usage_context: UsageContext,
361
ipc_path: str):