@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 6e0c9d6bd0
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 15:45 UTC

[Bugfix] Use heartbeats instead of health checks (#8583)

Joe Runde committed Sep 24, 2024 at 21:37 UTC 6e0c9d6bd07464b311eb098e2dac8196eed16721
4 files changed +87 -63
tests/mq_llm_engine/test_error_handling.py
+4 -11
@@ -153,27 +153,20 @@ async def test_failed_abort(tmp_socket):
153 await client.check_health()
154
155 # Trigger an abort on the client side.
156 - async def bad_abort_after_2s():
157 - await asyncio.sleep(2.0)
158 - await client.abort(request_id="foo")
156 + # This request ID does not exist, and will cause the engine to error
157 + await client.abort(request_id="foo")
158
160 - # Trigger an abort in 2s from now.
161 - abort_task = asyncio.create_task(bad_abort_after_2s())
162 -
163 - # Exception in abort() will happen during this generation.
164 - # This will kill the engine and should return ENGINE_DEAD_ERROR
159 + # Future generation requests will now fail
160 # with reference to the original KeyError("foo")
161 with pytest.raises(MQEngineDeadError) as execinfo:
162 async for _ in client.generate(
163 inputs="Hello my name is",
169 - sampling_params=SamplingParams(max_tokens=2000),
164 + sampling_params=SamplingParams(max_tokens=10),
165 request_id=uuid.uuid4()):
166 pass
167 assert "KeyError" in repr(execinfo.value)
168 assert client.errored
169
175 - await abort_task
176 -
170 # This should raise the original error.
171 with pytest.raises(RAISED_ERROR):
172 await client.check_health()
vllm/engine/multiprocessing/__init__.py
+1 -6
@@ -43,10 +43,6 @@ class RPCAbortRequest:
43 request_id: str
44
45
46 -class RPCHealthRequest:
47 - pass
48 -
49 -
46 class RPCStartupRequest(Enum):
47 IS_SERVER_READY = 1
48
@@ -56,8 +52,7 @@ class RPCStartupResponse:
52 tracing_enabled: bool
53
54
59 -RPC_REQUEST_T = Union[RPCProcessRequest, RPCAbortRequest, RPCHealthRequest,
60 - RPCStartupRequest]
55 +RPC_REQUEST_T = Union[RPCProcessRequest, RPCAbortRequest, RPCStartupRequest]
56
57 REQUEST_OUTPUTS_T = Union[List[RequestOutput], RPCError]
58
vllm/engine/multiprocessing/client.py
+22 -29
@@ -20,9 +20,8 @@ from vllm.engine.multiprocessing import (ENGINE_DEAD_ERROR, IPC_DATA_EXT,
20 IPC_HEALTH_EXT, IPC_INPUT_EXT,
21 IPC_OUTPUT_EXT, RPC_REQUEST_T,
22 VLLM_RPC_SUCCESS_STR, RPCAbortRequest,
23 - RPCError, RPCHealthRequest,
24 - RPCProcessRequest, RPCStartupRequest,
25 - RPCStartupResponse)
23 + RPCError, RPCProcessRequest,
24 + RPCStartupRequest, RPCStartupResponse)
25 # yapf: enable
26 from vllm.envs import VLLM_RPC_TIMEOUT
27 from vllm.inputs import PromptInputs
@@ -95,9 +94,9 @@ class MQLLMEngineClient:
94 self.output_socket: Socket = self.context.socket(zmq.constants.PULL)
95 self.output_socket.connect(f"{ipc_path}{IPC_OUTPUT_EXT}")
96
98 - # IPC path for ack of check_health requests.
99 - self.health_socket: Socket = self.context.socket(zmq.constants.PULL)
100 - self.health_socket.connect(f"{ipc_path}{IPC_HEALTH_EXT}")
97 + # IPC path for acking heartbeats.
98 + self.heartbeat_socket: Socket = self.context.socket(zmq.constants.PULL)
99 + self.heartbeat_socket.connect(f"{ipc_path}{IPC_HEALTH_EXT}")
100
101 # IPC path for the data socket.
102 self.data_ipc_path = f"{ipc_path}{IPC_DATA_EXT}"
@@ -124,34 +123,28 @@ class MQLLMEngineClient:
123 finally:
124 socket.close(linger=0)
125
127 - async def run_check_health_loop(self, timeout: int):
128 - """Background loop that continually probes the RPCServer for health.
129 -
130 - The loop sends CHECK_HEALTH requests to the INPUT_SOCKET, which
131 - the MQLLMEngine server is blocking on.
132 -
133 - The Server replies on the HEALTH_SOCKET (rather than on the
134 - OUTPUT_SOCKET such that the messages are not intermingled with
135 - output streaming).
126 + async def run_heartbeat_loop(self, timeout: int):
127 + """Background loop that continually listens to the RPCServer for
128 + heartbeats.
129 """
137 -
130 try:
131 while True:
140 - if await self.health_socket.poll(timeout=timeout) == 0:
141 - # Wakeup every N seconds and do a health probe.
142 - await self._send_one_way_rpc_request(
143 - RPCHealthRequest(), self.input_socket)
144 -
145 - # Wait for ack from the health socket.
146 - await self._await_ack(error_message="Health check failed.",
147 - socket=self.health_socket)
132 + if await self.heartbeat_socket.poll(timeout=timeout) == 0:
133 + # No heartbeat was received. Set error and exit the loop
134 + self._set_errored(
135 + TimeoutError("No heartbeat received "
136 + "from MQLLMEngine"))
137 + logger.debug("Shutting down MQLLMEngineClient check "
138 + "health loop due to timeout")
139 + break
140 +
141 else:
149 - # Server sent a health status message unprompted.
142 + # Heartbeat received- check the message
143 await self._check_success(
151 - error_message="Health check failed.",
152 - socket=self.health_socket)
144 + error_message="Heartbeat failed.",
145 + socket=self.heartbeat_socket)
146
154 - logger.debug("Health probe successful.")
147 + logger.debug("Heartbeat successful.")
148
149 except asyncio.CancelledError:
150 logger.debug("Shutting down MQLLMEngineClient check health loop.")
@@ -234,7 +227,7 @@ class MQLLMEngineClient:
227
228 # Start health_loop.
229 self.health_loop = asyncio.create_task(
237 - self.run_check_health_loop(timeout=VLLM_RPC_TIMEOUT))
230 + self.run_heartbeat_loop(timeout=VLLM_RPC_TIMEOUT))
231
232 def close(self):
233 """Destroy the ZeroMQ Context."""
vllm/engine/multiprocessing/engine.py
+60 -17
@@ -1,5 +1,7 @@
1 import pickle
2 import signal
3 +import threading
4 +import time
5 from contextlib import contextmanager
6 from typing import Iterator, List, Optional, Union
7
@@ -15,10 +17,10 @@ from vllm.engine.multiprocessing import (ENGINE_DEAD_ERROR, IPC_DATA_EXT,
17 IPC_HEALTH_EXT, IPC_INPUT_EXT,
18 IPC_OUTPUT_EXT, REQUEST_OUTPUTS_T,
19 VLLM_RPC_SUCCESS_STR, RPCAbortRequest,
18 - RPCError, RPCHealthRequest,
19 - RPCProcessRequest, RPCStartupRequest,
20 - RPCStartupResponse)
20 + RPCError, RPCProcessRequest,
21 + RPCStartupRequest, RPCStartupResponse)
22 # yapf: enable
23 +from vllm.envs import VLLM_RPC_TIMEOUT
24 from vllm.logger import init_logger
25 from vllm.outputs import RequestOutput
26 from vllm.usage.usage_lib import UsageContext
@@ -91,9 +93,9 @@ class MQLLMEngine:
93 self.output_socket = self.ctx.socket(zmq.constants.PUSH)
94 self.output_socket.bind(f"{ipc_path}{IPC_OUTPUT_EXT}")
95
94 - # Send health status back to client.
95 - self.health_socket = self.ctx.socket(zmq.constants.PUSH)
96 - self.health_socket.bind(f"{ipc_path}{IPC_HEALTH_EXT}")
96 + # Send heartbeats back to client.
97 + self.heartbeat_socket = self.ctx.socket(zmq.constants.PUSH)
98 + self.heartbeat_socket.bind(f"{ipc_path}{IPC_HEALTH_EXT}")
99
100 # IPC path for the data socket.
101 self.data_ipc_path = f"{ipc_path}{IPC_DATA_EXT}"
@@ -101,6 +103,20 @@ class MQLLMEngine:
103 # Error state.
104 self._errored_with: Optional[BaseException] = None
105
106 + # Heartbeat thread
107 + self.heartbeat_thread = threading.Thread(target=self._heartbeat_loop,
108 + daemon=True)
109 + self._heartbeat_stop_event = threading.Event()
110 + # The heartbeat needs to be faster than what the client will wait for
111 + # The VLLM_RPC_TIMEOUT duration is in ms, and we need one in seconds
112 + self.heartbeat_interval_seconds = VLLM_RPC_TIMEOUT / 5000.0
113 +
114 + self._last_alive_time = time.time()
115 + # The heartbeats can tolerate a long period of the engine chugging
116 + # away at a generation request.
117 + # The VLLM_RPC_TIMEOUT duration is in ms, and we need one in seconds
118 + self.last_alive_threshold = VLLM_RPC_TIMEOUT * 3.0 / 1000.0
119 +
120 @property
121 def dead_error(self) -> BaseException:
122 if self._errored_with is not None:
@@ -131,6 +147,8 @@ class MQLLMEngine:
147 try:
148 logger.debug("Starting Startup Loop.")
149 self.run_startup_loop()
150 + logger.debug("Starting heartbeat thread")
151 + self.heartbeat_thread.start()
152 logger.debug("Starting Engine Loop.")
153 self.run_engine_loop()
154 except Exception as e:
@@ -144,6 +162,7 @@ class MQLLMEngine:
162 def cleanup(self):
163 """Cleanup zeromq state on shutdown."""
164 # Closes all sockets and destroys context.
165 + self._heartbeat_stop_event.set()
166 self.ctx.destroy(linger=0)
167 del self.engine
168
@@ -182,9 +201,11 @@ class MQLLMEngine:
201 """Core busy loop of the LLMEngine."""
202
203 while True:
204 + self._alive()
205 if not self.engine.has_unfinished_requests():
206 # Poll until there is work to do.
207 while self.input_socket.poll(timeout=POLLING_TIMEOUT_MS) == 0:
208 + self._alive()
209 self.engine.do_log_stats()
210 logger.debug("Waiting for new requests in engine loop.")
211
@@ -200,7 +221,6 @@ class MQLLMEngine:
221
222 def engine_step(self) -> List[RequestOutput]:
223 """Engine step wrapper with error handling."""
203 -
224 try:
225 return self.engine.step()
226 except SystemExit:
@@ -229,10 +249,9 @@ class MQLLMEngine:
249 self._handle_process_request(request)
250 elif isinstance(request, RPCAbortRequest):
251 self._handle_abort_request(request)
232 - elif isinstance(request, RPCHealthRequest):
233 - self._handle_health_request()
252 else:
235 - raise ValueError("Unknown RPCRequest Type: {request}")
253 + raise ValueError("Unknown RPCRequest Type: "
254 + f"{type(request)}")
255
256 except Exception as e:
257 self._set_errored(e)
@@ -279,13 +298,32 @@ class MQLLMEngine:
298 if self.log_requests:
299 logger.info("Aborted request %s.", request.request_id)
300
282 - def _handle_health_request(self):
301 + def _heartbeat_loop(self):
302 + while not self._heartbeat_stop_event.wait(
303 + timeout=self.heartbeat_interval_seconds):
304 + # Loops until the stop event is set
305 + self._heartbeat()
306 +
307 + logger.debug("Exiting MQLLMEngine heartbeat thread")
308 +
309 + def _heartbeat(self):
310 + # Send unhealthy if engine has already errored
311 if self._errored_with is not None:
312 self._send_unhealthy(self._errored_with)
313
286 - # Raises error if unhealthy.
287 - self.engine.check_health()
288 - self._send_healthy()
314 + # Check for life of the main loop
315 + elif time.time() - self._last_alive_time > self.last_alive_threshold:
316 + self._send_unhealthy(RuntimeError("Engine loop has died"))
317 +
318 + else:
319 + # Otherwise- check health of the engine
320 + # self.engine.check_health() raises on unhealthy
321 + try:
322 + self.engine.check_health()
323 + self._send_healthy()
324 + except Exception as e:
325 + self._set_errored(e)
326 + self._send_unhealthy(e)
327
328 def _send_outputs(self, outputs: REQUEST_OUTPUTS_T):
329 """Send List of RequestOutput to RPCClient."""
@@ -295,12 +333,14 @@ class MQLLMEngine:
333
334 def _send_healthy(self):
335 """Send HEALTHY message to RPCClient."""
298 - self.health_socket.send_multipart(HEALTHY_RESPONSE, copy=False)
336 + if not self.heartbeat_socket.closed:
337 + self.heartbeat_socket.send_multipart(HEALTHY_RESPONSE, copy=False)
338
339 def _send_unhealthy(self, error: BaseException):
340 """Send UNHEALTHY message to RPCClient."""
302 - error_bytes = pickle.dumps(error)
303 - self.health_socket.send_multipart((error_bytes, ), copy=False)
341 + if not self.heartbeat_socket.closed:
342 + error_bytes = pickle.dumps(error)
343 + self.heartbeat_socket.send_multipart((error_bytes, ), copy=False)
344
345 def _async_socket_engine_callback(self,
346 request_outputs: REQUEST_OUTPUTS_T):
@@ -313,6 +353,9 @@ class MQLLMEngine:
353 if self._errored_with is None:
354 self._errored_with = e
355
356 + def _alive(self):
357 + self._last_alive_time = time.time()
358 +
359
360 def run_mp_engine(engine_args: AsyncEngineArgs, usage_context: UsageContext,
361 ipc_path: str):