@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / ff578cae54
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 04:15 UTC

Add health check, make async Engine more robust (#3015)

Co-authored-by: Zhuohan Li <zhuohan123@gmail.com>

Antoni Baum committed Mar 4, 2024 at 14:01 UTC ff578cae54d23812b53b6c9b94b8bd0bb293a1fe
4 files changed +138 -65
tests/async_engine/test_async_llm_engine.py
+16 -16
@@ -25,12 +25,8 @@ class MockEngine:
25 return [RequestOutput(
26 request_id=self.request_id)] if self.request_id else []
27
28 - async def encode_request_async(
29 - self,
30 - *args,
31 - **kwargs,
32 - ):
33 - return [1]
28 + async def encode_request_async(self, *args, **kwargs):
29 + pass
30
31 def generate(self, request_id):
32 self.request_id = request_id
@@ -43,13 +39,16 @@ class MockEngine:
39 self.add_request_calls += 1
40
41 async def add_request_async(self, **kwargs):
46 - del kwargs # Unused
42 self.add_request_calls += 1
43 + return
44
45 def abort_request(self, request_id):
46 del request_id # Unused
47 self.abort_request_calls += 1
48
49 + def has_unfinished_requests(self):
50 + return self.request_id is not None
51 +
52
53 class MockAsyncLLMEngine(AsyncLLMEngine):
54
@@ -72,20 +71,21 @@ async def test_new_requests_event():
71 await engine.add_request("2", "", None)
72 engine.engine.generate("2")
73 await asyncio.sleep(0)
75 - assert engine.engine.add_request_calls == 2
76 - assert engine.engine.step_calls == 2
74 await asyncio.sleep(0)
78 - assert engine.engine.step_calls == 3
75 + assert engine.engine.add_request_calls == 2
76 + assert engine.engine.step_calls >= 2
77 + await asyncio.sleep(0.001)
78 + assert engine.engine.step_calls >= 3
79 engine.engine.stop_generating()
80 - await asyncio.sleep(0)
81 - assert engine.engine.step_calls == 4
82 - await asyncio.sleep(0)
83 - assert engine.engine.step_calls == 4
80 + await asyncio.sleep(0.001)
81 + old_step_calls = engine.engine.step_calls
82 + await asyncio.sleep(0.001)
83 + assert engine.engine.step_calls == old_step_calls
84
85 await engine.add_request("3", "", None)
86 await asyncio.sleep(0.01)
87 assert engine.engine.add_request_calls == 3
88 - assert engine.engine.step_calls == 5
88 + assert engine.engine.step_calls == old_step_calls + 1
89 await asyncio.sleep(0.01)
90 assert engine.engine.add_request_calls == 3
91 - assert engine.engine.step_calls == 5
91 + assert engine.engine.step_calls == old_step_calls + 1
tests/async_engine/test_request_tracker.py
+15 -23
@@ -4,25 +4,14 @@ from vllm.engine.async_llm_engine import RequestTracker
4 from vllm.outputs import RequestOutput
5
6
7 -class DummyEvent:
8 -
9 - def __init__(self):
10 - self.flag = False
11 -
12 - def set(self):
13 - self.flag = True
14 -
15 - def clear(self):
16 - self.flag = False
17 -
18 -
19 -def test_request_tracker():
7 +@pytest.mark.asyncio
8 +async def test_request_tracker():
9 tracker = RequestTracker()
21 - tracker.new_requests_event = DummyEvent()
10 stream_1 = tracker.add_request("1")
23 - assert tracker.new_requests_event.flag
11 + assert tracker.new_requests_event.is_set()
12 + await tracker.wait_for_new_requests()
13 new, finished = tracker.get_new_and_finished_requests()
25 - assert not tracker.new_requests_event.flag
14 + assert not tracker.new_requests_event.is_set()
15 assert len(new) == 1
16 assert new[0]["request_id"] == "1"
17 assert not finished
@@ -30,9 +19,10 @@ def test_request_tracker():
19
20 stream_2 = tracker.add_request("2")
21 stream_3 = tracker.add_request("3")
33 - assert tracker.new_requests_event.flag
22 + assert tracker.new_requests_event.is_set()
23 + await tracker.wait_for_new_requests()
24 new, finished = tracker.get_new_and_finished_requests()
35 - assert not tracker.new_requests_event.flag
25 + assert not tracker.new_requests_event.is_set()
26 assert len(new) == 2
27 assert new[0]["request_id"] == "2"
28 assert new[1]["request_id"] == "3"
@@ -43,7 +33,7 @@ def test_request_tracker():
33 # request_ids must be unique
34 with pytest.raises(KeyError):
35 tracker.add_request("1")
46 - assert not tracker.new_requests_event.flag
36 + assert not tracker.new_requests_event.is_set()
37
38 tracker.abort_request("1")
39 new, finished = tracker.get_new_and_finished_requests()
@@ -54,7 +44,8 @@ def test_request_tracker():
44
45 stream_4 = tracker.add_request("4")
46 tracker.abort_request("4")
57 - assert tracker.new_requests_event.flag
47 + assert tracker.new_requests_event.is_set()
48 + await tracker.wait_for_new_requests()
49 new, finished = tracker.get_new_and_finished_requests()
50 assert len(finished) == 1
51 assert "4" in finished
@@ -62,11 +53,12 @@ def test_request_tracker():
53 assert stream_4.finished
54
55 stream_5 = tracker.add_request("5")
65 - assert tracker.new_requests_event.flag
56 + assert tracker.new_requests_event.is_set()
57 tracker.process_request_output(
67 - RequestOutput("2", "output", [], [], [], bool(finished)))
58 + RequestOutput("2", "output", [], [], [], finished=True))
59 + await tracker.wait_for_new_requests()
60 new, finished = tracker.get_new_and_finished_requests()
69 - assert not tracker.new_requests_event.flag
61 + assert not tracker.new_requests_event.is_set()
62 assert len(finished) == 1
63 assert "2" in finished
64 assert len(new) == 1
vllm/engine/async_llm_engine.py
+87 -26
@@ -1,8 +1,9 @@
1 import asyncio
2 +import os
3 import time
4 from functools import partial
5 from typing import (Any, Dict, Iterable, List, Optional, Set, Tuple, Type,
5 - Union, AsyncIterator)
6 + Union, AsyncIterator, Callable)
7
8 from vllm.lora.request import LoRARequest
9 from vllm.config import ModelConfig
@@ -14,28 +15,31 @@ from vllm.outputs import RequestOutput
15 from vllm.sampling_params import SamplingParams
16
17 logger = init_logger(__name__)
18 +ENGINE_ITERATION_TIMEOUT_S = int(
19 + os.environ.get("VLLM_ENGINE_ITERATION_TIMEOUT_S", "60"))
20
21
22 class AsyncEngineDeadError(RuntimeError):
23 pass
24
25
23 -def _raise_exception_on_finish(task: asyncio.Task,
24 - request_tracker: "RequestTracker") -> None:
26 +def _raise_exception_on_finish(
27 + task: asyncio.Task, error_callback: Callable[[Exception],
28 + None]) -> None:
29 msg = ("Task finished unexpectedly. This should never happen! "
30 "Please open an issue on Github.")
31 +
32 + exception = None
33 try:
28 - try:
29 - task.result()
30 - except asyncio.CancelledError:
31 - return
32 - except Exception as exc:
33 - raise AsyncEngineDeadError(
34 - msg + " See stack trace above for the actual cause.") from exc
34 + task.result()
35 + # NOTE: This will be thrown if task exits normally (which it should not)
36 raise AsyncEngineDeadError(msg)
36 - except Exception as exc:
37 - request_tracker.propagate_exception(exc)
38 - raise exc
37 + except Exception as e:
38 + exception = e
39 + logger.error("Engine background task failed", exc_info=e)
40 + error_callback(exception)
41 + raise AsyncEngineDeadError(
42 + msg + " See stack trace above for the actual cause.") from e
43
44
45 class AsyncStream:
@@ -78,13 +82,13 @@ class RequestTracker:
82 self._finished_requests: asyncio.Queue[str] = asyncio.Queue()
83 self._new_requests: asyncio.Queue[Tuple[AsyncStream,
84 dict]] = asyncio.Queue()
81 - self.new_requests_event = None
85 + self.new_requests_event = asyncio.Event()
86
87 def __contains__(self, item):
88 return item in self._request_streams
89
86 - def init_event(self):
87 - self.new_requests_event = asyncio.Event()
90 + def __len__(self) -> int:
91 + return len(self._request_streams)
92
93 def propagate_exception(self,
94 exc: Exception,
@@ -93,9 +97,11 @@ class RequestTracker:
97 (all if request_id is None)."""
98 if request_id is not None:
99 self._request_streams[request_id].put(exc)
100 + self.abort_request(request_id)
101 else:
97 - for stream in self._request_streams.values():
102 + for rid, stream in self._request_streams.items():
103 stream.put(exc)
104 + self.abort_request(rid)
105
106 def process_request_output(self,
107 request_output: RequestOutput,
@@ -172,12 +178,15 @@ class RequestTracker:
178 self._request_streams[stream.request_id] = stream
179 new_requests.append(new_request)
180
175 - self.new_requests_event.clear()
176 -
181 return new_requests, finished_requests
182
183 async def wait_for_new_requests(self):
180 - await self.new_requests_event.wait()
184 + if not self.has_new_requests():
185 + await self.new_requests_event.wait()
186 + self.new_requests_event.clear()
187 +
188 + def has_new_requests(self):
189 + return not self._new_requests.empty()
190
191
192 class _AsyncLLMEngine(LLMEngine):
@@ -285,6 +294,10 @@ class _AsyncLLMEngine(LLMEngine):
294 all_outputs = await asyncio.gather(*coros)
295 return all_outputs
296
297 + async def check_health_async(self):
298 + """Raises an error if engine is unhealthy."""
299 + self._check_if_any_actor_is_dead()
300 +
301
302 class AsyncLLMEngine:
303 """An asynchronous wrapper for LLMEngine.
@@ -335,27 +348,48 @@ class AsyncLLMEngine:
348 # collected
349 self._background_loop_unshielded = None
350 self.start_engine_loop = start_engine_loop
338 - self._request_tracker = RequestTracker()
351 + self._request_tracker: Optional[RequestTracker] = None
352 + self._errored_with: Optional[BaseException] = None
353
354 @property
355 def is_running(self) -> bool:
356 return (self.background_loop is not None
343 - and not self.background_loop.done())
357 + and not self._background_loop_unshielded.done())
358 +
359 + @property
360 + def is_stopped(self) -> bool:
361 + return self.errored or (self.background_loop is not None
362 + and self._background_loop_unshielded.done())
363 +
364 + @property
365 + def errored(self) -> bool:
366 + return self._errored_with is not None
367 +
368 + def set_errored(self, exc: Exception) -> None:
369 + self._errored_with = exc
370 +
371 + def _error_callback(self, exc: Exception) -> None:
372 + self.set_errored(exc)
373 + self._request_tracker.propagate_exception(exc)
374
375 def get_tokenizer(self):
376 return self.engine.tokenizer.tokenizer
377
378 def start_background_loop(self) -> None:
379 """Start the background loop."""
380 + if self.errored:
381 + raise AsyncEngineDeadError(
382 + "Background loop has errored already.") from self._errored_with
383 if self.is_running:
384 raise RuntimeError("Background loop is already running.")
352 - self._request_tracker.init_event()
385 + # Initialize the RequestTracker here so it uses the right event loop.
386 + self._request_tracker = RequestTracker()
387
388 self._background_loop_unshielded = asyncio.get_event_loop(
389 ).create_task(self.run_engine_loop())
390 self._background_loop_unshielded.add_done_callback(
391 partial(_raise_exception_on_finish,
358 - request_tracker=self._request_tracker))
392 + error_callback=self._error_callback))
393 self.background_loop = asyncio.shield(self._background_loop_unshielded)
394
395 def _init_engine(self, *args,
@@ -423,12 +457,23 @@ class AsyncLLMEngine:
457 self.engine.abort_request(request_ids)
458
459 async def run_engine_loop(self):
426 - # Initialize the RequestTracker here so it uses the right event loop.
460 has_requests_in_progress = False
461 while True:
462 if not has_requests_in_progress:
463 + logger.debug("Waiting for new requests...")
464 await self._request_tracker.wait_for_new_requests()
431 - has_requests_in_progress = await self.engine_step()
465 + logger.debug("Got new requests!")
466 +
467 + # Abort if iteration takes too long due to unrecoverable errors
468 + # (eg. NCCL timeouts).
469 + try:
470 + has_requests_in_progress = await asyncio.wait_for(
471 + self.engine_step(), ENGINE_ITERATION_TIMEOUT_S)
472 + except asyncio.TimeoutError as exc:
473 + logger.error(
474 + "Engine iteration timed out. This should never happen!")
475 + self.set_errored(exc)
476 + raise
477 await asyncio.sleep(0)
478
479 async def add_request(
@@ -647,3 +692,19 @@ class AsyncLLMEngine:
692 await self.engine.do_log_stats.remote()
693 else:
694 self.engine.do_log_stats()
695 +
696 + async def check_health(self):
697 + """Raises an error if engine is unhealthy."""
698 + t = time.perf_counter()
699 + logger.debug("Starting health check...")
700 + if self.is_stopped:
701 + raise AsyncEngineDeadError("Background loop is stopped.")
702 +
703 + if self.engine_use_ray:
704 + try:
705 + await self.engine.check_health.remote()
706 + except ray.exceptions.RayActorError as e:
707 + raise RuntimeError("Engine is dead.") from e
708 + else:
709 + await self.engine.check_health_async()
710 + logger.debug(f"Health check took {time.perf_counter()-t}s")
vllm/engine/llm_engine.py
+20
@@ -1119,3 +1119,23 @@ class LLMEngine:
1119 for worker in self.workers
1120 ])
1121 return forward_dag.experimental_compile()
1122 +
1123 + def check_health(self) -> None:
1124 + """Raises an error if engine is unhealthy."""
1125 + self._check_if_any_actor_is_dead()
1126 +
1127 + def _check_if_any_actor_is_dead(self):
1128 + if not self.parallel_config.worker_use_ray:
1129 + return
1130 +
1131 + if not self.workers:
1132 + return
1133 +
1134 + dead_actors = []
1135 + for actor in self.workers:
1136 + actor_state = ray.state.actors(actor._ray_actor_id.hex()) # pylint: disable=protected-access
1137 + if actor_state["State"] == "DEAD":
1138 + dead_actors.append(actor)
1139 + if dead_actors:
1140 + raise RuntimeError("At least one Worker is dead. "
1141 + f"Dead Workers: {dead_actors}. ")