A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 04:15 UTC
Add health check, make async Engine more robust (#3015)
Co-authored-by: Zhuohan Li <zhuohan123@gmail.com>
Antoni Baum committed
Mar 4, 2024 at 14:01 UTC
ff578cae54d23812b53b6c9b94b8bd0bb293a1fe
4 files changed
+138
-65
tests/async_engine/test_async_llm_engine.py
+16
-16
@@ -25,12 +25,8 @@ class MockEngine:
25
return [RequestOutput(
26
request_id=self.request_id)] if self.request_id else []
27
28
- async def encode_request_async(
29
- self,
30
- *args,
31
- **kwargs,
32
- ):
33
- return [1]
28
+ async def encode_request_async(self, *args, **kwargs):
29
+ pass
30
31
def generate(self, request_id):
32
self.request_id = request_id
@@ -43,13 +39,16 @@ class MockEngine:
39
self.add_request_calls += 1
40
41
async def add_request_async(self, **kwargs):
46
- del kwargs # Unused
42
self.add_request_calls += 1
43
+ return
44
45
def abort_request(self, request_id):
46
del request_id # Unused
47
self.abort_request_calls += 1
48
49
+ def has_unfinished_requests(self):
50
+ return self.request_id is not None
51
+
52
53
class MockAsyncLLMEngine(AsyncLLMEngine):
54
@@ -72,20 +71,21 @@ async def test_new_requests_event():
71
await engine.add_request("2", "", None)
72
engine.engine.generate("2")
73
await asyncio.sleep(0)
75
- assert engine.engine.add_request_calls == 2
76
- assert engine.engine.step_calls == 2
74
await asyncio.sleep(0)
78
- assert engine.engine.step_calls == 3
75
+ assert engine.engine.add_request_calls == 2
76
+ assert engine.engine.step_calls >= 2
77
+ await asyncio.sleep(0.001)
78
+ assert engine.engine.step_calls >= 3
79
engine.engine.stop_generating()
80
- await asyncio.sleep(0)
81
- assert engine.engine.step_calls == 4
82
- await asyncio.sleep(0)
83
- assert engine.engine.step_calls == 4
80
+ await asyncio.sleep(0.001)
81
+ old_step_calls = engine.engine.step_calls
82
+ await asyncio.sleep(0.001)
83
+ assert engine.engine.step_calls == old_step_calls
84
85
await engine.add_request("3", "", None)
86
await asyncio.sleep(0.01)
87
assert engine.engine.add_request_calls == 3
88
- assert engine.engine.step_calls == 5
88
+ assert engine.engine.step_calls == old_step_calls + 1
89
await asyncio.sleep(0.01)
90
assert engine.engine.add_request_calls == 3
91
- assert engine.engine.step_calls == 5
91
+ assert engine.engine.step_calls == old_step_calls + 1
tests/async_engine/test_request_tracker.py
+15
-23
@@ -4,25 +4,14 @@ from vllm.engine.async_llm_engine import RequestTracker
4
from vllm.outputs import RequestOutput
5
6
7
-class DummyEvent:
8
-
9
- def __init__(self):
10
- self.flag = False
11
-
12
- def set(self):
13
- self.flag = True
14
-
15
- def clear(self):
16
- self.flag = False
17
-
18
-
19
-def test_request_tracker():
7
+@pytest.mark.asyncio
8
+async def test_request_tracker():
9
tracker = RequestTracker()
21
- tracker.new_requests_event = DummyEvent()
10
stream_1 = tracker.add_request("1")
23
- assert tracker.new_requests_event.flag
11
+ assert tracker.new_requests_event.is_set()
12
+ await tracker.wait_for_new_requests()
13
new, finished = tracker.get_new_and_finished_requests()
25
- assert not tracker.new_requests_event.flag
14
+ assert not tracker.new_requests_event.is_set()
15
assert len(new) == 1
16
assert new[0]["request_id"] == "1"
17
assert not finished
@@ -30,9 +19,10 @@ def test_request_tracker():
19
20
stream_2 = tracker.add_request("2")
21
stream_3 = tracker.add_request("3")
33
- assert tracker.new_requests_event.flag
22
+ assert tracker.new_requests_event.is_set()
23
+ await tracker.wait_for_new_requests()
24
new, finished = tracker.get_new_and_finished_requests()
35
- assert not tracker.new_requests_event.flag
25
+ assert not tracker.new_requests_event.is_set()
26
assert len(new) == 2
27
assert new[0]["request_id"] == "2"
28
assert new[1]["request_id"] == "3"
@@ -43,7 +33,7 @@ def test_request_tracker():
33
# request_ids must be unique
34
with pytest.raises(KeyError):
35
tracker.add_request("1")
46
- assert not tracker.new_requests_event.flag
36
+ assert not tracker.new_requests_event.is_set()
37
38
tracker.abort_request("1")
39
new, finished = tracker.get_new_and_finished_requests()
@@ -54,7 +44,8 @@ def test_request_tracker():
44
45
stream_4 = tracker.add_request("4")
46
tracker.abort_request("4")
57
- assert tracker.new_requests_event.flag
47
+ assert tracker.new_requests_event.is_set()
48
+ await tracker.wait_for_new_requests()
49
new, finished = tracker.get_new_and_finished_requests()
50
assert len(finished) == 1
51
assert "4" in finished
@@ -62,11 +53,12 @@ def test_request_tracker():
53
assert stream_4.finished
54
55
stream_5 = tracker.add_request("5")
65
- assert tracker.new_requests_event.flag
56
+ assert tracker.new_requests_event.is_set()
57
tracker.process_request_output(
67
- RequestOutput("2", "output", [], [], [], bool(finished)))
58
+ RequestOutput("2", "output", [], [], [], finished=True))
59
+ await tracker.wait_for_new_requests()
60
new, finished = tracker.get_new_and_finished_requests()
69
- assert not tracker.new_requests_event.flag
61
+ assert not tracker.new_requests_event.is_set()
62
assert len(finished) == 1
63
assert "2" in finished
64
assert len(new) == 1
vllm/engine/async_llm_engine.py
+87
-26
@@ -1,8 +1,9 @@
1
import asyncio
2
+import os
3
import time
4
from functools import partial
5
from typing import (Any, Dict, Iterable, List, Optional, Set, Tuple, Type,
5
- Union, AsyncIterator)
6
+ Union, AsyncIterator, Callable)
7
8
from vllm.lora.request import LoRARequest
9
from vllm.config import ModelConfig
@@ -14,28 +15,31 @@ from vllm.outputs import RequestOutput
15
from vllm.sampling_params import SamplingParams
16
17
logger = init_logger(__name__)
18
+ENGINE_ITERATION_TIMEOUT_S = int(
19
+ os.environ.get("VLLM_ENGINE_ITERATION_TIMEOUT_S", "60"))
20
21
22
class AsyncEngineDeadError(RuntimeError):
23
pass
24
25
23
-def _raise_exception_on_finish(task: asyncio.Task,
24
- request_tracker: "RequestTracker") -> None:
26
+def _raise_exception_on_finish(
27
+ task: asyncio.Task, error_callback: Callable[[Exception],
28
+ None]) -> None:
29
msg = ("Task finished unexpectedly. This should never happen! "
30
"Please open an issue on Github.")
31
+
32
+ exception = None
33
try:
28
- try:
29
- task.result()
30
- except asyncio.CancelledError:
31
- return
32
- except Exception as exc:
33
- raise AsyncEngineDeadError(
34
- msg + " See stack trace above for the actual cause.") from exc
34
+ task.result()
35
+ # NOTE: This will be thrown if task exits normally (which it should not)
36
raise AsyncEngineDeadError(msg)
36
- except Exception as exc:
37
- request_tracker.propagate_exception(exc)
38
- raise exc
37
+ except Exception as e:
38
+ exception = e
39
+ logger.error("Engine background task failed", exc_info=e)
40
+ error_callback(exception)
41
+ raise AsyncEngineDeadError(
42
+ msg + " See stack trace above for the actual cause.") from e
43
44
45
class AsyncStream:
@@ -78,13 +82,13 @@ class RequestTracker:
82
self._finished_requests: asyncio.Queue[str] = asyncio.Queue()
83
self._new_requests: asyncio.Queue[Tuple[AsyncStream,
84
dict]] = asyncio.Queue()
81
- self.new_requests_event = None
85
+ self.new_requests_event = asyncio.Event()
86
87
def __contains__(self, item):
88
return item in self._request_streams
89
86
- def init_event(self):
87
- self.new_requests_event = asyncio.Event()
90
+ def __len__(self) -> int:
91
+ return len(self._request_streams)
92
93
def propagate_exception(self,
94
exc: Exception,
@@ -93,9 +97,11 @@ class RequestTracker:
97
(all if request_id is None)."""
98
if request_id is not None:
99
self._request_streams[request_id].put(exc)
100
+ self.abort_request(request_id)
101
else:
97
- for stream in self._request_streams.values():
102
+ for rid, stream in self._request_streams.items():
103
stream.put(exc)
104
+ self.abort_request(rid)
105
106
def process_request_output(self,
107
request_output: RequestOutput,
@@ -172,12 +178,15 @@ class RequestTracker:
178
self._request_streams[stream.request_id] = stream
179
new_requests.append(new_request)
180
175
- self.new_requests_event.clear()
176
-
181
return new_requests, finished_requests
182
183
async def wait_for_new_requests(self):
180
- await self.new_requests_event.wait()
184
+ if not self.has_new_requests():
185
+ await self.new_requests_event.wait()
186
+ self.new_requests_event.clear()
187
+
188
+ def has_new_requests(self):
189
+ return not self._new_requests.empty()
190
191
192
class _AsyncLLMEngine(LLMEngine):
@@ -285,6 +294,10 @@ class _AsyncLLMEngine(LLMEngine):
294
all_outputs = await asyncio.gather(*coros)
295
return all_outputs
296
297
+ async def check_health_async(self):
298
+ """Raises an error if engine is unhealthy."""
299
+ self._check_if_any_actor_is_dead()
300
+
301
302
class AsyncLLMEngine:
303
"""An asynchronous wrapper for LLMEngine.
@@ -335,27 +348,48 @@ class AsyncLLMEngine:
348
# collected
349
self._background_loop_unshielded = None
350
self.start_engine_loop = start_engine_loop
338
- self._request_tracker = RequestTracker()
351
+ self._request_tracker: Optional[RequestTracker] = None
352
+ self._errored_with: Optional[BaseException] = None
353
354
@property
355
def is_running(self) -> bool:
356
return (self.background_loop is not None
343
- and not self.background_loop.done())
357
+ and not self._background_loop_unshielded.done())
358
+
359
+ @property
360
+ def is_stopped(self) -> bool:
361
+ return self.errored or (self.background_loop is not None
362
+ and self._background_loop_unshielded.done())
363
+
364
+ @property
365
+ def errored(self) -> bool:
366
+ return self._errored_with is not None
367
+
368
+ def set_errored(self, exc: Exception) -> None:
369
+ self._errored_with = exc
370
+
371
+ def _error_callback(self, exc: Exception) -> None:
372
+ self.set_errored(exc)
373
+ self._request_tracker.propagate_exception(exc)
374
375
def get_tokenizer(self):
376
return self.engine.tokenizer.tokenizer
377
378
def start_background_loop(self) -> None:
379
"""Start the background loop."""
380
+ if self.errored:
381
+ raise AsyncEngineDeadError(
382
+ "Background loop has errored already.") from self._errored_with
383
if self.is_running:
384
raise RuntimeError("Background loop is already running.")
352
- self._request_tracker.init_event()
385
+ # Initialize the RequestTracker here so it uses the right event loop.
386
+ self._request_tracker = RequestTracker()
387
388
self._background_loop_unshielded = asyncio.get_event_loop(
389
).create_task(self.run_engine_loop())
390
self._background_loop_unshielded.add_done_callback(
391
partial(_raise_exception_on_finish,
358
- request_tracker=self._request_tracker))
392
+ error_callback=self._error_callback))
393
self.background_loop = asyncio.shield(self._background_loop_unshielded)
394
395
def _init_engine(self, *args,
@@ -423,12 +457,23 @@ class AsyncLLMEngine:
457
self.engine.abort_request(request_ids)
458
459
async def run_engine_loop(self):
426
- # Initialize the RequestTracker here so it uses the right event loop.
460
has_requests_in_progress = False
461
while True:
462
if not has_requests_in_progress:
463
+ logger.debug("Waiting for new requests...")
464
await self._request_tracker.wait_for_new_requests()
431
- has_requests_in_progress = await self.engine_step()
465
+ logger.debug("Got new requests!")
466
+
467
+ # Abort if iteration takes too long due to unrecoverable errors
468
+ # (eg. NCCL timeouts).
469
+ try:
470
+ has_requests_in_progress = await asyncio.wait_for(
471
+ self.engine_step(), ENGINE_ITERATION_TIMEOUT_S)
472
+ except asyncio.TimeoutError as exc:
473
+ logger.error(
474
+ "Engine iteration timed out. This should never happen!")
475
+ self.set_errored(exc)
476
+ raise
477
await asyncio.sleep(0)
478
479
async def add_request(
@@ -647,3 +692,19 @@ class AsyncLLMEngine:
692
await self.engine.do_log_stats.remote()
693
else:
694
self.engine.do_log_stats()
695
+
696
+ async def check_health(self):
697
+ """Raises an error if engine is unhealthy."""
698
+ t = time.perf_counter()
699
+ logger.debug("Starting health check...")
700
+ if self.is_stopped:
701
+ raise AsyncEngineDeadError("Background loop is stopped.")
702
+
703
+ if self.engine_use_ray:
704
+ try:
705
+ await self.engine.check_health.remote()
706
+ except ray.exceptions.RayActorError as e:
707
+ raise RuntimeError("Engine is dead.") from e
708
+ else:
709
+ await self.engine.check_health_async()
710
+ logger.debug(f"Health check took {time.perf_counter()-t}s")
vllm/engine/llm_engine.py
+20
@@ -1119,3 +1119,23 @@ class LLMEngine:
1119
for worker in self.workers
1120
])
1121
return forward_dag.experimental_compile()
1122
+
1123
+ def check_health(self) -> None:
1124
+ """Raises an error if engine is unhealthy."""
1125
+ self._check_if_any_actor_is_dead()
1126
+
1127
+ def _check_if_any_actor_is_dead(self):
1128
+ if not self.parallel_config.worker_use_ray:
1129
+ return
1130
+
1131
+ if not self.workers:
1132
+ return
1133
+
1134
+ dead_actors = []
1135
+ for actor in self.workers:
1136
+ actor_state = ray.state.actors(actor._ray_actor_id.hex()) # pylint: disable=protected-access
1137
+ if actor_state["State"] == "DEAD":
1138
+ dead_actors.append(actor)
1139
+ if dead_actors:
1140
+ raise RuntimeError("At least one Worker is dead. "
1141
+ f"Dead Workers: {dead_actors}. ")