A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 06:45 UTC
Remove AsyncLLMEngine busy loop, shield background task (#1059)
Antoni Baum committed
Sep 17, 2023 at 00:29 UTC
ff36139ffc66294c19b503c1e52dc42c2cd265f6
4 files changed
+154
-18
requirements-dev.txt
+1
@@ -11,3 +11,4 @@ types-setuptools
11
# testing
12
pytest
13
pytest-forked
14
+pytest-asyncio
tests/async_engine/test_async_llm_engine.py
new
+80
@@ -0,0 +1,80 @@
1
+import asyncio
2
+from dataclasses import dataclass
3
+
4
+import pytest
5
+
6
+from vllm.engine.async_llm_engine import AsyncLLMEngine
7
+
8
+
9
+@dataclass
10
+class RequestOutput:
11
+ request_id: int
12
+ finished: bool = False
13
+
14
+
15
+class MockEngine:
16
+
17
+ def __init__(self):
18
+ self.step_calls = 0
19
+ self.add_request_calls = 0
20
+ self.abort_request_calls = 0
21
+ self.request_id = None
22
+
23
+ async def step_async(self):
24
+ self.step_calls += 1
25
+ return [RequestOutput(
26
+ request_id=self.request_id)] if self.request_id else []
27
+
28
+ def generate(self, request_id):
29
+ self.request_id = request_id
30
+
31
+ def stop_generating(self):
32
+ self.request_id = None
33
+
34
+ def add_request(self, **kwargs):
35
+ self.add_request_calls += 1
36
+ return
37
+
38
+ def abort_request(self, request_id):
39
+ self.abort_request_calls += 1
40
+ return
41
+
42
+
43
+class MockAsyncLLMEngine(AsyncLLMEngine):
44
+
45
+ def _init_engine(self, *args, **kwargs):
46
+ return MockEngine()
47
+
48
+
49
+@pytest.mark.asyncio
50
+async def test_new_requests_event():
51
+ engine = MockAsyncLLMEngine(worker_use_ray=False, engine_use_ray=False)
52
+ engine.start_background_loop()
53
+ await asyncio.sleep(0.01)
54
+ assert engine.engine.step_calls == 0
55
+
56
+ await engine.add_request("1", "", None)
57
+ await asyncio.sleep(0.01)
58
+ assert engine.engine.add_request_calls == 1
59
+ assert engine.engine.step_calls == 1
60
+
61
+ await engine.add_request("2", "", None)
62
+ engine.engine.generate("2")
63
+ await asyncio.sleep(0)
64
+ assert engine.engine.add_request_calls == 2
65
+ assert engine.engine.step_calls == 2
66
+ await asyncio.sleep(0)
67
+ assert engine.engine.step_calls == 3
68
+ engine.engine.stop_generating()
69
+ await asyncio.sleep(0)
70
+ assert engine.engine.step_calls == 4
71
+ await asyncio.sleep(0)
72
+ assert engine.engine.step_calls == 4
73
+
74
+ await engine.add_request("3", "", None)
75
+ await asyncio.sleep(0.01)
76
+ assert engine.engine.add_request_calls == 3
77
+ assert engine.engine.step_calls == 5
78
+ await asyncio.sleep(0.01)
79
+ assert engine.engine.add_request_calls == 3
80
+ assert engine.engine.step_calls == 5
tests/async_engine/test_request_tracker.py
+21
@@ -4,10 +4,25 @@ from vllm.engine.async_llm_engine import RequestTracker
4
from vllm.outputs import RequestOutput
5
6
7
+class DummyEvent:
8
+
9
+ def __init__(self):
10
+ self._flag = False
11
+
12
+ def set(self):
13
+ self._flag = True
14
+
15
+ def clear(self):
16
+ self._flag = False
17
+
18
+
19
def test_request_tracker():
20
tracker = RequestTracker()
21
+ tracker.new_requests_event = DummyEvent()
22
stream_1 = tracker.add_request("1")
23
+ assert tracker.new_requests_event._flag
24
new, finished = tracker.get_new_and_finished_requests()
25
+ assert not tracker.new_requests_event._flag
26
assert len(new) == 1
27
assert new[0]["request_id"] == "1"
28
assert not finished
@@ -15,7 +30,9 @@ def test_request_tracker():
30
31
stream_2 = tracker.add_request("2")
32
stream_3 = tracker.add_request("3")
33
+ assert tracker.new_requests_event._flag
34
new, finished = tracker.get_new_and_finished_requests()
35
+ assert not tracker.new_requests_event._flag
36
assert len(new) == 2
37
assert new[0]["request_id"] == "2"
38
assert new[1]["request_id"] == "3"
@@ -26,6 +43,7 @@ def test_request_tracker():
43
# request_ids must be unique
44
with pytest.raises(KeyError):
45
tracker.add_request("1")
46
+ assert not tracker.new_requests_event._flag
47
48
tracker.abort_request("1")
49
new, finished = tracker.get_new_and_finished_requests()
@@ -36,6 +54,7 @@ def test_request_tracker():
54
55
stream_4 = tracker.add_request("4")
56
tracker.abort_request("4")
57
+ assert tracker.new_requests_event._flag
58
new, finished = tracker.get_new_and_finished_requests()
59
assert len(finished) == 1
60
assert "4" in finished
@@ -43,9 +62,11 @@ def test_request_tracker():
62
assert stream_4.finished
63
64
stream_5 = tracker.add_request("5")
65
+ assert tracker.new_requests_event._flag
66
tracker.process_request_output(
67
RequestOutput("2", "output", [], [], finished=True))
68
new, finished = tracker.get_new_and_finished_requests()
69
+ assert not tracker.new_requests_event._flag
70
assert len(finished) == 1
71
assert "2" in finished
72
assert len(new) == 1
vllm/engine/async_llm_engine.py
+52
-18
@@ -1,7 +1,8 @@
1
import asyncio
2
import time
3
from functools import partial
4
-from typing import Any, Dict, Iterable, List, Optional, Set, Tuple, Type, Union
4
+from typing import (Any, Dict, Iterable, List, Optional, Set, Tuple, Type,
5
+ Union)
6
7
from vllm.config import ModelConfig
8
from vllm.engine.arg_utils import AsyncEngineArgs
@@ -78,14 +79,24 @@ class RequestTracker:
79
self._finished_requests: asyncio.Queue[str] = asyncio.Queue()
80
self._new_requests: asyncio.Queue[Tuple[AsyncStream,
81
dict]] = asyncio.Queue()
82
+ self.new_requests_event = None
83
84
def __contains__(self, item):
85
return item in self._request_streams
86
85
- def propagate_exception(self, exc: Exception) -> None:
86
- """Propagate an exception to all request streams."""
87
- for stream in self._request_streams.values():
88
- stream.put(exc)
87
+ def init_event(self):
88
+ self.new_requests_event = asyncio.Event()
89
+
90
+ def propagate_exception(self,
91
+ exc: Exception,
92
+ request_id: Optional[str] = None) -> None:
93
+ """Propagate an exception to request streams
94
+ (all if request_id is None)."""
95
+ if request_id is not None:
96
+ self._request_streams[request_id].put(exc)
97
+ else:
98
+ for stream in self._request_streams.values():
99
+ stream.put(exc)
100
101
def process_request_output(self,
102
request_output: RequestOutput,
@@ -112,6 +123,9 @@ class RequestTracker:
123
"request_id": request_id,
124
**engine_add_request_kwargs
125
}))
126
+
127
+ self.new_requests_event.set()
128
+
129
return stream
130
131
def abort_request(self, request_id: str, *, verbose: bool = False) -> None:
@@ -148,8 +162,13 @@ class RequestTracker:
162
self._request_streams[stream.request_id] = stream
163
new_requests.append(new_request)
164
165
+ self.new_requests_event.clear()
166
+
167
return new_requests, finished_requests
168
169
+ async def wait_for_new_requests(self):
170
+ await self.new_requests_event.wait()
171
+
172
173
class _AsyncLLMEngine(LLMEngine):
174
"""Extension of LLMEngine to add async methods."""
@@ -251,9 +270,13 @@ class AsyncLLMEngine:
270
self.max_log_len = max_log_len
271
self.engine = self._init_engine(*args, **kwargs)
272
254
- self.request_tracker: RequestTracker = RequestTracker()
273
self.background_loop = None
274
+ # We need to keep a reference to unshielded
275
+ # task as well to prevent it from being garbage
276
+ # collected
277
+ self._background_loop_unshielded = None
278
self.start_engine_loop = start_engine_loop
279
+ self._request_tracker = RequestTracker()
280
281
@property
282
def is_running(self) -> bool:
@@ -264,11 +287,14 @@ class AsyncLLMEngine:
287
"""Start the background loop."""
288
if self.is_running:
289
raise RuntimeError("Background loop is already running.")
267
- self.background_loop = asyncio.get_event_loop().create_task(
268
- self.run_engine_loop())
269
- self.background_loop.add_done_callback(
290
+ self._request_tracker.init_event()
291
+
292
+ self._background_loop_unshielded = asyncio.get_event_loop(
293
+ ).create_task(self.run_engine_loop())
294
+ self._background_loop_unshielded.add_done_callback(
295
partial(_raise_exception_on_finish,
271
- request_tracker=self.request_tracker))
296
+ request_tracker=self._request_tracker))
297
+ self.background_loop = asyncio.shield(self._background_loop_unshielded)
298
299
def _init_engine(self, *args,
300
**kwargs) -> Union[_AsyncLLMEngine, "ray.ObjectRef"]:
@@ -280,11 +306,13 @@ class AsyncLLMEngine:
306
engine_class = ray.remote(num_gpus=1)(self._engine_class).remote
307
return engine_class(*args, **kwargs)
308
283
- async def engine_step(self):
284
- """Kick the engine to process the waiting requests."""
309
+ async def engine_step(self) -> bool:
310
+ """Kick the engine to process the waiting requests.
311
+
312
+ Returns True if there are in-progress requests."""
313
314
new_requests, finished_requests = (
287
- self.request_tracker.get_new_and_finished_requests())
315
+ self._request_tracker.get_new_and_finished_requests())
316
317
for new_request in new_requests:
318
# Add the request into the vLLM engine's waiting queue.
@@ -304,9 +332,11 @@ class AsyncLLMEngine:
332
333
# Put the outputs into the corresponding streams.
334
for request_output in request_outputs:
307
- self.request_tracker.process_request_output(
335
+ self._request_tracker.process_request_output(
336
request_output, verbose=self.log_requests)
337
338
+ return len(request_outputs) > 0
339
+
340
async def _engine_abort(self, request_ids: Iterable[str]):
341
if self.engine_use_ray:
342
await self.engine.abort_request.remote(request_ids)
@@ -314,8 +344,12 @@ class AsyncLLMEngine:
344
self.engine.abort_request(request_ids)
345
346
async def run_engine_loop(self):
347
+ # Initialize the RequestTracker here so it uses the right event loop.
348
+ has_requests_in_progress = False
349
while True:
318
- await self.engine_step()
350
+ if not has_requests_in_progress:
351
+ await self._request_tracker.wait_for_new_requests()
352
+ has_requests_in_progress = await self.engine_step()
353
await asyncio.sleep(0)
354
355
async def add_request(
@@ -350,7 +384,7 @@ class AsyncLLMEngine:
384
"error that caused the background loop to stop "
385
"(AsyncEngineDeadError).")
386
353
- stream = self.request_tracker.add_request(
387
+ stream = self._request_tracker.add_request(
388
request_id,
389
prompt=prompt,
390
sampling_params=sampling_params,
@@ -428,8 +462,8 @@ class AsyncLLMEngine:
462
Args:
463
request_id: The unique id of the request.
464
"""
431
- self.request_tracker.abort_request(request_id,
432
- verbose=self.log_requests)
465
+ self._request_tracker.abort_request(request_id,
466
+ verbose=self.log_requests)
467
468
async def get_model_config(self) -> ModelConfig:
469
"""Get the model configuration of the vLLM engine."""