@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / ff36139ffc
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 06:45 UTC

Remove AsyncLLMEngine busy loop, shield background task (#1059)

Antoni Baum committed Sep 17, 2023 at 00:29 UTC ff36139ffc66294c19b503c1e52dc42c2cd265f6
4 files changed +154 -18
requirements-dev.txt
+1
@@ -11,3 +11,4 @@ types-setuptools
11 # testing
12 pytest
13 pytest-forked
14 +pytest-asyncio
tests/async_engine/test_async_llm_engine.py new
+80
@@ -0,0 +1,80 @@
1 +import asyncio
2 +from dataclasses import dataclass
3 +
4 +import pytest
5 +
6 +from vllm.engine.async_llm_engine import AsyncLLMEngine
7 +
8 +
9 +@dataclass
10 +class RequestOutput:
11 + request_id: int
12 + finished: bool = False
13 +
14 +
15 +class MockEngine:
16 +
17 + def __init__(self):
18 + self.step_calls = 0
19 + self.add_request_calls = 0
20 + self.abort_request_calls = 0
21 + self.request_id = None
22 +
23 + async def step_async(self):
24 + self.step_calls += 1
25 + return [RequestOutput(
26 + request_id=self.request_id)] if self.request_id else []
27 +
28 + def generate(self, request_id):
29 + self.request_id = request_id
30 +
31 + def stop_generating(self):
32 + self.request_id = None
33 +
34 + def add_request(self, **kwargs):
35 + self.add_request_calls += 1
36 + return
37 +
38 + def abort_request(self, request_id):
39 + self.abort_request_calls += 1
40 + return
41 +
42 +
43 +class MockAsyncLLMEngine(AsyncLLMEngine):
44 +
45 + def _init_engine(self, *args, **kwargs):
46 + return MockEngine()
47 +
48 +
49 +@pytest.mark.asyncio
50 +async def test_new_requests_event():
51 + engine = MockAsyncLLMEngine(worker_use_ray=False, engine_use_ray=False)
52 + engine.start_background_loop()
53 + await asyncio.sleep(0.01)
54 + assert engine.engine.step_calls == 0
55 +
56 + await engine.add_request("1", "", None)
57 + await asyncio.sleep(0.01)
58 + assert engine.engine.add_request_calls == 1
59 + assert engine.engine.step_calls == 1
60 +
61 + await engine.add_request("2", "", None)
62 + engine.engine.generate("2")
63 + await asyncio.sleep(0)
64 + assert engine.engine.add_request_calls == 2
65 + assert engine.engine.step_calls == 2
66 + await asyncio.sleep(0)
67 + assert engine.engine.step_calls == 3
68 + engine.engine.stop_generating()
69 + await asyncio.sleep(0)
70 + assert engine.engine.step_calls == 4
71 + await asyncio.sleep(0)
72 + assert engine.engine.step_calls == 4
73 +
74 + await engine.add_request("3", "", None)
75 + await asyncio.sleep(0.01)
76 + assert engine.engine.add_request_calls == 3
77 + assert engine.engine.step_calls == 5
78 + await asyncio.sleep(0.01)
79 + assert engine.engine.add_request_calls == 3
80 + assert engine.engine.step_calls == 5
tests/async_engine/test_request_tracker.py
+21
@@ -4,10 +4,25 @@ from vllm.engine.async_llm_engine import RequestTracker
4 from vllm.outputs import RequestOutput
5
6
7 +class DummyEvent:
8 +
9 + def __init__(self):
10 + self._flag = False
11 +
12 + def set(self):
13 + self._flag = True
14 +
15 + def clear(self):
16 + self._flag = False
17 +
18 +
19 def test_request_tracker():
20 tracker = RequestTracker()
21 + tracker.new_requests_event = DummyEvent()
22 stream_1 = tracker.add_request("1")
23 + assert tracker.new_requests_event._flag
24 new, finished = tracker.get_new_and_finished_requests()
25 + assert not tracker.new_requests_event._flag
26 assert len(new) == 1
27 assert new[0]["request_id"] == "1"
28 assert not finished
@@ -15,7 +30,9 @@ def test_request_tracker():
30
31 stream_2 = tracker.add_request("2")
32 stream_3 = tracker.add_request("3")
33 + assert tracker.new_requests_event._flag
34 new, finished = tracker.get_new_and_finished_requests()
35 + assert not tracker.new_requests_event._flag
36 assert len(new) == 2
37 assert new[0]["request_id"] == "2"
38 assert new[1]["request_id"] == "3"
@@ -26,6 +43,7 @@ def test_request_tracker():
43 # request_ids must be unique
44 with pytest.raises(KeyError):
45 tracker.add_request("1")
46 + assert not tracker.new_requests_event._flag
47
48 tracker.abort_request("1")
49 new, finished = tracker.get_new_and_finished_requests()
@@ -36,6 +54,7 @@ def test_request_tracker():
54
55 stream_4 = tracker.add_request("4")
56 tracker.abort_request("4")
57 + assert tracker.new_requests_event._flag
58 new, finished = tracker.get_new_and_finished_requests()
59 assert len(finished) == 1
60 assert "4" in finished
@@ -43,9 +62,11 @@ def test_request_tracker():
62 assert stream_4.finished
63
64 stream_5 = tracker.add_request("5")
65 + assert tracker.new_requests_event._flag
66 tracker.process_request_output(
67 RequestOutput("2", "output", [], [], finished=True))
68 new, finished = tracker.get_new_and_finished_requests()
69 + assert not tracker.new_requests_event._flag
70 assert len(finished) == 1
71 assert "2" in finished
72 assert len(new) == 1
vllm/engine/async_llm_engine.py
+52 -18
@@ -1,7 +1,8 @@
1 import asyncio
2 import time
3 from functools import partial
4 -from typing import Any, Dict, Iterable, List, Optional, Set, Tuple, Type, Union
4 +from typing import (Any, Dict, Iterable, List, Optional, Set, Tuple, Type,
5 + Union)
6
7 from vllm.config import ModelConfig
8 from vllm.engine.arg_utils import AsyncEngineArgs
@@ -78,14 +79,24 @@ class RequestTracker:
79 self._finished_requests: asyncio.Queue[str] = asyncio.Queue()
80 self._new_requests: asyncio.Queue[Tuple[AsyncStream,
81 dict]] = asyncio.Queue()
82 + self.new_requests_event = None
83
84 def __contains__(self, item):
85 return item in self._request_streams
86
85 - def propagate_exception(self, exc: Exception) -> None:
86 - """Propagate an exception to all request streams."""
87 - for stream in self._request_streams.values():
88 - stream.put(exc)
87 + def init_event(self):
88 + self.new_requests_event = asyncio.Event()
89 +
90 + def propagate_exception(self,
91 + exc: Exception,
92 + request_id: Optional[str] = None) -> None:
93 + """Propagate an exception to request streams
94 + (all if request_id is None)."""
95 + if request_id is not None:
96 + self._request_streams[request_id].put(exc)
97 + else:
98 + for stream in self._request_streams.values():
99 + stream.put(exc)
100
101 def process_request_output(self,
102 request_output: RequestOutput,
@@ -112,6 +123,9 @@ class RequestTracker:
123 "request_id": request_id,
124 **engine_add_request_kwargs
125 }))
126 +
127 + self.new_requests_event.set()
128 +
129 return stream
130
131 def abort_request(self, request_id: str, *, verbose: bool = False) -> None:
@@ -148,8 +162,13 @@ class RequestTracker:
162 self._request_streams[stream.request_id] = stream
163 new_requests.append(new_request)
164
165 + self.new_requests_event.clear()
166 +
167 return new_requests, finished_requests
168
169 + async def wait_for_new_requests(self):
170 + await self.new_requests_event.wait()
171 +
172
173 class _AsyncLLMEngine(LLMEngine):
174 """Extension of LLMEngine to add async methods."""
@@ -251,9 +270,13 @@ class AsyncLLMEngine:
270 self.max_log_len = max_log_len
271 self.engine = self._init_engine(*args, **kwargs)
272
254 - self.request_tracker: RequestTracker = RequestTracker()
273 self.background_loop = None
274 + # We need to keep a reference to unshielded
275 + # task as well to prevent it from being garbage
276 + # collected
277 + self._background_loop_unshielded = None
278 self.start_engine_loop = start_engine_loop
279 + self._request_tracker = RequestTracker()
280
281 @property
282 def is_running(self) -> bool:
@@ -264,11 +287,14 @@ class AsyncLLMEngine:
287 """Start the background loop."""
288 if self.is_running:
289 raise RuntimeError("Background loop is already running.")
267 - self.background_loop = asyncio.get_event_loop().create_task(
268 - self.run_engine_loop())
269 - self.background_loop.add_done_callback(
290 + self._request_tracker.init_event()
291 +
292 + self._background_loop_unshielded = asyncio.get_event_loop(
293 + ).create_task(self.run_engine_loop())
294 + self._background_loop_unshielded.add_done_callback(
295 partial(_raise_exception_on_finish,
271 - request_tracker=self.request_tracker))
296 + request_tracker=self._request_tracker))
297 + self.background_loop = asyncio.shield(self._background_loop_unshielded)
298
299 def _init_engine(self, *args,
300 **kwargs) -> Union[_AsyncLLMEngine, "ray.ObjectRef"]:
@@ -280,11 +306,13 @@ class AsyncLLMEngine:
306 engine_class = ray.remote(num_gpus=1)(self._engine_class).remote
307 return engine_class(*args, **kwargs)
308
283 - async def engine_step(self):
284 - """Kick the engine to process the waiting requests."""
309 + async def engine_step(self) -> bool:
310 + """Kick the engine to process the waiting requests.
311 +
312 + Returns True if there are in-progress requests."""
313
314 new_requests, finished_requests = (
287 - self.request_tracker.get_new_and_finished_requests())
315 + self._request_tracker.get_new_and_finished_requests())
316
317 for new_request in new_requests:
318 # Add the request into the vLLM engine's waiting queue.
@@ -304,9 +332,11 @@ class AsyncLLMEngine:
332
333 # Put the outputs into the corresponding streams.
334 for request_output in request_outputs:
307 - self.request_tracker.process_request_output(
335 + self._request_tracker.process_request_output(
336 request_output, verbose=self.log_requests)
337
338 + return len(request_outputs) > 0
339 +
340 async def _engine_abort(self, request_ids: Iterable[str]):
341 if self.engine_use_ray:
342 await self.engine.abort_request.remote(request_ids)
@@ -314,8 +344,12 @@ class AsyncLLMEngine:
344 self.engine.abort_request(request_ids)
345
346 async def run_engine_loop(self):
347 + # Initialize the RequestTracker here so it uses the right event loop.
348 + has_requests_in_progress = False
349 while True:
318 - await self.engine_step()
350 + if not has_requests_in_progress:
351 + await self._request_tracker.wait_for_new_requests()
352 + has_requests_in_progress = await self.engine_step()
353 await asyncio.sleep(0)
354
355 async def add_request(
@@ -350,7 +384,7 @@ class AsyncLLMEngine:
384 "error that caused the background loop to stop "
385 "(AsyncEngineDeadError).")
386
353 - stream = self.request_tracker.add_request(
387 + stream = self._request_tracker.add_request(
388 request_id,
389 prompt=prompt,
390 sampling_params=sampling_params,
@@ -428,8 +462,8 @@ class AsyncLLMEngine:
462 Args:
463 request_id: The unique id of the request.
464 """
431 - self.request_tracker.abort_request(request_id,
432 - verbose=self.log_requests)
465 + self._request_tracker.abort_request(request_id,
466 + verbose=self.log_requests)
467
468 async def get_model_config(self) -> ModelConfig:
469 """Get the model configuration of the vLLM engine."""