A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 19:30 UTC
[Core] Streamline stream termination in `AsyncLLMEngine` (#7336)
Nick Hill committed
Aug 9, 2024 at 00:06 UTC
b4e9528f9569d6eb8c29624771a4058fe794cb5a
2 files changed
+26
-21
tests/async_engine/test_request_tracker.py
+4
-2
@@ -47,8 +47,10 @@ async def test_request_tracker():
47
assert tracker.new_requests_event.is_set()
48
await tracker.wait_for_new_requests()
49
new, aborted = tracker.get_new_and_aborted_requests()
50
- assert len(aborted) == 1
51
- assert "4" in aborted
50
+ # aborted new requests will cancel each other out -
51
+ # there's no need for them to propagate into the
52
+ # engine
53
+ assert not aborted
54
assert not new
55
assert stream_4.finished
56
vllm/engine/async_llm_engine.py
+22
-19
@@ -85,11 +85,14 @@ class AsyncStream:
85
return
86
self._queue.put_nowait(item)
87
88
- def finish(self, cancelled: bool = False) -> None:
88
+ def finish(
89
+ self,
90
+ exception: Optional[Union[BaseException, Type[BaseException]]] = None,
91
+ ) -> None:
92
if not self._finished:
93
self._finished = True
94
self._queue.put_nowait(
92
- asyncio.CancelledError if cancelled else STOP_ITERATION)
95
+ exception if exception is not None else STOP_ITERATION)
96
97
@property
98
def finished(self) -> bool:
@@ -133,14 +136,12 @@ class RequestTracker:
136
"""Propagate an exception to request streams
137
(all if request_id is None)."""
138
if request_id is not None:
136
- self._request_streams[request_id].put(exc)
137
- self.abort_request(request_id)
139
+ self.abort_request(request_id, exception=exc)
140
else:
139
- # NB: list() used here because self.abort_request pops the stream
141
+ # NB: tuple() used here because self.abort_request pops the stream
142
# out of self._request_streams, so we can't iterate on it directly
141
- for rid, stream in list(self._request_streams.items()):
142
- stream.put(exc)
143
- self.abort_request(rid)
143
+ for rid in tuple(self._request_streams.keys()):
144
+ self.abort_request(rid, exception=exc)
145
146
def process_request_output(self,
147
request_output: Union[RequestOutput,
@@ -167,14 +168,13 @@ class RequestTracker:
168
169
def process_exception(self,
170
request_id: str,
170
- exception: Exception,
171
+ exception: BaseException,
172
*,
173
verbose: bool = False) -> None:
174
"""Propagate an exception from the engine."""
174
- self._request_streams[request_id].put(exception)
175
if verbose:
176
logger.info("Finished request %s.", request_id)
177
- self.abort_request(request_id)
177
+ self.abort_request(request_id, exception=exception)
178
179
def add_request(self,
180
request_id: str,
@@ -203,7 +203,8 @@ class RequestTracker:
203
def abort_request(self,
204
request_id: str,
205
*,
206
- cancelled: bool = False,
206
+ exception: Optional[Union[BaseException,
207
+ Type[BaseException]]] = None,
208
verbose: bool = False) -> None:
209
"""Abort a request during next background loop iteration."""
210
if verbose:
@@ -213,7 +214,7 @@ class RequestTracker:
214
215
stream = self._request_streams.pop(request_id, None)
216
if stream is not None:
216
- stream.finish(cancelled=cancelled)
217
+ stream.finish(exception=exception)
218
219
def get_new_and_aborted_requests(self) -> Tuple[List[Dict], Set[str]]:
220
"""Get the new requests and finished requests to be
@@ -227,12 +228,14 @@ class RequestTracker:
228
229
while not self._new_requests.empty():
230
stream, new_request = self._new_requests.get_nowait()
230
- if stream.request_id in finished_requests:
231
+ request_id = stream.request_id
232
+ if request_id in finished_requests:
233
# The request has already been aborted.
232
- stream.finish(cancelled=True)
233
- continue
234
- self._request_streams[stream.request_id] = stream
235
- new_requests.append(new_request)
234
+ stream.finish(asyncio.CancelledError)
235
+ finished_requests.discard(request_id)
236
+ else:
237
+ self._request_streams[request_id] = stream
238
+ new_requests.append(new_request)
239
240
return new_requests, finished_requests
241
@@ -1015,7 +1018,7 @@ class AsyncLLMEngine:
1018
request_id: The unique id of the request.
1019
"""
1020
self._request_tracker.abort_request(request_id,
1018
- cancelled=True,
1021
+ exception=asyncio.CancelledError,
1022
verbose=self.log_requests)
1023
1024
async def get_model_config(self) -> ModelConfig: