@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / b4e9528f95
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 19:30 UTC

[Core] Streamline stream termination in `AsyncLLMEngine` (#7336)

Nick Hill committed Aug 9, 2024 at 00:06 UTC b4e9528f9569d6eb8c29624771a4058fe794cb5a
2 files changed +26 -21
tests/async_engine/test_request_tracker.py
+4 -2
@@ -47,8 +47,10 @@ async def test_request_tracker():
47 assert tracker.new_requests_event.is_set()
48 await tracker.wait_for_new_requests()
49 new, aborted = tracker.get_new_and_aborted_requests()
50 - assert len(aborted) == 1
51 - assert "4" in aborted
50 + # aborted new requests will cancel each other out -
51 + # there's no need for them to propagate into the
52 + # engine
53 + assert not aborted
54 assert not new
55 assert stream_4.finished
56
vllm/engine/async_llm_engine.py
+22 -19
@@ -85,11 +85,14 @@ class AsyncStream:
85 return
86 self._queue.put_nowait(item)
87
88 - def finish(self, cancelled: bool = False) -> None:
88 + def finish(
89 + self,
90 + exception: Optional[Union[BaseException, Type[BaseException]]] = None,
91 + ) -> None:
92 if not self._finished:
93 self._finished = True
94 self._queue.put_nowait(
92 - asyncio.CancelledError if cancelled else STOP_ITERATION)
95 + exception if exception is not None else STOP_ITERATION)
96
97 @property
98 def finished(self) -> bool:
@@ -133,14 +136,12 @@ class RequestTracker:
136 """Propagate an exception to request streams
137 (all if request_id is None)."""
138 if request_id is not None:
136 - self._request_streams[request_id].put(exc)
137 - self.abort_request(request_id)
139 + self.abort_request(request_id, exception=exc)
140 else:
139 - # NB: list() used here because self.abort_request pops the stream
141 + # NB: tuple() used here because self.abort_request pops the stream
142 # out of self._request_streams, so we can't iterate on it directly
141 - for rid, stream in list(self._request_streams.items()):
142 - stream.put(exc)
143 - self.abort_request(rid)
143 + for rid in tuple(self._request_streams.keys()):
144 + self.abort_request(rid, exception=exc)
145
146 def process_request_output(self,
147 request_output: Union[RequestOutput,
@@ -167,14 +168,13 @@ class RequestTracker:
168
169 def process_exception(self,
170 request_id: str,
170 - exception: Exception,
171 + exception: BaseException,
172 *,
173 verbose: bool = False) -> None:
174 """Propagate an exception from the engine."""
174 - self._request_streams[request_id].put(exception)
175 if verbose:
176 logger.info("Finished request %s.", request_id)
177 - self.abort_request(request_id)
177 + self.abort_request(request_id, exception=exception)
178
179 def add_request(self,
180 request_id: str,
@@ -203,7 +203,8 @@ class RequestTracker:
203 def abort_request(self,
204 request_id: str,
205 *,
206 - cancelled: bool = False,
206 + exception: Optional[Union[BaseException,
207 + Type[BaseException]]] = None,
208 verbose: bool = False) -> None:
209 """Abort a request during next background loop iteration."""
210 if verbose:
@@ -213,7 +214,7 @@ class RequestTracker:
214
215 stream = self._request_streams.pop(request_id, None)
216 if stream is not None:
216 - stream.finish(cancelled=cancelled)
217 + stream.finish(exception=exception)
218
219 def get_new_and_aborted_requests(self) -> Tuple[List[Dict], Set[str]]:
220 """Get the new requests and finished requests to be
@@ -227,12 +228,14 @@ class RequestTracker:
228
229 while not self._new_requests.empty():
230 stream, new_request = self._new_requests.get_nowait()
230 - if stream.request_id in finished_requests:
231 + request_id = stream.request_id
232 + if request_id in finished_requests:
233 # The request has already been aborted.
232 - stream.finish(cancelled=True)
233 - continue
234 - self._request_streams[stream.request_id] = stream
235 - new_requests.append(new_request)
234 + stream.finish(asyncio.CancelledError)
235 + finished_requests.discard(request_id)
236 + else:
237 + self._request_streams[request_id] = stream
238 + new_requests.append(new_request)
239
240 return new_requests, finished_requests
241
@@ -1015,7 +1018,7 @@ class AsyncLLMEngine:
1018 request_id: The unique id of the request.
1019 """
1020 self._request_tracker.abort_request(request_id,
1018 - cancelled=True,
1021 + exception=asyncio.CancelledError,
1022 verbose=self.log_requests)
1023
1024 async def get_model_config(self) -> ModelConfig: