A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 04:30 UTC
[V1][3/N] API Server: Reduce Task Switching + Handle Abort Properly (#11534)
Robert Shaw committed
Dec 27, 2024 at 00:19 UTC
1b875a0ef3767a7da7507943f46e4a53d3f552c9
3 files changed
+63
-153
vllm/v1/engine/async_llm.py
+62
-97
@@ -9,14 +9,13 @@ from vllm.inputs import INPUT_REGISTRY, InputRegistry, PromptType
9
from vllm.inputs.preprocess import InputPreprocessor
10
from vllm.logger import init_logger
11
from vllm.lora.request import LoRARequest
12
-from vllm.outputs import PoolingRequestOutput, RequestOutput
12
+from vllm.outputs import RequestOutput
13
from vllm.pooling_params import PoolingParams
14
from vllm.prompt_adapter.request import PromptAdapterRequest
15
from vllm.sampling_params import SamplingParams
16
from vllm.transformers_utils.tokenizer import AnyTokenizer
17
from vllm.transformers_utils.tokenizer_group import init_tokenizer_from_configs
18
from vllm.usage.usage_lib import UsageContext
19
-from vllm.v1.engine.async_stream import AsyncStream
19
from vllm.v1.engine.core_client import EngineCoreClient
20
from vllm.v1.engine.detokenizer import Detokenizer
21
from vllm.v1.engine.processor import Processor
@@ -54,10 +53,8 @@ class AsyncLLM(EngineClient):
53
lora_config=vllm_config.lora_config)
54
self.tokenizer.ping()
55
57
- # Request streams (map of request_id -> AsyncStream).
58
- self.request_streams: Dict[str, AsyncStream] = {}
59
- # List of cancelled request ids to be aborted.
60
- self.client_aborted_requests: List[str] = []
56
+ # Request streams (map of request_id -> queue).
57
+ self.rid_to_queue: Dict[str, asyncio.Queue] = {}
58
59
# Processor (converts Inputs --> EngineCoreRequests).
60
self.processor = Processor(
@@ -153,14 +150,13 @@ class AsyncLLM(EngineClient):
150
trace_headers: Optional[Mapping[str, str]] = None,
151
prompt_adapter_request: Optional[PromptAdapterRequest] = None,
152
priority: int = 0,
156
- ) -> AsyncGenerator[Union[RequestOutput, PoolingRequestOutput], None]:
153
+ ) -> asyncio.Queue[RequestOutput]:
154
"""Add new request to the AsyncLLM."""
155
159
- if self.detokenizer.is_request_active(request_id):
160
- raise ValueError(f"Request {request_id} already exists.")
161
-
162
- # 1) Create a new AsyncStream for the request.
163
- stream = self._add_request_to_streams(request_id)
156
+ # 1) Create a new output queue for the request.
157
+ if request_id in self.rid_to_queue:
158
+ raise ValueError(f"Request id {request_id} already running.")
159
+ self.rid_to_queue[request_id] = asyncio.Queue()
160
161
# 2) Convert input --> DetokenizerRequest / EngineCoreRequest.
162
detokenizer_req, engine_core_req = self.processor.process_inputs(
@@ -173,8 +169,10 @@ class AsyncLLM(EngineClient):
169
# 4) Add the EngineCoreRequest to EngineCore (separate process).
170
await self.engine_core.add_request_async(engine_core_req)
171
176
- # 5) Return the generator.
177
- return stream.generator()
172
+ if self.log_requests:
173
+ logger.info("Added request %s.", request_id)
174
+
175
+ return self.rid_to_queue[request_id]
176
177
# TODO: we should support multiple prompts in one call, as you
178
# can do with LLM.generate. So that for multi-prompt completion
@@ -194,7 +192,7 @@ class AsyncLLM(EngineClient):
192
"""
193
Main function called by the API server to kick off a request
194
* 1) Making an AsyncStream corresponding to the Request.
197
- # 2) Processing the Input.
195
+ * 2) Processing the Input.
196
* 3) Adding the Request to the Detokenizer.
197
* 4) Adding the Request to the EngineCore (separate process).
198
@@ -206,14 +204,15 @@ class AsyncLLM(EngineClient):
204
returning the RequestOutput back to the caller.
205
"""
206
209
- # We start the output_handler on the first call to generate() so that
210
- # we can call __init__ before the event loop starts, which enables us
211
- # to handle startup failure gracefully in the OpenAI server.
212
- if self.output_handler is None:
213
- self.output_handler = asyncio.create_task(
214
- self._run_output_handler())
215
-
216
- async for output in await self.add_request(
207
+ try:
208
+ # We start the output_handler on the first call to generate() so
209
+ # we can call __init__ before the event loop, which enables us
210
+ # to handle startup failure gracefully in the OpenAI server.
211
+ if self.output_handler is None:
212
+ self.output_handler = asyncio.create_task(
213
+ self._run_output_handler())
214
+
215
+ q = await self.add_request(
216
request_id,
217
prompt,
218
sampling_params,
@@ -221,79 +220,42 @@ class AsyncLLM(EngineClient):
220
trace_headers=trace_headers,
221
prompt_adapter_request=prompt_adapter_request,
222
priority=priority,
224
- ):
225
- yield output
226
-
227
- def _finish_stream(self, request_id: str):
228
- stream = self.request_streams.pop(request_id, None)
229
- if stream is not None:
230
- stream.finish()
231
-
232
- def _add_request_to_streams(
233
- self,
234
- request_id: str,
235
- ) -> AsyncStream:
236
-
237
- if request_id in self.request_streams:
238
- raise ValueError(f"Request id {request_id} already running.")
239
-
240
- # Avoid streams having circular ref to parent AsyncLLM object.
241
- aborted_reqs = self.client_aborted_requests
242
- stream = AsyncStream(request_id, aborted_reqs.append)
243
- self.request_streams[request_id] = stream
244
-
245
- if self.log_requests:
246
- logger.info("Added request %s.", request_id)
223
+ )
224
248
- return stream
249
-
250
- async def _process_cancellations(self) -> None:
251
- """
252
- Process requests cancelled from user disconnecting.
253
-
254
- When a client disconnects, AsyncStream._cancel() is called.
255
- We passed a callback to AsyncStream(), which appends to
256
- self.client_aborted_requests.
257
-
258
- As a result, if any requests are canceled from the user side
259
- the request_id will show up in self.client_aborted_requests.
260
- """
261
-
262
- # Avoid streams having circular ref to parent AsyncLLM object.
263
- if not self.client_aborted_requests:
264
- return
265
- reqs_to_abort = self.client_aborted_requests.copy()
266
- self.client_aborted_requests.clear()
267
-
268
- # Remove from Detokenizer.
269
- self.detokenizer.abort_requests(reqs_to_abort)
270
-
271
- # Remove from RequestStreams.
272
- for request_id in reqs_to_abort:
273
- if self.log_requests:
274
- logger.info("User-cancelled request %s.", request_id)
275
- self._finish_stream(request_id)
276
-
277
- # Remove from EngineCore.
278
- await self.engine_core.abort_requests_async(reqs_to_abort)
225
+ # The output_handler task pushes items into the queue.
226
+ # This task pulls from the queue and yields to caller.
227
+ while True:
228
+ # Note: drain queue without await if possible (avoids
229
+ # task switching under load which helps performance).
230
+ out = q.get_nowait() if q.qsize() > 0 else await q.get()
231
+
232
+ # Note: both Detokenizer and EngineCore handle their
233
+ # own request cleanup based on finished.
234
+ if out.finished:
235
+ del self.rid_to_queue[request_id]
236
+ yield out
237
+ break
238
+
239
+ yield out
240
+
241
+ # If the request is disconnected by the client, the
242
+ # generate() task will be canceled. So, we abort the
243
+ # request if we end up here.
244
+ except asyncio.CancelledError:
245
+ await self.abort(request_id)
246
+ raise
247
248
def _process_request_outputs(self, request_outputs: List[RequestOutput]):
281
- """Process outputs by putting them into per-request AsyncStreams."""
249
+ """Process outputs by putting them into per-request queues."""
250
251
for request_output in request_outputs:
252
request_id = request_output.request_id
285
- assert request_id in self.request_streams
253
287
- # Each request in the API server pulls from the per-request stream.
288
- stream = self.request_streams.get(request_id)
289
- if stream is not None:
290
- stream.put(request_output)
291
-
292
- # If finished, remove from the tracker.
293
- if request_output.finished:
294
- if self.log_requests:
295
- logger.info("Finished request %s.", request_id)
296
- self._finish_stream(request_id)
254
+ # Note: it is possible a request was aborted and removed from
255
+ # the state due to client cancellations, so if we encounter a
256
+ # request id not in the state, we skip.
257
+ if request_id in self.rid_to_queue:
258
+ self.rid_to_queue[request_id].put_nowait(request_output)
259
260
async def _run_output_handler(self):
261
"""Background loop: pulls from EngineCore and pushes to AsyncStreams."""
@@ -306,24 +268,27 @@ class AsyncLLM(EngineClient):
268
# 2) Detokenize based on the output.
269
request_outputs, reqs_to_abort = self.detokenizer.step(outputs)
270
309
- # 3) Put the RequestOutputs into the per-request AsyncStreams.
271
+ # 3) Put the RequestOutputs into the per-request queues.
272
self._process_request_outputs(request_outputs)
273
274
# 4) Abort any requests that finished due to stop strings.
275
await self.engine_core.abort_requests_async(reqs_to_abort)
276
315
- # 5) Abort any requests due to client cancellations.
316
- await self._process_cancellations()
317
-
277
except BaseException as e:
278
logger.error(e)
279
raise e
280
322
- # TODO: can we eliminate these?
323
-
281
async def abort(self, request_id: str) -> None:
325
- # Note: Who Calls this? I dont think this is actually used.
326
- raise ValueError("Not Supported on V1 yet.")
282
+ """Abort RequestId in self, detokenizer, and engine core."""
283
+
284
+ request_ids = [request_id]
285
+ await self.engine_core.abort_requests_async(request_ids)
286
+ self.detokenizer.abort_requests(request_ids)
287
+
288
+ # If a request finishes while we await then the request_id
289
+ # will be removed from the tracked queues before we get here.
290
+ if request_id in self.rid_to_queue:
291
+ del self.rid_to_queue[request_id]
292
293
def encode(
294
self,
vllm/v1/engine/async_stream.py
deleted
-55
@@ -1,55 +0,0 @@
1
-import asyncio
2
-from typing import Any, AsyncGenerator, Callable, Optional, Type, Union
3
-
4
-from vllm.outputs import PoolingRequestOutput, RequestOutput
5
-
6
-
7
-class AsyncStream:
8
- """A stream of RequestOutputs or PoolingRequestOutputs for a request
9
- that can be iterated over asynchronously via an async generator."""
10
-
11
- STOP_ITERATION = Exception() # Sentinel
12
-
13
- def __init__(self, request_id: str, cancel: Callable[[str], None]) -> None:
14
- self.request_id = request_id
15
- self._cancel = cancel
16
- self._queue: asyncio.Queue = asyncio.Queue()
17
- self._finished = False
18
-
19
- def put(self, item: Union[RequestOutput, PoolingRequestOutput,
20
- Exception]) -> None:
21
- if not self._finished:
22
- self._queue.put_nowait(item)
23
-
24
- def finish(
25
- self,
26
- exception: Optional[Union[BaseException, Type[BaseException]]] = None,
27
- ) -> None:
28
- if not self._finished:
29
- self._finished = True
30
- self._queue.put_nowait(exception if self._is_raisable(exception)
31
- else AsyncStream.STOP_ITERATION)
32
-
33
- async def generator(
34
- self
35
- ) -> AsyncGenerator[Union[RequestOutput, PoolingRequestOutput], None]:
36
- finished = False
37
- try:
38
- while True:
39
- result = await self._queue.get()
40
- if self._is_raisable(result):
41
- finished = True
42
- if result == AsyncStream.STOP_ITERATION:
43
- return
44
- raise result
45
- yield result
46
- finally:
47
- self._finished = True
48
- if not finished:
49
- self._cancel(self.request_id)
50
-
51
- @staticmethod
52
- def _is_raisable(value: Any):
53
- return isinstance(value, BaseException) or \
54
- (isinstance(value, type) and \
55
- issubclass(value, BaseException))
vllm/v1/engine/core.py
+1
-1
@@ -32,7 +32,7 @@ logger = init_logger(__name__)
32
33
POLLING_TIMEOUT_MS = 5000
34
POLLING_TIMEOUT_S = POLLING_TIMEOUT_MS // 1000
35
-LOGGING_TIME_S = 5000
35
+LOGGING_TIME_S = POLLING_TIMEOUT_S
36
37
38
class EngineCore: