@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 1b875a0ef3
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 04:30 UTC

[V1][3/N] API Server: Reduce Task Switching + Handle Abort Properly (#11534)

Robert Shaw committed Dec 27, 2024 at 00:19 UTC 1b875a0ef3767a7da7507943f46e4a53d3f552c9
3 files changed +63 -153
vllm/v1/engine/async_llm.py
+62 -97
@@ -9,14 +9,13 @@ from vllm.inputs import INPUT_REGISTRY, InputRegistry, PromptType
9 from vllm.inputs.preprocess import InputPreprocessor
10 from vllm.logger import init_logger
11 from vllm.lora.request import LoRARequest
12 -from vllm.outputs import PoolingRequestOutput, RequestOutput
12 +from vllm.outputs import RequestOutput
13 from vllm.pooling_params import PoolingParams
14 from vllm.prompt_adapter.request import PromptAdapterRequest
15 from vllm.sampling_params import SamplingParams
16 from vllm.transformers_utils.tokenizer import AnyTokenizer
17 from vllm.transformers_utils.tokenizer_group import init_tokenizer_from_configs
18 from vllm.usage.usage_lib import UsageContext
19 -from vllm.v1.engine.async_stream import AsyncStream
19 from vllm.v1.engine.core_client import EngineCoreClient
20 from vllm.v1.engine.detokenizer import Detokenizer
21 from vllm.v1.engine.processor import Processor
@@ -54,10 +53,8 @@ class AsyncLLM(EngineClient):
53 lora_config=vllm_config.lora_config)
54 self.tokenizer.ping()
55
57 - # Request streams (map of request_id -> AsyncStream).
58 - self.request_streams: Dict[str, AsyncStream] = {}
59 - # List of cancelled request ids to be aborted.
60 - self.client_aborted_requests: List[str] = []
56 + # Request streams (map of request_id -> queue).
57 + self.rid_to_queue: Dict[str, asyncio.Queue] = {}
58
59 # Processor (converts Inputs --> EngineCoreRequests).
60 self.processor = Processor(
@@ -153,14 +150,13 @@ class AsyncLLM(EngineClient):
150 trace_headers: Optional[Mapping[str, str]] = None,
151 prompt_adapter_request: Optional[PromptAdapterRequest] = None,
152 priority: int = 0,
156 - ) -> AsyncGenerator[Union[RequestOutput, PoolingRequestOutput], None]:
153 + ) -> asyncio.Queue[RequestOutput]:
154 """Add new request to the AsyncLLM."""
155
159 - if self.detokenizer.is_request_active(request_id):
160 - raise ValueError(f"Request {request_id} already exists.")
161 -
162 - # 1) Create a new AsyncStream for the request.
163 - stream = self._add_request_to_streams(request_id)
156 + # 1) Create a new output queue for the request.
157 + if request_id in self.rid_to_queue:
158 + raise ValueError(f"Request id {request_id} already running.")
159 + self.rid_to_queue[request_id] = asyncio.Queue()
160
161 # 2) Convert input --> DetokenizerRequest / EngineCoreRequest.
162 detokenizer_req, engine_core_req = self.processor.process_inputs(
@@ -173,8 +169,10 @@ class AsyncLLM(EngineClient):
169 # 4) Add the EngineCoreRequest to EngineCore (separate process).
170 await self.engine_core.add_request_async(engine_core_req)
171
176 - # 5) Return the generator.
177 - return stream.generator()
172 + if self.log_requests:
173 + logger.info("Added request %s.", request_id)
174 +
175 + return self.rid_to_queue[request_id]
176
177 # TODO: we should support multiple prompts in one call, as you
178 # can do with LLM.generate. So that for multi-prompt completion
@@ -194,7 +192,7 @@ class AsyncLLM(EngineClient):
192 """
193 Main function called by the API server to kick off a request
194 * 1) Making an AsyncStream corresponding to the Request.
197 - # 2) Processing the Input.
195 + * 2) Processing the Input.
196 * 3) Adding the Request to the Detokenizer.
197 * 4) Adding the Request to the EngineCore (separate process).
198
@@ -206,14 +204,15 @@ class AsyncLLM(EngineClient):
204 returning the RequestOutput back to the caller.
205 """
206
209 - # We start the output_handler on the first call to generate() so that
210 - # we can call __init__ before the event loop starts, which enables us
211 - # to handle startup failure gracefully in the OpenAI server.
212 - if self.output_handler is None:
213 - self.output_handler = asyncio.create_task(
214 - self._run_output_handler())
215 -
216 - async for output in await self.add_request(
207 + try:
208 + # We start the output_handler on the first call to generate() so
209 + # we can call __init__ before the event loop, which enables us
210 + # to handle startup failure gracefully in the OpenAI server.
211 + if self.output_handler is None:
212 + self.output_handler = asyncio.create_task(
213 + self._run_output_handler())
214 +
215 + q = await self.add_request(
216 request_id,
217 prompt,
218 sampling_params,
@@ -221,79 +220,42 @@ class AsyncLLM(EngineClient):
220 trace_headers=trace_headers,
221 prompt_adapter_request=prompt_adapter_request,
222 priority=priority,
224 - ):
225 - yield output
226 -
227 - def _finish_stream(self, request_id: str):
228 - stream = self.request_streams.pop(request_id, None)
229 - if stream is not None:
230 - stream.finish()
231 -
232 - def _add_request_to_streams(
233 - self,
234 - request_id: str,
235 - ) -> AsyncStream:
236 -
237 - if request_id in self.request_streams:
238 - raise ValueError(f"Request id {request_id} already running.")
239 -
240 - # Avoid streams having circular ref to parent AsyncLLM object.
241 - aborted_reqs = self.client_aborted_requests
242 - stream = AsyncStream(request_id, aborted_reqs.append)
243 - self.request_streams[request_id] = stream
244 -
245 - if self.log_requests:
246 - logger.info("Added request %s.", request_id)
223 + )
224
248 - return stream
249 -
250 - async def _process_cancellations(self) -> None:
251 - """
252 - Process requests cancelled from user disconnecting.
253 -
254 - When a client disconnects, AsyncStream._cancel() is called.
255 - We passed a callback to AsyncStream(), which appends to
256 - self.client_aborted_requests.
257 -
258 - As a result, if any requests are canceled from the user side
259 - the request_id will show up in self.client_aborted_requests.
260 - """
261 -
262 - # Avoid streams having circular ref to parent AsyncLLM object.
263 - if not self.client_aborted_requests:
264 - return
265 - reqs_to_abort = self.client_aborted_requests.copy()
266 - self.client_aborted_requests.clear()
267 -
268 - # Remove from Detokenizer.
269 - self.detokenizer.abort_requests(reqs_to_abort)
270 -
271 - # Remove from RequestStreams.
272 - for request_id in reqs_to_abort:
273 - if self.log_requests:
274 - logger.info("User-cancelled request %s.", request_id)
275 - self._finish_stream(request_id)
276 -
277 - # Remove from EngineCore.
278 - await self.engine_core.abort_requests_async(reqs_to_abort)
225 + # The output_handler task pushes items into the queue.
226 + # This task pulls from the queue and yields to caller.
227 + while True:
228 + # Note: drain queue without await if possible (avoids
229 + # task switching under load which helps performance).
230 + out = q.get_nowait() if q.qsize() > 0 else await q.get()
231 +
232 + # Note: both Detokenizer and EngineCore handle their
233 + # own request cleanup based on finished.
234 + if out.finished:
235 + del self.rid_to_queue[request_id]
236 + yield out
237 + break
238 +
239 + yield out
240 +
241 + # If the request is disconnected by the client, the
242 + # generate() task will be canceled. So, we abort the
243 + # request if we end up here.
244 + except asyncio.CancelledError:
245 + await self.abort(request_id)
246 + raise
247
248 def _process_request_outputs(self, request_outputs: List[RequestOutput]):
281 - """Process outputs by putting them into per-request AsyncStreams."""
249 + """Process outputs by putting them into per-request queues."""
250
251 for request_output in request_outputs:
252 request_id = request_output.request_id
285 - assert request_id in self.request_streams
253
287 - # Each request in the API server pulls from the per-request stream.
288 - stream = self.request_streams.get(request_id)
289 - if stream is not None:
290 - stream.put(request_output)
291 -
292 - # If finished, remove from the tracker.
293 - if request_output.finished:
294 - if self.log_requests:
295 - logger.info("Finished request %s.", request_id)
296 - self._finish_stream(request_id)
254 + # Note: it is possible a request was aborted and removed from
255 + # the state due to client cancellations, so if we encounter a
256 + # request id not in the state, we skip.
257 + if request_id in self.rid_to_queue:
258 + self.rid_to_queue[request_id].put_nowait(request_output)
259
260 async def _run_output_handler(self):
261 """Background loop: pulls from EngineCore and pushes to AsyncStreams."""
@@ -306,24 +268,27 @@ class AsyncLLM(EngineClient):
268 # 2) Detokenize based on the output.
269 request_outputs, reqs_to_abort = self.detokenizer.step(outputs)
270
309 - # 3) Put the RequestOutputs into the per-request AsyncStreams.
271 + # 3) Put the RequestOutputs into the per-request queues.
272 self._process_request_outputs(request_outputs)
273
274 # 4) Abort any requests that finished due to stop strings.
275 await self.engine_core.abort_requests_async(reqs_to_abort)
276
315 - # 5) Abort any requests due to client cancellations.
316 - await self._process_cancellations()
317 -
277 except BaseException as e:
278 logger.error(e)
279 raise e
280
322 - # TODO: can we eliminate these?
323 -
281 async def abort(self, request_id: str) -> None:
325 - # Note: Who Calls this? I dont think this is actually used.
326 - raise ValueError("Not Supported on V1 yet.")
282 + """Abort RequestId in self, detokenizer, and engine core."""
283 +
284 + request_ids = [request_id]
285 + await self.engine_core.abort_requests_async(request_ids)
286 + self.detokenizer.abort_requests(request_ids)
287 +
288 + # If a request finishes while we await then the request_id
289 + # will be removed from the tracked queues before we get here.
290 + if request_id in self.rid_to_queue:
291 + del self.rid_to_queue[request_id]
292
293 def encode(
294 self,
vllm/v1/engine/async_stream.py deleted
-55
@@ -1,55 +0,0 @@
1 -import asyncio
2 -from typing import Any, AsyncGenerator, Callable, Optional, Type, Union
3 -
4 -from vllm.outputs import PoolingRequestOutput, RequestOutput
5 -
6 -
7 -class AsyncStream:
8 - """A stream of RequestOutputs or PoolingRequestOutputs for a request
9 - that can be iterated over asynchronously via an async generator."""
10 -
11 - STOP_ITERATION = Exception() # Sentinel
12 -
13 - def __init__(self, request_id: str, cancel: Callable[[str], None]) -> None:
14 - self.request_id = request_id
15 - self._cancel = cancel
16 - self._queue: asyncio.Queue = asyncio.Queue()
17 - self._finished = False
18 -
19 - def put(self, item: Union[RequestOutput, PoolingRequestOutput,
20 - Exception]) -> None:
21 - if not self._finished:
22 - self._queue.put_nowait(item)
23 -
24 - def finish(
25 - self,
26 - exception: Optional[Union[BaseException, Type[BaseException]]] = None,
27 - ) -> None:
28 - if not self._finished:
29 - self._finished = True
30 - self._queue.put_nowait(exception if self._is_raisable(exception)
31 - else AsyncStream.STOP_ITERATION)
32 -
33 - async def generator(
34 - self
35 - ) -> AsyncGenerator[Union[RequestOutput, PoolingRequestOutput], None]:
36 - finished = False
37 - try:
38 - while True:
39 - result = await self._queue.get()
40 - if self._is_raisable(result):
41 - finished = True
42 - if result == AsyncStream.STOP_ITERATION:
43 - return
44 - raise result
45 - yield result
46 - finally:
47 - self._finished = True
48 - if not finished:
49 - self._cancel(self.request_id)
50 -
51 - @staticmethod
52 - def _is_raisable(value: Any):
53 - return isinstance(value, BaseException) or \
54 - (isinstance(value, type) and \
55 - issubclass(value, BaseException))
vllm/v1/engine/core.py
+1 -1
@@ -32,7 +32,7 @@ logger = init_logger(__name__)
32
33 POLLING_TIMEOUT_MS = 5000
34 POLLING_TIMEOUT_S = POLLING_TIMEOUT_MS // 1000
35 -LOGGING_TIME_S = 5000
35 +LOGGING_TIME_S = POLLING_TIMEOUT_S
36
37
38 class EngineCore: