@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 9a3f49ae07
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 19:30 UTC

[BugFix] Overhaul async request cancellation (#7111)

Nick Hill committed Aug 6, 2024 at 22:21 UTC 9a3f49ae07f9627db02b2ee377accb76b65d1d1e
11 files changed +226 -226
tests/async_engine/api_server_async_engine.py
+5 -4
@@ -1,5 +1,5 @@
1 """vllm.entrypoints.api_server with some extra logging for testing."""
2 -from typing import Any, Dict
2 +from typing import Any, Dict, Iterable
3
4 import uvicorn
5 from fastapi.responses import JSONResponse, Response
@@ -18,9 +18,10 @@ class AsyncLLMEngineWithStats(AsyncLLMEngine):
18 super().__init__(*args, **kwargs)
19 self._num_aborts = 0
20
21 - async def abort(self, request_id: str) -> None:
22 - await super().abort(request_id)
23 - self._num_aborts += 1
21 + async def _engine_abort(self, request_ids: Iterable[str]):
22 + ids = list(request_ids)
23 + self._num_aborts += len(ids)
24 + await super()._engine_abort(ids)
25
26 def testing_stats(self) -> Dict[str, Any]:
27 return {"num_aborted_requests": self._num_aborts}
tests/async_engine/test_request_tracker.py
+12 -13
@@ -10,23 +10,23 @@ async def test_request_tracker():
10 stream_1 = tracker.add_request("1")
11 assert tracker.new_requests_event.is_set()
12 await tracker.wait_for_new_requests()
13 - new, finished = tracker.get_new_and_finished_requests()
13 + new, aborted = tracker.get_new_and_aborted_requests()
14 assert not tracker.new_requests_event.is_set()
15 assert len(new) == 1
16 assert new[0]["request_id"] == "1"
17 - assert not finished
17 + assert not aborted
18 assert not stream_1.finished
19
20 stream_2 = tracker.add_request("2")
21 stream_3 = tracker.add_request("3")
22 assert tracker.new_requests_event.is_set()
23 await tracker.wait_for_new_requests()
24 - new, finished = tracker.get_new_and_finished_requests()
24 + new, aborted = tracker.get_new_and_aborted_requests()
25 assert not tracker.new_requests_event.is_set()
26 assert len(new) == 2
27 assert new[0]["request_id"] == "2"
28 assert new[1]["request_id"] == "3"
29 - assert not finished
29 + assert not aborted
30 assert not stream_2.finished
31 assert not stream_3.finished
32
@@ -36,9 +36,9 @@ async def test_request_tracker():
36 assert not tracker.new_requests_event.is_set()
37
38 tracker.abort_request("1")
39 - new, finished = tracker.get_new_and_finished_requests()
40 - assert len(finished) == 1
41 - assert "1" in finished
39 + new, aborted = tracker.get_new_and_aborted_requests()
40 + assert len(aborted) == 1
41 + assert "1" in aborted
42 assert not new
43 assert stream_1.finished
44
@@ -46,9 +46,9 @@ async def test_request_tracker():
46 tracker.abort_request("4")
47 assert tracker.new_requests_event.is_set()
48 await tracker.wait_for_new_requests()
49 - new, finished = tracker.get_new_and_finished_requests()
50 - assert len(finished) == 1
51 - assert "4" in finished
49 + new, aborted = tracker.get_new_and_aborted_requests()
50 + assert len(aborted) == 1
51 + assert "4" in aborted
52 assert not new
53 assert stream_4.finished
54
@@ -57,10 +57,9 @@ async def test_request_tracker():
57 tracker.process_request_output(
58 RequestOutput("2", "output", [], [], [], finished=True))
59 await tracker.wait_for_new_requests()
60 - new, finished = tracker.get_new_and_finished_requests()
60 + new, aborted = tracker.get_new_and_aborted_requests()
61 assert not tracker.new_requests_event.is_set()
62 - assert len(finished) == 1
63 - assert "2" in finished
62 + assert not aborted
63 assert len(new) == 1
64 assert new[0]["request_id"] == "5"
65 assert stream_2.finished
tests/test_utils.py
+3 -2
@@ -2,6 +2,7 @@ import asyncio
2 import os
3 import socket
4 import sys
5 +from functools import partial
6 from typing import (TYPE_CHECKING, Any, AsyncIterator, Awaitable, Protocol,
7 Tuple, TypeVar)
8
@@ -37,11 +38,11 @@ async def test_merge_async_iterators():
38 yield f"item from iterator {idx}"
39 await asyncio.sleep(0.1)
40 except asyncio.CancelledError:
40 - pass
41 + print(f"iterator {idx} cancelled")
42
43 iterators = [mock_async_iterator(i) for i in range(3)]
44 merged_iterator: AsyncIterator[Tuple[int, str]] = merge_async_iterators(
44 - *iterators)
45 + *iterators, is_cancelled=partial(asyncio.sleep, 0, result=False))
46
47 async def stream_output(generator: AsyncIterator[Tuple[int, str]]):
48 async for idx, output in generator:
vllm/engine/async_llm_engine.py
+67 -79
@@ -1,7 +1,7 @@
1 import asyncio
2 import time
3 from functools import partial
4 -from typing import (AsyncIterator, Callable, Dict, Iterable, List, Mapping,
4 +from typing import (AsyncGenerator, Callable, Dict, Iterable, List, Mapping,
5 Optional, Set, Tuple, Type, Union)
6
7 from transformers import PreTrainedTokenizer
@@ -62,12 +62,16 @@ def _log_task_completion(task: asyncio.Task,
62 "actual cause.") from e
63
64
65 +STOP_ITERATION = Exception() # Sentinel
66 +
67 +
68 class AsyncStream:
69 """A stream of RequestOutputs or EmbeddingRequestOutputs for a request
67 - that can be iterated over asynchronously."""
70 + that can be iterated over asynchronously via an async generator."""
71
69 - def __init__(self, request_id: str) -> None:
72 + def __init__(self, request_id: str, cancel: Callable[[str], None]) -> None:
73 self.request_id = request_id
74 + self._cancel = cancel
75 self._queue: asyncio.Queue = asyncio.Queue()
76 self._finished = False
77
@@ -77,22 +81,30 @@ class AsyncStream:
81 return
82 self._queue.put_nowait(item)
83
80 - def finish(self) -> None:
81 - self._queue.put_nowait(StopAsyncIteration())
82 - self._finished = True
84 + def finish(self, cancelled: bool = False) -> None:
85 + if not self._finished:
86 + self._finished = True
87 + self._queue.put_nowait(
88 + asyncio.CancelledError if cancelled else STOP_ITERATION)
89
90 @property
91 def finished(self) -> bool:
92 return self._finished
93
88 - def __aiter__(self):
89 - return self
90 -
91 - async def __anext__(self) -> Union[RequestOutput, EmbeddingRequestOutput]:
92 - result = await self._queue.get()
93 - if isinstance(result, Exception):
94 - raise result
95 - return result
94 + async def generator(
95 + self
96 + ) -> AsyncGenerator[Union[RequestOutput, EmbeddingRequestOutput], None]:
97 + try:
98 + while not self._finished:
99 + result = await self._queue.get()
100 + if isinstance(result, Exception):
101 + if result == STOP_ITERATION:
102 + return
103 + raise result
104 + yield result
105 + except GeneratorExit:
106 + self._cancel(self.request_id)
107 + raise asyncio.CancelledError from None
108
109
110 class RequestTracker:
@@ -100,7 +112,7 @@ class RequestTracker:
112
113 def __init__(self) -> None:
114 self._request_streams: Dict[str, AsyncStream] = {}
103 - self._finished_requests: asyncio.Queue[str] = asyncio.Queue()
115 + self._aborted_requests: asyncio.Queue[str] = asyncio.Queue()
116 self._new_requests: asyncio.Queue[Tuple[AsyncStream,
117 dict]] = asyncio.Queue()
118 self.new_requests_event = asyncio.Event()
@@ -131,15 +143,21 @@ class RequestTracker:
143 verbose: bool = False) -> None:
144 """Process a request output from the engine."""
145 request_id = request_output.request_id
146 + finished = request_output.finished
147
148 + if finished:
149 + stream = self._request_streams.pop(request_id, None)
150 + else:
151 + stream = self._request_streams.get(request_id)
152 # Guard against a KeyError which can occur if the request was aborted
153 # while the output was generated
137 - if (stream := self._request_streams.get(request_id)) is not None:
154 + if stream is not None:
155 stream.put(request_output)
139 - if request_output.finished:
140 - if verbose:
141 - logger.info("Finished request %s.", request_id)
142 - self.abort_request(request_id)
156 + if finished:
157 + stream.finish()
158 +
159 + if verbose and finished:
160 + logger.info("Finished request %s.", request_id)
161
162 def process_exception(self,
163 request_id: str,
@@ -162,7 +180,8 @@ class RequestTracker:
180 if request_id in self._request_streams:
181 raise KeyError(f"Request {request_id} already exists.")
182
165 - stream = AsyncStream(request_id)
183 + abort_request = partial(self.abort_request, verbose=verbose)
184 + stream = AsyncStream(request_id, abort_request)
185 self._new_requests.put_nowait((stream, {
186 "request_id": request_id,
187 **engine_add_request_kwargs
@@ -175,36 +194,36 @@ class RequestTracker:
194
195 return stream
196
178 - def abort_request(self, request_id: str, *, verbose: bool = False) -> None:
197 + def abort_request(self,
198 + request_id: str,
199 + *,
200 + cancelled: bool = False,
201 + verbose: bool = False) -> None:
202 """Abort a request during next background loop iteration."""
203 if verbose:
204 logger.info("Aborted request %s.", request_id)
205
183 - self._finished_requests.put_nowait(request_id)
206 + self._aborted_requests.put_nowait(request_id)
207
185 - if request_id not in self._request_streams or self._request_streams[
186 - request_id].finished:
187 - # The request has already finished or been aborted.
188 - return
189 -
190 - self._request_streams[request_id].finish()
208 + stream = self._request_streams.pop(request_id, None)
209 + if stream is not None:
210 + stream.finish(cancelled=cancelled)
211
192 - def get_new_and_finished_requests(self) -> Tuple[List[Dict], Set[str]]:
212 + def get_new_and_aborted_requests(self) -> Tuple[List[Dict], Set[str]]:
213 """Get the new requests and finished requests to be
214 sent to the engine."""
215 new_requests: List[Dict] = []
216 finished_requests: Set[str] = set()
217
198 - while not self._finished_requests.empty():
199 - request_id = self._finished_requests.get_nowait()
218 + while not self._aborted_requests.empty():
219 + request_id = self._aborted_requests.get_nowait()
220 finished_requests.add(request_id)
201 - self._request_streams.pop(request_id, None)
221
222 while not self._new_requests.empty():
223 stream, new_request = self._new_requests.get_nowait()
224 if stream.request_id in finished_requests:
225 # The request has already been aborted.
207 - stream.finish()
226 + stream.finish(cancelled=True)
227 continue
228 self._request_streams[stream.request_id] = stream
229 new_requests.append(new_request)
@@ -556,8 +575,8 @@ class AsyncLLMEngine:
575
576 Returns True if there are in-progress requests."""
577
559 - new_requests, finished_requests = (
560 - self._request_tracker.get_new_and_finished_requests())
578 + new_requests, aborted_requests = (
579 + self._request_tracker.get_new_and_aborted_requests())
580
581 for new_request in new_requests:
582 # Add the request into the vLLM engine's waiting queue.
@@ -576,8 +595,8 @@ class AsyncLLMEngine:
595 verbose=self.log_requests,
596 )
597
579 - if finished_requests:
580 - await self._engine_abort(finished_requests)
598 + if aborted_requests:
599 + await self._engine_abort(aborted_requests)
600
601 if self.engine_use_ray:
602 request_outputs = await self.engine.step.remote() # type: ignore
@@ -666,6 +685,8 @@ class AsyncLLMEngine:
685 raise
686 await asyncio.sleep(0)
687
688 + # This method does not need to be async, but kept that way
689 + # for backwards compatibility.
690 async def add_request(
691 self,
692 request_id: str,
@@ -675,7 +696,7 @@ class AsyncLLMEngine:
696 lora_request: Optional[LoRARequest] = None,
697 trace_headers: Optional[Mapping[str, str]] = None,
698 prompt_adapter_request: Optional[PromptAdapterRequest] = None
678 - ) -> AsyncStream:
699 + ) -> AsyncGenerator[Union[RequestOutput, EmbeddingRequestOutput], None]:
700 if not self.is_running:
701 if self.start_engine_loop:
702 self.start_background_loop()
@@ -686,20 +707,17 @@ class AsyncLLMEngine:
707 "error that caused the background loop to stop "
708 "(AsyncEngineDeadError).")
709
689 - if arrival_time is None:
690 - arrival_time = time.time()
691 -
710 stream = self._request_tracker.add_request(
711 request_id,
712 verbose=self.log_requests,
713 inputs=inputs,
714 params=params,
697 - arrival_time=arrival_time,
715 + arrival_time=arrival_time or time.time(),
716 lora_request=lora_request,
717 trace_headers=trace_headers,
718 prompt_adapter_request=prompt_adapter_request)
719
702 - return stream
720 + return stream.generator()
721
722 async def generate(
723 self,
@@ -709,7 +727,7 @@ class AsyncLLMEngine:
727 lora_request: Optional[LoRARequest] = None,
728 trace_headers: Optional[Mapping[str, str]] = None,
729 prompt_adapter_request: Optional[PromptAdapterRequest] = None
712 - ) -> AsyncIterator[RequestOutput]:
730 + ) -> AsyncGenerator[RequestOutput, None]:
731 """Generate outputs for a request.
732
733 Generate outputs for a request. This method is a coroutine. It adds the
@@ -774,7 +792,7 @@ class AsyncLLMEngine:
792 >>> # Process and return the final output
793 >>> ...
794 """
777 - async for output in self._process_request(
795 + async for output in await self.add_request(
796 request_id,
797 inputs,
798 sampling_params,
@@ -791,7 +809,7 @@ class AsyncLLMEngine:
809 request_id: str,
810 lora_request: Optional[LoRARequest] = None,
811 trace_headers: Optional[Mapping[str, str]] = None,
794 - ) -> AsyncIterator[EmbeddingRequestOutput]:
812 + ) -> AsyncGenerator[EmbeddingRequestOutput, None]:
813 """Generate outputs for a request from an embedding model.
814
815 Generate outputs for a request. This method is a coroutine. It adds the
@@ -852,7 +870,7 @@ class AsyncLLMEngine:
870 >>> # Process and return the final output
871 >>> ...
872 """
855 - async for output in self._process_request(
873 + async for output in await self.add_request(
874 request_id,
875 inputs,
876 pooling_params,
@@ -861,37 +879,6 @@ class AsyncLLMEngine:
879 ):
880 yield LLMEngine.validate_output(output, EmbeddingRequestOutput)
881
864 - async def _process_request(
865 - self,
866 - request_id: str,
867 - inputs: PromptInputs,
868 - params: Union[SamplingParams, PoolingParams],
869 - *,
870 - lora_request: Optional[LoRARequest] = None,
871 - trace_headers: Optional[Mapping[str, str]] = None,
872 - prompt_adapter_request: Optional[PromptAdapterRequest] = None,
873 - ) -> AsyncIterator[Union[RequestOutput, EmbeddingRequestOutput]]:
874 - """Common logic to process requests with SamplingParams or
875 - PoolingParams."""
876 - arrival_time = time.time()
877 -
878 - stream = await self.add_request(
879 - request_id,
880 - inputs,
881 - params,
882 - arrival_time=arrival_time,
883 - lora_request=lora_request,
884 - trace_headers=trace_headers,
885 - prompt_adapter_request=prompt_adapter_request,
886 - )
887 -
888 - try:
889 - async for request_output in stream:
890 - yield request_output
891 - except (Exception, asyncio.CancelledError) as e:
892 - self._abort(request_id)
893 - raise e
894 -
882 async def abort(self, request_id: str) -> None:
883 """Abort a request.
884
@@ -920,6 +907,7 @@ class AsyncLLMEngine:
907 request_id: The unique id of the request.
908 """
909 self._request_tracker.abort_request(request_id,
910 + cancelled=True,
911 verbose=self.log_requests)
912
913 async def get_model_config(self) -> ModelConfig:
vllm/engine/protocol.py
+5 -5
@@ -1,4 +1,4 @@
1 -from typing import (AsyncIterator, List, Mapping, Optional, Protocol,
1 +from typing import (AsyncGenerator, List, Mapping, Optional, Protocol,
2 runtime_checkable)
3
4 from transformers import PreTrainedTokenizer
@@ -30,7 +30,7 @@ class AsyncEngineClient(Protocol):
30 def errored(self) -> bool:
31 ...
32
33 - async def generate(
33 + def generate(
34 self,
35 inputs: PromptInputs,
36 sampling_params: SamplingParams,
@@ -38,17 +38,17 @@ class AsyncEngineClient(Protocol):
38 lora_request: Optional[LoRARequest] = None,
39 trace_headers: Optional[Mapping[str, str]] = None,
40 prompt_adapter_request: Optional[PromptAdapterRequest] = None
41 - ) -> AsyncIterator[RequestOutput]:
41 + ) -> AsyncGenerator[RequestOutput, None]:
42 """Generates outputs for a request"""
43
44 - async def encode(
44 + def encode(
45 self,
46 inputs: PromptInputs,
47 pooling_params: PoolingParams,
48 request_id: str,
49 lora_request: Optional[LoRARequest] = None,
50 trace_headers: Optional[Mapping[str, str]] = None,
51 - ) -> AsyncIterator[EmbeddingRequestOutput]:
51 + ) -> AsyncGenerator[EmbeddingRequestOutput, None]:
52 """Generate outputs for a request from an embedding model."""
53
54 async def abort(self, request_id: str) -> None:
vllm/entrypoints/api_server.py
+9 -7
@@ -20,7 +20,8 @@ from vllm.entrypoints.launcher import serve_http
20 from vllm.logger import init_logger
21 from vllm.sampling_params import SamplingParams
22 from vllm.usage.usage_lib import UsageContext
23 -from vllm.utils import FlexibleArgumentParser, random_uuid
23 +from vllm.utils import (FlexibleArgumentParser, iterate_with_cancellation,
24 + random_uuid)
25 from vllm.version import __version__ as VLLM_VERSION
26
27 logger = init_logger("vllm.entrypoints.api_server")
@@ -53,6 +54,8 @@ async def generate(request: Request) -> Response:
54
55 assert engine is not None
56 results_generator = engine.generate(prompt, sampling_params, request_id)
57 + results_generator = iterate_with_cancellation(
58 + results_generator, is_cancelled=request.is_disconnected)
59
60 # Streaming case
61 async def stream_results() -> AsyncGenerator[bytes, None]:
@@ -69,12 +72,11 @@ async def generate(request: Request) -> Response:
72
73 # Non-streaming case
74 final_output = None
72 - async for request_output in results_generator:
73 - if await request.is_disconnected():
74 - # Abort the request if the client disconnects.
75 - await engine.abort(request_id)
76 - return Response(status_code=499)
77 - final_output = request_output
75 + try:
76 + async for request_output in results_generator:
77 + final_output = request_output
78 + except asyncio.CancelledError:
79 + return Response(status_code=499)
80
81 assert final_output is not None
82 prompt = final_output.prompt
vllm/entrypoints/openai/rpc/client.py
+32 -30
@@ -1,5 +1,5 @@
1 from contextlib import contextmanager
2 -from typing import Any, AsyncIterator, Mapping, Optional
2 +from typing import Any, AsyncGenerator, Mapping, Optional
3
4 import cloudpickle
5 import zmq
@@ -190,36 +190,38 @@ class AsyncEngineRPCClient:
190 lora_request: Optional[LoRARequest] = None,
191 trace_headers: Optional[Mapping[str, str]] = None,
192 prompt_adapter_request: Optional[PromptAdapterRequest] = None
193 - ) -> AsyncIterator[RequestOutput]:
193 + ) -> AsyncGenerator[RequestOutput, None]:
194 """Send an RPCGenerateRequest to the RPCServer and stream responses."""
195
196 - with self.socket() as socket:
197 -
198 - # Send RPCGenerateRequest to the RPCServer.
199 - await socket.send_multipart([
200 - cloudpickle.dumps(
201 - RPCGenerateRequest(
202 - inputs=inputs,
203 - sampling_params=sampling_params,
204 - request_id=request_id,
205 - lora_request=lora_request,
206 - trace_headers=trace_headers,
207 - prompt_adapter_request=prompt_adapter_request))
208 - ])
209 -
210 - # Stream back the results from the RPC Server.
211 - while True:
212 - message = await socket.recv()
213 - request_output = cloudpickle.loads(message)
214 -
215 - if isinstance(request_output, Exception):
216 - raise request_output
217 -
218 - if request_output.finished:
219 - break
220 - yield request_output
221 -
222 - yield request_output
196 + finished = False
197 + try:
198 + with self.socket() as socket:
199 +
200 + # Send RPCGenerateRequest to the RPCServer.
201 + await socket.send_multipart([
202 + cloudpickle.dumps(
203 + RPCGenerateRequest(
204 + inputs=inputs,
205 + sampling_params=sampling_params,
206 + request_id=request_id,
207 + lora_request=lora_request,
208 + trace_headers=trace_headers,
209 + prompt_adapter_request=prompt_adapter_request))
210 + ])
211 +
212 + # Stream back the results from the RPC Server.
213 + while not finished:
214 + message = await socket.recv()
215 + request_output = cloudpickle.loads(message)
216 +
217 + if isinstance(request_output, Exception):
218 + raise request_output
219 +
220 + finished = request_output.finished
221 + yield request_output
222 + finally:
223 + if not finished:
224 + await self.abort(request_id)
225
226 async def check_health(self) -> None:
227 """Raise if unhealthy"""
@@ -243,6 +245,6 @@ class AsyncEngineRPCClient:
245 "f{health_message}")
246
247 async def encode(self, *args,
246 - **kwargs) -> AsyncIterator[EmbeddingRequestOutput]:
248 + **kwargs) -> AsyncGenerator[EmbeddingRequestOutput, None]:
249 raise NotImplementedError(
250 "Embeddings not supported with multiprocessing backend")
vllm/entrypoints/openai/serving_chat.py
+18 -16
@@ -1,3 +1,4 @@
1 +import asyncio
2 import time
3 from typing import AsyncGenerator, AsyncIterator, Dict, List, Optional
4 from typing import Sequence as GenericSequence
@@ -29,7 +30,7 @@ from vllm.outputs import RequestOutput
30 from vllm.sequence import Logprob
31 from vllm.tracing import (contains_trace_headers, extract_trace_headers,
32 log_tracing_disabled_warning)
32 -from vllm.utils import random_uuid
33 +from vllm.utils import iterate_with_cancellation, random_uuid
34
35 logger = init_logger(__name__)
36
@@ -176,18 +177,20 @@ class OpenAIServingChat(OpenAIServing):
177 # TODO: Use a vllm-specific Validation Error
178 return self.create_error_response(str(e))
179
180 + if raw_request:
181 + result_generator = iterate_with_cancellation(
182 + result_generator, raw_request.is_disconnected)
183 +
184 # Streaming response
185 if request.stream:
186 return self.chat_completion_stream_generator(
187 request, result_generator, request_id, conversation, tokenizer)
183 - else:
184 - try:
185 - return await self.chat_completion_full_generator(
186 - request, raw_request, result_generator, request_id,
187 - conversation, tokenizer)
188 - except ValueError as e:
189 - # TODO: Use a vllm-specific Validation Error
190 - return self.create_error_response(str(e))
188 + try:
189 + return await self.chat_completion_full_generator(
190 + request, result_generator, request_id, conversation, tokenizer)
191 + except ValueError as e:
192 + # TODO: Use a vllm-specific Validation Error
193 + return self.create_error_response(str(e))
194
195 def get_chat_request_role(self, request: ChatCompletionRequest) -> str:
196 if request.add_generation_prompt:
@@ -422,7 +425,6 @@ class OpenAIServingChat(OpenAIServing):
425 async def chat_completion_full_generator(
426 self,
427 request: ChatCompletionRequest,
425 - raw_request: Optional[Request],
428 result_generator: AsyncIterator[RequestOutput],
429 request_id: str,
430 conversation: List[ConversationMessage],
@@ -433,12 +435,12 @@ class OpenAIServingChat(OpenAIServing):
435 created_time = int(time.time())
436 final_res: Optional[RequestOutput] = None
437
436 - async for res in result_generator:
437 - if raw_request is not None and await raw_request.is_disconnected():
438 - # Abort the request if the client disconnects.
439 - await self.async_engine_client.abort(request_id)
440 - return self.create_error_response("Client disconnected")
441 - final_res = res
438 + try:
439 + async for res in result_generator:
440 + final_res = res
441 + except asyncio.CancelledError:
442 + return self.create_error_response("Client disconnected")
443 +
444 assert final_res is not None
445
446 choices: List[ChatCompletionResponseChoice] = []
vllm/entrypoints/openai/serving_completion.py
+6 -14
@@ -1,3 +1,4 @@
1 +import asyncio
2 import time
3 from typing import (AsyncGenerator, AsyncIterator, Callable, Dict, List,
4 Optional)
@@ -84,7 +85,7 @@ class OpenAIServingCompletion(OpenAIServing):
85 created_time = int(time.time())
86
87 # Schedule the request and get the result generator.
87 - generators: List[AsyncIterator[RequestOutput]] = []
88 + generators: List[AsyncGenerator[RequestOutput, None]] = []
89 try:
90 (
91 lora_request,
@@ -144,7 +145,8 @@ class OpenAIServingCompletion(OpenAIServing):
145 return self.create_error_response(str(e))
146
147 result_generator: AsyncIterator[Tuple[
147 - int, RequestOutput]] = merge_async_iterators(*generators)
148 + int, RequestOutput]] = merge_async_iterators(
149 + *generators, is_cancelled=raw_request.is_disconnected)
150
151 # Similar to the OpenAI API, when n != best_of, we do not stream the
152 # results. In addition, we do not stream the results when use
@@ -156,7 +158,6 @@ class OpenAIServingCompletion(OpenAIServing):
158 # Streaming response
159 if stream:
160 return self.completion_stream_generator(request,
159 - raw_request,
161 result_generator,
162 request_id,
163 created_time,
@@ -168,10 +169,6 @@ class OpenAIServingCompletion(OpenAIServing):
169 final_res_batch: List[Optional[RequestOutput]] = [None] * len(prompts)
170 try:
171 async for i, res in result_generator:
171 - if await raw_request.is_disconnected():
172 - # Abort the request if the client disconnects.
173 - await self.async_engine_client.abort(f"{request_id}-{i}")
174 - return self.create_error_response("Client disconnected")
172 final_res_batch[i] = res
173
174 for i, final_res in enumerate(final_res_batch):
@@ -194,6 +191,8 @@ class OpenAIServingCompletion(OpenAIServing):
191 model_name,
192 tokenizer,
193 )
194 + except asyncio.CancelledError:
195 + return self.create_error_response("Client disconnected")
196 except ValueError as e:
197 # TODO: Use a vllm-specific Validation Error
198 return self.create_error_response(str(e))
@@ -214,7 +213,6 @@ class OpenAIServingCompletion(OpenAIServing):
213 async def completion_stream_generator(
214 self,
215 request: CompletionRequest,
217 - raw_request: Request,
216 result_generator: AsyncIterator[Tuple[int, RequestOutput]],
217 request_id: str,
218 created_time: int,
@@ -230,12 +228,6 @@ class OpenAIServingCompletion(OpenAIServing):
228 try:
229 async for prompt_idx, res in result_generator:
230
233 - # Abort the request if the client disconnects.
234 - if await raw_request.is_disconnected():
235 - await self.async_engine_client.abort(
236 - f"{request_id}-{prompt_idx}")
237 - raise StopAsyncIteration()
238 -
231 for output in res.outputs:
232 i = output.index + prompt_idx * num_choices
233 # TODO(simon): optimize the performance by avoiding full
vllm/entrypoints/openai/serving_embedding.py
+7 -7
@@ -1,6 +1,7 @@
1 +import asyncio
2 import base64
3 import time
3 -from typing import AsyncIterator, List, Optional, Tuple, cast
4 +from typing import AsyncGenerator, AsyncIterator, List, Optional, Tuple, cast
5
6 import numpy as np
7 from fastapi import Request
@@ -92,7 +93,7 @@ class OpenAIServingEmbedding(OpenAIServing):
93 created_time = int(time.monotonic())
94
95 # Schedule the request and get the result generator.
95 - generators: List[AsyncIterator[EmbeddingRequestOutput]] = []
96 + generators: List[AsyncGenerator[EmbeddingRequestOutput, None]] = []
97 try:
98 (
99 lora_request,
@@ -138,17 +139,14 @@ class OpenAIServingEmbedding(OpenAIServing):
139 return self.create_error_response(str(e))
140
141 result_generator: AsyncIterator[Tuple[
141 - int, EmbeddingRequestOutput]] = merge_async_iterators(*generators)
142 + int, EmbeddingRequestOutput]] = merge_async_iterators(
143 + *generators, is_cancelled=raw_request.is_disconnected)
144
145 # Non-streaming response
146 final_res_batch: List[Optional[EmbeddingRequestOutput]]
147 final_res_batch = [None] * len(prompts)
148 try:
149 async for i, res in result_generator:
148 - if await raw_request.is_disconnected():
149 - # Abort the request if the client disconnects.
150 - await self.async_engine_client.abort(f"{request_id}-{i}")
151 - return self.create_error_response("Client disconnected")
150 final_res_batch[i] = res
151
152 for final_res in final_res_batch:
@@ -160,6 +158,8 @@ class OpenAIServingEmbedding(OpenAIServing):
158 response = request_output_to_embedding_response(
159 final_res_batch_checked, request_id, created_time, model_name,
160 encoding_format)
161 + except asyncio.CancelledError:
162 + return self.create_error_response("Client disconnected")
163 except ValueError as e:
164 # TODO: Use a vllm-specific Validation Error
165 return self.create_error_response(str(e))
vllm/utils.py
+62 -49
@@ -1,5 +1,6 @@
1 import argparse
2 import asyncio
3 +import contextlib
4 import datetime
5 import enum
6 import gc
@@ -11,10 +12,11 @@ import tempfile
12 import threading
13 import uuid
14 import warnings
15 +from asyncio import FIRST_COMPLETED, ensure_future
16 from collections import defaultdict
17 from functools import lru_cache, partial, wraps
18 from platform import uname
17 -from typing import (Any, AsyncIterator, Awaitable, Callable, Dict, Generic,
19 +from typing import (Any, AsyncGenerator, Awaitable, Callable, Dict, Generic,
20 Hashable, List, Optional, OrderedDict, Set, Tuple, TypeVar,
21 Union, overload)
22
@@ -373,63 +375,74 @@ def make_async(func: Callable[P, T]) -> Callable[P, Awaitable[T]]:
375 return _async_wrapper
376
377
376 -class ProducerFinished:
377 - pass
378 +async def iterate_with_cancellation(
379 + iterator: AsyncGenerator[T, None],
380 + is_cancelled: Callable[[], Awaitable[bool]],
381 +) -> AsyncGenerator[T, None]:
382 + """Convert async iterator into one that polls the provided function
383 + at least once per second to check for client cancellation.
384 + """
385
386 + # Can use anext() in python >= 3.10
387 + awaits = [ensure_future(iterator.__anext__())]
388 + while True:
389 + done, pending = await asyncio.wait(awaits, timeout=1)
390 + if await is_cancelled():
391 + with contextlib.suppress(BaseException):
392 + awaits[0].cancel()
393 + await iterator.aclose()
394 + raise asyncio.CancelledError("client cancelled")
395 + if done:
396 + try:
397 + item = await awaits[0]
398 + awaits[0] = ensure_future(iterator.__anext__())
399 + yield item
400 + except StopAsyncIteration:
401 + # we are done
402 + return
403
380 -def merge_async_iterators(
381 - *iterators: AsyncIterator[T]) -> AsyncIterator[Tuple[int, T]]:
404 +
405 +async def merge_async_iterators(
406 + *iterators: AsyncGenerator[T, None],
407 + is_cancelled: Callable[[], Awaitable[bool]],
408 +) -> AsyncGenerator[Tuple[int, T], None]:
409 """Merge multiple asynchronous iterators into a single iterator.
410
411 This method handle the case where some iterators finish before others.
412 When it yields, it yields a tuple (i, item) where i is the index of the
413 iterator that yields the item.
387 - """
388 - queue: asyncio.Queue[Union[Tuple[int, T], ProducerFinished,
389 - Exception]] = asyncio.Queue()
390 -
391 - producers = len(iterators)
392 -
393 - async def producer(i: int, iterator: AsyncIterator[T]):
394 - try:
395 - async for item in iterator:
396 - await queue.put((i, item))
397 - except Exception as e:
398 - await queue.put(e)
399 - # Signal to the consumer that we've finished
400 - await queue.put(ProducerFinished())
401 -
402 - _tasks = [
403 - asyncio.create_task(producer(i, iterator))
404 - for i, iterator in enumerate(iterators)
405 - ]
414
407 - async def consumer():
408 - remaining = producers
409 - try:
410 - while remaining or not queue.empty():
411 - # we think there is a race condition here
412 - item = await queue.get()
413 -
414 - if isinstance(item, ProducerFinished):
415 - # Signal that a producer finished- not a real item
416 - remaining -= 1
417 - continue
418 -
419 - if isinstance(item, Exception):
420 - raise item
421 - yield item
422 - except (Exception, asyncio.CancelledError) as e:
423 - for task in _tasks:
424 - if sys.version_info >= (3, 9):
425 - # msg parameter only supported in Python 3.9+
426 - task.cancel(e)
427 - else:
428 - task.cancel()
429 - raise e
430 - await asyncio.gather(*_tasks)
415 + It also polls the provided function at least once per second to check
416 + for client cancellation.
417 + """
418
432 - return consumer()
419 + # Can use anext() in python >= 3.10
420 + awaits = {
421 + ensure_future(pair[1].__anext__()): pair
422 + for pair in enumerate(iterators)
423 + }
424 + try:
425 + while awaits:
426 + done, pending = await asyncio.wait(awaits.keys(),
427 + return_when=FIRST_COMPLETED,
428 + timeout=1)
429 + if await is_cancelled():
430 + raise asyncio.CancelledError("client cancelled")
431 + for d in done:
432 + pair = awaits.pop(d)
433 + try:
434 + item = await d
435 + i, it = pair
436 + awaits[ensure_future(it.__anext__())] = pair
437 + yield i, item
438 + except StopAsyncIteration:
439 + pass
440 + finally:
441 + # Cancel any remaining iterators
442 + for f, (_, it) in awaits.items():
443 + with contextlib.suppress(BaseException):
444 + f.cancel()
445 + await it.aclose()
446
447
448 def get_ip() -> str: