A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 19:30 UTC
[BugFix] Overhaul async request cancellation (#7111)
Nick Hill committed
Aug 6, 2024 at 22:21 UTC
9a3f49ae07f9627db02b2ee377accb76b65d1d1e
11 files changed
+226
-226
tests/async_engine/api_server_async_engine.py
+5
-4
@@ -1,5 +1,5 @@
1
"""vllm.entrypoints.api_server with some extra logging for testing."""
2
-from typing import Any, Dict
2
+from typing import Any, Dict, Iterable
3
4
import uvicorn
5
from fastapi.responses import JSONResponse, Response
@@ -18,9 +18,10 @@ class AsyncLLMEngineWithStats(AsyncLLMEngine):
18
super().__init__(*args, **kwargs)
19
self._num_aborts = 0
20
21
- async def abort(self, request_id: str) -> None:
22
- await super().abort(request_id)
23
- self._num_aborts += 1
21
+ async def _engine_abort(self, request_ids: Iterable[str]):
22
+ ids = list(request_ids)
23
+ self._num_aborts += len(ids)
24
+ await super()._engine_abort(ids)
25
26
def testing_stats(self) -> Dict[str, Any]:
27
return {"num_aborted_requests": self._num_aborts}
tests/async_engine/test_request_tracker.py
+12
-13
@@ -10,23 +10,23 @@ async def test_request_tracker():
10
stream_1 = tracker.add_request("1")
11
assert tracker.new_requests_event.is_set()
12
await tracker.wait_for_new_requests()
13
- new, finished = tracker.get_new_and_finished_requests()
13
+ new, aborted = tracker.get_new_and_aborted_requests()
14
assert not tracker.new_requests_event.is_set()
15
assert len(new) == 1
16
assert new[0]["request_id"] == "1"
17
- assert not finished
17
+ assert not aborted
18
assert not stream_1.finished
19
20
stream_2 = tracker.add_request("2")
21
stream_3 = tracker.add_request("3")
22
assert tracker.new_requests_event.is_set()
23
await tracker.wait_for_new_requests()
24
- new, finished = tracker.get_new_and_finished_requests()
24
+ new, aborted = tracker.get_new_and_aborted_requests()
25
assert not tracker.new_requests_event.is_set()
26
assert len(new) == 2
27
assert new[0]["request_id"] == "2"
28
assert new[1]["request_id"] == "3"
29
- assert not finished
29
+ assert not aborted
30
assert not stream_2.finished
31
assert not stream_3.finished
32
@@ -36,9 +36,9 @@ async def test_request_tracker():
36
assert not tracker.new_requests_event.is_set()
37
38
tracker.abort_request("1")
39
- new, finished = tracker.get_new_and_finished_requests()
40
- assert len(finished) == 1
41
- assert "1" in finished
39
+ new, aborted = tracker.get_new_and_aborted_requests()
40
+ assert len(aborted) == 1
41
+ assert "1" in aborted
42
assert not new
43
assert stream_1.finished
44
@@ -46,9 +46,9 @@ async def test_request_tracker():
46
tracker.abort_request("4")
47
assert tracker.new_requests_event.is_set()
48
await tracker.wait_for_new_requests()
49
- new, finished = tracker.get_new_and_finished_requests()
50
- assert len(finished) == 1
51
- assert "4" in finished
49
+ new, aborted = tracker.get_new_and_aborted_requests()
50
+ assert len(aborted) == 1
51
+ assert "4" in aborted
52
assert not new
53
assert stream_4.finished
54
@@ -57,10 +57,9 @@ async def test_request_tracker():
57
tracker.process_request_output(
58
RequestOutput("2", "output", [], [], [], finished=True))
59
await tracker.wait_for_new_requests()
60
- new, finished = tracker.get_new_and_finished_requests()
60
+ new, aborted = tracker.get_new_and_aborted_requests()
61
assert not tracker.new_requests_event.is_set()
62
- assert len(finished) == 1
63
- assert "2" in finished
62
+ assert not aborted
63
assert len(new) == 1
64
assert new[0]["request_id"] == "5"
65
assert stream_2.finished
tests/test_utils.py
+3
-2
@@ -2,6 +2,7 @@ import asyncio
2
import os
3
import socket
4
import sys
5
+from functools import partial
6
from typing import (TYPE_CHECKING, Any, AsyncIterator, Awaitable, Protocol,
7
Tuple, TypeVar)
8
@@ -37,11 +38,11 @@ async def test_merge_async_iterators():
38
yield f"item from iterator {idx}"
39
await asyncio.sleep(0.1)
40
except asyncio.CancelledError:
40
- pass
41
+ print(f"iterator {idx} cancelled")
42
43
iterators = [mock_async_iterator(i) for i in range(3)]
44
merged_iterator: AsyncIterator[Tuple[int, str]] = merge_async_iterators(
44
- *iterators)
45
+ *iterators, is_cancelled=partial(asyncio.sleep, 0, result=False))
46
47
async def stream_output(generator: AsyncIterator[Tuple[int, str]]):
48
async for idx, output in generator:
vllm/engine/async_llm_engine.py
+67
-79
@@ -1,7 +1,7 @@
1
import asyncio
2
import time
3
from functools import partial
4
-from typing import (AsyncIterator, Callable, Dict, Iterable, List, Mapping,
4
+from typing import (AsyncGenerator, Callable, Dict, Iterable, List, Mapping,
5
Optional, Set, Tuple, Type, Union)
6
7
from transformers import PreTrainedTokenizer
@@ -62,12 +62,16 @@ def _log_task_completion(task: asyncio.Task,
62
"actual cause.") from e
63
64
65
+STOP_ITERATION = Exception() # Sentinel
66
+
67
+
68
class AsyncStream:
69
"""A stream of RequestOutputs or EmbeddingRequestOutputs for a request
67
- that can be iterated over asynchronously."""
70
+ that can be iterated over asynchronously via an async generator."""
71
69
- def __init__(self, request_id: str) -> None:
72
+ def __init__(self, request_id: str, cancel: Callable[[str], None]) -> None:
73
self.request_id = request_id
74
+ self._cancel = cancel
75
self._queue: asyncio.Queue = asyncio.Queue()
76
self._finished = False
77
@@ -77,22 +81,30 @@ class AsyncStream:
81
return
82
self._queue.put_nowait(item)
83
80
- def finish(self) -> None:
81
- self._queue.put_nowait(StopAsyncIteration())
82
- self._finished = True
84
+ def finish(self, cancelled: bool = False) -> None:
85
+ if not self._finished:
86
+ self._finished = True
87
+ self._queue.put_nowait(
88
+ asyncio.CancelledError if cancelled else STOP_ITERATION)
89
90
@property
91
def finished(self) -> bool:
92
return self._finished
93
88
- def __aiter__(self):
89
- return self
90
-
91
- async def __anext__(self) -> Union[RequestOutput, EmbeddingRequestOutput]:
92
- result = await self._queue.get()
93
- if isinstance(result, Exception):
94
- raise result
95
- return result
94
+ async def generator(
95
+ self
96
+ ) -> AsyncGenerator[Union[RequestOutput, EmbeddingRequestOutput], None]:
97
+ try:
98
+ while not self._finished:
99
+ result = await self._queue.get()
100
+ if isinstance(result, Exception):
101
+ if result == STOP_ITERATION:
102
+ return
103
+ raise result
104
+ yield result
105
+ except GeneratorExit:
106
+ self._cancel(self.request_id)
107
+ raise asyncio.CancelledError from None
108
109
110
class RequestTracker:
@@ -100,7 +112,7 @@ class RequestTracker:
112
113
def __init__(self) -> None:
114
self._request_streams: Dict[str, AsyncStream] = {}
103
- self._finished_requests: asyncio.Queue[str] = asyncio.Queue()
115
+ self._aborted_requests: asyncio.Queue[str] = asyncio.Queue()
116
self._new_requests: asyncio.Queue[Tuple[AsyncStream,
117
dict]] = asyncio.Queue()
118
self.new_requests_event = asyncio.Event()
@@ -131,15 +143,21 @@ class RequestTracker:
143
verbose: bool = False) -> None:
144
"""Process a request output from the engine."""
145
request_id = request_output.request_id
146
+ finished = request_output.finished
147
148
+ if finished:
149
+ stream = self._request_streams.pop(request_id, None)
150
+ else:
151
+ stream = self._request_streams.get(request_id)
152
# Guard against a KeyError which can occur if the request was aborted
153
# while the output was generated
137
- if (stream := self._request_streams.get(request_id)) is not None:
154
+ if stream is not None:
155
stream.put(request_output)
139
- if request_output.finished:
140
- if verbose:
141
- logger.info("Finished request %s.", request_id)
142
- self.abort_request(request_id)
156
+ if finished:
157
+ stream.finish()
158
+
159
+ if verbose and finished:
160
+ logger.info("Finished request %s.", request_id)
161
162
def process_exception(self,
163
request_id: str,
@@ -162,7 +180,8 @@ class RequestTracker:
180
if request_id in self._request_streams:
181
raise KeyError(f"Request {request_id} already exists.")
182
165
- stream = AsyncStream(request_id)
183
+ abort_request = partial(self.abort_request, verbose=verbose)
184
+ stream = AsyncStream(request_id, abort_request)
185
self._new_requests.put_nowait((stream, {
186
"request_id": request_id,
187
**engine_add_request_kwargs
@@ -175,36 +194,36 @@ class RequestTracker:
194
195
return stream
196
178
- def abort_request(self, request_id: str, *, verbose: bool = False) -> None:
197
+ def abort_request(self,
198
+ request_id: str,
199
+ *,
200
+ cancelled: bool = False,
201
+ verbose: bool = False) -> None:
202
"""Abort a request during next background loop iteration."""
203
if verbose:
204
logger.info("Aborted request %s.", request_id)
205
183
- self._finished_requests.put_nowait(request_id)
206
+ self._aborted_requests.put_nowait(request_id)
207
185
- if request_id not in self._request_streams or self._request_streams[
186
- request_id].finished:
187
- # The request has already finished or been aborted.
188
- return
189
-
190
- self._request_streams[request_id].finish()
208
+ stream = self._request_streams.pop(request_id, None)
209
+ if stream is not None:
210
+ stream.finish(cancelled=cancelled)
211
192
- def get_new_and_finished_requests(self) -> Tuple[List[Dict], Set[str]]:
212
+ def get_new_and_aborted_requests(self) -> Tuple[List[Dict], Set[str]]:
213
"""Get the new requests and finished requests to be
214
sent to the engine."""
215
new_requests: List[Dict] = []
216
finished_requests: Set[str] = set()
217
198
- while not self._finished_requests.empty():
199
- request_id = self._finished_requests.get_nowait()
218
+ while not self._aborted_requests.empty():
219
+ request_id = self._aborted_requests.get_nowait()
220
finished_requests.add(request_id)
201
- self._request_streams.pop(request_id, None)
221
222
while not self._new_requests.empty():
223
stream, new_request = self._new_requests.get_nowait()
224
if stream.request_id in finished_requests:
225
# The request has already been aborted.
207
- stream.finish()
226
+ stream.finish(cancelled=True)
227
continue
228
self._request_streams[stream.request_id] = stream
229
new_requests.append(new_request)
@@ -556,8 +575,8 @@ class AsyncLLMEngine:
575
576
Returns True if there are in-progress requests."""
577
559
- new_requests, finished_requests = (
560
- self._request_tracker.get_new_and_finished_requests())
578
+ new_requests, aborted_requests = (
579
+ self._request_tracker.get_new_and_aborted_requests())
580
581
for new_request in new_requests:
582
# Add the request into the vLLM engine's waiting queue.
@@ -576,8 +595,8 @@ class AsyncLLMEngine:
595
verbose=self.log_requests,
596
)
597
579
- if finished_requests:
580
- await self._engine_abort(finished_requests)
598
+ if aborted_requests:
599
+ await self._engine_abort(aborted_requests)
600
601
if self.engine_use_ray:
602
request_outputs = await self.engine.step.remote() # type: ignore
@@ -666,6 +685,8 @@ class AsyncLLMEngine:
685
raise
686
await asyncio.sleep(0)
687
688
+ # This method does not need to be async, but kept that way
689
+ # for backwards compatibility.
690
async def add_request(
691
self,
692
request_id: str,
@@ -675,7 +696,7 @@ class AsyncLLMEngine:
696
lora_request: Optional[LoRARequest] = None,
697
trace_headers: Optional[Mapping[str, str]] = None,
698
prompt_adapter_request: Optional[PromptAdapterRequest] = None
678
- ) -> AsyncStream:
699
+ ) -> AsyncGenerator[Union[RequestOutput, EmbeddingRequestOutput], None]:
700
if not self.is_running:
701
if self.start_engine_loop:
702
self.start_background_loop()
@@ -686,20 +707,17 @@ class AsyncLLMEngine:
707
"error that caused the background loop to stop "
708
"(AsyncEngineDeadError).")
709
689
- if arrival_time is None:
690
- arrival_time = time.time()
691
-
710
stream = self._request_tracker.add_request(
711
request_id,
712
verbose=self.log_requests,
713
inputs=inputs,
714
params=params,
697
- arrival_time=arrival_time,
715
+ arrival_time=arrival_time or time.time(),
716
lora_request=lora_request,
717
trace_headers=trace_headers,
718
prompt_adapter_request=prompt_adapter_request)
719
702
- return stream
720
+ return stream.generator()
721
722
async def generate(
723
self,
@@ -709,7 +727,7 @@ class AsyncLLMEngine:
727
lora_request: Optional[LoRARequest] = None,
728
trace_headers: Optional[Mapping[str, str]] = None,
729
prompt_adapter_request: Optional[PromptAdapterRequest] = None
712
- ) -> AsyncIterator[RequestOutput]:
730
+ ) -> AsyncGenerator[RequestOutput, None]:
731
"""Generate outputs for a request.
732
733
Generate outputs for a request. This method is a coroutine. It adds the
@@ -774,7 +792,7 @@ class AsyncLLMEngine:
792
>>> # Process and return the final output
793
>>> ...
794
"""
777
- async for output in self._process_request(
795
+ async for output in await self.add_request(
796
request_id,
797
inputs,
798
sampling_params,
@@ -791,7 +809,7 @@ class AsyncLLMEngine:
809
request_id: str,
810
lora_request: Optional[LoRARequest] = None,
811
trace_headers: Optional[Mapping[str, str]] = None,
794
- ) -> AsyncIterator[EmbeddingRequestOutput]:
812
+ ) -> AsyncGenerator[EmbeddingRequestOutput, None]:
813
"""Generate outputs for a request from an embedding model.
814
815
Generate outputs for a request. This method is a coroutine. It adds the
@@ -852,7 +870,7 @@ class AsyncLLMEngine:
870
>>> # Process and return the final output
871
>>> ...
872
"""
855
- async for output in self._process_request(
873
+ async for output in await self.add_request(
874
request_id,
875
inputs,
876
pooling_params,
@@ -861,37 +879,6 @@ class AsyncLLMEngine:
879
):
880
yield LLMEngine.validate_output(output, EmbeddingRequestOutput)
881
864
- async def _process_request(
865
- self,
866
- request_id: str,
867
- inputs: PromptInputs,
868
- params: Union[SamplingParams, PoolingParams],
869
- *,
870
- lora_request: Optional[LoRARequest] = None,
871
- trace_headers: Optional[Mapping[str, str]] = None,
872
- prompt_adapter_request: Optional[PromptAdapterRequest] = None,
873
- ) -> AsyncIterator[Union[RequestOutput, EmbeddingRequestOutput]]:
874
- """Common logic to process requests with SamplingParams or
875
- PoolingParams."""
876
- arrival_time = time.time()
877
-
878
- stream = await self.add_request(
879
- request_id,
880
- inputs,
881
- params,
882
- arrival_time=arrival_time,
883
- lora_request=lora_request,
884
- trace_headers=trace_headers,
885
- prompt_adapter_request=prompt_adapter_request,
886
- )
887
-
888
- try:
889
- async for request_output in stream:
890
- yield request_output
891
- except (Exception, asyncio.CancelledError) as e:
892
- self._abort(request_id)
893
- raise e
894
-
882
async def abort(self, request_id: str) -> None:
883
"""Abort a request.
884
@@ -920,6 +907,7 @@ class AsyncLLMEngine:
907
request_id: The unique id of the request.
908
"""
909
self._request_tracker.abort_request(request_id,
910
+ cancelled=True,
911
verbose=self.log_requests)
912
913
async def get_model_config(self) -> ModelConfig:
vllm/engine/protocol.py
+5
-5
@@ -1,4 +1,4 @@
1
-from typing import (AsyncIterator, List, Mapping, Optional, Protocol,
1
+from typing import (AsyncGenerator, List, Mapping, Optional, Protocol,
2
runtime_checkable)
3
4
from transformers import PreTrainedTokenizer
@@ -30,7 +30,7 @@ class AsyncEngineClient(Protocol):
30
def errored(self) -> bool:
31
...
32
33
- async def generate(
33
+ def generate(
34
self,
35
inputs: PromptInputs,
36
sampling_params: SamplingParams,
@@ -38,17 +38,17 @@ class AsyncEngineClient(Protocol):
38
lora_request: Optional[LoRARequest] = None,
39
trace_headers: Optional[Mapping[str, str]] = None,
40
prompt_adapter_request: Optional[PromptAdapterRequest] = None
41
- ) -> AsyncIterator[RequestOutput]:
41
+ ) -> AsyncGenerator[RequestOutput, None]:
42
"""Generates outputs for a request"""
43
44
- async def encode(
44
+ def encode(
45
self,
46
inputs: PromptInputs,
47
pooling_params: PoolingParams,
48
request_id: str,
49
lora_request: Optional[LoRARequest] = None,
50
trace_headers: Optional[Mapping[str, str]] = None,
51
- ) -> AsyncIterator[EmbeddingRequestOutput]:
51
+ ) -> AsyncGenerator[EmbeddingRequestOutput, None]:
52
"""Generate outputs for a request from an embedding model."""
53
54
async def abort(self, request_id: str) -> None:
vllm/entrypoints/api_server.py
+9
-7
@@ -20,7 +20,8 @@ from vllm.entrypoints.launcher import serve_http
20
from vllm.logger import init_logger
21
from vllm.sampling_params import SamplingParams
22
from vllm.usage.usage_lib import UsageContext
23
-from vllm.utils import FlexibleArgumentParser, random_uuid
23
+from vllm.utils import (FlexibleArgumentParser, iterate_with_cancellation,
24
+ random_uuid)
25
from vllm.version import __version__ as VLLM_VERSION
26
27
logger = init_logger("vllm.entrypoints.api_server")
@@ -53,6 +54,8 @@ async def generate(request: Request) -> Response:
54
55
assert engine is not None
56
results_generator = engine.generate(prompt, sampling_params, request_id)
57
+ results_generator = iterate_with_cancellation(
58
+ results_generator, is_cancelled=request.is_disconnected)
59
60
# Streaming case
61
async def stream_results() -> AsyncGenerator[bytes, None]:
@@ -69,12 +72,11 @@ async def generate(request: Request) -> Response:
72
73
# Non-streaming case
74
final_output = None
72
- async for request_output in results_generator:
73
- if await request.is_disconnected():
74
- # Abort the request if the client disconnects.
75
- await engine.abort(request_id)
76
- return Response(status_code=499)
77
- final_output = request_output
75
+ try:
76
+ async for request_output in results_generator:
77
+ final_output = request_output
78
+ except asyncio.CancelledError:
79
+ return Response(status_code=499)
80
81
assert final_output is not None
82
prompt = final_output.prompt
vllm/entrypoints/openai/rpc/client.py
+32
-30
@@ -1,5 +1,5 @@
1
from contextlib import contextmanager
2
-from typing import Any, AsyncIterator, Mapping, Optional
2
+from typing import Any, AsyncGenerator, Mapping, Optional
3
4
import cloudpickle
5
import zmq
@@ -190,36 +190,38 @@ class AsyncEngineRPCClient:
190
lora_request: Optional[LoRARequest] = None,
191
trace_headers: Optional[Mapping[str, str]] = None,
192
prompt_adapter_request: Optional[PromptAdapterRequest] = None
193
- ) -> AsyncIterator[RequestOutput]:
193
+ ) -> AsyncGenerator[RequestOutput, None]:
194
"""Send an RPCGenerateRequest to the RPCServer and stream responses."""
195
196
- with self.socket() as socket:
197
-
198
- # Send RPCGenerateRequest to the RPCServer.
199
- await socket.send_multipart([
200
- cloudpickle.dumps(
201
- RPCGenerateRequest(
202
- inputs=inputs,
203
- sampling_params=sampling_params,
204
- request_id=request_id,
205
- lora_request=lora_request,
206
- trace_headers=trace_headers,
207
- prompt_adapter_request=prompt_adapter_request))
208
- ])
209
-
210
- # Stream back the results from the RPC Server.
211
- while True:
212
- message = await socket.recv()
213
- request_output = cloudpickle.loads(message)
214
-
215
- if isinstance(request_output, Exception):
216
- raise request_output
217
-
218
- if request_output.finished:
219
- break
220
- yield request_output
221
-
222
- yield request_output
196
+ finished = False
197
+ try:
198
+ with self.socket() as socket:
199
+
200
+ # Send RPCGenerateRequest to the RPCServer.
201
+ await socket.send_multipart([
202
+ cloudpickle.dumps(
203
+ RPCGenerateRequest(
204
+ inputs=inputs,
205
+ sampling_params=sampling_params,
206
+ request_id=request_id,
207
+ lora_request=lora_request,
208
+ trace_headers=trace_headers,
209
+ prompt_adapter_request=prompt_adapter_request))
210
+ ])
211
+
212
+ # Stream back the results from the RPC Server.
213
+ while not finished:
214
+ message = await socket.recv()
215
+ request_output = cloudpickle.loads(message)
216
+
217
+ if isinstance(request_output, Exception):
218
+ raise request_output
219
+
220
+ finished = request_output.finished
221
+ yield request_output
222
+ finally:
223
+ if not finished:
224
+ await self.abort(request_id)
225
226
async def check_health(self) -> None:
227
"""Raise if unhealthy"""
@@ -243,6 +245,6 @@ class AsyncEngineRPCClient:
245
"f{health_message}")
246
247
async def encode(self, *args,
246
- **kwargs) -> AsyncIterator[EmbeddingRequestOutput]:
248
+ **kwargs) -> AsyncGenerator[EmbeddingRequestOutput, None]:
249
raise NotImplementedError(
250
"Embeddings not supported with multiprocessing backend")
vllm/entrypoints/openai/serving_chat.py
+18
-16
@@ -1,3 +1,4 @@
1
+import asyncio
2
import time
3
from typing import AsyncGenerator, AsyncIterator, Dict, List, Optional
4
from typing import Sequence as GenericSequence
@@ -29,7 +30,7 @@ from vllm.outputs import RequestOutput
30
from vllm.sequence import Logprob
31
from vllm.tracing import (contains_trace_headers, extract_trace_headers,
32
log_tracing_disabled_warning)
32
-from vllm.utils import random_uuid
33
+from vllm.utils import iterate_with_cancellation, random_uuid
34
35
logger = init_logger(__name__)
36
@@ -176,18 +177,20 @@ class OpenAIServingChat(OpenAIServing):
177
# TODO: Use a vllm-specific Validation Error
178
return self.create_error_response(str(e))
179
180
+ if raw_request:
181
+ result_generator = iterate_with_cancellation(
182
+ result_generator, raw_request.is_disconnected)
183
+
184
# Streaming response
185
if request.stream:
186
return self.chat_completion_stream_generator(
187
request, result_generator, request_id, conversation, tokenizer)
183
- else:
184
- try:
185
- return await self.chat_completion_full_generator(
186
- request, raw_request, result_generator, request_id,
187
- conversation, tokenizer)
188
- except ValueError as e:
189
- # TODO: Use a vllm-specific Validation Error
190
- return self.create_error_response(str(e))
188
+ try:
189
+ return await self.chat_completion_full_generator(
190
+ request, result_generator, request_id, conversation, tokenizer)
191
+ except ValueError as e:
192
+ # TODO: Use a vllm-specific Validation Error
193
+ return self.create_error_response(str(e))
194
195
def get_chat_request_role(self, request: ChatCompletionRequest) -> str:
196
if request.add_generation_prompt:
@@ -422,7 +425,6 @@ class OpenAIServingChat(OpenAIServing):
425
async def chat_completion_full_generator(
426
self,
427
request: ChatCompletionRequest,
425
- raw_request: Optional[Request],
428
result_generator: AsyncIterator[RequestOutput],
429
request_id: str,
430
conversation: List[ConversationMessage],
@@ -433,12 +435,12 @@ class OpenAIServingChat(OpenAIServing):
435
created_time = int(time.time())
436
final_res: Optional[RequestOutput] = None
437
436
- async for res in result_generator:
437
- if raw_request is not None and await raw_request.is_disconnected():
438
- # Abort the request if the client disconnects.
439
- await self.async_engine_client.abort(request_id)
440
- return self.create_error_response("Client disconnected")
441
- final_res = res
438
+ try:
439
+ async for res in result_generator:
440
+ final_res = res
441
+ except asyncio.CancelledError:
442
+ return self.create_error_response("Client disconnected")
443
+
444
assert final_res is not None
445
446
choices: List[ChatCompletionResponseChoice] = []
vllm/entrypoints/openai/serving_completion.py
+6
-14
@@ -1,3 +1,4 @@
1
+import asyncio
2
import time
3
from typing import (AsyncGenerator, AsyncIterator, Callable, Dict, List,
4
Optional)
@@ -84,7 +85,7 @@ class OpenAIServingCompletion(OpenAIServing):
85
created_time = int(time.time())
86
87
# Schedule the request and get the result generator.
87
- generators: List[AsyncIterator[RequestOutput]] = []
88
+ generators: List[AsyncGenerator[RequestOutput, None]] = []
89
try:
90
(
91
lora_request,
@@ -144,7 +145,8 @@ class OpenAIServingCompletion(OpenAIServing):
145
return self.create_error_response(str(e))
146
147
result_generator: AsyncIterator[Tuple[
147
- int, RequestOutput]] = merge_async_iterators(*generators)
148
+ int, RequestOutput]] = merge_async_iterators(
149
+ *generators, is_cancelled=raw_request.is_disconnected)
150
151
# Similar to the OpenAI API, when n != best_of, we do not stream the
152
# results. In addition, we do not stream the results when use
@@ -156,7 +158,6 @@ class OpenAIServingCompletion(OpenAIServing):
158
# Streaming response
159
if stream:
160
return self.completion_stream_generator(request,
159
- raw_request,
161
result_generator,
162
request_id,
163
created_time,
@@ -168,10 +169,6 @@ class OpenAIServingCompletion(OpenAIServing):
169
final_res_batch: List[Optional[RequestOutput]] = [None] * len(prompts)
170
try:
171
async for i, res in result_generator:
171
- if await raw_request.is_disconnected():
172
- # Abort the request if the client disconnects.
173
- await self.async_engine_client.abort(f"{request_id}-{i}")
174
- return self.create_error_response("Client disconnected")
172
final_res_batch[i] = res
173
174
for i, final_res in enumerate(final_res_batch):
@@ -194,6 +191,8 @@ class OpenAIServingCompletion(OpenAIServing):
191
model_name,
192
tokenizer,
193
)
194
+ except asyncio.CancelledError:
195
+ return self.create_error_response("Client disconnected")
196
except ValueError as e:
197
# TODO: Use a vllm-specific Validation Error
198
return self.create_error_response(str(e))
@@ -214,7 +213,6 @@ class OpenAIServingCompletion(OpenAIServing):
213
async def completion_stream_generator(
214
self,
215
request: CompletionRequest,
217
- raw_request: Request,
216
result_generator: AsyncIterator[Tuple[int, RequestOutput]],
217
request_id: str,
218
created_time: int,
@@ -230,12 +228,6 @@ class OpenAIServingCompletion(OpenAIServing):
228
try:
229
async for prompt_idx, res in result_generator:
230
233
- # Abort the request if the client disconnects.
234
- if await raw_request.is_disconnected():
235
- await self.async_engine_client.abort(
236
- f"{request_id}-{prompt_idx}")
237
- raise StopAsyncIteration()
238
-
231
for output in res.outputs:
232
i = output.index + prompt_idx * num_choices
233
# TODO(simon): optimize the performance by avoiding full
vllm/entrypoints/openai/serving_embedding.py
+7
-7
@@ -1,6 +1,7 @@
1
+import asyncio
2
import base64
3
import time
3
-from typing import AsyncIterator, List, Optional, Tuple, cast
4
+from typing import AsyncGenerator, AsyncIterator, List, Optional, Tuple, cast
5
6
import numpy as np
7
from fastapi import Request
@@ -92,7 +93,7 @@ class OpenAIServingEmbedding(OpenAIServing):
93
created_time = int(time.monotonic())
94
95
# Schedule the request and get the result generator.
95
- generators: List[AsyncIterator[EmbeddingRequestOutput]] = []
96
+ generators: List[AsyncGenerator[EmbeddingRequestOutput, None]] = []
97
try:
98
(
99
lora_request,
@@ -138,17 +139,14 @@ class OpenAIServingEmbedding(OpenAIServing):
139
return self.create_error_response(str(e))
140
141
result_generator: AsyncIterator[Tuple[
141
- int, EmbeddingRequestOutput]] = merge_async_iterators(*generators)
142
+ int, EmbeddingRequestOutput]] = merge_async_iterators(
143
+ *generators, is_cancelled=raw_request.is_disconnected)
144
145
# Non-streaming response
146
final_res_batch: List[Optional[EmbeddingRequestOutput]]
147
final_res_batch = [None] * len(prompts)
148
try:
149
async for i, res in result_generator:
148
- if await raw_request.is_disconnected():
149
- # Abort the request if the client disconnects.
150
- await self.async_engine_client.abort(f"{request_id}-{i}")
151
- return self.create_error_response("Client disconnected")
150
final_res_batch[i] = res
151
152
for final_res in final_res_batch:
@@ -160,6 +158,8 @@ class OpenAIServingEmbedding(OpenAIServing):
158
response = request_output_to_embedding_response(
159
final_res_batch_checked, request_id, created_time, model_name,
160
encoding_format)
161
+ except asyncio.CancelledError:
162
+ return self.create_error_response("Client disconnected")
163
except ValueError as e:
164
# TODO: Use a vllm-specific Validation Error
165
return self.create_error_response(str(e))
vllm/utils.py
+62
-49
@@ -1,5 +1,6 @@
1
import argparse
2
import asyncio
3
+import contextlib
4
import datetime
5
import enum
6
import gc
@@ -11,10 +12,11 @@ import tempfile
12
import threading
13
import uuid
14
import warnings
15
+from asyncio import FIRST_COMPLETED, ensure_future
16
from collections import defaultdict
17
from functools import lru_cache, partial, wraps
18
from platform import uname
17
-from typing import (Any, AsyncIterator, Awaitable, Callable, Dict, Generic,
19
+from typing import (Any, AsyncGenerator, Awaitable, Callable, Dict, Generic,
20
Hashable, List, Optional, OrderedDict, Set, Tuple, TypeVar,
21
Union, overload)
22
@@ -373,63 +375,74 @@ def make_async(func: Callable[P, T]) -> Callable[P, Awaitable[T]]:
375
return _async_wrapper
376
377
376
-class ProducerFinished:
377
- pass
378
+async def iterate_with_cancellation(
379
+ iterator: AsyncGenerator[T, None],
380
+ is_cancelled: Callable[[], Awaitable[bool]],
381
+) -> AsyncGenerator[T, None]:
382
+ """Convert async iterator into one that polls the provided function
383
+ at least once per second to check for client cancellation.
384
+ """
385
386
+ # Can use anext() in python >= 3.10
387
+ awaits = [ensure_future(iterator.__anext__())]
388
+ while True:
389
+ done, pending = await asyncio.wait(awaits, timeout=1)
390
+ if await is_cancelled():
391
+ with contextlib.suppress(BaseException):
392
+ awaits[0].cancel()
393
+ await iterator.aclose()
394
+ raise asyncio.CancelledError("client cancelled")
395
+ if done:
396
+ try:
397
+ item = await awaits[0]
398
+ awaits[0] = ensure_future(iterator.__anext__())
399
+ yield item
400
+ except StopAsyncIteration:
401
+ # we are done
402
+ return
403
380
-def merge_async_iterators(
381
- *iterators: AsyncIterator[T]) -> AsyncIterator[Tuple[int, T]]:
404
+
405
+async def merge_async_iterators(
406
+ *iterators: AsyncGenerator[T, None],
407
+ is_cancelled: Callable[[], Awaitable[bool]],
408
+) -> AsyncGenerator[Tuple[int, T], None]:
409
"""Merge multiple asynchronous iterators into a single iterator.
410
411
This method handle the case where some iterators finish before others.
412
When it yields, it yields a tuple (i, item) where i is the index of the
413
iterator that yields the item.
387
- """
388
- queue: asyncio.Queue[Union[Tuple[int, T], ProducerFinished,
389
- Exception]] = asyncio.Queue()
390
-
391
- producers = len(iterators)
392
-
393
- async def producer(i: int, iterator: AsyncIterator[T]):
394
- try:
395
- async for item in iterator:
396
- await queue.put((i, item))
397
- except Exception as e:
398
- await queue.put(e)
399
- # Signal to the consumer that we've finished
400
- await queue.put(ProducerFinished())
401
-
402
- _tasks = [
403
- asyncio.create_task(producer(i, iterator))
404
- for i, iterator in enumerate(iterators)
405
- ]
414
407
- async def consumer():
408
- remaining = producers
409
- try:
410
- while remaining or not queue.empty():
411
- # we think there is a race condition here
412
- item = await queue.get()
413
-
414
- if isinstance(item, ProducerFinished):
415
- # Signal that a producer finished- not a real item
416
- remaining -= 1
417
- continue
418
-
419
- if isinstance(item, Exception):
420
- raise item
421
- yield item
422
- except (Exception, asyncio.CancelledError) as e:
423
- for task in _tasks:
424
- if sys.version_info >= (3, 9):
425
- # msg parameter only supported in Python 3.9+
426
- task.cancel(e)
427
- else:
428
- task.cancel()
429
- raise e
430
- await asyncio.gather(*_tasks)
415
+ It also polls the provided function at least once per second to check
416
+ for client cancellation.
417
+ """
418
432
- return consumer()
419
+ # Can use anext() in python >= 3.10
420
+ awaits = {
421
+ ensure_future(pair[1].__anext__()): pair
422
+ for pair in enumerate(iterators)
423
+ }
424
+ try:
425
+ while awaits:
426
+ done, pending = await asyncio.wait(awaits.keys(),
427
+ return_when=FIRST_COMPLETED,
428
+ timeout=1)
429
+ if await is_cancelled():
430
+ raise asyncio.CancelledError("client cancelled")
431
+ for d in done:
432
+ pair = awaits.pop(d)
433
+ try:
434
+ item = await d
435
+ i, it = pair
436
+ awaits[ensure_future(it.__anext__())] = pair
437
+ yield i, item
438
+ except StopAsyncIteration:
439
+ pass
440
+ finally:
441
+ # Cancel any remaining iterators
442
+ for f, (_, it) in awaits.items():
443
+ with contextlib.suppress(BaseException):
444
+ f.cancel()
445
+ await it.aclose()
446
447
448
def get_ip() -> str: