A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 04:30 UTC
[Bugfix] Fix request cancellation without polling (#11190)
Joe Runde committed
Dec 17, 2024 at 13:26 UTC
2d1b9baa8f57fc59912c7bcd07fd630fb9d72c9d
12 files changed
+164
-103
tests/entrypoints/openai/test_basic.py
+51
@@ -1,6 +1,8 @@
1
+import asyncio
2
from http import HTTPStatus
3
from typing import List
4
5
+import openai
6
import pytest
7
import pytest_asyncio
8
import requests
@@ -103,3 +105,52 @@ async def test_check_health(server: RemoteOpenAIServer):
105
response = requests.get(server.url_for("health"))
106
107
assert response.status_code == HTTPStatus.OK
108
+
109
+
110
+@pytest.mark.parametrize(
111
+ "server_args",
112
+ [
113
+ pytest.param(["--max-model-len", "10100"],
114
+ id="default-frontend-multiprocessing"),
115
+ pytest.param(
116
+ ["--disable-frontend-multiprocessing", "--max-model-len", "10100"],
117
+ id="disable-frontend-multiprocessing")
118
+ ],
119
+ indirect=True,
120
+)
121
+@pytest.mark.asyncio
122
+async def test_request_cancellation(server: RemoteOpenAIServer):
123
+ # clunky test: send an ungodly amount of load in with short timeouts
124
+ # then ensure that it still responds quickly afterwards
125
+
126
+ chat_input = [{"role": "user", "content": "Write a long story"}]
127
+ client = server.get_async_client(timeout=0.5)
128
+ tasks = []
129
+ # Request about 2 million tokens
130
+ for _ in range(200):
131
+ task = asyncio.create_task(
132
+ client.chat.completions.create(messages=chat_input,
133
+ model=MODEL_NAME,
134
+ max_tokens=10000,
135
+ extra_body={"min_tokens": 10000}))
136
+ tasks.append(task)
137
+
138
+ done, pending = await asyncio.wait(tasks,
139
+ return_when=asyncio.ALL_COMPLETED)
140
+
141
+ # Make sure all requests were sent to the server and timed out
142
+ # (We don't want to hide other errors like 400s that would invalidate this
143
+ # test)
144
+ assert len(pending) == 0
145
+ for d in done:
146
+ with pytest.raises(openai.APITimeoutError):
147
+ d.result()
148
+
149
+ # If the server had not cancelled all the other requests, then it would not
150
+ # be able to respond to this one within the timeout
151
+ client = server.get_async_client(timeout=5)
152
+ response = await client.chat.completions.create(messages=chat_input,
153
+ model=MODEL_NAME,
154
+ max_tokens=10)
155
+
156
+ assert len(response.choices) == 1
tests/test_utils.py
+1
-5
@@ -1,7 +1,6 @@
1
import asyncio
2
import os
3
import socket
4
-from functools import partial
4
from typing import AsyncIterator, Tuple
5
6
import pytest
@@ -26,10 +25,7 @@ async def test_merge_async_iterators():
25
print(f"iterator {idx} cancelled")
26
27
iterators = [mock_async_iterator(i) for i in range(3)]
29
- merged_iterator = merge_async_iterators(*iterators,
30
- is_cancelled=partial(asyncio.sleep,
31
- 0,
32
- result=False))
28
+ merged_iterator = merge_async_iterators(*iterators)
29
30
async def stream_output(generator: AsyncIterator[Tuple[int, str]]):
31
async for idx, output in generator:
tests/utils.py
+5
-6
@@ -163,12 +163,11 @@ class RemoteOpenAIServer:
163
api_key=self.DUMMY_API_KEY,
164
)
165
166
- def get_async_client(self):
167
- return openai.AsyncOpenAI(
168
- base_url=self.url_for("v1"),
169
- api_key=self.DUMMY_API_KEY,
170
- max_retries=0,
171
- )
166
+ def get_async_client(self, **kwargs):
167
+ return openai.AsyncOpenAI(base_url=self.url_for("v1"),
168
+ api_key=self.DUMMY_API_KEY,
169
+ max_retries=0,
170
+ **kwargs)
171
172
173
def _test_completion(
vllm/engine/async_llm_engine.py
+27
-19
@@ -1065,16 +1065,20 @@ class AsyncLLMEngine(EngineClient):
1065
>>> # Process and return the final output
1066
>>> ...
1067
"""
1068
- async for output in await self.add_request(
1069
- request_id,
1070
- prompt,
1071
- sampling_params,
1072
- lora_request=lora_request,
1073
- trace_headers=trace_headers,
1074
- prompt_adapter_request=prompt_adapter_request,
1075
- priority=priority,
1076
- ):
1077
- yield LLMEngine.validate_output(output, RequestOutput)
1068
+ try:
1069
+ async for output in await self.add_request(
1070
+ request_id,
1071
+ prompt,
1072
+ sampling_params,
1073
+ lora_request=lora_request,
1074
+ trace_headers=trace_headers,
1075
+ prompt_adapter_request=prompt_adapter_request,
1076
+ priority=priority,
1077
+ ):
1078
+ yield LLMEngine.validate_output(output, RequestOutput)
1079
+ except asyncio.CancelledError:
1080
+ await self.abort(request_id)
1081
+ raise
1082
1083
async def encode(
1084
self,
@@ -1147,15 +1151,19 @@ class AsyncLLMEngine(EngineClient):
1151
>>> # Process and return the final output
1152
>>> ...
1153
"""
1150
- async for output in await self.add_request(
1151
- request_id,
1152
- prompt,
1153
- pooling_params,
1154
- lora_request=lora_request,
1155
- trace_headers=trace_headers,
1156
- priority=priority,
1157
- ):
1158
- yield LLMEngine.validate_output(output, PoolingRequestOutput)
1154
+ try:
1155
+ async for output in await self.add_request(
1156
+ request_id,
1157
+ prompt,
1158
+ pooling_params,
1159
+ lora_request=lora_request,
1160
+ trace_headers=trace_headers,
1161
+ priority=priority,
1162
+ ):
1163
+ yield LLMEngine.validate_output(output, PoolingRequestOutput)
1164
+ except asyncio.CancelledError:
1165
+ await self.abort(request_id)
1166
+ raise
1167
1168
async def abort(self, request_id: str) -> None:
1169
"""Abort a request.
vllm/entrypoints/api_server.py
+7
-4
@@ -17,11 +17,11 @@ from fastapi.responses import JSONResponse, Response, StreamingResponse
17
from vllm.engine.arg_utils import AsyncEngineArgs
18
from vllm.engine.async_llm_engine import AsyncLLMEngine
19
from vllm.entrypoints.launcher import serve_http
20
+from vllm.entrypoints.utils import with_cancellation
21
from vllm.logger import init_logger
22
from vllm.sampling_params import SamplingParams
23
from vllm.usage.usage_lib import UsageContext
23
-from vllm.utils import (FlexibleArgumentParser, iterate_with_cancellation,
24
- random_uuid)
24
+from vllm.utils import FlexibleArgumentParser, random_uuid
25
from vllm.version import __version__ as VLLM_VERSION
26
27
logger = init_logger("vllm.entrypoints.api_server")
@@ -47,6 +47,11 @@ async def generate(request: Request) -> Response:
47
- other fields: the sampling parameters (See `SamplingParams` for details).
48
"""
49
request_dict = await request.json()
50
+ return await _generate(request_dict, raw_request=request)
51
+
52
+
53
+@with_cancellation
54
+async def _generate(request_dict: dict, raw_request: Request) -> Response:
55
prompt = request_dict.pop("prompt")
56
stream = request_dict.pop("stream", False)
57
sampling_params = SamplingParams(**request_dict)
@@ -54,8 +59,6 @@ async def generate(request: Request) -> Response:
59
60
assert engine is not None
61
results_generator = engine.generate(prompt, sampling_params, request_id)
57
- results_generator = iterate_with_cancellation(
58
- results_generator, is_cancelled=request.is_disconnected)
62
63
# Streaming case
64
async def stream_results() -> AsyncGenerator[bytes, None]:
vllm/entrypoints/openai/api_server.py
+8
@@ -59,6 +59,7 @@ from vllm.entrypoints.openai.serving_score import OpenAIServingScores
59
from vllm.entrypoints.openai.serving_tokenization import (
60
OpenAIServingTokenization)
61
from vllm.entrypoints.openai.tool_parsers import ToolParserManager
62
+from vllm.entrypoints.utils import with_cancellation
63
from vllm.logger import init_logger
64
from vllm.usage.usage_lib import UsageContext
65
from vllm.utils import (FlexibleArgumentParser, get_open_zmq_ipc_path,
@@ -311,6 +312,7 @@ async def health(raw_request: Request) -> Response:
312
313
314
@router.post("/tokenize")
315
+@with_cancellation
316
async def tokenize(request: TokenizeRequest, raw_request: Request):
317
handler = tokenization(raw_request)
318
@@ -325,6 +327,7 @@ async def tokenize(request: TokenizeRequest, raw_request: Request):
327
328
329
@router.post("/detokenize")
330
+@with_cancellation
331
async def detokenize(request: DetokenizeRequest, raw_request: Request):
332
handler = tokenization(raw_request)
333
@@ -353,6 +356,7 @@ async def show_version():
356
357
358
@router.post("/v1/chat/completions")
359
+@with_cancellation
360
async def create_chat_completion(request: ChatCompletionRequest,
361
raw_request: Request):
362
handler = chat(raw_request)
@@ -373,6 +377,7 @@ async def create_chat_completion(request: ChatCompletionRequest,
377
378
379
@router.post("/v1/completions")
380
+@with_cancellation
381
async def create_completion(request: CompletionRequest, raw_request: Request):
382
handler = completion(raw_request)
383
if handler is None:
@@ -390,6 +395,7 @@ async def create_completion(request: CompletionRequest, raw_request: Request):
395
396
397
@router.post("/v1/embeddings")
398
+@with_cancellation
399
async def create_embedding(request: EmbeddingRequest, raw_request: Request):
400
handler = embedding(raw_request)
401
if handler is None:
@@ -407,6 +413,7 @@ async def create_embedding(request: EmbeddingRequest, raw_request: Request):
413
414
415
@router.post("/score")
416
+@with_cancellation
417
async def create_score(request: ScoreRequest, raw_request: Request):
418
handler = score(raw_request)
419
if handler is None:
@@ -424,6 +431,7 @@ async def create_score(request: ScoreRequest, raw_request: Request):
431
432
433
@router.post("/v1/score")
434
+@with_cancellation
435
async def create_score_v1(request: ScoreRequest, raw_request: Request):
436
logger.warning(
437
"To indicate that Score API is not part of standard OpenAI API, we "
vllm/entrypoints/openai/serving_chat.py
-5
@@ -32,7 +32,6 @@ from vllm.sampling_params import BeamSearchParams, SamplingParams
32
from vllm.sequence import Logprob
33
from vllm.transformers_utils.tokenizer import AnyTokenizer, MistralTokenizer
34
from vllm.transformers_utils.tokenizers import maybe_serialize_tool_calls
35
-from vllm.utils import iterate_with_cancellation
35
36
logger = init_logger(__name__)
37
@@ -234,10 +233,6 @@ class OpenAIServingChat(OpenAIServing):
233
assert len(generators) == 1
234
result_generator, = generators
235
237
- if raw_request:
238
- result_generator = iterate_with_cancellation(
239
- result_generator, raw_request.is_disconnected)
240
-
236
# Streaming response
237
if request.stream:
238
return self.chat_completion_stream_generator(
vllm/entrypoints/openai/serving_completion.py
+1
-2
@@ -159,8 +159,7 @@ class OpenAIServingCompletion(OpenAIServing):
159
# TODO: Use a vllm-specific Validation Error
160
return self.create_error_response(str(e))
161
162
- result_generator = merge_async_iterators(
163
- *generators, is_cancelled=raw_request.is_disconnected)
162
+ result_generator = merge_async_iterators(*generators)
163
164
model_name = self._get_model_name(lora_request)
165
num_prompts = len(engine_prompts)
vllm/entrypoints/openai/serving_embedding.py
+1
-4
@@ -202,10 +202,7 @@ class OpenAIServingEmbedding(OpenAIServing):
202
# TODO: Use a vllm-specific Validation Error
203
return self.create_error_response(str(e))
204
205
- result_generator = merge_async_iterators(
206
- *generators,
207
- is_cancelled=raw_request.is_disconnected if raw_request else None,
208
- )
205
+ result_generator = merge_async_iterators(*generators)
206
207
num_prompts = len(engine_prompts)
208
vllm/entrypoints/openai/serving_score.py
+1
-4
@@ -186,10 +186,7 @@ class OpenAIServingScores(OpenAIServing):
186
# TODO: Use a vllm-specific Validation Error
187
return self.create_error_response(str(e))
188
189
- result_generator = merge_async_iterators(
190
- *generators,
191
- is_cancelled=raw_request.is_disconnected if raw_request else None,
192
- )
189
+ result_generator = merge_async_iterators(*generators)
190
191
num_prompts = len(engine_prompts)
192
vllm/entrypoints/utils.py
new
+57
@@ -0,0 +1,57 @@
1
+import asyncio
2
+import functools
3
+
4
+from fastapi import Request
5
+
6
+
7
+async def listen_for_disconnect(request: Request) -> None:
8
+ """Returns if a disconnect message is received"""
9
+ while True:
10
+ message = await request.receive()
11
+ if message["type"] == "http.disconnect":
12
+ break
13
+
14
+
15
+def with_cancellation(handler_func):
16
+ """Decorator that allows a route handler to be cancelled by client
17
+ disconnections.
18
+
19
+ This does _not_ use request.is_disconnected, which does not work with
20
+ middleware. Instead this follows the pattern from
21
+ starlette.StreamingResponse, which simultaneously awaits on two tasks- one
22
+ to wait for an http disconnect message, and the other to do the work that we
23
+ want done. When the first task finishes, the other is cancelled.
24
+
25
+ A core assumption of this method is that the body of the request has already
26
+ been read. This is a safe assumption to make for fastapi handlers that have
27
+ already parsed the body of the request into a pydantic model for us.
28
+ This decorator is unsafe to use elsewhere, as it will consume and throw away
29
+ all incoming messages for the request while it looks for a disconnect
30
+ message.
31
+
32
+ In the case where a `StreamingResponse` is returned by the handler, this
33
+ wrapper will stop listening for disconnects and instead the response object
34
+ will start listening for disconnects.
35
+ """
36
+
37
+ # Functools.wraps is required for this wrapper to appear to fastapi as a
38
+ # normal route handler, with the correct request type hinting.
39
+ @functools.wraps(handler_func)
40
+ async def wrapper(*args, **kwargs):
41
+
42
+ # The request is either the second positional arg or `raw_request`
43
+ request = args[1] if len(args) > 1 else kwargs["raw_request"]
44
+
45
+ handler_task = asyncio.create_task(handler_func(*args, **kwargs))
46
+ cancellation_task = asyncio.create_task(listen_for_disconnect(request))
47
+
48
+ done, pending = await asyncio.wait([handler_task, cancellation_task],
49
+ return_when=asyncio.FIRST_COMPLETED)
50
+ for task in pending:
51
+ task.cancel()
52
+
53
+ if handler_task in done:
54
+ return handler_task.result()
55
+ return None
56
+
57
+ return wrapper
vllm/utils.py
+5
-54
@@ -20,7 +20,7 @@ import time
20
import uuid
21
import warnings
22
import weakref
23
-from asyncio import FIRST_COMPLETED, AbstractEventLoop, Future, Task
23
+from asyncio import FIRST_COMPLETED, AbstractEventLoop, Task
24
from collections import UserDict, defaultdict
25
from collections.abc import Iterable, Mapping
26
from dataclasses import dataclass, field
@@ -370,72 +370,23 @@ def _next_task(iterator: AsyncGenerator[T, None],
370
return loop.create_task(iterator.__anext__()) # type: ignore[arg-type]
371
372
373
-async def iterate_with_cancellation(
374
- iterator: AsyncGenerator[T, None],
375
- is_cancelled: Callable[[], Awaitable[bool]],
376
-) -> AsyncGenerator[T, None]:
377
- """Convert async iterator into one that polls the provided function
378
- at least once per second to check for client cancellation.
379
- """
380
-
381
- loop = asyncio.get_running_loop()
382
-
383
- awaits: List[Future[T]] = [_next_task(iterator, loop)]
384
- next_cancel_check: float = 0
385
- while True:
386
- done, pending = await asyncio.wait(awaits, timeout=1.5)
387
-
388
- # Check for cancellation at most once per second
389
- time_now = time.time()
390
- if time_now >= next_cancel_check:
391
- if await is_cancelled():
392
- with contextlib.suppress(BaseException):
393
- awaits[0].cancel()
394
- await iterator.aclose()
395
- raise asyncio.CancelledError("client cancelled")
396
- next_cancel_check = time_now + 1
397
-
398
- if done:
399
- try:
400
- item = await awaits[0]
401
- awaits[0] = _next_task(iterator, loop)
402
- yield item
403
- except StopAsyncIteration:
404
- # we are done
405
- return
406
-
407
-
373
async def merge_async_iterators(
409
- *iterators: AsyncGenerator[T, None],
410
- is_cancelled: Optional[Callable[[], Awaitable[bool]]] = None,
411
-) -> AsyncGenerator[Tuple[int, T], None]:
374
+ *iterators: AsyncGenerator[T,
375
+ None], ) -> AsyncGenerator[Tuple[int, T], None]:
376
"""Merge multiple asynchronous iterators into a single iterator.
377
378
This method handle the case where some iterators finish before others.
379
When it yields, it yields a tuple (i, item) where i is the index of the
380
iterator that yields the item.
417
-
418
- It also optionally polls a provided function at least once per second
419
- to check for client cancellation.
381
"""
382
383
loop = asyncio.get_running_loop()
384
385
awaits = {_next_task(pair[1], loop): pair for pair in enumerate(iterators)}
425
- timeout = None if is_cancelled is None else 1.5
426
- next_cancel_check: float = 0
386
try:
387
while awaits:
429
- done, pending = await asyncio.wait(awaits.keys(),
430
- return_when=FIRST_COMPLETED,
431
- timeout=timeout)
432
- if is_cancelled is not None:
433
- # Check for cancellation at most once per second
434
- time_now = time.time()
435
- if time_now >= next_cancel_check:
436
- if await is_cancelled():
437
- raise asyncio.CancelledError("client cancelled")
438
- next_cancel_check = time_now + 1
388
+ done, _ = await asyncio.wait(awaits.keys(),
389
+ return_when=FIRST_COMPLETED)
390
for d in done:
391
pair = awaits.pop(d)
392
try: