@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 2d1b9baa8f
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 04:30 UTC

[Bugfix] Fix request cancellation without polling (#11190)

Joe Runde committed Dec 17, 2024 at 13:26 UTC 2d1b9baa8f57fc59912c7bcd07fd630fb9d72c9d
12 files changed +164 -103
tests/entrypoints/openai/test_basic.py
+51
@@ -1,6 +1,8 @@
1 +import asyncio
2 from http import HTTPStatus
3 from typing import List
4
5 +import openai
6 import pytest
7 import pytest_asyncio
8 import requests
@@ -103,3 +105,52 @@ async def test_check_health(server: RemoteOpenAIServer):
105 response = requests.get(server.url_for("health"))
106
107 assert response.status_code == HTTPStatus.OK
108 +
109 +
110 +@pytest.mark.parametrize(
111 + "server_args",
112 + [
113 + pytest.param(["--max-model-len", "10100"],
114 + id="default-frontend-multiprocessing"),
115 + pytest.param(
116 + ["--disable-frontend-multiprocessing", "--max-model-len", "10100"],
117 + id="disable-frontend-multiprocessing")
118 + ],
119 + indirect=True,
120 +)
121 +@pytest.mark.asyncio
122 +async def test_request_cancellation(server: RemoteOpenAIServer):
123 + # clunky test: send an ungodly amount of load in with short timeouts
124 + # then ensure that it still responds quickly afterwards
125 +
126 + chat_input = [{"role": "user", "content": "Write a long story"}]
127 + client = server.get_async_client(timeout=0.5)
128 + tasks = []
129 + # Request about 2 million tokens
130 + for _ in range(200):
131 + task = asyncio.create_task(
132 + client.chat.completions.create(messages=chat_input,
133 + model=MODEL_NAME,
134 + max_tokens=10000,
135 + extra_body={"min_tokens": 10000}))
136 + tasks.append(task)
137 +
138 + done, pending = await asyncio.wait(tasks,
139 + return_when=asyncio.ALL_COMPLETED)
140 +
141 + # Make sure all requests were sent to the server and timed out
142 + # (We don't want to hide other errors like 400s that would invalidate this
143 + # test)
144 + assert len(pending) == 0
145 + for d in done:
146 + with pytest.raises(openai.APITimeoutError):
147 + d.result()
148 +
149 + # If the server had not cancelled all the other requests, then it would not
150 + # be able to respond to this one within the timeout
151 + client = server.get_async_client(timeout=5)
152 + response = await client.chat.completions.create(messages=chat_input,
153 + model=MODEL_NAME,
154 + max_tokens=10)
155 +
156 + assert len(response.choices) == 1
tests/test_utils.py
+1 -5
@@ -1,7 +1,6 @@
1 import asyncio
2 import os
3 import socket
4 -from functools import partial
4 from typing import AsyncIterator, Tuple
5
6 import pytest
@@ -26,10 +25,7 @@ async def test_merge_async_iterators():
25 print(f"iterator {idx} cancelled")
26
27 iterators = [mock_async_iterator(i) for i in range(3)]
29 - merged_iterator = merge_async_iterators(*iterators,
30 - is_cancelled=partial(asyncio.sleep,
31 - 0,
32 - result=False))
28 + merged_iterator = merge_async_iterators(*iterators)
29
30 async def stream_output(generator: AsyncIterator[Tuple[int, str]]):
31 async for idx, output in generator:
tests/utils.py
+5 -6
@@ -163,12 +163,11 @@ class RemoteOpenAIServer:
163 api_key=self.DUMMY_API_KEY,
164 )
165
166 - def get_async_client(self):
167 - return openai.AsyncOpenAI(
168 - base_url=self.url_for("v1"),
169 - api_key=self.DUMMY_API_KEY,
170 - max_retries=0,
171 - )
166 + def get_async_client(self, **kwargs):
167 + return openai.AsyncOpenAI(base_url=self.url_for("v1"),
168 + api_key=self.DUMMY_API_KEY,
169 + max_retries=0,
170 + **kwargs)
171
172
173 def _test_completion(
vllm/engine/async_llm_engine.py
+27 -19
@@ -1065,16 +1065,20 @@ class AsyncLLMEngine(EngineClient):
1065 >>> # Process and return the final output
1066 >>> ...
1067 """
1068 - async for output in await self.add_request(
1069 - request_id,
1070 - prompt,
1071 - sampling_params,
1072 - lora_request=lora_request,
1073 - trace_headers=trace_headers,
1074 - prompt_adapter_request=prompt_adapter_request,
1075 - priority=priority,
1076 - ):
1077 - yield LLMEngine.validate_output(output, RequestOutput)
1068 + try:
1069 + async for output in await self.add_request(
1070 + request_id,
1071 + prompt,
1072 + sampling_params,
1073 + lora_request=lora_request,
1074 + trace_headers=trace_headers,
1075 + prompt_adapter_request=prompt_adapter_request,
1076 + priority=priority,
1077 + ):
1078 + yield LLMEngine.validate_output(output, RequestOutput)
1079 + except asyncio.CancelledError:
1080 + await self.abort(request_id)
1081 + raise
1082
1083 async def encode(
1084 self,
@@ -1147,15 +1151,19 @@ class AsyncLLMEngine(EngineClient):
1151 >>> # Process and return the final output
1152 >>> ...
1153 """
1150 - async for output in await self.add_request(
1151 - request_id,
1152 - prompt,
1153 - pooling_params,
1154 - lora_request=lora_request,
1155 - trace_headers=trace_headers,
1156 - priority=priority,
1157 - ):
1158 - yield LLMEngine.validate_output(output, PoolingRequestOutput)
1154 + try:
1155 + async for output in await self.add_request(
1156 + request_id,
1157 + prompt,
1158 + pooling_params,
1159 + lora_request=lora_request,
1160 + trace_headers=trace_headers,
1161 + priority=priority,
1162 + ):
1163 + yield LLMEngine.validate_output(output, PoolingRequestOutput)
1164 + except asyncio.CancelledError:
1165 + await self.abort(request_id)
1166 + raise
1167
1168 async def abort(self, request_id: str) -> None:
1169 """Abort a request.
vllm/entrypoints/api_server.py
+7 -4
@@ -17,11 +17,11 @@ from fastapi.responses import JSONResponse, Response, StreamingResponse
17 from vllm.engine.arg_utils import AsyncEngineArgs
18 from vllm.engine.async_llm_engine import AsyncLLMEngine
19 from vllm.entrypoints.launcher import serve_http
20 +from vllm.entrypoints.utils import with_cancellation
21 from vllm.logger import init_logger
22 from vllm.sampling_params import SamplingParams
23 from vllm.usage.usage_lib import UsageContext
23 -from vllm.utils import (FlexibleArgumentParser, iterate_with_cancellation,
24 - random_uuid)
24 +from vllm.utils import FlexibleArgumentParser, random_uuid
25 from vllm.version import __version__ as VLLM_VERSION
26
27 logger = init_logger("vllm.entrypoints.api_server")
@@ -47,6 +47,11 @@ async def generate(request: Request) -> Response:
47 - other fields: the sampling parameters (See `SamplingParams` for details).
48 """
49 request_dict = await request.json()
50 + return await _generate(request_dict, raw_request=request)
51 +
52 +
53 +@with_cancellation
54 +async def _generate(request_dict: dict, raw_request: Request) -> Response:
55 prompt = request_dict.pop("prompt")
56 stream = request_dict.pop("stream", False)
57 sampling_params = SamplingParams(**request_dict)
@@ -54,8 +59,6 @@ async def generate(request: Request) -> Response:
59
60 assert engine is not None
61 results_generator = engine.generate(prompt, sampling_params, request_id)
57 - results_generator = iterate_with_cancellation(
58 - results_generator, is_cancelled=request.is_disconnected)
62
63 # Streaming case
64 async def stream_results() -> AsyncGenerator[bytes, None]:
vllm/entrypoints/openai/api_server.py
+8
@@ -59,6 +59,7 @@ from vllm.entrypoints.openai.serving_score import OpenAIServingScores
59 from vllm.entrypoints.openai.serving_tokenization import (
60 OpenAIServingTokenization)
61 from vllm.entrypoints.openai.tool_parsers import ToolParserManager
62 +from vllm.entrypoints.utils import with_cancellation
63 from vllm.logger import init_logger
64 from vllm.usage.usage_lib import UsageContext
65 from vllm.utils import (FlexibleArgumentParser, get_open_zmq_ipc_path,
@@ -311,6 +312,7 @@ async def health(raw_request: Request) -> Response:
312
313
314 @router.post("/tokenize")
315 +@with_cancellation
316 async def tokenize(request: TokenizeRequest, raw_request: Request):
317 handler = tokenization(raw_request)
318
@@ -325,6 +327,7 @@ async def tokenize(request: TokenizeRequest, raw_request: Request):
327
328
329 @router.post("/detokenize")
330 +@with_cancellation
331 async def detokenize(request: DetokenizeRequest, raw_request: Request):
332 handler = tokenization(raw_request)
333
@@ -353,6 +356,7 @@ async def show_version():
356
357
358 @router.post("/v1/chat/completions")
359 +@with_cancellation
360 async def create_chat_completion(request: ChatCompletionRequest,
361 raw_request: Request):
362 handler = chat(raw_request)
@@ -373,6 +377,7 @@ async def create_chat_completion(request: ChatCompletionRequest,
377
378
379 @router.post("/v1/completions")
380 +@with_cancellation
381 async def create_completion(request: CompletionRequest, raw_request: Request):
382 handler = completion(raw_request)
383 if handler is None:
@@ -390,6 +395,7 @@ async def create_completion(request: CompletionRequest, raw_request: Request):
395
396
397 @router.post("/v1/embeddings")
398 +@with_cancellation
399 async def create_embedding(request: EmbeddingRequest, raw_request: Request):
400 handler = embedding(raw_request)
401 if handler is None:
@@ -407,6 +413,7 @@ async def create_embedding(request: EmbeddingRequest, raw_request: Request):
413
414
415 @router.post("/score")
416 +@with_cancellation
417 async def create_score(request: ScoreRequest, raw_request: Request):
418 handler = score(raw_request)
419 if handler is None:
@@ -424,6 +431,7 @@ async def create_score(request: ScoreRequest, raw_request: Request):
431
432
433 @router.post("/v1/score")
434 +@with_cancellation
435 async def create_score_v1(request: ScoreRequest, raw_request: Request):
436 logger.warning(
437 "To indicate that Score API is not part of standard OpenAI API, we "
vllm/entrypoints/openai/serving_chat.py
-5
@@ -32,7 +32,6 @@ from vllm.sampling_params import BeamSearchParams, SamplingParams
32 from vllm.sequence import Logprob
33 from vllm.transformers_utils.tokenizer import AnyTokenizer, MistralTokenizer
34 from vllm.transformers_utils.tokenizers import maybe_serialize_tool_calls
35 -from vllm.utils import iterate_with_cancellation
35
36 logger = init_logger(__name__)
37
@@ -234,10 +233,6 @@ class OpenAIServingChat(OpenAIServing):
233 assert len(generators) == 1
234 result_generator, = generators
235
237 - if raw_request:
238 - result_generator = iterate_with_cancellation(
239 - result_generator, raw_request.is_disconnected)
240 -
236 # Streaming response
237 if request.stream:
238 return self.chat_completion_stream_generator(
vllm/entrypoints/openai/serving_completion.py
+1 -2
@@ -159,8 +159,7 @@ class OpenAIServingCompletion(OpenAIServing):
159 # TODO: Use a vllm-specific Validation Error
160 return self.create_error_response(str(e))
161
162 - result_generator = merge_async_iterators(
163 - *generators, is_cancelled=raw_request.is_disconnected)
162 + result_generator = merge_async_iterators(*generators)
163
164 model_name = self._get_model_name(lora_request)
165 num_prompts = len(engine_prompts)
vllm/entrypoints/openai/serving_embedding.py
+1 -4
@@ -202,10 +202,7 @@ class OpenAIServingEmbedding(OpenAIServing):
202 # TODO: Use a vllm-specific Validation Error
203 return self.create_error_response(str(e))
204
205 - result_generator = merge_async_iterators(
206 - *generators,
207 - is_cancelled=raw_request.is_disconnected if raw_request else None,
208 - )
205 + result_generator = merge_async_iterators(*generators)
206
207 num_prompts = len(engine_prompts)
208
vllm/entrypoints/openai/serving_score.py
+1 -4
@@ -186,10 +186,7 @@ class OpenAIServingScores(OpenAIServing):
186 # TODO: Use a vllm-specific Validation Error
187 return self.create_error_response(str(e))
188
189 - result_generator = merge_async_iterators(
190 - *generators,
191 - is_cancelled=raw_request.is_disconnected if raw_request else None,
192 - )
189 + result_generator = merge_async_iterators(*generators)
190
191 num_prompts = len(engine_prompts)
192
vllm/entrypoints/utils.py new
+57
@@ -0,0 +1,57 @@
1 +import asyncio
2 +import functools
3 +
4 +from fastapi import Request
5 +
6 +
7 +async def listen_for_disconnect(request: Request) -> None:
8 + """Returns if a disconnect message is received"""
9 + while True:
10 + message = await request.receive()
11 + if message["type"] == "http.disconnect":
12 + break
13 +
14 +
15 +def with_cancellation(handler_func):
16 + """Decorator that allows a route handler to be cancelled by client
17 + disconnections.
18 +
19 + This does _not_ use request.is_disconnected, which does not work with
20 + middleware. Instead this follows the pattern from
21 + starlette.StreamingResponse, which simultaneously awaits on two tasks- one
22 + to wait for an http disconnect message, and the other to do the work that we
23 + want done. When the first task finishes, the other is cancelled.
24 +
25 + A core assumption of this method is that the body of the request has already
26 + been read. This is a safe assumption to make for fastapi handlers that have
27 + already parsed the body of the request into a pydantic model for us.
28 + This decorator is unsafe to use elsewhere, as it will consume and throw away
29 + all incoming messages for the request while it looks for a disconnect
30 + message.
31 +
32 + In the case where a `StreamingResponse` is returned by the handler, this
33 + wrapper will stop listening for disconnects and instead the response object
34 + will start listening for disconnects.
35 + """
36 +
37 + # Functools.wraps is required for this wrapper to appear to fastapi as a
38 + # normal route handler, with the correct request type hinting.
39 + @functools.wraps(handler_func)
40 + async def wrapper(*args, **kwargs):
41 +
42 + # The request is either the second positional arg or `raw_request`
43 + request = args[1] if len(args) > 1 else kwargs["raw_request"]
44 +
45 + handler_task = asyncio.create_task(handler_func(*args, **kwargs))
46 + cancellation_task = asyncio.create_task(listen_for_disconnect(request))
47 +
48 + done, pending = await asyncio.wait([handler_task, cancellation_task],
49 + return_when=asyncio.FIRST_COMPLETED)
50 + for task in pending:
51 + task.cancel()
52 +
53 + if handler_task in done:
54 + return handler_task.result()
55 + return None
56 +
57 + return wrapper
vllm/utils.py
+5 -54
@@ -20,7 +20,7 @@ import time
20 import uuid
21 import warnings
22 import weakref
23 -from asyncio import FIRST_COMPLETED, AbstractEventLoop, Future, Task
23 +from asyncio import FIRST_COMPLETED, AbstractEventLoop, Task
24 from collections import UserDict, defaultdict
25 from collections.abc import Iterable, Mapping
26 from dataclasses import dataclass, field
@@ -370,72 +370,23 @@ def _next_task(iterator: AsyncGenerator[T, None],
370 return loop.create_task(iterator.__anext__()) # type: ignore[arg-type]
371
372
373 -async def iterate_with_cancellation(
374 - iterator: AsyncGenerator[T, None],
375 - is_cancelled: Callable[[], Awaitable[bool]],
376 -) -> AsyncGenerator[T, None]:
377 - """Convert async iterator into one that polls the provided function
378 - at least once per second to check for client cancellation.
379 - """
380 -
381 - loop = asyncio.get_running_loop()
382 -
383 - awaits: List[Future[T]] = [_next_task(iterator, loop)]
384 - next_cancel_check: float = 0
385 - while True:
386 - done, pending = await asyncio.wait(awaits, timeout=1.5)
387 -
388 - # Check for cancellation at most once per second
389 - time_now = time.time()
390 - if time_now >= next_cancel_check:
391 - if await is_cancelled():
392 - with contextlib.suppress(BaseException):
393 - awaits[0].cancel()
394 - await iterator.aclose()
395 - raise asyncio.CancelledError("client cancelled")
396 - next_cancel_check = time_now + 1
397 -
398 - if done:
399 - try:
400 - item = await awaits[0]
401 - awaits[0] = _next_task(iterator, loop)
402 - yield item
403 - except StopAsyncIteration:
404 - # we are done
405 - return
406 -
407 -
373 async def merge_async_iterators(
409 - *iterators: AsyncGenerator[T, None],
410 - is_cancelled: Optional[Callable[[], Awaitable[bool]]] = None,
411 -) -> AsyncGenerator[Tuple[int, T], None]:
374 + *iterators: AsyncGenerator[T,
375 + None], ) -> AsyncGenerator[Tuple[int, T], None]:
376 """Merge multiple asynchronous iterators into a single iterator.
377
378 This method handle the case where some iterators finish before others.
379 When it yields, it yields a tuple (i, item) where i is the index of the
380 iterator that yields the item.
417 -
418 - It also optionally polls a provided function at least once per second
419 - to check for client cancellation.
381 """
382
383 loop = asyncio.get_running_loop()
384
385 awaits = {_next_task(pair[1], loop): pair for pair in enumerate(iterators)}
425 - timeout = None if is_cancelled is None else 1.5
426 - next_cancel_check: float = 0
386 try:
387 while awaits:
429 - done, pending = await asyncio.wait(awaits.keys(),
430 - return_when=FIRST_COMPLETED,
431 - timeout=timeout)
432 - if is_cancelled is not None:
433 - # Check for cancellation at most once per second
434 - time_now = time.time()
435 - if time_now >= next_cancel_check:
436 - if await is_cancelled():
437 - raise asyncio.CancelledError("client cancelled")
438 - next_cancel_check = time_now + 1
388 + done, _ = await asyncio.wait(awaits.keys(),
389 + return_when=FIRST_COMPLETED)
390 for d in done:
391 pair = awaits.pop(d)
392 try: