A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 18:15 UTC
[Frontend] Improve Startup Failure UX (#7716)
Robert Shaw committed
Aug 21, 2024 at 15:53 UTC
970dfdc01d3453c83066e6156278d70bade0350c
2 files changed
+37
-19
tests/entrypoints/openai/test_mp_api_server.py
+16
-13
@@ -1,3 +1,5 @@
1
+import time
2
+
3
import pytest
4
5
from vllm.entrypoints.openai.api_server import build_async_engine_client
@@ -8,19 +10,20 @@ from vllm.utils import FlexibleArgumentParser
10
@pytest.mark.asyncio
11
async def test_mp_crash_detection():
12
11
- with pytest.raises(RuntimeError) as excinfo:
12
- parser = FlexibleArgumentParser(
13
- description="vLLM's remote OpenAI server.")
14
- parser = make_arg_parser(parser)
15
- args = parser.parse_args([])
16
- # use an invalid tensor_parallel_size to trigger the
17
- # error in the server
18
- args.tensor_parallel_size = 65536
19
-
20
- async with build_async_engine_client(args):
21
- pass
22
- assert "The server process died before responding to the readiness probe"\
23
- in str(excinfo.value)
13
+ parser = FlexibleArgumentParser(description="vLLM's remote OpenAI server.")
14
+ parser = make_arg_parser(parser)
15
+ args = parser.parse_args([])
16
+ # use an invalid tensor_parallel_size to trigger the
17
+ # error in the server
18
+ args.tensor_parallel_size = 65536
19
+
20
+ start = time.perf_counter()
21
+ async with build_async_engine_client(args):
22
+ pass
23
+ end = time.perf_counter()
24
+
25
+ assert end - start < 60, ("Expected vLLM to gracefully shutdown in <60s "
26
+ "if there is an error in the startup.")
27
28
29
@pytest.mark.asyncio
vllm/entrypoints/openai/api_server.py
+21
-6
@@ -8,7 +8,7 @@ import tempfile
8
from argparse import Namespace
9
from contextlib import asynccontextmanager
10
from http import HTTPStatus
11
-from typing import AsyncIterator, Set
11
+from typing import AsyncIterator, Optional, Set
12
13
from fastapi import APIRouter, FastAPI, Request
14
from fastapi.exceptions import RequestValidationError
@@ -60,6 +60,7 @@ openai_serving_embedding: OpenAIServingEmbedding
60
openai_serving_tokenization: OpenAIServingTokenization
61
prometheus_multiproc_dir: tempfile.TemporaryDirectory
62
63
+# Cannot use __name__ (https://github.com/vllm-project/vllm/pull/4765)
64
logger = init_logger('vllm.entrypoints.openai.api_server')
65
66
_running_tasks: Set[asyncio.Task] = set()
@@ -94,7 +95,15 @@ async def lifespan(app: FastAPI):
95
96
@asynccontextmanager
97
async def build_async_engine_client(
97
- args: Namespace) -> AsyncIterator[AsyncEngineClient]:
98
+ args: Namespace) -> AsyncIterator[Optional[AsyncEngineClient]]:
99
+ """
100
+ Create AsyncEngineClient, either:
101
+ - in-process using the AsyncLLMEngine Directly
102
+ - multiprocess using AsyncLLMEngine RPC
103
+
104
+ Returns the Client or None if the creation failed.
105
+ """
106
+
107
# Context manager to handle async_engine_client lifecycle
108
# Ensures everything is shutdown and cleaned up on error/exit
109
global engine_args
@@ -157,11 +166,13 @@ async def build_async_engine_client(
166
try:
167
await rpc_client.setup()
168
break
160
- except TimeoutError as e:
169
+ except TimeoutError:
170
if not rpc_server_process.is_alive():
162
- raise RuntimeError(
163
- "The server process died before "
164
- "responding to the readiness probe") from e
171
+ logger.error(
172
+ "RPCServer process died before responding "
173
+ "to readiness probe")
174
+ yield None
175
+ return
176
177
yield async_engine_client
178
finally:
@@ -410,6 +421,10 @@ async def run_server(args, **uvicorn_kwargs) -> None:
421
logger.info("args: %s", args)
422
423
async with build_async_engine_client(args) as async_engine_client:
424
+ # If None, creation of the client failed and we exit.
425
+ if async_engine_client is None:
426
+ return
427
+
428
app = await init_app(async_engine_client, args)
429
430
shutdown_task = await serve_http(