@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 970dfdc01d
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 18:15 UTC

[Frontend] Improve Startup Failure UX (#7716)

Robert Shaw committed Aug 21, 2024 at 15:53 UTC 970dfdc01d3453c83066e6156278d70bade0350c
2 files changed +37 -19
tests/entrypoints/openai/test_mp_api_server.py
+16 -13
@@ -1,3 +1,5 @@
1 +import time
2 +
3 import pytest
4
5 from vllm.entrypoints.openai.api_server import build_async_engine_client
@@ -8,19 +10,20 @@ from vllm.utils import FlexibleArgumentParser
10 @pytest.mark.asyncio
11 async def test_mp_crash_detection():
12
11 - with pytest.raises(RuntimeError) as excinfo:
12 - parser = FlexibleArgumentParser(
13 - description="vLLM's remote OpenAI server.")
14 - parser = make_arg_parser(parser)
15 - args = parser.parse_args([])
16 - # use an invalid tensor_parallel_size to trigger the
17 - # error in the server
18 - args.tensor_parallel_size = 65536
19 -
20 - async with build_async_engine_client(args):
21 - pass
22 - assert "The server process died before responding to the readiness probe"\
23 - in str(excinfo.value)
13 + parser = FlexibleArgumentParser(description="vLLM's remote OpenAI server.")
14 + parser = make_arg_parser(parser)
15 + args = parser.parse_args([])
16 + # use an invalid tensor_parallel_size to trigger the
17 + # error in the server
18 + args.tensor_parallel_size = 65536
19 +
20 + start = time.perf_counter()
21 + async with build_async_engine_client(args):
22 + pass
23 + end = time.perf_counter()
24 +
25 + assert end - start < 60, ("Expected vLLM to gracefully shutdown in <60s "
26 + "if there is an error in the startup.")
27
28
29 @pytest.mark.asyncio
vllm/entrypoints/openai/api_server.py
+21 -6
@@ -8,7 +8,7 @@ import tempfile
8 from argparse import Namespace
9 from contextlib import asynccontextmanager
10 from http import HTTPStatus
11 -from typing import AsyncIterator, Set
11 +from typing import AsyncIterator, Optional, Set
12
13 from fastapi import APIRouter, FastAPI, Request
14 from fastapi.exceptions import RequestValidationError
@@ -60,6 +60,7 @@ openai_serving_embedding: OpenAIServingEmbedding
60 openai_serving_tokenization: OpenAIServingTokenization
61 prometheus_multiproc_dir: tempfile.TemporaryDirectory
62
63 +# Cannot use __name__ (https://github.com/vllm-project/vllm/pull/4765)
64 logger = init_logger('vllm.entrypoints.openai.api_server')
65
66 _running_tasks: Set[asyncio.Task] = set()
@@ -94,7 +95,15 @@ async def lifespan(app: FastAPI):
95
96 @asynccontextmanager
97 async def build_async_engine_client(
97 - args: Namespace) -> AsyncIterator[AsyncEngineClient]:
98 + args: Namespace) -> AsyncIterator[Optional[AsyncEngineClient]]:
99 + """
100 + Create AsyncEngineClient, either:
101 + - in-process using the AsyncLLMEngine Directly
102 + - multiprocess using AsyncLLMEngine RPC
103 +
104 + Returns the Client or None if the creation failed.
105 + """
106 +
107 # Context manager to handle async_engine_client lifecycle
108 # Ensures everything is shutdown and cleaned up on error/exit
109 global engine_args
@@ -157,11 +166,13 @@ async def build_async_engine_client(
166 try:
167 await rpc_client.setup()
168 break
160 - except TimeoutError as e:
169 + except TimeoutError:
170 if not rpc_server_process.is_alive():
162 - raise RuntimeError(
163 - "The server process died before "
164 - "responding to the readiness probe") from e
171 + logger.error(
172 + "RPCServer process died before responding "
173 + "to readiness probe")
174 + yield None
175 + return
176
177 yield async_engine_client
178 finally:
@@ -410,6 +421,10 @@ async def run_server(args, **uvicorn_kwargs) -> None:
421 logger.info("args: %s", args)
422
423 async with build_async_engine_client(args) as async_engine_client:
424 + # If None, creation of the client failed and we exit.
425 + if async_engine_client is None:
426 + return
427 +
428 app = await init_app(async_engine_client, args)
429
430 shutdown_task = await serve_http(