A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 15:45 UTC
[BugFix] Nonzero exit code if MQLLMEngine startup fails (#8572)
Nick Hill committed
Sep 18, 2024 at 21:17 UTC
d9cd78eb718c233ebc5b84377fc2226af7ef0fa2
1 file changed
+5
-11
vllm/entrypoints/openai/api_server.py
+5
-11
@@ -11,7 +11,7 @@ from argparse import Namespace
11
from contextlib import asynccontextmanager
12
from functools import partial
13
from http import HTTPStatus
14
-from typing import AsyncIterator, Optional, Set
14
+from typing import AsyncIterator, Set
15
16
import uvloop
17
from fastapi import APIRouter, FastAPI, Request
@@ -95,7 +95,7 @@ async def lifespan(app: FastAPI):
95
96
@asynccontextmanager
97
async def build_async_engine_client(
98
- args: Namespace) -> AsyncIterator[Optional[EngineClient]]:
98
+ args: Namespace) -> AsyncIterator[EngineClient]:
99
100
# Context manager to handle engine_client lifecycle
101
# Ensures everything is shutdown and cleaned up on error/exit
@@ -110,7 +110,7 @@ async def build_async_engine_client(
110
async def build_async_engine_client_from_engine_args(
111
engine_args: AsyncEngineArgs,
112
disable_frontend_multiprocessing: bool = False,
113
-) -> AsyncIterator[Optional[EngineClient]]:
113
+) -> AsyncIterator[EngineClient]:
114
"""
115
Create EngineClient, either:
116
- in-process using the AsyncLLMEngine Directly
@@ -188,10 +188,8 @@ async def build_async_engine_client_from_engine_args(
188
break
189
except TimeoutError:
190
if not engine_process.is_alive():
191
- logger.error("Engine process died before responding "
192
- "to readiness probe")
193
- yield None
194
- return
191
+ raise RuntimeError(
192
+ "Engine process failed to start") from None
193
194
yield mp_engine_client # type: ignore[misc]
195
finally:
@@ -532,10 +530,6 @@ async def run_server(args, **uvicorn_kwargs) -> None:
530
signal.signal(signal.SIGTERM, signal_handler)
531
532
async with build_async_engine_client(args) as engine_client:
535
- # If None, creation of the client failed and we exit.
536
- if engine_client is None:
537
- return
538
-
533
app = build_app(args)
534
535
model_config = await engine_client.get_model_config()