@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / d9cd78eb71
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 15:45 UTC

[BugFix] Nonzero exit code if MQLLMEngine startup fails (#8572)

Nick Hill committed Sep 18, 2024 at 21:17 UTC d9cd78eb718c233ebc5b84377fc2226af7ef0fa2
1 file changed +5 -11
vllm/entrypoints/openai/api_server.py
+5 -11
@@ -11,7 +11,7 @@ from argparse import Namespace
11 from contextlib import asynccontextmanager
12 from functools import partial
13 from http import HTTPStatus
14 -from typing import AsyncIterator, Optional, Set
14 +from typing import AsyncIterator, Set
15
16 import uvloop
17 from fastapi import APIRouter, FastAPI, Request
@@ -95,7 +95,7 @@ async def lifespan(app: FastAPI):
95
96 @asynccontextmanager
97 async def build_async_engine_client(
98 - args: Namespace) -> AsyncIterator[Optional[EngineClient]]:
98 + args: Namespace) -> AsyncIterator[EngineClient]:
99
100 # Context manager to handle engine_client lifecycle
101 # Ensures everything is shutdown and cleaned up on error/exit
@@ -110,7 +110,7 @@ async def build_async_engine_client(
110 async def build_async_engine_client_from_engine_args(
111 engine_args: AsyncEngineArgs,
112 disable_frontend_multiprocessing: bool = False,
113 -) -> AsyncIterator[Optional[EngineClient]]:
113 +) -> AsyncIterator[EngineClient]:
114 """
115 Create EngineClient, either:
116 - in-process using the AsyncLLMEngine Directly
@@ -188,10 +188,8 @@ async def build_async_engine_client_from_engine_args(
188 break
189 except TimeoutError:
190 if not engine_process.is_alive():
191 - logger.error("Engine process died before responding "
192 - "to readiness probe")
193 - yield None
194 - return
191 + raise RuntimeError(
192 + "Engine process failed to start") from None
193
194 yield mp_engine_client # type: ignore[misc]
195 finally:
@@ -532,10 +530,6 @@ async def run_server(args, **uvicorn_kwargs) -> None:
530 signal.signal(signal.SIGTERM, signal_handler)
531
532 async with build_async_engine_client(args) as engine_client:
535 - # If None, creation of the client failed and we exit.
536 - if engine_client is None:
537 - return
538 -
533 app = build_app(args)
534
535 model_config = await engine_client.get_model_config()