A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
feat(api): Return 503 on /health when engine is dead (#24897)
Signed-off-by: dongbo910220 <1275604947@qq.com> Co-authored-by: Claude <noreply@anthropic.com>
dongbo910220 committed
Sep 18, 2025 at 22:29 UTC
67244c86f0f1ffc06fcab9cad5e78989695cc15f
1 file changed
+6
-2
vllm/entrypoints/openai/api_server.py
+6
-2
@@ -103,6 +103,7 @@ from vllm.transformers_utils.tokenizer import MistralTokenizer
103
from vllm.usage.usage_lib import UsageContext
104
from vllm.utils import (Device, FlexibleArgumentParser, decorate_logs,
105
is_valid_ipv6_address, set_ulimit)
106
+from vllm.v1.engine.exceptions import EngineDeadError
107
from vllm.v1.metrics.prometheus import get_prometheus_registry
108
from vllm.version import __version__ as VLLM_VERSION
109
@@ -351,8 +352,11 @@ def engine_client(request: Request) -> EngineClient:
352
@router.get("/health", response_class=Response)
353
async def health(raw_request: Request) -> Response:
354
"""Health check."""
354
- await engine_client(raw_request).check_health()
355
- return Response(status_code=200)
355
+ try:
356
+ await engine_client(raw_request).check_health()
357
+ return Response(status_code=200)
358
+ except EngineDeadError:
359
+ return Response(status_code=503)
360
361
362
@router.get("/load")