A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 04:15 UTC
Connect engine healthcheck to openai server (#3260)
Nick Hill committed
Mar 7, 2024 at 16:38 UTC
d2339d6840498397f6e373489ed120cd2cce8eb4
1 file changed
+1
vllm/entrypoints/openai/api_server.py
+1
@@ -160,6 +160,7 @@ async def validation_exception_handler(_, exc):
160
@app.get("/health")
161
async def health() -> Response:
162
"""Health check."""
163
+ await openai_serving_chat.engine.check_health()
164
return Response(status_code=200)
165
166