@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 7e90a2d117
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 06:45 UTC

Add `/health` Endpoint for both Servers (#1540)

Fluder-Paradyne committed Nov 1, 2023 at 22:59 UTC 7e90a2d11785b4cba5172f13178adb6d194a867f
2 files changed +13 -1
vllm/entrypoints/api_server.py
+6
@@ -17,6 +17,12 @@ app = FastAPI()
17 engine = None
18
19
20 +@app.get("/health")
21 +async def health() -> Response:
22 + """Health check."""
23 + return Response(status_code=200)
24 +
25 +
26 @app.post("/generate")
27 async def generate(request: Request) -> Response:
28 """Generate completion for the request.
vllm/entrypoints/openai/api_server.py
+7 -1
@@ -13,7 +13,7 @@ import uvicorn
13 from fastapi import Request
14 from fastapi.exceptions import RequestValidationError
15 from fastapi.middleware.cors import CORSMiddleware
16 -from fastapi.responses import JSONResponse, StreamingResponse
16 +from fastapi.responses import JSONResponse, StreamingResponse, Response
17 from packaging import version
18
19 from vllm.engine.arg_utils import AsyncEngineArgs
@@ -145,6 +145,12 @@ async def check_length(
145 return input_ids, None
146
147
148 +@app.get("/health")
149 +async def health() -> Response:
150 + """Health check."""
151 + return Response(status_code=200)
152 +
153 +
154 @app.get("/v1/models")
155 async def show_available_models():
156 """Show available models. Right now we only have one model."""