A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 06:45 UTC
Add `/health` Endpoint for both Servers (#1540)
Fluder-Paradyne committed
Nov 1, 2023 at 22:59 UTC
7e90a2d11785b4cba5172f13178adb6d194a867f
2 files changed
+13
-1
vllm/entrypoints/api_server.py
+6
@@ -17,6 +17,12 @@ app = FastAPI()
17
engine = None
18
19
20
+@app.get("/health")
21
+async def health() -> Response:
22
+ """Health check."""
23
+ return Response(status_code=200)
24
+
25
+
26
@app.post("/generate")
27
async def generate(request: Request) -> Response:
28
"""Generate completion for the request.
vllm/entrypoints/openai/api_server.py
+7
-1
@@ -13,7 +13,7 @@ import uvicorn
13
from fastapi import Request
14
from fastapi.exceptions import RequestValidationError
15
from fastapi.middleware.cors import CORSMiddleware
16
-from fastapi.responses import JSONResponse, StreamingResponse
16
+from fastapi.responses import JSONResponse, StreamingResponse, Response
17
from packaging import version
18
19
from vllm.engine.arg_utils import AsyncEngineArgs
@@ -145,6 +145,12 @@ async def check_length(
145
return input_ids, None
146
147
148
+@app.get("/health")
149
+async def health() -> Response:
150
+ """Health check."""
151
+ return Response(status_code=200)
152
+
153
+
154
@app.get("/v1/models")
155
async def show_available_models():
156
"""Show available models. Right now we only have one model."""