A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 08:00 UTC
[Fix] Change /generate response-type to json for non-streaming (#374)
Nicolas Frenay committed
Jul 6, 2023 at 20:15 UTC
be54f8e5c4083831aeaa4a1da9e248ddafcef54b
1 file changed
+2
-2
vllm/entrypoints/api_server.py
+2
-2
@@ -3,7 +3,7 @@ import json
3
from typing import AsyncGenerator
4
5
from fastapi import BackgroundTasks, FastAPI, Request
6
-from fastapi.responses import Response, StreamingResponse
6
+from fastapi.responses import JSONResponse, Response, StreamingResponse
7
import uvicorn
8
9
from vllm.engine.arg_utils import AsyncEngineArgs
@@ -64,7 +64,7 @@ async def generate(request: Request) -> Response:
64
prompt = final_output.prompt
65
text_outputs = [prompt + output.text for output in final_output.outputs]
66
ret = {"text": text_outputs}
67
- return Response(content=json.dumps(ret))
67
+ return JSONResponse(ret)
68
69
70
if __name__ == "__main__":