@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / be54f8e5c4
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 08:00 UTC

[Fix] Change /generate response-type to json for non-streaming (#374)

Nicolas Frenay committed Jul 6, 2023 at 20:15 UTC be54f8e5c4083831aeaa4a1da9e248ddafcef54b
1 file changed +2 -2
vllm/entrypoints/api_server.py
+2 -2
@@ -3,7 +3,7 @@ import json
3 from typing import AsyncGenerator
4
5 from fastapi import BackgroundTasks, FastAPI, Request
6 -from fastapi.responses import Response, StreamingResponse
6 +from fastapi.responses import JSONResponse, Response, StreamingResponse
7 import uvicorn
8
9 from vllm.engine.arg_utils import AsyncEngineArgs
@@ -64,7 +64,7 @@ async def generate(request: Request) -> Response:
64 prompt = final_output.prompt
65 text_outputs = [prompt + output.text for output in final_output.outputs]
66 ret = {"text": text_outputs}
67 - return Response(content=json.dumps(ret))
67 + return JSONResponse(ret)
68
69
70 if __name__ == "__main__":