A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 08:15 UTC
bugfix: fix the bug that stream generate not work (#2756)
Zhao Yingzhuo committed
Nov 9, 2024 at 18:09 UTC
9e372664208b4905f7343f1fc76aca758fbf6f8f
1 file changed
+1
-1
vllm/entrypoints/api_server.py
+1
-1
@@ -66,7 +66,7 @@ async def generate(request: Request) -> Response:
66
prompt + output.text for output in request_output.outputs
67
]
68
ret = {"text": text_outputs}
69
- yield (json.dumps(ret) + "\0").encode("utf-8")
69
+ yield (json.dumps(ret) + "\n").encode("utf-8")
70
71
if stream:
72
return StreamingResponse(stream_results())