@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / b396cb4998
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 08:00 UTC

fix: only response [DONE] once when streaming response. (#378)

Ricardo Lu committed Jul 7, 2023 at 09:08 UTC b396cb49982e740276b5dbd8c99b2f5a5d07aab0
1 file changed +2 -2
vllm/entrypoints/openai/api_server.py
+2 -2
@@ -269,7 +269,7 @@ async def create_chat_completion(raw_request: Request):
269 finish_reason=output.finish_reason,
270 )
271 yield f"data: {response_json}\n\n"
272 - yield "data: [DONE]\n\n"
272 + yield "data: [DONE]\n\n"
273
274 # Streaming response
275 if request.stream:
@@ -465,7 +465,7 @@ async def create_completion(raw_request: Request):
465 finish_reason=output.finish_reason,
466 )
467 yield f"data: {response_json}\n\n"
468 - yield "data: [DONE]\n\n"
468 + yield "data: [DONE]\n\n"
469
470 # Streaming response
471 if stream: