@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 16620f439d
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 22:00 UTC

do not exclude `object` field in CompletionStreamResponse (#6196)

kczimm committed Jul 7, 2024 at 21:32 UTC 16620f439db1f2cc91b5582b59fc8845cbb02881
1 file changed +2 -2
vllm/entrypoints/openai/serving_completion.py
+2 -2
@@ -301,7 +301,7 @@ class OpenAIServingCompletion(OpenAIServing):
301 else:
302 chunk.usage = None
303
304 - response_json = chunk.model_dump_json(exclude_unset=True)
304 + response_json = chunk.model_dump_json(exclude_unset=False)
305 yield f"data: {response_json}\n\n"
306
307 if (request.stream_options
@@ -314,7 +314,7 @@ class OpenAIServingCompletion(OpenAIServing):
314 usage=usage,
315 )
316 final_usage_data = (final_usage_chunk.model_dump_json(
317 - exclude_unset=True, exclude_none=True))
317 + exclude_unset=False, exclude_none=True))
318 yield f"data: {final_usage_data}\n\n"
319
320 except ValueError as e: