@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 27ca23dc00
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 04:15 UTC

Remove exclude_unset in streaming response (#3143)

Seonghyeon committed Mar 2, 2024 at 02:59 UTC 27ca23dc002e06eade014ac6b801dc2dcbea40f3
1 file changed +3 -3
vllm/entrypoints/openai/serving_completion.py
+3 -3
@@ -96,7 +96,7 @@ async def completion_stream_generator(
96 logprobs=logprobs,
97 finish_reason=finish_reason,
98 )
99 - ]).model_dump_json(exclude_unset=True)
99 + ]).model_dump_json()
100 yield f"data: {response_json}\n\n"
101
102 if output.finish_reason is not None: # return final usage
@@ -121,7 +121,7 @@ async def completion_stream_generator(
121 )
122 ],
123 usage=final_usage,
124 - ).model_dump_json(exclude_unset=True)
124 + ).model_dump_json()
125 yield f"data: {response_json}\n\n"
126
127 yield "data: [DONE]\n\n"
@@ -306,7 +306,7 @@ class OpenAIServingCompletion(OpenAIServing):
306 request, prompt=prompt)
307
308 generators.append(
309 - self.engine.generate(None,
309 + self.engine.generate(prompt,
310 sampling_params,
311 f"{request_id}-{i}",
312 prompt_token_ids=input_ids,