A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 04:15 UTC
Remove exclude_unset in streaming response (#3143)
Seonghyeon committed
Mar 2, 2024 at 02:59 UTC
27ca23dc002e06eade014ac6b801dc2dcbea40f3
1 file changed
+3
-3
vllm/entrypoints/openai/serving_completion.py
+3
-3
@@ -96,7 +96,7 @@ async def completion_stream_generator(
96
logprobs=logprobs,
97
finish_reason=finish_reason,
98
)
99
- ]).model_dump_json(exclude_unset=True)
99
+ ]).model_dump_json()
100
yield f"data: {response_json}\n\n"
101
102
if output.finish_reason is not None: # return final usage
@@ -121,7 +121,7 @@ async def completion_stream_generator(
121
)
122
],
123
usage=final_usage,
124
- ).model_dump_json(exclude_unset=True)
124
+ ).model_dump_json()
125
yield f"data: {response_json}\n\n"
126
127
yield "data: [DONE]\n\n"
@@ -306,7 +306,7 @@ class OpenAIServingCompletion(OpenAIServing):
306
request, prompt=prompt)
307
308
generators.append(
309
- self.engine.generate(None,
309
+ self.engine.generate(prompt,
310
sampling_params,
311
f"{request_id}-{i}",
312
prompt_token_ids=input_ids,