A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 15:45 UTC
[Bugfix] Include encoder prompts len to non-stream api usage response (#8861)
Pernekhan Utemuratov committed
Sep 26, 2024 at 15:47 UTC
93d364da3406f5523e5e4772ffbc3c72dac7bbf4
1 file changed
+2
vllm/entrypoints/openai/serving_chat.py
+2
@@ -726,6 +726,8 @@ class OpenAIServingChat(OpenAIServing):
726
727
assert final_res.prompt_token_ids is not None
728
num_prompt_tokens = len(final_res.prompt_token_ids)
729
+ if final_res.encoder_prompt_token_ids is not None:
730
+ num_prompt_tokens += len(final_res.encoder_prompt_token_ids)
731
num_generated_tokens = sum(
732
len(output.token_ids) for output in final_res.outputs)
733
usage = UsageInfo(