@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 93d364da34
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 15:45 UTC

[Bugfix] Include encoder prompts len to non-stream api usage response (#8861)

Pernekhan Utemuratov committed Sep 26, 2024 at 15:47 UTC 93d364da3406f5523e5e4772ffbc3c72dac7bbf4
1 file changed +2
vllm/entrypoints/openai/serving_chat.py
+2
@@ -726,6 +726,8 @@ class OpenAIServingChat(OpenAIServing):
726
727 assert final_res.prompt_token_ids is not None
728 num_prompt_tokens = len(final_res.prompt_token_ids)
729 + if final_res.encoder_prompt_token_ids is not None:
730 + num_prompt_tokens += len(final_res.encoder_prompt_token_ids)
731 num_generated_tokens = sum(
732 len(output.token_ids) for output in final_res.outputs)
733 usage = UsageInfo(