@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 1aa1361510
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 06:45 UTC

Fix OpenAI server completion_tokens referenced before assignment (#1996)

Jin Shang committed Dec 10, 2023 at 13:01 UTC 1aa13615103c2ea47e36710a9b2e17dfe1909143
1 file changed +3 -4
vllm/entrypoints/openai/api_server.py
+3 -4
@@ -332,8 +332,7 @@ async def create_chat_completion(request: ChatCompletionRequest,
332 # Send token-by-token response for each request.n
333 delta_text = output.text[len(previous_texts[i]):]
334 previous_texts[i] = output.text
335 - completion_tokens = len(output.token_ids)
336 - previous_num_tokens[i] = completion_tokens
335 + previous_num_tokens[i] = len(output.token_ids)
336 choice_data = ChatCompletionResponseStreamChoice(
337 index=i,
338 delta=DeltaMessage(content=delta_text),
@@ -351,8 +350,8 @@ async def create_chat_completion(request: ChatCompletionRequest,
350 prompt_tokens = len(res.prompt_token_ids)
351 final_usage = UsageInfo(
352 prompt_tokens=prompt_tokens,
354 - completion_tokens=completion_tokens,
355 - total_tokens=prompt_tokens + completion_tokens,
353 + completion_tokens=previous_num_tokens[i],
354 + total_tokens=prompt_tokens + previous_num_tokens[i],
355 )
356 choice_data = ChatCompletionResponseStreamChoice(
357 index=i, delta=[], finish_reason=output.finish_reason)