A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 06:45 UTC
Fix OpenAI server completion_tokens referenced before assignment (#1996)
Jin Shang committed
Dec 10, 2023 at 13:01 UTC
1aa13615103c2ea47e36710a9b2e17dfe1909143
1 file changed
+3
-4
vllm/entrypoints/openai/api_server.py
+3
-4
@@ -332,8 +332,7 @@ async def create_chat_completion(request: ChatCompletionRequest,
332
# Send token-by-token response for each request.n
333
delta_text = output.text[len(previous_texts[i]):]
334
previous_texts[i] = output.text
335
- completion_tokens = len(output.token_ids)
336
- previous_num_tokens[i] = completion_tokens
335
+ previous_num_tokens[i] = len(output.token_ids)
336
choice_data = ChatCompletionResponseStreamChoice(
337
index=i,
338
delta=DeltaMessage(content=delta_text),
@@ -351,8 +350,8 @@ async def create_chat_completion(request: ChatCompletionRequest,
350
prompt_tokens = len(res.prompt_token_ids)
351
final_usage = UsageInfo(
352
prompt_tokens=prompt_tokens,
354
- completion_tokens=completion_tokens,
355
- total_tokens=prompt_tokens + completion_tokens,
353
+ completion_tokens=previous_num_tokens[i],
354
+ total_tokens=prompt_tokens + previous_num_tokens[i],
355
)
356
choice_data = ChatCompletionResponseStreamChoice(
357
index=i, delta=[], finish_reason=output.finish_reason)