@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 2e8fc0d4c3
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 06:45 UTC

Fix completion API echo and logprob combo (#1992)

Simon Mo committed Dec 10, 2023 at 13:20 UTC 2e8fc0d4c3bf8374f1f55569069e59ef45d4bc98
1 file changed +9 -4
vllm/entrypoints/openai/api_server.py
+9 -4
@@ -567,17 +567,22 @@ async def create_completion(request: CompletionRequest, raw_request: Request):
567 i = output.index
568 delta_text = output.text[len(previous_texts[i]):]
569 token_ids = output.token_ids[previous_num_tokens[i]:]
570 - top_logprobs = output.logprobs[previous_num_tokens[i]:]
570 + if request.logprobs is not None:
571 + top_logprobs = output.logprobs[previous_num_tokens[i]:]
572 + else:
573 + top_logprobs = None
574 offsets = len(previous_texts[i])
575 if request.echo and not has_echoed[i]:
576 if not echo_without_generation:
577 delta_text = res.prompt + delta_text
578 token_ids = res.prompt_token_ids + token_ids
576 - top_logprobs = res.prompt_logprobs + top_logprobs
577 - else:
579 + if top_logprobs:
580 + top_logprobs = res.prompt_logprobs + top_logprobs
581 + else: # only just return the prompt
582 delta_text = res.prompt
583 token_ids = res.prompt_token_ids
580 - top_logprobs = res.prompt_logprobs
584 + if top_logprobs:
585 + top_logprobs = res.prompt_logprobs
586 has_echoed[i] = True
587 if request.logprobs is not None:
588 logprobs = create_logprobs(