A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 06:45 UTC
Fix completion API echo and logprob combo (#1992)
Simon Mo committed
Dec 10, 2023 at 13:20 UTC
2e8fc0d4c3bf8374f1f55569069e59ef45d4bc98
1 file changed
+9
-4
vllm/entrypoints/openai/api_server.py
+9
-4
@@ -567,17 +567,22 @@ async def create_completion(request: CompletionRequest, raw_request: Request):
567
i = output.index
568
delta_text = output.text[len(previous_texts[i]):]
569
token_ids = output.token_ids[previous_num_tokens[i]:]
570
- top_logprobs = output.logprobs[previous_num_tokens[i]:]
570
+ if request.logprobs is not None:
571
+ top_logprobs = output.logprobs[previous_num_tokens[i]:]
572
+ else:
573
+ top_logprobs = None
574
offsets = len(previous_texts[i])
575
if request.echo and not has_echoed[i]:
576
if not echo_without_generation:
577
delta_text = res.prompt + delta_text
578
token_ids = res.prompt_token_ids + token_ids
576
- top_logprobs = res.prompt_logprobs + top_logprobs
577
- else:
579
+ if top_logprobs:
580
+ top_logprobs = res.prompt_logprobs + top_logprobs
581
+ else: # only just return the prompt
582
delta_text = res.prompt
583
token_ids = res.prompt_token_ids
580
- top_logprobs = res.prompt_logprobs
584
+ if top_logprobs:
585
+ top_logprobs = res.prompt_logprobs
586
has_echoed[i] = True
587
if request.logprobs is not None:
588
logprobs = create_logprobs(