@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 665cbcec4b
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 06:45 UTC

Added echo function to OpenAI API server. (#1504)

Yunmo Chen committed Nov 27, 2023 at 13:29 UTC 665cbcec4b963f6ab7b696f3d7e3393a7909003d
2 files changed +71 -24
vllm/entrypoints/openai/api_server.py
+70 -22
@@ -160,16 +160,26 @@ async def show_available_models():
160 return ModelList(data=model_cards)
161
162
163 -def create_logprobs(token_ids: List[int],
164 - id_logprobs: List[Dict[int, float]],
165 - initial_text_offset: int = 0) -> LogProbs:
163 +def create_logprobs(
164 + token_ids: List[int],
165 + top_logprobs: Optional[List[Optional[Dict[int, float]]]] = None,
166 + num_output_top_logprobs: Optional[int] = None,
167 + initial_text_offset: int = 0,
168 +) -> LogProbs:
169 """Create OpenAI-style logprobs."""
170 logprobs = LogProbs()
171 last_token_len = 0
169 - for token_id, id_logprob in zip(token_ids, id_logprobs):
172 + if num_output_top_logprobs:
173 + logprobs.top_logprobs = []
174 + for i, token_id in enumerate(token_ids):
175 + step_top_logprobs = top_logprobs[i]
176 + if step_top_logprobs is not None:
177 + token_logprob = step_top_logprobs[token_id]
178 + else:
179 + token_logprob = None
180 token = tokenizer.convert_ids_to_tokens(token_id)
181 logprobs.tokens.append(token)
172 - logprobs.token_logprobs.append(id_logprob[token_id])
182 + logprobs.token_logprobs.append(token_logprob)
183 if len(logprobs.text_offset) == 0:
184 logprobs.text_offset.append(initial_text_offset)
185 else:
@@ -177,10 +187,11 @@ def create_logprobs(token_ids: List[int],
187 last_token_len)
188 last_token_len = len(token)
189
180 - logprobs.top_logprobs.append({
181 - tokenizer.convert_ids_to_tokens(i): p
182 - for i, p in id_logprob.items()
183 - })
190 + if num_output_top_logprobs:
191 + logprobs.top_logprobs.append({
192 + tokenizer.convert_ids_to_tokens(i): p
193 + for i, p in step_top_logprobs.items()
194 + } if step_top_logprobs else None)
195 return logprobs
196
197
@@ -371,8 +382,6 @@ async def create_completion(request: CompletionRequest, raw_request: Request):
382 for the API specification. This API mimics the OpenAI Completion API.
383
384 NOTE: Currently we do not support the following features:
374 - - echo (since the vLLM engine does not currently support
375 - getting the logprobs of prompt tokens)
385 - suffix (the language models we currently support do not support
386 suffix)
387 - logit_bias (to be supported by vLLM engine)
@@ -383,11 +392,8 @@ async def create_completion(request: CompletionRequest, raw_request: Request):
392 if error_check_ret is not None:
393 return error_check_ret
394
386 - if request.echo:
387 - # We do not support echo since the vLLM engine does not
388 - # currently support getting the logprobs of prompt tokens.
389 - return create_error_response(HTTPStatus.BAD_REQUEST,
390 - "echo is not currently supported")
395 + # OpenAI API supports echoing the prompt when max_tokens is 0.
396 + echo_without_generation = request.echo and request.max_tokens == 0
397
398 if request.suffix is not None:
399 # The language models we currently support do not support suffix.
@@ -443,9 +449,11 @@ async def create_completion(request: CompletionRequest, raw_request: Request):
449 stop=request.stop,
450 stop_token_ids=request.stop_token_ids,
451 ignore_eos=request.ignore_eos,
446 - max_tokens=request.max_tokens,
452 + max_tokens=request.max_tokens
453 + if not echo_without_generation else 1,
454 logprobs=request.logprobs,
455 use_beam_search=request.use_beam_search,
456 + prompt_logprobs=request.logprobs if request.echo else None,
457 skip_special_tokens=request.skip_special_tokens,
458 spaces_between_special_tokens=spaces_between_special_tokens,
459 )
@@ -495,24 +503,42 @@ async def create_completion(request: CompletionRequest, raw_request: Request):
503 async def completion_stream_generator() -> AsyncGenerator[str, None]:
504 previous_texts = [""] * request.n
505 previous_num_tokens = [0] * request.n
506 + has_echoed = [False] * request.n
507 async for res in result_generator:
508 res: RequestOutput
509 for output in res.outputs:
510 i = output.index
511 delta_text = output.text[len(previous_texts[i]):]
512 + token_ids = output.token_ids[previous_num_tokens[i]:]
513 + top_logprobs = output.logprobs[previous_num_tokens[i]:]
514 + offsets = len(previous_texts[i])
515 + if request.echo and not has_echoed[i]:
516 + if not echo_without_generation:
517 + delta_text = res.prompt + delta_text
518 + token_ids = res.prompt_token_ids + token_ids
519 + top_logprobs = res.prompt_logprobs + top_logprobs
520 + else:
521 + delta_text = res.prompt
522 + token_ids = res.prompt_token_ids
523 + top_logprobs = res.prompt_logprobs
524 + has_echoed[i] = True
525 if request.logprobs is not None:
526 logprobs = create_logprobs(
505 - output.token_ids[previous_num_tokens[i]:],
506 - output.logprobs[previous_num_tokens[i]:],
507 - len(previous_texts[i]))
527 + token_ids=token_ids,
528 + top_logprobs=top_logprobs,
529 + num_output_top_logprobs=request.logprobs,
530 + initial_text_offset=offsets,
531 + )
532 else:
533 logprobs = None
534 previous_texts[i] = output.text
535 previous_num_tokens[i] = len(output.token_ids)
536 + finish_reason = output.finish_reason
537 response_json = create_stream_response_json(
538 index=i,
539 text=delta_text,
540 logprobs=logprobs,
541 + finish_reason=finish_reason,
542 )
543 yield f"data: {response_json}\n\n"
544 if output.finish_reason is not None:
@@ -551,14 +577,36 @@ async def create_completion(request: CompletionRequest, raw_request: Request):
577 final_res = res
578 assert final_res is not None
579 choices = []
580 + prompt_token_ids = final_res.prompt_token_ids
581 + prompt_logprobs = final_res.prompt_logprobs
582 + prompt_text = final_res.prompt
583 for output in final_res.outputs:
584 if request.logprobs is not None:
556 - logprobs = create_logprobs(output.token_ids, output.logprobs)
585 + if not echo_without_generation:
586 + token_ids = output.token_ids
587 + top_logprobs = output.logprobs
588 + if request.echo:
589 + token_ids = prompt_token_ids + token_ids
590 + top_logprobs = prompt_logprobs + top_logprobs
591 + else:
592 + token_ids = prompt_token_ids
593 + top_logprobs = prompt_logprobs
594 + logprobs = create_logprobs(
595 + token_ids=token_ids,
596 + top_logprobs=top_logprobs,
597 + num_output_top_logprobs=request.logprobs,
598 + )
599 else:
600 logprobs = None
601 + if not echo_without_generation:
602 + output_text = output.text
603 + if request.echo:
604 + output_text = prompt_text + output_text
605 + else:
606 + output_text = prompt_text
607 choice_data = CompletionResponseChoice(
608 index=output.index,
561 - text=output.text,
609 + text=output_text,
610 logprobs=logprobs,
611 finish_reason=output.finish_reason,
612 )
vllm/entrypoints/openai/protocol.py
+1 -2
@@ -106,8 +106,7 @@ class LogProbs(BaseModel):
106 text_offset: List[int] = Field(default_factory=list)
107 token_logprobs: List[Optional[float]] = Field(default_factory=list)
108 tokens: List[str] = Field(default_factory=list)
109 - top_logprobs: List[Optional[Dict[str,
110 - float]]] = Field(default_factory=list)
109 + top_logprobs: Optional[List[Optional[Dict[int, float]]]] = None
110
111
112 class CompletionResponseChoice(BaseModel):