A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 07:00 UTC
Remove token-adding chat embedding params (#10551)
Signed-off-by: Noam Gat <noamgat@gmail.com>
Noam Gat committed
Nov 22, 2024 at 09:59 UTC
11fcf0e0661365f24bfff9591434a0cec640df6c
2 files changed
+4
-18
vllm/entrypoints/openai/protocol.py
-16
@@ -760,22 +760,6 @@ class EmbeddingChatRequest(OpenAIBaseModel):
760
# doc: end-chat-embedding-pooling-params
761
762
# doc: begin-chat-embedding-extra-params
763
- add_generation_prompt: bool = Field(
764
- default=True,
765
- description=
766
- ("If true, the generation prompt will be added to the chat template. "
767
- "This is a parameter used by chat template in tokenizer config of the "
768
- "model."),
769
- )
770
- continue_final_message: bool = Field(
771
- default=False,
772
- description=
773
- ("If this is set, the chat will be formatted so that the final "
774
- "message in the chat is open-ended, without any EOS tokens. The "
775
- "model will continue this message rather than starting a new one. "
776
- "This allows you to \"prefill\" part of the model's response for it. "
777
- "Cannot be used at the same time as `add_generation_prompt`."),
778
- )
763
add_special_tokens: bool = Field(
764
default=False,
765
description=(
vllm/entrypoints/openai/serving_embedding.py
+4
-2
@@ -148,8 +148,10 @@ class OpenAIServingEmbedding(OpenAIServing):
148
chat_template=request.chat_template or self.chat_template,
149
chat_template_content_format=self.
150
chat_template_content_format,
151
- add_generation_prompt=request.add_generation_prompt,
152
- continue_final_message=request.continue_final_message,
151
+ # In embedding requests, we are not generating tokens,
152
+ # so there is no need to append extra tokens to the input
153
+ add_generation_prompt=False,
154
+ continue_final_message=False,
155
truncate_prompt_tokens=truncate_prompt_tokens,
156
add_special_tokens=request.add_special_tokens,
157
)