@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 11fcf0e066
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 07:00 UTC

Remove token-adding chat embedding params (#10551)

Signed-off-by: Noam Gat <noamgat@gmail.com>

Noam Gat committed Nov 22, 2024 at 09:59 UTC 11fcf0e0661365f24bfff9591434a0cec640df6c
2 files changed +4 -18
vllm/entrypoints/openai/protocol.py
-16
@@ -760,22 +760,6 @@ class EmbeddingChatRequest(OpenAIBaseModel):
760 # doc: end-chat-embedding-pooling-params
761
762 # doc: begin-chat-embedding-extra-params
763 - add_generation_prompt: bool = Field(
764 - default=True,
765 - description=
766 - ("If true, the generation prompt will be added to the chat template. "
767 - "This is a parameter used by chat template in tokenizer config of the "
768 - "model."),
769 - )
770 - continue_final_message: bool = Field(
771 - default=False,
772 - description=
773 - ("If this is set, the chat will be formatted so that the final "
774 - "message in the chat is open-ended, without any EOS tokens. The "
775 - "model will continue this message rather than starting a new one. "
776 - "This allows you to \"prefill\" part of the model's response for it. "
777 - "Cannot be used at the same time as `add_generation_prompt`."),
778 - )
763 add_special_tokens: bool = Field(
764 default=False,
765 description=(
vllm/entrypoints/openai/serving_embedding.py
+4 -2
@@ -148,8 +148,10 @@ class OpenAIServingEmbedding(OpenAIServing):
148 chat_template=request.chat_template or self.chat_template,
149 chat_template_content_format=self.
150 chat_template_content_format,
151 - add_generation_prompt=request.add_generation_prompt,
152 - continue_final_message=request.continue_final_message,
151 + # In embedding requests, we are not generating tokens,
152 + # so there is no need to append extra tokens to the input
153 + add_generation_prompt=False,
154 + continue_final_message=False,
155 truncate_prompt_tokens=truncate_prompt_tokens,
156 add_special_tokens=request.add_special_tokens,
157 )