A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
Support add_generation_prompt in embeddings endpoint with chat request (#23931)
Signed-off-by: biba10 <jaksmid@seznam.cz>
Jakub Smid committed
Sep 3, 2025 at 12:47 UTC
28f350e147c4b5c050c0080ef0d924c15ab87635
2 files changed
+9
-3
vllm/entrypoints/openai/protocol.py
+8
@@ -1342,6 +1342,14 @@ class EmbeddingChatRequest(OpenAIBaseModel):
1342
truncate_prompt_tokens: Optional[Annotated[int, Field(ge=-1)]] = None
1343
1344
# --8<-- [start:chat-embedding-extra-params]
1345
+ add_generation_prompt: bool = Field(
1346
+ default=False,
1347
+ description=
1348
+ ("If true, the generation prompt will be added to the chat template. "
1349
+ "This is a parameter used by chat template in tokenizer config of the "
1350
+ "model."),
1351
+ )
1352
+
1353
add_special_tokens: bool = Field(
1354
default=False,
1355
description=(
vllm/entrypoints/openai/serving_embedding.py
+1
-3
@@ -93,9 +93,7 @@ class EmbeddingMixin(OpenAIServing):
93
or ctx.chat_template,
94
chat_template_content_format=ctx.
95
chat_template_content_format,
96
- # In embedding requests, we are not generating tokens,
97
- # so there is no need to append extra tokens to the input
98
- add_generation_prompt=False,
96
+ add_generation_prompt=ctx.request.add_generation_prompt,
97
continue_final_message=False,
98
add_special_tokens=ctx.request.add_special_tokens,
99
)