A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 10, 20:15 UTC
[Frontend] add continue_final_message parameter to /embeddings endpoint (#31497)
Signed-off-by: Kevin P-W <140451262+kevin-pw@users.noreply.github.com>
Kevin committed
Dec 29, 2025 at 23:21 UTC
51085c2aebe7df2c53dbe9a44d89bc3ee761793f
2 files changed
+11
-2
vllm/entrypoints/pooling/embed/protocol.py
+10
-1
@@ -97,7 +97,16 @@ class EmbeddingChatRequest(OpenAIBaseModel):
97
"model."
98
),
99
)
100
-
100
+ continue_final_message: bool = Field(
101
+ default=False,
102
+ description=(
103
+ "If this is set, the chat will be formatted so that the final "
104
+ "message in the chat is open-ended, without any EOS tokens. The "
105
+ "model will continue this message rather than starting a new one. "
106
+ 'This allows you to "prefill" part of the model\'s response for it. '
107
+ "Cannot be used at the same time as `add_generation_prompt`."
108
+ ),
109
+ )
110
add_special_tokens: bool = Field(
111
default=False,
112
description=(
vllm/entrypoints/pooling/embed/serving.py
+1
-1
@@ -89,7 +89,7 @@ class EmbeddingMixin(OpenAIServing):
89
chat_template=ctx.request.chat_template or ctx.chat_template,
90
chat_template_content_format=ctx.chat_template_content_format,
91
add_generation_prompt=ctx.request.add_generation_prompt,
92
- continue_final_message=False,
92
+ continue_final_message=ctx.request.continue_final_message,
93
add_special_tokens=ctx.request.add_special_tokens,
94
)
95
else: