A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 4, 22:30 UTC
[Anthropic][Frontend] Added chat_template_kwargs to /v1/messages (#40125)
Signed-off-by: Aleksandar Yanakiev <alexander.yanakiev@discretestack.com> Co-authored-by: Aleksandar Yanakiev <alexander.yanakiev@discretestack.com>
aleksandaryanakiev committed
Apr 20, 2026 at 16:10 UTC
b82fc1364d313a222bde7e9ac897fd847ad7b05b
2 files changed
+18
vllm/entrypoints/anthropic/protocol.py
+16
@@ -117,6 +117,13 @@ class AnthropicMessagesRequest(BaseModel):
117
default=None,
118
description="KVTransfer parameters used for disaggregated serving.",
119
)
120
+ chat_template_kwargs: dict[str, Any] | None = Field(
121
+ default=None,
122
+ description=(
123
+ "Additional keyword args to pass to the chat template renderer. "
124
+ "Will be accessible by the template."
125
+ ),
126
+ )
127
128
@field_validator("model")
129
@classmethod
@@ -212,6 +219,15 @@ class AnthropicCountTokensRequest(BaseModel):
219
tool_choice: AnthropicToolChoice | None = None
220
tools: list[AnthropicTool] | None = None
221
222
+ # vLLM-specific fields that are not in Anthropic spec
223
+ chat_template_kwargs: dict[str, Any] | None = Field(
224
+ default=None,
225
+ description=(
226
+ "Additional keyword args to pass to the chat template renderer. "
227
+ "Will be accessible by the template."
228
+ ),
229
+ )
230
+
231
@field_validator("model")
232
@classmethod
233
def validate_model(cls, v):
vllm/entrypoints/anthropic/serving.py
+2
@@ -323,6 +323,7 @@ class AnthropicServingMessages(OpenAIServingChat):
323
return ChatCompletionRequest(
324
model=anthropic_request.model,
325
messages=openai_messages,
326
+ chat_template_kwargs=anthropic_request.chat_template_kwargs,
327
)
328
329
return ChatCompletionRequest(
@@ -335,6 +336,7 @@ class AnthropicServingMessages(OpenAIServingChat):
336
top_p=anthropic_request.top_p,
337
top_k=anthropic_request.top_k,
338
kv_transfer_params=anthropic_request.kv_transfer_params,
339
+ chat_template_kwargs=anthropic_request.chat_template_kwargs,
340
)
341
342
@classmethod