A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 3, 22:45 UTC
[Frontend]Responses API supports chat_template_kwargs (#42272)
Signed-off-by: chaunceyjiang <chaunceyjiang@gmail.com>
Chauncey committed
May 11, 2026 at 19:59 UTC
617239b70c46d3d53fc16619718da1c347c43fb6
4 files changed
+29
-10
tests/entrypoints/openai/responses/test_function_call.py
+6
-1
@@ -325,8 +325,12 @@ async def test_function_calling_with_streaming_expected_arguments(
325
"tool_choice",
326
["auto", "required", {"type": "function", "name": "get_current_weather"}],
327
)
328
+@pytest.mark.parametrize(
329
+ "enable_thinking",
330
+ [True, False],
331
+)
332
async def test_function_calling_with_streaming_types(
329
- client: openai.AsyncOpenAI, model_name: str, tool_choice
333
+ client: openai.AsyncOpenAI, model_name: str, tool_choice, enable_thinking: bool
334
):
335
# this links the "done" type with the "start" type
336
# so every "done" type should have a corresponding "start" type
@@ -436,6 +440,7 @@ async def test_function_calling_with_streaming_types(
440
input=input_list,
441
tools=tools,
442
tool_choice=tool_choice,
443
+ extra_body={"chat_template_kwargs": {"enable_thinking": enable_thinking}},
444
stream=True,
445
)
446
vllm/entrypoints/openai/generate/api_router.py
+1
@@ -103,6 +103,7 @@ async def init_generate_state(
103
enable_prompt_tokens_details=args.enable_prompt_tokens_details,
104
enable_force_include_usage=args.enable_force_include_usage,
105
enable_log_outputs=args.enable_log_outputs,
106
+ default_chat_template_kwargs=args.default_chat_template_kwargs,
107
)
108
if "generate" in supported_tasks
109
else None
vllm/entrypoints/openai/responses/protocol.py
+8
-1
@@ -276,6 +276,13 @@ class ResponsesRequest(OpenAIBaseModel):
276
default=None,
277
description="KVTransfer parameters used for disaggregated serving.",
278
)
279
+ chat_template_kwargs: dict[str, Any] | None = Field(
280
+ default=None,
281
+ description=(
282
+ "Additional keyword args to pass to the chat template renderer. "
283
+ "Will be accessible by the template."
284
+ ),
285
+ )
286
# --8<-- [end:responses-extra-params]
287
288
def build_chat_params(
@@ -296,7 +303,7 @@ class ResponsesRequest(OpenAIBaseModel):
303
chat_template=default_template,
304
chat_template_content_format=default_template_content_format,
305
chat_template_kwargs=merge_kwargs( # To remove unset values
299
- {},
306
+ self.chat_template_kwargs,
307
dict(
308
add_generation_prompt=not continue_final,
309
continue_final_message=continue_final,
vllm/entrypoints/openai/responses/serving.py
+14
-8
@@ -167,6 +167,7 @@ class OpenAIServingResponses(OpenAIServing):
167
enable_prompt_tokens_details: bool = False,
168
enable_force_include_usage: bool = False,
169
enable_log_outputs: bool = False,
170
+ default_chat_template_kwargs: dict[str, Any] | None = None,
171
) -> None:
172
super().__init__(
173
engine_client=engine_client,
@@ -178,6 +179,7 @@ class OpenAIServingResponses(OpenAIServing):
179
self.openai_serving_render = openai_serving_render
180
self.chat_template = chat_template
181
self.chat_template_content_format: Final = chat_template_content_format
182
+ self.chat_template_kwargs = default_chat_template_kwargs or {}
183
self.enable_log_outputs = enable_log_outputs
184
185
# Set up the unified parser - either a unified parser or fall back to
@@ -254,10 +256,14 @@ class OpenAIServingResponses(OpenAIServing):
256
def _effective_chat_template_kwargs(
257
self, request: ResponsesRequest
258
) -> dict[str, Any]:
257
- return request.build_chat_params(
258
- self.chat_template,
259
- self.chat_template_content_format,
260
- ).chat_template_kwargs
259
+ return (
260
+ request.build_chat_params(
261
+ self.chat_template,
262
+ self.chat_template_content_format,
263
+ )
264
+ .with_defaults(self.chat_template_kwargs)
265
+ .chat_template_kwargs
266
+ )
267
268
def _validate_generator_input(
269
self,
@@ -601,13 +607,13 @@ class OpenAIServingResponses(OpenAIServing):
607
prev_msg=self.msg_store.get(prev_response.id) if prev_response else None,
608
prev_response_output=prev_response.output if prev_response else None,
609
)
604
-
610
+ chat_template_kwargs = self._effective_chat_template_kwargs(request)
611
_, engine_inputs = await self.openai_serving_render.preprocess_chat(
612
request,
613
messages,
614
default_template=self.chat_template,
615
default_template_content_format=self.chat_template_content_format,
610
- default_template_kwargs=None,
616
+ default_template_kwargs=chat_template_kwargs,
617
tool_dicts=tool_dicts,
618
tool_parser=self.parser.tool_parser_cls if self.parser else None,
619
reasoning_parser=self.parser.reasoning_parser_cls if self.parser else None,
@@ -626,13 +632,13 @@ class OpenAIServingResponses(OpenAIServing):
632
new_messages = construct_input_messages(
633
request_input=messages,
634
)
629
-
635
+ chat_template_kwargs = self._effective_chat_template_kwargs(request)
636
_, engine_inputs = await self.openai_serving_render.preprocess_chat(
637
request,
638
new_messages,
639
default_template=chat_template,
640
default_template_content_format=chat_template_content_format,
635
- default_template_kwargs=None,
641
+ default_template_kwargs=chat_template_kwargs,
642
tool_dicts=tool_dicts,
643
tool_parser=tool_parser,
644
reasoning_parser=self.parser.reasoning_parser_cls if self.parser else None,