A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 4, 20:00 UTC
[Bugfix] Pass effective chat template kwargs to reasoning parsers (#40460)
Signed-off-by: Bugen Zhao <i@bugenzhao.com>
Bugen Zhao committed
Apr 22, 2026 at 13:17 UTC
0210024ae796446a121f96d2d31053668ac0fd85
6 files changed
+71
-17
vllm/entrypoints/openai/chat_completion/batch_serving.py
+7
-4
@@ -114,12 +114,15 @@ class OpenAIServingChatBatch(OpenAIServingChat):
114
"""
115
tokenizer = self.renderer.tokenizer
116
assert tokenizer is not None
117
+ single_requests = [
118
+ request.to_chat_completion_request(messages)
119
+ for messages in request.messages
120
+ ]
121
122
reasoning_parser: ReasoningParser | None = None
123
if self.reasoning_parser_cls:
120
- chat_template_kwargs = self._prepare_extra_chat_template_kwargs(
121
- request.chat_template_kwargs,
122
- self.default_chat_template_kwargs,
124
+ chat_template_kwargs = self._effective_chat_template_kwargs(
125
+ single_requests[0]
126
)
127
reasoning_parser = self.reasoning_parser_cls(
128
tokenizer,
@@ -155,7 +158,7 @@ class OpenAIServingChatBatch(OpenAIServingChat):
158
self.default_sampling_params,
159
self.override_max_tokens,
160
)
158
- single_request = request.to_chat_completion_request(request.messages[i])
161
+ single_request = single_requests[i]
162
sampling_params = single_request.to_sampling_params(
163
max_tokens, self.default_sampling_params
164
)
vllm/entrypoints/openai/chat_completion/serving.py
+13
-4
@@ -189,6 +189,18 @@ class OpenAIServingChat(OpenAIServing):
189
)
190
)
191
192
+ def _effective_chat_template_kwargs(
193
+ self, request: ChatCompletionRequest
194
+ ) -> dict[str, Any]:
195
+ return (
196
+ request.build_chat_params(
197
+ self.chat_template,
198
+ self.chat_template_content_format,
199
+ )
200
+ .with_defaults(self.default_chat_template_kwargs)
201
+ .chat_template_kwargs
202
+ )
203
+
204
async def render_chat_request(
205
self,
206
request: ChatCompletionRequest,
@@ -231,10 +243,7 @@ class OpenAIServingChat(OpenAIServing):
243
# Streaming response
244
tokenizer = self.renderer.tokenizer
245
assert tokenizer is not None
234
- chat_template_kwargs = self._prepare_extra_chat_template_kwargs(
235
- request.chat_template_kwargs,
236
- self.default_chat_template_kwargs,
237
- )
246
+ chat_template_kwargs = self._effective_chat_template_kwargs(request)
247
reasoning_parser: ReasoningParser | None = None
248
if self.reasoning_parser_cls:
249
reasoning_parser = self.reasoning_parser_cls(
vllm/entrypoints/openai/parser/responses_parser.py
+29
-4
@@ -1,7 +1,7 @@
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
import logging
4
-from collections.abc import Callable
4
+from typing import Any
5
6
from openai.types.responses import ResponseFunctionToolCall, ResponseOutputItem
7
from openai.types.responses.response_function_tool_call_output_item import (
@@ -15,6 +15,7 @@ from openai.types.responses.response_reasoning_item import (
15
ResponseReasoningItem,
16
)
17
18
+from vllm.entrypoints.chat_utils import ChatTemplateContentFormatOption
19
from vllm.entrypoints.constants import MCP_PREFIX
20
from vllm.entrypoints.openai.responses.protocol import (
21
ResponseInputOutputItem,
@@ -36,10 +37,12 @@ class ResponsesParser:
37
self,
38
*,
39
tokenizer: TokenizerLike,
39
- reasoning_parser_cls: Callable[[TokenizerLike], ReasoningParser],
40
+ reasoning_parser_cls: type[ReasoningParser],
41
response_messages: list[ResponseInputOutputItem],
42
request: ResponsesRequest,
43
tool_parser_cls: type[ToolParser] | None,
44
+ chat_template: str | None,
45
+ chat_template_content_format: ChatTemplateContentFormatOption,
46
):
47
self.response_messages: list[ResponseInputOutputItem] = (
48
# TODO: initial messages may not be properly typed
@@ -49,7 +52,14 @@ class ResponsesParser:
52
self.tokenizer = tokenizer
53
self.request = request
54
52
- self.reasoning_parser_instance = reasoning_parser_cls(tokenizer)
55
+ self.reasoning_parser_instance = reasoning_parser_cls(
56
+ tokenizer,
57
+ chat_template_kwargs=_effective_chat_template_kwargs(
58
+ request,
59
+ chat_template=chat_template,
60
+ chat_template_content_format=chat_template_content_format,
61
+ ),
62
+ )
63
self.tool_parser_instance = None
64
if tool_parser_cls is not None:
65
self.tool_parser_instance = tool_parser_cls(tokenizer, request.tools)
@@ -159,10 +169,12 @@ class ResponsesParser:
169
def get_responses_parser_for_simple_context(
170
*,
171
tokenizer: TokenizerLike,
162
- reasoning_parser_cls: Callable[[TokenizerLike], ReasoningParser],
172
+ reasoning_parser_cls: type[ReasoningParser],
173
response_messages: list[ResponseInputOutputItem],
174
request: ResponsesRequest,
175
tool_parser_cls,
176
+ chat_template: str | None,
177
+ chat_template_content_format: ChatTemplateContentFormatOption,
178
) -> ResponsesParser:
179
"""Factory function to create a ResponsesParser with
180
optional reasoning parser.
@@ -176,4 +188,17 @@ def get_responses_parser_for_simple_context(
188
response_messages=response_messages,
189
request=request,
190
tool_parser_cls=tool_parser_cls,
191
+ chat_template=chat_template,
192
+ chat_template_content_format=chat_template_content_format,
193
)
194
+
195
+
196
+def _effective_chat_template_kwargs(
197
+ request: ResponsesRequest,
198
+ chat_template: str | None,
199
+ chat_template_content_format: ChatTemplateContentFormatOption,
200
+) -> dict[str, Any]:
201
+ return request.build_chat_params(
202
+ default_template=chat_template,
203
+ default_template_content_format=chat_template_content_format,
204
+ ).chat_template_kwargs
vllm/entrypoints/openai/responses/context.py
+3
-2
@@ -6,7 +6,6 @@ import copy
6
import json
7
import logging
8
from abc import ABC, abstractmethod
9
-from collections.abc import Callable
9
from contextlib import AsyncExitStack
10
from dataclasses import replace
11
from typing import TYPE_CHECKING, Any, Final, Union
@@ -273,7 +272,7 @@ class ParsableContext(ConversationContext):
272
*,
273
response_messages: list[ResponseInputOutputItem],
274
tokenizer: TokenizerLike,
276
- reasoning_parser_cls: Callable[[TokenizerLike], ReasoningParser] | None,
275
+ reasoning_parser_cls: type[ReasoningParser] | None,
276
request: ResponsesRequest,
277
available_tools: list[str] | None,
278
tool_parser_cls: type[ToolParser] | None,
@@ -296,6 +295,8 @@ class ParsableContext(ConversationContext):
295
response_messages=response_messages,
296
request=request,
297
tool_parser_cls=tool_parser_cls,
298
+ chat_template=chat_template,
299
+ chat_template_content_format=chat_template_content_format,
300
)
301
self.tool_parser_cls = tool_parser_cls
302
self.request = request
vllm/entrypoints/openai/responses/serving.py
+16
-2
@@ -267,6 +267,14 @@ class OpenAIServingResponses(OpenAIServing):
267
268
self.tool_server = tool_server
269
270
+ def _effective_chat_template_kwargs(
271
+ self, request: ResponsesRequest
272
+ ) -> dict[str, Any]:
273
+ return request.build_chat_params(
274
+ self.chat_template,
275
+ self.chat_template_content_format,
276
+ ).chat_template_kwargs
277
+
278
def _validate_generator_input(
279
self,
280
engine_input: EngineInput,
@@ -464,7 +472,10 @@ class OpenAIServingResponses(OpenAIServing):
472
context = SimpleContext()
473
474
if self.parser and self.parser.reasoning_parser_cls is not None:
467
- reasoning_parser = self.parser.reasoning_parser_cls(tokenizer)
475
+ reasoning_parser = self.parser.reasoning_parser_cls(
476
+ tokenizer,
477
+ chat_template_kwargs=self._effective_chat_template_kwargs(request),
478
+ )
479
if (
480
isinstance(
481
struct_out := sampling_params.structured_outputs,
@@ -835,7 +846,10 @@ class OpenAIServingResponses(OpenAIServing):
846
and self.parser.reasoning_parser_cls is not None
847
and isinstance(context, (SimpleContext, ParsableContext))
848
):
838
- reasoning_parser = self.parser.reasoning_parser_cls(tokenizer)
849
+ reasoning_parser = self.parser.reasoning_parser_cls(
850
+ tokenizer,
851
+ chat_template_kwargs=self._effective_chat_template_kwargs(request),
852
+ )
853
accumulated = getattr(context, "_accumulated_token_ids", []) or []
854
num_reasoning_tokens = reasoning_parser.count_reasoning_tokens(accumulated)
855
vllm/entrypoints/serve/render/serving.py
+3
-1
@@ -566,7 +566,9 @@ class OpenAIServingRender:
566
if reasoning_parser is not None:
567
tokenizer = renderer.get_tokenizer()
568
request = reasoning_parser(
569
- tokenizer, model_config=self.model_config
569
+ tokenizer,
570
+ model_config=self.model_config,
571
+ chat_template_kwargs=chat_params.chat_template_kwargs,
572
).adjust_request(request=request)
573
574
# tool parsing is done only if a tool_parser has been set and if