A high-throughput and memory-efficient inference and serving engine for LLMs
[Chore] Consolidate reasoning/tool parser attributes into unified Parser in chat serving (#45548)
Signed-off-by: sfeng33 <4florafeng@gmail.com>
Flora Feng committed
Jun 15, 2026 at 11:16 UTC
0d80979644e0237b6ef02ce0601dc0bd654e357b
4 files changed
+47
-45
tests/entrypoints/openai/chat_completion/test_serving_chat.py
+16
-1
@@ -1350,6 +1350,21 @@ class TestServingChatWithHarmony:
1350
else serving_chat.chat_completion_full_generator
1351
)
1352
1353
+ chat_template_kwargs = serving_chat._effective_chat_template_kwargs(req)
1354
+ if stream:
1355
+ extra_kwargs: dict[str, Any] = {
1356
+ "chat_template_kwargs": chat_template_kwargs,
1357
+ }
1358
+ else:
1359
+ parser = None
1360
+ if serving_chat.parser_cls is not None:
1361
+ parser = serving_chat.parser_cls(
1362
+ tokenizer,
1363
+ req.tools,
1364
+ chat_template_kwargs=chat_template_kwargs,
1365
+ )
1366
+ extra_kwargs = {"parser": parser}
1367
+
1368
result = generator_func(
1369
request=req,
1370
result_generator=result_generator(),
@@ -1361,7 +1376,7 @@ class TestServingChatWithHarmony:
1376
request_id=req.request_id,
1377
model_name=req.model,
1378
),
1364
- chat_template_kwargs=serving_chat._effective_chat_template_kwargs(req),
1379
+ **extra_kwargs,
1380
)
1381
1382
if stream:
vllm/entrypoints/openai/chat_completion/batch_serving.py
+11
-10
@@ -25,7 +25,7 @@ from vllm.entrypoints.serve.utils.api_utils import get_max_tokens
25
from vllm.inputs import EngineInput
26
from vllm.logger import init_logger
27
from vllm.outputs import RequestOutput
28
-from vllm.reasoning import ReasoningParser
28
+from vllm.parser.abstract_parser import Parser
29
from vllm.tokenizers import TokenizerLike
30
from vllm.utils.async_utils import merge_async_iterators
31
from vllm.utils.collection_utils import as_list
@@ -119,14 +119,15 @@ class OpenAIServingChatBatch(OpenAIServingChat):
119
for messages in request.messages
120
]
121
122
- reasoning_parser: ReasoningParser | None = None
123
- if self.reasoning_parser_cls:
122
+ parser: Parser | None = None
123
+ if self.parser_cls is not None:
124
chat_template_kwargs = self._effective_chat_template_kwargs(
125
single_requests[0]
126
)
127
- reasoning_parser = self.reasoning_parser_cls(
127
+ parser = self.parser_cls(
128
tokenizer,
129
- chat_template_kwargs=chat_template_kwargs, # type: ignore[call-arg]
129
+ None, # tools
130
+ chat_template_kwargs=chat_template_kwargs,
131
)
132
133
render_result = await self.render_batch_chat_request(request)
@@ -194,7 +195,7 @@ class OpenAIServingChatBatch(OpenAIServingChat):
195
all_conversations,
196
tokenizer,
197
request_metadata,
197
- reasoning_parser,
198
+ parser,
199
)
200
201
async def chat_completion_full_generator_batch(
@@ -206,7 +207,7 @@ class OpenAIServingChatBatch(OpenAIServingChat):
207
all_conversations: list[list[ConversationMessage]],
208
tokenizer: TokenizerLike,
209
request_metadata: RequestResponseMetadata,
209
- reasoning_parser: ReasoningParser | None = None,
210
+ parser: Parser | None = None,
211
) -> ErrorResponse | ChatCompletionResponse:
212
"""Handle batched (non-streaming) chat completions.
213
@@ -262,12 +263,12 @@ class OpenAIServingChatBatch(OpenAIServingChat):
263
else:
264
logprobs = None
265
265
- if reasoning_parser:
266
- reasoning, content = reasoning_parser.extract_reasoning(
266
+ if parser is not None:
267
+ reasoning, content, _ = parser.parse(
268
output.text,
269
request=request, # type: ignore[arg-type]
270
)
270
- if not getattr(request, "include_reasoning", True):
271
+ if not request.include_reasoning:
272
reasoning = None
273
else:
274
reasoning = None
vllm/entrypoints/openai/chat_completion/protocol.py
+2
@@ -955,6 +955,8 @@ class BatchChatCompletionRequest(OpenAIBaseModel):
955
temperature: float | None = 0.7
956
top_p: float | None = 1.0
957
user: str | None = None
958
+ tool_choice: Literal["none"] | None = "none"
959
+ include_reasoning: bool = True
960
961
# vLLM extensions
962
best_of: int | None = None
vllm/entrypoints/openai/chat_completion/serving.py
+18
-34
@@ -60,7 +60,6 @@ from vllm.logprobs import Logprob
60
from vllm.outputs import RequestOutput
61
from vllm.parser import ParserManager
62
from vllm.parser.abstract_parser import Parser
63
-from vllm.reasoning import ReasoningParser
63
from vllm.renderers import ChatParams
64
from vllm.sampling_params import BeamSearchParams, SamplingParams
65
from vllm.tokenizers import TokenizerLike
@@ -145,17 +144,7 @@ class OpenAIServingChat(OpenAIServing):
144
self.enable_log_outputs = enable_log_outputs
145
self.enable_log_deltas = enable_log_deltas
146
148
- # set up reasoning parser
149
- self.reasoning_parser_cls = ParserManager.get_reasoning_parser(
150
- reasoning_parser_name=reasoning_parser
151
- )
152
- # set up tool use
147
self.enable_auto_tools: bool = enable_auto_tools
154
- self.tool_parser = ParserManager.get_tool_parser(
155
- tool_parser_name=tool_parser,
156
- enable_auto_tools=enable_auto_tools,
157
- model_name=self.model_config.model,
158
- )
148
self.parser_cls = ParserManager.get_parser(
149
tool_parser_name=tool_parser,
150
reasoning_parser_name=reasoning_parser,
@@ -164,8 +153,9 @@ class OpenAIServingChat(OpenAIServing):
153
is_harmony=self.model_config.hf_config.model_type == "gpt_oss",
154
)
155
if (
167
- is_mistral_tool_parser(self.tool_parser)
168
- and self.reasoning_parser_cls is not None
156
+ self.parser_cls is not None
157
+ and is_mistral_tool_parser(self.parser_cls.tool_parser_cls)
158
+ and self.parser_cls.reasoning_parser_cls is not None
159
):
160
from vllm.tool_parsers.mistral_tool_parser import MistralToolParser
161
@@ -267,11 +257,12 @@ class OpenAIServingChat(OpenAIServing):
257
tokenizer = self.renderer.tokenizer
258
assert tokenizer is not None
259
chat_template_kwargs = self._effective_chat_template_kwargs(request)
270
- reasoning_parser: ReasoningParser | None = None
271
- if self.reasoning_parser_cls:
272
- reasoning_parser = self.reasoning_parser_cls(
260
+ parser: Parser | None = None
261
+ if self.parser_cls is not None:
262
+ parser = self.parser_cls(
263
tokenizer,
274
- chat_template_kwargs=chat_template_kwargs, # type: ignore[call-arg]
264
+ request.tools,
265
+ chat_template_kwargs=chat_template_kwargs,
266
)
267
result = await self.render_chat_request(request)
268
if isinstance(result, ErrorResponse):
@@ -359,10 +350,8 @@ class OpenAIServingChat(OpenAIServing):
350
# `think?` rule that handles both reasoning and
351
# non-reasoning outputs.
352
reasoning_ended = True
362
- elif reasoning_parser:
363
- reasoning_ended = reasoning_parser.is_reasoning_end(
364
- prompt_token_ids or []
365
- )
353
+ elif parser is not None and parser.reasoning_parser is not None:
354
+ reasoning_ended = parser.is_reasoning_end(prompt_token_ids or [])
355
else:
356
reasoning_ended = None
357
@@ -378,7 +367,7 @@ class OpenAIServingChat(OpenAIServing):
367
reasoning_parser_kwargs={
368
"chat_template_kwargs": chat_template_kwargs,
369
}
381
- if reasoning_parser
370
+ if parser is not None and parser.reasoning_parser is not None
371
else None,
372
)
373
@@ -408,7 +397,7 @@ class OpenAIServingChat(OpenAIServing):
397
conversation,
398
tokenizer,
399
request_metadata,
411
- chat_template_kwargs=chat_template_kwargs,
400
+ parser=parser,
401
mm_token_counts=mm_token_counts,
402
)
403
@@ -835,7 +824,7 @@ class OpenAIServingChat(OpenAIServing):
824
conversation: list[ConversationMessage],
825
tokenizer: TokenizerLike,
826
request_metadata: RequestResponseMetadata,
838
- chat_template_kwargs: dict[str, Any] | None = None,
827
+ parser: Parser | None = None,
828
mm_token_counts: dict[str, int] | None = None,
829
) -> ErrorResponse | ChatCompletionResponse:
830
created_time = int(time.time())
@@ -861,6 +850,9 @@ class OpenAIServingChat(OpenAIServing):
850
history_tool_call_cnt = 0
851
852
role = self.get_chat_request_role(request)
853
+ tool_parser_cls = (
854
+ self.parser_cls.tool_parser_cls if self.parser_cls is not None else None
855
+ )
856
for output in final_res.outputs:
857
# check for error finish reason and raise GenerationError
858
# finish_reason='error' indicates a retryable request-level internal error
@@ -880,14 +872,6 @@ class OpenAIServingChat(OpenAIServing):
872
else:
873
logprobs = None
874
883
- parser: Parser | None = None
884
- if self.parser_cls is not None:
885
- parser = self.parser_cls(
886
- tokenizer,
887
- request.tools,
888
- chat_template_kwargs=chat_template_kwargs,
889
- )
890
-
875
if parser is not None:
876
reasoning, content, tool_calls = parser.parse(
877
output.text,
@@ -904,7 +888,7 @@ class OpenAIServingChat(OpenAIServing):
888
889
auto_tools_called = False
890
907
- if (not self.enable_auto_tools or not self.tool_parser) and (
891
+ if (not self.enable_auto_tools or not tool_parser_cls) and (
892
not isinstance(request.tool_choice, ChatCompletionNamedToolChoiceParam)
893
and request.tool_choice != "required"
894
):
@@ -963,7 +947,7 @@ class OpenAIServingChat(OpenAIServing):
947
request.tools
948
and (request.tool_choice == "auto" or request.tool_choice is None)
949
and self.enable_auto_tools
966
- and self.tool_parser
950
+ and tool_parser_cls
951
):
952
auto_tools_called = tool_calls is not None and len(tool_calls) > 0
953
if tool_calls: