@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 782505ed8e
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 14, 05:30 UTC

[Model] Add reasoning_parser and tool_parser for Ernie45 thinking (#25027)

Signed-off-by: wangyafeng <wangyafeng@baidu.com>

CSWYF3634076 committed Oct 13, 2025 at 15:55 UTC 782505ed8eb4f1b27cccd009a8dc9b69f6ad6ebc
7 files changed +870
docs/features/reasoning_outputs.md
+2
@@ -11,6 +11,8 @@ vLLM currently supports the following reasoning models:
11 | Model Series | Parser Name | Structured Output Support | Tool Calling |
12 |--------------|-------------|------------------|-------------|
13 | [DeepSeek R1 series](https://huggingface.co/collections/deepseek-ai/deepseek-r1-678e1e131c0169c0bc89728d) | `deepseek_r1` | `json`, `regex` | ❌ |
14 +| [ERNIE-4.5-VL series](https://huggingface.co/baidu/ERNIE-4.5-VL-28B-A3B-PT) | `ernie45` | `json`, `regex` | ❌ |
15 +| [ERNIE-4.5-21B-A3B-Thinking](https://huggingface.co/baidu/ERNIE-4.5-21B-A3B-Thinking) | `ernie45` | `json`, `regex` | ✅ |
16 | [QwQ-32B](https://huggingface.co/Qwen/QwQ-32B) | `deepseek_r1` | `json`, `regex` | ✅ |
17 | [IBM Granite 3.2 language models](https://huggingface.co/collections/ibm-granite/granite-32-language-models-67b3bc8c13508f6d064cff9a) | `granite` | ❌ | ❌ |
18 | [Qwen3 series](https://huggingface.co/collections/Qwen/qwen3-67dd247413f0e2e4f653967f) | `qwen3` | `json`, `regex` | ✅ |
tests/reasoning/test_ernie45_reasoning_parser.py new
+124
@@ -0,0 +1,124 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +
4 +import pytest
5 +from transformers import AutoTokenizer
6 +
7 +from tests.reasoning.utils import run_reasoning_extraction
8 +from vllm.reasoning import ReasoningParser, ReasoningParserManager
9 +
10 +parser_name = "ernie45"
11 +
12 +REASONING_MODEL_NAME = "baidu/ERNIE-4.5-21B-A3B-Thinking"
13 +
14 +
15 +@pytest.fixture(scope="module")
16 +def ernie45_tokenizer():
17 + return AutoTokenizer.from_pretrained(REASONING_MODEL_NAME)
18 +
19 +
20 +# 带 </think>，非stream
21 +WITH_THINK = {
22 + "output": "abc</think>def",
23 + "reasoning_content": "abc",
24 + "content": "def",
25 +}
26 +# 带 </think>，stream
27 +WITH_THINK_STREAM = {
28 + "output": "abc</think>def",
29 + "reasoning_content": "abc",
30 + "content": "def",
31 +}
32 +# without </think>, all is reasoning_content
33 +WITHOUT_THINK = {
34 + "output": "abc",
35 + "reasoning_content": "abc",
36 + "content": None,
37 +}
38 +# without </think>, all is reasoning_content
39 +WITHOUT_THINK_STREAM = {
40 + "output": "abc",
41 + "reasoning_content": "abc",
42 + "content": None,
43 +}
44 +
45 +COMPLETE_REASONING = {
46 + "output": "abc</think>",
47 + "reasoning_content": "abc",
48 + "content": None,
49 +}
50 +MULTILINE_REASONING = {
51 + "output": "abc\nABC</think>def\nDEF",
52 + "reasoning_content": "abc\nABC",
53 + "content": "def\nDEF",
54 +}
55 +
56 +TEST_CASES = [
57 + pytest.param(
58 + False,
59 + WITH_THINK,
60 + id="with_think",
61 + ),
62 + pytest.param(
63 + True,
64 + WITH_THINK_STREAM,
65 + id="with_think_stream",
66 + ),
67 + pytest.param(
68 + False,
69 + WITHOUT_THINK,
70 + id="without_think",
71 + ),
72 + pytest.param(
73 + True,
74 + WITHOUT_THINK_STREAM,
75 + id="without_think_stream",
76 + ),
77 + pytest.param(
78 + False,
79 + COMPLETE_REASONING,
80 + id="complete_reasoning",
81 + ),
82 + pytest.param(
83 + True,
84 + COMPLETE_REASONING,
85 + id="complete_reasoning_stream",
86 + ),
87 + pytest.param(
88 + False,
89 + MULTILINE_REASONING,
90 + id="multiline_reasoning",
91 + ),
92 + pytest.param(
93 + True,
94 + MULTILINE_REASONING,
95 + id="multiline_reasoning_stream",
96 + ),
97 +]
98 +
99 +
100 +@pytest.mark.parametrize("streaming, param_dict", TEST_CASES)
101 +def test_reasoning(
102 + streaming: bool,
103 + param_dict: dict,
104 + ernie45_tokenizer,
105 +):
106 + output = ernie45_tokenizer.tokenize(param_dict["output"])
107 + output_tokens: list[str] = []
108 + for token in output:
109 + one_token = ernie45_tokenizer.convert_tokens_to_string([token])
110 + if one_token:
111 + output_tokens.append(one_token)
112 +
113 + parser: ReasoningParser = ReasoningParserManager.get_reasoning_parser(parser_name)(
114 + ernie45_tokenizer
115 + )
116 +
117 + reasoning, content = run_reasoning_extraction(
118 + parser, output_tokens, streaming=streaming
119 + )
120 +
121 + print()
122 +
123 + assert reasoning == param_dict["reasoning_content"]
124 + assert content == param_dict["content"]
tests/tool_use/test_ernie45_moe_tool_parser.py new
+359
@@ -0,0 +1,359 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +# ruff: noqa: E501
4 +
5 +import json
6 +from collections.abc import Generator
7 +
8 +import pytest
9 +
10 +from vllm.entrypoints.openai.protocol import (
11 + ChatCompletionRequest,
12 + DeltaMessage,
13 + FunctionCall,
14 + ToolCall,
15 +)
16 +from vllm.entrypoints.openai.tool_parsers import Ernie45ToolParser
17 +from vllm.transformers_utils.detokenizer_utils import detokenize_incrementally
18 +from vllm.transformers_utils.tokenizer import AnyTokenizer, get_tokenizer
19 +
20 +# Use a common model that is likely to be available
21 +MODEL = "baidu/ERNIE-4.5-21B-A3B-Thinking"
22 +
23 +
24 +@pytest.fixture(scope="module")
25 +def ernie45_tokenizer():
26 + return get_tokenizer(tokenizer_name=MODEL, trust_remote_code=True)
27 +
28 +
29 +@pytest.fixture
30 +def ernie45_tool_parser(ernie45_tokenizer):
31 + return Ernie45ToolParser(ernie45_tokenizer)
32 +
33 +
34 +def assert_tool_calls(
35 + actual_tool_calls: list[ToolCall], expected_tool_calls: list[ToolCall]
36 +):
37 + assert len(actual_tool_calls) == len(expected_tool_calls)
38 +
39 + for actual_tool_call, expected_tool_call in zip(
40 + actual_tool_calls, expected_tool_calls
41 + ):
42 + assert isinstance(actual_tool_call.id, str)
43 + assert len(actual_tool_call.id) > 0
44 +
45 + assert actual_tool_call.type == "function"
46 + assert actual_tool_call.function.name == expected_tool_call.function.name
47 + # Compare arguments as JSON objects to handle formatting differences
48 + actual_args = json.loads(actual_tool_call.function.arguments)
49 + expected_args = json.loads(expected_tool_call.function.arguments)
50 + assert actual_args == expected_args
51 +
52 +
53 +def test_extract_tool_calls_no_tools(ernie45_tool_parser):
54 + model_output = "This is a test"
55 + extracted_tool_calls = ernie45_tool_parser.extract_tool_calls(
56 + model_output, request=None
57 + ) # type: ignore[arg-type]
58 + assert not extracted_tool_calls.tools_called
59 + assert extracted_tool_calls.tool_calls == []
60 + assert extracted_tool_calls.content == model_output
61 +
62 +
63 +@pytest.mark.parametrize(
64 + ids=[
65 + "single_tool_call",
66 + "multiple_tool_calls",
67 + "tool_call_with_content_before",
68 + ],
69 + argnames=["model_output", "expected_tool_calls", "expected_content"],
70 + argvalues=[
71 + (
72 + """<tool_call>
73 +{"name": "get_current_temperature", "arguments": {"location": "Beijing"}}
74 +</tool_call>
75 +""",
76 + [
77 + ToolCall(
78 + function=FunctionCall(
79 + name="get_current_temperature",
80 + arguments=json.dumps(
81 + {
82 + "location": "Beijing",
83 + }
84 + ),
85 + )
86 + )
87 + ],
88 + None,
89 + ),
90 + (
91 + """<tool_call>
92 +{"name": "get_current_temperature", "arguments": {"location": "Beijing"}}
93 +</tool_call>
94 +<tool_call>
95 +{"name": "get_temperature_unit", "arguments": {"location": "Guangzhou", "unit": "c"}}
96 +</tool_call>
97 +""",
98 + [
99 + ToolCall(
100 + function=FunctionCall(
101 + name="get_current_temperature",
102 + arguments=json.dumps(
103 + {
104 + "location": "Beijing",
105 + }
106 + ),
107 + )
108 + ),
109 + ToolCall(
110 + function=FunctionCall(
111 + name="get_temperature_unit",
112 + arguments=json.dumps(
113 + {
114 + "location": "Guangzhou",
115 + "unit": "c",
116 + }
117 + ),
118 + )
119 + ),
120 + ],
121 + None,
122 + ),
123 + (
124 + """I need to call two tools to handle these two issues separately.
125 +</think>
126 +
127 +<tool_call>
128 +{"name": "get_current_temperature", "arguments": {"location": "Beijing"}}
129 +</tool_call>
130 +<tool_call>
131 +{"name": "get_temperature_unit", "arguments": {"location": "Guangzhou", "unit": "c"}}
132 +</tool_call>
133 +""",
134 + [
135 + ToolCall(
136 + function=FunctionCall(
137 + name="get_current_temperature",
138 + arguments=json.dumps(
139 + {
140 + "location": "Beijing",
141 + }
142 + ),
143 + )
144 + ),
145 + ToolCall(
146 + function=FunctionCall(
147 + name="get_temperature_unit",
148 + arguments=json.dumps(
149 + {
150 + "location": "Guangzhou",
151 + "unit": "c",
152 + }
153 + ),
154 + )
155 + ),
156 + ],
157 + "I need to call two tools to handle these two issues separately.\n</think>",
158 + ),
159 + ],
160 +)
161 +def test_extract_tool_calls(
162 + ernie45_tool_parser, model_output, expected_tool_calls, expected_content
163 +):
164 + extracted_tool_calls = ernie45_tool_parser.extract_tool_calls(
165 + model_output, request=None
166 + ) # type: ignore[arg-type]
167 + assert extracted_tool_calls.tools_called
168 +
169 + assert_tool_calls(extracted_tool_calls.tool_calls, expected_tool_calls)
170 +
171 + assert extracted_tool_calls.content == expected_content
172 +
173 +
174 +def stream_delta_message_generator(
175 + ernie45_tool_parser: Ernie45ToolParser,
176 + ernie45_tokenizer: AnyTokenizer,
177 + model_output: str,
178 + request: ChatCompletionRequest | None = None,
179 +) -> Generator[DeltaMessage, None, None]:
180 + all_token_ids = ernie45_tokenizer.encode(model_output, add_special_tokens=False)
181 +
182 + previous_text = ""
183 + previous_tokens = None
184 + prefix_offset = 0
185 + read_offset = 0
186 + for i, delta_token in enumerate(all_token_ids):
187 + delta_token_ids = [delta_token]
188 + previous_token_ids = all_token_ids[:i]
189 + current_token_ids = all_token_ids[: i + 1]
190 +
191 + (new_tokens, delta_text, new_prefix_offset, new_read_offset) = (
192 + detokenize_incrementally(
193 + tokenizer=ernie45_tokenizer,
194 + all_input_ids=current_token_ids,
195 + prev_tokens=previous_tokens,
196 + prefix_offset=prefix_offset,
197 + read_offset=read_offset,
198 + skip_special_tokens=False,
199 + spaces_between_special_tokens=True,
200 + )
201 + )
202 +
203 + current_text = previous_text + delta_text
204 +
205 + delta_message = ernie45_tool_parser.extract_tool_calls_streaming(
206 + previous_text,
207 + current_text,
208 + delta_text,
209 + previous_token_ids,
210 + current_token_ids,
211 + delta_token_ids,
212 + request=request,
213 + )
214 + if delta_message:
215 + yield delta_message
216 +
217 + previous_text = current_text
218 + previous_tokens = (
219 + previous_tokens + new_tokens if previous_tokens else new_tokens
220 + )
221 + prefix_offset = new_prefix_offset
222 + read_offset = new_read_offset
223 +
224 +
225 +@pytest.mark.parametrize(
226 + ids=[
227 + "single_tool_call",
228 + "multiple_tool_calls",
229 + "tool_call_with_content_before",
230 + ],
231 + argnames=["model_output", "expected_tool_calls", "expected_content"],
232 + argvalues=[
233 + (
234 + """<tool_call>
235 +{"name": "get_current_temperature", "arguments": {"location": "Beijing"}}
236 +</tool_call>
237 +""",
238 + [
239 + ToolCall(
240 + function=FunctionCall(
241 + name="get_current_temperature",
242 + arguments=json.dumps(
243 + {
244 + "location": "Beijing",
245 + }
246 + ),
247 + )
248 + )
249 + ],
250 + None,
251 + ),
252 + (
253 + """<tool_call>
254 +{"name": "get_current_temperature", "arguments": {"location": "Beijing"}}
255 +</tool_call>
256 +<tool_call>
257 +{"name": "get_temperature_unit", "arguments": {"location": "Guangzhou", "unit": "c"}}
258 +</tool_call>
259 +""",
260 + [
261 + ToolCall(
262 + function=FunctionCall(
263 + name="get_current_temperature",
264 + arguments=json.dumps(
265 + {
266 + "location": "Beijing",
267 + }
268 + ),
269 + )
270 + ),
271 + ToolCall(
272 + function=FunctionCall(
273 + name="get_temperature_unit",
274 + arguments=json.dumps(
275 + {
276 + "location": "Guangzhou",
277 + "unit": "c",
278 + }
279 + ),
280 + )
281 + ),
282 + ],
283 + None,
284 + ),
285 + (
286 + """I need to call two tools to handle these two issues separately.
287 +</think>
288 +
289 +<tool_call>
290 +{"name": "get_current_temperature", "arguments": {"location": "Beijing"}}
291 +</tool_call>
292 +<tool_call>
293 +{"name": "get_temperature_unit", "arguments": {"location": "Guangzhou", "unit": "c"}}
294 +</tool_call>
295 +""",
296 + [
297 + ToolCall(
298 + function=FunctionCall(
299 + name="get_current_temperature",
300 + arguments=json.dumps(
301 + {
302 + "location": "Beijing",
303 + }
304 + ),
305 + )
306 + ),
307 + ToolCall(
308 + function=FunctionCall(
309 + name="get_temperature_unit",
310 + arguments=json.dumps(
311 + {
312 + "location": "Guangzhou",
313 + "unit": "c",
314 + }
315 + ),
316 + )
317 + ),
318 + ],
319 + "I need to call two tools to handle these two issues separately.\n</think>",
320 + ),
321 + ],
322 +)
323 +def test_extract_tool_calls_streaming_incremental(
324 + ernie45_tool_parser,
325 + ernie45_tokenizer,
326 + model_output,
327 + expected_tool_calls,
328 + expected_content,
329 +):
330 + """Verify the Ernie45 Parser streaming behavior by verifying each chunk is as expected.""" # noqa: E501
331 + request = ChatCompletionRequest(model=MODEL, messages=[], tools=[])
332 +
333 + tool_calls_dict = {}
334 + for delta_message in stream_delta_message_generator(
335 + ernie45_tool_parser, ernie45_tokenizer, model_output, request
336 + ):
337 + if (
338 + delta_message.role is None
339 + and delta_message.content is None
340 + and delta_message.reasoning_content is None
341 + and len(delta_message.tool_calls) == 0
342 + ):
343 + continue
344 + tool_calls = delta_message.tool_calls
345 + for tool_call_chunk in tool_calls:
346 + index = tool_call_chunk.index
347 + if index not in tool_calls_dict:
348 + if tool_call_chunk.function.arguments is None:
349 + tool_call_chunk.function.arguments = ""
350 + tool_calls_dict[index] = tool_call_chunk
351 + else:
352 + tool_calls_dict[
353 + index
354 + ].function.arguments += tool_call_chunk.function.arguments
355 + actual_tool_calls = list(tool_calls_dict.values())
356 +
357 + assert len(actual_tool_calls) > 0
358 + # check tool call format
359 + assert_tool_calls(actual_tool_calls, expected_tool_calls)
vllm/entrypoints/openai/tool_parsers/__init__.py
+2
@@ -4,6 +4,7 @@
4 from .abstract_tool_parser import ToolParser, ToolParserManager
5 from .deepseekv3_tool_parser import DeepSeekV3ToolParser
6 from .deepseekv31_tool_parser import DeepSeekV31ToolParser
7 +from .ernie45_tool_parser import Ernie45ToolParser
8 from .glm4_moe_tool_parser import Glm4MoeModelToolParser
9 from .granite_20b_fc_tool_parser import Granite20bFCToolParser
10 from .granite_tool_parser import GraniteToolParser
@@ -42,6 +43,7 @@ __all__ = [
43 "Phi4MiniJsonToolParser",
44 "DeepSeekV3ToolParser",
45 "DeepSeekV31ToolParser",
46 + "Ernie45ToolParser",
47 "xLAMToolParser",
48 "MinimaxToolParser",
49 "KimiK2ToolParser",
vllm/entrypoints/openai/tool_parsers/ernie45_tool_parser.py new
+212
@@ -0,0 +1,212 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +
4 +import json
5 +from collections.abc import Sequence
6 +
7 +import regex as re
8 +
9 +from vllm.entrypoints.openai.protocol import (
10 + ChatCompletionRequest,
11 + DeltaFunctionCall,
12 + DeltaMessage,
13 + DeltaToolCall,
14 + ExtractedToolCallInformation,
15 + FunctionCall,
16 + ToolCall,
17 +)
18 +from vllm.entrypoints.openai.tool_parsers.abstract_tool_parser import (
19 + ToolParser,
20 + ToolParserManager,
21 +)
22 +from vllm.logger import init_logger
23 +from vllm.transformers_utils.tokenizer import AnyTokenizer
24 +
25 +logger = init_logger(__name__)
26 +
27 +
28 +@ToolParserManager.register_module("ernie45")
29 +class Ernie45ToolParser(ToolParser):
30 + def __init__(self, tokenizer: AnyTokenizer):
31 + """
32 + Ernie thinking model format:
33 + abc\n</think>\n\n\n<tool_call>\ndef\n</tool_call>\n
34 + """
35 + super().__init__(tokenizer)
36 + self.current_tool_name_sent = False
37 + self.prev_tool_call_arr: list[dict] = []
38 + self.current_tool_id = -1
39 + self.streamed_args_for_tool: list[str] = []
40 + self.think_end_token = "</think>"
41 + self.response_start_token: str = "<response>"
42 + self.response_end_token: str = "</response>"
43 + self.tool_call_start_token = "<tool_call>"
44 + self.tool_call_end_token = "</tool_call>"
45 + self.tool_calls_start_token = self.tool_call_start_token
46 + self.newline_token: str = "<0x0A>"
47 +
48 + self.tool_call_regex = re.compile(
49 + r"<tool_call>\s*(?P<json>\{.*?\})\s*</tool_call>", re.DOTALL
50 + )
51 +
52 + if not self.model_tokenizer:
53 + raise ValueError(
54 + "The model tokenizer must be passed to the ToolParser "
55 + "constructor during construction."
56 + )
57 +
58 + self.think_end_token_id = self.vocab.get(self.think_end_token)
59 + self.response_start_token_id = self.vocab.get(self.response_start_token)
60 + self.response_end_token_id = self.vocab.get(self.response_end_token)
61 + self.tool_call_start_token_id = self.vocab.get(self.tool_call_start_token)
62 + self.tool_call_end_token_id = self.vocab.get(self.tool_call_end_token)
63 + self.newline_token_id = self.vocab.get(self.newline_token)
64 + self.parser_token_ids = [
65 + self.think_end_token_id,
66 + self.response_start_token_id,
67 + self.response_end_token_id,
68 + ]
69 +
70 + self._buffer = ""
71 +
72 + def extract_tool_calls(
73 + self,
74 + model_output: str,
75 + request: ChatCompletionRequest,
76 + ) -> ExtractedToolCallInformation:
77 + # sanity check; avoid unnecessary processing
78 + if self.tool_calls_start_token not in model_output:
79 + return ExtractedToolCallInformation(
80 + tools_called=False, tool_calls=[], content=model_output
81 + )
82 +
83 + else:
84 + try:
85 + tool_call_json_list = self.tool_call_regex.findall(model_output)
86 +
87 + tool_calls = []
88 + for tool_call_json in tool_call_json_list:
89 + tool_call_dict = json.loads(tool_call_json)
90 + args_str = json.dumps(
91 + tool_call_dict.get("arguments", {}), ensure_ascii=False
92 + )
93 + tool_calls.append(
94 + ToolCall(
95 + type="function",
96 + function=FunctionCall(
97 + name=tool_call_dict.get("name", ""),
98 + arguments=args_str,
99 + ),
100 + )
101 + )
102 +
103 + content = model_output[
104 + : model_output.find(self.tool_calls_start_token)
105 + ].rstrip("\n")
106 + return ExtractedToolCallInformation(
107 + tools_called=True,
108 + tool_calls=tool_calls,
109 + content=content if content else None,
110 + )
111 +
112 + except Exception:
113 + logger.exception("Error in extracting tool call from response.")
114 + return ExtractedToolCallInformation(
115 + tools_called=False, tool_calls=[], content=model_output
116 + )
117 +
118 + def extract_tool_calls_streaming(
119 + self,
120 + previous_text: str,
121 + current_text: str,
122 + delta_text: str,
123 + previous_token_ids: Sequence[int],
124 + current_token_ids: Sequence[int],
125 + delta_token_ids: Sequence[int],
126 + request: ChatCompletionRequest,
127 + ) -> DeltaMessage | None:
128 + self._buffer += delta_text
129 + cur_text = self._buffer
130 + start_idx = cur_text.find(self.tool_call_start_token)
131 + if start_idx == -1:
132 + self._buffer = ""
133 + # At least one toolcall has been completed
134 + if self.current_tool_id > 0:
135 + cur_text = ""
136 + if self.current_tool_id == -1 and all(
137 + token_id == self.newline_token_id for token_id in previous_token_ids
138 + ):
139 + cur_text = cur_text.strip("\n")
140 +
141 + # handle <response> </response> when tool_call is not triggered
142 + # cur_text === delta_text
143 + content = cur_text
144 + if self.response_start_token_id in delta_token_ids:
145 + content = content.lstrip("\n")
146 + response_start_idx = content.find(self.response_start_token)
147 + content = content[response_start_idx + len(self.response_start_token) :]
148 + # if have </response>, remove it
149 + response_end_idx = content.rfind(self.response_end_token)
150 + if response_end_idx != -1:
151 + content = content[:response_end_idx]
152 + elif self.response_end_token_id in delta_token_ids:
153 + response_end_idx = content.rfind(self.response_end_token)
154 + content = content[:response_end_idx]
155 + # remove \n after </think> or <response> or </response>
156 + if (
157 + len(previous_token_ids) > 0
158 + and previous_token_ids[-1] in self.parser_token_ids
159 + ) and (
160 + len(delta_token_ids) > 0 and delta_token_ids[0] == self.newline_token_id
161 + ):
162 + content = content.lstrip("\n")
163 +
164 + return DeltaMessage(content=content if content else None)
165 + logger.debug("cur_text = %s", cur_text)
166 + end_idx = cur_text.find(self.tool_call_end_token)
167 + if end_idx != -1:
168 + if self.current_tool_id == -1:
169 + self.current_tool_id = 0
170 + self.prev_tool_call_arr = []
171 + self.streamed_args_for_tool = []
172 + while len(self.prev_tool_call_arr) <= self.current_tool_id:
173 + self.prev_tool_call_arr.append({})
174 + while len(self.streamed_args_for_tool) <= self.current_tool_id:
175 + self.streamed_args_for_tool.append("")
176 +
177 + extracted_tool_calls = self.extract_tool_calls(
178 + cur_text[: end_idx + len(self.tool_call_end_token)], request
179 + )
180 +
181 + if len(extracted_tool_calls.tool_calls) == 0:
182 + logger.warning("Failed to extract any tool calls.")
183 + return None
184 + tool_call = extracted_tool_calls.tool_calls[0]
185 + self.prev_tool_call_arr[self.current_tool_id] = {
186 + "name": tool_call.function.name,
187 + "arguments": json.loads(tool_call.function.arguments),
188 + }
189 + self.streamed_args_for_tool[self.current_tool_id] = (
190 + tool_call.function.arguments
191 + )
192 + delta = DeltaMessage(
193 + content=extracted_tool_calls.content,
194 + tool_calls=[
195 + DeltaToolCall(
196 + index=self.current_tool_id,
197 + id=tool_call.id,
198 + type=tool_call.type,
199 + function=DeltaFunctionCall(
200 + name=tool_call.function.name,
201 + arguments=tool_call.function.arguments,
202 + ),
203 + )
204 + ],
205 + )
206 + self.current_tool_id += 1
207 + self._buffer = cur_text[end_idx + len(self.tool_call_end_token) :]
208 + return delta
209 +
210 + self._buffer = cur_text[start_idx:]
211 + content = cur_text[:start_idx].rstrip("\n")
212 + return DeltaMessage(content=content if content else None)
vllm/reasoning/__init__.py
+2
@@ -4,6 +4,7 @@
4 from .abs_reasoning_parsers import ReasoningParser, ReasoningParserManager
5 from .basic_parsers import BaseThinkingReasoningParser
6 from .deepseek_r1_reasoning_parser import DeepSeekR1ReasoningParser
7 +from .ernie45_reasoning_parser import Ernie45ReasoningParser
8 from .glm4_moe_reasoning_parser import Glm4MoeModelReasoningParser
9 from .gptoss_reasoning_parser import GptOssReasoningParser
10 from .granite_reasoning_parser import GraniteReasoningParser
@@ -19,6 +20,7 @@ __all__ = [
20 "BaseThinkingReasoningParser",
21 "ReasoningParserManager",
22 "DeepSeekR1ReasoningParser",
23 + "Ernie45ReasoningParser",
24 "GraniteReasoningParser",
25 "HunyuanA13BReasoningParser",
26 "Qwen3ReasoningParser",
vllm/reasoning/ernie45_reasoning_parser.py new
+169
@@ -0,0 +1,169 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +
4 +from collections.abc import Sequence
5 +
6 +from transformers import PreTrainedTokenizerBase
7 +
8 +from vllm.entrypoints.openai.protocol import ChatCompletionRequest, DeltaMessage
9 +from vllm.logger import init_logger
10 +from vllm.reasoning import ReasoningParserManager
11 +from vllm.reasoning.basic_parsers import BaseThinkingReasoningParser
12 +
13 +logger = init_logger(__name__)
14 +
15 +
16 +@ReasoningParserManager.register_module("ernie45")
17 +class Ernie45ReasoningParser(BaseThinkingReasoningParser):
18 + """
19 + Reasoning parser for Ernie45 thinking model.
20 + The Ernie45 thinking model ouput format is
21 + abc\n</think>\n\n<response>\ndef\n</response>\n
22 + or abc\n</think>\ndef
23 + """
24 +
25 + response_start_token: str = "<response>"
26 + response_end_token: str = "</response>"
27 + newline_token: str = "<0x0A>"
28 +
29 + @property
30 + def start_token(self) -> str:
31 + """The token that starts reasoning content."""
32 + return "<think>"
33 +
34 + @property
35 + def end_token(self) -> str:
36 + """The token that ends reasoning content."""
37 + return "</think>"
38 +
39 + def __init__(self, tokenizer: PreTrainedTokenizerBase):
40 + super().__init__(tokenizer)
41 +
42 + if not self.model_tokenizer:
43 + raise ValueError(
44 + "The model tokenizer must be passed to the ReasoningParser "
45 + "constructor during construction."
46 + )
47 +
48 + self.start_token_id = self.vocab.get(self.start_token)
49 + self.end_token_id = self.vocab.get(self.end_token)
50 + self.response_start_token_id = self.vocab.get(self.response_start_token)
51 + self.response_end_token_id = self.vocab.get(self.response_end_token)
52 + self.newline_token_id = self.vocab.get(self.newline_token)
53 +
54 + self.parser_token_ids = [self.end_token_id, self.response_end_token_id]
55 +
56 + if self.start_token_id is None or self.end_token_id is None:
57 + raise RuntimeError(
58 + "Ernie45 reasoning parser could not locate think start/end "
59 + "tokens in the tokenizer!"
60 + )
61 +
62 + def extract_reasoning_content_streaming(
63 + self,
64 + previous_text: str,
65 + current_text: str,
66 + delta_text: str,
67 + previous_token_ids: Sequence[int],
68 + current_token_ids: Sequence[int],
69 + delta_token_ids: Sequence[int],
70 + ) -> DeltaMessage | None:
71 + """
72 + Extract reasoning content from a delta message.
73 + Handles streaming output where previous + delta = current.
74 + Uses token IDs for faster processing.
75 + The Ernie45 thinking model ouput format is
76 + abc\n</think>\n\n<response>\ndef\n</response>\n
77 + or abc\n</think>\ndef
78 + - 'abc' goes to reasoning_content
79 + - 'def' goes to content
80 + """
81 + # Skip single special tokens
82 + if len(delta_token_ids) == 1 and (
83 + delta_token_ids[0]
84 + in [
85 + self.start_token_id,
86 + self.end_token_id,
87 + self.response_start_token_id,
88 + self.response_end_token_id,
89 + ]
90 + ):
91 + return None
92 +
93 + # No <think> in previous or delta, also need to check for </think>.
94 + # Because the model may have generated </think> without <think>
95 + if self.end_token_id in delta_token_ids:
96 + # </think> in delta with more tokens,
97 + # extract reasoning content and content
98 + think_end_index = delta_text.find(self.end_token)
99 + reasoning_content = delta_text[:think_end_index]
100 + content = delta_text[think_end_index + len(self.end_token) :]
101 + content = content.lstrip("\n")
102 + response_start_idx = content.find(self.response_start_token)
103 + response_end_idx = content.rfind(self.response_end_token)
104 + if response_start_idx != -1:
105 + content = content[response_start_idx + len(self.response_start_token) :]
106 + if response_end_idx != -1:
107 + content = content[:response_end_idx]
108 + return DeltaMessage(
109 + reasoning_content=reasoning_content,
110 + content=content if content else None,
111 + )
112 + elif self.end_token_id in previous_token_ids:
113 + # </think> in previous, thinking content ends
114 + content = delta_text
115 + if self.response_start_token_id in delta_token_ids:
116 + content = content.lstrip("\n")
117 + response_start_idx = content.find(self.response_start_token)
118 + content = content[response_start_idx + len(self.response_start_token) :]
119 + # if have </response>, remove it
120 + response_end_idx = content.rfind(self.response_end_token)
121 + if response_end_idx != -1:
122 + content = content[:response_end_idx]
123 + elif self.response_end_token_id in delta_token_ids:
124 + response_end_idx = content.rfind(self.response_end_token)
125 + content = content[:response_end_idx]
126 + # remove \n after </think> or </response>
127 + if previous_token_ids[-1] in self.parser_token_ids and (
128 + len(delta_token_ids) > 0 and delta_token_ids[0] == self.newline_token_id
129 + ):
130 + content = content.lstrip("\n")
131 + # remove \n after </think>\n
132 + if (
133 + len(previous_token_ids) > 1
134 + and previous_token_ids[-2] == self.end_token_id
135 + ) and (
136 + len(delta_token_ids) > 0 and delta_token_ids[0] == self.newline_token_id
137 + ):
138 + content = content.lstrip("\n")
139 +
140 + return DeltaMessage(content=content if content else None)
141 + else:
142 + # no </think> in previous or delta, reasoning content continues
143 + return DeltaMessage(reasoning_content=delta_text)
144 +
145 + def extract_reasoning_content(
146 + self, model_output: str, request: ChatCompletionRequest
147 + ) -> tuple[str | None, str | None]:
148 + """
149 + Extract reasoning content from the model output.
150 + The Ernie45 thinking model ouput format is
151 + abc\n</think>\n\n\n<response>\ndef\n</response>\n
152 + or abc\n</think>\ndef
153 + - 'abc' goes to reasoning_content
154 + - 'def' goes to content
155 + Returns:
156 + tuple[Optional[str], Optional[str]]: reasoning content and content
157 + """
158 + reasoning_content, content = super().extract_reasoning_content(
159 + model_output, request
160 + )
161 + if content:
162 + start_idx = content.find(self.response_start_token)
163 + end_idx = content.rfind(self.response_end_token)
164 + # Simultaneously existing and in the correct order
165 + if start_idx != -1 and end_idx != -1 and start_idx < end_idx:
166 + content = content[start_idx + len(self.response_start_token) : end_idx]
167 + final_content = content or None
168 +
169 + return reasoning_content, final_content