@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / b140416abf
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 15, 02:15 UTC

[Model] Add reason parser for Hunyuan A13B Model. (#20625)

Signed-off-by: Asher Zhang <asherszhang@tencent.com>

Asher committed Jul 11, 2025 at 00:33 UTC b140416abf836dcf927eda6f50bba11dca860b20
3 files changed +402
tests/reasoning/test_hunyuan_reasoning_parser.py new
+162
@@ -0,0 +1,162 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +
4 +import pytest
5 +from transformers import AutoTokenizer
6 +
7 +from tests.reasoning.utils import run_reasoning_extraction
8 +from vllm.reasoning import ReasoningParser, ReasoningParserManager
9 +
10 +parser_name = "hunyuan_a13b"
11 +START_REASONING = "<think>\n"
12 +START_RESPONSE = "\n</think>\n<answer>\n"
13 +END_RESPONSE = "\n</answer>"
14 +
15 +NO_REASONING_QUICK_THROUGHT = {
16 + "output":
17 + f"{START_REASONING}{START_RESPONSE}This is the rest{END_RESPONSE}", #noqa: E501
18 + "reasoning_content": None,
19 + "content": "This is the rest",
20 +}
21 +
22 +SIMPLE_REASONING = {
23 + "output":
24 + f"{START_REASONING}This is a reasoning section{START_RESPONSE}This is the rest{END_RESPONSE}", #noqa: E501
25 + "reasoning_content": "This is a reasoning section",
26 + "content": "This is the rest",
27 +}
28 +COMPLETE_REASONING = {
29 + "output": f"{START_REASONING}This is a reasoning section{START_RESPONSE}",
30 + "reasoning_content": "This is a reasoning section",
31 + "content": None,
32 +}
33 +NO_REASONING = {
34 + "output": "This is content",
35 + "reasoning_content": None,
36 + "content": "This is content",
37 +}
38 +MULTIPLE_LINES = {
39 + "output":
40 + f"{START_REASONING}This\nThat{START_RESPONSE}This is the rest\nThat",
41 + "reasoning_content": "This\nThat",
42 + "content": "This is the rest\nThat",
43 +}
44 +REASONING_WITH_THINK = {
45 + "output":
46 + f"{START_REASONING}This is a reasoning section{START_RESPONSE}This is the rest", #noqa: E501
47 + "reasoning_content": "This is a reasoning section",
48 + "content": "This is the rest",
49 +}
50 +COMPLETE_REASONING_WITH_THINK = {
51 + "output": f"{START_REASONING}This is a reasoning section{START_RESPONSE}",
52 + "reasoning_content": "This is a reasoning section",
53 + "content": None,
54 +}
55 +MULTIPLE_LINES_WITH_THINK = {
56 + "output":
57 + f"{START_REASONING}This\nThat{START_RESPONSE}This is the rest\nThat",
58 + "reasoning_content": "This\nThat",
59 + "content": "This is the rest\nThat",
60 +}
61 +
62 +TEST_CASES = [
63 + pytest.param(
64 + False,
65 + SIMPLE_REASONING,
66 + id="simple_reasoning",
67 + ),
68 + pytest.param(
69 + False,
70 + COMPLETE_REASONING,
71 + id="complete_reasoning",
72 + ),
73 + pytest.param(
74 + False,
75 + NO_REASONING,
76 + id="no_reasoning",
77 + ),
78 + pytest.param(False, NO_REASONING_QUICK_THROUGHT, id="no_reasoning_quick"),
79 + pytest.param(
80 + False,
81 + MULTIPLE_LINES,
82 + id="multiple_lines",
83 + ),
84 + pytest.param(
85 + False,
86 + REASONING_WITH_THINK,
87 + id="reasoning_with_think",
88 + ),
89 + pytest.param(
90 + False,
91 + COMPLETE_REASONING_WITH_THINK,
92 + id="complete_reasoning_with_think",
93 + ),
94 + pytest.param(
95 + False,
96 + MULTIPLE_LINES_WITH_THINK,
97 + id="multiple_lines_with_think",
98 + ),
99 + pytest.param(
100 + True,
101 + SIMPLE_REASONING,
102 + id="simple_reasoning_streaming",
103 + ),
104 + pytest.param(
105 + True,
106 + COMPLETE_REASONING,
107 + id="complete_reasoning_streaming",
108 + ),
109 + pytest.param(
110 + True,
111 + NO_REASONING,
112 + id="no_reasoning_streaming",
113 + ),
114 + pytest.param(True,
115 + NO_REASONING_QUICK_THROUGHT,
116 + id="no_reasoning_quick_stream"),
117 + pytest.param(
118 + True,
119 + MULTIPLE_LINES,
120 + id="multiple_lines_streaming",
121 + ),
122 + pytest.param(
123 + True,
124 + REASONING_WITH_THINK,
125 + id="reasoning_with_think_streaming",
126 + ),
127 + pytest.param(
128 + True,
129 + COMPLETE_REASONING_WITH_THINK,
130 + id="complete_reasoning_with_think_streaming",
131 + ),
132 + pytest.param(
133 + True,
134 + MULTIPLE_LINES_WITH_THINK,
135 + id="multiple_lines_with_think_streaming",
136 + ),
137 +]
138 +
139 +# Global tokenizer initialization to avoid repeated loading
140 +tokenizer = AutoTokenizer.from_pretrained("tencent/Hunyuan-A13B-Instruct",
141 + trust_remote_code=True)
142 +
143 +
144 +@pytest.mark.parametrize("streaming, param_dict", TEST_CASES)
145 +def test_reasoning(
146 + streaming: bool,
147 + param_dict: dict,
148 +):
149 + output = tokenizer.tokenize(param_dict["output"])
150 + # decode everything to tokens
151 + output_tokens: list[str] = [
152 + tokenizer.convert_tokens_to_string([token]) for token in output
153 + ]
154 + parser: ReasoningParser = ReasoningParserManager.get_reasoning_parser(
155 + parser_name)(tokenizer)
156 +
157 + reasoning, content = run_reasoning_extraction(parser,
158 + output_tokens,
159 + streaming=streaming)
160 +
161 + assert reasoning == param_dict["reasoning_content"]
162 + assert content == param_dict["content"]
vllm/reasoning/__init__.py
+2
@@ -4,6 +4,7 @@
4 from .abs_reasoning_parsers import ReasoningParser, ReasoningParserManager
5 from .deepseek_r1_reasoning_parser import DeepSeekR1ReasoningParser
6 from .granite_reasoning_parser import GraniteReasoningParser
7 +from .hunyuan_a13b_reasoning_parser import HunyuanA13BReasoningParser
8 from .qwen3_reasoning_parser import Qwen3ReasoningParser
9
10 __all__ = [
@@ -11,5 +12,6 @@ __all__ = [
12 "ReasoningParserManager",
13 "DeepSeekR1ReasoningParser",
14 "GraniteReasoningParser",
15 + "HunyuanA13BReasoningParser",
16 "Qwen3ReasoningParser",
17 ]
vllm/reasoning/hunyuan_a13b_reasoning_parser.py new
+238
@@ -0,0 +1,238 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +
4 +import re
5 +from collections.abc import Sequence
6 +from typing import Optional, Union
7 +
8 +from transformers import PreTrainedTokenizerBase
9 +
10 +from vllm.entrypoints.openai.protocol import (ChatCompletionRequest,
11 + DeltaMessage)
12 +from vllm.logger import init_logger
13 +from vllm.reasoning import ReasoningParser, ReasoningParserManager
14 +
15 +logger = init_logger(__name__)
16 +
17 +
18 +@ReasoningParserManager.register_module("hunyuan_a13b")
19 +class HunyuanA13BReasoningParser(ReasoningParser):
20 + """
21 + Reasoning parser for Hunyuan A13B Model
22 +
23 + HunyuanReasoningParser
24 +
25 + This class implements a reasoning parser specifically designed
26 + for the Hunyuan A13B Model. It is responsible for parsing and
27 + extracting structured reasoning and answer segments from model
28 + outputs that follow a specific pattern.
29 +
30 + Key Features:
31 + - For non-stream output , Recognizes and extracts reasoning ("think")
32 + and answer ("answer") sections from text using regular expressions.
33 + - For stream process, it require a token id sequences to change the
34 + reasoning state and other state so it maintains internal state to
35 + manage parsing across multiple token.
36 +
37 +
38 + think start: "<think>\n": [14023, 771, 397]
39 + think ends: "\n</think>\n<answer>\n": [198, 524, 27963, 397, 27, 9399, 397]
40 + response ends: "\n</answer>": [524, 9399, 29]
41 + """
42 +
43 + def __init__(self, tokenizer: PreTrainedTokenizerBase):
44 + super().__init__(tokenizer)
45 + self.think_start_expr = r"<think>\n"
46 + self.think_end_expr = r"\n</think>\n"
47 +
48 + self.response_start_expr = r"\n</think>\n<answer>\n"
49 + self.response_end_expr = r"\n</answer>"
50 +
51 + self.full_match_reasoning_regex = re.compile(
52 + rf"(?:{self.think_start_expr}(.*?){self.response_start_expr})?(.*?){self.response_end_expr}",
53 + re.DOTALL)
54 +
55 + self.half_match_reasoning_regex = re.compile(
56 + rf"{self.think_start_expr}(.*?){self.response_start_expr}(.*)",
57 + re.DOTALL)
58 +
59 + self.think_start_ids = [14023, 771, 397]
60 + self.think_start_ids_fast = [14023, 771, 1363]
61 + self.response_start_ids = [198, 524, 27963, 397, 27, 9399, 397]
62 + self.response_start_ids_fast = [524, 27963, 397, 27, 9399, 397]
63 + self.response_end_ids = [198, 524, 9399, 29]
64 + self.fast_think_ids = [
65 + 14023, 771, 1363, 524, 27963, 397, 27, 9399, 397
66 + ]
67 +
68 + # when state change, send out all the buffered text in last state
69 + self.buffered_text = []
70 + self.buffered_ids = []
71 +
72 + self.current_state = "reasoning"
73 + self.all_states = ["reasoning", "response"]
74 +
75 + self.current_state = "idle"
76 + self.expected_sequence = self.think_start_ids
77 + # this sequence only for the think start, it has two way to start.
78 + self.expected_sequence_side = self.think_start_ids_fast
79 + self.sequence_index = 0
80 + self.token_buffer = []
81 + self.text_buffer = ""
82 +
83 + def is_reasoning_end(self, input_ids: list[int]) -> bool:
84 + return self.current_state == "response"
85 +
86 + def extract_reasoning_content(
87 + self, model_output: str, request: ChatCompletionRequest
88 + ) -> tuple[Optional[str], Optional[str]]:
89 + """Extract the reasoning content & content sections, respectively.
90 + If the sequence doesn't match what we expect, i.e., the model generates
91 + something else, all content is considered non-reasoning content.
92 +
93 + Args:
94 + model_output (str): Output of the model to be parsed.
95 + request (ChatCompletionRequest): Request being processed.
96 +
97 + Returns:
98 + tuple[Optional[str], Optional[str]]: Tuple pair containing the
99 + reasoning content and non-reasoning content.
100 + """
101 +
102 + re_match = self.full_match_reasoning_regex.findall(model_output)
103 + if re_match:
104 + reasoning_content, response_content = re_match[0]
105 + if len(reasoning_content) == 0:
106 + reasoning_content = None
107 + if len(response_content) == 0:
108 + response_content = None
109 + return reasoning_content, response_content
110 +
111 + fallback_regex = self.half_match_reasoning_regex
112 + fallback_match = fallback_regex.findall(model_output)
113 + if fallback_match:
114 + reasoning_content, response_content = fallback_match[0]
115 +
116 + if response_content.endswith(self.response_end_expr):
117 + response_content = response_content[:-len(self.
118 + response_end_expr)]
119 +
120 + if len(reasoning_content) == 0:
121 + reasoning_content = None
122 + if len(response_content) == 0:
123 + response_content = None
124 +
125 + return reasoning_content, response_content
126 +
127 + return None, model_output
128 +
129 + def _is_strict_increasing_subsequence(self, subsequence: Sequence[int],
130 + sequence: Sequence[int]) -> bool:
131 + if not subsequence:
132 + return False
133 +
134 + sub_idx = 0
135 + for num in sequence:
136 + if sub_idx < len(subsequence) and num == subsequence[sub_idx]:
137 + sub_idx += 1
138 + return sub_idx == len(subsequence)
139 +
140 + def extract_reasoning_content_streaming(
141 + self,
142 + previous_text: str,
143 + current_text: str,
144 + delta_text: str,
145 + previous_token_ids: Sequence[int],
146 + current_token_ids: Sequence[int],
147 + delta_token_ids: Sequence[int],
148 + ) -> Union[DeltaMessage, None]:
149 + """Extract content using token ID sequence state machine"""
150 + # Define sequences
151 + think_start_sequence = self.think_start_ids
152 + response_start_sequence = self.response_start_ids
153 + response_end_sequence = self.response_end_ids
154 +
155 + assert (len(delta_token_ids) == 1)
156 + # Process each token in the delta
157 + token = delta_token_ids[0]
158 +
159 + def check_token_with_sequence(token):
160 + if self.current_state == "idle" or self.current_state == "think":
161 + return (token == self.expected_sequence[self.sequence_index]
162 + or token == \
163 + self.expected_sequence_side[self.sequence_index])
164 + else:
165 + return token == self.expected_sequence[self.sequence_index]
166 +
167 + def check_last_token(token):
168 + if self.current_state == "idle" or self.current_state == "think":
169 + # only return true if it's judge using a side sequence.
170 + if (self.sequence_index - 1 < len(self.expected_sequence_side)
171 + and token
172 + == self.expected_sequence_side[self.sequence_index -
173 + 1]):
174 + return self.sequence_index == len(
175 + self.expected_sequence_side)
176 + else:
177 + return self.sequence_index == len(self.expected_sequence)
178 + else:
179 + return self.sequence_index == len(self.expected_sequence)
180 +
181 + # Check if token matches expected sequence
182 + token_in_state_seq = check_token_with_sequence(token)
183 +
184 + if token_in_state_seq:
185 + # Store matching token
186 + self.token_buffer.append(token)
187 + self.text_buffer += delta_text
188 + self.sequence_index += 1
189 + ## state change from idle->think->response->idle
190 +
191 + # Check if sequence fully matched
192 + if check_last_token(token):
193 + # State transition
194 + if self.current_state == "idle":
195 + self.current_state = "think"
196 + self.expected_sequence = response_start_sequence
197 + self.expected_sequence_side = self.response_start_ids_fast
198 + elif self.current_state == "think":
199 + self.current_state = "response"
200 + self.expected_sequence = response_end_sequence
201 + elif self.current_state == "response":
202 + self.current_state = "idle"
203 + self.expected_sequence = think_start_sequence
204 + self.expected_sequence_side = self.think_start_ids_fast
205 +
206 + # Reset matching state
207 + self.sequence_index = 0
208 + self.token_buffer = []
209 + self.text_buffer = ""
210 + # Do not send content for state transition texts.
211 + else:
212 + # Sequence broken - handle buffered content
213 + if self.token_buffer and len(self.token_buffer) > 0:
214 + # Send buffered tokens
215 + buffered_content = self.text_buffer + delta_text
216 + # Reset matching state
217 + self.sequence_index = 0
218 + self.token_buffer = []
219 + self.text_buffer = ""
220 +
221 + # Return content based on current state
222 + if self.current_state == "think":
223 + return DeltaMessage(reasoning_content=buffered_content,
224 + content=None)
225 + else:
226 + return DeltaMessage(reasoning_content=None,
227 + content=buffered_content)
228 + else:
229 + # No buffered content, send normally
230 + if self.current_state == "think":
231 + return DeltaMessage(reasoning_content=delta_text,
232 + content=None)
233 + else:
234 + return DeltaMessage(reasoning_content=None,
235 + content=delta_text)
236 +
237 + # If no content to send in this delta
238 + return None