A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 11, 23:45 UTC
[Model] Add Holo2 reasoning parser (#30048)
Signed-off-by: hdlj-h <hubert@hcompany.ai>
Hubert de La Jonquiere committed
Dec 5, 2025 at 03:38 UTC
befb59e5b102d2ffbdea85a85347eec8e2f7c27f
4 files changed
+277
docs/features/reasoning_outputs.md
+2
@@ -18,6 +18,7 @@ vLLM currently supports the following reasoning models:
18
| [ERNIE-4.5-VL series](https://huggingface.co/baidu/ERNIE-4.5-VL-28B-A3B-PT) | `ernie45` | `json`, `regex` | ❌ |
19
| [ERNIE-4.5-21B-A3B-Thinking](https://huggingface.co/baidu/ERNIE-4.5-21B-A3B-Thinking) | `ernie45` | `json`, `regex` | ✅ |
20
| [GLM-4.5 series](https://huggingface.co/collections/zai-org/glm-45-687c621d34bda8c9e4bf503b) | `glm45` | `json`, `regex` | ✅ |
21
+| [Holo2 series](https://huggingface.co/collections/Hcompany/holo2) | `holo2` | `json`, `regex` | ✅ |
22
| [Hunyuan A13B series](https://huggingface.co/collections/tencent/hunyuan-a13b-685ec38e5b46321e3ea7c4be) | `hunyuan_a13b` | `json`, `regex` | ✅ |
23
| [IBM Granite 3.2 language models](https://huggingface.co/collections/ibm-granite/granite-32-language-models-67b3bc8c13508f6d064cff9a) | `granite` | ❌ | ❌ |
24
| [MiniMax-M2](https://huggingface.co/MiniMaxAI/MiniMax-M2) | `minimax_m2_append_think` | `json`, `regex` | ✅ |
@@ -28,6 +29,7 @@ vLLM currently supports the following reasoning models:
29
IBM Granite 3.2 and DeepSeek-V3.1 reasoning is disabled by default; to enable it, you must also pass `thinking=True` in your `chat_template_kwargs`.
30
The reasoning feature for the Qwen3 series is enabled by default. To disable it, you must pass `enable_thinking=False` in your `chat_template_kwargs`.
31
DeepSeek-V3.1 tool calling is supported in non-thinking mode.
32
+ Holo2 reasoning is enabled by default. To disable it, you must also pass `thinking=False` in your `chat_template_kwargs`.
33
34
## Quickstart
35
tests/reasoning/test_holo2_reasoning_parser.py
new
+188
@@ -0,0 +1,188 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+import pytest
4
+from transformers import AutoTokenizer
5
+
6
+from tests.reasoning.utils import run_reasoning_extraction
7
+from vllm.reasoning import ReasoningParser, ReasoningParserManager
8
+from vllm.reasoning.deepseek_r1_reasoning_parser import DeepSeekR1ReasoningParser
9
+from vllm.reasoning.holo2_reasoning_parser import Holo2ReasoningParser
10
+from vllm.reasoning.identity_reasoning_parser import IdentityReasoningParser
11
+
12
+REASONING_MODEL_NAME = "HCompany/Holo2-4B"
13
+
14
+
15
+@pytest.fixture(scope="module")
16
+def tokenizer():
17
+ return AutoTokenizer.from_pretrained(REASONING_MODEL_NAME)
18
+
19
+
20
+@pytest.mark.parametrize(
21
+ "thinking,expected_parser_type",
22
+ [
23
+ (True, DeepSeekR1ReasoningParser),
24
+ (False, IdentityReasoningParser),
25
+ ],
26
+)
27
+def test_parser_selection(tokenizer, thinking, expected_parser_type):
28
+ parser = Holo2ReasoningParser(
29
+ tokenizer,
30
+ chat_template_kwargs={
31
+ "thinking": thinking,
32
+ },
33
+ )
34
+
35
+ assert isinstance(parser._parser, expected_parser_type)
36
+
37
+
38
+def test_holo2_default_parser_is_deepseekr1(tokenizer):
39
+ parser = Holo2ReasoningParser(tokenizer)
40
+
41
+ assert isinstance(parser._parser, DeepSeekR1ReasoningParser)
42
+
43
+
44
+def test_holo2_supports_structured_output(tokenizer):
45
+ # Structured output manager uses the reasoning parser to check if the
46
+ # reasoning content is ended before applying the grammar. The main function
47
+ # used is is_reasoning_end. This test checks if the parser is able to
48
+ # correctly identify the end of the reasoning content.
49
+
50
+ # important to not pass chat_template_kwargs here as it is done in the
51
+ # StructuredOutputManager
52
+ parser = Holo2ReasoningParser(tokenizer)
53
+
54
+ end_token_id = tokenizer.encode("</think>", add_special_tokens=False)[0]
55
+
56
+ assert parser.is_reasoning_end([1, 2, 4, end_token_id])
57
+ assert not parser.is_reasoning_end([1, 2, 4])
58
+ assert parser.is_reasoning_end([1, 2, 4, end_token_id, 5])
59
+
60
+
61
+# thinking is True, non-streaming
62
+WITH_THINK = {
63
+ "output": "This is a reasoning section</think>This is the rest",
64
+ "reasoning": "This is a reasoning section",
65
+ "content": "This is the rest",
66
+}
67
+# thinking is True, streaming
68
+WITH_THINK_STREAM = {
69
+ "output": "This is a reasoning section</think>This is the rest",
70
+ "reasoning": "This is a reasoning section",
71
+ "content": "This is the rest",
72
+}
73
+# thinking is False, non-streaming
74
+THINKING_DISABLED = {
75
+ "output": "This is the rest",
76
+ "reasoning": None,
77
+ "content": "This is the rest",
78
+}
79
+# thinking is False, streaming
80
+THINKING_DISABLED_STREAM = {
81
+ "output": "This is the rest",
82
+ "reasoning": None,
83
+ "content": "This is the rest",
84
+}
85
+# thinking is False but the model output </think>, non-streaming
86
+THINKING_DISABLED_WITH_CLOSE_TAG = {
87
+ "output": "</think>This is the rest",
88
+ "reasoning": None,
89
+ "content": "</think>This is the rest",
90
+}
91
+# thinking is False but the model output </think>, streaming
92
+THINKING_DISABLED_WITH_CLOSE_TAG_STREAM = {
93
+ "output": "some text</think>This is the rest",
94
+ "reasoning": None,
95
+ "content": "some text</think>This is the rest",
96
+}
97
+COMPLETE_REASONING = {
98
+ "output": "This is a reasoning section</think>",
99
+ "reasoning": "This is a reasoning section",
100
+ "content": None,
101
+}
102
+
103
+TEST_CASES = [
104
+ pytest.param(
105
+ False,
106
+ WITH_THINK,
107
+ None,
108
+ id="with_think",
109
+ ),
110
+ pytest.param(
111
+ True,
112
+ WITH_THINK_STREAM,
113
+ None,
114
+ id="with_think_stream",
115
+ ),
116
+ pytest.param(
117
+ False,
118
+ WITH_THINK,
119
+ {"thinking": True},
120
+ id="with_think_enabled",
121
+ ),
122
+ pytest.param(
123
+ True,
124
+ WITH_THINK_STREAM,
125
+ {"thinking": True},
126
+ id="with_think_stream_enabled",
127
+ ),
128
+ pytest.param(
129
+ False,
130
+ THINKING_DISABLED,
131
+ {"thinking": False},
132
+ id="thinking_disabled",
133
+ ),
134
+ pytest.param(
135
+ True,
136
+ THINKING_DISABLED_STREAM,
137
+ {"thinking": False},
138
+ id="thinking_disabled_stream",
139
+ ),
140
+ pytest.param(
141
+ False,
142
+ THINKING_DISABLED_WITH_CLOSE_TAG,
143
+ {"thinking": False},
144
+ id="thinking_disabled_with_close_tag",
145
+ ),
146
+ pytest.param(
147
+ True,
148
+ THINKING_DISABLED_WITH_CLOSE_TAG_STREAM,
149
+ {"thinking": False},
150
+ id="thinking_disabled_with_close_tag_stream",
151
+ ),
152
+ pytest.param(
153
+ False,
154
+ COMPLETE_REASONING,
155
+ None,
156
+ id="complete_reasoning",
157
+ ),
158
+ pytest.param(
159
+ True,
160
+ COMPLETE_REASONING,
161
+ None,
162
+ id="complete_reasoning_stream",
163
+ ),
164
+]
165
+
166
+
167
+@pytest.mark.parametrize("streaming, param_dict, chat_template_kwargs", TEST_CASES)
168
+def test_reasoning(
169
+ streaming: bool,
170
+ param_dict: dict,
171
+ chat_template_kwargs: dict | None,
172
+ tokenizer,
173
+):
174
+ output = tokenizer.tokenize(param_dict["output"])
175
+ output_tokens: list[str] = [
176
+ tokenizer.convert_tokens_to_string([token]) for token in output
177
+ ]
178
+ parser: ReasoningParser = ReasoningParserManager.get_reasoning_parser("holo2")(
179
+ tokenizer,
180
+ chat_template_kwargs=chat_template_kwargs,
181
+ )
182
+
183
+ reasoning, content = run_reasoning_extraction(
184
+ parser, output_tokens, streaming=streaming
185
+ )
186
+
187
+ assert reasoning == param_dict["reasoning"]
188
+ assert content == param_dict["content"]
vllm/reasoning/__init__.py
+4
@@ -44,6 +44,10 @@ _REASONING_PARSERS_TO_REGISTER = {
44
"granite_reasoning_parser",
45
"GraniteReasoningParser",
46
),
47
+ "holo2": (
48
+ "holo2_reasoning_parser",
49
+ "Holo2ReasoningParser",
50
+ ),
51
"hunyuan_a13b": (
52
"hunyuan_a13b_reasoning_parser",
53
"HunyuanA13BReasoningParser",
vllm/reasoning/holo2_reasoning_parser.py
new
+83
@@ -0,0 +1,83 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+
4
+from collections.abc import Sequence
5
+
6
+from vllm.entrypoints.openai.protocol import ChatCompletionRequest, DeltaMessage
7
+from vllm.logger import init_logger
8
+from vllm.reasoning import (
9
+ ReasoningParser,
10
+)
11
+from vllm.reasoning.deepseek_r1_reasoning_parser import DeepSeekR1ReasoningParser
12
+from vllm.reasoning.identity_reasoning_parser import IdentityReasoningParser
13
+from vllm.tokenizers import TokenizerLike
14
+
15
+logger = init_logger(__name__)
16
+
17
+
18
+class Holo2ReasoningParser(ReasoningParser):
19
+ """
20
+ Reasoning parser for the Holo2 models which are based on Qwen3.
21
+
22
+ The Holo2 model uses <think>...</think> tokens to denote reasoning text but <think>
23
+ is part of the chat template. This parser extracts the reasoning content until
24
+ </think> in the model's output.
25
+
26
+ The model provides a switch to enable or disable reasoning
27
+ output via the 'thinking=False' parameter.
28
+
29
+ Chat template args:
30
+ - thinking: Whether to enable reasoning output (default: True)
31
+
32
+
33
+ Parsing rules on model output:
34
+ - thinking == False
35
+ -> Model output is treated as purely the content |content|
36
+ - thinking == True
37
+ -> Model output is |reasoning_content|</think>|content|
38
+ """
39
+
40
+ def __init__(self, tokenizer: TokenizerLike, *args, **kwargs):
41
+ super().__init__(tokenizer, *args, **kwargs)
42
+
43
+ chat_kwargs = kwargs.get("chat_template_kwargs", {}) or {}
44
+ # Deepseek V3 and Holo2 are similar. However, Holo2 models think by default.
45
+ # this parser without user specified chat template args is initiated once for
46
+ # all requests in the structured output manager. So it is important that without
47
+ # user specified chat template args, the default thinking is True.
48
+
49
+ enable_thinking = bool(chat_kwargs.get("thinking", True))
50
+
51
+ if enable_thinking:
52
+ self._parser = DeepSeekR1ReasoningParser(tokenizer, *args, **kwargs)
53
+ else:
54
+ self._parser = IdentityReasoningParser(tokenizer, *args, **kwargs)
55
+
56
+ def is_reasoning_end(self, input_ids: Sequence[int]) -> bool:
57
+ return self._parser.is_reasoning_end(input_ids)
58
+
59
+ def extract_content_ids(self, input_ids: list[int]) -> list[int]:
60
+ return self._parser.extract_content_ids(input_ids)
61
+
62
+ def extract_reasoning(
63
+ self, model_output: str, request: ChatCompletionRequest
64
+ ) -> tuple[str | None, str | None]:
65
+ return self._parser.extract_reasoning(model_output, request)
66
+
67
+ def extract_reasoning_streaming(
68
+ self,
69
+ previous_text: str,
70
+ current_text: str,
71
+ delta_text: str,
72
+ previous_token_ids: Sequence[int],
73
+ current_token_ids: Sequence[int],
74
+ delta_token_ids: Sequence[int],
75
+ ) -> DeltaMessage | None:
76
+ return self._parser.extract_reasoning_streaming(
77
+ previous_text,
78
+ current_text,
79
+ delta_text,
80
+ previous_token_ids,
81
+ current_token_ids,
82
+ delta_token_ids,
83
+ )