A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 11, 18:45 UTC
[Structured Output][Reasoning] Improves decoding throughput for models using single-token reasoning endings. (#30056)
Hubert de La Jonquiere committed
Dec 9, 2025 at 11:54 UTC
c72ea1072313b30fc8b4701697520fdd0b56fc35
10 files changed
+89
-1
docs/features/reasoning_outputs.md
+3
@@ -299,6 +299,9 @@ Additionally, to enable structured output, you'll need to create a new `Reasoner
299
300
def is_reasoning_end(self, input_ids: list[int]) -> bool:
301
return self.end_token_id in input_ids
302
+
303
+ def is_reasoning_end_streaming(self, input_ids: list[int], delta_ids: list[int]) -> bool:
304
+ return self.end_token_id in delta_token_ids
305
...
306
```
307
tests/reasoning/test_base_thinking_reasoning_parser.py
+35
@@ -132,6 +132,41 @@ class TestBaseThinkingReasoningParserMethods:
132
is False
133
)
134
135
+ def test_is_reasoning_end_streaming(self, test_tokenizer):
136
+ """Test the is_reasoning_end_streaming method."""
137
+ parser = TestThinkingReasoningParser(test_tokenizer)
138
+ end_token_id = parser.end_token_id
139
+ start_token_id = parser.start_token_id
140
+
141
+ assert (
142
+ parser.is_reasoning_end_streaming([1, 2, end_token_id], [end_token_id])
143
+ is True
144
+ )
145
+ assert parser.is_reasoning_end_streaming([1, 2, 3, 4], [4]) is False
146
+ assert parser.is_reasoning_end_streaming([], []) is False
147
+ assert (
148
+ parser.is_reasoning_end_streaming(
149
+ [1, start_token_id, 2, end_token_id], [end_token_id]
150
+ )
151
+ is True
152
+ )
153
+ assert (
154
+ parser.is_reasoning_end_streaming([1, start_token_id, 2, 3], [3]) is False
155
+ )
156
+ assert (
157
+ parser.is_reasoning_end_streaming(
158
+ [1, start_token_id, 2, end_token_id, 2, start_token_id, 2],
159
+ [2],
160
+ )
161
+ is False
162
+ )
163
+ assert (
164
+ parser.is_reasoning_end_streaming(
165
+ [1, start_token_id, 2, end_token_id, 2, 2], [2]
166
+ )
167
+ is False
168
+ )
169
+
170
def test_extract_content_ids(self, test_tokenizer):
171
"""Test the extract_content_ids method."""
172
parser = TestThinkingReasoningParser(test_tokenizer)
tests/reasoning/test_deepseekv3_reasoning_parser.py
+1
@@ -40,6 +40,7 @@ def test_identity_reasoning_parser_basic(tokenizer):
40
input_tokens = tokenizer.tokenize(input_text)
41
input_ids = tokenizer.convert_tokens_to_ids(input_tokens)
42
assert parser.is_reasoning_end(input_ids) is True
43
+ assert parser.is_reasoning_end_streaming(input_ids, input_ids) is True
44
45
# Test extract_content_ids returns all input_ids
46
assert parser.extract_content_ids(input_ids) == input_ids
tests/v1/structured_output/test_reasoning_structured_output.py
+1
@@ -70,6 +70,7 @@ class TestReasoningStructuredOutput:
70
request.use_structured_output = True
71
request.prompt_token_ids = [1, 2, 3, 4, 5]
72
request.all_token_ids = [1, 2, 3, 4, 5, 6, 7, 8]
73
+ request.num_computed_tokens = 5
74
return request
75
76
def test_should_fill_bitmask_with_enable_in_reasoning(
vllm/reasoning/abs_reasoning_parsers.py
+25
@@ -63,6 +63,31 @@ class ReasoningParser:
63
True if the reasoning content ends in the input_ids.
64
"""
65
66
+ def is_reasoning_end_streaming(
67
+ self, input_ids: list[int], delta_ids: list[int]
68
+ ) -> bool:
69
+ """
70
+ Check if the reasoning content ends in the input_ids on a
71
+ decode step.
72
+
73
+ It is used in structured engines like `xgrammar` to check if the
74
+ reasoning content ends in the model output during a decode step.
75
+ `input_ids` the entire model output and `delta_ids` are the last few
76
+ computed tokens of the model output (like during a decode step).
77
+
78
+ Parameters:
79
+ input_ids: list[int]
80
+ The entire model output.
81
+ delta_ids: list[int]
82
+ The last few computed tokens of the model output at the current decode step.
83
+
84
+ Returns:
85
+ bool
86
+ True if the reasoning content ends in the `delta_ids` on a
87
+ decode step.
88
+ """
89
+ return self.is_reasoning_end(input_ids)
90
+
91
@abstractmethod
92
def extract_content_ids(self, input_ids: list[int]) -> list[int]:
93
"""
vllm/reasoning/basic_parsers.py
+6
@@ -74,6 +74,12 @@ class BaseThinkingReasoningParser(ReasoningParser):
74
return True
75
return False
76
77
+ def is_reasoning_end_streaming(
78
+ self, input_ids: list[int], delta_ids: list[int]
79
+ ) -> bool:
80
+ end_token_id = self.end_token_id
81
+ return end_token_id in delta_ids
82
+
83
def extract_content_ids(self, input_ids: list[int]) -> list[int]:
84
"""
85
Extract the content after the end tokens
vllm/reasoning/deepseek_v3_reasoning_parser.py
+5
@@ -35,6 +35,11 @@ class DeepSeekV3ReasoningParser(ReasoningParser):
35
def is_reasoning_end(self, input_ids: Sequence[int]) -> bool:
36
return self._parser.is_reasoning_end(input_ids)
37
38
+ def is_reasoning_end_streaming(
39
+ self, input_ids: list[int], delta_ids: list[int]
40
+ ) -> bool:
41
+ return self._parser.is_reasoning_end_streaming(input_ids, delta_ids)
42
+
43
def extract_content_ids(self, input_ids: list[int]) -> list[int]:
44
return self._parser.extract_content_ids(input_ids)
45
vllm/reasoning/holo2_reasoning_parser.py
+5
@@ -56,6 +56,11 @@ class Holo2ReasoningParser(ReasoningParser):
56
def is_reasoning_end(self, input_ids: Sequence[int]) -> bool:
57
return self._parser.is_reasoning_end(input_ids)
58
59
+ def is_reasoning_end_streaming(
60
+ self, input_ids: list[int], delta_ids: list[int]
61
+ ) -> bool:
62
+ return self._parser.is_reasoning_end_streaming(input_ids, delta_ids)
63
+
64
def extract_content_ids(self, input_ids: list[int]) -> list[int]:
65
return self._parser.extract_content_ids(input_ids)
66
vllm/reasoning/identity_reasoning_parser.py
+5
@@ -32,6 +32,11 @@ class IdentityReasoningParser(ReasoningParser):
32
# Always return True, since we never treat reasoning specially
33
return True
34
35
+ def is_reasoning_end_streaming(
36
+ self, input_ids: list[int], delta_ids: list[int]
37
+ ) -> bool:
38
+ return True
39
+
40
def extract_content_ids(self, input_ids: list[int]) -> list[int]:
41
# Identity: return all tokens as content
42
return input_ids
vllm/v1/structured_output/__init__.py
+3
-1
@@ -339,7 +339,9 @@ class StructuredOutputManager:
339
return True
340
341
# Check if reasoning ends in *this* step
342
- if self.reasoner.is_reasoning_end(request.all_token_ids):
342
+ if self.reasoner.is_reasoning_end_streaming(
343
+ request.all_token_ids, request.all_token_ids[request.num_computed_tokens :]
344
+ ):
345
# Reasoning just ended, so we shouldn't advance til
346
# next pass
347
structured_req.reasoning_ended = True