[Perf][Structured Output] Skip unused request-local reasoners (#52573)
Signed-off-by: Bugen Zhao <i@bugenzhao.com> Co-authored-by: OpenAI Codex <codex@openai.com> Co-authored-by: Nick Hill <nickhill123@gmail.com>
Bugen Zhao committed
Aug 18, 2026 at 09:30 UTC
5ae2d38821ba381ff2091b52b463296784b56217
2 files changed
+79
-26
tests/v1/structured_output/test_reasoning_structured_output.py
+59
-6
@@ -84,17 +84,62 @@ class TestReasoningStructuredOutput:
84
return manager
85
86
def test_should_fill_bitmask_with_enable_in_reasoning(
87
- self, mock_vllm_config, mock_request_with_structured_output
87
+ self, manager_with_reasoner, mock_request_with_structured_output
88
):
89
"""Test should_fill_bitmask when enable_in_reasoning is True."""
90
- # Enable enable_in_reasoning
91
- mock_vllm_config.structured_outputs_config.enable_in_reasoning = True
92
-
93
- manager = StructuredOutputManager(mock_vllm_config)
90
+ manager_with_reasoner.enable_in_reasoning = True
91
92
# Should always return True when enable_in_reasoning is enabled
96
- result = manager.should_fill_bitmask(mock_request_with_structured_output)
93
+ result = manager_with_reasoner.should_fill_bitmask(
94
+ mock_request_with_structured_output
95
+ )
96
+ assert result is True
97
+ assert (
98
+ mock_request_with_structured_output.structured_output_request.reasoner
99
+ is None
100
+ )
101
+
102
+ def test_should_fill_bitmask_reasoning_already_ended(
103
+ self,
104
+ manager_with_reasoner,
105
+ mock_request_with_structured_output,
106
+ ):
107
+ """An active grammar does not need a request-local reasoner."""
108
+ structured_req = mock_request_with_structured_output.structured_output_request
109
+ structured_req.reasoning_ended = True
110
+
111
+ result = manager_with_reasoner.should_fill_bitmask(
112
+ mock_request_with_structured_output
113
+ )
114
+
115
assert result is True
116
+ assert structured_req.reasoner is None
117
+
118
+ def test_grammar_bitmask_skips_reasoner_when_already_active(
119
+ self,
120
+ manager_with_reasoner,
121
+ mock_request_with_structured_output,
122
+ ):
123
+ """Bitmask generation skips reasoning-boundary detection."""
124
+ manager_with_reasoner.vllm_config.num_speculative_tokens = 1
125
+ manager_with_reasoner._grammar_bitmask = Mock()
126
+ manager_with_reasoner._grammar_bitmask.shape = (1, 1)
127
+ expected_bitmask = Mock()
128
+ manager_with_reasoner._grammar_bitmask.numpy.return_value = expected_bitmask
129
+ manager_with_reasoner._fill_bitmasks = Mock()
130
+
131
+ structured_req = mock_request_with_structured_output.structured_output_request
132
+ structured_req.reasoning_ended = True
133
+ request_id = mock_request_with_structured_output.request_id
134
+
135
+ bitmask = manager_with_reasoner.grammar_bitmask(
136
+ requests={request_id: mock_request_with_structured_output},
137
+ structured_output_request_ids=[request_id],
138
+ scheduled_spec_decode_tokens={},
139
+ )
140
+
141
+ assert bitmask is expected_bitmask
142
+ assert structured_req.reasoner is None
143
144
def test_should_fill_bitmask_without_enable_in_reasoning(
145
self,
@@ -171,6 +216,10 @@ class TestReasoningStructuredOutput:
216
mock_request_with_structured_output
217
)
218
assert result is True
219
+ assert (
220
+ mock_request_with_structured_output.structured_output_request.reasoner
221
+ is None
222
+ )
223
224
def test_should_advance_reasoning_not_ended(
225
self,
@@ -233,6 +282,10 @@ class TestReasoningStructuredOutput:
282
283
# Should return True since reasoning has ended
284
assert result is True
285
+ assert (
286
+ mock_request_with_structured_output.structured_output_request.reasoner
287
+ is None
288
+ )
289
290
def test_should_advance_uses_new_token_ids_when_provided(
291
self,
vllm/v1/structured_output/__init__.py
+20
-20
@@ -289,12 +289,8 @@ class StructuredOutputManager:
289
assert isinstance(grammar, StructuredOutputGrammar)
290
apply_bitmask = self.should_fill_bitmask(request)
291
292
- reasoner = self._get_reasoner(request)
293
- detect_reasoning_end = (
294
- not apply_bitmask
295
- and reasoner is not None
296
- and not self.enable_in_reasoning
297
- )
292
+ reasoner = None if apply_bitmask else self._get_reasoner(request)
293
+ detect_reasoning_end = reasoner is not None
294
simulated_buf: list[int] | None = None
295
history_len = 0
296
@@ -369,20 +365,24 @@ class StructuredOutputManager:
365
# NOTE (Hanchen) if enable_in_reasoning is True, it means that
366
# the model needs to be constrained in reasoning. So we should always
367
# enable the bitmask filling.
368
+ structured_req = request.structured_output_request
369
+ if self.enable_in_reasoning or (
370
+ structured_req is not None and structured_req.reasoning_ended is True
371
+ ):
372
+ return True
373
+
374
reasoner = self._get_reasoner(request)
375
if reasoner is not None:
374
- if self.enable_in_reasoning:
375
- return True
376
- assert request.structured_output_request is not None
377
- if request.structured_output_request.reasoning_ended is None:
376
+ assert structured_req is not None
377
+ if structured_req.reasoning_ended is None:
378
# This should be removed here, but since `openai_gptoss`
379
# is an independent code path, it is kept for now.
380
# After unifying the `openai_gptoss` and non-`openai_gptoss` styles,
381
# it can be removed.
382
- request.structured_output_request.reasoning_ended = (
383
- reasoner.is_reasoning_end(request.prompt_token_ids or [])
382
+ structured_req.reasoning_ended = reasoner.is_reasoning_end(
383
+ request.prompt_token_ids or []
384
)
385
- return request.structured_output_request.reasoning_ended
385
+ return structured_req.reasoning_ended
386
return True
387
388
def should_advance(
@@ -398,19 +398,19 @@ class StructuredOutputManager:
398
if TYPE_CHECKING:
399
assert request.structured_output_request is not None
400
assert request.structured_output_request.grammar is not None
401
+ structured_req = request.structured_output_request
402
+ if self.enable_in_reasoning or (
403
+ structured_req is not None and structured_req.reasoning_ended is True
404
+ ):
405
+ return True
406
+
407
# by default, we should always advance
408
# for cases that don't use thinking mode.
409
reasoner = self._get_reasoner(request)
410
if reasoner is None:
411
return True
412
407
- # if the model needs structured in reasoning, we should advance
408
- if self.enable_in_reasoning:
409
- return True
410
-
411
- structured_req = request.structured_output_request
412
- if structured_req.reasoning_ended:
413
- return True
413
+ assert structured_req is not None
414
415
# Check if reasoning ends in *this* step.
416
# When the caller passes new_token_ids (the tokens that were just