A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 4, 10:00 UTC
[Bugfix] Pass reasoning parser kwargs to structured output (#41199)
Signed-off-by: Bugen Zhao <i@bugenzhao.com> Co-authored-by: wang.yuqi <yuqi.wang@daocloud.io>
Bugen Zhao committed
May 1, 2026 at 14:38 UTC
a07642667db1284ad2128d7f9ef089e6b0d24a4c
9 files changed
+132
-56
tests/v1/structured_output/test_reasoning_structured_output.py
+70
-46
@@ -8,11 +8,16 @@ from unittest.mock import Mock
8
import pytest
9
10
from vllm.config import ModelConfig, SchedulerConfig, VllmConfig
11
-from vllm.reasoning import ReasoningParser
11
from vllm.v1.request import Request
12
from vllm.v1.structured_output import StructuredOutputManager
13
14
15
+class MockReasoner:
16
+ def __init__(self, tokenizer):
17
+ self.is_reasoning_end = Mock(return_value=False)
18
+ self.is_reasoning_end_streaming = Mock(return_value=False)
19
+
20
+
21
class TestReasoningStructuredOutput:
22
"""Test reasoning-aware structured output functionality."""
23
@@ -50,13 +55,6 @@ class TestReasoningStructuredOutput:
55
config.speculative_config = None
56
return config
57
53
- @pytest.fixture
54
- def mock_reasoning_parser(self):
55
- """Create a mock ReasoningParser."""
56
- parser = Mock(spec=ReasoningParser)
57
- parser.is_reasoning_end = Mock(return_value=False)
58
- return parser
59
-
58
@pytest.fixture
59
def mock_request_with_structured_output(self):
60
"""Create a mock request with structured output."""
@@ -64,6 +62,8 @@ class TestReasoningStructuredOutput:
62
request.structured_output_request = Mock()
63
request.structured_output_request.reasoning_ended = None
64
request.structured_output_request.grammar = Mock()
65
+ request.structured_output_request.reasoning_parser_kwargs = None
66
+ request.structured_output_request.reasoner = None
67
request.structured_output_request.grammar.is_terminated = Mock(
68
return_value=False
69
)
@@ -74,6 +74,13 @@ class TestReasoningStructuredOutput:
74
request.num_output_placeholders = 0
75
return request
76
77
+ @pytest.fixture
78
+ def manager_with_reasoner(self, mock_vllm_config):
79
+ manager = StructuredOutputManager(mock_vllm_config)
80
+ manager.reasoner_cls = MockReasoner
81
+ manager.tokenizer = Mock()
82
+ return manager
83
+
84
def test_should_fill_bitmask_with_enable_in_reasoning(
85
self, mock_vllm_config, mock_request_with_structured_output
86
):
@@ -89,22 +96,17 @@ class TestReasoningStructuredOutput:
96
97
def test_should_fill_bitmask_without_enable_in_reasoning(
98
self,
92
- mock_vllm_config,
99
+ manager_with_reasoner,
100
mock_request_with_structured_output,
94
- mock_reasoning_parser,
101
):
102
"""Test should_fill_bitmask when enable_in_reasoning is False."""
103
# Keep enable_in_reasoning as False (default)
98
- config = mock_vllm_config.structured_outputs_config
104
+ config = manager_with_reasoner.vllm_config.structured_outputs_config
105
assert config.enable_in_reasoning is False
106
101
- manager = StructuredOutputManager(mock_vllm_config)
102
- manager.reasoner = mock_reasoning_parser
103
-
104
- # Mock reasoning not ended
105
- mock_reasoning_parser.is_reasoning_end.return_value = False
106
-
107
- result = manager.should_fill_bitmask(mock_request_with_structured_output)
107
+ result = manager_with_reasoner.should_fill_bitmask(
108
+ mock_request_with_structured_output
109
+ )
110
111
# Should set reasoning_ended and return its value
112
assert (
@@ -118,68 +120,92 @@ class TestReasoningStructuredOutput:
120
):
121
"""Test should_fill_bitmask when no reasoner is configured."""
122
manager = StructuredOutputManager(mock_vllm_config)
121
- manager.reasoner = None
123
124
result = manager.should_fill_bitmask(mock_request_with_structured_output)
125
126
# Should default to True when no reasoner
127
assert result is True
128
129
+ def test_should_fill_bitmask_uses_request_reasoning_parser_kwargs(
130
+ self, mock_vllm_config, mock_request_with_structured_output
131
+ ):
132
+ """Test request-level parser kwargs override the default reasoner."""
133
+
134
+ class KwargReasoner:
135
+ def __init__(self, tokenizer, chat_template_kwargs=None):
136
+ self.chat_template_kwargs = chat_template_kwargs or {}
137
+
138
+ def is_reasoning_end(self, input_ids):
139
+ return not self.chat_template_kwargs.get("enable_thinking", False)
140
+
141
+ manager = StructuredOutputManager(mock_vllm_config)
142
+ manager.reasoner_cls = KwargReasoner
143
+ manager.tokenizer = Mock()
144
+
145
+ structured_req = mock_request_with_structured_output.structured_output_request
146
+ structured_req.reasoning_parser_kwargs = {
147
+ "chat_template_kwargs": {"enable_thinking": True}
148
+ }
149
+
150
+ result = manager.should_fill_bitmask(mock_request_with_structured_output)
151
+
152
+ assert result is False
153
+ assert (
154
+ mock_request_with_structured_output.structured_output_request.reasoner
155
+ is not None
156
+ )
157
+
158
def test_should_advance_with_enable_in_reasoning(
159
self,
130
- mock_vllm_config,
160
+ manager_with_reasoner,
161
mock_request_with_structured_output,
132
- mock_reasoning_parser,
162
):
163
"""Test should_advance when enable_in_reasoning is True."""
164
# Enable enable_in_reasoning
136
- mock_vllm_config.structured_outputs_config.enable_in_reasoning = True
137
-
138
- manager = StructuredOutputManager(mock_vllm_config)
139
- manager.reasoner = mock_reasoning_parser
165
+ manager_with_reasoner.enable_in_reasoning = True
166
167
# Should always return True when enable_in_reasoning is enabled
142
- result = manager.should_advance(mock_request_with_structured_output)
168
+ result = manager_with_reasoner.should_advance(
169
+ mock_request_with_structured_output
170
+ )
171
assert result is True
172
173
def test_should_advance_reasoning_not_ended(
174
self,
147
- mock_vllm_config,
175
+ manager_with_reasoner,
176
mock_request_with_structured_output,
149
- mock_reasoning_parser,
177
):
178
"""Test should_advance when reasoning has not ended."""
152
- manager = StructuredOutputManager(mock_vllm_config)
153
- manager.reasoner = mock_reasoning_parser
154
-
179
# Set reasoning as not ended
180
(
181
mock_request_with_structured_output.structured_output_request
182
).reasoning_ended = False
159
- mock_reasoning_parser.is_reasoning_end.return_value = False
183
161
- result = manager.should_advance(mock_request_with_structured_output)
184
+ result = manager_with_reasoner.should_advance(
185
+ mock_request_with_structured_output
186
+ )
187
188
# Should return False since reasoning hasn't ended
189
assert result is False
190
191
def test_should_advance_reasoning_just_ended(
192
self,
168
- mock_vllm_config,
193
+ manager_with_reasoner,
194
mock_request_with_structured_output,
170
- mock_reasoning_parser,
195
):
196
"""Test should_advance when reasoning ends in current step."""
173
- manager = StructuredOutputManager(mock_vllm_config)
174
- manager.reasoner = mock_reasoning_parser
175
-
197
# Set reasoning as not ended initially, but ends in this step
198
(
199
mock_request_with_structured_output.structured_output_request
200
).reasoning_ended = False
180
- mock_reasoning_parser.is_reasoning_end.return_value = True
201
+ reasoner = MockReasoner(tokenizer=Mock())
202
+ reasoner.is_reasoning_end_streaming.return_value = True
203
+ structured_req = mock_request_with_structured_output.structured_output_request
204
+ structured_req.reasoner = reasoner
205
182
- result = manager.should_advance(mock_request_with_structured_output)
206
+ result = manager_with_reasoner.should_advance(
207
+ mock_request_with_structured_output
208
+ )
209
210
# Should set reasoning_ended to True but return False for this step
211
assert (
@@ -190,20 +216,18 @@ class TestReasoningStructuredOutput:
216
217
def test_should_advance_reasoning_already_ended(
218
self,
193
- mock_vllm_config,
219
+ manager_with_reasoner,
220
mock_request_with_structured_output,
195
- mock_reasoning_parser,
221
):
222
"""Test should_advance when reasoning has already ended."""
198
- manager = StructuredOutputManager(mock_vllm_config)
199
- manager.reasoner = mock_reasoning_parser
200
-
223
# Set reasoning as already ended
224
(
225
mock_request_with_structured_output.structured_output_request
226
).reasoning_ended = True
227
206
- result = manager.should_advance(mock_request_with_structured_output)
228
+ result = manager_with_reasoner.should_advance(
229
+ mock_request_with_structured_output
230
+ )
231
232
# Should return True since reasoning has ended
233
assert result is True
vllm/engine/protocol.py
+1
@@ -78,6 +78,7 @@ class EngineClient(ABC):
78
priority: int = 0,
79
data_parallel_rank: int | None = None,
80
reasoning_ended: bool | None = None,
81
+ reasoning_parser_kwargs: dict[str, Any] | None = None,
82
) -> AsyncGenerator[RequestOutput, None]:
83
"""Generate outputs for a request."""
84
...
vllm/entrypoints/openai/chat_completion/serving.py
+5
@@ -347,6 +347,11 @@ class OpenAIServingChat(OpenAIServing):
347
priority=request.priority,
348
data_parallel_rank=data_parallel_rank,
349
reasoning_ended=reasoning_ended,
350
+ reasoning_parser_kwargs={
351
+ "chat_template_kwargs": chat_template_kwargs,
352
+ }
353
+ if reasoning_parser
354
+ else None,
355
)
356
357
generators.append(generator)
vllm/entrypoints/openai/responses/serving.py
+10
-1
@@ -456,9 +456,13 @@ class OpenAIServingResponses(OpenAIServing):
456
context = SimpleContext()
457
458
if self.parser and self.parser.reasoning_parser_cls is not None:
459
+ chat_template_kwargs = self._effective_chat_template_kwargs(request)
460
+ reasoning_parser_kwargs = {
461
+ "chat_template_kwargs": chat_template_kwargs,
462
+ }
463
reasoning_parser = self.parser.reasoning_parser_cls(
464
tokenizer,
461
- chat_template_kwargs=self._effective_chat_template_kwargs(request),
465
+ chat_template_kwargs=chat_template_kwargs,
466
)
467
if (
468
isinstance(
@@ -481,6 +485,9 @@ class OpenAIServingResponses(OpenAIServing):
485
lora_request=lora_request,
486
priority=request.priority,
487
trace_headers=trace_headers,
488
+ reasoning_parser_kwargs=reasoning_parser_kwargs
489
+ if self.parser and self.parser.reasoning_parser_cls is not None
490
+ else None,
491
)
492
generators.append(generator)
493
@@ -627,6 +634,7 @@ class OpenAIServingResponses(OpenAIServing):
634
lora_request: LoRARequest | None = None,
635
priority: int = 0,
636
trace_headers: Mapping[str, str] | None = None,
637
+ reasoning_parser_kwargs: dict[str, Any] | None = None,
638
):
639
max_model_len = self.model_config.max_model_len
640
@@ -650,6 +658,7 @@ class OpenAIServingResponses(OpenAIServing):
658
lora_request=lora_request,
659
trace_headers=trace_headers,
660
priority=priority,
661
+ reasoning_parser_kwargs=reasoning_parser_kwargs,
662
)
663
664
async for res in generator:
vllm/v1/engine/__init__.py
+1
@@ -120,6 +120,7 @@ class EngineCoreRequest(
120
external_req_id: str | None = None
121
122
reasoning_ended: bool | None = None
123
+ reasoning_parser_kwargs: dict[str, Any] | None = None
124
125
@property
126
def params(self) -> SamplingParams | PoolingParams:
vllm/v1/engine/async_llm.py
+6
-1
@@ -293,6 +293,7 @@ class AsyncLLM(EngineClient):
293
data_parallel_rank: int | None = None,
294
prompt_text: str | None = None,
295
reasoning_ended: bool | None = None,
296
+ reasoning_parser_kwargs: dict[str, Any] | None = None,
297
) -> RequestOutputCollector:
298
"""Add new request to the AsyncLLM."""
299
@@ -313,7 +314,7 @@ class AsyncLLM(EngineClient):
314
)
315
316
if isinstance(prompt, AsyncGenerator):
316
- if reasoning_ended is not None:
317
+ if reasoning_ended is not None or reasoning_parser_kwargs is not None:
318
raise NotImplementedError
319
320
# Streaming input case.
@@ -361,6 +362,8 @@ class AsyncLLM(EngineClient):
362
363
if reasoning_ended is not None:
364
request.reasoning_ended = reasoning_ended
365
+ if reasoning_parser_kwargs is not None:
366
+ request.reasoning_parser_kwargs = reasoning_parser_kwargs
367
368
self.input_processor.assign_request_id(request)
369
@@ -534,6 +537,7 @@ class AsyncLLM(EngineClient):
537
priority: int = 0,
538
data_parallel_rank: int | None = None,
539
reasoning_ended: bool | None = None,
540
+ reasoning_parser_kwargs: dict[str, Any] | None = None,
541
) -> AsyncGenerator[RequestOutput, None]:
542
"""
543
Main function called by the API server to kick off a request
@@ -563,6 +567,7 @@ class AsyncLLM(EngineClient):
567
data_parallel_rank=data_parallel_rank,
568
prompt_text=prompt_text,
569
reasoning_ended=reasoning_ended,
570
+ reasoning_parser_kwargs=reasoning_parser_kwargs,
571
)
572
573
# The output_handler task pushes items into the queue.
vllm/v1/request.py
+5
@@ -75,6 +75,7 @@ class Request:
75
block_hasher: Callable[["Request"], list["BlockHash"]] | None = None,
76
resumable: bool = False,
77
reasoning_ended: bool | None = None,
78
+ reasoning_parser_kwargs: dict[str, Any] | None = None,
79
) -> None:
80
self.request_id = request_id
81
self.client_index = client_index
@@ -87,6 +88,9 @@ class Request:
88
)
89
if self.structured_output_request is not None:
90
self.structured_output_request.reasoning_ended = reasoning_ended
91
+ self.structured_output_request.reasoning_parser_kwargs = (
92
+ reasoning_parser_kwargs
93
+ )
94
self.arrival_time = arrival_time if arrival_time is not None else time.time()
95
96
self.status = RequestStatus.WAITING
@@ -201,6 +205,7 @@ class Request:
205
block_hasher=block_hasher,
206
resumable=request.resumable,
207
reasoning_ended=request.reasoning_ended,
208
+ reasoning_parser_kwargs=request.reasoning_parser_kwargs,
209
)
210
211
def append_output_token_ids(
vllm/v1/structured_output/__init__.py
+26
-7
@@ -37,7 +37,10 @@ class StructuredOutputManager:
37
38
def __init__(self, vllm_config: VllmConfig):
39
self.backend: StructuredOutputBackend | None = None
40
- self.reasoner: ReasoningParser | None = None
40
+ # We only store the class of the reasoner in the manager.
41
+ # The parser instance is request-scoped because some reasoning parsers
42
+ # depend on per-request chat-template kwargs.
43
+ self.reasoner_cls: type[ReasoningParser] | None = None
44
self.vllm_config = vllm_config
45
46
# When in external_launcher mode, async grammar compilation causes deadlocks
@@ -85,15 +88,29 @@ class StructuredOutputManager:
88
self.vllm_config.structured_outputs_config.reasoning_parser
89
)
90
if reasoning_parser:
88
- reasoner_cls = ReasoningParserManager.get_reasoning_parser(
91
+ self.reasoner_cls = ReasoningParserManager.get_reasoning_parser(
92
reasoning_parser
93
)
91
- self.reasoner = reasoner_cls(tokenizer=self.tokenizer)
94
95
self.enable_in_reasoning = (
96
self.vllm_config.structured_outputs_config.enable_in_reasoning
97
)
98
99
+ def _get_reasoner(self, request: "Request") -> "ReasoningParser | None":
100
+ structured_req = request.structured_output_request
101
+ if structured_req is None or self.reasoner_cls is None:
102
+ return None
103
+
104
+ if structured_req.reasoner is None:
105
+ # Lazily build the request-local parser so the structured-output
106
+ # gate observes the same template kwargs used by the frontend.
107
+ parser_kwargs = structured_req.reasoning_parser_kwargs or {}
108
+ structured_req.reasoner = self.reasoner_cls(
109
+ tokenizer=self.tokenizer,
110
+ **parser_kwargs,
111
+ )
112
+ return structured_req.reasoner
113
+
114
def grammar_init(self, request: "Request") -> None:
115
if request.structured_output_request is None:
116
return
@@ -285,7 +302,8 @@ class StructuredOutputManager:
302
# NOTE (Hanchen) if enable_in_reasoning is True, it means that
303
# the model needs to be constrained in reasoning. So we should always
304
# enable the bitmask filling.
288
- if self.reasoner is not None:
305
+ reasoner = self._get_reasoner(request)
306
+ if reasoner is not None:
307
if self.enable_in_reasoning:
308
return True
309
assert request.structured_output_request is not None
@@ -295,7 +313,7 @@ class StructuredOutputManager:
313
# After unifying the `openai_gptoss` and non-`openai_gptoss` styles,
314
# it can be removed.
315
request.structured_output_request.reasoning_ended = (
298
- self.reasoner.is_reasoning_end(request.prompt_token_ids or [])
316
+ reasoner.is_reasoning_end(request.prompt_token_ids or [])
317
)
318
return request.structured_output_request.reasoning_ended
319
return True
@@ -311,7 +329,8 @@ class StructuredOutputManager:
329
assert request.structured_output_request.grammar is not None
330
# by default, we should always advance
331
# for cases that don't use thinking mode.
314
- if self.reasoner is None:
332
+ reasoner = self._get_reasoner(request)
333
+ if reasoner is None:
334
return True
335
336
# if the model needs structured in reasoning, we should advance
@@ -328,7 +347,7 @@ class StructuredOutputManager:
347
start = (
348
delta_from if delta_from >= 0 else max(len(all_token_ids) + delta_from, 0)
349
)
331
- if self.reasoner.is_reasoning_end_streaming(
350
+ if reasoner.is_reasoning_end_streaming(
351
all_token_ids, itertools.islice(all_token_ids, start, None)
352
):
353
# Reasoning just ended, so we shouldn't advance til
vllm/v1/structured_output/request.py
+8
-1
@@ -5,7 +5,7 @@ import functools
5
import json
6
from concurrent.futures import Future
7
from concurrent.futures._base import TimeoutError
8
-from typing import cast
8
+from typing import TYPE_CHECKING, Any, cast
9
10
from vllm.sampling_params import SamplingParams, StructuredOutputsParams
11
from vllm.v1.structured_output.backend_types import (
@@ -14,12 +14,19 @@ from vllm.v1.structured_output.backend_types import (
14
StructuredOutputOptions,
15
)
16
17
+if TYPE_CHECKING:
18
+ from vllm.reasoning import ReasoningParser
19
+
20
21
@dataclasses.dataclass
22
class StructuredOutputRequest:
23
params: StructuredOutputsParams
24
_grammar: Future[StructuredOutputGrammar] | StructuredOutputGrammar | None = None
25
reasoning_ended: bool | None = None
26
+ reasoning_parser_kwargs: dict[str, Any] | None = None
27
+ # Cached per request; do not share reasoning parsers across requests because
28
+ # their behavior can depend on reasoning_parser_kwargs.
29
+ reasoner: "ReasoningParser | None" = None
30
31
@staticmethod
32
def from_sampling_params(