[Bugfix][Structured Output][Spec Decode] Constrain bitmask and trim grammar advance at the reasoning boundary (#44297)
Signed-off-by: Allen.Yu <yuyue0225sc@163.com> Signed-off-by: yue.yu <yuyue0225sc@163.com> Co-authored-by: Benjamin Chislett <chislett.ben@gmail.com>
yue.yu committed
Jul 4, 2026 at 17:08 UTC
e7c9df944969c251bdceed7b9d7bbcc7fec3639b
6 files changed
+480
-16
tests/v1/core/test_async_scheduler.py
+3
@@ -272,6 +272,9 @@ def test_abort_request_when_structured_output_fsm_cannot_advance():
272
scheduler.connector = None
273
scheduler.structured_output_manager = Mock()
274
scheduler.structured_output_manager.should_advance.return_value = True
275
+ scheduler.structured_output_manager.trim_reasoning_for_advance.side_effect = (
276
+ lambda request, new_token_ids: new_token_ids
277
+ )
278
scheduler.requests = {request.request_id: request}
279
scheduler.running = [request]
280
scheduler.waiting = Mock()
tests/v1/core/test_scheduler.py
+3
@@ -2911,6 +2911,9 @@ def test_abort_request_when_structured_output_fsm_cannot_advance():
2911
scheduler.connector = None
2912
scheduler.structured_output_manager = Mock()
2913
scheduler.structured_output_manager.should_advance.return_value = True
2914
+ scheduler.structured_output_manager.trim_reasoning_for_advance.side_effect = (
2915
+ lambda request, new_token_ids: new_token_ids
2916
+ )
2917
scheduler.requests = {request.request_id: request}
2918
scheduler.running = [request]
2919
scheduler.waiting = Mock()
tests/v1/spec_decode/test_mtp_structured_output.py
new
+343
@@ -0,0 +1,343 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+"""grammar_bitmask under spec-decode draft padding (#44006)."""
4
+
5
+import pytest
6
+from transformers import AutoTokenizer
7
+
8
+from vllm.config import StructuredOutputsConfig, VllmConfig
9
+from vllm.config.model import ModelConfig
10
+from vllm.config.speculative import SpeculativeConfig
11
+from vllm.sampling_params import SamplingParams, StructuredOutputsParams
12
+from vllm.v1.request import Request
13
+from vllm.v1.structured_output import StructuredOutputManager
14
+
15
+TOKENIZER = "gpt2"
16
+NUM_SPEC_TOKENS = 4
17
+
18
+
19
+def _make_manager_and_request(backend: str, prompt_str: str = '{"a": "b"}'):
20
+ tokenizer = AutoTokenizer.from_pretrained(TOKENIZER)
21
+ prompt = tokenizer.encode(prompt_str)
22
+
23
+ vllm_config = VllmConfig(
24
+ model_config=ModelConfig(tokenizer=TOKENIZER),
25
+ structured_outputs_config=StructuredOutputsConfig(backend=backend),
26
+ speculative_config=SpeculativeConfig(
27
+ model="[ngram]", num_speculative_tokens=NUM_SPEC_TOKENS
28
+ ),
29
+ )
30
+ manager = StructuredOutputManager(vllm_config)
31
+
32
+ sampling_params = SamplingParams(
33
+ structured_outputs=StructuredOutputsParams(json='{"type": "object"}'),
34
+ )
35
+ sampling_params.structured_outputs._backend = backend
36
+ sampling_params.update_from_generation_config({}, tokenizer.eos_token_id)
37
+
38
+ request = Request(
39
+ "mtp_req",
40
+ prompt_token_ids=prompt,
41
+ sampling_params=sampling_params,
42
+ pooling_params=None,
43
+ )
44
+ manager.grammar_init(request)
45
+ while not request.structured_output_request._check_grammar_completion():
46
+ continue
47
+
48
+ return tokenizer, manager, request, prompt
49
+
50
+
51
+@pytest.mark.parametrize("backend", ["xgrammar", "guidance"])
52
+def test_bitmask_with_padded_invalid_drafts(backend):
53
+ """Bitmask handles -1 padded drafts and returns N+1 rows."""
54
+ tokenizer, manager, request, prompt = _make_manager_and_request(
55
+ backend, prompt_str='{"a"'
56
+ )
57
+ grammar = request.structured_output_request.grammar
58
+
59
+ assert grammar.accept_tokens(request.request_id, prompt)
60
+
61
+ valid_drafts = [tokenizer.encode(":")[0], tokenizer.encode(' "')[0]]
62
+ padded = valid_drafts + [-1, -1]
63
+
64
+ bitmask = manager.grammar_bitmask(
65
+ requests={request.request_id: request},
66
+ structured_output_request_ids=[request.request_id],
67
+ scheduled_spec_decode_tokens={request.request_id: padded},
68
+ )
69
+
70
+ assert bitmask is not None
71
+ assert bitmask.shape[0] == len(padded) + 1
72
+ assert not grammar.is_terminated()
73
+
74
+
75
+@pytest.mark.parametrize("backend", ["xgrammar", "guidance"])
76
+def test_bitmask_when_grammar_terminates_mid_window(backend):
77
+ """Drafts following an EOS that terminates the grammar are a no-op."""
78
+ tokenizer, manager, request, prompt = _make_manager_and_request(backend)
79
+ grammar = request.structured_output_request.grammar
80
+
81
+ assert grammar.accept_tokens(request.request_id, prompt)
82
+ eos = tokenizer.eos_token_id
83
+ drafts = [eos] + [tokenizer.encode(" ")[0]] * (NUM_SPEC_TOKENS - 1)
84
+
85
+ bitmask = manager.grammar_bitmask(
86
+ requests={request.request_id: request},
87
+ structured_output_request_ids=[request.request_id],
88
+ scheduled_spec_decode_tokens={request.request_id: drafts},
89
+ )
90
+
91
+ assert bitmask is not None
92
+ assert bitmask.shape[0] == NUM_SPEC_TOKENS + 1
93
+ assert not grammar.is_terminated()
94
+
95
+
96
+@pytest.mark.parametrize("backend", ["xgrammar", "guidance"])
97
+def test_bitmask_idempotent_across_calls(backend):
98
+ """Repeated calls with the same input return the same bitmask."""
99
+ tokenizer, manager, request, prompt = _make_manager_and_request(
100
+ backend, prompt_str='{"a"'
101
+ )
102
+ grammar = request.structured_output_request.grammar
103
+
104
+ assert grammar.accept_tokens(request.request_id, prompt)
105
+
106
+ drafts = [tokenizer.encode(":")[0], -1, -1, -1]
107
+
108
+ first = manager.grammar_bitmask(
109
+ requests={request.request_id: request},
110
+ structured_output_request_ids=[request.request_id],
111
+ scheduled_spec_decode_tokens={request.request_id: drafts},
112
+ )
113
+ second = manager.grammar_bitmask(
114
+ requests={request.request_id: request},
115
+ structured_output_request_ids=[request.request_id],
116
+ scheduled_spec_decode_tokens={request.request_id: drafts},
117
+ )
118
+
119
+ assert first is not None and second is not None
120
+ assert (first == second).all()
121
+ assert not grammar.is_terminated()
122
+
123
+
124
+@pytest.mark.parametrize("backend", ["xgrammar", "guidance"])
125
+def test_bonus_position_constrained_after_invalid_drafts(backend):
126
+ """Regression for #44006: bonus row stays constrained after -1 padding."""
127
+ tokenizer, manager, request, prompt = _make_manager_and_request(
128
+ backend, prompt_str='{"a"'
129
+ )
130
+ grammar = request.structured_output_request.grammar
131
+
132
+ assert grammar.accept_tokens(request.request_id, prompt)
133
+
134
+ valid = tokenizer.encode(":")[0]
135
+ drafts = [valid, -1, -1, -1]
136
+ bitmask = manager.grammar_bitmask(
137
+ requests={request.request_id: request},
138
+ structured_output_request_ids=[request.request_id],
139
+ scheduled_spec_decode_tokens={request.request_id: drafts},
140
+ )
141
+ assert bitmask is not None
142
+ assert bitmask.shape[0] == len(drafts) + 1
143
+
144
+ assert not (bitmask[-1] == -1).all()
145
+ assert not grammar.is_terminated()
146
+
147
+
148
+@pytest.mark.parametrize("backend", ["xgrammar", "guidance"])
149
+def test_bitmask_constrained_when_reasoning_ends_midwindow(backend):
150
+ """Drafts after a mid-window reasoning-end marker stay constrained."""
151
+ tokenizer, manager, request, prompt = _make_manager_and_request(backend)
152
+ grammar = request.structured_output_request.grammar
153
+
154
+ assert grammar.accept_tokens(request.request_id, prompt)
155
+
156
+ marker = tokenizer.encode("\n")[0]
157
+
158
+ class StubReasoner:
159
+ def __init__(self, *_, **__):
160
+ self.end_token_id = marker
161
+
162
+ def is_reasoning_end(self, input_ids):
163
+ return marker in list(input_ids)
164
+
165
+ def is_reasoning_end_streaming(self, input_ids, delta_ids):
166
+ return marker in list(delta_ids)
167
+
168
+ manager.reasoner_cls = StubReasoner
169
+ request.structured_output_request.reasoner = StubReasoner()
170
+ request.structured_output_request.reasoning_ended = False
171
+
172
+ pre = tokenizer.encode(" ")[0]
173
+ post = tokenizer.encode(",")[0]
174
+ drafts = [pre, marker, post]
175
+
176
+ bitmask = manager.grammar_bitmask(
177
+ requests={request.request_id: request},
178
+ structured_output_request_ids=[request.request_id],
179
+ scheduled_spec_decode_tokens={request.request_id: drafts},
180
+ )
181
+
182
+ assert bitmask is not None
183
+ assert bitmask.shape[0] == len(drafts) + 1
184
+ assert (bitmask[0] == -1).all()
185
+ assert (bitmask[1] == -1).all()
186
+ assert not (bitmask[2] == -1).all()
187
+ assert not (bitmask[-1] == -1).all()
188
+ assert not grammar.is_terminated()
189
+
190
+
191
+@pytest.mark.parametrize("backend", ["xgrammar", "guidance"])
192
+def test_bitmask_post_reasoning_end_drafts_skip_grammar_advance(backend):
193
+ """Post-marker drafts predate the bitmask and may be grammar-invalid;
194
+ grammar_bitmask must skip the grammar advance instead of asserting.
195
+ """
196
+ tokenizer, manager, request, prompt = _make_manager_and_request(
197
+ backend, prompt_str="{"
198
+ )
199
+ grammar = request.structured_output_request.grammar
200
+
201
+ assert grammar.accept_tokens(request.request_id, prompt)
202
+ assert not grammar.is_terminated()
203
+
204
+ marker = tokenizer.encode("\n")[0]
205
+
206
+ class StubReasoner:
207
+ def __init__(self, *_, **__):
208
+ self.end_token_id = marker
209
+
210
+ def is_reasoning_end(self, input_ids):
211
+ return marker in list(input_ids)
212
+
213
+ def is_reasoning_end_streaming(self, input_ids, delta_ids):
214
+ return marker in list(delta_ids)
215
+
216
+ manager.reasoner_cls = StubReasoner
217
+ request.structured_output_request.reasoner = StubReasoner()
218
+ request.structured_output_request.reasoning_ended = False
219
+
220
+ pre = tokenizer.encode(" ")[0]
221
+ # A token that the JSON grammar would reject as the first post-marker
222
+ # token; without the fix grammar.accept_tokens fires the assertion.
223
+ invalid_post = tokenizer.encode("z")[0]
224
+ drafts = [pre, marker, invalid_post]
225
+
226
+ bitmask = manager.grammar_bitmask(
227
+ requests={request.request_id: request},
228
+ structured_output_request_ids=[request.request_id],
229
+ scheduled_spec_decode_tokens={request.request_id: drafts},
230
+ )
231
+
232
+ assert bitmask is not None
233
+ assert bitmask.shape[0] == len(drafts) + 1
234
+ # Post-marker position is still bitmask-constrained.
235
+ assert not (bitmask[2] == -1).all()
236
+ # Grammar must not have advanced through the unvalidated draft.
237
+ assert not grammar.is_terminated()
238
+
239
+
240
+@pytest.mark.parametrize("backend", ["xgrammar", "guidance"])
241
+def test_validate_tokens_then_bitmask_round_trip(backend):
242
+ """validate_tokens -> pad with -1 -> grammar_bitmask must not assert."""
243
+ tokenizer, manager, request, prompt = _make_manager_and_request(backend)
244
+ grammar = request.structured_output_request.grammar
245
+
246
+ assert grammar.accept_tokens(request.request_id, prompt)
247
+
248
+ raw_drafts = [tokenizer.encode(",")[0], 99999, 12345, 67890]
249
+ accepted = grammar.validate_tokens(raw_drafts)
250
+ assert len(accepted) <= len(raw_drafts)
251
+
252
+ padded = accepted + [-1] * (len(raw_drafts) - len(accepted))
253
+ assert len(padded) == len(raw_drafts)
254
+
255
+ bitmask = manager.grammar_bitmask(
256
+ requests={request.request_id: request},
257
+ structured_output_request_ids=[request.request_id],
258
+ scheduled_spec_decode_tokens={request.request_id: padded},
259
+ )
260
+ assert bitmask is not None
261
+ assert bitmask.shape[0] == len(padded) + 1
262
+ assert not grammar.is_terminated()
263
+
264
+
265
+class _MarkerReasoner:
266
+ """Stub reasoner whose reasoning-end marker is a single fixed token."""
267
+
268
+ def __init__(self, marker: int):
269
+ self.marker = marker
270
+
271
+ def is_reasoning_end(self, input_ids):
272
+ return self.marker in list(input_ids)
273
+
274
+ def is_reasoning_end_streaming(self, input_ids, delta_ids):
275
+ return self.marker in list(delta_ids)
276
+
277
+
278
+def _setup_boundary_request(backend: str):
279
+ """Request with a structural-tag key and reasoning not yet ended."""
280
+ from vllm.v1.structured_output.backend_types import StructuredOutputOptions
281
+
282
+ tokenizer, manager, request, prompt = _make_manager_and_request(backend)
283
+ marker = tokenizer.encode("\n")[0]
284
+ structured_req = request.structured_output_request
285
+ # The grammar itself is JSON (cheap to build); only the key kind matters
286
+ # for the should_advance structural-tag branch, so pre-seed the cached
287
+ # property.
288
+ structured_req.__dict__["structured_output_key"] = (
289
+ StructuredOutputOptions.STRUCTURAL_TAG,
290
+ "",
291
+ )
292
+ manager.reasoner_cls = _MarkerReasoner
293
+ structured_req.reasoner = _MarkerReasoner(marker)
294
+ structured_req.reasoning_ended = False
295
+ return tokenizer, manager, request, prompt, marker
296
+
297
+
298
+def test_should_advance_records_reasoning_end_index():
299
+ """Regression for #44006 on post-#42452 main: the boundary step must
300
+ record where reasoning ends so the scheduler can trim before advancing.
301
+ """
302
+ tokenizer, manager, request, prompt, marker = _setup_boundary_request("xgrammar")
303
+ structured_req = request.structured_output_request
304
+
305
+ pre = tokenizer.encode(" ")[0]
306
+ post = tokenizer.encode("{")[0]
307
+ request.append_output_token_ids([pre, marker, post])
308
+
309
+ assert manager.should_advance(request)
310
+ assert structured_req.reasoning_ended
311
+ # Marker sits at absolute index len(prompt) + 1.
312
+ assert structured_req.reasoning_end_token_index == len(prompt) + 1
313
+
314
+
315
+def test_trim_reasoning_for_advance():
316
+ """trim drops the marker and everything before it; later steps and
317
+ requests without a recorded boundary pass through unchanged.
318
+ """
319
+ tokenizer, manager, request, prompt, marker = _setup_boundary_request("xgrammar")
320
+ structured_req = request.structured_output_request
321
+
322
+ pre = tokenizer.encode(" ")[0]
323
+ post = tokenizer.encode("{")[0]
324
+
325
+ # No boundary recorded yet: pass-through.
326
+ assert manager.trim_reasoning_for_advance(request, [pre]) == [pre]
327
+
328
+ # Boundary step: marker mid-step keeps only the suffix.
329
+ step_tokens = [pre, marker, post]
330
+ request.append_output_token_ids(step_tokens)
331
+ assert manager.should_advance(request)
332
+ assert manager.trim_reasoning_for_advance(request, step_tokens) == [post]
333
+
334
+ # Boundary step variant: marker last (the #44006 crash shape
335
+ # [198, </think>]) trims to empty -> scheduler skips accept_tokens.
336
+ structured_req.reasoning_end_token_index = len(request.all_token_ids) - 1
337
+ assert manager.trim_reasoning_for_advance(request, step_tokens) == []
338
+
339
+ # Later steps: tokens are past the boundary, returned unchanged.
340
+ structured_req.reasoning_end_token_index = len(prompt) + 1
341
+ next_step = [post, post]
342
+ request.append_output_token_ids(next_step)
343
+ assert manager.trim_reasoning_for_advance(request, next_step) == next_step
vllm/v1/core/sched/scheduler.py
+13
-4
@@ -1636,14 +1636,23 @@ class Scheduler(SchedulerInterface):
1636
if new_token_ids and self.structured_output_manager.should_advance(request):
1637
struct_output_request = request.structured_output_request
1638
assert struct_output_request is not None
1639
- assert struct_output_request.grammar is not None
1640
- if not struct_output_request.grammar.accept_tokens( # type: ignore[union-attr]
1641
- req_id, new_token_ids
1639
+ grammar = struct_output_request.grammar
1640
+ assert grammar is not None
1641
+ # new_token_ids can be a mixed block of reasoning content, then
1642
+ # the reasoning end marker, then the start of the grammar content.
1643
+ # Trim the reasoning content so the grammar only sees grammar content.
1644
+ advance_token_ids = (
1645
+ self.structured_output_manager.trim_reasoning_for_advance(
1646
+ request, new_token_ids
1647
+ )
1648
+ )
1649
+ if advance_token_ids and not grammar.accept_tokens(
1650
+ req_id, advance_token_ids
1651
):
1652
logger.error(
1653
"Unexpected: grammar rejected tokens %s for request %s. "
1654
"Terminating request.",
1646
- new_token_ids,
1655
+ advance_token_ids,
1656
req_id,
1657
)
1658
request.status = RequestStatus.FINISHED_ERROR
vllm/v1/structured_output/__init__.py
+112
-12
@@ -2,7 +2,7 @@
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
import itertools
4
import multiprocessing
5
-from collections.abc import Iterable
5
+from collections.abc import Iterable, Sequence
6
from concurrent.futures import Future, ThreadPoolExecutor
7
from typing import TYPE_CHECKING
8
@@ -272,23 +272,69 @@ class StructuredOutputManager:
272
grammar = structured_output_request.grammar
273
apply_bitmask = self.should_fill_bitmask(request)
274
275
+ reasoner = self._get_reasoner(request)
276
+ detect_reasoning_end = (
277
+ not apply_bitmask
278
+ and reasoner is not None
279
+ and not self.enable_in_reasoning
280
+ )
281
+ simulated_buf: list[int] | None = None
282
+ history_len = 0
283
+
284
state_advancements = 0
285
+ post_reasoning_end_in_window = False
286
req_tokens = scheduled_spec_decode_tokens.get(req_id, ())
277
- if self.vllm_config.model_config.is_diffusion and req_tokens:
278
- # Diffusion LLMs don't sample a bonus token after the
279
- # scheduled positions, so don't append the -1 placeholder.
280
- token_iter: Iterable[int] = req_tokens
281
- else:
282
- token_iter = itertools.chain(req_tokens, (-1,))
283
- for token in token_iter:
287
+ for i, token in enumerate(req_tokens):
288
self._fill_bitmasks(((grammar, cumulative_index, apply_bitmask),))
289
+ advance_grammar = apply_bitmask
290
if token == -1:
286
- # Stop advancing the grammar once we hit a padding token.
291
apply_bitmask = False
288
- if apply_bitmask and not grammar.is_terminated():
292
+ advance_grammar = False
293
+ elif (
294
+ detect_reasoning_end
295
+ and reasoner is not None
296
+ and not apply_bitmask
297
+ ):
298
+ if simulated_buf is None:
299
+ history = list(request.all_token_ids)
300
+ history_len = len(history)
301
+ simulated_buf = history + list(req_tokens)
302
+ simulated = simulated_buf[: history_len + i + 1]
303
+ if reasoner.is_reasoning_end_streaming(simulated, [token]):
304
+ # Reasoning ended mid-window. Constrain the rest
305
+ # of the window via bitmask. Skip grammar advance
306
+ # through the marker (it is reasoning content);
307
+ # try to advance through subsequent drafts so the
308
+ # next bitmask row reflects the post-advance state,
309
+ # but tolerate rejection since those drafts predate
310
+ # the bitmask and are not guaranteed valid.
311
+ apply_bitmask = True
312
+ advance_grammar = False
313
+ post_reasoning_end_in_window = True
314
+ if advance_grammar and not grammar.is_terminated():
315
accepted = grammar.accept_tokens(req_id, [token])
290
- assert accepted, (token, req_id, scheduled_spec_decode_tokens)
291
- state_advancements += 1
316
+ if accepted:
317
+ state_advancements += 1
318
+ elif not post_reasoning_end_in_window:
319
+ raise AssertionError(
320
+ (token, req_id, scheduled_spec_decode_tokens)
321
+ )
322
+ cumulative_index += 1
323
+ # Diffusion LLMs don't sample a bonus token after the
324
+ # scheduled positions, so skip its bitmask in that case.
325
+ if not (self.vllm_config.model_config.is_diffusion and req_tokens):
326
+ # bonus_apply must be True when the bonus-row position
327
+ # should be grammar-constrained. Two triggers:
328
+ # - should_fill_bitmask(request): reasoning was already
329
+ # over at step start (or no reasoner /
330
+ # enable_in_reasoning).
331
+ # - apply_bitmask: reasoning ended mid-window in this
332
+ # call and was flipped True after the marker;
333
+ # should_fill_bitmask still returns False here because
334
+ # reasoning_ended is only persisted later by
335
+ # should_advance.
336
+ bonus_apply = self.should_fill_bitmask(request) or apply_bitmask
337
+ self._fill_bitmasks(((grammar, cumulative_index, bonus_apply),))
338
cumulative_index += 1
339
if state_advancements > 0:
340
grammar.rollback(state_advancements)
@@ -368,10 +414,64 @@ class StructuredOutputManager:
414
and structured_req.structured_output_key[0]
415
== StructuredOutputOptions.STRUCTURAL_TAG
416
):
417
+ # The scheduler will advance the grammar with this step's
418
+ # tokens right away, but the step still contains reasoning
419
+ # content up to and including the end marker. Record where
420
+ # it ends so trim_reasoning_for_advance() can drop it.
421
+ structured_req.reasoning_end_token_index = (
422
+ self._find_reasoning_end_index(reasoner, all_token_ids, start)
423
+ )
424
return True
425
426
return False
427
428
+ @staticmethod
429
+ def _find_reasoning_end_index(
430
+ reasoner: "ReasoningParser", all_token_ids: Sequence[int], start: int
431
+ ) -> int:
432
+ """Locates the last reasoning token within ``all_token_ids[start:]``.
433
+
434
+ Returns:
435
+ The absolute index of the token at which
436
+ ``is_reasoning_end_streaming`` first fires. Falls back to the
437
+ final index when no single token triggers the detection (e.g.
438
+ a multi-token marker only recognized on the full delta), which
439
+ conservatively treats the whole step as reasoning content.
440
+ """
441
+ prefix = list(itertools.islice(all_token_ids, start))
442
+ for idx in range(start, len(all_token_ids)):
443
+ token = all_token_ids[idx]
444
+ prefix.append(token)
445
+ if reasoner.is_reasoning_end_streaming(prefix, [token]):
446
+ return idx
447
+ return len(all_token_ids) - 1
448
+
449
+ def trim_reasoning_for_advance(
450
+ self, request: "Request", new_token_ids: list[int]
451
+ ) -> list[int]:
452
+ """Drops reasoning content from tokens about to advance the grammar.
453
+
454
+ When reasoning ends mid-step (see should_advance), the step's output
455
+ still contains reasoning tokens up to and including the end marker.
456
+ Those are not grammar content: feeding them to accept_tokens makes
457
+ the grammar reject the marker and kills the request (#44006).
458
+
459
+ Returns:
460
+ The suffix of ``new_token_ids`` that follows the reasoning-end
461
+ marker. Steps fully after the boundary are returned unchanged.
462
+ """
463
+ structured_req = request.structured_output_request
464
+ if structured_req is None:
465
+ return new_token_ids
466
+ end_idx = structured_req.reasoning_end_token_index
467
+ if end_idx is None:
468
+ return new_token_ids
469
+ first_idx = len(request.all_token_ids) - len(new_token_ids)
470
+ num_reasoning = end_idx + 1 - first_idx
471
+ if num_reasoning <= 0:
472
+ return new_token_ids
473
+ return new_token_ids[num_reasoning:]
474
+
475
def clear_backend(self) -> None:
476
if self.backend is not None:
477
self.backend.destroy()
vllm/v1/structured_output/request.py
+6
@@ -23,6 +23,12 @@ class StructuredOutputRequest:
23
params: StructuredOutputsParams
24
_grammar: Future[StructuredOutputGrammar] | StructuredOutputGrammar | None = None
25
reasoning_ended: bool | None = None
26
+ # Absolute index into the request's all_token_ids of the last reasoning
27
+ # token (the reasoning-end marker). Tokens at or before this index are
28
+ # reasoning content and must never be fed to the grammar. Only set when
29
+ # reasoning ends in a step whose tokens the scheduler advances immediately
30
+ # (structural tags + speculative decoding, see #42452).
31
+ reasoning_end_token_index: int | None = None
32
reasoning_parser_kwargs: dict[str, Any] | None = None
33
# Cached per request; do not share reasoning parsers across requests because
34
# their behavior can depend on reasoning_parser_kwargs.