A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 3, 05:30 UTC
[Bug][Structured Outputs] Fix bug that leads to unconstrained generations with structural tags (#42452)
Signed-off-by: rishitdholakia13 <rishit+github@cohere.com> Co-authored-by: Cursor <cursoragent@cursor.com>
rishitdholakia13 committed
May 20, 2026 at 10:08 UTC
ded871201a424dd0d28a00aaf74c5786457a18ee
2 files changed
+42
-2
tests/v1/structured_output/test_reasoning_structured_output.py
+27
@@ -10,6 +10,7 @@ import pytest
10
from vllm.config import ModelConfig, SchedulerConfig, VllmConfig
11
from vllm.v1.request import Request
12
from vllm.v1.structured_output import StructuredOutputManager
13
+from vllm.v1.structured_output.backend_types import StructuredOutputOptions
14
15
16
class MockReasoner:
@@ -214,6 +215,32 @@ class TestReasoningStructuredOutput:
215
)
216
assert result is False
217
218
+ def test_should_advance_reasoning_just_ended_with_spec_decode_structural_tag(
219
+ self,
220
+ manager_with_reasoner,
221
+ mock_request_with_structured_output,
222
+ ):
223
+ """When reasoning ends this step, advance immediately for structural
224
+ tags with speculative decoding."""
225
+ structured_req = mock_request_with_structured_output.structured_output_request
226
+ structured_req.reasoning_ended = False
227
+ structured_req.structured_output_key = (
228
+ StructuredOutputOptions.STRUCTURAL_TAG,
229
+ "{}",
230
+ )
231
+ reasoner = MockReasoner(tokenizer=Mock())
232
+ reasoner.is_reasoning_end_streaming.return_value = True
233
+ structured_req.reasoner = reasoner
234
+
235
+ manager_with_reasoner.vllm_config.speculative_config = Mock()
236
+
237
+ result = manager_with_reasoner.should_advance(
238
+ mock_request_with_structured_output
239
+ )
240
+
241
+ assert structured_req.reasoning_ended is True
242
+ assert result is True
243
+
244
def test_should_advance_reasoning_already_ended(
245
self,
246
manager_with_reasoner,
vllm/v1/structured_output/__init__.py
+15
-2
@@ -15,6 +15,7 @@ from vllm.v1.structured_output.backend_guidance import GuidanceBackend
15
from vllm.v1.structured_output.backend_types import (
16
StructuredOutputBackend,
17
StructuredOutputGrammar,
18
+ StructuredOutputOptions,
19
)
20
from vllm.v1.structured_output.backend_xgrammar import XgrammarBackend
21
@@ -350,10 +351,22 @@ class StructuredOutputManager:
351
if reasoner.is_reasoning_end_streaming(
352
all_token_ids, itertools.islice(all_token_ids, start, None)
353
):
353
- # Reasoning just ended, so we shouldn't advance til
354
- # next pass
354
structured_req.reasoning_ended = True
355
356
+ # Reasoning just ended this step. Defer FSM advance until the next
357
+ # pass (see reasoning_ended check above) for JSON/regex/choice/grammar:
358
+ # advancing on the closing boundary token can accept tokens that still
359
+ # belong to the reasoning stream. Structural tags are the only safe
360
+ # same-step exception: they model phased output (e.g. thinking tag ->
361
+ # answer tag), and speculative decoding must run grammar.validate_tokens
362
+ # on draft tokens produced immediately after that transition.
363
+ if (
364
+ self.vllm_config.speculative_config is not None
365
+ and structured_req.structured_output_key[0]
366
+ == StructuredOutputOptions.STRUCTURAL_TAG
367
+ ):
368
+ return True
369
+
370
return False
371
372
def clear_backend(self) -> None: