[Structured Output] Keep invalid structured-output requests from stopping the engine (#51450)
Co-authored-by: Lucas Bourtoule <35483370+dhalf@users.noreply.github.com> Co-authored-by: Vadim Gimpelson <156319763+vadiklyutiy@users.noreply.github.com>
Clinton Thomas committed
Sep 9, 2026 at 11:32 UTC
c55e15a44ec4127832d4a86928a356fdd9e68dbd
6 files changed
+193
tests/v1/core/test_scheduler.py
+90
@@ -2,6 +2,7 @@
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
import dataclasses
4
from concurrent.futures import Future
5
+from types import SimpleNamespace
6
from unittest.mock import Mock
7
8
import numpy as np
@@ -6425,3 +6426,92 @@ def test_encoder_input_skipped_when_connector_already_has_the_item(ec_role: str)
6426
6427
assert output.num_scheduled_tokens[req_id] > 0
6428
assert not output.scheduled_encoder_inputs.get(req_id)
6429
+
6430
+
6431
+class _RecordingGrammar:
6432
+ """Records what reaches ``validate_tokens`` and raises on a negative id.
6433
+
6434
+ llguidance raises ``OverflowError`` on a negative token id, so this mirrors
6435
+ the real failure: the tests below fail on the unpatched scheduler (the -1
6436
+ padding reaches ``validate_tokens``) and pass once it is stripped first.
6437
+ """
6438
+
6439
+ def __init__(self) -> None:
6440
+ self.seen: list[list[int]] = []
6441
+
6442
+ def validate_tokens(self, tokens: list[int]) -> list[int]:
6443
+ if any(t < 0 for t in tokens):
6444
+ raise OverflowError("llguidance rejects negative token ids")
6445
+ self.seen.append(list(tokens))
6446
+ return tokens
6447
+
6448
+
6449
+def _decode_ready_request(scheduler):
6450
+ """Add one request and advance it out of prefill into decode."""
6451
+ request = create_requests(num_requests=1, num_tokens=1)[0]
6452
+ scheduler.add_request(request)
6453
+ output = scheduler.schedule()
6454
+ scheduler.update_from_output(
6455
+ output,
6456
+ ModelRunnerOutput(
6457
+ req_ids=[request.request_id],
6458
+ req_id_to_index={request.request_id: 0},
6459
+ sampled_token_ids=[[0]],
6460
+ logprobs=None,
6461
+ prompt_logprobs_dict={},
6462
+ pooler_output=[],
6463
+ ),
6464
+ )
6465
+ return request
6466
+
6467
+
6468
+def test_update_draft_token_ids_strips_ngram_padding(monkeypatch):
6469
+ """ngram_gpu pads unfilled draft slots with -1; update_draft_token_ids must
6470
+ strip them before grammar.validate_tokens (which otherwise raises)."""
6471
+ scheduler = create_scheduler(num_speculative_tokens=4)
6472
+ request = _decode_ready_request(scheduler)
6473
+
6474
+ grammar = _RecordingGrammar()
6475
+ request.structured_output_request = SimpleNamespace(grammar=grammar)
6476
+ monkeypatch.setattr(
6477
+ scheduler.structured_output_manager, "should_advance", lambda req: True
6478
+ )
6479
+
6480
+ scheduler.update_draft_token_ids(
6481
+ DraftTokenIds([request.request_id], [[10, 11, -1, -1]])
6482
+ )
6483
+
6484
+ assert grammar.seen == [[10, 11]]
6485
+ assert request.spec_token_ids == [10, 11]
6486
+
6487
+
6488
+def test_update_draft_token_ids_in_output_strips_padding(monkeypatch):
6489
+ """Same guard on the output path; the -1 pad-back for the rejected count
6490
+ is preserved (only the input to validate_tokens is stripped)."""
6491
+ scheduler = create_scheduler(num_speculative_tokens=4)
6492
+ request = _decode_ready_request(scheduler)
6493
+
6494
+ grammar = _RecordingGrammar()
6495
+ request.structured_output_request = SimpleNamespace(grammar=grammar)
6496
+ monkeypatch.setattr(
6497
+ scheduler.structured_output_manager, "should_advance", lambda req: True
6498
+ )
6499
+
6500
+ scheduler_output = SimpleNamespace(
6501
+ scheduled_spec_decode_tokens={request.request_id: [0, 0, 0, 0]}
6502
+ )
6503
+ scheduler.update_draft_token_ids_in_output(
6504
+ DraftTokenIds([request.request_id], [[10, 11, -1, -1]]),
6505
+ scheduler_output,
6506
+ )
6507
+
6508
+ # The grammar only saw the stripped prefix, never a -1.
6509
+ assert grammar.seen == [[10, 11]]
6510
+ # Two drafts were rejected (4 scheduled - 2 valid), padded back with -1.
6511
+ assert scheduler_output.scheduled_spec_decode_tokens[request.request_id] == [
6512
+ 10,
6513
+ 11,
6514
+ -1,
6515
+ -1,
6516
+ ]
6517
+ assert scheduler_output.num_invalid_spec_tokens == {request.request_id: 2}
tests/v1/structured_output/test_guidance_negative_draft_tokens.py
new
+61
@@ -0,0 +1,61 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+"""Guidance backend must not choke on speculative -1 padding.
4
+
5
+Speculative drafters (e.g. ngram_gpu) pad unfilled draft slots with a -1
6
+sentinel. llguidance raises ``OverflowError`` on a negative id, so
7
+``GuidanceGrammar.validate_tokens`` must stop at the first sentinel. The mock
8
+matcher below emulates that raise, so each test fails on the unpatched backend
9
+and passes with the guard.
10
+"""
11
+
12
+import pytest
13
+
14
+from vllm.v1.structured_output.backend_guidance import GuidanceGrammar
15
+
16
+pytestmark = pytest.mark.skip_global_cleanup
17
+
18
+
19
+class _Matcher:
20
+ """Stand-in for ``llguidance.LLMatcher`` that rejects negative ids."""
21
+
22
+ def __init__(self) -> None:
23
+ self.tokens: list[int] | None = None
24
+
25
+ def is_stopped(self) -> bool:
26
+ return False
27
+
28
+ def validate_tokens(self, tokens: list[int]) -> int:
29
+ if any(t < 0 for t in tokens):
30
+ raise OverflowError("llguidance rejects negative token ids")
31
+ self.tokens = tokens
32
+ return len(tokens)
33
+
34
+ def get_error(self):
35
+ return None
36
+
37
+
38
+class _Tokenizer:
39
+ eos_token = 0
40
+
41
+
42
+def _grammar(matcher: _Matcher) -> GuidanceGrammar:
43
+ return GuidanceGrammar(ll_matcher=matcher, ll_tokenizer=_Tokenizer(), vocab_size=16)
44
+
45
+
46
+def test_validate_tokens_stops_at_trailing_padding() -> None:
47
+ matcher = _Matcher()
48
+ assert _grammar(matcher).validate_tokens([3, 4, -1, -1]) == [3, 4]
49
+ assert matcher.tokens == [3, 4]
50
+
51
+
52
+def test_validate_tokens_single_negative_does_not_raise() -> None:
53
+ matcher = _Matcher()
54
+ assert _grammar(matcher).validate_tokens([-1]) == []
55
+ assert matcher.tokens is None
56
+
57
+
58
+def test_validate_tokens_all_padding_returns_empty() -> None:
59
+ matcher = _Matcher()
60
+ assert _grammar(matcher).validate_tokens([-1, -1]) == []
61
+ assert matcher.tokens is None
tests/v1/structured_output/test_scheduler_speculative_padding.py
new
+20
@@ -0,0 +1,20 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+
4
+from vllm.v1.structured_output.utils import strip_speculative_padding
5
+
6
+
7
+def test_strips_trailing_padding():
8
+ assert strip_speculative_padding([3, 4, -1, -1]) == [3, 4]
9
+
10
+
11
+def test_empty_when_leading_padding():
12
+ assert strip_speculative_padding([-1, -1]) == []
13
+
14
+
15
+def test_passthrough_without_padding():
16
+ assert strip_speculative_padding([3, 4]) == [3, 4]
17
+
18
+
19
+def test_truncates_at_first_sentinel():
20
+ assert strip_speculative_padding([3, -1, 4]) == [3]
vllm/v1/core/sched/scheduler.py
+3
@@ -71,6 +71,7 @@ from vllm.v1.request import Request, RequestStatus, StreamingUpdate
71
from vllm.v1.spec_decode.dynamic.utils import build_dynamic_sd_schedule_lookup
72
from vllm.v1.spec_decode.metrics import SpecDecodingStats
73
from vllm.v1.structured_output import StructuredOutputGrammar, StructuredOutputManager
74
+from vllm.v1.structured_output.utils import strip_speculative_padding
75
from vllm.v1.utils import record_function_or_nullcontext
76
77
logger = init_logger(__name__)
@@ -2424,6 +2425,7 @@ class Scheduler(SchedulerInterface):
2425
# Add newly generated spec token ids to the request.
2426
if self.structured_output_manager.should_advance(request):
2427
metadata = request.structured_output_request
2428
+ spec_token_ids = strip_speculative_padding(spec_token_ids)
2429
spec_token_ids = metadata.grammar.validate_tokens(spec_token_ids) # type: ignore[union-attr]
2430
request.spec_token_ids = spec_token_ids
2431
@@ -2453,6 +2455,7 @@ class Scheduler(SchedulerInterface):
2455
# Filter out spec tokens which do not adhere to the grammar.
2456
if self.structured_output_manager.should_advance(request):
2457
metadata = request.structured_output_request
2458
+ spec_token_ids = strip_speculative_padding(spec_token_ids)
2459
spec_token_ids = metadata.grammar.validate_tokens(spec_token_ids) # type: ignore[union-attr]
2460
# Pad to original number of spec tokens.
2461
num_invalid_tokens = orig_num_spec_tokens - len(spec_token_ids)
vllm/v1/structured_output/backend_guidance.py
+5
@@ -21,6 +21,7 @@ from vllm.v1.structured_output.backend_types import (
21
StructuredOutputOptions,
22
)
23
from vllm.v1.structured_output.request import get_structured_output_key
24
+from vllm.v1.structured_output.utils import strip_speculative_padding
25
26
if TYPE_CHECKING:
27
import llguidance
@@ -198,6 +199,10 @@ class GuidanceGrammar(StructuredOutputGrammar):
199
if self.ll_matcher.is_stopped():
200
return []
201
202
+ tokens = strip_speculative_padding(tokens)
203
+ if len(tokens) == 0:
204
+ return []
205
+
206
num_tokens = self.ll_matcher.validate_tokens(tokens)
207
208
self.check_error()
vllm/v1/structured_output/utils.py
+14
@@ -46,6 +46,20 @@ _T = TypeVar("_T")
46
CACHE = None
47
48
49
+def strip_speculative_padding(token_ids: list[int]) -> list[int]:
50
+ """Drop speculative-decoding padding from a token block.
51
+
52
+ ngram and other speculative backends pad rejected draft positions with a
53
+ -1 sentinel. Structured-output grammars treat every entry as a real token
54
+ id, so the sentinels (and everything after the first one) are removed here,
55
+ before tokens reach any backend, rather than inside a single backend.
56
+ """
57
+ for i, token_id in enumerate(token_ids):
58
+ if token_id < 0:
59
+ return token_ids[:i]
60
+ return token_ids
61
+
62
+
63
def compile_regex_with_timeout(fn: Callable[[str], _T], pattern: str) -> _T:
64
"""Run a regex compilation callable with a timeout.
65