[BugFix][Core] Make the structured-output grammar poll non-blocking (#55931)
Signed-off-by: ubwzwd <ubwzwd@gmail.com> Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com> Co-authored-by: Artem Perevedentsev <aperevedents@nvidia.com>
ubwzwd committed
Sep 22, 2026 at 19:39 UTC
9815732c16ca076c60b65fccd787167d6a9c5328
2 files changed
+33
-5
tests/v1/structured_output/test_request.py
new
+30
@@ -0,0 +1,30 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+"""Unit tests for StructuredOutputRequest grammar-future polling."""
4
+
5
+from concurrent.futures import Future
6
+
7
+from vllm.sampling_params import StructuredOutputsParams
8
+from vllm.v1.structured_output.request import StructuredOutputRequest
9
+
10
+
11
+def _request_with_future(fut: Future) -> StructuredOutputRequest:
12
+ req = StructuredOutputRequest(params=StructuredOutputsParams(regex="a+"))
13
+ req.grammar = fut
14
+ return req
15
+
16
+
17
+def test_pending_grammar_future_is_not_ready():
18
+ req = _request_with_future(Future())
19
+ assert not req.is_grammar_ready
20
+ assert req.grammar is None
21
+
22
+
23
+def test_grammar_compile_raising_timeout_error_is_a_failure():
24
+ """A compile that raised TimeoutError must be treated as a failed compile,
25
+ not as still-compiling (#53130)."""
26
+ fut: Future = Future()
27
+ fut.set_exception(TimeoutError("compiler timed out"))
28
+ req = _request_with_future(fut)
29
+ assert req.is_grammar_ready
30
+ assert isinstance(req.grammar, TimeoutError)
vllm/v1/structured_output/request.py
+3
-5
@@ -4,7 +4,6 @@ import dataclasses
4
import functools
5
import json
6
from concurrent.futures import Future
7
-from concurrent.futures._base import TimeoutError
7
from typing import TYPE_CHECKING, Any, cast
8
9
from vllm.sampling_params import SamplingParams, StructuredOutputsParams
@@ -43,11 +42,10 @@ class StructuredOutputRequest:
42
43
def _check_grammar_completion(self) -> bool:
44
if isinstance(self._grammar, Future):
46
- try:
47
- # We will check whether the future is ready within 100 us
48
- self._grammar = self._grammar.result(timeout=0.0001)
49
- except TimeoutError:
45
+ if not self._grammar.done():
46
return False
47
+ try:
48
+ self._grammar = self._grammar.result()
49
except Exception as e:
50
self._grammar = e
51
return True