@tduncan / Nvidia-Optimization-Stack-VLLM / commits / 12213c6795

[Bugfix] handle grammar compilation failures to avoid engine crash (#47312)

Signed-off-by: zhuhaoran <zhuhaoran.zhr@alibaba-inc.com> Signed-off-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: Nick Hill <nickhill123@gmail.com>

zhrrr committed Jul 23, 2026 at 18:13 UTC 12213c67951b71cd6d750cc059a87a53641e8730

7 files changed +123 -41

tests/v1/core/test_async_scheduler.py

+3 -1

@@ -9,6 +9,7 @@ from vllm.v1.core.sched.async_scheduler import AsyncScheduler

9 from vllm.v1.core.sched.output import CachedRequestData, SchedulerOutput

10 from vllm.v1.outputs import ModelRunnerOutput

11 from vllm.v1.request import RequestStatus

12 +from vllm.v1.structured_output import StructuredOutputGrammar

13 from vllm.v1.utils import ConstantList

15 from .utils import create_requests, create_scheduler

@@ -262,7 +263,7 @@ def test_abort_request_when_structured_output_fsm_cannot_advance():

263 scheduler = object.__new__(AsyncScheduler)

264 request = create_requests(num_requests=1, num_tokens=1)[0]

265 request.structured_output_request = Mock()

265 - request.structured_output_request.grammar = Mock()

266 + request.structured_output_request.grammar = Mock(spec=StructuredOutputGrammar)

267 request.structured_output_request.grammar.accept_tokens.return_value = False

268 request.status = RequestStatus.RUNNING

269 request.num_computed_tokens = request.num_tokens

@@ -284,6 +285,7 @@ def test_abort_request_when_structured_output_fsm_cannot_advance():

285 scheduler.kv_event_publisher = Mock()

286 scheduler.finished_req_ids = set()

287 scheduler.finished_req_ids_dict = None

288 + scheduler.grammar_compile_error_reqs = set()

289 scheduler.vllm_config = Mock()

290 scheduler.vllm_config.model_config.enable_return_routed_experts = False

291 scheduler.enable_return_routed_experts = False

tests/v1/core/test_scheduler.py

+56 -2

@@ -1,6 +1,7 @@

1 # SPDX-License-Identifier: Apache-2.0

2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project

3 import dataclasses

4 +from concurrent.futures import Future

5 from unittest.mock import Mock

7 import pytest

@@ -39,7 +40,7 @@ from vllm.v1.kv_cache_interface import (

40 )

41 from vllm.v1.outputs import DraftTokenIds, KVConnectorOutput, ModelRunnerOutput

42 from vllm.v1.request import Request, RequestStatus

42 -from vllm.v1.structured_output import StructuredOutputManager

43 +from vllm.v1.structured_output import StructuredOutputGrammar, StructuredOutputManager

45 from .utils import EOS_TOKEN_ID, create_requests, create_scheduler, mock_kv

@@ -3144,6 +3145,58 @@ def test_schedule_skip_tokenizer_init_structured_output_request():

3145 assert len(scheduler.skipped_waiting) == 1

3146

3147

3148 +@pytest.mark.parametrize("async_grammar", [True, False])

3149 +def test_grammar_compile_error_finishes_only_request(async_grammar: bool):

3150 + scheduler = create_scheduler()

3151 + manager = scheduler.structured_output_manager

3152 + manager.backend = Mock()

3153 + manager.backend.compile_grammar.side_effect = RuntimeError(

3154 + "forced FSM compilation error"

3155 + )

3156 + manager._use_async_grammar_compilation = async_grammar

3157 +

3158 + sampling_params = SamplingParams(

3159 + max_tokens=16,

3160 + structured_outputs=StructuredOutputsParams(json='{"type": "object"}'),

3161 + )

3162 + sampling_params.update_from_generation_config({}, EOS_TOKEN_ID)

3163 + request = Request(

3164 + request_id="grammar-error",

3165 + prompt_token_ids=[0, 1],

3166 + sampling_params=sampling_params,

3167 + pooling_params=None,

3168 + )

3169 +

3170 + manager.grammar_init(request)

3171 + assert request.structured_output_request is not None

3172 + grammar_future = request.structured_output_request._grammar

3173 + assert isinstance(grammar_future, Future)

3174 + assert isinstance(grammar_future.exception(timeout=5), RuntimeError)

3175 +

3176 + scheduler.add_request(request)

3177 + scheduler_output = scheduler.schedule()

3178 + assert not scheduler_output.num_scheduled_tokens

3179 +

3180 + engine_core_outputs = scheduler.update_from_output(

3181 + scheduler_output,

3182 + ModelRunnerOutput(req_ids=[], req_id_to_index={}),

3183 + )

3184 +

3185 + assert request.status == RequestStatus.FINISHED_ERROR

3186 + assert request.request_id not in scheduler.requests

3187 + output = engine_core_outputs[0].outputs[0]

3188 + assert output.request_id == request.request_id

3189 + assert output.finish_reason == FinishReason.ERROR

3190 + assert output.stop_reason is None

3191 +

3192 + healthy_request = create_requests(num_requests=1, req_ids=["healthy-request"])[0]

3193 + scheduler.add_request(healthy_request)

3194 + next_output = scheduler.schedule()

3195 + assert [req.req_id for req in next_output.scheduled_new_reqs] == [

3196 + healthy_request.request_id

3197 + ]

3198 +

3199 +

3200 def test_abort_request_when_structured_output_fsm_cannot_advance():

3201 scheduler = object.__new__(Scheduler)

3202 sampling_params = SamplingParams(ignore_eos=True, max_tokens=4)

@@ -3157,7 +3210,7 @@ def test_abort_request_when_structured_output_fsm_cannot_advance():

3210 pooling_params=None,

3211 )

3212 request.structured_output_request = Mock()

3160 - request.structured_output_request.grammar = Mock()

3213 + request.structured_output_request.grammar = Mock(spec=StructuredOutputGrammar)

3214 request.structured_output_request.grammar.accept_tokens.return_value = False

3215 request.status = RequestStatus.RUNNING

3216 request.num_computed_tokens = request.num_tokens

@@ -3178,6 +3231,7 @@ def test_abort_request_when_structured_output_fsm_cannot_advance():

3231 scheduler.kv_event_publisher = Mock()

3232 scheduler.finished_req_ids = set()

3233 scheduler.finished_req_ids_dict = None

3234 + scheduler.grammar_compile_error_reqs = set()

3235 scheduler.vllm_config = Mock()

3236 scheduler.vllm_config.model_config.enable_return_routed_experts = False

3237 scheduler.enable_return_routed_experts = False

vllm/v1/core/sched/interface.py

+3 -3

@@ -145,7 +145,7 @@ class SchedulerInterface(ABC):

145 self,

146 request_ids: str | Iterable[str] | None,

147 finished_status: "RequestStatus",

148 - ) -> list[tuple[str, int]]:

148 + ) -> "list[Request]":

149 """Finish the requests in the scheduler's internal queue. If the request

150 is not in the queue, this method will do nothing for that request.

151

@@ -159,8 +159,8 @@ class SchedulerInterface(ABC):

159 finished_status: The finished status of the given requests.

160

161 Returns:

162 - Tuple of (req_id, client_index) for requests that were aborted. Will not

163 - include any that were already finished.

162 + List of requests that were aborted. Will not include any that were

163 + already finished.

164 """

165 raise NotImplementedError

166

vllm/v1/core/sched/scheduler.py

+24 -12

@@ -60,7 +60,7 @@ from vllm.v1.outputs import DraftTokenIds, KVConnectorOutput, ModelRunnerOutput

60 from vllm.v1.request import Request, RequestStatus, StreamingUpdate

61 from vllm.v1.spec_decode.dynamic.utils import build_dynamic_sd_schedule_lookup

62 from vllm.v1.spec_decode.metrics import SpecDecodingStats

63 -from vllm.v1.structured_output import StructuredOutputManager

63 +from vllm.v1.structured_output import StructuredOutputGrammar, StructuredOutputManager

64 from vllm.v1.utils import record_function_or_nullcontext

66 logger = init_logger(__name__)

@@ -201,6 +201,10 @@ class Scheduler(SchedulerInterface):

201 self.finished_recving_kv_req_ids: set[str] = set()

202 self.failed_recving_kv_req_ids: set[str] = set()

203

204 + # Grammar compilation failures to finish as per-request errors in

205 + # update_from_output.

206 + self.grammar_compile_error_reqs: set[str] = set()

207 +

208 # Encoder-related.

209 # Calculate encoder cache size if applicable

210 supports_mm_inputs = mm_registry.supports_multimodal_inputs(

@@ -1716,7 +1720,7 @@ class Scheduler(SchedulerInterface):

1720 struct_output_request = request.structured_output_request

1721 assert struct_output_request is not None

1722 grammar = struct_output_request.grammar

1719 - assert grammar is not None

1723 + assert isinstance(grammar, StructuredOutputGrammar)

1724 # new_token_ids can be a mixed block of reasoning content, then

1725 # the reasoning end marker, then the start of the grammar content.

1726 # Trim the reasoning content so the grammar only sees grammar content.

@@ -1846,10 +1850,16 @@ class Scheduler(SchedulerInterface):

1850 # This is a rare case and unlikely to impact performance.

1851 self.waiting.remove_requests(stopped_preempted_reqs)

1852

1853 + error_req_ids = set(self.grammar_compile_error_reqs)

1854 + self.grammar_compile_error_reqs.clear()

1855 if failed_kv_load_req_ids and not self.recompute_kv_load_failures:

1850 - requests = [self.requests[req_id] for req_id in failed_kv_load_req_ids]

1851 - self.finish_requests(failed_kv_load_req_ids, RequestStatus.FINISHED_ERROR)

1852 - for request in requests:

1856 + error_req_ids.update(failed_kv_load_req_ids)

1857 +

1858 + if error_req_ids:

1859 + error_reqs = self.finish_requests(

1860 + error_req_ids, RequestStatus.FINISHED_ERROR

1861 + )

1862 + for request in error_reqs:

1863 outputs[request.client_index].append(

1864 EngineCoreOutput(

1865 request_id=request.request_id,

@@ -2079,8 +2089,7 @@ class Scheduler(SchedulerInterface):

2089 # Filter out spec tokens which do not adhere to the grammar.

2090 if self.structured_output_manager.should_advance(request):

2091 metadata = request.structured_output_request

2082 - assert metadata is not None and metadata.grammar is not None

2083 - spec_token_ids = metadata.grammar.validate_tokens(spec_token_ids)

2092 + spec_token_ids = metadata.grammar.validate_tokens(spec_token_ids) # type: ignore[union-attr]

2093 # Pad to original number of spec tokens.

2094 num_invalid_tokens = orig_num_spec_tokens - len(spec_token_ids)

2095 if num_invalid_tokens:

@@ -2121,7 +2130,7 @@ class Scheduler(SchedulerInterface):

2130

2131 def finish_requests(

2132 self, request_ids: str | Iterable[str] | None, finished_status: RequestStatus

2124 - ) -> list[tuple[str, int]]:

2133 + ) -> list[Request]:

2134 """Handles the finish signal from outside the scheduler.

2135

2136 For example, the API server can abort a request when the client

@@ -2130,8 +2139,8 @@ class Scheduler(SchedulerInterface):

2139 If request_ids is None, all requests will be finished.

2140

2141 Returns:

2133 - Tuple of (req_id, client_index) for requests that were aborted. Will not

2134 - include any that were already finished.

2142 + List of requests that were aborted. Will not include any that were

2143 + already finished.

2144 """

2145 assert RequestStatus.is_finished(finished_status)

2146 if isinstance(request_ids, str):

@@ -2180,7 +2189,7 @@ class Scheduler(SchedulerInterface):

2189 request.status = finished_status

2190 self._free_request(request, delay_free_blocks=delay_free_blocks)

2191

2183 - return [(r.request_id, r.client_index) for r in valid_requests]

2192 + return valid_requests

2193

2194 def _free_request(

2195 self, request: Request, delay_free_blocks: bool = False

@@ -2580,7 +2589,10 @@ class Scheduler(SchedulerInterface):

2589

2590 if request.status == RequestStatus.WAITING_FOR_STRUCTURED_OUTPUT_GRAMMAR:

2591 structured_output_req = request.structured_output_request

2583 - if not (structured_output_req and structured_output_req.grammar):

2592 + if not structured_output_req or structured_output_req.grammar is None:

2593 + return False

2594 + if isinstance(structured_output_req.grammar, Exception):

2595 + self.grammar_compile_error_reqs.add(request.request_id)

2596 return False

2597 request.status = RequestStatus.WAITING

2598 return True

vllm/v1/engine/core.py

+3 -3

@@ -1830,13 +1830,13 @@ class EngineCoreProc(EngineCore):

1830 ) -> None:

1831 self._send_finish_outputs_to_client(req_ids, client_index, FinishReason.ERROR)

1832

1833 - def _send_abort_outputs(self, aborted_reqs: list[tuple[str, int]]) -> None:

1833 + def _send_abort_outputs(self, aborted_reqs: list[Request]) -> None:

1834 # TODO(nick) this will be moved inside the scheduler

1835 if aborted_reqs:

1836 # Map client_index to list of request_ids that belong to that client.

1837 by_client = defaultdict[int, set[str]](set)

1838 - for req_id, client_index in aborted_reqs:

1839 - by_client[client_index].add(req_id)

1838 + for request in aborted_reqs:

1839 + by_client[request.client_index].add(request.request_id)

1840 for client_index, req_ids in by_client.items():

1841 self._send_abort_outputs_to_client(list(req_ids), client_index)

1842

vllm/v1/structured_output/__init__.py

+25 -14

@@ -164,24 +164,33 @@ class StructuredOutputManager:

164 else:

165 raise ValueError(f"Unsupported structured output backend: {backend}")

166

167 + grammar: Future[StructuredOutputGrammar] | StructuredOutputGrammar

168 if self._use_async_grammar_compilation:

169 grammar = self.executor.submit(self._create_grammar, request)

170 else:

170 - grammar = self._create_grammar(request) # type: ignore[assignment]

171 - request.structured_output_request.grammar = grammar # type: ignore[assignment]

171 + try:

172 + grammar = self._create_grammar(request)

173 + except Exception as e:

174 + grammar = Future()

175 + grammar.set_exception(e)

176 + request.structured_output_request.grammar = grammar

177

178 def _create_grammar(self, request: "Request") -> StructuredOutputGrammar:

174 - key = request.structured_output_request.structured_output_key # type: ignore[union-attr]

175 -

179 + struct_request = request.structured_output_request

180 + assert struct_request is not None

181 # Note that the request was validated in the engine core client,

177 - # so at this point we know it is a supported type of request.

178 - #

179 - # TODO: we still need to handle xgrammar compilation failures,

180 - # though it should be unlikely as we test that up front as well.

181 - request_type, grammar_spec = key

182 -

183 - assert self.backend is not None

184 - return self.backend.compile_grammar(request_type, grammar_spec)

182 + # so at this point we know it is a supported type of request. Grammar

183 + # compilation may still fail; the Future carries that error to the

184 + # scheduler so it can fail only this request.

185 + try:

186 + request_type, grammar_spec = struct_request.structured_output_key

187 + assert self.backend is not None

188 + return self.backend.compile_grammar(request_type, grammar_spec)

189 + except Exception:

190 + logger.exception(

191 + "Failed to compile grammar for request %s", request.request_id

192 + )

193 + raise

194

195 def _fill_bitmasks(

196 self, batch: Iterable[tuple[StructuredOutputGrammar, int, bool]]

@@ -244,8 +253,9 @@ class StructuredOutputManager:

253 structured_output_request = request.structured_output_request

254 if TYPE_CHECKING:

255 assert structured_output_request is not None

247 - assert structured_output_request.grammar is not None

256 grammar = structured_output_request.grammar

257 + if TYPE_CHECKING:

258 + assert isinstance(grammar, StructuredOutputGrammar)

259

260 apply_bitmask = self.should_fill_bitmask(request)

261 batch.append((grammar, cumulative_index, apply_bitmask))

@@ -268,8 +278,9 @@ class StructuredOutputManager:

278

279 if TYPE_CHECKING:

280 assert structured_output_request is not None

271 - assert structured_output_request.grammar is not None

281 grammar = structured_output_request.grammar

282 + if TYPE_CHECKING:

283 + assert isinstance(grammar, StructuredOutputGrammar)

284 apply_bitmask = self.should_fill_bitmask(request)

285

286 reasoner = self._get_reasoner(request)

vllm/v1/structured_output/request.py

+9 -6

@@ -21,7 +21,9 @@ if TYPE_CHECKING:

21 @dataclasses.dataclass

22 class StructuredOutputRequest:

23 params: StructuredOutputsParams

24 - _grammar: Future[StructuredOutputGrammar] | StructuredOutputGrammar | None = None

24 + _grammar: (

25 + Future[StructuredOutputGrammar] | StructuredOutputGrammar | Exception | None

26 + ) = None

27 reasoning_ended: bool | None = None

28 # Absolute index into the request's all_token_ids of the last reasoning

29 # token (the reasoning-end marker). Tokens at or before this index are

@@ -52,6 +54,8 @@ class StructuredOutputRequest:

54 self._grammar = self._grammar.result(timeout=0.0001)

55 except TimeoutError:

56 return False

57 + except Exception as e:

58 + self._grammar = e

59 return True

61 @property

@@ -59,11 +63,10 @@ class StructuredOutputRequest:

63 return self._check_grammar_completion()

65 @property

62 - def grammar(self) -> StructuredOutputGrammar | None:

63 - completed = self._check_grammar_completion()

64 - return (

65 - cast(StructuredOutputGrammar | None, self._grammar) if completed else None

66 - )

66 + def grammar(self) -> StructuredOutputGrammar | Exception | None:

67 + if not self._check_grammar_completion():

68 + return None

69 + return cast(StructuredOutputGrammar | Exception | None, self._grammar)

71 @grammar.setter

72 def grammar(