@tduncan / Nvidia-Optimization-Stack-VLLM / commits / 41f12a0daf

[Bugfix] Raise `VLLMValidationError` from structured output validators (#52394)

Signed-off-by: Jeffrey Wang <jeffreywang@anyscale.com> Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>

Jeffrey Wang committed Aug 16, 2026 at 01:04 UTC 41f12a0daf193c80f2e748f6a65f0735623be3a5

8 files changed +109 -30

tests/entrypoints/llm/test_struct_output_generate.py

+5 -2

@@ -13,6 +13,7 @@ from pydantic import BaseModel

14 from tests.reasoning.utils import run_reasoning_extraction

15 from vllm.config import StructuredOutputsConfig

16 +from vllm.exceptions import VLLMValidationError

17 from vllm.outputs import RequestOutput

18 from vllm.platforms import current_platform

19 from vllm.reasoning.abs_reasoning_parsers import ReasoningParserManager

@@ -326,7 +327,7 @@ def test_structured_output(

327 )

328 if backend.startswith("xgrammar"):

329 with pytest.raises(

329 - ValueError,

330 + VLLMValidationError,

331 match="The provided JSON schema contains features "

332 "not supported by xgrammar.",

333 ):

@@ -453,7 +454,9 @@ def test_structured_output(

454 max_tokens=1000,

455 structured_outputs=StructuredOutputsParams(grammar="not a grammar"),

456 )

456 - with pytest.raises(ValueError, match="Failed to convert the grammar "):

457 + with pytest.raises(

458 + VLLMValidationError, match="Failed to convert the grammar "

459 + ):

460 runner.llm.generate(

461 (

462 "Generate a sql statement that selects col_1 from "

tests/v1/structured_output/test_validation.py

+58 -1

@@ -5,7 +5,7 @@

5 import pytest

7 from vllm.config import StructuredOutputsConfig

8 -from vllm.exceptions import VLLMValidationError

8 +from vllm.exceptions import VLLMClientError, VLLMValidationError

9 from vllm.sampling_params import SamplingParams, StructuredOutputsParams

11 pytestmark = pytest.mark.cpu_test

@@ -103,3 +103,60 @@ def test_regex_with_nul_byte_rejected(regex):

103

104 with pytest.raises(ValueError, match="NUL"):

105 validate_xgrammar_grammar(params)

106 +

107 +

108 +INVALID_JSON_SCHEMA = {"type": "object", "properties": {"name": {"type": "str"}}}

109 +

110 +

111 +@pytest.mark.parametrize(

112 + "backend, structured_outputs",

113 + [

114 + ("xgrammar", StructuredOutputsParams(json=INVALID_JSON_SCHEMA)),

115 + ("outlines", StructuredOutputsParams(json=INVALID_JSON_SCHEMA)),

116 + ("auto", StructuredOutputsParams(json=INVALID_JSON_SCHEMA)),

117 + ("auto", StructuredOutputsParams(json='{"type": ')),

118 + ("xgrammar", StructuredOutputsParams(grammar="not a grammar")),

119 + ("guidance", StructuredOutputsParams(grammar="not a grammar")),

120 + ("lm-format-enforcer", StructuredOutputsParams(grammar="not a grammar")),

121 + ("outlines", StructuredOutputsParams(regex="(")),

122 + ("guidance", StructuredOutputsParams(structural_tag='{"nope": 1}')),

123 + ],

124 +)

125 +def test_unsupported_grammar_is_a_client_error(backend, structured_outputs):

126 + """Only `VLLMClientError` survives `AsyncLLM.generate` untouched; anything else

127 + is wrapped in `EngineGenerateError` and served as a 500 instead of a 400."""

128 + params = SamplingParams(structured_outputs=structured_outputs)

129 + with pytest.raises(VLLMClientError):

130 + params._validate_structured_outputs(

131 + _StubModelConfig(is_diffusion=False),

132 + StructuredOutputsConfig(backend=backend),

133 + tokenizer=object(),

134 + )

135 +

136 +

137 +@pytest.mark.parametrize(

138 + "schema, expected_backend",

139 + [

140 + # multipleOf is unsupported by xgrammar, patternProperties also by guidance.

141 + (

142 + {

143 + "type": "object",

144 + "properties": {"n": {"type": "integer", "multipleOf": 2}},

145 + },

146 + "guidance",

147 + ),

148 + (

149 + {"type": "object", "patternProperties": {"^a": {"type": "string"}}},

150 + "outlines",

151 + ),

152 + ],

153 +)

154 +def test_auto_backend_falls_back_on_unsupported_schema(schema, expected_backend):

155 + """`auto` falls back on rejection, so it must catch what the validators raise."""

156 + params = SamplingParams(structured_outputs=StructuredOutputsParams(json=schema))

157 + params._validate_structured_outputs(

158 + _StubModelConfig(is_diffusion=False),

159 + StructuredOutputsConfig(backend="auto"),

160 + tokenizer=object(),

161 + )

162 + assert params.structured_outputs._backend == expected_backend

vllm/entrypoints/openai/engine/protocol.py

+1 -1

@@ -266,7 +266,7 @@ def validate_structural_tag_payload(payload: Any, *, parameter: str) -> None:

266 structured_outputs=StructuredOutputsParams(structural_tag=payload)

267 )

268 )

269 - except (TypeError, ValueError) as exc:

269 + except (TypeError, ValueError, VLLMValidationError) as exc:

270 raise VLLMValidationError(

271 f"Invalid {parameter} structural_tag specification.",

272 parameter=parameter,

vllm/sampling_params.py

+7 -2

@@ -1092,7 +1092,7 @@ class SamplingParams(

1092 try:

1093 validate_xgrammar_grammar(self)

1094 self.structured_outputs._backend = "xgrammar"

1095 - except ValueError:

1095 + except VLLMValidationError:

1096 # The request either failed validation

1097 # or includes some jsonschema feature(s) that

1098 # are not supported in xgrammar.

@@ -1103,7 +1103,12 @@ class SamplingParams(

1103 so_params = self.structured_outputs

1104 if not skip_guidance and so_params.json:

1105 if isinstance(so_params.json, str):

1106 - schema = json_mod.loads(so_params.json)

1106 + try:

1107 + schema = json_mod.loads(so_params.json)

1108 + except json_mod.JSONDecodeError as e:

1109 + raise VLLMValidationError(

1110 + "Invalid JSON grammar specification."

1111 + ) from e

1112 else:

1113 schema = so_params.json

1114 skip_guidance = has_guidance_unsupported_json_features(schema)

vllm/v1/structured_output/backend_guidance.py

+10 -4

@@ -10,6 +10,7 @@ from typing import TYPE_CHECKING, Any

10 import torch

11 from transformers import MistralCommonBackend

13 +from vllm.exceptions import VLLMValidationError

14 from vllm.logger import init_logger

15 from vllm.sampling_params import SamplingParams

16 from vllm.utils.import_utils import LazyLoader

@@ -269,7 +270,7 @@ def serialize_guidance_grammar(

270 begin: str = s["begin"]

271 trig = next((t for t in triggers if begin.startswith(t)), None)

272 if trig is None:

272 - raise ValueError(

273 + raise VLLMValidationError(

274 f"Trigger {begin} not found in triggers {triggers}"

275 )

276 tags.append(

@@ -281,7 +282,9 @@ def serialize_guidance_grammar(

282 )

283 )

284 if not tags:

284 - raise ValueError("No structural tags found in the grammar spec.")

285 + raise VLLMValidationError(

286 + "No structural tags found in the grammar spec."

287 + )

288 return llguidance.StructTag.to_grammar(tags)

289 else:

290 logger.error(

@@ -300,7 +303,10 @@ def validate_guidance_grammar(

303 if sampling_params.structured_outputs is None:

304 return

305 tp, grm = get_structured_output_key(sampling_params.structured_outputs)

303 - guidance_grm = serialize_guidance_grammar(tp, grm)

306 + try:

307 + guidance_grm = serialize_guidance_grammar(tp, grm)

308 + except (ValueError, KeyError, TypeError) as e:

309 + raise VLLMValidationError(f"Invalid grammar specification: {e}") from e

310 err = llguidance.LLMatcher.validate_grammar(guidance_grm, tokenizer)

311 if err:

306 - raise ValueError(f"Grammar error: {err}")

312 + raise VLLMValidationError(f"Grammar error: {err}")

vllm/v1/structured_output/backend_lm_format_enforcer.py

+5 -4

@@ -9,6 +9,7 @@ from typing import TYPE_CHECKING

9 import torch

10 from transformers import PreTrainedTokenizerBase

12 +from vllm.exceptions import VLLMValidationError

13 from vllm.sampling_params import SamplingParams

14 from vllm.utils.import_utils import LazyLoader

15 from vllm.utils.torch_utils import PIN_MEMORY

@@ -166,7 +167,7 @@ def validate_structured_output_request_lm_format_enforcer(params: SamplingParams

167 so_params.regex,

168 )

169 except Exception as err:

169 - raise ValueError(

170 + raise VLLMValidationError(

171 f"Failed to compile regex for lm-format-enforcer: {err}"

172 ) from err

173 return

@@ -176,19 +177,19 @@ def validate_structured_output_request_lm_format_enforcer(params: SamplingParams

177 # make sure schema is valid json

178 json.loads(so_params.json)

179 except json.JSONDecodeError as e:

179 - raise ValueError("Invalid JSON grammar specification.") from e

180 + raise VLLMValidationError("Invalid JSON grammar specification.") from e

181 else:

182 try:

183 json.dumps(so_params.json)

184 except Exception as e:

184 - raise ValueError(

185 + raise VLLMValidationError(

186 f"Error serializing structured outputs jsonschema: {e}"

187 ) from e

188 return

189 elif so_params.choice:

190 return

191 elif so_params.grammar:

191 - raise ValueError(

192 + raise VLLMValidationError(

193 "LM Format Enforcer structured outputs backend "

194 "does not support grammar specifications"

195 )

vllm/v1/structured_output/backend_outlines.py

+13 -7

@@ -13,6 +13,7 @@ from typing import TYPE_CHECKING

13 import torch

14 from regex import escape as regex_escape

16 +from vllm.exceptions import VLLMValidationError

17 from vllm.sampling_params import SamplingParams

18 from vllm.utils.import_utils import LazyLoader

19 from vllm.utils.torch_utils import PIN_MEMORY

@@ -186,22 +187,27 @@ def validate_structured_output_request_outlines(params: SamplingParams):

187 json.loads(so_params.json)

188 schema = so_params.json

189 except json.JSONDecodeError as e:

189 - raise ValueError("Invalid JSON grammar specification.") from e

190 + raise VLLMValidationError("Invalid JSON grammar specification.") from e

191 else:

192 try:

193 schema = json.dumps(so_params.json)

194 except Exception as e:

194 - raise ValueError(

195 + raise VLLMValidationError(

196 f"Error serializing structured outputs jsonschema: {e}"

197 ) from e

197 - pattern = json_schema.build_regex_from_schema(schema)

198 + try:

199 + pattern = json_schema.build_regex_from_schema(schema)

200 + except Exception as e:

201 + raise VLLMValidationError(

202 + f"Failed to transform json schema into a regex: {e}"

203 + ) from e

204 validate_regex_is_buildable(pattern)

205 elif so_params.choice:

206 choices = [regex_escape(str(choice)) for choice in so_params.choice]

207 regex = "(" + "|".join(choices) + ")"

208 validate_regex_is_buildable(regex)

209 elif so_params.grammar:

204 - raise ValueError(

210 + raise VLLMValidationError(

211 "Outlines structured outputs backend "

212 "does not support grammar specifications"

213 )

@@ -315,19 +321,19 @@ def validate_regex_is_buildable(pattern: str) -> None:

321 parsed = sre_parse.parse(pattern)

322

323 except sre_constants.error as e:

318 - raise ValueError(f"Error parsing regex: {e}") from e

324 + raise VLLMValidationError(f"Error parsing regex: {e}") from e

325

326 try:

327 _check_unsupported(parsed)

328 except ValueError as e:

323 - raise ValueError(

329 + raise VLLMValidationError(

330 f"Regex uses unsupported feature for structured outputs: {e}. "

331 "Only basic matching constructs are supported—lookarounds, "

332 "backreferences, and unicode boundaries are not."

333 ) from e

334

335 if _prefix_needs_context(parsed):

330 - raise ValueError(

336 + raise VLLMValidationError(

337 "Regex does not have a anchored universal start state"

338 "This means that the Regex uses anchors (^) or look-arounds "

339 "in a way which requires context before any token is matched."

vllm/v1/structured_output/backend_xgrammar.py

+10 -9

@@ -8,6 +8,7 @@ from typing import TYPE_CHECKING, Any

8 import torch

10 import vllm.envs

11 +from vllm.exceptions import VLLMValidationError

12 from vllm.logger import init_logger

13 from vllm.sampling_params import SamplingParams

14 from vllm.utils.import_utils import LazyLoader

@@ -276,7 +277,7 @@ def has_xgrammar_unsupported_json_features(schema: dict[str, Any]) -> bool:

277 def validate_xgrammar_grammar(sampling_params: SamplingParams) -> None:

278 """Validate that the request is supported by structured output.

279

279 - Raises ValueError if the request is not supported.

280 + Raises VLLMValidationError if the request is not supported.

281 """

282 if sampling_params.structured_outputs is None:

283 return

@@ -298,7 +299,7 @@ def validate_xgrammar_grammar(sampling_params: SamplingParams) -> None:

299 so_params.regex,

300 )

301 except Exception as err:

301 - raise ValueError(

302 + raise VLLMValidationError(

303 f"Failed to transform regex into a grammar: {err}"

304 ) from err

305

@@ -307,7 +308,7 @@ def validate_xgrammar_grammar(sampling_params: SamplingParams) -> None:

308 try:

309 xgr.Grammar.from_ebnf(choice_grammar)

310 except Exception as err:

310 - raise ValueError(

311 + raise VLLMValidationError(

312 f"Failed to transform choices into a grammar: {err}"

313 ) from err

314 so_params.choice = None

@@ -319,19 +320,19 @@ def validate_xgrammar_grammar(sampling_params: SamplingParams) -> None:

320 try:

321 schema = json.loads(so_params.json)

322 except json.JSONDecodeError as e:

322 - raise ValueError("Invalid JSON grammar specification.") from e

323 + raise VLLMValidationError("Invalid JSON grammar specification.") from e

324 else:

325 schema = so_params.json

326

327 if has_xgrammar_unsupported_json_features(schema):

327 - raise ValueError(

328 + raise VLLMValidationError(

329 "The provided JSON schema contains features not supported by xgrammar."

330 )

331

332 try:

333 xgr.Grammar.from_json_schema(schema)

334 except Exception as err:

334 - raise ValueError(

335 + raise VLLMValidationError(

336 f"Failed to transform json schema into a grammar: {err}"

337 ) from err

338 return

@@ -342,7 +343,7 @@ def validate_xgrammar_grammar(sampling_params: SamplingParams) -> None:

343 try:

344 so_params.grammar = convert_lark_to_ebnf(so_params.grammar)

345 except ValueError as e:

345 - raise ValueError(

346 + raise VLLMValidationError(

347 "Failed to convert the grammar from Lark to EBNF. "

348 ) from e

349

@@ -351,7 +352,7 @@ def validate_xgrammar_grammar(sampling_params: SamplingParams) -> None:

352 # parse the grammar, but we aren't compiling it.

353 xgr.Grammar.from_ebnf(so_params.grammar)

354 except Exception as e:

354 - raise ValueError("Invalid grammar specification.") from e

355 + raise VLLMValidationError("Invalid grammar specification.") from e

356 return

357

358 if so_params.structural_tag:

@@ -372,4 +373,4 @@ def validate_xgrammar_grammar(sampling_params: SamplingParams) -> None:

373 else:

374 xgr.Grammar.from_structural_tag(so_params.structural_tag)

375 except Exception as e:

375 - raise ValueError("Invalid structural tag specification.") from e

376 + raise VLLMValidationError("Invalid structural tag specification.") from e