| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | """Request-time validation of structured output requests.""" |
| 4 | |
| 5 | import pytest |
| 6 | |
| 7 | from vllm.config import StructuredOutputsConfig |
| 8 | from vllm.exceptions import VLLMClientError, VLLMValidationError |
| 9 | from vllm.sampling_params import SamplingParams, StructuredOutputsParams |
| 10 | |
| 11 | pytestmark = pytest.mark.cpu_test |
| 12 | |
| 13 | JSON_SCHEMA = { |
| 14 | "type": "object", |
| 15 | "properties": { |
| 16 | "invoice_id": {"type": "string"}, |
| 17 | "customer": {"type": "string"}, |
| 18 | }, |
| 19 | "required": ["invoice_id", "customer"], |
| 20 | "additionalProperties": False, |
| 21 | } |
| 22 | |
| 23 | |
| 24 | class _StubModelConfig: |
| 25 | def __init__(self, is_diffusion: bool): |
| 26 | self.is_diffusion = is_diffusion |
| 27 | |
| 28 | |
| 29 | def test_structured_outputs_rejected_for_diffusion_models(): |
| 30 | """Diffusion LLMs denoise the canvas in parallel, which is incompatible |
| 31 | with the token-by-token grammar FSM. The request must fail with a clear |
| 32 | validation error instead of an FSM rejection mid-generation (#45436).""" |
| 33 | params = SamplingParams( |
| 34 | structured_outputs=StructuredOutputsParams(json=JSON_SCHEMA) |
| 35 | ) |
| 36 | with pytest.raises(VLLMValidationError, match="not yet supported for diffusion"): |
| 37 | params._validate_structured_outputs( |
| 38 | _StubModelConfig(is_diffusion=True), |
| 39 | StructuredOutputsConfig(), |
| 40 | tokenizer=None, |
| 41 | ) |
| 42 | |
| 43 | |
| 44 | def test_plain_request_allowed_for_diffusion_models(): |
| 45 | """Requests without structured outputs are unaffected by the guard.""" |
| 46 | params = SamplingParams() |
| 47 | params._validate_structured_outputs( |
| 48 | _StubModelConfig(is_diffusion=True), |
| 49 | StructuredOutputsConfig(), |
| 50 | tokenizer=None, |
| 51 | ) |
| 52 | |
| 53 | |
| 54 | @pytest.mark.parametrize( |
| 55 | "structured_outputs, match", |
| 56 | [ |
| 57 | (StructuredOutputsParams(json_object=False), "json_object must be True"), |
| 58 | (StructuredOutputsParams(json=""), "json cannot be an empty string"), |
| 59 | ( |
| 60 | StructuredOutputsParams(structural_tag=""), |
| 61 | "structural_tag cannot be an empty string", |
| 62 | ), |
| 63 | ], |
| 64 | ) |
| 65 | def test_degenerate_structured_outputs_rejected(structured_outputs, match): |
| 66 | """json_object=False and an empty json schema pass the `is not None` |
| 67 | exclusivity check but resolve to no structured-output key, so they must be |
| 68 | rejected at request validation (-> 400). Empty `structural_tag` is rejected |
| 69 | for the same reason: `json.loads("")` in `compile_grammar` would otherwise |
| 70 | raise JSONDecodeError and surface as a per-request engine error.""" |
| 71 | params = SamplingParams(structured_outputs=structured_outputs) |
| 72 | with pytest.raises(VLLMValidationError, match=match): |
| 73 | params._validate_structured_outputs( |
| 74 | _StubModelConfig(is_diffusion=False), |
| 75 | StructuredOutputsConfig(), |
| 76 | tokenizer=object(), |
| 77 | ) |
| 78 | |
| 79 | |
| 80 | @pytest.mark.parametrize( |
| 81 | "regex", |
| 82 | [ |
| 83 | "\x00", # a lone leading NUL |
| 84 | "\x00\x01\x02\x1f", # a NUL followed by other control chars |
| 85 | "[0-9]\x00", # an embedded NUL |
| 86 | ], |
| 87 | ) |
| 88 | def test_regex_with_nul_byte_rejected(regex): |
| 89 | """A NUL byte is never meaningful in a structured-outputs regex and is not |
| 90 | handled by xgrammar's native regex converter. It must be rejected at request |
| 91 | validation in every backend mode (a clean 400), instead of reaching that |
| 92 | native code or silently falling back to another backend in the default |
| 93 | 'auto' mode.""" |
| 94 | params = SamplingParams(structured_outputs=StructuredOutputsParams(regex=regex)) |
| 95 | |
| 96 | # Rejected before backend selection, so it is a 400 even in 'auto' mode |
| 97 | # (which would otherwise catch the error and fall back to another backend). |
| 98 | with pytest.raises(VLLMValidationError, match="NUL"): |
| 99 | params._validate_structured_outputs( |
| 100 | _StubModelConfig(is_diffusion=False), |
| 101 | StructuredOutputsConfig(), |
| 102 | tokenizer=object(), |
| 103 | ) |
| 104 | |
| 105 | # The xgrammar backend also rejects it directly (defense in depth), before |
| 106 | # the pattern reaches the native from_regex call. |
| 107 | from vllm.v1.structured_output.backend_xgrammar import validate_xgrammar_grammar |
| 108 | |
| 109 | with pytest.raises(ValueError, match="NUL"): |
| 110 | validate_xgrammar_grammar(params) |
| 111 | |
| 112 | |
| 113 | INVALID_JSON_SCHEMA = {"type": "object", "properties": {"name": {"type": "str"}}} |
| 114 | |
| 115 | |
| 116 | @pytest.mark.parametrize( |
| 117 | "backend, structured_outputs", |
| 118 | [ |
| 119 | ("xgrammar", StructuredOutputsParams(json=INVALID_JSON_SCHEMA)), |
| 120 | ("outlines", StructuredOutputsParams(json=INVALID_JSON_SCHEMA)), |
| 121 | ("auto", StructuredOutputsParams(json=INVALID_JSON_SCHEMA)), |
| 122 | ("auto", StructuredOutputsParams(json='{"type": ')), |
| 123 | ("xgrammar", StructuredOutputsParams(grammar="not a grammar")), |
| 124 | ("guidance", StructuredOutputsParams(grammar="not a grammar")), |
| 125 | ("lm-format-enforcer", StructuredOutputsParams(grammar="not a grammar")), |
| 126 | ("outlines", StructuredOutputsParams(regex="(")), |
| 127 | ("guidance", StructuredOutputsParams(structural_tag='{"nope": 1}')), |
| 128 | ], |
| 129 | ) |
| 130 | def test_unsupported_grammar_is_a_client_error(backend, structured_outputs): |
| 131 | """Only `VLLMClientError` survives `AsyncLLM.generate` untouched; anything else |
| 132 | is wrapped in `EngineGenerateError` and served as a 500 instead of a 400.""" |
| 133 | params = SamplingParams(structured_outputs=structured_outputs) |
| 134 | with pytest.raises(VLLMClientError): |
| 135 | params._validate_structured_outputs( |
| 136 | _StubModelConfig(is_diffusion=False), |
| 137 | StructuredOutputsConfig(backend=backend), |
| 138 | tokenizer=object(), |
| 139 | ) |
| 140 | |
| 141 | |
| 142 | @pytest.mark.parametrize( |
| 143 | "schema, expected_backend", |
| 144 | [ |
| 145 | # multipleOf is unsupported by xgrammar. |
| 146 | ( |
| 147 | { |
| 148 | "type": "object", |
| 149 | "properties": {"n": {"type": "integer", "multipleOf": 2}}, |
| 150 | }, |
| 151 | "guidance", |
| 152 | ), |
| 153 | ( |
| 154 | { |
| 155 | "type": "object", |
| 156 | "properties": {"n": {"type": ["number", "null"], "multipleOf": 3}}, |
| 157 | }, |
| 158 | "guidance", |
| 159 | ), |
| 160 | ( |
| 161 | { |
| 162 | "type": ["string", "null"], |
| 163 | "pattern": "^a+$", |
| 164 | "maxLength": 2, |
| 165 | }, |
| 166 | "guidance", |
| 167 | ), |
| 168 | # patternProperties + properties is also unsupported by guidance. |
| 169 | ( |
| 170 | { |
| 171 | "type": "object", |
| 172 | "properties": {"a": {"type": "string"}}, |
| 173 | "patternProperties": {"^a$": {"type": "string"}}, |
| 174 | }, |
| 175 | "outlines", |
| 176 | ), |
| 177 | ], |
| 178 | ) |
| 179 | def test_auto_backend_falls_back_on_unsupported_schema(schema, expected_backend): |
| 180 | """`auto` falls back on rejection, so it must catch what the validators raise.""" |
| 181 | params = SamplingParams(structured_outputs=StructuredOutputsParams(json=schema)) |
| 182 | params._validate_structured_outputs( |
| 183 | _StubModelConfig(is_diffusion=False), |
| 184 | StructuredOutputsConfig(backend="auto"), |
| 185 | tokenizer=object(), |
| 186 | ) |
| 187 | assert params.structured_outputs is not None |
| 188 | assert params.structured_outputs._backend == expected_backend |