| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | |
| 4 | import pytest |
| 5 | from xgrammar import Grammar |
| 6 | from xgrammar.testing import _is_grammar_accept_string |
| 7 | |
| 8 | from vllm.sampling_params import SamplingParams, StructuredOutputsParams |
| 9 | from vllm.v1.structured_output.backend_xgrammar import ( |
| 10 | has_xgrammar_unsupported_json_features, |
| 11 | validate_xgrammar_grammar, |
| 12 | ) |
| 13 | from vllm.v1.structured_output.utils import choice_as_grammar |
| 14 | |
| 15 | pytestmark = pytest.mark.cpu_test |
| 16 | |
| 17 | |
| 18 | def grammar_accepts(schema: dict, text: str) -> bool: |
| 19 | return _is_grammar_accept_string(Grammar.from_json_schema(schema), text) |
| 20 | |
| 21 | |
| 22 | @pytest.fixture |
| 23 | def unsupported_string_schemas(): |
| 24 | return [ |
| 25 | {"type": "string", "format": "non_existing_format"}, |
| 26 | # TODO(arpera): |
| 27 | # pattern/format is compiled but length bounds are silently dropped, |
| 28 | # so the combination must be rejected instead of producing quietly |
| 29 | # wrong output |
| 30 | # https://github.com/mlc-ai/xgrammar/issues/749 |
| 31 | {"type": "string", "pattern": "^a+$", "maxLength": 2}, |
| 32 | {"type": "string", "pattern": "^a+$", "minLength": 3}, |
| 33 | {"type": "string", "format": "email", "maxLength": 10}, |
| 34 | ] |
| 35 | |
| 36 | |
| 37 | @pytest.fixture |
| 38 | def unsupported_integer_schemas(): |
| 39 | return [ |
| 40 | {"type": "integer", "multipleOf": 120}, |
| 41 | ] |
| 42 | |
| 43 | |
| 44 | @pytest.fixture |
| 45 | def unsupported_number_schemas(): |
| 46 | return [ |
| 47 | {"type": "number", "multipleOf": 120}, |
| 48 | ] |
| 49 | |
| 50 | |
| 51 | @pytest.fixture |
| 52 | def unsupported_array_schemas(): |
| 53 | return [ |
| 54 | {"type": "array", "uniqueItems": True}, |
| 55 | {"type": "array", "contains": {"type": "string"}}, |
| 56 | {"type": "array", "minContains": 1}, |
| 57 | {"type": "array", "maxContains": 5}, |
| 58 | ] |
| 59 | |
| 60 | |
| 61 | @pytest.fixture |
| 62 | def unsupported_property_names_combinations(): |
| 63 | return [ |
| 64 | # TODO(arpera): |
| 65 | # this case is not covered by xgrammar, so we should report this bug to xgrammar |
| 66 | # xgrammar drops propertyNames whenever patternProperties is present, |
| 67 | # so "grade_12" matches the pattern and escapes the name constraint |
| 68 | { |
| 69 | "type": "object", |
| 70 | "patternProperties": {"^grade_[0-9]+$": {"type": "integer"}}, |
| 71 | "propertyNames": {"pattern": "^grade_[0-9]$"}, # does NOT match "grade_12" |
| 72 | }, |
| 73 | # propertyNames makes xgrammar discard the sibling additionalProperties |
| 74 | # value schema: https://github.com/mlc-ai/xgrammar/issues/826 |
| 75 | { |
| 76 | "type": "object", |
| 77 | "propertyNames": {"pattern": "^[a-z]+$"}, |
| 78 | "additionalProperties": {"type": "integer"}, |
| 79 | }, |
| 80 | # propertyNames is a string schema that conventionally omits "type", so |
| 81 | # it escapes the string check while xgrammar drops its length bound all |
| 82 | # the same: https://github.com/mlc-ai/xgrammar/issues/749 |
| 83 | { |
| 84 | "type": "object", |
| 85 | "propertyNames": {"pattern": "^a+$", "maxLength": 2}, |
| 86 | }, |
| 87 | # xgrammar emits named `properties` separately from `propertyNames` and |
| 88 | # only applies `propertyNames` in its additional-properties branch, so a |
| 89 | # key declared in `properties` escapes the name constraint regardless of |
| 90 | # what `propertyNames` contains. |
| 91 | { |
| 92 | "type": "object", |
| 93 | "properties": {"Bad": {"type": "integer"}}, |
| 94 | "required": ["Bad"], |
| 95 | "propertyNames": {"pattern": "^[a-z]+$"}, |
| 96 | }, |
| 97 | { |
| 98 | "type": "object", |
| 99 | "properties": {"Bad": {"type": "integer"}}, |
| 100 | "propertyNames": {"enum": ["good"]}, |
| 101 | }, |
| 102 | # xgrammar's propertyNames-only branch falls back to an unconstrained |
| 103 | # value type, so it drops unevaluatedProperties whether it restricts |
| 104 | # values (a schema) or forbids them (false). |
| 105 | { |
| 106 | "type": "object", |
| 107 | "propertyNames": {"pattern": "^[a-z]+$"}, |
| 108 | "unevaluatedProperties": {"type": "integer"}, |
| 109 | }, |
| 110 | { |
| 111 | "type": "object", |
| 112 | "propertyNames": {"pattern": "^[a-z]+$"}, |
| 113 | "unevaluatedProperties": False, |
| 114 | }, |
| 115 | ] |
| 116 | |
| 117 | |
| 118 | @pytest.fixture |
| 119 | def unsupported_pattern_properties_combinations(): |
| 120 | return [ |
| 121 | # JSON Schema requires a property matched by both `properties` and |
| 122 | # `patternProperties` to satisfy both (conjunction), but xgrammar |
| 123 | # compiles them as alternative branches, so satisfying either one is |
| 124 | # enough. |
| 125 | { |
| 126 | "type": "object", |
| 127 | "properties": {"x": {"type": "string"}}, |
| 128 | "patternProperties": {"^x$": {"type": "integer"}}, |
| 129 | }, |
| 130 | # The same alternative-branches problem applies to overlapping |
| 131 | # patternProperties patterns: a key matching both patterns only has to |
| 132 | # satisfy one of them. |
| 133 | { |
| 134 | "type": "object", |
| 135 | "patternProperties": { |
| 136 | "^a[a-z]*$": {"type": "string"}, |
| 137 | "^[a-z]*z$": {"type": "integer"}, |
| 138 | }, |
| 139 | }, |
| 140 | ] |
| 141 | |
| 142 | |
| 143 | @pytest.fixture |
| 144 | def supported_frankenstein_schema(): |
| 145 | # IMPORTANT(arpera): |
| 146 | # Do NOT add more keywords here! This schema is overcrowded enough that a new |
| 147 | # keyword can end up having no effect on the compiled grammar while the |
| 148 | # test still passes. Give a new keyword its own fixture and named test. |
| 149 | return { |
| 150 | "type": "object", |
| 151 | "properties": { |
| 152 | "name": {"type": "string"}, |
| 153 | "age": {"type": "integer"}, |
| 154 | "email": {"type": "string", "format": "email"}, |
| 155 | "status": {"type": "string"}, |
| 156 | "scores": {"type": "array", "items": {"type": "number"}}, |
| 157 | "car_type": {"type": "string", "enum": ["sedan", "suv", "truck"]}, |
| 158 | "car_brand": {"type": "string", "pattern": "^[a-zA-Z]+$"}, |
| 159 | "short_description": {"type": "string", "maxLength": 50}, |
| 160 | "mileage": {"type": "number", "minimum": 0, "maximum": 1000000}, |
| 161 | "model_year": { |
| 162 | "type": "integer", |
| 163 | "exclusiveMinimum": 1900, |
| 164 | "exclusiveMaximum": 2100, |
| 165 | }, |
| 166 | "long_description": {"type": "string", "minLength": 50, "maxLength": 2000}, |
| 167 | "address": { |
| 168 | "type": "object", |
| 169 | "properties": { |
| 170 | "street": {"type": "string"}, |
| 171 | "city": {"type": "string"}, |
| 172 | }, |
| 173 | }, |
| 174 | }, |
| 175 | "minProperties": 1, |
| 176 | "maxProperties": 100, |
| 177 | } |
| 178 | |
| 179 | |
| 180 | @pytest.fixture |
| 181 | def property_names_schema(): |
| 182 | return {"type": "object", "propertyNames": {"pattern": "^[a-z_]+$"}} |
| 183 | |
| 184 | |
| 185 | @pytest.fixture |
| 186 | def pattern_properties_schema(): |
| 187 | return { |
| 188 | "type": "object", |
| 189 | "patternProperties": {"^grade_[0-9]+$": {"type": "integer"}}, |
| 190 | } |
| 191 | |
| 192 | |
| 193 | class TestHasXGrammarUnsupportedJsonFeatures: |
| 194 | @pytest.mark.parametrize( |
| 195 | "schema_type", |
| 196 | [ |
| 197 | "unsupported_string_schemas", |
| 198 | "unsupported_integer_schemas", |
| 199 | "unsupported_number_schemas", |
| 200 | "unsupported_array_schemas", |
| 201 | "unsupported_property_names_combinations", |
| 202 | "unsupported_pattern_properties_combinations", |
| 203 | ], |
| 204 | ) |
| 205 | def test_unsupported_json_features_by_type(self, schema_type, request): |
| 206 | schemas = request.getfixturevalue(schema_type) |
| 207 | for schema in schemas: |
| 208 | assert has_xgrammar_unsupported_json_features(schema), ( |
| 209 | f"Schema should be unsupported: {schema}" |
| 210 | ) |
| 211 | |
| 212 | @pytest.mark.parametrize( |
| 213 | "schema_type", |
| 214 | [ |
| 215 | "supported_frankenstein_schema", |
| 216 | "pattern_properties_schema", |
| 217 | "property_names_schema", |
| 218 | ], |
| 219 | ) |
| 220 | def test_supported_json_features(self, schema_type, request): |
| 221 | schema = request.getfixturevalue(schema_type) |
| 222 | assert not has_xgrammar_unsupported_json_features(schema), ( |
| 223 | f"Schema should be supported: {schema}" |
| 224 | ) |
| 225 | |
| 226 | class TestPR48416Regressions: |
| 227 | @pytest.mark.parametrize( |
| 228 | "schema", |
| 229 | [ |
| 230 | {"type": ["number"], "multipleOf": 3}, |
| 231 | { |
| 232 | "type": ["string", "null"], |
| 233 | "pattern": "^a+$", |
| 234 | "maxLength": 2, |
| 235 | }, |
| 236 | { |
| 237 | "type": ["array", "null"], |
| 238 | "items": {"type": "integer"}, |
| 239 | "uniqueItems": True, |
| 240 | }, |
| 241 | { |
| 242 | "type": ["object", "null"], |
| 243 | "properties": {"Bad": {"type": "integer"}}, |
| 244 | "propertyNames": {"pattern": "^[a-z]+$"}, |
| 245 | }, |
| 246 | ], |
| 247 | ) |
| 248 | def test_list_type_does_not_bypass_unsupported_feature_check(self, schema): |
| 249 | assert has_xgrammar_unsupported_json_features(schema) |
| 250 | |
| 251 | @pytest.mark.parametrize( |
| 252 | "schema", |
| 253 | [ |
| 254 | {"type": ["string", "null"]}, |
| 255 | {"type": ["integer", "string"]}, |
| 256 | {"type": ["array", "null"], "items": {"type": "string"}}, |
| 257 | { |
| 258 | "type": ["object", "null"], |
| 259 | "propertyNames": {"pattern": "^[a-z_]+$"}, |
| 260 | }, |
| 261 | { |
| 262 | "type": ["object", "null"], |
| 263 | "patternProperties": {"^S": {"type": "string"}}, |
| 264 | }, |
| 265 | ], |
| 266 | ) |
| 267 | def test_supported_list_type_json_features(self, schema): |
| 268 | assert not has_xgrammar_unsupported_json_features(schema) |
| 269 | |
| 270 | |
| 271 | class TestIsGrammarAcceptString: |
| 272 | class TestPR42904Support: |
| 273 | def test_property_names_constrains_keys(self, property_names_schema): |
| 274 | assert grammar_accepts(property_names_schema, '{"score": 5}') |
| 275 | assert grammar_accepts(property_names_schema, '{"score": "seven"}') |
| 276 | assert not grammar_accepts(property_names_schema, '{"Score": 5}') |
| 277 | assert not grammar_accepts(property_names_schema, '{"score_1": 5}') |
| 278 | |
| 279 | def test_pattern_properties_constrains_keys_and_values( |
| 280 | self, pattern_properties_schema |
| 281 | ): |
| 282 | assert grammar_accepts(pattern_properties_schema, '{"grade_1": 5}') |
| 283 | assert not grammar_accepts(pattern_properties_schema, '{"other": 5}') |
| 284 | assert not grammar_accepts(pattern_properties_schema, '{"grade_1": "five"}') |
| 285 | |
| 286 | class TestPR48115Regressions: |
| 287 | @pytest.mark.parametrize("codepoint", [*range(0x20), 0x7F]) |
| 288 | def test_choice_as_grammar_preserves_control_characters(self, codepoint): |
| 289 | choice = f"a{chr(codepoint)}f" |
| 290 | grammar = Grammar.from_ebnf(choice_as_grammar([choice])) |
| 291 | |
| 292 | assert _is_grammar_accept_string(grammar, choice) |
| 293 | assert not _is_grammar_accept_string(grammar, "a") |
| 294 | assert not _is_grammar_accept_string(grammar, "af") |
| 295 | assert not _is_grammar_accept_string(grammar, f"a\\u{codepoint:04x}f") |
| 296 | |
| 297 | @pytest.mark.parametrize( |
| 298 | ("choice", "other"), |
| 299 | [ |
| 300 | (r"a\nf", "a\nf"), |
| 301 | ('a quote " and a backslash \\', 'a quote " and a backslash '), |
| 302 | ("café", "cafe"), |
| 303 | ("日本語", "日本"), |
| 304 | ("😀", "😁"), |
| 305 | ], |
| 306 | ) |
| 307 | def test_choice_as_grammar_preserves_literal_choices(self, choice, other): |
| 308 | grammar = Grammar.from_ebnf(choice_as_grammar(["yes", choice])) |
| 309 | |
| 310 | assert _is_grammar_accept_string(grammar, "yes") |
| 311 | assert _is_grammar_accept_string(grammar, choice) |
| 312 | assert not _is_grammar_accept_string(grammar, other) |
| 313 | assert not _is_grammar_accept_string(grammar, choice + "extra") |
| 314 | |
| 315 | def test_validate_xgrammar_preserves_multiline_choice(self): |
| 316 | structured_outputs = StructuredOutputsParams(choice=["yes", "no\nplease"]) |
| 317 | validate_xgrammar_grammar( |
| 318 | SamplingParams(structured_outputs=structured_outputs) |
| 319 | ) |
| 320 | |
| 321 | assert structured_outputs.choice is None |
| 322 | grammar = Grammar.from_ebnf(structured_outputs.grammar) |
| 323 | assert _is_grammar_accept_string(grammar, "yes") |
| 324 | assert _is_grammar_accept_string(grammar, "no\nplease") |
| 325 | assert not _is_grammar_accept_string(grammar, r"no\nplease") |