Composio powers 1000+ toolkits, tool search, context management, authentication, and a sandboxed workbench to help you build AI agents that turn intent into action.

next
py 440 lines 13.4 KB
Raw
1 """Regression matrix for whole-schema JSON Schema acceptance.
2
3 These cases combine keywords that previously took different conversion paths.
4 Every fixture is checked against the Draft 7 oracle before the public Pydantic
5 entry point, so a passing test cannot merely preserve two matching mistakes.
6 """
7
8 import json
9 import threading
10 import time
11 import typing as t
12 from http.server import BaseHTTPRequestHandler, HTTPServer
13
14 import pytest
15 from jsonschema import Draft7Validator
16 from pydantic import TypeAdapter, ValidationError
17
18 from composio.utils.schema_converter import json_schema_to_pydantic_type
19 from composio.utils.shared import (
20 json_schema_to_model,
21 pydantic_model_from_param_schema,
22 validate_and_serialize_tool_arguments,
23 )
24
25 REJECTION_CASES: tuple[tuple[str, dict[str, t.Any], t.Any], ...] = (
26 (
27 "typeless scalar dispatch must not discard object assertions",
28 {"minLength": 2, "required": ["x"]},
29 {},
30 ),
31 (
32 "anyOf must compose with adjacent assertions",
33 {
34 "type": "string",
35 "minLength": 2,
36 "anyOf": [{"const": "a"}, {"const": "abc"}],
37 },
38 "a",
39 ),
40 (
41 "allOf must compose with adjacent assertions",
42 {"type": "string", "maxLength": 1, "allOf": [{"pattern": "^ab"}]},
43 "ab",
44 ),
45 (
46 "oneOf must compose with adjacent assertions",
47 {
48 "type": "number",
49 "minimum": 10,
50 "oneOf": [{"maximum": 5}, {"minimum": 0}],
51 },
52 7,
53 ),
54 (
55 "not must compose with adjacent assertions",
56 {"type": "string", "minLength": 2, "not": {"const": "x"}},
57 "a",
58 ),
59 (
60 "if and then are a complete conditional without else",
61 {"if": {"type": "string"}, "then": {"minLength": 2}},
62 "a",
63 ),
64 (
65 "conditionals must compose with adjacent primitive assertions",
66 {
67 "type": "number",
68 "if": {"minimum": 0},
69 "then": {"maximum": 10},
70 "else": {"minimum": -10},
71 },
72 20,
73 ),
74 (
75 "contains must be enforced for typed arrays",
76 {"type": "array", "contains": {"const": 1}},
77 [2],
78 ),
79 (
80 "uniqueItems must be enforced for typed arrays",
81 {"type": "array", "uniqueItems": True},
82 [1, 1],
83 ),
84 (
85 "minItems must be enforced for typed arrays",
86 {"type": "array", "minItems": 2},
87 [1],
88 ),
89 (
90 "maxItems must be enforced for typed arrays",
91 {"type": "array", "maxItems": 1},
92 [1, 2],
93 ),
94 (
95 "propertyNames must be enforced for typed objects",
96 {"type": "object", "propertyNames": {"pattern": "^x"}},
97 {"bad": 1},
98 ),
99 (
100 "required must be enforced for property-less typed objects",
101 {"type": "object", "required": ["x"]},
102 {},
103 ),
104 (
105 "local references must constrain direct converter callers",
106 {
107 "$defs": {"positive": {"type": "number", "minimum": 1}},
108 "$ref": "#/$defs/positive",
109 },
110 0,
111 ),
112 )
113
114
115 @pytest.mark.unit
116 @pytest.mark.schema
117 @pytest.mark.parametrize(
118 ("name", "schema", "value"),
119 REJECTION_CASES,
120 ids=[case[0] for case in REJECTION_CASES],
121 )
122 def test_whole_schema_rejections_match_draft7(
123 name: str,
124 schema: dict[str, t.Any],
125 value: t.Any,
126 ) -> None:
127 del name
128 Draft7Validator.check_schema(schema)
129 assert not Draft7Validator(schema).is_valid(value)
130
131 adapter = TypeAdapter(json_schema_to_pydantic_type(schema))
132 with pytest.raises(ValidationError):
133 adapter.validate_python(value)
134
135
136 @pytest.mark.unit
137 @pytest.mark.schema
138 @pytest.mark.parametrize("literal_keyword", ["const", "enum"])
139 @pytest.mark.parametrize("value", [{}, {"x": "fixed"}])
140 def test_typed_object_compound_literals_do_not_fall_back_to_string(
141 literal_keyword: str,
142 value: dict[str, t.Any],
143 ) -> None:
144 literal = value if literal_keyword == "const" else [value]
145 schema = {
146 "type": "object",
147 "properties": {"x": {"type": "string", "default": "generated"}},
148 literal_keyword: literal,
149 }
150 Draft7Validator.check_schema(schema)
151
152 adapter = TypeAdapter(json_schema_to_pydantic_type(schema))
153 result = adapter.validate_python(value)
154 assert adapter.dump_python(result, mode="json") == value
155 assert adapter.json_schema() == schema
156 with pytest.raises(ValidationError):
157 adapter.validate_python({"x": "wrong"})
158
159 model = json_schema_to_model(
160 {"type": "object", "properties": {"payload": schema}, "required": ["payload"]}
161 )
162 assert validate_and_serialize_tool_arguments(model, {"payload": value}) == {
163 "payload": value
164 }
165 assert (
166 model.model_json_schema()["properties"]["payload"][literal_keyword] == literal
167 )
168
169
170 @pytest.mark.unit
171 @pytest.mark.schema
172 @pytest.mark.parametrize(
173 ("schema", "value"),
174 [
175 (
176 {
177 "type": "array",
178 "minimum": 0,
179 "exclusiveMinimum": True,
180 "contains": {"const": 1},
181 },
182 [2],
183 ),
184 (
185 {
186 "type": "object",
187 "minimum": 0,
188 "exclusiveMinimum": True,
189 "propertyNames": {"pattern": "^x"},
190 },
191 {"bad": 1},
192 ),
193 ],
194 ids=["contains", "property-names"],
195 )
196 def test_draft4_bounds_do_not_disable_draft7_siblings(
197 schema: dict[str, t.Any],
198 value: t.Any,
199 ) -> None:
200 adapter = TypeAdapter(json_schema_to_pydantic_type(schema))
201 with pytest.raises(ValidationError):
202 adapter.validate_python(value)
203
204
205 @pytest.mark.unit
206 @pytest.mark.schema
207 def test_scalar_allof_acceptance_survives_pydantic_materialization() -> None:
208 schema = {
209 "title": "ScalarAllOf",
210 "type": "object",
211 "properties": {
212 "value": {
213 "allOf": [
214 {"type": "string", "minLength": 2},
215 {"type": "string", "pattern": "^[A-Z]+$"},
216 ]
217 }
218 },
219 "required": ["value"],
220 }
221 Draft7Validator.check_schema(schema)
222 assert Draft7Validator(schema).is_valid({"value": "OK"})
223
224 annotations = (
225 json_schema_to_pydantic_type(schema),
226 json_schema_to_model(schema),
227 pydantic_model_from_param_schema(schema),
228 )
229 for annotation in annotations:
230 TypeAdapter(annotation).validate_python({"value": "OK"})
231
232
233 @pytest.mark.unit
234 @pytest.mark.schema
235 @pytest.mark.parametrize(
236 ("schema", "value"),
237 [
238 ({"properties": {"x": {"type": "string"}}}, 42),
239 ({"required": ["x"]}, 42),
240 ({"items": {"type": "string"}}, 42),
241 ({"minItems": 2}, {"not": "an array"}),
242 ({"propertyNames": {"pattern": "^x"}}, 42),
243 ({"dependencies": {"a": ["b"]}}, 42),
244 ({"not": {"type": "string"}}, 42),
245 ({"if": {"type": "string"}, "then": {"minLength": 2}}, 42),
246 ],
247 )
248 def test_typeless_assertions_accept_nonmatching_instance_types(
249 schema: dict[str, t.Any],
250 value: t.Any,
251 ) -> None:
252 Draft7Validator.check_schema(schema)
253 assert Draft7Validator(schema).is_valid(value)
254 TypeAdapter(json_schema_to_pydantic_type(schema)).validate_python(value)
255
256
257 @pytest.mark.unit
258 @pytest.mark.schema
259 def test_direct_local_ref_accepts_a_valid_referenced_value() -> None:
260 schema = {
261 "$defs": {"positive": {"type": "number", "minimum": 1}},
262 "$ref": "#/$defs/positive",
263 }
264 Draft7Validator.check_schema(schema)
265 assert Draft7Validator(schema).is_valid(2)
266 TypeAdapter(json_schema_to_pydantic_type(schema)).validate_python(2)
267
268
269 @pytest.mark.unit
270 @pytest.mark.schema
271 def test_nested_typeless_object_assertion_accepts_a_non_object_property() -> None:
272 schema = {
273 "title": "NestedTypelessObject",
274 "type": "object",
275 "properties": {
276 "value": {"properties": {"x": {"type": "string"}}},
277 },
278 "required": ["value"],
279 }
280 value = {"value": 42}
281 Draft7Validator.check_schema(schema)
282 assert Draft7Validator(schema).is_valid(value)
283
284 annotations = (
285 json_schema_to_pydantic_type(schema),
286 json_schema_to_model(schema),
287 pydantic_model_from_param_schema(schema),
288 )
289 for annotation in annotations:
290 TypeAdapter(annotation).validate_python(value)
291
292
293 @pytest.mark.unit
294 @pytest.mark.schema
295 @pytest.mark.parametrize("entry_point", ("pydantic_type", "model", "param_schema"))
296 def test_external_ref_is_refused_without_network_io(entry_point: str) -> None:
297 """A remote `$ref` must never make the validator fetch over the network."""
298 requests: list[str] = []
299
300 class Handler(BaseHTTPRequestHandler):
301 def do_GET(self) -> None: # noqa: N802 - http.server API
302 requests.append(self.path)
303 body = json.dumps({"type": "string"}).encode()
304 self.send_response(200)
305 self.send_header("Content-Type", "application/json")
306 self.send_header("Content-Length", str(len(body)))
307 self.end_headers()
308 self.wfile.write(body)
309
310 def log_message(self, *args: t.Any) -> None:
311 pass
312
313 server = HTTPServer(("127.0.0.1", 0), Handler)
314 thread = threading.Thread(target=server.serve_forever, daemon=True)
315 thread.start()
316 try:
317 url = f"http://127.0.0.1:{server.server_address[1]}/schema"
318 schema = {
319 "title": "ExternalRef",
320 "type": "object",
321 "properties": {"value": {"$ref": url}},
322 "required": ["value"],
323 }
324 converters: t.Dict[str, t.Callable[..., t.Any]] = {
325 "pydantic_type": json_schema_to_pydantic_type,
326 "model": json_schema_to_model,
327 "param_schema": pydantic_model_from_param_schema,
328 }
329 convert = converters[entry_point]
330 with pytest.raises(ValueError, match="External schema reference"):
331 annotation = convert(schema)
332 TypeAdapter(annotation).validate_python({"value": "x"})
333 finally:
334 server.shutdown()
335 server.server_close()
336 assert requests == []
337
338
339 @pytest.mark.unit
340 @pytest.mark.schema
341 def test_backtracking_pattern_rejects_in_linear_time() -> None:
342 schema = {
343 "type": "object",
344 "properties": {"value": {"type": "string", "pattern": "(a+)+$"}},
345 }
346 value = {"value": "a" * 24 + "!"}
347 assert not Draft7Validator(schema).is_valid(value)
348 model = json_schema_to_model(schema)
349
350 started = time.perf_counter()
351 with pytest.raises(ValidationError):
352 model.model_validate(value)
353 assert time.perf_counter() - started < 0.2
354
355 model.model_validate({"value": "aaa"})
356
357
358 @pytest.mark.unit
359 @pytest.mark.schema
360 @pytest.mark.parametrize("combiner", ("anyOf", "oneOf"))
361 def test_simple_union_rejects_bool_for_integer_or_null(combiner: str) -> None:
362 schema = {combiner: [{"type": "integer"}, {"type": "null"}]}
363 assert not Draft7Validator(schema).is_valid(True)
364
365 adapter = TypeAdapter(json_schema_to_pydantic_type(schema))
366 with pytest.raises(ValidationError):
367 adapter.validate_python(True)
368 assert adapter.validate_python(1) == 1
369 assert adapter.validate_python(None) is None
370
371
372 @pytest.mark.unit
373 @pytest.mark.schema
374 @pytest.mark.parametrize("combiner", ("anyOf", "oneOf"))
375 def test_toplevel_combiner_model_keeps_sibling_assertions(combiner: str) -> None:
376 schema = {
377 combiner: [{"type": "object", "properties": {"a": {"type": "string"}}}],
378 "required": ["a"],
379 }
380 assert not Draft7Validator(schema).is_valid({})
381
382 adapter = TypeAdapter(json_schema_to_pydantic_type(schema))
383 with pytest.raises(ValidationError):
384 adapter.validate_python({})
385 assert adapter.dump_python(adapter.validate_python({"a": "x"}), mode="json") == {
386 "a": "x"
387 }
388
389
390 @pytest.mark.unit
391 @pytest.mark.schema
392 def test_property_less_param_schema_keeps_assertions() -> None:
393 schema = {"title": "NonEmpty", "type": "object", "minProperties": 1}
394 assert not Draft7Validator(schema).is_valid({})
395
396 adapter = TypeAdapter(pydantic_model_from_param_schema(schema))
397 with pytest.raises(ValidationError):
398 adapter.validate_python({})
399 assert adapter.validate_python({"k": 1}) == {"k": 1}
400
401
402 @pytest.mark.unit
403 @pytest.mark.schema
404 def test_exact_validated_array_items_keep_materialized_defaults() -> None:
405 schema = {
406 "type": "object",
407 "title": "DynamicArrayArguments",
408 "patternProperties": {
409 "^items_": {
410 "type": "array",
411 "items": {
412 "type": "object",
413 "properties": {
414 "null_default": {
415 "anyOf": [{"type": "string"}, {"type": "null"}],
416 "default": None,
417 }
418 },
419 },
420 }
421 },
422 "additionalProperties": False,
423 }
424 adapter = TypeAdapter(json_schema_to_pydantic_type(schema))
425 assert adapter.dump_python(
426 adapter.validate_python({"items_a": [{}]}), mode="json"
427 ) == {"items_a": [{"null_default": None}]}
428
429
430 @pytest.mark.unit
431 @pytest.mark.schema
432 def test_property_less_param_schema_admits_non_objects() -> None:
433 schema = {"title": "Either", "anyOf": [{"type": "string"}, {"type": "object"}]}
434 assert Draft7Validator(schema).is_valid("x")
435
436 adapter = TypeAdapter(pydantic_model_from_param_schema(schema))
437 assert adapter.validate_python("x") == "x"
438 assert adapter.validate_python({"k": 1}) == {"k": 1}
439 with pytest.raises(ValidationError):
440 adapter.validate_python(1)