Composio powers 1000+ toolkits, tool search, context management, authentication, and a sandboxed workbench to help you build AI agents that turn intent into action.

next
py 215 lines 7.02 KB
Raw
1 """Property-based acceptance parity for scalar JSON Schema conversion.
2
3 The oracle is the `jsonschema` Draft 7 validator: for every generated scalar
4 schema and instance, the converted Pydantic artifacts must agree with it.
5
6 The generator deliberately stays inside the semantics the converters claim to
7 support today; shrinking an exclusion is the way to turn a fixed behavior into
8 a permanent regression guard. The remaining exclusions are deliberate
9 divergences or oracle limits, not open bugs:
10
11 - ``multipleOf`` is always an integer: the converters check decimal multiples
12 through decimal scaling (see the ``primitive-float-multiple-of`` corpus
13 case), while the oracle uses raw float modulo
14 - draft-4 boolean ``exclusiveMinimum``/``exclusiveMaximum`` are never
15 generated: they are not Draft 7, so the oracle cannot express them (the
16 ``primitive-draft4-boolean-exclusive-bounds`` corpus case covers them)
17 """
18
19 import typing as t
20
21 import pytest
22 from hypothesis import HealthCheck, given, settings
23 from hypothesis import strategies as st
24 from jsonschema import Draft7Validator
25 from pydantic import TypeAdapter, ValidationError
26
27 from composio.utils.schema_converter import json_schema_to_pydantic_type
28 from composio.utils.shared import pydantic_model_from_param_schema
29
30 SCALAR_TYPES = ("string", "integer", "number", "boolean", "null")
31
32 PATTERN_POOL = ("^[a-z]+$", "^[A-Z]{2}$", "[0-9]", "^a.*z$", "^(?=.*[a-y])a")
33
34 scalar_values = st.one_of(
35 st.none(),
36 st.booleans(),
37 st.integers(min_value=-100, max_value=100),
38 st.floats(
39 min_value=-100,
40 max_value=100,
41 allow_nan=False,
42 allow_infinity=False,
43 ),
44 st.text(
45 alphabet=st.characters(min_codepoint=97, max_codepoint=122),
46 max_size=6,
47 ),
48 )
49
50
51 @st.composite
52 def scalar_schemas(draw: st.DrawFn) -> dict[str, t.Any]:
53 schema: dict[str, t.Any] = {}
54 declared: tuple[str, ...] = ()
55
56 if draw(st.booleans()):
57 members = draw(
58 st.lists(st.sampled_from(SCALAR_TYPES), min_size=1, max_size=3, unique=True)
59 )
60 if len(members) == 1 and draw(st.booleans()):
61 schema["type"] = members[0]
62 declared = (members[0],)
63 else:
64 schema["type"] = members
65 declared = tuple(members)
66
67 literal_kind = draw(st.sampled_from(("none", "enum", "const", "both")))
68 if literal_kind in ("enum", "both"):
69 schema["enum"] = draw(
70 st.lists(scalar_values, min_size=1, max_size=4, unique_by=repr)
71 )
72 if literal_kind in ("const", "both"):
73 schema["const"] = draw(scalar_values)
74
75 # Constraints attach independently of the declared type (or its absence):
76 # Draft 7 scopes every scalar keyword to matching instance types anyway.
77 del declared
78 if draw(st.booleans()):
79 schema["minLength"] = draw(st.integers(min_value=0, max_value=4))
80 if draw(st.booleans()):
81 schema["maxLength"] = draw(st.integers(min_value=0, max_value=6))
82 if draw(st.booleans()):
83 schema["pattern"] = draw(st.sampled_from(PATTERN_POOL))
84 if draw(st.booleans()):
85 schema["minimum"] = draw(st.integers(min_value=-20, max_value=20))
86 if draw(st.booleans()):
87 schema["maximum"] = draw(st.integers(min_value=-20, max_value=20))
88 if draw(st.booleans()):
89 schema["exclusiveMinimum"] = draw(st.integers(min_value=-20, max_value=20))
90 if draw(st.booleans()):
91 schema["exclusiveMaximum"] = draw(st.integers(min_value=-20, max_value=20))
92 if draw(st.booleans()):
93 schema["multipleOf"] = draw(st.integers(min_value=1, max_value=5))
94
95 return schema
96
97
98 @st.composite
99 def schema_and_instance(draw: st.DrawFn) -> tuple[dict[str, t.Any], t.Any]:
100 schema = draw(scalar_schemas())
101
102 pools: list[st.SearchStrategy[t.Any]] = [scalar_values]
103 interesting: list[t.Any] = []
104 if "enum" in schema:
105 interesting.extend(schema["enum"])
106 if "const" in schema:
107 interesting.append(schema["const"])
108 for keyword in ("minimum", "maximum", "exclusiveMinimum", "exclusiveMaximum"):
109 if keyword in schema:
110 bound = schema[keyword]
111 interesting.extend((bound, bound + 1, bound - 1))
112 if interesting:
113 pools.append(st.sampled_from(interesting))
114
115 return schema, draw(st.one_of(pools))
116
117
118 def _oracle_accepts(object_schema: dict[str, t.Any], instance: t.Any) -> bool:
119 return Draft7Validator(object_schema).is_valid(instance)
120
121
122 def _pydantic_accepts(annotation: t.Any, instance: t.Any) -> bool:
123 try:
124 TypeAdapter(annotation).validate_python(instance)
125 return True
126 except ValidationError:
127 return False
128
129
130 def _wrap(schema: dict[str, t.Any]) -> dict[str, t.Any]:
131 return {
132 "type": "object",
133 "properties": {"value": schema},
134 "required": ["value"],
135 "title": "PropertyCase",
136 }
137
138
139 @pytest.mark.unit
140 @pytest.mark.schema
141 @settings(
142 max_examples=200,
143 deadline=None,
144 suppress_health_check=[HealthCheck.too_slow],
145 )
146 @given(schema_and_instance())
147 def test_object_conversion_matches_draft7_oracle(
148 case: tuple[dict[str, t.Any], t.Any],
149 ) -> None:
150 """`json_schema_to_pydantic_type` on an object schema must agree with Draft 7."""
151 schema, value = case
152 object_schema = _wrap(schema)
153 instance = {"value": value}
154
155 expected = _oracle_accepts(object_schema, instance)
156 actual = _pydantic_accepts(json_schema_to_pydantic_type(object_schema), instance)
157
158 assert actual == expected, (
159 f"object path disagreed with Draft 7 oracle for schema={schema!r} "
160 f"value={value!r}: oracle={expected} converted={actual}"
161 )
162
163
164 @pytest.mark.unit
165 @pytest.mark.schema
166 @settings(
167 max_examples=200,
168 deadline=None,
169 suppress_health_check=[HealthCheck.too_slow],
170 )
171 @given(schema_and_instance())
172 def test_legacy_model_matches_draft7_oracle(
173 case: tuple[dict[str, t.Any], t.Any],
174 ) -> None:
175 """`pydantic_model_from_param_schema` enforces exact Draft 7 acceptance."""
176 schema, value = case
177 object_schema = _wrap(schema)
178 instance = {"value": value}
179
180 expected = _oracle_accepts(object_schema, instance)
181 model = pydantic_model_from_param_schema(object_schema)
182 try:
183 model.model_validate(instance)
184 actual = True
185 except ValidationError:
186 actual = False
187
188 assert actual == expected, (
189 f"legacy path disagreed with Draft 7 oracle for schema={schema!r} "
190 f"value={value!r}: oracle={expected} converted={actual}"
191 )
192
193
194 @pytest.mark.unit
195 @pytest.mark.schema
196 @settings(
197 max_examples=200,
198 deadline=None,
199 suppress_health_check=[HealthCheck.too_slow],
200 )
201 @given(schema_and_instance())
202 def test_toplevel_conversion_never_rejects_draft7_valid_input(
203 case: tuple[dict[str, t.Any], t.Any],
204 ) -> None:
205 """The top-level scalar path may coerce, but must not over-reject."""
206 schema, value = case
207
208 if not Draft7Validator(schema).is_valid(value):
209 return
210
211 annotation = json_schema_to_pydantic_type(schema)
212 assert _pydantic_accepts(annotation, value), (
213 f"top-level path rejected a Draft 7-valid instance for schema={schema!r} "
214 f"value={value!r}"
215 )