@devries-cameron20 / composio Mirror · read-only
Fork 0

Composio powers 1000+ toolkits, tool search, context management, authentication, and a sandboxed workbench to help you build AI agents that turn intent into action.

Mirrored from https://github.com/Camo67/composio.git

Synced Nov 21, 03:37 UTC

next
py 222 lines 8.44 KB
Raw
1 """Property-based null semantics for strict mode.
2
3 The oracle is the `jsonschema` Draft 2020-12 validator. Strict mode makes every
4 optional parameter required and uses ``null`` to mean "omitted", so for every
5 generated optional property:
6
7 - the strict schema accepts ``null``
8 - the arguments left after :func:`omit_null_tool_arguments` validate against
9 the tool's own schema, and a ``null`` that schema accepts is kept
10 - the strict schema accepts every non-null value the tool's schema accepts,
11 and nothing more unless a ``oneOf`` was converted to ``anyOf``
12 - the rewrite is idempotent
13
14 The TypeScript counterpart is
15 ``ts/packages/core/test/utils/jsonSchema.strict.property.test.ts``; keep the
16 generators and invariants aligned.
17
18 Those generators never produce reference cycles: a cycle that consumes no input
19 makes the oracle recurse forever. Cycles are checked separately against the
20 least fixed point of the definitions, computed by plain iteration: a null is
21 accepted only when a finite chain of branches proves it.
22 """
23
24 import copy
25 import typing as t
26
27 from hypothesis import assume, given, settings
28 from hypothesis import strategies as st
29 from jsonschema import Draft202012Validator
30
31 from composio.utils.strict_schema import omit_null_tool_arguments, to_strict_json_schema
32
33 TYPES = ("string", "integer", "boolean", "null")
34 LITERALS: tuple[t.Any, ...] = ("asc", "desc", 1, True, None)
35 SAMPLES: tuple[t.Any, ...] = ("asc", "desc", "other", 1, 2, True, False)
36 DEFINITIONS: dict[str, t.Any] = {
37 "Direction": {"type": "string", "enum": ["asc", "desc"]},
38 "NullableDirection": {"enum": ["asc", None]},
39 "Text": {"type": "string"},
40 "NullableText": {"type": ["string", "null"]},
41 "Config": {"properties": {"url": {"type": "string"}}},
42 }
43 # The rewrite types this definition as an object, so the strict schema rejects
44 # the non-object values the tool's schema lets through.
45 TYPELESS_OBJECT_REF = "#/$defs/Config"
46 # Compositions strict mode supports, and the ones only the tool's schema uses.
47 STRICT_COMPOSITIONS = ("anyOf", "oneOf")
48 ALL_COMPOSITIONS = (*STRICT_COMPOSITIONS, "allOf", "not", "if")
49
50
51 @st.composite
52 def property_schemas(
53 draw: st.DrawFn, compositions: tuple[str, ...], depth: int = 0
54 ) -> dict[str, t.Any]:
55 schema: dict[str, t.Any] = {}
56 type_kind = draw(st.sampled_from(("none", "single", "list")))
57 if type_kind == "single":
58 schema["type"] = draw(st.sampled_from(TYPES))
59 elif type_kind == "list":
60 schema["type"] = draw(
61 st.lists(st.sampled_from(TYPES), min_size=1, max_size=3, unique=True)
62 )
63 literal_kind = draw(st.sampled_from(("none", "none", "enum", "const")))
64 if literal_kind == "enum":
65 schema["enum"] = draw(
66 st.lists(st.sampled_from(LITERALS), min_size=1, max_size=3, unique_by=repr)
67 )
68 elif literal_kind == "const":
69 schema["const"] = draw(st.sampled_from(LITERALS))
70 if draw(st.integers(0, 3)) == 0:
71 schema["$ref"] = "#/$defs/" + draw(st.sampled_from(sorted(DEFINITIONS)))
72 if draw(st.booleans()):
73 schema["description"] = "a parameter"
74
75 if depth >= 2:
76 return schema
77 nested = property_schemas(compositions, depth + 1)
78 keyword = draw(st.sampled_from((None, None, *compositions)))
79 if keyword in ("anyOf", "oneOf", "allOf"):
80 schema[keyword] = draw(st.lists(nested, min_size=1, max_size=3))
81 elif keyword == "not":
82 schema["not"] = draw(nested)
83 elif keyword == "if":
84 schema["if"] = draw(nested)
85 for branch in ("then", "else"):
86 if draw(st.booleans()):
87 schema[branch] = draw(nested)
88 return schema
89
90
91 def tool_schema(property_schema: dict[str, t.Any]) -> dict[str, t.Any]:
92 return {
93 "type": "object",
94 "properties": {"value": property_schema},
95 "$defs": copy.deepcopy(DEFINITIONS),
96 }
97
98
99 def mentions(node: t.Any, keyword: str, value: t.Any = None) -> bool:
100 """Whether any node carries ``keyword`` (with ``value``, when given)."""
101 if isinstance(node, list):
102 return any(mentions(item, keyword, value) for item in node)
103 if not isinstance(node, dict):
104 return False
105 if keyword in node and (value is None or node[keyword] == value):
106 return True
107 return any(mentions(child, keyword, value) for child in node.values())
108
109
110 @settings(max_examples=500, deadline=None)
111 @given(property_schemas(STRICT_COMPOSITIONS))
112 def test_null_stands_for_omission_without_changing_other_values(
113 property_schema: dict[str, t.Any],
114 ) -> None:
115 source = tool_schema(property_schema)
116 snapshot = copy.deepcopy(source)
117 result = to_strict_json_schema(source)
118 assert source == snapshot
119 assume(not result.unsupported)
120
121 original = Draft202012Validator(source)
122 strict = Draft202012Validator(result.schema)
123
124 assert strict.is_valid({"value": None})
125 sent = omit_null_tool_arguments({"value": None}, result.source)
126 assert original.is_valid(sent)
127 assert sent == ({"value": None} if original.is_valid({"value": None}) else {})
128
129 if not mentions(property_schema, "$ref", TYPELESS_OBJECT_REF):
130 for sample in SAMPLES:
131 arguments = {"value": sample}
132 if original.is_valid(arguments):
133 assert strict.is_valid(arguments)
134 elif not mentions(property_schema, "oneOf"):
135 assert not strict.is_valid(arguments)
136
137 again = to_strict_json_schema(result.schema)
138 assert again.schema == result.schema
139 assert again.changes == []
140
141
142 @settings(max_examples=500, deadline=None)
143 @given(property_schemas(ALL_COMPOSITIONS))
144 def test_null_is_kept_exactly_when_the_tool_schema_accepts_it(
145 property_schema: dict[str, t.Any],
146 ) -> None:
147 source = tool_schema(property_schema)
148 accepted = Draft202012Validator(source).is_valid({"value": None})
149 expected = {"value": None} if accepted else {}
150 assert omit_null_tool_arguments({"value": None}, source) == expected
151
152
153 LEAVES: tuple[dict[str, t.Any], ...] = ({"type": "null"}, {"type": "string"}, {})
154
155
156 @st.composite
157 def reference_graphs(draw: st.DrawFn) -> dict[str, dict[str, t.Any]]:
158 """Definitions that reference each other freely, cycles included."""
159 names = [f"d{index}" for index in range(draw(st.integers(1, 7)))]
160 references = st.lists(
161 st.sampled_from(names).map(lambda name: {"$ref": f"#/$defs/{name}"}),
162 min_size=1,
163 max_size=3,
164 )
165 definitions: dict[str, dict[str, t.Any]] = {}
166 for name in names:
167 kind = draw(st.sampled_from(("leaf", "anyOf", "anyOf", "allOf", "allOf")))
168 if kind == "leaf":
169 definitions[name] = dict(draw(st.sampled_from(LEAVES)))
170 else:
171 branches: list[dict[str, t.Any]] = draw(references)
172 if draw(st.booleans()):
173 branches.append(dict(draw(st.sampled_from(LEAVES))))
174 definitions[name] = {kind: branches}
175 return definitions
176
177
178 def least_fixed_point(definitions: dict[str, dict[str, t.Any]]) -> dict[str, bool]:
179 accepted = dict.fromkeys(definitions, False)
180
181 def holds(node: dict[str, t.Any]) -> bool:
182 if "$ref" in node:
183 return accepted[node["$ref"].rsplit("/", 1)[1]]
184 if "anyOf" in node:
185 return any(holds(branch) for branch in node["anyOf"])
186 if "allOf" in node:
187 return all(holds(branch) for branch in node["allOf"])
188 return node.get("type") != "string"
189
190 changed = True
191 while changed:
192 changed = False
193 for name, definition in definitions.items():
194 if not accepted[name] and holds(definition):
195 accepted[name] = changed = True
196 return accepted
197
198
199 @settings(max_examples=500, deadline=None)
200 @given(reference_graphs())
201 def test_cyclic_references_resolve_to_the_least_fixed_point(
202 definitions: dict[str, dict[str, t.Any]],
203 ) -> None:
204 source = {
205 "type": "object",
206 "properties": {name: {"$ref": f"#/$defs/{name}"} for name in definitions},
207 "$defs": definitions,
208 }
209 expected = least_fixed_point(definitions)
210 arguments = dict.fromkeys(definitions)
211 kept = omit_null_tool_arguments(arguments, source)
212 assert kept == {name: None for name in definitions if expected[name]}
213
214 # `allOf` makes the tool unsupported, but its properties are still widened.
215 result = to_strict_json_schema(source)
216 for name in definitions:
217 reference = {"$ref": f"#/$defs/{name}"}
218 widened = result.schema["properties"][name]
219 if expected[name]:
220 assert widened == reference
221 else:
222 assert widened == {"anyOf": [reference, {"type": "null"}]}