Composio powers 1000+ toolkits, tool search, context management, authentication, and a sandboxed workbench to help you build AI agents that turn intent into action.
Mirrored from https://github.com/Camo67/composio.git
Synced Nov 21, 03:37 UTC
| 1 | """Property-based null semantics for strict mode. |
| 2 | |
| 3 | The oracle is the `jsonschema` Draft 2020-12 validator. Strict mode makes every |
| 4 | optional parameter required and uses ``null`` to mean "omitted", so for every |
| 5 | generated optional property: |
| 6 | |
| 7 | - the strict schema accepts ``null`` |
| 8 | - the arguments left after :func:`omit_null_tool_arguments` validate against |
| 9 | the tool's own schema, and a ``null`` that schema accepts is kept |
| 10 | - the strict schema accepts every non-null value the tool's schema accepts, |
| 11 | and nothing more unless a ``oneOf`` was converted to ``anyOf`` |
| 12 | - the rewrite is idempotent |
| 13 | |
| 14 | The TypeScript counterpart is |
| 15 | ``ts/packages/core/test/utils/jsonSchema.strict.property.test.ts``; keep the |
| 16 | generators and invariants aligned. |
| 17 | |
| 18 | Those generators never produce reference cycles: a cycle that consumes no input |
| 19 | makes the oracle recurse forever. Cycles are checked separately against the |
| 20 | least fixed point of the definitions, computed by plain iteration: a null is |
| 21 | accepted only when a finite chain of branches proves it. |
| 22 | """ |
| 23 | |
| 24 | import copy |
| 25 | import typing as t |
| 26 | |
| 27 | from hypothesis import assume, given, settings |
| 28 | from hypothesis import strategies as st |
| 29 | from jsonschema import Draft202012Validator |
| 30 | |
| 31 | from composio.utils.strict_schema import omit_null_tool_arguments, to_strict_json_schema |
| 32 | |
| 33 | TYPES = ("string", "integer", "boolean", "null") |
| 34 | LITERALS: tuple[t.Any, ...] = ("asc", "desc", 1, True, None) |
| 35 | SAMPLES: tuple[t.Any, ...] = ("asc", "desc", "other", 1, 2, True, False) |
| 36 | DEFINITIONS: dict[str, t.Any] = { |
| 37 | "Direction": {"type": "string", "enum": ["asc", "desc"]}, |
| 38 | "NullableDirection": {"enum": ["asc", None]}, |
| 39 | "Text": {"type": "string"}, |
| 40 | "NullableText": {"type": ["string", "null"]}, |
| 41 | "Config": {"properties": {"url": {"type": "string"}}}, |
| 42 | } |
| 43 | # The rewrite types this definition as an object, so the strict schema rejects |
| 44 | # the non-object values the tool's schema lets through. |
| 45 | TYPELESS_OBJECT_REF = "#/$defs/Config" |
| 46 | # Compositions strict mode supports, and the ones only the tool's schema uses. |
| 47 | STRICT_COMPOSITIONS = ("anyOf", "oneOf") |
| 48 | ALL_COMPOSITIONS = (*STRICT_COMPOSITIONS, "allOf", "not", "if") |
| 49 | |
| 50 | |
| 51 | @st.composite |
| 52 | def property_schemas( |
| 53 | draw: st.DrawFn, compositions: tuple[str, ...], depth: int = 0 |
| 54 | ) -> dict[str, t.Any]: |
| 55 | schema: dict[str, t.Any] = {} |
| 56 | type_kind = draw(st.sampled_from(("none", "single", "list"))) |
| 57 | if type_kind == "single": |
| 58 | schema["type"] = draw(st.sampled_from(TYPES)) |
| 59 | elif type_kind == "list": |
| 60 | schema["type"] = draw( |
| 61 | st.lists(st.sampled_from(TYPES), min_size=1, max_size=3, unique=True) |
| 62 | ) |
| 63 | literal_kind = draw(st.sampled_from(("none", "none", "enum", "const"))) |
| 64 | if literal_kind == "enum": |
| 65 | schema["enum"] = draw( |
| 66 | st.lists(st.sampled_from(LITERALS), min_size=1, max_size=3, unique_by=repr) |
| 67 | ) |
| 68 | elif literal_kind == "const": |
| 69 | schema["const"] = draw(st.sampled_from(LITERALS)) |
| 70 | if draw(st.integers(0, 3)) == 0: |
| 71 | schema["$ref"] = "#/$defs/" + draw(st.sampled_from(sorted(DEFINITIONS))) |
| 72 | if draw(st.booleans()): |
| 73 | schema["description"] = "a parameter" |
| 74 | |
| 75 | if depth >= 2: |
| 76 | return schema |
| 77 | nested = property_schemas(compositions, depth + 1) |
| 78 | keyword = draw(st.sampled_from((None, None, *compositions))) |
| 79 | if keyword in ("anyOf", "oneOf", "allOf"): |
| 80 | schema[keyword] = draw(st.lists(nested, min_size=1, max_size=3)) |
| 81 | elif keyword == "not": |
| 82 | schema["not"] = draw(nested) |
| 83 | elif keyword == "if": |
| 84 | schema["if"] = draw(nested) |
| 85 | for branch in ("then", "else"): |
| 86 | if draw(st.booleans()): |
| 87 | schema[branch] = draw(nested) |
| 88 | return schema |
| 89 | |
| 90 | |
| 91 | def tool_schema(property_schema: dict[str, t.Any]) -> dict[str, t.Any]: |
| 92 | return { |
| 93 | "type": "object", |
| 94 | "properties": {"value": property_schema}, |
| 95 | "$defs": copy.deepcopy(DEFINITIONS), |
| 96 | } |
| 97 | |
| 98 | |
| 99 | def mentions(node: t.Any, keyword: str, value: t.Any = None) -> bool: |
| 100 | """Whether any node carries ``keyword`` (with ``value``, when given).""" |
| 101 | if isinstance(node, list): |
| 102 | return any(mentions(item, keyword, value) for item in node) |
| 103 | if not isinstance(node, dict): |
| 104 | return False |
| 105 | if keyword in node and (value is None or node[keyword] == value): |
| 106 | return True |
| 107 | return any(mentions(child, keyword, value) for child in node.values()) |
| 108 | |
| 109 | |
| 110 | @settings(max_examples=500, deadline=None) |
| 111 | @given(property_schemas(STRICT_COMPOSITIONS)) |
| 112 | def test_null_stands_for_omission_without_changing_other_values( |
| 113 | property_schema: dict[str, t.Any], |
| 114 | ) -> None: |
| 115 | source = tool_schema(property_schema) |
| 116 | snapshot = copy.deepcopy(source) |
| 117 | result = to_strict_json_schema(source) |
| 118 | assert source == snapshot |
| 119 | assume(not result.unsupported) |
| 120 | |
| 121 | original = Draft202012Validator(source) |
| 122 | strict = Draft202012Validator(result.schema) |
| 123 | |
| 124 | assert strict.is_valid({"value": None}) |
| 125 | sent = omit_null_tool_arguments({"value": None}, result.source) |
| 126 | assert original.is_valid(sent) |
| 127 | assert sent == ({"value": None} if original.is_valid({"value": None}) else {}) |
| 128 | |
| 129 | if not mentions(property_schema, "$ref", TYPELESS_OBJECT_REF): |
| 130 | for sample in SAMPLES: |
| 131 | arguments = {"value": sample} |
| 132 | if original.is_valid(arguments): |
| 133 | assert strict.is_valid(arguments) |
| 134 | elif not mentions(property_schema, "oneOf"): |
| 135 | assert not strict.is_valid(arguments) |
| 136 | |
| 137 | again = to_strict_json_schema(result.schema) |
| 138 | assert again.schema == result.schema |
| 139 | assert again.changes == [] |
| 140 | |
| 141 | |
| 142 | @settings(max_examples=500, deadline=None) |
| 143 | @given(property_schemas(ALL_COMPOSITIONS)) |
| 144 | def test_null_is_kept_exactly_when_the_tool_schema_accepts_it( |
| 145 | property_schema: dict[str, t.Any], |
| 146 | ) -> None: |
| 147 | source = tool_schema(property_schema) |
| 148 | accepted = Draft202012Validator(source).is_valid({"value": None}) |
| 149 | expected = {"value": None} if accepted else {} |
| 150 | assert omit_null_tool_arguments({"value": None}, source) == expected |
| 151 | |
| 152 | |
| 153 | LEAVES: tuple[dict[str, t.Any], ...] = ({"type": "null"}, {"type": "string"}, {}) |
| 154 | |
| 155 | |
| 156 | @st.composite |
| 157 | def reference_graphs(draw: st.DrawFn) -> dict[str, dict[str, t.Any]]: |
| 158 | """Definitions that reference each other freely, cycles included.""" |
| 159 | names = [f"d{index}" for index in range(draw(st.integers(1, 7)))] |
| 160 | references = st.lists( |
| 161 | st.sampled_from(names).map(lambda name: {"$ref": f"#/$defs/{name}"}), |
| 162 | min_size=1, |
| 163 | max_size=3, |
| 164 | ) |
| 165 | definitions: dict[str, dict[str, t.Any]] = {} |
| 166 | for name in names: |
| 167 | kind = draw(st.sampled_from(("leaf", "anyOf", "anyOf", "allOf", "allOf"))) |
| 168 | if kind == "leaf": |
| 169 | definitions[name] = dict(draw(st.sampled_from(LEAVES))) |
| 170 | else: |
| 171 | branches: list[dict[str, t.Any]] = draw(references) |
| 172 | if draw(st.booleans()): |
| 173 | branches.append(dict(draw(st.sampled_from(LEAVES)))) |
| 174 | definitions[name] = {kind: branches} |
| 175 | return definitions |
| 176 | |
| 177 | |
| 178 | def least_fixed_point(definitions: dict[str, dict[str, t.Any]]) -> dict[str, bool]: |
| 179 | accepted = dict.fromkeys(definitions, False) |
| 180 | |
| 181 | def holds(node: dict[str, t.Any]) -> bool: |
| 182 | if "$ref" in node: |
| 183 | return accepted[node["$ref"].rsplit("/", 1)[1]] |
| 184 | if "anyOf" in node: |
| 185 | return any(holds(branch) for branch in node["anyOf"]) |
| 186 | if "allOf" in node: |
| 187 | return all(holds(branch) for branch in node["allOf"]) |
| 188 | return node.get("type") != "string" |
| 189 | |
| 190 | changed = True |
| 191 | while changed: |
| 192 | changed = False |
| 193 | for name, definition in definitions.items(): |
| 194 | if not accepted[name] and holds(definition): |
| 195 | accepted[name] = changed = True |
| 196 | return accepted |
| 197 | |
| 198 | |
| 199 | @settings(max_examples=500, deadline=None) |
| 200 | @given(reference_graphs()) |
| 201 | def test_cyclic_references_resolve_to_the_least_fixed_point( |
| 202 | definitions: dict[str, dict[str, t.Any]], |
| 203 | ) -> None: |
| 204 | source = { |
| 205 | "type": "object", |
| 206 | "properties": {name: {"$ref": f"#/$defs/{name}"} for name in definitions}, |
| 207 | "$defs": definitions, |
| 208 | } |
| 209 | expected = least_fixed_point(definitions) |
| 210 | arguments = dict.fromkeys(definitions) |
| 211 | kept = omit_null_tool_arguments(arguments, source) |
| 212 | assert kept == {name: None for name in definitions if expected[name]} |
| 213 | |
| 214 | # `allOf` makes the tool unsupported, but its properties are still widened. |
| 215 | result = to_strict_json_schema(source) |
| 216 | for name in definitions: |
| 217 | reference = {"$ref": f"#/$defs/{name}"} |
| 218 | widened = result.schema["properties"][name] |
| 219 | if expected[name]: |
| 220 | assert widened == reference |
| 221 | else: |
| 222 | assert widened == {"anyOf": [reference, {"type": "null"}]} |