Composio powers 1000+ toolkits, tool search, context management, authentication, and a sandboxed workbench to help you build AI agents that turn intent into action.

next
py 257 lines 10.7 KB
Raw
1 """Inline internal JSON Schema ``$ref`` pointers.
2
3 Python counterpart of the TypeScript SDK's ``dereferenceJsonSchema``
4 (``ts/packages/core/src/utils/jsonSchema.ts``). Hand-rolled schema walkers in
5 the SDK (e.g. the file-upload/download substitution in
6 :mod:`composio.core.models._files`) recurse through ``properties``/``anyOf``/
7 ``oneOf``/``allOf``/``items`` but do not dereference ``$ref``. The Composio API
8 emits tool parameter schemas with ``$ref``/``$defs`` indirection (``GMAIL_GET_
9 ATTACHMENT`` is the canonical shape), so a ``file_uploadable``/``file_
10 downloadable`` flag reachable only through a reference is silently missed.
11
12 Resolving references *once* at the boundary — rather than teaching every walker
13 to dereference — keeps the walkers reference-agnostic and gives all of them
14 ``$ref`` support for free, including keywords they never special-cased
15 (``additionalProperties``, ``patternProperties``, ``prefixItems``, ``not``, …),
16 because :func:`dereference_json_schema` walks containers reflectively.
17 """
18
19 from __future__ import annotations
20
21 import typing as t
22
23 from composio.exceptions import JSONSchemaRefResolutionError
24 from composio.utils.logging import get as get_logger
25
26 logger = get_logger()
27
28 # A ``$ref`` chain longer than this, or object nesting deeper than this, is
29 # treated as pathological (cyclic data, a billion-laughs-style schema) and
30 # raises rather than exhausting the interpreter's recursion limit. The caps
31 # fire in both strict and lenient modes — leniency applies to *unresolvable*
32 # refs, not to runaway expansion.
33 MAX_REF_CHAIN_DEPTH = 100
34 MAX_NODE_DEPTH = 512
35
36 # In-band hint attached to the sentinel when lenient mode stands in for an
37 # unresolvable ``$ref``. Makes the degradation visible to an LLM reading the
38 # schema instead of leaving it a bare permissive object. Overridden by a
39 # caller-provided ``description`` sibling (Draft 2020-12 sibling-keyword merge).
40 UNRESOLVED_REF_DESCRIPTION = (
41 "Schema shape unresolved at the source — validate loosely. "
42 "See https://github.com/ComposioHQ/composio/issues/3307."
43 )
44
45 # Strategy when an internal ``$ref`` cannot be resolved.
46 # - ``"throw"`` (default): raise ``JSONSchemaRefResolutionError``. Right for
47 # first-party / custom-tool schemas where a dangling ``$ref`` is a bug.
48 # - ``"sentinel"``: replace the node with a permissive object. Right for
49 # schemas from an upstream service the caller cannot edit (the Composio API
50 # ships some ``output_parameters`` with a ``$ref`` into ``#/$defs/...`` but no
51 # ``$defs`` block — see https://github.com/ComposioHQ/composio/issues/3307).
52 UnresolvedRefStrategy = t.Literal["throw", "sentinel"]
53 UnresolvedRefReason = t.Literal["missing-target", "malformed-pointer"]
54
55 # Callback invoked once per replaced node in ``"sentinel"`` mode.
56 OnReplace = t.Callable[[str, "UnresolvedRefReason"], None]
57
58
59 def _cycle_break_sentinel() -> t.Dict[str, t.Any]:
60 """A fresh permissive object used to break cycles / stand in for danglers."""
61 return {"type": "object", "additionalProperties": True}
62
63
64 def _is_mapping(value: t.Any) -> bool:
65 return isinstance(value, dict)
66
67
68 def _decode_pointer_segment(segment: str) -> str:
69 # Order matters: ``~01`` must decode to ``~1`` (literal), not ``/``.
70 return segment.replace("~1", "/").replace("~0", "~")
71
72
73 class _Resolution(t.NamedTuple):
74 ok: bool
75 value: t.Any = None
76 reason: t.Optional[UnresolvedRefReason] = None
77 failed_at: t.Optional[str] = None
78
79
80 def _try_resolve_pointer(root: t.Dict[str, t.Any], pointer: str) -> _Resolution:
81 """Walk a local JSON Pointer (``#/a/b/0``) against ``root``.
82
83 Returns a tagged result so the caller can distinguish a malformed pointer
84 from a structurally valid pointer whose target is absent.
85 """
86 if pointer in ("#", ""):
87 return _Resolution(ok=True, value=root)
88 if not pointer.startswith("#/"):
89 return _Resolution(ok=False, reason="malformed-pointer")
90
91 cursor: t.Any = root
92 for raw_segment in pointer[2:].split("/"):
93 segment = _decode_pointer_segment(raw_segment)
94 if isinstance(cursor, list):
95 try:
96 index = int(segment)
97 except ValueError:
98 return _Resolution(ok=False, reason="missing-target", failed_at=segment)
99 if index < 0 or index >= len(cursor):
100 return _Resolution(ok=False, reason="missing-target", failed_at=segment)
101 cursor = cursor[index]
102 elif isinstance(cursor, dict):
103 if segment not in cursor:
104 return _Resolution(ok=False, reason="missing-target", failed_at=segment)
105 cursor = cursor[segment]
106 else:
107 return _Resolution(ok=False, reason="missing-target", failed_at=segment)
108 return _Resolution(ok=True, value=cursor)
109
110
111 def _raise_resolution_error(pointer: str, resolution: _Resolution) -> t.NoReturn:
112 if resolution.reason == "malformed-pointer":
113 raise JSONSchemaRefResolutionError(
114 f"Unsupported $ref pointer: {pointer}",
115 meta={"ref": pointer},
116 )
117 meta: t.Dict[str, t.Any] = {"ref": pointer}
118 if resolution.failed_at is not None:
119 meta["failed_at"] = resolution.failed_at
120 raise JSONSchemaRefResolutionError(
121 f"Cannot resolve $ref {pointer}",
122 meta=meta,
123 )
124
125
126 def dereference_json_schema(
127 schema: t.Any,
128 *,
129 on_unresolved: UnresolvedRefStrategy = "throw",
130 on_replace: t.Optional[OnReplace] = None,
131 ) -> t.Any:
132 """Inline internal ``$ref`` pointers (``#/$defs/...`` and legacy
133 ``#/definitions/...``), returning a new schema; the input is never mutated.
134
135 External refs (``http://``, ``https://``, …) are left untouched (and logged
136 once for audit, since a downstream resolver fetching them could enable SSRF
137 or local-file disclosure). Cycles — both ``$ref`` cycles and JS-object-style
138 identity cycles — are broken with a permissive ``{"type": "object",
139 "additionalProperties": True}`` sentinel. ``$defs``/``definitions`` are
140 stripped from the returned root once everything is inlined.
141
142 :param on_unresolved: see :data:`UnresolvedRefStrategy`.
143 :param on_replace: invoked once per replaced node in ``"sentinel"`` mode,
144 with the original pointer and the reason it could not be resolved.
145 :raises JSONSchemaRefResolutionError: on malformed pointers or missing
146 targets (strict mode only), or chains/nesting past the safety caps
147 (both modes).
148 """
149 if not _is_mapping(schema):
150 return schema
151
152 root = t.cast(t.Dict[str, t.Any], schema)
153 # Identity-based cycle guard for the *current* walk path. Python doesn't
154 # suffer JS-style prototype pollution, so unlike the TS port there's no
155 # need to filter ``__proto__``/``constructor`` keys while cloning.
156 visiting: t.Set[int] = set()
157
158 # ``walk`` uses explicit loops (not comprehensions) so each level of schema
159 # nesting costs exactly one Python stack frame. That keeps ``MAX_NODE_DEPTH``
160 # the binding limit under the interpreter's default recursion ceiling, so a
161 # pathologically deep schema fails with our typed error rather than a bare
162 # ``RecursionError`` (which is also caught at the call site as a backstop).
163 def walk(
164 node: t.Any,
165 visited_refs: t.FrozenSet[str],
166 chain_depth: int,
167 node_depth: int,
168 ) -> t.Any:
169 if node_depth >= MAX_NODE_DEPTH:
170 raise JSONSchemaRefResolutionError(
171 f"JSON Schema node depth exceeded cap ({MAX_NODE_DEPTH})"
172 )
173
174 if isinstance(node, list):
175 node_id = id(node)
176 if node_id in visiting:
177 return _cycle_break_sentinel()
178 visiting.add(node_id)
179 try:
180 cloned_list: t.List[t.Any] = []
181 for item in node:
182 cloned_list.append(
183 walk(item, visited_refs, chain_depth, node_depth + 1)
184 )
185 return cloned_list
186 finally:
187 visiting.discard(node_id)
188
189 if not _is_mapping(node):
190 return node
191
192 node_id = id(node)
193 if node_id in visiting:
194 return _cycle_break_sentinel()
195 visiting.add(node_id)
196 try:
197 ref = node.get("$ref")
198 ref = ref if isinstance(ref, str) else None
199
200 # External refs and non-``$ref`` nodes both pass through the same
201 # reflective clone path.
202 if ref is None or not ref.startswith("#"):
203 if ref is not None:
204 logger.warning("Leaving external $ref untouched: %s", ref)
205 cloned: t.Dict[str, t.Any] = {}
206 for key, value in node.items():
207 cloned[key] = walk(value, visited_refs, chain_depth, node_depth + 1)
208 return cloned
209
210 if chain_depth >= MAX_REF_CHAIN_DEPTH:
211 raise JSONSchemaRefResolutionError(
212 f"JSON Schema $ref chain exceeded depth cap "
213 f"({MAX_REF_CHAIN_DEPTH}): {ref}",
214 meta={"ref": ref},
215 )
216 if ref in visited_refs:
217 return _cycle_break_sentinel()
218
219 resolution = _try_resolve_pointer(root, ref)
220 if resolution.ok:
221 target: t.Any = resolution.value
222 elif on_unresolved == "sentinel":
223 if on_replace is not None and resolution.reason is not None:
224 on_replace(ref, resolution.reason)
225 target = {
226 **_cycle_break_sentinel(),
227 "description": UNRESOLVED_REF_DESCRIPTION,
228 }
229 else:
230 _raise_resolution_error(ref, resolution)
231
232 next_refs = visited_refs | {ref}
233 resolved = walk(target, next_refs, chain_depth + 1, node_depth + 1)
234
235 # Shallow-merge sibling keywords next to ``$ref`` (Draft 2020-12:
236 # siblings win on collision). Draft 7 ignores them, but the Composio
237 # tool surface admits both, so we honor siblings for safety.
238 siblings = {key: value for key, value in node.items() if key != "$ref"}
239 if not siblings or not _is_mapping(resolved):
240 return resolved
241 merged = dict(resolved)
242 for key, value in siblings.items():
243 merged[key] = walk(value, visited_refs, chain_depth, node_depth + 1)
244 return merged
245 finally:
246 visiting.discard(node_id)
247
248 try:
249 out = walk(root, frozenset(), 0, 0)
250 except RecursionError as exc: # pragma: no cover - depth cap normally fires first
251 raise JSONSchemaRefResolutionError(
252 "JSON Schema nesting too deep to dereference"
253 ) from exc
254 if _is_mapping(out):
255 out.pop("$defs", None)
256 out.pop("definitions", None)
257 return out