@devries-cameron20 / composio Mirror · read-only
Fork 0

Composio powers 1000+ toolkits, tool search, context management, authentication, and a sandboxed workbench to help you build AI agents that turn intent into action.

Mirrored from https://github.com/Camo67/composio.git

Synced Nov 21, 04:45 UTC

next
ts 207 lines 7.82 KB
Raw
1 /**
2 * Cross-provider `$ref` handling contract.
3 *
4 * The original bug was not that one provider mishandled `$ref` — it was that
5 * *six* did, silently, for months, because nothing enumerated the providers and
6 * asked the question. Two separate fixes (mastra, then anthropic) each solved it
7 * locally and neither surfaced the rest.
8 *
9 * This test enumerates every provider in the repo and requires each one to be
10 * classified below. Adding provider #11 without a classification fails the
11 * suite. That is the whole point: the taxonomy stops being tribal knowledge and
12 * becomes a build gate.
13 *
14 * It reads source text rather than importing the providers — importing all ten
15 * would drag in every vendor SDK as a test dependency of `@composio/core`. The
16 * behavioral assertions live in each provider's own package (see
17 * `providers/openai-agents/test/openai-agents-ref-schemas.test.ts`); this test
18 * owns *completeness*.
19 *
20 * Regenerated 2026-08-28 from its Python counterpart
21 * (`python/tests/test_provider_ref_contract.py`) after the original was lost in
22 * a session handoff; the original ratchets (`pending-plan-003`) were retired
23 * when the fix landed on `fix/deref-ref-schemas-in-providers`.
24 */
25 import { describe, it, expect } from 'vitest';
26 import { readdirSync, readFileSync, existsSync, statSync } from 'node:fs';
27 import path from 'node:path';
28 import { fileURLToPath } from 'node:url';
29
30 const here = path.dirname(fileURLToPath(import.meta.url));
31 /** ts/packages/core/test/providers -> ts/packages */
32 const packagesDir = path.resolve(here, '../../..');
33 /** ts/packages -> repo root */
34 const repoRoot = path.resolve(packagesDir, '../..');
35 const providersDir = path.join(packagesDir, 'providers');
36
37 /** A provider is "dereferencing" if its source calls this. */
38 const DEREF_MARK = 'dereferenceJsonSchema';
39
40 /**
41 * How each provider treats a tool's input schema.
42 *
43 * - `needs-deref`: the provider rebuilds the schema root from
44 * `properties`/`required`, or converts to Zod. Either way a `$ref` cannot
45 * survive intact, so the schema must be dereferenced first.
46 * - `passthrough`: the provider forwards the schema object whole. `$defs`
47 * travels with `$ref` and the vendor resolves it. Dereferencing here would
48 * cost tokens and, for vendors that support recursion, remove capability.
49 *
50 * `status` exists only for `needs-deref` entries that are not fixed yet; today
51 * there are none, but the ratchet below stays so the mechanism does not rot.
52 */
53 type Treatment = 'needs-deref' | 'passthrough';
54
55 interface Classification {
56 treatment: Treatment;
57 /** Present only while a needs-deref provider is known-broken. */
58 status?: `pending-plan-${string}`;
59 /** Why this provider is classified the way it is. */
60 reason: string;
61 }
62
63 const PROVIDER_PACKAGES: Record<string, Classification> = {
64 anthropic: {
65 treatment: 'needs-deref',
66 reason: 'sanitizes property keys, which requires keys behind a ref to be materialized',
67 },
68 mastra: {
69 treatment: 'needs-deref',
70 reason:
71 '@mastra/schema-compat (AJV) refuses unresolved $ref; its converter degrades refs to permissive anyOf',
72 },
73 langchain: {
74 treatment: 'needs-deref',
75 reason: 'converts to Zod; the converter has no $ref branch (degrades to z.any())',
76 },
77 llamaindex: {
78 treatment: 'needs-deref',
79 reason: 'converts to Zod; the converter has no $ref branch (degrades to z.any())',
80 },
81 'claude-agent-sdk': {
82 treatment: 'needs-deref',
83 reason: 'converts to Zod; the converter has no $ref branch (degrades to z.any())',
84 },
85 vercel: {
86 treatment: 'needs-deref',
87 reason: 'converts to Zod on every path, not just the strict rewrite',
88 },
89 google: {
90 treatment: 'needs-deref',
91 reason:
92 'rebuilds the root from properties/required; the @google/genai Schema proto has no ref/defs field',
93 },
94 'openai-agents': {
95 treatment: 'needs-deref',
96 reason:
97 'non-strict fallback rebuilds the root; the strict branch keeps refs on purpose (OpenAI resolves them natively)',
98 },
99 openai: {
100 treatment: 'passthrough',
101 reason:
102 'forwards the whole schema; OpenAI resolves $defs/$ref natively (strict mode asserts this)',
103 },
104 cloudflare: {
105 treatment: 'passthrough',
106 reason: 'forwards the whole schema',
107 },
108 typesafe: {
109 treatment: 'needs-deref',
110 reason:
111 'compiles each property into questions, so an enum behind a ref must be materialized to be asked',
112 },
113 };
114
115 /** Non-`providers/` sources that also hand schemas to vendors. */
116 const EXTRA_PASSTHROUGH_SOURCES: Record<string, string> = {
117 'ts/packages/core/src/provider/OpenAIProvider.ts': 'forwards the whole schema',
118 'ts/packages/experimental/src/eve/provider.ts': 'forwards the whole schema',
119 };
120
121 function listSourceFiles(dir: string): string[] {
122 const out: string[] = [];
123 for (const entry of readdirSync(dir)) {
124 const full = path.join(dir, entry);
125 if (statSync(full).isDirectory()) {
126 out.push(...listSourceFiles(full));
127 } else if (entry.endsWith('.ts') && !entry.endsWith('.d.ts')) {
128 out.push(full);
129 }
130 }
131 return out;
132 }
133
134 function packageText(provider: string): string {
135 return listSourceFiles(path.join(providersDir, provider, 'src'))
136 .map(file => readFileSync(file, 'utf8'))
137 .join('\n');
138 }
139
140 const DISCOVERED = readdirSync(providersDir).filter(
141 entry =>
142 statSync(path.join(providersDir, entry)).isDirectory() &&
143 existsSync(path.join(providersDir, entry, 'package.json')) &&
144 existsSync(path.join(providersDir, entry, 'src'))
145 );
146
147 describe('cross-provider $ref handling contract', () => {
148 it('discovers the provider packages', () => {
149 // Guards the test itself: a broken path would make everything below
150 // vacuously pass.
151 expect(DISCOVERED.length).toBeGreaterThanOrEqual(10);
152 });
153
154 it('classifies every provider package', () => {
155 const unclassified = DISCOVERED.filter(name => !(name in PROVIDER_PACKAGES));
156 expect(unclassified).toEqual([]);
157 });
158
159 it('does not classify providers that no longer exist', () => {
160 const stale = Object.keys(PROVIDER_PACKAGES).filter(name => !DISCOVERED.includes(name));
161 expect(stale).toEqual([]);
162 });
163
164 const isPending = (c: Classification) => c.status?.startsWith('pending-plan') ?? false;
165
166 // Fixed needs-deref providers only: a `pending-plan-*` entry is asserted by
167 // the ratchet at the bottom instead, so the two blocks never both claim it.
168 describe.each(
169 Object.entries(PROVIDER_PACKAGES).filter(
170 ([, c]) => c.treatment === 'needs-deref' && !isPending(c)
171 )
172 )('%s', (provider, classification) => {
173 it(`dereferences before translation (${classification.reason})`, () => {
174 expect(packageText(provider)).toContain(DEREF_MARK);
175 });
176 });
177
178 describe.each(Object.entries(PROVIDER_PACKAGES).filter(([, c]) => c.treatment === 'passthrough'))(
179 '%s',
180 (provider, classification) => {
181 it(`forwards the schema whole (${classification.reason})`, () => {
182 expect(packageText(provider)).not.toContain(DEREF_MARK);
183 });
184 }
185 );
186
187 describe.each(Object.entries(EXTRA_PASSTHROUGH_SOURCES))('%s', (relPath, reason) => {
188 it(`forwards the schema whole (${reason})`, () => {
189 const full = path.join(repoRoot, relPath);
190 expect(existsSync(full), `${relPath} not found — update this test`).toBe(true);
191 expect(readFileSync(full, 'utf8')).not.toContain(DEREF_MARK);
192 });
193 });
194
195 // Ratchet, currently empty: when a needs-deref provider is added without its
196 // fix, give it `status: 'pending-plan-NNN'` and it is asserted NOT to
197 // dereference here — so the fix cannot land and leave the taxonomy stale
198 // (the assertion flips the moment the dereference call appears).
199 describe.each(Object.entries(PROVIDER_PACKAGES).filter(([, c]) => isPending(c)))(
200 '%s',
201 (provider, classification) => {
202 it(`is still pending (${classification.status})`, () => {
203 expect(packageText(provider)).not.toContain(DEREF_MARK);
204 });
205 }
206 );
207 });