Composio powers 1000+ toolkits, tool search, context management, authentication, and a sandboxed workbench to help you build AI agents that turn intent into action.
Mirrored from https://github.com/Camo67/composio.git
Synced Nov 21, 04:45 UTC
| 1 | /** |
| 2 | * Cross-provider `$ref` handling contract. |
| 3 | * |
| 4 | * The original bug was not that one provider mishandled `$ref` — it was that |
| 5 | * *six* did, silently, for months, because nothing enumerated the providers and |
| 6 | * asked the question. Two separate fixes (mastra, then anthropic) each solved it |
| 7 | * locally and neither surfaced the rest. |
| 8 | * |
| 9 | * This test enumerates every provider in the repo and requires each one to be |
| 10 | * classified below. Adding provider #11 without a classification fails the |
| 11 | * suite. That is the whole point: the taxonomy stops being tribal knowledge and |
| 12 | * becomes a build gate. |
| 13 | * |
| 14 | * It reads source text rather than importing the providers — importing all ten |
| 15 | * would drag in every vendor SDK as a test dependency of `@composio/core`. The |
| 16 | * behavioral assertions live in each provider's own package (see |
| 17 | * `providers/openai-agents/test/openai-agents-ref-schemas.test.ts`); this test |
| 18 | * owns *completeness*. |
| 19 | * |
| 20 | * Regenerated 2026-08-28 from its Python counterpart |
| 21 | * (`python/tests/test_provider_ref_contract.py`) after the original was lost in |
| 22 | * a session handoff; the original ratchets (`pending-plan-003`) were retired |
| 23 | * when the fix landed on `fix/deref-ref-schemas-in-providers`. |
| 24 | */ |
| 25 | import { describe, it, expect } from 'vitest'; |
| 26 | import { readdirSync, readFileSync, existsSync, statSync } from 'node:fs'; |
| 27 | import path from 'node:path'; |
| 28 | import { fileURLToPath } from 'node:url'; |
| 29 | |
| 30 | const here = path.dirname(fileURLToPath(import.meta.url)); |
| 31 | /** ts/packages/core/test/providers -> ts/packages */ |
| 32 | const packagesDir = path.resolve(here, '../../..'); |
| 33 | /** ts/packages -> repo root */ |
| 34 | const repoRoot = path.resolve(packagesDir, '../..'); |
| 35 | const providersDir = path.join(packagesDir, 'providers'); |
| 36 | |
| 37 | /** A provider is "dereferencing" if its source calls this. */ |
| 38 | const DEREF_MARK = 'dereferenceJsonSchema'; |
| 39 | |
| 40 | /** |
| 41 | * How each provider treats a tool's input schema. |
| 42 | * |
| 43 | * - `needs-deref`: the provider rebuilds the schema root from |
| 44 | * `properties`/`required`, or converts to Zod. Either way a `$ref` cannot |
| 45 | * survive intact, so the schema must be dereferenced first. |
| 46 | * - `passthrough`: the provider forwards the schema object whole. `$defs` |
| 47 | * travels with `$ref` and the vendor resolves it. Dereferencing here would |
| 48 | * cost tokens and, for vendors that support recursion, remove capability. |
| 49 | * |
| 50 | * `status` exists only for `needs-deref` entries that are not fixed yet; today |
| 51 | * there are none, but the ratchet below stays so the mechanism does not rot. |
| 52 | */ |
| 53 | type Treatment = 'needs-deref' | 'passthrough'; |
| 54 | |
| 55 | interface Classification { |
| 56 | treatment: Treatment; |
| 57 | /** Present only while a needs-deref provider is known-broken. */ |
| 58 | status?: `pending-plan-${string}`; |
| 59 | /** Why this provider is classified the way it is. */ |
| 60 | reason: string; |
| 61 | } |
| 62 | |
| 63 | const PROVIDER_PACKAGES: Record<string, Classification> = { |
| 64 | anthropic: { |
| 65 | treatment: 'needs-deref', |
| 66 | reason: 'sanitizes property keys, which requires keys behind a ref to be materialized', |
| 67 | }, |
| 68 | mastra: { |
| 69 | treatment: 'needs-deref', |
| 70 | reason: |
| 71 | '@mastra/schema-compat (AJV) refuses unresolved $ref; its converter degrades refs to permissive anyOf', |
| 72 | }, |
| 73 | langchain: { |
| 74 | treatment: 'needs-deref', |
| 75 | reason: 'converts to Zod; the converter has no $ref branch (degrades to z.any())', |
| 76 | }, |
| 77 | llamaindex: { |
| 78 | treatment: 'needs-deref', |
| 79 | reason: 'converts to Zod; the converter has no $ref branch (degrades to z.any())', |
| 80 | }, |
| 81 | 'claude-agent-sdk': { |
| 82 | treatment: 'needs-deref', |
| 83 | reason: 'converts to Zod; the converter has no $ref branch (degrades to z.any())', |
| 84 | }, |
| 85 | vercel: { |
| 86 | treatment: 'needs-deref', |
| 87 | reason: 'converts to Zod on every path, not just the strict rewrite', |
| 88 | }, |
| 89 | google: { |
| 90 | treatment: 'needs-deref', |
| 91 | reason: |
| 92 | 'rebuilds the root from properties/required; the @google/genai Schema proto has no ref/defs field', |
| 93 | }, |
| 94 | 'openai-agents': { |
| 95 | treatment: 'needs-deref', |
| 96 | reason: |
| 97 | 'non-strict fallback rebuilds the root; the strict branch keeps refs on purpose (OpenAI resolves them natively)', |
| 98 | }, |
| 99 | openai: { |
| 100 | treatment: 'passthrough', |
| 101 | reason: |
| 102 | 'forwards the whole schema; OpenAI resolves $defs/$ref natively (strict mode asserts this)', |
| 103 | }, |
| 104 | cloudflare: { |
| 105 | treatment: 'passthrough', |
| 106 | reason: 'forwards the whole schema', |
| 107 | }, |
| 108 | typesafe: { |
| 109 | treatment: 'needs-deref', |
| 110 | reason: |
| 111 | 'compiles each property into questions, so an enum behind a ref must be materialized to be asked', |
| 112 | }, |
| 113 | }; |
| 114 | |
| 115 | /** Non-`providers/` sources that also hand schemas to vendors. */ |
| 116 | const EXTRA_PASSTHROUGH_SOURCES: Record<string, string> = { |
| 117 | 'ts/packages/core/src/provider/OpenAIProvider.ts': 'forwards the whole schema', |
| 118 | 'ts/packages/experimental/src/eve/provider.ts': 'forwards the whole schema', |
| 119 | }; |
| 120 | |
| 121 | function listSourceFiles(dir: string): string[] { |
| 122 | const out: string[] = []; |
| 123 | for (const entry of readdirSync(dir)) { |
| 124 | const full = path.join(dir, entry); |
| 125 | if (statSync(full).isDirectory()) { |
| 126 | out.push(...listSourceFiles(full)); |
| 127 | } else if (entry.endsWith('.ts') && !entry.endsWith('.d.ts')) { |
| 128 | out.push(full); |
| 129 | } |
| 130 | } |
| 131 | return out; |
| 132 | } |
| 133 | |
| 134 | function packageText(provider: string): string { |
| 135 | return listSourceFiles(path.join(providersDir, provider, 'src')) |
| 136 | .map(file => readFileSync(file, 'utf8')) |
| 137 | .join('\n'); |
| 138 | } |
| 139 | |
| 140 | const DISCOVERED = readdirSync(providersDir).filter( |
| 141 | entry => |
| 142 | statSync(path.join(providersDir, entry)).isDirectory() && |
| 143 | existsSync(path.join(providersDir, entry, 'package.json')) && |
| 144 | existsSync(path.join(providersDir, entry, 'src')) |
| 145 | ); |
| 146 | |
| 147 | describe('cross-provider $ref handling contract', () => { |
| 148 | it('discovers the provider packages', () => { |
| 149 | // Guards the test itself: a broken path would make everything below |
| 150 | // vacuously pass. |
| 151 | expect(DISCOVERED.length).toBeGreaterThanOrEqual(10); |
| 152 | }); |
| 153 | |
| 154 | it('classifies every provider package', () => { |
| 155 | const unclassified = DISCOVERED.filter(name => !(name in PROVIDER_PACKAGES)); |
| 156 | expect(unclassified).toEqual([]); |
| 157 | }); |
| 158 | |
| 159 | it('does not classify providers that no longer exist', () => { |
| 160 | const stale = Object.keys(PROVIDER_PACKAGES).filter(name => !DISCOVERED.includes(name)); |
| 161 | expect(stale).toEqual([]); |
| 162 | }); |
| 163 | |
| 164 | const isPending = (c: Classification) => c.status?.startsWith('pending-plan') ?? false; |
| 165 | |
| 166 | // Fixed needs-deref providers only: a `pending-plan-*` entry is asserted by |
| 167 | // the ratchet at the bottom instead, so the two blocks never both claim it. |
| 168 | describe.each( |
| 169 | Object.entries(PROVIDER_PACKAGES).filter( |
| 170 | ([, c]) => c.treatment === 'needs-deref' && !isPending(c) |
| 171 | ) |
| 172 | )('%s', (provider, classification) => { |
| 173 | it(`dereferences before translation (${classification.reason})`, () => { |
| 174 | expect(packageText(provider)).toContain(DEREF_MARK); |
| 175 | }); |
| 176 | }); |
| 177 | |
| 178 | describe.each(Object.entries(PROVIDER_PACKAGES).filter(([, c]) => c.treatment === 'passthrough'))( |
| 179 | '%s', |
| 180 | (provider, classification) => { |
| 181 | it(`forwards the schema whole (${classification.reason})`, () => { |
| 182 | expect(packageText(provider)).not.toContain(DEREF_MARK); |
| 183 | }); |
| 184 | } |
| 185 | ); |
| 186 | |
| 187 | describe.each(Object.entries(EXTRA_PASSTHROUGH_SOURCES))('%s', (relPath, reason) => { |
| 188 | it(`forwards the schema whole (${reason})`, () => { |
| 189 | const full = path.join(repoRoot, relPath); |
| 190 | expect(existsSync(full), `${relPath} not found — update this test`).toBe(true); |
| 191 | expect(readFileSync(full, 'utf8')).not.toContain(DEREF_MARK); |
| 192 | }); |
| 193 | }); |
| 194 | |
| 195 | // Ratchet, currently empty: when a needs-deref provider is added without its |
| 196 | // fix, give it `status: 'pending-plan-NNN'` and it is asserted NOT to |
| 197 | // dereference here — so the fix cannot land and leave the taxonomy stale |
| 198 | // (the assertion flips the moment the dereference call appears). |
| 199 | describe.each(Object.entries(PROVIDER_PACKAGES).filter(([, c]) => isPending(c)))( |
| 200 | '%s', |
| 201 | (provider, classification) => { |
| 202 | it(`is still pending (${classification.status})`, () => { |
| 203 | expect(packageText(provider)).not.toContain(DEREF_MARK); |
| 204 | }); |
| 205 | } |
| 206 | ); |
| 207 | }); |