@devries-cameron20 / composio Mirror · read-only
Fork 0

Composio powers 1000+ toolkits, tool search, context management, authentication, and a sandboxed workbench to help you build AI agents that turn intent into action.

Mirrored from https://github.com/Camo67/composio.git

Synced Nov 21, 03:37 UTC

next
ts 224 lines 7.98 KB
Raw
1 #!/usr/bin/env bun
2 // Skill-routing smoke test: a lightweight, deterministic eval that guards against
3 // SKILL.md description edits silently breaking which skill a task routes to.
4 //
5 // For each probe (a representative task plus its distinctive trigger phrases) we
6 // score every skill by how many of those phrases appear in its description, and
7 // assert the expected skill is the unique top scorer. This is NOT an LLM eval; it
8 // catches the common regression: a description loses the terms that made it the
9 // obvious match, or another skill grows ambiguous overlap. It also fails if a
10 // skill has no probe, so routing coverage tracks the taxonomy.
11 //
12 // Run: bun run validate:skill-routing (enforced in CI by
13 // .github/workflows/agent-substrate.yml).
14
15 import fs from 'node:fs';
16 import path from 'node:path';
17 import process from 'node:process';
18
19 const repoRoot = process.cwd();
20 const skillsRoot = path.join(repoRoot, '.agents/skills');
21
22 const readDescription = (skillName: string) => {
23 const file = path.join(skillsRoot, skillName, 'SKILL.md');
24 const content = fs.readFileSync(file, 'utf8');
25 const match = content.match(/^description:\s*(.*)$/m);
26 if (!match) {
27 throw new Error(`${skillName}: SKILL.md has no description`);
28 }
29 return match[1].replace(/^"(.*)"$/, '$1').trim();
30 };
31
32 // Each probe: a representative task and the distinctive trigger phrases that
33 // should make `expect` the obvious match. Phrases are matched case-insensitively
34 // as substrings against the skill description.
35 const probes = [
36 {
37 task: 'resume a task in another agent and verify revision-bound evidence',
38 expect: 'verify-change',
39 terms: [
40 'resumable task records',
41 'verification profiles',
42 'revision-bound receipts',
43 'carrying work between coding agents',
44 ],
45 },
46 {
47 task: 'reproduce a reported bug, find the root cause, and add a regression test',
48 expect: 'bug-fixing',
49 terms: ['root-cause', 'reproduction', 'regression test', 'incorrect SDK behavior'],
50 },
51 {
52 task: 'implement a new Effect CLI command and wire it into the command tree',
53 expect: 'cli-command',
54 terms: ['effect/cli', 'command wiring', 'CLI command UX', 'CLI source edits'],
55 },
56 {
57 task: 'write a Docker-based end-to-end test for the composio CLI binary',
58 expect: 'cli-e2e',
59 terms: ['Docker-based', 'end-to-end tests', 'binary invocation', 'fixture isolation'],
60 },
61 {
62 task: 'promote a tested Composio CLI beta to a stable first-party binary release',
63 expect: 'cli-release',
64 terms: [
65 'first-party Composio CLI binaries',
66 'promote-stable',
67 'beta-tag selection',
68 'failed release recovery',
69 ],
70 },
71 {
72 task: 'align TypeScript and Python behavior after a backend API contract change',
73 expect: 'cross-sdk-parity',
74 terms: ['backend API contract', 'both SDKs', 'generated client pins', 'comparing TS/Python'],
75 },
76 {
77 task: 'record an ADR and update Fumadocs documentation',
78 expect: 'docs-decisions',
79 terms: ['Fumadocs', 'ADR-style records', 'docs decisions', 'docs review guidance'],
80 },
81 {
82 task: 'define a new Context.Service with an explicit layer and a Schema.TaggedErrorClass, and bump the Effect beta pins',
83 expect: 'effect-v4',
84 terms: [
85 'Context.Service',
86 'Schema.TaggedErrorClass',
87 'vendored effect source',
88 'bumping the Effect beta pins',
89 ],
90 },
91 {
92 task: 'build or debug a durable backend agent with eve channels and schedules',
93 expect: 'eve',
94 terms: ['durable backend AI agents', 'eve framework', 'channels', 'schedules'],
95 },
96 {
97 task: 'draft a new guide in the house documentation voice',
98 expect: 'good-docs-writing',
99 terms: ['writing style guide', "modal's documentation voice", 'second-person', 'example-first'],
100 },
101 {
102 task: 'review a README for voice and tone violations and report suggested rewrites',
103 expect: 'good-docs-audit',
104 terms: ['report violations', 'critique', 'suggested rewrite', 'rule violated'],
105 },
106 {
107 task: 'create a Python provider adapter under python/providers with metadata',
108 expect: 'python-providers',
109 terms: [
110 'python/providers',
111 'Python provider adapters',
112 'provider metadata',
113 'framework-specific dependencies',
114 ],
115 },
116 {
117 task: 'publish the Python SDK to PyPI, bump the version and update the client pin',
118 expect: 'python-release',
119 terms: ['PyPI client pin', 'uv.lock', 'publish verification', 'packaging metadata'],
120 },
121 {
122 task: 'implement Python SDK toolkits, sessions and connected accounts under python/composio',
123 expect: 'python-sdk',
124 terms: ['python/composio', 'connected accounts', 'Python core runtime', 'shared Python models'],
125 },
126 {
127 task: 'run Python tests with nox, mypy, ruff and pytest markers',
128 expect: 'python-testing',
129 terms: ['pytest markers', 'mypy', 'sanity tests', 'Python SDK verification'],
130 },
131 {
132 task: 'navigate the monorepo layout and prepare a PR with a changeset',
133 expect: 'repo-guidance',
134 terms: ['monorepo', 'repo layout', 'changesets', 'generated-file boundaries'],
135 },
136 {
137 task: 'add or update an Agent Skill SKILL.md frontmatter and references',
138 expect: 'skill-maintenance',
139 terms: ['SKILL.md frontmatter', 'compatibility symlinks', 'agent skills', 'skill taxonomy'],
140 },
141 {
142 task: 'implement a TypeScript provider package adapter for OpenAI and Anthropic',
143 expect: 'typescript-providers',
144 terms: [
145 'ts/packages/providers',
146 'TypeScript provider packages',
147 'Claude Agent SDK',
148 'framework adapters',
149 ],
150 },
151 {
152 task: 'modify @composio/core tool and toolkit behavior and modifiers',
153 expect: 'typescript-sdk',
154 terms: ['@composio/core', 'modifiers', 'generated SDK surfaces', 'shared TypeScript packages'],
155 },
156 {
157 task: 'run Vitest suites, type checks and lint a TypeScript package',
158 expect: 'typescript-testing',
159 terms: ['Vitest', 'type checks', 'TypeScript SDK verification', 'runtime E2E tests'],
160 },
161 ];
162
163 const skillNames = fs
164 .readdirSync(skillsRoot, { withFileTypes: true })
165 .filter(entry => entry.isDirectory())
166 .map(entry => entry.name)
167 .sort();
168
169 const descriptions = new Map(skillNames.map(name => [name, readDescription(name).toLowerCase()]));
170
171 const failures: string[] = [];
172
173 // Coverage: every skill must have at least one probe so routing tracks the taxonomy.
174 const covered = new Set(probes.map(probe => probe.expect));
175 for (const name of skillNames) {
176 if (!covered.has(name)) {
177 failures.push(`skill "${name}" has no routing probe; add one to test-skill-routing.ts`);
178 }
179 }
180
181 for (const probe of probes) {
182 if (!descriptions.has(probe.expect)) {
183 failures.push(`probe "${probe.task}": expected skill "${probe.expect}" does not exist`);
184 continue;
185 }
186
187 const scores = [...descriptions].map(([name, description]) => {
188 const score = probe.terms.reduce(
189 (total, term) => total + (description.includes(term.toLowerCase()) ? 1 : 0),
190 0
191 );
192 return { name, score };
193 });
194
195 const maxScore = Math.max(...scores.map(entry => entry.score));
196 const winners = scores.filter(entry => entry.score === maxScore).map(entry => entry.name);
197 const expectScore = scores.find(entry => entry.name === probe.expect)?.score ?? 0;
198
199 if (expectScore === 0) {
200 failures.push(
201 `probe "${probe.task}": expected "${probe.expect}" matched none of its trigger phrases (description drifted?)`
202 );
203 } else if (winners.length !== 1 || winners[0] !== probe.expect) {
204 const top = scores
205 .filter(entry => entry.score > 0)
206 .sort((a, b) => b.score - a.score)
207 .slice(0, 4)
208 .map(entry => `${entry.name}=${entry.score}`)
209 .join(', ');
210 failures.push(
211 `probe "${probe.task}": expected unique winner "${probe.expect}" (score ${expectScore}) but top was [${top}]`
212 );
213 }
214 }
215
216 if (failures.length > 0) {
217 console.error('Skill-routing smoke test failed:');
218 console.error(failures.map(failure => `- ${failure}`).join('\n'));
219 process.exit(1);
220 }
221
222 console.log(
223 `Skill-routing smoke test passed (${probes.length} probes over ${skillNames.length} skills).`
224 );