Composio powers 1000+ toolkits, tool search, context management, authentication, and a sandboxed workbench to help you build AI agents that turn intent into action.
| 1 | #!/usr/bin/env bun |
| 2 | // Skill-routing smoke test: a lightweight, deterministic eval that guards against |
| 3 | // SKILL.md description edits silently breaking which skill a task routes to. |
| 4 | // |
| 5 | // For each probe (a representative task plus its distinctive trigger phrases) we |
| 6 | // score every skill by how many of those phrases appear in its description, and |
| 7 | // assert the expected skill is the unique top scorer. This is NOT an LLM eval; it |
| 8 | // catches the common regression: a description loses the terms that made it the |
| 9 | // obvious match, or another skill grows ambiguous overlap. It also fails if a |
| 10 | // skill has no probe, so routing coverage tracks the taxonomy. |
| 11 | // |
| 12 | // Run: bun run validate:skill-routing (enforced in CI by |
| 13 | // .github/workflows/agent-substrate.yml). |
| 14 | |
| 15 | import fs from 'node:fs'; |
| 16 | import path from 'node:path'; |
| 17 | import process from 'node:process'; |
| 18 | |
| 19 | const repoRoot = process.cwd(); |
| 20 | const skillsRoot = path.join(repoRoot, '.agents/skills'); |
| 21 | |
| 22 | const readDescription = (skillName: string) => { |
| 23 | const file = path.join(skillsRoot, skillName, 'SKILL.md'); |
| 24 | const content = fs.readFileSync(file, 'utf8'); |
| 25 | const match = content.match(/^description:\s*(.*)$/m); |
| 26 | if (!match) { |
| 27 | throw new Error(`${skillName}: SKILL.md has no description`); |
| 28 | } |
| 29 | return match[1].replace(/^"(.*)"$/, '$1').trim(); |
| 30 | }; |
| 31 | |
| 32 | // Each probe: a representative task and the distinctive trigger phrases that |
| 33 | // should make `expect` the obvious match. Phrases are matched case-insensitively |
| 34 | // as substrings against the skill description. |
| 35 | const probes = [ |
| 36 | { |
| 37 | task: 'resume a task in another agent and verify revision-bound evidence', |
| 38 | expect: 'verify-change', |
| 39 | terms: [ |
| 40 | 'resumable task records', |
| 41 | 'verification profiles', |
| 42 | 'revision-bound receipts', |
| 43 | 'carrying work between coding agents', |
| 44 | ], |
| 45 | }, |
| 46 | { |
| 47 | task: 'reproduce a reported bug, find the root cause, and add a regression test', |
| 48 | expect: 'bug-fixing', |
| 49 | terms: ['root-cause', 'reproduction', 'regression test', 'incorrect SDK behavior'], |
| 50 | }, |
| 51 | { |
| 52 | task: 'implement a new Effect CLI command and wire it into the command tree', |
| 53 | expect: 'cli-command', |
| 54 | terms: ['effect/cli', 'command wiring', 'CLI command UX', 'CLI source edits'], |
| 55 | }, |
| 56 | { |
| 57 | task: 'write a Docker-based end-to-end test for the composio CLI binary', |
| 58 | expect: 'cli-e2e', |
| 59 | terms: ['Docker-based', 'end-to-end tests', 'binary invocation', 'fixture isolation'], |
| 60 | }, |
| 61 | { |
| 62 | task: 'promote a tested Composio CLI beta to a stable first-party binary release', |
| 63 | expect: 'cli-release', |
| 64 | terms: [ |
| 65 | 'first-party Composio CLI binaries', |
| 66 | 'promote-stable', |
| 67 | 'beta-tag selection', |
| 68 | 'failed release recovery', |
| 69 | ], |
| 70 | }, |
| 71 | { |
| 72 | task: 'align TypeScript and Python behavior after a backend API contract change', |
| 73 | expect: 'cross-sdk-parity', |
| 74 | terms: ['backend API contract', 'both SDKs', 'generated client pins', 'comparing TS/Python'], |
| 75 | }, |
| 76 | { |
| 77 | task: 'record an ADR and update Fumadocs documentation', |
| 78 | expect: 'docs-decisions', |
| 79 | terms: ['Fumadocs', 'ADR-style records', 'docs decisions', 'docs review guidance'], |
| 80 | }, |
| 81 | { |
| 82 | task: 'define a new Context.Service with an explicit layer and a Schema.TaggedErrorClass, and bump the Effect beta pins', |
| 83 | expect: 'effect-v4', |
| 84 | terms: [ |
| 85 | 'Context.Service', |
| 86 | 'Schema.TaggedErrorClass', |
| 87 | 'vendored effect source', |
| 88 | 'bumping the Effect beta pins', |
| 89 | ], |
| 90 | }, |
| 91 | { |
| 92 | task: 'build or debug a durable backend agent with eve channels and schedules', |
| 93 | expect: 'eve', |
| 94 | terms: ['durable backend AI agents', 'eve framework', 'channels', 'schedules'], |
| 95 | }, |
| 96 | { |
| 97 | task: 'draft a new guide in the house documentation voice', |
| 98 | expect: 'good-docs-writing', |
| 99 | terms: ['writing style guide', "modal's documentation voice", 'second-person', 'example-first'], |
| 100 | }, |
| 101 | { |
| 102 | task: 'review a README for voice and tone violations and report suggested rewrites', |
| 103 | expect: 'good-docs-audit', |
| 104 | terms: ['report violations', 'critique', 'suggested rewrite', 'rule violated'], |
| 105 | }, |
| 106 | { |
| 107 | task: 'create a Python provider adapter under python/providers with metadata', |
| 108 | expect: 'python-providers', |
| 109 | terms: [ |
| 110 | 'python/providers', |
| 111 | 'Python provider adapters', |
| 112 | 'provider metadata', |
| 113 | 'framework-specific dependencies', |
| 114 | ], |
| 115 | }, |
| 116 | { |
| 117 | task: 'publish the Python SDK to PyPI, bump the version and update the client pin', |
| 118 | expect: 'python-release', |
| 119 | terms: ['PyPI client pin', 'uv.lock', 'publish verification', 'packaging metadata'], |
| 120 | }, |
| 121 | { |
| 122 | task: 'implement Python SDK toolkits, sessions and connected accounts under python/composio', |
| 123 | expect: 'python-sdk', |
| 124 | terms: ['python/composio', 'connected accounts', 'Python core runtime', 'shared Python models'], |
| 125 | }, |
| 126 | { |
| 127 | task: 'run Python tests with nox, mypy, ruff and pytest markers', |
| 128 | expect: 'python-testing', |
| 129 | terms: ['pytest markers', 'mypy', 'sanity tests', 'Python SDK verification'], |
| 130 | }, |
| 131 | { |
| 132 | task: 'navigate the monorepo layout and prepare a PR with a changeset', |
| 133 | expect: 'repo-guidance', |
| 134 | terms: ['monorepo', 'repo layout', 'changesets', 'generated-file boundaries'], |
| 135 | }, |
| 136 | { |
| 137 | task: 'add or update an Agent Skill SKILL.md frontmatter and references', |
| 138 | expect: 'skill-maintenance', |
| 139 | terms: ['SKILL.md frontmatter', 'compatibility symlinks', 'agent skills', 'skill taxonomy'], |
| 140 | }, |
| 141 | { |
| 142 | task: 'implement a TypeScript provider package adapter for OpenAI and Anthropic', |
| 143 | expect: 'typescript-providers', |
| 144 | terms: [ |
| 145 | 'ts/packages/providers', |
| 146 | 'TypeScript provider packages', |
| 147 | 'Claude Agent SDK', |
| 148 | 'framework adapters', |
| 149 | ], |
| 150 | }, |
| 151 | { |
| 152 | task: 'modify @composio/core tool and toolkit behavior and modifiers', |
| 153 | expect: 'typescript-sdk', |
| 154 | terms: ['@composio/core', 'modifiers', 'generated SDK surfaces', 'shared TypeScript packages'], |
| 155 | }, |
| 156 | { |
| 157 | task: 'run Vitest suites, type checks and lint a TypeScript package', |
| 158 | expect: 'typescript-testing', |
| 159 | terms: ['Vitest', 'type checks', 'TypeScript SDK verification', 'runtime E2E tests'], |
| 160 | }, |
| 161 | ]; |
| 162 | |
| 163 | const skillNames = fs |
| 164 | .readdirSync(skillsRoot, { withFileTypes: true }) |
| 165 | .filter(entry => entry.isDirectory()) |
| 166 | .map(entry => entry.name) |
| 167 | .sort(); |
| 168 | |
| 169 | const descriptions = new Map(skillNames.map(name => [name, readDescription(name).toLowerCase()])); |
| 170 | |
| 171 | const failures: string[] = []; |
| 172 | |
| 173 | // Coverage: every skill must have at least one probe so routing tracks the taxonomy. |
| 174 | const covered = new Set(probes.map(probe => probe.expect)); |
| 175 | for (const name of skillNames) { |
| 176 | if (!covered.has(name)) { |
| 177 | failures.push(`skill "${name}" has no routing probe; add one to test-skill-routing.ts`); |
| 178 | } |
| 179 | } |
| 180 | |
| 181 | for (const probe of probes) { |
| 182 | if (!descriptions.has(probe.expect)) { |
| 183 | failures.push(`probe "${probe.task}": expected skill "${probe.expect}" does not exist`); |
| 184 | continue; |
| 185 | } |
| 186 | |
| 187 | const scores = [...descriptions].map(([name, description]) => { |
| 188 | const score = probe.terms.reduce( |
| 189 | (total, term) => total + (description.includes(term.toLowerCase()) ? 1 : 0), |
| 190 | 0 |
| 191 | ); |
| 192 | return { name, score }; |
| 193 | }); |
| 194 | |
| 195 | const maxScore = Math.max(...scores.map(entry => entry.score)); |
| 196 | const winners = scores.filter(entry => entry.score === maxScore).map(entry => entry.name); |
| 197 | const expectScore = scores.find(entry => entry.name === probe.expect)?.score ?? 0; |
| 198 | |
| 199 | if (expectScore === 0) { |
| 200 | failures.push( |
| 201 | `probe "${probe.task}": expected "${probe.expect}" matched none of its trigger phrases (description drifted?)` |
| 202 | ); |
| 203 | } else if (winners.length !== 1 || winners[0] !== probe.expect) { |
| 204 | const top = scores |
| 205 | .filter(entry => entry.score > 0) |
| 206 | .sort((a, b) => b.score - a.score) |
| 207 | .slice(0, 4) |
| 208 | .map(entry => `${entry.name}=${entry.score}`) |
| 209 | .join(', '); |
| 210 | failures.push( |
| 211 | `probe "${probe.task}": expected unique winner "${probe.expect}" (score ${expectScore}) but top was [${top}]` |
| 212 | ); |
| 213 | } |
| 214 | } |
| 215 | |
| 216 | if (failures.length > 0) { |
| 217 | console.error('Skill-routing smoke test failed:'); |
| 218 | console.error(failures.map(failure => `- ${failure}`).join('\n')); |
| 219 | process.exit(1); |
| 220 | } |
| 221 | |
| 222 | console.log( |
| 223 | `Skill-routing smoke test passed (${probes.length} probes over ${skillNames.length} skills).` |
| 224 | ); |