Composio powers 1000+ toolkits, tool search, context management, authentication, and a sandboxed workbench to help you build AI agents that turn intent into action.
Mirrored from https://github.com/Camo67/composio.git
Synced Nov 21, 04:45 UTC
| 1 | /** |
| 2 | * The commands the skill teaches, checked against the binary. |
| 3 | * |
| 4 | * Every `composio …` line in a code block of the built skill is reduced to its |
| 5 | * command path and looked up with `composio help <path>`. A path the binary |
| 6 | * does not know is a line that sends an agent into a parse error on its first |
| 7 | * try. |
| 8 | */ |
| 9 | |
| 10 | import { afterAll, beforeAll, describe, expect, it } from 'bun:test'; |
| 11 | import { startCliRig, type CliRig } from '@e2e-tests/cli-fake-backend/rig'; |
| 12 | import { buildSkill } from './src/skill'; |
| 13 | |
| 14 | const RIG_TIMEOUT_MS = 1_200_000; |
| 15 | |
| 16 | /** `composio dev toolkits info github` -> `dev toolkits info github`: the words before any flag, quote, or slug. */ |
| 17 | const leadingWords = (line: string): string[] => { |
| 18 | const words: string[] = []; |
| 19 | for (const token of line.trim().split(/\s+/).slice(1)) { |
| 20 | if (!/^[a-z][a-z-]*$/.test(token)) break; |
| 21 | words.push(token); |
| 22 | } |
| 23 | return words; |
| 24 | }; |
| 25 | |
| 26 | const taughtCommands = (): Array<{ file: string; line: string; words: string[] }> => { |
| 27 | const skill = buildSkill(); |
| 28 | const documents = { 'SKILL.md': skill.skillMarkdown, ...skill.references }; |
| 29 | return Object.entries(documents).flatMap(([file, markdown]) => |
| 30 | [...markdown.matchAll(/```bash\n([\s\S]*?)```/g)].flatMap(block => |
| 31 | (block[1] ?? '') |
| 32 | .split('\n') |
| 33 | .filter(line => line.startsWith('composio ')) |
| 34 | .map(line => ({ file, line, words: leadingWords(line) })) |
| 35 | ) |
| 36 | ); |
| 37 | }; |
| 38 | |
| 39 | describe('skill: every command it teaches exists', () => { |
| 40 | let rig: CliRig; |
| 41 | const unknown: Array<{ file: string; line: string; error: string }> = []; |
| 42 | const commands = taughtCommands(); |
| 43 | |
| 44 | beforeAll(async () => { |
| 45 | rig = await startCliRig({ label: 'skill-commands' }); |
| 46 | const paths = new Map(commands.map(command => [command.words.join(' '), command])); |
| 47 | for (const [, command] of paths) { |
| 48 | // `composio help` rejects an argument and an unknown subcommand differently. |
| 49 | // Trailing words that are arguments (`link gmail`) are dropped one at a |
| 50 | // time; a word that is not a subcommand stops the search. `<path> --help` |
| 51 | // cannot be used: it prints the parent's help and exits 0 for both. |
| 52 | for (let length = command.words.length; length > 0; length -= 1) { |
| 53 | const result = await rig.run(`composio help ${command.words.slice(0, length).join(' ')}`); |
| 54 | if (result.exitCode === 0) break; |
| 55 | if (!result.stderr.includes('Unexpected positional argument')) { |
| 56 | const error = /Unknown subcommand[^\n]*/.exec(result.stderr)?.[0] ?? result.stderr.trim(); |
| 57 | unknown.push({ file: command.file, line: command.line, error }); |
| 58 | break; |
| 59 | } |
| 60 | } |
| 61 | } |
| 62 | }, RIG_TIMEOUT_MS); |
| 63 | |
| 64 | afterAll(async () => { |
| 65 | await rig.stop(); |
| 66 | }); |
| 67 | |
| 68 | it('finds command lines in the skill and its references', () => { |
| 69 | expect(commands.length).toBeGreaterThan(20); |
| 70 | expect(new Set(commands.map(command => command.file))).toEqual( |
| 71 | new Set(['SKILL.md', 'composio-dev.md', 'power-user-examples.md', 'troubleshooting.md']) |
| 72 | ); |
| 73 | }); |
| 74 | |
| 75 | it('SKILL.md teaches only commands the binary has', () => { |
| 76 | expect(unknown.filter(command => command.file === 'SKILL.md')).toEqual([]); |
| 77 | }); |
| 78 | |
| 79 | it('the references teach only commands the binary has', () => { |
| 80 | expect( |
| 81 | unknown |
| 82 | .filter(command => command.file !== 'SKILL.md') |
| 83 | .map(command => `${command.file}: ${command.line}`) |
| 84 | ).toEqual([]); |
| 85 | }); |
| 86 | }); |