Composio powers 1000+ toolkits, tool search, context management, authentication, and a sandboxed workbench to help you build AI agents that turn intent into action.

next
ts 86 lines 3.31 KB
Raw
1 /**
2 * The commands the skill teaches, checked against the binary.
3 *
4 * Every `composio …` line in a code block of the built skill is reduced to its
5 * command path and looked up with `composio help <path>`. A path the binary
6 * does not know is a line that sends an agent into a parse error on its first
7 * try.
8 */
9
10 import { afterAll, beforeAll, describe, expect, it } from 'bun:test';
11 import { startCliRig, type CliRig } from '@e2e-tests/cli-fake-backend/rig';
12 import { buildSkill } from './src/skill';
13
14 const RIG_TIMEOUT_MS = 1_200_000;
15
16 /** `composio dev toolkits info github` -> `dev toolkits info github`: the words before any flag, quote, or slug. */
17 const leadingWords = (line: string): string[] => {
18 const words: string[] = [];
19 for (const token of line.trim().split(/\s+/).slice(1)) {
20 if (!/^[a-z][a-z-]*$/.test(token)) break;
21 words.push(token);
22 }
23 return words;
24 };
25
26 const taughtCommands = (): Array<{ file: string; line: string; words: string[] }> => {
27 const skill = buildSkill();
28 const documents = { 'SKILL.md': skill.skillMarkdown, ...skill.references };
29 return Object.entries(documents).flatMap(([file, markdown]) =>
30 [...markdown.matchAll(/```bash\n([\s\S]*?)```/g)].flatMap(block =>
31 (block[1] ?? '')
32 .split('\n')
33 .filter(line => line.startsWith('composio '))
34 .map(line => ({ file, line, words: leadingWords(line) }))
35 )
36 );
37 };
38
39 describe('skill: every command it teaches exists', () => {
40 let rig: CliRig;
41 const unknown: Array<{ file: string; line: string; error: string }> = [];
42 const commands = taughtCommands();
43
44 beforeAll(async () => {
45 rig = await startCliRig({ label: 'skill-commands' });
46 const paths = new Map(commands.map(command => [command.words.join(' '), command]));
47 for (const [, command] of paths) {
48 // `composio help` rejects an argument and an unknown subcommand differently.
49 // Trailing words that are arguments (`link gmail`) are dropped one at a
50 // time; a word that is not a subcommand stops the search. `<path> --help`
51 // cannot be used: it prints the parent's help and exits 0 for both.
52 for (let length = command.words.length; length > 0; length -= 1) {
53 const result = await rig.run(`composio help ${command.words.slice(0, length).join(' ')}`);
54 if (result.exitCode === 0) break;
55 if (!result.stderr.includes('Unexpected positional argument')) {
56 const error = /Unknown subcommand[^\n]*/.exec(result.stderr)?.[0] ?? result.stderr.trim();
57 unknown.push({ file: command.file, line: command.line, error });
58 break;
59 }
60 }
61 }
62 }, RIG_TIMEOUT_MS);
63
64 afterAll(async () => {
65 await rig.stop();
66 });
67
68 it('finds command lines in the skill and its references', () => {
69 expect(commands.length).toBeGreaterThan(20);
70 expect(new Set(commands.map(command => command.file))).toEqual(
71 new Set(['SKILL.md', 'composio-dev.md', 'power-user-examples.md', 'troubleshooting.md'])
72 );
73 });
74
75 it('SKILL.md teaches only commands the binary has', () => {
76 expect(unknown.filter(command => command.file === 'SKILL.md')).toEqual([]);
77 });
78
79 it('the references teach only commands the binary has', () => {
80 expect(
81 unknown
82 .filter(command => command.file !== 'SKILL.md')
83 .map(command => `${command.file}: ${command.line}`)
84 ).toEqual([]);
85 });
86 });