Composio powers 1000+ toolkits, tool search, context management, authentication, and a sandboxed workbench to help you build AI agents that turn intent into action.

next
ts 169 lines 8.57 KB
Raw
1 #!/usr/bin/env bun
2 /**
3 * Local end-to-end run of KB publication, docs side: `bun run test:kb-publish` from docs/.
4 *
5 * It replays what happens after the support portal sends `kb-guide-approved`, using the real
6 * steps from .github/workflows/docs-kb-publish.yml (same scripts, same bash flags as Actions):
7 * 1. two portal dispatches, each applied on its own branch off this checkout's HEAD (= two KB PRs);
8 * 2. both "PRs" merged into the base in turn (proves open KB PRs don't conflict);
9 * 3. the merged KB regenerated, and the new guides looked up through Ask AI's search_docs and read_doc.
10 * Everything happens in a throwaway git worktree; this checkout is never modified.
11 * Pass --keep to leave the worktree for previewing (it prints the path). `next dev` there needs a real
12 * node_modules, not this symlink: `rm node_modules && cp -cR <docs>/node_modules node_modules` on macOS.
13 */
14 import { spawnSync } from 'node:child_process';
15 import { mkdtempSync, readFileSync, rmSync, symlinkSync } from 'node:fs';
16 import { tmpdir } from 'node:os';
17 import { join, resolve } from 'node:path';
18
19 const repo = resolve(import.meta.dir, '../../..');
20 const keep = process.argv.includes('--keep');
21 const runDir = mkdtempSync(join(tmpdir(), 'kb-e2e-'));
22 const work = join(runDir, 'repo');
23 // Branches live in the shared repo, so each run gets its own names.
24 const prefix = `kb-e2e/${runDir.split('-').at(-1)}`;
25 const base = `${prefix}/base`;
26 const prBranch = (draftId: number) => `${prefix}/docs/kb-front-desk-${draftId}`;
27 const branches = () => [base, ...dispatches.map(({ draft_id }) => prBranch(draft_id))];
28 const docs = join(work, 'docs');
29 const runnerTemp = mkdtempSync(join(tmpdir(), 'kb-e2e-runner-'));
30
31 // The guide objects exactly as core/knowledge/kb.py `_checked()` builds them for client_payload.guide.
32 const dispatches = [
33 {
34 draft_id: 9001,
35 guide: {
36 action: 'upsert', slug: 'sdk-e2e-webhook-retries', area: 'sdk',
37 title: 'Webhook delivery retries',
38 description: 'How trigger webhook deliveries are retried after a failed response.',
39 body: '## When a delivery is retried\n\nA trigger webhook delivery is retried when your endpoint returns a 5xx status or times out.\n\n## Making your handler idempotent\n\nUse the event ID to ignore a delivery you already processed.',
40 topics: ['workflows'], tags: ['triggers', 'webhooks'],
41 },
42 },
43 {
44 draft_id: 9002,
45 guide: {
46 action: 'upsert', slug: 'platform-e2e-session-limits', area: 'platform',
47 title: 'Session tool call limits',
48 description: 'What happens when a session reaches its tool call limit.',
49 body: '## Hitting the limit\n\nA session that reaches its tool call limit returns a limit error for further calls.\n\n## Starting a new session\n\nCreate a new session for the user to continue.',
50 topics: ['sessions-and-execution'], tags: ['sessions'],
51 },
52 },
53 ];
54
55 type Step = { name: string; run?: string };
56 const workflow = Bun.YAML.parse(
57 readFileSync(join(repo, '.github/workflows/docs-kb-publish.yml'), 'utf8'),
58 ) as { jobs: { publish: { steps: Step[] } } };
59
60 function sh(command: 'git' | 'bun', args: string[], cwd: string, env: Record<string, string> = {}) {
61 const result = spawnSync(command, args, {
62 cwd,
63 encoding: 'utf8',
64 env: { ...process.env, ...env },
65 shell: false,
66 });
67 return { ok: result.status === 0, out: `${result.stdout ?? ''}${result.stderr ?? ''}`.trim() };
68 }
69 function must(command: 'git' | 'bun', args: string[], cwd = work, env: Record<string, string> = {}) {
70 const result = sh(command, args, cwd, env);
71 if (!result.ok) throw new Error(`${command} ${args.join(' ')} failed:\n${result.out}`);
72 return result.out;
73 }
74 function workflowStep(name: string, env: Record<string, string>) {
75 const script = workflow.jobs.publish.steps.find(step => step.name === name)?.run;
76 if (!script) throw new Error(`docs-kb-publish.yml has no step "${name}"`);
77 // The job's default shell is bash, which Actions runs with -e -o pipefail.
78 const result = spawnSync('bash', ['--noprofile', '--norc', '-e', '-o', 'pipefail', '-s'], {
79 cwd: docs,
80 encoding: 'utf8',
81 env: { ...process.env, RUNNER_TEMP: runnerTemp, ...env },
82 input: script,
83 shell: false,
84 });
85 return { ok: result.status === 0, out: `${result.stdout ?? ''}${result.stderr ?? ''}`.trim() };
86 }
87
88 let failed = false;
89 const check = (ok: boolean, label: string, detail = '') => {
90 console.log(`${ok ? 'PASS' : 'FAIL'} ${label}${detail ? `\n ${detail.replaceAll('\n', '\n ')}` : ''}`);
91 if (!ok) failed = true;
92 };
93
94 try {
95 must('git', ['-C', repo, 'worktree', 'add', '-q', '--detach', work, 'HEAD'], repo);
96 symlinkSync(join(repo, 'docs/node_modules'), join(docs, 'node_modules'));
97 must('git', ['switch', '-q', '-c', base]);
98 const git = ['-c', 'user.name=kb-e2e', '-c', 'user.email=kb-e2e@localhost', '-c', 'commit.gpgsign=false'];
99
100 // 1. Each dispatch becomes its own PR branch, built by the workflow's own steps.
101 for (const { draft_id, guide } of dispatches) {
102 const branch = prBranch(draft_id);
103 must('git', ['switch', '-q', '-c', branch, base]);
104 const apply = workflowStep('Apply the approved guide', { GUIDE: JSON.stringify(guide) });
105 check(apply.ok, `${branch}: "Apply the approved guide"`, apply.ok ? '' : apply.out.slice(-1500));
106 const regenerate = workflowStep('Regenerate KB pages', { GUIDE_SLUG: guide.slug });
107 const errors = regenerate.ok ? [] : (() => {
108 try {
109 const report = JSON.parse(readFileSync(join(runnerTemp, 'kb-verify.json'), 'utf8')) as {
110 findings: { severity: string; kind: string }[];
111 };
112 const counts = new Map<string, number>();
113 for (const finding of report.findings.filter(item => item.severity === 'error')) {
114 counts.set(finding.kind, (counts.get(finding.kind) ?? 0) + 1);
115 }
116 return [...counts].map(([kind, count]) => `${count} x ${kind}`);
117 } catch {
118 return [regenerate.out.slice(-1500)];
119 }
120 })();
121 check(regenerate.ok, `${branch}: "Regenerate KB pages" (the PR is only opened if this passes)`,
122 regenerate.ok ? '' : `verify:kb errors: ${errors.join(', ')}`);
123 // Stage what create-pull-request would (add-paths: docs/kb/).
124 must('git', ['add', 'docs/kb']);
125 const files = must('git', ['diff', '--cached', '--name-only']).split('\n').filter(Boolean);
126 const ownPaths = [guide.slug, `${guide.area}/${guide.slug.slice(guide.area.length + 1)}/`];
127 check(files.every(file => ownPaths.some(path => file.includes(path))),
128 `${branch}: the PR only touches this guide's own files`, files.join('\n'));
129 must('git', [...git, 'commit', '-q', '-m', `docs(kb): ${guide.title}`]);
130 }
131
132 // 2. Both PRs wait for review side by side, then merge in turn.
133 must('git', ['switch', '-q', base]);
134 for (const { draft_id } of dispatches) {
135 const merge = sh('git', [...git, 'merge', '-q', '--no-edit', prBranch(draft_id)], work);
136 check(merge.ok, `merge ${prBranch(draft_id)} after the other`, merge.ok ? '' : merge.out);
137 }
138
139 // 3. After merge, the build regenerates content/kb and Ask AI's index and finds the guides.
140 must('bun', ['run', 'generate:kb'], docs);
141 must('bun', ['scripts/build-agent-index.ts'], docs);
142 const probe = must('bun', ['-e', `
143 const { searchDocs } = await import('./agent/lib/docs-search.ts');
144 const { readPageByUrl } = await import('./agent/lib/docs.ts');
145 const queries = ${JSON.stringify(dispatches.map(({ guide }) => [guide.slug, guide.title]))};
146 console.log(JSON.stringify(queries.map(([slug, title]) => ({
147 slug,
148 found: searchDocs(title, { limit: 5 }).results.some(r => r.url === '/kb/guide/' + slug),
149 readable: readPageByUrl('/kb/guide/' + slug)?.title === title,
150 }))));
151 `], docs, { EVE_FORCE_BUNDLE: '1' });
152 for (const result of JSON.parse(probe.split('\n').at(-1)!) as { slug: string; found: boolean; readable: boolean }[]) {
153 check(result.found, `Ask AI search_docs finds /kb/guide/${result.slug} (deployed bundle)`);
154 check(result.readable, `Ask AI read_doc opens /kb/guide/${result.slug}`);
155 }
156 } catch (error) {
157 check(false, 'run aborted', String(error));
158 } finally {
159 if (keep) {
160 console.log(`\nWorktree kept at ${work}\nPreview: cd ${docs} && bun run dev, then open /kb/guide/${dispatches[0]!.guide.slug}`);
161 console.log(`Remove with: git -C ${repo} worktree remove --force ${work} && git -C ${repo} branch -D ${branches().join(' ')}`);
162 } else {
163 sh('git', ['-C', repo, 'worktree', 'remove', '--force', work], repo);
164 sh('git', ['-C', repo, 'branch', '-D', ...branches()], repo);
165 }
166 rmSync(runnerTemp, { recursive: true, force: true });
167 }
168 console.log(failed ? '\nE2E FAILED' : '\nE2E PASSED');
169 process.exit(failed ? 1 : 0);