Composio powers 1000+ toolkits, tool search, context management, authentication, and a sandboxed workbench to help you build AI agents that turn intent into action.
| 1 | #!/usr/bin/env bun |
| 2 | /** |
| 3 | * Local end-to-end run of KB publication, docs side: `bun run test:kb-publish` from docs/. |
| 4 | * |
| 5 | * It replays what happens after the support portal sends `kb-guide-approved`, using the real |
| 6 | * steps from .github/workflows/docs-kb-publish.yml (same scripts, same bash flags as Actions): |
| 7 | * 1. two portal dispatches, each applied on its own branch off this checkout's HEAD (= two KB PRs); |
| 8 | * 2. both "PRs" merged into the base in turn (proves open KB PRs don't conflict); |
| 9 | * 3. the merged KB regenerated, and the new guides looked up through Ask AI's search_docs and read_doc. |
| 10 | * Everything happens in a throwaway git worktree; this checkout is never modified. |
| 11 | * Pass --keep to leave the worktree for previewing (it prints the path). `next dev` there needs a real |
| 12 | * node_modules, not this symlink: `rm node_modules && cp -cR <docs>/node_modules node_modules` on macOS. |
| 13 | */ |
| 14 | import { spawnSync } from 'node:child_process'; |
| 15 | import { mkdtempSync, readFileSync, rmSync, symlinkSync } from 'node:fs'; |
| 16 | import { tmpdir } from 'node:os'; |
| 17 | import { join, resolve } from 'node:path'; |
| 18 | |
| 19 | const repo = resolve(import.meta.dir, '../../..'); |
| 20 | const keep = process.argv.includes('--keep'); |
| 21 | const runDir = mkdtempSync(join(tmpdir(), 'kb-e2e-')); |
| 22 | const work = join(runDir, 'repo'); |
| 23 | // Branches live in the shared repo, so each run gets its own names. |
| 24 | const prefix = `kb-e2e/${runDir.split('-').at(-1)}`; |
| 25 | const base = `${prefix}/base`; |
| 26 | const prBranch = (draftId: number) => `${prefix}/docs/kb-front-desk-${draftId}`; |
| 27 | const branches = () => [base, ...dispatches.map(({ draft_id }) => prBranch(draft_id))]; |
| 28 | const docs = join(work, 'docs'); |
| 29 | const runnerTemp = mkdtempSync(join(tmpdir(), 'kb-e2e-runner-')); |
| 30 | |
| 31 | // The guide objects exactly as core/knowledge/kb.py `_checked()` builds them for client_payload.guide. |
| 32 | const dispatches = [ |
| 33 | { |
| 34 | draft_id: 9001, |
| 35 | guide: { |
| 36 | action: 'upsert', slug: 'sdk-e2e-webhook-retries', area: 'sdk', |
| 37 | title: 'Webhook delivery retries', |
| 38 | description: 'How trigger webhook deliveries are retried after a failed response.', |
| 39 | body: '## When a delivery is retried\n\nA trigger webhook delivery is retried when your endpoint returns a 5xx status or times out.\n\n## Making your handler idempotent\n\nUse the event ID to ignore a delivery you already processed.', |
| 40 | topics: ['workflows'], tags: ['triggers', 'webhooks'], |
| 41 | }, |
| 42 | }, |
| 43 | { |
| 44 | draft_id: 9002, |
| 45 | guide: { |
| 46 | action: 'upsert', slug: 'platform-e2e-session-limits', area: 'platform', |
| 47 | title: 'Session tool call limits', |
| 48 | description: 'What happens when a session reaches its tool call limit.', |
| 49 | body: '## Hitting the limit\n\nA session that reaches its tool call limit returns a limit error for further calls.\n\n## Starting a new session\n\nCreate a new session for the user to continue.', |
| 50 | topics: ['sessions-and-execution'], tags: ['sessions'], |
| 51 | }, |
| 52 | }, |
| 53 | ]; |
| 54 | |
| 55 | type Step = { name: string; run?: string }; |
| 56 | const workflow = Bun.YAML.parse( |
| 57 | readFileSync(join(repo, '.github/workflows/docs-kb-publish.yml'), 'utf8'), |
| 58 | ) as { jobs: { publish: { steps: Step[] } } }; |
| 59 | |
| 60 | function sh(command: 'git' | 'bun', args: string[], cwd: string, env: Record<string, string> = {}) { |
| 61 | const result = spawnSync(command, args, { |
| 62 | cwd, |
| 63 | encoding: 'utf8', |
| 64 | env: { ...process.env, ...env }, |
| 65 | shell: false, |
| 66 | }); |
| 67 | return { ok: result.status === 0, out: `${result.stdout ?? ''}${result.stderr ?? ''}`.trim() }; |
| 68 | } |
| 69 | function must(command: 'git' | 'bun', args: string[], cwd = work, env: Record<string, string> = {}) { |
| 70 | const result = sh(command, args, cwd, env); |
| 71 | if (!result.ok) throw new Error(`${command} ${args.join(' ')} failed:\n${result.out}`); |
| 72 | return result.out; |
| 73 | } |
| 74 | function workflowStep(name: string, env: Record<string, string>) { |
| 75 | const script = workflow.jobs.publish.steps.find(step => step.name === name)?.run; |
| 76 | if (!script) throw new Error(`docs-kb-publish.yml has no step "${name}"`); |
| 77 | // The job's default shell is bash, which Actions runs with -e -o pipefail. |
| 78 | const result = spawnSync('bash', ['--noprofile', '--norc', '-e', '-o', 'pipefail', '-s'], { |
| 79 | cwd: docs, |
| 80 | encoding: 'utf8', |
| 81 | env: { ...process.env, RUNNER_TEMP: runnerTemp, ...env }, |
| 82 | input: script, |
| 83 | shell: false, |
| 84 | }); |
| 85 | return { ok: result.status === 0, out: `${result.stdout ?? ''}${result.stderr ?? ''}`.trim() }; |
| 86 | } |
| 87 | |
| 88 | let failed = false; |
| 89 | const check = (ok: boolean, label: string, detail = '') => { |
| 90 | console.log(`${ok ? 'PASS' : 'FAIL'} ${label}${detail ? `\n ${detail.replaceAll('\n', '\n ')}` : ''}`); |
| 91 | if (!ok) failed = true; |
| 92 | }; |
| 93 | |
| 94 | try { |
| 95 | must('git', ['-C', repo, 'worktree', 'add', '-q', '--detach', work, 'HEAD'], repo); |
| 96 | symlinkSync(join(repo, 'docs/node_modules'), join(docs, 'node_modules')); |
| 97 | must('git', ['switch', '-q', '-c', base]); |
| 98 | const git = ['-c', 'user.name=kb-e2e', '-c', 'user.email=kb-e2e@localhost', '-c', 'commit.gpgsign=false']; |
| 99 | |
| 100 | // 1. Each dispatch becomes its own PR branch, built by the workflow's own steps. |
| 101 | for (const { draft_id, guide } of dispatches) { |
| 102 | const branch = prBranch(draft_id); |
| 103 | must('git', ['switch', '-q', '-c', branch, base]); |
| 104 | const apply = workflowStep('Apply the approved guide', { GUIDE: JSON.stringify(guide) }); |
| 105 | check(apply.ok, `${branch}: "Apply the approved guide"`, apply.ok ? '' : apply.out.slice(-1500)); |
| 106 | const regenerate = workflowStep('Regenerate KB pages', { GUIDE_SLUG: guide.slug }); |
| 107 | const errors = regenerate.ok ? [] : (() => { |
| 108 | try { |
| 109 | const report = JSON.parse(readFileSync(join(runnerTemp, 'kb-verify.json'), 'utf8')) as { |
| 110 | findings: { severity: string; kind: string }[]; |
| 111 | }; |
| 112 | const counts = new Map<string, number>(); |
| 113 | for (const finding of report.findings.filter(item => item.severity === 'error')) { |
| 114 | counts.set(finding.kind, (counts.get(finding.kind) ?? 0) + 1); |
| 115 | } |
| 116 | return [...counts].map(([kind, count]) => `${count} x ${kind}`); |
| 117 | } catch { |
| 118 | return [regenerate.out.slice(-1500)]; |
| 119 | } |
| 120 | })(); |
| 121 | check(regenerate.ok, `${branch}: "Regenerate KB pages" (the PR is only opened if this passes)`, |
| 122 | regenerate.ok ? '' : `verify:kb errors: ${errors.join(', ')}`); |
| 123 | // Stage what create-pull-request would (add-paths: docs/kb/). |
| 124 | must('git', ['add', 'docs/kb']); |
| 125 | const files = must('git', ['diff', '--cached', '--name-only']).split('\n').filter(Boolean); |
| 126 | const ownPaths = [guide.slug, `${guide.area}/${guide.slug.slice(guide.area.length + 1)}/`]; |
| 127 | check(files.every(file => ownPaths.some(path => file.includes(path))), |
| 128 | `${branch}: the PR only touches this guide's own files`, files.join('\n')); |
| 129 | must('git', [...git, 'commit', '-q', '-m', `docs(kb): ${guide.title}`]); |
| 130 | } |
| 131 | |
| 132 | // 2. Both PRs wait for review side by side, then merge in turn. |
| 133 | must('git', ['switch', '-q', base]); |
| 134 | for (const { draft_id } of dispatches) { |
| 135 | const merge = sh('git', [...git, 'merge', '-q', '--no-edit', prBranch(draft_id)], work); |
| 136 | check(merge.ok, `merge ${prBranch(draft_id)} after the other`, merge.ok ? '' : merge.out); |
| 137 | } |
| 138 | |
| 139 | // 3. After merge, the build regenerates content/kb and Ask AI's index and finds the guides. |
| 140 | must('bun', ['run', 'generate:kb'], docs); |
| 141 | must('bun', ['scripts/build-agent-index.ts'], docs); |
| 142 | const probe = must('bun', ['-e', ` |
| 143 | const { searchDocs } = await import('./agent/lib/docs-search.ts'); |
| 144 | const { readPageByUrl } = await import('./agent/lib/docs.ts'); |
| 145 | const queries = ${JSON.stringify(dispatches.map(({ guide }) => [guide.slug, guide.title]))}; |
| 146 | console.log(JSON.stringify(queries.map(([slug, title]) => ({ |
| 147 | slug, |
| 148 | found: searchDocs(title, { limit: 5 }).results.some(r => r.url === '/kb/guide/' + slug), |
| 149 | readable: readPageByUrl('/kb/guide/' + slug)?.title === title, |
| 150 | })))); |
| 151 | `], docs, { EVE_FORCE_BUNDLE: '1' }); |
| 152 | for (const result of JSON.parse(probe.split('\n').at(-1)!) as { slug: string; found: boolean; readable: boolean }[]) { |
| 153 | check(result.found, `Ask AI search_docs finds /kb/guide/${result.slug} (deployed bundle)`); |
| 154 | check(result.readable, `Ask AI read_doc opens /kb/guide/${result.slug}`); |
| 155 | } |
| 156 | } catch (error) { |
| 157 | check(false, 'run aborted', String(error)); |
| 158 | } finally { |
| 159 | if (keep) { |
| 160 | console.log(`\nWorktree kept at ${work}\nPreview: cd ${docs} && bun run dev, then open /kb/guide/${dispatches[0]!.guide.slug}`); |
| 161 | console.log(`Remove with: git -C ${repo} worktree remove --force ${work} && git -C ${repo} branch -D ${branches().join(' ')}`); |
| 162 | } else { |
| 163 | sh('git', ['-C', repo, 'worktree', 'remove', '--force', work], repo); |
| 164 | sh('git', ['-C', repo, 'branch', '-D', ...branches()], repo); |
| 165 | } |
| 166 | rmSync(runnerTemp, { recursive: true, force: true }); |
| 167 | } |
| 168 | console.log(failed ? '\nE2E FAILED' : '\nE2E PASSED'); |
| 169 | process.exit(failed ? 1 : 0); |