Composio powers 1000+ toolkits, tool search, context management, authentication, and a sandboxed workbench to help you build AI agents that turn intent into action.

next
ts 338 lines 14.1 KB
Raw
1 import { describe, expect, test } from 'bun:test';
2 import { existsSync, readFileSync } from 'node:fs';
3 import { resolve } from 'node:path';
4
5 const workflowPath = resolve(process.cwd(), '..', '.github', 'workflows', 'docs-update-kb.yml');
6 const semanticRefreshWorkflowPath = resolve(
7 process.cwd(),
8 '..',
9 '.github',
10 'workflows',
11 'docs-rebuild-kb-semantic.yml',
12 );
13 const docsTestsWorkflowPath = resolve(
14 process.cwd(),
15 '..',
16 '.github',
17 'workflows',
18 'docs-tests.yml',
19 );
20 const dataWorkflowPath = resolve(
21 process.cwd(),
22 '..',
23 '.github',
24 'workflows',
25 'docs-update-data.yml',
26 );
27
28 interface WorkflowStep {
29 name?: string;
30 id?: string;
31 if?: string;
32 uses?: string;
33 'continue-on-error'?: boolean;
34 'working-directory'?: string;
35 env?: Record<string, unknown>;
36 with?: Record<string, unknown>;
37 run?: string;
38 }
39
40 interface WorkflowJob {
41 needs?: string | string[];
42 if?: string;
43 outputs?: Record<string, string>;
44 env?: Record<string, unknown>;
45 permissions?: Record<string, string>;
46 steps?: WorkflowStep[];
47 }
48
49 function workflowSteps(path: string, job: string): WorkflowStep[] {
50 const workflow = Bun.YAML.parse(readFileSync(path, 'utf8')) as {
51 jobs?: Record<string, { steps?: WorkflowStep[] }>;
52 };
53 return workflow.jobs?.[job]?.steps ?? [];
54 }
55
56 describe('KB semantic artifact workflows', () => {
57 test('allows stale embeddings in PR checks without ignoring artifact errors', () => {
58 const check = workflowSteps(docsTestsWorkflowPath, 'test')
59 .find(step => step.name === 'Check KB semantic artifact');
60 expect(check?.run).toBe('bun run check:kb-semantic --allow-stale');
61 expect(check?.['continue-on-error']).not.toBe(true);
62 });
63
64 test('rebuilds stale semantic artifacts from default-branch code only for trusted current pull requests', () => {
65 const workflow = existsSync(semanticRefreshWorkflowPath)
66 ? Bun.YAML.parse(readFileSync(semanticRefreshWorkflowPath, 'utf8')) as {
67 on?: {
68 workflow_run?: { workflows?: string[]; types?: string[] };
69 pull_request?: unknown;
70 pull_request_target?: unknown;
71 };
72 permissions?: Record<string, string>;
73 env?: Record<string, unknown>;
74 jobs?: Record<string, WorkflowJob>;
75 }
76 : {};
77 const authorize = workflow.jobs?.authorize;
78 const authorizeStep = authorize?.steps?.find(
79 step => step.name === 'Authorize originating pull request',
80 );
81 const rebuild = workflow.jobs?.rebuild;
82 const steps = rebuild?.steps ?? [];
83 const checkout = steps.find(step => step.name === 'Checkout pull request');
84 const freshness = steps.find(step => step.name === 'Check semantic artifact freshness');
85 const build = steps.find(step => step.name === 'Rebuild semantic artifact');
86 const verify = steps.find(step => step.name === 'Verify rebuilt artifact');
87 const writeToken = steps.find(step => step.name === 'Generate write token');
88 const commit = steps.find(step => step.name === 'Commit rebuilt artifact');
89
90 expect(workflow.on?.workflow_run?.workflows).toEqual(['Docs - Tests']);
91 expect(workflow.on?.workflow_run?.types).toEqual(['completed']);
92 expect(workflow.on).not.toHaveProperty('pull_request');
93 expect(workflow.on).not.toHaveProperty('pull_request_target');
94 expect(workflow.permissions?.contents).toBe('read');
95 expect(workflow.env).toBeUndefined();
96
97 expect(authorize?.permissions).toEqual({
98 actions: 'read',
99 contents: 'read',
100 'pull-requests': 'read',
101 });
102 expect(authorize?.outputs).toEqual({
103 trusted: '${{ steps.trust.outputs.trusted }}',
104 head_sha: '${{ steps.trust.outputs.head_sha }}',
105 head_ref: '${{ steps.trust.outputs.head_ref }}',
106 });
107 expect(authorize?.env).toBeUndefined();
108 expect(authorizeStep?.env).toEqual({
109 GH_TOKEN: '${{ github.token }}',
110 REPOSITORY: '${{ github.repository }}',
111 RUN_ID: '${{ github.event.workflow_run.id }}',
112 });
113 expect(authorize?.steps?.map(step => step.name)).toEqual([
114 'Authorize originating pull request',
115 ]);
116 expect(authorizeStep?.run?.trim()).toBe(`echo "trusted=false" >> "$GITHUB_OUTPUT"
117
118 run_json=$(gh api "repos/$REPOSITORY/actions/runs/$RUN_ID")
119 event_name=$(jq -r '.event' <<< "$run_json")
120 repository_id=$(jq -r '.repository.id' <<< "$run_json")
121 head_repository_id=$(jq -r '.head_repository.id // empty' <<< "$run_json")
122 head_branch=$(jq -r '.head_branch // empty' <<< "$run_json")
123 head_sha=$(jq -r '.head_sha // empty' <<< "$run_json")
124
125 if [ "$event_name" != "pull_request" ] ||
126 [ -z "$head_repository_id" ] ||
127 [ "$head_repository_id" != "$repository_id" ] ||
128 [ -z "$head_branch" ] ||
129 [ -z "$head_sha" ]; then
130 echo "Skipping a non-pull-request or fork workflow run."
131 exit 0
132 fi
133
134 pull_matches=$(jq -c \\
135 --arg head_repository_id "$head_repository_id" \\
136 --arg head_branch "$head_branch" \\
137 --arg head_sha "$head_sha" \\
138 '[
139 (.pull_requests // [])[]
140 | select(
141 (.head.repo.id | tostring) == $head_repository_id and
142 .head.ref == $head_branch and
143 .head.sha == $head_sha
144 )
145 ]' <<< "$run_json")
146
147 if [ "$(jq 'length' <<< "$pull_matches")" != "1" ]; then
148 echo "Skipping a workflow run without exactly one matching pull request."
149 exit 0
150 fi
151
152 pull_number=$(jq -r '.[0].number' <<< "$pull_matches")
153 pull_json=$(gh api "repos/$REPOSITORY/pulls/$pull_number")
154 association=$(jq -r '.author_association' <<< "$pull_json")
155 live_head_repository_id=$(jq -r '.head.repo.id // empty' <<< "$pull_json")
156 live_head_sha=$(jq -r '.head.sha // empty' <<< "$pull_json")
157 live_head_ref=$(jq -r '.head.ref // empty' <<< "$pull_json")
158 state=$(jq -r '.state' <<< "$pull_json")
159
160 if [ "$state" != "open" ] ||
161 [ "$live_head_repository_id" != "$repository_id" ] ||
162 [ "$live_head_sha" != "$head_sha" ] ||
163 [ "$live_head_ref" != "$head_branch" ]; then
164 echo "Skipping a closed or stale pull request."
165 exit 0
166 fi
167
168 case "$association" in
169 MEMBER|OWNER) ;;
170 *)
171 echo "Skipping a pull request whose author is not a repository member or owner."
172 exit 0
173 ;;
174 esac
175
176 echo "trusted=true" >> "$GITHUB_OUTPUT"
177 echo "head_sha=$live_head_sha" >> "$GITHUB_OUTPUT"
178 echo "head_ref=$live_head_ref" >> "$GITHUB_OUTPUT"`);
179
180 expect(rebuild?.needs).toBe('authorize');
181 expect(rebuild?.if).toBe("needs.authorize.outputs.trusted == 'true'");
182 expect(rebuild?.permissions?.contents).toBe('read');
183 expect(rebuild?.env).toBeUndefined();
184 expect(steps.map(step => step.name)).toEqual([
185 'Checkout pull request',
186 'Setup Node.js, pnpm, Bun',
187 'Cache Bun dependencies',
188 'Install dependencies',
189 'Check semantic artifact freshness',
190 'Rebuild semantic artifact',
191 'Verify rebuilt artifact',
192 'Generate write token',
193 'Resolve bot identity',
194 'Commit rebuilt artifact',
195 ]);
196 expect(checkout?.with?.ref).toBe('${{ needs.authorize.outputs.head_sha }}');
197 expect(checkout?.with?.['persist-credentials']).toBe(false);
198 expect(freshness?.['continue-on-error']).toBe(true);
199 expect(freshness?.run).toBe('bun run check:kb-semantic');
200 expect(build?.if).toBe("steps.freshness.outcome == 'failure'");
201 expect(build?.env).toEqual({ OPENAI_API_KEY: '${{ secrets.OPENAI_API_KEY }}' });
202 expect(verify?.if).toBe("steps.freshness.outcome == 'failure'");
203 expect(verify?.run).toBe('bun run check:kb-semantic');
204 expect(writeToken?.if).toBe("steps.freshness.outcome == 'failure'");
205 expect(writeToken?.with).toEqual({
206 'client-id': '${{ vars.RELEASE_BOT_CLIENT_ID }}',
207 'private-key': '${{ secrets.RELEASE_BOT_APP_PRIVATE_KEY }}',
208 owner: 'ComposioHQ',
209 repositories: 'composio',
210 'permission-contents': 'write',
211 });
212 expect(commit?.if).toBe("steps.freshness.outcome == 'failure'");
213 expect(commit?.env?.HEAD_REF).toBe('${{ needs.authorize.outputs.head_ref }}');
214 expect(commit?.run).toContain('git push origin "HEAD:$HEAD_REF"');
215 expect(commit?.run).not.toMatch(/git push[^\n]*(?:--force|\s-f(?:\s|$))/);
216 expect(steps.indexOf(writeToken)).toBeGreaterThan(steps.indexOf(verify));
217
218 expect(
219 steps.filter(step => JSON.stringify(step).includes('secrets.OPENAI_API_KEY'))
220 .map(step => step.name),
221 ).toEqual(['Rebuild semantic artifact']);
222 expect(
223 steps.filter(step => JSON.stringify(step).includes('RELEASE_BOT_'))
224 .map(step => step.name),
225 ).toEqual(['Generate write token']);
226
227 const allSteps = Object.values(workflow.jobs ?? {}).flatMap(job => job.steps ?? []);
228 expect(
229 allSteps.filter(step => JSON.stringify(step).includes('${{ secrets.'))
230 .map(step => step.name),
231 ).toEqual(['Rebuild semantic artifact', 'Generate write token']);
232
233 const docsTestsWorkflow = Bun.YAML.parse(readFileSync(docsTestsWorkflowPath, 'utf8')) as {
234 on?: { pull_request?: { paths?: string[] } };
235 };
236 expect(docsTestsWorkflow.on?.pull_request?.paths).toContain(
237 '.github/workflows/docs-rebuild-kb-semantic.yml',
238 );
239 });
240
241 test('repairs the semantic artifact on a schedule without any upstream source', () => {
242 const workflow = readFileSync(workflowPath, 'utf8');
243 const parsed = Bun.YAML.parse(workflow) as { on?: Record<string, unknown> };
244
245 expect(Object.keys(parsed.on ?? {}).sort()).toEqual(['schedule', 'workflow_dispatch']);
246 expect(workflow).not.toContain('support-knowledge');
247 expect(workflow).toContain('bun run build:kb-semantic');
248 expect(workflow).toContain('bun run check:kb-semantic');
249 expect(workflow).toContain('branch: docs/auto-update-kb');
250 expect(workflow).toContain('base: next');
251 expect(workflow).toContain('persist-credentials: false');
252 });
253
254 test('serializes both KB writers and pins artifacts to their docs base', () => {
255 const workflow = Bun.YAML.parse(readFileSync(workflowPath, 'utf8')) as {
256 concurrency?: { group?: string; 'cancel-in-progress'?: boolean; queue?: string };
257 jobs?: Record<string, WorkflowJob>;
258 };
259 const publish = Bun.YAML.parse(readFileSync(
260 resolve(process.cwd(), '..', '.github/workflows/docs-kb-publish.yml'), 'utf8',
261 )) as { concurrency: { group: string }; jobs: Record<string, WorkflowJob> };
262 expect(workflow.concurrency?.group).toBe(publish.concurrency.group);
263 expect(workflow.concurrency?.['cancel-in-progress']).toBe(false);
264 expect(workflow.concurrency?.queue).toBe('max');
265 for (const job of [workflow.jobs?.refresh, publish.jobs.publish]) {
266 const checkout = job?.steps?.find(step => step.name === 'Checkout docs repository');
267 expect(checkout?.with?.ref).toBe('next');
268 }
269 const proposalCheckout = workflow.jobs?.propose?.steps?.find(
270 step => step.name === 'Checkout docs repository',
271 );
272 expect(workflow.jobs?.refresh?.outputs?.docs_base_sha).toBe('${{ steps.docs-base.outputs.sha }}');
273 expect(proposalCheckout?.with?.ref).toBe('${{ needs.refresh.outputs.docs_base_sha }}');
274 });
275
276 test('tracks failures until both rebuild and PR proposal recover', () => {
277 const workflow = Bun.YAML.parse(readFileSync(workflowPath, 'utf8')) as {
278 jobs?: Record<string, WorkflowJob>;
279 };
280 const reporter = workflow.jobs?.report;
281 const failure = reporter?.steps?.find(step => step.name === 'Open KB repair failure issue');
282 const recovery = reporter?.steps?.find(step => step.name === 'Close KB repair failure issue');
283
284 expect(reporter?.needs).toEqual(['refresh', 'propose']);
285 expect(reporter?.if).toBe('always()');
286 expect(reporter?.permissions?.issues).toBe('write');
287 expect(failure?.if).toContain("needs.refresh.result == 'failure'");
288 expect(failure?.if).toContain("needs.propose.result == 'failure'");
289 expect(failure?.env?.GH_TOKEN).toBe('${{ secrets.GITHUB_TOKEN }}');
290 expect(failure?.env?.LABEL).toBe('docs-kb-refresh-failure');
291 expect(failure?.run).toContain('gh issue list');
292 expect(failure?.run).toContain('gh issue create');
293 expect(recovery?.if).toContain("needs.refresh.result == 'success'");
294 expect(recovery?.if).toContain("needs.refresh.outputs.changed != 'true'");
295 expect(recovery?.if).toContain("needs.propose.result == 'success'");
296 expect(recovery?.env?.LABEL).toBe('docs-kb-refresh-failure');
297 expect(recovery?.run).toContain('gh issue close');
298 });
299
300 test('transfers the semantic index to the exact path staged by the repair PR', () => {
301 const upload = workflowSteps(workflowPath, 'refresh').find(
302 step => step.name === 'Upload verified semantic artifact',
303 );
304 const steps = workflowSteps(workflowPath, 'propose');
305 const download = steps.find(step => step.name === 'Download verified semantic artifact');
306 const createPullRequest = steps.find(step => step.name === 'Create repair pull request');
307
308 expect(upload?.with?.name).toBe('kb-semantic-repair');
309 expect(download?.with?.name).toBe(upload?.with?.name);
310 expect(upload?.with?.path).toBe('docs/kb/semantic-index.json');
311 expect(download?.with?.path).toBe('docs/kb');
312 expect(String(createPullRequest?.with?.['add-paths']).trim()).toBe(upload?.with?.path);
313 });
314
315 test('treats keyword-only production retrieval as a semantic health failure', () => {
316 const steps = workflowSteps(
317 resolve(process.cwd(), '..', '.github', 'workflows', 'docs.health-check.yml'),
318 'health-check',
319 );
320 const endpointCheck = steps.find(step => step.name === 'Check endpoints');
321
322 expect(endpointCheck?.run).toContain('.mode == "hybrid"');
323 expect(endpointCheck?.run).not.toContain('.mode == "keyword"');
324 });
325
326 test('keeps verifier crashes separate from published-content freshness findings', () => {
327 const steps = workflowSteps(dataWorkflowPath, 'update-data');
328 const verifierHealth = steps.find(step => step.name === 'Track KB freshness verifier health');
329 const freshnessFindings = steps.find(step => step.name === 'Track KB freshness findings');
330
331 expect(verifierHealth?.if).toContain("steps.verify-kb.outputs.exit_code != ''");
332 expect(verifierHealth?.env?.LABEL).toBe('kb-freshness-check-failure');
333 expect(verifierHealth?.run).toContain('[ "$VERIFY_EXIT" = "0" ] || [ "$VERIFY_EXIT" = "1" ]');
334 expect(freshnessFindings?.if).toContain("steps.verify-kb.outputs.exit_code == '0'");
335 expect(freshnessFindings?.if).toContain("steps.verify-kb.outputs.exit_code == '1'");
336 expect(freshnessFindings?.if).not.toContain("steps.verify-kb.outputs.exit_code != ''");
337 });
338 });