Composio powers 1000+ toolkits, tool search, context management, authentication, and a sandboxed workbench to help you build AI agents that turn intent into action.
Mirrored from https://github.com/Camo67/composio.git
Synced Nov 21, 03:37 UTC
| 1 | import { describe, expect, test } from 'bun:test'; |
| 2 | import { existsSync, readFileSync } from 'node:fs'; |
| 3 | import { resolve } from 'node:path'; |
| 4 | |
| 5 | const workflowPath = resolve(process.cwd(), '..', '.github', 'workflows', 'docs-update-kb.yml'); |
| 6 | const semanticRefreshWorkflowPath = resolve( |
| 7 | process.cwd(), |
| 8 | '..', |
| 9 | '.github', |
| 10 | 'workflows', |
| 11 | 'docs-rebuild-kb-semantic.yml', |
| 12 | ); |
| 13 | const docsTestsWorkflowPath = resolve( |
| 14 | process.cwd(), |
| 15 | '..', |
| 16 | '.github', |
| 17 | 'workflows', |
| 18 | 'docs-tests.yml', |
| 19 | ); |
| 20 | const dataWorkflowPath = resolve( |
| 21 | process.cwd(), |
| 22 | '..', |
| 23 | '.github', |
| 24 | 'workflows', |
| 25 | 'docs-update-data.yml', |
| 26 | ); |
| 27 | |
| 28 | interface WorkflowStep { |
| 29 | name?: string; |
| 30 | id?: string; |
| 31 | if?: string; |
| 32 | uses?: string; |
| 33 | 'continue-on-error'?: boolean; |
| 34 | 'working-directory'?: string; |
| 35 | env?: Record<string, unknown>; |
| 36 | with?: Record<string, unknown>; |
| 37 | run?: string; |
| 38 | } |
| 39 | |
| 40 | interface WorkflowJob { |
| 41 | needs?: string | string[]; |
| 42 | if?: string; |
| 43 | outputs?: Record<string, string>; |
| 44 | env?: Record<string, unknown>; |
| 45 | permissions?: Record<string, string>; |
| 46 | steps?: WorkflowStep[]; |
| 47 | } |
| 48 | |
| 49 | function workflowSteps(path: string, job: string): WorkflowStep[] { |
| 50 | const workflow = Bun.YAML.parse(readFileSync(path, 'utf8')) as { |
| 51 | jobs?: Record<string, { steps?: WorkflowStep[] }>; |
| 52 | }; |
| 53 | return workflow.jobs?.[job]?.steps ?? []; |
| 54 | } |
| 55 | |
| 56 | describe('KB semantic artifact workflows', () => { |
| 57 | test('allows stale embeddings in PR checks without ignoring artifact errors', () => { |
| 58 | const check = workflowSteps(docsTestsWorkflowPath, 'test') |
| 59 | .find(step => step.name === 'Check KB semantic artifact'); |
| 60 | expect(check?.run).toBe('bun run check:kb-semantic --allow-stale'); |
| 61 | expect(check?.['continue-on-error']).not.toBe(true); |
| 62 | }); |
| 63 | |
| 64 | test('rebuilds stale semantic artifacts from default-branch code only for trusted current pull requests', () => { |
| 65 | const workflow = existsSync(semanticRefreshWorkflowPath) |
| 66 | ? Bun.YAML.parse(readFileSync(semanticRefreshWorkflowPath, 'utf8')) as { |
| 67 | on?: { |
| 68 | workflow_run?: { workflows?: string[]; types?: string[] }; |
| 69 | pull_request?: unknown; |
| 70 | pull_request_target?: unknown; |
| 71 | }; |
| 72 | permissions?: Record<string, string>; |
| 73 | env?: Record<string, unknown>; |
| 74 | jobs?: Record<string, WorkflowJob>; |
| 75 | } |
| 76 | : {}; |
| 77 | const authorize = workflow.jobs?.authorize; |
| 78 | const authorizeStep = authorize?.steps?.find( |
| 79 | step => step.name === 'Authorize originating pull request', |
| 80 | ); |
| 81 | const rebuild = workflow.jobs?.rebuild; |
| 82 | const steps = rebuild?.steps ?? []; |
| 83 | const checkout = steps.find(step => step.name === 'Checkout pull request'); |
| 84 | const freshness = steps.find(step => step.name === 'Check semantic artifact freshness'); |
| 85 | const build = steps.find(step => step.name === 'Rebuild semantic artifact'); |
| 86 | const verify = steps.find(step => step.name === 'Verify rebuilt artifact'); |
| 87 | const writeToken = steps.find(step => step.name === 'Generate write token'); |
| 88 | const commit = steps.find(step => step.name === 'Commit rebuilt artifact'); |
| 89 | |
| 90 | expect(workflow.on?.workflow_run?.workflows).toEqual(['Docs - Tests']); |
| 91 | expect(workflow.on?.workflow_run?.types).toEqual(['completed']); |
| 92 | expect(workflow.on).not.toHaveProperty('pull_request'); |
| 93 | expect(workflow.on).not.toHaveProperty('pull_request_target'); |
| 94 | expect(workflow.permissions?.contents).toBe('read'); |
| 95 | expect(workflow.env).toBeUndefined(); |
| 96 | |
| 97 | expect(authorize?.permissions).toEqual({ |
| 98 | actions: 'read', |
| 99 | contents: 'read', |
| 100 | 'pull-requests': 'read', |
| 101 | }); |
| 102 | expect(authorize?.outputs).toEqual({ |
| 103 | trusted: '${{ steps.trust.outputs.trusted }}', |
| 104 | head_sha: '${{ steps.trust.outputs.head_sha }}', |
| 105 | head_ref: '${{ steps.trust.outputs.head_ref }}', |
| 106 | }); |
| 107 | expect(authorize?.env).toBeUndefined(); |
| 108 | expect(authorizeStep?.env).toEqual({ |
| 109 | GH_TOKEN: '${{ github.token }}', |
| 110 | REPOSITORY: '${{ github.repository }}', |
| 111 | RUN_ID: '${{ github.event.workflow_run.id }}', |
| 112 | }); |
| 113 | expect(authorize?.steps?.map(step => step.name)).toEqual([ |
| 114 | 'Authorize originating pull request', |
| 115 | ]); |
| 116 | expect(authorizeStep?.run?.trim()).toBe(`echo "trusted=false" >> "$GITHUB_OUTPUT" |
| 117 | |
| 118 | run_json=$(gh api "repos/$REPOSITORY/actions/runs/$RUN_ID") |
| 119 | event_name=$(jq -r '.event' <<< "$run_json") |
| 120 | repository_id=$(jq -r '.repository.id' <<< "$run_json") |
| 121 | head_repository_id=$(jq -r '.head_repository.id // empty' <<< "$run_json") |
| 122 | head_branch=$(jq -r '.head_branch // empty' <<< "$run_json") |
| 123 | head_sha=$(jq -r '.head_sha // empty' <<< "$run_json") |
| 124 | |
| 125 | if [ "$event_name" != "pull_request" ] || |
| 126 | [ -z "$head_repository_id" ] || |
| 127 | [ "$head_repository_id" != "$repository_id" ] || |
| 128 | [ -z "$head_branch" ] || |
| 129 | [ -z "$head_sha" ]; then |
| 130 | echo "Skipping a non-pull-request or fork workflow run." |
| 131 | exit 0 |
| 132 | fi |
| 133 | |
| 134 | pull_matches=$(jq -c \\ |
| 135 | --arg head_repository_id "$head_repository_id" \\ |
| 136 | --arg head_branch "$head_branch" \\ |
| 137 | --arg head_sha "$head_sha" \\ |
| 138 | '[ |
| 139 | (.pull_requests // [])[] |
| 140 | | select( |
| 141 | (.head.repo.id | tostring) == $head_repository_id and |
| 142 | .head.ref == $head_branch and |
| 143 | .head.sha == $head_sha |
| 144 | ) |
| 145 | ]' <<< "$run_json") |
| 146 | |
| 147 | if [ "$(jq 'length' <<< "$pull_matches")" != "1" ]; then |
| 148 | echo "Skipping a workflow run without exactly one matching pull request." |
| 149 | exit 0 |
| 150 | fi |
| 151 | |
| 152 | pull_number=$(jq -r '.[0].number' <<< "$pull_matches") |
| 153 | pull_json=$(gh api "repos/$REPOSITORY/pulls/$pull_number") |
| 154 | association=$(jq -r '.author_association' <<< "$pull_json") |
| 155 | live_head_repository_id=$(jq -r '.head.repo.id // empty' <<< "$pull_json") |
| 156 | live_head_sha=$(jq -r '.head.sha // empty' <<< "$pull_json") |
| 157 | live_head_ref=$(jq -r '.head.ref // empty' <<< "$pull_json") |
| 158 | state=$(jq -r '.state' <<< "$pull_json") |
| 159 | |
| 160 | if [ "$state" != "open" ] || |
| 161 | [ "$live_head_repository_id" != "$repository_id" ] || |
| 162 | [ "$live_head_sha" != "$head_sha" ] || |
| 163 | [ "$live_head_ref" != "$head_branch" ]; then |
| 164 | echo "Skipping a closed or stale pull request." |
| 165 | exit 0 |
| 166 | fi |
| 167 | |
| 168 | case "$association" in |
| 169 | MEMBER|OWNER) ;; |
| 170 | *) |
| 171 | echo "Skipping a pull request whose author is not a repository member or owner." |
| 172 | exit 0 |
| 173 | ;; |
| 174 | esac |
| 175 | |
| 176 | echo "trusted=true" >> "$GITHUB_OUTPUT" |
| 177 | echo "head_sha=$live_head_sha" >> "$GITHUB_OUTPUT" |
| 178 | echo "head_ref=$live_head_ref" >> "$GITHUB_OUTPUT"`); |
| 179 | |
| 180 | expect(rebuild?.needs).toBe('authorize'); |
| 181 | expect(rebuild?.if).toBe("needs.authorize.outputs.trusted == 'true'"); |
| 182 | expect(rebuild?.permissions?.contents).toBe('read'); |
| 183 | expect(rebuild?.env).toBeUndefined(); |
| 184 | expect(steps.map(step => step.name)).toEqual([ |
| 185 | 'Checkout pull request', |
| 186 | 'Setup Node.js, pnpm, Bun', |
| 187 | 'Cache Bun dependencies', |
| 188 | 'Install dependencies', |
| 189 | 'Check semantic artifact freshness', |
| 190 | 'Rebuild semantic artifact', |
| 191 | 'Verify rebuilt artifact', |
| 192 | 'Generate write token', |
| 193 | 'Resolve bot identity', |
| 194 | 'Commit rebuilt artifact', |
| 195 | ]); |
| 196 | expect(checkout?.with?.ref).toBe('${{ needs.authorize.outputs.head_sha }}'); |
| 197 | expect(checkout?.with?.['persist-credentials']).toBe(false); |
| 198 | expect(freshness?.['continue-on-error']).toBe(true); |
| 199 | expect(freshness?.run).toBe('bun run check:kb-semantic'); |
| 200 | expect(build?.if).toBe("steps.freshness.outcome == 'failure'"); |
| 201 | expect(build?.env).toEqual({ OPENAI_API_KEY: '${{ secrets.OPENAI_API_KEY }}' }); |
| 202 | expect(verify?.if).toBe("steps.freshness.outcome == 'failure'"); |
| 203 | expect(verify?.run).toBe('bun run check:kb-semantic'); |
| 204 | expect(writeToken?.if).toBe("steps.freshness.outcome == 'failure'"); |
| 205 | expect(writeToken?.with).toEqual({ |
| 206 | 'client-id': '${{ vars.RELEASE_BOT_CLIENT_ID }}', |
| 207 | 'private-key': '${{ secrets.RELEASE_BOT_APP_PRIVATE_KEY }}', |
| 208 | owner: 'ComposioHQ', |
| 209 | repositories: 'composio', |
| 210 | 'permission-contents': 'write', |
| 211 | }); |
| 212 | expect(commit?.if).toBe("steps.freshness.outcome == 'failure'"); |
| 213 | expect(commit?.env?.HEAD_REF).toBe('${{ needs.authorize.outputs.head_ref }}'); |
| 214 | expect(commit?.run).toContain('git push origin "HEAD:$HEAD_REF"'); |
| 215 | expect(commit?.run).not.toMatch(/git push[^\n]*(?:--force|\s-f(?:\s|$))/); |
| 216 | expect(steps.indexOf(writeToken)).toBeGreaterThan(steps.indexOf(verify)); |
| 217 | |
| 218 | expect( |
| 219 | steps.filter(step => JSON.stringify(step).includes('secrets.OPENAI_API_KEY')) |
| 220 | .map(step => step.name), |
| 221 | ).toEqual(['Rebuild semantic artifact']); |
| 222 | expect( |
| 223 | steps.filter(step => JSON.stringify(step).includes('RELEASE_BOT_')) |
| 224 | .map(step => step.name), |
| 225 | ).toEqual(['Generate write token']); |
| 226 | |
| 227 | const allSteps = Object.values(workflow.jobs ?? {}).flatMap(job => job.steps ?? []); |
| 228 | expect( |
| 229 | allSteps.filter(step => JSON.stringify(step).includes('${{ secrets.')) |
| 230 | .map(step => step.name), |
| 231 | ).toEqual(['Rebuild semantic artifact', 'Generate write token']); |
| 232 | |
| 233 | const docsTestsWorkflow = Bun.YAML.parse(readFileSync(docsTestsWorkflowPath, 'utf8')) as { |
| 234 | on?: { pull_request?: { paths?: string[] } }; |
| 235 | }; |
| 236 | expect(docsTestsWorkflow.on?.pull_request?.paths).toContain( |
| 237 | '.github/workflows/docs-rebuild-kb-semantic.yml', |
| 238 | ); |
| 239 | }); |
| 240 | |
| 241 | test('repairs the semantic artifact on a schedule without any upstream source', () => { |
| 242 | const workflow = readFileSync(workflowPath, 'utf8'); |
| 243 | const parsed = Bun.YAML.parse(workflow) as { on?: Record<string, unknown> }; |
| 244 | |
| 245 | expect(Object.keys(parsed.on ?? {}).sort()).toEqual(['schedule', 'workflow_dispatch']); |
| 246 | expect(workflow).not.toContain('support-knowledge'); |
| 247 | expect(workflow).toContain('bun run build:kb-semantic'); |
| 248 | expect(workflow).toContain('bun run check:kb-semantic'); |
| 249 | expect(workflow).toContain('branch: docs/auto-update-kb'); |
| 250 | expect(workflow).toContain('base: next'); |
| 251 | expect(workflow).toContain('persist-credentials: false'); |
| 252 | }); |
| 253 | |
| 254 | test('serializes both KB writers and pins artifacts to their docs base', () => { |
| 255 | const workflow = Bun.YAML.parse(readFileSync(workflowPath, 'utf8')) as { |
| 256 | concurrency?: { group?: string; 'cancel-in-progress'?: boolean; queue?: string }; |
| 257 | jobs?: Record<string, WorkflowJob>; |
| 258 | }; |
| 259 | const publish = Bun.YAML.parse(readFileSync( |
| 260 | resolve(process.cwd(), '..', '.github/workflows/docs-kb-publish.yml'), 'utf8', |
| 261 | )) as { concurrency: { group: string }; jobs: Record<string, WorkflowJob> }; |
| 262 | expect(workflow.concurrency?.group).toBe(publish.concurrency.group); |
| 263 | expect(workflow.concurrency?.['cancel-in-progress']).toBe(false); |
| 264 | expect(workflow.concurrency?.queue).toBe('max'); |
| 265 | for (const job of [workflow.jobs?.refresh, publish.jobs.publish]) { |
| 266 | const checkout = job?.steps?.find(step => step.name === 'Checkout docs repository'); |
| 267 | expect(checkout?.with?.ref).toBe('next'); |
| 268 | } |
| 269 | const proposalCheckout = workflow.jobs?.propose?.steps?.find( |
| 270 | step => step.name === 'Checkout docs repository', |
| 271 | ); |
| 272 | expect(workflow.jobs?.refresh?.outputs?.docs_base_sha).toBe('${{ steps.docs-base.outputs.sha }}'); |
| 273 | expect(proposalCheckout?.with?.ref).toBe('${{ needs.refresh.outputs.docs_base_sha }}'); |
| 274 | }); |
| 275 | |
| 276 | test('tracks failures until both rebuild and PR proposal recover', () => { |
| 277 | const workflow = Bun.YAML.parse(readFileSync(workflowPath, 'utf8')) as { |
| 278 | jobs?: Record<string, WorkflowJob>; |
| 279 | }; |
| 280 | const reporter = workflow.jobs?.report; |
| 281 | const failure = reporter?.steps?.find(step => step.name === 'Open KB repair failure issue'); |
| 282 | const recovery = reporter?.steps?.find(step => step.name === 'Close KB repair failure issue'); |
| 283 | |
| 284 | expect(reporter?.needs).toEqual(['refresh', 'propose']); |
| 285 | expect(reporter?.if).toBe('always()'); |
| 286 | expect(reporter?.permissions?.issues).toBe('write'); |
| 287 | expect(failure?.if).toContain("needs.refresh.result == 'failure'"); |
| 288 | expect(failure?.if).toContain("needs.propose.result == 'failure'"); |
| 289 | expect(failure?.env?.GH_TOKEN).toBe('${{ secrets.GITHUB_TOKEN }}'); |
| 290 | expect(failure?.env?.LABEL).toBe('docs-kb-refresh-failure'); |
| 291 | expect(failure?.run).toContain('gh issue list'); |
| 292 | expect(failure?.run).toContain('gh issue create'); |
| 293 | expect(recovery?.if).toContain("needs.refresh.result == 'success'"); |
| 294 | expect(recovery?.if).toContain("needs.refresh.outputs.changed != 'true'"); |
| 295 | expect(recovery?.if).toContain("needs.propose.result == 'success'"); |
| 296 | expect(recovery?.env?.LABEL).toBe('docs-kb-refresh-failure'); |
| 297 | expect(recovery?.run).toContain('gh issue close'); |
| 298 | }); |
| 299 | |
| 300 | test('transfers the semantic index to the exact path staged by the repair PR', () => { |
| 301 | const upload = workflowSteps(workflowPath, 'refresh').find( |
| 302 | step => step.name === 'Upload verified semantic artifact', |
| 303 | ); |
| 304 | const steps = workflowSteps(workflowPath, 'propose'); |
| 305 | const download = steps.find(step => step.name === 'Download verified semantic artifact'); |
| 306 | const createPullRequest = steps.find(step => step.name === 'Create repair pull request'); |
| 307 | |
| 308 | expect(upload?.with?.name).toBe('kb-semantic-repair'); |
| 309 | expect(download?.with?.name).toBe(upload?.with?.name); |
| 310 | expect(upload?.with?.path).toBe('docs/kb/semantic-index.json'); |
| 311 | expect(download?.with?.path).toBe('docs/kb'); |
| 312 | expect(String(createPullRequest?.with?.['add-paths']).trim()).toBe(upload?.with?.path); |
| 313 | }); |
| 314 | |
| 315 | test('treats keyword-only production retrieval as a semantic health failure', () => { |
| 316 | const steps = workflowSteps( |
| 317 | resolve(process.cwd(), '..', '.github', 'workflows', 'docs.health-check.yml'), |
| 318 | 'health-check', |
| 319 | ); |
| 320 | const endpointCheck = steps.find(step => step.name === 'Check endpoints'); |
| 321 | |
| 322 | expect(endpointCheck?.run).toContain('.mode == "hybrid"'); |
| 323 | expect(endpointCheck?.run).not.toContain('.mode == "keyword"'); |
| 324 | }); |
| 325 | |
| 326 | test('keeps verifier crashes separate from published-content freshness findings', () => { |
| 327 | const steps = workflowSteps(dataWorkflowPath, 'update-data'); |
| 328 | const verifierHealth = steps.find(step => step.name === 'Track KB freshness verifier health'); |
| 329 | const freshnessFindings = steps.find(step => step.name === 'Track KB freshness findings'); |
| 330 | |
| 331 | expect(verifierHealth?.if).toContain("steps.verify-kb.outputs.exit_code != ''"); |
| 332 | expect(verifierHealth?.env?.LABEL).toBe('kb-freshness-check-failure'); |
| 333 | expect(verifierHealth?.run).toContain('[ "$VERIFY_EXIT" = "0" ] || [ "$VERIFY_EXIT" = "1" ]'); |
| 334 | expect(freshnessFindings?.if).toContain("steps.verify-kb.outputs.exit_code == '0'"); |
| 335 | expect(freshnessFindings?.if).toContain("steps.verify-kb.outputs.exit_code == '1'"); |
| 336 | expect(freshnessFindings?.if).not.toContain("steps.verify-kb.outputs.exit_code != ''"); |
| 337 | }); |
| 338 | }); |