Composio powers 1000+ toolkits, tool search, context management, authentication, and a sandboxed workbench to help you build AI agents that turn intent into action.

next
yml 280 lines 11.7 KB
Raw
1 # Docs Agent Eval — label-triggered CI check (V1, informational)
2 #
3 # Add the `run-docs-tests` label to a docs PR to run an agent-buildability
4 # eval against the PR's Vercel docs preview: a deterministic router picks
5 # no-eval / Smoke (1 agent) / Deep (5 agents), fresh isolated agents follow
6 # the proposed docs through a rewrite proxy, and the result is posted back
7 # here as a PR comment. `run-docs-tests-deep` forces Deep.
8 #
9 # Two entry paths, so label-then-push and push-then-label both work:
10 # - deployment_status (Vercel finished a docs preview): runs if the PR
11 # already carries a run-docs-tests label.
12 # - pull_request labeled/reopened: runs if the docs preview already
13 # succeeded; otherwise skips, and the deployment_status event re-enters.
14 #
15 # The eval logic lives in the PRIVATE repo ComposioHQ/docs-agent-eval-ci.
16 # A public repo cannot `uses:` a private repo's reusable workflow, so instead
17 # a GitHub App token checks that repo out and this job runs it inline. See its
18 # ARCHITECTURE.md. Informational check — must NOT be a required status in V1.
19 #
20 # Required repo/org secrets: COMPOSIO_ORG_API_KEY_FOR_CODING_AGENT_EVAL, DEEPSEEK_API_KEY,
21 # DOCS_EVAL_APP_ID, DOCS_EVAL_APP_PRIVATE_KEY (GitHub App that can read the
22 # private engine repo).
23 name: Docs Agent Eval
24
25 on:
26 pull_request:
27 types: [labeled, reopened]
28 deployment_status:
29
30 env:
31 ENGINE_REPO: ComposioHQ/docs-agent-eval-ci
32 # Full-SHA pin of the engine code — bump on docs-agent-eval-ci releases.
33 ENGINE_REF: 8bd5993a85d60d4b4b70f171d41816b91ab67295
34
35 jobs:
36 # Decides IF an eval should start and FOR WHICH PR. deployment_status
37 # events know the commit but not the PR (deployments attach to commits),
38 # so this job maps SHA -> open PR before any label check is possible.
39 route:
40 if: |
41 (github.event_name == 'pull_request' &&
42 (contains(github.event.pull_request.labels.*.name, 'run-docs-tests') ||
43 contains(github.event.pull_request.labels.*.name, 'run-docs-tests-deep')) &&
44 (github.event.action != 'labeled' || startsWith(github.event.label.name, 'run-docs-tests'))) ||
45 (github.event_name == 'deployment_status' &&
46 github.event.deployment_status.state == 'success' &&
47 startsWith(github.event.deployment_status.environment_url, 'https://docs-') &&
48 contains(github.event.deployment_status.environment_url, '.preview.composio.dev'))
49 runs-on: ubuntu-latest
50 timeout-minutes: 5
51 permissions:
52 contents: read
53 deployments: read
54 pull-requests: read
55 outputs:
56 should_run: ${{ steps.decide.outputs.should_run }}
57 pr_number: ${{ steps.decide.outputs.pr_number }}
58 mode: ${{ steps.decide.outputs.mode }}
59 steps:
60 - name: Decide whether to run, and for which PR
61 id: decide
62 shell: bash
63 env:
64 GH_TOKEN: ${{ github.token }}
65 REPO: ${{ github.repository }}
66 run: |
67 set -euo pipefail
68 pr="" assoc="" labels="" preview_ok=false
69
70 if [ "$GITHUB_EVENT_NAME" = deployment_status ]; then
71 # Trap 2: the event payload has no PR number — map commit -> PR.
72 SHA=$(jq -r .deployment.sha "$GITHUB_EVENT_PATH")
73 PRJSON=$(gh api "repos/$REPO/commits/$SHA/pulls" \
74 --jq '[.[] | select(.state=="open")][0] // empty')
75 if [ -n "$PRJSON" ]; then
76 pr=$(jq -r .number <<<"$PRJSON")
77 assoc=$(jq -r .author_association <<<"$PRJSON")
78 labels=$(jq -r '[.labels[].name] | join(",")' <<<"$PRJSON")
79 fi
80 preview_ok=true # this event IS the successful preview
81 else
82 pr=$(jq -r .pull_request.number "$GITHUB_EVENT_PATH")
83 assoc=$(jq -r .pull_request.author_association "$GITHUB_EVENT_PATH")
84 labels=$(jq -r '[.pull_request.labels[].name] | join(",")' "$GITHUB_EVENT_PATH")
85 SHA=$(jq -r .pull_request.head.sha "$GITHUB_EVENT_PATH")
86 # Trap 1: label may predate the preview. Only run now if a docs
87 # preview already succeeded for this exact commit; otherwise skip
88 # — the deployment_status event re-enters when the preview lands.
89 for d in $(gh api "repos/$REPO/deployments?sha=$SHA&per_page=10" --jq '.[].id'); do
90 URL=$(gh api "repos/$REPO/deployments/$d/statuses" \
91 --jq '[.[] | select(.state=="success")][0].environment_url // empty')
92 case "$URL" in
93 https://docs-*.preview.composio.dev*) preview_ok=true; break ;;
94 esac
95 done
96 fi
97
98 haslabel=false
99 case ",$labels," in
100 *,run-docs-tests,*|*,run-docs-tests-deep,*) haslabel=true ;;
101 esac
102
103 # Employee gate: PR author must be org-affiliated (public repo).
104 trusted=false
105 case "$assoc" in MEMBER|OWNER|COLLABORATOR) trusted=true ;; esac
106
107 mode=Auto
108 case ",$labels," in *,run-docs-tests-deep,*) mode=Deep ;; esac
109
110 should=false
111 if [ -n "$pr" ] && $haslabel && $trusted && $preview_ok; then
112 should=true
113 fi
114 echo "event=$GITHUB_EVENT_NAME pr=${pr:-none} labels=[$labels] assoc=${assoc:-none} preview_ok=$preview_ok -> should_run=$should"
115
116 {
117 echo "should_run=$should"
118 echo "pr_number=$pr"
119 echo "mode=$mode"
120 } >> "$GITHUB_OUTPUT"
121
122 # Runs the eval inline: mint an App token, check out the private engine with
123 # it, then run the engine's code on this runner. (A public repo can't `uses:`
124 # a private reusable workflow, so the App token + checkout replaces it.)
125 eval:
126 needs: route
127 if: needs.route.outputs.should_run == 'true'
128 concurrency:
129 group: docs-agent-eval-pr-${{ needs.route.outputs.pr_number }}
130 cancel-in-progress: true
131 runs-on: ubuntu-latest
132 timeout-minutes: 55
133 permissions:
134 contents: read
135 deployments: read
136 pull-requests: write # the result comment (posted with the built-in token)
137 env:
138 SOURCE_REPO: ${{ github.repository }}
139 PR_NUMBER: ${{ needs.route.outputs.pr_number }}
140 MODE: ${{ needs.route.outputs.mode }}
141 COMPOSIO_ORG_API_KEY_FOR_CODING_AGENT_EVAL: ${{ secrets.COMPOSIO_ORG_API_KEY_FOR_CODING_AGENT_EVAL }}
142 # Builder runs on DeepSeek V4 Flash via the Anthropic-compatible endpoint.
143 ANTHROPIC_BASE_URL: https://api.deepseek.com/anthropic
144 ANTHROPIC_AUTH_TOKEN: ${{ secrets.DEEPSEEK_API_KEY }}
145 DEEP_RUNS: '5'
146 POST_PR_COMMENT: 'true'
147 steps:
148 - name: Mint a GitHub App token (read the private engine repo)
149 id: app-token
150 uses: actions/create-github-app-token@64ab729a6294000f5aa4a4bc0b73d1f32bc45510 # v3.3.0
151 with:
152 app-id: ${{ secrets.DOCS_EVAL_APP_ID }}
153 private-key: ${{ secrets.DOCS_EVAL_APP_PRIVATE_KEY }}
154 owner: ComposioHQ
155 repositories: docs-agent-eval-ci
156
157 - name: Check out the private engine
158 uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
159 with:
160 repository: ${{ env.ENGINE_REPO }}
161 ref: ${{ env.ENGINE_REF }}
162 token: ${{ steps.app-token.outputs.token }}
163 path: engine
164 # Don't write the App token into engine/.git/config — the eval then
165 # runs untrusted agent code with filesystem access on this runner,
166 # which could read a persisted token. We only need it for this fetch.
167 persist-credentials: false
168
169 - name: Setup Python
170 uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 # v7.1.0
171 with:
172 python-version: '3.12'
173 cache: pip
174 cache-dependency-path: engine/ci/requirements.txt
175
176 - name: Install harness dependencies
177 working-directory: engine
178 run: |
179 python -m pip install --upgrade pip
180 python -m pip install -r ci/requirements.txt
181
182 - name: Cache npm downloads
183 uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
184 with:
185 path: ~/.npm
186 key: npm-claude-code-${{ runner.os }}
187
188 - name: Install Claude Code
189 run: npm install --global @anthropic-ai/claude-code
190
191 - name: Validate required credentials
192 shell: bash
193 run: |
194 test -n "$COMPOSIO_ORG_API_KEY_FOR_CODING_AGENT_EVAL" || { echo "COMPOSIO_ORG_API_KEY_FOR_CODING_AGENT_EVAL is not configured"; exit 1; }
195 test -n "$ANTHROPIC_AUTH_TOKEN" || { echo "DEEPSEEK_API_KEY is not configured"; exit 1; }
196
197 - name: Route (deterministic no-eval / Smoke / Deep)
198 id: extent
199 working-directory: engine
200 shell: bash
201 env:
202 GITHUB_TOKEN: ${{ github.token }}
203 run: |
204 mkdir -p ci-output
205 python ci/route_pr.py --repo "$SOURCE_REPO" --pr "$PR_NUMBER" > ci-output/route.json
206 cat ci-output/route.json
207 EXTENT=$(python -c 'import json; print(json.load(open("ci-output/route.json"))["validation_extent"])')
208 HEAD=$(python -c 'import json; print(json.load(open("ci-output/route.json"))["head_sha"])')
209 # run-docs-tests-deep forces Deep regardless of the router.
210 [ "$MODE" = Deep ] && EXTENT=deep
211 echo "extent=$EXTENT" >> "$GITHUB_OUTPUT"
212 echo "head_sha=$HEAD" >> "$GITHUB_OUTPUT"
213
214 - name: Record no-eval outcome
215 if: steps.extent.outputs.extent == 'none'
216 working-directory: engine
217 shell: bash
218 run: |
219 python - <<'EOF'
220 import json
221 route = json.load(open("ci-output/route.json"))
222 json.dump({"status": "NO_EVAL", "reasons": route["reasons"]}, open("ci-output/outcome.json", "w"), indent=2)
223 lines = "\n".join(f"- {r}" for r in route["reasons"])
224 open("ci-output/comment.md", "w").write(
225 f"## Docs Agent Eval — ✓ NOT APPLICABLE\n\nNo agent-impacting docs change detected; no eval run.\n\n{lines}\n")
226 EOF
227
228 - name: Run exact-HEAD docs agent eval
229 if: steps.extent.outputs.extent != 'none'
230 continue-on-error: true
231 working-directory: engine
232 env:
233 GITHUB_TOKEN: ${{ github.token }}
234 run: |
235 RUNMODE=Smoke
236 [ '${{ steps.extent.outputs.extent }}' = deep ] && RUNMODE=Deep
237 python ci/run_full_eval.py \
238 --repo "$SOURCE_REPO" \
239 --head-sha '${{ steps.extent.outputs.head_sha }}' \
240 --mode "$RUNMODE"
241
242 - name: Upload eval artifacts
243 if: always()
244 uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
245 with:
246 name: docs-agent-eval-artifacts
247 path: engine/ci-output/
248 if-no-files-found: warn
249 retention-days: 14
250
251 - name: Publish job summary
252 if: always()
253 working-directory: engine
254 shell: bash
255 run: |
256 if [ -f ci-output/comment.md ]; then
257 cat ci-output/comment.md >> "$GITHUB_STEP_SUMMARY"
258 else
259 echo '## Docs Agent Eval — ✕ ERROR' >> "$GITHUB_STEP_SUMMARY"
260 echo 'The CI spine failed before it could produce a report.' >> "$GITHUB_STEP_SUMMARY"
261 fi
262
263 - name: Post result comment on the PR
264 if: always()
265 working-directory: engine
266 shell: bash
267 env:
268 GH_TOKEN: ${{ github.token }}
269 run: |
270 test -f ci-output/comment.md || exit 0
271 gh pr comment "$PR_NUMBER" --repo "$SOURCE_REPO" --body-file ci-output/comment.md
272
273 - name: Set check result
274 if: always()
275 working-directory: engine
276 shell: bash
277 run: |
278 test -f ci-output/outcome.json || exit 1
279 STATUS=$(python -c 'import json; print(json.load(open("ci-output/outcome.json"))["status"])')
280 test "$STATUS" = PASS || test "$STATUS" = NO_EVAL