10
# - deployment_status (Vercel finished a docs preview): runs if the PR
11
# already carries a run-docs-tests label.
12
# - pull_request labeled/reopened: runs if the docs preview already
13
-# succeeded; otherwise skips, and the deployment_status event re-enters
14
-# when the preview lands. (`synchronize` removed: new pushes surface as
15
-# new deployments, so deployment_status covers them.)
13
+# succeeded; otherwise skips, and the deployment_status event re-enters.
14
#
17
-# All eval logic lives in ComposioHQ/docs-agent-eval-ci (reusable workflow) —
18
-# see its ARCHITECTURE.md. This check is informational and must NOT be made
19
-# a required status check in V1.
15
+# The eval logic lives in the PRIVATE repo ComposioHQ/docs-agent-eval-ci.
16
+# A public repo cannot `uses:` a private repo's reusable workflow, so instead
17
+# a GitHub App token checks that repo out and this job runs it inline. See its
18
+# ARCHITECTURE.md. Informational check — must NOT be a required status in V1.
19
#
21
-# Required repo/org secrets: COMPOSIO_ORG_API_KEY, DEEPSEEK_API_KEY.
22
-# No Vercel credentials (decision 2026-08-19): the engine resolves previews
23
-# from GitHub's deployment records with the per-run token.
20
+# Required repo/org secrets: COMPOSIO_ORG_API_KEY, DEEPSEEK_API_KEY,
21
+# DOCS_EVAL_APP_ID, DOCS_EVAL_APP_PRIVATE_KEY (GitHub App that can read the
22
+# private engine repo).
23
name: Docs Agent Eval
24
25
on:
27
types: [labeled, reopened]
28
deployment_status:
29
30
+env:
31
+ ENGINE_REPO: ComposioHQ/docs-agent-eval-ci
32
+ # Full-SHA pin of the engine code — bump on docs-agent-eval-ci releases.
33
+ ENGINE_REF: d299de8061151997548a3650e13ebfc63c03d08f
34
+
35
jobs:
36
# Decides IF an eval should start and FOR WHICH PR. deployment_status
37
# events know the commit but not the PR (deployments attach to commits),
118
echo "mode=$mode"
119
} >> "$GITHUB_OUTPUT"
120
121
+ # Runs the eval inline: mint an App token, check out the private engine with
122
+ # it, then run the engine's code on this runner. (A public repo can't `uses:`
123
+ # a private reusable workflow, so the App token + checkout replaces it.)
124
eval:
125
needs: route
126
if: needs.route.outputs.should_run == 'true'
120
- # Concurrency here, after routing: a skipped/filtered event never enters
121
- # the group, so unrelated triggers cannot cancel a running eval. A newer
122
- # run for the same PR (new preview, re-label) still supersedes the stale
123
- # one.
127
concurrency:
128
group: docs-agent-eval-pr-${{ needs.route.outputs.pr_number }}
129
cancel-in-progress: true
130
+ runs-on: ubuntu-latest
131
+ timeout-minutes: 55
132
permissions:
133
contents: read
129
- pull-requests: write # the result comment
130
- # Full-SHA pin required by org policy — bump on docs-agent-eval releases.
131
- uses: ComposioHQ/docs-agent-eval-ci/.github/workflows/eval-pr.yml@21146fbe43e6d6eb0db4b46ef9b09faab6110edd
132
- with:
133
- pr_number: ${{ needs.route.outputs.pr_number }}
134
- mode: ${{ needs.route.outputs.mode }}
135
- post_comment: true
136
- # Explicit and minimal on purpose — inherit would hand the engine every
137
- # secret this repo holds (review decision 2).
138
- secrets:
134
+ deployments: read
135
+ pull-requests: write # the result comment (posted with the built-in token)
136
+ env:
137
+ SOURCE_REPO: ${{ github.repository }}
138
+ PR_NUMBER: ${{ needs.route.outputs.pr_number }}
139
+ MODE: ${{ needs.route.outputs.mode }}
140
COMPOSIO_ORG_API_KEY: ${{ secrets.COMPOSIO_ORG_API_KEY }}
140
- DEEPSEEK_API_KEY: ${{ secrets.DEEPSEEK_API_KEY }}
141
+ # Builder runs on DeepSeek V4 Flash via the Anthropic-compatible endpoint.
142
+ ANTHROPIC_BASE_URL: https://api.deepseek.com/anthropic
143
+ ANTHROPIC_AUTH_TOKEN: ${{ secrets.DEEPSEEK_API_KEY }}
144
+ DEEP_RUNS: '5'
145
+ POST_PR_COMMENT: 'true'
146
+ steps:
147
+ - name: Mint a GitHub App token (read the private engine repo)
148
+ id: app-token
149
+ uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0
150
+ with:
151
+ app-id: ${{ secrets.DOCS_EVAL_APP_ID }}
152
+ private-key: ${{ secrets.DOCS_EVAL_APP_PRIVATE_KEY }}
153
+ owner: ComposioHQ
154
+ repositories: docs-agent-eval-ci
155
+
156
+ - name: Check out the private engine
157
+ uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
158
+ with:
159
+ repository: ${{ env.ENGINE_REPO }}
160
+ ref: ${{ env.ENGINE_REF }}
161
+ token: ${{ steps.app-token.outputs.token }}
162
+ path: engine
163
+
164
+ - name: Setup Python
165
+ uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
166
+ with:
167
+ python-version: '3.12'
168
+ cache: pip
169
+ cache-dependency-path: engine/ci/requirements.txt
170
+
171
+ - name: Install harness dependencies
172
+ working-directory: engine
173
+ run: |
174
+ python -m pip install --upgrade pip
175
+ python -m pip install -r ci/requirements.txt
176
+
177
+ - name: Cache npm downloads
178
+ uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4
179
+ with:
180
+ path: ~/.npm
181
+ key: npm-claude-code-${{ runner.os }}
182
+
183
+ - name: Install Claude Code
184
+ run: npm install --global @anthropic-ai/claude-code
185
+
186
+ - name: Validate required credentials
187
+ shell: bash
188
+ run: |
189
+ test -n "$COMPOSIO_ORG_API_KEY" || { echo "COMPOSIO_ORG_API_KEY is not configured"; exit 1; }
190
+ test -n "$ANTHROPIC_AUTH_TOKEN" || { echo "DEEPSEEK_API_KEY is not configured"; exit 1; }
191
+
192
+ - name: Route (deterministic no-eval / Smoke / Deep)
193
+ id: extent
194
+ working-directory: engine
195
+ shell: bash
196
+ env:
197
+ GITHUB_TOKEN: ${{ github.token }}
198
+ run: |
199
+ mkdir -p ci-output
200
+ python ci/route_pr.py --repo "$SOURCE_REPO" --pr "$PR_NUMBER" > ci-output/route.json
201
+ cat ci-output/route.json
202
+ EXTENT=$(python -c 'import json; print(json.load(open("ci-output/route.json"))["validation_extent"])')
203
+ HEAD=$(python -c 'import json; print(json.load(open("ci-output/route.json"))["head_sha"])')
204
+ # run-docs-tests-deep forces Deep regardless of the router.
205
+ [ "$MODE" = Deep ] && EXTENT=deep
206
+ echo "extent=$EXTENT" >> "$GITHUB_OUTPUT"
207
+ echo "head_sha=$HEAD" >> "$GITHUB_OUTPUT"
208
+
209
+ - name: Record no-eval outcome
210
+ if: steps.extent.outputs.extent == 'none'
211
+ working-directory: engine
212
+ shell: bash
213
+ run: |
214
+ python - <<'EOF'
215
+ import json
216
+ route = json.load(open("ci-output/route.json"))
217
+ json.dump({"status": "NO_EVAL", "reasons": route["reasons"]}, open("ci-output/outcome.json", "w"), indent=2)
218
+ lines = "\n".join(f"- {r}" for r in route["reasons"])
219
+ open("ci-output/comment.md", "w").write(
220
+ f"## Docs Agent Eval — ✓ NOT APPLICABLE\n\nNo agent-impacting docs change detected; no eval run.\n\n{lines}\n")
221
+ EOF
222
+
223
+ - name: Run exact-HEAD docs agent eval
224
+ if: steps.extent.outputs.extent != 'none'
225
+ continue-on-error: true
226
+ working-directory: engine
227
+ env:
228
+ GITHUB_TOKEN: ${{ github.token }}
229
+ run: |
230
+ RUNMODE=Smoke
231
+ [ '${{ steps.extent.outputs.extent }}' = deep ] && RUNMODE=Deep
232
+ python ci/run_full_eval.py \
233
+ --repo "$SOURCE_REPO" \
234
+ --head-sha '${{ steps.extent.outputs.head_sha }}' \
235
+ --mode "$RUNMODE"
236
+
237
+ - name: Upload eval artifacts
238
+ if: always()
239
+ uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
240
+ with:
241
+ name: docs-agent-eval-artifacts
242
+ path: engine/ci-output/
243
+ if-no-files-found: warn
244
+ retention-days: 14
245
+
246
+ - name: Publish job summary
247
+ if: always()
248
+ working-directory: engine
249
+ shell: bash
250
+ run: |
251
+ if [ -f ci-output/comment.md ]; then
252
+ cat ci-output/comment.md >> "$GITHUB_STEP_SUMMARY"
253
+ else
254
+ echo '## Docs Agent Eval — ✕ ERROR' >> "$GITHUB_STEP_SUMMARY"
255
+ echo 'The CI spine failed before it could produce a report.' >> "$GITHUB_STEP_SUMMARY"
256
+ fi
257
+
258
+ - name: Post result comment on the PR
259
+ if: always()
260
+ working-directory: engine
261
+ shell: bash
262
+ env:
263
+ GH_TOKEN: ${{ github.token }}
264
+ run: |
265
+ test -f ci-output/comment.md || exit 0
266
+ gh pr comment "$PR_NUMBER" --repo "$SOURCE_REPO" --body-file ci-output/comment.md
267
+
268
+ - name: Set check result
269
+ if: always()
270
+ working-directory: engine
271
+ shell: bash
272
+ run: |
273
+ test -f ci-output/outcome.json || exit 1
274
+ STATUS=$(python -c 'import json; print(json.load(open("ci-output/outcome.json"))["status"])')
275
+ test "$STATUS" = PASS || test "$STATUS" = NO_EVAL