Composio powers 1000+ toolkits, tool search, context management, authentication, and a sandboxed workbench to help you build AI agents that turn intent into action.
| 1 | # Docs Agent Eval — label-triggered CI check (V1, informational) |
| 2 | # |
| 3 | # Add the `run-docs-tests` label to a docs PR to run an agent-buildability |
| 4 | # eval against the PR's Vercel docs preview: a deterministic router picks |
| 5 | # no-eval / Smoke (1 agent) / Deep (5 agents), fresh isolated agents follow |
| 6 | # the proposed docs through a rewrite proxy, and the result is posted back |
| 7 | # here as a PR comment. `run-docs-tests-deep` forces Deep. |
| 8 | # |
| 9 | # Two entry paths, so label-then-push and push-then-label both work: |
| 10 | # - deployment_status (Vercel finished a docs preview): runs if the PR |
| 11 | # already carries a run-docs-tests label. |
| 12 | # - pull_request labeled/reopened: runs if the docs preview already |
| 13 | # succeeded; otherwise skips, and the deployment_status event re-enters. |
| 14 | # |
| 15 | # The eval logic lives in the PRIVATE repo ComposioHQ/docs-agent-eval-ci. |
| 16 | # A public repo cannot `uses:` a private repo's reusable workflow, so instead |
| 17 | # a GitHub App token checks that repo out and this job runs it inline. See its |
| 18 | # ARCHITECTURE.md. Informational check — must NOT be a required status in V1. |
| 19 | # |
| 20 | # Required repo/org secrets: COMPOSIO_ORG_API_KEY_FOR_CODING_AGENT_EVAL, DEEPSEEK_API_KEY, |
| 21 | # DOCS_EVAL_APP_ID, DOCS_EVAL_APP_PRIVATE_KEY (GitHub App that can read the |
| 22 | # private engine repo). |
| 23 | name: Docs Agent Eval |
| 24 | |
| 25 | on: |
| 26 | pull_request: |
| 27 | types: [labeled, reopened] |
| 28 | deployment_status: |
| 29 | |
| 30 | env: |
| 31 | ENGINE_REPO: ComposioHQ/docs-agent-eval-ci |
| 32 | # Full-SHA pin of the engine code — bump on docs-agent-eval-ci releases. |
| 33 | ENGINE_REF: 8bd5993a85d60d4b4b70f171d41816b91ab67295 |
| 34 | |
| 35 | jobs: |
| 36 | # Decides IF an eval should start and FOR WHICH PR. deployment_status |
| 37 | # events know the commit but not the PR (deployments attach to commits), |
| 38 | # so this job maps SHA -> open PR before any label check is possible. |
| 39 | route: |
| 40 | if: | |
| 41 | (github.event_name == 'pull_request' && |
| 42 | (contains(github.event.pull_request.labels.*.name, 'run-docs-tests') || |
| 43 | contains(github.event.pull_request.labels.*.name, 'run-docs-tests-deep')) && |
| 44 | (github.event.action != 'labeled' || startsWith(github.event.label.name, 'run-docs-tests'))) || |
| 45 | (github.event_name == 'deployment_status' && |
| 46 | github.event.deployment_status.state == 'success' && |
| 47 | startsWith(github.event.deployment_status.environment_url, 'https://docs-') && |
| 48 | contains(github.event.deployment_status.environment_url, '.preview.composio.dev')) |
| 49 | runs-on: ubuntu-latest |
| 50 | timeout-minutes: 5 |
| 51 | permissions: |
| 52 | contents: read |
| 53 | deployments: read |
| 54 | pull-requests: read |
| 55 | outputs: |
| 56 | should_run: ${{ steps.decide.outputs.should_run }} |
| 57 | pr_number: ${{ steps.decide.outputs.pr_number }} |
| 58 | mode: ${{ steps.decide.outputs.mode }} |
| 59 | steps: |
| 60 | - name: Decide whether to run, and for which PR |
| 61 | id: decide |
| 62 | shell: bash |
| 63 | env: |
| 64 | GH_TOKEN: ${{ github.token }} |
| 65 | REPO: ${{ github.repository }} |
| 66 | run: | |
| 67 | set -euo pipefail |
| 68 | pr="" assoc="" labels="" preview_ok=false |
| 69 | |
| 70 | if [ "$GITHUB_EVENT_NAME" = deployment_status ]; then |
| 71 | # Trap 2: the event payload has no PR number — map commit -> PR. |
| 72 | SHA=$(jq -r .deployment.sha "$GITHUB_EVENT_PATH") |
| 73 | PRJSON=$(gh api "repos/$REPO/commits/$SHA/pulls" \ |
| 74 | --jq '[.[] | select(.state=="open")][0] // empty') |
| 75 | if [ -n "$PRJSON" ]; then |
| 76 | pr=$(jq -r .number <<<"$PRJSON") |
| 77 | assoc=$(jq -r .author_association <<<"$PRJSON") |
| 78 | labels=$(jq -r '[.labels[].name] | join(",")' <<<"$PRJSON") |
| 79 | fi |
| 80 | preview_ok=true # this event IS the successful preview |
| 81 | else |
| 82 | pr=$(jq -r .pull_request.number "$GITHUB_EVENT_PATH") |
| 83 | assoc=$(jq -r .pull_request.author_association "$GITHUB_EVENT_PATH") |
| 84 | labels=$(jq -r '[.pull_request.labels[].name] | join(",")' "$GITHUB_EVENT_PATH") |
| 85 | SHA=$(jq -r .pull_request.head.sha "$GITHUB_EVENT_PATH") |
| 86 | # Trap 1: label may predate the preview. Only run now if a docs |
| 87 | # preview already succeeded for this exact commit; otherwise skip |
| 88 | # — the deployment_status event re-enters when the preview lands. |
| 89 | for d in $(gh api "repos/$REPO/deployments?sha=$SHA&per_page=10" --jq '.[].id'); do |
| 90 | URL=$(gh api "repos/$REPO/deployments/$d/statuses" \ |
| 91 | --jq '[.[] | select(.state=="success")][0].environment_url // empty') |
| 92 | case "$URL" in |
| 93 | https://docs-*.preview.composio.dev*) preview_ok=true; break ;; |
| 94 | esac |
| 95 | done |
| 96 | fi |
| 97 | |
| 98 | haslabel=false |
| 99 | case ",$labels," in |
| 100 | *,run-docs-tests,*|*,run-docs-tests-deep,*) haslabel=true ;; |
| 101 | esac |
| 102 | |
| 103 | # Employee gate: PR author must be org-affiliated (public repo). |
| 104 | trusted=false |
| 105 | case "$assoc" in MEMBER|OWNER|COLLABORATOR) trusted=true ;; esac |
| 106 | |
| 107 | mode=Auto |
| 108 | case ",$labels," in *,run-docs-tests-deep,*) mode=Deep ;; esac |
| 109 | |
| 110 | should=false |
| 111 | if [ -n "$pr" ] && $haslabel && $trusted && $preview_ok; then |
| 112 | should=true |
| 113 | fi |
| 114 | echo "event=$GITHUB_EVENT_NAME pr=${pr:-none} labels=[$labels] assoc=${assoc:-none} preview_ok=$preview_ok -> should_run=$should" |
| 115 | |
| 116 | { |
| 117 | echo "should_run=$should" |
| 118 | echo "pr_number=$pr" |
| 119 | echo "mode=$mode" |
| 120 | } >> "$GITHUB_OUTPUT" |
| 121 | |
| 122 | # Runs the eval inline: mint an App token, check out the private engine with |
| 123 | # it, then run the engine's code on this runner. (A public repo can't `uses:` |
| 124 | # a private reusable workflow, so the App token + checkout replaces it.) |
| 125 | eval: |
| 126 | needs: route |
| 127 | if: needs.route.outputs.should_run == 'true' |
| 128 | concurrency: |
| 129 | group: docs-agent-eval-pr-${{ needs.route.outputs.pr_number }} |
| 130 | cancel-in-progress: true |
| 131 | runs-on: ubuntu-latest |
| 132 | timeout-minutes: 55 |
| 133 | permissions: |
| 134 | contents: read |
| 135 | deployments: read |
| 136 | pull-requests: write # the result comment (posted with the built-in token) |
| 137 | env: |
| 138 | SOURCE_REPO: ${{ github.repository }} |
| 139 | PR_NUMBER: ${{ needs.route.outputs.pr_number }} |
| 140 | MODE: ${{ needs.route.outputs.mode }} |
| 141 | COMPOSIO_ORG_API_KEY_FOR_CODING_AGENT_EVAL: ${{ secrets.COMPOSIO_ORG_API_KEY_FOR_CODING_AGENT_EVAL }} |
| 142 | # Builder runs on DeepSeek V4 Flash via the Anthropic-compatible endpoint. |
| 143 | ANTHROPIC_BASE_URL: https://api.deepseek.com/anthropic |
| 144 | ANTHROPIC_AUTH_TOKEN: ${{ secrets.DEEPSEEK_API_KEY }} |
| 145 | DEEP_RUNS: '5' |
| 146 | POST_PR_COMMENT: 'true' |
| 147 | steps: |
| 148 | - name: Mint a GitHub App token (read the private engine repo) |
| 149 | id: app-token |
| 150 | uses: actions/create-github-app-token@64ab729a6294000f5aa4a4bc0b73d1f32bc45510 # v3.3.0 |
| 151 | with: |
| 152 | app-id: ${{ secrets.DOCS_EVAL_APP_ID }} |
| 153 | private-key: ${{ secrets.DOCS_EVAL_APP_PRIVATE_KEY }} |
| 154 | owner: ComposioHQ |
| 155 | repositories: docs-agent-eval-ci |
| 156 | |
| 157 | - name: Check out the private engine |
| 158 | uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 |
| 159 | with: |
| 160 | repository: ${{ env.ENGINE_REPO }} |
| 161 | ref: ${{ env.ENGINE_REF }} |
| 162 | token: ${{ steps.app-token.outputs.token }} |
| 163 | path: engine |
| 164 | # Don't write the App token into engine/.git/config — the eval then |
| 165 | # runs untrusted agent code with filesystem access on this runner, |
| 166 | # which could read a persisted token. We only need it for this fetch. |
| 167 | persist-credentials: false |
| 168 | |
| 169 | - name: Setup Python |
| 170 | uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 # v7.1.0 |
| 171 | with: |
| 172 | python-version: '3.12' |
| 173 | cache: pip |
| 174 | cache-dependency-path: engine/ci/requirements.txt |
| 175 | |
| 176 | - name: Install harness dependencies |
| 177 | working-directory: engine |
| 178 | run: | |
| 179 | python -m pip install --upgrade pip |
| 180 | python -m pip install -r ci/requirements.txt |
| 181 | |
| 182 | - name: Cache npm downloads |
| 183 | uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 |
| 184 | with: |
| 185 | path: ~/.npm |
| 186 | key: npm-claude-code-${{ runner.os }} |
| 187 | |
| 188 | - name: Install Claude Code |
| 189 | run: npm install --global @anthropic-ai/claude-code |
| 190 | |
| 191 | - name: Validate required credentials |
| 192 | shell: bash |
| 193 | run: | |
| 194 | test -n "$COMPOSIO_ORG_API_KEY_FOR_CODING_AGENT_EVAL" || { echo "COMPOSIO_ORG_API_KEY_FOR_CODING_AGENT_EVAL is not configured"; exit 1; } |
| 195 | test -n "$ANTHROPIC_AUTH_TOKEN" || { echo "DEEPSEEK_API_KEY is not configured"; exit 1; } |
| 196 | |
| 197 | - name: Route (deterministic no-eval / Smoke / Deep) |
| 198 | id: extent |
| 199 | working-directory: engine |
| 200 | shell: bash |
| 201 | env: |
| 202 | GITHUB_TOKEN: ${{ github.token }} |
| 203 | run: | |
| 204 | mkdir -p ci-output |
| 205 | python ci/route_pr.py --repo "$SOURCE_REPO" --pr "$PR_NUMBER" > ci-output/route.json |
| 206 | cat ci-output/route.json |
| 207 | EXTENT=$(python -c 'import json; print(json.load(open("ci-output/route.json"))["validation_extent"])') |
| 208 | HEAD=$(python -c 'import json; print(json.load(open("ci-output/route.json"))["head_sha"])') |
| 209 | # run-docs-tests-deep forces Deep regardless of the router. |
| 210 | [ "$MODE" = Deep ] && EXTENT=deep |
| 211 | echo "extent=$EXTENT" >> "$GITHUB_OUTPUT" |
| 212 | echo "head_sha=$HEAD" >> "$GITHUB_OUTPUT" |
| 213 | |
| 214 | - name: Record no-eval outcome |
| 215 | if: steps.extent.outputs.extent == 'none' |
| 216 | working-directory: engine |
| 217 | shell: bash |
| 218 | run: | |
| 219 | python - <<'EOF' |
| 220 | import json |
| 221 | route = json.load(open("ci-output/route.json")) |
| 222 | json.dump({"status": "NO_EVAL", "reasons": route["reasons"]}, open("ci-output/outcome.json", "w"), indent=2) |
| 223 | lines = "\n".join(f"- {r}" for r in route["reasons"]) |
| 224 | open("ci-output/comment.md", "w").write( |
| 225 | f"## Docs Agent Eval — ✓ NOT APPLICABLE\n\nNo agent-impacting docs change detected; no eval run.\n\n{lines}\n") |
| 226 | EOF |
| 227 | |
| 228 | - name: Run exact-HEAD docs agent eval |
| 229 | if: steps.extent.outputs.extent != 'none' |
| 230 | continue-on-error: true |
| 231 | working-directory: engine |
| 232 | env: |
| 233 | GITHUB_TOKEN: ${{ github.token }} |
| 234 | run: | |
| 235 | RUNMODE=Smoke |
| 236 | [ '${{ steps.extent.outputs.extent }}' = deep ] && RUNMODE=Deep |
| 237 | python ci/run_full_eval.py \ |
| 238 | --repo "$SOURCE_REPO" \ |
| 239 | --head-sha '${{ steps.extent.outputs.head_sha }}' \ |
| 240 | --mode "$RUNMODE" |
| 241 | |
| 242 | - name: Upload eval artifacts |
| 243 | if: always() |
| 244 | uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 |
| 245 | with: |
| 246 | name: docs-agent-eval-artifacts |
| 247 | path: engine/ci-output/ |
| 248 | if-no-files-found: warn |
| 249 | retention-days: 14 |
| 250 | |
| 251 | - name: Publish job summary |
| 252 | if: always() |
| 253 | working-directory: engine |
| 254 | shell: bash |
| 255 | run: | |
| 256 | if [ -f ci-output/comment.md ]; then |
| 257 | cat ci-output/comment.md >> "$GITHUB_STEP_SUMMARY" |
| 258 | else |
| 259 | echo '## Docs Agent Eval — ✕ ERROR' >> "$GITHUB_STEP_SUMMARY" |
| 260 | echo 'The CI spine failed before it could produce a report.' >> "$GITHUB_STEP_SUMMARY" |
| 261 | fi |
| 262 | |
| 263 | - name: Post result comment on the PR |
| 264 | if: always() |
| 265 | working-directory: engine |
| 266 | shell: bash |
| 267 | env: |
| 268 | GH_TOKEN: ${{ github.token }} |
| 269 | run: | |
| 270 | test -f ci-output/comment.md || exit 0 |
| 271 | gh pr comment "$PR_NUMBER" --repo "$SOURCE_REPO" --body-file ci-output/comment.md |
| 272 | |
| 273 | - name: Set check result |
| 274 | if: always() |
| 275 | working-directory: engine |
| 276 | shell: bash |
| 277 | run: | |
| 278 | test -f ci-output/outcome.json || exit 1 |
| 279 | STATUS=$(python -c 'import json; print(json.load(open("ci-output/outcome.json"))["status"])') |
| 280 | test "$STATUS" = PASS || test "$STATUS" = NO_EVAL |