6
# the proposed docs through a rewrite proxy, and the result is posted back
7
# here as a PR comment. `run-docs-tests-deep` forces Deep.
8
#
9
-# All eval logic lives in ComposioHQ/docs-agent-eval (reusable workflow) —
9
+# Two entry paths, so label-then-push and push-then-label both work:
10
+# - deployment_status (Vercel finished a docs preview): runs if the PR
11
+# already carries a run-docs-tests label.
12
+# - pull_request labeled/reopened: runs if the docs preview already
13
+# succeeded; otherwise skips, and the deployment_status event re-enters
14
+# when the preview lands. (`synchronize` removed: new pushes surface as
15
+# new deployments, so deployment_status covers them.)
16
+#
17
+# All eval logic lives in ComposioHQ/docs-agent-eval-ci (reusable workflow) —
18
# see its ARCHITECTURE.md. This check is informational and must NOT be made
19
# a required status check in V1.
20
#
13
-# Required repo/org secrets: COMPOSIO_ORG_API_KEY, DEEPSEEK_API_KEY,
14
-# VERCEL_PROJECT_ID,
15
-# VERCEL_TOKEN (see the PR description for provisioning notes).
21
+# Required repo/org secrets: COMPOSIO_ORG_API_KEY, DEEPSEEK_API_KEY.
22
+# No Vercel credentials (decision 2026-08-19): the engine resolves previews
23
+# from GitHub's deployment records with the per-run token.
24
name: Docs Agent Eval
25
26
on:
27
pull_request:
20
- types: [labeled, synchronize, reopened]
28
+ types: [labeled, reopened]
29
+ deployment_status:
30
31
jobs:
23
- eval:
24
- # Run when the PR carries the label — but for `labeled` events, only when
25
- # the label just added is one of ours (any other label addition would
26
- # otherwise cancel-and-restart an in-flight eval; Cursor Bugbot finding).
27
- # Either label triggers on its own (run-docs-tests-deep standalone forces
28
- # Deep); labeled events still only count when the added label is ours.
32
+ # Decides IF an eval should start and FOR WHICH PR. deployment_status
33
+ # events know the commit but not the PR (deployments attach to commits),
34
+ # so this job maps SHA -> open PR before any label check is possible.
35
+ route:
36
if: |
30
- (contains(github.event.pull_request.labels.*.name, 'run-docs-tests') ||
31
- contains(github.event.pull_request.labels.*.name, 'run-docs-tests-deep')) &&
32
- (github.event.action != 'labeled' || startsWith(github.event.label.name, 'run-docs-tests'))
33
- # Job-level on purpose: a skipped job never enters the concurrency group,
34
- # so unrelated triggers cannot cancel a running eval. New commits still
35
- # supersede stale runs.
37
+ (github.event_name == 'pull_request' &&
38
+ (contains(github.event.pull_request.labels.*.name, 'run-docs-tests') ||
39
+ contains(github.event.pull_request.labels.*.name, 'run-docs-tests-deep')) &&
40
+ (github.event.action != 'labeled' || startsWith(github.event.label.name, 'run-docs-tests'))) ||
41
+ (github.event_name == 'deployment_status' &&
42
+ github.event.deployment_status.state == 'success' &&
43
+ contains(github.event.deployment_status.environment_url, '.preview.composio.dev'))
44
+ runs-on: ubuntu-latest
45
+ timeout-minutes: 5
46
+ permissions:
47
+ contents: read
48
+ deployments: read
49
+ pull-requests: read
50
+ outputs:
51
+ should_run: ${{ steps.decide.outputs.should_run }}
52
+ pr_number: ${{ steps.decide.outputs.pr_number }}
53
+ mode: ${{ steps.decide.outputs.mode }}
54
+ steps:
55
+ - name: Decide whether to run, and for which PR
56
+ id: decide
57
+ shell: bash
58
+ env:
59
+ GH_TOKEN: ${{ github.token }}
60
+ REPO: ${{ github.repository }}
61
+ run: |
62
+ set -euo pipefail
63
+ pr="" assoc="" labels="" preview_ok=false
64
+
65
+ if [ "$GITHUB_EVENT_NAME" = deployment_status ]; then
66
+ # Trap 2: the event payload has no PR number — map commit -> PR.
67
+ SHA=$(jq -r .deployment.sha "$GITHUB_EVENT_PATH")
68
+ PRJSON=$(gh api "repos/$REPO/commits/$SHA/pulls" \
69
+ --jq '[.[] | select(.state=="open")][0] // empty')
70
+ if [ -n "$PRJSON" ]; then
71
+ pr=$(jq -r .number <<<"$PRJSON")
72
+ assoc=$(jq -r .author_association <<<"$PRJSON")
73
+ labels=$(jq -r '[.labels[].name] | join(",")' <<<"$PRJSON")
74
+ fi
75
+ preview_ok=true # this event IS the successful preview
76
+ else
77
+ pr=$(jq -r .pull_request.number "$GITHUB_EVENT_PATH")
78
+ assoc=$(jq -r .pull_request.author_association "$GITHUB_EVENT_PATH")
79
+ labels=$(jq -r '[.pull_request.labels[].name] | join(",")' "$GITHUB_EVENT_PATH")
80
+ SHA=$(jq -r .pull_request.head.sha "$GITHUB_EVENT_PATH")
81
+ # Trap 1: label may predate the preview. Only run now if a docs
82
+ # preview already succeeded for this exact commit; otherwise skip
83
+ # — the deployment_status event re-enters when the preview lands.
84
+ for d in $(gh api "repos/$REPO/deployments?sha=$SHA&per_page=10" --jq '.[].id'); do
85
+ URL=$(gh api "repos/$REPO/deployments/$d/statuses" \
86
+ --jq '[.[] | select(.state=="success")][0].environment_url // empty')
87
+ case "$URL" in
88
+ https://docs-*.preview.composio.dev*) preview_ok=true; break ;;
89
+ esac
90
+ done
91
+ fi
92
+
93
+ haslabel=false
94
+ case ",$labels," in
95
+ *,run-docs-tests,*|*,run-docs-tests-deep,*) haslabel=true ;;
96
+ esac
97
+
98
+ # Employee gate: PR author must be org-affiliated (public repo).
99
+ trusted=false
100
+ case "$assoc" in MEMBER|OWNER|COLLABORATOR) trusted=true ;; esac
101
+
102
+ mode=Auto
103
+ case ",$labels," in *,run-docs-tests-deep,*) mode=Deep ;; esac
104
+
105
+ should=false
106
+ if [ -n "$pr" ] && $haslabel && $trusted && $preview_ok; then
107
+ should=true
108
+ fi
109
+ echo "event=$GITHUB_EVENT_NAME pr=${pr:-none} labels=[$labels] assoc=${assoc:-none} preview_ok=$preview_ok -> should_run=$should"
110
+
111
+ {
112
+ echo "should_run=$should"
113
+ echo "pr_number=$pr"
114
+ echo "mode=$mode"
115
+ } >> "$GITHUB_OUTPUT"
116
+
117
+ eval:
118
+ needs: route
119
+ if: needs.route.outputs.should_run == 'true'
120
+ # Concurrency here, after routing: a skipped/filtered event never enters
121
+ # the group, so unrelated triggers cannot cancel a running eval. A newer
122
+ # run for the same PR (new preview, re-label) still supersedes the stale
123
+ # one.
124
concurrency:
37
- group: docs-agent-eval-pr-${{ github.event.pull_request.number }}
125
+ group: docs-agent-eval-pr-${{ needs.route.outputs.pr_number }}
126
cancel-in-progress: true
127
permissions:
128
contents: read
129
pull-requests: write # the result comment
130
# Full-SHA pin required by org policy — bump on docs-agent-eval releases.
43
- uses: ComposioHQ/docs-agent-eval-ci/.github/workflows/eval-pr.yml@48e1a1d2f8baf12a7a073b8141afede3baee94f7
131
+ uses: ComposioHQ/docs-agent-eval-ci/.github/workflows/eval-pr.yml@2a831429ee6d8be6b9b4526f78bf5a451881cf8f
132
with:
45
- pr_number: ${{ github.event.pull_request.number }}
46
- mode: ${{ contains(github.event.pull_request.labels.*.name, 'run-docs-tests-deep') && 'Deep' || 'Auto' }}
133
+ pr_number: ${{ needs.route.outputs.pr_number }}
134
+ mode: ${{ needs.route.outputs.mode }}
135
post_comment: true
136
# Explicit and minimal on purpose — inherit would hand the engine every
137
# secret this repo holds (review decision 2).
138
secrets:
139
COMPOSIO_ORG_API_KEY: ${{ secrets.COMPOSIO_ORG_API_KEY }}
140
DEEPSEEK_API_KEY: ${{ secrets.DEEPSEEK_API_KEY }}
53
- VERCEL_TOKEN: ${{ secrets.VERCEL_TOKEN }}
54
- VERCEL_PROJECT_ID: ${{ secrets.VERCEL_PROJECT_ID }}