Composio powers 1000+ toolkits, tool search, context management, authentication, and a sandboxed workbench to help you build AI agents that turn intent into action.

next
yml 338 lines 16 KB
Raw
1 name: Docs - Update Data
2
3 on:
4 schedule:
5 - cron: '0 */5 * * *'
6 repository_dispatch:
7 types: [apollo-production-deploy, mercury-production-deploy]
8 workflow_dispatch:
9
10 permissions:
11 contents: read
12
13 jobs:
14 update-data:
15 runs-on: ubuntu-latest
16 defaults:
17 run:
18 working-directory: ./docs
19 permissions:
20 contents: write
21 pull-requests: write
22 issues: write
23 env:
24 # Git 2.54 moved automatic cleanup from gc.auto to maintenance.auto, but
25 # actions/checkout@v7.0.1 only disables the former. Disable maintenance
26 # for every Git process in this job so its background task cannot race the
27 # shallow fetch used to update docs/auto-update-data.
28 # https://github.com/actions/checkout/issues/2437
29 GIT_CONFIG_COUNT: '1'
30 GIT_CONFIG_KEY_0: maintenance.auto
31 GIT_CONFIG_VALUE_0: 'false'
32 # Deliberately NOT secrets.COMPOSIO_API_KEY. That secret is shared with
33 # ts.test-e2e, py.test, py.check and ts.examples-nightly, all of which run
34 # against staging — it is a staging-scoped credential. This job is the only
35 # consumer that must reach production (scripts/production-api.mjs rejects any
36 # non-production base URL), so it needs its own production key. Pointing the
37 # shared secret at production instead would break the staging suites.
38 COMPOSIO_API_KEY: ${{ secrets.COMPOSIO_DOCS_API_KEY }}
39
40 steps:
41 - name: Generate GitHub App token
42 id: app-token
43 uses: actions/create-github-app-token@64ab729a6294000f5aa4a4bc0b73d1f32bc45510 # v3.3.0
44 with:
45 client-id: ${{ vars.RELEASE_BOT_CLIENT_ID }}
46 private-key: ${{ secrets.RELEASE_BOT_APP_PRIVATE_KEY }}
47 owner: ComposioHQ
48 repositories: composio
49 # Same-repository issue tracking uses github.token under the job-level
50 # issues grant; do not widen this app token for issue operations.
51 permission-contents: write
52 permission-pull-requests: write
53
54 - name: Checkout repository
55 uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
56 with:
57 token: ${{ steps.app-token.outputs.token }}
58 persist-credentials: false
59
60 - name: Log trigger source
61 env:
62 EVENT_NAME: ${{ github.event_name }}
63 DISPATCH_ACTION: ${{ github.event.action }}
64 APOLLO_COMMIT: ${{ github.event.client_payload.apollo_commit || github.event.client_payload.hermes_commit }}
65 MERCURY_COMMIT: ${{ github.event.client_payload.mercury_commit }}
66 DEPLOY_TIMESTAMP: ${{ github.event.client_payload.timestamp }}
67 run: |
68 echo "Workflow triggered by: $EVENT_NAME"
69 if [ "$EVENT_NAME" = "repository_dispatch" ]; then
70 echo "Repository dispatch action: $DISPATCH_ACTION"
71 if [ "$DISPATCH_ACTION" = "mercury-production-deploy" ]; then
72 echo "Triggered by Mercury production registry sync"
73 echo "Mercury commit: $MERCURY_COMMIT"
74 else
75 echo "Triggered by Apollo production deployment"
76 echo "Apollo commit: $APOLLO_COMMIT"
77 fi
78 echo "Timestamp: $DEPLOY_TIMESTAMP"
79 fi
80
81 - name: Setup Node.js, pnpm, Bun
82 uses: ./.github/actions/setup-node-pnpm-bun
83
84 # Docs must reflect PRODUCTION. No base-URL override is set here: the
85 # Toolkit and OpenAPI generators default to production (docs/scripts/production-api.mjs).
86 # Previously this step pointed the fetch at STAGING, which published staging
87 # hosts (and unreleased content) into the committed docs.
88 # NOTE: requires COMPOSIO_DOCS_API_KEY to have PRODUCTION read access.
89
90 - name: Cache bun dependencies
91 uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
92 with:
93 path: ~/.bun/install/cache
94 key: ${{ runner.os }}-bun-${{ hashFiles('docs/bun.lock') }}
95 restore-keys: |
96 ${{ runner.os }}-bun-
97
98 - name: Install dependencies
99 run: bun install
100
101 - name: Generate toolkits data
102 run: bun run generate:toolkits
103
104 - name: Fetch OpenAPI spec
105 run: bun run scripts/fetch-openapi.mjs
106
107 - name: Generate API index pages
108 run: bun run generate:api-index
109
110 - name: Generate meta tools reference
111 env:
112 COMPOSIO_META_TOOLS_SCHEMA_URL: ${{ vars.COMPOSIO_META_TOOLS_SCHEMA_URL }}
113 run: bun run generate:meta-tools
114
115 # The published KB makes claims about tool slugs, toolkits, and provider docs.
116 # Those claims are checkable against the catalog this job just refreshed, so
117 # freshness is verified on every production deploy rather than on a review
118 # calendar. Findings never block the data sync — they are reported instead.
119 - name: Verify KB freshness against refreshed data
120 id: verify-kb
121 continue-on-error: true
122 run: |
123 set +e
124 bun scripts/verify-kb.ts --check-links \
125 --markdown "$RUNNER_TEMP/kb-freshness.md"
126 echo "exit_code=$?" >> "$GITHUB_OUTPUT"
127 cat "$RUNNER_TEMP/kb-freshness.md" >> "$GITHUB_STEP_SUMMARY"
128
129 - name: Check for changes
130 id: changes
131 run: |
132 cd ..
133 git add -N docs/public/data/ docs/public/openapi.json docs/public/openapi-v3.json docs/public/openapi-webhooks.json docs/content/reference/api-reference/ docs/content/reference/v3/api-reference/ docs/content/toolkits/meta-tools/ 2>/dev/null || true
134 if git diff --quiet docs/public/data/ docs/public/openapi.json docs/public/openapi-v3.json docs/public/openapi-webhooks.json docs/content/reference/api-reference/ docs/content/reference/v3/api-reference/ docs/content/toolkits/meta-tools/ 2>/dev/null; then
135 echo "has_changes=false" >> "$GITHUB_OUTPUT"
136 echo "No changes detected"
137 else
138 echo "has_changes=true" >> "$GITHUB_OUTPUT"
139 echo "Changes detected:"
140 git diff --stat docs/public/data/ docs/public/openapi.json docs/public/openapi-v3.json docs/public/openapi-webhooks.json docs/content/reference/api-reference/ docs/content/reference/v3/api-reference/ docs/content/toolkits/meta-tools/ 2>/dev/null || true
141 fi
142
143 - name: Create Pull Request
144 id: create-pr
145 if: steps.changes.outputs.has_changes == 'true'
146 uses: peter-evans/create-pull-request@3ffccacd46053766916f087ce41d627c0b8cf2d1 # v8.1.3
147 with:
148 token: ${{ steps.app-token.outputs.token }}
149 commit-message: 'docs: update toolkits and API data'
150 title: 'docs: update toolkits, API spec, and meta tools data'
151 body: |
152 ## Summary
153 Automated sync of backend data into the docs site.
154
155 - Trigger: `${{ github.event_name }}`
156 - Dispatch action: `${{ github.event.action || 'n/a' }}`
157 - Source commit: `${{ github.event.client_payload.mercury_commit || github.event.client_payload.apollo_commit || github.event.client_payload.hermes_commit || 'n/a' }}`
158
159 ## What changed
160 - **Toolkit catalog** (`docs/public/data/toolkits.json`, `toolkits-list.json`) — refreshed list of available toolkits, auth schemes, and tools from the backend API
161 - **OpenAPI specs** (`docs/public/openapi.json`, `docs/public/openapi-v3.json`, `docs/public/openapi-webhooks.json`) — latest v3.1 and v3.0 API specifications plus the webhook-events spec, fetched from production
162 - **API reference pages** (`docs/content/reference/api-reference/`, `docs/content/reference/v3/api-reference/`) — regenerated index pages for both API versions
163 - **Meta tools reference** (`docs/public/data/meta-tools.json`, `docs/content/toolkits/meta-tools/*.mdx`) — generated from the bundled schema snapshot or the configured Platform export
164 branch: docs/auto-update-data
165 base: next
166 add-paths: |
167 docs/public/data/
168 docs/public/openapi.json
169 docs/public/openapi-v3.json
170 docs/public/openapi-webhooks.json
171 docs/content/reference/api-reference/
172 docs/content/reference/v3/api-reference/
173 docs/content/toolkits/meta-tools/
174
175 - name: Request review from trigger actor
176 if: steps.create-pr.outputs.pull-request-number
177 continue-on-error: true
178 env:
179 GH_TOKEN: ${{ steps.app-token.outputs.token }}
180 PR_NUMBER: ${{ steps.create-pr.outputs.pull-request-number }}
181 AUTHOR: ${{ github.actor }}
182 run: |
183 gh pr edit "$PR_NUMBER" --add-reviewer "$AUTHOR" || \
184 gh pr edit "$PR_NUMBER" --add-reviewer "Sushmithamallesh"
185
186 # A silent failure here is invisible: the workflow stops refreshing
187 # public/data/toolkits.json, the docs site keeps building from the last
188 # good commit, and nothing 500s — new toolkits simply 404. That is how a
189 # credential failure went unnoticed for 60 consecutive runs. File one
190 # tracking issue and close it after the next successful run proves recovery.
191 - name: Open tracking issue on failure
192 if: failure()
193 continue-on-error: true
194 # Override the job-level `./docs` default. That path only exists after
195 # checkout, so a failure before it (e.g. the app-token step) would leave
196 # the runner unable to start this shell — and continue-on-error would
197 # swallow that, losing the alert for precisely the earliest failures.
198 working-directory: ${{ github.workspace }}
199 env:
200 GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
201 GH_REPO: ${{ github.repository }}
202 LABEL: docs-data-sync-failure
203 RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
204 EVENT_NAME: ${{ github.event_name }}
205 run: |
206 gh label create "$LABEL" \
207 --color B60205 \
208 --description "Scheduled docs data sync is failing" 2>/dev/null || true
209
210 existing=$(gh issue list --label "$LABEL" --state open --limit 1 --json number --jq '.[0].number // empty')
211 if [ -n "$existing" ]; then
212 echo "Issue #$existing is already open for this failure; not filing a duplicate."
213 exit 0
214 fi
215
216 gh issue create \
217 --title "Docs data sync is failing" \
218 --label "$LABEL" \
219 --body "The scheduled \`Docs - Update Data\` workflow failed.
220
221 - Run: $RUN_URL
222 - Trigger: \`$EVENT_NAME\`
223
224 **Impact:** while this is red, \`docs/public/data/toolkits.json\` stops being
225 refreshed. docs.composio.dev/toolkits is statically generated from that file, so
226 toolkits added to production after the last successful run have no page and
227 return 404. The site stays up, which is why this fails silently.
228
229 **First thing to check:** the \`COMPOSIO_DOCS_API_KEY\` secret must have
230 **production** read access against \`backend.composio.dev\`. Note this job does
231 not use the shared \`COMPOSIO_API_KEY\`, which is staging-scoped.
232
233 This issue is filed once, not re-filed on every run, and automatically closes
234 after the next successful sync proves recovery."
235
236 - name: Close tracking issue on recovery
237 if: success()
238 continue-on-error: true
239 env:
240 GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
241 GH_REPO: ${{ github.repository }}
242 LABEL: docs-data-sync-failure
243 run: |
244 existing=$(gh issue list --label "$LABEL" --state open --limit 1 \
245 --json number --jq '.[0].number // empty')
246 if [ -n "$existing" ]; then
247 gh issue close "$existing" \
248 --comment "The docs data sync recovered in run ${{ github.run_id }}: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}."
249 fi
250
251 # A verifier crash is a tooling failure, not evidence that published guides
252 # contradict production. Track it separately and close it after the verifier
253 # returns either of its expected outcomes: clean (0) or findings (1).
254 - name: Track KB freshness verifier health
255 if: always() && steps.verify-kb.outputs.exit_code != ''
256 continue-on-error: true
257 env:
258 GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
259 VERIFY_EXIT: ${{ steps.verify-kb.outputs.exit_code }}
260 LABEL: kb-freshness-check-failure
261 TITLE: 'KB freshness verifier is failing'
262 run: |
263 set -euo pipefail
264 existing=$(gh issue list --state open --label "$LABEL" \
265 --json number --jq '.[0].number // empty' 2>/dev/null || true)
266
267 if [ "$VERIFY_EXIT" = "0" ] || [ "$VERIFY_EXIT" = "1" ]; then
268 if [ -n "$existing" ]; then
269 gh issue close "$existing" \
270 --comment "The verifier recovered in run ${{ github.run_id }} (exit $VERIFY_EXIT)."
271 fi
272 exit 0
273 fi
274
275 gh label create "$LABEL" --color B60205 \
276 --description "KB freshness verifier is failing" 2>/dev/null || true
277
278 {
279 echo "The KB freshness verifier exited $VERIFY_EXIT before it could classify published content."
280 echo
281 echo "This is a verifier or workflow failure, not evidence that published guides contradict production."
282 echo
283 echo "_Observed in [run ${{ github.run_id }}](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }})._"
284 } > "$RUNNER_TEMP/kb-verifier-issue.md"
285
286 if [ -n "$existing" ]; then
287 gh issue edit "$existing" --body-file "$RUNNER_TEMP/kb-verifier-issue.md"
288 else
289 gh issue create --title "$TITLE" --label "$LABEL" \
290 --body-file "$RUNNER_TEMP/kb-verifier-issue.md"
291 fi
292
293 # One long-lived issue, rewritten each run. A new issue per sweep would train
294 # everyone to ignore the label; a single issue that closes itself when the KB
295 # is clean stays worth reading. Runs regardless of whether data changed —
296 # a guide can rot while the catalog is unchanged.
297 - name: Track KB freshness findings
298 if: always() && (steps.verify-kb.outputs.exit_code == '0' || steps.verify-kb.outputs.exit_code == '1')
299 continue-on-error: true
300 env:
301 GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
302 VERIFY_EXIT: ${{ steps.verify-kb.outputs.exit_code }}
303 TITLE: 'KB freshness: published guides contradict production data'
304 run: |
305 set -euo pipefail
306 # Exactly one issue carries this label, so listing by label is the lookup.
307 existing=$(gh issue list --state open --label kb-freshness \
308 --json number --jq '.[0].number // empty' 2>/dev/null || true)
309
310 if [ "$VERIFY_EXIT" = "0" ]; then
311 if [ -n "$existing" ]; then
312 gh issue close "$existing" \
313 --comment "Verified clean against the current production catalog by run ${{ github.run_id }}."
314 fi
315 exit 0
316 fi
317
318 {
319 echo "\`verify:kb\` found published KB guides whose claims no longer match production."
320 echo
321 echo "Fix the guide, or demote it to \`state: needs-review\` in its \`docs/kb/guides/<slug>.json\`"
322 echo "so it stops serving readers while it is wrong."
323 echo
324 cat "$RUNNER_TEMP/kb-freshness.md" 2>/dev/null \
325 || echo "Report unavailable — \`verify:kb\` exited $VERIFY_EXIT before writing it."
326 echo
327 echo "_Regenerated by [run ${{ github.run_id }}](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }})._"
328 } > "$RUNNER_TEMP/kb-issue.md"
329
330 gh label create kb-freshness --color FBCA04 \
331 --description "Published KB guides contradicting production data" 2>/dev/null || true
332
333 if [ -n "$existing" ]; then
334 gh issue edit "$existing" --body-file "$RUNNER_TEMP/kb-issue.md"
335 else
336 gh issue create --title "$TITLE" --label kb-freshness \
337 --body-file "$RUNNER_TEMP/kb-issue.md"
338 fi