Composio powers 1000+ toolkits, tool search, context management, authentication, and a sandboxed workbench to help you build AI agents that turn intent into action.
Mirrored from https://github.com/Camo67/composio.git
Synced Nov 21, 03:37 UTC
| 1 | name: Docs - Update Data |
| 2 | |
| 3 | on: |
| 4 | schedule: |
| 5 | - cron: '0 */5 * * *' |
| 6 | repository_dispatch: |
| 7 | types: [apollo-production-deploy, mercury-production-deploy] |
| 8 | workflow_dispatch: |
| 9 | |
| 10 | permissions: |
| 11 | contents: read |
| 12 | |
| 13 | jobs: |
| 14 | update-data: |
| 15 | runs-on: ubuntu-latest |
| 16 | defaults: |
| 17 | run: |
| 18 | working-directory: ./docs |
| 19 | permissions: |
| 20 | contents: write |
| 21 | pull-requests: write |
| 22 | issues: write |
| 23 | env: |
| 24 | # Git 2.54 moved automatic cleanup from gc.auto to maintenance.auto, but |
| 25 | # actions/checkout@v7.0.1 only disables the former. Disable maintenance |
| 26 | # for every Git process in this job so its background task cannot race the |
| 27 | # shallow fetch used to update docs/auto-update-data. |
| 28 | # https://github.com/actions/checkout/issues/2437 |
| 29 | GIT_CONFIG_COUNT: '1' |
| 30 | GIT_CONFIG_KEY_0: maintenance.auto |
| 31 | GIT_CONFIG_VALUE_0: 'false' |
| 32 | # Deliberately NOT secrets.COMPOSIO_API_KEY. That secret is shared with |
| 33 | # ts.test-e2e, py.test, py.check and ts.examples-nightly, all of which run |
| 34 | # against staging — it is a staging-scoped credential. This job is the only |
| 35 | # consumer that must reach production (scripts/production-api.mjs rejects any |
| 36 | # non-production base URL), so it needs its own production key. Pointing the |
| 37 | # shared secret at production instead would break the staging suites. |
| 38 | COMPOSIO_API_KEY: ${{ secrets.COMPOSIO_DOCS_API_KEY }} |
| 39 | |
| 40 | steps: |
| 41 | - name: Generate GitHub App token |
| 42 | id: app-token |
| 43 | uses: actions/create-github-app-token@64ab729a6294000f5aa4a4bc0b73d1f32bc45510 # v3.3.0 |
| 44 | with: |
| 45 | client-id: ${{ vars.RELEASE_BOT_CLIENT_ID }} |
| 46 | private-key: ${{ secrets.RELEASE_BOT_APP_PRIVATE_KEY }} |
| 47 | owner: ComposioHQ |
| 48 | repositories: composio |
| 49 | # Same-repository issue tracking uses github.token under the job-level |
| 50 | # issues grant; do not widen this app token for issue operations. |
| 51 | permission-contents: write |
| 52 | permission-pull-requests: write |
| 53 | |
| 54 | - name: Checkout repository |
| 55 | uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 |
| 56 | with: |
| 57 | token: ${{ steps.app-token.outputs.token }} |
| 58 | persist-credentials: false |
| 59 | |
| 60 | - name: Log trigger source |
| 61 | env: |
| 62 | EVENT_NAME: ${{ github.event_name }} |
| 63 | DISPATCH_ACTION: ${{ github.event.action }} |
| 64 | APOLLO_COMMIT: ${{ github.event.client_payload.apollo_commit || github.event.client_payload.hermes_commit }} |
| 65 | MERCURY_COMMIT: ${{ github.event.client_payload.mercury_commit }} |
| 66 | DEPLOY_TIMESTAMP: ${{ github.event.client_payload.timestamp }} |
| 67 | run: | |
| 68 | echo "Workflow triggered by: $EVENT_NAME" |
| 69 | if [ "$EVENT_NAME" = "repository_dispatch" ]; then |
| 70 | echo "Repository dispatch action: $DISPATCH_ACTION" |
| 71 | if [ "$DISPATCH_ACTION" = "mercury-production-deploy" ]; then |
| 72 | echo "Triggered by Mercury production registry sync" |
| 73 | echo "Mercury commit: $MERCURY_COMMIT" |
| 74 | else |
| 75 | echo "Triggered by Apollo production deployment" |
| 76 | echo "Apollo commit: $APOLLO_COMMIT" |
| 77 | fi |
| 78 | echo "Timestamp: $DEPLOY_TIMESTAMP" |
| 79 | fi |
| 80 | |
| 81 | - name: Setup Node.js, pnpm, Bun |
| 82 | uses: ./.github/actions/setup-node-pnpm-bun |
| 83 | |
| 84 | # Docs must reflect PRODUCTION. No base-URL override is set here: the |
| 85 | # Toolkit and OpenAPI generators default to production (docs/scripts/production-api.mjs). |
| 86 | # Previously this step pointed the fetch at STAGING, which published staging |
| 87 | # hosts (and unreleased content) into the committed docs. |
| 88 | # NOTE: requires COMPOSIO_DOCS_API_KEY to have PRODUCTION read access. |
| 89 | |
| 90 | - name: Cache bun dependencies |
| 91 | uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 |
| 92 | with: |
| 93 | path: ~/.bun/install/cache |
| 94 | key: ${{ runner.os }}-bun-${{ hashFiles('docs/bun.lock') }} |
| 95 | restore-keys: | |
| 96 | ${{ runner.os }}-bun- |
| 97 | |
| 98 | - name: Install dependencies |
| 99 | run: bun install |
| 100 | |
| 101 | - name: Generate toolkits data |
| 102 | run: bun run generate:toolkits |
| 103 | |
| 104 | - name: Fetch OpenAPI spec |
| 105 | run: bun run scripts/fetch-openapi.mjs |
| 106 | |
| 107 | - name: Generate API index pages |
| 108 | run: bun run generate:api-index |
| 109 | |
| 110 | - name: Generate meta tools reference |
| 111 | env: |
| 112 | COMPOSIO_META_TOOLS_SCHEMA_URL: ${{ vars.COMPOSIO_META_TOOLS_SCHEMA_URL }} |
| 113 | run: bun run generate:meta-tools |
| 114 | |
| 115 | # The published KB makes claims about tool slugs, toolkits, and provider docs. |
| 116 | # Those claims are checkable against the catalog this job just refreshed, so |
| 117 | # freshness is verified on every production deploy rather than on a review |
| 118 | # calendar. Findings never block the data sync — they are reported instead. |
| 119 | - name: Verify KB freshness against refreshed data |
| 120 | id: verify-kb |
| 121 | continue-on-error: true |
| 122 | run: | |
| 123 | set +e |
| 124 | bun scripts/verify-kb.ts --check-links \ |
| 125 | --markdown "$RUNNER_TEMP/kb-freshness.md" |
| 126 | echo "exit_code=$?" >> "$GITHUB_OUTPUT" |
| 127 | cat "$RUNNER_TEMP/kb-freshness.md" >> "$GITHUB_STEP_SUMMARY" |
| 128 | |
| 129 | - name: Check for changes |
| 130 | id: changes |
| 131 | run: | |
| 132 | cd .. |
| 133 | git add -N docs/public/data/ docs/public/openapi.json docs/public/openapi-v3.json docs/public/openapi-webhooks.json docs/content/reference/api-reference/ docs/content/reference/v3/api-reference/ docs/content/toolkits/meta-tools/ 2>/dev/null || true |
| 134 | if git diff --quiet docs/public/data/ docs/public/openapi.json docs/public/openapi-v3.json docs/public/openapi-webhooks.json docs/content/reference/api-reference/ docs/content/reference/v3/api-reference/ docs/content/toolkits/meta-tools/ 2>/dev/null; then |
| 135 | echo "has_changes=false" >> "$GITHUB_OUTPUT" |
| 136 | echo "No changes detected" |
| 137 | else |
| 138 | echo "has_changes=true" >> "$GITHUB_OUTPUT" |
| 139 | echo "Changes detected:" |
| 140 | git diff --stat docs/public/data/ docs/public/openapi.json docs/public/openapi-v3.json docs/public/openapi-webhooks.json docs/content/reference/api-reference/ docs/content/reference/v3/api-reference/ docs/content/toolkits/meta-tools/ 2>/dev/null || true |
| 141 | fi |
| 142 | |
| 143 | - name: Create Pull Request |
| 144 | id: create-pr |
| 145 | if: steps.changes.outputs.has_changes == 'true' |
| 146 | uses: peter-evans/create-pull-request@3ffccacd46053766916f087ce41d627c0b8cf2d1 # v8.1.3 |
| 147 | with: |
| 148 | token: ${{ steps.app-token.outputs.token }} |
| 149 | commit-message: 'docs: update toolkits and API data' |
| 150 | title: 'docs: update toolkits, API spec, and meta tools data' |
| 151 | body: | |
| 152 | ## Summary |
| 153 | Automated sync of backend data into the docs site. |
| 154 | |
| 155 | - Trigger: `${{ github.event_name }}` |
| 156 | - Dispatch action: `${{ github.event.action || 'n/a' }}` |
| 157 | - Source commit: `${{ github.event.client_payload.mercury_commit || github.event.client_payload.apollo_commit || github.event.client_payload.hermes_commit || 'n/a' }}` |
| 158 | |
| 159 | ## What changed |
| 160 | - **Toolkit catalog** (`docs/public/data/toolkits.json`, `toolkits-list.json`) — refreshed list of available toolkits, auth schemes, and tools from the backend API |
| 161 | - **OpenAPI specs** (`docs/public/openapi.json`, `docs/public/openapi-v3.json`, `docs/public/openapi-webhooks.json`) — latest v3.1 and v3.0 API specifications plus the webhook-events spec, fetched from production |
| 162 | - **API reference pages** (`docs/content/reference/api-reference/`, `docs/content/reference/v3/api-reference/`) — regenerated index pages for both API versions |
| 163 | - **Meta tools reference** (`docs/public/data/meta-tools.json`, `docs/content/toolkits/meta-tools/*.mdx`) — generated from the bundled schema snapshot or the configured Platform export |
| 164 | branch: docs/auto-update-data |
| 165 | base: next |
| 166 | add-paths: | |
| 167 | docs/public/data/ |
| 168 | docs/public/openapi.json |
| 169 | docs/public/openapi-v3.json |
| 170 | docs/public/openapi-webhooks.json |
| 171 | docs/content/reference/api-reference/ |
| 172 | docs/content/reference/v3/api-reference/ |
| 173 | docs/content/toolkits/meta-tools/ |
| 174 | |
| 175 | - name: Request review from trigger actor |
| 176 | if: steps.create-pr.outputs.pull-request-number |
| 177 | continue-on-error: true |
| 178 | env: |
| 179 | GH_TOKEN: ${{ steps.app-token.outputs.token }} |
| 180 | PR_NUMBER: ${{ steps.create-pr.outputs.pull-request-number }} |
| 181 | AUTHOR: ${{ github.actor }} |
| 182 | run: | |
| 183 | gh pr edit "$PR_NUMBER" --add-reviewer "$AUTHOR" || \ |
| 184 | gh pr edit "$PR_NUMBER" --add-reviewer "Sushmithamallesh" |
| 185 | |
| 186 | # A silent failure here is invisible: the workflow stops refreshing |
| 187 | # public/data/toolkits.json, the docs site keeps building from the last |
| 188 | # good commit, and nothing 500s — new toolkits simply 404. That is how a |
| 189 | # credential failure went unnoticed for 60 consecutive runs. File one |
| 190 | # tracking issue and close it after the next successful run proves recovery. |
| 191 | - name: Open tracking issue on failure |
| 192 | if: failure() |
| 193 | continue-on-error: true |
| 194 | # Override the job-level `./docs` default. That path only exists after |
| 195 | # checkout, so a failure before it (e.g. the app-token step) would leave |
| 196 | # the runner unable to start this shell — and continue-on-error would |
| 197 | # swallow that, losing the alert for precisely the earliest failures. |
| 198 | working-directory: ${{ github.workspace }} |
| 199 | env: |
| 200 | GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} |
| 201 | GH_REPO: ${{ github.repository }} |
| 202 | LABEL: docs-data-sync-failure |
| 203 | RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} |
| 204 | EVENT_NAME: ${{ github.event_name }} |
| 205 | run: | |
| 206 | gh label create "$LABEL" \ |
| 207 | --color B60205 \ |
| 208 | --description "Scheduled docs data sync is failing" 2>/dev/null || true |
| 209 | |
| 210 | existing=$(gh issue list --label "$LABEL" --state open --limit 1 --json number --jq '.[0].number // empty') |
| 211 | if [ -n "$existing" ]; then |
| 212 | echo "Issue #$existing is already open for this failure; not filing a duplicate." |
| 213 | exit 0 |
| 214 | fi |
| 215 | |
| 216 | gh issue create \ |
| 217 | --title "Docs data sync is failing" \ |
| 218 | --label "$LABEL" \ |
| 219 | --body "The scheduled \`Docs - Update Data\` workflow failed. |
| 220 | |
| 221 | - Run: $RUN_URL |
| 222 | - Trigger: \`$EVENT_NAME\` |
| 223 | |
| 224 | **Impact:** while this is red, \`docs/public/data/toolkits.json\` stops being |
| 225 | refreshed. docs.composio.dev/toolkits is statically generated from that file, so |
| 226 | toolkits added to production after the last successful run have no page and |
| 227 | return 404. The site stays up, which is why this fails silently. |
| 228 | |
| 229 | **First thing to check:** the \`COMPOSIO_DOCS_API_KEY\` secret must have |
| 230 | **production** read access against \`backend.composio.dev\`. Note this job does |
| 231 | not use the shared \`COMPOSIO_API_KEY\`, which is staging-scoped. |
| 232 | |
| 233 | This issue is filed once, not re-filed on every run, and automatically closes |
| 234 | after the next successful sync proves recovery." |
| 235 | |
| 236 | - name: Close tracking issue on recovery |
| 237 | if: success() |
| 238 | continue-on-error: true |
| 239 | env: |
| 240 | GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} |
| 241 | GH_REPO: ${{ github.repository }} |
| 242 | LABEL: docs-data-sync-failure |
| 243 | run: | |
| 244 | existing=$(gh issue list --label "$LABEL" --state open --limit 1 \ |
| 245 | --json number --jq '.[0].number // empty') |
| 246 | if [ -n "$existing" ]; then |
| 247 | gh issue close "$existing" \ |
| 248 | --comment "The docs data sync recovered in run ${{ github.run_id }}: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}." |
| 249 | fi |
| 250 | |
| 251 | # A verifier crash is a tooling failure, not evidence that published guides |
| 252 | # contradict production. Track it separately and close it after the verifier |
| 253 | # returns either of its expected outcomes: clean (0) or findings (1). |
| 254 | - name: Track KB freshness verifier health |
| 255 | if: always() && steps.verify-kb.outputs.exit_code != '' |
| 256 | continue-on-error: true |
| 257 | env: |
| 258 | GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} |
| 259 | VERIFY_EXIT: ${{ steps.verify-kb.outputs.exit_code }} |
| 260 | LABEL: kb-freshness-check-failure |
| 261 | TITLE: 'KB freshness verifier is failing' |
| 262 | run: | |
| 263 | set -euo pipefail |
| 264 | existing=$(gh issue list --state open --label "$LABEL" \ |
| 265 | --json number --jq '.[0].number // empty' 2>/dev/null || true) |
| 266 | |
| 267 | if [ "$VERIFY_EXIT" = "0" ] || [ "$VERIFY_EXIT" = "1" ]; then |
| 268 | if [ -n "$existing" ]; then |
| 269 | gh issue close "$existing" \ |
| 270 | --comment "The verifier recovered in run ${{ github.run_id }} (exit $VERIFY_EXIT)." |
| 271 | fi |
| 272 | exit 0 |
| 273 | fi |
| 274 | |
| 275 | gh label create "$LABEL" --color B60205 \ |
| 276 | --description "KB freshness verifier is failing" 2>/dev/null || true |
| 277 | |
| 278 | { |
| 279 | echo "The KB freshness verifier exited $VERIFY_EXIT before it could classify published content." |
| 280 | echo |
| 281 | echo "This is a verifier or workflow failure, not evidence that published guides contradict production." |
| 282 | echo |
| 283 | echo "_Observed in [run ${{ github.run_id }}](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }})._" |
| 284 | } > "$RUNNER_TEMP/kb-verifier-issue.md" |
| 285 | |
| 286 | if [ -n "$existing" ]; then |
| 287 | gh issue edit "$existing" --body-file "$RUNNER_TEMP/kb-verifier-issue.md" |
| 288 | else |
| 289 | gh issue create --title "$TITLE" --label "$LABEL" \ |
| 290 | --body-file "$RUNNER_TEMP/kb-verifier-issue.md" |
| 291 | fi |
| 292 | |
| 293 | # One long-lived issue, rewritten each run. A new issue per sweep would train |
| 294 | # everyone to ignore the label; a single issue that closes itself when the KB |
| 295 | # is clean stays worth reading. Runs regardless of whether data changed — |
| 296 | # a guide can rot while the catalog is unchanged. |
| 297 | - name: Track KB freshness findings |
| 298 | if: always() && (steps.verify-kb.outputs.exit_code == '0' || steps.verify-kb.outputs.exit_code == '1') |
| 299 | continue-on-error: true |
| 300 | env: |
| 301 | GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} |
| 302 | VERIFY_EXIT: ${{ steps.verify-kb.outputs.exit_code }} |
| 303 | TITLE: 'KB freshness: published guides contradict production data' |
| 304 | run: | |
| 305 | set -euo pipefail |
| 306 | # Exactly one issue carries this label, so listing by label is the lookup. |
| 307 | existing=$(gh issue list --state open --label kb-freshness \ |
| 308 | --json number --jq '.[0].number // empty' 2>/dev/null || true) |
| 309 | |
| 310 | if [ "$VERIFY_EXIT" = "0" ]; then |
| 311 | if [ -n "$existing" ]; then |
| 312 | gh issue close "$existing" \ |
| 313 | --comment "Verified clean against the current production catalog by run ${{ github.run_id }}." |
| 314 | fi |
| 315 | exit 0 |
| 316 | fi |
| 317 | |
| 318 | { |
| 319 | echo "\`verify:kb\` found published KB guides whose claims no longer match production." |
| 320 | echo |
| 321 | echo "Fix the guide, or demote it to \`state: needs-review\` in its \`docs/kb/guides/<slug>.json\`" |
| 322 | echo "so it stops serving readers while it is wrong." |
| 323 | echo |
| 324 | cat "$RUNNER_TEMP/kb-freshness.md" 2>/dev/null \ |
| 325 | || echo "Report unavailable — \`verify:kb\` exited $VERIFY_EXIT before writing it." |
| 326 | echo |
| 327 | echo "_Regenerated by [run ${{ github.run_id }}](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }})._" |
| 328 | } > "$RUNNER_TEMP/kb-issue.md" |
| 329 | |
| 330 | gh label create kb-freshness --color FBCA04 \ |
| 331 | --description "Published KB guides contradicting production data" 2>/dev/null || true |
| 332 | |
| 333 | if [ -n "$existing" ]; then |
| 334 | gh issue edit "$existing" --body-file "$RUNNER_TEMP/kb-issue.md" |
| 335 | else |
| 336 | gh issue create --title "$TITLE" --label kb-freshness \ |
| 337 | --body-file "$RUNNER_TEMP/kb-issue.md" |
| 338 | fi |