@devries-cameron20 / composio Mirror · read-only / commits / 8e8ce95eb
Fork 0

Composio powers 1000+ toolkits, tool search, context management, authentication, and a sandboxed workbench to help you build AI agents that turn intent into action.

Mirrored from https://github.com/Camo67/composio.git

Synced Nov 21, 19:45 UTC

fix(docs): paginate toolkits fetch so the full catalog reaches the site (#3745)

This PR: - fixes the `docs.composio.dev/toolkits` search returning 0 results for toolkits like Workday that exist in the catalog - paginates `fetchToolkits()` in `generate-toolkits.ts` by following `next_cursor`: `GET /api/v3/toolkits` silently caps `limit` at 1000 per page and defaults to usage ordering, so the previous single-request fetch shipped only the top-1000 toolkits — about half of the ~2.1k catalog - dedupes across pages by slug (pages can overlap when the catalog shifts between cursor fetches) and throws instead of silently truncating if the catalog ever outgrows the page guard, mirroring the dashboard's `listToolkits` procedure - fixes the `docs-update-data.yml` workflow, broken since the sessions-first docs rewrite (https://github.com/ComposioHQ/composio/pull/3637) moved the generated meta-tools MDX to `docs/content/toolkits/meta-tools/`: `create-pull-request` aborted on the stale `docs/content/reference/meta-tools/` pathspec, so every scheduled data sync since then failed at the Create Pull Request step and docs data went stale - updates the request-volume notes in `fetch-with-retry.ts` and `scripts/README.md` (~3000 → ~6500 requests per run); the existing 429 backoff already absorbs the larger run ## Context The toolkits landing page filters a build-time snapshot (`public/data/toolkits-list.json`) client-side, so every toolkit missing from the snapshot is unsearchable and its detail page has no data. The snapshot regenerates via the scheduled `docs-update-data.yml` workflow; with both fixes in place the automated data PR picks up the full catalog and the "Browse N toolkits" count grows from 1000 to the real catalog size. Verified offline by running the script against a stubbed paginated backend (1500 toolkits across 2 pages): all pages are fetched, ordering is preserved, no duplicates. `bun run types:check` and eslint pass. A manual `workflow_dispatch` of the fixed workflow on this branch produced the full-catalog data PR.

Alberto Schiabel committed Jul 3, 2026 at 17:40 UTC 8e8ce95eb3fb7eb4ef246633769402b31ca64727
4 files changed +56 -19
.github/workflows/docs-update-data.yml
+5 -5
@@ -96,14 +96,14 @@ jobs:
96 id: changes
97 run: |
98 cd ..
99 - git add -N docs/public/data/ docs/public/openapi.json docs/public/openapi-v3.json docs/content/reference/api-reference/ docs/content/reference/v3/api-reference/ docs/content/reference/meta-tools/ 2>/dev/null || true
100 - if git diff --quiet docs/public/data/ docs/public/openapi.json docs/public/openapi-v3.json docs/content/reference/api-reference/ docs/content/reference/v3/api-reference/ docs/content/reference/meta-tools/ 2>/dev/null; then
99 + git add -N docs/public/data/ docs/public/openapi.json docs/public/openapi-v3.json docs/content/reference/api-reference/ docs/content/reference/v3/api-reference/ docs/content/toolkits/meta-tools/ 2>/dev/null || true
100 + if git diff --quiet docs/public/data/ docs/public/openapi.json docs/public/openapi-v3.json docs/content/reference/api-reference/ docs/content/reference/v3/api-reference/ docs/content/toolkits/meta-tools/ 2>/dev/null; then
101 echo "has_changes=false" >> "$GITHUB_OUTPUT"
102 echo "No changes detected"
103 else
104 echo "has_changes=true" >> "$GITHUB_OUTPUT"
105 echo "Changes detected:"
106 - git diff --stat docs/public/data/ docs/public/openapi.json docs/public/openapi-v3.json docs/content/reference/api-reference/ docs/content/reference/v3/api-reference/ docs/content/reference/meta-tools/ 2>/dev/null || true
106 + git diff --stat docs/public/data/ docs/public/openapi.json docs/public/openapi-v3.json docs/content/reference/api-reference/ docs/content/reference/v3/api-reference/ docs/content/toolkits/meta-tools/ 2>/dev/null || true
107 fi
108
109 - name: Create Pull Request
@@ -122,7 +122,7 @@ jobs:
122 - **Toolkit catalog** (`docs/public/data/toolkits.json`, `toolkits-list.json`) — refreshed list of available toolkits, auth schemes, and tools from the backend API
123 - **OpenAPI specs** (`docs/public/openapi.json`, `docs/public/openapi-v3.json`) — latest v3.1 and v3.0 API specifications fetched from staging
124 - **API reference pages** (`docs/content/reference/api-reference/`, `docs/content/reference/v3/api-reference/`) — regenerated index pages for both API versions
125 - - **Meta tools reference** (`docs/public/data/meta-tools.json`, `docs/content/reference/meta-tools/*.mdx`) — updated meta tool schemas and reference docs
125 + - **Meta tools reference** (`docs/public/data/meta-tools.json`, `docs/content/toolkits/meta-tools/*.mdx`) — updated meta tool schemas and reference docs
126 branch: docs/auto-update-data
127 base: next
128 add-paths: |
@@ -131,7 +131,7 @@ jobs:
131 docs/public/openapi-v3.json
132 docs/content/reference/api-reference/
133 docs/content/reference/v3/api-reference/
134 - docs/content/reference/meta-tools/
134 + docs/content/toolkits/meta-tools/
135
136 - name: Request review from trigger actor
137 if: steps.create-pr.outputs.pull-request-number
docs/scripts/README.md
+3 -2
@@ -9,8 +9,9 @@ responses, honors the `Retry-After` header when present, otherwise falls back to
9 exponential backoff with jitter, and caps attempts so CI still fails fast when
10 the backend is genuinely down.
11
12 -This matters because `generate-toolkits.ts` issues ~3000 requests per run
13 -(2 + 3 per toolkit), which exceeds the staging limit of 2000 requests/minute.
12 +This matters because `generate-toolkits.ts` issues ~6500 requests per run
13 +(a few catalog pages + 3 per toolkit across a ~2.1k catalog), which exceeds the
14 +staging limit of 2000 requests/minute.
15 Before this helper, runs failed with `429`, and `generate-meta-tools.ts` — which
16 runs immediately after — inherited the exhausted rate-limit window.
17
docs/scripts/fetch-with-retry.ts
+3 -2
@@ -13,8 +13,9 @@
13 * Network-level errors thrown by `fetch` (DNS, connection reset, …) are retried
14 * the same way and re-thrown once attempts are exhausted.
15 *
16 - * Why this exists: `generate-toolkits.ts` issues ~3000 requests per run, which
17 - * exceeds the staging limit of 2000 requests/minute. Without backoff the run
16 + * Why this exists: `generate-toolkits.ts` issues ~6500 requests per run (3 per
17 + * toolkit across a ~2.1k catalog), which exceeds the staging limit of 2000
18 + * requests/minute. Without backoff the run
19 * fails with `429`, and `generate-meta-tools.ts` (which runs next) inherits the
20 * exhausted window. See docs/scripts/README.md.
21 *
docs/scripts/generate-toolkits.ts
+45 -10
@@ -74,22 +74,57 @@ interface Toolkit {
74 authConfigDetails?: AuthConfigDetail[];
75 }
76
77 +// The backend silently caps `limit` at 1000 per page and defaults to usage
78 +// ordering, so a single request returns only the top-1000 toolkits — about half
79 +// the catalog. Request the cap and follow `next_cursor` until exhausted.
80 +// MAX_PAGES is a runaway guard well above the real catalog (~2.1k → 3 pages).
81 +const TOOLKITS_PAGE_LIMIT = 1000;
82 +const TOOLKITS_MAX_PAGES = 12;
83 +
84 async function fetchToolkits(): Promise<any[]> {
85 console.log('Fetching toolkits from API...');
86
80 - const response = await fetchWithRetry(`${API_BASE}/toolkits`, {
81 - headers: {
82 - 'Content-Type': 'application/json',
83 - 'x-api-key': API_KEY!,
84 - },
85 - });
87 + const items: any[] = [];
88 + // Pages can overlap when the catalog shifts between cursor fetches; keep the
89 + // first occurrence so API-provided ordering stays stable.
90 + const seen = new Set<string>();
91 + let cursor: string | undefined;
92
87 - if (!response.ok) {
88 - throw new Error(`Failed to fetch toolkits: ${response.status} ${response.statusText}`);
93 + for (let page = 0; page < TOOLKITS_MAX_PAGES; page++) {
94 + const params = new URLSearchParams({ limit: String(TOOLKITS_PAGE_LIMIT) });
95 + if (cursor) params.set('cursor', cursor);
96 +
97 + const response = await fetchWithRetry(`${API_BASE}/toolkits?${params}`, {
98 + headers: {
99 + 'Content-Type': 'application/json',
100 + 'x-api-key': API_KEY!,
101 + },
102 + });
103 +
104 + if (!response.ok) {
105 + throw new Error(`Failed to fetch toolkits: ${response.status} ${response.statusText}`);
106 + }
107 +
108 + const data = await response.json();
109 + const pageItems: any[] = data.items || data;
110 + for (const item of pageItems) {
111 + const slug = item?.slug?.toLowerCase();
112 + if (slug) {
113 + if (seen.has(slug)) continue;
114 + seen.add(slug);
115 + }
116 + items.push(item);
117 + }
118 +
119 + cursor = data.next_cursor ?? undefined;
120 + if (!cursor) return items;
121 }
122
91 - const data = await response.json();
92 - return data.items || data;
123 + // Failing beats silently publishing a truncated catalog — that is the exact
124 + // bug this pagination loop exists to prevent.
125 + throw new Error(
126 + `Toolkit catalog exceeds ${TOOLKITS_MAX_PAGES} pages of ${TOOLKITS_PAGE_LIMIT}; raise TOOLKITS_MAX_PAGES`
127 + );
128 }
129
130 async function fetchToolkitChangelog(): Promise<Map<string, string>> {