@devries-cameron20 / composio-1 / commits / 82896858e
Fork 0

Composio powers 1000+ toolkits, tool search, context management, authentication, and a sandboxed workbench to help you build AI agents that turn intent into action.

fix(docs): retry 429s in data sync scripts and lower concurrency (#3608)

This PR: - adds a shared `fetchWithRetry` helper for the docs data-sync scripts that retries `429` and transient 5xx responses, honoring `Retry-After` (with jitter) and falling back to exponential backoff - routes every request in `generate-toolkits.ts` and `generate-meta-tools.ts` through it, so the `Docs - Update Data` workflow rides out the staging rate limit (2000 req/min) instead of hard-failing - lowers toolkit fetch `batchSize` 10 → 5 (~30 → ~15 concurrent requests) to soften burst pressure on the rate limit - caps retry attempts so CI still fails fast when the backend is genuinely down - fixes the current `429 Too Many Requests` failure in `Generate meta tools reference`, which inherited the rate-limit window exhausted by the high-volume `generate-toolkits.ts` run - verified with `bun run types:check` and an offline retry/backoff behavior test (real end-to-end run needs the staging key in CI: `gh workflow run docs-update-data.yml`) - follow-up (separate PR): reduce per-toolkit request volume in `generate-toolkits.ts` — left out here because it needs consumer and search-quality validation

Alberto Schiabel committed Jun 18, 2026 at 20:37 UTC 82896858ecc3697b1d9a4e8ad554c7cdd3a97bb2
4 files changed +167 -9
docs/scripts/README.md
+14
@@ -1,5 +1,19 @@
1 # OpenAPI Scripts
2
3 +## fetch-with-retry.ts
4 +
5 +Shared `fetchWithRetry` helper used by the data-generation scripts
6 +(`generate-toolkits.ts`, `generate-meta-tools.ts`). It wraps the global `fetch`
7 +with rate-limit-aware retry/backoff: it retries `429` and transient 5xx
8 +responses, honors the `Retry-After` header when present, otherwise falls back to
9 +exponential backoff with jitter, and caps attempts so CI still fails fast when
10 +the backend is genuinely down.
11 +
12 +This matters because `generate-toolkits.ts` issues ~3000 requests per run
13 +(2 + 3 per toolkit), which exceeds the staging limit of 2000 requests/minute.
14 +Before this helper, runs failed with `429`, and `generate-meta-tools.ts` — which
15 +runs immediately after — inherited the exhausted rate-limit window.
16 +
17 ## fetch-openapi.mjs
18
19 Fetches the Composio OpenAPI spec and filters it for use in Fumadocs API reference documentation.
docs/scripts/fetch-with-retry.ts new
+139
@@ -0,0 +1,139 @@
1 +/**
2 + * Shared fetch helper for docs generation scripts.
3 + *
4 + * Wraps the global `fetch` with rate-limit-aware retry/backoff:
5 + * - Retries `429 Too Many Requests` and transient 5xx responses.
6 + * - Honors the `Retry-After` header (delta-seconds or HTTP-date) when present.
7 + * - Falls back to exponential backoff with full jitter.
8 + * - Logs the endpoint and retry count so CI logs explain the delay.
9 + * - Caps attempts so CI still fails fast when the backend is genuinely down —
10 + * the final failing `Response` is returned to the caller, which keeps existing
11 + * `if (!response.ok)` handling intact.
12 + *
13 + * Network-level errors thrown by `fetch` (DNS, connection reset, …) are retried
14 + * the same way and re-thrown once attempts are exhausted.
15 + *
16 + * Why this exists: `generate-toolkits.ts` issues ~3000 requests per run, which
17 + * exceeds the staging limit of 2000 requests/minute. Without backoff the run
18 + * fails with `429`, and `generate-meta-tools.ts` (which runs next) inherits the
19 + * exhausted window. See docs/scripts/README.md.
20 + *
21 + * Deliberately hand-rolled rather than using Effect's `Schedule` (which would be
22 + * the idiomatic fit elsewhere): the docs package has no Effect dependency and
23 + * these are plain bun scripts. Effect is reserved for `ts/packages/cli`. Keep
24 + * this helper dependency-free.
25 + */
26 +
27 +const DEFAULT_RETRY_STATUSES = [429, 500, 502, 503, 504];
28 +
29 +export interface FetchWithRetryOptions extends RequestInit {
30 + /** Maximum number of attempts, including the first. Default 6. */
31 + maxAttempts?: number;
32 + /** Base delay in ms for exponential backoff. Default 1000. */
33 + baseDelayMs?: number;
34 + /** Upper bound for a single backoff wait in ms. Default 60000. */
35 + maxDelayMs?: number;
36 + /** HTTP status codes that trigger a retry. Default [429, 500, 502, 503, 504]. */
37 + retryStatuses?: number[];
38 +}
39 +
40 +function sleep(ms: number): Promise<void> {
41 + return new Promise((resolve) => setTimeout(resolve, ms));
42 +}
43 +
44 +/** Parse a `Retry-After` header (delta-seconds or HTTP-date) into ms, or null. */
45 +function parseRetryAfter(header: string | null): number | null {
46 + if (!header) return null;
47 +
48 + const seconds = Number(header);
49 + if (Number.isFinite(seconds)) {
50 + return Math.max(0, seconds * 1000);
51 + }
52 +
53 + const dateMs = Date.parse(header);
54 + if (!Number.isNaN(dateMs)) {
55 + return Math.max(0, dateMs - Date.now());
56 + }
57 +
58 + return null;
59 +}
60 +
61 +/** Exponential backoff with full jitter. `attempt` is 1-based. */
62 +function backoffDelay(attempt: number, baseDelayMs: number, maxDelayMs: number): number {
63 + const ceiling = Math.min(maxDelayMs, baseDelayMs * 2 ** (attempt - 1));
64 + return Math.random() * ceiling;
65 +}
66 +
67 +/** Collapse a URL to `origin + pathname` for tidy log lines (drops query noise). */
68 +function loggableEndpoint(url: string): string {
69 + try {
70 + const parsed = new URL(url);
71 + return `${parsed.origin}${parsed.pathname}`;
72 + } catch {
73 + return url;
74 + }
75 +}
76 +
77 +export async function fetchWithRetry(
78 + url: string,
79 + options: FetchWithRetryOptions = {}
80 +): Promise<Response> {
81 + const {
82 + maxAttempts = 6,
83 + baseDelayMs = 1000,
84 + maxDelayMs = 60_000,
85 + retryStatuses = DEFAULT_RETRY_STATUSES,
86 + ...init
87 + } = options;
88 +
89 + const endpoint = loggableEndpoint(url);
90 + let lastError: unknown;
91 +
92 + for (let attempt = 1; attempt <= maxAttempts; attempt++) {
93 + let response: Response;
94 +
95 + try {
96 + response = await fetch(url, init);
97 + } catch (error) {
98 + lastError = error;
99 + if (attempt >= maxAttempts) break;
100 + const delay = backoffDelay(attempt, baseDelayMs, maxDelayMs);
101 + console.warn(
102 + ` [fetch] ${endpoint} network error (attempt ${attempt}/${maxAttempts}), ` +
103 + `retrying in ${Math.round(delay)}ms: ${(error as Error).message}`
104 + );
105 + await sleep(delay);
106 + continue;
107 + }
108 +
109 + // Success, or a non-retryable status: hand it back to the caller.
110 + if (!retryStatuses.includes(response.status)) {
111 + return response;
112 + }
113 +
114 + // Out of attempts: surface the failing response so callers still fail.
115 + if (attempt >= maxAttempts) {
116 + return response;
117 + }
118 +
119 + const retryAfterMs = parseRetryAfter(response.headers.get('retry-after'));
120 + // Add jitter on top of Retry-After to avoid a synchronized retry burst.
121 + const delay =
122 + retryAfterMs != null
123 + ? retryAfterMs + Math.random() * 1000
124 + : backoffDelay(attempt, baseDelayMs, maxDelayMs);
125 +
126 + console.warn(
127 + ` [fetch] ${endpoint} -> ${response.status} (attempt ${attempt}/${maxAttempts}), ` +
128 + `retrying in ${Math.round(delay)}ms${retryAfterMs != null ? ' (Retry-After)' : ''}`
129 + );
130 +
131 + // Drain the body so the connection can be reused.
132 + await response.arrayBuffer().catch(() => {});
133 + await sleep(delay);
134 + }
135 +
136 + throw lastError instanceof Error
137 + ? lastError
138 + : new Error(`fetchWithRetry: exhausted ${maxAttempts} attempts for ${endpoint}`);
139 +}
docs/scripts/generate-meta-tools.ts
+3 -2
@@ -18,6 +18,7 @@
18
19 import { writeFile, mkdir, readdir, unlink } from 'fs/promises';
20 import { join } from 'path';
21 +import { fetchWithRetry } from './fetch-with-retry';
22
23 const API_BASE = process.env.COMPOSIO_API_BASE || 'https://backend.composio.dev/api/v3';
24 const API_KEY = process.env.COMPOSIO_API_KEY;
@@ -33,7 +34,7 @@ const CONTENT_DIR = join(process.cwd(), 'content/reference/meta-tools');
34 async function createSession(): Promise<string> {
35 console.log('Creating session...');
36
36 - const response = await fetch(`${API_BASE}/tool_router/session`, {
37 + const response = await fetchWithRetry(`${API_BASE}/tool_router/session`, {
38 method: 'POST',
39 headers: {
40 'Content-Type': 'application/json',
@@ -69,7 +70,7 @@ async function createSession(): Promise<string> {
70 async function fetchMetaTools(sessionId: string): Promise<any[]> {
71 console.log('Fetching meta tools with schemas...');
72
72 - const response = await fetch(`${API_BASE}/tool_router/session/${sessionId}/tools`, {
73 + const response = await fetchWithRetry(`${API_BASE}/tool_router/session/${sessionId}/tools`, {
74 headers: {
75 'x-api-key': API_KEY!,
76 },
docs/scripts/generate-toolkits.ts
+11 -7
@@ -10,6 +10,7 @@
10
11 import { mkdir, writeFile } from 'fs/promises';
12 import { join } from 'path';
13 +import { fetchWithRetry } from './fetch-with-retry';
14
15 const API_BASE = process.env.COMPOSIO_API_BASE || 'https://backend.composio.dev/api/v3';
16 const API_KEY = process.env.COMPOSIO_API_KEY;
@@ -76,7 +77,7 @@ interface Toolkit {
77 async function fetchToolkits(): Promise<any[]> {
78 console.log('Fetching toolkits from API...');
79
79 - const response = await fetch(`${API_BASE}/toolkits`, {
80 + const response = await fetchWithRetry(`${API_BASE}/toolkits`, {
81 headers: {
82 'Content-Type': 'application/json',
83 'x-api-key': API_KEY!,
@@ -94,7 +95,7 @@ async function fetchToolkits(): Promise<any[]> {
95 async function fetchToolkitChangelog(): Promise<Map<string, string>> {
96 console.log('Fetching toolkit changelog...');
97
97 - const response = await fetch(`${API_BASE}/toolkits/changelog`, {
98 + const response = await fetchWithRetry(`${API_BASE}/toolkits/changelog`, {
99 headers: {
100 'Content-Type': 'application/json',
101 'x-api-key': API_KEY!,
@@ -124,7 +125,7 @@ async function fetchToolkitChangelog(): Promise<Map<string, string>> {
125 }
126
127 async function fetchToolsForToolkit(slug: string): Promise<Tool[]> {
127 - const response = await fetch(`${API_BASE}/tools?toolkit_slug=${slug}&toolkit_versions=latest&limit=1000`, {
128 + const response = await fetchWithRetry(`${API_BASE}/tools?toolkit_slug=${slug}&toolkit_versions=latest&limit=1000`, {
129 headers: {
130 'Content-Type': 'application/json',
131 'x-api-key': API_KEY!,
@@ -145,7 +146,7 @@ async function fetchToolsForToolkit(slug: string): Promise<Tool[]> {
146 }
147
148 async function fetchTriggersForToolkit(slug: string): Promise<Trigger[]> {
148 - const response = await fetch(`${API_BASE}/triggers_types?toolkit_slugs=${slug}&toolkit_versions=latest`, {
149 + const response = await fetchWithRetry(`${API_BASE}/triggers_types?toolkit_slugs=${slug}&toolkit_versions=latest`, {
150 headers: {
151 'Content-Type': 'application/json',
152 'x-api-key': API_KEY!,
@@ -166,7 +167,7 @@ async function fetchTriggersForToolkit(slug: string): Promise<Trigger[]> {
167 }
168
169 async function fetchAuthConfigDetails(slug: string): Promise<AuthConfigDetail[]> {
169 - const response = await fetch(`${API_BASE}/toolkits/${slug}`, {
170 + const response = await fetchWithRetry(`${API_BASE}/toolkits/${slug}`, {
171 headers: {
172 'Content-Type': 'application/json',
173 'x-api-key': API_KEY!,
@@ -263,9 +264,12 @@ async function main() {
264 toolkit.version = versionMap.get(toolkit.slug) || null;
265 }
266
266 - // Fetch tools, triggers, and auth config details for each toolkit in batches
267 + // Fetch tools, triggers, and auth config details for each toolkit in batches.
268 + // Each toolkit fires 3 requests in parallel, so batchSize N = ~3N concurrent
269 + // requests. Kept low to soften burst pressure on the staging rate limit
270 + // (2000 req/min); fetchWithRetry handles the remaining 429s with backoff.
271 console.log('Fetching tools, triggers, and auth config details...');
268 - const batchSize = 10;
272 + const batchSize = 5;
273 let completed = 0;
274
275 for (let i = 0; i < toolkits.length; i += batchSize) {