Composio powers 1000+ toolkits, tool search, context management, authentication, and a sandboxed workbench to help you build AI agents that turn intent into action.
| 1 | /** |
| 2 | * Shared fetch helper for docs generation scripts. |
| 3 | * |
| 4 | * Wraps the global `fetch` with rate-limit-aware retry/backoff: |
| 5 | * - Retries `429 Too Many Requests` and transient 5xx responses. |
| 6 | * - Honors the `Retry-After` header (delta-seconds or HTTP-date) when present. |
| 7 | * - Falls back to exponential backoff with full jitter. |
| 8 | * - Logs the endpoint and retry count so CI logs explain the delay. |
| 9 | * - Caps attempts so CI still fails fast when the backend is genuinely down — |
| 10 | * the final failing `Response` is returned to the caller, which keeps existing |
| 11 | * `if (!response.ok)` handling intact. |
| 12 | * |
| 13 | * Network-level errors thrown by `fetch` (DNS, connection reset, …) are retried |
| 14 | * the same way and re-thrown once attempts are exhausted. |
| 15 | * |
| 16 | * Why this exists: `generate-toolkits.ts` issues ~6500 requests per run (3 per |
| 17 | * toolkit across a ~2.1k catalog), which can exceed the backend request limit. |
| 18 | * Without backoff the run can fail with `429`. See docs/scripts/README.md. |
| 19 | * |
| 20 | * Deliberately hand-rolled rather than using Effect's `Schedule` (which would be |
| 21 | * the idiomatic fit elsewhere): the docs package has no Effect dependency and |
| 22 | * these are plain bun scripts. Effect is reserved for `ts/packages/cli`. Keep |
| 23 | * this helper dependency-free. |
| 24 | */ |
| 25 | |
| 26 | const DEFAULT_RETRY_STATUSES = [429, 500, 502, 503, 504]; |
| 27 | |
| 28 | export interface FetchWithRetryOptions extends RequestInit { |
| 29 | /** Maximum number of attempts, including the first. Default 6. */ |
| 30 | maxAttempts?: number; |
| 31 | /** Base delay in ms for exponential backoff. Default 1000. */ |
| 32 | baseDelayMs?: number; |
| 33 | /** Upper bound for a single backoff wait in ms. Default 60000. */ |
| 34 | maxDelayMs?: number; |
| 35 | /** HTTP status codes that trigger a retry. Default [429, 500, 502, 503, 504]. */ |
| 36 | retryStatuses?: number[]; |
| 37 | } |
| 38 | |
| 39 | function sleep(ms: number): Promise<void> { |
| 40 | return new Promise((resolve) => setTimeout(resolve, ms)); |
| 41 | } |
| 42 | |
| 43 | /** Parse a `Retry-After` header (delta-seconds or HTTP-date) into ms, or null. */ |
| 44 | function parseRetryAfter(header: string | null): number | null { |
| 45 | if (!header) return null; |
| 46 | |
| 47 | const seconds = Number(header); |
| 48 | if (Number.isFinite(seconds)) { |
| 49 | return Math.max(0, seconds * 1000); |
| 50 | } |
| 51 | |
| 52 | const dateMs = Date.parse(header); |
| 53 | if (!Number.isNaN(dateMs)) { |
| 54 | return Math.max(0, dateMs - Date.now()); |
| 55 | } |
| 56 | |
| 57 | return null; |
| 58 | } |
| 59 | |
| 60 | /** Exponential backoff with full jitter. `attempt` is 1-based. */ |
| 61 | function backoffDelay(attempt: number, baseDelayMs: number, maxDelayMs: number): number { |
| 62 | const ceiling = Math.min(maxDelayMs, baseDelayMs * 2 ** (attempt - 1)); |
| 63 | return Math.random() * ceiling; |
| 64 | } |
| 65 | |
| 66 | /** Collapse a URL to `origin + pathname` for tidy log lines (drops query noise). */ |
| 67 | function loggableEndpoint(url: string): string { |
| 68 | try { |
| 69 | const parsed = new URL(url); |
| 70 | return `${parsed.origin}${parsed.pathname}`; |
| 71 | } catch { |
| 72 | return url; |
| 73 | } |
| 74 | } |
| 75 | |
| 76 | export async function fetchWithRetry( |
| 77 | url: string, |
| 78 | options: FetchWithRetryOptions = {} |
| 79 | ): Promise<Response> { |
| 80 | const { |
| 81 | maxAttempts = 6, |
| 82 | baseDelayMs = 1000, |
| 83 | maxDelayMs = 60_000, |
| 84 | retryStatuses = DEFAULT_RETRY_STATUSES, |
| 85 | ...init |
| 86 | } = options; |
| 87 | |
| 88 | const endpoint = loggableEndpoint(url); |
| 89 | let lastError: unknown; |
| 90 | |
| 91 | for (let attempt = 1; attempt <= maxAttempts; attempt++) { |
| 92 | let response: Response; |
| 93 | |
| 94 | try { |
| 95 | response = await fetch(url, init); |
| 96 | } catch (error) { |
| 97 | lastError = error; |
| 98 | if (attempt >= maxAttempts) break; |
| 99 | const delay = backoffDelay(attempt, baseDelayMs, maxDelayMs); |
| 100 | console.warn( |
| 101 | ` [fetch] ${endpoint} network error (attempt ${attempt}/${maxAttempts}), ` + |
| 102 | `retrying in ${Math.round(delay)}ms: ${(error as Error).message}` |
| 103 | ); |
| 104 | await sleep(delay); |
| 105 | continue; |
| 106 | } |
| 107 | |
| 108 | // Success, or a non-retryable status: hand it back to the caller. |
| 109 | if (!retryStatuses.includes(response.status)) { |
| 110 | return response; |
| 111 | } |
| 112 | |
| 113 | // Out of attempts: surface the failing response so callers still fail. |
| 114 | if (attempt >= maxAttempts) { |
| 115 | return response; |
| 116 | } |
| 117 | |
| 118 | const retryAfterMs = parseRetryAfter(response.headers.get('retry-after')); |
| 119 | // Add jitter on top of Retry-After to avoid a synchronized retry burst. |
| 120 | const delay = |
| 121 | retryAfterMs != null |
| 122 | ? retryAfterMs + Math.random() * 1000 |
| 123 | : backoffDelay(attempt, baseDelayMs, maxDelayMs); |
| 124 | |
| 125 | console.warn( |
| 126 | ` [fetch] ${endpoint} -> ${response.status} (attempt ${attempt}/${maxAttempts}), ` + |
| 127 | `retrying in ${Math.round(delay)}ms${retryAfterMs != null ? ' (Retry-After)' : ''}` |
| 128 | ); |
| 129 | |
| 130 | // Drain the body so the connection can be reused. |
| 131 | await response.arrayBuffer().catch(() => {}); |
| 132 | await sleep(delay); |
| 133 | } |
| 134 | |
| 135 | throw lastError instanceof Error |
| 136 | ? lastError |
| 137 | : new Error(`fetchWithRetry: exhausted ${maxAttempts} attempts for ${endpoint}`); |
| 138 | } |