@devries-cameron20 / composio Mirror · read-only
Fork 0

Composio powers 1000+ toolkits, tool search, context management, authentication, and a sandboxed workbench to help you build AI agents that turn intent into action.

Mirrored from https://github.com/Camo67/composio.git

Synced Nov 21, 03:37 UTC

next
ts 138 lines 4.68 KB
Raw
1 /**
2 * Shared fetch helper for docs generation scripts.
3 *
4 * Wraps the global `fetch` with rate-limit-aware retry/backoff:
5 * - Retries `429 Too Many Requests` and transient 5xx responses.
6 * - Honors the `Retry-After` header (delta-seconds or HTTP-date) when present.
7 * - Falls back to exponential backoff with full jitter.
8 * - Logs the endpoint and retry count so CI logs explain the delay.
9 * - Caps attempts so CI still fails fast when the backend is genuinely down —
10 * the final failing `Response` is returned to the caller, which keeps existing
11 * `if (!response.ok)` handling intact.
12 *
13 * Network-level errors thrown by `fetch` (DNS, connection reset, …) are retried
14 * the same way and re-thrown once attempts are exhausted.
15 *
16 * Why this exists: `generate-toolkits.ts` issues ~6500 requests per run (3 per
17 * toolkit across a ~2.1k catalog), which can exceed the backend request limit.
18 * Without backoff the run can fail with `429`. See docs/scripts/README.md.
19 *
20 * Deliberately hand-rolled rather than using Effect's `Schedule` (which would be
21 * the idiomatic fit elsewhere): the docs package has no Effect dependency and
22 * these are plain bun scripts. Effect is reserved for `ts/packages/cli`. Keep
23 * this helper dependency-free.
24 */
25
26 const DEFAULT_RETRY_STATUSES = [429, 500, 502, 503, 504];
27
28 export interface FetchWithRetryOptions extends RequestInit {
29 /** Maximum number of attempts, including the first. Default 6. */
30 maxAttempts?: number;
31 /** Base delay in ms for exponential backoff. Default 1000. */
32 baseDelayMs?: number;
33 /** Upper bound for a single backoff wait in ms. Default 60000. */
34 maxDelayMs?: number;
35 /** HTTP status codes that trigger a retry. Default [429, 500, 502, 503, 504]. */
36 retryStatuses?: number[];
37 }
38
39 function sleep(ms: number): Promise<void> {
40 return new Promise((resolve) => setTimeout(resolve, ms));
41 }
42
43 /** Parse a `Retry-After` header (delta-seconds or HTTP-date) into ms, or null. */
44 function parseRetryAfter(header: string | null): number | null {
45 if (!header) return null;
46
47 const seconds = Number(header);
48 if (Number.isFinite(seconds)) {
49 return Math.max(0, seconds * 1000);
50 }
51
52 const dateMs = Date.parse(header);
53 if (!Number.isNaN(dateMs)) {
54 return Math.max(0, dateMs - Date.now());
55 }
56
57 return null;
58 }
59
60 /** Exponential backoff with full jitter. `attempt` is 1-based. */
61 function backoffDelay(attempt: number, baseDelayMs: number, maxDelayMs: number): number {
62 const ceiling = Math.min(maxDelayMs, baseDelayMs * 2 ** (attempt - 1));
63 return Math.random() * ceiling;
64 }
65
66 /** Collapse a URL to `origin + pathname` for tidy log lines (drops query noise). */
67 function loggableEndpoint(url: string): string {
68 try {
69 const parsed = new URL(url);
70 return `${parsed.origin}${parsed.pathname}`;
71 } catch {
72 return url;
73 }
74 }
75
76 export async function fetchWithRetry(
77 url: string,
78 options: FetchWithRetryOptions = {}
79 ): Promise<Response> {
80 const {
81 maxAttempts = 6,
82 baseDelayMs = 1000,
83 maxDelayMs = 60_000,
84 retryStatuses = DEFAULT_RETRY_STATUSES,
85 ...init
86 } = options;
87
88 const endpoint = loggableEndpoint(url);
89 let lastError: unknown;
90
91 for (let attempt = 1; attempt <= maxAttempts; attempt++) {
92 let response: Response;
93
94 try {
95 response = await fetch(url, init);
96 } catch (error) {
97 lastError = error;
98 if (attempt >= maxAttempts) break;
99 const delay = backoffDelay(attempt, baseDelayMs, maxDelayMs);
100 console.warn(
101 ` [fetch] ${endpoint} network error (attempt ${attempt}/${maxAttempts}), ` +
102 `retrying in ${Math.round(delay)}ms: ${(error as Error).message}`
103 );
104 await sleep(delay);
105 continue;
106 }
107
108 // Success, or a non-retryable status: hand it back to the caller.
109 if (!retryStatuses.includes(response.status)) {
110 return response;
111 }
112
113 // Out of attempts: surface the failing response so callers still fail.
114 if (attempt >= maxAttempts) {
115 return response;
116 }
117
118 const retryAfterMs = parseRetryAfter(response.headers.get('retry-after'));
119 // Add jitter on top of Retry-After to avoid a synchronized retry burst.
120 const delay =
121 retryAfterMs != null
122 ? retryAfterMs + Math.random() * 1000
123 : backoffDelay(attempt, baseDelayMs, maxDelayMs);
124
125 console.warn(
126 ` [fetch] ${endpoint} -> ${response.status} (attempt ${attempt}/${maxAttempts}), ` +
127 `retrying in ${Math.round(delay)}ms${retryAfterMs != null ? ' (Retry-After)' : ''}`
128 );
129
130 // Drain the body so the connection can be reused.
131 await response.arrayBuffer().catch(() => {});
132 await sleep(delay);
133 }
134
135 throw lastError instanceof Error
136 ? lastError
137 : new Error(`fetchWithRetry: exhausted ${maxAttempts} attempts for ${endpoint}`);
138 }