1
+/**
2
+ * Shared fetch helper for docs generation scripts.
3
+ *
4
+ * Wraps the global `fetch` with rate-limit-aware retry/backoff:
5
+ * - Retries `429 Too Many Requests` and transient 5xx responses.
6
+ * - Honors the `Retry-After` header (delta-seconds or HTTP-date) when present.
7
+ * - Falls back to exponential backoff with full jitter.
8
+ * - Logs the endpoint and retry count so CI logs explain the delay.
9
+ * - Caps attempts so CI still fails fast when the backend is genuinely down —
10
+ * the final failing `Response` is returned to the caller, which keeps existing
11
+ * `if (!response.ok)` handling intact.
12
+ *
13
+ * Network-level errors thrown by `fetch` (DNS, connection reset, …) are retried
14
+ * the same way and re-thrown once attempts are exhausted.
15
+ *
16
+ * Why this exists: `generate-toolkits.ts` issues ~3000 requests per run, which
17
+ * exceeds the staging limit of 2000 requests/minute. Without backoff the run
18
+ * fails with `429`, and `generate-meta-tools.ts` (which runs next) inherits the
19
+ * exhausted window. See docs/scripts/README.md.
20
+ *
21
+ * Deliberately hand-rolled rather than using Effect's `Schedule` (which would be
22
+ * the idiomatic fit elsewhere): the docs package has no Effect dependency and
23
+ * these are plain bun scripts. Effect is reserved for `ts/packages/cli`. Keep
24
+ * this helper dependency-free.
25
+ */
26
+
27
+const DEFAULT_RETRY_STATUSES = [429, 500, 502, 503, 504];
28
+
29
+export interface FetchWithRetryOptions extends RequestInit {
30
+ /** Maximum number of attempts, including the first. Default 6. */
31
+ maxAttempts?: number;
32
+ /** Base delay in ms for exponential backoff. Default 1000. */
33
+ baseDelayMs?: number;
34
+ /** Upper bound for a single backoff wait in ms. Default 60000. */
35
+ maxDelayMs?: number;
36
+ /** HTTP status codes that trigger a retry. Default [429, 500, 502, 503, 504]. */
37
+ retryStatuses?: number[];
38
+}
39
+
40
+function sleep(ms: number): Promise<void> {
41
+ return new Promise((resolve) => setTimeout(resolve, ms));
42
+}
43
+
44
+/** Parse a `Retry-After` header (delta-seconds or HTTP-date) into ms, or null. */
45
+function parseRetryAfter(header: string | null): number | null {
46
+ if (!header) return null;
47
+
48
+ const seconds = Number(header);
49
+ if (Number.isFinite(seconds)) {
50
+ return Math.max(0, seconds * 1000);
51
+ }
52
+
53
+ const dateMs = Date.parse(header);
54
+ if (!Number.isNaN(dateMs)) {
55
+ return Math.max(0, dateMs - Date.now());
56
+ }
57
+
58
+ return null;
59
+}
60
+
61
+/** Exponential backoff with full jitter. `attempt` is 1-based. */
62
+function backoffDelay(attempt: number, baseDelayMs: number, maxDelayMs: number): number {
63
+ const ceiling = Math.min(maxDelayMs, baseDelayMs * 2 ** (attempt - 1));
64
+ return Math.random() * ceiling;
65
+}
66
+
67
+/** Collapse a URL to `origin + pathname` for tidy log lines (drops query noise). */
68
+function loggableEndpoint(url: string): string {
69
+ try {
70
+ const parsed = new URL(url);
71
+ return `${parsed.origin}${parsed.pathname}`;
72
+ } catch {
73
+ return url;
74
+ }
75
+}
76
+
77
+export async function fetchWithRetry(
78
+ url: string,
79
+ options: FetchWithRetryOptions = {}
80
+): Promise<Response> {
81
+ const {
82
+ maxAttempts = 6,
83
+ baseDelayMs = 1000,
84
+ maxDelayMs = 60_000,
85
+ retryStatuses = DEFAULT_RETRY_STATUSES,
86
+ ...init
87
+ } = options;
88
+
89
+ const endpoint = loggableEndpoint(url);
90
+ let lastError: unknown;
91
+
92
+ for (let attempt = 1; attempt <= maxAttempts; attempt++) {
93
+ let response: Response;
94
+
95
+ try {
96
+ response = await fetch(url, init);
97
+ } catch (error) {
98
+ lastError = error;
99
+ if (attempt >= maxAttempts) break;
100
+ const delay = backoffDelay(attempt, baseDelayMs, maxDelayMs);
101
+ console.warn(
102
+ ` [fetch] ${endpoint} network error (attempt ${attempt}/${maxAttempts}), ` +
103
+ `retrying in ${Math.round(delay)}ms: ${(error as Error).message}`
104
+ );
105
+ await sleep(delay);
106
+ continue;
107
+ }
108
+
109
+ // Success, or a non-retryable status: hand it back to the caller.
110
+ if (!retryStatuses.includes(response.status)) {
111
+ return response;
112
+ }
113
+
114
+ // Out of attempts: surface the failing response so callers still fail.
115
+ if (attempt >= maxAttempts) {
116
+ return response;
117
+ }
118
+
119
+ const retryAfterMs = parseRetryAfter(response.headers.get('retry-after'));
120
+ // Add jitter on top of Retry-After to avoid a synchronized retry burst.
121
+ const delay =
122
+ retryAfterMs != null
123
+ ? retryAfterMs + Math.random() * 1000
124
+ : backoffDelay(attempt, baseDelayMs, maxDelayMs);
125
+
126
+ console.warn(
127
+ ` [fetch] ${endpoint} -> ${response.status} (attempt ${attempt}/${maxAttempts}), ` +
128
+ `retrying in ${Math.round(delay)}ms${retryAfterMs != null ? ' (Retry-After)' : ''}`
129
+ );
130
+
131
+ // Drain the body so the connection can be reused.
132
+ await response.arrayBuffer().catch(() => {});
133
+ await sleep(delay);
134
+ }
135
+
136
+ throw lastError instanceof Error
137
+ ? lastError
138
+ : new Error(`fetchWithRetry: exhausted ${maxAttempts} attempts for ${endpoint}`);
139
+}