@devries-cameron20 / t3code Mirror · read-only
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

Mirrored from https://github.com/Camo67/t3code.git

Synced Nov 21, 06:00 UTC

main
ts 205 lines 7.45 KB
Raw
1 /**
2 * Standalone cold-start baseline for the plain wrangler + `@cloudflare/containers`
3 * worker — no Alchemy involved. Mirrors the methodology of the Alchemy driver
4 * (`benchmark/container/test/bench.test.ts`) so the two are directly
5 * comparable, but uses zero Alchemy code — just `fetch` against the deployed
6 * worker.
7 *
8 * Usage (from this directory):
9 * bun x wrangler deploy # build + push the image, deploy the worker
10 * WORKER_URL=https://<worker>.workers.dev bun bench.ts
11 * BENCH_N=2 BENCH_CONCURRENCY=2 WORKER_URL=... bun bench.ts
12 * BENCH_CRASH=1 WORKER_URL=... bun bench.ts # crash-loop fail-fast baseline
13 * bun x wrangler delete # tear down
14 *
15 * `wrangler` and `@cloudflare/containers` resolve from the `alchemy` package's
16 * devDependencies (catalog), so no separate install is needed.
17 */
18
19 const WORKER_URL = (process.env.WORKER_URL ?? process.argv[2] ?? "").replace(/\/$/, "");
20 const N = Number(process.env.BENCH_N ?? 2);
21 const CONCURRENCY = Number(process.env.BENCH_CONCURRENCY ?? N);
22 const REQUEST_TIMEOUT_MS = Number(process.env.BENCH_REQUEST_TIMEOUT_MS ?? 240_000);
23 // When set, benchmark how fast native surfaces a fatal crash (the baseline
24 // Alchemy's fail-fast behaviour is compared against) instead of cold start.
25 const CRASH = process.env.BENCH_CRASH === "1";
26 const CRASH_N = Number(process.env.BENCH_CRASH_N ?? 10);
27
28 if (!WORKER_URL) {
29 console.error(
30 "Set WORKER_URL to the deployed worker, e.g. WORKER_URL=https://x.workers.dev bun bench.ts",
31 );
32 process.exit(1);
33 }
34
35 interface Sample {
36 outside: number;
37 inside: number | undefined;
38 }
39
40 const wait = async (url: string) => {
41 for (let i = 0; i < 30; i++) {
42 try {
43 const res = await fetch(url, { headers: { connection: "close" } });
44 const body = await res.text();
45 if (res.status === 200 && body.includes("ok")) return;
46 } catch {
47 // retry
48 }
49 await new Promise((r) => setTimeout(r, Math.min(3000, 500 * 2 ** i)));
50 }
51 throw new Error("worker never became ready");
52 };
53
54 const boot = async (name: string): Promise<{ sample?: Sample; failure?: string }> => {
55 const controller = new AbortController();
56 const timer = setTimeout(() => controller.abort(), REQUEST_TIMEOUT_MS);
57 const start = Date.now();
58 try {
59 const res = await fetch(`${WORKER_URL}/start?name=${encodeURIComponent(name)}`, {
60 headers: { connection: "close" },
61 signal: controller.signal,
62 });
63 const body = await res.text();
64 const outside = Date.now() - start;
65 if (res.status !== 200) {
66 return { failure: `${name}: HTTP ${res.status} ${body.slice(0, 200)}` };
67 }
68 let inside: number | undefined;
69 try {
70 inside = (JSON.parse(body) as { ms?: number }).ms;
71 } catch {
72 inside = undefined;
73 }
74 return { sample: { outside, inside } };
75 } catch (err) {
76 return { failure: `${name}: ${String(err)}` };
77 } finally {
78 clearTimeout(timer);
79 }
80 };
81
82 // Simple bounded-concurrency pool.
83 const runPool = async (
84 names: string[],
85 concurrency: number,
86 ): Promise<Array<{ sample?: Sample; failure?: string }>> => {
87 const results: Array<{ sample?: Sample; failure?: string }> = [];
88 let next = 0;
89 const workers = Array.from({ length: Math.min(concurrency, names.length) }, async () => {
90 while (next < names.length) {
91 const i = next++;
92 results[i] = await boot(names[i]);
93 }
94 });
95 await Promise.all(workers);
96 return results;
97 };
98
99 const stats = (xs: number[]) => {
100 if (xs.length === 0) return { min: 0, max: 0, mean: 0, p50: 0, p90: 0, p95: 0, p99: 0 };
101 const s = [...xs].sort((a, b) => a - b);
102 const pct = (p: number) => s[Math.min(s.length - 1, Math.floor((p / 100) * s.length))];
103 return {
104 min: s[0],
105 max: s[s.length - 1],
106 mean: Math.round(s.reduce((a, b) => a + b, 0) / s.length),
107 p50: pct(50),
108 p90: pct(90),
109 p95: pct(95),
110 p99: pct(99),
111 };
112 };
113
114 // One crash-loop attempt: time how long until the failure surfaces. `ok` is
115 // true only on a 2xx (never, for a crash-looping container).
116 const crashAttempt = async (
117 name: string,
118 ): Promise<{ outside: number; inside: number | undefined; ok: boolean }> => {
119 const controller = new AbortController();
120 const timer = setTimeout(() => controller.abort(), REQUEST_TIMEOUT_MS);
121 const start = Date.now();
122 try {
123 const res = await fetch(`${WORKER_URL}/crashloop?name=${encodeURIComponent(name)}`, {
124 headers: { connection: "close" },
125 signal: controller.signal,
126 });
127 const body = await res.text();
128 const outside = Date.now() - start;
129 const parsed = JSON.parse(body) as { ms?: number; ok?: boolean };
130 return { outside, inside: parsed.ms, ok: parsed.ok ?? false };
131 } catch {
132 return { outside: Date.now() - start, inside: undefined, ok: false };
133 } finally {
134 clearTimeout(timer);
135 }
136 };
137
138 const runCrash = async () => {
139 const nonce = crypto.randomUUID().slice(0, 8);
140 const names = Array.from({ length: CRASH_N }, (_, i) => `crash-${nonce}-${i}`);
141 const results = await Promise.all(names.map(crashAttempt));
142 const outside = stats(results.map((r) => r.outside));
143 const inside = stats(
144 results.map((r) => r.inside).filter((m): m is number => typeof m === "number"),
145 );
146 const sec = (n: number) => `${(n / 1000).toFixed(1)}s`;
147 const failed = results.filter((r) => !r.ok).length;
148 console.log(
149 [
150 "",
151 `wrangler + @cloudflare/containers crash-loop fail-fast (N=${CRASH_N})`,
152 `── fatal crash (container exits immediately) ──`,
153 ` detected-as-failed: ${failed}/${CRASH_N}`,
154 ` time-to-fail inside (worker):`,
155 ` min ${sec(inside.min)} p50 ${sec(inside.p50)} p95 ${sec(inside.p95)} max ${sec(inside.max)} mean ${sec(inside.mean)}`,
156 ` time-to-fail outside (client):`,
157 ` min ${sec(outside.min)} p50 ${sec(outside.p50)} p95 ${sec(outside.p95)} max ${sec(outside.max)} mean ${sec(outside.mean)}`,
158 "",
159 ].join("\n"),
160 );
161 };
162
163 const main = async () => {
164 await wait(WORKER_URL);
165
166 if (CRASH) {
167 await runCrash();
168 return;
169 }
170
171 const nonce = crypto.randomUUID().slice(0, 8);
172 const names = Array.from({ length: N }, (_, i) => `${nonce}-${i}`);
173 const outcomes = await runPool(names, CONCURRENCY);
174
175 const samples = outcomes.map((o) => o.sample).filter((s): s is Sample => s !== undefined);
176 const failures = outcomes.map((o) => o.failure).filter((f): f is string => f !== undefined);
177
178 const outside = stats(samples.map((s) => s.outside));
179 const inside = stats(
180 samples.map((s) => s.inside).filter((m): m is number => typeof m === "number"),
181 );
182 const sec = (n: number) => `${(n / 1000).toFixed(1)}s`;
183
184 console.log(
185 [
186 "",
187 `wrangler + @cloudflare/containers cold-start benchmark (N=${N}, concurrency=${CONCURRENCY})`,
188 `── non-effectful (oven/bun:latest, plain wrangler) ──`,
189 ` ok: ${samples.length}/${N} failed: ${failures.length}`,
190 ` outside (client round-trip):`,
191 ` min ${sec(outside.min)} p50 ${sec(outside.p50)} p90 ${sec(outside.p90)} p95 ${sec(outside.p95)} p99 ${sec(outside.p99)} max ${sec(outside.max)} mean ${sec(outside.mean)}`,
192 ` inside (worker start→reachable):`,
193 ` min ${sec(inside.min)} p50 ${sec(inside.p50)} p90 ${sec(inside.p90)} p95 ${sec(inside.p95)} p99 ${sec(inside.p99)} max ${sec(inside.max)} mean ${sec(inside.mean)}`,
194 ...(failures.length > 0
195 ? [" failures:", ...failures.slice(0, 5).map((f) => ` - ${f}`)]
196 : []),
197 "",
198 ].join("\n"),
199 );
200 };
201
202 main().catch((err) => {
203 console.error(err);
204 process.exit(1);
205 });