The best way to control Claude Code, Codex and more from any computer or phone
Mirrored from https://github.com/Camo67/t3code.git
Synced Nov 21, 06:00 UTC
| 1 | /** |
| 2 | * Standalone cold-start baseline for the plain wrangler + `@cloudflare/containers` |
| 3 | * worker — no Alchemy involved. Mirrors the methodology of the Alchemy driver |
| 4 | * (`benchmark/container/test/bench.test.ts`) so the two are directly |
| 5 | * comparable, but uses zero Alchemy code — just `fetch` against the deployed |
| 6 | * worker. |
| 7 | * |
| 8 | * Usage (from this directory): |
| 9 | * bun x wrangler deploy # build + push the image, deploy the worker |
| 10 | * WORKER_URL=https://<worker>.workers.dev bun bench.ts |
| 11 | * BENCH_N=2 BENCH_CONCURRENCY=2 WORKER_URL=... bun bench.ts |
| 12 | * BENCH_CRASH=1 WORKER_URL=... bun bench.ts # crash-loop fail-fast baseline |
| 13 | * bun x wrangler delete # tear down |
| 14 | * |
| 15 | * `wrangler` and `@cloudflare/containers` resolve from the `alchemy` package's |
| 16 | * devDependencies (catalog), so no separate install is needed. |
| 17 | */ |
| 18 | |
| 19 | const WORKER_URL = (process.env.WORKER_URL ?? process.argv[2] ?? "").replace(/\/$/, ""); |
| 20 | const N = Number(process.env.BENCH_N ?? 2); |
| 21 | const CONCURRENCY = Number(process.env.BENCH_CONCURRENCY ?? N); |
| 22 | const REQUEST_TIMEOUT_MS = Number(process.env.BENCH_REQUEST_TIMEOUT_MS ?? 240_000); |
| 23 | // When set, benchmark how fast native surfaces a fatal crash (the baseline |
| 24 | // Alchemy's fail-fast behaviour is compared against) instead of cold start. |
| 25 | const CRASH = process.env.BENCH_CRASH === "1"; |
| 26 | const CRASH_N = Number(process.env.BENCH_CRASH_N ?? 10); |
| 27 | |
| 28 | if (!WORKER_URL) { |
| 29 | console.error( |
| 30 | "Set WORKER_URL to the deployed worker, e.g. WORKER_URL=https://x.workers.dev bun bench.ts", |
| 31 | ); |
| 32 | process.exit(1); |
| 33 | } |
| 34 | |
| 35 | interface Sample { |
| 36 | outside: number; |
| 37 | inside: number | undefined; |
| 38 | } |
| 39 | |
| 40 | const wait = async (url: string) => { |
| 41 | for (let i = 0; i < 30; i++) { |
| 42 | try { |
| 43 | const res = await fetch(url, { headers: { connection: "close" } }); |
| 44 | const body = await res.text(); |
| 45 | if (res.status === 200 && body.includes("ok")) return; |
| 46 | } catch { |
| 47 | // retry |
| 48 | } |
| 49 | await new Promise((r) => setTimeout(r, Math.min(3000, 500 * 2 ** i))); |
| 50 | } |
| 51 | throw new Error("worker never became ready"); |
| 52 | }; |
| 53 | |
| 54 | const boot = async (name: string): Promise<{ sample?: Sample; failure?: string }> => { |
| 55 | const controller = new AbortController(); |
| 56 | const timer = setTimeout(() => controller.abort(), REQUEST_TIMEOUT_MS); |
| 57 | const start = Date.now(); |
| 58 | try { |
| 59 | const res = await fetch(`${WORKER_URL}/start?name=${encodeURIComponent(name)}`, { |
| 60 | headers: { connection: "close" }, |
| 61 | signal: controller.signal, |
| 62 | }); |
| 63 | const body = await res.text(); |
| 64 | const outside = Date.now() - start; |
| 65 | if (res.status !== 200) { |
| 66 | return { failure: `${name}: HTTP ${res.status} ${body.slice(0, 200)}` }; |
| 67 | } |
| 68 | let inside: number | undefined; |
| 69 | try { |
| 70 | inside = (JSON.parse(body) as { ms?: number }).ms; |
| 71 | } catch { |
| 72 | inside = undefined; |
| 73 | } |
| 74 | return { sample: { outside, inside } }; |
| 75 | } catch (err) { |
| 76 | return { failure: `${name}: ${String(err)}` }; |
| 77 | } finally { |
| 78 | clearTimeout(timer); |
| 79 | } |
| 80 | }; |
| 81 | |
| 82 | // Simple bounded-concurrency pool. |
| 83 | const runPool = async ( |
| 84 | names: string[], |
| 85 | concurrency: number, |
| 86 | ): Promise<Array<{ sample?: Sample; failure?: string }>> => { |
| 87 | const results: Array<{ sample?: Sample; failure?: string }> = []; |
| 88 | let next = 0; |
| 89 | const workers = Array.from({ length: Math.min(concurrency, names.length) }, async () => { |
| 90 | while (next < names.length) { |
| 91 | const i = next++; |
| 92 | results[i] = await boot(names[i]); |
| 93 | } |
| 94 | }); |
| 95 | await Promise.all(workers); |
| 96 | return results; |
| 97 | }; |
| 98 | |
| 99 | const stats = (xs: number[]) => { |
| 100 | if (xs.length === 0) return { min: 0, max: 0, mean: 0, p50: 0, p90: 0, p95: 0, p99: 0 }; |
| 101 | const s = [...xs].sort((a, b) => a - b); |
| 102 | const pct = (p: number) => s[Math.min(s.length - 1, Math.floor((p / 100) * s.length))]; |
| 103 | return { |
| 104 | min: s[0], |
| 105 | max: s[s.length - 1], |
| 106 | mean: Math.round(s.reduce((a, b) => a + b, 0) / s.length), |
| 107 | p50: pct(50), |
| 108 | p90: pct(90), |
| 109 | p95: pct(95), |
| 110 | p99: pct(99), |
| 111 | }; |
| 112 | }; |
| 113 | |
| 114 | // One crash-loop attempt: time how long until the failure surfaces. `ok` is |
| 115 | // true only on a 2xx (never, for a crash-looping container). |
| 116 | const crashAttempt = async ( |
| 117 | name: string, |
| 118 | ): Promise<{ outside: number; inside: number | undefined; ok: boolean }> => { |
| 119 | const controller = new AbortController(); |
| 120 | const timer = setTimeout(() => controller.abort(), REQUEST_TIMEOUT_MS); |
| 121 | const start = Date.now(); |
| 122 | try { |
| 123 | const res = await fetch(`${WORKER_URL}/crashloop?name=${encodeURIComponent(name)}`, { |
| 124 | headers: { connection: "close" }, |
| 125 | signal: controller.signal, |
| 126 | }); |
| 127 | const body = await res.text(); |
| 128 | const outside = Date.now() - start; |
| 129 | const parsed = JSON.parse(body) as { ms?: number; ok?: boolean }; |
| 130 | return { outside, inside: parsed.ms, ok: parsed.ok ?? false }; |
| 131 | } catch { |
| 132 | return { outside: Date.now() - start, inside: undefined, ok: false }; |
| 133 | } finally { |
| 134 | clearTimeout(timer); |
| 135 | } |
| 136 | }; |
| 137 | |
| 138 | const runCrash = async () => { |
| 139 | const nonce = crypto.randomUUID().slice(0, 8); |
| 140 | const names = Array.from({ length: CRASH_N }, (_, i) => `crash-${nonce}-${i}`); |
| 141 | const results = await Promise.all(names.map(crashAttempt)); |
| 142 | const outside = stats(results.map((r) => r.outside)); |
| 143 | const inside = stats( |
| 144 | results.map((r) => r.inside).filter((m): m is number => typeof m === "number"), |
| 145 | ); |
| 146 | const sec = (n: number) => `${(n / 1000).toFixed(1)}s`; |
| 147 | const failed = results.filter((r) => !r.ok).length; |
| 148 | console.log( |
| 149 | [ |
| 150 | "", |
| 151 | `wrangler + @cloudflare/containers crash-loop fail-fast (N=${CRASH_N})`, |
| 152 | `── fatal crash (container exits immediately) ──`, |
| 153 | ` detected-as-failed: ${failed}/${CRASH_N}`, |
| 154 | ` time-to-fail inside (worker):`, |
| 155 | ` min ${sec(inside.min)} p50 ${sec(inside.p50)} p95 ${sec(inside.p95)} max ${sec(inside.max)} mean ${sec(inside.mean)}`, |
| 156 | ` time-to-fail outside (client):`, |
| 157 | ` min ${sec(outside.min)} p50 ${sec(outside.p50)} p95 ${sec(outside.p95)} max ${sec(outside.max)} mean ${sec(outside.mean)}`, |
| 158 | "", |
| 159 | ].join("\n"), |
| 160 | ); |
| 161 | }; |
| 162 | |
| 163 | const main = async () => { |
| 164 | await wait(WORKER_URL); |
| 165 | |
| 166 | if (CRASH) { |
| 167 | await runCrash(); |
| 168 | return; |
| 169 | } |
| 170 | |
| 171 | const nonce = crypto.randomUUID().slice(0, 8); |
| 172 | const names = Array.from({ length: N }, (_, i) => `${nonce}-${i}`); |
| 173 | const outcomes = await runPool(names, CONCURRENCY); |
| 174 | |
| 175 | const samples = outcomes.map((o) => o.sample).filter((s): s is Sample => s !== undefined); |
| 176 | const failures = outcomes.map((o) => o.failure).filter((f): f is string => f !== undefined); |
| 177 | |
| 178 | const outside = stats(samples.map((s) => s.outside)); |
| 179 | const inside = stats( |
| 180 | samples.map((s) => s.inside).filter((m): m is number => typeof m === "number"), |
| 181 | ); |
| 182 | const sec = (n: number) => `${(n / 1000).toFixed(1)}s`; |
| 183 | |
| 184 | console.log( |
| 185 | [ |
| 186 | "", |
| 187 | `wrangler + @cloudflare/containers cold-start benchmark (N=${N}, concurrency=${CONCURRENCY})`, |
| 188 | `── non-effectful (oven/bun:latest, plain wrangler) ──`, |
| 189 | ` ok: ${samples.length}/${N} failed: ${failures.length}`, |
| 190 | ` outside (client round-trip):`, |
| 191 | ` min ${sec(outside.min)} p50 ${sec(outside.p50)} p90 ${sec(outside.p90)} p95 ${sec(outside.p95)} p99 ${sec(outside.p99)} max ${sec(outside.max)} mean ${sec(outside.mean)}`, |
| 192 | ` inside (worker start→reachable):`, |
| 193 | ` min ${sec(inside.min)} p50 ${sec(inside.p50)} p90 ${sec(inside.p90)} p95 ${sec(inside.p95)} p99 ${sec(inside.p99)} max ${sec(inside.max)} mean ${sec(inside.mean)}`, |
| 194 | ...(failures.length > 0 |
| 195 | ? [" failures:", ...failures.slice(0, 5).map((f) => ` - ${f}`)] |
| 196 | : []), |
| 197 | "", |
| 198 | ].join("\n"), |
| 199 | ); |
| 200 | }; |
| 201 | |
| 202 | main().catch((err) => { |
| 203 | console.error(err); |
| 204 | process.exit(1); |
| 205 | }); |