The best way to control Claude Code, Codex and more from any computer or phone

main
ts 504 lines 18.7 KB
Raw
1 import { expect } from "bun:test";
2 import * as AWS from "@alchemy.run/aws";
3 import * as Cloudflare from "@alchemy.run/cloudflare";
4 import * as Test from "@alchemy.run/test/Bun";
5 import * as Alchemy from "alchemy";
6 import * as Effect from "effect/Effect";
7 import * as FileSystem from "effect/FileSystem";
8 import * as HttpClient from "effect/http/HttpClient";
9 import * as HttpClientRequest from "effect/http/HttpClientRequest";
10 import * as Layer from "effect/Layer";
11 import * as Path from "effect/Path";
12 import * as Schedule from "effect/Schedule";
13 import Stack, { benchMicrovm } from "../alchemy.run.ts";
14
15 /**
16 * Cold-start benchmark driver. Deploys the {@link Stack} once, drives a
17 * batched-concurrent boot/shutdown load against every variant, records each
18 * sample DIRECTLY (no stdout parsing), then writes:
19 *
20 * - `data/samples-<run>.csv` — one row per boot (raw, for plotting)
21 * - `report/summary.csv` — per-variant aggregate stats
22 * - `report/report.md` — human-readable markdown report
23 *
24 * Methodology: targets run sequentially so they never contend for shared
25 * quotas. Each round uses distinct keys → distinct instances; a round boots its
26 * instances, waits for each to reach usable service, then shuts them all down
27 * before the next round, and the round-over-round trend shows image/edge warm-up.
28 *
29 * - Containers are measured UNDER CONCURRENT LOAD: BATCHES rounds × CONCURRENCY
30 * simultaneous boots (no comparable per-instance API throttle).
31 * - MicroVMs are measured INDEPENDENTLY: MICROVM_BOOTS serial boots at
32 * concurrency 1. The Lambda MicroVM API is throttled per account/Region
33 * (RunMicrovm 5 TPS/burst 5, TerminateMicrovm 10 TPS), so booting >1 at once
34 * would measure API admission, not VM cold start.
35 *
36 * MicroVM variants (Lambda + cross-cloud Worker hosts) only run when the stack
37 * was deployed with `BENCH_MICROVM=1`. Set `NO_DESTROY=1` to keep the deploy
38 * between runs while iterating.
39 */
40 const { test, beforeAll, afterAll, deploy, destroy } = Test.make({
41 providers: Layer.mergeAll(AWS.providers(), Cloudflare.providers()),
42 state: Alchemy.localState(),
43 });
44
45 const HOOK_TIMEOUT = 1_500_000;
46 const TEST_TIMEOUT = 3_000_000;
47
48 // Containers tolerate high concurrency (no comparable per-instance API limit),
49 // so they're measured under concurrent load: CONCURRENCY boots per round.
50 const CONCURRENCY = Number(process.env.BENCH_CONCURRENCY ?? 10);
51 const BATCHES = Number(process.env.BENCH_BATCHES ?? 10);
52
53 // MicroVM cold starts MUST be measured independently. The Lambda MicroVM API is
54 // throttled per account/Region — RunMicrovm is 5 TPS / burst 5, SuspendMicrovm
55 // 2 TPS, TerminateMicrovm 10 TPS
56 // (https://docs.aws.amazon.com/lambda/latest/dg/gettingstarted-limits.html#microvms-quotas).
57 // Booting N>1 at once means boots 2..N partly measure API admission/throttling
58 // instead of VM cold start. So MicroVM defaults to concurrency 1 (each boot is a
59 // fully isolated launch→ready→terminate), sampled MICROVM_BOOTS times serially.
60 const MICROVM_CONCURRENCY = Number(process.env.BENCH_MICROVM_CONCURRENCY ?? 1);
61 const MICROVM_BOOTS = Number(process.env.BENCH_MICROVM_BOOTS ?? 25);
62
63 // Optional comma-separated variant filter (e.g. `BENCH_VARIANTS=opencode`) to
64 // re-run a subset without re-measuring everything.
65 const VARIANT_FILTER = process.env.BENCH_VARIANTS
66 ? new Set(process.env.BENCH_VARIANTS.split(","))
67 : undefined;
68 const REQUEST_TIMEOUT = "240 seconds";
69
70 const DEPLOY_PLACEHOLDER = "Alchemy worker is being deployed...";
71
72 // Force `Connection: close` so each request opens a fresh connection rather
73 // than pinning to one edge/host over a pooled keep-alive socket.
74 const freshConn = HttpClient.mapRequest(HttpClientRequest.setHeader("connection", "close"));
75
76 interface Target {
77 /** Coarse environment, e.g. `container`, `lambda→microvm`, `worker→microvm`. */
78 readonly env: string;
79 readonly host: string;
80 readonly variant: string;
81 readonly label: string;
82 /** Query param naming the instance: containers use `name`, MicroVMs use `key`. */
83 readonly keyParam: "name" | "key";
84 /** Shutdown addresses by the boot key (`name`) or by the returned id (`id`). */
85 readonly shutdownBy: "name" | "id";
86 /** Instances booted simultaneously per round (1 = fully isolated boots). */
87 readonly concurrency: number;
88 /** Number of rounds; total samples = rounds × concurrency. */
89 readonly rounds: number;
90 }
91
92 interface Sample {
93 readonly env: string;
94 readonly variant: string;
95 readonly label: string;
96 readonly batch: number;
97 readonly key: string;
98 /** Inside the host: start → usable service. */
99 readonly readyMs: number | undefined;
100 /** Wall-clock latency of the boot request, measured by the client (outside). */
101 readonly outside: number;
102 readonly ok: boolean;
103 readonly error: string | undefined;
104 }
105
106 const waitForHost = (url: string) =>
107 Effect.gen(function* () {
108 const client = freshConn(yield* HttpClient.HttpClient);
109 yield* client.get(url).pipe(
110 Effect.flatMap((r) =>
111 r.status !== 200
112 ? Effect.fail(new Error(`host not ready: ${r.status}`))
113 : Effect.flatMap(r.text, (body) =>
114 body.includes(DEPLOY_PLACEHOLDER) || !body.includes("ok")
115 ? Effect.fail(new Error(`not ready: ${body.slice(0, 80)}`))
116 : Effect.succeed(body),
117 ),
118 ),
119 Effect.timeout("30 seconds"),
120 Effect.retry({
121 schedule: Schedule.min([Schedule.exponential("500 millis"), Schedule.spaced("3 seconds")]),
122 times: 30,
123 }),
124 );
125 });
126
127 // Boot ONE fresh instance and time it from the outside. Returns the recorded
128 // sample plus the instance id (when the host reports one, for id-based
129 // shutdown). Does NOT shut down — the batch tears its instances down together.
130 const bootOne = (t: Target, batch: number, key: string) =>
131 Effect.gen(function* () {
132 const client = freshConn(yield* HttpClient.HttpClient);
133 const q = `variant=${t.variant}&${t.keyParam}=${encodeURIComponent(key)}`;
134 const start = yield* Effect.sync(() => Date.now());
135 const result = yield* client.get(`${t.host}/boot?${q}`).pipe(
136 Effect.flatMap((r) => Effect.map(r.text, (body) => ({ status: r.status, body }))),
137 Effect.timeout(REQUEST_TIMEOUT),
138 Effect.map((res) => ({ ok: true as const, ...res })),
139 Effect.catch((err) => Effect.succeed({ ok: false as const, error: String(err) })),
140 );
141 const outside = (yield* Effect.sync(() => Date.now())) - start;
142
143 const base = {
144 env: t.env,
145 variant: t.variant,
146 label: t.label,
147 batch,
148 key,
149 outside,
150 };
151
152 if (!result.ok) {
153 return {
154 sample: { ...base, readyMs: undefined, ok: false, error: result.error },
155 id: undefined,
156 };
157 }
158 if (result.status !== 200) {
159 return {
160 sample: {
161 ...base,
162 readyMs: undefined,
163 ok: false,
164 error: `HTTP ${result.status} ${result.body.slice(0, 160)}`,
165 },
166 id: undefined,
167 };
168 }
169 const parsed = (() => {
170 try {
171 return JSON.parse(result.body) as { id?: string; readyMs?: number };
172 } catch {
173 return {} as { id?: string; readyMs?: number };
174 }
175 })();
176 return {
177 sample: { ...base, readyMs: parsed.readyMs, ok: true, error: undefined },
178 id: parsed.id,
179 };
180 });
181
182 // Stop one instance (best-effort, untimed) so the next batch is a fresh cold
183 // start and we never hold more than `concurrency` instances at once.
184 const shutdownOne = (t: Target, key: string, id: string | undefined) =>
185 Effect.gen(function* () {
186 const client = freshConn(yield* HttpClient.HttpClient);
187 const target = t.shutdownBy === "id" ? id : key;
188 if (target === undefined) return;
189 const q = `variant=${t.variant}&${t.shutdownBy}=${encodeURIComponent(target)}`;
190 yield* client.get(`${t.host}/shutdown?${q}`).pipe(Effect.timeout("60 seconds"), Effect.ignore);
191 });
192
193 const runTarget = (t: Target, nonce: string) =>
194 Effect.gen(function* () {
195 // Untimed warm-up: the FIRST pull of a freshly-pushed image onto a cold
196 // edge/host costs tens of seconds — a one-time DEPLOY artifact, not a
197 // per-cold-start cost. Boot+shutdown once before timing so the image is
198 // distributed and we measure container/VM cold start, not first-pull.
199 const warm = `${nonce}-${t.env}-${t.variant}-warm`;
200 const { id: warmId } = yield* bootOne(t, 0, warm);
201 yield* shutdownOne(t, warm, warmId);
202
203 const samples: Sample[] = [];
204 for (let b = 1; b <= t.rounds; b++) {
205 const keys = Array.from(
206 { length: t.concurrency },
207 (_, i) => `${nonce}-${t.env}-${t.variant}-b${b}-${i}`,
208 );
209 const outcomes = yield* Effect.forEach(keys, (key) => bootOne(t, b, key), {
210 concurrency: t.concurrency,
211 });
212 samples.push(...outcomes.map((o) => o.sample));
213 yield* Effect.forEach(
214 outcomes.map((o, i) => ({ key: keys[i], id: o.id })),
215 ({ key, id }) => shutdownOne(t, key, id),
216 { concurrency: t.concurrency },
217 );
218 }
219 return samples;
220 });
221
222 const stats = (xs: ReadonlyArray<number>) => {
223 if (xs.length === 0) {
224 return { n: 0, min: 0, max: 0, mean: 0, p50: 0, p95: 0 };
225 }
226 const sorted = [...xs].sort((a, b) => a - b);
227 const pct = (p: number) =>
228 sorted[Math.min(sorted.length - 1, Math.floor((p / 100) * sorted.length))];
229 return {
230 n: sorted.length,
231 min: sorted[0],
232 max: sorted[sorted.length - 1],
233 mean: Math.round(sorted.reduce((a, b) => a + b, 0) / sorted.length),
234 p50: pct(50),
235 p95: pct(95),
236 };
237 };
238
239 const sN = (n: number) => `${(n / 1000).toFixed(1)}s`;
240
241 const csvEscape = (v: string | number | boolean | undefined) => {
242 const s = v === undefined ? "" : String(v);
243 return /[",\n]/.test(s) ? `"${s.replaceAll('"', '""')}"` : s;
244 };
245
246 const buildSamplesCsv = (samples: ReadonlyArray<Sample>) => {
247 const header = [
248 "env",
249 "variant",
250 "label",
251 "batch",
252 "key",
253 "readyMs",
254 "outside",
255 "ok",
256 "error",
257 ].join(",");
258 const rows = samples.map((s) =>
259 [s.env, s.variant, s.label, s.batch, s.key, s.readyMs ?? "", s.outside, s.ok, s.error]
260 .map(csvEscape)
261 .join(","),
262 );
263 return [header, ...rows].join("\n") + "\n";
264 };
265
266 // One aggregate row per (env, variant): ready-to-service stats + per-batch
267 // mean trend, computed from the readyMs of successful samples.
268 const summarize = (samples: ReadonlyArray<Sample>) => {
269 const groups = new Map<string, Sample[]>();
270 for (const s of samples) {
271 const k = `${s.env}\u0000${s.variant}`;
272 (groups.get(k) ?? groups.set(k, []).get(k)!).push(s);
273 }
274 return [...groups.values()].map((g) => {
275 const ready = stats(g.map((s) => s.readyMs).filter((m): m is number => typeof m === "number"));
276 const outside = stats(g.filter((s) => s.ok).map((s) => s.outside));
277 // Variable round count per target (containers vs serial MicroVM), so derive
278 // the span from the samples themselves rather than a global batch count.
279 const maxBatch = g.reduce((m, s) => Math.max(m, s.batch), 0);
280 const byBatch: Array<number | undefined> = [];
281 for (let b = 1; b <= maxBatch; b++) {
282 const xs = g
283 .filter((s) => s.batch === b)
284 .map((s) => s.readyMs)
285 .filter((m): m is number => typeof m === "number");
286 byBatch.push(xs.length > 0 ? stats(xs).mean : undefined);
287 }
288 return {
289 env: g[0].env,
290 variant: g[0].variant,
291 label: g[0].label,
292 ok: g.filter((s) => s.ok).length,
293 total: g.length,
294 ready,
295 outside,
296 byBatch,
297 };
298 });
299 };
300
301 type Summary = ReturnType<typeof summarize>[number];
302
303 const buildSummaryCsv = (rows: ReadonlyArray<Summary>) => {
304 const header = [
305 "env",
306 "variant",
307 "ok",
308 "total",
309 "ready_min_ms",
310 "ready_p50_ms",
311 "ready_p95_ms",
312 "ready_mean_ms",
313 "ready_max_ms",
314 // Per-round mean trend (semicolon-separated; round count varies by target).
315 "round_means_ms",
316 ].join(",");
317 const lines = rows.map((r) =>
318 [
319 r.env,
320 r.variant,
321 r.ok,
322 r.total,
323 r.ready.min,
324 r.ready.p50,
325 r.ready.p95,
326 r.ready.mean,
327 r.ready.max,
328 r.byBatch.map((b) => b ?? "").join(";"),
329 ]
330 .map(csvEscape)
331 .join(","),
332 );
333 return [header, ...lines].join("\n") + "\n";
334 };
335
336 const buildReport = (rows: ReadonlyArray<Summary>, meta: { runId: string; total: number }) => {
337 const head =
338 "| environment | variant | ok | ready p50 | ready p95 | ready mean | ready max | first → last round (mean) |";
339 const sep = "| --- | --- | --- | --- | --- | --- | --- | --- |";
340 const body = rows.map((r) => {
341 const first = r.byBatch[0];
342 const last = [...r.byBatch].reverse().find((b) => b !== undefined);
343 const trend = first !== undefined && last !== undefined ? `${sN(first)} → ${sN(last)}` : "—";
344 return `| ${r.env} | ${r.variant} | ${r.ok}/${r.total} | ${sN(r.ready.p50)} | ${sN(r.ready.p95)} | ${sN(r.ready.mean)} | ${sN(r.ready.max)} | ${trend} |`;
345 });
346 return [
347 `# Cold-start benchmark`,
348 ``,
349 `Run \`${meta.runId}\` — ${meta.total} samples. Metric: **time to usable service** (host start → first successful request).`,
350 ``,
351 `Methodology: targets run sequentially so they never contend for shared quotas. Containers are measured under concurrent load (${BATCHES} rounds × ${CONCURRENCY} simultaneous boots). MicroVMs are measured **independently** — ${MICROVM_BOOTS} serial boots at concurrency ${MICROVM_CONCURRENCY} — because the Lambda MicroVM API is throttled per account/Region (RunMicrovm 5 TPS/burst 5, TerminateMicrovm 10 TPS), so concurrent boots would measure API admission rather than VM cold start. Each boot launches, waits until the service is usable, then terminates before the next; the image is pre-warmed once (untimed) so we measure cold start, not first-pull distribution.`,
352 ``,
353 head,
354 sep,
355 ...body,
356 ``,
357 `Raw per-boot samples: \`data/samples-${meta.runId}.csv\`. Aggregates: \`report/summary.csv\`.`,
358 ``,
359 ].join("\n");
360 };
361
362 const buildTargets = (outputs: {
363 containerWorkerUrl: string;
364 lambdaUrl?: string;
365 microvmWorkerUrl?: string;
366 }): Target[] => {
367 const clean = (u: string) => u.replace(/\/+$/, "");
368 const targets: Target[] = [
369 {
370 env: "container",
371 host: clean(outputs.containerWorkerUrl),
372 variant: "effectful",
373 label: "Cloudflare container (bundled Effect image)",
374 keyParam: "name",
375 shutdownBy: "name",
376 concurrency: CONCURRENCY,
377 rounds: BATCHES,
378 },
379 {
380 env: "container",
381 host: clean(outputs.containerWorkerUrl),
382 variant: "bun",
383 label: "Cloudflare container (oven/bun, no Effect bundle)",
384 keyParam: "name",
385 shutdownBy: "name",
386 concurrency: CONCURRENCY,
387 rounds: BATCHES,
388 },
389 {
390 env: "container",
391 host: clean(outputs.containerWorkerUrl),
392 variant: "remote",
393 label: "Cloudflare container (remote echo image)",
394 keyParam: "name",
395 shutdownBy: "name",
396 concurrency: CONCURRENCY,
397 rounds: BATCHES,
398 },
399 {
400 env: "container",
401 host: clean(outputs.containerWorkerUrl),
402 variant: "opencode",
403 label: "Cloudflare container (opencode server, eager entrypoint)",
404 keyParam: "name",
405 shutdownBy: "name",
406 concurrency: CONCURRENCY,
407 rounds: BATCHES,
408 },
409 ];
410 if (benchMicrovm && outputs.lambdaUrl && outputs.microvmWorkerUrl) {
411 const lambda = clean(outputs.lambdaUrl);
412 const worker = clean(outputs.microvmWorkerUrl);
413 // Per host, the node-vs-bun matrix: effectful (alchemy/Effect bundle) on
414 // each runtime, the raw baseline on each runtime, plus the Python external.
415 const microvmVariants: ReadonlyArray<{ variant: string; label: string }> = [
416 { variant: "effectful-bun", label: "effectful (Effect bundle, bun)" },
417 { variant: "effectful-node", label: "effectful (Effect bundle, node)" },
418 { variant: "bun", label: "bun baseline (raw Bun.serve)" },
419 { variant: "node", label: "node baseline (raw http)" },
420 { variant: "external", label: "external (Python Dockerfile)" },
421 // The MicroVM image build starts opencode and snapshots the running
422 // memory, so each boot resumes with the server already up.
423 { variant: "opencode", label: "opencode (snapshotted eager server)" },
424 ];
425 for (const host of [
426 { env: "lambda→microvm", url: lambda },
427 { env: "worker→microvm", url: worker },
428 ]) {
429 for (const v of microvmVariants) {
430 targets.push({
431 env: host.env,
432 host: host.url,
433 variant: v.variant,
434 label: `${host.env} (${v.label})`,
435 keyParam: "key",
436 shutdownBy: "id",
437 // Isolated, serial boots — see the MICROVM_CONCURRENCY rationale above.
438 concurrency: MICROVM_CONCURRENCY,
439 rounds: MICROVM_BOOTS,
440 });
441 }
442 }
443 }
444 return targets;
445 };
446
447 const stack = beforeAll(deploy(Stack), { timeout: HOOK_TIMEOUT });
448 afterAll.skipIf(!!process.env.NO_DESTROY)(destroy(Stack), {
449 timeout: HOOK_TIMEOUT,
450 });
451
452 test(
453 "cold-start benchmark → data/*.csv + report/report.md",
454 Effect.gen(function* () {
455 const outputs = (yield* stack) as {
456 containerWorkerUrl: string;
457 lambdaUrl?: string;
458 microvmWorkerUrl?: string;
459 };
460 const targets = buildTargets(outputs).filter(
461 (t) => VARIANT_FILTER === undefined || VARIANT_FILTER.has(t.variant),
462 );
463
464 // Wait for each distinct host to answer before timing.
465 const hosts = [...new Set(targets.map((t) => t.host))];
466 yield* Effect.forEach(hosts, waitForHost, { concurrency: hosts.length });
467
468 const nonce = yield* Effect.sync(() => crypto.randomUUID().slice(0, 8));
469
470 const samples: Sample[] = [];
471 for (const t of targets) {
472 samples.push(...(yield* runTarget(t, nonce)));
473 }
474
475 const rows = summarize(samples);
476
477 const fs = yield* FileSystem.FileSystem;
478 const path = yield* Path.Path;
479 const root = path.join(import.meta.dirname, "..");
480 const dataDir = path.join(root, "data");
481 const reportDir = path.join(root, "report");
482 yield* fs.makeDirectory(dataDir, { recursive: true });
483 yield* fs.makeDirectory(reportDir, { recursive: true });
484
485 const runId = new Date().toISOString().replace(/[:.]/g, "-");
486 yield* fs.writeFileString(path.join(dataDir, `samples-${runId}.csv`), buildSamplesCsv(samples));
487 yield* fs.writeFileString(path.join(reportDir, "summary.csv"), buildSummaryCsv(rows));
488 yield* fs.writeFileString(
489 path.join(reportDir, "report.md"),
490 buildReport(rows, { runId, total: samples.length }),
491 );
492
493 yield* Effect.sync(() =>
494 console.log(
495 `\nWrote ${samples.length} samples → data/samples-${runId}.csv` +
496 `\nReport → report/report.md\n`,
497 ),
498 );
499
500 // Informational, but a run where nothing started indicates a broken deploy.
501 expect(samples.filter((s) => s.ok).length).toBeGreaterThan(0);
502 }),
503 { timeout: TEST_TIMEOUT },
504 );