The best way to control Claude Code, Codex and more from any computer or phone

main
ts 462 lines 16.3 KB
Raw
1 /**
2 * Runs every example's test suite, then drives each example through the CLI
3 * lifecycle (`scripts/test-example-cli.ts`), then checks formatting.
4 *
5 * Usage: node scripts/test-examples.ts [--profile <name>]
6 */
7 import * as NodeRuntime from "@effect/platform-node/NodeRuntime";
8 import * as NodeServices from "@effect/platform-node/NodeServices";
9 import { Command, Flag } from "effect/cli";
10 import * as Console from "effect/Console";
11 import * as Data from "effect/Data";
12 import * as Effect from "effect/Effect";
13 import * as Option from "effect/Option";
14 import * as Path from "effect/Path";
15 import { ChildProcess } from "effect/process";
16 import { ChildProcessSpawner } from "effect/process/ChildProcessSpawner";
17 import * as Runtime from "effect/Runtime";
18 import * as Schedule from "effect/Schedule";
19 import * as Semaphore from "effect/Semaphore";
20 import * as Stream from "effect/Stream";
21 import { preferLocalFlociImage } from "./floci-image.ts";
22
23 /** Ends the run with `code` after the reason has already been printed. */
24 class ExitWith extends Data.TaggedError("ExitWith")<{ readonly code: number }> {
25 readonly [Runtime.errorReported] = false;
26 get [Runtime.errorExitCode]() {
27 return this.code;
28 }
29 }
30
31 const examples = [
32 "./examples/cloudflare-dev",
33 "./examples/cloudflare-worker",
34 "./examples/cloudflare-worker-async",
35 "./examples/cloudflare-website-tanstack-start",
36 "./examples/cloudflare-tanstack-start-solid",
37 "./examples/cloudflare-neon-drizzle",
38 "./examples/cloudflare-vue",
39 "./examples/cloudflare-website-solidstart",
40 "./examples/cloudflare-foldkit-spa",
41 "./examples/cloudflare-foldkit-ssr",
42 "./examples/cloudflare-foldkit-ssg",
43 "./examples/cloudflare-foldkit-hybrid",
44 "./examples/cloudflare-octane",
45 "./examples/cloudflare-website-astro",
46 "./examples/cloudflare-website-nextjs",
47 "./examples/cloudflare-website-nuxt",
48 "./examples/cloudflare-website-react-router",
49 "./examples/cloudflare-website-sveltekit",
50 "./examples/cloudflare-website-service-bindings",
51 "./examples/cloudflare-website-vite",
52 "./examples/cloudflare-website-vinext",
53 "./examples/cloudflare-website-waku",
54 "./examples/cloudflare-website-vocs",
55 "./examples/cloudflare-agents",
56 "./examples/cloudflare-flue",
57 "./examples/aws-dev",
58 // "./examples/aws-ecs",
59 "./examples/aws-lambda",
60 // Cloudfront is too slow
61 // "./examples/aws-website-astro",
62 // "./examples/aws-website-foldkit",
63 // "./examples/aws-website-nextjs",
64 // "./examples/aws-website-nuxt",
65 // "./examples/aws-website-react-router",
66 // "./examples/aws-website-solidstart",
67 // "./examples/aws-website-sveltekit",
68 // "./examples/aws-website-tanstack-start",
69 // "./examples/aws-website-vite",
70 // "./examples/aws-website-vinext",
71 // "./examples/aws-website-waku",
72 "./examples/fly-app",
73 "./examples/fly-service",
74 "./examples/fly-website-vite",
75 "./examples/hetzner-website-vite",
76 // Railway examples are gated out of test:examples for now: repeated
77 // Railway platform outages (deployment queue backlogs, e.g. the
78 // 2026-08-28 "Deployments slow to start" incident) and general
79 // flakiness working with Railway's API make these suites too
80 // unreliable for CI. Run them from the example directory with
81 // `bun test` when Railway is healthy.
82 // "./examples/railway-project",
83 // "./examples/railway-service",
84 // "./examples/railway-website-vite",
85 // Card-app Website composites (same apps as the AWS/Cloudflare
86 // *-website-* examples). Commented so CI does not provision extra
87 // sites; run them from the example directory with `bun test`.
88 // "./examples/fly-website-astro",
89 // "./examples/fly-website-foldkit",
90 // "./examples/fly-website-nextjs",
91 // "./examples/fly-website-nuxt",
92 // "./examples/fly-website-react-router",
93 // "./examples/fly-website-solidstart",
94 // "./examples/fly-website-sveltekit",
95 // "./examples/fly-website-tanstack-start",
96 // "./examples/fly-website-vinext",
97 // "./examples/fly-website-waku",
98 // "./examples/fly-website-vocs",
99 // "./examples/hetzner-website-astro",
100 // "./examples/hetzner-website-foldkit",
101 // "./examples/hetzner-website-nextjs",
102 // "./examples/hetzner-website-nuxt",
103 // "./examples/hetzner-website-react-router",
104 // "./examples/hetzner-website-solidstart",
105 // "./examples/hetzner-website-sveltekit",
106 // "./examples/hetzner-website-tanstack-start",
107 // "./examples/hetzner-website-vinext",
108 // "./examples/hetzner-website-waku",
109 // "./examples/hetzner-website-vocs",
110 // "./examples/railway-website-astro",
111 // "./examples/railway-website-foldkit",
112 // "./examples/railway-website-nextjs",
113 // "./examples/railway-website-nuxt",
114 // "./examples/railway-website-react-router",
115 // "./examples/railway-website-solidstart",
116 // "./examples/railway-website-sveltekit",
117 // "./examples/railway-website-tanstack-start",
118 // "./examples/railway-website-vinext",
119 // "./examples/railway-website-waku",
120 // "./examples/railway-website-vocs",
121 "./examples/fly-sprite",
122 "./examples/fly-redis",
123 "./examples/fly-bucket",
124 "./examples/fly-postgres",
125 ] as const;
126
127 // The AWS examples share one floci emulator container (`alchemy-floci`):
128 // two `alchemy dev` sessions ensuring it at the same time race to recreate
129 // it on an image bump and hot-swap each other's Lambda code, so they run
130 // one at a time. Everything else stays concurrent.
131 const serialGroup = (example: string): string | undefined =>
132 example.startsWith("./examples/aws-") ? "floci" : undefined;
133
134 type CommandResult = {
135 label: string;
136 command: readonly string[];
137 exitCode: number | null;
138 stdout: string;
139 stderr: string;
140 };
141
142 type Task = {
143 label: string;
144 command: readonly string[];
145 /** Relative to the repository root; defaults to it. */
146 cwd?: string;
147 /** Tasks sharing a key run one at a time, in list order. */
148 serial?: string;
149 };
150
151 type TaskState = Task & {
152 status: "pending" | "running" | "ok" | "failed";
153 startedAt?: number;
154 endedAt?: number;
155 exitCode?: number | null;
156 };
157
158 const elapsedSeconds = (state: TaskState): string => {
159 if (state.startedAt === undefined) {
160 return "0s";
161 }
162 const endedAt = state.endedAt ?? performance.now();
163 return `${Math.round((endedAt - state.startedAt) / 1000)}s`;
164 };
165
166 const makeStatusRenderer = (title: string, states: readonly TaskState[]) => {
167 const interactive = process.stdout.isTTY === true;
168 let renderedRows = 0;
169
170 const taskLine = (state: TaskState) => {
171 const icon = state.status === "ok" ? "ok" : state.status === "failed" ? "failed" : state.status;
172 const exit =
173 state.exitCode === undefined || state.exitCode === 0
174 ? ""
175 : ` exit ${state.exitCode ?? "signal"}`;
176 return ` ${icon.padEnd(7)} ${state.label} ${elapsedSeconds(state)}${exit}`;
177 };
178
179 // Physical rows a line occupies once the terminal wraps it.
180 const rowsFor = (line: string) => {
181 const columns = process.stdout.columns || 80;
182 return Math.max(1, Math.ceil(line.length / columns));
183 };
184 const rowsForAll = (output: readonly string[]) =>
185 output.reduce((total, line) => total + rowsFor(line), 0);
186
187 const fullLines = () => [title, ...states.map(taskLine)];
188
189 // The in-place repaint moves the cursor up with `\x1b[NF`, which cannot
190 // climb above the top of the viewport: if a paint is taller than the
191 // terminal (or lines wrap), the write scrolls and every repaint leaks its
192 // topmost rows into scrollback. Keep each paint strictly shorter than the
193 // viewport by collapsing finished tasks into the header once space runs
194 // out (`+ 1` accounts for the cursor parking on the row below the paint).
195 const lines = () => {
196 const rows = process.stdout.rows || 24;
197 const full = fullLines();
198 if (rowsForAll(full) + 1 <= rows) {
199 return full;
200 }
201 const done = states.filter((state) => state.status === "ok").length;
202 const active = states.filter((state) => state.status !== "ok");
203 const header = `${title} (${done}/${states.length} ok)`;
204 const activeLines = active.map(taskLine);
205 let shown = activeLines.length;
206 const fits = (count: number) => {
207 const overflow = count < activeLines.length ? 1 : 0;
208 return rowsForAll([header, ...activeLines.slice(0, count)]) + overflow + 1 <= rows;
209 };
210 while (shown > 0 && !fits(shown)) {
211 shown--;
212 }
213 return shown === activeLines.length
214 ? [header, ...activeLines]
215 : [header, ...activeLines.slice(0, shown), ` … ${activeLines.length - shown} more`];
216 };
217
218 return {
219 failure(result: CommandResult) {
220 if (interactive && renderedRows > 0) {
221 process.stdout.write(`\x1b[${renderedRows}F\x1b[J`);
222 renderedRows = 0;
223 }
224 const output = [
225 `\nFailed: ${result.label} (exit ${result.exitCode ?? "signal"}): ${result.command.join(" ")}`,
226 `--- ${result.label} stdout ---`,
227 result.stdout.trimEnd() || "(empty)",
228 `--- ${result.label} stderr ---`,
229 result.stderr.trimEnd() || "(empty)",
230 "",
231 ].join("\n");
232 // Use the TUI's stream so its next repaint starts below the logs.
233 (interactive ? process.stdout : process.stderr).write(output);
234 this.render();
235 },
236 render() {
237 if (!interactive) {
238 return;
239 }
240 if (renderedRows > 0) {
241 process.stdout.write(`\x1b[${renderedRows}F\x1b[J`);
242 }
243 const output = lines();
244 process.stdout.write(`${output.join("\n")}\n`);
245 renderedRows = rowsForAll(output);
246 },
247 finish() {
248 if (interactive && renderedRows > 0) {
249 process.stdout.write(`\x1b[${renderedRows}F\x1b[J`);
250 renderedRows = 0;
251 }
252 // Always end with the full table (the live view may be compacted).
253 for (const line of fullLines()) {
254 console.log(line);
255 }
256 },
257 };
258 };
259
260 const runTask = Effect.fn(function* (
261 state: TaskState,
262 env: Record<string, string>,
263 render: Effect.Effect<void>,
264 ) {
265 const spawner = yield* ChildProcessSpawner;
266 const path = yield* Path.Path;
267 const repoRoot = path.resolve(import.meta.dirname, "..");
268
269 yield* Effect.sync(() => {
270 state.status = "running";
271 state.startedAt = performance.now();
272 });
273 yield* render;
274
275 const { exitCode, stdout, stderr } = yield* Effect.scoped(
276 Effect.gen(function* () {
277 const child = yield* spawner.spawn(
278 ChildProcess.make(state.command[0]!, state.command.slice(1), {
279 cwd: path.resolve(repoRoot, state.cwd ?? "."),
280 env,
281 extendEnv: true,
282 // Not detached: terminal Ctrl-C reaches every task directly.
283 detached: false,
284 stdin: "inherit",
285 stdout: "pipe",
286 stderr: "pipe",
287 }),
288 );
289 return yield* Effect.all(
290 {
291 exitCode: child.exitCode.pipe(
292 Effect.map((code): number | null => code),
293 // Terminated by a signal.
294 Effect.orElseSucceed(() => null),
295 ),
296 stdout: child.stdout.pipe(Stream.decodeText, Stream.mkString),
297 stderr: child.stderr.pipe(Stream.decodeText, Stream.mkString),
298 },
299 { concurrency: "unbounded" },
300 );
301 }),
302 ).pipe(
303 // A command that cannot start (e.g. missing binary) fails the task.
304 Effect.catch((error) =>
305 Effect.succeed({ exitCode: null, stdout: "", stderr: String(error.message ?? error) }),
306 ),
307 );
308
309 yield* Effect.sync(() => {
310 state.status = exitCode === 0 ? "ok" : "failed";
311 state.exitCode = exitCode;
312 state.endedAt = performance.now();
313 });
314 yield* render;
315
316 return {
317 label: state.label,
318 command: state.command,
319 exitCode,
320 stdout,
321 stderr,
322 } satisfies CommandResult;
323 });
324
325 const runParallel = Effect.fn(function* (
326 title: string,
327 tasks: readonly Task[],
328 env: Record<string, string>,
329 options?: { readonly concurrency?: number },
330 ) {
331 const states = tasks.map((task): TaskState => ({
332 ...task,
333 status: "pending",
334 }));
335 const renderer = yield* Effect.sync(() => makeStatusRenderer(title, states));
336 const render = Effect.sync(() => renderer.render());
337
338 // At most `concurrency` tasks run at once; the rest wait for a slot.
339 // Serial tasks queue on their group before taking a slot, in list order.
340 const slots = yield* Semaphore.make(options?.concurrency ?? Number.MAX_SAFE_INTEGER);
341 const serial = new Map<string, Semaphore.Semaphore>();
342 for (const task of tasks) {
343 if (task.serial !== undefined && !serial.has(task.serial)) {
344 serial.set(task.serial, yield* Semaphore.make(1));
345 }
346 }
347
348 const run = Effect.forEach(
349 states,
350 (state) => {
351 const start = runTask(state, env, render).pipe(
352 Effect.tap((result) =>
353 result.exitCode === 0 ? Effect.void : Effect.sync(() => renderer.failure(result)),
354 ),
355 Semaphore.withPermits(slots, 1),
356 );
357 return state.serial === undefined
358 ? start
359 : start.pipe(Semaphore.withPermits(serial.get(state.serial)!, 1));
360 },
361 { concurrency: "unbounded" },
362 );
363
364 yield* render;
365 return yield* run.pipe(
366 Effect.raceFirst(
367 render.pipe(Effect.repeat(Schedule.spaced("1 second")), Effect.andThen(Effect.never)),
368 ),
369 Effect.ensuring(Effect.sync(() => renderer.finish())),
370 );
371 });
372
373 const command = Command.make(
374 "test-examples",
375 {
376 profile: Flag.String("profile").pipe(
377 Flag.withDescription(
378 "Alchemy profile the example suites and CLI runs use (sets ALCHEMY_PROFILE)",
379 ),
380 Flag.optional,
381 ),
382 },
383 Effect.fn(function* ({ profile }) {
384 // The example suites read the profile from `ALCHEMY_PROFILE`
385 // (`Test.make({ profile: process.env.ALCHEMY_PROFILE })`), so translate
386 // the flag into the env every spawned `bun test` inherits.
387 const env: Record<string, string> = Option.isSome(profile)
388 ? { ALCHEMY_PROFILE: profile.value }
389 : {};
390
391 // The AWS examples run against the floci emulator: use the locally built
392 // image when there is one so an emulator fix is testable before its
393 // release image lands on GHCR.
394 yield* preferLocalFlociImage("test:examples");
395
396 const testResults = yield* runParallel(
397 "Example tests",
398 examples.map((example) => ({
399 label: example,
400 // Run `bun test` IN the example directory. Concurrent
401 // `bun run --filter <workspace> test` all contend on bun's workspace
402 // graph (each sits in `openat` walking the monorepo) and none of them
403 // ever spawn the actual test process.
404 command: ["bun", "test"] as const,
405 cwd: example,
406 serial: serialGroup(example),
407 })),
408 env,
409 );
410 const failedTests = testResults.filter((result) => result.exitCode !== 0);
411
412 if (failedTests.length > 0) {
413 yield* Console.error("\nFailed example tests:");
414 for (const failure of failedTests) {
415 const exit = failure.exitCode === null ? "signal" : failure.exitCode;
416 yield* Console.error(`- ${failure.label} (exit ${exit}): ${failure.command.join(" ")}`);
417 }
418 return yield* new ExitWith({ code: 1 });
419 }
420
421 const cliResults = yield* runParallel(
422 "Example CLI lifecycle",
423 examples.map((example) => ({
424 label: `${example} CLI lifecycle`,
425 command: ["node", "scripts/test-example-cli.ts", example],
426 serial: serialGroup(example),
427 })),
428 env,
429 // Each `alchemy dev` session is a CLI, an exec child, sidecars, workerd
430 // and a bundler watch loop; 30 at once starve each other past the
431 // 4-minute readiness timeout.
432 { concurrency: 8 },
433 );
434 const failedCliTests = cliResults.filter((result) => result.exitCode !== 0);
435
436 if (failedCliTests.length > 0) {
437 yield* Console.error("\nFailed example CLI lifecycle tests:");
438 for (const failure of failedCliTests) {
439 const exit = failure.exitCode === null ? "signal" : failure.exitCode;
440 yield* Console.error(`- ${failure.label} (exit ${exit})`);
441 }
442 return yield* new ExitWith({ code: 1 });
443 }
444
445 const [formatFailure] = yield* runParallel(
446 "Format",
447 [{ label: "format", command: ["bun", "run", "format"] }],
448 env,
449 );
450 if (formatFailure!.exitCode !== 0) {
451 yield* Console.error("\nFormat failed:");
452 if (formatFailure!.stdout.length > 0) yield* Console.error(formatFailure!.stdout.trimEnd());
453 if (formatFailure!.stderr.length > 0) yield* Console.error(formatFailure!.stderr.trimEnd());
454 return yield* new ExitWith({ code: formatFailure!.exitCode ?? 1 });
455 }
456 }),
457 ).pipe(Command.withDescription("Run the example suites, CLI lifecycles and format check"));
458
459 Command.run(command, { version: "0.0.0" }).pipe(
460 Effect.provide(NodeServices.layer),
461 NodeRuntime.runMain,
462 );