@devries-cameron20 / t3code Mirror · read-only
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

Mirrored from https://github.com/Camo67/t3code.git

Synced Nov 21, 06:00 UTC

main
ts 494 lines 17.6 KB
Raw
1 /**
2 * Scaffolding for the `alchemy dev` stress suite.
3 *
4 * Everything here exists to make one thing possible: run the REAL CLI
5 * against a throwaway copy of this project and then rewrite that copy's
6 * files underneath it, while keeping every assertion independent of what
7 * the CLI happens to be printing at the time.
8 */
9 import { spawn, spawnSync, type ChildProcess } from "node:child_process";
10 import * as fs from "node:fs";
11 import * as path from "node:path";
12
13 /** The checked-in example project (the copy source). */
14 export const EXAMPLE_ROOT = path.resolve(import.meta.dirname, "..");
15
16 /**
17 * Spawn the CLI entry directly rather than through `bun run` / `cli.js`, so
18 * signals reach the actual CLI process whose scope teardown kills the exec
19 * child and the provider sidecars.
20 */
21 export const ALCHEMY_BIN = path.join(EXAMPLE_ROOT, "node_modules", "alchemy", "bin", "alchemy.js");
22
23 /** Entries never copied into a scratch project. */
24 const SKIP_COPY = new Set([
25 "node_modules",
26 "test",
27 ".stress",
28 ".alchemy",
29 "dist",
30 "tsconfig.json",
31 "bun.lock",
32 ]);
33
34 /**
35 * Copy the checked-in project into `.stress/<name>/`.
36 *
37 * The scratch dir lives INSIDE the example so Node/Bun resolution still
38 * walks up into `examples/dev-stress/node_modules` and finds the workspace
39 * `alchemy` link — while every mutation the suite makes stays out of the
40 * source-controlled tree, and survives a mid-suite failure without leaving
41 * the repo dirty.
42 */
43 /**
44 * Reset the floci emulator to a clean slate before a run. A KILLED previous
45 * run (bun's timeout, Ctrl-C on the test process) never reaches the CLI's
46 * graceful shutdown, so its emulated ECS services stay registered in the
47 * long-lived `alchemy-floci` container — their reconcilers re-spawn task
48 * containers forever and squat the suite's pinned host ports, deadlocking
49 * every subsequent run. The emulator is disposable by design (in-memory
50 * state, local image), so a fresh one per run makes the suite self-healing
51 * regardless of how the previous run died.
52 */
53 export const resetFlociEmulator = (): void => {
54 const names = spawnSync(
55 "docker",
56 ["ps", "-aq", "--filter", "name=^(alchemy-floci$|floci-ec2|floci-ecs|floci-microvm)"],
57 { encoding: "utf8", timeout: 30_000 },
58 )
59 .stdout?.trim()
60 .split("\n")
61 .filter(Boolean);
62 if (names && names.length > 0) {
63 spawnSync("docker", ["rm", "-f", ...names], {
64 stdio: "ignore",
65 timeout: 120_000,
66 });
67 }
68 };
69
70 /**
71 * Free the suite's pinned ports before a run. A KILLED previous run leaves
72 * orphaned workerd / dev-child processes listening on them (they are not
73 * children of the test process, so bun's cleanup never reaches them), and
74 * every `strictPort` worker then fails its first apply. Scoped strictly to
75 * the ports this suite owns.
76 */
77 export const freePinnedPorts = (ports: readonly number[]): void => {
78 const pids = new Set<string>();
79 for (const port of ports) {
80 const out = spawnSync("lsof", ["-ti", `tcp:${port}`, "-sTCP:LISTEN"], {
81 encoding: "utf8",
82 timeout: 15_000,
83 }).stdout;
84 for (const pid of (out ?? "").trim().split("\n")) {
85 if (pid) pids.add(pid);
86 }
87 }
88 if (pids.size === 0) return;
89 spawnSync("kill", [...pids], { stdio: "ignore", timeout: 15_000 });
90 // Grace, then force anything still holding a port.
91 spawnSync("sleep", ["2"], { stdio: "ignore", timeout: 5_000 });
92 spawnSync("kill", ["-9", ...pids], { stdio: "ignore", timeout: 15_000 });
93 };
94
95 export const makeScratchProject = (name: string): string => {
96 const dir = path.join(EXAMPLE_ROOT, ".stress", name);
97 fs.rmSync(dir, { recursive: true, force: true });
98 fs.mkdirSync(dir, { recursive: true });
99 for (const entry of fs.readdirSync(EXAMPLE_ROOT)) {
100 if (SKIP_COPY.has(entry) || entry.startsWith(".git")) continue;
101 fs.cpSync(path.join(EXAMPLE_ROOT, entry), path.join(dir, entry), {
102 recursive: true,
103 });
104 }
105 return dir;
106 };
107
108 export interface DevServerOptions {
109 /** Scratch project directory the CLI runs in. */
110 readonly cwd: string;
111 /** Stage name — isolates local state from any other run. */
112 readonly stage: string;
113 /** Extra environment for the CLI process. */
114 readonly env?: Record<string, string>;
115 }
116
117 /**
118 * Hermetic environment for the CLI: an alchemy profile that exists nowhere
119 * on disk plus stripped AWS credentials, so the run can only resolve to the
120 * local emulator, and placeholder Cloudflare credentials so the local
121 * Cloudflare providers' env lookups are satisfied without any cloud call
122 * being possible.
123 *
124 * `CI=1` is what makes the placeholders usable: without it a profile with
125 * no configured Cloudflare section refuses to fall back to environment
126 * credentials in a non-interactive process.
127 */
128 export const hermeticEnv = (extra?: Record<string, string>) => {
129 const env: NodeJS.ProcessEnv = {
130 ...process.env,
131 CI: "1",
132 ALCHEMY_PROFILE: "dev-stress-cli-test",
133 CLOUDFLARE_ACCOUNT_ID: "0123456789abcdef0123456789abcdef",
134 CLOUDFLARE_API_TOKEN: "dev-stress-placeholder",
135 ...extra,
136 };
137 delete env.AWS_ACCESS_KEY_ID;
138 delete env.AWS_SECRET_ACCESS_KEY;
139 delete env.AWS_SESSION_TOKEN;
140 delete env.AWS_PROFILE;
141 return env;
142 };
143
144 export class DevServer {
145 readonly proc: ChildProcess;
146 readonly cwd: string;
147 readonly stage: string;
148 /** Everything the CLI has written to stdout+stderr, in order. */
149 output = "";
150 private readonly logPath: string;
151 private readonly logStream: fs.WriteStream;
152
153 constructor(options: DevServerOptions) {
154 this.cwd = options.cwd;
155 this.stage = options.stage;
156 this.logPath = path.join(options.cwd, "dev-stress.log");
157 this.logStream = fs.createWriteStream(this.logPath, { flags: "a" });
158 this.proc = spawn("bun", [ALCHEMY_BIN, "dev", "--stage", options.stage], {
159 cwd: options.cwd,
160 // Own process group so teardown can deliver Ctrl-C to the whole
161 // tree (CLI + `--watch` exec child + provider sidecars) the way a
162 // terminal would.
163 detached: true,
164 stdio: ["ignore", "pipe", "pipe"],
165 env: hermeticEnv(options.env),
166 });
167 const pump = (stream: NodeJS.ReadableStream | null) =>
168 stream?.on("data", (chunk: Buffer) => {
169 const text = chunk.toString();
170 this.output += text;
171 this.logStream.write(text);
172 if (process.env.DEBUG) process.stderr.write(text);
173 });
174 pump(this.proc.stdout);
175 pump(this.proc.stderr);
176 }
177
178 /** `true` while the CLI process itself is still running. */
179 get alive(): boolean {
180 return this.proc.exitCode === null && this.proc.signalCode === null;
181 }
182
183 /** A cursor into {@link output}, for "did anything new happen since?". */
184 mark(): number {
185 return this.output.length;
186 }
187
188 /** Output written since `cursor`. */
189 since(cursor: number): string {
190 return this.output.slice(cursor);
191 }
192
193 /** Last `n` characters of output — for failure messages. */
194 tail(n = 6_000): string {
195 return this.output.slice(-n);
196 }
197
198 /**
199 * Number of plan renders. `alchemy dev` prints one `Plan:` line per run
200 * of the stack, so this counts EXEC-CHILD RESTARTS.
201 *
202 * It is the discriminator between the two dev reload paths. Both paths
203 * put new code in front of a request within a couple of seconds — the
204 * local Worker provider's bundler watch loop always wins the race — but
205 * only a file inside the STACK's import graph also makes `bun --watch`
206 * re-run the stack, which shows up here a beat later. Measured on this
207 * fixture: a bundler-only edit swaps at ~2s and never re-plans; a
208 * stack-graph edit swaps at ~2s and re-plans at ~4s.
209 */
210 get planCount(): number {
211 return this.output.match(/^Plan:/gm)?.length ?? 0;
212 }
213
214 /** Number of applies that ran to completion (`Done: N succeeded`). */
215 get doneCount(): number {
216 return this.output.match(/^Done:/gm)?.length ?? 0;
217 }
218
219 /**
220 * Resolve once the CLI has rendered more than `count` plans — i.e. the
221 * stack has re-run. The re-plan trails the bundler's hot swap, so this
222 * must be WAITED for, never sampled right after new code starts serving.
223 */
224 waitForPlanAfter(count: number, options?: { tries?: number }): Promise<true> {
225 return pollUntil(
226 `the stack to re-run (plan #${count + 1})`,
227 () => (this.planCount > count ? (true as const) : undefined),
228 { tries: options?.tries ?? 120, delayMs: 500, server: this },
229 );
230 }
231
232 /**
233 * Resolve once no new plan has appeared for `quietMs`. Phases that edit
234 * a file inside the STACK's import graph leave a re-plan trailing the
235 * hot swap by a few seconds; a later phase that baselines `planCount`
236 * (to assert "this path never re-plans") must absorb that trail first
237 * or the trailing plan lands inside its window and reads as a spurious
238 * re-run.
239 */
240 async settlePlans(quietMs = 8_000, capMs = 60_000): Promise<void> {
241 const deadline = Date.now() + capMs;
242 let last = this.planCount;
243 let quietSince = Date.now();
244 while (Date.now() < deadline) {
245 await Bun.sleep(500);
246 const current = this.planCount;
247 if (current !== last) {
248 last = current;
249 quietSince = Date.now();
250 } else if (Date.now() - quietSince >= quietMs) {
251 return;
252 }
253 }
254 }
255
256 /**
257 * Throws (with an output tail) unless the CLI process is still running.
258 * The single most important invariant of the whole suite: nothing the
259 * suite does to the project may kill the dev server.
260 */
261 assertAlive(context: string): void {
262 if (this.alive) return;
263 throw new Error(
264 `alchemy dev died during: ${context}\n` +
265 `exit=${this.proc.exitCode} signal=${this.proc.signalCode}\n` +
266 `--- output tail ---\n${this.tail()}`,
267 );
268 }
269
270 /**
271 * What the CLI has printed since `cursor`, with ANSI colour stripped —
272 * `bun test` output is not a TTY, but the exec child still colours its
273 * plan lines when stdout is a pipe it believes supports colour.
274 */
275 plain(cursor = 0): string {
276 // eslint-disable-next-line no-control-regex
277 return this.output.slice(cursor).replaceAll(/\x1b\[[0-9;]*m/g, "");
278 }
279
280 /** Extract a plain value for `key` from the stack outputs the CLI printed. */
281 outputValue(key: string): string | undefined {
282 const matches = this.output.match(new RegExp(`${key}:\\s*['\"]?([^\\s'\",]+)`, "g"));
283 // The newest print wins: outputs are re-printed on every re-apply.
284 return matches?.at(-1)?.match(new RegExp(`${key}:\\s*['\"]?([^\\s'\",]+)`))?.[1];
285 }
286
287 /** Extract a URL for `key` from the stack outputs the CLI printed. */
288 outputUrl(key: string): string | undefined {
289 const matches = this.output.match(new RegExp(`${key}:\\s*['"]?(http[^\\s'",]+)`, "g"));
290 // The newest print wins: outputs are re-printed on every re-apply.
291 return matches?.at(-1)?.match(/(http[^\s'",]+)/)?.[1];
292 }
293
294 // ── file mutation ────────────────────────────────────────────────────
295
296 abs(relative: string): string {
297 return path.join(this.cwd, relative);
298 }
299
300 read(relative: string): string {
301 return fs.readFileSync(this.abs(relative), "utf8");
302 }
303
304 write(relative: string, contents: string): void {
305 const file = this.abs(relative);
306 fs.mkdirSync(path.dirname(file), { recursive: true });
307 fs.writeFileSync(file, contents);
308 }
309
310 remove(relative: string): void {
311 fs.rmSync(this.abs(relative), { force: true, recursive: true });
312 }
313
314 move(from: string, to: string): void {
315 const target = this.abs(to);
316 fs.mkdirSync(path.dirname(target), { recursive: true });
317 fs.renameSync(this.abs(from), target);
318 }
319
320 exists(relative: string): boolean {
321 return fs.existsSync(this.abs(relative));
322 }
323
324 /**
325 * Replace the body of a `// <<NAME>> … // <</NAME>>` region. Surgical
326 * region edits keep the rest of the stack byte-identical, so a diff the
327 * suite makes is exactly the change under test — no regex drift.
328 */
329 patchRegion(relative: string, name: string, body: string): void {
330 const source = this.read(relative);
331 const pattern = new RegExp(`([ \\t]*// <<${name}>>\\n)[\\s\\S]*?([ \\t]*// <<\\/${name}>>)`);
332 if (!pattern.test(source)) {
333 throw new Error(`region <<${name}>> not found in ${relative}`);
334 }
335 this.write(relative, source.replace(pattern, `$1${body}$2`));
336 }
337
338 // ── teardown ─────────────────────────────────────────────────────────
339
340 /**
341 * Ctrl-C the whole process GROUP, then escalate. Signalling only the CLI
342 * orphans the exec child, which keeps holding the stack's state lock.
343 */
344 async shutdown(timeoutMs = 30_000): Promise<void> {
345 if (!this.proc.pid) return;
346 const killGroup = (signal: NodeJS.Signals) => {
347 try {
348 process.kill(-this.proc.pid!, signal);
349 } catch {
350 // group already gone
351 }
352 };
353 const exited = new Promise((resolve) => this.proc.once("exit", resolve));
354 killGroup("SIGINT");
355 await Promise.race([exited, Bun.sleep(timeoutMs)]);
356 if (this.alive) {
357 killGroup("SIGKILL");
358 await Promise.race([exited, Bun.sleep(5_000)]);
359 }
360 this.logStream.end();
361 }
362
363 /** Tear the stack's local resources down out of band. */
364 destroyStack(timeoutMs = 180_000): void {
365 spawnSync("bun", [ALCHEMY_BIN, "destroy", "--stage", this.stage, "--yes"], {
366 cwd: this.cwd,
367 stdio: process.env.DEBUG ? "inherit" : "ignore",
368 timeout: timeoutMs,
369 env: hermeticEnv(),
370 });
371 }
372 }
373
374 // ── polling helpers ─────────────────────────────────────────────────────
375
376 export class PollTimeout extends Error {}
377
378 /**
379 * Every request the suite makes carries a deadline. A local Worker that is
380 * hot-swapped while a request is in flight can drop that request on the
381 * floor — workerd cancels it and the dev proxy never answers the client —
382 * and a bare `fetch` would then wait forever, wedging the whole suite
383 * (observed: a `/roundtrip` in flight across a MicrovmWorker bundle swap).
384 * The deadline is generous enough for a MicroVM boot-to-terminate
385 * roundtrip, which is the slowest single request here.
386 */
387 export const REQUEST_TIMEOUT_MS = 120_000;
388
389 export const fetchWithDeadline = (
390 url: string | URL,
391 init?: RequestInit,
392 timeoutMs = REQUEST_TIMEOUT_MS,
393 ): Promise<Response> =>
394 fetch(url, {
395 ...init,
396 signal: init?.signal ?? AbortSignal.timeout(timeoutMs),
397 });
398
399 /** Bounded poll for a (possibly async) producer to yield a value. */
400 export const pollUntil = async <T>(
401 what: string,
402 f: () => T | undefined | Promise<T | undefined>,
403 {
404 tries = 60,
405 delayMs = 500,
406 server,
407 }: { tries?: number; delayMs?: number; server?: DevServer } = {},
408 ): Promise<T> => {
409 for (let i = 0; i < tries; i++) {
410 // A dead CLI can never satisfy the predicate — fail immediately with
411 // the crash context instead of burning the whole budget.
412 server?.assertAlive(what);
413 const value = await f();
414 if (value !== undefined) return value;
415 await Bun.sleep(delayMs);
416 }
417 throw new PollTimeout(
418 `Timed out after ${(tries * delayMs) / 1000}s waiting for ${what}.` +
419 (server ? `\n--- alchemy dev output (tail) ---\n${server.tail()}` : ""),
420 );
421 };
422
423 /** Fetch with retries — a freshly (re)started worker takes a moment. */
424 export const fetchOk = async (
425 url: string | URL,
426 init?: RequestInit,
427 { tries = 40, delayMs = 500 }: { tries?: number; delayMs?: number } = {},
428 ): Promise<Response> => {
429 let last: Response | undefined;
430 let error: unknown;
431 for (let i = 0; i < tries; i++) {
432 try {
433 last = await fetchWithDeadline(url, init);
434 if (last.ok) return last;
435 } catch (cause) {
436 error = cause; // not listening yet, mid-swap, or dropped
437 }
438 await Bun.sleep(delayMs);
439 }
440 throw new Error(
441 `${init?.method ?? "GET"} ${url} never returned 2xx ` +
442 `(last status: ${last?.status}, last error: ${String(error)})`,
443 );
444 };
445
446 export const fetchJson = async <T>(
447 url: string | URL,
448 init?: RequestInit,
449 options?: { tries?: number; delayMs?: number },
450 ): Promise<T> => (await fetchOk(url, init, options)).json() as Promise<T>;
451
452 /**
453 * Poll a URL until its JSON body satisfies `predicate`. This is how the
454 * suite observes a reload landing: not by reading the CLI's logs, but by
455 * asking the running worker what code it is currently executing.
456 */
457 export const waitForJson = async <T>(
458 what: string,
459 url: string | URL,
460 predicate: (body: T) => boolean,
461 options?: { tries?: number; delayMs?: number; server?: DevServer },
462 ): Promise<T> => {
463 // What the endpoint last said, so a timeout explains itself: "served the
464 // OLD marker for 240s" and "answered 502 the whole time" are different
465 // bugs, and a bare timeout hides which one it was.
466 let last = "no response yet";
467 try {
468 return await pollUntil(
469 what,
470 async () => {
471 try {
472 const res = await fetchWithDeadline(url);
473 const text = await res.text();
474 last = `${res.status} ${text.slice(0, 300)}`;
475 if (!res.ok) return undefined;
476 const body = JSON.parse(text) as T;
477 return predicate(body) ? body : undefined;
478 } catch (cause) {
479 last = `error: ${String(cause).slice(0, 300)}`;
480 return undefined; // mid-restart, or dropped
481 }
482 },
483 { tries: 120, delayMs: 500, ...options },
484 );
485 } catch (cause) {
486 if (cause instanceof PollTimeout) {
487 cause.message = `${cause.message.split("\n")[0]}\n--- last response from ${url} ---\n${last}\n${cause.message.split("\n").slice(1).join("\n")}`;
488 }
489 throw cause;
490 }
491 };
492
493 /** `http://localhost:<port>` */
494 export const at = (port: number, path = "/"): URL => new URL(path, `http://localhost:${port}`);